From 5dd2cc3a295b09d5632e541f8f17fbc6073502e9 Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Wed, 29 Nov 2023 18:46:02 -0300 Subject: [PATCH] fix(basic.gblib): Databases. #392 @othonlima. --- package.json | 1 + packages/basic.gblib/services/GBVMService.ts | 241 +++++++++++------- packages/core.gbapp/services/GBCoreService.ts | 7 +- packages/core.gbapp/services/GBDeployer.ts | 5 +- src/app.ts | 2 +- 5 files changed, 154 insertions(+), 102 deletions(-) diff --git a/package.json b/package.json index d092f005..8cedfb74 100644 --- a/package.json +++ b/package.json @@ -125,6 +125,7 @@ "lodash": "4.17.21", "luxon": "3.1.0", "mammoth": "1.5.1", + "mariadb": "3.2.2", "mime-types": "2.1.35", "moment": "1.3.0", "ms-rest-azure": "3.0.0", diff --git a/packages/basic.gblib/services/GBVMService.ts b/packages/basic.gblib/services/GBVMService.ts index abe8ad47..a320d84a 100644 --- a/packages/basic.gblib/services/GBVMService.ts +++ b/packages/basic.gblib/services/GBVMService.ts @@ -32,7 +32,7 @@ 'use strict'; -import { GBMinInstance, GBService, IGBCoreService, GBDialogStep, GBLog } from 'botlib'; +import { GBMinInstance, GBService, IGBCoreService, GBDialogStep, GBLog, GBError } from 'botlib'; import * as Fs from 'fs'; import { GBServer } from '../../../src/app.js'; import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; @@ -205,56 +205,79 @@ export class GBVMService extends GBService { const tableDef = JSON.parse(Fs.readFileSync(tablesFile, 'utf8')) as any; - const getTypeBasedOnCondition = (t) => { - switch (t) { - case 'string': - return { key: 'STRING' }; - case 'guid': - return { key: 'UUID' }; - case 'key': - return { key: 'STRING' }; // Assuming key is a string data type - case 'number': - return { key: 'BIGINT' }; - case 'integer': - return { key: 'INTEGER' }; - case 'double': - return { key: 'FLOAT' }; - case 'float': - return { key: 'FLOAT' }; - case 'date': - return { key: 'DATE' }; - case 'boolean': - return { key: 'BOOLEAN' }; - default: - return { key: 'TABLE', name: t }; + const getTypeBasedOnCondition = (t, size) => { + + if (1) { + switch (t) { + case 'string': + return `varchar(${size})`; + case 'guid': + return 'UUID'; + case 'key': + return `varchar(${size})`; + case 'number': + return 'BIGINT'; + case 'integer': + return 'INTEGER'; + case 'double': + return 'FLOAT'; + case 'float': + return 'FLOAT'; + case 'date': + return 'DATE'; + case 'boolean': + return 'BOOLEAN'; + default: + return { type: 'TABLE', name: t }; + } + + } else { + switch (t) { + case 'string': + return { key: 'STRING' }; + case 'guid': + return { key: 'UUID' }; + case 'key': + return { key: 'STRING' }; // Assuming key is a string data type + case 'number': + return { key: 'BIGINT' }; + case 'integer': + return { key: 'INTEGER' }; + case 'double': + return { key: 'FLOAT' }; + case 'float': + return { key: 'FLOAT' }; + case 'date': + return { key: 'DATE' }; + case 'boolean': + return { key: 'BOOLEAN' }; + default: + return { key: 'TABLE', name: t }; + } + } }; const associations = []; - const tables = await min.core.sequelize.query(`SELECT table_name, table_schema - FROM information_schema.tables - WHERE table_type = 'BASE TABLE' - ORDER BY table_name ASC`, { - type: QueryTypes.RAW - }) - // Loads storage custom connections. const path = DialogKeywords.getGBAIPath(min.botId, null); const filePath = Path.join('work', path, 'connections.json'); let connections = null; - if(Fs.existsSync(filePath)){ - connections = Fs.readFileSync(filePath, 'utf8'); + if (Fs.existsSync(filePath)) { + connections = JSON.parse(Fs.readFileSync(filePath, 'utf8')); + } - tableDef.forEach(t => { - const tableName = t.name; - + tableDef.forEach(async t => { + + const tableName = t.name.trim(); + // Determines autorelationship. Object.keys(t.fields).forEach(key => { let obj = t.fields[key]; - obj.type = getTypeBasedOnCondition(obj.type); + obj.type = getTypeBasedOnCondition(obj.type, obj.size); if (obj.type.key === "TABLE") { obj.type.key = "BIGINT" associations.push({ from: tableName, to: obj.type.name }); @@ -267,16 +290,17 @@ export class GBVMService extends GBService { // Cutom connection for TABLE. const connectionName = t.connection; - if (connectionName) { + let con; - const con = connections[connectionName]; + if (connectionName && connections) { + con = connections.filter(p => p.name === connectionName)[0]; - const dialect = con['storageDialect']; - const host = con['storageHost']; + const dialect = con['storageDriver']; + const host = con['storageServer']; const port = con['storagePort']; const storageName = con['storageName']; const username = con['storageUsername']; - const password = con['password']; + const password = con['storagePassword']; const logging: boolean | Function = GBConfigService.get('STORAGE_LOGGING') === 'true' @@ -312,73 +336,98 @@ export class GBVMService extends GBService { } }; - min[connectionName] = new Sequelize(storageName, username, password, sequelizeOptions); + if (!min[connectionName]) { + GBLogEx.info(min, `Loading custom connection ${connectionName}...`); + min[connectionName] = new Sequelize(storageName, username, password, sequelizeOptions); + } + } + + if (!con) { + throw new Error(`Invalid connection specified: ${connectionName}.`); } // Field checking, syncs if there is any difference. - const model = min[connectionName] ? min[connectionName].models[tableName] : minBoot.core.sequelize; - if (model) { + const seq = min[connectionName] ? min[connectionName] + : minBoot.core.sequelize; - // Except Id, checks if has same number of fields. + if (seq) { - let equals = 0; - Object.keys(t.fields).forEach(key => { - let obj1 = t.fields[key]; - let obj2 = model['fieldRawAttributesMap'][key]; + const model = seq.models[tableName]; + if (model) { + // Except Id, checks if has same number of fields. - if (key !== "id") { - if (obj1 && obj2) { - equals++; + let equals = 0; + Object.keys(t.fields).forEach(key => { + let obj1 = t.fields[key]; + let obj2 = model['fieldRawAttributesMap'][key]; + + if (key !== "id") { + if (obj1 && obj2) { + equals++; + } } + }); + + + if (equals != Object.keys(t.fields).length) { + sync = true; + } + } + + seq.define(tableName, t.fields); + + // New table checking, if needs sync. + let tables; + + if (con.storageDriver === 'mssql') { + tables = await seq.query(`SELECT table_name, table_schema + FROM information_schema.tables + WHERE table_type = 'BASE TABLE' + ORDER BY table_name ASC`, { + type: QueryTypes.RAW + })[0] + } + else if (con.storageDriver === 'mariadb') { + tables = await seq.getQueryInterface().showAllTables(); + } + + let found = false; + tables.forEach((storageTable) => { + if (storageTable['table_name'] === tableName) { + found = true; } }); + sync = sync ? sync : !found; - if (equals != Object.keys(t.fields).length) { - sync = true; + + associations.forEach(e => { + const from = seq.models[e.from]; + const to = seq.models[e.to]; + + try { + to.hasMany(from); + } catch (error) { + throw new Error(`BASIC: Invalid relationship in ${mainName}: from ${e.from} to ${e.to} (${min.botId})... ${error.message}`); + } + + }); + + if (sync) { + GBLogEx.info(min, `BASIC: Syncing changes for TABLE ${connectionName} ${tableName} keyword (${min.botId})...`); + + await seq.sync({ + alter: true, + force: false // Keep it false due to data loss danger. + }); + GBLogEx.info(min, `BASIC: Done sync for ${min.botId} ${connectionName} ${tableName} storage table...`); + } + else { + GBLogEx.verbose(min, `BASIC: TABLE ${tableName} keywords already up to date ${connectionName} (${min.botId})...`); } } - - minBoot.core.sequelize.define(tableName, t.fields); - - // New table checking, if needs sync. - - let found = false; - tables[0].forEach((storageTable) => { - if (storageTable['table_name'] === tableName) { - found = true; - } - }); - - sync = sync ? sync : !found; - }); - - associations.forEach(e => { - const from = minBoot.core.sequelize.models[e.from]; - const to = minBoot.core.sequelize.models[e.to]; - - try { - to.hasMany(from); - } catch (error) { - throw new Error(`BASIC: Invalid relationship in ${mainName}: from ${e.from} to ${e.to} (${min.botId})... ${error.message}`); - } - - }); - - if (sync) { - GBLogEx.info(min, `BASIC: Syncing changes for TABLE keywords (${min.botId})...`); - - await minBoot.core.sequelize.sync({ - alter: true, - force: false // Keep it false due to data loss danger. - }); - GBLogEx.info(min, `BASIC: Done sync for ${min.botId} storage tables...`); - } - else { - GBLogEx.verbose(min, `BASIC: TABLE keywords already up to date (${min.botId})...`); - } } const parsedCode: string = Fs.readFileSync(jsfile, 'utf8'); @@ -658,7 +707,7 @@ export class GBVMService extends GBService { definition['type'] = t; if (reg[3]) { - definition['size'] = Number.parseInt(reg[3] === 'max' ? '4000' : reg[3]); + definition['size'] = Number.parseInt(reg[3] === 'max' ? '4000' : reg[3]); } return { name, definition }; @@ -721,7 +770,7 @@ export class GBVMService extends GBService { if (endTableReg && table) { tables.push({ - name: table, fields: fields, connection:connection + name: table, fields: fields, connection: connection }); @@ -743,7 +792,7 @@ export class GBVMService extends GBService { let tableReg = tableKeyword.exec(line); if (tableReg && !table) { table = tableReg[1]; - connection= tableReg[2]; + connection = tableReg[2]; emmit = false; } diff --git a/packages/core.gbapp/services/GBCoreService.ts b/packages/core.gbapp/services/GBCoreService.ts index 15f35a8d..8e07b72e 100644 --- a/packages/core.gbapp/services/GBCoreService.ts +++ b/packages/core.gbapp/services/GBCoreService.ts @@ -675,7 +675,8 @@ ENDPOINT_UPDATE=true public getParam(instance: IGBInstance, name: string, defaultValue?: T): any { let value = null; if (instance.params) { - const params = JSON.parse(instance.params); + + const params = typeof (instance.params) === 'object' ? instance.params: JSON.parse(instance.params); value = params ? params[name] : defaultValue; } if (typeof defaultValue === 'boolean') { @@ -696,7 +697,7 @@ ENDPOINT_UPDATE=true } } if (value === undefined) { - value = null; + value = null; } return value; @@ -709,7 +710,7 @@ ENDPOINT_UPDATE=true let params = null; const list = []; if (instance.params) { - params = JSON.parse(instance.params); + params = typeof (instance.params) === 'object' ? instance.params : JSON.parse(instance.params); } Object.keys(params).forEach(e => { diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index 73ebf249..fb4f0711 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -619,10 +619,11 @@ export class GBDeployer implements IGBDeployer { // Find all tokens in .gbot Config. const strFind = ' Driver'; - const tokens = await min.core['findParam'](min.instance, strFind); - await CollectionUtil.asyncForEach(tokens,async t => { + const conns = await min.core['findParam'](min.instance, strFind); + await CollectionUtil.asyncForEach(conns, async t => { const connectionName = t.replace(strFind, ''); let con = {}; + con['name'] = connectionName; con['storageServer']= min.core.getParam(min.instance, `${connectionName} Server`, null), con['storageName']= min.core.getParam(min.instance, `${connectionName} Name`, null), con['storageUsername']= min.core.getParam(min.instance, `${connectionName} Username`, null), diff --git a/src/app.ts b/src/app.ts index 29ff8893..1ff7b7da 100644 --- a/src/app.ts +++ b/src/app.ts @@ -111,7 +111,7 @@ export class GBServer { process.on('unhandledRejection', (err, p) => { err = err['e'] ? err['e'] : err; - GBLog.error(`UNHANDLED_REJECTION(promises): ${err.toString()} ${err['stack'] ? '\n' + err['stack'] : ''}`); + GBLog.error(`UNHANDLED_REJECTION(promises): ${err.toString()} ${err['stack'] ? '\n' + err['stack'] : ''} ${err['cause'] ? '\n' + err['cause']?.message : ''}`); if (err['response']?.obj?.httpStatusCode === 404) { GBLog.warn(`Check reverse proxy: ${process.env.BOT_URL} as it seems to be invalid.`); }