From c1db8be0c074da0ccb10eb5968e79040343f9588 Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (pragmatismo.io)" Date: Mon, 26 Nov 2018 15:54:34 -0200 Subject: [PATCH] fix(core): Moved logic from app to core. --- packages/core.gbapp/services/GBAPIService.ts | 24 +-- packages/core.gbapp/services/GBCoreService.ts | 181 ++++++++++++++++-- packages/core.gbapp/services/GBDeployer.ts | 11 +- packages/core.gbapp/services/GBVMService.ts | 49 ++--- packages/core.gbapp/tests/vm.test.ts | 2 +- src/app.ts | 153 +++------------ 6 files changed, 221 insertions(+), 199 deletions(-) diff --git a/packages/core.gbapp/services/GBAPIService.ts b/packages/core.gbapp/services/GBAPIService.ts index aa6308b8..e79025c7 100644 --- a/packages/core.gbapp/services/GBAPIService.ts +++ b/packages/core.gbapp/services/GBAPIService.ts @@ -32,31 +32,21 @@ 'use strict'; -import { BotAdapter } from 'botbuilder'; -import { GBError } from 'botlib'; -import { IGBPackage } from 'botlib'; -import * as fs from 'fs'; -import { Messages } from '../strings'; -const logger = require('../../../src/logger'); import { WaterfallDialog } from 'botbuilder-dialogs'; -import { IGBCoreService, IGBInstance } from 'botlib'; -import { resolve } from 'bluebird'; -const util = require('util'); -const vm = require('vm'); +import { IGBInstance, IGBPackage } from 'botlib'; /** * @fileoverview General Bots server core. */ export class DialogClass { - public step: any; public min: IGBInstance; constructor(min: IGBInstance) { this.min = min; } - public async expectMessage(text: string): Promise { + public async expectMessage(text: string): Promise { return new Promise((resolve, reject) => { this.min.dialogs.add( new WaterfallDialog('/vmExpect', [ @@ -67,8 +57,8 @@ export class DialogClass { async step => { resolve(step.result); return await step.next(); - }, - ]), + } + ]) ); }); } @@ -79,10 +69,8 @@ export class DialogClass { async step => { await step.context.sendActivity(text); return await step.next(); - }, - ]), + } + ]) ); } - - } diff --git a/packages/core.gbapp/services/GBCoreService.ts b/packages/core.gbapp/services/GBCoreService.ts index 6043c892..88f6690f 100644 --- a/packages/core.gbapp/services/GBCoreService.ts +++ b/packages/core.gbapp/services/GBCoreService.ts @@ -36,14 +36,23 @@ 'use strict'; -import { IGBCoreService, IGBInstance } from 'botlib'; +import { IGBCoreService, IGBInstance, IGBPackage } from 'botlib'; import * as fs from 'fs'; -import processExists = require('process-exists'); import { Sequelize } from 'sequelize-typescript'; -const logger = require('../../../src/logger'); import { GBAdminService } from '../../admin.gbapp/services/GBAdminService'; import { GuaribasInstance } from '../models/GBModel'; import { GBConfigService } from './GBConfigService'; +import { AzureDeployerService } from 'packages/azuredeployer.gbapp/services/AzureDeployerService'; +import { GBAnalyticsPackage } from 'packages/analytics.gblib'; +import { GBAdminPackage } from 'packages/admin.gbapp/index'; +import { GBCorePackage } from 'packages/core.gbapp'; +import { GBCustomerSatisfactionPackage } from 'packages/customer-satisfaction.gbapp'; +import { GBKBPackage } from 'packages/kb.gbapp'; +import { GBSecurityPackage } from 'packages/security.gblib'; +import { GBWhatsappPackage } from 'packages/whatsapp.gblib/index'; + +const logger = require('../../../src/logger'); +const opn = require('opn'); /** * Core service layer. @@ -70,7 +79,7 @@ export class GBCoreService implements IGBCoreService { private createTableQuery: ( tableName: string, attributes: any, - options: any + options: any, ) => string; /** @@ -136,15 +145,15 @@ export class GBCoreService implements IGBCoreService { dialect: this.dialect, storage: storage, dialectOptions: { - encrypt: encrypt + encrypt: encrypt, }, pool: { max: 32, min: 8, idle: 40000, evict: 40000, - acquire: 40000 - } + acquire: 40000, + }, }); if (this.dialect === 'mssql') { @@ -153,7 +162,7 @@ export class GBCoreService implements IGBCoreService { this.queryGenerator.createTableQuery = ( tableName, attributes, - options + options, ) => this.createTableQueryOverride(tableName, attributes, options); this.changeColumnQuery = this.queryGenerator.changeColumnQuery; this.queryGenerator.changeColumnQuery = (tableName, attributes) => @@ -163,7 +172,7 @@ export class GBCoreService implements IGBCoreService { } catch (error) { reject(error); } - } + }, ); } @@ -174,7 +183,7 @@ export class GBCoreService implements IGBCoreService { logger.info('Syncing database...'); return this.sequelize.sync({ alter: alter, - force: force + force: force, }); } else { const msg = 'Database synchronization is disabled.'; @@ -257,7 +266,7 @@ export class GBCoreService implements IGBCoreService { let sql: string = this.createTableQuery.apply(this.queryGenerator, [ tableName, attributes, - options + options, ]); const re1 = /CREATE\s+TABLE\s+\[([^\]]*)\]/; const matches = re1.exec(sql); @@ -268,7 +277,7 @@ export class GBCoreService implements IGBCoreService { re2, (match: string, ...args: any[]): string => { return 'CONSTRAINT [' + table + '_pk] ' + match; - } + }, ); const re3 = /FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g; const re4 = /\[([^\]]*)\]/g; @@ -283,7 +292,7 @@ export class GBCoreService implements IGBCoreService { matches = re4.exec(fkcols); } return 'CONSTRAINT [' + fkname + '_fk] FOREIGN KEY (' + fkcols + ')'; - } + }, ); } return sql; @@ -300,7 +309,7 @@ export class GBCoreService implements IGBCoreService { private changeColumnQueryOverride(tableName, attributes): string { let sql: string = this.changeColumnQuery.apply(this.queryGenerator, [ tableName, - attributes + attributes, ]); const re1 = /ALTER\s+TABLE\s+\[([^\]]*)\]/; const matches = re1.exec(sql); @@ -326,9 +335,151 @@ export class GBCoreService implements IGBCoreService { fkcols + ')' ); - } + }, ); } return sql; } + + /** + * Loads all bot instances from object storage, if it's formatted. + * + * @param core + * @param azureDeployer + * @param proxyAddress + */ + public async loadAllInstances( + core: GBCoreService, + azureDeployer: AzureDeployerService, + proxyAddress: string, + ) { + logger.info(`Loading instances from storage...`); + let instances: GuaribasInstance[]; + try { + instances = await core.loadInstances(); + const instance = instances[0]; + if (process.env.NODE_ENV === 'development') { + logger.info(`Updating bot endpoint to local reverse proxy (ngrok)...`); + await azureDeployer.updateBotProxy( + instance.botId, + instance.botId, + `${proxyAddress}/api/messages/${instance.botId}`, + ); + } + } catch (error) { + if (error.parent.code === 'ELOGIN') { + const group = GBConfigService.get('CLOUD_GROUP'); + const serverName = GBConfigService.get('STORAGE_SERVER').split( + '.database.windows.net', + )[0]; + await azureDeployer.openStorageFirewall(group, serverName); + } else { + // Check if storage is empty and needs formatting. + const isInvalidObject = + error.parent.number == 208 || error.parent.errno == 1; // MSSQL or SQLITE. + if (isInvalidObject) { + if (GBConfigService.get('STORAGE_SYNC') != 'true') { + throw new Error( + `Operating storage is out of sync or there is a storage connection error. Try setting STORAGE_SYNC to true in .env file. Error: ${ + error.message + }.`, + ); + } else { + logger.info( + `Storage is empty. After collecting storage structure from all .gbapps it will get synced.`, + ); + } + } else { + throw new Error( + `Cannot connect to operating storage: ${error.message}.`, + ); + } + } + } + return instances; + } + + /** + * If instances is undefined here it's because storage has been formatted. + * Load all instances from .gbot found on deploy package directory. + * @param instances + * @param bootInstance + * @param core + */ + public async ensureInstances( + instances: GuaribasInstance[], + bootInstance: any, + core: GBCoreService, + ) { + if (!instances) { + const saveInstance = new GuaribasInstance(bootInstance); + await saveInstance.save(); + instances = await core.loadInstances(); + } + return instances; + } + + public loadSysPackages(core: GBCoreService) { + // NOTE: if there is any code before this line a semicolon + // will be necessary before this line. + // Loads all system packages. + + [ + GBAdminPackage, + GBAnalyticsPackage, + GBCorePackage, + GBSecurityPackage, + GBKBPackage, + GBCustomerSatisfactionPackage, + GBWhatsappPackage, + ].forEach(e => { + logger.info(`Loading sys package: ${e.name}...`); + const p = Object.create(e.prototype) as IGBPackage; + p.loadPackage(core, core.sequelize); + }); + } + + public ensureAdminIsSecured() { + const password = GBConfigService.get('ADMIN_PASS'); + if (!GBAdminService.StrongRegex.test(password)) { + throw new Error( + 'Please, define a really strong password in ADMIN_PASS environment variable before running the server.', + ); + } + } + + public async createBootInstance( + core: GBCoreService, + azureDeployer: AzureDeployerService, + proxyAddress: string, + ) { + let bootInstance: IGBInstance; + try { + await core.initDatabase(); + } catch (error) { + logger.info( + `Deploying cognitive infrastructure (on the cloud / on premises)...`, + ); + try { + bootInstance = await azureDeployer.deployFarm(proxyAddress); + } catch (error) { + logger.warn( + 'In case of error, please cleanup any infrastructure objects created during this procedure and .env before running again.', + ); + throw error; + } + core.writeEnv(bootInstance); + logger.info(`File .env written, starting General Bots...`); + GBConfigService.init(); + await core.initDatabase(); + } + return bootInstance; + } + + public openBrowserInDevelopment() { + if (process.env.NODE_ENV === 'development') { + opn('http://localhost:4242'); + } + } + } diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index 35581bae..3eca1b85 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -52,6 +52,7 @@ import { GuaribasInstance, GuaribasPackage } from '../models/GBModel'; import { KBService } from './../../kb.gbapp/services/KBService'; import { GBConfigService } from './GBConfigService'; import { GBImporter } from './GBImporterService'; +import { GBVMService } from './GBVMService'; /** Deployer service for bots, themes, ai and more. */ export class GBDeployer { @@ -266,6 +267,10 @@ export class GBDeployer { }); } + public deployScriptToStorage(instanceId: number, localPath: string) { + + } + public deployTheme(localPath: string) { // DISABLED: Until completed, "/ui/public". // FsExtra.copy(localPath, this.workDir + packageName) @@ -297,14 +302,12 @@ export class GBDeployer { break; case '.gbdialog': - const vm = new VMService(this.core.sequelize); + const vm = new GBVMService(); return service.deployKb(this.core, this, localPath); - break; - default: const err = GBError.create( - `GuaribasBusinessError: Unknow package type: ${packageType}.`, + `GuaribasBusinessError: Unknown package type: ${packageType}.` ); Promise.reject(err); break; diff --git a/packages/core.gbapp/services/GBVMService.ts b/packages/core.gbapp/services/GBVMService.ts index 6eaa4a83..42da3fc2 100644 --- a/packages/core.gbapp/services/GBVMService.ts +++ b/packages/core.gbapp/services/GBVMService.ts @@ -36,13 +36,17 @@ import { IGBCoreService, IGBInstance } from 'botlib'; import { GBError } from 'botlib'; import { IGBPackage } from 'botlib'; const logger = require('../../../src/logger'); -import * as fs from 'fs'; import { BotAdapter } from 'botbuilder'; import { WaterfallDialog } from 'botbuilder-dialogs'; +import * as fs from 'fs'; import { Messages } from '../strings'; +import { DialogClass } from './GBAPIService'; import { GBDeployer } from './GBDeployer'; const util = require('util'); const vm = require('vm'); +import processExists = require('process-exists'); +import { Sequelize } from 'sequelize-typescript'; +const UrlJoin = require('url-join'); /** * @fileoverview General Bots server core. @@ -50,50 +54,27 @@ const vm = require('vm'); export class GBVMService implements IGBCoreService { - public static setup(bot: BotAdapter, min: IGBInstance) { + private script = new vm.Script(); - } - - public loadJS( + public async loadJS( filename: string, min: IGBInstance, core: IGBCoreService, deployer: GBDeployer, localPath: string - ) { + ): Promise { - const sandbox = { - animal: 'cat', - count: 2, - }; + const code = fs.readFileSync(UrlJoin(localPath, filename), 'utf8'); + const sandbox = new DialogClass(min); - const script = new vm.Script('count += 1; name = "kitty";'); const context = vm.createContext(sandbox); - - for (let i = 0; i < 10; ++i) { - script.runInContext(context); - } - + this.script.runInContext(context); console.log(util.inspect(sandbox)); - // { animal: 'cat', count: 12, name: 'kitty' } - - const packageType = Path.extname(localPath); - const packageName = Path.basename(localPath); - logger.info(`[GBDeployer] Opening package: ${localPath}`); - const packageObject = JSON.parse( - Fs.readFileSync(UrlJoin(localPath, 'package.json'), 'utf8'), + await deployer.deployScriptToStorage( + min.instanceId, + filename ); - - const instance = await core.loadInstance(packageObject.botId); - logger.info(`[GBDeployer] Importing: ${localPath}`); - const p = await deployer.deployPackageToStorage( - instance.instanceId, - packageName, - ); - await this.importKbPackage(localPath, p, instance); - - deployer.rebuildIndex(instance); - logger.info(`[GBDeployer] Finished import of ${localPath}`); + logger.info(`[GBVMService] Finished loading of ${filename}`); } } diff --git a/packages/core.gbapp/tests/vm.test.ts b/packages/core.gbapp/tests/vm.test.ts index 4102b513..acade433 100644 --- a/packages/core.gbapp/tests/vm.test.ts +++ b/packages/core.gbapp/tests/vm.test.ts @@ -43,7 +43,7 @@ describe('Load function', () => { it('should fail on invalid file', () => { try { const service = new GBVMService(); - service.loadJS('invalid.file'); + service.loadJS('invalid.file', null, null, null, null); } catch (error) { expect(error).to.equal(0); } diff --git a/src/app.ts b/src/app.ts index 6014b637..c0d174e0 100644 --- a/src/app.ts +++ b/src/app.ts @@ -40,14 +40,9 @@ const logger = require('./logger'); const express = require('express'); const bodyParser = require('body-parser'); -const opn = require('opn'); - import { IGBInstance, IGBPackage } from 'botlib'; -import { GBAdminPackage } from '../packages/admin.gbapp/index'; import { GBAdminService } from '../packages/admin.gbapp/services/GBAdminService'; -import { GBAnalyticsPackage } from '../packages/analytics.gblib'; import { AzureDeployerService } from '../packages/azuredeployer.gbapp/services/AzureDeployerService'; -import { GBCorePackage } from '../packages/core.gbapp'; import { GuaribasInstance } from '../packages/core.gbapp/models/GBModel'; import { GBConfigService } from '../packages/core.gbapp/services/GBConfigService'; import { GBConversationalService } from '../packages/core.gbapp/services/GBConversationalService'; @@ -55,10 +50,6 @@ import { GBCoreService } from '../packages/core.gbapp/services/GBCoreService'; import { GBDeployer } from '../packages/core.gbapp/services/GBDeployer'; import { GBImporter } from '../packages/core.gbapp/services/GBImporterService'; import { GBMinService } from '../packages/core.gbapp/services/GBMinService'; -import { GBCustomerSatisfactionPackage } from '../packages/customer-satisfaction.gbapp'; -import { GBKBPackage } from '../packages/kb.gbapp'; -import { GBSecurityPackage } from '../packages/security.gblib'; -import { GBWhatsappPackage } from './../packages/whatsapp.gblib/index'; const appPackages = new Array(); @@ -66,13 +57,11 @@ const appPackages = new Array(); * General Bots open-core entry point. */ export class GBServer { - /** * Program entry-point. */ public static run() { - logger.info(`The Bot Server is in STARTING mode...`); // Creates a basic HTTP server that will serve several URL, one for each @@ -86,8 +75,8 @@ export class GBServer { server.use( bodyParser.urlencoded({ // to support URL-encoded bodies - extended: true - }) + extended: true, + }), ); let bootInstance: IGBInstance; @@ -106,133 +95,42 @@ export class GBServer { logger.info(`Establishing a development local proxy (ngrok)...`); const proxyAddress = await core.ensureProxy(port); + logger.info(`Deploying packages...`); const deployer = new GBDeployer(core, new GBImporter(core)); const azureDeployer = new AzureDeployerService(deployer); - - try { - await core.initDatabase(); - } catch (error) { - logger.info(`Deploying cognitive infrastructure (on the cloud / on premises)...`); - try { - bootInstance = await azureDeployer.deployFarm(proxyAddress); - } catch (error) { - logger.warn( - 'In case of error, please cleanup any infrastructure objects created during this procedure and .env before running again.' - ); - throw error; - } - core.writeEnv(bootInstance); - logger.info(`File .env written, starting General Bots...`); - GBConfigService.init(); - - await core.initDatabase(); - } - - // TODO: Get .gb* templates from GitHub and download do additional deploy folder. - - // Check admin password. - - const conversationalService = new GBConversationalService(core); const adminService = new GBAdminService(core); - const password = GBConfigService.get('ADMIN_PASS'); - - if (!GBAdminService.StrongRegex.test(password)) { - throw new Error( - 'Please, define a really strong password in ADMIN_PASS environment variable before running the server.' - ); - } - - // NOTE: the semicolon is necessary before this line. - // Loads all system packages. - - [ - GBAdminPackage, - GBAnalyticsPackage, - GBCorePackage, - GBSecurityPackage, - GBKBPackage, - GBCustomerSatisfactionPackage, - GBWhatsappPackage - ].forEach(e => { - logger.info(`Loading sys package: ${e.name}...`); - const p = Object.create(e.prototype) as IGBPackage; - p.loadPackage(core, core.sequelize); - }); - - // Loads all bot instances from object storage, if it's formatted. - - logger.info(`Loading instances from storage...`); - let instances: GuaribasInstance[]; - try { - instances = await core.loadInstances(); - const instance = instances[0]; - - if (process.env.NODE_ENV === 'development') { - logger.info(`Updating bot endpoint to local reverse proxy (ngrok)...`); - - await azureDeployer.updateBotProxy( - instance.botId, - instance.botId, - `${proxyAddress}/api/messages/${instance.botId}` - ); - } - } catch (error) { - if (error.parent.code === 'ELOGIN') { - const group = GBConfigService.get('CLOUD_GROUP'); - const serverName = GBConfigService.get('STORAGE_SERVER').split( - '.database.windows.net' - )[0]; - await azureDeployer.openStorageFirewall(group, serverName); - } else { - // Check if storage is empty and needs formatting. - - const isInvalidObject = - error.parent.number == 208 || error.parent.errno == 1; // MSSQL or SQLITE. - - if (isInvalidObject) { - if (GBConfigService.get('STORAGE_SYNC') != 'true') { - throw new Error(`Operating storage is out of sync or there is a storage connection error. Try setting STORAGE_SYNC to true in .env file. Error: ${ - error.message - }.`); - } else { - logger.info( - `Storage is empty. After collecting storage structure from all .gbapps it will get synced.` - ); - } - } else { - throw new Error(`Cannot connect to operating storage: ${error.message}.`); - } - } - } - - // Deploy packages and format object store according to .gbapp storage models. - - logger.info(`Deploying packages...`); + const conversationalService = new GBConversationalService(core); + bootInstance = await core.createBootInstance( + core, + azureDeployer, + proxyAddress, + ); + core.ensureAdminIsSecured(); + core.loadSysPackages(core); await deployer.deployPackages(core, server, appPackages); - // If instances is undefined here it's because storage has been formatted. - // Load all instances from .gbot found on deploy package directory. - if (!instances) { - const saveInstance = new GuaribasInstance(bootInstance); - await saveInstance.save(); - instances = await core.loadInstances(); - } - - // Setup server dynamic (per bot instance) resources and listeners. - logger.info(`Publishing instances...`); + let instances: GuaribasInstance[] = await core.loadAllInstances( + core, + azureDeployer, + proxyAddress, + ); + instances = await core.ensureInstances( + instances, + bootInstance, + core + ); + const minService = new GBMinService( core, conversationalService, adminService, - deployer + deployer, ); await minService.buildMin(server, appPackages, instances); - logger.info(`The Bot Server is in RUNNING mode...`); - if (process.env.NODE_ENV === 'development') { - opn('http://localhost:4242'); - } + logger.info(`The Bot Server is in RUNNING mode...`); + core.openBrowserInDevelopment(); return core; } catch (err) { @@ -243,6 +141,7 @@ export class GBServer { }); } } + // First line to run.