GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix(core): Moved logic from app to core.

Browse source
This commit is contained in:
Rodrigo Rodriguez (pragmatismo.io) 2018-11-26 15:54:34 -02:00
parent 09715bcfc0
commit c1db8be0c0
6 changed files with 221 additions and 199 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
packages/core.gbapp/services/GBAPIService.ts
View file

@ -32,31 +32,21 @@
'use strict';
import { BotAdapter } from 'botbuilder';
import { GBError } from 'botlib';
import { IGBPackage } from 'botlib';
import * as fs from 'fs';
import { Messages } from '../strings';
const logger = require('../../../src/logger');
import { WaterfallDialog } from 'botbuilder-dialogs';
import { IGBCoreService, IGBInstance } from 'botlib';
import { resolve } from 'bluebird';
const util = require('util');
const vm = require('vm');
import { IGBInstance, IGBPackage } from 'botlib';
/**
* @fileoverview General Bots server core.
*/
export class DialogClass {
public step: any;
public min: IGBInstance;
constructor(min: IGBInstance) {
this.min = min;
}
public async expectMessage(text: string): Promise<string> {
public async expectMessage(text: string): Promise<any> {
return new Promise((resolve, reject) => {
this.min.dialogs.add(
new WaterfallDialog('/vmExpect', [
@ -67,8 +57,8 @@ export class DialogClass {
async step => {
resolve(step.result);
return await step.next();
},
]),
}
])
);
});
}
@ -79,10 +69,8 @@ export class DialogClass {
async step => {
await step.context.sendActivity(text);
return await step.next();
},
]),
}
])
);
}
}

181
packages/core.gbapp/services/GBCoreService.ts
View file

@ -36,14 +36,23 @@
'use strict';
import { IGBCoreService, IGBInstance } from 'botlib';
import { IGBCoreService, IGBInstance, IGBPackage } from 'botlib';
import * as fs from 'fs';
import processExists = require('process-exists');
import { Sequelize } from 'sequelize-typescript';
const logger = require('../../../src/logger');
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService';
import { GuaribasInstance } from '../models/GBModel';
import { GBConfigService } from './GBConfigService';
import { AzureDeployerService } from 'packages/azuredeployer.gbapp/services/AzureDeployerService';
import { GBAnalyticsPackage } from 'packages/analytics.gblib';
import { GBAdminPackage } from 'packages/admin.gbapp/index';
import { GBCorePackage } from 'packages/core.gbapp';
import { GBCustomerSatisfactionPackage } from 'packages/customer-satisfaction.gbapp';
import { GBKBPackage } from 'packages/kb.gbapp';
import { GBSecurityPackage } from 'packages/security.gblib';
import { GBWhatsappPackage } from 'packages/whatsapp.gblib/index';
const logger = require('../../../src/logger');
const opn = require('opn');
/**
* Core service layer.
@ -70,7 +79,7 @@ export class GBCoreService implements IGBCoreService {
private createTableQuery: (
tableName: string,
attributes: any,
options: any
options: any,
) => string;
/**
@ -136,15 +145,15 @@ export class GBCoreService implements IGBCoreService {
dialect: this.dialect,
storage: storage,
dialectOptions: {
encrypt: encrypt
encrypt: encrypt,
},
pool: {
max: 32,
min: 8,
idle: 40000,
evict: 40000,
acquire: 40000
}
acquire: 40000,
},
});
if (this.dialect === 'mssql') {
@ -153,7 +162,7 @@ export class GBCoreService implements IGBCoreService {
this.queryGenerator.createTableQuery = (
tableName,
attributes,
options
options,
) => this.createTableQueryOverride(tableName, attributes, options);
this.changeColumnQuery = this.queryGenerator.changeColumnQuery;
this.queryGenerator.changeColumnQuery = (tableName, attributes) =>
@ -163,7 +172,7 @@ export class GBCoreService implements IGBCoreService {
} catch (error) {
reject(error);
}
}
},
);
}
@ -174,7 +183,7 @@ export class GBCoreService implements IGBCoreService {
logger.info('Syncing database...');
return this.sequelize.sync({
alter: alter,
force: force
force: force,
});
} else {
const msg = 'Database synchronization is disabled.';
@ -257,7 +266,7 @@ export class GBCoreService implements IGBCoreService {
let sql: string = this.createTableQuery.apply(this.queryGenerator, [
tableName,
attributes,
options
options,
]);
const re1 = /CREATE\s+TABLE\s+\[([^\]]*)\]/;
const matches = re1.exec(sql);
@ -268,7 +277,7 @@ export class GBCoreService implements IGBCoreService {
re2,
(match: string, ...args: any[]): string => {
return 'CONSTRAINT [' + table + '_pk] ' + match;
}
},
);
const re3 = /FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g;
const re4 = /\[([^\]]*)\]/g;
@ -283,7 +292,7 @@ export class GBCoreService implements IGBCoreService {
matches = re4.exec(fkcols);
}
return 'CONSTRAINT [' + fkname + '_fk] FOREIGN KEY (' + fkcols + ')';
}
},
);
}
return sql;
@ -300,7 +309,7 @@ export class GBCoreService implements IGBCoreService {
private changeColumnQueryOverride(tableName, attributes): string {
let sql: string = this.changeColumnQuery.apply(this.queryGenerator, [
tableName,
attributes
attributes,
]);
const re1 = /ALTER\s+TABLE\s+\[([^\]]*)\]/;
const matches = re1.exec(sql);
@ -326,9 +335,151 @@ export class GBCoreService implements IGBCoreService {
fkcols +
')'
);
}
},
);
}
return sql;
}
/**
* Loads all bot instances from object storage, if it's formatted.
*
* @param core
* @param azureDeployer
* @param proxyAddress
*/
public async loadAllInstances(
core: GBCoreService,
azureDeployer: AzureDeployerService,
proxyAddress: string,
) {
logger.info(`Loading instances from storage...`);
let instances: GuaribasInstance[];
try {
instances = await core.loadInstances();
const instance = instances[0];
if (process.env.NODE_ENV === 'development') {
logger.info(`Updating bot endpoint to local reverse proxy (ngrok)...`);
await azureDeployer.updateBotProxy(
instance.botId,
instance.botId,
`${proxyAddress}/api/messages/${instance.botId}`,
);
}
} catch (error) {
if (error.parent.code === 'ELOGIN') {
const group = GBConfigService.get('CLOUD_GROUP');
const serverName = GBConfigService.get('STORAGE_SERVER').split(
'.database.windows.net',
)[0];
await azureDeployer.openStorageFirewall(group, serverName);
} else {
// Check if storage is empty and needs formatting.
const isInvalidObject =
error.parent.number == 208 || error.parent.errno == 1; // MSSQL or SQLITE.
if (isInvalidObject) {
if (GBConfigService.get('STORAGE_SYNC') != 'true') {
throw new Error(
`Operating storage is out of sync or there is a storage connection error. Try setting STORAGE_SYNC to true in .env file. Error: ${
error.message
}.`,
);
} else {
logger.info(
`Storage is empty. After collecting storage structure from all .gbapps it will get synced.`,
);
}
} else {
throw new Error(
`Cannot connect to operating storage: ${error.message}.`,
);
}
}
}
return instances;
}
/**
* If instances is undefined here it's because storage has been formatted.
* Load all instances from .gbot found on deploy package directory.
* @param instances
* @param bootInstance
* @param core
*/
public async ensureInstances(
instances: GuaribasInstance[],
bootInstance: any,
core: GBCoreService,
) {
if (!instances) {
const saveInstance = new GuaribasInstance(bootInstance);
await saveInstance.save();
instances = await core.loadInstances();
}
return instances;
}
public loadSysPackages(core: GBCoreService) {
// NOTE: if there is any code before this line a semicolon
// will be necessary before this line.
// Loads all system packages.
[
GBAdminPackage,
GBAnalyticsPackage,
GBCorePackage,
GBSecurityPackage,
GBKBPackage,
GBCustomerSatisfactionPackage,
GBWhatsappPackage,
].forEach(e => {
logger.info(`Loading sys package: ${e.name}...`);
const p = Object.create(e.prototype) as IGBPackage;
p.loadPackage(core, core.sequelize);
});
}
public ensureAdminIsSecured() {
const password = GBConfigService.get('ADMIN_PASS');
if (!GBAdminService.StrongRegex.test(password)) {
throw new Error(
'Please, define a really strong password in ADMIN_PASS environment variable before running the server.',
);
}
}
public async createBootInstance(
core: GBCoreService,
azureDeployer: AzureDeployerService,
proxyAddress: string,
) {
let bootInstance: IGBInstance;
try {
await core.initDatabase();
} catch (error) {
logger.info(
`Deploying cognitive infrastructure (on the cloud / on premises)...`,
);
try {
bootInstance = await azureDeployer.deployFarm(proxyAddress);
} catch (error) {
logger.warn(
'In case of error, please cleanup any infrastructure objects created during this procedure and .env before running again.',
);
throw error;
}
core.writeEnv(bootInstance);
logger.info(`File .env written, starting General Bots...`);
GBConfigService.init();
await core.initDatabase();
}
return bootInstance;
}
public openBrowserInDevelopment() {
if (process.env.NODE_ENV === 'development') {
opn('http://localhost:4242');
}
}
}

11
packages/core.gbapp/services/GBDeployer.ts
View file

@ -52,6 +52,7 @@ import { GuaribasInstance, GuaribasPackage } from '../models/GBModel';
import { KBService } from './../../kb.gbapp/services/KBService';
import { GBConfigService } from './GBConfigService';
import { GBImporter } from './GBImporterService';
import { GBVMService } from './GBVMService';
/** Deployer service for bots, themes, ai and more. */
export class GBDeployer {
@ -266,6 +267,10 @@ export class GBDeployer {
});
}
public deployScriptToStorage(instanceId: number, localPath: string) {
}
public deployTheme(localPath: string) {
// DISABLED: Until completed, "/ui/public".
// FsExtra.copy(localPath, this.workDir + packageName)
@ -297,14 +302,12 @@ export class GBDeployer {
break;
case '.gbdialog':
const vm = new VMService(this.core.sequelize);
const vm = new GBVMService();
return service.deployKb(this.core, this, localPath);
break;
default:
const err = GBError.create(
`GuaribasBusinessError: Unknow package type: ${packageType}.`,
`GuaribasBusinessError: Unknown package type: ${packageType}.`
);
Promise.reject(err);
break;

49
packages/core.gbapp/services/GBVMService.ts
View file

@ -36,13 +36,17 @@ import { IGBCoreService, IGBInstance } from 'botlib';
import { GBError } from 'botlib';
import { IGBPackage } from 'botlib';
const logger = require('../../../src/logger');
import * as fs from 'fs';
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import * as fs from 'fs';
import { Messages } from '../strings';
import { DialogClass } from './GBAPIService';
import { GBDeployer } from './GBDeployer';
const util = require('util');
const vm = require('vm');
import processExists = require('process-exists');
import { Sequelize } from 'sequelize-typescript';
const UrlJoin = require('url-join');
/**
* @fileoverview General Bots server core.
@ -50,50 +54,27 @@ const vm = require('vm');
export class GBVMService implements IGBCoreService {
public static setup(bot: BotAdapter, min: IGBInstance) {
private script = new vm.Script();
}
public loadJS(
public async loadJS(
filename: string,
min: IGBInstance,
core: IGBCoreService,
deployer: GBDeployer,
localPath: string
) {
): Promise<void> {
const sandbox = {
animal: 'cat',
count: 2,
};
const code = fs.readFileSync(UrlJoin(localPath, filename), 'utf8');
const sandbox = new DialogClass(min);
const script = new vm.Script('count += 1; name = "kitty";');
const context = vm.createContext(sandbox);
for (let i = 0; i < 10; ++i) {
script.runInContext(context);
}
this.script.runInContext(context);
console.log(util.inspect(sandbox));
// { animal: 'cat', count: 12, name: 'kitty' }
const packageType = Path.extname(localPath);
const packageName = Path.basename(localPath);
logger.info(`[GBDeployer] Opening package: ${localPath}`);
const packageObject = JSON.parse(
Fs.readFileSync(UrlJoin(localPath, 'package.json'), 'utf8'),
await deployer.deployScriptToStorage(
min.instanceId,
filename
);
const instance = await core.loadInstance(packageObject.botId);
logger.info(`[GBDeployer] Importing: ${localPath}`);
const p = await deployer.deployPackageToStorage(
instance.instanceId,
packageName,
);
await this.importKbPackage(localPath, p, instance);
deployer.rebuildIndex(instance);
logger.info(`[GBDeployer] Finished import of ${localPath}`);
logger.info(`[GBVMService] Finished loading of ${filename}`);
}
}

2
packages/core.gbapp/tests/vm.test.ts
View file

@ -43,7 +43,7 @@ describe('Load function', () => {
it('should fail on invalid file', () => {
try {
const service = new GBVMService();
service.loadJS('invalid.file');
service.loadJS('invalid.file', null, null, null, null);
} catch (error) {
expect(error).to.equal(0);
}

153
src/app.ts
View file

@ -40,14 +40,9 @@
const logger = require('./logger');
const express = require('express');
const bodyParser = require('body-parser');
const opn = require('opn');
import { IGBInstance, IGBPackage } from 'botlib';
import { GBAdminPackage } from '../packages/admin.gbapp/index';
import { GBAdminService } from '../packages/admin.gbapp/services/GBAdminService';
import { GBAnalyticsPackage } from '../packages/analytics.gblib';
import { AzureDeployerService } from '../packages/azuredeployer.gbapp/services/AzureDeployerService';
import { GBCorePackage } from '../packages/core.gbapp';
import { GuaribasInstance } from '../packages/core.gbapp/models/GBModel';
import { GBConfigService } from '../packages/core.gbapp/services/GBConfigService';
import { GBConversationalService } from '../packages/core.gbapp/services/GBConversationalService';
@ -55,10 +50,6 @@ import { GBCoreService } from '../packages/core.gbapp/services/GBCoreService';
import { GBDeployer } from '../packages/core.gbapp/services/GBDeployer';
import { GBImporter } from '../packages/core.gbapp/services/GBImporterService';
import { GBMinService } from '../packages/core.gbapp/services/GBMinService';
import { GBCustomerSatisfactionPackage } from '../packages/customer-satisfaction.gbapp';
import { GBKBPackage } from '../packages/kb.gbapp';
import { GBSecurityPackage } from '../packages/security.gblib';
import { GBWhatsappPackage } from './../packages/whatsapp.gblib/index';
const appPackages = new Array<IGBPackage>();
@ -66,13 +57,11 @@ const appPackages = new Array<IGBPackage>();
* General Bots open-core entry point.
*/
export class GBServer {
/**
* Program entry-point.
*/
public static run() {
logger.info(`The Bot Server is in STARTING mode...`);
// Creates a basic HTTP server that will serve several URL, one for each
@ -86,8 +75,8 @@ export class GBServer {
server.use(
bodyParser.urlencoded({
// to support URL-encoded bodies
extended: true
})
extended: true,
}),
);
let bootInstance: IGBInstance;
@ -106,133 +95,42 @@ export class GBServer {
logger.info(`Establishing a development local proxy (ngrok)...`);
const proxyAddress = await core.ensureProxy(port);
logger.info(`Deploying packages...`);
const deployer = new GBDeployer(core, new GBImporter(core));
const azureDeployer = new AzureDeployerService(deployer);
try {
await core.initDatabase();
} catch (error) {
logger.info(`Deploying cognitive infrastructure (on the cloud / on premises)...`);
try {
bootInstance = await azureDeployer.deployFarm(proxyAddress);
} catch (error) {
logger.warn(
'In case of error, please cleanup any infrastructure objects created during this procedure and .env before running again.'
);
throw error;
}
core.writeEnv(bootInstance);
logger.info(`File .env written, starting General Bots...`);
GBConfigService.init();
await core.initDatabase();
}
// TODO: Get .gb* templates from GitHub and download do additional deploy folder.
// Check admin password.
const conversationalService = new GBConversationalService(core);
const adminService = new GBAdminService(core);
const password = GBConfigService.get('ADMIN_PASS');
if (!GBAdminService.StrongRegex.test(password)) {
throw new Error(
'Please, define a really strong password in ADMIN_PASS environment variable before running the server.'
);
}
// NOTE: the semicolon is necessary before this line.
// Loads all system packages.
[
GBAdminPackage,
GBAnalyticsPackage,
GBCorePackage,
GBSecurityPackage,
GBKBPackage,
GBCustomerSatisfactionPackage,
GBWhatsappPackage
].forEach(e => {
logger.info(`Loading sys package: ${e.name}...`);
const p = Object.create(e.prototype) as IGBPackage;
p.loadPackage(core, core.sequelize);
});
// Loads all bot instances from object storage, if it's formatted.
logger.info(`Loading instances from storage...`);
let instances: GuaribasInstance[];
try {
instances = await core.loadInstances();
const instance = instances[0];
if (process.env.NODE_ENV === 'development') {
logger.info(`Updating bot endpoint to local reverse proxy (ngrok)...`);
await azureDeployer.updateBotProxy(
instance.botId,
instance.botId,
`${proxyAddress}/api/messages/${instance.botId}`
);
}
} catch (error) {
if (error.parent.code === 'ELOGIN') {
const group = GBConfigService.get('CLOUD_GROUP');
const serverName = GBConfigService.get('STORAGE_SERVER').split(
'.database.windows.net'
)[0];
await azureDeployer.openStorageFirewall(group, serverName);
} else {
// Check if storage is empty and needs formatting.
const isInvalidObject =
error.parent.number == 208 || error.parent.errno == 1; // MSSQL or SQLITE.
if (isInvalidObject) {
if (GBConfigService.get('STORAGE_SYNC') != 'true') {
throw new Error(`Operating storage is out of sync or there is a storage connection error. Try setting STORAGE_SYNC to true in .env file. Error: ${
error.message
}.`);
} else {
logger.info(
`Storage is empty. After collecting storage structure from all .gbapps it will get synced.`
);
}
} else {
throw new Error(`Cannot connect to operating storage: ${error.message}.`);
}
}
}
// Deploy packages and format object store according to .gbapp storage models.
logger.info(`Deploying packages...`);
const conversationalService = new GBConversationalService(core);
bootInstance = await core.createBootInstance(
core,
azureDeployer,
proxyAddress,
);
core.ensureAdminIsSecured();
core.loadSysPackages(core);
await deployer.deployPackages(core, server, appPackages);
// If instances is undefined here it's because storage has been formatted.
// Load all instances from .gbot found on deploy package directory.
if (!instances) {
const saveInstance = new GuaribasInstance(bootInstance);
await saveInstance.save();
instances = await core.loadInstances();
}
// Setup server dynamic (per bot instance) resources and listeners.
logger.info(`Publishing instances...`);
let instances: GuaribasInstance[] = await core.loadAllInstances(
core,
azureDeployer,
proxyAddress,
);
instances = await core.ensureInstances(
instances,
bootInstance,
core
);
const minService = new GBMinService(
core,
conversationalService,
adminService,
deployer
deployer,
);
await minService.buildMin(server, appPackages, instances);
logger.info(`The Bot Server is in RUNNING mode...`);
if (process.env.NODE_ENV === 'development') {
opn('http://localhost:4242');
}
logger.info(`The Bot Server is in RUNNING mode...`);
core.openBrowserInDevelopment();
return core;
} catch (err) {
@ -243,6 +141,7 @@ export class GBServer {
});
}
}
// First line to run.