fix(core.gbapp): Fixing loading of instances.

This commit is contained in:
Rodrigo Rodriguez 2019-05-14 23:02:21 -03:00
parent 5b320e5c04
commit bc9c58828b
10 changed files with 2992 additions and 2157 deletions

View file

@ -5,5 +5,4 @@
"arrowParens": "avoid",
"semi": true,
"singleQuote": true
}

View file

@ -5,12 +5,10 @@ node_js:
notifications:
email: false
before_script:
- npm run build
script:
- npm run coveralls
branches:
only:

4782
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -47,90 +47,90 @@
"commit": "git-cz"
},
"dependencies": {
"@microsoft/microsoft-graph-client": "1.5.2",
"@semantic-release/exec": "^3.3.2",
"@microsoft/microsoft-graph-client": "1.6.0",
"@semantic-release/exec": "3.3.2",
"adal-node": "0.1.28",
"async": "2.6.2",
"async-promises": "0.2.1",
"azure-arm-cognitiveservices": "2.4.1",
"async-promises": "0.2.2",
"azure-arm-cognitiveservices": "3.0.0",
"azure-arm-resource": "7.3.0",
"azure-arm-search": "^1.3.0-preview",
"azure-arm-sql": "5.6.0",
"azure-arm-search": "1.3.0-preview",
"azure-arm-sql": "5.7.0",
"azure-arm-website": "5.7.0",
"bluebird": "^3.5.4",
"body-parser": "1.18.3",
"botbuilder": "4.3.4",
"botbuilder-ai": "4.3.4",
"botbuilder-azure": "4.3.4",
"bluebird": "3.5.4",
"body-parser": "1.19.0",
"botbuilder": "4.4.0",
"botbuilder-ai": "4.4.0",
"botbuilder-azure": "4.4.0",
"botbuilder-choices": "4.0.0-preview1.2",
"botbuilder-dialogs": "4.3.4",
"botbuilder-dialogs": "4.4.0",
"botbuilder-prompts": "4.0.0-preview1.2",
"botlib": "0.1.24",
"botlib": "1.2.0",
"chai": "4.2.0",
"child_process": "^1.0.2",
"chokidar": "2.1.5",
"cli-spinner": "^0.2.10",
"csv-parse": "4.3.4",
"child_process": "1.0.2",
"chokidar": "3.0.0",
"cli-spinner": "0.2.10",
"csv-parse": "4.4.1",
"dotenv-extended": "2.4.0",
"express": "4.16.4",
"express-promise-router": "3.0.3",
"fs-extra": "7.0.1",
"ip": "^1.1.5",
"js-beautify": "^1.9.1",
"fs-extra": "8.0.0",
"ip": "1.1.5",
"js-beautify": "1.10.0",
"localize": "0.4.7",
"marked": "0.6.2",
"mocha": "6.0.2",
"mocha": "6.1.4",
"mocha-typescript": "1.1.17",
"ms": "2.1.1",
"ms-rest-azure": "2.6.0",
"ms-rest-js": "^1.0.1",
"ms-rest-js": "1.0.1",
"nexmo": "2.4.1",
"ngrok": "3.1.1",
"nyc": "13.3.0",
"nyc": "14.1.1",
"opn": "6.0.0",
"pragmatismo-io-framework": "1.0.19",
"process-exists": "3.1.0",
"public-ip": "^3.0.0",
"public-ip": "3.1.0",
"reflect-metadata": "0.1.13",
"request-promise": "4.2.4",
"request-promise-native": "1.0.7",
"scanf": "^1.0.2",
"sequelize": "^5.2.12",
"sequelize-typescript": "0.6.9",
"scanf": "1.0.2",
"sequelize": "5.8.6",
"sequelize-typescript": "0.6.10",
"shx": "0.3.2",
"simple-git": "1.110.0",
"sqlite3": "4.0.6",
"strict-password-generator": "^1.1.2",
"simple-git": "1.113.0",
"sqlite3": "4.0.8",
"strict-password-generator": "1.1.2",
"swagger-client": "3.8.25",
"tedious": "6.1.0",
"temperature-js": "^0.1.0",
"ts-node": "8.0.3",
"tedious": "6.1.1",
"temperature-js": "0.1.0",
"ts-node": "8.1.0",
"typedoc": "0.14.2",
"typedoc-plugin-external-module-name": "^2.0.0",
"typedoc-plugin-markdown": "^1.1.27",
"typescript": "3.4.1",
"typedoc-plugin-external-module-name": "2.1.0",
"typedoc-plugin-markdown": "1.2.0",
"typescript": "3.4.5",
"url-join": "4.0.0",
"vbscript-to-typescript": "^1.0.8",
"vbscript-to-typescript": "1.0.8",
"wait-until": "0.0.2",
"walk-promise": "0.2.0",
"winston": "3.2.1"
},
"devDependencies": {
"@semantic-release/changelog": "^3.0.2",
"@semantic-release/commit-analyzer": "^6.1.0",
"@semantic-release/git": "^7.0.8",
"@semantic-release/github": "^5.2.10",
"@semantic-release/npm": "^5.1.4",
"@semantic-release/release-notes-generator": "^7.1.4",
"@semantic-release/changelog": "3.0.2",
"@semantic-release/commit-analyzer": "6.1.0",
"@semantic-release/git": "7.0.8",
"@semantic-release/github": "5.2.10",
"@semantic-release/npm": "5.1.7",
"@semantic-release/release-notes-generator": "7.1.4",
"@types/chai": "4.1.7",
"@types/mocha": "5.2.6",
"@types/sequelize": "4.27.46",
"@types/sequelize": "4.27.49",
"@types/url-join": "4.0.0",
"@types/winston": "2.4.4",
"ban-sensitive-files": "1.9.2",
"commitizen": "^3.0.7",
"coveralls": "^3.0.3",
"cz-conventional-changelog": "^2.1.0",
"commitizen": "3.1.1",
"coveralls": "3.0.3",
"cz-conventional-changelog": "2.1.0",
"dependency-check": "3.3.0",
"deps-ok": "1.4.1",
"git-issues": "1.3.1",
@ -138,12 +138,12 @@
"nsp": "3.2.1",
"pre-git": "3.17.1",
"prettier-standard": "9.1.1",
"semantic-release": "^15.13.3",
"semantic-release": "15.13.12",
"standard": "12.0.1",
"travis-deploy-once": "5.0.11",
"ts-loader": "^5.3.3",
"tslint": "^5.15.0",
"tslint-microsoft-contrib": "^6.1.0"
"ts-loader": "6.0.0",
"tslint": "5.16.0",
"tslint-microsoft-contrib": "6.1.1"
},
"eslintConfig": {
"env": {

View file

@ -201,6 +201,37 @@ export class AzureDeployerService implements IGBInstallationDeployer {
};
}
public async botExists(botId, group, endpoint) {
const baseUrl = `https://management.azure.com/`;
const username = GBConfigService.get('CLOUD_USERNAME');
const password = GBConfigService.get('CLOUD_PASSWORD');
const subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID');
const accessToken = await GBAdminService.getADALTokenFromUsername(username, password);
const httpClient = new ServiceClient();
const parameters = {
properties: {
endpoint: endpoint
}
};
const query = `subscriptions/${subscriptionId}/resourceGroups/${group}/providers/${
this.provider
}/botServices/${botId}?api-version=${this.apiVersion}`;
const url = urlJoin(baseUrl, query);
const req = AzureDeployerService.createRequestObject(url, accessToken, 'GET', JSON.stringify(parameters));
const res = await httpClient.sendRequest(req);
// CHECK
if (!JSON.parse(res.bodyAsText).id) {
return false;
}
else {
return true;
}
}
public async updateBotProxy(botId, group, endpoint) {
const baseUrl = `https://management.azure.com/`;
const username = GBConfigService.get('CLOUD_USERNAME');
@ -229,6 +260,39 @@ export class AzureDeployerService implements IGBInstallationDeployer {
GBLog.info(`Bot proxy updated at: ${endpoint}.`);
}
public async updateBot(botId: string, group: string, name: string,
description: string, endpoint: string, iconUrl: string) {
const baseUrl = `https://management.azure.com/`;
const username = GBConfigService.get('CLOUD_USERNAME');
const password = GBConfigService.get('CLOUD_PASSWORD');
const subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID');
const accessToken = await GBAdminService.getADALTokenFromUsername(username, password);
const httpClient = new ServiceClient();
const parameters = {
properties: {
description: description,
displayName: name,
endpoint: endpoint,
iconUrl: iconUrl
}
};
const query = `subscriptions/${subscriptionId}/resourceGroups/${group}/providers/${
this.provider
}/botServices/${botId}?api-version=${this.apiVersion}`;
const url = urlJoin(baseUrl, query);
const req = AzureDeployerService.createRequestObject(url, accessToken, 'PATCH', JSON.stringify(parameters));
const res = await httpClient.sendRequest(req);
// CHECK
if (!JSON.parse(res.bodyAsText).id) {
throw res.bodyAsText;
}
GBLog.info(`Bot proxy updated at: ${endpoint}.`);
}
public async openStorageFirewall(groupName, serverName) {
const username = GBConfigService.get('CLOUD_USERNAME');
const password = GBConfigService.get('CLOUD_PASSWORD');
@ -296,7 +360,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
instance.searchIndex = 'azuresql-index';
instance.searchIndexer = 'azuresql-indexer';
instance.searchKey = searchKeys.primaryKey;
this.deployer.rebuildIndex(instance, this.deployer);
this.deployer.rebuildIndex(instance, this.getKBSearchSchema(instance.searchIndex));
GBLog.info(`Deploying Speech...`);
const speech = await this.createSpeech(name, `${name}-speech`, instance.cloudLocation);
@ -378,43 +442,10 @@ export class AzureDeployerService implements IGBInstallationDeployer {
this.deployFarm(url, instance, credentials, subscriptionId);
}
private initServices(credentials: any, subscriptionId: string) {
this.resourceClient = new ResourceManagementClient.default(credentials, subscriptionId);
this.webSiteClient = new WebSiteManagementClient(credentials, subscriptionId);
this.storageClient = new SqlManagementClient(credentials, subscriptionId);
this.cognitiveClient = new CognitiveServicesManagementClient(credentials, subscriptionId);
this.searchClient = new SearchManagementClient(credentials, subscriptionId);
this.accessToken = credentials.tokenCache._entries[0].accessToken;
}
private async createStorageServer(group, name, administratorLogin, administratorPassword, serverName, location) {
const params = {
location: location,
administratorLogin: administratorLogin,
administratorLoginPassword: administratorPassword,
fullyQualifiedDomainName: `${serverName}.database.windows.net`
};
return this.storageClient.servers.createOrUpdate(group, name, params);
}
private async registerProviders(subscriptionId, baseUrl, accessToken) {
const query = `subscriptions/${subscriptionId}/providers/${this.provider}/register?api-version=2018-02-01`;
const requestUrl = urlJoin(baseUrl, query);
const req = new WebResource();
req.method = 'POST';
req.url = requestUrl;
req.headers = <any>{};
req.headers['Content-Type'] = 'application/json; charset=utf-8';
req.headers['accept-language'] = '*';
(req.headers as any).Authorization = `Bearer ${accessToken}`;
}
/**
* @see https://github.com/Azure/azure-rest-api-specs/blob/master/specification/botservice/resource-manager/Microsoft.BotService/preview/2017-12-01/botservice.json
*/
private async internalDeployBot(
public async internalDeployBot(
instance,
accessToken,
botId,
@ -490,6 +521,39 @@ export class AzureDeployerService implements IGBInstallationDeployer {
});
}
private initServices(credentials: any, subscriptionId: string) {
this.resourceClient = new ResourceManagementClient.default(credentials, subscriptionId);
this.webSiteClient = new WebSiteManagementClient(credentials, subscriptionId);
this.storageClient = new SqlManagementClient(credentials, subscriptionId);
this.cognitiveClient = new CognitiveServicesManagementClient(credentials, subscriptionId);
this.searchClient = new SearchManagementClient(credentials, subscriptionId);
this.accessToken = credentials.tokenCache._entries[0].accessToken;
}
private async createStorageServer(group, name, administratorLogin, administratorPassword, serverName, location) {
const params = {
location: location,
administratorLogin: administratorLogin,
administratorLoginPassword: administratorPassword,
fullyQualifiedDomainName: `${serverName}.database.windows.net`
};
return this.storageClient.servers.createOrUpdate(group, name, params);
}
private async registerProviders(subscriptionId, baseUrl, accessToken) {
const query = `subscriptions/${subscriptionId}/providers/${this.provider}/register?api-version=2018-02-01`;
const requestUrl = urlJoin(baseUrl, query);
const req = new WebResource();
req.method = 'POST';
req.url = requestUrl;
req.headers = <any>{};
req.headers['Content-Type'] = 'application/json; charset=utf-8';
req.headers['accept-language'] = '*';
(req.headers as any).Authorization = `Bearer ${accessToken}`;
}
private async createNLPService(
name: string,
description: string,

View file

@ -102,6 +102,9 @@ export class GBConfigService {
case 'STORAGE_STORAGE':
value = './guaribas.sqlite';
break;
case 'GBKB_AUTO_DEPLOY':
value = false;
break;
case 'ADDITIONAL_DEPLOY_PATH':
value = undefined;
break;

View file

@ -46,8 +46,10 @@ const graph = require('@microsoft/microsoft-graph-client');
import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBInstance, IGBPackage } from 'botlib';
import { AzureSearch } from 'pragmatismo-io-framework';
import { GBServer } from '../../../src/app';
import { GuaribasPackage } from '../models/GBModel';
import { GBAdminService } from './../../admin.gbapp/services/GBAdminService';
import { AzureDeployerService } from './../../azuredeployer.gbapp/services/AzureDeployerService';
import { KBService } from './../../kb.gbapp/services/KBService';
import { GBConfigService } from './GBConfigService';
import { GBImporter } from './GBImporterService';
@ -70,9 +72,13 @@ export class GBDeployer {
}
public static getConnectionStringFromInstance(instance: IGBInstance) {
return `Server=tcp:torageServer}.database.windows.net,1433;Database=${instance.storageName};User ID=${
return `Server=tcp:${instance.storageServer}.database.windows.net,1433;Database=${
instance.storageName
};User ID=${
instance.storageUsername
};Password=${instance.storagePassword};Trusted_Connection=False;Encrypt=True;Connection Timeout=30;`;
};Password=${
instance.storagePassword
};Trusted_Connection=False;Encrypt=True;Connection Timeout=30;`;
}
/**
@ -139,6 +145,7 @@ export class GBDeployer {
GBLog.info(`App Package deployment done.`);
({ generalPackages, totalPackages } = await this.deployDataPackages(
core,
botPackages,
_this,
@ -157,10 +164,49 @@ export class GBDeployer {
* Deploys a bot to the storage.
*/
public async deployBot(localPath: string): Promise<IGBInstance> {
public async deployBot(localPath: string, proxyAddress: string): Promise<IGBInstance> {
const packageName = Path.basename(localPath);
return await this.importer.importIfNotExistsBotPackage(undefined, packageName, localPath);
const service = new AzureDeployerService(this);
let instance = await this.importer.importIfNotExistsBotPackage(undefined, packageName, localPath);
const username = GBConfigService.get('CLOUD_USERNAME');
const password = GBConfigService.get('CLOUD_PASSWORD');
const group = GBConfigService.get('CLOUD_GROUP');
const subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID');
const accessToken = await GBAdminService.getADALTokenFromUsername(username, password);
if (await service.botExists(instance.botId, group, proxyAddress)) {
instance = await service.updateBot(
instance,
accessToken,
instance.title,
instance.description,
proxyAddress,
''
);
}
else {
instance = await service.internalDeployBot(
instance,
accessToken,
instance.botId,
instance.title,
group,
instance.description,
`${proxyAddress}/api/messages/${instance.botId}`,
'global',
instance.nlpAppId,
instance.nlpKey,
instance.marketplaceId,
instance.marketplacePassword,
subscriptionId
);
}
return instance;
}
public async deployPackageToStorage(instanceId: number, packageName: string): Promise<GuaribasPackage> {
@ -188,7 +234,7 @@ export class GBDeployer {
switch (packageType) {
case '.gbot':
return this.deployBot(localPath);
return this.deployBot(localPath, min.proxyAddress);
case '.gbkb':
const service = new KBService(this.core.sequelize);
@ -284,6 +330,7 @@ export class GBDeployer {
}
private async deployDataPackages(
core: IGBCoreService,
botPackages: string[],
_this: this,
@ -304,7 +351,7 @@ export class GBDeployer {
botPackages.forEach(e => {
if (e !== 'packages\\boot.gbot') {
GBLog.info(`Deploying bot: ${e}...`);
_this.deployBot(e);
_this.deployBot(e, GBServer.globals.proxyAddress);
GBLog.info(`Bot: ${e} deployed...`);
}
});

View file

@ -36,7 +36,7 @@
'use strict';
import { IGBCoreService } from 'botlib';
import { IGBCoreService, IGBInstance } from 'botlib';
import fs = require('fs');
import urlJoin = require('url-join');
import { GuaribasInstance } from '../models/GBModel';
@ -57,14 +57,12 @@ export class GBImporter {
botId = packageJson.botId;
}
const instance = await this.core.loadInstance(botId);
if (instance !== null) {
return instance;
} else {
return await this.createInstanceInternal(botId, localPath, packageJson);
}
return await this.createOrUpdateInstanceInternal(instance, botId, localPath, packageJson);
}
private async createInstanceInternal(botId: string, localPath: string, packageJson: any) {
private async createOrUpdateInstanceInternal(instance: IGBInstance,
botId: string, localPath: string, packageJson: any) {
const settings = JSON.parse(fs.readFileSync(urlJoin(localPath, 'settings.json'), 'utf8'));
const servicesJson = JSON.parse(fs.readFileSync(urlJoin(localPath, 'services.json'), 'utf8'));
@ -74,6 +72,12 @@ export class GBImporter {
packageJson.botId = botId;
}
return GuaribasInstance.create(packageJson);
if (instance !== null) {
instance = { ...instance, ...packageJson, ...settings, ...servicesJson };
return this.core.saveInstance(instance);
} else {
return GuaribasInstance.create(packageJson);
}
}
}

View file

@ -112,7 +112,8 @@ export class GBMinService {
server: any,
appPackages: IGBPackage[],
instances: IGBInstance[],
deployer: GBDeployer
deployer: GBDeployer,
proxyAddress: string
) {
// Serves default UI on root address '/'.
@ -136,11 +137,11 @@ export class GBMinService {
// Build bot adapter.
const { min, adapter, conversationState } = await this.buildBotAdapter(instance);
const { min, adapter, conversationState } = await this.buildBotAdapter(instance, proxyAddress);
// Install default VBA module.
// DISABLED: deployer.deployPackage(min, 'packages/default.gbdialog');
// DISABLED: deployer.deployPackage(min, 'packages/default.gbdialog');
// Call the loadBot context.activity for all packages.
@ -220,7 +221,7 @@ export class GBMinService {
);
authorizationUrl = `${authorizationUrl}?response_type=code&client_id=${
min.instance.authenticatorClientId
}&redirect_uri=${urlJoin(min.instance.botEndpoint, min.instance.botId, 'token')}`;
}&redirect_uri=${urlJoin(min.instance.botEndpoint, min.instance.botId, 'token')}`;
res.redirect(authorizationUrl);
});
}
@ -281,7 +282,7 @@ export class GBMinService {
} catch (error) {
const msg = `[botId:${
instance.botId
}] Error calling Direct Line client, verify Bot endpoint on the cloud. Error is: ${error}.`;
}] Error calling Direct Line client, verify Bot endpoint on the cloud. Error is: ${error}.`;
return Promise.reject(new Error(msg));
}
@ -312,7 +313,7 @@ export class GBMinService {
}
}
private async buildBotAdapter(instance: any) {
private async buildBotAdapter(instance: any, proxyAddress: string) {
const adapter = new BotFrameworkAdapter({
appId: instance.marketplaceId,
appPassword: instance.marketplacePassword
@ -346,7 +347,7 @@ export class GBMinService {
}
private invokeLoadBot(appPackages: any[], min: GBMinInstance, server: any) {
const sysPackages : IGBPackage[] = [];
const sysPackages: IGBPackage[] = [];
// NOTE: A semicolon is necessary before this line.
[
GBCorePackage,
@ -366,7 +367,7 @@ export class GBMinService {
(p as any).channel.received(req, res);
});
}
}, this);
}, this);
appPackages.forEach(p => {
p.sysPackages = sysPackages;
@ -377,7 +378,7 @@ export class GBMinService {
min.dialogs.add(new WaterfallDialog(dialog.name, dialog.waterfall));
});
}
}, this);
}, this);
}
/**
@ -415,7 +416,7 @@ export class GBMinService {
GBLog.info(
`User>: ${context.activity.text} (${context.activity.type}, ${context.activity.name}, ${
context.activity.channelId
context.activity.channelId
}, {context.activity.value})`
);
if (context.activity.type === 'conversationUpdate' && context.activity.membersAdded.length > 0) {

View file

@ -52,10 +52,20 @@ import { GBMinService } from '../packages/core.gbapp/services/GBMinService';
const appPackages: IGBPackage[] = [];
/**
* Global shared server data;
*/
export class RootData {
public proxyAddress: string;
}
/**
* General Bots open-core entry point.
*/
export class GBServer {
public static globals: RootData;
/**
* Program entry-point.
*/
@ -66,6 +76,7 @@ export class GBServer {
// Creates a basic HTTP server that will serve several URL, one for each
// bot instance.
GBServer.globals = new RootData();
const port = GBConfigService.getServerPort();
const server = express();
server.use(bodyParser.json());
@ -94,7 +105,7 @@ export class GBServer {
// Ensure that local development proxy is setup.
GBLog.info(`Establishing a development local proxy (ngrok)...`);
const proxyAddress: string = await core.ensureProxy(port);
GBServer.globals.proxyAddress = await core.ensureProxy(port);
// Creates a boot instance or load it from storage.
@ -103,7 +114,7 @@ export class GBServer {
await core.initStorage();
} catch (error) {
GBLog.verbose(`Error initializing storage: ${error}`);
bootInstance = await core.createBootInstance(core, azureDeployer, proxyAddress);
bootInstance = await core.createBootInstance(core, azureDeployer, GBServer.globals.proxyAddress);
await core.initStorage();
}
@ -116,7 +127,7 @@ export class GBServer {
await core.checkStorage(azureDeployer);
await deployer.deployPackages(core, server, appPackages);
// Loads all bot instances.
// Loads boot bot and other instances.
GBLog.info(`Publishing instances...`);
const packageInstance = await importer.importIfNotExistsBotPackage(
@ -130,7 +141,8 @@ export class GBServer {
// tslint:disable-next-line:prefer-object-spread
const fullInstance = Object.assign(packageInstance, bootInstance);
await core.saveInstance(fullInstance);
let instances: IGBInstance[] = await core.loadAllInstances(core, azureDeployer, proxyAddress);
let instances: IGBInstance[] = await core.loadAllInstances(core, azureDeployer,
GBServer.globals.proxyAddress);
instances = await core.ensureInstances(instances, bootInstance, core);
if (bootInstance !== undefined) {
bootInstance = instances[0];
@ -139,7 +151,8 @@ export class GBServer {
// Builds minimal service infrastructure.
const minService: GBMinService = new GBMinService(core, conversationalService, adminService, deployer);
await minService.buildMin(bootInstance, server, appPackages, instances, deployer);
await minService.buildMin(bootInstance, server, appPackages, instances,
deployer, GBServer.globals.proxyAddress);
// Deployment of local applications for the first time.