From 7ef4e22764584e9a58b20702a108e10f83affa1c Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (pragmatismo.io)" Date: Sun, 14 Oct 2018 19:58:54 -0300 Subject: [PATCH] New tasks on Azure Deployer and start of Bot Farm deployer. --- deploy/admin.gbapp/dialogs/AdminDialog.ts | 10 +- .../dialogs/BotFarmDialog.ts | 65 +++ .../index.ts | 4 - .../services/AzureDeployerService.ts | 457 ++++++++++++++++++ deploy/azuredeployer.gbapp/strings.ts | 22 + .../services/AzureDeployerService.ts | 304 ------------ deploy/core.gbapp/models/GBModel.ts | 38 +- .../services/GBConversationalService.ts | 4 +- deploy/core.gbapp/services/GBCoreService.ts | 182 ++++--- deploy/core.gbapp/services/GBMinService.ts | 42 +- package.json | 2 + src/app.ts | 10 +- 12 files changed, 717 insertions(+), 423 deletions(-) create mode 100644 deploy/azuredeployer.gbapp/dialogs/BotFarmDialog.ts rename deploy/{azuredeployer.gblib => azuredeployer.gbapp}/index.ts (97%) create mode 100644 deploy/azuredeployer.gbapp/services/AzureDeployerService.ts create mode 100644 deploy/azuredeployer.gbapp/strings.ts delete mode 100644 deploy/azuredeployer.gblib/services/AzureDeployerService.ts diff --git a/deploy/admin.gbapp/dialogs/AdminDialog.ts b/deploy/admin.gbapp/dialogs/AdminDialog.ts index 45a86f64..8544e300 100644 --- a/deploy/admin.gbapp/dialogs/AdminDialog.ts +++ b/deploy/admin.gbapp/dialogs/AdminDialog.ts @@ -33,21 +33,24 @@ "use strict"; const UrlJoin = require("url-join"); -import { AzureSearch } from "pragmatismo-io-framework"; import { GBMinInstance } from "botlib"; import { IGBDialog } from "botlib"; import { GBDeployer } from "../../core.gbapp/services/GBDeployer"; import { GBImporter } from "../../core.gbapp/services/GBImporter"; import { GBConfigService } from "../../core.gbapp/services/GBConfigService"; -import { KBService } from "./../../kb.gbapp/services/KBService"; import { BotAdapter } from "botbuilder"; import { GBAdminService } from "../services/GBAdminService"; import { Messages } from "../strings"; + /** * Dialogs for administration tasks. */ export class AdminDialog extends IGBDialog { + + static async createFarmCommand(text: any, min: GBMinInstance) { + } + static async undeployPackageCommand(text: any, min: GBMinInstance) { let packageName = text.split(" ")[1]; let importer = new GBImporter(min.core); @@ -106,6 +109,9 @@ export class AdminDialog extends IGBDialog { if (text === "quit") { await dc.replace("/"); + } else if (cmdName === "createFarm") { + await AdminDialog.createFarmCommand(text, deployer); + await dc.replace("/admin", { firstRun: false }); } else if (cmdName === "deployPackage") { await AdminDialog.deployPackageCommand(text, deployer); await dc.replace("/admin", { firstRun: false }); diff --git a/deploy/azuredeployer.gbapp/dialogs/BotFarmDialog.ts b/deploy/azuredeployer.gbapp/dialogs/BotFarmDialog.ts new file mode 100644 index 00000000..cd84084b --- /dev/null +++ b/deploy/azuredeployer.gbapp/dialogs/BotFarmDialog.ts @@ -0,0 +1,65 @@ +/*****************************************************************************\ +| ( )_ _ | +| _ _ _ __ _ _ __ ___ ___ _ _ | ,_)(_) ___ ___ _ | +| ( '_`\ ( '__)/'_` ) /'_ `\/' _ ` _ `\ /'_` )| | | |/',__)/' _ `\ /'_`\ | +| | (_) )| | ( (_| |( (_) || ( ) ( ) |( (_| || |_ | |\__, \| ( ) |( (_) ) | +| | ,__/'(_) `\__,_)`\__ |(_) (_) (_)`\__,_)`\__)(_)(____/(_) (_)`\___/' | +| | | ( )_) | | +| (_) \___/' | +| | +| General Bots Copyright (c) Pragmatismo.io. All rights reserved. | +| Licensed under the AGPL-3.0. | +| | +| According to our dual licensing model, this program can be used either | +| under the terms of the GNU Affero General Public License, version 3, | +| or under a proprietary license. | +| | +| The texts of the GNU Affero General Public License with an additional | +| permission and of our proprietary license can be found at and | +| in the LICENSE file you have received along with this program. | +| | +| This program is distributed in the hope that it will be useful, | +| but WITHOUT ANY WARRANTY, without even the implied warranty of | +| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | +| GNU Affero General Public License for more details. | +| | +| "General Bots" is a registered trademark of Pragmatismo.io. | +| The licensing of the program under the AGPLv3 does not imply a | +| trademark license. Therefore any rights, title and interest in | +| our trademarks remain entirely with us. | +| | +\*****************************************************************************/ + +"use strict"; + +import { GBMinInstance } from "botlib"; +import { IGBDialog } from "botlib"; +import { BotAdapter } from "botbuilder"; +import { Messages } from "../strings"; + +export class BotFarmDialog extends IGBDialog { + /** + * Setup dialogs flows and define services call. + * + * @param bot The bot adapter. + * @param min The minimal bot instance data. + */ + static setup(bot: BotAdapter, min: GBMinInstance) { + min.dialogs.add("/createBotFarm", [ + async dc => { + let locale = dc.context.activity.locale; + await dc.prompt("choicePrompt", Messages[locale].what_about_me, [ + "1", + "2", + "3", + "4", + "5" + ]); + }, + async (dc, value) => { + let locale = dc.context.activity.locale; + await dc.context.sendActivity(Messages[locale].thanks); + } + ]); + } +} diff --git a/deploy/azuredeployer.gblib/index.ts b/deploy/azuredeployer.gbapp/index.ts similarity index 97% rename from deploy/azuredeployer.gblib/index.ts rename to deploy/azuredeployer.gbapp/index.ts index 29706a09..2acf2d22 100644 --- a/deploy/azuredeployer.gblib/index.ts +++ b/deploy/azuredeployer.gbapp/index.ts @@ -33,12 +33,8 @@ "use strict" const UrlJoin = require("url-join") - - import { GBMinInstance, IGBPackage, IGBCoreService } from "botlib" - import { Sequelize } from "sequelize-typescript" -import { AzureDeployerService } from "./services/AzureDeployerService" export class GBWhatsappPackage implements IGBPackage { diff --git a/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts b/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts new file mode 100644 index 00000000..30208535 --- /dev/null +++ b/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts @@ -0,0 +1,457 @@ +/*****************************************************************************\ +| ( )_ _ | +| _ _ _ __ _ _ __ ___ ___ _ _ | ,_)(_) ___ ___ _ | +| ( '_`\ ( '__)/'_` ) /'_ `\/' _ ` _ `\ /'_` )| | | |/',__)/' _ `\ /'_`\ | +| | (_) )| | ( (_| |( (_) || ( ) ( ) |( (_| || |_ | |\__, \| ( ) |( (_) ) | +| | ,__/'(_) `\__,_)`\__ |(_) (_) (_)`\__,_)`\__)(_)(____/(_) (_)`\___/' | +| | | ( )_) | | +| (_) \___/' | +| | +| General Bots Copyright (c) Pragmatismo.io. All rights reserved. | +| Licensed under the AGPL-3.0. | +| | +| According to our dual licensing model, this program can be used either | +| under the terms of the GNU Affero General Public License, version 3, | +| or under a proprietary license. | +| | +| The texts of the GNU Affero General Public License with an additional | +| permission and of our proprietary license can be found at and | +| in the LICENSE file you have received along with this program. | +| | +| This program is distributed in the hope that it will be useful, | +| but WITHOUT ANY WARRANTY, without even the implied warranty of | +| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | +| GNU Affero General Public License for more details. | +| | +| "General Bots" is a registered trademark of Pragmatismo.io. | +| The licensing of the program under the AGPLv3 does not imply a | +| trademark license. Therefore any rights, title and interest in | +| our trademarks remain entirely with us. | +| | +\*****************************************************************************/ + +"use strict"; + +import { GBService, IGBInstance } from "botlib"; +const msRestAzure = require("ms-rest-azure"); +import { + ResourceManagementClient, + SubscriptionClient +} from "azure-arm-resource"; +import { WebSiteManagementClient } from "azure-arm-website"; +import { SqlManagementClient } from "azure-arm-sql"; +import { CognitiveServicesManagementClient } from "azure-arm-cognitiveservices"; +import { CognitiveServicesAccount } from "azure-arm-cognitiveservices/lib/models"; +import { SearchManagementClient } from "azure-arm-search"; +import { WebResource, ServiceClient } from "ms-rest-js"; +import * as simplegit from "simple-git/promise"; +import { AppServicePlan } from "azure-arm-website/lib/models"; +const git = simplegit(); +const logger = require("../../../src/logger"); +const UrlJoin = require("url-join"); +const PasswordGenerator = require("strict-password-generator").default; + +export class AzureDeployerService extends GBService { + instance: IGBInstance; + resourceClient: ResourceManagementClient.ResourceManagementClient; + webSiteClient: WebSiteManagementClient; + storageClient: SqlManagementClient; + cognitiveClient: CognitiveServicesManagementClient; + searchClient: SearchManagementClient; + provider = "Microsoft.BotService"; + subscriptionClient: SubscriptionClient.SubscriptionClient; + + constructor(credentials, subscriptionId) { + super(); + this.resourceClient = new ResourceManagementClient.default( + credentials, + subscriptionId + ); + this.webSiteClient = new WebSiteManagementClient( + credentials, + subscriptionId + ); + this.storageClient = new SqlManagementClient(credentials, subscriptionId); + this.cognitiveClient = new CognitiveServicesManagementClient( + credentials, + subscriptionId + ); + this.searchClient = new SearchManagementClient(credentials, subscriptionId); + this.subscriptionClient = new SubscriptionClient.default(credentials); + } + + public async getSubscriptions() { + this.subscriptionClient.subscriptions.list(); + } + + public async deploy( + instance: IGBInstance, + location: string + ): Promise { + + logger.info(`Creating Deploy...`); + await this.createDeploy(name, location); + + logger.info(`Creating Server...`); + let serverFarm = await this.createHostingPlan( + name, + `${name}-server-plan`, + location + ); + await this.createServer(serverFarm.id, name, `${name}-server`, location); + + let administratorLogin = AzureDeployerService.getRndAdminAccount(); + let administratorPassword = AzureDeployerService.getRndPassword(); + + logger.info(`Creating Storage...`); + let storageServerName = `${name}-storage`; + await this.createStorageServer( + name, + `${storageServerName}-server`, + administratorLogin, + administratorPassword, + storageServerName, + location + ); + + await this.createStorage( + name, + storageServerName, + `${name}-storage`, + location + ); + instance.storageUsername = administratorLogin; + instance.storagePassword = administratorPassword; + instance.storageName = storageServerName; + instance.storageDialect = "mssql"; + instance.storageServerName = storageServerName; + + logger.info(`Creating Search...`); + let search = await this.createSearch(name, `${name}-search`, location); + + logger.info(`Creating Bot...`); + //await this.createBot(credentials.tokenCache._entries[0].accessToken, + // name, name, name, 'global', subscriptionId, tenantId); + + logger.info(`Creating NLP...`); + let nlp = await this.createNLP(name, `${name}-nlp`, location); + let keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name); + instance.nlpEndpoint = nlp.endpoint; + instance.nlpKey = keys.key1; + + logger.info(`Creating Speech...`); + let speech = await this.createSpeech(name, `${name}-speech`, location); + keys = await this.cognitiveClient.accounts.listKeys(name, speech.name); + instance.speechKeyEndpoint = speech.endpoint; + instance.speechKey = keys.key1; + + logger.info(`Creating SpellChecker...`); + let spellChecker = await this.createSpellChecker( + name, + `${name}-spellchecker`, + location + ); + keys = await this.cognitiveClient.accounts.listKeys( + name, + spellChecker.name + ); + instance.spellCheckerKey = keys.key1; + instance.spellCheckerEndpoint = spellChecker.endpoint; + + logger.info(`Creating Text Analytics...`); + let textAnalytics = await this.createTextAnalytics( + name, + `${name}-textanalytics`, + location + ); + keys = await this.cognitiveClient.accounts.listKeys( + name, + textAnalytics.name + ); + instance.textAnalyticsServerUrl = textAnalytics.endpoint; + instance.textAnalyticsKey = keys.key1; + + logger.info(`Cleaning Deploy it can take a while...`); + // DISABLED: await this.dangerouslyDeleteDeploy(name); + } + + private async dangerouslyDeleteDeploy(name) { + return this.resourceClient.resourceGroups.deleteMethod(name); + } + + private async createStorageServer( + group, + name, + administratorLogin, + administratorPassword, + serverName, + location + ) { + var params = { + location: location, + administratorLogin: administratorLogin, + administratorLoginPassword: administratorPassword, + fullyQualifiedDomainName: `${serverName}.database.windows.net` + }; + + return this.storageClient.servers.createOrUpdate(group, name, params); + } + + private async registerProviders(subscriptionId, baseUrl, accessToken) { + let query = `subscriptions/${subscriptionId}/providers/${ + this.provider + }/register?api-version=2018-02-01`; + let requestUrl = UrlJoin(baseUrl, query); + + let req = new WebResource(); + req.method = "POST"; + req.url = requestUrl; + req.headers = {}; + req.headers["Content-Type"] = "application/json; charset=utf-8"; + req.headers["accept-language"] = "*"; + req.headers["x-ms-client-request-id"] = msRestAzure.generateUuid(); + req.headers["Authorization"] = "Bearer " + accessToken; + + let httpClient = new ServiceClient(); + let res = await httpClient.sendRequest(req); + } + + private async createBot( + accessToken, + botId, + group, + name, + location, + subscriptionId, + tenantId + ) { + let baseUrl = `https://management.azure.com/`; + let appId = ""; + let description = ""; + let endpoint = ""; + let nlpKey = ""; + let nlpAppId = "3"; + + let parameters = { + parameters: { + location: location, + sku: { + name: "F0" + }, + name: name, + //"type": "sampletype", + id: botId, + kind: "sdk", + properties: { + description: description, + displayName: name, + endpoint: endpoint, + iconUrl: "http://myicon", + luisAppIds: [nlpAppId], + luisKey: nlpKey, + msaAppId: appId + } + } + }; + + let query = `subscriptions/${subscriptionId}/resourceGroups/${group}/providers/${ + this.provider + }/botServices/${botId}?api-version=2017-12-01`; + let requestUrl = UrlJoin(baseUrl, query); + + let req = new WebResource(); + req.method = "PUT"; + req.url = requestUrl; + req.headers = {}; + req.headers["Content-Type"] = "application/json"; + req.headers["accept-language"] = "*"; + //req.headers['x-ms-client-request-id'] = msRestAzure.generateUuid(); + req.headers["Authorization"] = "Bearer " + accessToken; + + let requestContent = JSON.stringify(parameters); + req.body = requestContent; + + let httpClient = new ServiceClient(); + let res = await httpClient.sendRequest(req); + } + + private async createSearch(group, name, location) { + var params = { + sku: { name: "free" }, + location: location + }; + + return this.searchClient.services.createOrUpdate(group, name, params); + } + + private async createStorage(group, serverName, name, location) { + var params = { + sku: { name: "Free" }, + createMode: "Default", + location: location + }; + + return this.storageClient.databases.createOrUpdate( + group, + serverName, + name, + params + ); + } + + private async createCognitiveServices( + group, + name, + location, + kind + ): Promise { + // * 'Bing.Autosuggest.v7', 'Bing.CustomSearch', + // * 'Bing.Search.v7', 'Bing.Speech', 'Bing.SpellCheck.v7', 'ComputerVision', + // * 'ContentModerator', 'CustomSpeech', 'CustomVision.Prediction', + // * 'CustomVision.Training', 'Emotion', 'Face', 'LUIS', 'QnAMaker', + // * 'SpeakerRecognition', 'SpeechTranslation', 'TextAnalytics', + // * 'TextTranslation', 'WebLM' + + let params = { + sku: { name: "F0" }, + createMode: "Default", + location: location, + kind: kind, + properties: {} + }; + + return await this.cognitiveClient.accounts.create(group, name, params); + } + + private async createSpeech( + group, + name, + location + ): Promise { + return await this.createCognitiveServices( + group, + name, + location, + "SpeechServices" + ); + } + + private async createNLP( + group, + name, + location + ): Promise { + return await this.createCognitiveServices(group, name, location, "LUIS"); + } + + private async createSpellChecker( + group, + name, + location + ): Promise { + return await this.createCognitiveServices( + group, + name, + "global", + "Bing.SpellCheck.v7" + ); + } + + private async createTextAnalytics( + group, + name, + location + ): Promise { + return await this.createCognitiveServices( + group, + name, + location, + "TextAnalytics" + ); + } + + private async createDeploy(name, location) { + var params = { location: location }; + return this.resourceClient.resourceGroups.createOrUpdate(name, params); + } + + private async createHostingPlan( + group, + name, + location + ): Promise { + let params = { + serverFarmWithRichSkuName: name, + location: location, + sku: { + name: "F1", + capacity: 1, + tier: "Free" + } + }; + + return this.webSiteClient.appServicePlans.createOrUpdate( + group, + name, + params + ); + } + + private async createServer(farmId, group, name, location) { + var parameters = { + location: location, + serverFarmId: farmId + }; + return this.webSiteClient.webApps.createOrUpdate(group, name, parameters); + } + + private async updateWebisteConfig(group, serverFarmId, name, location) { + var siteConfig = { + location: location, + serverFarmId: serverFarmId, + numberOfWorkers: 1, + phpVersion: "5.5" + }; + return this.webSiteClient.webApps.createOrUpdateConfiguration( + group, + name, + siteConfig + ); + } + + private deleteDeploy(name) { + return this.resourceClient.resourceGroups.deleteMethod(name); + } + + async deployGeneralBotsToAzure() { + let status = await git.status(); + } + + private static getRndAdminAccount() { + const passwordGenerator = new PasswordGenerator(); + const options = { + upperCaseAlpha: true, + lowerCaseAlpha: true, + number: true, + specialCharacter: true, + minimumLength: 8, + maximumLength: 8 + }; + let password = passwordGenerator.generatePassword(options); + return `sa${password}`; + } + + private static getRndPassword() { + const passwordGenerator = new PasswordGenerator(); + const options = { + upperCaseAlpha: true, + lowerCaseAlpha: true, + number: true, + specialCharacter: true, + minimumLength: 8, + maximumLength: 8 + }; + let password = passwordGenerator.generatePassword(options); + return password; + } +} diff --git a/deploy/azuredeployer.gbapp/strings.ts b/deploy/azuredeployer.gbapp/strings.ts new file mode 100644 index 00000000..35481519 --- /dev/null +++ b/deploy/azuredeployer.gbapp/strings.ts @@ -0,0 +1,22 @@ +export const Messages = { + "en-US": { + about_suggestions: "Suggestions are welcomed and improve my quality...", + what_about_service: "What about my service?", + glad_you_liked: "I'm glad you liked. I'm here for you.", + we_will_improve: "Let's take note of that, thanks for sharing.", + what_about_me: "What about the service, please rate between 1 and 5.", + thanks: "Thanks!", + im_sorry_lets_try: "I'm sorry. Let's try again...", + great_thanks: "Great, thanks for sharing your thoughts." + }, + "pt-BR": { + about_suggestions: "Sugestões melhoram muito minha qualidade...", + what_about_service:"O que achou do meu atendimento?", + glad_you_liked: "Bom saber que você gostou. Conte comigo.", + we_will_improve: "Vamos registrar sua questão, obrigado pela sinceridade.", + what_about_me: "O que achou do meu atendimento, de 1 a 5?", + thanks: "Obrigado!", + im_sorry_lets_try: "Desculpe-me, vamos tentar novamente.", + great_thanks: "Ótimo, obrigado por contribuir com sua resposta." + } +}; diff --git a/deploy/azuredeployer.gblib/services/AzureDeployerService.ts b/deploy/azuredeployer.gblib/services/AzureDeployerService.ts deleted file mode 100644 index 0294f474..00000000 --- a/deploy/azuredeployer.gblib/services/AzureDeployerService.ts +++ /dev/null @@ -1,304 +0,0 @@ -/*****************************************************************************\ -| ( )_ _ | -| _ _ _ __ _ _ __ ___ ___ _ _ | ,_)(_) ___ ___ _ | -| ( '_`\ ( '__)/'_` ) /'_ `\/' _ ` _ `\ /'_` )| | | |/',__)/' _ `\ /'_`\ | -| | (_) )| | ( (_| |( (_) || ( ) ( ) |( (_| || |_ | |\__, \| ( ) |( (_) ) | -| | ,__/'(_) `\__,_)`\__ |(_) (_) (_)`\__,_)`\__)(_)(____/(_) (_)`\___/' | -| | | ( )_) | | -| (_) \___/' | -| | -| General Bots Copyright (c) Pragmatismo.io. All rights reserved. | -| Licensed under the AGPL-3.0. | -| | -| According to our dual licensing model, this program can be used either | -| under the terms of the GNU Affero General Public License, version 3, | -| or under a proprietary license. | -| | -| The texts of the GNU Affero General Public License with an additional | -| permission and of our proprietary license can be found at and | -| in the LICENSE file you have received along with this program. | -| | -| This program is distributed in the hope that it will be useful, | -| but WITHOUT ANY WARRANTY, without even the implied warranty of | -| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | -| GNU Affero General Public License for more details. | -| | -| "General Bots" is a registered trademark of Pragmatismo.io. | -| The licensing of the program under the AGPLv3 does not imply a | -| trademark license. Therefore any rights, title and interest in | -| our trademarks remain entirely with us. | -| | -\*****************************************************************************/ - -'use strict'; - -import { GBService, IGBInstance } from "botlib" -const msRestAzure = require('ms-rest-azure'); -import { ResourceManagementClient } from 'azure-arm-resource' -import { WebSiteManagementClient } from 'azure-arm-website'; -import { SqlManagementClient } from "azure-arm-sql"; -import { CognitiveServicesManagementClient } from "azure-arm-cognitiveservices"; -import { CognitiveServicesAccount } from "azure-arm-cognitiveservices/lib/models"; -import { SearchManagementClient } from "azure-arm-search"; -import { BotConfiguration, BotService, EndpointService, IBotService, IConnectedService, ServiceTypes } from 'botframework-config'; -import { WebResource, ServiceClient } from "ms-rest-js"; -import * as simplegit from 'simple-git/promise'; -import { AppServicePlan } from "azure-arm-website/lib/models"; - -const git = simplegit(); -const logger = require("../../../src/logger"); -const UrlJoin = require("url-join") - -export class AzureDeployerService extends GBService { - - instance: IGBInstance - resourceClient: ResourceManagementClient.ResourceManagementClient; - webSiteClient: WebSiteManagementClient; - storageClient: SqlManagementClient; - cognitiveClient: CognitiveServicesManagementClient; - searchClient: SearchManagementClient; - provider = 'Microsoft.BotService'; - - - public async process(username: any, password: any, instance: IGBInstance, - subscriptionId: string, location: string) { - let _this = this; - msRestAzure.loginWithUsernamePassword(username, password, async (err, credentials) => { - - _this.resourceClient = new ResourceManagementClient.default(credentials, subscriptionId); - _this.webSiteClient = new WebSiteManagementClient(credentials, subscriptionId); - _this.storageClient = new SqlManagementClient(credentials, subscriptionId); - _this.cognitiveClient = new CognitiveServicesManagementClient(credentials, subscriptionId); - _this.searchClient = new SearchManagementClient(credentials, subscriptionId); - - let name = "generalbots"; - let administratorLogin = "" - let administratorPassword = "" - let serverName = name + ""; - let tenantId = ''; - - logger.info(`Creating Deploy...`); - let deploymentName = await this.createDeploy(name, location); - - logger.info(`Creating Server...`); - let serverFarm = await this.createHostingPlan(name, `${name}-server-plan`, location); - await this.createServer(serverFarm.id, name, `${name}-server`, location); - - logger.info(`Creating Storage...`); - //await this.createStorageServer(name, `${name}-storage-server`, administratorLogin, administratorPassword, serverName, location); - //await this.createStorage(name, name, `${name}-storage`, location); - - logger.info(`Creating NLP...`); - //await this.createNLP(name, `${name}-nlp`, location); - - logger.info(`Creating Speech...`); - //await this.createSpeech(name, `${name}-speech`, location); - - logger.info(`Creating SpellChecker...`); - //await this.createSpellChecker(name, `${name}-spellchecker`, location); - - logger.info(`Creating Text Analytics...`); - //await this.createTextAnalytics(name, `${name}-textanalytics`, location); - - logger.info(`Creating Search...`); - //await this.createSearch(name, `${name}-search`, location); - - logger.info(`Creating Bot...`); - //await this.createBot(credentials.tokenCache._entries[0].accessToken, - // name, name, name, 'global', subscriptionId, tenantId); - - - logger.info(`Cleaning Deploy it can take a while...`); - // DISABLED: await this.dangerouslyDeleteDeploy(name); - }); - } - - private async dangerouslyDeleteDeploy(name) { - - return this.resourceClient.resourceGroups.deleteMethod(name); - } - - private async createStorageServer(group, name, administratorLogin, - administratorPassword, serverName, location) { - - var params = { - location: location, - administratorLogin: administratorLogin, - administratorLoginPassword: administratorPassword, - fullyQualifiedDomainName: `${serverName}.database.windows.net` - }; - - return this.storageClient.servers.createOrUpdate(group, name, params); - } - - private async registerProviders(subscriptionId, baseUrl, accessToken, ){ - - let query = `subscriptions/${subscriptionId}/providers/${this.provider}/register?api-version=2018-02-01` - let requestUrl = UrlJoin(baseUrl, query); - - let req = new WebResource(); - req.method = 'POST'; - req.url = requestUrl; - req.headers = {}; - req.headers['Content-Type'] = 'application/json; charset=utf-8'; - req.headers["accept-language"] = '*' - req.headers['x-ms-client-request-id'] = msRestAzure.generateUuid(); - req.headers['Authorization'] = 'Bearer ' + accessToken; - - let httpClient = new ServiceClient(); - let res = await httpClient.sendRequest(req); - - } - - private async createBot(accessToken, botId, group, name, location, subscriptionId, tenantId) { - - let baseUrl = `https://management.azure.com/`; - let appId = '2cac4573-0aea-442a-a222-dcc340000000'; - let description = 'description'; - let endpoint = 'http://localhost:4242/'; - let nlpKey = 'c5869c6c13854434a3f228aad2d6dfb6'; - let nlpAppId = "3e431b4f-96a4-4bdb-b2d5-3ea462ddb773"; - - let parameters = { parameters:{ - "location": location, - "sku": { - "name": "F0" - }, - "name": name, - //"type": "sampletype", - "id": botId, - "kind": "sdk", - "properties": { - "description": description, - "displayName": name, - "endpoint": endpoint, - "iconUrl": "http://myicon", - "luisAppIds": [ - nlpAppId, - ], - "luisKey": nlpKey, - "msaAppId": appId - } - }} - - let query = `subscriptions/${subscriptionId}/resourceGroups/${group}/providers/${this.provider}/botServices/${botId}?api-version=2017-12-01`; - let requestUrl = UrlJoin(baseUrl, query); - - let req = new WebResource(); - req.method = 'PUT'; - req.url = requestUrl; - req.headers = {}; - req.headers['Content-Type'] = 'application/json'; - req.headers["accept-language"] = '*' - //req.headers['x-ms-client-request-id'] = msRestAzure.generateUuid(); - req.headers['Authorization'] = 'Bearer ' + accessToken; - - let requestContent = JSON.stringify(parameters); - req.body = requestContent; - - let httpClient = new ServiceClient(); - let res = await httpClient.sendRequest(req); - } - - private async createSearch(group, name, location) { - var params = { - sku: { name: 'free' }, - location: location - }; - - return this.searchClient.services.createOrUpdate(group, name, params); - } - - private async createStorage(group, serverName, name, location) { - - var params = { - sku: { name: 'Free' }, - createMode: 'Default', - location: location - }; - - return this.storageClient.databases.createOrUpdate(group, - serverName, name, params); - } - - private async createCognitiveServices(group, name, location, kind): Promise { - - // * 'Bing.Autosuggest.v7', 'Bing.CustomSearch', - // * 'Bing.Search.v7', 'Bing.Speech', 'Bing.SpellCheck.v7', 'ComputerVision', - // * 'ContentModerator', 'CustomSpeech', 'CustomVision.Prediction', - // * 'CustomVision.Training', 'Emotion', 'Face', 'LUIS', 'QnAMaker', - // * 'SpeakerRecognition', 'SpeechTranslation', 'TextAnalytics', - // * 'TextTranslation', 'WebLM' - - let params = { - sku: { name: 'F0' }, - createMode: 'Default', - location: location, - kind: kind, - properties: {} - }; - - return await this.cognitiveClient.accounts.create(group, name, params); - } - - private async createSpeech(group, name, location): Promise { - return await this.createCognitiveServices(group, name, location, 'SpeechServices'); - } - - private async createNLP(group, name, location): Promise { - return await this.createCognitiveServices(group, name, location, 'LUIS'); - } - - private async createSpellChecker(group, name, location): Promise { - return await this.createCognitiveServices(group, name, 'global', 'Bing.SpellCheck.v7'); - } - - private async createTextAnalytics(group, name, location): Promise { - return await this.createCognitiveServices(group, name, location, 'TextAnalytics'); - } - - private async createDeploy(name, location) { - var params = { location: location }; - return this.resourceClient.resourceGroups.createOrUpdate(name, params); - } - - private async createHostingPlan(group, name, location):Promise { - let params = { - serverFarmWithRichSkuName: name, - location: location, - sku: { - name: 'F1', - capacity: 1, - tier: 'Free' - } - }; - - return this.webSiteClient.appServicePlans.createOrUpdate(group, name, params); - } - - private async createServer(farmId, group, name, location) { - var parameters = { - location: location, - serverFarmId: farmId - }; - return this.webSiteClient.webApps.createOrUpdate(group, name, parameters); - } - - private async updateWebisteConfig(group, serverFarmId, name, location) { - var siteConfig = { - location: location, - serverFarmId: serverFarmId, - numberOfWorkers: 1, - phpVersion: '5.5' - }; - return this.webSiteClient.webApps.createOrUpdateConfiguration(group, name, siteConfig); - } - - private deleteDeploy(name) { - return this.resourceClient.resourceGroups.deleteMethod(name); - } - - async deployGeneralBotsToAzure(){ - let status = await git.status(); - } -} \ No newline at end of file diff --git a/deploy/core.gbapp/models/GBModel.ts b/deploy/core.gbapp/models/GBModel.ts index 2b45910a..e22f3e96 100644 --- a/deploy/core.gbapp/models/GBModel.ts +++ b/deploy/core.gbapp/models/GBModel.ts @@ -65,9 +65,9 @@ export class GuaribasInstance extends Model @AutoIncrement @Column instanceId: number; - + @Column - botServerUrl:string; + botServerUrl: string; @Column whoAmIVideo: string; @@ -109,10 +109,10 @@ export class GuaribasInstance extends Model @Column authenticatorTenant: string; - + @Column authenticatorAuthorityHostUrl: string; - + @Column authenticatorClientId: string; @@ -148,13 +148,19 @@ export class GuaribasInstance extends Model @Column smsServiceNumber: string; - + @Column speechKey: string; + @Column + speechKeyEndpoint: string; + @Column spellcheckerKey: string; + @Column + spellcheckerEndpoint: string; + @Column theme: string; @@ -168,11 +174,11 @@ export class GuaribasInstance extends Model nlpAppId: string; @Column - nlpSubscriptionKey: string; + nlpKey: string; @Column @Column({ type: DataType.STRING(512) }) - nlpServerUrl: string; + nlpEndpoint: string; @Column searchHost: string; @@ -186,6 +192,24 @@ export class GuaribasInstance extends Model @Column searchIndexer: string; + @Column + storageUsername: string; + + @Column + storagePassword: string; + + @Column + storageName: string; + + @Column + storageServer: string; + + @Column + storageDialect: string; + + @Column + storagePath: string; + /* Settings section of bot.json */ @Column(DataType.FLOAT) diff --git a/deploy/core.gbapp/services/GBConversationalService.ts b/deploy/core.gbapp/services/GBConversationalService.ts index fd382478..02de13da 100644 --- a/deploy/core.gbapp/services/GBConversationalService.ts +++ b/deploy/core.gbapp/services/GBConversationalService.ts @@ -100,8 +100,8 @@ export class GBConversationalService implements IGBConversationalService { const model = new LuisRecognizer({ applicationId: min.instance.nlpAppId, - endpointKey: min.instance.nlpSubscriptionKey, - endpoint: min.instance.nlpServerUrl + endpointKey: min.instance.nlpKey, + endpoint: min.instance.nlpEndpoint }); let nlp: any; diff --git a/deploy/core.gbapp/services/GBCoreService.ts b/deploy/core.gbapp/services/GBCoreService.ts index 1ec2437a..db24f7d8 100644 --- a/deploy/core.gbapp/services/GBCoreService.ts +++ b/deploy/core.gbapp/services/GBCoreService.ts @@ -30,55 +30,83 @@ | | \*****************************************************************************/ -"use strict" +"use strict"; -const logger = require("../../../src/logger") -import { Sequelize } from "sequelize-typescript" -import { GBConfigService } from "./GBConfigService" -import { IGBInstance, IGBCoreService } from "botlib" -import { GuaribasInstance } from "../models/GBModel" +const logger = require("../../../src/logger"); +import { Sequelize } from "sequelize-typescript"; +import { GBConfigService } from "./GBConfigService"; +import { IGBInstance, IGBCoreService } from "botlib"; +import { GuaribasInstance } from "../models/GBModel"; import { GBAdminService } from "../../admin.gbapp/services/GBAdminService"; +import * as fs from "fs"; +import { AzureDeployerService } from "../../azuredeployer.gbapp/services/AzureDeployerService"; +const msRestAzure = require("ms-rest-azure"); /** * Core service layer. */ export class GBCoreService implements IGBCoreService { + async ensureCloud() { + if (!fs.existsSync(".env")) { + return; + } + + logger.warn( + "This mechanism will only work for organizational ids and ids that are not 2FA enabled." + ); + + let credentials = await msRestAzure.loginWithUsernamePassword( + "", + "" + ); + let subscriptionId = ""; + + let s = new AzureDeployerService(credentials, subscriptionId); + let instance = new GuaribasInstance(); + await s.deploy(instance, "westus"); + instance.save(); + + let content = `STORAGE_HOST = ${instance.storageServer}\n + STORAGE_NAME, STORAGE_USERNAME, STORAGE_PASSWORD, STORAGE_DIALECT`; + + fs.writeFileSync(".env", content); + } /** * Data access layer instance. */ - public sequelize: Sequelize + public sequelize: Sequelize; /** * Administrative services. */ - public adminService: GBAdminService + public adminService: GBAdminService; /** * Allows filtering on SQL generated before send to the database. */ - private queryGenerator: any + private queryGenerator: any; /** * Custom create table query. */ - private createTableQuery: (tableName, attributes, options) => string + private createTableQuery: (tableName, attributes, options) => string; /** * Custom change column query. */ - private changeColumnQuery: (tableName, attributes) => string + private changeColumnQuery: (tableName, attributes) => string; /** * Dialect used. Tested: mssql and sqlite. */ - private dialect: string + private dialect: string; /** * Constructor retrieves default values. */ constructor() { - this.dialect = GBConfigService.get("STORAGE_DIALECT") - this.adminService = new GBAdminService(this) + this.dialect = GBConfigService.get("STORAGE_DIALECT"); + this.adminService = new GBAdminService(this); } /** @@ -87,29 +115,29 @@ export class GBCoreService implements IGBCoreService { async initDatabase() { return new Promise((resolve, reject) => { try { - let host: string | undefined - let database: string | undefined - let username: string | undefined - let password: string | undefined - let storage: string | undefined + let host: string | undefined; + let database: string | undefined; + let username: string | undefined; + let password: string | undefined; + let storage: string | undefined; if (this.dialect === "mssql") { - host = GBConfigService.get("STORAGE_HOST") - database = GBConfigService.get("STORAGE_NAME") - username = GBConfigService.get("STORAGE_USERNAME") - password = GBConfigService.get("STORAGE_PASSWORD") + host = GBConfigService.get("STORAGE_HOST"); + database = GBConfigService.get("STORAGE_NAME"); + username = GBConfigService.get("STORAGE_USERNAME"); + password = GBConfigService.get("STORAGE_PASSWORD"); } else if (this.dialect === "sqlite") { - storage = GBConfigService.get("STORAGE_STORAGE") + storage = GBConfigService.get("STORAGE_STORAGE"); } let logging = GBConfigService.get("STORAGE_LOGGING") === "true" ? (str: string) => { - logger.info(str) - } - : false + logger.info(str); + } + : false; - let encrypt = GBConfigService.get("STORAGE_ENCRYPT") === "true" + let encrypt = GBConfigService.get("STORAGE_ENCRYPT") === "true"; this.sequelize = new Sequelize({ host: host, @@ -130,30 +158,30 @@ export class GBCoreService implements IGBCoreService { evict: 40000, acquire: 40000 } - }) + }); if (this.dialect === "mssql") { - this.queryGenerator = this.sequelize.getQueryInterface().QueryGenerator - this.createTableQuery = this.queryGenerator.createTableQuery + this.queryGenerator = this.sequelize.getQueryInterface().QueryGenerator; + this.createTableQuery = this.queryGenerator.createTableQuery; this.queryGenerator.createTableQuery = ( tableName, attributes, options - ) => this.createTableQueryOverride(tableName, attributes, options) - this.changeColumnQuery = this.queryGenerator.changeColumnQuery + ) => this.createTableQueryOverride(tableName, attributes, options); + this.changeColumnQuery = this.queryGenerator.changeColumnQuery; this.queryGenerator.changeColumnQuery = (tableName, attributes) => - this.changeColumnQueryOverride(tableName, attributes) + this.changeColumnQueryOverride(tableName, attributes); } - resolve() + resolve(); } catch (error) { - reject(error) + reject(error); } - }) + }); } /** * SQL: - * + * * // let sql: string = '' + * // 'IF OBJECT_ID(\'[UserGroup]\', \'U\') IS NULL\n' + * // 'CREATE TABLE [UserGroup] (\n' + @@ -171,38 +199,37 @@ export class GBCoreService implements IGBCoreService { tableName, attributes, options - ]) - const re1 = /CREATE\s+TABLE\s+\[([^\]]*)\]/ - const matches = re1.exec(sql) + ]); + const re1 = /CREATE\s+TABLE\s+\[([^\]]*)\]/; + const matches = re1.exec(sql); if (matches) { - const table = matches[1] - const re2 = /PRIMARY\s+KEY\s+\(\[[^\]]*\](?:,\s*\[[^\]]*\])*\)/ + const table = matches[1]; + const re2 = /PRIMARY\s+KEY\s+\(\[[^\]]*\](?:,\s*\[[^\]]*\])*\)/; sql = sql.replace( re2, (match: string, ...args: any[]): string => { - return "CONSTRAINT [" + table + "_pk] " + match + return "CONSTRAINT [" + table + "_pk] " + match; } - ) - const re3 = /FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g - const re4 = /\[([^\]]*)\]/g + ); + const re3 = /FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g; + const re4 = /\[([^\]]*)\]/g; sql = sql.replace( re3, (match: string, ...args: any[]): string => { - const fkcols = args[0] - let fkname = table - let matches = re4.exec(fkcols) + const fkcols = args[0]; + let fkname = table; + let matches = re4.exec(fkcols); while (matches != null) { - fkname += "_" + matches[1] - matches = re4.exec(fkcols) + fkname += "_" + matches[1]; + matches = re4.exec(fkcols); } - return "CONSTRAINT [" + fkname + "_fk] FOREIGN KEY (" + fkcols + ")" + return "CONSTRAINT [" + fkname + "_fk] FOREIGN KEY (" + fkcols + ")"; } - ) + ); } - return sql + return sql; } - /** * SQL: * let sql = '' + @@ -215,22 +242,22 @@ export class GBCoreService implements IGBCoreService { let sql: string = this.changeColumnQuery.apply(this.queryGenerator, [ tableName, attributes - ]) - const re1 = /ALTER\s+TABLE\s+\[([^\]]*)\]/ - const matches = re1.exec(sql) + ]); + const re1 = /ALTER\s+TABLE\s+\[([^\]]*)\]/; + const matches = re1.exec(sql); if (matches) { - const table = matches[1] - const re2 = /(ADD\s+)?CONSTRAINT\s+\[([^\]]*)\]\s+FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g - const re3 = /\[([^\]]*)\]/g + const table = matches[1]; + const re2 = /(ADD\s+)?CONSTRAINT\s+\[([^\]]*)\]\s+FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g; + const re3 = /\[([^\]]*)\]/g; sql = sql.replace( re2, (match: string, ...args: any[]): string => { - const fkcols = args[2] - let fkname = table - let matches = re3.exec(fkcols) + const fkcols = args[2]; + let fkname = table; + let matches = re3.exec(fkcols); while (matches != null) { - fkname += "_" + matches[1] - matches = re3.exec(fkcols) + fkname += "_" + matches[1]; + matches = re3.exec(fkcols); } return ( (args[0] ? args[0] : "") + @@ -239,25 +266,25 @@ export class GBCoreService implements IGBCoreService { "_fk] FOREIGN KEY (" + fkcols + ")" - ) + ); } - ) + ); } - return sql + return sql; } async syncDatabaseStructure() { if (GBConfigService.get("STORAGE_SYNC") === "true") { - const alter = GBConfigService.get("STORAGE_SYNC_ALTER") === "true" - const force = GBConfigService.get("STORAGE_SYNC_FORCE") === "true" - logger.info("Syncing database...") + const alter = GBConfigService.get("STORAGE_SYNC_ALTER") === "true"; + const force = GBConfigService.get("STORAGE_SYNC_FORCE") === "true"; + logger.info("Syncing database..."); return this.sequelize.sync({ alter: alter, force: force }); } else { let msg = "Database synchronization is disabled."; - logger.info(msg) + logger.info(msg); } } @@ -268,12 +295,11 @@ export class GBCoreService implements IGBCoreService { return GuaribasInstance.findAll({}); } - /** * Loads just one Bot instance by its internal Id. */ async loadInstanceById(instanceId: string): Promise { - let options = { where: {instanceId: instanceId} } + let options = { where: { instanceId: instanceId } }; return GuaribasInstance.findOne(options); } @@ -281,10 +307,10 @@ export class GBCoreService implements IGBCoreService { * Loads just one Bot instance. */ async loadInstance(botId: string): Promise { - let options = { where: {} } + let options = { where: {} }; if (botId != "[default]") { - options.where = { botId: botId } + options.where = { botId: botId }; } return GuaribasInstance.findOne(options); diff --git a/deploy/core.gbapp/services/GBMinService.ts b/deploy/core.gbapp/services/GBMinService.ts index 0735ac0d..b89c2eba 100644 --- a/deploy/core.gbapp/services/GBMinService.ts +++ b/deploy/core.gbapp/services/GBMinService.ts @@ -37,7 +37,7 @@ const UrlJoin = require("url-join"); const express = require("express"); const logger = require("../../../src/logger"); const request = require("request-promise-native"); -const ngrok = require('ngrok'); +const ngrok = require("ngrok"); var crypto = require("crypto"); var AuthenticationContext = require("adal-node").AuthenticationContext; @@ -67,16 +67,18 @@ import { import { GuaribasInstance } from "../models/GBModel"; import { Messages } from "../strings"; + /** Minimal service layer for a bot. */ export class GBMinService { + core: IGBCoreService; conversationalService: IGBConversationalService; adminService: IGBAdminService; deployer: GBDeployer; corePackage = "core.gbai"; - + /** * Static initialization of minimal instance. * @@ -108,7 +110,7 @@ export class GBMinService { async buildMin( server: any, appPackages: Array, - instances:GuaribasInstance[] + instances: GuaribasInstance[] ): Promise { // Serves default UI on root address '/'. @@ -117,7 +119,7 @@ export class GBMinService { "/", express.static(UrlJoin(GBDeployer.deployFolder, uiPackage, "build")) ); - + Promise.all( instances.map(async instance => { // Gets the authorization key for each instance from Bot Service. @@ -298,11 +300,11 @@ export class GBMinService { ); } -private async ngrokRefresh(){ - const url = await ngrok.connect(9090); // https://757c1652.ngrok.io -> http://localhost:9090 - // TODO: Persist to storage and refresh each 8h. - // TODO: Update all bots definition in azure. -} + private async ngrokRefresh() { + const url = await ngrok.connect(9090); // https://757c1652.ngrok.io -> http://localhost:9090 + // TODO: Persist to storage and refresh each 8h. + // TODO: Update all bots definition in azure. + } private async buildBotAdapter(instance: any) { let adapter = new BotFrameworkAdapter({ @@ -372,15 +374,12 @@ private async ngrokRefresh(){ instance: any, appPackages: any[] ) { - return adapter.processActivity(req, res, async context => { - const state = conversationState.get(context); const dc = min.dialogs.createContext(context, state); dc.context.activity.locale = "en-US"; // TODO: Make dynamic. - + try { - const user = min.userState.get(dc.context); if (!user.loaded) { @@ -472,14 +471,13 @@ private async ngrokRefresh(){ } } } catch (error) { - let msg = `ERROR: ${error.message} ${ - error.stack ? error.stack : "" - }`; - logger.error(msg); - - await dc.context.sendActivity(Messages[dc.context.activity.locale].very_sorry_about_error) - await dc.begin("/ask", { isReturning: true }); - + let msg = `ERROR: ${error.message} ${error.stack ? error.stack : ""}`; + logger.error(msg); + + await dc.context.sendActivity( + Messages[dc.context.activity.locale].very_sorry_about_error + ); + await dc.begin("/ask", { isReturning: true }); } }); } @@ -506,7 +504,7 @@ private async ngrokRefresh(){ let msg = `Error calling Direct Line client, verify Bot endpoint on the cloud. Error is: ${error}.`; return Promise.reject(new Error(msg)); } - } + } /** * Gets a Speech to Text / Text to Speech token from the provider. diff --git a/package.json b/package.json index f94b21a6..0ca31e3f 100644 --- a/package.json +++ b/package.json @@ -71,10 +71,12 @@ "pragmatismo-io-framework": "1.0.17", "reflect-metadata": "0.1.12", "request-promise-native": "1.0.5", + "scanf": "^1.0.2", "sequelize": "4.39.0", "sequelize-typescript": "0.6.6", "simple-git": "^1.105.0", "sqlite3": "4.0.2", + "strict-password-generator": "^1.1.1", "swagger-client": "3.8.21", "tedious": "2.6.4", "ts-node": "7.0.1", diff --git a/src/app.ts b/src/app.ts index ecba6dbd..8a5641e4 100644 --- a/src/app.ts +++ b/src/app.ts @@ -33,13 +33,11 @@ "use strict"; -const UrlJoin = require("url-join"); const logger = require("./logger"); const express = require("express"); const bodyParser = require("body-parser"); -const MicrosoftGraph = require("@microsoft/microsoft-graph-client"); +const scanf = require('scanf'); -import { Sequelize } from "sequelize-typescript"; import { GBConfigService } from "../deploy/core.gbapp/services/GBConfigService"; import { GBConversationalService } from "../deploy/core.gbapp/services/GBConversationalService"; import { GBMinService } from "../deploy/core.gbapp/services/GBMinService"; @@ -56,7 +54,8 @@ import { GBCustomerSatisfactionPackage } from "../deploy/customer-satisfaction.g import { IGBPackage } from "botlib"; import { GBAdminService } from "../deploy/admin.gbapp/services/GBAdminService"; import { GuaribasInstance } from "../deploy/core.gbapp/models/GBModel"; -import { AzureDeployerService } from "../deploy/azuredeployer.gblib/services/AzureDeployerService"; +import { AzureDeployerService } from "deploy/azuredeployer.gbapp/services/AzureDeployerService"; + let appPackages = new Array(); @@ -94,7 +93,10 @@ export class GBServer { GBConfigService.init(); let core = new GBCoreService(); + let instance = await core.ensureCloud(); + await core.initDatabase(); + // Boot a bot package if any.