diff --git a/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts b/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts index 973554d0..b0d2fc66 100644 --- a/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts +++ b/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts @@ -56,12 +56,6 @@ const PasswordGenerator = require("strict-password-generator").default; const iconUrl = "https://github.com/pragmatismo-io/BotServer/blob/master/docs/images/generalbots-logo-squared.png"; - class x implements IGBInstance { - - }; - - - export class AzureDeployerService extends GBService { instance: IGBInstance; resourceClient: ResourceManagementClient.ResourceManagementClient; @@ -73,6 +67,7 @@ export class AzureDeployerService extends GBService { subscriptionClient: SubscriptionClient.SubscriptionClient; accessToken: string; location: string; + public subscriptionId: string; constructor(credentials, subscriptionId, location) { super(); @@ -92,6 +87,7 @@ export class AzureDeployerService extends GBService { this.searchClient = new SearchManagementClient(credentials, subscriptionId); this.accessToken = credentials.tokenCache._entries[0].accessToken; this.location = location; + this.subscriptionId = subscriptionId; } public static async getSubscriptions(credentials) { @@ -101,16 +97,32 @@ export class AzureDeployerService extends GBService { public async deployFarm( name: string, - location: string + location: string, + proxyAddress: string ): Promise { + let instance: any = {}; + let culture = "en-us"; - let instance:any = {}; - logger.info(`Starting infrastructure deployment...`); - // TODO: REMOVE THIS********* - - //await this.dangerouslyDeleteDeploy(name); + let keys: any; + logger.info(`Deploying NLP...`); + let nlp = await this.createNLP(name, `${name}-nlp`, location); + keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name); + let nlpAppId = await this.createLUISApp(name, name, location, culture); + + + await this.deployBootBot( + instance, + name, + proxyAddress, + nlpAppId, + keys.key1, + this.subscriptionId + ); + instance.nlpEndpoint = nlp.endpoint; + instance.nlpKey = keys.key1; + instance.nlpAppId = nlpAppId; logger.info(`Deploying Deploy Group...`); await this.createDeployGroup(name, location); @@ -151,19 +163,18 @@ export class AzureDeployerService extends GBService { logger.info(`Deploying Search...`); let searchName = `${name}-search`; - let search = await this.createSearch(name,searchName , location); - let searchKeys = await this.searchClient.queryKeys.listBySearchService(name, searchName) + + await this.createSearch(name, searchName, location); + let searchKeys = await this.searchClient.queryKeys.listBySearchService( + name, + searchName + ); instance.searchHost = `${searchName}.search.windows.net`; instance.searchIndex = "azuresql-index"; instance.searchIndexer = "azuresql-indexer"; - instance.searchKey = searchKeys[0]; + instance.searchKey = searchKeys[0].key; - logger.info(`Deploying NLP...`); - let nlp = await this.createNLP(name, `${name}-nlp`, location); - let keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name); - instance.nlpEndpoint = nlp.endpoint; - instance.nlpKey = keys.key1; - instance.nlpAppId = "0ff1ceb4f-96a4-4bdb-b2d5-3ea462ddb773"; + ////////////// LUS logger.info(`Deploying Speech...`); let speech = await this.createSpeech(name, `${name}-speech`, location); @@ -200,7 +211,7 @@ export class AzureDeployerService extends GBService { return instance; } - public async deployBot( + public async deployBootBot( instance, name, endpoint, @@ -229,7 +240,7 @@ export class AzureDeployerService extends GBService { } private async dangerouslyDeleteDeploy(name) { - return this.resourceClient.resourceGroups.deleteMethod(name); + return await this.resourceClient.resourceGroups.deleteMethod(name); } private async createStorageServer( @@ -282,27 +293,25 @@ export class AzureDeployerService extends GBService { subscriptionId ) { let baseUrl = `https://management.azure.com/`; - + this.registerProviders(subscriptionId, baseUrl, accessToken); let appId = msRestAzure.generateUuid(); let parameters = { - parameters: { - location: location, - sku: { - name: "F0" - }, - name: name, - id: botId, - kind: "sdk", - properties: { - description: description, - displayName: name, - endpoint: endpoint, - iconUrl: iconUrl, - luisAppIds: [nlpAppId], - luisKey: nlpKey, - msaAppId: appId - } + location: location, + sku: { + name: "F0" + }, + name: name, + id: botId, + kind: "sdk", + properties: { + description: description, + displayName: name, + endpoint: endpoint, + iconUrl: iconUrl, + luisAppIds: [nlpAppId], + luisKey: nlpKey, + msaAppId: appId } }; @@ -328,6 +337,50 @@ export class AzureDeployerService extends GBService { return JSON.parse(res.bodyAsJson as string); } + private async createLUISApp( + name: string, + description: string, + location: string, + culture: string + ) { + let parameters = { + name: name, + description: description, + culture: culture + }; + let requestUrl = `https://${location}.api.cognitive.microsoft.com/luis/api/v2.0/apps/`; + let req = new WebResource(); + + req.method = "POST"; + req.url = requestUrl; + req.headers = {}; + req.headers["Content-Type"] = "application/json"; + req.headers["accept-language"] = "*"; + + let authoringKey; + let retriveAuthoringKey = () => { + if (!authoringKey) { + process.stdout.write( + "Due to this opened issue: https://github.com/Microsoft/botbuilder-tools/issues/550\n" + ); + process.stdout.write("Please enter your LUIS Authoring Key:"); + authoringKey = scanf("%s"); + } + }; + + while (!authoringKey) { + retriveAuthoringKey(); + } + + req.headers["Ocp-Apim-Subscription-Key"] = authoringKey; + req.body = JSON.stringify(parameters); + + let httpClient = new ServiceClient(); + let res = await httpClient.sendRequest(req); + + return res.bodyAsText; + } + private async createSearch(group, name, location) { var params = { sku: { name: "free" }, @@ -358,13 +411,6 @@ export class AzureDeployerService extends GBService { location, kind ): Promise { - // * 'Bing.Autosuggest.v7', 'Bing.CustomSearch', - // * 'Bing.Search.v7', 'Bing.Speech', 'Bing.SpellCheck.v7', 'ComputerVision', - // * 'ContentModerator', 'CustomSpeech', 'CustomVision.Prediction', - // * 'CustomVision.Training', 'Emotion', 'Face', 'LUIS', 'QnAMaker', - // * 'SpeakerRecognition', 'SpeechTranslation', 'TextAnalytics', - // * 'TextTranslation', 'WebLM' - let params = { sku: { name: "F0" }, createMode: "Default", @@ -490,8 +536,8 @@ export class AzureDeployerService extends GBService { minimumLength: 8, maximumLength: 8 }; - let password = passwordGenerator.generatePassword(options); - return `sa${password}`; + let generated = passwordGenerator.generatePassword(options); + return `sa${generated}`; } private static getRndPassword() { @@ -591,4 +637,105 @@ export class AzureDeployerService extends GBService { return new AzureDeployerService(credentials, subscriptionId, location); } + + static getKBSearchSchema(indexName) { + return { + name: indexName, + fields: [ + { + name: "questionId", + type: "Edm.String", + searchable: false, + filterable: false, + retrievable: true, + sortable: false, + facetable: false, + key: true + }, + { + name: "subject1", + type: "Edm.String", + searchable: true, + filterable: false, + retrievable: false, + sortable: false, + facetable: false, + key: false + }, + { + name: "subject2", + type: "Edm.String", + searchable: true, + filterable: false, + retrievable: false, + sortable: false, + facetable: false, + key: false + }, + { + name: "subject3", + type: "Edm.String", + searchable: true, + filterable: false, + retrievable: false, + sortable: false, + facetable: false, + key: false + }, + { + name: "subject4", + type: "Edm.String", + searchable: true, + filterable: false, + retrievable: false, + sortable: false, + facetable: false, + key: false + }, + { + name: "content", + type: "Edm.String", + searchable: true, + filterable: false, + retrievable: false, + sortable: false, + facetable: false, + key: false + }, + { + name: "answerId", + type: "Edm.Int32", + searchable: false, + filterable: false, + retrievable: true, + sortable: false, + facetable: false, + key: false + }, + { + name: "instanceId", + type: "Edm.Int32", + searchable: false, + filterable: true, + retrievable: true, + sortable: false, + facetable: false, + key: false + }, + { + name: "packageId", + type: "Edm.Int32", + searchable: false, + filterable: true, + retrievable: true, + sortable: false, + facetable: false, + key: false + } + ], + scoringProfiles: [], + defaultScoringProfile: null, + corsOptions: null + }; + } } diff --git a/deploy/core.gbapp/services/GBCoreService.ts b/deploy/core.gbapp/services/GBCoreService.ts index 787476c9..ca3180a1 100644 --- a/deploy/core.gbapp/services/GBCoreService.ts +++ b/deploy/core.gbapp/services/GBCoreService.ts @@ -39,7 +39,7 @@ import { IGBInstance, IGBCoreService } from "botlib"; import { GuaribasInstance } from "../models/GBModel"; import { GBAdminService } from "../../admin.gbapp/services/GBAdminService"; const processExists = require("process-exists"); - +const TextDecoder = require("util").TextDecoder; /** * Core service layer. @@ -303,40 +303,47 @@ export class GBCoreService implements IGBCoreService { } public async ensureProxy(): Promise { - let expiresOn = new Date( - await this.adminService.getValue(0, "proxyExpiresOn") - ); - let proxyAddress; - if (expiresOn.getTime() > new Date().getTime()) { - proxyAddress = await this.adminService.getValue( - GBAdminService.masterBotInstanceId, - "proxyAddress" - ); - return Promise.resolve(proxyAddress); - } else { - if (await processExists("ngrok")) { - logger.warn("ngrok is already running."); - } else { - const { spawn } = require("child_process"); - const child = spawn("node_modules\ngrok\bin\ngrok"); - child.stdout.on("data", data => { - console.log(`child stdout:\n${data}`); - }); - } + let proxyAddress: string; + const ngrok = require('ngrok'); + return await ngrok.connect(); + - await this.adminService.setValue( - GBAdminService.masterBotInstanceId, - "proxyAddress", - proxyAddress - ); - let now = new Date(); - let expiresOn = now.setHours(now.getHours()); - await this.adminService.setValue( - GBAdminService.masterBotInstanceId, - "proxyExpiresOn", - expiresOn.toString() - ); - return Promise.resolve(proxyAddress); - } + // let expiresOn = new Date( + // await this.adminService.getValue(0, "proxyExpiresOn") + // ); + // let proxyAddress; + // if (expiresOn.getTime() > new Date().getTime()) { + // proxyAddress = await this.adminService.getValue( + // GBAdminService.masterBotInstanceId, + // "proxyAddress" + // ); + // return Promise.resolve(proxyAddress); + // } else { + // if (await processExists("ngrok.exe")) { + // logger.warn("ngrok is already running."); + // } else { + // const { exec } = require("child_process"); + // const child = exec( + // "node_modules\\ngrok\\bin\\ngrok http 4242", + // (error, stdout, stderr) => { + // console.log(`child stdout:\n${stdout}`); + // } + // ); + // } + + // await this.adminService.setValue( + // GBAdminService.masterBotInstanceId, + // "proxyAddress", + // proxyAddress + // ); + // let now = new Date(); + // let expiresOn = now.setHours(now.getHours()); + // await this.adminService.setValue( + // GBAdminService.masterBotInstanceId, + // "proxyExpiresOn", + // expiresOn.toString() + // ); + return Promise.resolve(proxyAddress); + // } } } diff --git a/deploy/core.gbapp/services/GBDeployer.ts b/deploy/core.gbapp/services/GBDeployer.ts index 7548dc9a..63871085 100644 --- a/deploy/core.gbapp/services/GBDeployer.ts +++ b/deploy/core.gbapp/services/GBDeployer.ts @@ -47,6 +47,7 @@ import { GBError } from "botlib"; import { GuaribasPackage, GuaribasInstance } from "../models/GBModel"; import { IGBPackage } from "botlib"; import { AzureSearch } from "pragmatismo-io-framework"; +import { AzureDeployerService } from "../../azuredeployer.gbapp/services/AzureDeployerService"; /** Deployer service for bots, themes, ai and more. */ export class GBDeployer { @@ -251,7 +252,6 @@ export class GBDeployer { }); } - deployTheme(localPath: string) { // DISABLED: Until completed, "/ui/public". // FsExtra.copy(localPath, this.workDir + packageName) @@ -322,6 +322,16 @@ export class GBDeployer { } } + public static getConnectionStringFromInstance(instance: GuaribasInstance) { + return `Server=tcp:${ + instance.storageServer + }.database.windows.net,1433;Database=${instance.storageName};User ID=${ + instance.storageUsername + };Password=${ + instance.storagePassword + };Trusted_Connection=False;Encrypt=True;Connection Timeout=30;`; + } + public async rebuildIndex(instance: GuaribasInstance) { let search = new AzureSearch( instance.searchKey, @@ -329,11 +339,22 @@ export class GBDeployer { instance.searchIndex, instance.searchIndexer ); + + let connectionString = GBDeployer.getConnectionStringFromInstance(instance); + + const dsName = "gb"; + await search.deleteDatasource(dsName); + await search.createDatasource( + dsName, + dsName, + "GuaribasQuestion", + "azuresql", + connectionString + ); await search.deleteIndex(); - let kbService = new KBService(this.core.sequelize); await search.createIndex( - kbService.getSearchSchema(instance.searchIndex), - "gb" + AzureDeployerService.getKBSearchSchema(instance.searchIndex), + dsName ); } diff --git a/deploy/kb.gbapp/services/KBService.ts b/deploy/kb.gbapp/services/KBService.ts index 7a2fd923..deb8a81a 100644 --- a/deploy/kb.gbapp/services/KBService.ts +++ b/deploy/kb.gbapp/services/KBService.ts @@ -183,106 +183,6 @@ export class KBService { } } - getSearchSchema(indexName) { - return { - name: indexName, - fields: [ - { - name: "questionId", - type: "Edm.String", - searchable: false, - filterable: false, - retrievable: true, - sortable: false, - facetable: false, - key: true - }, - { - name: "subject1", - type: "Edm.String", - searchable: true, - filterable: false, - retrievable: false, - sortable: false, - facetable: false, - key: false - }, - { - name: "subject2", - type: "Edm.String", - searchable: true, - filterable: false, - retrievable: false, - sortable: false, - facetable: false, - key: false - }, - { - name: "subject3", - type: "Edm.String", - searchable: true, - filterable: false, - retrievable: false, - sortable: false, - facetable: false, - key: false - }, - { - name: "subject4", - type: "Edm.String", - searchable: true, - filterable: false, - retrievable: false, - sortable: false, - facetable: false, - key: false - }, - { - name: "content", - type: "Edm.String", - searchable: true, - filterable: false, - retrievable: false, - sortable: false, - facetable: false, - key: false - }, - { - name: "answerId", - type: "Edm.Int32", - searchable: false, - filterable: false, - retrievable: true, - sortable: false, - facetable: false, - key: false - }, - { - name: "instanceId", - type: "Edm.Int32", - searchable: false, - filterable: true, - retrievable: true, - sortable: false, - facetable: false, - key: false - }, - { - name: "packageId", - type: "Edm.Int32", - searchable: false, - filterable: true, - retrievable: true, - sortable: false, - facetable: false, - key: false - } - ], - scoringProfiles: [], - defaultScoringProfile: null, - corsOptions: null - } - } static getFormattedSubjectItems(subjects: GuaribasSubject[]) { if (!subjects) return "" diff --git a/src/app.ts b/src/app.ts index 4feec590..5d66e9c4 100644 --- a/src/app.ts +++ b/src/app.ts @@ -94,24 +94,20 @@ export class GBServer { GBConfigService.init(); let core = new GBCoreService(); - + // Ensures cloud / on-premises infrastructure is setup. logger.info(`Connecting to the infrastructure...`); + let proxyAddress = await core.ensureProxy(); let cloudDeployer = await AzureDeployerService.ensureDeployer(); - let instance = await cloudDeployer.deployFarm('gbot', 'westus'); + let instance = await cloudDeployer.deployFarm('gbot', 'westus', proxyAddress); // TODO: Get .gb* templates from GitHub and download do additional deploy folder. await core.initDatabase(); - // Boot a bot package if any. - - logger.info(`Starting instances...`); - let deployer = new GBDeployer(core, new GBImporter(core)); - - // Build a minimal bot instance for each .gbot deployment. - + // Check admin password. + let conversationalService = new GBConversationalService(core); let adminService = new GBAdminService(core); let password = GBConfigService.get("ADMIN_PASS"); @@ -122,15 +118,6 @@ export class GBServer { ); } - // Creates the minimal service shared across all .gbapps. - - let minService = new GBMinService( - core, - conversationalService, - adminService, - deployer - ); - // NOTE: the semicolon is necessary before this line. // Loads all system packages. @@ -177,7 +164,8 @@ export class GBServer { // Deploy packages and format object store according to .gbapp storage models. - logger.info(`Deploying packages.`); + logger.info(`Deploying packages...`); + let deployer = new GBDeployer(core, new GBImporter(core)); await deployer.deployPackages(core, server, appPackages); // If instances is undefined here it's because storage has been formatted. @@ -190,7 +178,13 @@ export class GBServer { // Setup server dynamic (per bot instance) resources and listeners. - logger.info(`Building minimal instances.`); + logger.info(`Building instances.`); + let minService = new GBMinService( + core, + conversationalService, + adminService, + deployer + ); await minService.buildMin(server, appPackages, instances); logger.info(`The Bot Server is in RUNNING mode...`);