diff --git a/VERSION.md b/VERSION.md index ba961bc2..c763091e 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1,5 +1,11 @@ # Release History +## Version 0.1.7 + +* Azure Deployer +* Strategy to replicate itself in several subscriptions. +* Nkrok experiments to allow 100% automated development environement setup. + ## Version 0.1.6 * Updated packages references. diff --git a/deploy/admin.gbapp/dialogs/AdminDialog.ts b/deploy/admin.gbapp/dialogs/AdminDialog.ts index 8544e300..8f644bed 100644 --- a/deploy/admin.gbapp/dialogs/AdminDialog.ts +++ b/deploy/admin.gbapp/dialogs/AdminDialog.ts @@ -156,7 +156,7 @@ export class AdminDialog extends IGBDialog { min.instance.authenticatorTenant }/oauth2/authorize?client_id=${ min.instance.authenticatorClientId - }&response_type=code&redirect_uri=${min.instance.botServerUrl}/${ + }&response_type=code&redirect_uri=${min.instance.botEndpoint}/${ min.instance.botId }/token&state=${state}&response_mode=query`; diff --git a/deploy/admin.gbapp/services/GBAdminService.ts b/deploy/admin.gbapp/services/GBAdminService.ts index 6583d117..58a35b05 100644 --- a/deploy/admin.gbapp/services/GBAdminService.ts +++ b/deploy/admin.gbapp/services/GBAdminService.ts @@ -37,7 +37,11 @@ import { IGBCoreService } from "botlib"; import { AuthenticationContext, TokenResponse } from "adal-node"; const UrlJoin = require("url-join"); +const ngrok = require("ngrok"); + export class GBAdminService { + static masterBotInstanceId = 0; + public static StrongRegex = new RegExp( "^(?=.*[a-z])(?=.*[A-Z])(?=.*[0-9])(?=.*[!@#\$%\^&\*])(?=.{8,})" ); diff --git a/deploy/azuredeployer.gbapp/dialogs/BotFarmDialog.ts b/deploy/azuredeployer.gbapp/dialogs/BotFarmDialog.ts index cd84084b..0a740854 100644 --- a/deploy/azuredeployer.gbapp/dialogs/BotFarmDialog.ts +++ b/deploy/azuredeployer.gbapp/dialogs/BotFarmDialog.ts @@ -38,6 +38,8 @@ import { BotAdapter } from "botbuilder"; import { Messages } from "../strings"; export class BotFarmDialog extends IGBDialog { + + /** * Setup dialogs flows and define services call. * diff --git a/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts b/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts index 30208535..f5f48100 100644 --- a/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts +++ b/deploy/azuredeployer.gbapp/services/AzureDeployerService.ts @@ -46,11 +46,16 @@ import { SearchManagementClient } from "azure-arm-search"; import { WebResource, ServiceClient } from "ms-rest-js"; import * as simplegit from "simple-git/promise"; import { AppServicePlan } from "azure-arm-website/lib/models"; +import { GBConfigService } from "deploy/core.gbapp/services/GBConfigService"; +const scanf = require("scanf"); const git = simplegit(); const logger = require("../../../src/logger"); const UrlJoin = require("url-join"); const PasswordGenerator = require("strict-password-generator").default; - +const iconUrl = + "https://github.com/pragmatismo-io/BotServer/blob/master/docs/images/generalbots-logo-squared.png"; + + export class AzureDeployerService extends GBService { instance: IGBInstance; resourceClient: ResourceManagementClient.ResourceManagementClient; @@ -60,8 +65,10 @@ export class AzureDeployerService extends GBService { searchClient: SearchManagementClient; provider = "Microsoft.BotService"; subscriptionClient: SubscriptionClient.SubscriptionClient; + accessToken: string; + location: string; - constructor(credentials, subscriptionId) { + constructor(credentials, subscriptionId, location) { super(); this.resourceClient = new ResourceManagementClient.default( credentials, @@ -77,17 +84,20 @@ export class AzureDeployerService extends GBService { subscriptionId ); this.searchClient = new SearchManagementClient(credentials, subscriptionId); - this.subscriptionClient = new SubscriptionClient.default(credentials); + this.accessToken = credentials.tokenCache._entries[0].accessToken; + this.location= location; } - public async getSubscriptions() { - this.subscriptionClient.subscriptions.list(); + public static async getSubscriptions(credentials) { + let subscriptionClient = new SubscriptionClient.default(credentials); + return subscriptionClient.subscriptions.list(); } - public async deploy( - instance: IGBInstance, + public async deployFarm( + name: string, location: string ): Promise { + let instance = new IGBInstance(); logger.info(`Creating Deploy...`); await this.createDeploy(name, location); @@ -128,16 +138,17 @@ export class AzureDeployerService extends GBService { logger.info(`Creating Search...`); let search = await this.createSearch(name, `${name}-search`, location); - - logger.info(`Creating Bot...`); - //await this.createBot(credentials.tokenCache._entries[0].accessToken, - // name, name, name, 'global', subscriptionId, tenantId); + instance.searchHost = "generalbots.search.windows.net"; + instance.searchIndex = "azuresql-index"; + instance.searchIndexer = "azuresql-indexer"; + instance.searchKey = "0FF1CE27564C208555A22B6E278289813"; logger.info(`Creating NLP...`); let nlp = await this.createNLP(name, `${name}-nlp`, location); let keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name); instance.nlpEndpoint = nlp.endpoint; instance.nlpKey = keys.key1; + instance.nlpAppId = "0ff1ceb4f-96a4-4bdb-b2d5-3ea462ddb773"; logger.info(`Creating Speech...`); let speech = await this.createSpeech(name, `${name}-speech`, location); @@ -168,11 +179,32 @@ export class AzureDeployerService extends GBService { name, textAnalytics.name ); - instance.textAnalyticsServerUrl = textAnalytics.endpoint; + instance.textAnalyticsEndpoint = textAnalytics.endpoint; instance.textAnalyticsKey = keys.key1; - logger.info(`Cleaning Deploy it can take a while...`); - // DISABLED: await this.dangerouslyDeleteDeploy(name); + return instance; + } + + public async deployBot(instance, name, endpoint, nlpAppId, nlpKey, subscriptionId) { + + logger.info(`Creating Bot...`); + await this.internalDeployBot( + this.accessToken, + name, + name, + name, + "General BootBot", + endpoint, + "global", + nlpAppId, + nlpKey, + subscriptionId + ); + instance.webchatKey = "********"; + instance.marketplaceId = "0ff1ce73-0aea-442a-a222-dcc340eca294"; + instance.marketplacePassword = "************"; + + return instance; } private async dangerouslyDeleteDeploy(name) { @@ -216,21 +248,21 @@ export class AzureDeployerService extends GBService { let res = await httpClient.sendRequest(req); } - private async createBot( + private async internalDeployBot( accessToken, botId, group, name, + description, + endpoint, location, - subscriptionId, - tenantId + nlpAppId, + nlpKey, + subscriptionId ) { let baseUrl = `https://management.azure.com/`; - let appId = ""; - let description = ""; - let endpoint = ""; - let nlpKey = ""; - let nlpAppId = "3"; + + let appId = msRestAzure.generateUuid(); let parameters = { parameters: { @@ -239,14 +271,13 @@ export class AzureDeployerService extends GBService { name: "F0" }, name: name, - //"type": "sampletype", id: botId, kind: "sdk", properties: { description: description, displayName: name, endpoint: endpoint, - iconUrl: "http://myicon", + iconUrl: iconUrl, luisAppIds: [nlpAppId], luisKey: nlpKey, msaAppId: appId @@ -265,7 +296,6 @@ export class AzureDeployerService extends GBService { req.headers = {}; req.headers["Content-Type"] = "application/json"; req.headers["accept-language"] = "*"; - //req.headers['x-ms-client-request-id'] = msRestAzure.generateUuid(); req.headers["Authorization"] = "Bearer " + accessToken; let requestContent = JSON.stringify(parameters); @@ -273,6 +303,8 @@ export class AzureDeployerService extends GBService { let httpClient = new ServiceClient(); let res = await httpClient.sendRequest(req); + + return JSON.parse(res.bodyAsJson as string); } private async createSearch(group, name, location) { @@ -454,4 +486,90 @@ export class AzureDeployerService extends GBService { let password = passwordGenerator.generatePassword(options); return password; } + + static async ensureDeployer() { + + // Tries do get information from .env file otherwise asks in command-line. + + let username = GBConfigService.get("CLOUD_USERNAME"); + let password = GBConfigService.get("CLOUD_PASSWORD"); + let subscriptionId = GBConfigService.get("CLOUD_SUBSCRIPTIONID"); + let cloudLocation = GBConfigService.get("CLOUD_LOCATION"); + + // No .env so asks for cloud credentials to start a new farm. + + if (!username || !password || !subscriptionId || !cloudLocation) { + process.stdout.write( + "FIRST RUN: A empty enviroment is detected. Please, enter credentials to create a new General Bots Farm." + ); + } + + let retriveUsername = () => { + if (!username) { + process.stdout.write("CLOUD_USERNAME:"); + username = scanf("%s"); + } + }; + + let retrivePassword = () => { + if (!password) { + process.stdout.write("CLOUD_PASSWORD:"); + password = scanf("%s"); + } + }; + + while (!username) { + retriveUsername(); + } + + while (!password) { + retrivePassword(); + } + + // Connects to the cloud and retrives subscriptions. + + let credentials = await msRestAzure.loginWithUsernamePassword( + username, + password + ); + let list = await AzureDeployerService.getSubscriptions(credentials); + + let map = {}; + let index = 1; + list.forEach(element => { + console.log( + `${index}: ${element.displayName} (${element.subscriptionId})` + ); + map[index++] = element; + }); + + let subscriptionIndex; + let retrieveSubscription = () => { + if (!subscriptionIndex) { + process.stdout.write("CLOUD_SUBSCRIPTIONID (type a number):"); + subscriptionIndex = scanf("%d"); + } + }; + if (!subscriptionId) { + while (!subscriptionIndex) { + retrieveSubscription(); + } + subscriptionId = map[subscriptionIndex].subscriptionId; + } + + + let retriveLocation = () => { + if (!location) { + process.stdout.write("CLOUD_LOCATION:"); + location = scanf("%s"); + } + }; + + while (!location) { + retriveLocation(); + } + + return new AzureDeployerService(credentials, subscriptionId, location); + } + } diff --git a/deploy/core.gbapp/models/GBModel.ts b/deploy/core.gbapp/models/GBModel.ts index e22f3e96..3cd3587e 100644 --- a/deploy/core.gbapp/models/GBModel.ts +++ b/deploy/core.gbapp/models/GBModel.ts @@ -67,7 +67,7 @@ export class GuaribasInstance extends Model instanceId: number; @Column - botServerUrl: string; + botEndpoint: string; @Column whoAmIVideo: string; @@ -99,7 +99,7 @@ export class GuaribasInstance extends Model textAnalyticsKey: string; @Column - textAnalyticsServerUrl: string; + textAnalyticsEndpoint: string; @Column marketplacePassword: string; diff --git a/deploy/core.gbapp/services/GBConfigService.ts b/deploy/core.gbapp/services/GBConfigService.ts index bc2a0aff..2676c1b1 100644 --- a/deploy/core.gbapp/services/GBConfigService.ts +++ b/deploy/core.gbapp/services/GBConfigService.ts @@ -30,11 +30,13 @@ | | \*****************************************************************************/ -const logger = require("../../../src/logger") +const logger = require("../../../src/logger"); +import * as fs from "fs"; -"use strict" +"use strict"; export class GBConfigService { + static init(): any { try { require("dotenv-extended").load({ @@ -42,55 +44,66 @@ export class GBConfigService { errorOnMissing: true, errorOnExtra: false, overrideProcessEnv: true - }) + }); } catch (e) { - console.error(e.message) - process.exit(3) + console.error(e.message); + process.exit(3); + } + } + + public writeEntry(name, value) { + + if (fs.exists) { + fs.appendFileSync('.env',`${name}=${value}`); } } static get(key: string): string | undefined { - let value = process.env["container:" + key] - - if (!value) { - value = process.env[key] - } + let value = GBConfigService.tryGet(key); if (!value) { switch (key) { case "STORAGE_DIALECT": - value = "sqlite" - break + value = "sqlite"; + break; case "STORAGE_STORAGE": - value = "./guaribas.sqlite" - break + value = "./guaribas.sqlite"; + break; case "ADDITIONAL_DEPLOY_PATH": - value = undefined - break + value = undefined; + break; case "STORAGE_SYNC": case "STORAGE_SYNC_ALTER": case "STORAGE_SYNC_FORCE": - value = "false" - break + value = "false"; + break; case "STORAGE_LOGGING": - value = "false" - break + value = "false"; + break; case "STORAGE_ENCRYPT": - value = "true" - break + value = "true"; + break; default: logger.info( `Guaribas General Error: Invalid key on .env file: '${key}'` - ) - break + ); + break; } } - return value + return value; + } + + public static tryGet(key: string) { + let value = process.env["container:" + key]; + if (!value) { + value = process.env[key]; + } + return value; } } diff --git a/deploy/core.gbapp/services/GBConversationalService.ts b/deploy/core.gbapp/services/GBConversationalService.ts index 02de13da..2f3daae2 100644 --- a/deploy/core.gbapp/services/GBConversationalService.ts +++ b/deploy/core.gbapp/services/GBConversationalService.ts @@ -146,7 +146,7 @@ export class GBConversationalService implements IGBConversationalService { async checkLanguage(dc, min, text) { let locale = await AzureText.getLocale( min.instance.textAnalyticsKey, - min.instance.textAnalyticsServerUrl, + min.instance.textAnalyticsEndpoint, text ); if (locale != dc.context.activity.locale.split("-")[0]) { diff --git a/deploy/core.gbapp/services/GBCoreService.ts b/deploy/core.gbapp/services/GBCoreService.ts index db24f7d8..787476c9 100644 --- a/deploy/core.gbapp/services/GBCoreService.ts +++ b/deploy/core.gbapp/services/GBCoreService.ts @@ -38,38 +38,24 @@ import { GBConfigService } from "./GBConfigService"; import { IGBInstance, IGBCoreService } from "botlib"; import { GuaribasInstance } from "../models/GBModel"; import { GBAdminService } from "../../admin.gbapp/services/GBAdminService"; -import * as fs from "fs"; -import { AzureDeployerService } from "../../azuredeployer.gbapp/services/AzureDeployerService"; -const msRestAzure = require("ms-rest-azure"); +const processExists = require("process-exists"); + /** * Core service layer. */ export class GBCoreService implements IGBCoreService { - async ensureCloud() { - if (!fs.existsSync(".env")) { - return; - } + isCloudSetup() { + return GBConfigService.tryGet("STORAGE_DIALECT"); + } - logger.warn( - "This mechanism will only work for organizational ids and ids that are not 2FA enabled." - ); - - let credentials = await msRestAzure.loginWithUsernamePassword( - "", - "" - ); - let subscriptionId = ""; - - let s = new AzureDeployerService(credentials, subscriptionId); + async ensureCloud(cloudDeployer) { let instance = new GuaribasInstance(); - await s.deploy(instance, "westus"); + await cloudDeployer.deploy(instance, "westus"); instance.save(); let content = `STORAGE_HOST = ${instance.storageServer}\n STORAGE_NAME, STORAGE_USERNAME, STORAGE_PASSWORD, STORAGE_DIALECT`; - - fs.writeFileSync(".env", content); } /** * Data access layer instance. @@ -315,4 +301,42 @@ export class GBCoreService implements IGBCoreService { return GuaribasInstance.findOne(options); } + + public async ensureProxy(): Promise { + let expiresOn = new Date( + await this.adminService.getValue(0, "proxyExpiresOn") + ); + let proxyAddress; + if (expiresOn.getTime() > new Date().getTime()) { + proxyAddress = await this.adminService.getValue( + GBAdminService.masterBotInstanceId, + "proxyAddress" + ); + return Promise.resolve(proxyAddress); + } else { + if (await processExists("ngrok")) { + logger.warn("ngrok is already running."); + } else { + const { spawn } = require("child_process"); + const child = spawn("node_modules\ngrok\bin\ngrok"); + child.stdout.on("data", data => { + console.log(`child stdout:\n${data}`); + }); + } + + await this.adminService.setValue( + GBAdminService.masterBotInstanceId, + "proxyAddress", + proxyAddress + ); + let now = new Date(); + let expiresOn = now.setHours(now.getHours()); + await this.adminService.setValue( + GBAdminService.masterBotInstanceId, + "proxyExpiresOn", + expiresOn.toString() + ); + return Promise.resolve(proxyAddress); + } + } } diff --git a/deploy/core.gbapp/services/GBMinService.ts b/deploy/core.gbapp/services/GBMinService.ts index b89c2eba..a75ad445 100644 --- a/deploy/core.gbapp/services/GBMinService.ts +++ b/deploy/core.gbapp/services/GBMinService.ts @@ -37,8 +37,6 @@ const UrlJoin = require("url-join"); const express = require("express"); const logger = require("../../../src/logger"); const request = require("request-promise-native"); -const ngrok = require("ngrok"); -var crypto = require("crypto"); var AuthenticationContext = require("adal-node").AuthenticationContext; import { @@ -210,7 +208,7 @@ export class GBMinService { ); authorizationUrl = `${authorizationUrl}?response_type=code&client_id=${ min.instance.authenticatorClientId - }&redirect_uri=${min.instance.botServerUrl}/${ + }&redirect_uri=${min.instance.botEndpoint}/${ min.instance.botId }/token`; @@ -245,7 +243,7 @@ export class GBMinService { authenticationContext.acquireTokenWithAuthorizationCode( req.query.code, - UrlJoin(instance.botServerUrl, min.instance.botId, "/token"), + UrlJoin(instance.botEndpoint, min.instance.botId, "/token"), resource, instance.authenticatorClientId, instance.authenticatorClientSecret, @@ -276,7 +274,7 @@ export class GBMinService { null ); - res.redirect(min.instance.botServerUrl); + res.redirect(min.instance.botEndpoint); } } ); @@ -300,12 +298,6 @@ export class GBMinService { ); } - private async ngrokRefresh() { - const url = await ngrok.connect(9090); // https://757c1652.ngrok.io -> http://localhost:9090 - // TODO: Persist to storage and refresh each 8h. - // TODO: Update all bots definition in azure. - } - private async buildBotAdapter(instance: any) { let adapter = new BotFrameworkAdapter({ appId: instance.marketplaceId, diff --git a/deploy/customer-satisfaction.gbapp/dialogs/FeedbackDialog.ts b/deploy/customer-satisfaction.gbapp/dialogs/FeedbackDialog.ts index 6e7c79d7..920f1ac2 100644 --- a/deploy/customer-satisfaction.gbapp/dialogs/FeedbackDialog.ts +++ b/deploy/customer-satisfaction.gbapp/dialogs/FeedbackDialog.ts @@ -82,7 +82,7 @@ export class FeedbackDialog extends IGBDialog { let locale = dc.context.activity.locale; let rate = await AzureText.getSentiment( min.instance.textAnalyticsKey, - min.instance.textAnalyticsServerUrl, + min.instance.textAnalyticsEndpoint, min.conversationalService.getCurrentLanguage(dc), value ); diff --git a/package.json b/package.json index 0ca31e3f..02db9120 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "botbuilder-prompts": "4.0.0-preview1.2", "botlib": "0.1.3", "chai": "4.2.0", + "child_process": "^1.0.2", "chokidar": "2.0.4", "csv-parse": "3.1.3", "dotenv-extended": "2.3.0", @@ -69,6 +70,7 @@ "nexmo": "2.4.0", "ngrok": "^3.1.0", "pragmatismo-io-framework": "1.0.17", + "process-exists": "^3.1.0", "reflect-metadata": "0.1.12", "request-promise-native": "1.0.5", "scanf": "^1.0.2", diff --git a/src/app.ts b/src/app.ts index 8a5641e4..d1e6e246 100644 --- a/src/app.ts +++ b/src/app.ts @@ -36,7 +36,6 @@ const logger = require("./logger"); const express = require("express"); const bodyParser = require("body-parser"); -const scanf = require('scanf'); import { GBConfigService } from "../deploy/core.gbapp/services/GBConfigService"; import { GBConversationalService } from "../deploy/core.gbapp/services/GBConversationalService"; @@ -54,8 +53,7 @@ import { GBCustomerSatisfactionPackage } from "../deploy/customer-satisfaction.g import { IGBPackage } from "botlib"; import { GBAdminService } from "../deploy/admin.gbapp/services/GBAdminService"; import { GuaribasInstance } from "../deploy/core.gbapp/models/GBModel"; -import { AzureDeployerService } from "deploy/azuredeployer.gbapp/services/AzureDeployerService"; - +import { AzureDeployerService } from "../deploy/azuredeployer.gbapp/services/AzureDeployerService"; let appPackages = new Array(); @@ -63,10 +61,13 @@ let appPackages = new Array(); * General Bots open-core entry point. */ export class GBServer { + /** * Program entry-point. */ + static MASTERBOT_PREFIX = "generalbots-masterbot" + static run() { // Creates a basic HTTP server that will serve several URL, one for each // bot instance. This allows the same server to attend multiple Bot on @@ -93,10 +94,18 @@ export class GBServer { GBConfigService.init(); let core = new GBCoreService(); - let instance = await core.ensureCloud(); + + // Ensures cloud / on-premises infrastructure is setup. + + let cloudDeployer = await AzureDeployerService.ensureDeployer(); + let masterBotName = `${GBServer.MASTERBOT_PREFIX}-${Math.floor( + Math.random() * 1000000000 + )}`; + cloudDeployer.deployFarm(masterBotName, 'westus'); + + // TODO: Get .gb* templates from GitHub and download do additional deploy folder. await core.initDatabase(); - // Boot a bot package if any. @@ -148,7 +157,6 @@ export class GBServer { try { instances = await core.loadInstances(); } catch (error) { - // Check if storage is empty and needs formatting. let isInvalidObject = @@ -157,11 +165,12 @@ export class GBServer { if (isInvalidObject) { if (GBConfigService.get("STORAGE_SYNC") != "true") { throw `Operating storage is out of sync or there is a storage connection error. Try setting STORAGE_SYNC to true in .env file. Error: ${ - error.message + error.message }.`; - } - else { - logger.info(`Storage is empty. After collecting storage structure from all .gbapps it will get synced.`); + } else { + logger.info( + `Storage is empty. After collecting storage structure from all .gbapps it will get synced.` + ); } } else { throw `Cannot connect to operating storage: ${error.message}.`; @@ -179,6 +188,8 @@ export class GBServer { instances = await core.loadInstances(); } + await core.ensureCloud(cloudDeployer); + // Setup server dynamic (per bot instance) resources and listeners. logger.info(`Building minimal instances.`); @@ -198,4 +209,3 @@ export class GBServer { // First line to run. GBServer.run(); -