Advancements in ARM and other repeatable stuff automation.

VERSION.md

@@ -1,5 +1,11 @@
 # Release History
 
+## Version 0.1.7
+
+* Azure Deployer
+* Strategy to replicate itself in several subscriptions.
+* Nkrok experiments to allow 100% automated development environement setup.
+
 ## Version 0.1.6
 
 * Updated packages references.

deploy/admin.gbapp/dialogs/AdminDialog.ts

@@ -156,7 +156,7 @@ export class AdminDialog extends IGBDialog {
       min.instance.authenticatorTenant
     }/oauth2/authorize?client_id=${
       min.instance.authenticatorClientId
-    }&response_type=code&redirect_uri=${min.instance.botServerUrl}/${
+    }&response_type=code&redirect_uri=${min.instance.botEndpoint}/${
       min.instance.botId
     }/token&state=${state}&response_mode=query`;
 

deploy/admin.gbapp/services/GBAdminService.ts

@@ -37,7 +37,11 @@ import { IGBCoreService } from "botlib";
 import { AuthenticationContext, TokenResponse } from "adal-node";
 const UrlJoin = require("url-join");
 
+const ngrok = require("ngrok");
+
 export class GBAdminService {
+  static masterBotInstanceId = 0;
+
   public static StrongRegex = new RegExp(
     "^(?=.*[a-z])(?=.*[A-Z])(?=.*[0-9])(?=.*[!@#\$%\^&\*])(?=.{8,})"
   );

deploy/azuredeployer.gbapp/dialogs/BotFarmDialog.ts

@@ -38,6 +38,8 @@ import { BotAdapter } from "botbuilder";
 import { Messages } from "../strings";
 
 export class BotFarmDialog extends IGBDialog {
+
+  
   /**
    * Setup dialogs flows and define services call.
    *

deploy/azuredeployer.gbapp/services/AzureDeployerService.ts

@@ -46,10 +46,15 @@ import { SearchManagementClient } from "azure-arm-search";
 import { WebResource, ServiceClient } from "ms-rest-js";
 import * as simplegit from "simple-git/promise";
 import { AppServicePlan } from "azure-arm-website/lib/models";
+import { GBConfigService } from "deploy/core.gbapp/services/GBConfigService";
+const scanf = require("scanf");
 const git = simplegit();
 const logger = require("../../../src/logger");
 const UrlJoin = require("url-join");
 const PasswordGenerator = require("strict-password-generator").default;
+const iconUrl =
+  "https://github.com/pragmatismo-io/BotServer/blob/master/docs/images/generalbots-logo-squared.png";
+  
   
 export class AzureDeployerService extends GBService {
   instance: IGBInstance;
@@ -60,8 +65,10 @@ export class AzureDeployerService extends GBService {
   searchClient: SearchManagementClient;
   provider = "Microsoft.BotService";
   subscriptionClient: SubscriptionClient.SubscriptionClient;
+  accessToken: string;
+  location: string;
 
-  constructor(credentials, subscriptionId) {
+  constructor(credentials, subscriptionId, location) {
     super();
     this.resourceClient = new ResourceManagementClient.default(
       credentials,
@@ -77,17 +84,20 @@ export class AzureDeployerService extends GBService {
       subscriptionId
     );
     this.searchClient = new SearchManagementClient(credentials, subscriptionId);
-    this.subscriptionClient = new SubscriptionClient.default(credentials);
+    this.accessToken = credentials.tokenCache._entries[0].accessToken;
+    this.location= location;
   }
 
-  public async getSubscriptions() {
+  public static async getSubscriptions(credentials) {
-    this.subscriptionClient.subscriptions.list();
+    let subscriptionClient = new SubscriptionClient.default(credentials);
+    return subscriptionClient.subscriptions.list();
   }
 
-  public async deploy(
+  public async deployFarm(
-    instance: IGBInstance,
+    name: string,
     location: string
   ): Promise<IGBInstance> {
+    let instance = new IGBInstance();
 
     logger.info(`Creating Deploy...`);
     await this.createDeploy(name, location);
@@ -128,16 +138,17 @@ export class AzureDeployerService extends GBService {
 
     logger.info(`Creating Search...`);
     let search = await this.createSearch(name, `${name}-search`, location);
-
+    instance.searchHost = "generalbots.search.windows.net";
-    logger.info(`Creating Bot...`);
+    instance.searchIndex = "azuresql-index";
-    //await this.createBot(credentials.tokenCache._entries[0].accessToken,
+    instance.searchIndexer = "azuresql-indexer";
-    //  name, name, name, 'global', subscriptionId, tenantId);
+    instance.searchKey = "0FF1CE27564C208555A22B6E278289813";
 
     logger.info(`Creating NLP...`);
     let nlp = await this.createNLP(name, `${name}-nlp`, location);
     let keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name);
     instance.nlpEndpoint = nlp.endpoint;
     instance.nlpKey = keys.key1;
+    instance.nlpAppId = "0ff1ceb4f-96a4-4bdb-b2d5-3ea462ddb773";
 
     logger.info(`Creating Speech...`);
     let speech = await this.createSpeech(name, `${name}-speech`, location);
@@ -168,11 +179,32 @@ export class AzureDeployerService extends GBService {
       name,
       textAnalytics.name
     );
-    instance.textAnalyticsServerUrl = textAnalytics.endpoint;
+    instance.textAnalyticsEndpoint = textAnalytics.endpoint;
     instance.textAnalyticsKey = keys.key1;
 
-    logger.info(`Cleaning Deploy it can take a while...`);
+    return instance;
-    // DISABLED: await this.dangerouslyDeleteDeploy(name);
+  }
+
+  public async deployBot(instance, name, endpoint, nlpAppId, nlpKey, subscriptionId) {
+
+    logger.info(`Creating Bot...`);
+    await this.internalDeployBot(
+      this.accessToken,
+      name,
+      name,
+      name,
+      "General BootBot",
+      endpoint,
+      "global",
+      nlpAppId,
+      nlpKey,
+      subscriptionId
+    );
+    instance.webchatKey = "********";
+    instance.marketplaceId = "0ff1ce73-0aea-442a-a222-dcc340eca294";
+    instance.marketplacePassword = "************";
+
+    return instance;
   }
 
   private async dangerouslyDeleteDeploy(name) {
@@ -216,21 +248,21 @@ export class AzureDeployerService extends GBService {
     let res = await httpClient.sendRequest(req);
   }
 
-  private async createBot(
+  private async internalDeployBot(
     accessToken,
     botId,
     group,
     name,
+    description,
+    endpoint,
     location,
-    subscriptionId,
+    nlpAppId,
-    tenantId
+    nlpKey,
+    subscriptionId
   ) {
     let baseUrl = `https://management.azure.com/`;
-    let appId = "";
+
-    let description = "";
+    let appId = msRestAzure.generateUuid();
-    let endpoint = "";
-    let nlpKey = "";
-    let nlpAppId = "3";
 
     let parameters = {
       parameters: {
@@ -239,14 +271,13 @@ export class AzureDeployerService extends GBService {
           name: "F0"
         },
         name: name,
-        //"type": "sampletype",
         id: botId,
         kind: "sdk",
         properties: {
           description: description,
           displayName: name,
           endpoint: endpoint,
-          iconUrl: "http://myicon",
+          iconUrl: iconUrl,
           luisAppIds: [nlpAppId],
           luisKey: nlpKey,
           msaAppId: appId
@@ -265,7 +296,6 @@ export class AzureDeployerService extends GBService {
     req.headers = {};
     req.headers["Content-Type"] = "application/json";
     req.headers["accept-language"] = "*";
-    //req.headers['x-ms-client-request-id'] = msRestAzure.generateUuid();
     req.headers["Authorization"] = "Bearer " + accessToken;
 
     let requestContent = JSON.stringify(parameters);
@@ -273,6 +303,8 @@ export class AzureDeployerService extends GBService {
 
     let httpClient = new ServiceClient();
     let res = await httpClient.sendRequest(req);
+
+    return JSON.parse(res.bodyAsJson as string);
   }
 
   private async createSearch(group, name, location) {
@@ -454,4 +486,90 @@ export class AzureDeployerService extends GBService {
     let password = passwordGenerator.generatePassword(options);
     return password;
   }
+  
+  static async ensureDeployer() {
+    
+    // Tries do get information from .env file otherwise asks in command-line.
+
+    let username = GBConfigService.get("CLOUD_USERNAME");
+    let password = GBConfigService.get("CLOUD_PASSWORD");
+    let subscriptionId = GBConfigService.get("CLOUD_SUBSCRIPTIONID");
+    let cloudLocation = GBConfigService.get("CLOUD_LOCATION");
+
+    // No .env so asks for cloud credentials to start a new farm.
+
+    if (!username || !password || !subscriptionId || !cloudLocation) {
+      process.stdout.write(
+        "FIRST RUN: A empty enviroment is detected. Please, enter credentials to create a new General Bots Farm."
+      );
+    }
+
+    let retriveUsername = () => {
+      if (!username) {
+        process.stdout.write("CLOUD_USERNAME:");
+        username = scanf("%s");
+      }
+    };
+
+    let retrivePassword = () => {
+      if (!password) {
+        process.stdout.write("CLOUD_PASSWORD:");
+        password = scanf("%s");
+      }
+    };
+
+    while (!username) {
+      retriveUsername();
+    }
+
+    while (!password) {
+      retrivePassword();
+    }
+
+    // Connects to the cloud and retrives subscriptions.
+
+    let credentials = await msRestAzure.loginWithUsernamePassword(
+      username,
+      password
+    );
+    let list = await AzureDeployerService.getSubscriptions(credentials);
+
+    let map = {};
+    let index = 1;
+    list.forEach(element => {
+      console.log(
+        `${index}: ${element.displayName} (${element.subscriptionId})`
+      );
+      map[index++] = element;
+    });
+
+    let subscriptionIndex;
+    let retrieveSubscription = () => {
+      if (!subscriptionIndex) {
+        process.stdout.write("CLOUD_SUBSCRIPTIONID (type a number):");
+        subscriptionIndex = scanf("%d");
+      }
+    };
+    if (!subscriptionId) {
+      while (!subscriptionIndex) {
+        retrieveSubscription();
+      }
+      subscriptionId = map[subscriptionIndex].subscriptionId;
+    }
+
+
+    let retriveLocation = () => {
+      if (!location) {
+        process.stdout.write("CLOUD_LOCATION:");
+        location = scanf("%s");
+      }
+    };
+
+    while (!location) {
+      retriveLocation();
+    }
+
+    return new AzureDeployerService(credentials, subscriptionId, location);
+  }
+  
 }

deploy/core.gbapp/models/GBModel.ts

@@ -67,7 +67,7 @@ export class GuaribasInstance extends Model<GuaribasInstance>
   instanceId: number;
 
   @Column
-  botServerUrl: string;
+  botEndpoint: string;
 
   @Column
   whoAmIVideo: string;
@@ -99,7 +99,7 @@ export class GuaribasInstance extends Model<GuaribasInstance>
   textAnalyticsKey: string;
 
   @Column
-  textAnalyticsServerUrl: string;
+  textAnalyticsEndpoint: string;
 
   @Column
   marketplacePassword: string;

deploy/core.gbapp/services/GBConfigService.ts

@@ -30,11 +30,13 @@
 |                                                                             |
 \*****************************************************************************/
 
-const logger = require("../../../src/logger")
+const logger = require("../../../src/logger");
+import * as fs from "fs";
 
-"use strict"
+"use strict";
 
 export class GBConfigService {
+
   static init(): any {
     try {
       require("dotenv-extended").load({
@@ -42,55 +44,66 @@ export class GBConfigService {
         errorOnMissing: true,
         errorOnExtra: false,
         overrideProcessEnv: true
-      })
+      });
     } catch (e) {
-      console.error(e.message)
+      console.error(e.message);
-      process.exit(3)
+      process.exit(3);
+    }
+  }
+
+  public writeEntry(name, value) {
+
+    if (fs.exists) {
+      fs.appendFileSync('.env',`${name}=${value}`);
     }
   }
 
   static get(key: string): string | undefined {
-    let value = process.env["container:" + key]
+    let value = GBConfigService.tryGet(key);
-
-    if (!value) {
-      value = process.env[key]
-    }
 
     if (!value) {
       switch (key) {
         case "STORAGE_DIALECT":
-          value = "sqlite"
+          value = "sqlite";
-          break
+          break;
 
         case "STORAGE_STORAGE":
-          value = "./guaribas.sqlite"
+          value = "./guaribas.sqlite";
-          break
+          break;
 
         case "ADDITIONAL_DEPLOY_PATH":
-          value = undefined
+          value = undefined;
-          break
+          break;
 
         case "STORAGE_SYNC":
         case "STORAGE_SYNC_ALTER":
         case "STORAGE_SYNC_FORCE":
-          value = "false"
+          value = "false";
-          break
+          break;
 
         case "STORAGE_LOGGING":
-          value = "false"
+          value = "false";
-          break
+          break;
 
         case "STORAGE_ENCRYPT":
-          value = "true"
+          value = "true";
-          break
+          break;
 
         default:
           logger.info(
             `Guaribas General Error: Invalid key on .env file: '${key}'`
-          )
+          );
-          break
+          break;
       }
     }
-    return value
+    return value;
+  }
+
+  public static tryGet(key: string) {
+    let value = process.env["container:" + key];
+    if (!value) {
+      value = process.env[key];
+    }
+    return value;
   }
 }

deploy/core.gbapp/services/GBConversationalService.ts

@@ -146,7 +146,7 @@ export class GBConversationalService implements IGBConversationalService {
   async checkLanguage(dc, min, text) {
     let locale = await AzureText.getLocale(
       min.instance.textAnalyticsKey,
-      min.instance.textAnalyticsServerUrl,
+      min.instance.textAnalyticsEndpoint,
       text
     );
     if (locale != dc.context.activity.locale.split("-")[0]) {

deploy/core.gbapp/services/GBCoreService.ts

@@ -38,38 +38,24 @@ import { GBConfigService } from "./GBConfigService";
 import { IGBInstance, IGBCoreService } from "botlib";
 import { GuaribasInstance } from "../models/GBModel";
 import { GBAdminService } from "../../admin.gbapp/services/GBAdminService";
-import * as fs from "fs";
+const processExists = require("process-exists");
-import { AzureDeployerService } from "../../azuredeployer.gbapp/services/AzureDeployerService";
+
-const msRestAzure = require("ms-rest-azure");
 
 /**
  *  Core service layer.
  */
 export class GBCoreService implements IGBCoreService {
-  async ensureCloud() {
+  isCloudSetup() {
-    if (!fs.existsSync(".env")) {
+    return GBConfigService.tryGet("STORAGE_DIALECT");
-      return;
   }
 
-    logger.warn(
+  async ensureCloud(cloudDeployer) {
-      "This mechanism will only work for organizational ids and ids that are not 2FA enabled."
-    );
-
-    let credentials = await msRestAzure.loginWithUsernamePassword(
-      "",
-      ""
-    );
-    let subscriptionId = "";
-
-    let s = new AzureDeployerService(credentials, subscriptionId);
     let instance = new GuaribasInstance();
-    await s.deploy(instance, "westus");
+    await cloudDeployer.deploy(instance, "westus");
     instance.save();
 
     let content = `STORAGE_HOST = ${instance.storageServer}\n
     STORAGE_NAME, STORAGE_USERNAME, STORAGE_PASSWORD, STORAGE_DIALECT`;
-
-    fs.writeFileSync(".env", content);
   }
   /**
    * Data access layer instance.
@@ -315,4 +301,42 @@ export class GBCoreService implements IGBCoreService {
 
     return GuaribasInstance.findOne(options);
   }
+
+  public async ensureProxy(): Promise<string> {
+    let expiresOn = new Date(
+      await this.adminService.getValue(0, "proxyExpiresOn")
+    );
+    let proxyAddress;
+    if (expiresOn.getTime() > new Date().getTime()) {
+      proxyAddress = await this.adminService.getValue(
+        GBAdminService.masterBotInstanceId,
+        "proxyAddress"
+      );
+      return Promise.resolve(proxyAddress);
+    } else {
+      if (await processExists("ngrok")) {
+        logger.warn("ngrok is already running.");
+      } else {
+        const { spawn } = require("child_process");
+        const child = spawn("node_modules\ngrok\bin\ngrok");
+        child.stdout.on("data", data => {
+          console.log(`child stdout:\n${data}`);
+        });
+      }
+
+      await this.adminService.setValue(
+        GBAdminService.masterBotInstanceId,
+        "proxyAddress",
+        proxyAddress
+      );
+      let now = new Date();
+      let expiresOn = now.setHours(now.getHours());
+      await this.adminService.setValue(
+        GBAdminService.masterBotInstanceId,
+        "proxyExpiresOn",
+        expiresOn.toString()
+      );
+      return Promise.resolve(proxyAddress);
+    }
+  }
 }

deploy/core.gbapp/services/GBMinService.ts

@@ -37,8 +37,6 @@ const UrlJoin = require("url-join");
 const express = require("express");
 const logger = require("../../../src/logger");
 const request = require("request-promise-native");
-const ngrok = require("ngrok");
-var crypto = require("crypto");
 var AuthenticationContext = require("adal-node").AuthenticationContext;
 
 import {
@@ -210,7 +208,7 @@ export class GBMinService {
           );
           authorizationUrl = `${authorizationUrl}?response_type=code&client_id=${
             min.instance.authenticatorClientId
-          }&redirect_uri=${min.instance.botServerUrl}/${
+          }&redirect_uri=${min.instance.botEndpoint}/${
             min.instance.botId
           }/token`;
 
@@ -245,7 +243,7 @@ export class GBMinService {
 
           authenticationContext.acquireTokenWithAuthorizationCode(
             req.query.code,
-            UrlJoin(instance.botServerUrl, min.instance.botId, "/token"),
+            UrlJoin(instance.botEndpoint, min.instance.botId, "/token"),
             resource,
             instance.authenticatorClientId,
             instance.authenticatorClientSecret,
@@ -276,7 +274,7 @@ export class GBMinService {
                   null
                 );
 
-                res.redirect(min.instance.botServerUrl);
+                res.redirect(min.instance.botEndpoint);
               }
             }
           );
@@ -300,12 +298,6 @@ export class GBMinService {
     );
   }
 
-  private async ngrokRefresh() {
-    const url = await ngrok.connect(9090); // https://757c1652.ngrok.io -> http://localhost:9090
-    // TODO: Persist to storage and refresh each 8h.
-    // TODO: Update all bots definition in azure.
-  }
-
   private async buildBotAdapter(instance: any) {
     let adapter = new BotFrameworkAdapter({
       appId: instance.marketplaceId,

deploy/customer-satisfaction.gbapp/dialogs/FeedbackDialog.ts

@@ -82,7 +82,7 @@ export class FeedbackDialog extends IGBDialog {
         let locale = dc.context.activity.locale;
         let rate = await AzureText.getSentiment(
           min.instance.textAnalyticsKey,
-          min.instance.textAnalyticsServerUrl,
+          min.instance.textAnalyticsEndpoint,
           min.conversationalService.getCurrentLanguage(dc),
           value
         );

package.json

@@ -53,6 +53,7 @@
     "botbuilder-prompts": "4.0.0-preview1.2",
     "botlib": "0.1.3",
     "chai": "4.2.0",
+    "child_process": "^1.0.2",
     "chokidar": "2.0.4",
     "csv-parse": "3.1.3",
     "dotenv-extended": "2.3.0",
@@ -69,6 +70,7 @@
     "nexmo": "2.4.0",
     "ngrok": "^3.1.0",
     "pragmatismo-io-framework": "1.0.17",
+    "process-exists": "^3.1.0",
     "reflect-metadata": "0.1.12",
     "request-promise-native": "1.0.5",
     "scanf": "^1.0.2",

src/app.ts

@@ -36,7 +36,6 @@
 const logger = require("./logger");
 const express = require("express");
 const bodyParser = require("body-parser");
-const scanf = require('scanf');
 
 import { GBConfigService } from "../deploy/core.gbapp/services/GBConfigService";
 import { GBConversationalService } from "../deploy/core.gbapp/services/GBConversationalService";
@@ -54,8 +53,7 @@ import { GBCustomerSatisfactionPackage } from "../deploy/customer-satisfaction.g
 import { IGBPackage } from "botlib";
 import { GBAdminService } from "../deploy/admin.gbapp/services/GBAdminService";
 import { GuaribasInstance } from "../deploy/core.gbapp/models/GBModel";
-import { AzureDeployerService } from "deploy/azuredeployer.gbapp/services/AzureDeployerService";
+import { AzureDeployerService } from "../deploy/azuredeployer.gbapp/services/AzureDeployerService";
-
 
 let appPackages = new Array<IGBPackage>();
 
@@ -63,10 +61,13 @@ let appPackages = new Array<IGBPackage>();
  * General Bots open-core entry point.
  */
 export class GBServer {
+
   /**
    *  Program entry-point.
    */
 
+  static MASTERBOT_PREFIX = "generalbots-masterbot"
+
   static run() {
     // Creates a basic HTTP server that will serve several URL, one for each
     // bot instance. This allows the same server to attend multiple Bot on
@@ -93,11 +94,19 @@ export class GBServer {
 
           GBConfigService.init();
           let core = new GBCoreService();
-          let instance = await core.ensureCloud();
+
+          // Ensures cloud / on-premises infrastructure is setup.
+
+          let cloudDeployer = await AzureDeployerService.ensureDeployer();
+          let masterBotName = `${GBServer.MASTERBOT_PREFIX}-${Math.floor(
+            Math.random() * 1000000000
+          )}`;
+          cloudDeployer.deployFarm(masterBotName, 'westus');
+          
+          // TODO: Get .gb* templates from GitHub and download do additional deploy folder.
 
           await core.initDatabase();
 
-
           // Boot a bot package if any.
 
           logger.info(`Starting instances...`);
@@ -148,7 +157,6 @@ export class GBServer {
           try {
             instances = await core.loadInstances();
           } catch (error) {
-
             // Check if storage is empty and needs formatting.
 
             let isInvalidObject =
@@ -159,9 +167,10 @@ export class GBServer {
                 throw `Operating storage is out of sync or there is a storage connection error. Try setting STORAGE_SYNC to true in .env file. Error: ${
                   error.message
                 }.`;
-              }
+              } else {
-              else {
+                logger.info(
-                logger.info(`Storage is empty. After collecting storage structure from all .gbapps it will get synced.`);
+                  `Storage is empty. After collecting storage structure from all .gbapps it will get synced.`
+                );
               }
             } else {
               throw `Cannot connect to operating storage: ${error.message}.`;
@@ -179,6 +188,8 @@ export class GBServer {
             instances = await core.loadInstances();
           }
 
+          await core.ensureCloud(cloudDeployer);
+
           // Setup server dynamic (per bot instance) resources and listeners.
 
           logger.info(`Building minimal instances.`);
@@ -198,4 +209,3 @@ export class GBServer {
 // First line to run.
 
 GBServer.run();
-