Advancements in ARM and other repeatable stuff automation.

This commit is contained in:
Rodrigo Rodriguez (pragmatismo.io) 2018-10-15 19:05:43 -03:00
parent 7ef4e22764
commit a7142c5cfe
13 changed files with 269 additions and 98 deletions

View file

@ -1,5 +1,11 @@
# Release History
## Version 0.1.7
* Azure Deployer
* Strategy to replicate itself in several subscriptions.
* Nkrok experiments to allow 100% automated development environement setup.
## Version 0.1.6
* Updated packages references.

View file

@ -156,7 +156,7 @@ export class AdminDialog extends IGBDialog {
min.instance.authenticatorTenant
}/oauth2/authorize?client_id=${
min.instance.authenticatorClientId
}&response_type=code&redirect_uri=${min.instance.botServerUrl}/${
}&response_type=code&redirect_uri=${min.instance.botEndpoint}/${
min.instance.botId
}/token&state=${state}&response_mode=query`;

View file

@ -37,7 +37,11 @@ import { IGBCoreService } from "botlib";
import { AuthenticationContext, TokenResponse } from "adal-node";
const UrlJoin = require("url-join");
const ngrok = require("ngrok");
export class GBAdminService {
static masterBotInstanceId = 0;
public static StrongRegex = new RegExp(
"^(?=.*[a-z])(?=.*[A-Z])(?=.*[0-9])(?=.*[!@#\$%\^&\*])(?=.{8,})"
);

View file

@ -38,6 +38,8 @@ import { BotAdapter } from "botbuilder";
import { Messages } from "../strings";
export class BotFarmDialog extends IGBDialog {
/**
* Setup dialogs flows and define services call.
*

View file

@ -46,11 +46,16 @@ import { SearchManagementClient } from "azure-arm-search";
import { WebResource, ServiceClient } from "ms-rest-js";
import * as simplegit from "simple-git/promise";
import { AppServicePlan } from "azure-arm-website/lib/models";
import { GBConfigService } from "deploy/core.gbapp/services/GBConfigService";
const scanf = require("scanf");
const git = simplegit();
const logger = require("../../../src/logger");
const UrlJoin = require("url-join");
const PasswordGenerator = require("strict-password-generator").default;
const iconUrl =
"https://github.com/pragmatismo-io/BotServer/blob/master/docs/images/generalbots-logo-squared.png";
export class AzureDeployerService extends GBService {
instance: IGBInstance;
resourceClient: ResourceManagementClient.ResourceManagementClient;
@ -60,8 +65,10 @@ export class AzureDeployerService extends GBService {
searchClient: SearchManagementClient;
provider = "Microsoft.BotService";
subscriptionClient: SubscriptionClient.SubscriptionClient;
accessToken: string;
location: string;
constructor(credentials, subscriptionId) {
constructor(credentials, subscriptionId, location) {
super();
this.resourceClient = new ResourceManagementClient.default(
credentials,
@ -77,17 +84,20 @@ export class AzureDeployerService extends GBService {
subscriptionId
);
this.searchClient = new SearchManagementClient(credentials, subscriptionId);
this.subscriptionClient = new SubscriptionClient.default(credentials);
this.accessToken = credentials.tokenCache._entries[0].accessToken;
this.location= location;
}
public async getSubscriptions() {
this.subscriptionClient.subscriptions.list();
public static async getSubscriptions(credentials) {
let subscriptionClient = new SubscriptionClient.default(credentials);
return subscriptionClient.subscriptions.list();
}
public async deploy(
instance: IGBInstance,
public async deployFarm(
name: string,
location: string
): Promise<IGBInstance> {
let instance = new IGBInstance();
logger.info(`Creating Deploy...`);
await this.createDeploy(name, location);
@ -128,16 +138,17 @@ export class AzureDeployerService extends GBService {
logger.info(`Creating Search...`);
let search = await this.createSearch(name, `${name}-search`, location);
logger.info(`Creating Bot...`);
//await this.createBot(credentials.tokenCache._entries[0].accessToken,
// name, name, name, 'global', subscriptionId, tenantId);
instance.searchHost = "generalbots.search.windows.net";
instance.searchIndex = "azuresql-index";
instance.searchIndexer = "azuresql-indexer";
instance.searchKey = "0FF1CE27564C208555A22B6E278289813";
logger.info(`Creating NLP...`);
let nlp = await this.createNLP(name, `${name}-nlp`, location);
let keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name);
instance.nlpEndpoint = nlp.endpoint;
instance.nlpKey = keys.key1;
instance.nlpAppId = "0ff1ceb4f-96a4-4bdb-b2d5-3ea462ddb773";
logger.info(`Creating Speech...`);
let speech = await this.createSpeech(name, `${name}-speech`, location);
@ -168,11 +179,32 @@ export class AzureDeployerService extends GBService {
name,
textAnalytics.name
);
instance.textAnalyticsServerUrl = textAnalytics.endpoint;
instance.textAnalyticsEndpoint = textAnalytics.endpoint;
instance.textAnalyticsKey = keys.key1;
logger.info(`Cleaning Deploy it can take a while...`);
// DISABLED: await this.dangerouslyDeleteDeploy(name);
return instance;
}
public async deployBot(instance, name, endpoint, nlpAppId, nlpKey, subscriptionId) {
logger.info(`Creating Bot...`);
await this.internalDeployBot(
this.accessToken,
name,
name,
name,
"General BootBot",
endpoint,
"global",
nlpAppId,
nlpKey,
subscriptionId
);
instance.webchatKey = "********";
instance.marketplaceId = "0ff1ce73-0aea-442a-a222-dcc340eca294";
instance.marketplacePassword = "************";
return instance;
}
private async dangerouslyDeleteDeploy(name) {
@ -216,21 +248,21 @@ export class AzureDeployerService extends GBService {
let res = await httpClient.sendRequest(req);
}
private async createBot(
private async internalDeployBot(
accessToken,
botId,
group,
name,
description,
endpoint,
location,
subscriptionId,
tenantId
nlpAppId,
nlpKey,
subscriptionId
) {
let baseUrl = `https://management.azure.com/`;
let appId = "";
let description = "";
let endpoint = "";
let nlpKey = "";
let nlpAppId = "3";
let appId = msRestAzure.generateUuid();
let parameters = {
parameters: {
@ -239,14 +271,13 @@ export class AzureDeployerService extends GBService {
name: "F0"
},
name: name,
//"type": "sampletype",
id: botId,
kind: "sdk",
properties: {
description: description,
displayName: name,
endpoint: endpoint,
iconUrl: "http://myicon",
iconUrl: iconUrl,
luisAppIds: [nlpAppId],
luisKey: nlpKey,
msaAppId: appId
@ -265,7 +296,6 @@ export class AzureDeployerService extends GBService {
req.headers = {};
req.headers["Content-Type"] = "application/json";
req.headers["accept-language"] = "*";
//req.headers['x-ms-client-request-id'] = msRestAzure.generateUuid();
req.headers["Authorization"] = "Bearer " + accessToken;
let requestContent = JSON.stringify(parameters);
@ -273,6 +303,8 @@ export class AzureDeployerService extends GBService {
let httpClient = new ServiceClient();
let res = await httpClient.sendRequest(req);
return JSON.parse(res.bodyAsJson as string);
}
private async createSearch(group, name, location) {
@ -454,4 +486,90 @@ export class AzureDeployerService extends GBService {
let password = passwordGenerator.generatePassword(options);
return password;
}
static async ensureDeployer() {
// Tries do get information from .env file otherwise asks in command-line.
let username = GBConfigService.get("CLOUD_USERNAME");
let password = GBConfigService.get("CLOUD_PASSWORD");
let subscriptionId = GBConfigService.get("CLOUD_SUBSCRIPTIONID");
let cloudLocation = GBConfigService.get("CLOUD_LOCATION");
// No .env so asks for cloud credentials to start a new farm.
if (!username || !password || !subscriptionId || !cloudLocation) {
process.stdout.write(
"FIRST RUN: A empty enviroment is detected. Please, enter credentials to create a new General Bots Farm."
);
}
let retriveUsername = () => {
if (!username) {
process.stdout.write("CLOUD_USERNAME:");
username = scanf("%s");
}
};
let retrivePassword = () => {
if (!password) {
process.stdout.write("CLOUD_PASSWORD:");
password = scanf("%s");
}
};
while (!username) {
retriveUsername();
}
while (!password) {
retrivePassword();
}
// Connects to the cloud and retrives subscriptions.
let credentials = await msRestAzure.loginWithUsernamePassword(
username,
password
);
let list = await AzureDeployerService.getSubscriptions(credentials);
let map = {};
let index = 1;
list.forEach(element => {
console.log(
`${index}: ${element.displayName} (${element.subscriptionId})`
);
map[index++] = element;
});
let subscriptionIndex;
let retrieveSubscription = () => {
if (!subscriptionIndex) {
process.stdout.write("CLOUD_SUBSCRIPTIONID (type a number):");
subscriptionIndex = scanf("%d");
}
};
if (!subscriptionId) {
while (!subscriptionIndex) {
retrieveSubscription();
}
subscriptionId = map[subscriptionIndex].subscriptionId;
}
let retriveLocation = () => {
if (!location) {
process.stdout.write("CLOUD_LOCATION:");
location = scanf("%s");
}
};
while (!location) {
retriveLocation();
}
return new AzureDeployerService(credentials, subscriptionId, location);
}
}

View file

@ -67,7 +67,7 @@ export class GuaribasInstance extends Model<GuaribasInstance>
instanceId: number;
@Column
botServerUrl: string;
botEndpoint: string;
@Column
whoAmIVideo: string;
@ -99,7 +99,7 @@ export class GuaribasInstance extends Model<GuaribasInstance>
textAnalyticsKey: string;
@Column
textAnalyticsServerUrl: string;
textAnalyticsEndpoint: string;
@Column
marketplacePassword: string;

View file

@ -30,11 +30,13 @@
| |
\*****************************************************************************/
const logger = require("../../../src/logger")
const logger = require("../../../src/logger");
import * as fs from "fs";
"use strict"
"use strict";
export class GBConfigService {
static init(): any {
try {
require("dotenv-extended").load({
@ -42,55 +44,66 @@ export class GBConfigService {
errorOnMissing: true,
errorOnExtra: false,
overrideProcessEnv: true
})
});
} catch (e) {
console.error(e.message)
process.exit(3)
console.error(e.message);
process.exit(3);
}
}
public writeEntry(name, value) {
if (fs.exists) {
fs.appendFileSync('.env',`${name}=${value}`);
}
}
static get(key: string): string | undefined {
let value = process.env["container:" + key]
if (!value) {
value = process.env[key]
}
let value = GBConfigService.tryGet(key);
if (!value) {
switch (key) {
case "STORAGE_DIALECT":
value = "sqlite"
break
value = "sqlite";
break;
case "STORAGE_STORAGE":
value = "./guaribas.sqlite"
break
value = "./guaribas.sqlite";
break;
case "ADDITIONAL_DEPLOY_PATH":
value = undefined
break
value = undefined;
break;
case "STORAGE_SYNC":
case "STORAGE_SYNC_ALTER":
case "STORAGE_SYNC_FORCE":
value = "false"
break
value = "false";
break;
case "STORAGE_LOGGING":
value = "false"
break
value = "false";
break;
case "STORAGE_ENCRYPT":
value = "true"
break
value = "true";
break;
default:
logger.info(
`Guaribas General Error: Invalid key on .env file: '${key}'`
)
break
);
break;
}
}
return value
return value;
}
public static tryGet(key: string) {
let value = process.env["container:" + key];
if (!value) {
value = process.env[key];
}
return value;
}
}

View file

@ -146,7 +146,7 @@ export class GBConversationalService implements IGBConversationalService {
async checkLanguage(dc, min, text) {
let locale = await AzureText.getLocale(
min.instance.textAnalyticsKey,
min.instance.textAnalyticsServerUrl,
min.instance.textAnalyticsEndpoint,
text
);
if (locale != dc.context.activity.locale.split("-")[0]) {

View file

@ -38,38 +38,24 @@ import { GBConfigService } from "./GBConfigService";
import { IGBInstance, IGBCoreService } from "botlib";
import { GuaribasInstance } from "../models/GBModel";
import { GBAdminService } from "../../admin.gbapp/services/GBAdminService";
import * as fs from "fs";
import { AzureDeployerService } from "../../azuredeployer.gbapp/services/AzureDeployerService";
const msRestAzure = require("ms-rest-azure");
const processExists = require("process-exists");
/**
* Core service layer.
*/
export class GBCoreService implements IGBCoreService {
async ensureCloud() {
if (!fs.existsSync(".env")) {
return;
}
isCloudSetup() {
return GBConfigService.tryGet("STORAGE_DIALECT");
}
logger.warn(
"This mechanism will only work for organizational ids and ids that are not 2FA enabled."
);
let credentials = await msRestAzure.loginWithUsernamePassword(
"",
""
);
let subscriptionId = "";
let s = new AzureDeployerService(credentials, subscriptionId);
async ensureCloud(cloudDeployer) {
let instance = new GuaribasInstance();
await s.deploy(instance, "westus");
await cloudDeployer.deploy(instance, "westus");
instance.save();
let content = `STORAGE_HOST = ${instance.storageServer}\n
STORAGE_NAME, STORAGE_USERNAME, STORAGE_PASSWORD, STORAGE_DIALECT`;
fs.writeFileSync(".env", content);
}
/**
* Data access layer instance.
@ -315,4 +301,42 @@ export class GBCoreService implements IGBCoreService {
return GuaribasInstance.findOne(options);
}
public async ensureProxy(): Promise<string> {
let expiresOn = new Date(
await this.adminService.getValue(0, "proxyExpiresOn")
);
let proxyAddress;
if (expiresOn.getTime() > new Date().getTime()) {
proxyAddress = await this.adminService.getValue(
GBAdminService.masterBotInstanceId,
"proxyAddress"
);
return Promise.resolve(proxyAddress);
} else {
if (await processExists("ngrok")) {
logger.warn("ngrok is already running.");
} else {
const { spawn } = require("child_process");
const child = spawn("node_modules\ngrok\bin\ngrok");
child.stdout.on("data", data => {
console.log(`child stdout:\n${data}`);
});
}
await this.adminService.setValue(
GBAdminService.masterBotInstanceId,
"proxyAddress",
proxyAddress
);
let now = new Date();
let expiresOn = now.setHours(now.getHours());
await this.adminService.setValue(
GBAdminService.masterBotInstanceId,
"proxyExpiresOn",
expiresOn.toString()
);
return Promise.resolve(proxyAddress);
}
}
}

View file

@ -37,8 +37,6 @@ const UrlJoin = require("url-join");
const express = require("express");
const logger = require("../../../src/logger");
const request = require("request-promise-native");
const ngrok = require("ngrok");
var crypto = require("crypto");
var AuthenticationContext = require("adal-node").AuthenticationContext;
import {
@ -210,7 +208,7 @@ export class GBMinService {
);
authorizationUrl = `${authorizationUrl}?response_type=code&client_id=${
min.instance.authenticatorClientId
}&redirect_uri=${min.instance.botServerUrl}/${
}&redirect_uri=${min.instance.botEndpoint}/${
min.instance.botId
}/token`;
@ -245,7 +243,7 @@ export class GBMinService {
authenticationContext.acquireTokenWithAuthorizationCode(
req.query.code,
UrlJoin(instance.botServerUrl, min.instance.botId, "/token"),
UrlJoin(instance.botEndpoint, min.instance.botId, "/token"),
resource,
instance.authenticatorClientId,
instance.authenticatorClientSecret,
@ -276,7 +274,7 @@ export class GBMinService {
null
);
res.redirect(min.instance.botServerUrl);
res.redirect(min.instance.botEndpoint);
}
}
);
@ -300,12 +298,6 @@ export class GBMinService {
);
}
private async ngrokRefresh() {
const url = await ngrok.connect(9090); // https://757c1652.ngrok.io -> http://localhost:9090
// TODO: Persist to storage and refresh each 8h.
// TODO: Update all bots definition in azure.
}
private async buildBotAdapter(instance: any) {
let adapter = new BotFrameworkAdapter({
appId: instance.marketplaceId,

View file

@ -82,7 +82,7 @@ export class FeedbackDialog extends IGBDialog {
let locale = dc.context.activity.locale;
let rate = await AzureText.getSentiment(
min.instance.textAnalyticsKey,
min.instance.textAnalyticsServerUrl,
min.instance.textAnalyticsEndpoint,
min.conversationalService.getCurrentLanguage(dc),
value
);

View file

@ -53,6 +53,7 @@
"botbuilder-prompts": "4.0.0-preview1.2",
"botlib": "0.1.3",
"chai": "4.2.0",
"child_process": "^1.0.2",
"chokidar": "2.0.4",
"csv-parse": "3.1.3",
"dotenv-extended": "2.3.0",
@ -69,6 +70,7 @@
"nexmo": "2.4.0",
"ngrok": "^3.1.0",
"pragmatismo-io-framework": "1.0.17",
"process-exists": "^3.1.0",
"reflect-metadata": "0.1.12",
"request-promise-native": "1.0.5",
"scanf": "^1.0.2",

View file

@ -36,7 +36,6 @@
const logger = require("./logger");
const express = require("express");
const bodyParser = require("body-parser");
const scanf = require('scanf');
import { GBConfigService } from "../deploy/core.gbapp/services/GBConfigService";
import { GBConversationalService } from "../deploy/core.gbapp/services/GBConversationalService";
@ -54,8 +53,7 @@ import { GBCustomerSatisfactionPackage } from "../deploy/customer-satisfaction.g
import { IGBPackage } from "botlib";
import { GBAdminService } from "../deploy/admin.gbapp/services/GBAdminService";
import { GuaribasInstance } from "../deploy/core.gbapp/models/GBModel";
import { AzureDeployerService } from "deploy/azuredeployer.gbapp/services/AzureDeployerService";
import { AzureDeployerService } from "../deploy/azuredeployer.gbapp/services/AzureDeployerService";
let appPackages = new Array<IGBPackage>();
@ -63,10 +61,13 @@ let appPackages = new Array<IGBPackage>();
* General Bots open-core entry point.
*/
export class GBServer {
/**
* Program entry-point.
*/
static MASTERBOT_PREFIX = "generalbots-masterbot"
static run() {
// Creates a basic HTTP server that will serve several URL, one for each
// bot instance. This allows the same server to attend multiple Bot on
@ -93,10 +94,18 @@ export class GBServer {
GBConfigService.init();
let core = new GBCoreService();
let instance = await core.ensureCloud();
// Ensures cloud / on-premises infrastructure is setup.
let cloudDeployer = await AzureDeployerService.ensureDeployer();
let masterBotName = `${GBServer.MASTERBOT_PREFIX}-${Math.floor(
Math.random() * 1000000000
)}`;
cloudDeployer.deployFarm(masterBotName, 'westus');
// TODO: Get .gb* templates from GitHub and download do additional deploy folder.
await core.initDatabase();
// Boot a bot package if any.
@ -148,7 +157,6 @@ export class GBServer {
try {
instances = await core.loadInstances();
} catch (error) {
// Check if storage is empty and needs formatting.
let isInvalidObject =
@ -157,11 +165,12 @@ export class GBServer {
if (isInvalidObject) {
if (GBConfigService.get("STORAGE_SYNC") != "true") {
throw `Operating storage is out of sync or there is a storage connection error. Try setting STORAGE_SYNC to true in .env file. Error: ${
error.message
error.message
}.`;
}
else {
logger.info(`Storage is empty. After collecting storage structure from all .gbapps it will get synced.`);
} else {
logger.info(
`Storage is empty. After collecting storage structure from all .gbapps it will get synced.`
);
}
} else {
throw `Cannot connect to operating storage: ${error.message}.`;
@ -179,6 +188,8 @@ export class GBServer {
instances = await core.loadInstances();
}
await core.ensureCloud(cloudDeployer);
// Setup server dynamic (per bot instance) resources and listeners.
logger.info(`Building minimal instances.`);
@ -198,4 +209,3 @@ export class GBServer {
// First line to run.
GBServer.run();