Plug and play dev. environment in progress.

This commit is contained in:
Rodrigo Rodriguez (pragmatismo.io) 2018-10-22 15:33:23 -03:00
parent 4011edfb19
commit c3d49e3288
5 changed files with 278 additions and 209 deletions

View file

@ -56,12 +56,6 @@ const PasswordGenerator = require("strict-password-generator").default;
const iconUrl =
"https://github.com/pragmatismo-io/BotServer/blob/master/docs/images/generalbots-logo-squared.png";
class x implements IGBInstance {
};
export class AzureDeployerService extends GBService {
instance: IGBInstance;
resourceClient: ResourceManagementClient.ResourceManagementClient;
@ -73,6 +67,7 @@ export class AzureDeployerService extends GBService {
subscriptionClient: SubscriptionClient.SubscriptionClient;
accessToken: string;
location: string;
public subscriptionId: string;
constructor(credentials, subscriptionId, location) {
super();
@ -92,6 +87,7 @@ export class AzureDeployerService extends GBService {
this.searchClient = new SearchManagementClient(credentials, subscriptionId);
this.accessToken = credentials.tokenCache._entries[0].accessToken;
this.location = location;
this.subscriptionId = subscriptionId;
}
public static async getSubscriptions(credentials) {
@ -101,16 +97,32 @@ export class AzureDeployerService extends GBService {
public async deployFarm(
name: string,
location: string
location: string,
proxyAddress: string
): Promise<IGBInstance> {
let instance:any = {};
let instance: any = {};
let culture = "en-us";
logger.info(`Starting infrastructure deployment...`);
// TODO: REMOVE THIS*********
let keys: any;
logger.info(`Deploying NLP...`);
let nlp = await this.createNLP(name, `${name}-nlp`, location);
keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name);
let nlpAppId = await this.createLUISApp(name, name, location, culture);
//await this.dangerouslyDeleteDeploy(name);
await this.deployBootBot(
instance,
name,
proxyAddress,
nlpAppId,
keys.key1,
this.subscriptionId
);
instance.nlpEndpoint = nlp.endpoint;
instance.nlpKey = keys.key1;
instance.nlpAppId = nlpAppId;
logger.info(`Deploying Deploy Group...`);
await this.createDeployGroup(name, location);
@ -151,19 +163,18 @@ export class AzureDeployerService extends GBService {
logger.info(`Deploying Search...`);
let searchName = `${name}-search`;
let search = await this.createSearch(name,searchName , location);
let searchKeys = await this.searchClient.queryKeys.listBySearchService(name, searchName)
await this.createSearch(name, searchName, location);
let searchKeys = await this.searchClient.queryKeys.listBySearchService(
name,
searchName
);
instance.searchHost = `${searchName}.search.windows.net`;
instance.searchIndex = "azuresql-index";
instance.searchIndexer = "azuresql-indexer";
instance.searchKey = searchKeys[0];
instance.searchKey = searchKeys[0].key;
logger.info(`Deploying NLP...`);
let nlp = await this.createNLP(name, `${name}-nlp`, location);
let keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name);
instance.nlpEndpoint = nlp.endpoint;
instance.nlpKey = keys.key1;
instance.nlpAppId = "0ff1ceb4f-96a4-4bdb-b2d5-3ea462ddb773";
////////////// LUS
logger.info(`Deploying Speech...`);
let speech = await this.createSpeech(name, `${name}-speech`, location);
@ -200,7 +211,7 @@ export class AzureDeployerService extends GBService {
return instance;
}
public async deployBot(
public async deployBootBot(
instance,
name,
endpoint,
@ -229,7 +240,7 @@ export class AzureDeployerService extends GBService {
}
private async dangerouslyDeleteDeploy(name) {
return this.resourceClient.resourceGroups.deleteMethod(name);
return await this.resourceClient.resourceGroups.deleteMethod(name);
}
private async createStorageServer(
@ -282,27 +293,25 @@ export class AzureDeployerService extends GBService {
subscriptionId
) {
let baseUrl = `https://management.azure.com/`;
this.registerProviders(subscriptionId, baseUrl, accessToken);
let appId = msRestAzure.generateUuid();
let parameters = {
parameters: {
location: location,
sku: {
name: "F0"
},
name: name,
id: botId,
kind: "sdk",
properties: {
description: description,
displayName: name,
endpoint: endpoint,
iconUrl: iconUrl,
luisAppIds: [nlpAppId],
luisKey: nlpKey,
msaAppId: appId
}
location: location,
sku: {
name: "F0"
},
name: name,
id: botId,
kind: "sdk",
properties: {
description: description,
displayName: name,
endpoint: endpoint,
iconUrl: iconUrl,
luisAppIds: [nlpAppId],
luisKey: nlpKey,
msaAppId: appId
}
};
@ -328,6 +337,50 @@ export class AzureDeployerService extends GBService {
return JSON.parse(res.bodyAsJson as string);
}
private async createLUISApp(
name: string,
description: string,
location: string,
culture: string
) {
let parameters = {
name: name,
description: description,
culture: culture
};
let requestUrl = `https://${location}.api.cognitive.microsoft.com/luis/api/v2.0/apps/`;
let req = new WebResource();
req.method = "POST";
req.url = requestUrl;
req.headers = {};
req.headers["Content-Type"] = "application/json";
req.headers["accept-language"] = "*";
let authoringKey;
let retriveAuthoringKey = () => {
if (!authoringKey) {
process.stdout.write(
"Due to this opened issue: https://github.com/Microsoft/botbuilder-tools/issues/550\n"
);
process.stdout.write("Please enter your LUIS Authoring Key:");
authoringKey = scanf("%s");
}
};
while (!authoringKey) {
retriveAuthoringKey();
}
req.headers["Ocp-Apim-Subscription-Key"] = authoringKey;
req.body = JSON.stringify(parameters);
let httpClient = new ServiceClient();
let res = await httpClient.sendRequest(req);
return res.bodyAsText;
}
private async createSearch(group, name, location) {
var params = {
sku: { name: "free" },
@ -358,13 +411,6 @@ export class AzureDeployerService extends GBService {
location,
kind
): Promise<CognitiveServicesAccount> {
// * 'Bing.Autosuggest.v7', 'Bing.CustomSearch',
// * 'Bing.Search.v7', 'Bing.Speech', 'Bing.SpellCheck.v7', 'ComputerVision',
// * 'ContentModerator', 'CustomSpeech', 'CustomVision.Prediction',
// * 'CustomVision.Training', 'Emotion', 'Face', 'LUIS', 'QnAMaker',
// * 'SpeakerRecognition', 'SpeechTranslation', 'TextAnalytics',
// * 'TextTranslation', 'WebLM'
let params = {
sku: { name: "F0" },
createMode: "Default",
@ -490,8 +536,8 @@ export class AzureDeployerService extends GBService {
minimumLength: 8,
maximumLength: 8
};
let password = passwordGenerator.generatePassword(options);
return `sa${password}`;
let generated = passwordGenerator.generatePassword(options);
return `sa${generated}`;
}
private static getRndPassword() {
@ -591,4 +637,105 @@ export class AzureDeployerService extends GBService {
return new AzureDeployerService(credentials, subscriptionId, location);
}
static getKBSearchSchema(indexName) {
return {
name: indexName,
fields: [
{
name: "questionId",
type: "Edm.String",
searchable: false,
filterable: false,
retrievable: true,
sortable: false,
facetable: false,
key: true
},
{
name: "subject1",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "subject2",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "subject3",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "subject4",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "content",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "answerId",
type: "Edm.Int32",
searchable: false,
filterable: false,
retrievable: true,
sortable: false,
facetable: false,
key: false
},
{
name: "instanceId",
type: "Edm.Int32",
searchable: false,
filterable: true,
retrievable: true,
sortable: false,
facetable: false,
key: false
},
{
name: "packageId",
type: "Edm.Int32",
searchable: false,
filterable: true,
retrievable: true,
sortable: false,
facetable: false,
key: false
}
],
scoringProfiles: [],
defaultScoringProfile: null,
corsOptions: null
};
}
}

View file

@ -39,7 +39,7 @@ import { IGBInstance, IGBCoreService } from "botlib";
import { GuaribasInstance } from "../models/GBModel";
import { GBAdminService } from "../../admin.gbapp/services/GBAdminService";
const processExists = require("process-exists");
const TextDecoder = require("util").TextDecoder;
/**
* Core service layer.
@ -303,40 +303,47 @@ export class GBCoreService implements IGBCoreService {
}
public async ensureProxy(): Promise<string> {
let expiresOn = new Date(
await this.adminService.getValue(0, "proxyExpiresOn")
);
let proxyAddress;
if (expiresOn.getTime() > new Date().getTime()) {
proxyAddress = await this.adminService.getValue(
GBAdminService.masterBotInstanceId,
"proxyAddress"
);
return Promise.resolve(proxyAddress);
} else {
if (await processExists("ngrok")) {
logger.warn("ngrok is already running.");
} else {
const { spawn } = require("child_process");
const child = spawn("node_modules\ngrok\bin\ngrok");
child.stdout.on("data", data => {
console.log(`child stdout:\n${data}`);
});
}
let proxyAddress: string;
const ngrok = require('ngrok');
return await ngrok.connect();
await this.adminService.setValue(
GBAdminService.masterBotInstanceId,
"proxyAddress",
proxyAddress
);
let now = new Date();
let expiresOn = now.setHours(now.getHours());
await this.adminService.setValue(
GBAdminService.masterBotInstanceId,
"proxyExpiresOn",
expiresOn.toString()
);
return Promise.resolve(proxyAddress);
}
// let expiresOn = new Date(
// await this.adminService.getValue(0, "proxyExpiresOn")
// );
// let proxyAddress;
// if (expiresOn.getTime() > new Date().getTime()) {
// proxyAddress = await this.adminService.getValue(
// GBAdminService.masterBotInstanceId,
// "proxyAddress"
// );
// return Promise.resolve(proxyAddress);
// } else {
// if (await processExists("ngrok.exe")) {
// logger.warn("ngrok is already running.");
// } else {
// const { exec } = require("child_process");
// const child = exec(
// "node_modules\\ngrok\\bin\\ngrok http 4242",
// (error, stdout, stderr) => {
// console.log(`child stdout:\n${stdout}`);
// }
// );
// }
// await this.adminService.setValue(
// GBAdminService.masterBotInstanceId,
// "proxyAddress",
// proxyAddress
// );
// let now = new Date();
// let expiresOn = now.setHours(now.getHours());
// await this.adminService.setValue(
// GBAdminService.masterBotInstanceId,
// "proxyExpiresOn",
// expiresOn.toString()
// );
return Promise.resolve(proxyAddress);
// }
}
}

View file

@ -47,6 +47,7 @@ import { GBError } from "botlib";
import { GuaribasPackage, GuaribasInstance } from "../models/GBModel";
import { IGBPackage } from "botlib";
import { AzureSearch } from "pragmatismo-io-framework";
import { AzureDeployerService } from "../../azuredeployer.gbapp/services/AzureDeployerService";
/** Deployer service for bots, themes, ai and more. */
export class GBDeployer {
@ -251,7 +252,6 @@ export class GBDeployer {
});
}
deployTheme(localPath: string) {
// DISABLED: Until completed, "/ui/public".
// FsExtra.copy(localPath, this.workDir + packageName)
@ -322,6 +322,16 @@ export class GBDeployer {
}
}
public static getConnectionStringFromInstance(instance: GuaribasInstance) {
return `Server=tcp:${
instance.storageServer
}.database.windows.net,1433;Database=${instance.storageName};User ID=${
instance.storageUsername
};Password=${
instance.storagePassword
};Trusted_Connection=False;Encrypt=True;Connection Timeout=30;`;
}
public async rebuildIndex(instance: GuaribasInstance) {
let search = new AzureSearch(
instance.searchKey,
@ -329,11 +339,22 @@ export class GBDeployer {
instance.searchIndex,
instance.searchIndexer
);
let connectionString = GBDeployer.getConnectionStringFromInstance(instance);
const dsName = "gb";
await search.deleteDatasource(dsName);
await search.createDatasource(
dsName,
dsName,
"GuaribasQuestion",
"azuresql",
connectionString
);
await search.deleteIndex();
let kbService = new KBService(this.core.sequelize);
await search.createIndex(
kbService.getSearchSchema(instance.searchIndex),
"gb"
AzureDeployerService.getKBSearchSchema(instance.searchIndex),
dsName
);
}

View file

@ -183,106 +183,6 @@ export class KBService {
}
}
getSearchSchema(indexName) {
return {
name: indexName,
fields: [
{
name: "questionId",
type: "Edm.String",
searchable: false,
filterable: false,
retrievable: true,
sortable: false,
facetable: false,
key: true
},
{
name: "subject1",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "subject2",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "subject3",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "subject4",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "content",
type: "Edm.String",
searchable: true,
filterable: false,
retrievable: false,
sortable: false,
facetable: false,
key: false
},
{
name: "answerId",
type: "Edm.Int32",
searchable: false,
filterable: false,
retrievable: true,
sortable: false,
facetable: false,
key: false
},
{
name: "instanceId",
type: "Edm.Int32",
searchable: false,
filterable: true,
retrievable: true,
sortable: false,
facetable: false,
key: false
},
{
name: "packageId",
type: "Edm.Int32",
searchable: false,
filterable: true,
retrievable: true,
sortable: false,
facetable: false,
key: false
}
],
scoringProfiles: [],
defaultScoringProfile: null,
corsOptions: null
}
}
static getFormattedSubjectItems(subjects: GuaribasSubject[]) {
if (!subjects) return ""

View file

@ -98,19 +98,15 @@ export class GBServer {
// Ensures cloud / on-premises infrastructure is setup.
logger.info(`Connecting to the infrastructure...`);
let proxyAddress = await core.ensureProxy();
let cloudDeployer = await AzureDeployerService.ensureDeployer();
let instance = await cloudDeployer.deployFarm('gbot', 'westus');
let instance = await cloudDeployer.deployFarm('gbot', 'westus', proxyAddress);
// TODO: Get .gb* templates from GitHub and download do additional deploy folder.
await core.initDatabase();
// Boot a bot package if any.
logger.info(`Starting instances...`);
let deployer = new GBDeployer(core, new GBImporter(core));
// Build a minimal bot instance for each .gbot deployment.
// Check admin password.
let conversationalService = new GBConversationalService(core);
let adminService = new GBAdminService(core);
@ -122,15 +118,6 @@ export class GBServer {
);
}
// Creates the minimal service shared across all .gbapps.
let minService = new GBMinService(
core,
conversationalService,
adminService,
deployer
);
// NOTE: the semicolon is necessary before this line.
// Loads all system packages.
@ -177,7 +164,8 @@ export class GBServer {
// Deploy packages and format object store according to .gbapp storage models.
logger.info(`Deploying packages.`);
logger.info(`Deploying packages...`);
let deployer = new GBDeployer(core, new GBImporter(core));
await deployer.deployPackages(core, server, appPackages);
// If instances is undefined here it's because storage has been formatted.
@ -190,7 +178,13 @@ export class GBServer {
// Setup server dynamic (per bot instance) resources and listeners.
logger.info(`Building minimal instances.`);
logger.info(`Building instances.`);
let minService = new GBMinService(
core,
conversationalService,
adminService,
deployer
);
await minService.buildMin(server, appPackages, instances);
logger.info(`The Bot Server is in RUNNING mode...`);