From b92fbca72ae819d2b7b417231d8b35d23ef7ec34 Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Fri, 14 Jul 2023 18:45:17 -0300 Subject: [PATCH] fix(all): Create bot working again. --- package.json | 4 +- .../dialogs/StartDialog.ts | 10 +++- .../services/AzureDeployerService.ts | 56 +++++++++++-------- .../basic.gblib/services/SystemKeywords.ts | 20 ++++--- .../core.gbapp/services/GBConfigService.ts | 9 +++ packages/core.gbapp/services/GBCoreService.ts | 36 ++++++++---- packages/core.gbapp/services/GBSSR.ts | 2 +- .../security.gbapp/dialogs/ProfileDialog.ts | 11 ++-- src/app.ts | 48 ++++++++-------- 9 files changed, 119 insertions(+), 77 deletions(-) diff --git a/package.json b/package.json index 16f987f6..e556438e 100644 --- a/package.json +++ b/package.json @@ -182,7 +182,8 @@ "winston": "3.8.2", "winston-logs-display": "1.0.0", "ws": "8.12.1", - "yarn": "1.22.19" + "yarn": "1.22.19", + "ngrok": "4.3.3" }, "devDependencies": { "@types/qrcode": "1.5.0", @@ -193,7 +194,6 @@ "dependency-check": "4.1.0", "git-issues": "1.0.0", "license-checker": "25.0.1", - "ngrok": "4.3.3", "prettier-standard": "15.0.1", "semantic-release": "17.2.4", "simple-commit-message": "4.0.13", diff --git a/packages/azuredeployer.gbapp/dialogs/StartDialog.ts b/packages/azuredeployer.gbapp/dialogs/StartDialog.ts index 0325d74b..c0381114 100644 --- a/packages/azuredeployer.gbapp/dialogs/StartDialog.ts +++ b/packages/azuredeployer.gbapp/dialogs/StartDialog.ts @@ -41,12 +41,13 @@ import * as Fs from 'fs'; import { GBAdminService } from '../../../packages/admin.gbapp/services/GBAdminService.js'; import { GBConfigService } from '../../../packages/core.gbapp/services/GBConfigService.js'; import scanf from 'scanf'; +import { AzureDeployerService } from '../services/AzureDeployerService.js'; /** * Handles command-line dialog for getting info for Boot Bot. */ export class StartDialog { - public static async createBaseInstance (installationDeployer: IGBInstallationDeployer) { + public static async createBaseInstance (deployer, freeTier) { // No .env so asks for cloud credentials to start a new farm. if (!Fs.existsSync(`.env`)) { @@ -76,10 +77,13 @@ export class StartDialog { let subscriptionId: string; while (subscriptionId === undefined) { - const list = await installationDeployer.getSubscriptions(credentials); + const list = await (new AzureDeployerService()).getSubscriptions(credentials); subscriptionId = this.retrieveSubscriptionId(list); } + const installationDeployer = await AzureDeployerService.createInstanceWithADALCredentials( + deployer, freeTier, subscriptionId, credentials); + let location: string; while (location === undefined) { location = this.retrieveLocation(); @@ -108,7 +112,7 @@ export class StartDialog { instance.marketplacePassword = appPassword; instance.adminPass = GBAdminService.getRndPassword(); - return { instance, credentials, subscriptionId }; + return { instance, credentials, subscriptionId , installationDeployer}; } private static retrieveUsername () { diff --git a/packages/azuredeployer.gbapp/services/AzureDeployerService.ts b/packages/azuredeployer.gbapp/services/AzureDeployerService.ts index a4e8202c..f614bc51 100644 --- a/packages/azuredeployer.gbapp/services/AzureDeployerService.ts +++ b/packages/azuredeployer.gbapp/services/AzureDeployerService.ts @@ -84,17 +84,22 @@ export class AzureDeployerService implements IGBInstallationDeployer { await this.deployer.rebuildIndex(instance, this.getKBSearchSchema(instance.searchIndex)); } + public static async createInstance(deployer: GBDeployer, freeTier: boolean = true): Promise { const username = GBConfigService.get('CLOUD_USERNAME'); const password = GBConfigService.get('CLOUD_PASSWORD'); const subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID'); + return await this.createInstanceWithCredentials(deployer, freeTier, subscriptionId, username, password); + } + + public static async createInstanceWithADALCredentials(deployer: GBDeployer, freeTier: boolean = true, + subscriptionId: string, credentials): Promise { const service = new AzureDeployerService(); service.core = deployer.core; service.deployer = deployer; service.freeTier = freeTier; - const credentials = await GBAdminService.getADALCredentialsFromUsername(username, password); const token = credentials['tokenCache']._entries[0]; await service.initServices(token.accessToken, token.expiresOn, subscriptionId); @@ -102,6 +107,13 @@ export class AzureDeployerService implements IGBInstallationDeployer { return service; } + public static async createInstanceWithCredentials(deployer: GBDeployer, freeTier: boolean = true, + subscriptionId: string, username: string, password: string): Promise { + const service = new AzureDeployerService(); + const credentials = await GBAdminService.getADALCredentialsFromUsername(username, password); + return await this.createInstanceWithADALCredentials(deployer, freeTier, subscriptionId, credentials); + } + private static createRequestObject(url: string, accessToken: string, verb: HttpMethods, body: string) { const req = new WebResource(); req.method = verb; @@ -399,24 +411,24 @@ export class AzureDeployerService implements IGBInstallationDeployer { instance.searchIndexer = 'azuresql-indexer'; instance.searchKey = searchKeys.primaryKey; - GBLog.info(`Deploying Speech...`); - const speech = await this.createSpeech(name, `${name}speech`, instance.cloudLocation); - keys = await this.cognitiveClient.accounts.listKeys(name, speech.name); - instance.speechEndpoint = speech.properties.endpoint; - instance.speechKey = keys.key1; + // GBLog.info(`Deploying Speech...`); + // const speech = await this.createSpeech(name, `${name}speech`, instance.cloudLocation); + // keys = await this.cognitiveClient.accounts.listKeys(name, speech.name); + // instance.speechEndpoint = speech.properties.endpoint; + // instance.speechKey = keys.key1; - GBLog.info(`Deploying Text Analytics...`); - const textAnalytics = await this.createTextAnalytics(name, `${name}-textanalytics`, instance.cloudLocation); - instance.textAnalyticsEndpoint = textAnalytics.properties.endpoint.replace(`/text/analytics/v2.0`, ''); + // GBLog.info(`Deploying Text Analytics...`); + // const textAnalytics = await this.createTextAnalytics(name, `${name}-textanalytics`, instance.cloudLocation); + // instance.textAnalyticsEndpoint = textAnalytics.properties.endpoint.replace(`/text/analytics/v2.0`, ''); GBLog.info(`Deploying SpellChecker...`); const spellChecker = await this.createSpellChecker(name, `${name}-spellchecker`); instance.spellcheckerEndpoint = spellChecker.properties.endpoint; - GBLog.info(`Deploying NLP...`); - const nlp = await this.createNLP(name, `${name}-nlp`, instance.cloudLocation); - const nlpa = await this.createNLPAuthoring(name, `${name}-nlpa`, instance.cloudLocation); - instance.nlpEndpoint = nlp.properties.endpoint; + // GBLog.info(`Deploying NLP...`); + // const nlp = await this.createNLP(name, `${name}-nlp`, instance.cloudLocation); + // const nlpa = await this.createNLPAuthoring(name, `${name}-nlpa`, instance.cloudLocation); + // instance.nlpEndpoint = nlp.properties.endpoint; const sleep = ms => { return new Promise(resolve => { @@ -443,19 +455,19 @@ export class AzureDeployerService implements IGBInstallationDeployer { instance.cloudSubscriptionId ); - GBLog.info(`Waiting one minute to finishing NLP service and keys creation...`); + GBLog.info(`Waiting one minute to finish NLP service and keys creation...`); await sleep(60000); - keys = await this.cognitiveClient.accounts.listKeys(name, textAnalytics.name); - instance.textAnalyticsKey = keys.key1; + // keys = await this.cognitiveClient.accounts.listKeys(name, textAnalytics.name); + // instance.textAnalyticsKey = keys.key1; keys = await this.cognitiveClient.accounts.listKeys(name, spellChecker.name); instance.spellcheckerKey = keys.key1; - let authoringKeys = await this.cognitiveClient.accounts.listKeys(name, nlpa.name); - keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name); - instance.nlpKey = keys.key1; + // let authoringKeys = await this.cognitiveClient.accounts.listKeys(name, nlpa.name); + // keys = await this.cognitiveClient.accounts.listKeys(name, nlp.name); + // instance.nlpKey = keys.key1; - instance.nlpAuthoringKey = authoringKeys.key1; - const nlpAppId = await this.createNLPService(name, name, instance.cloudLocation, culture, instance.nlpAuthoringKey); - instance.nlpAppId = nlpAppId; + // instance.nlpAuthoringKey = authoringKeys.key1; + // const nlpAppId = await this.createNLPService(name, name, instance.cloudLocation, culture, instance.nlpAuthoringKey); + // instance.nlpAppId = nlpAppId; GBLog.info('Updating server environment variables...'); await this.updateWebisteConfig(name, serverName, serverFarm.id, instance); diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index 87ec14d7..86da2e6a 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -51,7 +51,7 @@ import PizZip from 'pizzip'; import Docxtemplater from 'docxtemplater'; import pptxTemplaterModule from 'pptxtemplater'; import _ from 'lodash'; -import { pdfToPng, PngPageOutput } from 'pdf-to-png-converter'; +// TODO: canvas error import { pdfToPng, PngPageOutput } from 'pdf-to-png-converter'; import sharp from 'sharp'; import ImageModule from 'open-docxtemplater-image-module'; import DynamicsWebApi from 'dynamics-web-api'; @@ -350,14 +350,16 @@ export class SystemKeywords { // Converts the PDF to PNG. - const pngPages: PngPageOutput[] = await pdfToPng(gbfile.data, { - disableFontFace: false, - useSystemFonts: false, - viewportScale: 2.0, - pagesToProcess: [1], - strictPagesToProcess: false, - verbosityLevel: 0 - }); + const pngPages /*: PngPageOutput*/ = []; + // TODO: https://github.com/GeneralBots/BotServer/issues/350 + // await pdfToPng(gbfile.data, { + // disableFontFace: false, + // useSystemFonts: false, + // viewportScale: 2.0, + // pagesToProcess: [1], + // strictPagesToProcess: false, + // verbosityLevel: 0 + // }); // Prepare an image on cache and return the GBFILE information. diff --git a/packages/core.gbapp/services/GBConfigService.ts b/packages/core.gbapp/services/GBConfigService.ts index b7257e2d..46468b4e 100644 --- a/packages/core.gbapp/services/GBConfigService.ts +++ b/packages/core.gbapp/services/GBConfigService.ts @@ -158,6 +158,15 @@ export class GBConfigService { case 'DEV_GBAI': value = undefined; break; + case 'FREE_TIER': + value = true; + break; + case 'BOT_URL': + value = undefined; + break; + case 'STORAGE_SERVER': + value = undefined; + break; default: GBLog.warn(`Invalid key on .env file: '${key}'`); break; diff --git a/packages/core.gbapp/services/GBCoreService.ts b/packages/core.gbapp/services/GBCoreService.ts index a9348659..28c6a392 100644 --- a/packages/core.gbapp/services/GBCoreService.ts +++ b/packages/core.gbapp/services/GBCoreService.ts @@ -50,7 +50,7 @@ import { GBCustomerSatisfactionPackage } from '../../customer-satisfaction.gbapp import { GBKBPackage } from '../../kb.gbapp/index.js'; import { GBSecurityPackage } from '../../security.gbapp/index.js'; import { GBWhatsappPackage } from '../../whatsapp.gblib/index.js'; -import { GuaribasInstance, GuaribasLog} from '../models/GBModel.js'; +import { GuaribasInstance, GuaribasLog } from '../models/GBModel.js'; import { GBConfigService } from './GBConfigService.js'; import { GBAzureDeployerPackage } from '../../azuredeployer.gbapp/index.js'; import { GBSharePointPackage } from '../../sharepoint.gblib/index.js'; @@ -105,7 +105,7 @@ export class GBCoreService implements IGBCoreService { constructor() { this.adminService = new GBAdminService(this); } - public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) {} + public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) { } /** * Gets database config and connect to storage. Currently two databases @@ -134,8 +134,8 @@ export class GBCoreService implements IGBCoreService { const logging: boolean | Function = GBConfigService.get('STORAGE_LOGGING') === 'true' ? (str: string): void => { - GBLog.info(str); - } + GBLog.info(str); + } : false; const encrypt: boolean = GBConfigService.get('STORAGE_ENCRYPT') === 'true'; @@ -318,10 +318,10 @@ ENDPOINT_UPDATE=true */ public async ensureProxy(port): Promise { try { - if (Fs.existsSync('node_modules/ngrok/bin/ngrok.exe') || Fs.existsSync('node_modules/ngrok/bin/ngrok')) { + if (Fs.existsSync('node_modules/ngrok/bin/ngrok.exe') || Fs.existsSync('node_modules/.bin/ngrok')) { return await ngrok.connect({ port: port }); } else { - GBLog.warn('ngrok executable not found (only tested on Windows). Check installation or node_modules folder.'); + GBLog.warn('ngrok executable not found. Check installation or node_modules folder.'); return 'https://localhost'; } @@ -500,20 +500,36 @@ ENDPOINT_UPDATE=true } } + + public async createBootInstance( + core: GBCoreService, + installationDeployer: IGBInstallationDeployer, + proxyAddress: string + ) { + return await this.createBootInstanceEx( + core, + installationDeployer, + proxyAddress, null, + GBConfigService.get('FREE_TIER')); + + } /** * Creates the first bot instance (boot instance) used to "boot" the server. * At least one bot is required to perform conversational administrative tasks. * So a base main bot is always deployed and will act as root bot for * configuration tree with three levels: .env > root bot > all other bots. */ - public async createBootInstance( + public async createBootInstanceEx( core: GBCoreService, installationDeployer: IGBInstallationDeployer, - proxyAddress: string + proxyAddress: string, + deployer, + freeTier ) { GBLog.info(`Deploying cognitive infrastructure (on the cloud / on premises)...`); try { - const { instance, credentials, subscriptionId } = await StartDialog.createBaseInstance(installationDeployer); + const { instance, credentials, subscriptionId, installationDeployer } + = await StartDialog.createBaseInstance(deployer, freeTier); installationDeployer['core'] = this; const changedInstance = await installationDeployer.deployFarm( proxyAddress, @@ -528,7 +544,7 @@ ENDPOINT_UPDATE=true await this.openStorageFrontier(installationDeployer); await this.initStorage(); - return changedInstance; + return [changedInstance, installationDeployer]; } catch (error) { GBLog.warn( `There is an error being thrown, so please cleanup any infrastructure objects diff --git a/packages/core.gbapp/services/GBSSR.ts b/packages/core.gbapp/services/GBSSR.ts index 487133d3..764ae097 100644 --- a/packages/core.gbapp/services/GBSSR.ts +++ b/packages/core.gbapp/services/GBSSR.ts @@ -336,7 +336,7 @@ export class GBSSR { if (GBServer.globals.wwwroot && url === '/') { path = GBServer.globals.wwwroot + "/index.html"; // TODO. } - if (!min && !url.startsWith("/static")) { + if (!min && !url.startsWith("/static") && GBServer.globals.wwwroot) { path = Path.join(GBServer.globals.wwwroot, url); } if (Fs.existsSync(path)) { diff --git a/packages/security.gbapp/dialogs/ProfileDialog.ts b/packages/security.gbapp/dialogs/ProfileDialog.ts index d6442ff4..d4a951b3 100644 --- a/packages/security.gbapp/dialogs/ProfileDialog.ts +++ b/packages/security.gbapp/dialogs/ProfileDialog.ts @@ -39,7 +39,7 @@ import { GBLog, GBMinInstance, IGBDialog } from 'botlib'; import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; import { Messages } from '../strings.js'; -import * as phone from 'google-libphonenumber'; +import libphonenumber from 'google-libphonenumber'; /** * Dialogs for handling Menu control. @@ -101,17 +101,16 @@ export class ProfileDialog extends IGBDialog { }, async step => { const locale = step.context.activity.locale; - let phoneNumber; + let phoneNumber = step.context.activity.text; try { - let p = phone.PhoneNumberUtil.getInstance(); - phoneNumber = p(step.result, 'BRA')[0]; // https://github.com/GeneralBots/BotServer/issues/307 - phoneNumber = phone.phoneUtil.parse(phoneNumber); + let p = libphonenumber.PhoneNumberUtil.getInstance(); + phoneNumber = p.parse(phoneNumber); } catch (error) { await step.context.sendActivity(Messages[locale].validation_enter_valid_mobile); return await step.replaceDialog('/profile_mobile', step.activeDialog.state.options); } - if (!phone.phoneUtil.isPossibleNumber(phoneNumber)) { + if (!libphonenumber.phoneUtil.isPossibleNumber(phoneNumber)) { await step.context.sendActivity(Messages[locale].validation_enter_valid_mobile); return await step.replaceDialog('/profile_mobile', step.activeDialog.state.options); diff --git a/src/app.ts b/src/app.ts index 7e962fa7..616f5d58 100644 --- a/src/app.ts +++ b/src/app.ts @@ -140,16 +140,16 @@ export class GBServer { const core: IGBCoreService = new GBCoreService(); const importer: GBImporter = new GBImporter(core); const deployer: GBDeployer = new GBDeployer(core, importer); - const azureDeployer: AzureDeployerService = await AzureDeployerService.createInstance(deployer); - const adminService: GBAdminService = new GBAdminService(core); + const subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID'); + let azureDeployer: AzureDeployerService; + + // Ensure that local proxy is setup. if (process.env.NODE_ENV === 'development') { const proxy = GBConfigService.get('BOT_URL'); if (proxy !== undefined) { GBServer.globals.publicAddress = proxy; } else { - // Ensure that local development proxy is setup. - GBLog.info(`Establishing a development local proxy (proxy) on BOT_URL...`); GBServer.globals.publicAddress = await core.ensureProxy(port); } @@ -159,16 +159,19 @@ export class GBServer { GBServer.globals.publicAddress = serverAddress; } + // Creates a boot instance or load it from storage. - try { + if (GBConfigService.get('STORAGE_SERVER')) { + azureDeployer = await AzureDeployerService.createInstance(deployer); await core.initStorage(); - } catch (error) { - GBLog.verbose(`Error initializing storage: ${error}`); - GBServer.globals.bootInstance = await core.createBootInstance( + } else { + [GBServer.globals.bootInstance, azureDeployer] = await core['createBootInstanceEx']( core, - azureDeployer, - GBServer.globals.publicAddress + null, + GBServer.globals.publicAddress, + deployer, + GBConfigService.get('FREE_TIER') ); } @@ -216,6 +219,7 @@ export class GBServer { // Builds minimal service infrastructure. const conversationalService: GBConversationalService = new GBConversationalService(core); + const adminService: GBAdminService = new GBAdminService(core); const minService: GBMinService = new GBMinService(core, conversationalService, adminService, deployer); GBServer.globals.minService = minService; await minService.buildMin(instances); @@ -231,11 +235,6 @@ export class GBServer { } } - // let s = new GBVMService(); - // await s.translateBASIC('work/gptA.vbs', GBServer.globals.minBoot ); - // await s.translateBASIC('work/gptB.vbs', GBServer.globals.minBoot ); - // await s.translateBASIC('work/gptC.vbs', GBServer.globals.minBoot ); - // process.exit(9); if (process.env.ENABLE_WEBLOG) { // If global log enabled, reorders transports adding web logging. @@ -280,15 +279,16 @@ export class GBServer { // Setups unsecure http redirect. - const server1 = http.createServer((req, res) => { - const host = req.headers.host.startsWith('www.') ? - req.headers.host.substring(4) : req.headers.host; - res.writeHead(301, { - Location: "https://" + host + req.url - }).end(); - }); - - server1.listen(80); + if (process.env.NODE_ENV === 'production') { + const server1 = http.createServer((req, res) => { + const host = req.headers.host.startsWith('www.') ? + req.headers.host.substring(4) : req.headers.host; + res.writeHead(301, { + Location: "https://" + host + req.url + }).end(); + }); + server1.listen(80); + } if (process.env.CERTIFICATE_PFX) { const options1 = {