diff --git a/packages/admin.gbapp/services/GBAdminService.ts b/packages/admin.gbapp/services/GBAdminService.ts index 90dd7034..65543819 100644 --- a/packages/admin.gbapp/services/GBAdminService.ts +++ b/packages/admin.gbapp/services/GBAdminService.ts @@ -113,7 +113,7 @@ export class GBAdminService implements IGBAdminService { maximumLength: 14 }; let password = passwordGenerator.generatePassword(options); - password = password.replace(/[\@\[\=\:\;\?]/gi, '#'); + password = password.replace(/[\@\[\=\:\;\?\"\'\#]/gi, '1'); return password; } diff --git a/packages/azuredeployer.gbapp/dialogs/StartDialog.ts b/packages/azuredeployer.gbapp/dialogs/StartDialog.ts index b95cee40..4ead3d48 100644 --- a/packages/azuredeployer.gbapp/dialogs/StartDialog.ts +++ b/packages/azuredeployer.gbapp/dialogs/StartDialog.ts @@ -146,6 +146,11 @@ cannot start or end with or contain consecutive dashes and having 4 to 42 charac return botId; } + + /** + * + * Update Manifest in Azure: "signInAudience": "AzureADandPersonalMicrosoftAccount" and "accessTokenAcceptedVersion": 2. + */ private static retrieveAppId() { let appId = GBConfigService.get('MARKETPLACE_ID'); if (appId === undefined) { @@ -154,6 +159,7 @@ cannot start or end with or contain consecutive dashes and having 4 to 42 charac please go to https://apps.dev.microsoft.com/portal/register-app to generate manually an App ID and App Secret.\n` ); + process.stdout.write('Generated Application Id (MARKETPLACE_ID):'); appId = scanf('%s').replace(/(\n|\r)+$/, ''); } diff --git a/packages/azuredeployer.gbapp/services/AzureDeployerService.ts b/packages/azuredeployer.gbapp/services/AzureDeployerService.ts index ef4816f0..e3951558 100644 --- a/packages/azuredeployer.gbapp/services/AzureDeployerService.ts +++ b/packages/azuredeployer.gbapp/services/AzureDeployerService.ts @@ -44,7 +44,7 @@ import { SqlManagementClient } from 'azure-arm-sql'; import { WebSiteManagementClient } from 'azure-arm-website'; //tslint:disable-next-line:no-submodule-imports import { AppServicePlan, Site, SiteConfigResource, SiteLogsConfig, SiteSourceControl } from 'azure-arm-website/lib/models'; -import { GBLog, IGBInstallationDeployer, IGBInstance, IGBDeployer } from 'botlib'; +import { GBLog, IGBInstallationDeployer, IGBInstance, IGBDeployer, IGBCoreService } from 'botlib'; import { GBAdminService } from '../../../packages/admin.gbapp/services/GBAdminService'; import { GBCorePackage } from '../../../packages/core.gbapp'; import { GBConfigService } from '../../../packages/core.gbapp/services/GBConfigService'; @@ -82,13 +82,18 @@ export class AzureDeployerService implements IGBInstallationDeployer { public subscriptionId: string; public farmName: any; public deployer: IGBDeployer; + public core: IGBCoreService; private freeTier: boolean; - constructor(deployer: IGBDeployer, freeTier: boolean = true) { + constructor(deployer: IGBDeployer, freeTier: boolean = false) { this.deployer = deployer; this.freeTier = freeTier; } + public async runSearch(instance: IGBInstance) { + await this.deployer.rebuildIndex(instance, this.getKBSearchSchema(instance.searchIndex)); + } + public static async createInstance(deployer: GBDeployer): Promise { const username = GBConfigService.get('CLOUD_USERNAME'); @@ -97,6 +102,7 @@ export class AzureDeployerService implements IGBInstallationDeployer { const subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID'); const service = new AzureDeployerService(deployer); + service.core = deployer.core; service.initServices(credentials, subscriptionId); return service; @@ -350,15 +356,6 @@ export class AzureDeployerService implements IGBInstallationDeployer { }; await storageClient.firewallRules.createOrUpdate(groupName, serverName, 'gb', params); - // AllowAllWindowsAzureIps must be created that way, so the Azure Search can - // access SQL Database to index its contents. - - params = { - startIpAddress: '0.0.0.0', - endIpAddress: '0.0.0.0' - }; - await storageClient.firewallRules.createOrUpdate(groupName, serverName, 'AllowAllWindowsAzureIps', params); - } public async deployFarm( @@ -396,10 +393,19 @@ export class AzureDeployerService implements IGBInstallationDeployer { await this.createStorage(name, storageServer, storageName, instance.cloudLocation); instance.storageUsername = administratorLogin; instance.storagePassword = administratorPassword; - instance.storageName = storageServer; + instance.storageName = storageName; instance.storageDialect = 'mssql'; instance.storageServer = `${storageServer}.database.windows.net`; + GBLog.info(`Deploying Search...`); + const searchName = `${name}-search`.toLowerCase(); + await this.createSearch(name, searchName, instance.cloudLocation); + const searchKeys = await this.searchClient.adminKeys.get(name, searchName); + instance.searchHost = `${searchName}.search.windows.net`; + instance.searchIndex = 'azuresql-index'; + instance.searchIndexer = 'azuresql-indexer'; + instance.searchKey = searchKeys.primaryKey; + GBLog.info(`Deploying Speech...`); const speech = await this.createSpeech(name, `${name}-speech`, instance.cloudLocation); keys = await this.cognitiveClient.accounts.listKeys(name, speech.name); @@ -409,7 +415,6 @@ export class AzureDeployerService implements IGBInstallationDeployer { GBLog.info(`Deploying Text Analytics...`); const textAnalytics = await this.createTextAnalytics(name, `${name}-textanalytics`, instance.cloudLocation); keys = await this.cognitiveClient.accounts.listKeys(name, textAnalytics.name); - instance.textAnalyticsEndpoint = textAnalytics.endpoint.replace(`/text/analytics/v2.0`, ''); instance.textAnalyticsKey = keys.key1; @@ -419,17 +424,6 @@ export class AzureDeployerService implements IGBInstallationDeployer { instance.spellcheckerKey = keys.key1; instance.spellcheckerEndpoint = spellChecker.endpoint; - - GBLog.info(`Deploying Search...`); - const searchName = `${name}-search`.toLowerCase(); - await this.createSearch(name, searchName, instance.cloudLocation); - const searchKeys = await this.searchClient.adminKeys.get(name, searchName); - instance.searchHost = `${searchName}.search.windows.net`; - instance.searchIndex = 'azuresql-index'; - instance.searchIndexer = 'azuresql-indexer'; - instance.searchKey = searchKeys.primaryKey; - await this.deployer.rebuildIndex(instance, this.getKBSearchSchema(instance.searchIndex)); - GBLog.info(`Deploying NLP...`); const nlp = await this.createNLP(name, `${name}-nlp`, instance.cloudLocation); const nlpa = await this.createNLPAuthoring(name, `${name}-nlpa`, instance.cloudLocation); @@ -441,7 +435,7 @@ export class AzureDeployerService implements IGBInstallationDeployer { instance.nlpEndpoint = nlp.endpoint; instance.nlpKey = keys.key1; instance.nlpAppId = nlpAppId; - + GBLog.info(`Deploying Bot...`); instance.botEndpoint = this.defaultEndPoint; @@ -459,7 +453,7 @@ export class AzureDeployerService implements IGBInstallationDeployer { GBLog.info('Opening your browser with default.gbui...'); const opn = require('opn'); - opn(`https://${serverName}.azurewebsites.net`); + opn(`http://localhost:4242`); return instance; } @@ -591,7 +585,18 @@ export class AzureDeployerService implements IGBInstallationDeployer { fullyQualifiedDomainName: serverName }; - return await this.storageClient.servers.createOrUpdate(group, name, params); + const database = await this.storageClient.servers.createOrUpdate(group, name, params); + + // AllowAllWindowsAzureIps must be created that way, so the Azure Search can + // access SQL Database to index its contents. + + const paramsFirewall = { + startIpAddress: '0.0.0.0', + endIpAddress: '0.0.0.0' + }; + await this.storageClient.firewallRules.createOrUpdate(group, name, 'AllowAllWindowsAzureIps', paramsFirewall); + + return database; } public async createApplication(token: string, name: string) { @@ -668,7 +673,19 @@ export class AzureDeployerService implements IGBInstallationDeployer { const body = JSON.stringify(parameters); const apps = await this.makeNlpRequest(location, authoringKey, undefined, 'GET', 'apps'); - const app = JSON.parse(apps.bodyAsText).filter(x => x.name === name)[0]; + + let app = null; + if (apps.bodyAsText && apps.bodyAsText !== '[]') { + const result = JSON.parse(apps.bodyAsText) + if (result.error) { + if (result.error.code !== "401") { + throw new Error(result.error); + } + } + else { + app = result.filter(x => x.name === name)[0]; + } + } let id: string; if (!app) { const res = await this.makeNlpRequest(location, authoringKey, body, 'POST', 'apps'); @@ -847,7 +864,7 @@ export class AzureDeployerService implements IGBInstallationDeployer { } }; const server = await this.webSiteClient.webApps.createOrUpdate(group, name, parameters); - + const siteLogsConfig: SiteLogsConfig = { applicationLogs: { fileSystem: { level: 'Error' } diff --git a/packages/core.gbapp/services/GBConfigService.ts b/packages/core.gbapp/services/GBConfigService.ts index 3edf9ebd..19449af6 100644 --- a/packages/core.gbapp/services/GBConfigService.ts +++ b/packages/core.gbapp/services/GBConfigService.ts @@ -116,7 +116,7 @@ export class GBConfigService { value = undefined; break; case 'STORAGE_SYNC': - value = 'false'; + value = 'true'; break; case 'STORAGE_SYNC_ALTER': value = 'false'; diff --git a/packages/core.gbapp/services/GBCoreService.ts b/packages/core.gbapp/services/GBCoreService.ts index 95aeea1f..1c49b505 100644 --- a/packages/core.gbapp/services/GBCoreService.ts +++ b/packages/core.gbapp/services/GBCoreService.ts @@ -105,8 +105,8 @@ export class GBCoreService implements IGBCoreService { constructor() { this.adminService = new GBAdminService(this); } - public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) {} - + public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) { } + /** * Gets database config and connect to storage. Currently two databases * are available: SQL Server and SQLite. @@ -134,8 +134,8 @@ export class GBCoreService implements IGBCoreService { const logging: boolean | Function = GBConfigService.get('STORAGE_LOGGING') === 'true' ? (str: string): void => { - GBLog.info(str); - } + GBLog.info(str); + } : false; const encrypt: boolean = GBConfigService.get('STORAGE_ENCRYPT') === 'true'; @@ -168,7 +168,7 @@ export class GBCoreService implements IGBCoreService { this.sequelize = new Sequelize(database, username, password, sequelizeOptions); // Specifies custom setup for MSFT... - + if (this.dialect === 'mssql') { this.queryGenerator = this.sequelize.getQueryInterface().QueryGenerator; // tslint:disable:no-unsafe-any @@ -201,15 +201,16 @@ export class GBCoreService implements IGBCoreService { } } + /** - * Syncronizes structure between model and tables in storage. - */ + * Syncronizes structure between model and tables in storage. + */ public async syncDatabaseStructure() { if (GBConfigService.get('STORAGE_SYNC') === 'true') { const alter = GBConfigService.get('STORAGE_SYNC_ALTER') === 'true'; GBLog.info('Syncing database...'); - return this.sequelize.sync({ + return await this.sequelize.sync({ alter: alter, force: false // Keep it false this due to data loss danger. }); @@ -274,7 +275,8 @@ export class GBCoreService implements IGBCoreService { * full base environment. */ public async writeEnv(instance: IGBInstance) { - const env = `ADDITIONAL_DEPLOY_PATH= + const env = ` +ADDITIONAL_DEPLOY_PATH= ADMIN_PASS=${instance.adminPass} BOT_ID=${instance.botId} CLOUD_SUBSCRIPTIONID=${instance.cloudSubscriptionId} @@ -290,6 +292,7 @@ STORAGE_NAME=${instance.storageName} STORAGE_USERNAME=${instance.storageUsername} STORAGE_PASSWORD=${instance.storagePassword} STORAGE_SYNC=true +ENDPOINT_UPDATE=true `; fs.writeFileSync('.env', env); @@ -499,15 +502,19 @@ STORAGE_SYNC=true GBLog.info(`Deploying cognitive infrastructure (on the cloud / on premises)...`); try { const { instance, credentials, subscriptionId } = await StartDialog.createBaseInstance(installationDeployer); + installationDeployer['core'] = this; const changedInstance = await installationDeployer.deployFarm( proxyAddress, instance, credentials, subscriptionId ); - core.writeEnv(changedInstance); - GBLog.info(`File .env written, starting General Bots...`); + await this.writeEnv(changedInstance); GBConfigService.init(); + + GBLog.info(`File .env written. Preparing storage and search for the first time...`); + await this.openStorageFrontier(installationDeployer); + await this.initStorage(); return changedInstance; } catch (error) { @@ -639,9 +646,9 @@ STORAGE_SYNC=true value = instance['dataValues'][name]; if (value === null) { const minBoot = GBServer.globals.minBoot as any; - if (minBoot.instance && minBoot.instance.datavalues){ + if (minBoot.instance && minBoot.instance.datavalues) { value = minBoot.instance.datavalues[name]; - } + } } } @@ -678,7 +685,7 @@ STORAGE_SYNC=true options ); GBLog.info(`Running .gbdialog word ${item.name} on:${item.schedule}...`); - } catch (error) {} + } catch (error) { } }); } } catch (error) { diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index eb6b925a..e03f94dc 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -738,7 +738,7 @@ export class GBDeployer implements IGBDeployer { // If it is a 404 there is nothing to delete as it is the first creation. - if (err.code !== 404 || err.code !== "OperationNotAllowed") { + if (err.code !== 404 && err.code !== "OperationNotAllowed") { throw err; } } diff --git a/packages/core.gbapp/services/GBMinService.ts b/packages/core.gbapp/services/GBMinService.ts index bd8ad5a0..f91d6b26 100644 --- a/packages/core.gbapp/services/GBMinService.ts +++ b/packages/core.gbapp/services/GBMinService.ts @@ -635,7 +635,7 @@ export class GBMinService { // If there is WhatsApp configuration specified, initialize // infrastructure objects. - if (min.instance.whatsappServiceKey !== null) { + if (min.instance.whatsappServiceUrl !== null) { min.whatsAppDirectLine = new WhatsappDirectLine( min, min.botId, @@ -647,7 +647,7 @@ export class GBMinService { await min.whatsAppDirectLine.setup(true); } else { const minBoot = GBServer.globals.minBoot as any; - if (minBoot.instance.whatsappServiceKey) { + if (minBoot.instance.whatsappServiceUrl) { min.whatsAppDirectLine = new WhatsappDirectLine( min, min.botId, diff --git a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts index 96589226..9601624f 100644 --- a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts +++ b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts @@ -87,6 +87,7 @@ export class WhatsappDirectLine extends GBService { } public async setup(setUrl) { + this.directLineClient = new Swagger({ spec: JSON.parse(fs.readFileSync('directline-3.0.json', 'utf8')), diff --git a/src/app.ts b/src/app.ts index e8f0f7a2..c6898af5 100644 --- a/src/app.ts +++ b/src/app.ts @@ -140,7 +140,7 @@ export class GBServer { GBLog.verbose(`Error initializing storage: ${error}`); GBServer.globals.bootInstance = await core.createBootInstance(core, azureDeployer, GBServer.globals.publicAddress); - await core.initStorage(); + } core.ensureAdminIsSecured(); @@ -150,6 +150,7 @@ export class GBServer { GBLog.info(`Deploying packages...`); GBServer.globals.sysPackages = await core.loadSysPackages(core); await core.checkStorage(azureDeployer); + await core.syncDatabaseStructure(); await deployer.deployPackages(core, server, GBServer.globals.appPackages); GBLog.info(`Publishing instances...`); @@ -160,6 +161,7 @@ export class GBServer { ); if (instances.length === 0) { + const instance = await importer.importIfNotExistsBotPackage( GBConfigService.get('BOT_ID'), 'boot.gbot', @@ -168,6 +170,8 @@ export class GBServer { ); await deployer.deployBotFull(instance, GBServer.globals.publicAddress); instances.push(instance); + + await azureDeployer['runSearch'](instance); } GBServer.globals.bootInstance = instances[0]; @@ -186,7 +190,7 @@ export class GBServer { } GBLog.info(`The Bot Server is in RUNNING mode...`); - + // Opens Navigator. // TODO: Config: core.openBrowserInDevelopment();