diff --git a/packages/admin.gbapp/services/GBAdminService.ts b/packages/admin.gbapp/services/GBAdminService.ts index d5ee23b5..8a274194 100644 --- a/packages/admin.gbapp/services/GBAdminService.ts +++ b/packages/admin.gbapp/services/GBAdminService.ts @@ -1,5 +1,3 @@ - - /*****************************************************************************\ | █████ █████ ██ █ █████ █████ ████ ██ ████ █████ █████ ███ ® | | ██ █ ███ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ | @@ -49,7 +47,7 @@ import { GBSharePointService } from '../../sharepoint.gblib/services/SharePointS import { GuaribasAdmin } from '../models/AdminModel.js'; import msRestAzure from 'ms-rest-azure'; import Path from 'path'; -import { caseSensitive_Numbs_SpecialCharacters_PW, lowercase_PW } from 'super-strong-password-generator' +import { caseSensitive_Numbs_SpecialCharacters_PW, lowercase_PW } from 'super-strong-password-generator'; import crypto from 'crypto'; import Fs from 'fs'; import { GBServer } from '../../../src/app.js'; @@ -89,45 +87,40 @@ export class GBAdminService implements IGBAdminService { } public static async getADALCredentialsFromUsername(username: string, password: string) { - return await msRestAzure.loginWithUsernamePassword(username, password); } public static getMobileCode() { - return this.getNumberIdentifier(6); } public static getRndPassword(): string { - let password = caseSensitive_Numbs_SpecialCharacters_PW(15); password = password.replace(/[\@\[\=\:\;\?\"\'\#]/gi, '*'); const removeRepeatedChars = (s, r) => { - let res = '', last = null, counter = 0; + let res = '', + last = null, + counter = 0; s.split('').forEach(char => { - if (char == last) - counter++; - else { - counter = 0; - last = char; - } - if (counter < r) - res += char; - }); + if (char == last) counter++; + else { + counter = 0; + last = char; + } + if (counter < r) res += char; + }); return res; - } + }; return removeRepeatedChars(password, 1); } public static getRndReadableIdentifier(): string { - return lowercase_PW(14); } public static getNumberIdentifier(digits: number = 14): string { - if (digits <= 0) { throw new Error('Number of digits should be greater than 0.'); } @@ -155,7 +148,6 @@ export class GBAdminService implements IGBAdminService { } public static async undeployPackageCommand(text: string, min: GBMinInstance) { - const packageName = text.split(' ')[1]; const importer = new GBImporter(min.core); const deployer = new GBDeployer(min.core, importer); @@ -167,7 +159,12 @@ export class GBAdminService implements IGBAdminService { public static isSharePointPath(path: string) { return path.indexOf('sharepoint.com') !== -1; } - public static async deployPackageCommand(min: GBMinInstance, user: GuaribasUser, text: string, deployer: IGBDeployer) { + public static async deployPackageCommand( + min: GBMinInstance, + user: GuaribasUser, + text: string, + deployer: IGBDeployer + ) { const packageName = text.split(' ')[1]; if (!this.isSharePointPath(packageName)) { @@ -190,19 +187,21 @@ export class GBAdminService implements IGBAdminService { await deployer['cleanupPackage'](min.instance, packageName); } - await deployer['downloadFolder'](min, - Path.join('work', `${gbai}`), - Path.basename(localFolder)); + if (process.env.STORAGE_FILE) { + const path = Path.join(process.env.STORAGE_LIBRARY, gbaiPath); + Fs.cpSync(path, localFolder, { errorOnExist: false, force: true, recursive: true}); + } else { + await deployer['downloadFolder'](min, Path.join('work', `${gbai}`), Path.basename(localFolder)); + } await deployer['deployPackage2'](min, user, localFolder); } } public static async rebuildIndexPackageCommand(min: GBMinInstance, deployer: GBDeployer) { const service = await AzureDeployerService.createInstance(deployer); - const searchIndex = min.instance.searchIndex ? min.instance.searchIndex : GBServer.globals.minBoot.instance.searchIndex; - await deployer.rebuildIndex( - min.instance, - service.getKBSearchSchema(searchIndex) - ); + const searchIndex = min.instance.searchIndex + ? min.instance.searchIndex + : GBServer.globals.minBoot.instance.searchIndex; + await deployer.rebuildIndex(min.instance, service.getKBSearchSchema(searchIndex)); } public static async syncBotServerCommand(min: GBMinInstance, deployer: GBDeployer) { @@ -245,15 +244,15 @@ export class GBAdminService implements IGBAdminService { return obj.value; } - public async acquireElevatedToken(instanceId: number, root: boolean = false, + public async acquireElevatedToken( + instanceId: number, + root: boolean = false, tokenName: string = '', clientId: string = null, clientSecret: string = null, host: string = null, tenant: string = null ): Promise { - - if (root) { const minBoot = GBServer.globals.minBoot; instanceId = minBoot.instance.instanceId; @@ -267,7 +266,6 @@ export class GBAdminService implements IGBAdminService { throw new Error(`/setupSecurity is required before running /publish.`); } - return new Promise(async (resolve, reject) => { const instance = await this.core.loadInstanceById(instanceId); @@ -276,14 +274,10 @@ export class GBAdminService implements IGBAdminService { const accessToken = await this.getValue(instanceId, `${tokenName}accessToken`); resolve(accessToken); } else { - if (tokenName && !root) { - const refreshToken = await this.getValue(instanceId, `${tokenName}refreshToken`); - let url = urlJoin( - host, - tenant, 'oauth/token'); + let url = urlJoin(host, tenant, 'oauth/token'); let buff = new Buffer(`${clientId}:${clientSecret}`); const base64 = buff.toString('base64'); @@ -295,8 +289,8 @@ export class GBAdminService implements IGBAdminService { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ - 'grant_type': 'refresh_token', - 'refresh_token': refreshToken + grant_type: 'refresh_token', + refresh_token: refreshToken }) }; const result = await fetch(url, options); @@ -313,15 +307,15 @@ export class GBAdminService implements IGBAdminService { await this.setValue(instanceId, `${tokenName}accessToken`, token['access_token']); await this.setValue(instanceId, `${tokenName}refreshToken`, token['refresh_token']); - await this.setValue(instanceId, `${tokenName}expiresOn`, - new Date(Date.now() + (token['expires_in'] * 1000)).toString()); + await this.setValue( + instanceId, + `${tokenName}expiresOn`, + new Date(Date.now() + token['expires_in'] * 1000).toString() + ); await this.setValue(instanceId, `${tokenName}AntiCSRFAttackState`, null); resolve(token['access_token']); - - } - else { - + } else { const oauth2 = tokenName ? 'oauth' : 'oauth2'; const authorizationUrl = urlJoin( tokenName ? host : instance.authenticatorAuthorityHostUrl, @@ -359,5 +353,5 @@ export class GBAdminService implements IGBAdminService { }); } - public async publish(min: GBMinInstance, packageName: string, republish: boolean): Promise { } + public async publish(min: GBMinInstance, packageName: string, republish: boolean): Promise {} } diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index 93b17716..c1ac2a58 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -45,6 +45,8 @@ import { AzureSearch } from 'pragmatismo-io-framework'; import { CollectionUtil } from 'pragmatismo-io-framework'; import { GBServer } from '../../../src/app.js'; import { GBVMService } from '../../basic.gblib/services/GBVMService.js'; +import Excel from 'exceljs'; +import asyncPromise from 'async-promises'; import { GuaribasPackage } from '../models/GBModel.js'; import { GBAdminService } from './../../admin.gbapp/services/GBAdminService.js'; import { AzureDeployerService } from './../../azuredeployer.gbapp/services/AzureDeployerService.js'; @@ -286,7 +288,6 @@ export class GBDeployer implements IGBDeployer { `${publicAddress}/api/messages/${instance.botId}` ); } else { - // Internally create resources on cloud provider. instance = await service.internalDeployBot( @@ -326,7 +327,6 @@ export class GBDeployer implements IGBDeployer { let embedding; if (!azureOpenAIEmbeddingModel) { - return; } @@ -337,14 +337,13 @@ export class GBDeployer implements IGBDeployer { azureOpenAIApiVersion: azureOpenAIVersion, azureOpenAIApiInstanceName: azureOpenAIApiInstanceName }); - + try { vectorStore = await HNSWLib.load(min['vectorStorePath'], embedding); } catch { vectorStore = new HNSWLib(embedding, { space: 'cosine' }); - } return vectorStore; } @@ -418,64 +417,37 @@ export class GBDeployer implements IGBDeployer { /** * Loads all para from tabular file Config.xlsx. */ - public async loadParamsFromTabular(min: GBMinInstance): Promise { - const siteId = process.env.STORAGE_SITE_ID; - const libraryId = process.env.STORAGE_LIBRARY; + public async loadParamsFromTabular(min: GBMinInstance, path): Promise { + const workbook = new Excel.Workbook(); + const data = await workbook.xlsx.readFile(Path.join(path, 'Config.xlsx')); - GBLogEx.info(min, `Connecting to Config.xslx (siteId: ${siteId}, libraryId: ${libraryId})...`); + let worksheet: any; + for (let t = 0; t < data.worksheets.length; t++) { + worksheet = data.worksheets[t]; + if (worksheet) { + break; + } + } + const rows = worksheet._rows; + GBLogEx.info(min, `Processing ${rows.length} rows from Config file ${path}...`); + let list = []; - // Connects to MSFT storage. + await asyncPromise.eachSeries(rows, async line => { + // Skips the first line. - const token = await (min.adminService as any)['acquireElevatedToken'](min.instance.instanceId, true); + if (line != undefined && line._cells[0] !== undefined && line._cells[1] !== undefined) { + // Extracts values from columns in the current line. - const client = MicrosoftGraph.Client.init({ - authProvider: done => { - done(null, token); + let obj = {}; + obj[line._cells[0].text] = line._cells[1].text; + list.push(obj); } }); - // Retrieves all files in .bot folder. - - const botId = min.instance.botId; - const path = DialogKeywords.getGBAIPath(botId, 'gbot'); - let url = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/root:/${path}:/children`; - - GBLogEx.info(min, `Loading .gbot from Excel: ${url}`); - const res = await client.api(url).get(); - - // Finds Config.xlsx. - - const document = res.value.filter(m => { - return m.name === 'Config.xlsx'; - }); - if (document === undefined || document.length === 0) { - GBLogEx.info(min, `Config.xlsx not found on .bot folder, check the package.`); - - return null; - } - - // Reads all rows in Config.xlsx that contains a pair of name/value - // and fills an object that is returned to be saved in params instance field. - - const results = await client - .api( - `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/items/${document[0].id}/workbook/worksheets('General')/range(address='A7:B100')` - ) - .get(); - let index = 0, - obj = {}; - for (; index < results.text.length; index++) { - if (results.text[index][0] === '') { - return obj; - } - obj[results.text[index][0]] = results.text[index][1]; - } - - return obj; + return list; } /** - * Loads all para from tabular file Config.xlsx. */ public async downloadFolder( min: GBMinInstance, @@ -632,14 +604,7 @@ export class GBDeployer implements IGBDeployer { case '.gbot': // Extracts configuration information from .gbot files. - if (process.env.ENABLE_PARAMS_ONLINE === 'false') { - if (Fs.existsSync(localPath)) { - GBLogEx.info(min, `Loading .gbot from ${localPath}.`); - await this.deployBotFromLocalPath(localPath, GBServer.globals.publicAddress); - } - } else { - min.instance.params = await this.loadParamsFromTabular(min); - } + min.instance.params = await this.loadParamsFromTabular(min, localPath); let connections = []; diff --git a/packages/core.gbapp/services/GBMinService.ts b/packages/core.gbapp/services/GBMinService.ts index 911326e6..78556c36 100644 --- a/packages/core.gbapp/services/GBMinService.ts +++ b/packages/core.gbapp/services/GBMinService.ts @@ -316,15 +316,15 @@ export class GBMinService { if (!Fs.existsSync(dir)) { mkdirp.sync(dir); } - - dir = Path.join(process.env.PWD, 'work', gbai); - const server = new webdav.WebDAVServer(); - server.setFileSystem(`/${botId}`, - new webdav.PhysicalFileSystem(dir), (success) => { + if (process.env.STORAGE_FILE) { + dir = Path.join(process.env.STORAGE_LIBRARY, 'work', gbai); + + const server = new webdav.WebDAVServer(); + server.setFileSystem(`/${botId}`, new webdav.PhysicalFileSystem(dir), success => { server.start(() => console.log('WEBDAV READY')); - }) - + }); + } // Loads Named Entity data for this bot. // TODO: await KBService.RefreshNER(min); @@ -665,7 +665,6 @@ export class GBMinService { if (instance !== null) { // Gets the webchat token, speech token and theme. - const webchatTokenContainer = await this.getWebchatToken(instance); const speechToken = instance.speechKey != undefined ? await this.getSTSToken(instance) : null; let theme = instance.theme; @@ -674,13 +673,12 @@ export class GBMinService { if (!theme) { theme = `default.gbtheme`; } - + let config = { instanceId: instance.instanceId, botId: botId, theme: theme, - speechToken: speechToken, - conversationId: webchatTokenContainer.conversationId, + speechToken: speechToken, authenticatorTenant: instance.authenticatorTenant, authenticatorClientId: instance.marketplaceId, paramLogoImageUrl: this.core.getParam(instance, 'Logo Image Url', null), @@ -696,6 +694,8 @@ export class GBMinService { if (process.env.STORAGE_FILE) { config['domain'] = `http://localhost:${process.env.PORT}/directline`; } else { + const webchatTokenContainer = await this.getWebchatToken(instance); + config['conversationId']= webchatTokenContainer.conversationId, config['webchatToken'] = webchatTokenContainer.token; } @@ -711,7 +711,7 @@ export class GBMinService { * Gets Webchat token from Bot Service. */ private async getWebchatToken(instance: any) { - const url = 'https://directline.botframework.com/v3/directline/tokens/generate'; + const url = `http://localhost:${process.env.PORT}/v3/directline/tokens/generate`; const options = { method: 'POST', headers: { @@ -1397,7 +1397,8 @@ export class GBMinService { context.activity.text = context.activity.text.trim(); const member = context.activity.from; - let memberId = null, email = null; + let memberId = null, + email = null; // Processes e-mail from id in case of Teams messages. @@ -1669,7 +1670,7 @@ export class GBMinService { if (script === 'start') { pid = GBVMService.createProcessInfo(user, min, 'api', null); - const client = await GBUtil.getDirectLineClient(min); + const client = await GBUtil.getDirectLineClient(min); const response = await client.apis.Conversations.Conversations_StartConversation(); min['apiConversations'][pid] = { conversation: response.obj, client: client }; diff --git a/packages/kb.gbapp/services/KBService.ts b/packages/kb.gbapp/services/KBService.ts index fbc9d82d..7829c3f2 100644 --- a/packages/kb.gbapp/services/KBService.ts +++ b/packages/kb.gbapp/services/KBService.ts @@ -746,11 +746,11 @@ export class KBService implements IGBKBService { let doc = await loader.load(); let content = doc[0].pageContent; - if (file.name.endsWith('zap.docx')){ + if (file.name.endsWith('zap.docx')) { await min.whatsAppDirectLine.createOrUpdateTemplate(min, file.name, content); } - - const answer = { + + const answer = { instanceId: instance.instanceId, content: content, format: '.md', @@ -1369,10 +1369,12 @@ export class KBService implements IGBKBService { const p = await deployer.deployPackageToStorage(instance.instanceId, packageName); await this.importKbPackage(min, localPath, p, instance); GBDeployer.mountGBKBAssets(packageName, min.botId, localPath); - const service = await AzureDeployerService.createInstance(deployer); - const searchIndex = instance.searchIndex ? instance.searchIndex : GBServer.globals.minBoot.instance.searchIndex; - await deployer.rebuildIndex(instance, service.getKBSearchSchema(searchIndex)); + if (!process.env.STORAGE_FILE) { + const service = await AzureDeployerService.createInstance(deployer); + const searchIndex = instance.searchIndex ? instance.searchIndex : GBServer.globals.minBoot.instance.searchIndex; + await deployer.rebuildIndex(instance, service.getKBSearchSchema(searchIndex)); + } min['groupCache'] = await KBService.getGroupReplies(instance.instanceId); await KBService.RefreshNER(min);