fix (all): path and fs normalization.

packages/admin.gbapp/services/GBAdminService.ts

@@ -173,21 +173,7 @@ export class GBAdminService implements IGBAdminService {
     const gbaiPath = GBUtil.getGBAIPath(min.instance.botId, packageType, null);
     const localFolder = path.join('work', gbaiPath);
 
-    // .gbot packages are handled using storage API, so no download
-    // of local resources is required.
-    const gbai = GBUtil.getGBAIPath(min.instance.botId);
-
-    if (packageType === 'gbkb') {
-      await deployer['cleanupPackage'](min.instance, packageName);
-    }
-
-    if (!GBConfigService.get('STORAGE_NAME')) {
-      const filePath = path.join(GBConfigService.get('STORAGE_LIBRARY'), gbaiPath);
-      GBUtil.copyIfNewerRecursive(filePath, localFolder);
-    } else {
-      await deployer['downloadFolder'](min, path.join('work', `${gbai}`), path.basename(localFolder));
-    }
-    await deployer['deployPackage2'](min, user, localFolder);
+    await deployer['deployPackage2'](min, user, localFolder, true);
 
   }
   public static async rebuildIndexPackageCommand(min: GBMinInstance, deployer: GBDeployer) {

packages/basic.gblib/services/DialogKeywords.ts

@@ -1500,7 +1500,7 @@ export class DialogKeywords {
         'cache',
         `${fileOnly.replace(/\s/gi, '')}-${GBAdminService.getNumberIdentifier()}.${ext}`
       );
-      fs.writeFile(localName1, buf, { encoding: null });
+  await fs.writeFile(localName1, buf, { encoding: null });
 
       url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName1));
     }
@@ -1513,7 +1513,7 @@ export class DialogKeywords {
       const buf = await fs.readFile(filename);
       const gbaiName = GBUtil.getGBAIPath(min.botId);
       const localName = path.join('work', gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.${ext}`);
-      fs.writeFile(localName, buf, { encoding: null });
+  await fs.writeFile(localName, buf, { encoding: null });
       url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
     }
 
@@ -1546,7 +1546,7 @@ export class DialogKeywords {
 
     const gbaiName = GBUtil.getGBAIPath(min.botId);
     const localName = path.join('work', gbaiName, 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png`);
-    fs.writeFile(localName, buf, { encoding: null });
+await fs.writeFile(localName, buf, { encoding: null });
     const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
 
     return { data: data, localName: localName, url: url };

packages/basic.gblib/services/GBVMService.ts

@@ -155,7 +155,7 @@ export class GBVMService extends GBService {
 
       // Write VBS file without pragma keywords.
 
-      fs.writeFile(urlJoin(folder, vbsFile), text);
+  await fs.writeFile(urlJoin(folder, vbsFile), text);
     }
 
     // Process node_modules install.
@@ -215,7 +215,7 @@ export class GBVMService extends GBService {
                 "async-retry": "1.3.3"
               }
             }`;
-      fs.writeFile(urlJoin(folder, 'package.json'), packageJson);
+  await fs.writeFile(urlJoin(folder, 'package.json'), packageJson);
 
       GBLogEx.info(min, `Installing .gbdialog node_modules for ${min.botId}...`);
       const npmPath = urlJoin(process.env.PWD, 'node_modules', '.bin', 'npm');
@@ -498,10 +498,10 @@ export class GBVMService extends GBService {
     // Generates function JSON metadata to be used later.
 
     const jsonFile = `${filename}.json`;
-    fs.writeFile(jsonFile, JSON.stringify(metadata));
+await fs.writeFile(jsonFile, JSON.stringify(metadata));
 
     const mapFile = `${filename}.map`;
-    fs.writeFile(mapFile, JSON.stringify(map));
+await fs.writeFile(mapFile, JSON.stringify(map));
 
     // Execute off-line code tasks
 
@@ -711,7 +711,7 @@ export class GBVMService extends GBService {
 
     code = ji.default(code, '  ');
 
-    fs.writeFile(jsfile, code);
+await fs.writeFile(jsfile, code);
     GBLogEx.info(min, `[GBVMService] Finished loading of ${filename}, JavaScript from Word: \n ${code}`);
   }
 
@@ -723,7 +723,7 @@ export class GBVMService extends GBService {
         // Creates an empty object that will receive Sequelize fields.
 
         const tablesFile = `${task.file}.tables.json`;
-        fs.writeFile(tablesFile, JSON.stringify(task.tables));
+    await fs.writeFile(tablesFile, JSON.stringify(task.tables));
       }
     }
   }

packages/basic.gblib/services/SystemKeywords.ts

@@ -414,7 +414,7 @@ export class SystemKeywords {
         const buffer = pngPages[0].content;
         const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
 
-        fs.writeFile(localName, buffer, { encoding: null });
+    await fs.writeFile(localName, buffer, { encoding: null });
 
         return { localName: localName, url: url, data: buffer };
       }
@@ -707,7 +707,7 @@ export class SystemKeywords {
     // Writes it to disk and calculate hash.
 
     const data = await response.arrayBuffer();
-    fs.writeFile(localName, Buffer.from(data), { encoding: null });
+await fs.writeFile(localName, Buffer.from(data), { encoding: null });
     const hash = new Uint8Array(md5.array(data));
 
     // Performs uploading passing local hash.
@@ -1114,7 +1114,7 @@ export class SystemKeywords {
       const localName = path.join('work', gbaiName, 'cache', `csv${GBAdminService.getRndReadableIdentifier()}.csv`);
       const url = file['@microsoft.graph.downloadUrl'];
       const response = await fetch(url);
-      fs.writeFile(localName, Buffer.from(await response.arrayBuffer()), { encoding: null });
+  await fs.writeFile(localName, Buffer.from(await response.arrayBuffer()), { encoding: null });
 
       var workbook = new Excel.Workbook();
       let worksheet = await workbook.csv.readFile(localName);
@@ -1490,7 +1490,7 @@ export class SystemKeywords {
         user.userSystemId,
         'systemPrompt.txt'
       );
-      fs.writeFile(systemPromptFile, text);
+  await fs.writeFile(systemPromptFile, text);
     }
   }
 
@@ -1981,7 +1981,7 @@ export class SystemKeywords {
     const res = await fetch(url);
     let buf: any = Buffer.from(await res.arrayBuffer());
     localName = path.join('work', gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.docx`);
-    fs.writeFile(localName, buf, { encoding: null });
+await fs.writeFile(localName, buf, { encoding: null });
 
     // Replace image path on all elements of data.
 
@@ -2019,7 +2019,7 @@ export class SystemKeywords {
             );
             const response = await fetch(url);
             const buf = Buffer.from(await response.arrayBuffer());
-            fs.writeFile(imageName, buf, { encoding: null });
+        await fs.writeFile(imageName, buf, { encoding: null });
 
             const getNormalSize = ({ width, height, orientation }) => {
               return (orientation || 0) >= 5 ? [height, width] : [width, height];
@@ -2068,7 +2068,7 @@ export class SystemKeywords {
     doc.setData(data).render();
 
     buf = doc.getZip().generate({ type: 'nodebuffer', compression: 'DEFLATE' });
-    fs.writeFile(localName, buf, { encoding: null });
+await fs.writeFile(localName, buf, { encoding: null });
 
     return { localName: localName, url: url, data: buf };
   }
@@ -2526,7 +2526,7 @@ export class SystemKeywords {
 
     const buf = Buffer.from(data.Payment.QrCodeBase64Image, 'base64');
     const localName = path.join('work', gbaiName, 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png`);
-    fs.writeFile(localName, buf, { encoding: null });
+await fs.writeFile(localName, buf, { encoding: null });
     const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
 
     GBLogEx.info(min, `GBPay: ${data.MerchantOrderId} OK: ${url}.`);
@@ -2752,7 +2752,7 @@ export class SystemKeywords {
 
     // Criação de um arquivo temporário para enviar
     const tempFilePath = path.resolve('temp_image.jpg');
-    fs.writeFile(tempFilePath, imageBuffer);
+await fs.writeFile(tempFilePath, imageBuffer);
 
     // Publicação da imagem
     const page = new Page(pageId);

packages/core.gbapp/services/GBConfigService.ts

@@ -100,6 +100,7 @@ export class GBConfigService {
         case 'CLOUD_USERNAME':
           value = undefined;
           break;
+
         case 'CLOUD_PASSWORD':
           value = undefined;
           break;

packages/core.gbapp/services/GBConversationalService.ts

@@ -452,7 +452,7 @@ export class GBConversationalService {
         const waveFilename = `work/tmp${name}.pcm`;
 
         let audio = await textToSpeech.repairWavHeaderStream(res.result as any);
-        fs.writeFile(waveFilename, audio);
+    await fs.writeFile(waveFilename, audio);
 
         const oggFilenameOnly = `tmp${name}.ogg`;
         const oggFilename = `work/${oggFilenameOnly}`;
@@ -482,7 +482,7 @@ export class GBConversationalService {
 
         const dest = `work/tmp${name}.wav`;
         const src = `work/tmp${name}.ogg`;
-        fs.writeFile(src, oggFile.read());
+    await fs.writeFile(src, oggFile.read());
 
         const makeMp3 = shell([
           'node_modules/ffmpeg-static/ffmpeg', // TODO: .exe on MSWin.

packages/core.gbapp/services/GBCoreService.ts

@@ -35,7 +35,7 @@
 'use strict';
 
 import { GBLog, GBMinInstance, IGBCoreService, IGBInstallationDeployer, IGBInstance, IGBPackage } from 'botlib';
-import fs from 'fs/promises'; 
+import fs from 'fs/promises';
 import { Sequelize, SequelizeOptions } from 'sequelize-typescript';
 import { Op, Dialect } from 'sequelize';
 import { GBServer } from '../../../src/app.js';
@@ -135,7 +135,7 @@ export class GBCoreService implements IGBCoreService {
     } else if (this.dialect === 'sqlite') {
       storage = GBConfigService.get('STORAGE_FILE');
 
-      if (!await GBUtil.exists(storage)) {
+      if (!(await GBUtil.exists(storage))) {
         process.env.STORAGE_SYNC = 'true';
       }
     } else {
@@ -313,7 +313,7 @@ STORAGE_SYNC_ALTER=true
 ENDPOINT_UPDATE=true
 `;
 
-    fs.writeFile('.env', env);
+await fs.writeFile('.env', env);
   }
 
   /**
@@ -323,7 +323,10 @@ ENDPOINT_UPDATE=true
    */
   public async ensureProxy(port): Promise<string> {
     try {
-      if (await GBUtil.exists('node_modules/ngrok/bin/ngrok.exe') || await GBUtil.exists('node_modules/.bin/ngrok')) {
+      if (
+        (await GBUtil.exists('node_modules/ngrok/bin/ngrok.exe')) ||
+        (await GBUtil.exists('node_modules/.bin/ngrok'))
+      ) {
         return await ngrok.connect({ port: port });
       } else {
         GBLog.warn('ngrok executable not found. Check installation or node_modules folder.');
@@ -825,15 +828,15 @@ ENDPOINT_UPDATE=true
   public async ensureFolders(instances, deployer: GBDeployer) {
     let libraryPath = GBConfigService.get('STORAGE_LIBRARY');
 
-    if (!await GBUtil.exists(libraryPath)) {
+    if (!(await GBUtil.exists(libraryPath))) {
       mkdirp.sync(libraryPath);
     }
 
     await this.syncBotStorage(instances, 'default', deployer, libraryPath);
 
-    const files = fs.readdir(libraryPath);
+    const files = await fs.readdir(libraryPath);
     await CollectionUtil.asyncForEach(files, async file => {
-      if (file.trim().toLowerCase() !== 'default.gbai') {
+      if (file.trim().toLowerCase() !== 'default.gbai' && file.charAt(0) !== '_') {
         let botId = file.replace(/\.gbai/, '');
 
         await this.syncBotStorage(instances, botId, deployer, libraryPath);
@@ -855,7 +858,7 @@ ENDPOINT_UPDATE=true
       instance = await deployer.deployBlankBot(botId, mobile, email);
       const gbaiPath = path.join(libraryPath, `${botId}.gbai`);
 
-      if (!await GBUtil.exists(gbaiPath)) {
+      if (!(await GBUtil.exists(gbaiPath))) {
         fs.mkdir(gbaiPath, { recursive: true });
 
         const base = path.join(process.env.PWD, 'templates', 'default.gbai');

packages/core.gbapp/services/GBDeployer.ts

@@ -39,7 +39,7 @@ import express from 'express';
 import child_process from 'child_process';
 import { rimraf } from 'rimraf';
 import urlJoin from 'url-join';
-import fs from 'fs/promises'; 
+import fs from 'fs/promises';
 import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBDeployer, IGBInstance, IGBPackage } from 'botlib';
 import { AzureSearch } from 'pragmatismo-io-framework';
 import { CollectionUtil } from 'pragmatismo-io-framework';
@@ -152,9 +152,7 @@ export class GBDeployer implements IGBDeployer {
 
       const isDirectory = async source => (await fs.lstat(source)).isDirectory();
       const getDirectories = async source =>
-        (await fs.readdir(source))
-          .map(name => path.join(source, name))
-          .filter(isDirectory);
+        (await fs.readdir(source)).map(name => path.join(source, name)).filter(isDirectory);
       const dirs = await getDirectories(directory);
       await CollectionUtil.asyncForEach(dirs, async element => {
         // For each folder, checks its extensions looking for valid packages.
@@ -472,7 +470,7 @@ export class GBDeployer implements IGBDeployer {
       }
     });
 
-    GBLogEx.info(min, `Processing ${rows.length} rows from ${filePath}...`);
+    GBLogEx.info(min, `Processing ${rows.length} rows from ${path.basename(filePath)}...`);
     rows = null;
     return obj;
   }
@@ -497,13 +495,13 @@ export class GBDeployer implements IGBDeployer {
       // Creates each subfolder.
 
       let pathBase = localPath;
-      if (!await GBUtil.exists(pathBase)) {
+      if (!(await GBUtil.exists(pathBase))) {
         fs.mkdir(pathBase);
       }
 
       await CollectionUtil.asyncForEach(parts, async item => {
         pathBase = packagePath.join(pathBase, item);
-        if (!await GBUtil.exists(pathBase)) {
+        if (!(await GBUtil.exists(pathBase))) {
           fs.mkdir(pathBase);
         }
       });
@@ -514,7 +512,7 @@ export class GBDeployer implements IGBDeployer {
       packagePath = urlJoin(packagePath, remotePath);
       let url = `${baseUrl}/drive/root:/${packagePath}:/children`;
 
-      GBLogEx.info(min, `Download URL: ${url}`);
+      GBLogEx.info(min, `Downloading: ${url}`);
 
       const res = await client.api(url).get();
       const documents = res.value;
@@ -529,7 +527,7 @@ export class GBDeployer implements IGBDeployer {
         const itemPath = packagePath.join(localPath, remotePath, item.name);
 
         if (item.folder) {
-          if (!await GBUtil.exists(itemPath)) {
+          if (!(await GBUtil.exists(itemPath))) {
             fs.mkdir(itemPath);
           }
           const nextFolder = urlJoin(remotePath, item.name);
@@ -538,7 +536,7 @@ export class GBDeployer implements IGBDeployer {
           let download = true;
 
           if (await GBUtil.exists(itemPath)) {
-            const dt =await  fs.stat(itemPath);
+            const dt = await fs.stat(itemPath);
             if (new Date(dt.mtime) >= new Date(item.lastModifiedDateTime)) {
               download = false;
             }
@@ -549,10 +547,10 @@ export class GBDeployer implements IGBDeployer {
             const url = item['@microsoft.graph.downloadUrl'];
 
             const response = await fetch(url);
-            fs.writeFile(itemPath, Buffer.from(await response.arrayBuffer()), { encoding: null });
+        await fs.writeFile(itemPath, Buffer.from(await response.arrayBuffer()), { encoding: null });
             fs.utimes(itemPath, new Date(), new Date(item.lastModifiedDateTime));
           } else {
-            GBLogEx.info(min, `Local is up to date: ${itemPath}...`);
+            GBLogEx.info(min, `Local is up to date: ${path.basename(itemPath)}...`);
           }
         }
       });
@@ -594,11 +592,27 @@ export class GBDeployer implements IGBDeployer {
   /**
    * Deploys a folder into the bot storage.
    */
-  public async deployPackage2(min: GBMinInstance, user, localPath: string) {
-    const packageType = path.extname(localPath);
+  public async deployPackage2(min: GBMinInstance, user, packageWorkFolder: string, download = false) {
+    const packageName = path.basename(packageWorkFolder);
+    const packageType = path.extname(packageWorkFolder);
     let handled = false;
     let pck = null;
 
+    const gbai = GBUtil.getGBAIPath(min.instance.botId);
+
+    if (download) {
+      if (packageType === 'gbkb') {
+        await this.cleanupPackage(min.instance, packageName);
+      }
+
+      if (!GBConfigService.get('STORAGE_NAME')) {
+        const filePath = path.join(GBConfigService.get('STORAGE_LIBRARY'), gbai);
+        GBUtil.copyIfNewerRecursive(filePath, packageWorkFolder);
+      } else {
+        await this.downloadFolder(min, path.join('work', `${gbai}`), packageName);
+      }
+    }
+
     // Asks for each .gbapp if it will handle the package publishing.
 
     const _this = this;
@@ -608,7 +622,7 @@ export class GBDeployer implements IGBDeployer {
 
       if (
         (pck = await e.onExchangeData(min, 'handlePackage', {
-          name: localPath,
+          name: packageWorkFolder,
           createPackage: async packageName => {
             return await _this.deployPackageToStorage(min.instance.instanceId, packageName);
           },
@@ -634,7 +648,7 @@ export class GBDeployer implements IGBDeployer {
       case '.gbot':
         // Extracts configuration information from .gbot files.
 
-        min.instance.params = await this.loadParamsFromTabular(min, localPath);
+        min.instance.params = await this.loadParamsFromTabular(min, packageWorkFolder);
         if (min.instance.params) {
           let connections = [];
 
@@ -667,12 +681,11 @@ export class GBDeployer implements IGBDeployer {
 
           const packagePath = GBUtil.getGBAIPath(min.botId, null);
           const localFolder = path.join('work', packagePath, 'connections.json');
-          fs.writeFile(localFolder, JSON.stringify(connections), { encoding: null });
+      await fs.writeFile(localFolder, JSON.stringify(connections), { encoding: null });
 
           // Updates instance object.
 
           await this.core.saveInstance(min.instance);
-          GBLogEx.info(min, `Reloading bot ${min.botId}...`);
           GBServer.globals.minService.unmountBot(min.botId);
           GBServer.globals.minService.mountBot(min.instance);
           GBLogEx.info(min, `Bot ${min.botId} reloaded.`);
@@ -683,7 +696,7 @@ export class GBDeployer implements IGBDeployer {
         // Deploys .gbkb into the storage.
 
         const service = new KBService(this.core.sequelize);
-        await service.deployKb(this.core, this, localPath, min);
+        await service.deployKb(this.core, this, packageWorkFolder, min);
         break;
 
       case '.gbdialog':
@@ -691,15 +704,14 @@ export class GBDeployer implements IGBDeployer {
         // it to the VM.
 
         const vm = new GBVMService();
-        await vm.loadDialogPackage(localPath, min, this.core, this);
+        await vm.loadDialogPackage(packageWorkFolder, min, this.core, this);
         GBLogEx.verbose(min, `Dialogs (.gbdialog) for ${min.botId} loaded.`);
         break;
 
       case '.gbtheme':
         // Updates server listeners to serve theme files in .gbtheme.
 
-        const packageName = path.basename(localPath);
-        GBServer.globals.server.use(`/themes/${packageName}`, express.static(localPath));
+        GBServer.globals.server.use(`/themes/${packageName}`, express.static(packageWorkFolder));
         GBLogEx.verbose(min, `Theme (.gbtheme) assets accessible at: /themes/${packageName}.`);
 
         break;
@@ -707,13 +719,13 @@ export class GBDeployer implements IGBDeployer {
       case '.gbapp':
         // Dynamically compiles and loads .gbapp packages (Node.js packages).
 
-        await this.callGBAppCompiler(localPath, this.core);
+        await this.callGBAppCompiler(packageWorkFolder, this.core);
         break;
 
       case '.gblib':
         // Dynamically compiles and loads .gblib packages (Node.js packages).
 
-        await this.callGBAppCompiler(localPath, this.core);
+        await this.callGBAppCompiler(packageWorkFolder, this.core);
         break;
 
       default:
@@ -736,7 +748,6 @@ export class GBDeployer implements IGBDeployer {
   public async undeployPackageFromPackageName(instance: IGBInstance, packageName: string) {
     // Gets information about the package.
 
-    
     const p = await this.getStoragePackageByName(instance.instanceId, packageName);
 
     const packagePath = GBUtil.getGBAIPath(instance.botId, null, packageName);
@@ -881,7 +892,7 @@ export class GBDeployer implements IGBDeployer {
     if (!(await GBUtil.exists(`${root}/build`)) && process.env.DISABLE_WEB !== 'true') {
       // Write a .env required to fix some bungs in create-react-app tool.
 
-      fs.writeFile(`${root}/.env`, 'SKIP_PREFLIGHT_CHECK=true');
+  await fs.writeFile(`${root}/.env`, 'SKIP_PREFLIGHT_CHECK=true');
 
       // Install modules and compiles the web app.
 
@@ -930,10 +941,9 @@ export class GBDeployer implements IGBDeployer {
       express.static(urlJoin('work', gbaiName, filename, 'videos'))
     );
     GBServer.globals.server.use(`/${botId}/cache`, express.static(urlJoin('work', gbaiName, 'cache')));
-    
-    
+
     // FEAT-A7B1F6
-    
+
     GBServer.globals.server.use(
       `/${gbaiName}/${botId}.gbdrive/public`,
       express.static(urlJoin('work', gbaiName, `${botId}.gbdata`, 'public'))
@@ -956,8 +966,8 @@ export class GBDeployer implements IGBDeployer {
     GBLogEx.info(0, `Deploying General Bots Application (.gbapp) or Library (.gblib): ${path.basename(gbappPath)}...`);
     let folder = path.join(gbappPath, 'node_modules');
     if (process.env.GBAPP_DISABLE_COMPILE !== 'true') {
-      if (!await GBUtil.exists(folder)) {
-        GBLogEx.info(0, `Installing modules for ${gbappPath}...`);
+      if (!(await GBUtil.exists(folder))) {
+        GBLogEx.info(0, `Installing modules for ${path.basename(gbappPath)}...`);
         child_process.execSync('npm install', { cwd: gbappPath });
       }
     }
@@ -967,7 +977,7 @@ export class GBDeployer implements IGBDeployer {
       // Runs TSC in .gbapp folder.
 
       if (process.env.GBAPP_DISABLE_COMPILE !== 'true') {
-        GBLogEx.info(0, `Compiling: ${gbappPath}.`);
+        GBLogEx.info(0, `Compiling: ${path.basename(gbappPath)}.`);
         child_process.execSync(path.join(process.env.PWD, 'node_modules/.bin/tsc'), { cwd: gbappPath });
       }
 

packages/core.gbapp/services/GBMinService.ts

@@ -36,6 +36,10 @@
 import { createRpcServer } from '@push-rpc/core';
 import AuthenticationContext from 'adal-node';
 import arrayBufferToBuffer from 'arraybuffer-to-buffer';
+import { watch } from 'fs';
+import debounce from 'lodash.debounce';
+
+import chokidar from 'chokidar';
 import {
   AutoSaveStateMiddleware,
   BotFrameworkAdapter,
@@ -45,7 +49,14 @@ import {
   UserState
 } from 'botbuilder';
 import { FacebookAdapter } from 'botbuilder-adapter-facebook';
-import { AttachmentPrompt, ConfirmPrompt, DialogSet, OAuthPrompt, TextPrompt, WaterfallDialog } from 'botbuilder-dialogs';
+import {
+  AttachmentPrompt,
+  ConfirmPrompt,
+  DialogSet,
+  OAuthPrompt,
+  TextPrompt,
+  WaterfallDialog
+} from 'botbuilder-dialogs';
 import { MicrosoftAppCredentials } from 'botframework-connector';
 import {
   GBDialogStep,
@@ -313,39 +324,48 @@ export class GBMinService {
     let dir = `work/${gbai}/cache`;
     const botId = gbai.replace(/\.[^/.]+$/, '');
 
-    if (!await GBUtil.exists(dir)) {
+    if (!(await GBUtil.exists(dir))) {
       mkdirp.sync(dir);
     }
     dir = `work/${gbai}/profile`;
-    if (!await GBUtil.exists(dir)) {
+    if (!(await GBUtil.exists(dir))) {
       mkdirp.sync(dir);
     }
     dir = `work/${gbai}/uploads`;
-    if (!await GBUtil.exists(dir)) {
+    if (!(await GBUtil.exists(dir))) {
       mkdirp.sync(dir);
     }
+
     dir = `work/${gbai}/${botId}.gbkb`;
-    if (!await GBUtil.exists(dir)) {
+    if (!(await GBUtil.exists(dir))) {
       mkdirp.sync(dir);
     }
+    await this.watchPackages(min, 'gbkb');
+
     dir = `work/${gbai}/${botId}.gbkb/docs-vectorized`;
-    if (!await GBUtil.exists(dir)) {
+    if (!(await GBUtil.exists(dir))) {
       mkdirp.sync(dir);
     }
+
     dir = `work/${gbai}/${botId}.gbdialog`;
-    if (!await GBUtil.exists(dir)) {
+    if (!(await GBUtil.exists(dir))) {
       mkdirp.sync(dir);
     }
+    await this.watchPackages(min, 'gbdialog');
+
     dir = `work/${gbai}/${botId}.gbot`;
-    if (!await GBUtil.exists(dir)) {
+    if (!(await GBUtil.exists(dir))) {
       mkdirp.sync(dir);
     }
+    await this.watchPackages(min, 'gbot');
+
     dir = `work/${gbai}/${botId}.gbui`;
-    if (!await GBUtil.exists(dir)) {
+    if (!(await GBUtil.exists(dir))) {
       mkdirp.sync(dir);
     }
+
     dir = `work/${gbai}/users`;
-    if (!await GBUtil.exists(dir)) {
+    if (!(await GBUtil.exists(dir))) {
       mkdirp.sync(dir);
     }
 
@@ -354,7 +374,7 @@ export class GBMinService {
     await this.invokeLoadBot(min.appPackages, GBServer.globals.sysPackages, min);
     const receiver = async (req, res) => {
       let path = /(http[s]?:\/\/)?([^\/\s]+\/)(.*)/gi;
-      const botId = req.url.substr(req.url.lastIndexOf('/') +1);
+      const botId = req.url.substr(req.url.lastIndexOf('/') + 1);
 
       const min = GBServer.globals.minInstances.filter(p => p.instance.botId == botId)[0];
 
@@ -387,10 +407,9 @@ export class GBMinService {
 
     const manifest = `${instance.botId}-Teams.zip`;
     const packageTeams = urlJoin(`work`, GBUtil.getGBAIPath(instance.botId), manifest);
-    if (!await GBUtil.exists(packageTeams)) {
-      GBLogEx.info(min, 'Generating MS Teams manifest....');
+    if (!(await GBUtil.exists(packageTeams))) {
       const data = await this.deployer.getBotManifest(instance);
-      fs.writeFile(packageTeams, data);
+  await fs.writeFile(packageTeams, data);
     }
 
     // Serves individual URL for each bot user interface.
@@ -468,7 +487,7 @@ export class GBMinService {
 
     return min;
   }
-
+   
   public static getProviderName(req: any, res: any) {
     if (!res) {
       return 'GeneralBots';
@@ -1098,7 +1117,7 @@ export class GBMinService {
             const folder = `work/${path}/cache`;
             const filename = `${GBAdminService.generateUuid()}.png`;
 
-            fs.writeFile(urlJoin(folder, filename), data);
+        await fs.writeFile(urlJoin(folder, filename), data);
             step.context.activity.text = urlJoin(
               GBServer.globals.publicAddress,
               `${min.instance.botId}`,
@@ -1286,7 +1305,7 @@ export class GBMinService {
       buffer = arrayBufferToBuffer(await res.arrayBuffer());
     }
 
-    fs.writeFile(localFileName, buffer);
+await fs.writeFile(localFileName, buffer);
 
     return {
       fileName: attachment.name,
@@ -1723,4 +1742,65 @@ export class GBMinService {
 
     createRpcServer(proxies, GBServer.globals.server.apiServer, opts);
   }
+
+    // Map to track recent changes with timestamps
+    private  recentChanges: Map<string, number> = new Map();
+
+  private async watchPackages(min: GBMinInstance, packageType) {
+    if (!GBConfigService.get('STORAGE_NAME')) {
+      const packagePath = GBUtil.getGBAIPath(min.botId, packageType);
+      const dirPath = path.join(GBConfigService.get('STORAGE_LIBRARY'), packagePath);
+  
+      const watcher = chokidar.watch(dirPath, {
+        persistent: true,
+        ignoreInitial: true, // Ignore initial add events
+        depth: 99, // Watch subdirectories
+        awaitWriteFinish: {
+          stabilityThreshold: 500, // Wait for 500ms to ensure file is written completely
+        }
+      });
+  
+      const WRITE_INTERVAL = 5000; // 5 seconds interval
+  
+      // Function to handle file changes and prevent multiple calls within the 5-second window
+      const handleFileChange = async (filePath) => {
+        const now = Date.now();
+  
+        // Add or update the file in the recent changes list
+        this.recentChanges.set(filePath, now);
+  
+        // Clean up entries older than 5 seconds
+        for (const [key, timestamp] of this.recentChanges.entries()) {
+          if (now - timestamp > WRITE_INTERVAL) {
+            this.recentChanges.delete(key);
+          }
+        }
+  
+        // If we have recent changes, deploy the package only once
+        if (this.recentChanges.size > 0) {
+          
+          try {
+                        
+            await this.deployer.deployPackage2(min, null, dirPath);
+
+          } catch (error) {
+            GBLogEx.error(min, `Error deploying package: ${GBUtil.toYAML(error)}`);
+          }
+  
+          // Delay further processing to prevent multiple deployments within 5 seconds
+          await new Promise(resolve => setTimeout(resolve, WRITE_INTERVAL));
+          
+          // After the delay, clear only entries that were processed
+          this.recentChanges.clear();
+        }
+      };
+  
+      // Watch for file changes
+      watcher.on('change', (filePath) => {
+        handleFileChange(filePath).catch(error => console.error('Error processing file change:', error));
+      });
+  
+    }
+  }
+  
 }

packages/core.gbapp/services/GBSSR.ts

@@ -110,7 +110,7 @@ export class GBSSR {
         const file = await fs.readFile(preferences, 'utf8');
         const data = JSON.parse(file);
         data['profile']['exit_type'] = 'none';
-        fs.writeFile(preferences, JSON.stringify(data));
+    await fs.writeFile(preferences, JSON.stringify(data));
       }
     }
 

packages/kb.gbapp/services/KBService.ts

@@ -441,9 +441,15 @@ export class KBService implements IGBKBService {
     min: GBMinInstance,
     packageId: number
   ): Promise<GuaribasQuestion[]> {
-    GBLogEx.info(min, `Now reading file ${filePath}...`);
+    GBLogEx.info(min, `Now reading file ${path.basename(filePath)}...`);
     const workbook = new Excel.Workbook();
-    const data = await workbook.xlsx.readFile(filePath);
+
+    let data;
+    if (filePath.endsWith('.xlsx')) {
+      data = await workbook.xlsx.readFile(filePath);
+    } else if (filePath.endsWith('.csv')) {
+      data = await workbook.csv.readFile(filePath);
+    }
 
     let lastQuestionId: number;
     let lastAnswer: GuaribasAnswer;
@@ -451,11 +457,13 @@ export class KBService implements IGBKBService {
     // Finds a valid worksheet because Excel returns empty slots
     // when loading worksheets collection.
 
-    let worksheet: any;
-    for (let t = 0; t < data.worksheets.length; t++) {
-      worksheet = data.worksheets[t];
-      if (worksheet) {
-        break;
+    let worksheet = data;
+    if (!worksheet) {
+      for (let t = 0; t < data.worksheets.length; t++) {
+        worksheet = data.worksheets[t];
+        if (worksheet) {
+          break;
+        }
       }
     }
 
@@ -778,7 +786,7 @@ export class KBService implements IGBKBService {
               path.basename(localName)
             );
             const buffer = await image.read();
-            fs.writeFile(localName, buffer, { encoding: null });
+            await fs.writeFile(localName, buffer, { encoding: null });
             return { src: url };
           }
         };
@@ -1098,15 +1106,12 @@ export class KBService implements IGBKBService {
         } catch (error) {
           GBLogEx.info(min, `Ignore processing of ${file}. ${GBUtil.toYAML(error)}`);
         }
-
       });
     }
 
     files = await walkPromise(urlJoin(localPath, 'docs'));
 
-    if (!files[0]) {
-      GBLogEx.info(min, `[GBDeployer] docs folder not created yet in .gbkb neither a website in .gbot.`);
-    } else {
+    if (files[0]) {
       await CollectionUtil.asyncForEach(files, async file => {
         let content = null;
         let filePath = path.join(file.root, file.name);
@@ -1181,7 +1186,7 @@ export class KBService implements IGBKBService {
     const files = await walkPromise(localPath);
 
     await CollectionUtil.asyncForEach(files, async file => {
-      if (file !== null && file.name.endsWith('.xlsx')) {
+      if (file !== null && (file.name.endsWith('.xlsx') || file.name.endsWith('.csv'))) {
         return await this.importKbTabularFile(urlJoin(file.root, file.name), min, packageId);
       }
     });
@@ -1340,7 +1345,7 @@ export class KBService implements IGBKBService {
   public async deployKb(core: IGBCoreService, deployer: GBDeployer, localPath: string, min: GBMinInstance) {
     const packageName = path.basename(localPath);
     const instance = await core.loadInstanceByBotId(min.botId);
-    GBLogEx.info(min, `[GBDeployer] Importing: ${localPath}`);
+    GBLogEx.info(min, `Publishing: ${path.basename(localPath)}`);
 
     const p = await deployer.deployPackageToStorage(instance.instanceId, packageName);
     await this.importKbPackage(min, localPath, p, instance);
@@ -1354,14 +1359,18 @@ export class KBService implements IGBKBService {
     min['groupCache'] = await KBService.getGroupReplies(instance.instanceId);
     await KBService.RefreshNER(min);
 
-    GBLogEx.info(min, `[GBDeployer] Start Bot Server Side Rendering... ${localPath}`);
-    const html = await GBSSR.getHTML(min);
-    let packagePath = GBUtil.getGBAIPath(min.botId, `gbui`);
-    packagePath = path.join(process.env.PWD, 'work', packagePath, 'index.html');
-    GBLogEx.info(min, `[GBDeployer] Saving SSR HTML in ${packagePath}.`);
-    fs.writeFile(packagePath, html, 'utf8');
+    const ssr = min.core.getParam<boolean>(min.instance, 'SSR', false);
 
-    GBLogEx.info(min, `[GBDeployer] Finished import of ${localPath}`);
+    if (ssr) {
+      GBLogEx.info(min, `Start Bot Server Side Rendering... ${localPath}`);
+      const html = await GBSSR.getHTML(min);
+      let packagePath = GBUtil.getGBAIPath(min.botId, `gbui`);
+      packagePath = path.join(process.env.PWD, 'work', packagePath, 'index.html');
+      GBLogEx.info(min, `Saving SSR HTML in ${packagePath}.`);
+      await fs.writeFile(packagePath, html, 'utf8');
+    }
+
+    GBLogEx.info(min, `Done publishing of: ${localPath}.`);
   }
 
   private async playAudio(
@@ -1434,7 +1443,6 @@ export class KBService implements IGBKBService {
     directoryPath: string
   ): Promise<string | null> {
     try {
-      
       // Check if the directory exists, create it if not.
 
       const directoryExists = await GBUtil.exists(directoryPath);
@@ -1443,7 +1451,7 @@ export class KBService implements IGBKBService {
       }
 
       // Check if the URL is for a downloadable file (e.g., .pdf).
-      
+
       if (
         url.endsWith('.pdf') ||
         url.endsWith('.docx') ||

packages/llm.gblib/services/ChatServices.ts

@@ -188,7 +188,7 @@ export class ChatServices {
       const gbaiName = GBUtil.getGBAIPath(min.botId, null);
       const localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`);
       const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
-      fs.writeFile(localName, buffer, { encoding: null });
+  await fs.writeFile(localName, buffer, { encoding: null });
       return { localName: localName, url: url, data: buffer };
     }
   }

packages/llm.gblib/services/ImageServices.ts

@@ -73,7 +73,7 @@ export class ImageServices {
       const url = response.data[0].url;
       const res = await fetch(url);
       let buf: any = Buffer.from(await res.arrayBuffer());
-      fs.writeFile(localName, buf, { encoding: null });
+  await fs.writeFile(localName, buf, { encoding: null });
 
       GBLogEx.info(min, `DALL-E image generated at ${url}.`);
 

packages/whatsapp.gblib/services/WhatsappDirectLine.ts

@@ -188,7 +188,7 @@ export class WhatsappDirectLine extends GBService {
                 'cache',
                 `qr${GBAdminService.getRndReadableIdentifier()}.png`
               );
-              fs.writeFile(localName, qrBuf.data);
+          await fs.writeFile(localName, qrBuf.data);
               const url = urlJoin(GBServer.globals.publicAddress, this.min.botId, 'cache', path.basename(localName));
 
               if (adminNumber) {
@@ -331,7 +331,7 @@ export class WhatsappDirectLine extends GBService {
             'cache',
             `tmp${GBAdminService.getRndReadableIdentifier()}.docx`
           );
-          fs.writeFile(localName, buf, { encoding: null });
+      await fs.writeFile(localName, buf, { encoding: null });
           const url = urlJoin(GBServer.globals.publicAddress, this.min.botId, 'cache', path.basename(localName));
 
           attachments = [];

templates/default.gbai/default.gbot/config.csv

@@ -39,6 +39,7 @@ NLP Score,
 Notes,
 Open AI Key,
 Search Score,
+SSR,
 Start Dialog,
 Store Answer Score,
 Synchronize Database,