Merge branch 'main' of https://github.com/GeneralBots/BotServer

.travis.yml

@@ -1,7 +1,7 @@
 dist: focal
 language: node_js
 node_js:
-  - 19.5.0
+  - 19.6.0
 
 
 notifications:

.vscode/launch.json

@@ -9,10 +9,13 @@
       "program": "${workspaceRoot}/boot.mjs",
       "cwd": "${workspaceRoot}",
       "env": {
-        "NODE_ENV": "development"
+        "NODE_ENV": "development",
+        "NODE_NO_WARNINGS":"1"
       },
       "args": [
-        "--no-deprecation"
+        "--no-deprecation", 
+        "--loader ts-node/esm", 
+        "--require ${workspaceRoot}/suppress-node-warnings.cjs", 
       ],
       "skipFiles": [
         "node_modules/**/*.js"

boot.mjs

@@ -3,63 +3,56 @@
 import Fs from 'fs';
 import Path from 'path';
 import { exec } from 'child_process';
-import pjson from './package.json' assert { type: "json" };
+import pjson from './package.json' assert { type: 'json' };
-import * as GBServer from "./dist/src/app.js";
 
 // Displays version of Node JS being used at runtime and others attributes.
 
-console.log(`[GB Runtime] BotServer        = v${pjson.version}`);
-console.log(`[GB Runtime] BotLib           = v${pjson.dependencies.botlib}`);
-console.log(`[GB Runtime] BotBuilder (MS)  = v${pjson.dependencies.botbuilder}`);
-console.log(`[GB Runtime] NodeJS           = ${process.version}`);
-console.log(`[GB Runtime] platform         = ${process.platform}`);
-console.log(`[GB Runtime] architecture     = ${process.arch}`);
-console.log(`[GB Runtime] argv             = ${process.argv}`);
-console.log(`[GB Runtime] debugPort        = ${process.debugPort}`);
 
+console.log(``);
+console.log(``);
+console.log(``);
+console.log(` █████   ██████ ███  ██ ██████ █████   █████  ██      █████   █████  ██████   ███ ® `);
+console.log(` ██  ███ ██ █   ██ █ ██ ██ █   ██ ██  ███ ███ ██      ██████ █ * * █   ██     █     `);
+console.log(` ██████  ██████ ██  ███ ██████ ██  ██ ██   ██ █████   █████   █████    ██  ████  3.0`);
+console.log(``);
+console.debug(`botserver@${pjson.version}, botlib@${pjson.dependencies.botlib}, botbuilder@${pjson.dependencies.botbuilder}, nodeJS: ${process.version}, platform: ${process.platform}, architecture: ${process.arch}.`);
 var now = () => {
-    return (new Date()).toISOString().replace(/T/, ' ').replace(/\..+/, '') + ' UTC';
+  return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '') + ' UTC';
-}
+};
 var __dirname = process.env.PWD;
 try {
-
+  var run = () => {
-    var run = () => {
+    import('./dist/src/app.js').then((gb)=> gb.GBServer.run());
-        
+  };
-        console.log(`[GB Runtime] Initializing General Bots (BotServer)...`);
+  var processDist = () => {
-        console.log(`[GB Runtime] ${now()} - Running on '${import.meta.url}'`);
+    if (!Fs.existsSync('dist')) {
-        GBServer.GBServer.run();
+      console.log(`${now()} - Compiling...`);
-    }
+      exec(Path.join(__dirname, 'node_modules/.bin/tsc'), (err, stdout, stderr) => {
-    var processDist = () => {
+        if (err) {
-        if (!Fs.existsSync('dist')) {
+          console.error(err);
-            console.log(`${now()} - Compiling...`);
+          return;
-            exec(Path.join(__dirname, 'node_modules/.bin/tsc'), (err, stdout, stderr) => {
-                if (err) {
-                    console.error(err);
-                    return;
-                }
-                run();
-            });
         }
-        else {
+        run();
-            run();
+      });
-        }
+    } else {
-    };
+      run();
-
-    // Installing modules if it has not been done yet.
-
-    if (!Fs.existsSync('node_modules')) {
-        console.log(`${now()} - Installing modules for the first time, please wait...`);
-        exec('npm install', (err, stdout, stderr) => {
-            if (err) {
-                console.error(err);
-                return;
-            }
-            processDist();
-        });
-    }
-    else {
-        processDist();
     }
+  };
+
+  // Installing modules if it has not been done yet.
+
+  if (!Fs.existsSync('node_modules')) {
+    console.log(`${now()} - Installing modules for the first time, please wait...`);
+    exec('npm install', (err, stdout, stderr) => {
+      if (err) {
+        console.error(err);
+        return;
+      }
+      processDist();
+    });
+  } else {
+    processDist();
+  }
 } catch (e) {
-    console.log(e);
+  console.log(e);
 }

gbot.sh

@@ -1,5 +1,2 @@
-echo General Bots
+echo Starting General Bots...
-echo Installing modules for the first time...
+npm run start
-
-npm i
-node .

package.json

@@ -14,7 +14,7 @@
     "Dário Vieira <dario.junior3@gmail.com>"
   ],
   "engines": {
-    "node": "=19.5.0"
+    "node": "=19.6.0"
   },
   "license": "AGPL-3.0",
   "preferGlobal": true,
@@ -34,7 +34,7 @@
     "build-gbui": "cd packages/default.gbui && echo SKIP_PREFLIGHT_CHECK=true >.env &&  npm install && npm run build",
     "build-docs": "typedoc --options typedoc.json src/",
     "test": "node test.js",
-    "start": "node ./boot.cjs",
+    "start": "NODE_NO_WARNINGS=1 node ./boot.mjs --loader ts-node/esm --require ./suppress-node-warnings.cjs ",
     "reverse-proxy": "node_modules/.bin/ngrok http 4242",
     "watch:build": "tsc --watch",
     "posttypedoc": "shx cp .nojekyll docs/reference/.nojekyll",
@@ -75,7 +75,8 @@
     "adm-zip": "0.5.9",
     "alasql": "2.1.6",
     "any-shell-escape": "0.1.1",
-    "arraybuffer-to-buffer": "^0.0.7",
+    "arraybuffer-to-buffer": "0.0.7",
+    "async-mutex": "0.4.0",
     "async-promises": "0.2.3",
     "basic-auth": "2.0.1",
     "billboard.js": "3.6.3",
@@ -86,7 +87,7 @@
     "botbuilder-ai": "4.18.0",
     "botbuilder-dialogs": "4.18.0",
     "botframework-connector": "4.18.0",
-    "botlib": "3.0.2",
+    "botlib": "3.0.5",
     "c3-chart-maker": "0.2.8",
     "chatgpt": "2.4.2",
     "chrome-remote-interface": "0.31.3",
@@ -106,6 +107,7 @@
     "google-libphonenumber": "3.2.31",
     "googleapis": "109.0.1",
     "ibm-watson": "7.1.2",
+    "join-images-updated": "1.1.4",
     "keyv": "4.5.2",
     "koa": "2.13.4",
     "koa-body": "6.0.1",
@@ -117,22 +119,22 @@
     "ms-rest-azure": "3.0.0",
     "nexmo": "2.9.1",
     "node-cron": "3.0.2",
+    "node-html-parser": "6.1.5",
     "node-nlp": "4.24.0",
     "node-tesseract-ocr": "2.2.1",
-    "npm": "9.1.2",
     "open": "8.4.0",
-    "open-docxtemplater-image-module": "^1.0.3",
+    "open-docxtemplater-image-module": "1.0.3",
     "pdf-extraction": "1.0.2",
-    "pdf-to-png-converter": "^2.7.1",
+    "pdf-to-png-converter": "2.7.1",
     "pdfkit": "0.13.0",
     "phone": "3.1.30",
     "pizzip": "3.1.3",
     "pptxtemplater": "1.0.5",
-    "pragmatismo-io-framework": "1.0.20",
+    "pragmatismo-io-framework": "1.1.0",
     "prism-media": "1.3.4",
     "public-ip": "6.0.1",
     "punycode": "2.1.1",
-    "puppeteer": "19.2.2",
+    "puppeteer": "19.6.3",
     "puppeteer-extra": "3.3.4",
     "puppeteer-extra-plugin-stealth": "2.11.1",
     "qrcode": "1.5.1",
@@ -142,11 +144,11 @@
     "rimraf": "3.0.2",
     "safe-buffer": "5.2.1",
     "scanf": "1.1.2",
-    "sequelize": "6.25.7",
+    "sequelize": "6.28.2",
-    "sequelize-cli": "6.5.2",
+    "sequelize-cli": "6.6.0",
     "sequelize-typescript": "2.1.5",
-    "sharp": "^0.31.3",
+    "sharp": "0.31.3",
-    "simple-git": "3.15.0",
+    "simple-git": "3.16.0",
     "speakingurl": "14.0.1",
     "ssr-for-bots": "1.0.1-c",
     "strict-password-generator": "1.1.2",
@@ -155,7 +157,7 @@
     "tedious": "15.1.2",
     "textract": "2.5.0",
     "twitter-api-v2": "1.12.9",
-    "typescript": "4.9.3",
+    "typescript": "4.9.5",
     "typescript-rest-rpc": "1.0.7",
     "url-join": "5.0.0",
     "vbscript-to-typescript": "1.0.8",
@@ -164,7 +166,7 @@
     "vm2-process": "2.1.1",
     "walk-promise": "0.2.0",
     "washyourmouthoutwithsoap": "1.0.2",
-    "whatsapp-web.js": "1.18.3",
+    "whatsapp-web.js": "github:pedroslopez/whatsapp-web.js#fix-buttons-list",
     "winston": "3.8.2",
     "winston-logs-display": "1.0.0",
     "yarn": "1.22.19"
@@ -181,8 +183,8 @@
     "prettier-standard": "15.0.1",
     "semantic-release": "17.2.4",
     "simple-commit-message": "4.0.13",
-    "super-strong-password-generator": "^2.0.2",
+    "super-strong-password-generator": "2.0.2",
-    "super-strong-password-generator-es": "^2.0.2",
+    "super-strong-password-generator-es": "2.0.2",
     "travis-deploy-once": "5.0.11",
     "ts-node": "10.9.1",
     "tslint": "6.1.3"

packages/admin.gbapp/dialogs/AdminDialog.ts

@@ -238,6 +238,24 @@ export class AdminDialog extends IGBDialog {
       ])
     );
 
+
+    min.dialogs.add(
+      new WaterfallDialog('/logs', [
+        async step => {
+          if (step.context.activity.channelId !== 'msteams' && process.env.ENABLE_AUTH) {
+            return await step.beginDialog('/auth');
+          } else {
+            return await step.next(step.options);
+          }
+        },
+        async step => {
+          const logs = await min.core['getLatestLogs'](); 
+          await min.conversationalService.sendText(min, step, logs);
+          return await step.replaceDialog('/ask', { isReturning: true });
+        }
+      ]));
+
+
     min.dialogs.add(
       new WaterfallDialog('/publish', [
         async step => {

packages/basic.gblib/index.ts

@@ -60,7 +60,8 @@ export class GBBasicPackage implements IGBPackage {
 
   public async loadPackage (core: IGBCoreService, sequelize: Sequelize): Promise<void> {
     core.sequelize.addModels([GuaribasSchedule]);
-    app.use(koaBody.koaBody({ multipart: true }));
+    //app.use(koaBody.koaBody({ multipart: true,  }));
+
     app.listen(1111);
   }
 
@@ -97,5 +98,11 @@ export class GBBasicPackage implements IGBPackage {
     app.use(waRouter.routes());
     app.use(dbgRouter.routes());
     app.use(imgRouter.routes());
+    app.use(async (ctx, next) => {
+      if(ctx['status'] === 404){
+         ctx.status = 404
+         ctx.body = {msg:'emmmmmmm, seems 404'};
+      }
+  });
   }
 }

packages/basic.gblib/services/ChartServices.ts

@@ -32,7 +32,7 @@
 
 'use strict';
 
-import { createBrowser } from '../../core.gbapp/services/GBSSR.js';
+import { GBSSR }from '../../core.gbapp/services/GBSSR.js';
 
 export class ChartServices {
   /**
@@ -41,7 +41,7 @@ export class ChartServices {
    * @param {string} path screenshot image full path with file name
    */
   public static async screenshot (args, path) {
-    const browser = await createBrowser(null);
+    const browser = await GBSSR.createBrowser(null);
     const page = await browser.newPage();
 
     // load billboard.js assets from CDN.

packages/basic.gblib/services/DialogKeywords.ts

@@ -58,6 +58,7 @@ import mammoth from 'mammoth';
 import qrcode from 'qrcode';
 import { json } from 'body-parser';
 import { WebAutomationServices } from './WebAutomationServices.js';
+import urljoin from 'url-join';
 
 /**
  * Default check interval for user replay
@@ -562,7 +563,7 @@ export class DialogKeywords {
    *
    */
   public async sendFile({ pid, filename, caption }) {
-    const mobile = await this.userMobile({pid});
+    const mobile = await this.userMobile({ pid });
     GBLog.info(`BASIC: SEND FILE (current: ${mobile},filename '${filename}'.`);
     return await this.internalSendFile({ pid, mobile, filename, caption });
   }
@@ -606,10 +607,8 @@ export class DialogKeywords {
     return names.indexOf(name) > -1;
   }
 
-
+  private async setOption({ pid, name, value }) {
-  private async setOption({pid, name, value})
+    if (this.isUserSystemParam(name)) {
-  {
-    if (this.isUserSystemParam(name)){
       throw new Error(`Not possible to define ${name} as it is a reserved system param name.`);
     }
     const process = GBServer.globals.processes[pid];
@@ -620,9 +619,8 @@ export class DialogKeywords {
     return { min, user, params };
   }
 
-  private async getOption({pid, name})
+  private async getOption({ pid, name }) {
-  {
+    if (this.isUserSystemParam(name)) {
-    if (this.isUserSystemParam(name)){
       throw new Error(`Not possible to retrieve ${name} system param.`);
     }
     const process = GBServer.globals.processes[pid];
@@ -638,7 +636,7 @@ export class DialogKeywords {
    *
    */
   public async setMaxLines({ pid, count }) {
-    await this.setOption({pid, name: "maxLines", value: count});
+    await this.setOption({ pid, name: 'maxLines', value: count });
   }
 
   /**
@@ -647,8 +645,8 @@ export class DialogKeywords {
    * @example SET PARAM name AS value
    *
    */
-   public async setUserParam({ pid, name, value }) {
+  public async setUserParam({ pid, name, value }) {
-    await this.setOption({pid, name, value});
+    await this.setOption({ pid, name, value });
   }
 
   /**
@@ -657,11 +655,10 @@ export class DialogKeywords {
    * @example GET PARAM name
    *
    */
-   public async getUserParam({ pid, name }) {
+  public async getUserParam({ pid, name }) {
-    await this.getOption({pid, name});
+    await this.getOption({ pid, name });
   }
 
-
   /**
    * Defines the maximum lines to scan in spreedsheets.
    *
@@ -669,7 +666,7 @@ export class DialogKeywords {
    *
    */
   public async setMaxColumns({ pid, count }) {
-    await this.setOption({pid, name: "setMaxColumns", value: count});
+    await this.setOption({ pid, name: 'setMaxColumns', value: count });
   }
 
   /**
@@ -679,8 +676,8 @@ export class DialogKeywords {
    *
    */
   public async setWholeWord({ pid, on }) {
-    const value = (on.trim() === "on");
+    const value = on.trim() === 'on';
-    await this.setOption({pid, name: "wholeWord", value: value});
+    await this.setOption({ pid, name: 'wholeWord', value: value });
   }
 
   /**
@@ -691,7 +688,7 @@ export class DialogKeywords {
    */
   public async setTheme({ pid, theme }) {
     const value = theme.trim();
-    await this.setOption({pid, name: "theme", value: value});
+    await this.setOption({ pid, name: 'theme', value: value });
   }
 
   /**
@@ -701,14 +698,14 @@ export class DialogKeywords {
    *
    */
   public async setTranslatorOn({ pid, on }) {
-    const value = (on.trim() === "on");
+    const value = on.trim() === 'on';
-    await this.setOption({pid, name: "translatorOn", value: value});
+    await this.setOption({ pid, name: 'translatorOn', value: value });
   }
 
   /**
    * Returns the name of the user acquired by WhatsApp API.
    */
-  public async userName({pid}) {
+  public async userName({ pid }) {
     let { min, user, params } = await DialogKeywords.getProcessInfo(pid);
     return user.userName;
   }
@@ -716,7 +713,7 @@ export class DialogKeywords {
   /**
    * Returns current mobile number from user in conversation.
    */
-  public async userMobile({pid}) {
+  public async userMobile({ pid }) {
     let { min, user, params } = await DialogKeywords.getProcessInfo(pid);
     return user.userSystemId;
   }
@@ -732,7 +729,6 @@ export class DialogKeywords {
     // return await beginDialog('/menu');
   }
 
-
   /**
    * Performs the transfer of the conversation to a human agent.
    *
@@ -744,7 +740,7 @@ export class DialogKeywords {
     // return await beginDialog('/t',{ to: to });
   }
 
-  public static getFileByHandle (hash) {
+  public static getFileByHandle(hash) {
     return GBServer.globals.files[hash];
   }
 
@@ -804,11 +800,11 @@ export class DialogKeywords {
         // await CollectionUtil.asyncForEach(args, async arg => {
         //   i++;
         //   list.sections[0].rows.push({ title: arg, id: `button${i}` });
-        //   await this.talk(arg);
+        //   await this.getTalk(arg);
         // });
 
         // const button = new wpp.Buttons(Messages[locale].choices, choices, ' ', ' ');
-        // await this.talk(button);
+        // await this.getTalk(button);
 
         GBLog.info(`BASIC: HEAR with [${args.toString()}] (Asking for input).`);
       } else {
@@ -822,7 +818,7 @@ export class DialogKeywords {
           setTimeout(resolve, ms);
         });
       };
-      min.cbMap[userId] = {}; 
+      min.cbMap[userId] = {};
       min.cbMap[userId]['promise'] = '!GBHEAR';
 
       while (min.cbMap[userId].promise === '!GBHEAR') {
@@ -831,7 +827,68 @@ export class DialogKeywords {
 
       const answer = min.cbMap[userId].promise;
 
-      if (kind === 'file') {
+      if (kind === 'sheet') {
+        
+        // Retrieves the .xlsx file associated with the HEAR var AS file.xlsx.
+
+        let { baseUrl, client } = await GBDeployer.internalGetDriveClient(this.min);
+        const botId = min.instance.botId;
+        const path = urljoin(`${botId}.gbai`, `${botId}.gbdata`);
+        let url = `${baseUrl}/drive/root:/${path}:/children`;
+
+        GBLog.info(`Loading HEAR AS .xlsx options from Sheet: ${url}`);
+        const res = await client.api(url).get();
+
+        // Finds .xlsx specified by arg.
+
+        const document = res.value.filter(m => {
+          return m.name === arg;
+        });
+        if (document === undefined || document.length === 0) {
+          GBLog.info(`${arg} not found on .gbdata folder, check the package.`);
+          return null;
+        }
+
+        // Reads all rows to be used as menu items in HEAR validation.
+
+        let sheets = await client.api(`${baseUrl}/drive/items/${document[0].id}/workbook/worksheets`).get();
+        const results = await client
+          .api(
+            `${baseUrl}/drive/items/${document[0].id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A1:A256')`
+          )
+          .get();
+
+        // Builds an array of items found in sheet file.
+
+        let index = 0;
+        let list = [];
+        for (; index < results.text.length; index++) {
+          if (results.text[index][0] !== '') {
+            list.push( results.text[index][0]);
+          }
+          else
+          {
+            break;
+          }          
+        }
+
+        // Search the answer in one of valid list items loaded from sheeet.
+
+        result = null;
+        await CollectionUtil.asyncForEach(list, async item => {
+          if (GBConversationalService.kmpSearch(answer, item) != -1) {
+            result = item;
+          }
+        });
+
+        // In case of unmatch, asks the person to try again.
+
+        if (result === null) {
+          await this.getTalk({ pid, text: `Escolha por favor um dos itens sugeridos.` });
+          return await this.getHear({ pid, kind, arg });
+        }
+
+      } else if (kind === 'file') {
         GBLog.info(`BASIC (${min.botId}): Upload done for ${answer.filename}.`);
         const handle = WebAutomationServices.cyrb53(this.min.botId + answer.filename);
         GBServer.globals.files[handle] = answer;
@@ -850,7 +907,7 @@ export class DialogKeywords {
         const value = extractEntity(answer);
 
         if (value === null) {
-          await this.talk({ pid, text: 'Por favor, digite um e-mail válido.' });
+          await this.getTalk({ pid, text: 'Por favor, digite um e-mail válido.' });
           return await this.getHear({ pid, kind, arg });
         }
 
@@ -863,7 +920,7 @@ export class DialogKeywords {
         const value = extractEntity(answer);
 
         if (value === null || value.length != 1) {
-          await this.talk({ pid, text: 'Por favor, digite um nome válido.' });
+          await this.getTalk({ pid, text: 'Por favor, digite um nome válido.' });
           return await this.getHear({ pid, kind, arg });
         }
 
@@ -876,7 +933,7 @@ export class DialogKeywords {
         const value = extractEntity(answer);
 
         if (value === null || value.length != 1) {
-          await this.talk({ pid, text: 'Por favor, digite um número válido.' });
+          await this.getTalk({ pid, text: 'Por favor, digite um número válido.' });
           return await this.getHear({ pid, kind, arg });
         }
 
@@ -891,7 +948,7 @@ export class DialogKeywords {
         const value = extractEntity(answer);
 
         if (value === null || value.length != 1) {
-          await this.talk({ pid, text: 'Por favor, digite uma data no formato 12/12/2020.' });
+          await this.getTalk({ pid, text: 'Por favor, digite uma data no formato 12/12/2020.' });
           return await this.getHear({ pid, kind, arg });
         }
 
@@ -904,7 +961,7 @@ export class DialogKeywords {
         const value = extractEntity(answer);
 
         if (value === null || value.length != 1) {
-          await this.talk({ pid, text: 'Por favor, digite um horário no formato hh:ss.' });
+          await this.getTalk({ pid, text: 'Por favor, digite um horário no formato hh:ss.' });
           return await this.getHear({ pid, kind, arg });
         }
 
@@ -923,7 +980,7 @@ export class DialogKeywords {
         const value = extractEntity(answer);
 
         if (value === null || value.length != 1) {
-          await this.talk({ pid, text: 'Por favor, digite um valor monetário.' });
+          await this.getTalk({ pid, text: 'Por favor, digite um valor monetário.' });
           return await this.getHear({ pid, kind, arg });
         }
 
@@ -935,12 +992,12 @@ export class DialogKeywords {
           phoneNumber = phone(answer, { country: 'BRA' })[0];
           phoneNumber = phoneUtil.parse(phoneNumber);
         } catch (error) {
-          await this.talk({ pid, text: Messages[locale].validation_enter_valid_mobile });
+          await this.getTalk({ pid, text: Messages[locale].validation_enter_valid_mobile });
 
           return await this.getHear({ pid, kind, arg });
         }
         if (!phoneUtil.isPossibleNumber(phoneNumber)) {
-          await this.talk({ pid, text: 'Por favor, digite um número de telefone válido.' });
+          await this.getTalk({ pid, text: 'Por favor, digite um número de telefone válido.' });
           return await this.getHear({ pid, kind, arg });
         }
 
@@ -960,7 +1017,7 @@ export class DialogKeywords {
         const value = extractEntity(answer);
 
         if (value === null || value.length != 1) {
-          await this.talk({ pid, text: 'Por favor, digite um CEP válido.' });
+          await this.getTalk({ pid, text: 'Por favor, digite um CEP válido.' });
           return await this.getHear({ pid, kind, arg });
         }
 
@@ -975,7 +1032,7 @@ export class DialogKeywords {
         });
 
         if (result === null) {
-          await this.talk({ pid, text: `Escolha por favor um dos itens sugeridos.` });
+          await this.getTalk({ pid, text: `Escolha por favor um dos itens sugeridos.` });
           return await this.getHear({ pid, kind, arg });
         }
       } else if (kind === 'language') {
@@ -1007,7 +1064,7 @@ export class DialogKeywords {
         });
 
         if (result === null) {
-          await this.talk({ pid, text: `Escolha por favor um dos itens sugeridos.` });
+          await this.getTalk({ pid, text: `Escolha por favor um dos itens sugeridos.` });
           return await this.getHear({ pid, kind, arg });
         }
       }
@@ -1064,9 +1121,10 @@ export class DialogKeywords {
   /**
    * Talks to the user by using the specified text.
    */
-  public async talk({ pid, text }) {
+  public async getTalk({ pid, text }) {
     GBLog.info(`BASIC: TALK '${text}'.`);
     const { min, user } = await DialogKeywords.getProcessInfo(pid);
+
     if (user) {
       // TODO: const translate = this.user ? this.user.basicOptions.translatorOn : false;
 

packages/basic.gblib/services/GBVMService.ts

@@ -32,7 +32,7 @@
 
 'use strict';
 
-import { GBLog, GBMinInstance, GBService, IGBCoreService, GBDialogStep } from 'botlib';
+import { GBMinInstance, GBService, IGBCoreService, GBDialogStep } from 'botlib';
 import * as Fs from 'fs';
 import { GBServer } from '../../../src/app.js';
 import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
@@ -47,12 +47,10 @@ import walkPromise from 'walk-promise';
 import child_process from 'child_process';
 import Path from 'path';
 import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
-import pkg from 'swagger-client';
 import { DialogKeywords } from './DialogKeywords.js';
 import { KeywordsExpressions } from './KeywordsExpressions.js';
-const { Swagger } = pkg;
+import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
-
+import { GuaribasUser } from '../../security.gbapp/models/index.js';
-
 
 /**
  * @fileoverview  Decision was to priorize security(isolation) and debugging,
@@ -63,10 +61,8 @@ const { Swagger } = pkg;
  * Basic services for BASIC manipulation.
  */
 export class GBVMService extends GBService {
-
   private static DEBUGGER_PORT = 9222;
 
-
   public async loadDialogPackage(folder: string, min: GBMinInstance, core: IGBCoreService, deployer: GBDeployer) {
     const files = await walkPromise(folder);
 
@@ -78,41 +74,46 @@ export class GBVMService extends GBService {
       let filename: string = file.name;
 
       if (filename.endsWith('.docx')) {
-        const wordFile = filename;
+        filename = await this.loadDialog(filename, folder, min);
-        const vbsFile = filename.substr(0, filename.indexOf('docx')) + 'vbs';
+      }
-        const fullVbsFile = urlJoin(folder, vbsFile);
+    });
-        const docxStat = Fs.statSync(urlJoin(folder, wordFile));
+  }
-        const interval = 3000; // If compiled is older 30 seconds, then recompile.
-        let writeVBS = true;
-        if (Fs.existsSync(fullVbsFile)) {
-          const vbsStat = Fs.statSync(fullVbsFile);
-          if (docxStat['mtimeMs'] < vbsStat['mtimeMs'] + interval) {
-            writeVBS = false;
-          }
-        }
-        filename = vbsFile;
-        let mainName = GBVMService.getMethodNameFromVBSFilename(filename);
-        min.scriptMap[filename] = mainName;
 
-        if (writeVBS) {
+  public async loadDialog(filename: string, folder: string, min: GBMinInstance) {
-          let text = await this.getTextFromWord(folder, wordFile);
+    const wordFile = filename;
+    const vbsFile = filename.substr(0, filename.indexOf('docx')) + 'vbs';
+    const fullVbsFile = urlJoin(folder, vbsFile);
+    const docxStat = Fs.statSync(urlJoin(folder, wordFile));
+    const interval = 3000; // If compiled is older 30 seconds, then recompile.
+    let writeVBS = true;
+    if (Fs.existsSync(fullVbsFile)) {
+      const vbsStat = Fs.statSync(fullVbsFile);
+      if (docxStat['mtimeMs'] < vbsStat['mtimeMs'] + interval) {
+        writeVBS = false;
+      }
+    }
+    filename = vbsFile;
+    let mainName = GBVMService.getMethodNameFromVBSFilename(filename);
+    min.scriptMap[filename] = mainName;
 
-          const schedule = GBVMService.getSetScheduleKeywordArgs(text);
+    if (writeVBS) {
-          const s = new ScheduleServices();
+      let text = await this.getTextFromWord(folder, wordFile);
-          if (schedule) {
-            await s.createOrUpdateSchedule(min, schedule, mainName);
-          } else {
-            await s.deleteScheduleIfAny(min, mainName);
-          }
-          text = text.replace(/^\s*SET SCHEDULE (.*)/gim, '');
-          Fs.writeFileSync(urlJoin(folder, vbsFile), text);
-        }
 
-        // Process node_modules install.
+      const schedule = GBVMService.getSetScheduleKeywordArgs(text);
+      const s = new ScheduleServices();
+      if (schedule) {
+        await s.createOrUpdateSchedule(min, schedule, mainName);
+      } else {
+        await s.deleteScheduleIfAny(min, mainName);
+      }
+      text = text.replace(/^\s*SET SCHEDULE (.*)/gim, '');
+      Fs.writeFileSync(urlJoin(folder, vbsFile), text);
+    }
 
-        const node_modules = urlJoin(folder, 'node_modules');
+    // Process node_modules install.
-        if (!Fs.existsSync(node_modules)) {
+    const node_modules = urlJoin(folder, 'node_modules');
-          const packageJson = `
+    if (!Fs.existsSync(node_modules)) {
+      const packageJson = `
             {
               "name": "${min.botId}.gbdialog",
               "version": "1.0.0",
@@ -127,43 +128,41 @@ export class GBVMService extends GBService {
                 "vm2": "3.9.11"
               }
             }`;
-          Fs.writeFileSync(urlJoin(folder, 'package.json'), packageJson);
+      Fs.writeFileSync(urlJoin(folder, 'package.json'), packageJson);
 
-          GBLog.info(`BASIC: Installing .gbdialog node_modules for ${min.botId}...`);
+      GBLogEx.info(min, `BASIC: Installing .gbdialog node_modules for ${min.botId}...`);
-          const npmPath = urlJoin(process.env.PWD, 'node_modules', '.bin', 'npm');
+      const npmPath = urlJoin(process.env.PWD, 'node_modules', '.bin', 'npm');
-          child_process.execSync(`${npmPath} install`, { cwd: folder });
+      child_process.execSync(`${npmPath} install`, { cwd: folder });
-        }
+    }
 
-        // Hot swap for .vbs files.
+    // Hot swap for .vbs files.
-
+    const fullFilename = urlJoin(folder, filename);
-        const fullFilename = urlJoin(folder, filename);
+    if (process.env.GBDIALOG_HOTSWAP) {
-        if (process.env.GBDIALOG_HOTSWAP) {
+      Fs.watchFile(fullFilename, async () => {
-          Fs.watchFile(fullFilename, async () => {
+        await this.translateBASIC(fullFilename, mainName, min);
-            await this.translateBASIC(fullFilename, mainName, min.botId);
-            const parsedCode: string = Fs.readFileSync(jsfile, 'utf8');
-            min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode;
-          });
-        }
-
-        const compiledAt = Fs.statSync(fullFilename);
-        const jsfile = urlJoin(folder, `${filename}.js`);
-
-        if (Fs.existsSync(jsfile)) {
-          const jsStat = Fs.statSync(jsfile);
-          const interval = 30000; // If compiled is older 30 seconds, then recompile.
-          if (compiledAt.isFile() && compiledAt['mtimeMs'] > jsStat['mtimeMs'] + interval) {
-            await this.translateBASIC(fullFilename, mainName, min.botId);
-          }
-        } else {
-          await this.translateBASIC(fullFilename, mainName, min.botId);
-        }
         const parsedCode: string = Fs.readFileSync(jsfile, 'utf8');
         min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode;
+      });
+    }
+
+    const compiledAt = Fs.statSync(fullFilename);
+    const jsfile = urlJoin(folder, `${filename}.js`);
+
+    if (Fs.existsSync(jsfile)) {
+      const jsStat = Fs.statSync(jsfile);
+      const interval = 30000; // If compiled is older 30 seconds, then recompile.
+      if (compiledAt.isFile() && compiledAt['mtimeMs'] > jsStat['mtimeMs'] + interval) {
+        await this.translateBASIC(fullFilename, mainName, min);
       }
-    });
+    } else {
+      await this.translateBASIC(fullFilename, mainName, min);
+    }
+    const parsedCode: string = Fs.readFileSync(jsfile, 'utf8');
+    min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode;
+    return filename;
   }
 
-  public async translateBASIC(filename: any, mainName: string, botId: string) {
+  public async translateBASIC(filename: any, mainName: string, min: GBMinInstance) {
     // Converts General Bots BASIC into regular VBS
 
     let basicCode: string = Fs.readFileSync(filename, 'utf8');
@@ -204,10 +203,10 @@ export class GBVMService extends GBService {
 
       // Interprocess communication from local HTTP to the BotServer.
 
-      const dk = rest.createClient('http://localhost:1111/api/v2/${botId}/dialog');
+      const dk = rest.createClient('http://localhost:1111/api/v2/${min.botId}/dialog');
-      const sys = rest.createClient('http://localhost:1111/api/v2/${botId}/system');
+      const sys = rest.createClient('http://localhost:1111/api/v2/${min.botId}/system');
-      const wa = rest.createClient('http://localhost:1111/api/v2/${botId}/webautomation');
+      const wa = rest.createClient('http://localhost:1111/api/v2/${min.botId}/webautomation');
-      const img = rest.createClient('http://localhost:1111/api/v2/${botId}/imagprocessing');
+      const img = rest.createClient('http://localhost:1111/api/v2/${min.botId}/imagprocessing');
               
       // Local variables.
 
@@ -241,11 +240,13 @@ export class GBVMService extends GBService {
 
       ${code}
 
+      await wa.getCloseHandles({pid: pid});
+
     })(); 
   
 `;
     Fs.writeFileSync(jsfile, code);
-    GBLog.info(`[GBVMService] Finished loading of ${filename}, JavaScript from Word: \n ${code}`);
+    GBLogEx.info(min, `[GBVMService] Finished loading of ${filename}, JavaScript from Word: \n ${code}`);
   }
 
   public static getMethodNameFromVBSFilename(filename: string) {
@@ -262,22 +263,32 @@ export class GBVMService extends GBService {
 
   private async getTextFromWord(folder: string, filename: string) {
     return new Promise<string>(async (resolve, reject) => {
-      textract.fromFileWithPath(urlJoin(folder, filename), { preserveLineBreaks: true }, (error, text) => {
+      const path = urlJoin(folder, filename);
+      textract.fromFileWithPath(path, { preserveLineBreaks: true }, (error, text) => {
         if (error) {
-          reject(error);
+          if (error.message.startsWith('File not correctly recognized as zip file')) {
-        } else {
+            text = Fs.readFileSync(path, 'utf8');
-          text = text.replace('¨', '"');
+          } else {
-          text = text.replace('“', '"');
+            reject(error);
-          text = text.replace('”', '"');
+          }
-          text = text.replace('‘', "'");
-          text = text.replace('’', "'");
-
-          resolve(text);
         }
+
+        if (text) {
+          text = GBVMService.normalizeQuotes(text);
+        }
+        resolve(text);
       });
     });
   }
 
+  public static normalizeQuotes(text: any) {
+    text = text.replace('¨', '"');
+    text = text.replace('“', '"');
+    text = text.replace('”', '"');
+    text = text.replace('‘', "'");
+    text = text.replace('’', "'");
+    return text;
+  }
 
   /**
    * Converts General Bots BASIC
@@ -288,7 +299,7 @@ export class GBVMService extends GBService {
   public async convert(code: string) {
     // Start and End of VB2TS tags of processing.
 
-    code = process.env.ENABLE_AUTH ? `hear gbLogin as login\n${code}` : code;
+    code = process.env.ENABLE_AUTH ? `hear GBLogExin as login\n${code}` : code;
     var lines = code.split('\n');
     const keywords = KeywordsExpressions.getKeywords();
     let current = 41;
@@ -313,13 +324,18 @@ export class GBVMService extends GBService {
   /**
    * Executes the converted JavaScript from BASIC code inside execution context.
    */
-  public static async callVM(text: string, min: GBMinInstance, step, deployer: GBDeployer, debug: boolean) {
+  public static async callVM(
-
+    text: string,
+    min: GBMinInstance,
+    step,
+    user: GuaribasUser,
+    deployer: GBDeployer,
+    debug: boolean = false
+  ) {
     // Creates a class DialogKeywords which is the *this* pointer
     // in BASIC.
 
-    const user = step ? await min.userProfile.get(step.context, {}) : null;
+    const dk = new DialogKeywords(min, deployer, null);
-    const dk = new DialogKeywords(min, deployer, user);
     const sandbox = {};
     const contentLocale = min.core.getParam<string>(
       min.instance,
@@ -327,17 +343,18 @@ export class GBVMService extends GBService {
       GBConfigService.get('DEFAULT_CONTENT_LANGUAGE')
     );
 
+    // TODO: https://github.com/GeneralBots/BotServer/issues/217
     // Auto-NLP generates BASIC variables related to entities.
 
-    if (step && step.context.activity['originalText']) {
+    // if (step && step.context.activity['originalText'] && min['nerEngine']) {
-      const entities = await min['nerEngine'].findEntities(step.context.activity['originalText'], contentLocale);
+    //   const entities = await min['nerEngine'].findEntities(step.context.activity['originalText'], contentLocale);
 
-      for (let i = 0; i < entities.length; i++) {
+    //   for (let i = 0; i < entities.length; i++) {
-        const v = entities[i];
+    //     const v = entities[i];
-        const variableName = `${v.entity}`;
+    //     const variableName = `${v.entity}`;
-        sandbox[variableName] = v.option;
+    //     sandbox[variableName] = v.option;
-      }
+    //   }
-    }
+    // }
 
     const botId = min.botId;
     const gbdialogPath = urlJoin(process.cwd(), 'work', `${botId}.gbai`, `${botId}.gbdialog`);
@@ -353,9 +370,9 @@ export class GBVMService extends GBService {
     };
 
     sandbox['id'] = dk.sys().getRandomId();
-    sandbox['username'] = await dk.userName({pid});
+    sandbox['username'] = await dk.userName({ pid });
-    sandbox['mobile'] = await dk.userMobile({pid});
+    sandbox['mobile'] = await dk.userMobile({ pid });
-    sandbox['from'] = await dk.userMobile({pid});
+    sandbox['from'] = await dk.userMobile({ pid });
     sandbox['ENTER'] = String.fromCharCode(13);
     sandbox['headers'] = {};
     sandbox['data'] = {};
@@ -364,8 +381,10 @@ export class GBVMService extends GBService {
     sandbox['httpPs'] = '';
     sandbox['pid'] = pid;
 
-    if (GBConfigService.get('GBVM') === 'false') {
+    let result;
-      try {
+
+    try {
+      if (GBConfigService.get('GBVM') === 'false') {
         const vm1 = new NodeVM({
           allowAsync: true,
           sandbox: sandbox,
@@ -379,23 +398,18 @@ export class GBVMService extends GBService {
           }
         });
         const s = new VMScript(code, { filename: scriptPath });
-        let x = vm1.run(s);
+        result = vm1.run(s);
-        return x;
+      } else {
-      } catch (error) {
+        const runnerPath = urlJoin(
-        throw new Error(`BASIC RUNTIME ERR: ${error.message ? error.message : error}\n Stack:${error.stack}`);
+          process.cwd(),
-      }
+          'dist',
-    } else {
+          'packages',
-      const runnerPath = urlJoin(
+          'basic.gblib',
-        process.cwd(),
+          'services',
-        'dist',
+          'vm2-process',
-        'packages',
+          'vm2ProcessRunner.js'
-        'basic.gblib',
+        );
-        'services',
-        'vm2-process',
-        'vm2ProcessRunner.js'
-      );
 
-      try {
         const { run } = createVm2Pool({
           min: 0,
           max: 0,
@@ -409,12 +423,13 @@ export class GBVMService extends GBService {
           script: runnerPath
         });
 
-        const result = await run(code, { filename: scriptPath, sandbox: sandbox });
+        result = await run(code, { filename: scriptPath, sandbox: sandbox });
-
-        return result;
-      } catch (error) {
-        throw new Error(`BASIC RUNTIME ERR: ${error.message ? error.message : error}\n Stack:${error.stack}`);
       }
+    } catch (error) {
+      throw new Error(`BASIC RUNTIME ERR: ${error.message ? error.message : error}\n Stack:${error.stack}`);
+    } finally {
     }
+
+    return result;
   }
 }

packages/basic.gblib/services/ImageProcessingServices.ts

@@ -32,9 +32,15 @@
 
 'use strict';
 
+import Path from 'path';
 import { GBLog, GBMinInstance } from 'botlib';
 import { DialogKeywords } from './DialogKeywords.js';
 import sharp from 'sharp';
+import joinImages from 'join-images-updated';
+import { CollectionUtil } from 'pragmatismo-io-framework';
+import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
+import urlJoin from 'url-join';
+import { GBServer } from '../../../src/app.js';
 
 /**
  * Image processing services of conversation to be called by BASIC.
@@ -95,6 +101,32 @@ export class ImageProcessingServices {
     return;
   }
 
+  /**
+   * SET ORIENTATION VERTICAL
+   * 
+   * file = MERGE file1, file2, file3 
+   */
+  public async mergeImage({pid, files})
+  {
+    const { min, user } = await DialogKeywords.getProcessInfo(pid);
+
+    let paths = [];
+    await CollectionUtil.asyncForEach(files, async file => {
+      const gbfile = DialogKeywords.getFileByHandle(file);  
+      paths.push(gbfile.path);
+    });
+
+    const botId = this.min.instance.botId;
+    const gbaiName = `${botId}.gbai`;
+    const img = await joinImages(paths);
+    const localName = Path.join('work', gbaiName, 'cache', `img-mrg${GBAdminService.getRndReadableIdentifier()}.png`);
+    const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
+    img.toFile(localName);
+
+    return { localName: localName, url: url, data: null };
+
+  }
+
   /**
    * Sharpen the image.
    *

packages/basic.gblib/services/KeywordsExpressions.ts

@@ -36,8 +36,7 @@
  * Image processing services of conversation to be called by BASIC.
  */
 export class KeywordsExpressions {
-
+  private static getParams = (text: string, names) => {
-  private static  getParams = (text: string, names) => {
     let ret = {};
     const splitParamsButIgnoreCommasInDoublequotes = (str: string) => {
       return str.split(',').reduce(
@@ -74,7 +73,6 @@ export class KeywordsExpressions {
    * Returns the list of BASIC keyword and their JS match.
    */
   public static getKeywords() {
-    
     // Keywords from General Bots BASIC.
 
     let keywords = [];
@@ -140,12 +138,28 @@ export class KeywordsExpressions {
     keywords[i++] = [
       /^\s*open\s*(.*)/gim,
       ($0, $1, $2) => {
+        let sessionName;
+        let kind = '';
+        let pos;
+
+        if (pos = $1.match(/\s*AS\s*\#/)) {
+          kind = '"AS"';
+        } else if (pos = $1.match(/\s*WITH\s*\#/)) {
+          kind = '"WITH"';
+        }
+
+        if (pos) {
+          let part = $1.substr($1.lastIndexOf(pos[0]));
+          sessionName = `"${part.substr(part.indexOf('#') + 1)}"`;
+          $1 = $1.substr(0, $1.lastIndexOf(pos[0]));
+        }
+
         if (!$1.startsWith('"') && !$1.startsWith("'")) {
           $1 = `"${$1}"`;
         }
         const params = this.getParams($1, ['url', 'username', 'password']);
 
-        return `page = await wa.getPage({pid: pid,${params}})`;
+        return `page = await wa.getPage({pid: pid, sessionKind: ${kind}, sessionName: ${sessionName}, ${params}})`;
       }
     ];
 
@@ -156,6 +170,13 @@ export class KeywordsExpressions {
       }
     ];
 
+    keywords[i++] = [
+      /^\s*hear (\w+) as (\w+( \w+)*.xlsx)/gim,
+      ($0, $1, $2) => {
+        return `${$1} = await dk.getHear({pid: pid, kind:"sheet", arg: "${$2}"})`;
+      }
+    ];
+
     keywords[i++] = [
       /^\s*hear (\w+) as\s*login/gim,
       ($0, $1) => {
@@ -576,7 +597,7 @@ export class KeywordsExpressions {
         if ($3.substr(0, 1) !== '"') {
           $3 = `"${$3}"`;
         }
-        return `await dk.talk ({pid: pid, text: ${$3}})`;
+        return `await dk.getTalk ({pid: pid, text: ${$3}})`;
       }
     ];
 
@@ -677,6 +698,13 @@ export class KeywordsExpressions {
       }
     ];
 
+    keywords[i++] = [
+      /^\s*(MERGE)(\s*)(.*)/gim,
+      ($0, $1, $2, $3) => {
+        return `await img.mergeImage({pid: pid, files: [${$3}]})`;
+      }
+    ];
+
     keywords[i++] = [
       /^\s*PRESS\s*(.*)/gim,
       ($0, $1, $2) => {

packages/basic.gblib/services/ScheduleServices.ts

@@ -139,7 +139,7 @@ export class ScheduleServices extends GBService {
             let min: GBMinInstance = GBServer.globals.minInstances.filter(
               p => p.instance.instanceId === item.instanceId
             )[0];
-            await GBVMService.callVM(script, min, null, null, false);
+            await GBVMService.callVM(script, min, null, null, null, false);
           };
           (async () => {
             await finalData();

packages/basic.gblib/services/SystemKeywords.ts

@@ -39,7 +39,7 @@ import { DialogKeywords } from './DialogKeywords.js';
 import { GBServer } from '../../../src/app.js';
 import { GBVMService } from './GBVMService.js';
 import Fs from 'fs';
-import { createBrowser } from '../../core.gbapp/services/GBSSR.js';
+import { GBSSR }from '../../core.gbapp/services/GBSSR.js';
 import urlJoin from 'url-join';
 import Excel from 'exceljs';
 import { TwitterApi } from 'twitter-api-v2';
@@ -93,11 +93,11 @@ export class SystemKeywords {
   }
 
   public async callVM({ pid, text }) {
-    const min = null;
+    const { min, user } = await DialogKeywords.getProcessInfo(pid);
     const step = null;
     const deployer = null;
 
-    return await GBVMService.callVM(text, min, step, deployer, false);
+    return await GBVMService.callVM(text, min, step, user, deployer, false);
   }
 
   public async append({ pid, args }) {
@@ -259,7 +259,7 @@ export class SystemKeywords {
 
     const { min, user } = await DialogKeywords.getProcessInfo(pid);
     const gbaiName = `${min.botId}.gbai`;
-    const browser = await createBrowser(null);
+    const browser = await GBSSR.createBrowser(null);
     const page = await browser.newPage();
 
     // Includes the associated CSS related to current theme.
@@ -1462,6 +1462,7 @@ export class SystemKeywords {
     const images = [];
     let index = 0;
     path = Path.join(gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.docx`);
+    url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
 
     const traverseDataToInjectImageUrl = async o => {
       for (var i in o) {

packages/basic.gblib/services/WebAutomationServices.ts

@@ -32,19 +32,20 @@
 
 'use strict';
 
-import { GBLog, GBMinInstance } from 'botlib';
-import { GBServer } from '../../../src/app.js';
-import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
-import { createBrowser } from '../../core.gbapp/services/GBSSR.js';
-import { GuaribasUser } from '../../security.gbapp/models/index.js';
-import { DialogKeywords } from './DialogKeywords.js';
-
-import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
 import urlJoin from 'url-join';
 import Fs from 'fs';
 import Path from 'path';
 import url from 'url';
-import { pid } from 'process';
+
+import { GBLog, GBMinInstance } from 'botlib';
+import { GBServer } from '../../../src/app.js';
+import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
+import { GBSSR }from '../../core.gbapp/services/GBSSR.js';
+import { GuaribasUser } from '../../security.gbapp/models/index.js';
+import { DialogKeywords } from './DialogKeywords.js';
+import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
+import { Mutex } from 'async-mutex';
+import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
 
 /**
  * Web Automation services of conversation to be called by BASIC.
@@ -86,8 +87,6 @@ export class WebAutomationServices {
    */
   maxLines: number = 2000;
 
-  pageMap = {};
-
   public static cyrb53 = (str, seed = 0) => {
     let h1 = 0xdeadbeef ^ seed,
       h2 = 0x41c6ce57 ^ seed;
@@ -107,7 +106,7 @@ export class WebAutomationServices {
    * When creating this keyword facade,a bot instance is
    * specified among the deployer service.
    */
-  constructor (min: GBMinInstance, user, dk) {
+  constructor(min: GBMinInstance, user, dk) {
     this.min = min;
     this.user = user;
     this.dk = dk;
@@ -115,31 +114,106 @@ export class WebAutomationServices {
     this.debugWeb = this.min.core.getParam<boolean>(this.min.instance, 'Debug Web Automation', false);
   }
 
+  public async getCloseHandles({ pid }) {
+    const { min, user } = await DialogKeywords.getProcessInfo(pid);
+    // Releases previous allocated OPEN semaphores.
+
+    let keys = Object.keys(GBServer.globals.webSessions);
+    for (let i = 0; i < keys.length; i++) {
+      const session = GBServer.globals.webSessions[keys[i]];
+      if (session.activePid === pid) {
+        session.semaphore.release();
+        GBLogEx.info(min, `Release for PID: ${pid} done.`);
+      }
+    }
+  }
+
   /**
    * Returns the page object.
    *
    * @example OPEN "https://wikipedia.org"
    */
-  public async getPage ({ pid, url, username, password }) {
+
-    GBLog.info(`BASIC: Web Automation GET PAGE ${url}.`);
+  public async getPage({ pid, sessionKind, sessionName, url, username, password }) {
-    if (!this.browser) {
+    GBLog.info(`BASIC: Web Automation GET PAGE ${sessionName ? sessionName : ''} ${url}.`);
-      this.browser = await createBrowser(null);
+    const { min, user } = await DialogKeywords.getProcessInfo(pid);
+
+    let handle;
+
+    // Try to find an existing handle.
+
+    let session;
+    let keys = Object.keys(GBServer.globals.webSessions);
+    for (let i = 0; i < keys.length; i++) {
+      if (GBServer.globals.webSessions[keys[i]].sessionName === sessionName) {
+        session = GBServer.globals.webSessions[keys[i]];
+        handle = keys[i];
+        break;
+      }
     }
-    const page = (await this.browser.pages())[0];
+
-    if (username || password) {
+    // Semaphore logic to block multiple entries on the same session.
-      await page.authenticate({pid, username: username, password: password });
+
+    let page;
+    if (session) {
+      GBLogEx.info(min, `Acquiring (1) for PID: ${pid}...`);
+      const release = await session.semaphore.acquire();
+      GBLogEx.info(min, `Acquire (1) for PID: ${pid} done.`);
+      try {
+        session.activePid = pid;
+        session.release = release;
+        page = session.page;
+      } catch {
+        release();
+      }
     }
+
+    // Creates the page if it is the first time.
+
+    let browser;
+    if (!page) {
+      browser = await GBSSR.createBrowser(null);
+      page = (await browser.pages())[0];
+      if (username || password) {
+        await page.authenticate({ pid, username: username, password: password });
+      }
+    }
+
+    // There is no session yet.
+
+    if (!session && sessionKind === 'AS') {
+
+      // A new web session is being created.
+
+      handle = WebAutomationServices.cyrb53(this.min.botId + url);
+      GBServer.globals.webSessions[handle] = session = {};
+      session.sessionName = sessionName;
+      
+      session.page = page;
+      session.browser = browser;
+      session.semaphore = new Mutex();
+      GBLogEx.info(min, `Acquiring (2) for PID: ${pid}...`);
+      const release = await session.semaphore.acquire();
+      GBLogEx.info(min, `Acquire (2) for PID: ${pid} done.`);
+      session.release = release;
+      session.activePid = pid;
+
+    }
+
+    // WITH is only valid in a previously defined session.
+
+    if (!session && sessionKind == 'WITH') {
+      const error = `NULL session for OPEN WITH #${sessionName}.`;
+      GBLogEx.error(min, error);
+    }
+
     await page.goto(url);
 
-    const handle = WebAutomationServices.cyrb53(this.min.botId + url);
-
-    this.pageMap[handle] = page;
-
     return handle;
   }
 
-  public getPageByHandle (hash) {
+  public getPageByHandle(handle) {
-    return this.pageMap[hash];
+    return GBServer.globals.webSessions[handle].page;
   }
 
   /**
@@ -147,7 +221,7 @@ export class WebAutomationServices {
    *
    * @example GET "selector"
    */
-  public async getBySelector ({ handle, selector }) {
+  public async getBySelector({ handle, selector }) {
     const page = this.getPageByHandle(handle);
     GBLog.info(`BASIC: Web Automation GET element: ${selector}.`);
     await page.waitForSelector(selector);
@@ -170,7 +244,7 @@ export class WebAutomationServices {
    *
    * @example GET page,"frameSelector,"elementSelector"
    */
-  public async getByFrame ({ handle, frame, selector }) {
+  public async getByFrame({ handle, frame, selector }) {
     const page = this.getPageByHandle(handle);
     GBLog.info(`BASIC: Web Automation GET element by frame: ${selector}.`);
     await page.waitForSelector(frame);
@@ -190,7 +264,7 @@ export class WebAutomationServices {
   /**
    * Simulates a mouse hover an web page element.
    */
-  public async hover ({ pid, handle, selector }) {
+  public async hover({ pid, handle, selector }) {
     const page = this.getPageByHandle(handle);
     GBLog.info(`BASIC: Web Automation HOVER element: ${selector}.`);
     await this.getBySelector({ handle, selector: selector });
@@ -203,7 +277,7 @@ export class WebAutomationServices {
    *
    * @example CLICK page,"#idElement"
    */
-  public async click ({ pid, handle, frameOrSelector, selector }) {
+  public async click({ pid, handle, frameOrSelector, selector }) {
     const page = this.getPageByHandle(handle);
     GBLog.info(`BASIC: Web Automation CLICK element: ${frameOrSelector}.`);
     if (selector) {
@@ -219,7 +293,7 @@ export class WebAutomationServices {
     await this.debugStepWeb(pid, page);
   }
 
-  private async debugStepWeb (pid, page) {
+  private async debugStepWeb(pid, page) {
     let refresh = true;
     if (this.lastDebugWeb) {
       refresh = new Date().getTime() - this.lastDebugWeb.getTime() > 5000;
@@ -229,7 +303,7 @@ export class WebAutomationServices {
       const mobile = this.min.core.getParam(this.min.instance, 'Bot Admin Number', null);
       const filename = page;
       if (mobile) {
-        await this.dk.sendFileTo({pid: pid,  mobile, filename, caption: 'General Bots Debugger' });
+        await this.dk.sendFileTo({ pid: pid, mobile, filename, caption: 'General Bots Debugger' });
       }
       this.lastDebugWeb = new Date();
     }
@@ -240,7 +314,7 @@ export class WebAutomationServices {
    *
    * @example PRESS ENTER ON page
    */
-  public async pressKey ({ handle, char, frame }) {
+  public async pressKey({ handle, char, frame }) {
     const page = this.getPageByHandle(handle);
     GBLog.info(`BASIC: Web Automation PRESS ${char} ON element: ${frame}.`);
     if (char.toLowerCase() === 'enter') {
@@ -256,7 +330,7 @@ export class WebAutomationServices {
     }
   }
 
-  public async linkByText ({ pid, handle, text, index }) {
+  public async linkByText({ pid, handle, text, index }) {
     const page = this.getPageByHandle(handle);
     GBLog.info(`BASIC: Web Automation CLICK LINK TEXT: ${text} ${index}.`);
     if (!index) {
@@ -272,7 +346,7 @@ export class WebAutomationServices {
    *
    * @example file = SCREENSHOT page
    */
-  public async screenshot ({ handle, selector }) {
+  public async screenshot({ handle, selector }) {
     const page = this.getPageByHandle(handle);
     GBLog.info(`BASIC: Web Automation SCREENSHOT ${selector}.`);
 
@@ -292,7 +366,7 @@ export class WebAutomationServices {
    *
    * @example SET page,"selector","text"
    */
-  public async setElementText ({ pid, handle, selector, text }) {
+  public async setElementText({ pid, handle, selector, text }) {
     const page = this.getPageByHandle(handle);
     GBLog.info(`BASIC: Web Automation TYPE on ${selector}: ${text}.`);
     const e = await this.getBySelector({ handle, selector });
@@ -307,9 +381,9 @@ export class WebAutomationServices {
    *
    * @example file = DOWNLOAD element, folder
    */
-  public async download ({ handle, selector, folder }) {
+  public async download({ handle, selector, folder }) {
     const page = this.getPageByHandle(handle);
-    
+
     const element = await this.getBySelector({ handle, selector });
     // https://github.com/GeneralBots/BotServer/issues/311
     const container = element['_frame'] ? element['_frame'] : element['_page'];
@@ -386,4 +460,20 @@ export class WebAutomationServices {
 
     return file;
   }
+
+  private async  recursiveFindInFrames (inputFrame, selector) {
+    const frames = inputFrame.childFrames();
+    const results = await Promise.all(
+      frames.map(async frame => {
+        const el = await frame.$(selector);
+        if (el) return el;
+        if (frame.childFrames().length > 0) {
+          return await this.recursiveFindInFrames(frame, selector);
+        }
+        return null;
+      })
+    );
+    return results.find(Boolean);
+  }
+  
 }

packages/core.gbapp/dialogs/WelcomeDialog.ts

@@ -72,16 +72,14 @@ export class WelcomeDialog extends IGBDialog {
             return step.replaceDialog(GBServer.globals.entryPointDialog);
           }
 
-          const user = await min.userProfile.get(step.context, {});
           const locale = step.context.activity.locale;
 
           if (
-            !user.once &&
+            //  TODO: https://github.com/GeneralBots/BotServer/issues/9            !user.once &&
             step.context.activity.channelId === 'webchat' &&
             min.core.getParam<boolean>(min.instance, 'HelloGoodX', true) === 'true'
           ) {
-            user.once = true;
+            // user.once = true;
-            await min.userProfile.set(step.context, user);
             const a = new Date();
             const date = a.getHours();
             const msg =

packages/core.gbapp/index.ts

@@ -43,7 +43,7 @@ import { LanguageDialog } from './dialogs/LanguageDialog.js';
 import { SwitchBotDialog } from './dialogs/SwitchBot.js';
 import { WelcomeDialog } from './dialogs/WelcomeDialog.js';
 import { WhoAmIDialog } from './dialogs/WhoAmIDialog.js';
-import { GuaribasChannel, GuaribasException, GuaribasInstance, GuaribasPackage } from './models/GBModel.js';
+import { GuaribasChannel, GuaribasInstance, GuaribasLog, GuaribasPackage } from './models/GBModel.js';
 
 /**
  * Package for core.gbapp.
@@ -53,7 +53,7 @@ export class GBCorePackage implements IGBPackage {
   public CurrentEngineName = 'guaribas-1.0.0';
 
   public async loadPackage (core: IGBCoreService, sequelize: Sequelize): Promise<void> {
-    core.sequelize.addModels([GuaribasInstance, GuaribasPackage, GuaribasChannel, GuaribasException]);
+    core.sequelize.addModels([GuaribasInstance, GuaribasPackage, GuaribasChannel, GuaribasLog]);
   }
 
   public async getDialogs (min: GBMinInstance) {

packages/core.gbapp/models/GBModel.ts

@@ -326,15 +326,18 @@ export class GuaribasChannel extends Model<GuaribasChannel> {
  */
 @Table
 //tslint:disable-next-line:max-classes-per-file
-export class GuaribasException extends Model<GuaribasException> {
+export class GuaribasLog extends Model<GuaribasLog> {
   @PrimaryKey
   @AutoIncrement
   @Column(DataType.INTEGER)
-  declare  exceptionId: number;
+  declare  logId: number;
 
-  @Column(DataType.STRING(255))
+  @Column(DataType.STRING(1024))
   declare  message: string;
 
+  @Column(DataType.STRING(1))
+  declare  kind: string;
+  
   @ForeignKey(() => GuaribasInstance)
   @Column(DataType.INTEGER)
   declare  instanceId: number;

packages/core.gbapp/services/GBConversationalService.ts

@@ -581,14 +581,14 @@ export class GBConversationalService {
       await this.sendMarkdownToMobile(min, step, mobile, text);
     } else if (GBConfigService.get('DISABLE_WEB') !== 'true') {
       const html = marked(text);
-      await this.sendMarkdownToWeb(min, step, html, answer);
+      await this.sendHTMLToWeb(min, step, html, answer);
     } else {
       const html = marked(text);
       await min.conversationalService.sendText(min, step, html);
     }
   }
 
-  private async sendMarkdownToWeb (min, step: GBDialogStep, html: string, answer: string) {
+  private async sendHTMLToWeb (min, step: GBDialogStep, html: string, answer: string) {
     const locale = step.context.activity.locale;
 
     html = html.replace(/src\=\"kb\//gi, `src=\"../kb/`);

packages/core.gbapp/services/GBCoreService.ts

@@ -50,7 +50,7 @@ import { GBCustomerSatisfactionPackage } from '../../customer-satisfaction.gbapp
 import { GBKBPackage } from '../../kb.gbapp/index.js';
 import { GBSecurityPackage } from '../../security.gbapp/index.js';
 import { GBWhatsappPackage } from '../../whatsapp.gblib/index.js';
-import { GuaribasInstance } from '../models/GBModel.js';
+import { GuaribasInstance, GuaribasLog} from '../models/GBModel.js';
 import { GBConfigService } from './GBConfigService.js';
 import { GBAzureDeployerPackage } from '../../azuredeployer.gbapp/index.js';
 import { GBSharePointPackage } from '../../sharepoint.gblib/index.js';
@@ -102,16 +102,16 @@ export class GBCoreService implements IGBCoreService {
   /**
    *
    */
-  constructor () {
+  constructor() {
     this.adminService = new GBAdminService(this);
   }
-  public async ensureInstances (instances: IGBInstance[], bootInstance: any, core: IGBCoreService) {}
+  public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) {}
 
   /**
    * Gets database config and connect to storage. Currently two databases
    * are available: SQL Server and SQLite.
    */
-  public async initStorage (): Promise<any> {
+  public async initStorage(): Promise<any> {
     this.dialect = GBConfigService.get('STORAGE_DIALECT');
 
     let host: string | undefined;
@@ -177,7 +177,7 @@ export class GBCoreService implements IGBCoreService {
    * Checks wheather storage is acessible or not and opens firewall
    * in case of any connection block.
    */
-  public async checkStorage (installationDeployer: IGBInstallationDeployer) {
+  public async checkStorage(installationDeployer: IGBInstallationDeployer) {
     try {
       await this.sequelize.authenticate();
     } catch (error) {
@@ -195,7 +195,7 @@ export class GBCoreService implements IGBCoreService {
   /**
    * Syncronizes structure between model and tables in storage.
    */
-  public async syncDatabaseStructure () {
+  public async syncDatabaseStructure() {
     if (GBConfigService.get('STORAGE_SYNC') === 'true') {
       const alter = GBConfigService.get('STORAGE_SYNC_ALTER') === 'true';
       GBLog.info('Syncing database...');
@@ -213,7 +213,29 @@ export class GBCoreService implements IGBCoreService {
   /**
    * Loads all items to start several listeners.
    */
-  public async loadInstances (): Promise<IGBInstance[]> {
+  public async getLatestLogs(instanceId: number): Promise<string> {
+    const options = {
+      where: {
+        instanceId: instanceId,
+        state: 'active',
+        created: {
+          [Op.gt]: new Date(Date.now() - 60 * 60 * 1000 * 48) // Latest 48 hours.
+        }
+      }
+    };
+    const list = await GuaribasLog.findAll(options);
+    let out = 'General Bots Log\n';
+    await CollectionUtil.asyncForEach(list, async e => {
+      out = `${out}\n${e.createdAt} - ${e.message}`;
+    });
+    return out;
+  }
+
+
+  /**
+   * Loads all items to start several listeners.
+   */
+  public async loadInstances(): Promise<IGBInstance[]> {
     if (process.env.LOAD_ONLY !== undefined) {
       const bots = process.env.LOAD_ONLY.split(`;`);
       const and = [];
@@ -236,7 +258,7 @@ export class GBCoreService implements IGBCoreService {
   /**
    * Loads just one Bot instance by its internal Id.
    */
-  public async loadInstanceById (instanceId: number): Promise<IGBInstance> {
+  public async loadInstanceById(instanceId: number): Promise<IGBInstance> {
     const options = { where: { instanceId: instanceId, state: 'active' } };
 
     return await GuaribasInstance.findOne(options);
@@ -244,7 +266,7 @@ export class GBCoreService implements IGBCoreService {
   /**
    * Loads just one Bot instance.
    */
-  public async loadInstanceByActivationCode (code: string): Promise<IGBInstance> {
+  public async loadInstanceByActivationCode(code: string): Promise<IGBInstance> {
     let options = { where: { activationCode: code, state: 'active' } };
 
     return await GuaribasInstance.findOne(options);
@@ -252,7 +274,7 @@ export class GBCoreService implements IGBCoreService {
   /**
    * Loads just one Bot instance.
    */
-  public async loadInstanceByBotId (botId: string): Promise<IGBInstance> {
+  public async loadInstanceByBotId(botId: string): Promise<IGBInstance> {
     const options = { where: {} };
     options.where = { botId: botId, state: 'active' };
 
@@ -264,7 +286,7 @@ export class GBCoreService implements IGBCoreService {
    * first startup, when user is asked some questions to create the
    * full base environment.
    */
-  public async writeEnv (instance: IGBInstance) {
+  public async writeEnv(instance: IGBInstance) {
     const env = `
 ADDITIONAL_DEPLOY_PATH=
 ADMIN_PASS=${instance.adminPass}
@@ -294,7 +316,7 @@ ENDPOINT_UPDATE=true
    * when calling back from web services. This ensures that reverse proxy is
    * established.
    */
-  public async ensureProxy (port): Promise<string> {
+  public async ensureProxy(port): Promise<string> {
     try {
       if (Fs.existsSync('node_modules/ngrok/bin/ngrok.exe') || Fs.existsSync('node_modules/ngrok/bin/ngrok')) {
         return await ngrok.connect({ port: port });
@@ -315,7 +337,7 @@ ENDPOINT_UPDATE=true
    * Setup generic web hooks so .gbapps can expose application logic
    * and get called on demand.
    */
-  public installWebHook (isGet: boolean, url: string, callback: any) {
+  public installWebHook(isGet: boolean, url: string, callback: any) {
     if (isGet) {
       GBServer.globals.server.get(url, (req, res) => {
         callback(req, res);
@@ -331,7 +353,7 @@ ENDPOINT_UPDATE=true
    * Defines the entry point dialog to be called whenever a user
    * starts talking to the bot.
    */
-  public setEntryPointDialog (dialogName: string) {
+  public setEntryPointDialog(dialogName: string) {
     GBServer.globals.entryPointDialog = dialogName;
   }
 
@@ -339,14 +361,14 @@ ENDPOINT_UPDATE=true
    * Replaces the default web application root path used to start the GB
    * with a custom home page.
    */
-  public setWWWRoot (localPath: string) {
+  public setWWWRoot(localPath: string) {
     GBServer.globals.wwwroot = localPath;
   }
 
   /**
    * Removes a bot instance from storage.
    */
-  public async deleteInstance (botId: string) {
+  public async deleteInstance(botId: string) {
     const options = { where: {} };
     options.where = { botId: botId };
     await GuaribasInstance.destroy(options);
@@ -356,7 +378,7 @@ ENDPOINT_UPDATE=true
    * Saves a bot instance object to the storage handling
    * multi-column JSON based store 'params' field.
    */
-  public async saveInstance (fullInstance: any) {
+  public async saveInstance(fullInstance: any) {
     const options = { where: {} };
     options.where = { botId: fullInstance.botId };
     let instance = await GuaribasInstance.findOne(options);
@@ -377,7 +399,7 @@ ENDPOINT_UPDATE=true
   /**
    * Loads all bot instances from object storage, if it's formatted.
    */
-  public async loadAllInstances (
+  public async loadAllInstances(
     core: IGBCoreService,
     installationDeployer: IGBInstallationDeployer,
     proxyAddress: string
@@ -431,7 +453,7 @@ ENDPOINT_UPDATE=true
   /**
    * Loads all system packages from 'packages' folder.
    */
-  public async loadSysPackages (core: GBCoreService): Promise<IGBPackage[]> {
+  public async loadSysPackages(core: GBCoreService): Promise<IGBPackage[]> {
     // NOTE: if there is any code before this line a semicolon
     // will be necessary before this line.
     // Loads all system packages.
@@ -469,7 +491,7 @@ ENDPOINT_UPDATE=true
    * Verifies that an complex global password has been specified
    * before starting the server.
    */
-  public ensureAdminIsSecured () {
+  public ensureAdminIsSecured() {
     const password = GBConfigService.get('ADMIN_PASS');
     if (!GBAdminService.StrongRegex.test(password)) {
       throw new Error(
@@ -484,7 +506,7 @@ ENDPOINT_UPDATE=true
    * So a base main bot is always deployed and will act as root bot for
    * configuration tree with three levels: .env > root bot > all other bots.
    */
-  public async createBootInstance (
+  public async createBootInstance(
     core: GBCoreService,
     installationDeployer: IGBInstallationDeployer,
     proxyAddress: string
@@ -519,7 +541,7 @@ ENDPOINT_UPDATE=true
   /**
    * Helper to get the web browser onpened in UI interfaces.
    */
-  public openBrowserInDevelopment () {
+  public openBrowserInDevelopment() {
     if (process.env.NODE_ENV === 'development') {
       open('http://localhost:4242');
     }
@@ -540,35 +562,29 @@ ENDPOINT_UPDATE=true
    * //   '  FOREIGN KEY ([groupId1], [groupId2]) REFERENCES [Group] ([groupId1], [groupId1]) ON DELETE NO ACTION,' +
    * //   '  FOREIGN KEY ([instanceId]) REFERENCES [Instance] ([instanceId]) ON DELETE NO ACTION)'
    */
-  private createTableQueryOverride (tableName, attributes, options): string {
+  private createTableQueryOverride(tableName, attributes, options): string {
     let sql: string = this.createTableQuery.apply(this.queryGenerator, [tableName, attributes, options]);
     const re1 = /CREATE\s+TABLE\s+\[([^\]]*)\]/;
     const matches = re1.exec(sql);
     if (matches !== null) {
       const table = matches[1];
       const re2 = /PRIMARY\s+KEY\s+\(\[[^\]]*\](?:,\s*\[[^\]]*\])*\)/;
-      sql = sql.replace(
+      sql = sql.replace(re2, (match: string, ...args: any[]): string => {
-        re2,
+        return `CONSTRAINT [${table}_pk] ${match}`;
-        (match: string, ...args: any[]): string => {
+      });
-          return `CONSTRAINT [${table}_pk] ${match}`;
-        }
-      );
       const re3 = /FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g;
       const re4 = /\[([^\]]*)\]/g;
-      sql = sql.replace(
+      sql = sql.replace(re3, (match: string, ...args: any[]): string => {
-        re3,
+        const fkcols = args[0];
-        (match: string, ...args: any[]): string => {
+        let fkname = table;
-          const fkcols = args[0];
+        let matches2 = re4.exec(fkcols);
-          let fkname = table;
+        while (matches2 !== null) {
-          let matches2 = re4.exec(fkcols);
+          fkname += `_${matches2[1]}`;
-          while (matches2 !== null) {
+          matches2 = re4.exec(fkcols);
-            fkname += `_${matches2[1]}`;
-            matches2 = re4.exec(fkcols);
-          }
-
-          return `CONSTRAINT [${fkname}_fk] FOREIGN KEY (${fkcols})`;
         }
-      );
+
+        return `CONSTRAINT [${fkname}_fk] FOREIGN KEY (${fkcols})`;
+      });
     }
 
     return sql;
@@ -582,7 +598,7 @@ ENDPOINT_UPDATE=true
    * '      CONSTRAINT [invalid2] FOREIGN KEY ([groupId1], [groupId2]) REFERENCES [Group] ([groupId1], [groupId2]) ON DELETE NO ACTION, ' +
    * '      CONSTRAINT [invalid3] FOREIGN KEY ([instanceId1]) REFERENCES [Instance] ([instanceId1]) ON DELETE NO ACTION'
    */
-  private changeColumnQueryOverride (tableName, attributes): string {
+  private changeColumnQueryOverride(tableName, attributes): string {
     let sql: string = this.changeColumnQuery.apply(this.queryGenerator, [tableName, attributes]);
     const re1 = /ALTER\s+TABLE\s+\[([^\]]*)\]/;
     const matches = re1.exec(sql);
@@ -590,20 +606,17 @@ ENDPOINT_UPDATE=true
       const table = matches[1];
       const re2 = /(ADD\s+)?CONSTRAINT\s+\[([^\]]*)\]\s+FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g;
       const re3 = /\[([^\]]*)\]/g;
-      sql = sql.replace(
+      sql = sql.replace(re2, (match: string, ...args: any[]): string => {
-        re2,
+        const fkcols = args[2];
-        (match: string, ...args: any[]): string => {
+        let fkname = table;
-          const fkcols = args[2];
+        let matches2 = re3.exec(fkcols);
-          let fkname = table;
+        while (matches2 !== null) {
-          let matches2 = re3.exec(fkcols);
+          fkname += `_${matches2[1]}`;
-          while (matches2 !== null) {
+          matches2 = re3.exec(fkcols);
-            fkname += `_${matches2[1]}`;
-            matches2 = re3.exec(fkcols);
-          }
-
-          return `${args[0] ? args[0] : ''}CONSTRAINT [${fkname}_fk] FOREIGN KEY (${fkcols})`;
         }
-      );
+
+        return `${args[0] ? args[0] : ''}CONSTRAINT [${fkname}_fk] FOREIGN KEY (${fkcols})`;
+      });
     }
 
     return sql;
@@ -612,7 +625,7 @@ ENDPOINT_UPDATE=true
   /**
    * Opens storage firewall used by the server when starting to get root bot instance.
    */
-  private async openStorageFrontier (installationDeployer: IGBInstallationDeployer) {
+  private async openStorageFrontier(installationDeployer: IGBInstallationDeployer) {
     const group = GBConfigService.get('CLOUD_GROUP');
     const serverName = GBConfigService.get('STORAGE_SERVER').split('.database.windows.net')[0];
     await installationDeployer.openStorageFirewall(group, serverName);
@@ -625,7 +638,7 @@ ENDPOINT_UPDATE=true
    * @param name Name of param to get from instance.
    * @param defaultValue Value returned when no param is defined in Config.xlsx.
    */
-  public getParam<T> (instance: IGBInstance, name: string, defaultValue?: T): any {
+  public getParam<T>(instance: IGBInstance, name: string, defaultValue?: T): any {
     let value = null;
     if (instance.params) {
       const params = JSON.parse(instance.params);

packages/core.gbapp/services/GBDeployer.ts

@@ -223,7 +223,7 @@ export class GBDeployer implements IGBDeployer {
     instance.activationCode = instance.botId;
     instance.state = 'active';
     instance.nlpScore = 0.8;
-    instance.searchScore = 0.45;
+    instance.searchScore = 0.25;
     instance.whatsappServiceKey = null;
     instance.whatsappServiceNumber = null;
     instance.whatsappServiceUrl = null;
@@ -713,6 +713,7 @@ export class GBDeployer implements IGBDeployer {
   public async rebuildIndex(instance: IGBInstance, searchSchema: any) {
     // Prepares search.
 
+
     const search = new AzureSearch(
       instance.searchKey,
       instance.searchHost,
@@ -721,40 +722,28 @@ export class GBDeployer implements IGBDeployer {
     );
     const connectionString = GBDeployer.getConnectionStringFromInstance(instance);
     const dsName = 'gb';
-
-    // Removes any previous index.
-
     try {
-      await search.deleteDataSource(dsName);
+      await search.createDataSource(dsName, dsName, 'GuaribasQuestion', 'azuresql', connectionString);
     } catch (err) {
-      // If it is a 404 there is nothing to delete as it is the first creation.
+      GBLog.error(err);
 
-      if (err.code !== 404) {
-        throw err;
-      }
     }
 
     // Removes the index.
 
     try {
-      await search.deleteIndex();
+      await search.createIndex(searchSchema, dsName);
     } catch (err) {
       // If it is a 404 there is nothing to delete as it is the first creation.
 
       if (err.code !== 404 && err.code !== 'OperationNotAllowed') {
-        throw err;
+
       }
     }
 
-    // Creates the data source and index on the cloud.
 
-    try {
+    await search.rebuildIndex(instance.searchIndexer);
-      await search.createDataSource(dsName, dsName, 'GuaribasQuestion', 'azuresql', connectionString);
+
-    } catch (err) {
-      GBLog.error(err);
-      throw err;
-    }
-    await search.createIndex(searchSchema, dsName);
   }
 
   /**

packages/core.gbapp/services/GBLogEx.ts

@@ -0,0 +1,86 @@
+/*****************************************************************************\
+|                                               ( )_  _                       |
+|    _ _    _ __   _ _    __    ___ ___     _ _ | ,_)(_)  ___  _   _    _     |
+|   ( '_`\ ( '__)/'_` ) /'_ `\/' _ ` _ `\ /'_` )| |  | |/',__)/ \ /`\ /'_`\   |
+|   | (_) )| |  ( (_| |( (_) || ( ) ( ) |( (_| || |_ | |\__, \| |*| |( (_) )  |
+|   | ,__/'(_)  `\__,_)`\__  |(_) (_) (_)`\__,_)`\__)(_)(____/(_) (_)`\___/'  |
+|   | |                ( )_) |                                                |
+|   (_)                 \___/'                                                |
+|                                                                             |
+| General Bots Copyright (c) Pragmatismo.io. All rights reserved.             |
+| Licensed under the AGPL-3.0.                                                |
+|                                                                             |
+| According to our dual licensing model, this program can be used either      |
+| under the terms of the GNU Affero General Public License, version 3,        |
+| or under a proprietary license.                                             |
+|                                                                             |
+| The texts of the GNU Affero General Public License with an additional       |
+| permission and of our proprietary license can be found at and               |
+| in the LICENSE file you have received along with this program.              |
+|                                                                             |
+| This program is distributed in the hope that it will be useful,             |
+| but WITHOUT ANY WARRANTY, without even the implied warranty of              |
+| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                |
+| GNU Affero General Public License for more details.                         |
+|                                                                             |
+| "General Bots" is a registered trademark of Pragmatismo.io.                 |
+| The licensing of the program under the AGPLv3 does not imply a              |
+| trademark license. Therefore any rights, title and interest in              |
+| our trademarks remain entirely with us.                                     |
+|                                                                             |
+\*****************************************************************************/
+
+/**
+ * @fileoverview General Bots server core.
+ */
+
+'use strict';
+
+import { GBLog, IGBInstance } from 'botlib';
+import { GuaribasLog } from '../models/GBModel.js';
+
+export class GBLogEx {
+  public static async error(minOrInstanceId: any, message: string) {
+    if (typeof minOrInstanceId === 'object') {
+      minOrInstanceId = minOrInstanceId.instance.instanceId;
+    }
+    GBLog.error(`${minOrInstanceId}: ${message}.`);
+    await this.log(minOrInstanceId, 'e', message);
+  }
+
+  public static async debug(minOrInstanceId: any, message: string) {
+    if (typeof minOrInstanceId === 'object') {
+      minOrInstanceId = minOrInstanceId.instance.instanceId;
+    }
+    GBLog.debug(`${minOrInstanceId}: ${message}.`);
+    await this.log(minOrInstanceId, 'd', message);
+  }
+
+  public static async info(minOrInstanceId: any, message: string) {
+    if (typeof minOrInstanceId === 'object') {
+      minOrInstanceId = minOrInstanceId.instance.instanceId;
+    }
+    GBLog.info(`${minOrInstanceId}: ${message}.`);
+    await this.log(minOrInstanceId, 'i', message);
+  }
+
+  public static async verbose(minOrInstanceId: any, message: string) {
+    if (typeof minOrInstanceId === 'object') {
+      minOrInstanceId = minOrInstanceId.instance.instanceId;
+    }
+    GBLog.verbose(`${minOrInstanceId}: ${message}.`);
+    await this.log(minOrInstanceId, 'v', message);
+  }
+
+  /**
+   * Finds and update user agent information to a next available person.
+   */
+  public static async log(instance: IGBInstance, kind: string, message: string): Promise<GuaribasLog> {
+    message = message ? message.substring(0, 1023) : null;
+    return await GuaribasLog.create(<GuaribasLog>{
+      instanceId: instance ? instance.instanceId : 1,
+      message: message,
+      kind: kind
+    });
+  }
+}

packages/core.gbapp/services/GBMinService.ts

@@ -87,6 +87,7 @@ import { GoogleChatDirectLine } from '../../google-chat.gblib/services/GoogleCha
 import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js';
 import * as nlp from 'node-nlp';
 import Path from 'path';
+import { GBSSR } from './GBSSR.js';
 
 /**
  * Minimal service layer for a bot and encapsulation of BOT Framework calls.
@@ -95,7 +96,7 @@ export class GBMinService {
   /**
    * Default General Bots User Interface package.
    */
-  private static uiPackage = 'default.gbui';
+  public static uiPackage = 'default.gbui';
 
   /**
    * Main core service attached to this bot service.
@@ -141,23 +142,11 @@ export class GBMinService {
     // Servers default UI on root address '/' if web enabled.
 
     if (process.env.DISABLE_WEB !== 'true') {
-      // SSR processing.
+      // SSR processing and default.gbui access definition.
 
-      const defaultOptions = {
+      GBServer.globals.server.get('/', async (req, res, next) => {
-        prerender: [],
+        await GBSSR.ssrFilter(req, res, next);
-        exclude: ['/api/', '/instances/', '/webhooks/'],
+      });
-        useCache: true,
-        cacheRefreshRate: 86400
-      };
-      // GBServer.globals.server.use(ssrForBots(defaultOptions));
-
-      const url = GBServer.globals.wwwroot
-        ? GBServer.globals.wwwroot
-        : urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build');
-
-      // default.gbui access definition.
-
-      GBServer.globals.server.use('/', express.static(url));
 
       // Servers the bot information object via HTTP so clients can get
       // instance information stored on server.
@@ -240,6 +229,39 @@ export class GBMinService {
     removeRoute(GBServer.globals.server, uiUrl);
 
     GBServer.globals.minInstances = GBServer.globals.minInstances.filter(p => p.instance.botId !== botId);
+  }
+
+  /**
+   * Mount the bot web site (default.gbui) secure domain.
+   */
+  public async loadDomain(min: GBMinInstance) {
+    // TODO: https://github.com/GeneralBots/BotServer/issues/321
+    const options = {
+      passphrase: process.env.CERTIFICATE2_PASSPHRASE,
+      pfx: Fs.readFileSync(process.env.CERTIFICATE2_PFX)
+    };
+
+    const domain = min.core.getParam(min.instance, 'Domain', null);
+    if (domain) {
+      GBServer.globals.server.get(domain, async (req, res, next) => {
+        await GBSSR.ssrFilter(req, res, next);
+      });
+      GBLog.verbose(`Bot UI ${GBMinService.uiPackage} accessible at custom domain: ${domain}.`);
+    }
+
+    
+    GBServer.globals.httpsServer.addContext(process.env.CERTIFICATE2_DOMAIN, options);
+  }
+
+  /**
+   * Unmounts the bot web site (default.gbui) secure domain, if any.
+   */
+  public async unloadDomain(instance: IGBInstance) {
+
+
+
+
+    
   }
 
   /**
@@ -295,6 +317,10 @@ export class GBMinService {
     if (!Fs.existsSync(dir)) {
       mkdirp.sync(dir);
     }
+    dir = `work/${min.botId}.gbai/${min.botId}.gbui`;
+    if (!Fs.existsSync(dir)) {
+      mkdirp.sync(dir);
+    }
 
     // Loads Named Entity data for this bot.
 
@@ -370,23 +396,15 @@ export class GBMinService {
 
     if (process.env.DISABLE_WEB !== 'true') {
       const uiUrl = `/${instance.botId}`;
+
+      GBServer.globals.server.get(uiUrl, async (req, res, next) => {
+        await GBSSR.ssrFilter(req, res, next);
+      });
       const uiUrlAlt = `/${instance.activationCode}`;
-      GBServer.globals.server.use(
+      GBServer.globals.server.get(uiUrlAlt, async (req, res, next) => {
-        uiUrl,
+        await GBSSR.ssrFilter(req, res, next);
-        express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'))
+      });
-      );
+
-      GBServer.globals.server.use(
-        uiUrlAlt,
-        express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'))
-      );
-      const domain = min.core.getParam(min.instance, 'Domain', null);
-      if (domain) {
-        GBServer.globals.server.use(
-          domain,
-          express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'))
-        );
-        GBLog.verbose(`Bot UI ${GBMinService.uiPackage} accessible at custom domain: ${domain}.`);
-      }
       GBLog.verbose(`Bot UI ${GBMinService.uiPackage} accessible at: ${uiUrl} and ${uiUrlAlt}.`);
     }
 
@@ -599,6 +617,7 @@ export class GBMinService {
    * Gets a Speech to Text / Text to Speech token from the provider.
    */
   private async getSTSToken(instance: any) {
+    return null; // TODO: https://github.com/GeneralBots/BotServer/issues/332
     const options = {
       method: 'POST',
       headers: {
@@ -884,12 +903,7 @@ export class GBMinService {
               data: data.slice(0, 10)
             });
           }
-
-          // Saves session user (persisted GuaribasUser is inside).
-
-          await min.userProfile.set(step.context, user);
         }
-
         // Required for MSTEAMS handling of persisted conversations.
 
         if (step.context.activity.channelId === 'msteams') {
@@ -927,7 +941,7 @@ export class GBMinService {
             if (startDialog) {
               await sec.setParam(userId, 'welcomed', 'true');
               GBLog.info(`Auto start (teams) dialog is now being called: ${startDialog} for ${min.instance.botId}...`);
-              await GBVMService.callVM(startDialog.toLowerCase(), min, step, this.deployer, false);
+              await GBVMService.callVM(startDialog.toLowerCase(), min, step, user, this.deployer, false);
             }
           }
         }
@@ -973,7 +987,7 @@ export class GBMinService {
                 GBLog.info(
                   `Auto start (web 1) dialog is now being called: ${startDialog} for ${min.instance.instanceId}...`
                 );
-                await GBVMService.callVM(startDialog.toLowerCase(), min, step, this.deployer, false);
+                await GBVMService.callVM(startDialog.toLowerCase(), min, step, user, this.deployer, false);
               }
             }
           } else {
@@ -987,11 +1001,10 @@ export class GBMinService {
               ) {
                 await sec.setParam(userId, 'welcomed', 'true');
                 min['conversationWelcomed'][step.context.activity.conversation.id] = true;
-                await min.userProfile.set(step.context, user);
                 GBLog.info(
                   `Auto start (whatsapp) dialog is now being called: ${startDialog} for ${min.instance.instanceId}...`
                 );
-                await GBVMService.callVM(startDialog.toLowerCase(), min, step, this.deployer, false);
+                await GBVMService.callVM(startDialog.toLowerCase(), min, step, user, this.deployer, false);
               }
             }
           }
@@ -1004,10 +1017,6 @@ export class GBMinService {
 
           await this.processEventActivity(min, user, context, step);
         }
-
-        // Saves conversation state for later use.
-
-        await conversationState.saveChanges(context, true);
       } catch (error) {
         const msg = `ERROR: ${error.message} ${error.stack ? error.stack : ''}`;
         GBLog.error(msg);
@@ -1051,7 +1060,7 @@ export class GBMinService {
       if (startDialog && !min['conversationWelcomed'][step.context.activity.conversation.id]) {
         user.welcomed = true;
         GBLog.info(`Auto start (web 2) dialog is now being called: ${startDialog} for ${min.instance.instanceId}...`);
-        await GBVMService.callVM(startDialog.toLowerCase(), min, step, this.deployer, false);
+        await GBVMService.callVM(startDialog.toLowerCase(), min, step, user, this.deployer, false);
       }
     } else if (context.activity.name === 'updateToken') {
       const token = context.activity.data;
@@ -1133,6 +1142,7 @@ export class GBMinService {
     const member = context.activity.from;
 
     let user = await sec.ensureUser(min.instance.instanceId, member.id, member.name, '', 'web', member.name, null);
+
     const userId = user.userId;
     const params = user.params ? JSON.parse(user.params) : {};
 
@@ -1155,6 +1165,9 @@ export class GBMinService {
           userId,
           context.activity.text
         );
+
+        const conversationReference = JSON.stringify(TurnContext.getConversationReference(context.activity));
+        await sec.updateConversationReferenceById(userId, conversationReference);
       }
     }
 
@@ -1202,7 +1215,7 @@ export class GBMinService {
 
     const isVMCall = Object.keys(min.scriptMap).find(key => min.scriptMap[key] === context.activity.text) !== undefined;
     if (isVMCall) {
-      await GBVMService.callVM(context.activity.text, min, step, this.deployer, false);
+      await GBVMService.callVM(context.activity.text, min, step, user, this.deployer, false);
     } else if (context.activity.text.charAt(0) === '/') {
       const text = context.activity.text;
       const parts = text.split(' ');
@@ -1212,16 +1225,13 @@ export class GBMinService {
       if (cmdOrDialogName === '/start') {
         // Reset user.
 
-        const user = await min.userProfile.get(context, {});
         await min.conversationalService.sendEvent(min, step, 'loadInstance', {});
-        user.loaded = false;
-        await min.userProfile.set(step.context, user);
       } else if (cmdOrDialogName === '/call') {
-        await GBVMService.callVM(args, min, step, this.deployer, false);
+        await GBVMService.callVM(args, min, step, user, this.deployer, false);
       } else if (cmdOrDialogName === '/callsch') {
-        await GBVMService.callVM(args, min, null, null, false);
+        await GBVMService.callVM(args, min, null, null, null, false);
       } else if (cmdOrDialogName === '/calldbg') {
-        await GBVMService.callVM(args, min, step, this.deployer, true);
+        await GBVMService.callVM(args, min, step, user, this.deployer, true);
       } else {
         await step.beginDialog(cmdOrDialogName, { args: args });
       }

packages/core.gbapp/services/GBSSR.ts

@@ -36,290 +36,285 @@
 
 'use strict';
 
-import puppeteer from 'puppeteer-extra';
+import { createRequire } from 'module';
+const require = createRequire(import.meta.url);
+
+import Path from 'path';
 import Fs from 'fs';
-
-// const StealthPlugin from 'puppeteer-extra-plugin-stealth')
-// puppeteer.use(StealthPlugin());
-
 import { NextFunction, Request, Response } from 'express';
 import urljoin from 'url-join';
+import { GBMinInstance } from 'botlib';
+import { GBServer } from '../../../src/app.js';
+import { GBLogEx } from './GBLogEx.js';
+import urlJoin from 'url-join';
+import { GBDeployer } from './GBDeployer.js';
+import { GBMinService } from './GBMinService.js';
+const puppeteer = require('puppeteer-extra');
+const hidden = require('puppeteer-extra-plugin-stealth');
+const { executablePath } = require('puppeteer');
 
-// https://hackernoon.com/tips-and-tricks-for-web-scraping-with-puppeteer-ed391a63d952
+export class GBSSR {
-// Dont download all resources, we just need the HTML
+  // https://hackernoon.com/tips-and-tricks-for-web-scraping-with-puppeteer-ed391a63d952
-// Also, this is huge performance/response time boost
+  // Dont download all resources, we just need the HTML
-const blockedResourceTypes = ['image', 'media', 'font', 'texttrack', 'object', 'beacon', 'csp_report', 'imageset'];
+  // Also, this is huge performance/response time boost
-// const whitelist = ["document", "script", "xhr", "fetch"];
+  private static blockedResourceTypes = [
-const skippedResources = [
+    'image',
-  'quantserve',
+    'media',
-  'adzerk',
+    'font',
-  'doubleclick',
+    'texttrack',
-  'adition',
+    'object',
-  'exelator',
+    'beacon',
-  'sharethrough',
+    'csp_report',
-  'cdn.api.twitter',
+    'imageset'
-  'google-analytics',
-  'googletagmanager',
-  'google',
-  'fontawesome',
-  'facebook',
-  'analytics',
-  'optimizely',
-  'clicktale',
-  'mixpanel',
-  'zedo',
-  'clicksor',
-  'tiqcdn'
-];
-
-const RENDER_CACHE = new Map();
-
-async function createBrowser (profilePath): Promise<any> {
-  let args = [
-    '--check-for-update-interval=2592000',
-    '--disable-accelerated-2d-canvas',
-    '--disable-dev-shm-usage',
-    '--disable-features=site-per-process',
-    '--disable-gpu',
-    '--no-first-run',
-    '--no-default-browser-check'
   ];
 
-  if (profilePath) {
+  // const whitelist = ["document", "script", "xhr", "fetch"];
-    args.push(`--user-data-dir=${profilePath}`);
+  private static skippedResources = [
+    'quantserve',
+    'adzerk',
+    'doubleclick',
+    'adition',
+    'exelator',
+    'sharethrough',
+    'cdn.api.twitter',
+    'google-analytics',
+    'googletagmanager',
+    'google',
+    'fontawesome',
+    'facebook',
+    'analytics',
+    'optimizely',
+    'clicktale',
+    'mixpanel',
+    'zedo',
+    'clicksor',
+    'tiqcdn'
+  ];
 
-    const preferences = urljoin(profilePath, 'Default', 'Preferences');
+  public static async createBrowser(profilePath): Promise<any> {
-    if (Fs.existsSync(preferences)) {
+    let args = [
-      const file = Fs.readFileSync(preferences, 'utf8');
+      '--check-for-update-interval=2592000',
-      const data = JSON.parse(file);
+      '--disable-accelerated-2d-canvas',
-      data['profile']['exit_type'] = 'none';
+      '--disable-dev-shm-usage',
-      Fs.writeFileSync(preferences, JSON.stringify(data));
+      '--disable-features=site-per-process',
+      '--disable-gpu',
+      '--no-first-run',
+      '--no-default-browser-check'
+    ];
+
+    if (profilePath) {
+      args.push(`--user-data-dir=${profilePath}`);
+
+      const preferences = urljoin(profilePath, 'Default', 'Preferences');
+      if (Fs.existsSync(preferences)) {
+        const file = Fs.readFileSync(preferences, 'utf8');
+        const data = JSON.parse(file);
+        data['profile']['exit_type'] = 'none';
+        Fs.writeFileSync(preferences, JSON.stringify(data));
+      }
     }
+    puppeteer.use(hidden());
+    const browser = await puppeteer.launch({
+      args: args,
+      ignoreHTTPSErrors: true,
+      headless: false,
+      defaultViewport: null,
+      executablePath: executablePath(),
+      ignoreDefaultArgs: ['--enable-automation', '--enable-blink-features=IdleDetection']
+    });
+    return browser;
   }
 
-  const browser = await puppeteer.launch({
+  /**
-    args: args,
+   * Return the HTML of bot default.gbui.
-    ignoreHTTPSErrors: true,
+   */
-    headless: false,
+  public static async getHTML(min: GBMinInstance) {
-    defaultViewport: null,
+    const url = urljoin(GBServer.globals.publicAddress, min.botId);
-    ignoreDefaultArgs: ['--enable-automation', '--enable-blink-features=IdleDetection']
+    const browser = await GBSSR.createBrowser(null);
-  });
+    const stylesheetContents = {};
-  return browser;
+    let html;
-}
 
-async function recursiveFindInFrames (inputFrame, selector) {
+    try {
-  const frames = inputFrame.childFrames();
+      const page = await browser.newPage();
-  const results = await Promise.all(
+      await page.setUserAgent(
-    frames.map(async frame => {
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
-      const el = await frame.$(selector);
+      );
-      if (el) return el;
+      await page.setRequestInterception(true);
-      if (frame.childFrames().length > 0) {
+      page.on('request', request => {
-        return await recursiveFindInFrames(frame, selector);
+        const requestUrl = request.url().split('?')[0].split('#')[0];
-      }
+        if (
-      return null;
+          GBSSR.blockedResourceTypes.indexOf(request.resourceType()) !== -1 ||
-    })
+          GBSSR.skippedResources.some(resource => requestUrl.indexOf(resource) !== -1)
-  );
+        ) {
-  return results.find(Boolean);
+          request.abort();
-}
+        } else {
-
+          request.continue();
-/**
+        }
- * https://developers.google.com/web/tools/puppeteer/articles/ssr#reuseinstance
- * @param {string} url URL to prerender.
- */
-async function ssr (url: string, useCache: boolean, cacheRefreshRate: number) {
-  if (RENDER_CACHE.has(url) && useCache) {
-    const cached = RENDER_CACHE.get(url);
-    if (Date.now() - cached.renderedAt > cacheRefreshRate && !(cacheRefreshRate <= 0)) {
-      RENDER_CACHE.delete(url);
-    } else {
-      return {
-        html: cached.html,
-        status: 200
-      };
-    }
-  }
-  const browser = await createBrowser(null);
-  const stylesheetContents = {};
-
-  try {
-    const page = await browser.newPage();
-    await page.setUserAgent(
-      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
-    );
-    await page.setRequestInterception(true);
-    page.on('request', request => {
-      const requestUrl = request
-        .url()
-        .split('?')[0]
-        .split('#')[0];
-      if (
-        blockedResourceTypes.indexOf(request.resourceType()) !== -1 ||
-        skippedResources.some(resource => requestUrl.indexOf(resource) !== -1)
-      ) {
-        request.abort();
-      } else {
-        request.continue();
-      }
-    });
-
-    page.on('response', async resp => {
-      const responseUrl = resp.url();
-      const sameOrigin = new URL(responseUrl).origin === new URL(url).origin;
-      const isStylesheet = resp.request().resourceType() === 'stylesheet';
-      if (sameOrigin && isStylesheet) {
-        stylesheetContents[responseUrl] = await resp.text();
-      }
-    });
-
-    const response = await page.goto(url, {
-      timeout: 120000,
-      waitUntil: 'networkidle0'
-    });
-
-    const sleep = ms => {
-      return new Promise(resolve => {
-        setTimeout(resolve, ms);
       });
-    };
-    await sleep(45000);
 
-    // Inject <base> on page to relative resources load properly.
+      page.on('response', async resp => {
-    await page.evaluate(url => {
+        const responseUrl = resp.url();
-      const base = document.createElement('base');
+        const sameOrigin = new URL(responseUrl).origin === new URL(url).origin;
-      base.href = url;
+        const isStylesheet = resp.request().resourceType() === 'stylesheet';
-      // Add to top of head, before all other resources.
+        if (sameOrigin && isStylesheet) {
-      document.head.prepend(base);
+          stylesheetContents[responseUrl] = await resp.text();
-    }, url);
+        }
-
-    // Remove scripts and html imports. They've already executed.
-    await page.evaluate(() => {
-      const elements = document.querySelectorAll('script, link[rel="import"]');
-      elements.forEach(e => {
-        e.remove();
       });
-    });
 
-    // Replace stylesheets in the page with their equivalent <style>.
+      await page.setExtraHTTPHeaders({
-    await page.$$eval(
+        'ngrok-skip-browser-warning': '1'
-      'link[rel="stylesheet"]',
+      });
-      (links, content) => {
+      const response = await page.goto(url, {
-        links.forEach((link: any) => {
+        timeout: 120000,
-          const cssText = content[link.href];
+        waitUntil: 'networkidle0'
-          if (cssText) {
+      });
-            const style = document.createElement('style');
+
-            style.textContent = cssText;
+      const sleep = ms => {
-            link.replaceWith(style);
+        return new Promise(resolve => {
-          }
+          setTimeout(resolve, ms);
         });
-      },
+      };
-      stylesheetContents
-    );
 
-    const html = await page.content();
+      await sleep(15000);
 
-    // Close the page we opened here (not the browser).
+      // Inject <base> on page to relative resources load properly.
-    await page.close();
+
-    if (useCache) {
+      await page.evaluate(url => {
-      RENDER_CACHE.set(url, { html, renderedAt: Date.now() });
+        const base = document.createElement('base');
+        base.href = url;
+        // Add to top of head, beeeEEEfore all other resources.
+        document.head.prepend(base);
+      }, url);
+
+      // Remove scripts and html imports. They've already executed.
+
+      await page.evaluate(() => {
+        const elements = document.querySelectorAll('script, link[rel="import"]');
+        elements.forEach(e => {
+          e.remove();
+        });
+      });
+
+      // Replace stylesheets in the page with their equivalent <style>.
+
+      await page.$$eval(
+        'link[rel="stylesheet"]',
+        (links, content) => {
+          links.forEach((link: any) => {
+            const cssText = content[link.href];
+            if (cssText) {
+              const style = document.createElement('style');
+              style.textContent = cssText;
+              link.replaceWith(style);
+            }
+          });
+        },
+        stylesheetContents
+      );
+
+      html = await page.content();
+
+      // Close the page we opened here (not the browser).
+
+      await page.close();
+    } catch (e) {
+      const html = e.toString();
+      GBLogEx.error(min, `URL: ${url} Failed with message: ${html}`);
+    } finally {
+      await browser.close();
     }
-    return { html, status: response!.status() };
+    return html;
-  } catch (e) {
-    const html = e.toString();
-    console.warn({ message: `URL: ${url} Failed with message: ${html}` });
-    return { html, status: 500 };
-  } finally {
-    await browser.close();
   }
-}
 
-function clearCache () {
+  public static async ssrFilter(req: Request, res: Response, next) {
-  RENDER_CACHE.clear();
+    let applyOptions = {
-}
-
-interface Options {
-  prerender?: Array<string>;
-  exclude?: Array<string>;
-  useCache?: boolean;
-  cacheRefreshRate?: number;
-}
-
-function ssrForBots (
-  options: Options = {
-    prerender: [], // Array containing the user-agents that will trigger the ssr service
-    exclude: [], // Array containing paths and/or extentions that will be excluded from being prerendered by the ssr service
-    useCache: true, // Variable that determins if we will use page caching or not
-    cacheRefreshRate: 86400 // Seconds of which the cache will be kept alive, pass 0 or negative value for infinite lifespan
-  }
-) {
-  let applyOptions = Object.assign(
-    {
       prerender: [], // Array containing the user-agents that will trigger the ssr service
       exclude: [], // Array containing paths and/or extentions that will be excluded from being prerendered by the ssr service
       useCache: true, // Variable that determins if we will use page caching or not
       cacheRefreshRate: 86400 // Seconds of which the cache will be kept alive, pass 0 or negative value for infinite lifespan
-    },
+    };
-    options
-  );
 
-  // Default user agents
+    // Default user agents
-  const prerenderArray = [
+    const prerenderArray = [
-    'bot',
+      'bot',
-    'googlebot',
+      'googlebot',
-    'Chrome-Lighthouse',
+      'Chrome-Lighthouse',
-    'DuckDuckBot',
+      'DuckDuckBot',
-    'ia_archiver',
+      'ia_archiver',
-    'bingbot',
+      'bingbot',
-    'yandex',
+      'yandex',
-    'baiduspider',
+      'baiduspider',
-    'Facebot',
+      'Facebot',
-    'facebookexternalhit',
+      'facebookexternalhit',
-    'facebookexternalhit/1.1',
+      'facebookexternalhit/1.1',
-    'twitterbot',
+      'twitterbot',
-    'rogerbot',
+      'rogerbot',
-    'linkedinbot',
+      'linkedinbot',
-    'embedly',
+      'embedly',
-    'quora link preview',
+      'quora link preview',
-    'showyoubot',
+      'showyoubot',
-    'outbrain',
+      'outbrain',
-    'pinterest',
+      'pinterest',
-    'slackbot',
+      'slackbot',
-    'vkShare',
+      'vkShare',
-    'W3C_Validator'
+      'W3C_Validator'
-  ];
+    ];
 
-  // default exclude array
+    // default exclude array
-  const excludeArray = ['.xml', '.ico', '.txt', '.json'];
+    const excludeArray = ['.xml', '.ico', '.txt', '.json'];
+    const userAgent: string = req.headers['user-agent'] || '';
+    const prerender = new RegExp([...prerenderArray, ...applyOptions.prerender].join('|').slice(0, -1), 'i').test(
+      userAgent
+    );
+    const exclude = !new RegExp([...excludeArray, ...applyOptions.exclude].join('|').slice(0, -1)).test(
+      req.originalUrl
+    );
 
-  function ssrOnDemand (req: Request, res: Response, next: NextFunction) {
+    // Reads from static HTML when a bot is crawling.
-    Promise.resolve(() => {
+
+    const botId = req.originalUrl ? req.originalUrl.substr(1) : GBServer.globals.minInstances[0].botId; // TODO: Get only bot.
+    let min: GBMinInstance = req.url ===  '/'?
+    GBServer.globals.minInstances[0]:
+    GBServer.globals.minInstances.filter(p => p.instance.botId === botId)[0];
+    
+    if (min && req.originalUrl && prerender && exclude) {
+      const path = Path.join(
+        process.env.PWD,
+        'work',
+        `${min.instance.botId}.gbai`,
+        `${min.instance.botId}.gbui`,
+        'index.html'
+      );
+      const html = Fs.readFileSync(path, 'utf8');
+      res.status(200).send(html);
       return true;
-    })
+    } else {
-      .then(async () => {
-        const userAgent: string = req.headers['user-agent'] || '';
 
-        const prerender = new RegExp([...prerenderArray, ...applyOptions.prerender].join('|').slice(0, -1), 'i').test(
+      const path = Path.join(
-          userAgent
+        process.env.PWD,
-        );
+        GBDeployer.deployFolder,
-
+        GBMinService.uiPackage,
-        const exclude = !new RegExp([...excludeArray, ...applyOptions.exclude].join('|').slice(0, -1)).test(
+        'build',
-          req.originalUrl
+        min ? `index.html` : req.url
-        );
+      ); 
-
+      if (Fs.existsSync(path)) {
-        if (req.originalUrl && prerender && exclude) {
+        if (min){   
-          const { html, status } = await ssr(
+          let html = Fs.readFileSync(path, 'utf8');
-            req.protocol + '://' + req.get('host') + req.originalUrl,
+          html = html.replace(/\{botId\}/gi, min.botId);
-            applyOptions.useCache,
+          html = html.replace(/\{theme\}/gi, min.instance.theme);
-            applyOptions.cacheRefreshRate
+          html = html.replace(/\{title\}/gi, min.instance.title);
-          );
+          res.send(html).end();
-          return res.status(status).send(html);
-        } else {
-          return next();
         }
-      })
+        else
-      .catch(next);
+        {
+          res.sendFile(path);
+        }        
+        return true;
+      } else {
+        GBLogEx.info(min, `HTTP 404: ${req.url}.`);
+        res.status(404);
+        res.end();
+      }
+    }
   }
-
-  return ssrOnDemand;
 }
-
-export { createBrowser, ssr, clearCache, ssrForBots };

packages/customer-satisfaction.gbapp/dialogs/FeedbackDialog.ts

@@ -170,8 +170,7 @@ export class FeedbackDialog extends IGBDialog {
             await sec.updateHumanAgent(userSystemId, min.instance.instanceId, null);
             await sec.updateHumanAgent(manualUser.userSystemId, min.instance.instanceId, null);
 
-            user = await sec.getUserFromSystemId(userSystemId);
+            
-            await min.userProfile.set(step.context, user);
           } else if (user.agentMode === 'human') {
             const agent = await sec.getUserFromSystemId(user.agentSystemId);
 
@@ -200,9 +199,7 @@ export class FeedbackDialog extends IGBDialog {
 
             await sec.updateHumanAgent(user.userSystemId, min.instance.instanceId, null);
             await sec.updateHumanAgent(agent.userSystemId, min.instance.instanceId, null);
-
+            
-            user = await sec.getUserFromSystemId(userSystemId);
-            await min.userProfile.set(step.context, user);
           } else {
             if (user.userSystemId.charAt(2) === ':' || userSystemId.indexOf('@') > -1) {
               // Agent is from Teams or Google Chat.

packages/default.gbui/public/index.html

@@ -38,11 +38,18 @@
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <meta http-equiv="X-UA-Compatible" content="IE=edge" />
   <link rel="shortcut icon" href="%PUBLIC_URL%/favicon.ico">
-  <link rel="sylesheet" href="%PUBLIC_URL%/ssr-delay">
-
   <link rel="stylesheet" type="text/css" href="./css/pragmatismo.css" />
+  <link rel="stylesheet" type="text/css" href="/themes/{theme}/css/ChatPane.css" />
+  <link rel="stylesheet" type="text/css" href="/themes/{theme}/css/Content.css" />
+  <link rel="stylesheet" type="text/css" href="/themes/{theme}/css/Footer.css" />
+  <link rel="stylesheet" type="text/css" href="/themes/{theme}/css/GifPlayer.css" />
+  <link rel="stylesheet" type="text/css" href="/themes/{theme}/css/MediaPlayer.css" />
+  <link rel="stylesheet" type="text/css" href="/themes/{theme}/css/NavBar.css" />
+  <link rel="stylesheet" type="text/css" href="/themes/{theme}/css/App.css" />
+  <link rel="stylesheet" type="text/css" href="/themes/{theme}/css/SideBarMenu.css"  />
+
   <script src="https://cdn.botframework.com/botframework-webchat/latest/webchat.js"></script>
-  <title>General Bots Community Edition | pragmatismo.io</title>
+  <title>{title} - General Bots Community Edition</title>
 
   <style>
     .loader {

packages/kb.gbapp/dialogs/AskDialog.ts

@@ -101,7 +101,7 @@ export class AskDialog extends IGBDialog {
         if (step.options && step.options.firstTime) {
           text = Messages[locale].ask_first_time;
         } else if (step.options && step.options.isReturning) {
-          text = ''; // REMOVED: Messages[locale].anything_else;
+          text = Messages[locale].anything_else;
         } else if (step.options && step.options.emptyPrompt) {
           text = '';
         } else if (user.subjects.length > 0) {
@@ -169,7 +169,9 @@ export class AskDialog extends IGBDialog {
       },
       async step => {
         let answer: GuaribasAnswer = null;
-        const user = await min.userProfile.get(step.context, {});
+        const member = step.context.activity.from;
+        const sec = new SecService();
+        let user = await sec.ensureUser(min.instance.instanceId, member.id, member.name, '', 'web', member.name, null);
 
         const minBoot = GBServer.globals.minBoot as any;
 
@@ -182,7 +184,7 @@ export class AskDialog extends IGBDialog {
         if (!text) {
           const startDialog = min.core.getParam(min.instance, 'Start Dialog', null);
           if (startDialog) {
-            await GBVMService.callVM(startDialog.toLowerCase().trim(), min, step, this.deployer, false);
+            await GBVMService.callVM(startDialog.toLowerCase().trim(), min, step, user, this.deployer, false);
           }
 
           return step.endDialog();
@@ -210,54 +212,54 @@ export class AskDialog extends IGBDialog {
           min.instance.searchScore ? min.instance.searchScore : minBoot.instance.searchScore
         );
 
-        user.lastQuestion = text;
+        // TODO: https://github.com/GeneralBots/BotServer/issues/9 user.lastQuestion = text;
-        await min.userProfile.set(step.context, user);
 
-        const resultsA = await service.ask(min.instance, text, searchScore, user.subjects);
+        const resultsA = await service.ask(min.instance, text, searchScore, null /* user.subjects */ );
 
         // If there is some result, answer immediately.
 
         if (resultsA !== undefined && resultsA.answer !== undefined) {
           // Saves some context info.
 
-          user.isAsking = false;
+          // user.isAsking = false;
-          user.lastQuestionId = resultsA.questionId;
+          // user.lastQuestionId = resultsA.questionId;
-          await min.userProfile.set(step.context, user);
+
 
           // Sends the answer to all outputs, including projector.
 
           answer = resultsA.answer;
 
           // If this search was restricted to some subjects...
-        } else if (user.subjects && user.subjects.length > 0) {
+        } 
-          // ...second time running Search, now with no filter.
+        // TODO: https://github.com/GeneralBots/BotServer/issues/9
+        // else if (user.subjects && user.subjects.length > 0) {
+        //   // ...second time running Search, now with no filter.
 
-          const resultsB = await service.ask(min.instance, text, searchScore, undefined);
+        //   const resultsB = await service.ask(min.instance, text, searchScore, undefined);
 
-          // If there is some result, answer immediately.
+        //   // If there is some result, answer immediately.
 
-          if (resultsB !== undefined && resultsB.answer !== undefined) {
+        //   if (resultsB !== undefined && resultsB.answer !== undefined) {
-            // Saves some context info.
+        //     // Saves some context info.
 
-            const user2 = await min.userProfile.get(step.context, {});
+        //     // user2.isAsking = false;
-            user2.isAsking = false;
+        //     // user2.lastQuestionId = resultsB.questionId;
-            user2.lastQuestionId = resultsB.questionId;
+           
-            await min.userProfile.set(step.context, user2);
 
-            // Informs user that a broader search will be used.
+        //     // Informs user that a broader search will be used.
 
-            if (user2.subjects.length > 0) {
+        //     if (user2.subjects.length > 0) {
-              await min.conversationalService.sendText(min, step, Messages[locale].wider_answer);
+        //       await min.conversationalService.sendText(min, step, Messages[locale].wider_answer);
-            }
+        //     }
 
-            answer = resultsB.answer;
+        //     answer = resultsB.answer;
-          }
+        //   }
-        }
+        // }
 
         // Try to answer by search.
 
         if (answer) {
-          return await AskDialog.handleAnswer(service, min, step, answer);
+          return await AskDialog.handleAnswer(service, min, step, user, answer);
         }
 
         // Tries to answer by NLP.
@@ -322,11 +324,11 @@ export class AskDialog extends IGBDialog {
     ];
   }
 
-  private static async handleAnswer(service: KBService, min: GBMinInstance, step: any, answer: GuaribasAnswer) {
+  private static async handleAnswer(service: KBService, min: GBMinInstance, step: any, user, answer: GuaribasAnswer) {
     const text = answer.content;
     if (text.endsWith('.docx')) {
       const mainName = GBVMService.getMethodNameFromVBSFilename(text);
-      return await GBVMService.callVM(mainName, min, step, this.deployer, false);
+      return await GBVMService.callVM(mainName, min, step, user, this.deployer, false);
     } else {
       await service.sendAnswer(min, AskDialog.getChannel(step), step, answer);
       return await step.replaceDialog('/ask', { isReturning: true });

packages/kb.gbapp/services/KBService.ts

@@ -62,8 +62,17 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
 import { CSService } from '../../customer-satisfaction.gbapp/services/CSService.js';
 import { GuaribasAnswer, GuaribasQuestion, GuaribasSubject } from '../models/index.js';
 import { GBConfigService } from './../../core.gbapp/services/GBConfigService.js';
+import { parse } from 'node-html-parser';
 import textract from 'textract';
 import pdf from 'pdf-extraction';
+import { GBSSR } from '../../core.gbapp/services/GBSSR.js';
+import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
+import mammoth from 'mammoth';
+import { url } from 'inspector';
+import { min } from 'lodash';
+import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
+import { text } from 'body-parser';
+import { GBVMService } from '../../basic.gblib/services/GBVMService.js';
 
 /**
  * Result for quey on KB data.
@@ -295,7 +304,7 @@ export class KBService implements IGBKBService {
         subject4: string;
       }
 
-      const client = new SearchClient<SearchResults>('https://' + instance.searchHost, 'azuresql-index', {
+      const client = new SearchClient<any>('https://' + instance.searchHost, 'azuresql-index', {
         key: instance.searchKey
       } as any);
 
@@ -307,15 +316,14 @@ export class KBService implements IGBKBService {
         top: 1
       });
 
-      const values = results.results;
       let returnedScore = 0;
 
       // Searches via Search (Azure Search).
 
       let found = false;
-      for await (const result of values) {
+      for await (const result of results.results) {
         found = true;
-        returnedScore = result['@search.score'];
+        returnedScore = result.score;
         if (returnedScore >= searchScore) {
           const value = await this.getAnswerById(instance.instanceId, result.document.answerId);
           if (value !== null) {
@@ -402,7 +410,7 @@ export class KBService implements IGBKBService {
 
   public async importKbTabularFile(
     filePath: string,
-    instanceId: number,
+    min: GBMinInstance,
     packageId: number
   ): Promise<GuaribasQuestion[]> {
     GBLog.info(`Now reading file ${filePath}...`);
@@ -514,10 +522,27 @@ export class KBService implements IGBKBService {
             return false;
           }
 
+          // In case  of code cell, compiles it and associate with the answer.
+
+          answer = GBVMService.normalizeQuotes(answer);
+          const isBasic = answer.toLowerCase().startsWith('/basic');
+          if (/TALK\s*\".*\"/gi.test(answer) || isBasic) {
+            const code = isBasic ? answer.substr(6) : answer;
+            const gbaiName = `${min.instance.botId}.gbai`;
+            const gbdialogName = `${min.instance.botId}.gbdialog`;
+            const scriptName = `tmp${GBAdminService.getRndReadableIdentifier()}.docx`;
+            const localName = Path.join('work', gbaiName, gbdialogName, `${scriptName}`);
+            Fs.writeFileSync(localName, code, { encoding: null });
+            answer = scriptName;
+
+            const vm = new GBVMService();
+            await vm.loadDialog(Path.basename(localName), Path.dirname(localName), min);
+          }
+
           // Now with all the data ready, creates entities in the store.
 
           const answer1 = {
-            instanceId: instanceId,
+            instanceId: min.instance.instanceId,
             content: answer,
             format: format,
             media: media,
@@ -535,7 +560,7 @@ export class KBService implements IGBKBService {
             subject3: subject3,
             subject4: subject4,
             content: question.replace(/["]+/g, ''),
-            instanceId: instanceId,
+            instanceId: min.instance.instanceId,
             skipIndex: question.charAt(0) === '"',
             packageId: packageId
           };
@@ -610,14 +635,15 @@ export class KBService implements IGBKBService {
     // Imports subjects tree into database and return it.
 
     const subjectFile = urlJoin(localPath, 'subjects.json');
+    const menuFile = urlJoin(localPath, 'menu.xlsx');
 
-    if (Fs.existsSync(subjectFile)) {
+    if (Fs.existsSync(subjectFile) || Fs.existsSync(menuFile)) {
-      await this.importSubjectFile(packageStorage.packageId, subjectFile, instance);
+      await this.importSubjectFile(packageStorage.packageId, subjectFile, menuFile, instance);
     }
 
     // Import tabular files in the tabular directory.
 
-    await this.importKbTabularDirectory(localPath, instance, packageStorage.packageId);
+    await this.importKbTabularDirectory(localPath, min, packageStorage.packageId);
 
     // Import remaining .md files in articles directory.
 
@@ -633,6 +659,7 @@ export class KBService implements IGBKBService {
    */
   public async importRemainingArticles(localPath: string, instance: IGBInstance, packageId: number): Promise<any> {
     const files = await walkPromise(urlJoin(localPath, 'articles'));
+    const data = { questions: [], answers: [] };
 
     await CollectionUtil.asyncForEach(files, async file => {
       if (file !== null && file.name.endsWith('.md')) {
@@ -651,7 +678,103 @@ export class KBService implements IGBKBService {
             prevId: 0 // https://github.com/GeneralBots/BotServer/issues/312
           });
         }
+      } else if (file !== null && file.name.endsWith('.docx')) {
+        const gbaiName = `${instance.botId}.gbai`;
+        const gbkbName = `${instance.botId}.gbkb`;
+        const localName = Path.join('work', gbaiName, gbkbName, 'articles', file.name);
+        const buffer = Fs.readFileSync(localName, { encoding: null });
+        var options = {
+          buffer: buffer,
+          convertImage: async image => {
+            const localName = Path.join(
+              'work',
+              gbaiName,
+              'cache',
+              `img-docx${GBAdminService.getRndReadableIdentifier()}.png`
+            );
+            const url = urlJoin(GBServer.globals.publicAddress, instance.botId, 'cache', Path.basename(localName));
+            const buffer = await image.read();
+            Fs.writeFileSync(localName, buffer, { encoding: null });
+            return { src: url };
+          }
+        };
+
+        let state = 0;
+        let previousState = state;
+        const next = (root, el, data) => {
+          // If it is root, change to the first item.
+
+          if (el.parentNode == null) {
+            el = el.firstChild;
+          }
+          let value = el.innerHTML;
+          const isHeader = el => el.rawTagName.startsWith('h') && el.rawTagName.length === 2;
+
+          // Handle questions from H* elements.
+
+          if (state === 0) {
+            const question = {
+              from: 'document',
+              to: '',
+              subject1: '',
+              subject2: '',
+              subject3: '',
+              subject4: '',
+              content: value.replace(/["]+/g, ''),
+              instanceId: instance.instanceId,
+              skipIndex: 0,
+              packageId: packageId
+            };
+            data.questions.push(question);
+            previousState = state;
+            state = 1;
+
+            // Everything else is content for that Header.
+          } else if (state === 1) {
+            // If next element is null, the tree has been passed, so
+            // finish the append of other elements between the last Header
+            // and the end of the document.
+
+            if (!el.nextSibling || isHeader(el.nextSibling)) {
+              const answer = {
+                instanceId: instance.instanceId,
+                content: value,
+                format: '.html',
+                media: file.name,
+                packageId: packageId,
+                prevId: 0
+              };
+
+              data.answers.push(answer);
+
+              state = 0;
+
+              // Otherwise, just append content to insert later.
+            } else {
+              value += value;
+            }
+          }
+
+          // Goes to the next node, as it is all same level nodes.
+
+          if (el.nextSibling) {
+            next(root, el.nextSibling, data);
+          }
+        };
+
+        const html = await mammoth.convertToHtml(options);
+        const root = parse(html.value);
+        next(root, root, data);
       }
+
+      // Persist to storage.
+
+      const answersCreated = await GuaribasAnswer.bulkCreate(data.answers);
+      let i = 0;
+      await CollectionUtil.asyncForEach(data.questions, async question => {
+        question.answerId = answersCreated[i++].answerId;
+      });
+      return await GuaribasQuestion.bulkCreate(data.questions);
     });
   }
 
@@ -696,18 +819,93 @@ export class KBService implements IGBKBService {
     }
   }
 
-  public async importKbTabularDirectory(localPath: string, instance: IGBInstance, packageId: number): Promise<any> {
+  public async importKbTabularDirectory(localPath: string, min: GBMinInstance, packageId: number): Promise<any> {
     const files = await walkPromise(localPath);
 
     await CollectionUtil.asyncForEach(files, async file => {
       if (file !== null && file.name.endsWith('.xlsx')) {
-        return await this.importKbTabularFile(urlJoin(file.root, file.name), instance.instanceId, packageId);
+        return await this.importKbTabularFile(urlJoin(file.root, file.name), min, packageId);
       }
     });
   }
 
-  public async importSubjectFile(packageId: number, filename: string, instance: IGBInstance): Promise<any> {
+  public async importSubjectFile(
-    const subjectsLoaded = JSON.parse(Fs.readFileSync(filename, 'utf8'));
+    packageId: number,
+    filename: string,
+    menuFile: string,
+    instance: IGBInstance
+  ): Promise<any> {
+    let subjectsLoaded;
+    if (menuFile) {
+      // Loads menu.xlsx and finds worksheet.
+
+      const workbook = new Excel.Workbook();
+      const data = await workbook.xlsx.readFile(menuFile);
+      let worksheet: any;
+      for (let t = 0; t < data.worksheets.length; t++) {
+        worksheet = data.worksheets[t];
+        if (worksheet) {
+          break;
+        }
+      }
+
+      const MAX_LEVEL = 4; // Max column level to reach menu items in plan.
+      // Iterates over all items.
+
+      let rows = worksheet._rows;
+      rows.length = 24;
+      let lastLevel = 0;
+      let subjects = { children: [] };
+      let childrenNode = subjects.children;
+      let activeObj = null;
+
+      let activeChildrenGivenLevel = [childrenNode];
+
+      await asyncPromise.eachSeries(rows, async row => {
+        if (!row) return;
+        let menu;
+
+        // Detect menu level by skipping blank cells on left.
+        
+        let level;
+        for (level = 0; level < MAX_LEVEL; level++) {
+          menu = row._cells[level];
+          if (menu && menu.text) {
+            break;
+          }
+        }
+
+        // Tree hierarchy calculation.        
+
+         if (level > lastLevel) {
+          childrenNode = activeObj.children;
+        } else if (level < lastLevel) {
+          childrenNode = activeChildrenGivenLevel[level];
+        }
+
+        /// Keeps the record of last subroots for each level, to
+        // changel levels greater than one (return to main menu),
+        // can exists between leaf nodes and roots.
+
+        activeChildrenGivenLevel[level] = childrenNode;
+
+        // Insert the object into JSON.
+
+        activeObj = {
+          title: menu,
+          description: row._cells[level + 1],
+          id: menu,
+          children: []
+        };
+        activeChildrenGivenLevel[level].push(activeObj);
+
+        lastLevel = level;
+      });
+
+      subjectsLoaded = subjects;
+    } else {
+      subjectsLoaded = JSON.parse(Fs.readFileSync(filename, 'utf8'));
+    }
 
     const doIt = async (subjects: GuaribasSubject[], parentSubjectId: number) => {
       return asyncPromise.eachSeries(subjects, async item => {
@@ -775,8 +973,6 @@ export class KBService implements IGBKBService {
    */
   public async deployKb(core: IGBCoreService, deployer: GBDeployer, localPath: string, min: GBMinInstance) {
     const packageName = Path.basename(localPath);
-    GBLog.info(`[GBDeployer] Opening package: ${localPath}`);
-
     const instance = await core.loadInstanceByBotId(min.botId);
     GBLog.info(`[GBDeployer] Importing: ${localPath}`);
     const p = await deployer.deployPackageToStorage(instance.instanceId, packageName);
@@ -788,6 +984,18 @@ export class KBService implements IGBKBService {
     min['groupCache'] = await KBService.getGroupReplies(instance.instanceId);
     await KBService.RefreshNER(min);
 
+    GBLog.info(`[GBDeployer] Opening package: ${localPath}`);
+    const html = await GBSSR.getHTML(min);
+    const path = Path.join(
+      process.env.PWD,
+      'work',
+      `${min.instance.botId}.gbai`,
+      `${min.instance.botId}.gbui`,
+      'index.html'
+    );
+    GBLogEx.info(min, `[GBDeployer] Generating SSR HTML in ${path}.`);
+    Fs.writeFileSync(path, html, 'utf8');
+
     GBLog.info(`[GBDeployer] Finished import of ${localPath}`);
   }
 

packages/whatsapp.gblib/services/WhatsappDirectLine.ts

@@ -43,11 +43,13 @@ import { Messages } from '../strings.js';
 import { GuaribasUser } from '../../security.gbapp/models/index.js';
 import { GBMinService } from '../../core.gbapp/services/GBMinService.js';
 import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js';
-import * as wpp from 'whatsapp-web.js';
 import qrcode from 'qrcode-terminal';
 import express from 'express';
 import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
 import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
+import { method } from 'lodash';
+import pkg from 'whatsapp-web.js';
+const { Buttons, Client, MessageMedia } = pkg;
 
 /**
  * Support for Whatsapp.
@@ -101,6 +103,8 @@ export class WhatsappDirectLine extends GBService {
         ? 'GeneralBots'
         : whatsappServiceNumber.indexOf(';') > -1
         ? 'maytapi'
+        : whatsappServiceKey !== 'internal'
+        ? 'graphapi'
         : 'chatapi';
     this.groupId = groupId;
   }
@@ -111,6 +115,14 @@ export class WhatsappDirectLine extends GBService {
     }
   }
 
+  public async sendButton() {
+    let url = 'https://wwebjs.dev/logo.png';
+    const media = await MessageMedia.fromUrl(url);
+    media.mimetype = 'image/png';
+    media.filename = 'hello.png';
+    let btnClickableMenu = new Buttons(media as any, [{ id: 'customId', body: 'button1' }, { body: 'button2' }]);
+    await this.sendToDevice("5521996049063",btnClickableMenu as any,null)
+  }
   public async setup(setUrl: boolean) {
     this.directLineClient = new Swagger({
       spec: JSON.parse(Fs.readFileSync('directline-3.0.json', 'utf8')),
@@ -120,136 +132,106 @@ export class WhatsappDirectLine extends GBService {
     let url: string;
     let body: any;
 
-    client.clientAuthorizations.add(
+    /*client.clientAuthorizations.add(
       'AuthorizationBotConnector',
       new Swagger.ApiKeyAuthorization('Authorization', `Bearer ${this.directLineSecret}`, 'header')
-    );
+    );*/
     let options: any;
+    const phoneId = this.whatsappServiceNumber.split(';')[0];
 
     switch (this.provider) {
       case 'GeneralBots':
-        const minBoot = GBServer.globals.minBoot as any;
+        const minBoot = GBServer.globals.minBoot;
-
+        // TODO:  REMOVE THIS.
-        // Initialize the browser using a local profile for each bot.
+        if (!setUrl) {
-
+          this.customClient = minBoot.whatsAppDirectLine.customClient;
-        const gbaiName = `${this.min.botId}.gbai`;
+        } else {
-        const localName = Path.join('work', gbaiName, 'profile');
+          // Initialize the browser using a local profile for each bot.
-
+          const gbaiName = `${this.min.botId}.gbai`;
-        const createClient = async browserWSEndpoint => {
+          const localName = Path.join('work', gbaiName, 'profile');
-          let puppeteer: any = {
+          const createClient = async browserWSEndpoint => {
-            headless: false,
+            let puppeteer = { headless: false, args: ['--no-sandbox', '--disable-dev-shm-usage'] };
-            args: [
+            if (browserWSEndpoint) {
-              '--no-sandbox',
+              // puppeteer.browserWSEndpoint = browserWSEndpoint ;
-              '--disable-setuid-sandbox',
+            }
-              '--disable-dev-shm-usage',
+            const client = (this.customClient = new Client({
-              '--disable-accelerated-2d-canvas',
+              puppeteer: puppeteer
-              '--no-first-run',
+            }));
-              '--no-zygote',
+            client.on(
-              '--single-process',
+              'message',
-              '--disable-gpu',
+              (async message => {
-              '--disable-infobars',
+                await this.WhatsAppCallback(message, null);
-              '--disable-features=site-per-process',
-              `--user-data-dir=${localName}`
-            ]
-          };
-          if (browserWSEndpoint) {
-            puppeteer = { browserWSEndpoint: browserWSEndpoint };
-          }
-
-          const client = (this.customClient = new wpp.Client({
-            authStrategy: new wpp.LocalAuth({
-              clientId: this.min.botId,
-              dataPath: localName
-            }),
-            puppeteer: puppeteer
-          }));
-
-          client.on(
-            'message',
-            (async (message: string) => {
-              await this.WhatsAppCallback(message, null);
-            }).bind(this)
-          );
-
-          client.on(
-            'qr',
-            (async qr => {
-              const adminNumber = this.min.core.getParam(this.min.instance, 'Bot Admin Number', null);
-              const adminEmail = this.min.core.getParam(this.min.instance, 'Bot Admin E-mail', null);
-
-              // Sends QR Code to boot bot admin.
-
-              const msg = `Please, scan QR Code with for bot ${this.botId}.`;
-              GBLog.info(msg);
-              qrcode.generate(qr, { small: true, scale: 0.5 });
-
-              // While handling other bots uses boot instance of this class to send QR Codes.
-
-              const s = new DialogKeywords(this.min, null, null);
-              const qrBuf = await s.getQRCode(qr);
-              const gbaiName = `${this.min.botId}.gbai`;
-              const localName = Path.join(
-                'work',
-                gbaiName,
-                'cache',
-                `qr${GBAdminService.getRndReadableIdentifier()}.png`
-              );
-              Fs.writeFileSync(localName, qrBuf);
-              const url = urlJoin(GBServer.globals.publicAddress, this.min.botId, 'cache', Path.basename(localName));
-              GBServer.globals.minBoot.whatsAppDirectLine.sendFileToDevice(
-                adminNumber,
-                url,
-                Path.basename(localName),
-                msg
-              );
-
-              s.sendEmail({pid: 0, to: adminEmail, subject: `Check your WhatsApp for bot ${this.botId}`, body: msg });
-            }).bind(this)
-          );
-
-          client.on('authenticated', async () => {
-            this.browserWSEndpoint = client.pupBrowser.wsEndpoint();
-            GBLog.verbose(`GBWhatsApp: QR Code authenticated for ${this.botId}.`);
-          });
-
-          client.on('ready', async () => {
-            client.pupBrowser.on(
-              'disconnected',
-              (async () => {
-                GBLog.info(`Browser terminated. Restarting ${this.min.botId} WhatsApp native provider.`);
-                await createClient.bind(this)(null);
               }).bind(this)
             );
-
+            client.on(
-            GBLog.verbose(`GBWhatsApp: Emptying chat list for ${this.botId}...`);
+              'qr',
-
+              (async qr => {
-            // Keeps the chat list cleaned.
+                const adminNumber = this.min.core.getParam(this.min.instance, 'Bot Admin Number', null);
-
+                const adminEmail = this.min.core.getParam(this.min.instance, 'Bot Admin E-mail', null);
-            const chats = await client.getChats();
+                // Sends QR Code to boot bot admin.
-            await CollectionUtil.asyncForEach(chats, async chat => {
+                const msg = `Please, scan QR Code with for bot ${this.botId}.`;
-              const sleep = (ms: number) => {
+                GBLog.info(msg);
-                return new Promise(resolve => {
+                qrcode.generate(qr, { small: true, scale: 0.5 });
-                  setTimeout(resolve, ms);
+                // While handling other bots uses boot instance of this class to send QR Codes.
-                });
+                // const s = new DialogKeywords(min., null, null, null);
-              };
+                // const qrBuf = await s.getQRCode(qr);
-              const wait = Math.floor(Math.random() * 5000) + 1000;
+                // const gbaiName = `${this.min.botId}.gbai`;
-              await sleep(wait);
+                // const localName = Path.join('work', gbaiName, 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png`);
-              if (chat.isGroup) {
+                // fs.writeFileSync(localName, qrBuf);
-                await chat.clearMessages();
+                // const url = urlJoin(
-              } else if (!chat.pinned) {
+                //   GBServer.globals.publicAddress,
-                await chat.delete();
+                //   this.min.botId,
-              }
+                //   'cache',
+                //   Path.basename(localName)
+                // );
+                // GBServer.globals.minBoot.whatsAppDirectLine.sendFileToDevice(adminNumber, url, Path.basename(localName), msg);
+                // s.sendEmail(adminEmail, `Check your WhatsApp for bot ${this.botId}`, msg);
+              }).bind(this)
+            );
+            client.on('authenticated', async () => {
+              this.browserWSEndpoint = client.pupBrowser.wsEndpoint();
+              GBLog.verbose(`GBWhatsApp: QR Code authenticated for ${this.botId}.`);
             });
-          });
+            client.on('ready', async () => {
-
+              GBLog.verbose(`GBWhatsApp: Emptying chat list for ${this.botId}...`);
-          client.initialize();
+              // Keeps the chat list cleaned.
-        };
+              const chats = await client.getChats();
-        await createClient.bind(this)(this.browserWSEndpoint);
+              await CollectionUtil.asyncForEach(chats, async chat => {
-
+                const sleep = ms => {
-        setUrl = false;
+                  return new Promise(resolve => {
-
+                    setTimeout(resolve, ms);
+                  });
+                };
+                const wait = Math.floor(Math.random() * 5000) + 1000;
+                await sleep(wait);
+                if (chat.isGroup) {
+                  await chat.clearMessages();
+                } else if (!chat.pinned) {
+                  await chat.delete();
+                }
+              });
+            });
+            client.initialize();
+          };
+          await createClient.bind(this)(this.browserWSEndpoint);
+          setUrl = false;
+        }
+        break;
+      case 'chatapi':
+        options = {
+          method: 'POST',
+          url: urlJoin(this.whatsappServiceUrl, 'webhook'),
+          timeout: 10000,
+          qs: {
+            token: this.whatsappServiceKey,
+            webhookUrl: `${GBServer.globals.publicAddress}/webhooks/whatsapp/${this.botId}`,
+            set: true
+          },
+          headers: {
+            'cache-control': 'no-cache'
+          }
+        };
         break;
-
       case 'chatapi':
         url = urlJoin(this.whatsappServiceUrl, 'webhook');
         options = {
@@ -268,7 +250,6 @@ export class WhatsappDirectLine extends GBService {
 
         break;
       case 'maytapi':
-        let phoneId = this.whatsappServiceNumber.split(';')[0];
         let productId = this.whatsappServiceNumber.split(';')[1];
         url = `${this.INSTANCE_URL}/${productId}/${phoneId}/config`;
         body = {
@@ -327,7 +308,7 @@ export class WhatsappDirectLine extends GBService {
   }
 
   public static providerFromRequest(req: any) {
-    return req.body.messages ? 'chatapi' : req.body.message ? 'maytapi' : 'GeneralBots';
+    return req.body.messages ? 'chatapi' : req.body.message ? 'maytapi' : req.body.message ? 'graphapi' : 'GeneralBots';
   }
 
   public async received(req, res) {
@@ -379,6 +360,8 @@ export class WhatsappDirectLine extends GBService {
         }
 
         break;
+      case 'graphapi':
+        break;
 
       case 'maytapi':
         message = req.body.message;
@@ -621,7 +604,7 @@ export class WhatsappDirectLine extends GBService {
     }
   }
 
-  private async endTransfer(id: any, locale: string, user: GuaribasUser, agent: GuaribasUser, sec: SecService) {
+  private async endTransfer(id: string, locale: string, user: GuaribasUser, agent: GuaribasUser, sec: SecService) {
     await this.sendToDeviceEx(id, Messages[this.locale].notify_end_transfer(this.min.instance.botId), locale, null);
 
     if (user.agentSystemId.charAt(2) === ':') {
@@ -643,7 +626,7 @@ export class WhatsappDirectLine extends GBService {
     await sec.updateHumanAgent(id, this.min.instance.instanceId, null);
   }
 
-  public inputMessage(client, conversationId, text, from, fromName, group, attachments) {
+  public inputMessage(client, conversationId: string, text: string, from, fromName: string, group, attachments: File) {
     return client.Conversations.Conversations_PostActivity({
       conversationId: conversationId,
       activity: {
@@ -738,7 +721,7 @@ export class WhatsappDirectLine extends GBService {
     let options;
     switch (this.provider) {
       case 'GeneralBots':
-        const attachment = await wpp.MessageMedia.fromUrl(url);
+        const attachment = await MessageMedia.fromUrl(url);
         if (to.indexOf('@') == -1) {
           if (to.length == 18) {
             to = to + '@g.us';
@@ -794,7 +777,24 @@ export class WhatsappDirectLine extends GBService {
         };
 
         break;
+
+      case 'graphapi':
+        url = `https://graph.facebook.com/v15.0/${phoneId}/messages`;
+        options = {
+          method: 'POST',
+          timeout: 10000,
+          headers: {
+            token: `Bearer `,
+            'Content-Type': 'application/json'
+          },
+          body: {
+            messaging_product: 'whatsapp',
+            recipient_type: 'individual',
+            to: phoneId
+          }
+        };
     }
+
     if (options) {
       try {
         // tslint:disable-next-line: await-promise
@@ -810,7 +810,7 @@ export class WhatsappDirectLine extends GBService {
     let options;
     switch (this.provider) {
       case 'GeneralBots':
-        const attachment = wpp.MessageMedia.fromUrl(url);
+        const attachment = MessageMedia.fromUrl(url);
         await this.customClient.sendMessage(to, attachment);
 
         break;
@@ -845,7 +845,7 @@ export class WhatsappDirectLine extends GBService {
     }
   }
 
-  public async sendTextAsAudioToDevice(to, msg, chatId) {
+  public async sendTextAsAudioToDevice(to, msg: string, chatId) {
     const url = await GBConversationalService.getAudioBufferFromText(msg);
 
     await this.sendFileToDevice(to, url, 'Audio', msg, chatId);
@@ -906,6 +906,7 @@ export class WhatsappDirectLine extends GBService {
             }
           };
           break;
+        case 'graphapi':
       }
 
       if (options) {

src/RootData.ts

@@ -29,6 +29,7 @@
 | our trademarks remain entirely with us.                                     |
 |                                                                             |
 \*****************************************************************************/
+
 /**
  * @fileoverview General Bots server core.
  */
@@ -41,10 +42,12 @@ import { GBMinService } from '../packages/core.gbapp/services/GBMinService.js';
  */
 
 export class RootData {
+  public webSessions: {}  // List of Web Automation sessions.
   public processes: {}; // List of .gbdialog active executions.
   public files: {}; // List of uploaded files handled.
   public publicAddress: string; // URI for BotServer.
   public server: any; // Express reference.
+  public httpsServer: any; // Express reference (HTTPS).
   public sysPackages: any[]; // Loaded system package list.
   public appPackages: any[]; // Loaded .gbapp package list.
   public minService: GBMinService; // Minimalist service core.

src/app.ts

@@ -55,6 +55,7 @@ import auth from 'basic-auth';
 import child_process from 'child_process';
 import * as winston from 'winston-logs-display';
 import { RootData } from './RootData.js';
+import { GBSSR } from '../packages/core.gbapp/services/GBSSR.js';
 
 /**
  * General Bots open-core entry point.
@@ -84,6 +85,8 @@ export class GBServer {
     const server = express();
 
     GBServer.globals.server = server;
+    GBServer.globals.httpsServer = null;
+    GBServer.globals.webSessions = {};
     GBServer.globals.processes = {};
     GBServer.globals.files = {};
     GBServer.globals.appPackages = [];
@@ -126,7 +129,7 @@ export class GBServer {
           const azureDeployer: AzureDeployerService = await AzureDeployerService.createInstance(deployer);
           const adminService: GBAdminService = new GBAdminService(core);
 
-          if (process.env.NODE_ENV === 'development' ) {
+          if (process.env.NODE_ENV === 'development') {
             const proxy = GBConfigService.get('BOT_URL');
             if (proxy !== undefined) {
               GBServer.globals.publicAddress = proxy;
@@ -228,6 +231,10 @@ export class GBServer {
             winston.default(server, loggers[1]);
           }
 
+          server.get('*', async (req, res, next) => {
+            await GBSSR.ssrFilter(req, res, next);
+          });
+
           GBLog.info(`The Bot Server is in RUNNING mode...`);
 
           // Opens Navigator.
@@ -250,7 +257,7 @@ export class GBServer {
         pfx: Fs.readFileSync(process.env.CERTIFICATE_PFX)
       };
       const httpsServer = https.createServer(options1, server).listen(port, mainCallback);
-
+      GBServer.globals.httpsServer = httpsServer;
       if (process.env.CERTIFICATE2_PFX) {
         const options2 = {
           passphrase: process.env.CERTIFICATE2_PASSPHRASE,

suppress-node-warnings.cjs

@@ -0,0 +1,11 @@
+// inspired by 
+// https://github.com/nodejs/node/issues/30810#issuecomment-1383184769
+const { emit: originalEmit } = process;
+
+function suppresser(event, error) {
+  return event === 'warning' && error.name === 'ExperimentalWarning'
+    ? false
+    : originalEmit.apply(process, arguments);
+}
+
+process.emit = suppresser;

swagger.json

@@ -13,7 +13,7 @@
         "status": "Production"
       }
     },
-    "host": "generalbots.ai",
+    "host": "f993e4828c0d50.lhr.life",
     "basePath": "/",
     "schemes": [
       "https"
@@ -21,34 +21,20 @@
     "consumes": [],
     "produces": [],
     "paths": {
-      "/api/talkTo": {
+      "/api/v2/dev-perdomo/dialog/talk": {
         "post": {
+          "consumes":[
+            "text/plain; charset=utf-8"
+          ],
           "summary": "Talk to the user.",
           "description": "Talk to the user.",
           "x-ms-no-generic-test": true,
-          "operationId": "talkTo",
+          "operationId": "talk",
-          "consumes": [
-            "application/json"
-          ],
-          "produces": [
-            "application/json"
-          ],
-          "parameters": [
-            {
-              "name": "content",
-              "in": "body",
-              "required": true,
-              "schema": {
-                "$ref": "#/definitions/TalkRequest"
-              }
-            }
-          ],
           "responses": {
             "200": {
               "description": "OK"
             }
-          },
+          }
-          "deprecated": false
         }
       }   
     },
@@ -58,9 +44,6 @@
           "properties": {
             "pid": {
               "type": "string"
-            },
-            "mobile": {
-                "type": "string"
               },
             "text": {
               "type": "string"

tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+   
     "allowJs": true,
     "downlevelIteration": true,
     "baseUrl": "./",