new(core.gbapp): FIND BASIC keyword is now much more faster.

package.json

@@ -67,9 +67,7 @@
     "botbuilder-dialogs": "4.7.0",
     "botframework-connector": "4.7.0",
     "botlib": "1.5.1",
-    "chai": "4.2.0",
     "cli-spinner": "0.2.10",
-    "csv-parse": "4.8.3",
     "dotenv-extended": "2.7.1",
     "exceljs": "3.5.0",
     "express": "4.17.1",
@@ -79,12 +77,10 @@
     "js-beautify": "1.10.2",
     "marked": "0.8.0",
     "microsoft-cognitiveservices-speech-sdk": "1.11.0",
-    "mocha": "6.2.2",
     "ms-rest-azure": "3.0.0",
     "nexmo": "2.5.2",
     "ngrok": "3.2.7",
     "npm": "6.13.4",
-    "officeparser": "^2.1.1",
     "opn": "6.0.0",
     "pragmatismo-io-framework": "1.0.20",
     "prism-media": "1.2.1",
@@ -94,27 +90,24 @@
     "request-promise": "4.2.5",
     "request-promise-native": "1.0.8",
     "rimraf": "3.0.0",
-    "safe-buffer": "^5.2.0",
+    "safe-buffer": "5.2.0",
     "scanf": "1.1.1",
     "sequelize": "5.21.5",
     "sequelize-typescript": "1.1.0",
-    "shx": "0.3.2",
     "simple-git": "1.129.0",
     "sppull": "2.5.1",
     "strict-password-generator": "1.1.2",
     "swagger-client": "2.1.18",
     "tedious": "6.6.5",
-    "typedoc": "0.15.6",
+    "textract": "2.5.0",
     "typescript": "3.7.4",
     "url-join": "4.0.1",
     "vbscript-to-typescript": "1.0.8",
-    "wait-until": "0.0.2",
     "walk-promise": "0.2.0",
     "washyourmouthoutwithsoap": "1.0.2"
   },
   "devDependencies": {
-    "@types/chai": "4.2.7",
+    "typedoc": "0.15.6",
-    "@types/mocha": "5.2.7",
     "@types/url-join": "4.0.0",
     "@types/winston": "2.4.4",
     "ban-sensitive-files": "1.9.2",

packages/admin.gbapp/dialogs/AdminDialog.ts

@@ -154,21 +154,22 @@ export class AdminDialog extends IGBDialog {
     min.dialogs.add(
       new WaterfallDialog('/publish', [
         async step => {
+          const botId = min.instance.botId;
           const locale = step.context.activity.locale;
           await step.context.sendActivity(Messages[locale].working('Publishing'));
-          let unknownCommand = false;
+          
           step.activeDialog.state.options.args = (step.options as any).args;
           let args = step.activeDialog.state.options.args.split(' ');
           let filename = args[0];
           const packages = [];
           if (filename === null) {
             await step.context.sendActivity(`Starting publishing for all bot packages...`);
-            packages.push(`${min.instance.botId}.gbkb`);
+            packages.push(`${botId}.gbkb`);
-            packages.push(`${min.instance.botId}.gbdialog`);
+            packages.push(`${botId}.gbdialog`);
-            packages.push(`${min.instance.botId}.gbot`);
+            packages.push(`${botId}.gbot`);
-            packages.push(`${min.instance.botId}.gbtheme`);
+            packages.push(`${botId}.gbtheme`);
-            packages.push(`${min.instance.botId}.gbapp`);
+            packages.push(`${botId}.gbapp`);
-            packages.push(`${min.instance.botId}.gblib`);
+            packages.push(`${botId}.gblib`);
           } else {
             await step.context.sendActivity(`Starting publishing for ${filename}...`);
             packages.push(filename);
@@ -178,7 +179,7 @@ export class AdminDialog extends IGBDialog {
 
             await CollectionUtil.asyncForEach(packages, async packageName => {
 
-              const cmd1 = `deployPackage ${process.env.STORAGE_SITE} /${process.env.STORAGE_LIBRARY}/${min.instance.botId}.gbai/${packageName}`;
+              const cmd1 = `deployPackage ${process.env.STORAGE_SITE} /${process.env.STORAGE_LIBRARY}/${botId}.gbai/${packageName}`;
 
               if (await (deployer as any).getStoragePackageByName(min.instance.instanceId,
                 packageName) !== null) { // TODO: Move to interface.
@@ -193,7 +194,7 @@ export class AdminDialog extends IGBDialog {
               await step.context.sendActivity(`Finished publishing ${packageName}.`);
             });
 
-            return await step.replaceDialog('/ask', { firstRun: false });
+            return await step.replaceDialog('/ask', { isReturning: true });
 
           } catch (error) {
             await step.context.sendActivity(error.message);

packages/core.gbapp/services/GBAPIService.ts

@@ -42,7 +42,7 @@ import { AzureDeployerService } from '../../azuredeployer.gbapp/services/AzureDe
 import { GBDeployer } from './GBDeployer';
 const MicrosoftGraph = require("@microsoft/microsoft-graph-client");
 import { Messages } from "../strings";
-import { sys } from 'typescript';
+import { GBServer } from '../../../src/app';
 const request = require('request-promise-native');
 
 /**
@@ -160,69 +160,64 @@ class SysClass {
 
     const path = `/${botId}.gbai/${botId}.gbdata`;
 
-    let res = await client.api(
+    try {
-      `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/root:${path}:/children`)
-      .get();
 
 
-    // Performs validation.
+      let res = await client.api(
-
+        `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/root:${path}:/children`)
-    let document = res.value.filter(m => {
+        .get();
-      return m.name === file
-    });
 
 
-    if (document === undefined) {
+      // Performs validation.
-      throw `File '${file}' specified on save GBasic command FIND not found. Check the .gbdata or the .gbdialog associated.`;
-    }
-    if (args.length > 1) {
-      throw `File '${file}' has a FIND call with more than 1 arguments. Check the .gbdialog associated.`;
-    }
 
-    // Creates workbook session that will be discarded.
+      let document = res.value.filter(m => {
+        return m.name === file
+      });
 
-    const columnName = "CPF";
+      if (document === undefined) {
-    const value = "99988877766";
+        throw `File '${file}' specified on save GBasic command FIND not found. Check the .gbdata or the .gbdialog associated.`;
-
+      }
-    let body = { "persistChanges": false };
+      if (args.length > 1) {
-
+        throw `File '${file}' has a FIND call with more than 1 arguments. Check the .gbdialog associated.`;
-    const session = await client.api(
-      `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/items/${document[0].id}/workbook/createSession`)
-      .post(body);
-
-    // Applies filtering.
-
-    const bodyFilter = {
-      criteria:
-      {
-        filterOn: "Custom",
-        criterion1: `=${value}`
       }
-    };
 
-    await client.api(
+      // Creates workbook session that will be discarded.
-      `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/items/${document[0].id}/workbook/worksheets('Sheet1')/tables(id='Table1')/columns('${columnName}')/filter/apply`)
-      .headers("workbook-session-id", session.id)
-      .post(bodyFilter);
 
-    // Get the filtered values.
+      const filter = args[0].split('=');
+      const columnName = filter[0];
+      const value = filter[1];
+      let results = await client.api(
+        `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/items/${document[0].id}/workbook/worksheets('Sheet1')/range(address='A1:Z100')`)
+        .get();
 
-    let results = await client.api(
+      let columnIndex = 0;
-      `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/items/${document[0].id}/workbook/worksheets('Sheet1')/tables('Table1')/range/visibleView/rows?$select=values`)
+      const header = results.text[0];
-      .headers("workbook-session-id", session.id)
+      for (; columnIndex < header.length; columnIndex++) {
-      .get();
+        if (header[columnIndex] === columnName) {
+          break;
+        }
+      }
 
-
+      let foundIndex = 0;
-      // Translate an array into a readable BASIC object.
+      for (; foundIndex < results.text.length; foundIndex++) {
-
+        if (results.text[foundIndex][columnIndex] === value) {
-      let output = {};         
+          break;
-      results.value[1].values[0]
+        }
-      const firstRow = results.value[0];
+      }
-      for (let index = 0; index < firstRow.values[0].length; index++) {
+      if (foundIndex === results.text.length) {
-        output[firstRow.values[0][index]] = results.value[1].values[0][index];
+        return null;
-      }      
+      }
-
+      else {
-    return output;
+        let output = {};
+        const row = results.text[foundIndex];
+        for (let colIndex = 0; colIndex < row.length; colIndex++) {
+          output[header[colIndex]] = row[colIndex];
+        }
+        return output;
+      }
+    } catch (error) {
+      GBLog.error(error);
+    }
   }
 
   public generatePassword() {
@@ -351,6 +346,12 @@ export class DialogClass {
     }
   }
 
+  public async sendFile(step, filename, caption) {
+    let url = urlJoin(GBServer.globals.publicAddress, 'kb', this.min.botId + '.gbkb', 'assets', filename);
+    await this.min.conversationalService.sendFile(this.min, step,
+      null, url, caption);
+  }
+
   public async getFrom(step) {
     return step.context.activity.from.id;
   }

packages/core.gbapp/services/GBConversationalService.ts

@@ -91,10 +91,14 @@ export class GBConversationalService {
   public async sendFile(min: GBMinInstance, step: GBDialogStep, mobile: string, url: string, caption: string): Promise<any> {
     if (step !== null) {
       mobile = step.context.activity.from.id;
+      if (step.context.activity.channelId === 'whatsapp') {
+        const filename = url.substring(url.lastIndexOf('/') + 1);
+        await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename, caption);
+      }
+      else {
+        await step.context.sendActivity(url);
+      }
     }
-    const filename = url.substring(url.lastIndexOf('/') + 1);
-    await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename, caption);
-
   }
 
   public async sendAudio(min: GBMinInstance, step: GBDialogStep, url: string): Promise<any> {

packages/core.gbapp/services/GBDeployer.ts

@@ -39,7 +39,6 @@
 const Path = require('path');
 import urlJoin = require('url-join');
 const Fs = require('fs');
-const WaitUntil = require('wait-until');
 const express = require('express');
 const child_process = require('child_process');
 const graph = require('@microsoft/microsoft-graph-client');
@@ -92,67 +91,63 @@ export class GBDeployer implements IGBDeployer {
   public async deployPackages(core: IGBCoreService, server: any, appPackages: IGBPackage[]) {
     const _this = this;
 
-    return new Promise(
+
-      async (resolve: any, reject: any) => {
+    GBLog.info(`PWD ${process.env.PWD}...`);
-        GBLog.info(`PWD ${process.env.PWD}...`);
+    let totalPackages = 0;
-        let totalPackages = 0;
+    let paths = [urlJoin(process.env.PWD, GBDeployer.deployFolder), urlJoin(process.env.PWD, GBDeployer.workFolder)];
-        let paths = [urlJoin(process.env.PWD, GBDeployer.deployFolder), urlJoin(process.env.PWD, GBDeployer.workFolder)];
+    const additionalPath = GBConfigService.get('ADDITIONAL_DEPLOY_PATH');
-        const additionalPath = GBConfigService.get('ADDITIONAL_DEPLOY_PATH');
+    if (additionalPath !== undefined && additionalPath !== '') {
-        if (additionalPath !== undefined && additionalPath !== '') {
+      paths = paths.concat(additionalPath.toLowerCase().split(';'));
-          paths = paths.concat(additionalPath.toLowerCase().split(';'));
+    }
+    const botPackages: string[] = [];
+    const gbappPackages: string[] = [];
+    let generalPackages: string[] = [];
+
+    async function scanPackageDirectory(path) {
+      const isDirectory = source => Fs.lstatSync(source).isDirectory();
+      const getDirectories = source =>
+        Fs.readdirSync(source)
+          .map(name => Path.join(source, name))
+          .filter(isDirectory);
+
+      const dirs = getDirectories(path);
+      await CollectionUtil.asyncForEach(dirs, async element => {
+        if (element === '.') {
+          GBLog.info(`Ignoring ${element}...`);
+        } else {
+          if (element.endsWith('.gbot')) {
+            botPackages.push(element);
+          } else if (element.endsWith('.gbapp')) {
+            gbappPackages.push(element);
+          } else {
+            generalPackages.push(element);
+          }
         }
-        const botPackages: string[] = [];
+      });
-        const gbappPackages: string[] = [];
+    }
-        let generalPackages: string[] = [];
 
-        async function scanPackageDirectory(path) {
+    GBLog.info(`Starting looking for packages (.gbot, .gbtheme, .gbkb, .gbapp)...`);
-          const isDirectory = source => Fs.lstatSync(source).isDirectory();
+    await CollectionUtil.asyncForEach(paths, async e => {
-          const getDirectories = source =>
+      GBLog.info(`Looking in: ${e}...`);
-            Fs.readdirSync(source)
+      await scanPackageDirectory(e);
-              .map(name => Path.join(source, name))
+    });
-              .filter(isDirectory);
 
-          const dirs = getDirectories(path);
+    // Deploys all .gbapp files first.
-          await CollectionUtil.asyncForEach(dirs, async element => {
-            if (element === '.') {
-              GBLog.info(`Ignoring ${element}...`);
-            } else {
-              if (element.endsWith('.gbot')) {
-                botPackages.push(element);
-              } else if (element.endsWith('.gbapp')) {
-                gbappPackages.push(element);
-              } else {
-                generalPackages.push(element);
-              }
-            }
-          });
-        }
 
-        GBLog.info(`Starting looking for packages (.gbot, .gbtheme, .gbkb, .gbapp)...`);
+    await this.deployAppPackages(gbappPackages, core, appPackages);
-        await CollectionUtil.asyncForEach(paths, async e => {
-          GBLog.info(`Looking in: ${e}...`);
-          await scanPackageDirectory(e);
-        });
 
-        // Deploys all .gbapp files first.
+    GBLog.info(`App Package deployment done.`);
 
-        const appPackagesProcessed = await this.deployAppPackages(gbappPackages, core, appPackages);
+    ({ generalPackages, totalPackages } = await this.deployDataPackages(
 
-        GBLog.info(`App Package deployment done.`);
+      core,
+      botPackages,
+      _this,
+      generalPackages,
+      server,
+      totalPackages
+    ));
 
-        ({ generalPackages, totalPackages } = await this.deployDataPackages(
-
-          core,
-          botPackages,
-          _this,
-          generalPackages,
-          server,
-          reject,
-          totalPackages,
-          resolve
-        ));
-      }
-    );
   }
 
   public async deployBlankBot(botId: string) {
@@ -317,7 +312,7 @@ export class GBDeployer implements IGBDeployer {
 
       case '.gbtheme':
         const packageName = Path.basename(localPath);
-        GBServer.globals.server.use(`/themes/${packageName}`, express.static(packageName));
+        GBServer.globals.server.use(`/themes/${packageName}`, express.static(localPath));
         GBLog.info(`Theme (.gbtheme) assets accessible at: /themes/${packageName}.`);
 
         break;
@@ -446,9 +441,8 @@ export class GBDeployer implements IGBDeployer {
     _this: this,
     generalPackages: string[],
     server: any,
-    reject: any,
+    totalPackages: number
-    totalPackages: number,
+
-    resolve: any
   ) {
     try {
       await core.syncDatabaseStructure();
@@ -489,27 +483,13 @@ export class GBDeployer implements IGBDeployer {
       } else {
         // Unknown package format.
         const err = new Error(`Package type not handled: ${filename}.`);
-        reject(err);
+        throw err;
+
       }
       totalPackages++;
     });
 
-    WaitUntil()
+    GBLog.info(`Package deployment done.`);
-      .interval(100)
-      .times(5)
-      .condition(cb => {
-        GBLog.info(`Waiting for package deployment...`);
-        cb(totalPackages === generalPackages.length);
-      })
-      .done(() => {
-        if (botPackages.length === 0) {
-          GBLog.info('Use ADDITIONAL_DEPLOY_PATH to point to a .gbai package folder (no external packages).');
-        } else {
-          GBLog.info(`Package deployment done.`);
-        }
-        resolve();
-      });
-
     return { generalPackages, totalPackages };
   }
 

packages/core.gbapp/services/GBMinService.ts

@@ -210,7 +210,8 @@ export class GBMinService {
     const { min, adapter, conversationState } = await this.buildBotAdapter(instance, GBServer.globals.sysPackages);
     GBServer.globals.minInstances.push(min);
 
-    this.deployer.deployPackage(min, 'packages/default.gbdialog');
+    await this.deployer.deployPackage(min, 'packages/default.gbdialog');
+    await this.deployer.deployPackage(min, 'packages/default.gbtheme');
 
     // Install per bot deployed packages.
 

packages/core.gbapp/services/GBVMService.ts

@@ -43,10 +43,11 @@ const walkPromise = require('walk-promise');
 const vm = require('vm');
 import urlJoin = require('url-join');
 import { DialogClass } from './GBAPIService';
+import { Messages } from '../strings';
 //tslint:disable-next-line:no-submodule-imports
 const vb2ts = require('vbscript-to-typescript/dist/converter');
 const beautify = require('js-beautify').js;
-const parseOffice = require('bluebird').promisify(require('officeparser').parseOffice);
+var textract = require('textract');
 
 /**
  * @fileoverview Virtualization services for emulation of BASIC.
@@ -70,7 +71,7 @@ export class GBVMService extends GBService {
     return Promise.all(
       files.map(async file => {
 
-        let filename:string = file.name;
+        let filename: string = file.name;
 
         if (
           filename.endsWith('.vbs') ||
@@ -81,21 +82,19 @@ export class GBVMService extends GBService {
         ) {
 
           if (filename.endsWith('.docx')) {
-            const text = await parseOffice(filename);
+            let text = await this.getTextFromWord(folder, filename);
-
+            filename = filename.substr(0, filename.indexOf('docx')) + 'vbs';
-            filename = filename.substr(0, file.indexOf('docx')) + '.vbs';
-
             fs.writeFileSync(urlJoin(folder, filename), text);
-
           }
 
-          const mainName = filename.replace(/\-|\./g, '');
+          const mainName = filename.replace(/\s|\-/g, '').split('.')[0];
           min.scriptMap[filename] = mainName;
 
           const fullFilename = urlJoin(folder, filename);
-          fs.watchFile(fullFilename, async () => {
+          // TODO: Implement in development mode, how swap for .vbs files
-            await this.run(fullFilename, min, deployer, mainName);
+          // fs.watchFile(fullFilename, async () => {
-          });
+          //   await this.run(fullFilename, min, deployer, mainName);
+          // });
 
           await this.run(fullFilename, min, deployer, mainName);
         }
@@ -103,6 +102,20 @@ export class GBVMService extends GBService {
     );
   }
 
+  private async getTextFromWord(folder: string, filename: string) {
+    return new Promise<string>(async (resolve, reject) => {
+      textract.fromFileWithPath(urlJoin(folder, filename), { preserveLineBreaks: true },
+        (error, text) => {
+          if (error) {
+            reject(error);
+          }
+          else {
+            resolve(text);
+          }
+        });
+    });
+  }
+
   /**
    * Converts General Bots BASIC
    *
@@ -152,6 +165,9 @@ export class GBVMService extends GBService {
     code = code.replace(/(talk)(\s)(.*)/g, ($0, $1, $2, $3) => {
       return `talk (step, ${$3})\n`;
     });
+    code = code.replace(/(send file)(\s*)(.*)/g, ($0, $1, $2, $3) => {
+      return `sendFile (step, ${$3})\n`;
+    });
 
     code = code.replace(/(save)(\s)(.*)/g, ($0, $1, $2, $3) => {
       return `sys().save(${$3})\n`;
@@ -282,6 +298,10 @@ export class GBVMService extends GBService {
     code = code.replace(/("[^"]*"|'[^']*')|\baskEmail\b/g, ($0, $1) => {
       return $1 === undefined ? 'this.askEmail' : $1;
     });
+    code = code.replace(/("[^"]*"|'[^']*')|\bsendFile\b/g, ($0, $1) => {
+      return $1 === undefined ? 'this.sendFile' : $1;
+    });
+
 
     // await insertion.
 
@@ -305,9 +325,17 @@ export class GBVMService extends GBService {
           const promise = min.cbMap[cbId].promise;
 
           delete min.cbMap[cbId];
-          const opts = await promise(step, step.result);
+          try {
+            const opts = await promise(step, step.result);
 
-          return await step.replaceDialog('/hear', opts);
+            return await step.replaceDialog('/hear', opts);
+          } catch (error) {
+            GBLog.error(`Error running BASIC code: ${error}`);
+            const locale = step.context.activity.locale;
+            step.context.sendActivity(Messages[locale].very_sorry_about_error);
+
+            return await step.replaceDialog('/ask', { isReturning: true });
+          }
         }
       ])
     );

packages/core.gbapp/services/TSCompiler.ts

@@ -79,10 +79,15 @@ export class TSCompiler {
         const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n');
 
         if (diagnostic.file !== undefined) {
-          const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
+          if (
-          GBLog.error(`${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`);
+            diagnostic.file.fileName.indexOf('readable-stream') == -1 &&
+            diagnostic.file.fileName.indexOf('request-promise') == -1
+          ) {
+            const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
+            GBLog.warn(`BASIC error: ${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`);
+          }
         } else {
-          GBLog.error(`${message}`);
+          GBLog.warn(`BASIC error: ${message}`);
         }
       }
     });

packages/default.gbdialog/find-on-excel.vbs

@@ -1,4 +1,30 @@
-cpf = "999888777-66"
+talk "Para conseguir te ajudar, você quer informar o CPF ou código?"
-Linha = find "Cadastro.xlsx", "CPF=" + cpf 
+hear consulta
-talk Linha.Nome
 
+if consulta = "cpf" then
+    talk "Qual seu CPF?"
+    hear cpf
+    talk "Aguarde alguns instantes que eu localizo seu cadastro..."
+    row = find "Cadastro.xlsx", "CPF=" + cpf 
+    if row != null then
+        talk "Oi, " + row.Nome +  "Tudo bem? " 
+        talk "Seu código de cliente é " + row.Cod
+        talk "Vamos te enviar o pedido para seu endereço em: "  + row.Endereço
+        send file "boleta.pdf", "Pague já e evite multas!"
+    else
+        talk "Tente novamente."
+    end if
+else
+    talk "Qual seria seu código?"
+    hear cod
+    talk "Aguarde alguns instantes que eu localizo seu cadastro..."
+    row = find "Cadastro.xlsx", "Cod=" + cod
+    if row != null then
+        talk "Oi, " + row.Nome +  "Tudo bem? " 
+        talk "Seu CPF é " + row.CPF
+        talk "Vamos te enviar o pedido para seu endereço em: "  + row.Endereço
+        send file "boleta.pdf", "Pague já e evite multas!"
+    else
+        talk "Tente novamente."
+    end if
+end if

packages/kb.gbapp/services/KBService.ts

@@ -34,7 +34,6 @@
  * @fileoverview Knowledge base services and logic.
  */
 
-var Excel = require('exceljs');
 const Path = require('path');
 const Fs = require('fs');
 const urlJoin = require('url-join');
@@ -43,9 +42,8 @@ const path = require('path');
 const asyncPromise = require('async-promises');
 const walkPromise = require('walk-promise');
 // tslint:disable-next-line:newline-per-chained-call
-const parse = require('bluebird').promisify(require('csv-parse'));
 const { SearchService } = require('azure-search-client');
-
+var Excel = require('exceljs');
 import { IGBKBService, GBDialogStep, GBLog, IGBConversationalService, IGBCoreService, IGBInstance, GBMinInstance } from 'botlib';
 import { Op } from 'sequelize';
 import { Sequelize } from 'sequelize-typescript';