new(all): OCR and Web Manipulation.

package-lock.json

@@ -14448,6 +14448,11 @@
       "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz",
       "integrity": "sha1-WuVUHQJGRdMqWPzdyc7s6nrjrC8="
     },
+    "node-tesseract-ocr": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/node-tesseract-ocr/-/node-tesseract-ocr-2.2.1.tgz",
+      "integrity": "sha512-Q9cD79JGpPNQBxbi1fV+OAsTxYKLpx22sagsxSyKbu1u+t6UarApf5m32uVc8a5QAP1Wk7fIPN0aJFGGEE9DyQ=="
+    },
     "nodesecurity-npm-utils": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/nodesecurity-npm-utils/-/nodesecurity-npm-utils-6.0.0.tgz",

package.json

@@ -95,6 +95,7 @@
     "ms-rest-azure": "3.0.0",
     "nexmo": "2.9.1",
     "node-cron": "3.0.0",
+    "node-tesseract-ocr": "^2.2.1",
     "npm": "7.21.0",
     "opn": "6.0.0",
     "pdf-extraction": "1.0.2",

packages/basic.gblib/services/DialogKeywords.ts

@@ -42,8 +42,8 @@ import { SystemKeywords } from './SystemKeywords';
 import { GBMinService } from '../../core.gbapp/services/GBMinService';
 import { HubSpotServices } from '../../hubspot.gblib/services/HubSpotServices';
 import { WhatsappDirectLine } from '../../whatsapp.gblib/services/WhatsappDirectLine';
-var DateDiff = require('date-diff');
-
+const DateDiff = require('date-diff');
+const puppeteer = require('puppeteer');
 
 /**
  * Base services of conversation to be called by BASIC which
@@ -66,6 +66,16 @@ export class DialogKeywords {
    */
   public user;
 
+  /**
+   * HTML browser for conversation over page interaction.
+   */
+  browser: any;
+
+  /**
+   * The number used in this execution for HEAR calls (useful for SET SCHEDULE).
+   */
+   hrOn: string;
+
   /**
    * When creating this keyword facade, a bot instance is
    * specified among the deployer service.
@@ -74,6 +84,9 @@ export class DialogKeywords {
     this.min = min;
     this.user = user;
     this.internalSys = new SystemKeywords(min, deployer, this);
+    (async () => {
+      this.browser = await puppeteer.launch();
+    });
   }
 
   /**
@@ -84,6 +97,115 @@ export class DialogKeywords {
     return this.internalSys;
   }
 
+  /**
+   * Returns the page object.
+   *
+   * @example x = GET PAGE
+   */
+  public async getPage(step, url) {
+    const page = await this.browser.newPage();
+    await page.goto(url);
+    return page;
+  }
+
+  /**
+   * Find element on page DOM.
+   *
+   * @example GET page, "elementName", "text"
+   */
+  private async getByIDOrName(page, elementName) {
+    return await page.$(`[name="${elementName}"]`);
+  }
+
+  /**
+   * Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy.
+   *
+   * @example x = TODAY
+   */
+  public async click(step, page, idOrName) {
+    const e = await this.getByIDOrName(page, idOrName);
+
+    await Promise.all([
+      page.waitForNavigation(),
+      page.click(e.name)
+    ]);
+  }
+
+
+  /**
+   * Returns the screenshot of page or element
+   *
+   * @example file = SCREENSHOT page
+   */
+  public async screenshot(step, page, idOrName, localName) {
+    const e = await this.getByIDOrName(page, idOrName);
+    await e.screenshot({ path: localName });
+  }
+
+  /**
+   * Returns the screenshot of page or element
+   *
+   * @example file = SCREENSHOT page
+   */
+  public async captcha(step, page, idOrName) {
+    const e = await this.getByIDOrName(page, idOrName);
+
+
+
+  }
+
+  /**
+   * Performs the download to the .gbdrive Download folder.
+   *
+   * @example file = DOWNLOAD page, "tableName", row
+   */
+  public async download(step, page, idOrName, localName) {
+
+    const e = await this.getByIDOrName(page, idOrName);
+    const context = await this.browser.newContext({ acceptDownloads: true });
+
+    var cells = e.rows[0].cells;
+
+    const [download] = await Promise.all([
+      page.waitForEvent('download'),
+      page.click(cells[0])
+    ]);
+
+    const path = await download.path();
+
+    console.log(path);
+
+  }
+
+
+
+  /**
+   * Types the text into the text field.
+   *
+   * @example TYPE page, "elementName", "text"
+   */
+  public async type(step, page, idOrName, text) {
+    const e = await this.getByIDOrName(page, idOrName);
+    await e.type(text);
+  }
+
+  /**
+   * Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy.
+   *
+   * @example x = TODAY
+   */
+  public async getOCR(step, localFile) {
+    const tesseract = require("node-tesseract-ocr")
+
+    const config = {
+      lang: "eng",
+      oem: 1,
+      psm: 3,
+    }
+
+    return await tesseract.recognize(localFile, config);
+  }
+
   /**
    * Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy.
    *
@@ -122,6 +244,9 @@ export class DialogKeywords {
    * @example EXIT
    */
   public async exit(step) {
+    if (this.browser) {
+      await this.browser.close();
+    }
     await step.endDialog();
   }
 
@@ -411,6 +536,7 @@ export class DialogKeywords {
     this.user = user;
   }
 
+
   /**
    * Returns the name of the user acquired by WhatsApp API.
    */
@@ -470,13 +596,42 @@ export class DialogKeywords {
     this.min.cbMap[idPromise] = {};
     this.min.cbMap[idPromise].promise = promise;
 
-    const opts = { id: idPromise, previousResolve: previousResolve, kind: kind, args };
+    let opts = { id: idPromise, previousResolve: previousResolve, kind: kind, args };
+
+    if (this.hrOn) {
+
+      let sleep = ms => {
+        return new Promise(resolve => {
+          setTimeout(resolve, ms);
+        });
+      };
+
+      // Waits for next message in HEAR delegated context.
+      
+      const mobile = await this.userMobile(step);
+      while (true){
+        if (WhatsappDirectLine.state[mobile] === 3)
+        {
+          break;
+        }
+        sleep (5000);
+      }
+      const result = WhatsappDirectLine.lastMessage[mobile];
+      opts = await promise(step, result);
+
+      if (previousResolve !== undefined) {
+        previousResolve(opts);
+      }
+    }
+    else {
+
       if (previousResolve !== undefined) {
         previousResolve(opts);
       } else {
         await step.beginDialog('/hear', opts);
       }
     }
+  }
 
   /**
    * Prepares the next dialog to be shown to the specified user.

packages/basic.gblib/services/GBVMService.ts

@@ -196,6 +196,13 @@ export class GBVMService extends GBService {
 
     // Keywords from General Bots BASIC.
     
+    code = code.replace(/(\w+)\s*\=\s*get html\s*(.*)/gi, ($0, $1, $2, $3) => {
+      return `${$1} = sys().getPage ("${$2}")\n`;
+    });
+    code = code.replace(/(set hear on)(\s*)(.*)/gi, ($0, $1, $2, $3) => {
+      return `hrOn = ${$3}\n`;
+    });
+
     code = code.replace(/hear (\w+) as login/gi, ($0, $1) => {
       return `${$1} = hear("login")`;
     });
@@ -297,14 +304,20 @@ export class GBVMService extends GBService {
     });
 
     code = code.replace(/(get stock for )(.*)/gi, ($0, $1, $2) => {
-      return `let stock = sys().getStock(${$2})`;
+      return `stock = sys().getStock(${$2})`;
     });
 
-    code = code.replace(/(\w+)\s*\=\s*get\s(.*)/gi, ($0, $1, $2) => {
+    code = code.replace(/(\w+)\s*\=\s*get\s(.*)/gi, ($0, $1, $2, $3) => {
       if ($2.indexOf('http') !== -1) {
-        return `let ${$1} = sys().getByHttp (${$2}, headers, httpUsername, httpPs)`;
+        return `${$1} = sys().getByHttp (${$2}, headers, httpUsername, httpPs)`;
       } else {
-        return `let ${$1} = sys().get (${$2})`;
+        if ($2.indexOf(',') !== -1) {
+          const values = $2.split(',');
+          return `${$1} = getByIDOrName(${values[0]}, ${values[1]} )`;
+        }
+        else {
+          return `${$1} = sys().get (${$2})`;
+        }
       }
     });
 
@@ -349,7 +362,11 @@ export class GBVMService extends GBService {
     });
 
     code = code.replace(/(\w+)\s*\=\s*post\s*(.*),\s*(.*)/gi, ($0, $1, $2, $3) => {
-      return `let ${$1} = sys().httpPost (${$2}, ${$3})`;
+      return `${$1} = sys().httpPost (${$2}, ${$3})`;
+    });
+
+    code = code.replace(/(\w+)\s*\=\s*download\s*(.*),\s*(.*)/gi, ($0, $1, $2, $3) => {
+      return `${$1} = sys().download (${$2}, ${$3})`;
     });
 
     code = code.replace(/(create a bot farm using)(\s)(.*)/gi, ($0, $1, $2, $3) => {
@@ -384,6 +401,10 @@ export class GBVMService extends GBService {
       return `sendFileTo (step, ${$3})\n`;
     });
 
+    code = code.replace(/(click)(\s*)(.*)/gi, ($0, $1, $2, $3) => {
+      return `click (step, ${$3})\n`;
+    });
+
     code = code.replace(/(send file)(\s*)(.*)/gi, ($0, $1, $2, $3) => {
       return `sendFile (step, ${$3})\n`;
     });

packages/basic.gblib/services/SystemKeywords.ts

@@ -432,6 +432,8 @@ export class SystemKeywords {
     const botId = this.min.instance.botId;
     const path = `/${botId}.gbai/${botId}.gbdata`;
 
+    // TODO: if (typeof(file) === 
+
     let document = await this.internalGetDocument(client, baseUrl, path, file);
     let maxLines = 1000;
     if (this.dk.user && this.dk.user.basicOptions && this.dk.user.basicOptions.maxLines) {

packages/core.gbapp/services/GBMinService.ts

@@ -163,7 +163,7 @@ export class GBMinService {
             setTimeout(resolve, ms);
           });
         };
-        await sleep(20000);
+        await sleep(1);
         res.status(200);
         res.end();
       });

packages/whatsapp.gblib/services/WhatsappDirectLine.ts

@@ -52,6 +52,8 @@ export class WhatsappDirectLine extends GBService {
   public static mobiles = {};
   public static chatIds = {};
   public static usernames = {};
+  public static state = {}; // 2: Waiting, 3: MessageArrived.
+  public static lastMessage = {}; // 2: Waiting, 3: MessageArrived.
 
   public pollInterval = 3000;
   public directLineClientName = 'DirectLineClient';
@@ -240,6 +242,7 @@ export class WhatsappDirectLine extends GBService {
     }
 
     
+
     await CollectionUtil.asyncForEach(this.min.appPackages, async (e: IGBPackage) => {
       await e.onExchangeData(this.min, 'whatsappMessage', message);
     });
@@ -285,6 +288,8 @@ export class WhatsappDirectLine extends GBService {
 
     const client = await this.directLineClient;
 
+    WhatsappDirectLine.lastMessage[from] = message;
+
 
     // Check if this message is from a Human Agent itself.
 

src/app.ts

@@ -119,8 +119,6 @@ export class GBServer {
           const azureDeployer: AzureDeployerService = new AzureDeployerService(deployer);
           const adminService: GBAdminService = new GBAdminService(core);
 
-
-
           if (process.env.NODE_ENV === 'development' && !process.env.BOT_URL) {
             const proxy = GBConfigService.get('REVERSE_PROXY');
             if (proxy !== undefined) {