new(all): OCR and Web Manipulation.

package-lock.json

@@ -14448,6 +14448,11 @@
       "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz",
       "integrity": "sha1-WuVUHQJGRdMqWPzdyc7s6nrjrC8="
     },
+    "node-tesseract-ocr": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/node-tesseract-ocr/-/node-tesseract-ocr-2.2.1.tgz",
+      "integrity": "sha512-Q9cD79JGpPNQBxbi1fV+OAsTxYKLpx22sagsxSyKbu1u+t6UarApf5m32uVc8a5QAP1Wk7fIPN0aJFGGEE9DyQ=="
+    },
     "nodesecurity-npm-utils": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/nodesecurity-npm-utils/-/nodesecurity-npm-utils-6.0.0.tgz",

package.json

@@ -95,6 +95,7 @@
     "ms-rest-azure": "3.0.0",
     "nexmo": "2.9.1",
     "node-cron": "3.0.0",
+    "node-tesseract-ocr": "^2.2.1",
     "npm": "7.21.0",
     "opn": "6.0.0",
     "pdf-extraction": "1.0.2",

packages/basic.gblib/services/DialogKeywords.ts

@@ -42,8 +42,8 @@ import { SystemKeywords } from './SystemKeywords';
 import { GBMinService } from '../../core.gbapp/services/GBMinService';
 import { HubSpotServices } from '../../hubspot.gblib/services/HubSpotServices';
 import { WhatsappDirectLine } from '../../whatsapp.gblib/services/WhatsappDirectLine';
-var DateDiff = require('date-diff');
-
+const DateDiff = require('date-diff');
+const puppeteer = require('puppeteer');
 
 /**
  * Base services of conversation to be called by BASIC which
@@ -66,6 +66,16 @@ export class DialogKeywords {
    */
   public user;
 
+  /**
+   * HTML browser for conversation over page interaction.
+   */
+  browser: any;
+
+  /**
+   * The number used in this execution for HEAR calls (useful for SET SCHEDULE).
+   */
+   hrOn: string;
+
   /**
    * When creating this keyword facade, a bot instance is
    * specified among the deployer service.
@@ -74,6 +84,9 @@ export class DialogKeywords {
     this.min = min;
     this.user = user;
     this.internalSys = new SystemKeywords(min, deployer, this);
+    (async () => {
+      this.browser = await puppeteer.launch();
+    });
   }
 
   /**
@@ -84,6 +97,115 @@ export class DialogKeywords {
     return this.internalSys;
   }
 
+  /**
+   * Returns the page object.
+   *
+   * @example x = GET PAGE
+   */
+  public async getPage(step, url) {
+    const page = await this.browser.newPage();
+    await page.goto(url);
+    return page;
+  }
+
+  /**
+   * Find element on page DOM.
+   *
+   * @example GET page, "elementName", "text"
+   */
+  private async getByIDOrName(page, elementName) {
+    return await page.$(`[name="${elementName}"]`);
+  }
+
+  /**
+   * Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy.
+   *
+   * @example x = TODAY
+   */
+  public async click(step, page, idOrName) {
+    const e = await this.getByIDOrName(page, idOrName);
+
+    await Promise.all([
+      page.waitForNavigation(),
+      page.click(e.name)
+    ]);
+  }
+
+
+  /**
+   * Returns the screenshot of page or element
+   *
+   * @example file = SCREENSHOT page
+   */
+  public async screenshot(step, page, idOrName, localName) {
+    const e = await this.getByIDOrName(page, idOrName);
+    await e.screenshot({ path: localName });
+  }
+
+  /**
+   * Returns the screenshot of page or element
+   *
+   * @example file = SCREENSHOT page
+   */
+  public async captcha(step, page, idOrName) {
+    const e = await this.getByIDOrName(page, idOrName);
+
+
+
+  }
+
+  /**
+   * Performs the download to the .gbdrive Download folder.
+   *
+   * @example file = DOWNLOAD page, "tableName", row
+   */
+  public async download(step, page, idOrName, localName) {
+
+    const e = await this.getByIDOrName(page, idOrName);
+    const context = await this.browser.newContext({ acceptDownloads: true });
+
+    var cells = e.rows[0].cells;
+
+    const [download] = await Promise.all([
+      page.waitForEvent('download'),
+      page.click(cells[0])
+    ]);
+
+    const path = await download.path();
+
+    console.log(path);
+
+  }
+
+
+
+  /**
+   * Types the text into the text field.
+   *
+   * @example TYPE page, "elementName", "text"
+   */
+  public async type(step, page, idOrName, text) {
+    const e = await this.getByIDOrName(page, idOrName);
+    await e.type(text);
+  }
+
+  /**
+   * Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy.
+   *
+   * @example x = TODAY
+   */
+  public async getOCR(step, localFile) {
+    const tesseract = require("node-tesseract-ocr")
+
+    const config = {
+      lang: "eng",
+      oem: 1,
+      psm: 3,
+    }
+
+    return await tesseract.recognize(localFile, config);
+  }
+
   /**
    * Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy.
    *
@@ -122,6 +244,9 @@ export class DialogKeywords {
    * @example EXIT
    */
   public async exit(step) {
+    if (this.browser) {
+      await this.browser.close();
+    }
     await step.endDialog();
   }
 
@@ -151,7 +276,7 @@ export class DialogKeywords {
    *
    * @example list = FIND CONTACT "Sandra"
    */
-   public async fndContact(name) {
+  public async fndContact(name) {
     let s = new HubSpotServices(null, null, process.env.HUBSPOT_KEY);
     return await s.searchContact(name);
   }
@@ -323,13 +448,13 @@ export class DialogKeywords {
     );
 
     const nowUTC = new Date();
-    const now= typeof nowUTC === 'string' ?
+    const now = typeof nowUTC === 'string' ?
       new Date(nowUTC) :
       nowUTC;
-      
-   const nowText = now.toLocaleString(this.getContentLocaleWithCulture(contentLocale),
-        { timeZone: process.env.DEFAULT_TIMEZONE });
-    
+
+    const nowText = now.toLocaleString(this.getContentLocaleWithCulture(contentLocale),
+      { timeZone: process.env.DEFAULT_TIMEZONE });
+
     return /\b([0-9]|0[0-9]|1?[0-9]|2[0-3]):[0-5]?[0-9]/.exec(nowText)[0];
   }
 
@@ -411,6 +536,7 @@ export class DialogKeywords {
     this.user = user;
   }
 
+
   /**
    * Returns the name of the user acquired by WhatsApp API.
    */
@@ -470,11 +596,40 @@ export class DialogKeywords {
     this.min.cbMap[idPromise] = {};
     this.min.cbMap[idPromise].promise = promise;
 
-    const opts = { id: idPromise, previousResolve: previousResolve, kind: kind, args };
-    if (previousResolve !== undefined) {
-      previousResolve(opts);
-    } else {
-      await step.beginDialog('/hear', opts);
+    let opts = { id: idPromise, previousResolve: previousResolve, kind: kind, args };
+
+    if (this.hrOn) {
+
+      let sleep = ms => {
+        return new Promise(resolve => {
+          setTimeout(resolve, ms);
+        });
+      };
+
+      // Waits for next message in HEAR delegated context.
+      
+      const mobile = await this.userMobile(step);
+      while (true){
+        if (WhatsappDirectLine.state[mobile] === 3)
+        {
+          break;
+        }
+        sleep (5000);
+      }
+      const result = WhatsappDirectLine.lastMessage[mobile];
+      opts = await promise(step, result);
+
+      if (previousResolve !== undefined) {
+        previousResolve(opts);
+      }
+    }
+    else {
+
+      if (previousResolve !== undefined) {
+        previousResolve(opts);
+      } else {
+        await step.beginDialog('/hear', opts);
+      }
     }
   }
 

packages/basic.gblib/services/GBVMService.ts

@@ -189,12 +189,19 @@ export class GBVMService extends GBService {
     httpUsername = "";
     httpPs = "";
 
-    ${process.env.ENABLE_AUTH? `hear gbLogin as login`:``}
+    ${process.env.ENABLE_AUTH ? `hear gbLogin as login` : ``}
 
     ${code}
     `;
 
     // Keywords from General Bots BASIC.
+    
+    code = code.replace(/(\w+)\s*\=\s*get html\s*(.*)/gi, ($0, $1, $2, $3) => {
+      return `${$1} = sys().getPage ("${$2}")\n`;
+    });
+    code = code.replace(/(set hear on)(\s*)(.*)/gi, ($0, $1, $2, $3) => {
+      return `hrOn = ${$3}\n`;
+    });
 
     code = code.replace(/hear (\w+) as login/gi, ($0, $1) => {
       return `${$1} = hear("login")`;
@@ -275,7 +282,7 @@ export class GBVMService extends GBService {
     code = code.replace(/(\w)\s*\=\s*active tasks/gi, ($0, $1) => {
       return `${$1} = getActiveTasks()\n`;
     });
-    
+
     code = code.replace(/(\w)\s*\=\s*append\s*(.*)/gi, ($0, $1, $2, $3) => {
       return `${$1} = sys().append(${$2})\n`;
     });
@@ -297,14 +304,20 @@ export class GBVMService extends GBService {
     });
 
     code = code.replace(/(get stock for )(.*)/gi, ($0, $1, $2) => {
-      return `let stock = sys().getStock(${$2})`;
+      return `stock = sys().getStock(${$2})`;
     });
 
-    code = code.replace(/(\w+)\s*\=\s*get\s(.*)/gi, ($0, $1, $2) => {
+    code = code.replace(/(\w+)\s*\=\s*get\s(.*)/gi, ($0, $1, $2, $3) => {
       if ($2.indexOf('http') !== -1) {
-        return `let ${$1} = sys().getByHttp (${$2}, headers, httpUsername, httpPs)`;
+        return `${$1} = sys().getByHttp (${$2}, headers, httpUsername, httpPs)`;
       } else {
-        return `let ${$1} = sys().get (${$2})`;
+        if ($2.indexOf(',') !== -1) {
+          const values = $2.split(',');
+          return `${$1} = getByIDOrName(${values[0]}, ${values[1]} )`;
+        }
+        else {
+          return `${$1} = sys().get (${$2})`;
+        }
       }
     });
 
@@ -349,7 +362,11 @@ export class GBVMService extends GBService {
     });
 
     code = code.replace(/(\w+)\s*\=\s*post\s*(.*),\s*(.*)/gi, ($0, $1, $2, $3) => {
-      return `let ${$1} = sys().httpPost (${$2}, ${$3})`;
+      return `${$1} = sys().httpPost (${$2}, ${$3})`;
+    });
+
+    code = code.replace(/(\w+)\s*\=\s*download\s*(.*),\s*(.*)/gi, ($0, $1, $2, $3) => {
+      return `${$1} = sys().download (${$2}, ${$3})`;
     });
 
     code = code.replace(/(create a bot farm using)(\s)(.*)/gi, ($0, $1, $2, $3) => {
@@ -384,6 +401,10 @@ export class GBVMService extends GBService {
       return `sendFileTo (step, ${$3})\n`;
     });
 
+    code = code.replace(/(click)(\s*)(.*)/gi, ($0, $1, $2, $3) => {
+      return `click (step, ${$3})\n`;
+    });
+
     code = code.replace(/(send file)(\s*)(.*)/gi, ($0, $1, $2, $3) => {
       return `sendFile (step, ${$3})\n`;
     });
@@ -625,7 +646,7 @@ export class GBVMService extends GBService {
             if (step.context.activity.channelId !== 'msteams' && process.env.ENABLE_AUTH) {
               GBLog.info('BASIC: Authenticating beforing running General Bots BASIC code.');
               return await step.beginDialog('/auth');
-          }
+            }
           }
           return await step.next(step.options);
         },

packages/basic.gblib/services/SystemKeywords.ts

@@ -432,6 +432,8 @@ export class SystemKeywords {
     const botId = this.min.instance.botId;
     const path = `/${botId}.gbai/${botId}.gbdata`;
 
+    // TODO: if (typeof(file) === 
+
     let document = await this.internalGetDocument(client, baseUrl, path, file);
     let maxLines = 1000;
     if (this.dk.user && this.dk.user.basicOptions && this.dk.user.basicOptions.maxLines) {

packages/core.gbapp/services/GBMinService.ts

@@ -163,7 +163,7 @@ export class GBMinService {
             setTimeout(resolve, ms);
           });
         };
-        await sleep(20000);
+        await sleep(1);
         res.status(200);
         res.end();
       });

packages/whatsapp.gblib/services/WhatsappDirectLine.ts

@@ -52,6 +52,8 @@ export class WhatsappDirectLine extends GBService {
   public static mobiles = {};
   public static chatIds = {};
   public static usernames = {};
+  public static state = {}; // 2: Waiting, 3: MessageArrived.
+  public static lastMessage = {}; // 2: Waiting, 3: MessageArrived.
 
   public pollInterval = 3000;
   public directLineClientName = 'DirectLineClient';
@@ -239,6 +241,7 @@ export class WhatsappDirectLine extends GBService {
       }
     }
 
+    
 
     await CollectionUtil.asyncForEach(this.min.appPackages, async (e: IGBPackage) => {
       await e.onExchangeData(this.min, 'whatsappMessage', message);
@@ -285,6 +288,8 @@ export class WhatsappDirectLine extends GBService {
 
     const client = await this.directLineClient;
 
+    WhatsappDirectLine.lastMessage[from] = message;
+
 
     // Check if this message is from a Human Agent itself.
 

src/app.ts

@@ -119,8 +119,6 @@ export class GBServer {
           const azureDeployer: AzureDeployerService = new AzureDeployerService(deployer);
           const adminService: GBAdminService = new GBAdminService(core);
 
-
-
           if (process.env.NODE_ENV === 'development' && !process.env.BOT_URL) {
             const proxy = GBConfigService.get('REVERSE_PROXY');
             if (proxy !== undefined) {