new(basic.gblib): DOWNLOAD keyword for web automation and more.

packages/basic.gblib/services/DialogKeywords.ts

@@ -49,6 +49,7 @@ const DateDiff = require('date-diff');
 const puppeteer = require('puppeteer');
 const Path = require('path');
 import bb, { area, bar, zoom } from "billboard.js";
+import * as request from 'request-promise-native';
 
 /**
  * Base services of conversation to be called by BASIC which
@@ -201,6 +202,8 @@ export class DialogKeywords {
     const f = await frameHandle.contentFrame();
     await f.waitForSelector(selector);
     const element = await f.$(selector);
+    element['originalSelector'] = selector;
+    element['frame'] = f;
     return element;
   }
 
@@ -218,27 +221,9 @@ export class DialogKeywords {
     ]);
   }
 
-  /**
-   * Returns the screenshot of page or element
-   *
-   * @example file = SCREENSHOT page
-   */
-   public async getTableData(step, page, selector) {
-
-    const data = await page.evaluate(() => {
-      const rows = document.querySelectorAll(`${selector} tr`);
-      return Array.from(rows, row => {
-        const columns = row.querySelectorAll('td');
-        return Array.from(columns, column => column.innerText);
-      });
-    });
-
-    //You will now have an array of strings
-    //[ 'One', 'Two', 'Three', 'Four' ]
-    console.log(data);
 
 
-  }
+
 
   /**
    * Returns the screenshot of page or element
@@ -250,27 +235,6 @@ export class DialogKeywords {
     await e.screenshot({ path: localName });
   }
 
-  /**
-   * Performs the download to the .gbdrive Download folder.
-   *
-   * @example file = DOWNLOAD page, "tableName", row
-   */
-  public async download(step, page, idOrName, localName) {
-
-    const e = await this.getBySelector(page, idOrName);
-    const context = await this.browser.newContext({ acceptDownloads: true });
-
-    var cells = e.rows[0].cells;
-
-    const [download] = await Promise.all([
-      page.waitForEvent('download'),
-      page.click(cells[0])
-    ]);
-
-    const path = await download.path();
-
-    console.log(path);
-  }
 
   /**
    * Types the text into the text field.

packages/basic.gblib/services/GBVMService.ts

@@ -395,7 +395,7 @@ export class GBVMService extends GBService {
     });
 
     code = code.replace(/(\w+)\s*\=\s*download\s*(.*),\s*(.*)/gi, ($0, $1, $2, $3) => {
-      return `${$1} = download (${$2}, ${$3})`;
+      return `${$1} = sys().download (${$2}, ${$3})`;
     });
 
     code = code.replace(/(create a bot farm using)(\s)(.*)/gi, ($0, $1, $2, $3) => {
@@ -696,9 +696,6 @@ export class GBVMService extends GBService {
     code = code.replace(/("[^"]*"|'[^']*')|\bclick\b/gi, ($0, $1) => {
       return $1 === undefined ? 'this.click' : $1;
     });
-    code = code.replace(/("[^"]*"|'[^']*')|\bdownload\b/gi, ($0, $1) => {
-      return $1 === undefined ? 'this.download' : $1;
-    });
     // await insertion.
 
     code = code.replace(/this\./gm, 'await this.');

packages/basic.gblib/services/SystemKeywords.ts

@@ -39,14 +39,19 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer';
 import { DialogKeywords } from './DialogKeywords';
 import { Tabulator } from 'tabulator-tables';
 import { GBServer } from '../../../src/app';
+import * as fs from 'fs';
+const Fs = require('fs');
+const Excel = require('exceljs');
 
 const urlJoin = require('url-join');
+const url = require('url');
 const puppeteer = require('puppeteer')
 const Path = require('path');
 const sgMail = require('@sendgrid/mail');
 const ComputerVisionClient = require('@azure/cognitiveservices-computervision').ComputerVisionClient;
 const ApiKeyCredentials = require('@azure/ms-rest-js').ApiKeyCredentials;
 const alasql = require('alasql');
+const DateDiff = require('date-diff');
 
 
 /**
@@ -569,13 +574,12 @@ export class SystemKeywords {
    */
   public async find(file: string, ...args): Promise<any> {
     GBLog.info(`BASIC: FIND running on ${file} and args: ${JSON.stringify(args)}...`);
-    let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min);
+
     const botId = this.min.instance.botId;
     const path = `/${botId}.gbai/${botId}.gbdata`;
 
-    // TODO: if (typeof(file) === 
+    // MAX LINES property.
 
-    let document = await this.internalGetDocument(client, baseUrl, path, file);
     let maxLines = 1000;
     if (this.dk.user && this.dk.user.basicOptions && this.dk.user.basicOptions.maxLines) {
       if (this.dk.user.basicOptions.maxLines.toString().toLowerCase() !== "default") {
@@ -583,16 +587,94 @@ export class SystemKeywords {
       }
     }
 
+    // Choose data sources based on file type (HTML Table, data variable or sheet file)
+
+    let results;
+    let header, rows;
+
+    if (file['$eval']) {
+      const container = file['frame'] ? file['frame'] : file['_page'];
+      const originalSelector = file['originalSelector'];
+
+      // Transforms table
+
+      const resultH = await container.evaluate((originalSelector) => {
+        const rows = document.querySelectorAll(`${originalSelector} tr`);
+        return Array.from(rows, row => {
+          const columns = row.querySelectorAll('th');
+          return Array.from(columns, column => column.innerText);
+        });
+      }, originalSelector);
+
+      const result = await container.evaluate((originalSelector) => {
+        const rows = document.querySelectorAll(`${originalSelector} tr`);
+        return Array.from(rows, row => {
+          const columns = row.querySelectorAll('td');
+          return Array.from(columns, column => column.innerText);
+        });
+      }, originalSelector);
+
+      header = [];
+      for (let i = 0; i < resultH[0].length; i++) {
+        header[i] = resultH[0][i];
+      }
+
+      rows = [];
+      rows[0] = header;
+      for (let i = 1; i < result.length; i++) {
+        rows[i] = result[i];
+      }
+
+
+    } else if (file['cTag']) {
+
+      const gbaiName = `${this.min.botId}.gbai`;
+      const localName = Path.join('work', gbaiName, 'cache', `csv${GBAdminService.getRndReadableIdentifier()}.csv`);
+      const url = file['@microsoft.graph.downloadUrl'];
+      const response = await request({ uri: url, encoding: null });
+      Fs.writeFileSync(localName, response, { encoding: null });
+
+      var workbook = new Excel.Workbook();
+      const worksheet = await workbook.csv.readFile(localName);
+      header = [];
+      rows = [];
+
+      for (let i = 0; i < worksheet._rows.length; i++) {
+        const r = worksheet._rows[i];
+        let outRow = [];
+        for (let j = 0; j < r._cells.length; j++) {
+          outRow.push(r._cells[j].text);
+        }
+
+        if (i == 0) {
+          header = outRow;
+        }
+        else {
+          rows.push(outRow);
+        }
+      }
+
+    } else {
+
+      let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min);
+
+      let document
+      document = await this.internalGetDocument(client, baseUrl, path, file);
+
       // Creates workbook session that will be discarded.
 
       let sheets = await client
         .api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets`)
         .get();
 
-    let results = await client
+      results = await client
         .api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A1:Z${maxLines}')`)
         .get();
 
+      header = results.text[0];
+      rows = results.text;
+    }
+
     let getFilter = async (text) => {
       let filter;
       const operators = [/\<\=/, /\>\=/, /\</, /\>/, /\bnot in\b/, /\bin\b/, /\=/];
@@ -632,7 +714,7 @@ export class SystemKeywords {
     // Increments columnIndex by looping until find a column match.
 
     const filters = [];
-    const header = results.text[0];
+
     await CollectionUtil.asyncForEach(args, async arg => {
       const filter = await getFilter(arg);
       if (!filter) {
@@ -670,12 +752,12 @@ export class SystemKeywords {
     let foundIndex = 1;
 
     // Fills the row variable.
-
+    let rowCount = 0;
-    for (; foundIndex < results.text.length; foundIndex++) {
+    for (; foundIndex < rows.length; foundIndex++) {
       let filterAcceptCount = 0;
       await CollectionUtil.asyncForEach(filters, async filter => {
 
-        let result = results.text[foundIndex][filter.columnIndex];
+        let result = rows[foundIndex][filter.columnIndex];
         let wholeWord = true;
         if (this.dk.user && this.dk.user.basicOptions && this.dk.user.basicOptions.wholeWord) {
           wholeWord = this.dk.user.basicOptions.wholeWord;
@@ -760,9 +842,13 @@ export class SystemKeywords {
               result = result.substr(1);
             }
             const resultDate = SystemKeywords.getDateFromLocaleString(result, contentLocale);
+            if (filter.value['dateOnly']) {
+              resultDate.setHours(0, 0, 0, 0);
+            }
             if (resultDate) {
               switch (filter.operator) {
                 case '=':
+
                   if (resultDate.getTime() == filter.value.getTime())
                     filterAcceptCount++;
                   break;
@@ -789,8 +875,9 @@ export class SystemKeywords {
       });
 
       if (filterAcceptCount === filters.length) {
+        rowCount++;
         let row = {};
-        const xlRow = results.text[foundIndex];
+        const xlRow = rows[foundIndex];
         for (let colIndex = 0; colIndex < xlRow.length; colIndex++) {
           const propertyName = header[colIndex];
           let value = xlRow[colIndex];
@@ -801,7 +888,8 @@ export class SystemKeywords {
           }
           row[propertyName] = value;
         }
-        row['line'] = foundIndex + 1;
+        row['line'] = rowCount + 1;
+        row['originalLine'] = foundIndex + 1;
         table.push(row);
       }
 
@@ -814,36 +902,137 @@ export class SystemKeywords {
       GBLog.info(`BASIC: FIND returned single result: ${table[0]}.`);
       return table[1];
     } else {
-      GBLog.info(`BASIC: FIND returned multiple results (Count): ${table.length}.`);
+      GBLog.info(`BASIC: FIND returned multiple results (Count): ${table.length - 1}.`);
       return table;
     }
   }
 
   public static getDateFromLocaleString(date: any, contentLocale: any) {
+    let ret = null;
     let parts = /^([0-3]?[0-9]).([0-3]?[0-9]).((?:[0-9]{2})?[0-9]{2})\s*(10|11|12|0?[1-9]):([0-5][0-9])/gi.exec(date);
     if (parts && parts[5]) {
+
       switch (contentLocale) {
         case 'pt':
-          return new Date(Number.parseInt(parts[3]), Number.parseInt(parts[2]) - 1, Number.parseInt(parts[1]),
+          ret = new Date(Number.parseInt(parts[3]), Number.parseInt(parts[2]) - 1, Number.parseInt(parts[1]),
             Number.parseInt(parts[4]), Number.parseInt(parts[5]), 0, 0);
+          break;
         case 'en':
-          return new Date(Number.parseInt(parts[3]), Number.parseInt(parts[1]) - 1, Number.parseInt(parts[2]),
+          ret = new Date(Number.parseInt(parts[3]), Number.parseInt(parts[1]) - 1, Number.parseInt(parts[2]),
             Number.parseInt(parts[4]), Number.parseInt(parts[5]), 0, 0);
+          break;
       }
+
+      ret['dateOnly'] = false;
     }
 
     parts = /^([0-3]?[0-9]).([0-3]?[0-9]).((?:[0-9]{2})?[0-9]{2})$/gi.exec(date);
     if (parts && parts[3]) {
+
       switch (contentLocale) {
         case 'pt':
-          return new Date(Number.parseInt(parts[2]), Number.parseInt(parts[1]) - 1, Number.parseInt(parts[3]), 0, 0, 0, 0);
+          ret = new Date(Number.parseInt(parts[3]), Number.parseInt(parts[2]) - 1, Number.parseInt(parts[1]), 0, 0, 0, 0);
+          break;
         case 'en':
-          return new Date(Number.parseInt(parts[1]), Number.parseInt(parts[2]) - 1, Number.parseInt(parts[3]), 0, 0, 0, 0);
+          ret = new Date(Number.parseInt(parts[3]), Number.parseInt(parts[1]) - 1, Number.parseInt(parts[2]), 0, 0, 0, 0);
+          break;
       }
+
+      ret['dateOnly'] = true;
     }
-    return null;
+    return ret;
   }
 
+  /**
+   * Performs the download to the .gbdrive Download folder.
+   *
+   * @example file = DOWNLOAD element, folder
+   */
+  public async download(element, folder) {
+
+    const page = element['_page'];
+    const container = element['_frame'] ? element['_frame'] : element['_page'];
+
+    await page.setRequestInterception(true);
+    await container.click(element.originalSelector);
+
+    const xRequest = await new Promise(resolve => {
+      page.on('request', interceptedRequest => {
+        interceptedRequest.abort();     //stop intercepting requests
+        resolve(interceptedRequest);
+      });
+    });
+
+    const options = {
+      encoding: null,
+      method: xRequest['._method'],
+      uri: xRequest['_url'],
+      body: xRequest['_postData'],
+      headers: xRequest['_headers']
+    }
+
+    const cookies = await page.cookies();
+    options.headers.Cookie = cookies.map(ck => ck.name + '=' + ck.value).join(';');
+    GBLog.info(`BASIC: DOWNLOADING '${options.uri}...'`);
+
+    let local;
+    let filename;
+    if (options.uri.indexOf('file://') != -1) {
+      local = url.fileURLToPath(options.uri);
+      filename = Path.basename(local);
+    }
+    else {
+      const getBasenameFormUrl = (urlStr) => {
+        const url = new URL(urlStr)
+        return Path.basename(url.pathname)
+      };
+      filename = getBasenameFormUrl(options.uri);
+    }
+
+    let result: Buffer;
+    if (local) {
+      result = fs.readFileSync(local);
+    } else {
+      result = await request.get(options);
+    }
+    let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min);
+    const botId = this.min.instance.botId;
+
+    // Normalizes all slashes.
+
+    folder = folder.replace(/\\/gi, '/');
+
+    // Determines full path at source and destination.
+
+    const root = urlJoin(`/${botId}.gbai/${botId}.gbdrive`);
+    const dstPath = urlJoin(root, folder, filename);
+
+    // Checks if the destination contains subfolders that
+    // need to be created.
+
+    folder = await this.createFolder(folder);
+
+    // Performs the conversion operation getting a reference
+    // to the source and calling /content on drive API.
+    let file;
+    try {
+
+      file = await client
+        .api(`${baseUrl}/drive/root:/${dstPath}:/content`)
+        .put(result);
+
+    } catch (error) {
+
+      if (error.code === "nameAlreadyExists") {
+        GBLog.info(`BASIC: DOWNLOAD destination file already exists: ${dstPath}.`);
+      }
+      throw error;
+    }
+
+    return file;
+  }
+
+
   /**
    * Creates a folder in the bot instance drive.
    *
@@ -854,7 +1043,7 @@ export class SystemKeywords {
 
     let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min);
     const botId = this.min.instance.botId;
-    let path = `/${botId}.gbai/${botId}.gbdata`;
+    let path = `/${botId}.gbai/${botId}.gbdrive`;
 
     // Extracts each part of path to call create folder to each
     // one of them.