new(basic.gblib): DOWNLOAD keyword for web automation and more.
This commit is contained in:
Rodrigo Rodriguez
parent
90cf855810
commit
100cf31250
3 changed files with 533 additions and 383 deletions
|
|
@ -49,6 +49,7 @@ const DateDiff = require('date-diff');
|
|||
const puppeteer = require('puppeteer');
|
||||
const Path = require('path');
|
||||
import bb, { area, bar, zoom } from "billboard.js";
|
||||
import * as request from 'request-promise-native';
|
||||
|
||||
/**
|
||||
* Base services of conversation to be called by BASIC which
|
||||
|
|
@ -201,6 +202,8 @@ export class DialogKeywords {
|
|||
const f = await frameHandle.contentFrame();
|
||||
await f.waitForSelector(selector);
|
||||
const element = await f.$(selector);
|
||||
element['originalSelector'] = selector;
|
||||
element['frame'] = f;
|
||||
return element;
|
||||
}
|
||||
|
||||
|
|
@ -218,27 +221,9 @@ export class DialogKeywords {
|
|||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the screenshot of page or element
|
||||
*
|
||||
* @example file = SCREENSHOT page
|
||||
*/
|
||||
public async getTableData(step, page, selector) {
|
||||
|
||||
const data = await page.evaluate(() => {
|
||||
const rows = document.querySelectorAll(`${selector} tr`);
|
||||
return Array.from(rows, row => {
|
||||
const columns = row.querySelectorAll('td');
|
||||
return Array.from(columns, column => column.innerText);
|
||||
});
|
||||
});
|
||||
|
||||
//You will now have an array of strings
|
||||
//[ 'One', 'Two', 'Three', 'Four' ]
|
||||
console.log(data);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the screenshot of page or element
|
||||
|
|
@ -250,27 +235,6 @@ export class DialogKeywords {
|
|||
await e.screenshot({ path: localName });
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs the download to the .gbdrive Download folder.
|
||||
*
|
||||
* @example file = DOWNLOAD page, "tableName", row
|
||||
*/
|
||||
public async download(step, page, idOrName, localName) {
|
||||
|
||||
const e = await this.getBySelector(page, idOrName);
|
||||
const context = await this.browser.newContext({ acceptDownloads: true });
|
||||
|
||||
var cells = e.rows[0].cells;
|
||||
|
||||
const [download] = await Promise.all([
|
||||
page.waitForEvent('download'),
|
||||
page.click(cells[0])
|
||||
]);
|
||||
|
||||
const path = await download.path();
|
||||
|
||||
console.log(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Types the text into the text field.
|
||||
|
|
|
|||
|
|
@ -395,7 +395,7 @@ export class GBVMService extends GBService {
|
|||
});
|
||||
|
||||
code = code.replace(/(\w+)\s*\=\s*download\s*(.*),\s*(.*)/gi, ($0, $1, $2, $3) => {
|
||||
return `${$1} = download (${$2}, ${$3})`;
|
||||
return `${$1} = sys().download (${$2}, ${$3})`;
|
||||
});
|
||||
|
||||
code = code.replace(/(create a bot farm using)(\s)(.*)/gi, ($0, $1, $2, $3) => {
|
||||
|
|
@ -696,9 +696,6 @@ export class GBVMService extends GBService {
|
|||
code = code.replace(/("[^"]*"|'[^']*')|\bclick\b/gi, ($0, $1) => {
|
||||
return $1 === undefined ? 'this.click' : $1;
|
||||
});
|
||||
code = code.replace(/("[^"]*"|'[^']*')|\bdownload\b/gi, ($0, $1) => {
|
||||
return $1 === undefined ? 'this.download' : $1;
|
||||
});
|
||||
// await insertion.
|
||||
|
||||
code = code.replace(/this\./gm, 'await this.');
|
||||
|
|
|
|||
|
|
@ -39,14 +39,19 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer';
|
|||
import { DialogKeywords } from './DialogKeywords';
|
||||
import { Tabulator } from 'tabulator-tables';
|
||||
import { GBServer } from '../../../src/app';
|
||||
import * as fs from 'fs';
|
||||
const Fs = require('fs');
|
||||
const Excel = require('exceljs');
|
||||
|
||||
const urlJoin = require('url-join');
|
||||
const url = require('url');
|
||||
const puppeteer = require('puppeteer')
|
||||
const Path = require('path');
|
||||
const sgMail = require('@sendgrid/mail');
|
||||
const ComputerVisionClient = require('@azure/cognitiveservices-computervision').ComputerVisionClient;
|
||||
const ApiKeyCredentials = require('@azure/ms-rest-js').ApiKeyCredentials;
|
||||
const alasql = require('alasql');
|
||||
const DateDiff = require('date-diff');
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -569,13 +574,12 @@ export class SystemKeywords {
|
|||
*/
|
||||
public async find(file: string, ...args): Promise<any> {
|
||||
GBLog.info(`BASIC: FIND running on ${file} and args: ${JSON.stringify(args)}...`);
|
||||
let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min);
|
||||
|
||||
const botId = this.min.instance.botId;
|
||||
const path = `/${botId}.gbai/${botId}.gbdata`;
|
||||
|
||||
// TODO: if (typeof(file) ===
|
||||
// MAX LINES property.
|
||||
|
||||
let document = await this.internalGetDocument(client, baseUrl, path, file);
|
||||
let maxLines = 1000;
|
||||
if (this.dk.user && this.dk.user.basicOptions && this.dk.user.basicOptions.maxLines) {
|
||||
if (this.dk.user.basicOptions.maxLines.toString().toLowerCase() !== "default") {
|
||||
|
|
@ -583,16 +587,94 @@ export class SystemKeywords {
|
|||
}
|
||||
}
|
||||
|
||||
// Choose data sources based on file type (HTML Table, data variable or sheet file)
|
||||
|
||||
let results;
|
||||
let header, rows;
|
||||
|
||||
if (file['$eval']) {
|
||||
const container = file['frame'] ? file['frame'] : file['_page'];
|
||||
const originalSelector = file['originalSelector'];
|
||||
|
||||
// Transforms table
|
||||
|
||||
const resultH = await container.evaluate((originalSelector) => {
|
||||
const rows = document.querySelectorAll(`${originalSelector} tr`);
|
||||
return Array.from(rows, row => {
|
||||
const columns = row.querySelectorAll('th');
|
||||
return Array.from(columns, column => column.innerText);
|
||||
});
|
||||
}, originalSelector);
|
||||
|
||||
const result = await container.evaluate((originalSelector) => {
|
||||
const rows = document.querySelectorAll(`${originalSelector} tr`);
|
||||
return Array.from(rows, row => {
|
||||
const columns = row.querySelectorAll('td');
|
||||
return Array.from(columns, column => column.innerText);
|
||||
});
|
||||
}, originalSelector);
|
||||
|
||||
header = [];
|
||||
for (let i = 0; i < resultH[0].length; i++) {
|
||||
header[i] = resultH[0][i];
|
||||
}
|
||||
|
||||
rows = [];
|
||||
rows[0] = header;
|
||||
for (let i = 1; i < result.length; i++) {
|
||||
rows[i] = result[i];
|
||||
}
|
||||
|
||||
|
||||
} else if (file['cTag']) {
|
||||
|
||||
const gbaiName = `${this.min.botId}.gbai`;
|
||||
const localName = Path.join('work', gbaiName, 'cache', `csv${GBAdminService.getRndReadableIdentifier()}.csv`);
|
||||
const url = file['@microsoft.graph.downloadUrl'];
|
||||
const response = await request({ uri: url, encoding: null });
|
||||
Fs.writeFileSync(localName, response, { encoding: null });
|
||||
|
||||
var workbook = new Excel.Workbook();
|
||||
const worksheet = await workbook.csv.readFile(localName);
|
||||
header = [];
|
||||
rows = [];
|
||||
|
||||
for (let i = 0; i < worksheet._rows.length; i++) {
|
||||
const r = worksheet._rows[i];
|
||||
let outRow = [];
|
||||
for (let j = 0; j < r._cells.length; j++) {
|
||||
outRow.push(r._cells[j].text);
|
||||
}
|
||||
|
||||
if (i == 0) {
|
||||
header = outRow;
|
||||
}
|
||||
else {
|
||||
rows.push(outRow);
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min);
|
||||
|
||||
let document
|
||||
document = await this.internalGetDocument(client, baseUrl, path, file);
|
||||
|
||||
// Creates workbook session that will be discarded.
|
||||
|
||||
let sheets = await client
|
||||
.api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets`)
|
||||
.get();
|
||||
|
||||
let results = await client
|
||||
results = await client
|
||||
.api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A1:Z${maxLines}')`)
|
||||
.get();
|
||||
|
||||
header = results.text[0];
|
||||
rows = results.text;
|
||||
}
|
||||
|
||||
let getFilter = async (text) => {
|
||||
let filter;
|
||||
const operators = [/\<\=/, /\>\=/, /\</, /\>/, /\bnot in\b/, /\bin\b/, /\=/];
|
||||
|
|
@ -632,7 +714,7 @@ export class SystemKeywords {
|
|||
// Increments columnIndex by looping until find a column match.
|
||||
|
||||
const filters = [];
|
||||
const header = results.text[0];
|
||||
|
||||
await CollectionUtil.asyncForEach(args, async arg => {
|
||||
const filter = await getFilter(arg);
|
||||
if (!filter) {
|
||||
|
|
@ -670,12 +752,12 @@ export class SystemKeywords {
|
|||
let foundIndex = 1;
|
||||
|
||||
// Fills the row variable.
|
||||
|
||||
for (; foundIndex < results.text.length; foundIndex++) {
|
||||
let rowCount = 0;
|
||||
for (; foundIndex < rows.length; foundIndex++) {
|
||||
let filterAcceptCount = 0;
|
||||
await CollectionUtil.asyncForEach(filters, async filter => {
|
||||
|
||||
let result = results.text[foundIndex][filter.columnIndex];
|
||||
let result = rows[foundIndex][filter.columnIndex];
|
||||
let wholeWord = true;
|
||||
if (this.dk.user && this.dk.user.basicOptions && this.dk.user.basicOptions.wholeWord) {
|
||||
wholeWord = this.dk.user.basicOptions.wholeWord;
|
||||
|
|
@ -760,9 +842,13 @@ export class SystemKeywords {
|
|||
result = result.substr(1);
|
||||
}
|
||||
const resultDate = SystemKeywords.getDateFromLocaleString(result, contentLocale);
|
||||
if (filter.value['dateOnly']) {
|
||||
resultDate.setHours(0, 0, 0, 0);
|
||||
}
|
||||
if (resultDate) {
|
||||
switch (filter.operator) {
|
||||
case '=':
|
||||
|
||||
if (resultDate.getTime() == filter.value.getTime())
|
||||
filterAcceptCount++;
|
||||
break;
|
||||
|
|
@ -789,8 +875,9 @@ export class SystemKeywords {
|
|||
});
|
||||
|
||||
if (filterAcceptCount === filters.length) {
|
||||
rowCount++;
|
||||
let row = {};
|
||||
const xlRow = results.text[foundIndex];
|
||||
const xlRow = rows[foundIndex];
|
||||
for (let colIndex = 0; colIndex < xlRow.length; colIndex++) {
|
||||
const propertyName = header[colIndex];
|
||||
let value = xlRow[colIndex];
|
||||
|
|
@ -801,7 +888,8 @@ export class SystemKeywords {
|
|||
}
|
||||
row[propertyName] = value;
|
||||
}
|
||||
row['line'] = foundIndex + 1;
|
||||
row['line'] = rowCount + 1;
|
||||
row['originalLine'] = foundIndex + 1;
|
||||
table.push(row);
|
||||
}
|
||||
|
||||
|
|
@ -814,36 +902,137 @@ export class SystemKeywords {
|
|||
GBLog.info(`BASIC: FIND returned single result: ${table[0]}.`);
|
||||
return table[1];
|
||||
} else {
|
||||
GBLog.info(`BASIC: FIND returned multiple results (Count): ${table.length}.`);
|
||||
GBLog.info(`BASIC: FIND returned multiple results (Count): ${table.length - 1}.`);
|
||||
return table;
|
||||
}
|
||||
}
|
||||
|
||||
public static getDateFromLocaleString(date: any, contentLocale: any) {
|
||||
let ret = null;
|
||||
let parts = /^([0-3]?[0-9]).([0-3]?[0-9]).((?:[0-9]{2})?[0-9]{2})\s*(10|11|12|0?[1-9]):([0-5][0-9])/gi.exec(date);
|
||||
if (parts && parts[5]) {
|
||||
|
||||
switch (contentLocale) {
|
||||
case 'pt':
|
||||
return new Date(Number.parseInt(parts[3]), Number.parseInt(parts[2]) - 1, Number.parseInt(parts[1]),
|
||||
ret = new Date(Number.parseInt(parts[3]), Number.parseInt(parts[2]) - 1, Number.parseInt(parts[1]),
|
||||
Number.parseInt(parts[4]), Number.parseInt(parts[5]), 0, 0);
|
||||
break;
|
||||
case 'en':
|
||||
return new Date(Number.parseInt(parts[3]), Number.parseInt(parts[1]) - 1, Number.parseInt(parts[2]),
|
||||
ret = new Date(Number.parseInt(parts[3]), Number.parseInt(parts[1]) - 1, Number.parseInt(parts[2]),
|
||||
Number.parseInt(parts[4]), Number.parseInt(parts[5]), 0, 0);
|
||||
break;
|
||||
}
|
||||
|
||||
ret['dateOnly'] = false;
|
||||
}
|
||||
|
||||
parts = /^([0-3]?[0-9]).([0-3]?[0-9]).((?:[0-9]{2})?[0-9]{2})$/gi.exec(date);
|
||||
if (parts && parts[3]) {
|
||||
|
||||
switch (contentLocale) {
|
||||
case 'pt':
|
||||
return new Date(Number.parseInt(parts[2]), Number.parseInt(parts[1]) - 1, Number.parseInt(parts[3]), 0, 0, 0, 0);
|
||||
ret = new Date(Number.parseInt(parts[3]), Number.parseInt(parts[2]) - 1, Number.parseInt(parts[1]), 0, 0, 0, 0);
|
||||
break;
|
||||
case 'en':
|
||||
return new Date(Number.parseInt(parts[1]), Number.parseInt(parts[2]) - 1, Number.parseInt(parts[3]), 0, 0, 0, 0);
|
||||
ret = new Date(Number.parseInt(parts[3]), Number.parseInt(parts[1]) - 1, Number.parseInt(parts[2]), 0, 0, 0, 0);
|
||||
break;
|
||||
}
|
||||
|
||||
ret['dateOnly'] = true;
|
||||
}
|
||||
return null;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs the download to the .gbdrive Download folder.
|
||||
*
|
||||
* @example file = DOWNLOAD element, folder
|
||||
*/
|
||||
public async download(element, folder) {
|
||||
|
||||
const page = element['_page'];
|
||||
const container = element['_frame'] ? element['_frame'] : element['_page'];
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
await container.click(element.originalSelector);
|
||||
|
||||
const xRequest = await new Promise(resolve => {
|
||||
page.on('request', interceptedRequest => {
|
||||
interceptedRequest.abort(); //stop intercepting requests
|
||||
resolve(interceptedRequest);
|
||||
});
|
||||
});
|
||||
|
||||
const options = {
|
||||
encoding: null,
|
||||
method: xRequest['._method'],
|
||||
uri: xRequest['_url'],
|
||||
body: xRequest['_postData'],
|
||||
headers: xRequest['_headers']
|
||||
}
|
||||
|
||||
const cookies = await page.cookies();
|
||||
options.headers.Cookie = cookies.map(ck => ck.name + '=' + ck.value).join(';');
|
||||
GBLog.info(`BASIC: DOWNLOADING '${options.uri}...'`);
|
||||
|
||||
let local;
|
||||
let filename;
|
||||
if (options.uri.indexOf('file://') != -1) {
|
||||
local = url.fileURLToPath(options.uri);
|
||||
filename = Path.basename(local);
|
||||
}
|
||||
else {
|
||||
const getBasenameFormUrl = (urlStr) => {
|
||||
const url = new URL(urlStr)
|
||||
return Path.basename(url.pathname)
|
||||
};
|
||||
filename = getBasenameFormUrl(options.uri);
|
||||
}
|
||||
|
||||
let result: Buffer;
|
||||
if (local) {
|
||||
result = fs.readFileSync(local);
|
||||
} else {
|
||||
result = await request.get(options);
|
||||
}
|
||||
let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min);
|
||||
const botId = this.min.instance.botId;
|
||||
|
||||
// Normalizes all slashes.
|
||||
|
||||
folder = folder.replace(/\\/gi, '/');
|
||||
|
||||
// Determines full path at source and destination.
|
||||
|
||||
const root = urlJoin(`/${botId}.gbai/${botId}.gbdrive`);
|
||||
const dstPath = urlJoin(root, folder, filename);
|
||||
|
||||
// Checks if the destination contains subfolders that
|
||||
// need to be created.
|
||||
|
||||
folder = await this.createFolder(folder);
|
||||
|
||||
// Performs the conversion operation getting a reference
|
||||
// to the source and calling /content on drive API.
|
||||
let file;
|
||||
try {
|
||||
|
||||
file = await client
|
||||
.api(`${baseUrl}/drive/root:/${dstPath}:/content`)
|
||||
.put(result);
|
||||
|
||||
} catch (error) {
|
||||
|
||||
if (error.code === "nameAlreadyExists") {
|
||||
GBLog.info(`BASIC: DOWNLOAD destination file already exists: ${dstPath}.`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Creates a folder in the bot instance drive.
|
||||
*
|
||||
|
|
@ -854,7 +1043,7 @@ export class SystemKeywords {
|
|||
|
||||
let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min);
|
||||
const botId = this.min.instance.botId;
|
||||
let path = `/${botId}.gbai/${botId}.gbdata`;
|
||||
let path = `/${botId}.gbai/${botId}.gbdrive`;
|
||||
|
||||
// Extracts each part of path to call create folder to each
|
||||
// one of them.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue