From 3249da0c93c2959c887916b94969b28661c3381f Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Fri, 16 Aug 2024 12:20:27 -0300 Subject: [PATCH] new(basic.gblib): New batch features. --- packages/basic.gblib/services/GBVMService.ts | 2 +- .../basic.gblib/services/SystemKeywords.ts | 1372 ----------------- 2 files changed, 1 insertion(+), 1373 deletions(-) diff --git a/packages/basic.gblib/services/GBVMService.ts b/packages/basic.gblib/services/GBVMService.ts index e5ded8ab..8a8cc2cd 100644 --- a/packages/basic.gblib/services/GBVMService.ts +++ b/packages/basic.gblib/services/GBVMService.ts @@ -542,7 +542,7 @@ export class GBVMService extends GBService { let col = 1; let index = 1; - const mid(str, start, length) { + const mid = (str, start, length) => { start = start - 1; if (length === undefined) { return str.substring(start); diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index c25b69cc..9235e33d 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -80,1378 +80,6 @@ import { GBUtil } from '../../../src/util.js'; * BASIC system class for extra manipulation of bot behaviour. */ export class SystemKeywords { - /** - * @tags System - */ - public async callVM({ pid, text }) { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - const step = null; - const deployer = null; - - return await GBVMService.callVM(text, min, step, pid, false, [text]); - } - - public async append({ pid, args }) { - let array = [].concat(...args); - return array.filter(function (item, pos) { - return item; - }); - } - - /** - * - * @example SEE CAPTION OF url AS variable - * - */ - public async seeCaption({ pid, url }) { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - const computerVisionClient = new ComputerVisionClient.ComputerVisionClient( - new ApiKeyCredentials.ApiKeyCredentials({ inHeader: { 'Ocp-Apim-Subscription-Key': process.env.VISION_KEY } }), - process.env.VISION_ENDPOINT - ); - - let caption = (await computerVisionClient.describeImage(url)).captions[0]; - - const contentLocale = min.core.getParam( - min.instance, - 'Default Content Language', - GBConfigService.get('DEFAULT_CONTENT_LANGUAGE') - ); - GBLogEx.info(min, `GBVision (caption): '${caption.text}' (Confidence: ${caption.confidence.toFixed(2)})`); - - return await min.conversationalService.translate(min, caption.text, contentLocale); - } - - /** - * - * @example SEE TEXT OF url AS variable - * - */ - public async seeText({ pid, url }) { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - const computerVisionClient = new ComputerVisionClient.ComputerVisionClient( - new ApiKeyCredentials.ApiKeyCredentials({ inHeader: { 'Ocp-Apim-Subscription-Key': process.env.VISION_KEY } }), - process.env.VISION_ENDPOINT - ); - - const result = await computerVisionClient.recognizePrintedText(true, url); - const text = result.regions[0].lines[0].words[0].text; - let final = ''; - - for (let i = 0; i < result.regions.length; i++) { - const region = result.regions[i]; - - for (let j = 0; j < region.lines.length; j++) { - const line = region.lines[j]; - - for (let k = 0; k < line.words.length; k++) { - final += `${line.words[k].text} `; - } - } - } - - GBLogEx.info(min, `GBVision (text): '${final}'`); - return final; - } - - public async sortBy({ pid, array, memberName }) { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - memberName = memberName.trim(); - const contentLocale = min.core.getParam( - min.instance, - 'Default Content Language', - GBConfigService.get('DEFAULT_CONTENT_LANGUAGE') - ); - - // Detects data type from the first element of array. - - let dt = array[0] ? array[0][memberName] : null; - let date = SystemKeywords.getDateFromLocaleString(pid, dt, contentLocale); - if (date) { - return array - ? array.sort((a, b) => { - const c = new Date(a[memberName]); - const d = new Date(b[memberName]); - return c.getTime() - d.getTime(); - }) - : null; - } else { - return array - ? array.sort((a, b) => { - if (a[memberName] < b[memberName]) { - return -1; - } - if (a[memberName] > b[memberName]) { - return 1; - } - return 0; - }) - : array; - } - } - - public static JSONAsGBTable(data, headers) { - try { - let output = []; - let isObject = false; - if (data[0].gbarray) { - return data; - } // Already GB Table. - if (Array.isArray(data)) { - isObject = Object.keys(data[1]) !== null; - } else { - isObject = true; - } - - if (isObject || JSON.parse(data) !== null) { - // Copies data from JSON format into simple array. - - if (!Array.isArray(data)) { - // If data is a single object, wrap it in an array - data = [data]; - } - - // Ensure that keys is an array of strings representing the object keys - const keys = Object.keys(data[0]); - - if (headers) { - output[0] = []; - - // Copies headers as the first element. - - for (let i = 0; i < keys.length; i++) { - output[0][i] = keys[i]; - } - } else { - output.push({ gbarray: '0' }); - } - - for (let i = 0; i < data.length; i++) { - output[i + 1] = []; - for (let j = 0; j < keys.length; j++) { - output[i + 1][j] = data[i][keys[j]]; - } - } - return output; - } - } catch (error) { - GBLog.error(error); - return data; - } - } - - /** - * - * @param data - * @param renderImage - * @returns - * - * @see http://tabulator.info/examples/5.2 - */ - private async renderTable(pid, data, renderPDF, renderImage) { - if (data.length && !data[1]) { - return null; - } - - data = SystemKeywords.JSONAsGBTable(data, true); - - // Detects if it is a collection with repeated - // headers. - - const { min, user } = await DialogKeywords.getProcessInfo(pid); - const gbaiName = DialogKeywords.getGBAIPath(min.botId); - const browser = await GBSSR.createBrowser(null); - const page = await browser.newPage(); - await page.minimize(); - - // Includes the associated CSS related to current theme. - - const theme: string = await DialogKeywords.getOption({ pid, name: 'theme', root: true }); - switch (theme) { - case 'white': - await page.addStyleTag({ path: 'node_modules/tabulator-tables/dist/css/tabulator_simple.min.css' }); - break; - case 'dark': - await page.addStyleTag({ path: 'node_modules/tabulator-tables/dist/css/tabulator_midnight.min.css' }); - break; - case 'blue': - await page.addStyleTag({ path: 'node_modules/tabulator-tables/dist/css/tabulator_modern.min.css' }); - break; - default: - break; - } - - await page.addScriptTag({ path: 'node_modules/tabulator-tables/dist/js/tabulator.min.js' }); - - // Removes internal hidden element used to hold one-based index arrays. - - data.shift(); - - // Guess fields from data variable into Tabulator fields collection. - - let fields = []; - let keys = Object.keys(data[0]); - for (let i = 0; i < keys.length; i++) { - fields.push({ field: keys[i], title: keys[i] }); - } - - // Adds DIV for Tabulator. - - await page.evaluate(() => { - const el = document.createElement('div'); - el.setAttribute('id', 'table'); - document.body.appendChild(el); - }); - - const code = ` - var table = new Tabulator("#table", { - height:"auto", - layout:"fitDataStretch", - data: ${JSON.stringify(data)}, - columns: ${JSON.stringify(fields)} - }); - `; - await page.evaluate(code); - await page.waitForSelector('#table'); - - // Handles image generation. - - let url; - let localName; - if (renderImage) { - localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`); - await page.screenshot({ path: localName, fullPage: true }); - url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName)); - GBLogEx.info(min, `BASIC: Table image generated at ${url} .`); - } - - // Handles PDF generation. - - if (renderPDF) { - localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.pdf`); - url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName)); - let pdf = await page.pdf({ format: 'A4' }); - GBLogEx.info(min, `BASIC: Table PDF generated at ${url} .`); - } - - await browser.close(); - return { url, localName }; - } - - public async closeHandles({ pid }) { - delete this.cachedMerge[pid]; - } - - public async asPDF({ pid, data }) { - let file = await this.renderTable(pid, data, true, false); - return file; - } - - public async asImage({ pid, data }) { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - - // Checks if it is a GBFILE. - - if (data.data) { - const gbfile = data.data; - - let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min); - const botId = min.instance.botId; - const gbaiName = DialogKeywords.getGBAIPath(min.botId); - const tmpDocx = urlJoin(gbaiName, `${botId}.gbdrive`, `tmp${GBAdminService.getRndReadableIdentifier()}.docx`); - - // Performs the conversion operation. - - await client.api(`${baseUrl}/drive/root:/${tmpDocx}:/content`).put(data.data); - const res = await client.api(`${baseUrl}/drive/root:/${tmpDocx}:/content?format=pdf`).get(); - await client.api(`${baseUrl}/drive/root:/${tmpDocx}:/content`).delete(); - - const streamToBuffer = stream => { - const chunks = []; - return new Promise((resolve, reject) => { - stream.on('data', chunk => chunks.push(chunk)); - stream.on('error', reject); - stream.on('end', () => resolve(Buffer.concat(chunks))); - }); - }; - - gbfile.data = await streamToBuffer(res); - - // Converts the PDF to PNG. - - const pngPages: PngPageOutput[] = await pdfToPng(gbfile.data, { - disableFontFace: false, - useSystemFonts: false, - viewportScale: 2.0, - pagesToProcess: [1], - strictPagesToProcess: false, - verbosityLevel: 0 - }); - - // Prepare an image on cache and return the GBFILE information. - - const localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`); - if (pngPages.length > 0) { - const buffer = pngPages[0].content; - const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName)); - - Fs.writeFileSync(localName, buffer, { encoding: null }); - - return { localName: localName, url: url, data: buffer }; - } - } else { - let file = await this.renderTable(pid, data, false, true); - return file; - } - } - - public async executeSQL({ pid, data, sql }) { - const { min } = await DialogKeywords.getProcessInfo(pid); - if (!data || !data[0]) { - return data; - } - let objectMode = false; - if (data[0].gbarray) { - objectMode = true; - } - - let first; - if (objectMode) { - first = data.shift(); - } - GBLogEx.info(min, `Executing SQL: ${sql}`); - data = alasql(sql, [data]); - if (objectMode) { - data.unshift(first); - } - return data; - } - - /** - * Retrives the content of a given URL. - */ - public async getFileContents({ pid, url, headers }) { - const options = { - method: 'GET', - encoding: 'binary', - headers: headers - }; - return await fetch(url, options); - } - - /** - * Retrives a random id with a length of five, every time it is called. - */ - public getRandomId() { - const idGeneration = '1v'; // TODO: this.dk['idGeneration']; - if (idGeneration && idGeneration.trim().toLowerCase() === 'number') { - return GBAdminService.getNumberIdentifier(); - } else { - return GBAdminService.getRndReadableIdentifier().substr(5); - } - } - - /** - * Retrives stock inforation for a given symbol. - */ - public async getStock({ pid, symbol }) { - const url = `http://live-nse.herokuapp.com/?symbol=${symbol}`; - let data = await fetch(url); - return data; - } - - /** - * Holds script execution for the number of seconds specified. - * - * @example WAIT 5 ' This will wait five seconds. - * - */ - public async wait({ pid, seconds }) { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - // tslint:disable-next-line no-string-based-set-timeout - GBLogEx.info(min, `BASIC: WAIT for ${seconds} second(s).`); - const timeout = async (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); - await timeout(seconds * 1000); - } - - /** - * Sends a text message to the mobile number specified. - * - * @example TALK TO "+199988887777", "Message text here" - * - */ - public async talkTo({ pid, mobile, message }) { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - GBLogEx.info(min, `BASIC: Talking '${message}' to a specific user (${mobile}) (TALK TO). `); - await min.conversationalService.sendMarkdownToMobile(min, null, mobile, message); - } - - /** - * Get a user object from a alias. - * - * @example user = USER "someone" - * - */ - public async getUser({ pid, username }) { - const { min } = await DialogKeywords.getProcessInfo(pid); - let sec = new SecService(); - const user = await sec.getUserFromUsername(min.instance.instanceId, username); - - return { displayName: user.displayName, mobile: user.userSystemId, email: user.email }; - } - - /** - * Sends a SMS message to the mobile number specified. - * - * @example SEND SMS TO "+199988887777", "Message text here" - * - */ - public async sendSmsTo({ pid, mobile, message }) { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - GBLogEx.info(min, `BASIC: SEND SMS TO '${mobile}', message '${message}'.`); - await min.conversationalService.sendSms(min, mobile, message); - } - - /** - * 1. Defines a cell value in the tabular file. - * 2. Defines an element text on HTML page. - * - * @example SET "file.xlsx", "A2", 4500 - * - * @example SET page, "elementHTMLSelector", "text" - - */ - public async set({ pid, handle, file, address, value, name = null }): Promise { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - - // Handles calls for HTML stuff - - if (handle && WebAutomationServices.isSelector(file)) { - GBLogEx.info(min, `BASIC: Web automation SET ${file}' to '${address}' . `); - await new WebAutomationServices().setElementText({ pid, handle, selector: file, text: address }); - - return; - } - - // TODO: Add a semaphore between FILTER and SET. - - // Processes FILTER option to ensure parallel SET calls. - - const filter = await DialogKeywords.getOption({ pid, name: 'filter' }); - if (filter) { - const row = this.find({ pid, handle: null, args: [filter] }); - address += row['line']; - } - - // Handles calls for BASIC persistence on sheet files. - - GBLogEx.info(min, `BASIC: Defining '${address}' in '${file}' to '${value}' (SET). `); - - let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min); - - const botId = min.instance.botId; - const path = DialogKeywords.getGBAIPath(botId, 'gbdata'); - let document = await this.internalGetDocument(client, baseUrl, path, file); - let sheets = await client.api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets`).get(); - let body = { values: [[]] }; - - // Processes FILTER option to ensure parallel SET calls. - - let titleAddress; - - if (filter) { - // Transforms address number (col index) to letter based. - // Eg.: REM This is A column and index automatically specified by filter. - // SET file.xlsx, 1, 4000 - - if (KeywordsExpressions.isNumber(address)) { - address = `${this.numberToLetters(address)}`; - titleAddress = `${address}1:${address}1`; - } - - // Processes SET FILTER directive to calculate address. - - body.values[0][0] = 'id'; - const addressId = 'A1:A1'; - await client - .api( - `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${addressId}')` - ) - .patch(body); - - const row = await this.find({ pid, handle: null, args: [file, filter] }); - if (row) { - address += row['line']; // Eg.: "A" + 1 = "A1". - } - } - address = address.indexOf(':') !== -1 ? address : address + ':' + address; - - if (titleAddress) { - body.values[0][0] = name.trim().replace(/[^a-zA-Z]/gi, ''); - - await client - .api( - `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${titleAddress}')` - ) - .patch(body); - } - - body.values[0][0] = value; - await client - .api( - `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${address}')` - ) - .patch(body); - } - - /** - * Retrives a document from the drive, given a path and filename. - */ - public async internalGetDocument(client: any, baseUrl: any, path: string, file: string) { - let res = await client.api(`${baseUrl}/drive/root:/${path}:/children`).get(); - - let documents = res.value.filter(m => { - return m.name.toLowerCase() === file.toLowerCase(); - }); - - if (!documents || documents.length === 0) { - throw new Error( - `File '${file}' specified on GBasic command not found. Check the .gbdata or the .gbdialog associated.`, - { cause: 404 } - ); - } - - return documents[0]; - } - - /** - * Saves the content of variable into the file in .gbdata default folder. - * - * @exaple SAVE variable as "my.txt" - * - */ - public async saveFile({ pid, file, data }): Promise { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - GBLogEx.info(min, `BASIC: Saving '${file}' (SAVE file).`); - let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min); - const botId = min.instance.botId; - const path = DialogKeywords.getGBAIPath(min.botId, `gbdrive`); - - // Checks if it is a GB FILE object. - - if (data.data && data.filename) { - data = data.data; - } - - try { - data = GBServer.globals.files[data].data; // TODO - await client.api(`${baseUrl}/drive/root:/${path}/${file}:/content`).put(data); - } catch (error) { - if (error.code === 'itemNotFound') { - GBLogEx.info(min, `BASIC: BASIC source file not found: ${file}.`); - } else if (error.code === 'nameAlreadyExists') { - GBLogEx.info(min, `BASIC: BASIC destination file already exists: ${file}.`); - } - throw error; - } - } - - /** - * Saves the content of variable into BLOB storage. - * - * MSFT uses MD5, see https://katelynsills.com/law/the-curious-case-of-md5. - * - * @exaple UPLOAD file. - * - */ - public async uploadFile({ pid, file }): Promise { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - GBLogEx.info(min, `BASIC: UPLOAD '${file.name}' ${file.size} bytes.`); - - // Checks if it is a GB FILE object. - - const accountName = min.core.getParam(min.instance, 'Blob Account'); - const accountKey = min.core.getParam(min.instance, 'Blob Key'); - - const sharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); - const baseUrl = `https://${accountName}.blob.core.windows.net`; - - const blobServiceClient = new BlobServiceClient(`${baseUrl}`, sharedKeyCredential); - - // It is an SharePoint object that needs to be downloaded. - - const gbaiName = DialogKeywords.getGBAIPath(min.botId); - const localName = Path.join('work', gbaiName, 'cache', `${GBAdminService.getRndReadableIdentifier()}.tmp`); - const url = file['url']; - const response = await fetch(url); - - // Writes it to disk and calculate hash. - - const data = await response.arrayBuffer(); - Fs.writeFileSync(localName, Buffer.from(data), { encoding: null }); - const hash = new Uint8Array(md5.array(data)); - - // Performs uploading passing local hash. - - const container = blobServiceClient.getContainerClient(accountName); - const blockBlobClient: BlockBlobClient = container.getBlockBlobClient(file.path); - const res = await blockBlobClient.uploadFile(localName, { - blobHTTPHeaders: { - blobContentMD5: hash - } - }); - - // If upload is OK including hash check, removes the temporary file. - - if (res._response.status === 201 && new Uint8Array(res.contentMD5).toString() === hash.toString()) { - Fs.rmSync(localName); - - file['md5'] = hash.toString(); - - return file; - } else { - GBLog.error(`BASIC: BLOB HTTP ${res.errorCode} ${res._response.status} .`); - } - } - - /** - * Takes note inside a notes.xlsx of .gbdata. - * - * @example NOTE "text" - * - */ - public async note({ pid, text }): Promise { - await this.save({ pid, file: 'Notes.xlsx', args: [text] }); - } - - /** - * Saves variables to storage, not a worksheet. - * - */ - public async saveToStorageBatch({ pid, table, rows }): Promise { - const { min } = await DialogKeywords.getProcessInfo(pid); - GBLogEx.info(min, `BASIC: Saving batch to storage '${table}' (SAVE).`); - - if (rows.length === 0) { - return; - } - - const definition = this.getTableFromName(table, min); - const rowsDest = []; - - rows.forEach(row => { - const dst = {}; - let i = 0; - Object.keys(row).forEach(column => { - const field = column.charAt(0).toUpperCase() + column.slice(1); - dst[field] = row[column]; - i++; - }); - rowsDest.push(dst); - }); - - await retry( - async bail => { - await definition.bulkCreate(GBUtil.caseInsensitive(rowsDest)); - }, - { - retries: 5, - onRetry: err => { - GBLog.error(`Retrying SaveToStorageBatch due to: ${JSON.stringify(err)}.`); - } - } - ); - } - - /** - * Saves variables to storage, not a worksheet. - * - * @example SAVE "Billing", columnName1, columnName2 - * - */ - public async saveToStorage({ pid, table, fieldsValues, fieldsNames }): Promise { - if (!fieldsValues || fieldsValues.length === 0 || !fieldsValues[0]) { - return; - } - - const { min } = await DialogKeywords.getProcessInfo(pid); - GBLogEx.info(min, `BASIC: Saving to storage '${table}' (SAVE).`); - - const definition = this.getTableFromName(table, min); - - let dst = {}; - // Uppercases fields. - let i = 0; - Object.keys(fieldsValues).forEach(fieldSrc => { - const field = fieldsNames[i].charAt(0).toUpperCase() + fieldsNames[i].slice(1); - - dst[field] = fieldsValues[fieldSrc]; - - i++; - }); - - let item; - await retry( - async bail => { - item = await definition.create(dst); - }, - { - retries: 5, - onRetry: err => { - GBLog.error(`Retrying SaveToStorage due to: ${err.message}.`); - } - } - ); - return item; - } - - public async saveToStorageWithJSON({ pid, table, fieldsValues, fieldsNames }): Promise { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - GBLogEx.info(min, `BASIC: Saving to storage '${table}' (SAVE).`); - const minBoot = GBServer.globals.minBoot as any; - const definition = minBoot.core.sequelize.models[table]; - - let out = []; - let data = {}, - data2 = {}; - - // Flattern JSON to a table. - - data = this.flattenJSON(fieldsValues); - - // Uppercases fields. - - Object.keys(data).forEach(field => { - const field2 = field.charAt(0).toUpperCase() + field.slice(1); - data2[field2] = data[field]; - }); - - return await definition.create(data2); - } - - /** - * Saves the content of several variables to a new row in a tabular file. - * - * @example SAVE "customers.xlsx", name, email, phone, address, city, state, country - * - */ - public async save({ pid, file, args }): Promise { - if (!args) { - return; - } - - const { min } = await DialogKeywords.getProcessInfo(pid); - GBLogEx.info(min, `BASIC: Saving '${file}' (SAVE). Args: ${args.join(',')}.`); - let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min); - const botId = min.instance.botId; - const path = DialogKeywords.getGBAIPath(botId, 'gbdata'); - - let sheets; - let document; - try { - document = await this.internalGetDocument(client, baseUrl, path, file); - sheets = await client.api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets`).get(); - } catch (e) { - if (e.cause === 404) { - // Creates the file. - - const blank = Path.join(process.env.PWD, 'blank.xlsx'); - const data = Fs.readFileSync(blank); - await client.api(`${baseUrl}/drive/root:/${path}/${file}:/content`).put(data); - - // Tries to open again. - - document = await this.internalGetDocument(client, baseUrl, path, file); - sheets = await client.api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets`).get(); - } else { - throw e; - } - } - - let address; - let body = { values: [[]] }; - - // Processes FILTER option to ensure parallel SET calls. - - const filter = await DialogKeywords.getOption({ pid, name: 'filter' }); - if (filter) { - // Creates id row. - - body.values[0][0] = 'id'; - const addressId = 'A1:A1'; - await client - .api( - `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${addressId}')` - ) - .patch(body); - body.values[0][0] = undefined; - - // FINDs the filtered row to be updated. - - const row = await this.find({ pid, handle: null, args: [file, filter] }); - if (row) { - address = `A${row['line']}:${this.numberToLetters(args.length)}${row['line']}`; - } - } - - // Editing or saving detection. - - if (!address) { - await client - .api( - `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A2:DX2')/insert` - ) - .post({}); - address = `A2:${this.numberToLetters(args.length - 1)}2`; - } - - // Fills rows object to call sheet API. - - for (let index = 0; index < args.length; index++) { - let value = args[index]; - if (value && (await this.isValidDate({ pid, dt: value }))) { - value = `'${value}`; - } - - // If filter is defined, skips id column. - - body.values[0][filter ? index + 1 : index] = value; - } - - await retry( - async bail => { - const result = await client - .api( - `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${address}')` - ) - .patch(body); - - if (result.status != 200) { - GBLogEx.info(min, `Waiting 5 secs. before retrying HTTP ${result.status} GET: ${result.url}`); - await GBUtil.sleep(5 * 1000); - throw new Error(`BASIC: HTTP:${result.status} retry: ${result.statusText}.`); - } - }, - { - retries: 5, - onRetry: err => { - GBLog.error(`Retrying HTTP GET due to: ${err.message}.`); - } - } - ); - } - - /** - * Retrives the content of a cell in a tabular file. - * - * @example value = GET "file.xlsx", "A2" - * - */ - public async getHttp({ pid, file, addressOrHeaders, httpUsername, httpPs, qs, streaming }): Promise { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - if (file.startsWith('http')) { - return await this.getByHttp({ - pid, - url: file, - headers: addressOrHeaders, - username: httpUsername, - ps: httpPs, - qs - }); - } else { - GBLogEx.info(min, `BASIC: GET '${addressOrHeaders}' in '${file}'.`); - let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min); - const botId = min.instance.botId; - (''); - const path = DialogKeywords.getGBAIPath(botId, 'gbdata'); - - let document = await this.internalGetDocument(client, baseUrl, path, file); - - // Creates workbook session that will be discarded. - - let sheets = await client.api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets`).get(); - - let results = await client - .api( - `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${addressOrHeaders}')` - ) - .get(); - - let val = results.text[0][0]; - GBLogEx.info(min, `BASIC: Getting '${file}' (GET). Value= ${val}.`); - return val; - } - } - - public async isValidDate({ pid, dt }) { - const { min, user } = await DialogKeywords.getProcessInfo(pid); - const contentLocale = min.core.getParam( - min.instance, - 'Default Content Language', - GBConfigService.get('DEFAULT_CONTENT_LANGUAGE') - ); - - let date = SystemKeywords.getDateFromLocaleString(pid, dt, contentLocale); - if (!date) { - return false; - } - - if (!(date instanceof Date)) { - date = new Date(date); - } - - return !isNaN(date.valueOf()); - } - - public async isValidNumber({ pid, number }) { - return KeywordsExpressions.isNumber(number); - } - - public isValidHour({ pid, value }) { - return /^([01]?[0-9]|2[0-3]):[0-5][0-9]$/.test(value); - } - - public static async getFilter(text) { - let filter; - const operators = [/\<\=/, /\<\>/, /\>\=/, /\/, /\bnot in\b/, /\bin\b/, /\=/]; - let done = false; - await CollectionUtil.asyncForEach(operators, async op => { - var re = new RegExp(op, 'gi'); - const parts = text.split(re); - - if (parts.length === 2 && !done) { - filter = { - columnName: parts[0].trim(), - operator: op.toString().replace(/\\b/g, '').replace(/\//g, '').replace(/\\/g, '').replace(/\b/g, ''), - value: parts[1].trim() - }; - - // Swaps values and names in case of IN operators. - - if (filter.operator === 'not in' || filter.operator === 'in') { - const columnName = filter.columnName; - filter.columnName = filter.value; - filter.value = columnName; - } - - done = true; - } - }); - - return filter; - } - - /** - * Finds a value or multi-value results in a tabular file. - * - * @example - * - * rows = FIND "file.xlsx", "A2=active", "A2 < 12/06/2010 15:00" - * i = 1 - * do while i <= ubound(row) - * row = rows[i] - * send sms to "+" + row.mobile, "Hello " + row.name + "! " - * loop - * @see NPM package data-forge - * - */ - public async find({ pid, handle, args }): Promise { - const { min, user, params } = await DialogKeywords.getProcessInfo(pid); - const file = args[0]; - args.shift(); - - const botId = min.instance.botId; - const path = DialogKeywords.getGBAIPath(botId, 'gbdata'); - - // MAX LINES property. - - let maxLines = 5000; - if (params && params.maxLines) { - if (params.maxLines.toString().toLowerCase() !== 'default') { - maxLines = Number.parseInt(params.maxLines).valueOf(); - } - } else { - maxLines = maxLines; - } - GBLogEx.info(min, `BASIC: FIND running on ${file} (maxLines: ${maxLines}) and args: ${JSON.stringify(args)}...`); - - // Choose data sources based on file type (HTML Table, data variable or sheet file) - - let results; - let header, rows; - let page; - if (handle) { - page = WebAutomationServices.getPageByHandle(handle); - } - - if (handle && page['$eval'] && WebAutomationServices.isSelector(file)) { - const container = page['frame'] ? page['frame'] : page; - const originalSelector = file; - - // Transforms table - - const resultH = await container.evaluate(originalSelector => { - const rows = document.querySelectorAll(`${originalSelector} tr`); - return Array.from(rows, row => { - const columns = row.querySelectorAll('th'); - return Array.from(columns, column => column.innerText); - }); - }, originalSelector); - - const result = await container.evaluate(originalSelector => { - const rows = document.querySelectorAll(`${originalSelector} tr`); - return Array.from(rows, row => { - const columns = row.querySelectorAll('td'); - return Array.from(columns, column => column.innerText); - }); - }, originalSelector); - - header = []; - for (let i = 0; i < resultH[0].length; i++) { - header[i] = resultH[0][i]; - } - - rows = []; - rows[0] = header; - for (let i = 1; i < result.length; i++) { - rows[i] = result[i]; - } - } else if (file['cTag']) { - const gbaiName = DialogKeywords.getGBAIPath(min.botId); - const localName = Path.join('work', gbaiName, 'cache', `csv${GBAdminService.getRndReadableIdentifier()}.csv`); - const url = file['@microsoft.graph.downloadUrl']; - const response = await fetch(url); - Fs.writeFileSync(localName, Buffer.from(await response.arrayBuffer()), { encoding: null }); - - var workbook = new Excel.Workbook(); - const worksheet = await workbook.csv.readFile(localName); - header = []; - rows = []; - - for (let i = 0; i < worksheet.rowCount; i++) { - const r = worksheet.getRow(i + 1); - let outRow = []; - let hasValue = false; - for (let j = 0; j < r.cellCount; j++) { - const value = r.getCell(j + 1).text; - if (value) { - hasValue = true; - } - outRow.push(value); - } - - if (i == 0) { - header = outRow; - } else if (hasValue) { - rows.push(outRow); - } - } - } else if (file.indexOf('.xlsx') !== -1) { - let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min); - - let document; - document = await this.internalGetDocument(client, baseUrl, path, file); - - // Creates workbook session that will be discarded. - - let sheets = await client.api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets`).get(); - - results = await client - .api( - `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A1:CZ${maxLines}')` - ) - .get(); - - header = results.text[0]; - rows = results.text; - } else { - const t = this.getTableFromName(file, min); - - if (!t) { - throw new Error(`TABLE ${file} not found. Check TABLE keywords.`); - } - - const systemFilter = await SystemKeywords.getFilter(args[0]); - let filter = {}; - filter[systemFilter.columnName] = systemFilter.value; - const res = await t.findAll({ where: filter }); - - return res.length > 1 ? res : res[0]; - } - - const contentLocale = min.core.getParam( - min.instance, - 'Default Content Language', - GBConfigService.get('DEFAULT_CONTENT_LANGUAGE') - ); - - // Increments columnIndex by looping until find a column match. - - const filters = []; - let predefinedFilterTypes; - if (params.filterTypes) { - predefinedFilterTypes = params.filterTypes.split(','); - } - - let filterIndex = 0; - await CollectionUtil.asyncForEach(args, async arg => { - const filter = await SystemKeywords.getFilter(arg); - if (!filter) { - throw new Error(`BASIC: FIND filter has an error: ${arg} check this and publish .gbdialog again.`); - } - - let columnIndex = 0; - for (; columnIndex < header.length; columnIndex++) { - if (header[columnIndex].toLowerCase() === filter.columnName.toLowerCase()) { - break; - } - } - filter.columnIndex = columnIndex; - const fixed = predefinedFilterTypes ? predefinedFilterTypes[filterIndex] : null; - - if (this.isValidHour(filter.value)) { - filter.dataType = fixed ? fixed : 'hourInterval'; - } else if (await this.isValidDate({ pid, dt: filter.value })) { - filter.value = SystemKeywords.getDateFromLocaleString(pid, filter.value, contentLocale); - filter.dataType = fixed ? fixed : 'date'; - } else if (await this.isValidNumber({ pid, number: filter.value })) { - filter.value = Number.parseInt(filter.value); - filter.dataType = fixed ? fixed : 'number'; - } else { - filter.value = filter.value; - filter.dataType = fixed ? fixed : 'string'; - } - filters.push(filter); - filterIndex++; - }); - - // As BASIC uses arrays starting with 1 (one) as index, - // a ghost element is added at 0 (zero) position. - - let table = []; - table.push({ gbarray: '0' }); - let foundIndex = 1; - - // Fills the row variable. - - let rowCount = 0; - for (; foundIndex < rows.length; foundIndex++) { - let filterAcceptCount = 0; - await CollectionUtil.asyncForEach(filters, async filter => { - let result = rows[foundIndex][filter.columnIndex]; - let wholeWord = true; - if (user && params && params.wholeWord) { - wholeWord = params.wholeWord; - } - if (!result) { - return; - } - - switch (filter.dataType) { - case 'string': - const v1 = GBConversationalService.removeDiacritics(result.toLowerCase().trim()); - const v2 = GBConversationalService.removeDiacritics(filter.value.toLowerCase().trim()); - GBLogEx.info(min, `FIND filter: ${v1} ${filter.operator} ${v2}.`); - - switch (filter.operator) { - case '=': - if (v1 === v2) { - filterAcceptCount++; - } - break; - case '<>': - if (v1 !== v2) { - filterAcceptCount++; - } - break; - case 'not in': - if (v1.indexOf(v2) === -1) { - filterAcceptCount++; - } - break; - case 'in': - if (wholeWord) { - if (v1 === v2) { - filterAcceptCount++; - } - } else { - if (v1.indexOf(v2) > -1) { - filterAcceptCount++; - } - } - break; - } - break; - case 'number': - switch (filter.operator) { - case '=': - if (Number.parseInt(result) === filter.value) { - filterAcceptCount++; - } - break; - } - break; - - case 'hourInterval': - switch (filter.operator) { - case '=': - if (v1 === v2) { - filterAcceptCount++; - } - break; - case 'in': - const e = result.split(';'); - const hr = Number.parseInt(filter.value.split(':')[0]); - let lastHour = Number.parseInt(e[0]); - let found = false; - await CollectionUtil.asyncForEach(e, async hour => { - if (!found && lastHour <= hr && hr <= hour) { - filterAcceptCount++; - found = true; - } - lastHour = hour; - }); - break; - } - break; - - case 'date': - if (result.charAt(0) === "'") { - result = result.substr(1); - } - const resultDate = SystemKeywords.getDateFromLocaleString(pid, result, contentLocale); - if (resultDate) { - if (filter.value['dateOnly']) { - resultDate.setHours(0, 0, 0, 0); - } - switch (filter.operator) { - case '=': - if (resultDate.getTime() == filter.value.getTime()) filterAcceptCount++; - break; - case '<': - if (resultDate.getTime() < filter.value.getTime()) filterAcceptCount++; - break; - case '>': - if (resultDate.getTime() > filter.value.getTime()) filterAcceptCount++; - break; - case '<=': - if (resultDate.getTime() <= filter.value.getTime()) filterAcceptCount++; - break; - case '>=': - if (resultDate.getTime() >= filter.value.getTime()) filterAcceptCount++; - break; - } - break; - } - } - }); - - if (filterAcceptCount === filters.length) { - rowCount++; - let row = {}; - const xlRow = rows[foundIndex]; - let hasValue = false; - for (let colIndex = 0; colIndex < xlRow.length; colIndex++) { - const propertyName = header[colIndex].trim(); - - let value = xlRow[colIndex]; - if (value) { - hasValue = true; - value = value.trim(); - if (value.charAt(0) === "'") { - if (await this.isValidDate({ pid, dt: value.substr(1) })) { - value = value.substr(1); - } - } - } - - row[propertyName] = value; - } - row['ordinal'] = rowCount; - row['line'] = foundIndex + 1; - if (hasValue) { - table.push(row); - } - } - } - - const outputArray = await DialogKeywords.getOption({ pid, name: 'output' }); - - if (table.length === 1) { - GBLogEx.info(min, `BASIC: FIND returned no results (zero rows).`); - return null; - } else if (table.length === 2 && !outputArray) { - GBLogEx.info(min, `BASIC: FIND returned single result: ${table[0]}.`); - return table[1]; - } else { - GBLogEx.info(min, `BASIC: FIND returned multiple results (Count): ${table.length - 1}.`); - return table; - } - } - - public static getDateFromLocaleString(pid, date: any, contentLocale: any) { - let ret = null; - let parts = /^([0-3]?[0-9]).([0-3]?[0-9]).((?:[0-9]{2})?[0-9]{2})\s*(10|11|12|0?[1-9]):([0-5][0-9])/gi.exec(date); - if (parts && parts[5]) { - switch (contentLocale) { - case 'pt': - ret = new Date( - Number.parseInt(parts[3]), - Number.parseInt(parts[2]) - 1, - Number.parseInt(parts[1]), - Number.parseInt(parts[4]), - Number.parseInt(parts[5]), - 0, - 0 - ); - break; - case 'en': - ret = new Date( - Number.parseInt(parts[3]), - Number.parseInt(parts[1]) - 1, - Number.parseInt(parts[2]), - Number.parseInt(parts[4]), - Number.parseInt(parts[5]), - 0, - 0 - ); - break; - } - - ret['dateOnly'] = false; - } - - parts = /^([0-3]?[0-9]).([0-3]?[0-9]).((?:[0-9]{2})?[0-9]{2})$/gi.exec(date); - if (parts && parts[3]) { - switch (contentLocale) { - case 'pt': - ret = new Date( - Number.parseInt(parts[3]), - Number.parseInt(parts[2]) - 1, - Number.parseInt(parts[1]), - 0, - 0, - 0, - 0 - ); - break; - case 'en': - ret = new Date( - Number.parseInt(parts[3]), - Number.parseInt(parts[1]) - 1, - Number.parseInt(parts[2]), - 0, - 0, - 0, - 0 - ); - break; - } - - ret['dateOnly'] = true; - } - return ret; - } public async setSystemPrompt({ pid, text }) { let { min, user } = await DialogKeywords.getProcessInfo(pid);