diff --git a/package-lock.json b/package-lock.json index da77ce88..6f48ee73 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14448,6 +14448,11 @@ "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz", "integrity": "sha1-WuVUHQJGRdMqWPzdyc7s6nrjrC8=" }, + "node-tesseract-ocr": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-tesseract-ocr/-/node-tesseract-ocr-2.2.1.tgz", + "integrity": "sha512-Q9cD79JGpPNQBxbi1fV+OAsTxYKLpx22sagsxSyKbu1u+t6UarApf5m32uVc8a5QAP1Wk7fIPN0aJFGGEE9DyQ==" + }, "nodesecurity-npm-utils": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/nodesecurity-npm-utils/-/nodesecurity-npm-utils-6.0.0.tgz", diff --git a/package.json b/package.json index ad148672..25d7b4d7 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "ms-rest-azure": "3.0.0", "nexmo": "2.9.1", "node-cron": "3.0.0", + "node-tesseract-ocr": "^2.2.1", "npm": "7.21.0", "opn": "6.0.0", "pdf-extraction": "1.0.2", diff --git a/packages/basic.gblib/services/DialogKeywords.ts b/packages/basic.gblib/services/DialogKeywords.ts index c57ed9b4..c66f2892 100644 --- a/packages/basic.gblib/services/DialogKeywords.ts +++ b/packages/basic.gblib/services/DialogKeywords.ts @@ -42,8 +42,8 @@ import { SystemKeywords } from './SystemKeywords'; import { GBMinService } from '../../core.gbapp/services/GBMinService'; import { HubSpotServices } from '../../hubspot.gblib/services/HubSpotServices'; import { WhatsappDirectLine } from '../../whatsapp.gblib/services/WhatsappDirectLine'; -var DateDiff = require('date-diff'); - +const DateDiff = require('date-diff'); +const puppeteer = require('puppeteer'); /** * Base services of conversation to be called by BASIC which @@ -66,6 +66,16 @@ export class DialogKeywords { */ public user; + /** + * HTML browser for conversation over page interaction. + */ + browser: any; + + /** + * The number used in this execution for HEAR calls (useful for SET SCHEDULE). + */ + hrOn: string; + /** * When creating this keyword facade, a bot instance is * specified among the deployer service. @@ -74,6 +84,9 @@ export class DialogKeywords { this.min = min; this.user = user; this.internalSys = new SystemKeywords(min, deployer, this); + (async () => { + this.browser = await puppeteer.launch(); + }); } /** @@ -84,6 +97,115 @@ export class DialogKeywords { return this.internalSys; } + /** + * Returns the page object. + * + * @example x = GET PAGE + */ + public async getPage(step, url) { + const page = await this.browser.newPage(); + await page.goto(url); + return page; + } + + /** + * Find element on page DOM. + * + * @example GET page, "elementName", "text" + */ + private async getByIDOrName(page, elementName) { + return await page.$(`[name="${elementName}"]`); + } + + /** + * Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy. + * + * @example x = TODAY + */ + public async click(step, page, idOrName) { + const e = await this.getByIDOrName(page, idOrName); + + await Promise.all([ + page.waitForNavigation(), + page.click(e.name) + ]); + } + + + /** + * Returns the screenshot of page or element + * + * @example file = SCREENSHOT page + */ + public async screenshot(step, page, idOrName, localName) { + const e = await this.getByIDOrName(page, idOrName); + await e.screenshot({ path: localName }); + } + + /** + * Returns the screenshot of page or element + * + * @example file = SCREENSHOT page + */ + public async captcha(step, page, idOrName) { + const e = await this.getByIDOrName(page, idOrName); + + + + } + + /** + * Performs the download to the .gbdrive Download folder. + * + * @example file = DOWNLOAD page, "tableName", row + */ + public async download(step, page, idOrName, localName) { + + const e = await this.getByIDOrName(page, idOrName); + const context = await this.browser.newContext({ acceptDownloads: true }); + + var cells = e.rows[0].cells; + + const [download] = await Promise.all([ + page.waitForEvent('download'), + page.click(cells[0]) + ]); + + const path = await download.path(); + + console.log(path); + + } + + + + /** + * Types the text into the text field. + * + * @example TYPE page, "elementName", "text" + */ + public async type(step, page, idOrName, text) { + const e = await this.getByIDOrName(page, idOrName); + await e.type(text); + } + + /** + * Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy. + * + * @example x = TODAY + */ + public async getOCR(step, localFile) { + const tesseract = require("node-tesseract-ocr") + + const config = { + lang: "eng", + oem: 1, + psm: 3, + } + + return await tesseract.recognize(localFile, config); + } + /** * Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy. * @@ -122,6 +244,9 @@ export class DialogKeywords { * @example EXIT */ public async exit(step) { + if (this.browser) { + await this.browser.close(); + } await step.endDialog(); } @@ -151,7 +276,7 @@ export class DialogKeywords { * * @example list = FIND CONTACT "Sandra" */ - public async fndContact(name) { + public async fndContact(name) { let s = new HubSpotServices(null, null, process.env.HUBSPOT_KEY); return await s.searchContact(name); } @@ -323,13 +448,13 @@ export class DialogKeywords { ); const nowUTC = new Date(); - const now= typeof nowUTC === 'string' ? + const now = typeof nowUTC === 'string' ? new Date(nowUTC) : nowUTC; - - const nowText = now.toLocaleString(this.getContentLocaleWithCulture(contentLocale), - { timeZone: process.env.DEFAULT_TIMEZONE }); - + + const nowText = now.toLocaleString(this.getContentLocaleWithCulture(contentLocale), + { timeZone: process.env.DEFAULT_TIMEZONE }); + return /\b([0-9]|0[0-9]|1?[0-9]|2[0-3]):[0-5]?[0-9]/.exec(nowText)[0]; } @@ -411,6 +536,7 @@ export class DialogKeywords { this.user = user; } + /** * Returns the name of the user acquired by WhatsApp API. */ @@ -470,11 +596,40 @@ export class DialogKeywords { this.min.cbMap[idPromise] = {}; this.min.cbMap[idPromise].promise = promise; - const opts = { id: idPromise, previousResolve: previousResolve, kind: kind, args }; - if (previousResolve !== undefined) { - previousResolve(opts); - } else { - await step.beginDialog('/hear', opts); + let opts = { id: idPromise, previousResolve: previousResolve, kind: kind, args }; + + if (this.hrOn) { + + let sleep = ms => { + return new Promise(resolve => { + setTimeout(resolve, ms); + }); + }; + + // Waits for next message in HEAR delegated context. + + const mobile = await this.userMobile(step); + while (true){ + if (WhatsappDirectLine.state[mobile] === 3) + { + break; + } + sleep (5000); + } + const result = WhatsappDirectLine.lastMessage[mobile]; + opts = await promise(step, result); + + if (previousResolve !== undefined) { + previousResolve(opts); + } + } + else { + + if (previousResolve !== undefined) { + previousResolve(opts); + } else { + await step.beginDialog('/hear', opts); + } } } diff --git a/packages/basic.gblib/services/GBVMService.ts b/packages/basic.gblib/services/GBVMService.ts index 86112e77..3d130de7 100644 --- a/packages/basic.gblib/services/GBVMService.ts +++ b/packages/basic.gblib/services/GBVMService.ts @@ -189,12 +189,19 @@ export class GBVMService extends GBService { httpUsername = ""; httpPs = ""; - ${process.env.ENABLE_AUTH? `hear gbLogin as login`:``} + ${process.env.ENABLE_AUTH ? `hear gbLogin as login` : ``} ${code} `; // Keywords from General Bots BASIC. + + code = code.replace(/(\w+)\s*\=\s*get html\s*(.*)/gi, ($0, $1, $2, $3) => { + return `${$1} = sys().getPage ("${$2}")\n`; + }); + code = code.replace(/(set hear on)(\s*)(.*)/gi, ($0, $1, $2, $3) => { + return `hrOn = ${$3}\n`; + }); code = code.replace(/hear (\w+) as login/gi, ($0, $1) => { return `${$1} = hear("login")`; @@ -275,7 +282,7 @@ export class GBVMService extends GBService { code = code.replace(/(\w)\s*\=\s*active tasks/gi, ($0, $1) => { return `${$1} = getActiveTasks()\n`; }); - + code = code.replace(/(\w)\s*\=\s*append\s*(.*)/gi, ($0, $1, $2, $3) => { return `${$1} = sys().append(${$2})\n`; }); @@ -297,14 +304,20 @@ export class GBVMService extends GBService { }); code = code.replace(/(get stock for )(.*)/gi, ($0, $1, $2) => { - return `let stock = sys().getStock(${$2})`; + return `stock = sys().getStock(${$2})`; }); - code = code.replace(/(\w+)\s*\=\s*get\s(.*)/gi, ($0, $1, $2) => { + code = code.replace(/(\w+)\s*\=\s*get\s(.*)/gi, ($0, $1, $2, $3) => { if ($2.indexOf('http') !== -1) { - return `let ${$1} = sys().getByHttp (${$2}, headers, httpUsername, httpPs)`; + return `${$1} = sys().getByHttp (${$2}, headers, httpUsername, httpPs)`; } else { - return `let ${$1} = sys().get (${$2})`; + if ($2.indexOf(',') !== -1) { + const values = $2.split(','); + return `${$1} = getByIDOrName(${values[0]}, ${values[1]} )`; + } + else { + return `${$1} = sys().get (${$2})`; + } } }); @@ -349,7 +362,11 @@ export class GBVMService extends GBService { }); code = code.replace(/(\w+)\s*\=\s*post\s*(.*),\s*(.*)/gi, ($0, $1, $2, $3) => { - return `let ${$1} = sys().httpPost (${$2}, ${$3})`; + return `${$1} = sys().httpPost (${$2}, ${$3})`; + }); + + code = code.replace(/(\w+)\s*\=\s*download\s*(.*),\s*(.*)/gi, ($0, $1, $2, $3) => { + return `${$1} = sys().download (${$2}, ${$3})`; }); code = code.replace(/(create a bot farm using)(\s)(.*)/gi, ($0, $1, $2, $3) => { @@ -384,6 +401,10 @@ export class GBVMService extends GBService { return `sendFileTo (step, ${$3})\n`; }); + code = code.replace(/(click)(\s*)(.*)/gi, ($0, $1, $2, $3) => { + return `click (step, ${$3})\n`; + }); + code = code.replace(/(send file)(\s*)(.*)/gi, ($0, $1, $2, $3) => { return `sendFile (step, ${$3})\n`; }); @@ -625,7 +646,7 @@ export class GBVMService extends GBService { if (step.context.activity.channelId !== 'msteams' && process.env.ENABLE_AUTH) { GBLog.info('BASIC: Authenticating beforing running General Bots BASIC code.'); return await step.beginDialog('/auth'); - } + } } return await step.next(step.options); }, diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index c16d4dd2..dbdcb770 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -432,6 +432,8 @@ export class SystemKeywords { const botId = this.min.instance.botId; const path = `/${botId}.gbai/${botId}.gbdata`; + // TODO: if (typeof(file) === + let document = await this.internalGetDocument(client, baseUrl, path, file); let maxLines = 1000; if (this.dk.user && this.dk.user.basicOptions && this.dk.user.basicOptions.maxLines) { diff --git a/packages/core.gbapp/services/GBMinService.ts b/packages/core.gbapp/services/GBMinService.ts index 6cd7d09f..5d99c82b 100644 --- a/packages/core.gbapp/services/GBMinService.ts +++ b/packages/core.gbapp/services/GBMinService.ts @@ -163,7 +163,7 @@ export class GBMinService { setTimeout(resolve, ms); }); }; - await sleep(20000); + await sleep(1); res.status(200); res.end(); }); diff --git a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts index ecbdde69..a066d66d 100644 --- a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts +++ b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts @@ -52,6 +52,8 @@ export class WhatsappDirectLine extends GBService { public static mobiles = {}; public static chatIds = {}; public static usernames = {}; + public static state = {}; // 2: Waiting, 3: MessageArrived. + public static lastMessage = {}; // 2: Waiting, 3: MessageArrived. public pollInterval = 3000; public directLineClientName = 'DirectLineClient'; @@ -239,6 +241,7 @@ export class WhatsappDirectLine extends GBService { } } + await CollectionUtil.asyncForEach(this.min.appPackages, async (e: IGBPackage) => { await e.onExchangeData(this.min, 'whatsappMessage', message); @@ -285,6 +288,8 @@ export class WhatsappDirectLine extends GBService { const client = await this.directLineClient; + WhatsappDirectLine.lastMessage[from] = message; + // Check if this message is from a Human Agent itself. diff --git a/src/app.ts b/src/app.ts index d14a8938..05edcfc4 100644 --- a/src/app.ts +++ b/src/app.ts @@ -119,8 +119,6 @@ export class GBServer { const azureDeployer: AzureDeployerService = new AzureDeployerService(deployer); const adminService: GBAdminService = new GBAdminService(core); - - if (process.env.NODE_ENV === 'development' && !process.env.BOT_URL) { const proxy = GBConfigService.get('REVERSE_PROXY'); if (proxy !== undefined) {