fix(all): Fixes in production.

This commit is contained in:
rodrigorodriguez 2023-03-19 20:09:54 -03:00
parent 3e68858bb0
commit 56b687792b
15 changed files with 213 additions and 161 deletions

1
.vscode/launch.json vendored
View file

@ -15,6 +15,7 @@
"args": [
"--no-deprecation",
"--loader ts-node/esm",
"--openssl-legacy-provider",
"--require ${workspaceRoot}/suppress-node-warnings.cjs",
],
"skipFiles": [

0
gbot.sh Executable file → Normal file
View file

View file

@ -144,6 +144,7 @@
"punycode": "2.1.1",
"puppeteer": "19.6.3",
"puppeteer-extra": "3.3.4",
"puppeteer-extra-plugin-minmax": "1.1.2",
"puppeteer-extra-plugin-stealth": "2.11.1",
"qr-scanner": "1.4.2",
"qrcode": "1.5.1",

View file

@ -109,49 +109,6 @@ export class GBBasicPackage implements IGBPackage {
}
public async loadBot(min: GBMinInstance): Promise<void> {
const botId = min.botId;
const opts = {
pingSendTimeout: null,
keepAliveTimeout: null,
listeners: {
unsubscribed(subscriptions: number): void {},
subscribed(subscriptions: number): void {},
disconnected(remoteId: string, connections: number): void {},
connected(remoteId: string, connections: number): void {},
messageIn(...params): void {
GBLogEx.info(min, '[IN] ' + params);
},
messageOut(...params): void {
GBLogEx.info(min, '[OUT] ' + params);
}
}
};
function getRemoteId(ctx: Koa.Context) {
return '1'; // share a single session for now, real impl could use cookies or some other meaning for HTTP sessions
}
let instances: IGBInstance[];
instances = await min.core.loadInstances();
let proxies = {};
await CollectionUtil.asyncForEach(instances, async instance => {
const proxy = {
dk: new DialogKeywords(),
wa: new WebAutomationServices(),
sys: new SystemKeywords(),
dbg: new DebuggerService(),
img: new ImageProcessingServices()
};
proxies[instance.botId] = proxy;
});
GBServer.globals.server.dk = createRpcServer(
proxies,
createKoaHttpServer(GBVMService.API_PORT, getRemoteId, { prefix: `api/v3` }),
opts
);
GBLogEx.info(min, 'API RPC HTTP Server started at http://localhost:' + GBVMService.API_PORT);
GBServer.globals.debuggers[botId] = {};
GBServer.globals.debuggers[botId].state = 0;
GBServer.globals.debuggers[botId].breaks = [];

View file

@ -551,9 +551,9 @@ export class DialogKeywords {
}
private async setOption({ pid, name, value }) {
if (this.isUserSystemParam(name)) {
throw new Error(`Not possible to define ${name} as it is a reserved system param name.`);
}
// if (this.isUserSystemParam(name)) {
// throw new Error(`Not possible to define ${name} as it is a reserved system param name.`);
// }
let { min, user, params } = await DialogKeywords.getProcessInfo(pid);
const sec = new SecService();
await sec.setParam(user.userId, name, value);

View file

@ -297,11 +297,12 @@ export class GBVMService extends GBService {
}
public static normalizeQuotes(text: any) {
text = text.replace('¨', '"');
text = text.replace('“', '"');
text = text.replace('”', '"');
text = text.replace('', "'");
text = text.replace('', "'");
text = text.replace(/\¨/gm, '"');
text = text.replace(/\“/gm, '"');
text = text.replace(/\”/gm, '"');
text = text.replace(/\/gm, "'");
text = text.replace(/\/gm, "'");
return text;
}

View file

@ -32,6 +32,8 @@
'use strict';
import { GBVMService } from "./GBVMService.js";
/**
* Image processing services of conversation to be called by BASIC.
*/
@ -292,9 +294,9 @@ export class KeywordsExpressions {
keywords[i++] = [
/^\s*((?:[a-z]+.?)(?:(?:\w+).)(?:\w+)*)\s*=\s*find\s*(.*)\s*or talk\s*(.*)/gim,
($0, $1, $2, $3) => {
return `${$1} = await sys.find({pid: pid, handle: page, args:[${$2}])\n
if (!${$1}) {s
await dk.talk ({pid: pid, ${$3}})\n;
return `${$1} = await sys.find({pid: pid, handle: page, args:[${$2}]})\n
if (!${$1}) {
await dk.talk ({pid: pid, text: ${$3}})\n;
return -1;
}
`;
@ -601,7 +603,12 @@ export class KeywordsExpressions {
keywords[i++] = [
/^\s*(talk)(\s*)(.*)/gim,
($0, $1, $2, $3) => {
if ($3.substr(0, 1) !== '"') {
$3 = GBVMService.normalizeQuotes($3);
// Uses auto quote if this is a frase with more then one word.
if (/\s/.test($3) && $3.substr(0, 1) !== '"') {
$3 = `"${$3}"`;
}
return `await dk.talk ({pid: pid, text: ${$3}})`;
@ -755,15 +762,19 @@ export class KeywordsExpressions {
];
keywords[i++] = [
/^\s*save\s*(.*)\s*as\s*(.*)/gim,
/^\s*save\s*(\w+)\s*as\s*(.*)/gim,
($0, $1, $2, $3) => {
return `await sys.saveFile({pid: pid, file: ${$2}, data: ${$1}})`;
}
];
keywords[i++] = [
/^\s*(save)(\s*)(.*)/gim,
($0, $1, $2, $3) => {
return `await sys.save({pid: pid, args: [${$3}]})`;
/^\s*(save)(\s*)(.*\.xlsx)(.*)/gim,
($0, $1, $2, $3, $4) => {
$3 = $3.replace (/\'/g, "")
$3 = $3.replace (/\"/g, "")
$4 = $4.substr(2)
return `await sys.save({pid: pid,file: "${$3}" , args: [${$4}]})`;
}
];

View file

@ -236,6 +236,7 @@ export class SystemKeywords {
const gbaiName = DialogKeywords.getGBAIPath(min.botId);
const browser = await GBSSR.createBrowser(null);
const page = await browser.newPage();
await page.minimize();
// Includes the associated CSS related to current theme.
@ -544,7 +545,7 @@ export class SystemKeywords {
}
try {
data = GBServer.globals.files[data].data;
data = GBServer.globals.files[data].data; // TODO
await client.api(`${baseUrl}/drive/root:/${path}/${file}:/content`).put(data);
} catch (error) {
if (error.code === 'itemNotFound') {
@ -562,9 +563,8 @@ export class SystemKeywords {
* @exaple SAVE "customers.xlsx", name, email, phone, address, city, state, country
*
*/
public async save({ pid, args }): Promise<any> {
public async save({ pid,file, args }): Promise<any> {
const { min, user } = await DialogKeywords.getProcessInfo(pid);
const file = args[0];
args.shift();
GBLog.info(`BASIC: Saving '${file}' (SAVE). Args: ${args.join(',')}.`);
let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min);
@ -589,7 +589,7 @@ export class SystemKeywords {
const address = `A2:${this.numberToLetters(args.length - 1)}2`;
for (let index = 0; index < args.length; index++) {
let value = args[index];
if (value && this.isValidDate(value)) {
if (value && await this.isValidDate({pid, dt:value})) {
value = `'${value}`;
}
body.values[0][index] = value;
@ -699,8 +699,8 @@ export class SystemKeywords {
// MAX LINES property.
let maxLines;
if (user && params && params.maxLines) {
let maxLines = 5000;
if (params && params.maxLines) {
if (params.maxLines.toString().toLowerCase() !== 'default') {
maxLines = Number.parseInt(params.maxLines).valueOf();
}
@ -718,7 +718,7 @@ export class SystemKeywords {
page = WebAutomationServices.getPageByHandle(handle);
}
if (page['$eval'] && WebAutomationServices.isSelector(file)) {
if (handle &&page['$eval'] && WebAutomationServices.isSelector(file)) {
const container = page['frame'] ? page['frame'] : page;
const originalSelector = file;
@ -857,10 +857,10 @@ export class SystemKeywords {
if (this.isValidHour(filter.value)) {
filter.dataType = fixed ? fixed : 'hourInterval';
} else if (this.isValidDate(filter.value)) {
} else if (await this.isValidDate({pid, dt: filter.value})) {
filter.value = SystemKeywords.getDateFromLocaleString(pid, filter.value, contentLocale);
filter.dataType = fixed ? fixed : 'date';
} else if (this.isValidNumber(filter.value)) {
} else if (await this.isValidNumber({pid, number: filter.value})) {
filter.value = Number.parseInt(filter.value);
filter.dataType = fixed ? fixed : 'number';
} else {
@ -893,7 +893,7 @@ export class SystemKeywords {
switch (filter.dataType) {
case 'string':
const v1 = GBConversationalService.removeDiacritics(result.toLowerCase().trim());
const v2 = GBConversationalService.removeDiacritics(filter.toLowerCase().trim());
const v2 = GBConversationalService.removeDiacritics(filter.value.toLowerCase().trim());
switch (filter.operator) {
case '=':
@ -1003,7 +1003,7 @@ export class SystemKeywords {
const propertyName = header[colIndex];
let value = xlRow[colIndex];
if (value && value.charAt(0) === "'") {
if (this.isValidDate(value.substr(1))) {
if (await this.isValidDate({pid, dt:value.substr(1)})) {
value = value.substr(1);
}
}
@ -1650,7 +1650,7 @@ export class SystemKeywords {
const propertyName = header[colIndex];
let value = xlRow[colIndex];
if (value && value.charAt(0) === "'") {
if (this.isValidDate(value.substr(1))) {
if (await this.isValidDate({pid, dt:value.substr(1)})) {
value = value.substr(1);
}
}
@ -1702,13 +1702,13 @@ export class SystemKeywords {
}
}
} else {
let args = [file];
let args = [];
let keys = Object.keys(row);
for (let j = 0; j < keys.length; j++) {
args.push(row[keys[j]]);
}
await this.save({ pid, args });
await this.save({ pid,file, args });
adds++;
}
}

View file

@ -43,10 +43,10 @@ import * as en from 'dotenv-extended';
* Base configuration for the server like storage.
*/
export class GBConfigService {
public static getBoolean (value: string): boolean {
public static getBoolean(value: string): boolean {
return (this.get(value) as unknown) as boolean;
}
public static getServerPort (): string {
public static getServerPort(): string {
if (process.env.PORT) {
return process.env.PORT;
}
@ -57,7 +57,7 @@ export class GBConfigService {
return '4242';
}
public static init (): any {
public static init(): any {
try {
en.load({
encoding: 'utf8',
@ -78,7 +78,7 @@ export class GBConfigService {
}
}
public static get (key: string): string | undefined {
public static get(key: string): string | undefined {
let value = GBConfigService.tryGet(key);
if (value === undefined) {
@ -155,6 +155,9 @@ export class GBConfigService {
case 'ENABLE_SPELLING_CHECKER':
value = false;
break;
case 'DEV_GBAI':
value = undefined;
break;
default:
GBLog.warn(`Invalid key on .env file: '${key}'`);
break;
@ -164,7 +167,7 @@ export class GBConfigService {
return value;
}
public static tryGet (key: string): any {
public static tryGet(key: string): any {
let value = process.env[`container:${key}`];
if (value === undefined) {
value = process.env[key];

View file

@ -900,6 +900,7 @@ export class GBConversationalService {
}
public async translate(min: GBMinInstance, text: string, language: string): Promise<string> {
const translatorEnabled = () => {
if (min.instance.params) {
const params = JSON.parse(min.instance.params);

View file

@ -77,10 +77,11 @@ export class GBLogEx {
*/
public static async log(instance: IGBInstance, kind: string, message: string): Promise<GuaribasLog> {
message = message ? message.substring(0, 1023) : null;
return await GuaribasLog.create(<GuaribasLog>{
instanceId: instance ? instance.instanceId : 1,
message: message,
kind: kind
});
// return await GuaribasLog.create(<GuaribasLog>{
// instanceId: instance ? instance.instanceId : 1,
// message: message,
// kind: kind
// });
return null;
}
}

View file

@ -48,6 +48,11 @@ import mkdirp from 'mkdirp';
import Fs from 'fs';
import arrayBufferToBuffer from 'arraybuffer-to-buffer';
import { NlpManager } from 'node-nlp';
import Koa from 'koa';
import cors from '@koa/cors';
import { createRpcServer } from '@push-rpc/core';
import { createHttpKoaMiddleware } from '@push-rpc/http';
import { HttpServerOptions } from '@push-rpc/http/dist/server.js';
import {
AutoSaveStateMiddleware,
BotFrameworkAdapter,
@ -88,6 +93,11 @@ import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js';
import Path from 'path';
import { GBSSR } from './GBSSR.js';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
import { GBLogEx } from './GBLogEx.js';
import { WebAutomationServices } from '../../basic.gblib/services/WebAutomationServices.js';
import { createKoaHttpServer } from '../../basic.gblib/index.js';
import { DebuggerService } from '../../basic.gblib/services/DebuggerService.js';
import { ImageProcessingServices } from '../../basic.gblib/services/ImageProcessingServices.js';
/**
* Minimal service layer for a bot and encapsulation of BOT Framework calls.
@ -156,6 +166,7 @@ export class GBMinService {
// Calls mountBot event to all bots.
let i = 1;
if (instances.length > 1) {
this.bar1 = new cliProgress.SingleBar(
{
@ -168,45 +179,65 @@ export class GBMinService {
this.bar1.start(instances.length, i, { botId: 'Boot' });
}
const throttledPromiseAll = async promises => {
const MAX_IN_PROCESS = 20;
const results = new Array(promises.length);
async function doBlock(startIndex) {
// Shallow-copy a block of promises to work on
const currBlock = promises.slice(startIndex, startIndex + MAX_IN_PROCESS);
// Await the completion. If any fail, it will throw and that's good.
const blockResults = await Promise.all(currBlock);
// Assuming all succeeded, copy the results into the results array
for (let ix = 0; ix < blockResults.length; ix++) {
results[ix + startIndex] = blockResults[ix];
}
}
for (let iBlock = 0; iBlock < promises.length; iBlock += MAX_IN_PROCESS) {
await doBlock(iBlock);
}
return results;
};
await throttledPromiseAll(
instances.map(
(async instance => {
await CollectionUtil.asyncForEach(instances, (async instance => {
try {
await this['mountBot'](instance);
if (this.bar1) {
this.bar1.update(i++, { botId: instance.botId });
}
} catch (error) {
GBLog.error(`Error mounting bot ${instance.botId}: ${error.message}\n${error.stack}`);
}
}).bind(this)
)
);
finally {
if (this.bar1) {
this.bar1.update(i++, { botId: instance.botId });
}
}
}).bind(this));
if (this.bar1) {
this.bar1.stop();
}
const opts = {
pingSendTimeout: null,
keepAliveTimeout: null,
listeners: {
unsubscribed(subscriptions: number): void { },
subscribed(subscriptions: number): void { },
disconnected(remoteId: string, connections: number): void { },
connected(remoteId: string, connections: number): void { },
messageIn(...params): void {
GBLogEx.info(0, '[IN] ' + params);
},
messageOut(...params): void {
GBLogEx.info(0, '[OUT] ' + params);
}
}
};
function getRemoteId(ctx: Koa.Context) {
return '1'; // share a single session for now, real impl could use cookies or some other meaning for HTTP sessions
}
let proxies = {};
await CollectionUtil.asyncForEach(instances, async instance => {
const proxy = {
dk: new DialogKeywords(),
wa: new WebAutomationServices(),
sys: new SystemKeywords(),
dbg: new DebuggerService(),
img: new ImageProcessingServices()
};
proxies[instance.botId] = proxy;
});
GBServer.globals.server.dk = createRpcServer(
proxies,
createKoaHttpServer(GBVMService.API_PORT, getRemoteId, { prefix: `api/v3` }),
opts
);
GBLogEx.info(0, 'API RPC HTTP Server started.');
// // Loads schedules.
// GBLog.info(`Preparing SET SCHEDULE dialog calls...`);
@ -255,7 +286,7 @@ export class GBMinService {
/**
* Unmounts the bot web site (default.gbui) secure domain, if any.
*/
public async unloadDomain(instance: IGBInstance) {}
public async unloadDomain(instance: IGBInstance) { }
/**
* Mount the instance by creating an BOT Framework bot object,
@ -332,7 +363,7 @@ export class GBMinService {
// Loads Named Entity data for this bot.
await KBService.RefreshNER(min);
// TODO: await KBService.RefreshNER(min);
// Calls the loadBot context.activity for all packages.
@ -452,7 +483,7 @@ export class GBMinService {
if (min.whatsAppDirectLine != undefined && instance.whatsappServiceKey !== null) {
if (!(await min.whatsAppDirectLine.check(min))) {
const error = `WhatsApp API lost connection.`;
const error = `WhatsApp API lost connection for: ${min.botId}.`;
GBLog.error(error);
res.status(500).send(error);
@ -532,8 +563,7 @@ export class GBMinService {
min.instance.authenticatorTenant,
'/oauth2/authorize'
);
authorizationUrl = `${authorizationUrl}?response_type=code&client_id=${
min.instance.marketplaceId
authorizationUrl = `${authorizationUrl}?response_type=code&client_id=${min.instance.marketplaceId
}&redirect_uri=${urlJoin(min.instance.botEndpoint, min.instance.botId, 'token')}`;
GBLog.info(`HandleOAuthRequests: ${authorizationUrl}.`);
res.redirect(authorizationUrl);
@ -743,7 +773,6 @@ export class GBMinService {
await min.whatsAppDirectLine.setup(true);
} else {
const minBoot = GBServer.globals.minBoot as any;
if (minBoot.whatsappServiceKey) {
min.whatsAppDirectLine = new WhatsappDirectLine(
min,
min.botId,
@ -755,7 +784,6 @@ export class GBMinService {
);
await min.whatsAppDirectLine.setup(false);
}
}
// Setups default BOT Framework dialogs.
@ -1035,8 +1063,7 @@ export class GBMinService {
await this.processEventActivity(min, user, context, step);
}
} catch (error) {
const msg = `ERROR: ${error.message} ${error.error ? error.error.body : ''} ${
error.error ? (error.error.stack ? error.error.stack : '') : ''
const msg = `ERROR: ${error.message} ${error.error ? error.error.body : ''} ${error.error ? (error.error.stack ? error.error.stack : '') : ''
}`;
GBLog.error(msg);
@ -1053,7 +1080,7 @@ export class GBMinService {
try {
await adapter['processActivity'](req, res, handler);
} catch (error) {
if (error.code === 401){
if (error.code === 401) {
GBLog.error('Calling processActivity due to Signing Key could not be retrieved error.');
await adapter['processActivity'](req, res, handler);
}

View file

@ -125,9 +125,11 @@ export class GBSSR {
};
}
public static async createBrowser(profilePath): Promise<any> {
const opts = this.preparePuppeteer(profilePath);
puppeteer.use(hidden());
puppeteer.use(require("puppeteer-extra-plugin-minmax")());
const browser = await puppeteer.launch(opts);
return browser;
}
@ -143,6 +145,8 @@ export class GBSSR {
try {
const page = await browser.newPage();
await page.minimize();
await page.setUserAgent(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
);
@ -277,35 +281,61 @@ export class GBSSR {
req.originalUrl
);
// Reads from static HTML when a bot is crawling.
// Tries to find botId from URL.
const botId =
req.originalUrl || req.originalUrl === '/' ? req.originalUrl.substr(1) : GBServer.globals.minInstances[0].botId;
const minBoot = GBServer.globals.minInstances[0];
let botId =
req.originalUrl && req.originalUrl === '/' ?
minBoot.botId :
/\/([A-Za-z0-9\-\_]+)\/*/.exec(req.originalUrl)[1]
let min: GBMinInstance =
req.url === '/'
? GBServer.globals.minInstances[0]
: GBServer.globals.minInstances.filter(p => p.instance.botId === botId)[0];
? minBoot
: GBServer.globals.minInstances.filter(p => p.instance.botId.toLowerCase() === botId.toLowerCase())[0];
if (!min) {
min = req.url === '/'
? minBoot
: GBServer.globals.minInstances.filter(p => p.instance.activationCode.toLowerCase() === botId.toLowerCase())[0];
}
if (!min) {
botId = minBoot.botId;
}
let path = DialogKeywords.getGBAIPath(botId, `gbui`);
// Checks if the bot has an .gbui published or use default.gbui.
if (!Fs.existsSync(path)) {
path = DialogKeywords.getGBAIPath(minBoot.botId, `gbui`);
}
const url = req.url.replace(`/${botId}`, '');
if (min && req.originalUrl && prerender && exclude) {
// Reads from static HTML when a bot is crawling.
path = Path.join(process.env.PWD, 'work', path, 'index.html');
const html = Fs.readFileSync(path, 'utf8');
res.status(200).send(html);
return true;
} else {
// Servers default.gbui web application.
path = Path.join(
process.env.PWD,
GBDeployer.deployFolder,
GBMinService.uiPackage,
'build',
min ? `index.html` : req.url
url === '/' || url === '' ? `index.html` : url
);
if (Fs.existsSync(path)) {
if (min) {
let html = Fs.readFileSync(path, 'utf8');
html = html.replace(/\{botId\}/gi, min.botId);
html = html.replace(/\{theme\}/gi, min.instance.theme);
html = html.replace(/\{theme\}/gi, min.instance.theme ? min.instance.theme :
'default.gbtheme');
html = html.replace(/\{title\}/gi, min.instance.title);
res.send(html).end();
} else {

View file

@ -172,7 +172,7 @@ export class WhatsappDirectLine extends GBService {
});
client.on('ready', async () => {
GBLog.verbose(`GBWhatsApp: Emptying chat list for ${this.botId}...`);
// TODO: await client.pupPage['minimize']();
// Keeps the chat list cleaned.
const chats = await client.getChats();
await CollectionUtil.asyncForEach(chats, async chat => {
@ -192,8 +192,15 @@ export class WhatsappDirectLine extends GBService {
});
client.initialize();
};
if (setUrl){
createClient.bind(this)();
}
else
{
this.customClient = minBoot.whatsAppDirectLine.customClient;
}
setUrl = false;
break;
case 'chatapi':
options = {
@ -650,8 +657,7 @@ export class WhatsappDirectLine extends GBService {
await this.printMessages(response.obj.activities, conversationId, from, fromName);
} catch (err) {
GBLog.error(
`Error calling printMessages on Whatsapp channel ${err.data === undefined ? err : err.data} ${
err.errObj ? err.errObj.message : ''
`Error calling printMessages on Whatsapp channel ${err.data === undefined ? err : err.data} ${err.errObj ? err.errObj.message : ''
}`
);
}
@ -941,7 +947,7 @@ export class WhatsappDirectLine extends GBService {
id = req.from.split('@')[0];
senderName = req._data.notifyName;
text = req.body;
botId = this.botId;
break;
case 'chatapi':
@ -992,6 +998,9 @@ export class WhatsappDirectLine extends GBService {
let urlMin: any = GBServer.globals.minInstances.filter(p => p.instance.botId === botId)[0];
const botNumber = urlMin ? urlMin.core.getParam(urlMin.instance, 'Bot Number', null) : null;
if (botNumber){
user = await sec.updateUserInstance(user.userSystemId, urlMin.instance.instanceId);
}
let activeMin;
// Processes group behaviour.
@ -1120,15 +1129,25 @@ export class WhatsappDirectLine extends GBService {
res.end();
}
} else {
let t;
activeMin = GBServer.globals.minInstances.filter(p => p.instance.instanceId === user.instanceId)[0];
if (activeMin === undefined) {
activeMin = GBServer.globals.minBoot;
await (activeMin as any).whatsAppDirectLine.sendToDevice(
t = (activeMin as any).whatsAppDirectLine;
await t.sendToDevice(
id,
`O outro Bot que você estava falando(${user.instanceId}), não está mais disponível. Agora você está falando comigo, ${activeMin.instance.title}...`
);
}
await (activeMin as any).whatsAppDirectLine.received(req, res);
else {
if ((activeMin as any).whatsAppDirectLine) {
t = (activeMin as any).whatsAppDirectLine;
} else {
t = (GBServer.globals.minBoot as any).whatsAppDirectLine;
}
}
t.received(req, res);
}
}
} else {