GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

- Warming removal.
Some checks failed
GBCI / build (push) Failing after 2m51s
Details

Browse source
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-09-21 16:49:03 -03:00
parent 30663a608e
commit 904a33a409
72 changed files with 782 additions and 51722 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
.test-init.ts
View file

@ -1,16 +1,15 @@
import { expect, test } from 'vitest';
import { GBServer } from './src/app';
import { RootData } from './src/RootData';
import { GBMinInstance } from 'botlib';
import { GBMinInstance } from 'botlib-legacy';
import { Mutex } from 'async-mutex';
export default function init() {
const min = {
packages: null,
appPackages: null,
botId: 'gbtest',
instance: {botId: 'gbtest'},
instance: { botId: 'gbtest' },
core: {},
conversationalService: {},
kbService: {},
@ -26,14 +25,13 @@ export default function init() {
scriptMap: {},
sandBoxMap: {},
gbappServices: {}
}
};
GBServer.globals = new RootData();
GBServer.globals.server = null;
GBServer.globals.httpsServer = null;
GBServer.globals.webSessions = {};
GBServer.globals.processes = [0, { pid: 1, proc: {step: {}}}];
GBServer.globals.processes = [0, { pid: 1, proc: { step: {} } }];
GBServer.globals.files = {};
GBServer.globals.appPackages = [];
GBServer.globals.sysPackages = [];
@ -43,5 +41,5 @@ export default function init() {
GBServer.globals.entryPointDialog = null;
GBServer.globals.debuggers = [];
GBServer.globals.indexSemaphore = new Mutex();
GBServer.globals.users = {1: {userId: 1}};
GBServer.globals.users = { 1: { userId: 1 } };
}

21
additional-packages.cmd
View file

@ -1,21 +0,0 @@
npm install \
@azure/ms-rest-js@2.7.0 \
@azure/msal-node@3.7.3 \
@google-cloud/pubsub@5.2.0 \
@google-cloud/translate@9.2.0 \
botlib \
c3-chart-maker@0.2.8 \
docximager@0.0.4 \
docxtemplater@3.66.3 \
exceljs@4.4.0 \
get-image-colors@4.0.1 \
googleapis@159.0.0 \
ibm-watson@11.0.0 \
node-nlp@4.27.0 \
office-text-extractor@3.0.3 \
open-docxtemplater-image-module@1.0.3 \
pragmatismo-fram \
puppeteer \
svg2img@1.0.0-beta.2 \
vm2@3.9.19 \
whatsapp-web.js@1.34.1

48942
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

298
package.json
View file

@ -1,6 +1,6 @@
{
"name": "botserver",
"version": "5.0.0",
"version": "5.1.0",
"description": "General Bot Community Edition open-core server.",
"main": "./boot.mjs",
"type": "module",
@ -18,7 +18,7 @@
"disableAutoBuild": "1"
},
"engines": {
"node": "=22.9.0"
"node": "=22.19.0"
},
"license": "AGPL-3.0",
"preferGlobal": true,
@ -41,19 +41,7 @@
"test": "vitest",
"start": "NODE_NO_WARNINGS=1 node ./boot.mjs --loader ts-node/esm --require ./suppress-node-warnings.cjs",
"debug": "NODE_NO_WARNINGS=1 node ./boot.mjs --loader ts-node/esm --require ./suppress-node-warnings.cjs --inspect",
"reverse-proxy": "node_modules/.bin/ngrok http 4242",
"watch:build": "tsc --watch",
"posttypedoc": "shx cp .nojekyll docs/reference/.nojekyll",
"ban": "ban",
"issues": "git-issues",
"license": "license-checker --production --onlyunknown --csv",
"pretty": "prettier-standard 'src/*.ts' 'packages/**/*.ts'",
"secure": "nsp check",
"size": "t=\"$(npm pack .)\"; wc -c \"${t}\"; tar tvf \"${t}\"; rm \"${t}\";",
"unused-deps": "dependency-check --unused --no-dev ./package.json",
"travis-deploy-once": "travis-deploy-once --pro",
"semantic-release": "semantic-release",
"commit": "git-cz"
"watch:build": "tsc --watch"
},
"jest": {
"workerIdleMemoryLimit": "4096MB",
@ -69,212 +57,171 @@
]
},
"dependencies": {
"@azure/arm-appservice": "15.0.0",
"@azure/arm-cognitiveservices": "7.5.0",
"@azure/arm-resources": "5.2.0",
"@azure/arm-search": "3.2.0",
"@azure/arm-sql": "10.0.0",
"@azure/arm-subscriptions": "5.1.0",
"@azure/cognitiveservices-computervision": "8.2.0",
"@azure/keyvault-keys": "4.8.0",
"@azure/ms-rest-js": "2.7.0",
"@azure/msal-node": "2.13.1",
"@azure/openai": "2.0.0-beta.1",
"@azure/msal-node": "^3.7.3",
"@azure/openai": "2.0.0",
"@azure/search-documents": "12.1.0",
"@azure/storage-blob": "12.24.0",
"@google-cloud/pubsub": "4.7.0",
"@google-cloud/translate": "8.5.0",
"@hubspot/api-client": "11.2.0",
"@azure/storage-blob": "12.28.0",
"@google-cloud/pubsub": "^5.2.0",
"@google-cloud/translate": "^9.2.0",
"@koa/cors": "5.0.0",
"@langchain/anthropic": "^0.3.7",
"@langchain/community": "0.2.31",
"@langchain/core": "^0.3.17",
"@langchain/openai": "0.2.8",
"@koa/router": "14.0.0",
"@langchain/anthropic": "0.3.27",
"@langchain/community": "0.3.55",
"@langchain/core": "0.3.75",
"@langchain/openai": "0.6.11",
"@microsoft/microsoft-graph-client": "3.0.7",
"@nlpjs/basic": "4.27.0",
"@nosferatu500/textract": "3.1.3",
"@push-rpc/core": "1.9.0",
"@push-rpc/http": "1.9.0",
"@push-rpc/openapi": "1.9.0",
"@push-rpc/websocket": "1.9.0",
"@semantic-release/changelog": "6.0.3",
"@semantic-release/exec": "6.0.3",
"@semantic-release/git": "10.0.1",
"@sendgrid/mail": "8.1.3",
"@sequelize/core": "7.0.0-alpha.37",
"@sequelize/postgres": "^7.0.0-alpha.43",
"@types/validator": "13.12.1",
"@push-rpc/core": "1.9.3",
"@push-rpc/http": "1.9.3",
"@push-rpc/openapi": "1.9.3",
"@push-rpc/websocket": "1.9.3",
"@sequelize/core": "7.0.0-alpha.46",
"@sequelize/postgres": "7.0.0-alpha.46",
"@types/validator": "13.15.3",
"adm-zip": "0.5.16",
"ai2html": "^0.121.1",
"alasql": "4.5.1",
"alasql": "4.6.6",
"any-shell-escape": "0.1.1",
"arraybuffer-to-buffer": "0.0.7",
"async-mutex": "0.5.0",
"async-promises": "0.2.3",
"async-retry": "1.3.3",
"basic-auth": "2.0.1",
"bcrypt": "^5.1.1",
"billboard.js": "3.13.0",
"bcrypt": "6.0.0",
"billboard.js": "3.16.0",
"bluebird": "3.7.2",
"body-parser": "1.20.2",
"botbuilder": "4.23.0",
"body-parser": "2.2.0",
"botbuilder": "4.23.3",
"botbuilder-adapter-facebook": "1.0.12",
"botbuilder-ai": "4.23.0",
"botbuilder-dialogs": "4.23.0",
"botframework-connector": "4.23.0",
"botlib": "5.0.0",
"c3-chart-maker": "0.2.8",
"botbuilder-ai": "4.23.3",
"botbuilder-dialogs": "4.23.3",
"botframework-connector": "4.23.3",
"botlib-legacy": "5.1.1",
"cd": "0.3.3",
"chalk-animation": "2.0.3",
"chatgpt": "5.2.5",
"chrome-remote-interface": "0.33.2",
"chrome-remote-interface": "0.33.3",
"cli-progress": "3.12.0",
"cli-spinner": "0.2.10",
"core-js": "3.38.1",
"core-js": "3.45.1",
"cors": "2.8.5",
"csv-database": "0.9.2",
"data-forge": "1.10.2",
"data-forge": "1.10.4",
"date-diff": "1.0.2",
"docximager": "0.0.4",
"docxtemplater": "3.50.0",
"docximager": "^0.0.4",
"docxtemplater": "^3.66.3",
"dotenv-extended": "2.9.0",
"electron": "32.0.1",
"exceljs": "4.4.0",
"express": "4.19.2",
"exceljs": "^4.4.0",
"express": "5.1.0",
"express-remove-route": "1.0.0",
"facebook-nodejs-business-sdk": "^20.0.2",
"facebook-nodejs-business-sdk": "23.0.1",
"ffmpeg-static": "5.2.0",
"formidable": "^3.5.1",
"get-image-colors": "4.0.1",
"glob": "^11.0.0",
"google-libphonenumber": "3.2.38",
"googleapis": "143.0.0",
"final-stream": "^2.0.4",
"formidable": "3.5.4",
"get-image-colors": "^4.0.1",
"glob": "11.0.3",
"google-libphonenumber": "3.2.42",
"googleapis": "^159.0.0",
"hnswlib-node": "3.0.0",
"html-to-md": "0.8.6",
"html-to-md": "0.8.8",
"http-proxy": "1.18.1",
"ibm-watson": "9.1.0",
"icojs": "^0.19.4",
"instagram-private-api": "1.46.1",
"iso-639-1": "3.1.3",
"ibm-watson": "^11.0.0",
"icojs": "0.19.5",
"iso-639-1": "3.1.5",
"isomorphic-fetch": "3.0.0",
"jimp": "1.6.0",
"js-md5": "0.8.3",
"json-schema-to-zod": "2.4.0",
"jsqr": "^1.4.0",
"json-schema-to-zod": "2.6.1",
"jsqr": "1.4.0",
"just-indent": "0.0.1",
"keyv": "5.0.1",
"koa": "2.15.3",
"keyv": "5.5.1",
"koa": "3.0.1",
"koa-body": "6.0.1",
"koa-ratelimit": "5.1.0",
"koa-router": "12.0.1",
"langchain": "0.2.17",
"language-tags": "1.0.9",
"koa-ratelimit": "6.0.0",
"langchain": "0.3.33",
"language-tags": "2.1.0",
"line-replace": "2.0.1",
"livekit-server-sdk": "^2.12.0",
"livekit-server-sdk": "2.13.3",
"lodash": "4.17.21",
"luxon": "3.5.0",
"mammoth": "1.8.0",
"mariadb": "3.3.1",
"mime-types": "2.1.35",
"minio": "^8.0.4",
"luxon": "3.7.2",
"mammoth": "1.10.0",
"mariadb": "3.4.5",
"mime-types": "3.0.1",
"minio": "8.0.6",
"moment": "2.30.1",
"ms-rest-azure": "3.0.2",
"mysql": "^2.18.1",
"nexmo": "2.9.1",
"ngrok": "5.0.0-beta.2",
"node-cron": "3.0.3",
"node-html-parser": "6.1.13",
"node-nlp": "4.27.0",
"node-tesseract-ocr": "2.2.1",
"nodemailer": "6.10.1",
"nodemon": "^3.1.7",
"npm": "10.8.3",
"open": "10.1.0",
"open-docxtemplater-image-module": "1.0.3",
"openai": "4.57.0",
"ms-rest-azure": "^3.0.2",
"mysql": "2.18.1",
"node-cron": "4.2.1",
"node-html-parser": "7.0.1",
"node-nlp": "^4.27.0",
"nodemailer": "7.0.6",
"nodemon": "3.1.10",
"npm": "11.6.0",
"office-text-extractor": "^3.0.3",
"open": "10.2.0",
"open-docxtemplater-image-module": "^1.0.3",
"openai": "4.62.1",
"pdf-extraction": "1.0.2",
"pdf-parse": "1.1.1",
"pdf-to-png-converter": "3.3.0",
"pdfjs-dist": "4.6.82",
"pg": "^8.13.1",
"phone": "3.1.50",
"pizzip": "3.1.7",
"pdf-to-png-converter": "3.7.1",
"pdfjs-dist": "5.4.149",
"pg": "8.16.3",
"phone": "3.1.67",
"pizzip": "3.2.0",
"pptxtemplater": "1.0.5",
"pragmatismo-io-framework": "1.1.1",
"prism-media": "1.3.5",
"public-ip": "7.0.1",
"punycode": "2.3.1",
"puppeteer": "23.2.2",
"puppeteer-extra": "3.3.6",
"puppeteer-extra-plugin-minmax": "1.1.2",
"puppeteer-extra-plugin-stealth": "2.11.2",
"puppeteer": "24.20.0",
"qr-scanner": "1.4.2",
"qrcode": "1.5.4",
"qrcode-reader": "^1.0.4",
"qrcode-reader": "1.0.4",
"qrcode-terminal": "0.12.0",
"readline": "1.3.0",
"reflect-metadata": "0.2.2",
"rimraf": "6.0.1",
"safe-buffer": "5.2.1",
"scanf": "1.2.0",
"sequelize": "6.37.3",
"sequelize-cli": "6.6.2",
"sequelize": "6.37.7",
"sequelize-cli": "6.6.3",
"sequelize-typescript": "2.1.6",
"simple-git": "3.26.0",
"simple-git": "3.28.0",
"speakingurl": "14.0.1",
"sqlite3": "5.1.7",
"ssr-for-bots": "1.0.1-c",
"strict-password-generator": "1.1.2",
"stripe": "^18.0.0",
"stripe": "18.5.0",
"super-strong-password-generator": "2.0.2",
"super-strong-password-generator-es": "2.0.2",
"svg2img": "^1.0.0-beta.2",
"swagger-client": "3.29.2",
"swagger-ui-dist": "5.17.14",
"tabulator-tables": "6.2.5",
"swagger-client": "3.35.6",
"swagger-ui-dist": "5.29.0",
"tabulator-tables": "6.3.1",
"tedious": "18.6.1",
"textract": "2.5.0",
"twilio": "5.2.3",
"twitter-api-v2": "1.17.2",
"typeorm": "0.3.20",
"typescript": "5.5.4",
"url-join": "5.0.0",
"textract": "^2.5.0",
"twilio": "5.9.0",
"twitter-api-v2": "1.27.0",
"typeorm": "0.3.26",
"typescript": "5.9.2",
"url-join": "^5.0.0",
"vhost": "3.0.2",
"vm2": "3.9.19",
"vm2-process": "2.1.5",
"vm2": "github:n8n-io/vm2",
"walk-promise": "0.2.0",
"washyourmouthoutwithsoap": "1.0.2",
"webdav-server": "2.6.2",
"webp-converter": "^2.3.3",
"whatsapp-cloud-api": "0.3.1",
"whatsapp-web.js": "1.26.1-alpha.1",
"winston": "3.14.2",
"ws": "8.18.0",
"yaml": "2.5.0",
"whatsapp-web.js": "^1.34.1",
"ws": "8.18.3",
"yaml": "2.8.1",
"yarn": "1.22.22",
"zod-to-json-schema": "3.23.2"
"zod-to-json-schema": "3.24.6"
},
"devDependencies": {
"@types/lodash": "^4.17.20",
"@types/node": "^24.1.0",
"@types/node-fetch": "^2.6.12",
"@types/lodash": "4.17.20",
"@types/node": "24.3.3",
"@types/qrcode": "1.5.5",
"@types/url-join": "4.0.3",
"@typescript-eslint/eslint-plugin": "8.4.0",
"@typescript-eslint/parser": "8.4.0",
"ban-sensitive-files": "1.10.5",
"commitizen": "4.3.0",
"cz-conventional-changelog": "3.3.0",
"dependency-check": "4.1.0",
"git-issues": "1.3.1",
"license-checker": "25.0.1",
"prettier-standard": "16.4.1",
"semantic-release": "24.1.0",
"simple-commit-message": "4.1.3",
"travis-deploy-once": "5.0.11",
"tslint": "6.1.3",
"tsx": "^4.19.1",
"vitest": "2.0.5"
"@typescript-eslint/eslint-plugin": "8.43.0",
"@typescript-eslint/parser": "8.43.0",
"tsx": "4.20.5",
"vitest": "3.2.4"
},
"eslintConfig": {
"env": {
@ -368,5 +315,44 @@
"post-checkout": [],
"post-merge": []
}
},
"overrides": {
"lodash.trimend": "npm:lodash@4.17.21",
"lodash.isequal": "npm:lodash@4.17.21",
"node-domexception": "npm:whatwg-url@^11.0.0",
"csv-database": {
"fast-csv": "4.3.6"
},
"sequelize-typescript": {
"glob": "~9.0.0"
},
"botbuilder-adapter-facebook": {
"node-domexception": "npm:whatwg-url@^11.0.0"
},
"tough-cookie": "4.1.3",
"phin": "3.7.1",
"xmldom": "npm:@xmldom/xmldom@^0.8.10",
"form-data": "2.5.5",
"uuid": "9.0.1",
"har-validator": "5.1.5",
"request": "2.88.2",
"yaeti": "npm:events@^3.3.0",
"text-encoding": "npm:text-encoder-lite@^1.0.1",
"docximager": {
"xml2js": "^0.5.0"
},
"tar-fs": "3.1.0",
"ws": "8.18.3",
"xml2js": "^0.6.2",
"inflight": "npm:lru-cache@^10.2.0",
"rimraf": "6.0.1",
"glob": "11.0.3",
"fstream": "npm:fs-extra@^11.0.0",
"puppeteer": "24.20.0",
"fluent-ffmpeg": "npm:ffmpeg-static@^5.0.0",
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz",
"@nlpjs/xtables": {
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
}
}
}

63
packages/admin.gbapp/dialogs/AdminDialog.ts
View file

@ -37,19 +37,18 @@
import crypto from 'crypto';
import urlJoin from 'url-join';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import { GBImporter } from '../../core.gbapp/services/GBImporterService.js';
import { Messages } from '../strings.js';
import { GBAdminService } from '../services/GBAdminService.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { SecService } from '../../security.gbapp/services/SecService.js';
import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js';
import { GBServer } from '../../../src/app.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { GBUtil } from '../../../src/util.js';
class AdminDialog extends IGBDialog {
public static isIntentYes(locale, utterance) {
return utterance.toLowerCase().match(Messages[locale].affirmative_sentences);
@ -87,7 +86,7 @@ class AdminDialog extends IGBDialog {
const locale = step.context.activity.locale;
const sensitive = step.context.activity['originalText'];
if (await GBUtil.comparePassword( sensitive, min.instance.adminPass)) {
if (await GBUtil.comparePassword(sensitive, min.instance.adminPass)) {
await min.conversationalService.sendText(min, step, Messages[locale].welcome);
return await step.endDialog(true);
@ -121,7 +120,7 @@ class AdminDialog extends IGBDialog {
const locale = step.context.activity.locale;
const sensitive = step.context.activity['originalText'];
if (await GBUtil.comparePassword( sensitive, min.instance.adminPass)) {
if (await GBUtil.comparePassword(sensitive, min.instance.adminPass)) {
await min.conversationalService.sendText(min, step, Messages[locale].welcome);
return await min.conversationalService.prompt(min, step, Messages[locale].which_task);
@ -224,7 +223,8 @@ class AdminDialog extends IGBDialog {
await min.conversationalService.sendText(min, step, logs);
return await step.replaceDialog('/ask', { isReturning: true });
}
]));
])
);
min.dialogs.add(
new WaterfallDialog('/publish', [
@ -302,7 +302,7 @@ class AdminDialog extends IGBDialog {
packages.push(filename);
}
await CollectionUtil.asyncForEach(packages, async packageName => {
await GBUtil.asyncForEach(packages, async packageName => {
let cmd1;
if (
@ -330,26 +330,17 @@ class AdminDialog extends IGBDialog {
}
let sec = new SecService();
const member = step.context.activity.from;
const user = await sec.ensureUser(
min,
member.id,
member.name,
'',
'web',
member.name,
null
);
const user = await sec.ensureUser(min, member.id, member.name, '', 'web', member.name, null);
await GBAdminService.deployPackageCommand(min, user, cmd1, deployer);
// .gbot updates severals keys in instantece, so min must be updated.
const activeMin = GBServer.globals.minInstances.find(p=> p.botId === min.botId);
const activeMin = GBServer.globals.minInstances.find(p => p.botId === min.botId);
if (activeMin){
min = activeMin;
if (activeMin) {
min = activeMin;
}
});
await min.conversationalService.sendText(min, step, `Training is finished.`);
@ -389,11 +380,15 @@ class AdminDialog extends IGBDialog {
new WaterfallDialog('/setupSecurity', [
async step => {
min = GBServer.globals.minInstances.find(p => p.botId === min.botId);
const tokenName = step.activeDialog.state.tokenName = step.options['args'];
const tokenName = (step.activeDialog.state.tokenName = step.options['args']);
if (tokenName) {
step.activeDialog.state.clientId = min.core.getParam<string>(min.instance, `${tokenName} Client ID`, null),
step.activeDialog.state.host = min.core.getParam<string>(min.instance, `${tokenName} Host`, null),
step.activeDialog.state.tenant = min.core.getParam<string>(min.instance, `${tokenName} Tenant`, null)
((step.activeDialog.state.clientId = min.core.getParam<string>(
min.instance,
`${tokenName} Client ID`,
null
)),
(step.activeDialog.state.host = min.core.getParam<string>(min.instance, `${tokenName} Host`, null)),
(step.activeDialog.state.tenant = min.core.getParam<string>(min.instance, `${tokenName} Tenant`, null)));
}
if (step.context.activity.channelId !== 'msteams' && process.env.ENABLE_AUTH) {
return await step.beginDialog('/auth');
@ -416,7 +411,7 @@ class AdminDialog extends IGBDialog {
min = GBServer.globals.minInstances.find(p => p.botId === min.botId);
if (step.activeDialog.state.tokenName) {
return await step.next(step.options);
}
}
step.activeDialog.state.authenticatorTenant = step.context.activity['originalText'];
const locale = step.context.activity.locale;
const prompt = Messages[locale].enter_authenticator_authority_host_url;
@ -445,13 +440,19 @@ class AdminDialog extends IGBDialog {
min.adminService.setValue(min.instance.instanceId, `${tokenName}AntiCSRFAttackState`, state);
const redirectUri = urlJoin(process.env.BOT_URL, min.instance.botId,
tokenName ? `/token?value=${tokenName}` : '/token');
const redirectUri = urlJoin(
process.env.BOT_URL,
min.instance.botId,
tokenName ? `/token?value=${tokenName}` : '/token'
);
const scope = tokenName ? '' : 'https://graph.microsoft.com/.default';
const host = tokenName ? step.activeDialog.state.host : 'https://login.microsoftonline.com'
const host = tokenName ? step.activeDialog.state.host : 'https://login.microsoftonline.com';
const tenant = tokenName ? step.activeDialog.state.tenant : min.instance.authenticatorTenant;
const clientId = tokenName ? step.activeDialog.state.clientId : (min.instance.marketplaceId ?
min.instance.marketplaceId : GBConfigService.get('MARKETPLACE_ID'));
const clientId = tokenName
? step.activeDialog.state.clientId
: min.instance.marketplaceId
? min.instance.marketplaceId
: GBConfigService.get('MARKETPLACE_ID');
const oauth2 = tokenName ? 'oauth' : 'oauth2';
const url = `${host}/${tenant}/${oauth2}/authorize?client_id=${clientId}&response_type=code&redirect_uri=${redirectUri}&scope=${scope}&state=${state}&response_mode=query`;
@ -464,4 +465,4 @@ class AdminDialog extends IGBDialog {
}
}
export { AdminDialog };
export { AdminDialog };

2
packages/admin.gbapp/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
import { AdminDialog } from './dialogs/AdminDialog.js';
import { GuaribasAdmin } from './models/AdminModel.js';

44
packages/admin.gbapp/services/GBAdminService.ts
View file

@ -34,28 +34,46 @@
'use strict';
import { AuthenticationContext, TokenResponse } from 'adal-node';
import { GBError, GBLog, GBMinInstance, IGBAdminService, IGBCoreService, IGBDeployer, IGBInstance } from 'botlib';
import {
GBError,
GBLog,
GBMinInstance,
IGBAdminService,
IGBCoreService,
IGBDeployer,
IGBInstance
} from 'botlib-legacy';
import { FindOptions } from 'sequelize/types';
import urlJoin from 'url-join';
import { AzureDeployerService } from '../../azuredeployer.gbapp/services/AzureDeployerService.js';
import { GuaribasInstance } from '../../core.gbapp/models/GBModel.js';
import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js';
import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import { GBImporter } from '../../core.gbapp/services/GBImporterService.js';
import { GBSharePointService } from '../../sharepoint.gblib/services/SharePointService.js';
import { GuaribasAdmin } from '../models/AdminModel.js';
import msRestAzure from 'ms-rest-azure';
import path from 'path';
import { caseSensitive_Numbs_SpecialCharacters_PW, lowercase_PW } from 'super-strong-password-generator';
import crypto from 'crypto';
import fs from 'fs/promises';
import fs from 'fs/promises';
import { GBServer } from '../../../src/app.js';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { GBUtil } from '../../../src/util.js';
let msRestAzure: any = null;
try {
const msRestAzure = await import('ms-rest-azure');
} catch (error) {}
let AuthenticationContext: any = null;
try {
const adal = await import('adal-node');
AuthenticationContext = adal.AuthenticationContext;
} catch (error) {}
/**
* Services for server administration.
*/
@ -174,20 +192,6 @@ export class GBAdminService implements IGBAdminService {
const localFolder = path.join('work', gbaiPath);
await deployer['deployPackage2'](min, user, localFolder, true);
}
public static async rebuildIndexPackageCommand(min: GBMinInstance, deployer: GBDeployer) {
const service = await AzureDeployerService.createInstance(deployer);
const searchIndex = min.instance.searchIndex
? min.instance.searchIndex
: GBServer.globals.minBoot.instance.searchIndex;
await deployer.rebuildIndex(min.instance, service.getKBSearchSchema(searchIndex));
}
public static async syncBotServerCommand(min: GBMinInstance, deployer: GBDeployer) {
const serverName = `${min.instance.botId}-server`;
const service = await AzureDeployerService.createInstance(deployer);
service.syncBotServerRepository(min.instance.botId, serverName);
}
public async setValue(instanceId: number, key: string, value: string) {
@ -316,7 +320,7 @@ export class GBAdminService implements IGBAdminService {
if (err !== null) {
reject(err);
} else {
const token = res as TokenResponse;
const token = res;
try {
await this.setValue(instanceId, `${tokenName}accessToken`, token.accessToken);
await this.setValue(instanceId, `${tokenName}refreshToken`, token.refreshToken);

2
packages/analytics.gblib/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
import { GuaribasConversation, GuaribasConversationMessage } from './models/index.js';

28
packages/analytics.gblib/services/AnalyticsService.ts
View file

@ -32,7 +32,6 @@
* @fileoverview General Bots server core.
*/
import { AzureText } from 'pragmatismo-io-framework';
import { FindOptions } from 'sequelize/types';
import { GBServer } from '../../../src/app.js';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
@ -42,7 +41,7 @@ import { GuaribasConversation, GuaribasConversationMessage } from '../models/ind
* Base services for Bot Analytics.
*/
export class AnalyticsService {
public async createConversation (user: GuaribasUser): Promise<GuaribasConversation> {
public async createConversation(user: GuaribasUser): Promise<GuaribasConversation> {
const conversation = new GuaribasConversation();
conversation.startedBy = user;
conversation.startedByUserId = user.userId;
@ -51,41 +50,22 @@ export class AnalyticsService {
return await conversation.save();
}
public async updateConversationSuggestion (
public async updateConversationSuggestion(
instanceId: number,
conversationId: string,
feedback: string,
locale: string
): Promise<number> {
const minBoot = GBServer.globals.minBoot as any;
const rate = await AzureText.getSentiment(
minBoot.instance.textAnalyticsKey ? minBoot.instance.textAnalyticsKey : minBoot.instance.textAnalyticsKey,
minBoot.instance.textAnalyticsEndpoint
? minBoot.instance.textAnalyticsEndpoint
: minBoot.instance.textAnalyticsEndpoint,
locale,
feedback
);
const options = <FindOptions>{ where: {} };
options.where = { conversationId: conversationId, instanceId: instanceId };
const item = await GuaribasConversation.findOne(options);
item.feedback = feedback;
item.rate = rate;
item.rateDate = new Date();
await item.save();
return rate;
return 0;
}
public async createMessage (
public async createMessage(
instanceId: number,
conversationId: number,
userId: number,
content: string
): Promise<GuaribasConversationMessage> {
const message = GuaribasConversationMessage.build();
message.content = typeof content === 'object' ? JSON.stringify(content) : content;
message.instanceId = instanceId;

215
packages/azuredeployer.gbapp/dialogs/StartDialog.ts
View file

@ -1,215 +0,0 @@
/*****************************************************************************\
| █████ █████ ██ █ █████ █████ ████ ██ ████ █████ █████ ███ ® |
| ██ █ ███ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| ██ ███ ████ █ ██ █ ████ █████ ██████ ██ ████ █ █ █ ██ |
| ██ ██ █ █ ██ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| █████ █████ █ ███ █████ ██ ██ ██ ██ █████ ████ █████ █ ███ |
| |
| General Bots Copyright (c) pragmatismo.com.br. All rights reserved. |
| Licensed under the AGPL-3.0. |
| |
| According to our dual licensing model, this program can be used either |
| under the terms of the GNU Affero General Public License, version 3, |
| or under a proprietary license. |
| |
| The texts of the GNU Affero General Public License with an additional |
| permission and of our proprietary license can be found at and |
| in the LICENSE file you have received along with this program. |
| |
| This program is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY, without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| GNU Affero General Public License for more details. |
| |
| "General Bots" is a registered trademark of pragmatismo.com.br. |
| The licensing of the program under the AGPLv3 does not imply a |
| trademark license. Therefore any rights, title and interest in |
| our trademarks remain entirely with us. |
| |
\*****************************************************************************/
/**
* @fileoverview General Bots server core.
*/
'use strict';
import { GBLog, IGBInstallationDeployer, IGBInstance } from 'botlib';
import fs from 'fs/promises';
import { GBAdminService } from '../../../packages/admin.gbapp/services/GBAdminService.js';
import { GBConfigService } from '../../../packages/core.gbapp/services/GBConfigService.js';
import scanf from 'scanf';
import { AzureDeployerService } from '../services/AzureDeployerService.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { GBUtil } from '../../../src/util.js';
/**
* Handles command-line dialog for getting info for Boot Bot.
*/
export class StartDialog {
public static async createBaseInstance (deployer, freeTier) {
// No .env so asks for cloud credentials to start a new farm.
if (!await GBUtil.exists(`.env`)) {
process.stdout.write(
'A empty enviroment is detected. To start automatic deploy, please enter some information:\n'
);
}
let botId: string;
while (botId === undefined) {
botId = this.retrieveBotId();
}
let username: string;
while (username === undefined) {
username = this.retrieveUsername();
}
let password: string;
while (password === undefined) {
password = this.retrievePassword();
}
// Connects to the cloud and retrieves subscriptions.
const credentials = await GBAdminService.getADALCredentialsFromUsername(username, password);
let subscriptionId: string;
while (subscriptionId === undefined) {
const list = await (new AzureDeployerService()).getSubscriptions(credentials);
subscriptionId = this.retrieveSubscriptionId(list);
}
const installationDeployer = await AzureDeployerService.createInstanceWithADALCredentials(
deployer, freeTier, subscriptionId, credentials);
let location: string;
while (location === undefined) {
location = this.retrieveLocation();
}
let appId: string;
while (appId === undefined) {
appId = this.retrieveAppId();
}
let appPassword: string;
while (appPassword === undefined) {
appPassword = this.retrieveAppPassword();
}
// Prepares the first instance on bot farm.
const instance = <IGBInstance>{};
instance.botId = botId;
instance.state = 'active';
instance.cloudUsername = username;
instance.cloudPassword = password;
instance.cloudSubscriptionId = subscriptionId;
instance.cloudLocation = location;
instance.marketplaceId = appId;
instance.marketplacePassword = appPassword;
instance.adminPass = await GBUtil.hashPassword(GBAdminService.getRndPassword());
return { instance, credentials, subscriptionId , installationDeployer};
}
private static retrieveUsername () {
let value = GBConfigService.get('CLOUD_USERNAME');
if (value === undefined) {
process.stdout.write(`${GBAdminService.GB_PROMPT}CLOUD_USERNAME:`);
value = scanf('%s').replace(/(\n|\r)+$/, '');
}
return value;
}
private static retrievePassword () {
let password = GBConfigService.get('CLOUD_PASSWORD');
if (password === undefined) {
process.stdout.write(`${GBAdminService.GB_PROMPT}CLOUD_PASSWORD:`);
password = scanf('%s').replace(/(\n|\r)+$/, '');
}
return password;
}
private static retrieveBotId () {
let botId = GBConfigService.get('BOT_ID');
if (botId === undefined) {
process.stdout.write(
`${GBAdminService.GB_PROMPT}Choose a unique bot Id containing lowercase letters, digits or
dashes (cannot use dash as the first two or last one characters),
cannot start or end with or contain consecutive dashes and having 4 to 42 characters long.\n`
);
process.stdout.write(`${GBAdminService.GB_PROMPT}BOT_ID:`);
botId = scanf('%s').replace(/(\n|\r)+$/, '');
}
return botId;
}
/**
*
* Update Manifest in Azure: "signInAudience": "AzureADandPersonalMicrosoftAccount" and "accessTokenAcceptedVersion": 2.
*/
private static retrieveAppId () {
let appId = GBConfigService.get('MARKETPLACE_ID');
if (appId === undefined) {
process.stdout.write(
`Sorry, this part cannot be automated yet due to Microsoft schedule,
please go to https://apps.dev.microsoft.com/portal/register-app to
generate manually an App ID and App Secret.\n`
);
process.stdout.write('Generated Application Id (MARKETPLACE_ID):');
appId = scanf('%s').replace(/(\n|\r)+$/, '');
}
return appId;
}
private static retrieveAppPassword () {
let appPassword = GBConfigService.get('MARKETPLACE_SECRET');
if (appPassword === undefined) {
process.stdout.write('Generated Password (MARKETPLACE_SECRET):');
appPassword = scanf('%s').replace(/(\n|\r)+$/, '');
}
return appPassword;
}
private static retrieveSubscriptionId (list) {
let subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID');
if (subscriptionId){
return subscriptionId;
}
const map = {};
let index = 1;
list.forEach(element => {
GBLogEx.info(0, `${index}: ${element.displayName} (${element.subscriptionId})`);
map[index++] = element;
});
let subscriptionIndex;
if (!subscriptionIndex && subscriptionId === undefined) {
process.stdout.write('CLOUD_SUBSCRIPTIONID (type a number):');
subscriptionIndex = scanf('%d');
subscriptionId = map[subscriptionIndex].subscriptionId;
}
return subscriptionId;
}
private static retrieveLocation () {
let location = GBConfigService.get('CLOUD_LOCATION');
if (location === undefined) {
process.stdout.write('CLOUD_LOCATION (eg. westus):');
location = scanf('%s');
}
return location;
}
}

66
packages/azuredeployer.gbapp/index.ts
View file

@ -1,66 +0,0 @@
/*****************************************************************************\
| █████ █████ ██ █ █████ █████ ████ ██ ████ █████ █████ ███ ® |
| ██ █ ███ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| ██ ███ ████ █ ██ █ ████ █████ ██████ ██ ████ █ █ █ ██ |
| ██ ██ █ █ ██ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| █████ █████ █ ███ █████ ██ ██ ██ ██ █████ ████ █████ █ ███ |
| |
| General Bots Copyright (c) pragmatismo.com.br. All rights reserved. |
| Licensed under the AGPL-3.0. |
| |
| According to our dual licensing model, this program can be used either |
| under the terms of the GNU Affero General Public License, version 3, |
| or under a proprietary license. |
| |
| The texts of the GNU Affero General Public License with an additional |
| permission and of our proprietary license can be found at and |
| in the LICENSE file you have received along with this program. |
| |
| This program is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY, without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| GNU Affero General Public License for more details. |
| |
| "General Bots" is a registered trademark of pragmatismo.com.br. |
| The licensing of the program under the AGPLv3 does not imply a |
| trademark license. Therefore any rights, title and interest in |
| our trademarks remain entirely with us. |
| |
\*****************************************************************************/
/**
* @fileoverview General Bots server core.
*/
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { Sequelize } from 'sequelize-typescript';
/**
* Package for Azure Deployer.
*/
export class GBAzureDeployerPackage implements IGBPackage {
public sysPackages: IGBPackage[];
public async getDialogs (min: GBMinInstance) {
GBLog.verbose(`getDialogs called.`);
}
public async loadPackage (core: IGBCoreService, sequelize: Sequelize): Promise<void> {
GBLog.verbose(`loadPackage called.`);
}
public async unloadPackage (core: IGBCoreService): Promise<void> {
GBLog.verbose(`unloadPackage called.`);
}
public async loadBot (min: GBMinInstance): Promise<void> {
GBLog.verbose(`loadBot called.`);
}
public async unloadBot (min: GBMinInstance): Promise<void> {
GBLog.verbose(`unloadBot called.`);
}
public async onNewSession (min: GBMinInstance, step: GBDialogStep): Promise<void> {
GBLog.verbose(`onNewSession called.`);
}
public async onExchangeData (min: GBMinInstance, kind: string, data: any) {
GBLog.verbose(`onExchangeData called.`);
}
}

1008
packages/azuredeployer.gbapp/services/AzureDeployerService.ts
View file

File diff suppressed because it is too large Load diff

8
packages/azuredeployer.gbapp/strings.ts
View file

@ -1,8 +0,0 @@
export const Messages = {
'en-US': {
about_suggestions: 'Suggestions are welcomed and improve my quality...'
},
'pt-BR': {
about_suggestions: 'Sugestões melhoram muito minha qualidade...'
}
};

2
packages/basic.gblib/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { GuaribasSchedule } from '../core.gbapp/models/GBModel.js';
import { Sequelize } from 'sequelize-typescript';
import Koa from 'koa';

2
packages/basic.gblib/services/DebuggerService.ts
View file

@ -30,7 +30,7 @@
'use strict';
import { GBLog, GBMinInstance } from 'botlib';
import { GBLog, GBMinInstance } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import fs from 'fs/promises';
import SwaggerClient from 'swagger-client';

106
packages/basic.gblib/services/DialogKeywords.ts
View file

@ -31,7 +31,7 @@
'use strict';
import { ActivityTypes } from 'botbuilder';
import { GBLog, GBMinInstance } from 'botlib';
import { GBLog, GBMinInstance } from 'botlib-legacy';
import * as df from 'date-diff';
import fs from 'fs/promises';
import libphonenumber from 'google-libphonenumber';
@ -39,9 +39,8 @@ import { Jimp } from 'jimp';
import jsQR from 'jsqr';
import mammoth from 'mammoth';
import mime from 'mime-types';
import tesseract from 'node-tesseract-ocr';
import path from 'path';
import { CollectionUtil } from 'pragmatismo-io-framework';
import puppeteer, { executablePath } from 'puppeteer';
import qrcode from 'qrcode';
import urlJoin from 'url-join';
@ -64,7 +63,6 @@ import { Client } from 'minio';
import nodemailer from 'nodemailer';
const { List, Buttons } = pkg;
/**
* Default check interval for user replay
*/
@ -82,9 +80,9 @@ export class DialogKeywords {
const llmPrompt = `
You are given the following data: ${JSON.stringify(data)}.
Based on this data, generate a configuration for a Billboard.js chart. The output should be valid JSON, following Billboard.js conventions. Ensure the JSON is returned without markdown formatting, explanations, or comments.
The chart should be ${prompt}. Return only the one-line only JSON configuration, nothing else.`;
// Send the prompt to the LLM and get the response
@ -96,9 +94,8 @@ export class DialogKeywords {
const browser = await puppeteer.launch({
headless: process.env.CHROME_HEADLESS === 'true',
executablePath: process.env.CHROME_PATH ? process.env.CHROME_PATH : executablePath(),
}
);
executablePath: process.env.CHROME_PATH ? process.env.CHROME_PATH : executablePath()
});
const page = await browser.newPage();
// Load Billboard.js styles and scripts
@ -115,7 +112,7 @@ export class DialogKeywords {
const content = await page.$('.bb');
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = path.join('work', gbaiName, 'cache', `chart${GBAdminService.getRndReadableIdentifier()}.jpg`);
await content.screenshot({ path: localName, omitBackground: true });
await content.screenshot({ path: localName, omitBackground: true } as any);
await browser.close();
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
GBLogEx.info(min, `Visualization: Chart generated at ${url}.`);
@ -203,18 +200,7 @@ export class DialogKeywords {
* Returns the OCR of image file.
*
*/
public async getOCR({ pid, localFile }) {
const { min, user } = await DialogKeywords.getProcessInfo(pid);
GBLogEx.info(min, `OCR processing on ${localFile}.`);
const config = {
lang: 'eng',
oem: 1,
psm: 3
};
return await tesseract.recognize(localFile, config);
}
public async getOCR({ pid, localFile }) {}
/**
* Returns the today data filled in dd/mm/yyyy or mm/dd/yyyy.
@ -258,28 +244,28 @@ export class DialogKeywords {
*
* @example EXIT
*/
public async exit({ }) { }
public async exit({}) {}
/**
* Get active tasks.
*
* @example list = ACTIVE TASKS
*/
public async getActiveTasks({ pid }) { }
public async getActiveTasks({ pid }) {}
/**
* Creates a new deal.
*
* @example CREATE DEAL dealname,contato,empresa,amount
*/
public async createDeal({ pid, dealName, contact, company, amount }) { }
public async createDeal({ pid, dealName, contact, company, amount }) {}
/**
* Finds contacts in XRM.
*
* @example list = FIND CONTACT "Sandra"
*/
public async fndContact({ pid, name }) { }
public async fndContact({ pid, name }) {}
public getContentLocaleWithCulture(contentLocale) {
switch (contentLocale) {
@ -587,18 +573,17 @@ export class DialogKeywords {
}
if (GBConfigService.get('GB_MODE') !== 'legacy') {
const transporter = nodemailer.createTransport({
host: process.env.EMAIL_SERVER || 'localhost',
port: parseInt(process.env.EMAIL_PORT || '587', 10),
secure: false,
auth: {
user: process.env.EMAIL_USER ,
pass: process.env.EMAIL_PASS ,
user: process.env.EMAIL_USER,
pass: process.env.EMAIL_PASS
},
tls: {
rejectUnauthorized: (process.env.EMAIL_REJECT_UNAUTHORIZED === 'true'),
},
rejectUnauthorized: process.env.EMAIL_REJECT_UNAUTHORIZED === 'true'
}
});
const mailOptions = {
@ -606,16 +591,15 @@ export class DialogKeywords {
to: to,
subject: subject,
text: body,
html: body,
html: body
// headers: {
// 'List-Unsubscribe': `<mailto:${config.unsubscribeEmail}?subject=Unsubscribe>`,
// 'List-Unsubscribe-Post': 'List-Unsubscribe=One-Click'
// }
};
};
await transporter.sendMail(mailOptions);
GBLogEx.info(min, `E-mail ${to} (${subject}) sent via NodeMailer.`);
} else {
let { client } = await GBDeployer.internalGetDriveClient(min);
@ -725,7 +709,7 @@ export class DialogKeywords {
proc.roles = role;
// Checks access.
const file = process.env.GB_MODE === 'legacy' ? 'People.xlsx' : 'people.csv';
const filters = [file, `${role}=x`, `id=${user.userSystemId}`];
const people = await sys.find({ pid, handle: null, args: filters });
@ -953,7 +937,7 @@ export class DialogKeywords {
* @example MENU
*
*/
public async showMenu({ }) {
public async showMenu({}) {
// https://github.com/GeneralBots/BotServer/issues/237
// return await beginDialog('/menu');
}
@ -1016,7 +1000,7 @@ export class DialogKeywords {
if (args.length > 3) {
let section = { title: '', rows: [] };
await CollectionUtil.asyncForEach(args, async arg => {
await GBUtil.asyncForEach(args, async arg => {
i++;
section.rows.push({ title: arg, id: `button${i}` });
});
@ -1024,7 +1008,7 @@ export class DialogKeywords {
await this.talk({ pid: pid, text: list });
} else {
let buttons = [];
await CollectionUtil.asyncForEach(args, async arg => {
await GBUtil.asyncForEach(args, async arg => {
i++;
buttons.push({ body: arg, id: `button${i}` });
});
@ -1095,7 +1079,7 @@ export class DialogKeywords {
// Search the answer in one of valid list items loaded from sheeet.
result = null;
await CollectionUtil.asyncForEach(list, async item => {
await GBUtil.asyncForEach(list, async item => {
if (GBConversationalService.kmpSearch(answer, item) != -1) {
result = item;
}
@ -1239,14 +1223,13 @@ export class DialogKeywords {
const imageData = {
data: new Uint8ClampedArray(image.bitmap.data),
width: image.bitmap.width,
height: image.bitmap.height,
height: image.bitmap.height
};
// Use jsQR to decode the QR code
const decodedQR = jsQR(imageData.data, imageData.width, imageData.height);
result = decodedQR.data;
} else if (kind === 'zipcode') {
const extractEntity = (text: string) => {
text = text.replace(/\-/gi, '');
@ -1270,7 +1253,7 @@ export class DialogKeywords {
} else if (kind === 'menu') {
const list = args;
result = null;
await CollectionUtil.asyncForEach(list, async item => {
await GBUtil.asyncForEach(list, async item => {
if (GBConversationalService.kmpSearch(answer, item) != -1) {
result = item;
}
@ -1299,7 +1282,7 @@ export class DialogKeywords {
{ name: 'alemão', code: 'de' }
];
await CollectionUtil.asyncForEach(list, async item => {
await GBUtil.asyncForEach(list, async item => {
if (
GBConversationalService.kmpSearch(answer.toLowerCase(), item.name.toLowerCase()) != -1 ||
GBConversationalService.kmpSearch(answer.toLowerCase(), item.code.toLowerCase()) != -1
@ -1408,9 +1391,7 @@ export class DialogKeywords {
const conversation = min['apiConversations'][pid];
const client = await GBUtil.getDirectLineClient(min);
conversation.client = client;
const response = await client.apis.Conversations.Conversations_StartConversation(
);
const response = await client.apis.Conversations.Conversations_StartConversation();
conversation.conversationId = response.obj.conversationId;
return await GBVMService.callVM('start', min, null, pid);
@ -1440,7 +1421,6 @@ export class DialogKeywords {
GBLogEx.info(min, `TALK '${text} step:${step}'.`);
if (user) {
text = await min.conversationalService.translate(
min,
text,
@ -1475,7 +1455,6 @@ export class DialogKeywords {
let localName;
const gbaiName = GBUtil.getGBAIPath(min.botId);
// Web automation.
if (element) {
@ -1510,10 +1489,7 @@ export class DialogKeywords {
// .gbdrive direct sending.
else {
if (GBConfigService.get('GB_MODE') === 'legacy') {
const ext = path.extname(filename);
const gbaiName = GBUtil.getGBAIPath(min.botId);
@ -1541,10 +1517,7 @@ export class DialogKeywords {
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
} else if (GBConfigService.get('GB_MODE') === 'gbcluster') {
const ext = path.extname(filename);
const fileUrl = urlJoin('/', `${min.botId}.gbdrive`, filename);
GBLogEx.info(min, `Direct send from .gbdrive: ${fileUrl} to ${mobile}.`);
@ -1556,27 +1529,17 @@ export class DialogKeywords {
port: parseInt(process.env.DRIVE_PORT || '9000', 10),
useSSL: process.env.DRIVE_USE_SSL === 'true',
accessKey: process.env.DRIVE_ACCESSKEY,
secretKey: process.env.DRIVE_SECRET,
secretKey: process.env.DRIVE_SECRET
});
const bucketName = (process.env.DRIVE_ORG_PREFIX + min.botId + '.gbai').toLowerCase();
localName = path.join(
'work',
gbaiName,
'cache',
`${GBAdminService.getNumberIdentifier()}-${fileOnly}`
);
localName = path.join('work', gbaiName, 'cache', `${GBAdminService.getNumberIdentifier()}-${fileOnly}`);
await minioClient.fGetObject(bucketName, fileUrl, localName);
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
GBLogEx.info(min, `Exposing ${localName} to ${url}...`);
}
else {
} else {
const gbdriveName = GBUtil.getGBAIPath(min.botId, 'gbdrive');
localName = path.join(GBConfigService.get('STORAGE_LIBRARY'), gbdriveName, filename);
}
@ -1585,7 +1548,7 @@ export class DialogKeywords {
GBLogEx.info(min, `Converting PDF to images...`);
const pngs = await GBUtil.pdfPageAsImage(min, localName, undefined);
await CollectionUtil.asyncForEach(pngs, async png => {
await GBUtil.asyncForEach(pngs, async png => {
await GBUtil.sleepRandom();
// Prepare a cache to be referenced by Bot Framework.
@ -1600,9 +1563,6 @@ export class DialogKeywords {
contentUrl: url
});
if (!isNaN(mobile)) {
await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename, caption, undefined, true, true);
} else {
@ -1612,7 +1572,6 @@ export class DialogKeywords {
return;
}
}
if (!url) {
@ -1625,7 +1584,6 @@ export class DialogKeywords {
const localName = path.join('work', gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.${ext}`);
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
await fs.writeFile(localName, new Uint8Array(buf), { encoding: null });
}

57
packages/basic.gblib/services/GBVMService.ts
View file

@ -30,16 +30,15 @@
'use strict';
import { GBMinInstance, GBService, IGBCoreService, GBLog } from 'botlib';
import { GBMinInstance, GBService, IGBCoreService, GBLog } from 'botlib-legacy';
import fs from 'fs/promises';
import * as ji from 'just-indent';
import { GBServer } from '../../../src/app.js';
import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { ScheduleServices } from './ScheduleServices.js';
import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js';
import urlJoin from 'url-join';
import { PostgresDialect } from '@sequelize/postgres';
import { NodeVM, VMScript } from 'vm2';
import { createVm2Pool } from './vm2-process/index.js';
import { watch } from 'fs';
@ -53,7 +52,7 @@ import { KeywordsExpressions } from './KeywordsExpressions.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
import { SystemKeywords } from './SystemKeywords.js';
import { Sequelize, QueryTypes } from '@sequelize/core';
import { Sequelize, SequelizeOptions } from 'sequelize-typescript';
import { z } from 'zod';
import { zodToJsonSchema } from 'zod-to-json-schema';
import { GBUtil } from '../../../src/util.js';
@ -73,7 +72,7 @@ export class GBVMService extends GBService {
const ignore = path.join('work', GBUtil.getGBAIPath(min.botId, 'gbdialog'), 'node_modules');
const files = await walkPromise(folder, { ignore: [ignore] });
await CollectionUtil.asyncForEach(files, async file => {
await GBUtil.asyncForEach(files, async file => {
if (!file) {
return;
}
@ -205,7 +204,7 @@ export class GBVMService extends GBService {
"author": "${min.botId} owner.",
"license": "ISC",
"dependencies": {
"encoding": "0.1.13",
"isomorphic-fetch": "3.0.0",
"punycode": "2.1.1",
@ -245,17 +244,16 @@ export class GBVMService extends GBService {
const logging: boolean | Function =
GBConfigService.get('STORAGE_LOGGING') === 'true'
? (str: string): void => {
GBLogEx.info(min, str);
}
GBLogEx.info(min, str);
}
: false;
const encrypt: boolean = GBConfigService.get('STORAGE_ENCRYPT') === 'true';
const acquire = parseInt(GBConfigService.get('STORAGE_ACQUIRE_TIMEOUT'));
let sequelizeOptions;
// Simple function to convert all object keys to lowercase
const toLowerCase = (obj) => {
const toLowerCase = obj => {
if (!obj) return obj;
if (typeof obj !== 'object') return obj;
@ -263,10 +261,9 @@ export class GBVMService extends GBService {
acc[key.toLowerCase()] = obj[key];
return acc;
}, {});
}
};
if (dialect === 'postgres') {
sequelizeOptions = {
host: host,
port: port,
@ -278,8 +275,7 @@ export class GBVMService extends GBService {
connectTimeout: 10000,
query_timeout: 10000,
statement_timeout: 10000,
idle_in_transaction_session_timeout: 10000,
idle_in_transaction_session_timeout: 10000
},
pool: {
max: 1,
@ -300,17 +296,17 @@ export class GBVMService extends GBService {
// Convert all table names to lowercase
freezeTableName: true,
hooks: {
beforeSave: (options) => {
beforeSave: options => {
if (options.where) {
options.where = toLowerCase(options.where);
}
},
beforeDestroy: (options) => {
beforeDestroy: options => {
if (options.where) {
options.where = toLowerCase(options.where);
}
},
beforeFind: (options) => {
beforeFind: options => {
if (options.where) {
options.where = toLowerCase(options.where);
}
@ -324,7 +320,8 @@ export class GBVMService extends GBService {
options.tableName = options.tableName.toLowerCase();
} else {
options.tableName = options.modelName.toLowerCase();
} for (const attr in attributes) {
}
for (const attr in attributes) {
const lowered = attr.toLowerCase();
if (attr !== lowered) {
attributes[lowered] = attributes[attr];
@ -333,14 +330,9 @@ export class GBVMService extends GBService {
}
}
}
},
}
};
}
else {
} else {
sequelizeOptions = {
define: {
charset: 'utf8',
@ -463,7 +455,6 @@ export class GBVMService extends GBService {
if (!con) {
GBLogEx.debug(min, `Invalid connection specified: ${min.bot} ${tableName} ${connectionName}.`);
} else {
// Field checking, syncs if there is any difference.
const seq = con ? con : minBoot.core.sequelize;
@ -471,7 +462,6 @@ export class GBVMService extends GBService {
if (seq) {
const model = seq.models[tableName];
if (model) {
// Except Id, checks if has same number of fields.
let equals = 0;
@ -543,7 +533,6 @@ export class GBVMService extends GBService {
}
public async translateBASIC(mainName, filename: any, min: GBMinInstance) {
// Converts General Bots BASIC into regular VBS
let basicCode: string = await fs.readFile(filename, 'utf8');
@ -557,7 +546,7 @@ export class GBVMService extends GBService {
await s.deleteScheduleIfAny(min, mainName);
let i = 1;
await CollectionUtil.asyncForEach(schedules, async syntax => {
await GBUtil.asyncForEach(schedules, async syntax => {
if (s) {
await s.createOrUpdateSchedule(min, syntax, `${mainName};${i++}`);
}
@ -610,7 +599,6 @@ export class GBVMService extends GBService {
code = ji.default(code, ' ');
await fs.writeFile(jsfile, code);
}
private async executeTasks(min, tasks) {
@ -812,10 +800,9 @@ export class GBVMService extends GBService {
const oldLine = line;
line = line.replace(keywords[j][0], keywords[j][1]);
if(line != oldLine){
if (line != oldLine) {
break;
}
}
}
@ -1009,7 +996,7 @@ export class GBVMService extends GBService {
const strFind = ' Client ID';
const tokens = await min.core['findParam'](min.instance, strFind);
let tokensList = [];
await CollectionUtil.asyncForEach(tokens, async t => {
await GBUtil.asyncForEach(tokens, async t => {
const tokenName = t.replace(strFind, '');
tokensList.push(tokenName);
});
@ -1036,10 +1023,10 @@ export class GBVMService extends GBService {
try {
if (GBConfigService.get('GBVM') !== false) {
return await (async () => {
return await new Promise((resolve) => {
return await new Promise(resolve => {
sandbox['resolve'] = resolve;
// TODO: #411 sandbox['reject'] = reject;
sandbox['reject'] = () => { };
sandbox['reject'] = () => {};
const vm1 = new NodeVM({
allowAsync: true,

6
packages/basic.gblib/services/ImageProcessingServices.ts
View file

@ -31,9 +31,9 @@
'use strict';
import path from 'path';
import { GBLog, GBMinInstance } from 'botlib';
import { GBLog, GBMinInstance } from 'botlib-legacy';
import { DialogKeywords } from './DialogKeywords.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import urlJoin from 'url-join';
import { GBServer } from '../../../src/app.js';
@ -71,7 +71,7 @@ export class ImageProcessingServices {
const { min, user } = await DialogKeywords.getProcessInfo(pid);
let paths = [];
await CollectionUtil.asyncForEach(files, async file => {
await GBUtil.asyncForEach(files, async file => {
const gbfile = DialogKeywords.getFileByHandle(file);
paths.push(gbfile.path);
});

7
packages/basic.gblib/services/ScheduleServices.ts
View file

@ -30,14 +30,15 @@
'use strict';
import { GBLog, GBMinInstance, GBService } from 'botlib';
import { GBLog, GBMinInstance, GBService } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBVMService } from '../../basic.gblib/services/GBVMService.js';
import { GuaribasSchedule } from '../../core.gbapp/models/GBModel.js';
import cron from 'node-cron';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { GBUtil } from '../../../src/util.js';
/**
* @fileoverview Schedule Services.
@ -110,7 +111,7 @@ export class ScheduleServices extends GBService {
let i = 0;
let lastName = '';
await CollectionUtil.asyncForEach(schedules, async item => {
await GBUtil.asyncForEach(schedules, async item => {
if (item.name === lastName) {
item.name = item.name + ++i;
} else {

59
packages/basic.gblib/services/SystemKeywords.ts
View file

@ -33,16 +33,14 @@ import ComputerVisionClient from '@azure/cognitiveservices-computervision';
import ApiKeyCredentials from '@azure/ms-rest-js';
import { BlobServiceClient, BlockBlobClient, StorageSharedKeyCredential } from '@azure/storage-blob';
import { DataTypes, Sequelize } from '@sequelize/core';
import ai2html from 'ai2html';
import alasql from 'alasql';
import retry from 'async-retry';
import { GBLog } from 'botlib';
import { GBLog } from 'botlib-legacy';
import csvdb from 'csv-database';
import Docxtemplater from 'docxtemplater';
import Excel from 'exceljs';
import { Page } from 'facebook-nodejs-business-sdk';
import fs from 'fs/promises';
import { IgApiClient } from 'instagram-private-api';
import { BufferWindowMemory } from 'langchain/memory';
import _ from 'lodash';
import mime from 'mime-types';
@ -51,7 +49,7 @@ import path from 'path';
import { pdfToPng, PngPageOutput } from 'pdf-to-png-converter';
import PizZip from 'pizzip';
import pptxTemplaterModule from 'pptxtemplater';
import { CollectionUtil } from 'pragmatismo-io-framework';
import urlJoin from 'url-join';
import { setFlagsFromString } from 'v8';
import { runInNewContext } from 'vm';
@ -442,7 +440,6 @@ export class SystemKeywords {
useSystemFonts: false,
viewportScale: 2.0,
pagesToProcess: [1],
strictPagesToProcess: false,
verbosityLevel: 0
});
@ -1162,7 +1159,7 @@ export class SystemKeywords {
let filter;
const operators = [/\<\=/, /\<\>/, /\>\=/, /\</, /\>/, /\blike\b/, /\bnot in\b/, /\bin\b/, /\=/];
let done = false;
await CollectionUtil.asyncForEach(operators, async op => {
await GBUtil.asyncForEach(operators, async op => {
var re = new RegExp(op, 'gi');
const parts = text.split(re);
@ -1404,7 +1401,7 @@ export class SystemKeywords {
}
let filterIndex = 0;
await CollectionUtil.asyncForEach(args, async arg => {
await GBUtil.asyncForEach(args, async arg => {
const filter = await SystemKeywords.getFilter(arg);
if (!filter) {
throw new Error(`FIND filter has an error: ${arg} check this and publish .gbdialog again.`);
@ -1447,7 +1444,7 @@ export class SystemKeywords {
let rowCount = 0;
for (; foundIndex < rows.length; foundIndex++) {
let filterAcceptCount = 0;
await CollectionUtil.asyncForEach(filters, async filter => {
await GBUtil.asyncForEach(filters, async filter => {
let result = rows[foundIndex][filter.columnIndex];
let wholeWord = true;
if (user && params && params.wholeWord) {
@ -1514,7 +1511,7 @@ export class SystemKeywords {
const hr = Number.parseInt(filter.value.split(':')[0]);
let lastHour = Number.parseInt(e[0]);
let found = false;
await CollectionUtil.asyncForEach(e, async hour => {
await GBUtil.asyncForEach(e, async hour => {
if (!found && lastHour <= hr && hr <= hour) {
filterAcceptCount++;
found = true;
@ -1709,7 +1706,7 @@ export class SystemKeywords {
// Creates each subfolder.
await CollectionUtil.asyncForEach(parts, async item => {
await GBUtil.asyncForEach(parts, async item => {
// Calls drive API.
const body = {
@ -2902,7 +2899,7 @@ export class SystemKeywords {
// Navigate files / directory to recurse.
await CollectionUtil.asyncForEach(documents, async item => {
await GBUtil.asyncForEach(documents, async item => {
if (item.folder) {
remotePath = urlJoin(remotePath, item.name);
array = [...array, ...(await this.dirFolder({ pid, remotePath, baseUrl, client, array }))];
@ -3020,18 +3017,6 @@ export class SystemKeywords {
fs.unlink(tempFilePath);
}
public async postToInstagram({ pid, username, password, imagePath, caption }) {
const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
const ig = new IgApiClient();
ig.state.generateDevice(username);
await ig.account.login(username, password);
const imageBuffer = await fs.readFile(imagePath);
const publishResult = await ig.publish.photo({ file: imageBuffer, caption });
GBLogEx.info(min, `Image posted on IG: ${publishResult}`);
}
public async setAnswerMode({ pid, mode }) {
const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
@ -3116,30 +3101,6 @@ export class SystemKeywords {
await this.setMemoryContext({ pid, erase: true });
}
public async convertAI2HTML(aiFilePath) {
// Convert the AI file to HTML and assets
const result = await ai2html.convertFile(aiFilePath, {
outputFormat: 'html',
outputWriteMethod: 'write-file',
outputPath: path.dirname(aiFilePath),
useDocumentSettings: true
});
// Get the generated HTML file path
const htmlFilePath = result.outputFiles.find(file => file.endsWith('.html')).filePath;
// Read the HTML content
const htmlContent = await fs.readFile(htmlFilePath, 'utf8');
// Save the HTML and assets to a cache directory
const cacheDir = path.join('work', 'cache');
await fs.mkdir(cacheDir, { recursive: true });
const cacheFilePath = path.join(cacheDir, path.basename(htmlFilePath));
await fs.writeFile(cacheFilePath, htmlContent);
return cacheFilePath;
}
public async refreshDataSourceCache({ pid, connectionName }) {
const { min, user, params, step } = await DialogKeywords.getProcessInfo(pid);
@ -3153,8 +3114,8 @@ export class SystemKeywords {
// Step 2: Connect to SQLite (Local)
const sqlite = new Sequelize({
dialect: 'sqlite',
storage: sqliteFilePath
dialect: 'sqlite3',
url: `file://${sqliteFilePath}`
});
// Get the connection details from the min object

2
packages/basic.gblib/services/WebAutomationServices.ts
View file

@ -34,7 +34,7 @@ import urlJoin from 'url-join';
import fs from 'fs/promises';
import path from 'path';
import url from 'url';
import { GBLog } from 'botlib';
import { GBLog } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { GBSSR } from '../../core.gbapp/services/GBSSR.js';

15
packages/basic.gblib/services/vm2-process/index.ts
View file

@ -3,12 +3,17 @@ import { spawn } from 'child_process';
import CDP from 'chrome-remote-interface';
import {} from 'child_process';
import net from 'net';
import { GBLog } from 'botlib';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBLog } from 'botlib-legacy';
import { GBServer } from '../../../../src/app.js';
import { DebuggerService } from '../DebuggerService.js';
import finalStream from 'final-stream';
import { GBLogEx } from '../../../core.gbapp/services/GBLogEx.js';
import { GBUtil } from '../../../../src/util.js';
let finalStream: any = null;
try {
finalStream = await import('final-stream');
} catch {}
const waitUntil = condition => {
if (condition()) {
@ -208,7 +213,7 @@ export const createVm2Pool = ({ min, max, ...limits }) => {
const variables = await Runtime.getProperties({ objectId: scopeObjectId });
let variablesText = '';
if (variables && variables.result) {
await CollectionUtil.asyncForEach(variables.result, async v => {
await GBUtil.asyncForEach(variables.result, async v => {
if (!systemVariables.filter(x => x === v.name)[0]) {
if (v.value.value) {
variablesText = `${variablesText} \n ${v.name}: ${v.value.value}`;
@ -229,7 +234,7 @@ export const createVm2Pool = ({ min, max, ...limits }) => {
GBLog.verbose(`Configuring breakpoints if any for ${limits.botId}...`);
// Waits for debugger and setup breakpoints.
await CollectionUtil.asyncForEach(GBServer.globals.debuggers[limits.botId].breaks, async brk => {
await GBUtil.asyncForEach(GBServer.globals.debuggers[limits.botId].breaks, async brk => {
try {
const { breakpointId } = await client.Debugger.setBreakpoint({
location: {

2
packages/core.gbapp/dialogs/BroadcastDialog.ts
View file

@ -36,7 +36,7 @@
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { Messages } from '../strings.js';
import { SecService } from '../../security.gbapp/services/SecService.js';
import { GBServer } from '../../../src/app.js';

13
packages/core.gbapp/dialogs/LanguageDialog.ts
View file

@ -36,12 +36,13 @@
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { Messages } from '../strings.js';
import { SecService } from '../../security.gbapp/services/SecService.js';
import { GBServer } from '../../../src/app.js';
import { GBConversationalService } from '../services/GBConversationalService.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBUtil } from '../../../src/util.js';
/**
* Dialog for the bot explains about itself.
*/
@ -52,7 +53,7 @@ export class LanguageDialog extends IGBDialog {
* @param bot The bot adapter.
* @param min The minimal bot instance data.
*/
public static setup (bot: BotAdapter, min: GBMinInstance) {
public static setup(bot: BotAdapter, min: GBMinInstance) {
min.dialogs.add(
new WaterfallDialog('/language', [
async step => {
@ -69,7 +70,7 @@ export class LanguageDialog extends IGBDialog {
return await min.conversationalService.prompt(min, step, Messages[locale].which_language);
},
async step => {
const locale = step.context.activity.locale;
const locale = step.context.activity.locale;
const list = [
{ name: 'english', code: 'en' },
@ -90,7 +91,7 @@ export class LanguageDialog extends IGBDialog {
let translatorLocale = null;
const text = step.context.activity['originalText'];
await CollectionUtil.asyncForEach(list, async item => {
await GBUtil.asyncForEach(list, async item => {
if (
GBConversationalService.kmpSearch(text.toLowerCase(), item.name.toLowerCase()) != -1 ||
GBConversationalService.kmpSearch(text.toLowerCase(), item.code.toLowerCase()) != -1
@ -100,7 +101,7 @@ export class LanguageDialog extends IGBDialog {
});
let sec = new SecService();
let user = await sec.getUserFromSystemId(step.context.activity.from.id);
let user = await sec.getUserFromSystemId(step.context.activity.from.id);
user = await sec.updateUserLocale(user.userId, translatorLocale);
await min.conversationalService.sendText(min, step, Messages[locale].language_chosen);

2
packages/core.gbapp/dialogs/SwitchBot.ts
View file

@ -36,7 +36,7 @@
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import { SecService } from '../../security.gbapp/services/SecService.js';
import { GBConversationalService } from '../services/GBConversationalService.js';

2
packages/core.gbapp/dialogs/WelcomeDialog.ts
View file

@ -36,7 +36,7 @@
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import { Messages } from '../strings.js';
import { GBLogEx } from '../services/GBLogEx.js';

4
packages/core.gbapp/dialogs/WhoAmIDialog.ts
View file

@ -36,7 +36,7 @@
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { GBConversationalService } from '../services/GBConversationalService.js';
import { Messages } from '../strings.js';
/**
@ -49,7 +49,7 @@ export class WhoAmIDialog extends IGBDialog {
* @param bot The bot adapter.
* @param min The minimal bot instance data.
*/
public static setup (bot: BotAdapter, min: GBMinInstance) {
public static setup(bot: BotAdapter, min: GBMinInstance) {
min.dialogs.add(
new WaterfallDialog('/whoAmI', [
async step => {

2
packages/core.gbapp/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
import { BroadcastDialog } from './dialogs/BroadcastDialog.js';
import { LanguageDialog } from './dialogs/LanguageDialog.js';

2
packages/core.gbapp/models/GBModel.ts
View file

@ -47,7 +47,7 @@ import {
UpdatedAt
} from 'sequelize-typescript';
import { IGBInstance } from 'botlib';
import { IGBInstance } from 'botlib-legacy';
/**
* Base instance data for a bot.

2
packages/core.gbapp/services/GBConfigService.ts
View file

@ -30,7 +30,7 @@
'use strict';
import { GBLog } from 'botlib';
import { GBLog } from 'botlib-legacy';
import * as en from 'dotenv-extended';
import path from 'path';

85
packages/core.gbapp/services/GBConversationalService.ts
View file

@ -36,7 +36,7 @@
import { MessageFactory, RecognizerResult, TurnContext } from 'botbuilder';
import { LuisRecognizer } from 'botbuilder-ai';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import { Readable } from 'stream';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
@ -45,20 +45,16 @@ import { AnalyticsService } from '../../analytics.gblib/services/AnalyticsServic
import { MicrosoftAppCredentials } from 'botframework-connector';
import { DocxLoader } from '@langchain/community/document_loaders/fs/docx';
import { GBConfigService } from './GBConfigService.js';
import { CollectionUtil, AzureText } from 'pragmatismo-io-framework';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
import { GBMinService } from './GBMinService.js';
import urlJoin from 'url-join';
import { createWriteStream, createReadStream } from 'fs';
import fs from 'fs/promises';
import twilio from 'twilio';
import Nexmo from 'nexmo';
import path, { join } from 'path';
import shell from 'any-shell-escape';
import { exec } from 'child_process';
import prism from 'prism-media';
import SpeechToTextV1 from 'ibm-watson/speech-to-text/v1.js';
import TextToSpeechV1 from 'ibm-watson/text-to-speech/v1.js';
import { IamAuthenticator } from 'ibm-watson/auth/index.js';
@ -67,6 +63,8 @@ import Translate from '@google-cloud/translate';
import { GBUtil } from '../../../src/util.js';
import { GBLogEx } from './GBLogEx.js';
import shell from 'any-shell-escape';
/**
* Provides basic services for handling messages and dispatching to back-end
* services like NLP or Search.
@ -401,25 +399,6 @@ export class GBConversationalService {
return Promise.reject(new Error(msg));
}
} else {
if (min.instance.smsKey && min.instance.smsSecret) {
return new Promise((resolve: any, reject: any): any => {
const nexmo = new Nexmo({
apiKey: min.instance.smsKey,
apiSecret: min.instance.smsSecret
});
// tslint:disable-next-line:no-unsafe-any
nexmo.message.sendSms(min.instance.smsServiceNumber, mobile, text, {}, (err, data) => {
const message = data.messages ? data.messages[0] : {};
if (err || message['error-text']) {
GBLog.error(`error sending SMS to ${mobile}: ${message['error-text']}`);
reject(message['error-text']);
} else {
resolve(data);
}
});
});
}
}
}
@ -474,7 +453,7 @@ export class GBConversationalService {
return new Promise<string>(async (resolve, reject) => {
try {
const oggFile = new Readable();
oggFile._read = () => { }; // _read is required but you can noop it
oggFile._read = () => {}; // _read is required but you can noop it
oggFile.push(buffer);
oggFile.push(null);
@ -602,7 +581,7 @@ export class GBConversationalService {
renderer: renderer,
gfm: true,
breaks: false,
pedantic: false,
pedantic: false
});
// MSFT Translator breaks markdown, so we need to manually fix it:
@ -944,7 +923,7 @@ export class GBConversationalService {
// FIX MSFT NLP issue.
if (nlp.entities) {
await CollectionUtil.asyncForEach(Object.keys(nlp.entities), async key => {
await GBUtil.asyncForEach(Object.keys(nlp.entities), async key => {
if (key !== '$instance') {
let entity = nlp.entities[key];
if (Array.isArray(entity[0])) {
@ -964,31 +943,12 @@ export class GBConversationalService {
}
public async getLanguage(min: GBMinInstance, text: string): Promise<string> {
const key = min.core.getParam<string>(min.instance, 'textAnalyticsKey', null);
if (!key) {
return process.env.DEFAULT_USER_LANGUAGE;
}
let language = await AzureText.getLocale(
key,
min.core.getParam<string>(min.instance, 'textAnalyticsEndpoint', null),
text
);
return language === '(Unknown)' ? 'en' : language;
// TODO: Azure removed.
return process.env.DEFAULT_USER_LANGUAGE;
}
public async spellCheck(min: GBMinInstance, text: string): Promise<string> {
const key = min.core.getParam<string>(min.instance, 'spellcheckerKey', null);
if (key) {
text = text.charAt(0).toUpperCase() + text.slice(1);
const data = await AzureText.getSpelledText(key, text);
if (data !== text) {
GBLogEx.info(min, `Spelling>: ${data}`);
text = data;
}
}
// TODO: Azure removed.
return text;
}
@ -1091,7 +1051,7 @@ export class GBConversationalService {
keepTextList = keepTextList.concat(keepText.split(';'));
}
const replacements = [];
await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
await GBUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
const result = await e.onExchangeData(min, 'getKeepText', {});
if (result) {
keepTextList = keepTextList.concat(result);
@ -1118,7 +1078,7 @@ export class GBConversationalService {
if (keepTextList) {
keepTextList = keepTextList.filter(p => p.trim() !== '');
let i = 0;
await CollectionUtil.asyncForEach(keepTextList, item => {
await GBUtil.asyncForEach(keepTextList, item => {
const it = GBConversationalService.removeDiacritics(item);
const noAccentText = GBConversationalService.removeDiacritics(textProcessed);
@ -1175,7 +1135,7 @@ export class GBConversationalService {
if (keepTextList) {
let i = 0;
await CollectionUtil.asyncForEach(replacements, item => {
await GBUtil.asyncForEach(replacements, item => {
i++;
text = text.replace(new RegExp(`${item.replacementToken}`, 'gi'), item.text);
});
@ -1219,7 +1179,6 @@ export class GBConversationalService {
}
public async sendTextWithOptionsAndUser(min: GBMinInstance, user, step, text, translate, keepTextList) {
let replacements = [];
// To fix MSFT bug.
@ -1227,7 +1186,7 @@ export class GBConversationalService {
if (keepTextList) {
keepTextList = keepTextList.filter(p => p.trim() !== '');
let i = 0;
await CollectionUtil.asyncForEach(keepTextList, item => {
await GBUtil.asyncForEach(keepTextList, item => {
if (text.toLowerCase().indexOf(item.toLowerCase()) != -1) {
const replacementToken = GBAdminService['getNumberIdentifier']();
replacements[i] = { text: item, replacementToken: replacementToken };
@ -1246,7 +1205,7 @@ export class GBConversationalService {
if (keepTextList) {
let i = 0;
await CollectionUtil.asyncForEach(replacements, item => {
await GBUtil.asyncForEach(replacements, item => {
i++;
text = text.replace(new RegExp(`${item.replacementToken}`, 'gi'), item.text);
});
@ -1263,12 +1222,9 @@ export class GBConversationalService {
}
if (!isNaN(user.userSystemId)) {
await min.whatsAppDirectLine.sendToDevice(user.userSystemId, text);
}
else {
} else {
await step.context.sendActivity(text);
}
}
public async broadcast(min: GBMinInstance, message: string) {
@ -1276,7 +1232,7 @@ export class GBConversationalService {
const service = new SecService();
const users = await service.getAllUsers(min.instance.instanceId);
await CollectionUtil.asyncForEach(users, async user => {
await GBUtil.asyncForEach(users, async user => {
if (user.conversationReference) {
await this.sendOnConversation(min, user, message);
} else {
@ -1299,21 +1255,18 @@ export class GBConversationalService {
await t2.sendActivity(message);
});
});
} else {
const ref = JSON.parse(user.conversationReference);
await min.bot['continueConversation'](ref, async (t1) => {
await min.bot['continueConversation'](ref, async t1 => {
const ref2 = TurnContext.getConversationReference(t1.activity);
await min.bot.continueConversation(ref2, async (t2) => {
await min.bot.continueConversation(ref2, async t2 => {
await t2.sendActivity(message);
});
});
if (message['buttons'] || message['sections']) {
await min['whatsAppDirectLine'].sendToDevice(user.userSystemId, message, user.conversationReference);
}
else if (user.conversationReference && user.conversationReference.startsWith('spaces')) {
} else if (user.conversationReference && user.conversationReference.startsWith('spaces')) {
await min['googleDirectLine'].sendToDevice(user.userSystemId, null, user.conversationReference, message);
}
}

192
packages/core.gbapp/services/GBCoreService.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBLog, GBMinInstance, IGBCoreService, IGBInstallationDeployer, IGBInstance, IGBPackage } from 'botlib';
import { GBLog, GBMinInstance, IGBCoreService, IGBInstallationDeployer, IGBInstance, IGBPackage } from 'botlib-legacy';
import fs from 'fs/promises';
import { Sequelize, SequelizeOptions } from 'sequelize-typescript';
import { Op, Dialect } from 'sequelize';
@ -42,7 +42,6 @@ import { GBServer } from '../../../src/app.js';
import { GBAdminPackage } from '../../admin.gbapp/index.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { GBAnalyticsPackage } from '../../analytics.gblib/index.js';
import { StartDialog } from '../../azuredeployer.gbapp/dialogs/StartDialog.js';
import { GBCorePackage } from '../../core.gbapp/index.js';
import { GBCustomerSatisfactionPackage } from '../../customer-satisfaction.gbapp/index.js';
import { GBKBPackage } from '../../kb.gbapp/index.js';
@ -52,14 +51,10 @@ import { GBWhatsappPackage } from '../../whatsapp.gblib/index.js';
import { GuaribasApplications, GuaribasInstance, GuaribasLog } from '../models/GBModel.js';
import { GBConfigService } from './GBConfigService.js';
import mkdirp from 'mkdirp';
import { GBAzureDeployerPackage } from '../../azuredeployer.gbapp/index.js';
import { GBSharePointPackage } from '../../sharepoint.gblib/index.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBBasicPackage } from '../../basic.gblib/index.js';
import { GBGoogleChatPackage } from '../../google-chat.gblib/index.js';
import { GBHubSpotPackage } from '../../hubspot.gblib/index.js';
import open from 'open';
import ngrok from 'ngrok';
import path from 'path';
import { GBUtil } from '../../../src/util.js';
import { GBLogEx } from './GBLogEx.js';
@ -113,7 +108,7 @@ export class GBCoreService implements IGBCoreService {
constructor() {
this.adminService = new GBAdminService(this);
}
public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) { }
public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) {}
/**
* Gets database config and connect to storage. Currently two databases
@ -129,12 +124,10 @@ export class GBCoreService implements IGBCoreService {
let password: string | undefined;
let storage: string | undefined;
if (!['mssql', 'postgres', 'sqlite'].includes(this.dialect)) {
throw new Error(`Unknown or unsupported dialect: ${this.dialect}.`);
}
if (this.dialect === 'mssql' || this.dialect === 'postgres') {
host = GBConfigService.get('STORAGE_SERVER');
database = GBConfigService.get('STORAGE_NAME');
@ -159,26 +152,22 @@ export class GBCoreService implements IGBCoreService {
}
}
const logging: boolean | Function =
GBConfigService.get('STORAGE_LOGGING') === 'true'
? (str: string): void => {
GBLogEx.info(0, str);
}
GBLogEx.info(0, str);
}
: false;
const encrypt: boolean = GBConfigService.get('STORAGE_ENCRYPT') === 'true';
const acquireStr = GBConfigService.get('STORAGE_ACQUIRE_TIMEOUT');
const acquire = acquireStr ? parseInt(acquireStr, 10) : 10000; // Valor padrão de 10 segundos
const sequelizeOptions: SequelizeOptions = {
define: {
freezeTableName: true,
timestamps: false,
timestamps: false
},
host: host,
port: port,
@ -186,22 +175,24 @@ export class GBCoreService implements IGBCoreService {
dialect: this.dialect as Dialect,
storage: storage,
quoteIdentifiers: this.dialect === 'postgres',
dialectOptions: this.dialect === 'mssql' ? {
options: {
trustServerCertificate: true,
encrypt: encrypt,
},
} : {},
dialectOptions:
this.dialect === 'mssql'
? {
options: {
trustServerCertificate: true,
encrypt: encrypt
}
}
: {},
pool: {
max: 5,
min: 0,
idle: 10000,
evict: 10000,
acquire: acquire,
},
acquire: acquire
}
};
this.sequelize = new Sequelize(database, username, password, sequelizeOptions);
}
@ -257,7 +248,7 @@ export class GBCoreService implements IGBCoreService {
};
const list = await GuaribasLog.findAll(options);
let out = 'General Bots Log\n';
await CollectionUtil.asyncForEach(list, async e => {
await GBUtil.asyncForEach(list, async e => {
out = `${out}\n${e.createdAt} - ${e.message}`;
});
return out;
@ -270,7 +261,7 @@ export class GBCoreService implements IGBCoreService {
if (process.env.LOAD_ONLY) {
const bots = process.env.LOAD_ONLY.split(`;`);
const and = [];
await CollectionUtil.asyncForEach(bots, async e => {
await GBUtil.asyncForEach(bots, async e => {
and.push({ botId: e });
});
@ -282,9 +273,11 @@ export class GBCoreService implements IGBCoreService {
};
return await GuaribasInstance.findAll(options as any);
} else {
const options = { where: { state: 'active' } ,
order: [['instanceId', 'ASC']]};
const options = {
where: { state: 'active' },
order: [['instanceId', 'ASC']]
};
return await GuaribasInstance.findAll(options as any);
}
}
@ -343,31 +336,6 @@ ENDPOINT_UPDATE=true
await fs.writeFile('.env', env);
}
/**
* Certifies that network servers will reach back the development machine
* when calling back from web services. This ensures that reverse proxy is
* established.
*/
public async ensureProxy(port): Promise<string> {
try {
if (
(await GBUtil.exists('node_modules/ngrok/bin/ngrok.exe')) ||
(await GBUtil.exists('node_modules/.bin/ngrok'))
) {
return await ngrok.connect({ port: port });
} else {
GBLog.warn('ngrok executable not found. Check installation or node_modules folder.');
return 'https://localhost';
}
} catch (error) {
// There are false positive from ngrok regarding to no memory, but it's just
// lack of connection.
throw new Error(`Error connecting to remote ngrok server, please check network connection. ${error.msg}`);
}
}
/**
* Setup generic web hooks so .gbapps can expose application logic
* and get called on demand.
@ -439,7 +407,7 @@ ENDPOINT_UPDATE=true
const apps = await GuaribasApplications.findAll(options);
let matchingAppPackages = [];
await CollectionUtil.asyncForEach(appPackages, async appPackage => {
await GBUtil.asyncForEach(appPackages, async appPackage => {
const filenameOnly = path.basename(appPackage.name);
const matchedApp = apps.find(app => app.name === filenameOnly);
if (matchedApp || filenameOnly.endsWith('.gblib')) {
@ -464,7 +432,7 @@ ENDPOINT_UPDATE=true
instances = await core.loadInstances();
if (process.env.ENDPOINT_UPDATE === 'true') {
const group = GBConfigService.get('CLOUD_GROUP') ?? GBConfigService.get('BOT_ID');
await CollectionUtil.asyncForEach(instances, async instance => {
await GBUtil.asyncForEach(instances, async instance => {
GBLogEx.info(instance.instanceId, `Updating bot endpoint for ${instance.botId}...`);
try {
await installationDeployer.updateBotProxy(
@ -517,7 +485,7 @@ ENDPOINT_UPDATE=true
// Loads all system packages.
const sysPackages: IGBPackage[] = [];
await CollectionUtil.asyncForEach(
await GBUtil.asyncForEach(
[
GBAdminPackage,
GBCorePackage,
@ -526,11 +494,9 @@ ENDPOINT_UPDATE=true
GBCustomerSatisfactionPackage,
GBAnalyticsPackage,
GBWhatsappPackage,
GBAzureDeployerPackage,
GBSharePointPackage,
GBGoogleChatPackage,
GBBasicPackage,
GBHubSpotPackage,
SaaSPackage
],
async e => {
@ -550,88 +516,16 @@ ENDPOINT_UPDATE=true
* Verifies that an complex global password has been specified
* before starting the server.
*/
public ensureAdminIsSecured() { }
public ensureAdminIsSecured() {}
public async createBootInstance(
core: GBCoreService,
installationDeployer: IGBInstallationDeployer,
proxyAddress: string
) {
return await this.createBootInstanceEx(
core,
installationDeployer,
proxyAddress,
null,
GBConfigService.get('FREE_TIER')
);
}
/**
* Creates the first bot instance (boot instance) used to "boot" the server.
* At least one bot is required to perform conversational administrative tasks.
* So a base main bot is always deployed and will act as root bot for
* configuration tree with three levels: .env > root bot > all other bots.
*/
public async createBootInstanceEx(
core: GBCoreService,
installationDeployer: IGBInstallationDeployer,
proxyAddress: string,
deployer,
freeTier
) {
GBLogEx.info(0, `Deploying cognitive infrastructure (on the cloud / on premises)...`);
try {
const { instance, credentials, subscriptionId, installationDeployer } = await StartDialog.createBaseInstance(
deployer,
freeTier
);
installationDeployer['core'] = this;
const changedInstance = await installationDeployer['deployFarm2'](
proxyAddress,
instance,
credentials,
subscriptionId
);
await this.writeEnv(changedInstance);
GBConfigService.init();
) {}
GBLogEx.info(0, `File .env written. Preparing storage and search for the first time...`);
await this.openStorageFrontier(installationDeployer);
await this.initStorage();
public async openBrowserInDevelopment() {}
return [changedInstance, installationDeployer];
} catch (error) {
GBLog.warn(
`There is an error being thrown, so please cleanup any infrastructure objects
created during this procedure and .env before running again.`
);
throw error;
}
}
/**
* Helper to get the web browser onpened in UI interfaces.
*/
public openBrowserInDevelopment() {
if (process.env.NODE_ENV === 'development') {
open('http://localhost:4242');
}
}
/**
* SQL:
*
* // let sql: string = '' +
* // 'IF OBJECT_ID(\'[UserGroup]\', \'U\') IS NULL' +
* // 'CREATE TABLE [UserGroup] (' +
* // ' [id] INTEGER NOT NULL IDENTITY(1,1),' +
* // ' [userId] INTEGER NULL,' +
* // ' [groupId] INTEGER NULL,' +
* // ' [instanceId] INTEGER NULL,' +
* // ' PRIMARY KEY ([id1], [id2]),' +
* // ' FOREIGN KEY ([userId1], [userId2], [userId3]) REFERENCES [User] ([userId1], [userId2], [userId3]) ON DELETE NO ACTION,' +
* // ' FOREIGN KEY ([groupId1], [groupId2]) REFERENCES [Group] ([groupId1], [groupId1]) ON DELETE NO ACTION,' +
* // ' FOREIGN KEY ([instanceId]) REFERENCES [Instance] ([instanceId]) ON DELETE NO ACTION)'
*/
private createTableQueryOverride(tableName, attributes, options): string {
let sql: string = this.createTableQuery.apply(this.queryGenerator, [tableName, attributes, options]);
const re1 = /CREATE\s+TABLE\s+\[([^\]]*)\]/;
@ -718,7 +612,9 @@ ENDPOINT_UPDATE=true
// Get the current rows in column A
let results = await client
.api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A1:A${maxLines}')`)
.api(
`${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A1:A${maxLines}')`
)
.get();
const rows = results.values;
@ -748,11 +644,12 @@ ENDPOINT_UPDATE=true
// Update or add the new value in the found address
await client
.api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${address}')`)
.api(
`${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${address}')`
)
.patch(body);
}
else if (GBConfigService.get('GB_MODE') === 'local') {
let packagePath = GBUtil.getGBAIPath(min.botId, `gbot`);
} else if (GBConfigService.get('GB_MODE') === 'local') {
let packagePath = GBUtil.getGBAIPath(min.botId, `gbot`);
const config = path.join(GBConfigService.get('STORAGE_LIBRARY'), packagePath, 'config.csv');
const db = await csvdb(config, ['name', 'value'], ',');
@ -862,6 +759,10 @@ ENDPOINT_UPDATE=true
return list;
}
public async ensureProxy(port: any): Promise<string> {
return '';
} // Azure removed.
public async ensureFolders(instances, deployer: GBDeployer) {
const storageMode = process.env.GB_MODE;
let libraryPath = GBConfigService.get('STORAGE_LIBRARY');
@ -872,7 +773,7 @@ ENDPOINT_UPDATE=true
port: parseInt(process.env.DRIVE_PORT),
useSSL: process.env.DRIVE_USE_SSL === 'true',
accessKey: process.env.DRIVE_ACCESSKEY,
secretKey: process.env.DRIVE_SECRET,
secretKey: process.env.DRIVE_SECRET
});
await this.syncBotStorage(instances, 'default', deployer, libraryPath);
@ -881,10 +782,8 @@ ENDPOINT_UPDATE=true
for await (const bucket of bucketStream) {
if (bucket.name.endsWith('.gbai') && bucket.name.startsWith(process.env.DRIVE_ORG_PREFIX)) {
const botId = bucket.name.replace('.gbai', '').replace(process.env.DRIVE_ORG_PREFIX, '');
await this.syncBotStorage(instances, botId, deployer, libraryPath);
}
}
} else {
@ -895,19 +794,16 @@ ENDPOINT_UPDATE=true
await this.syncBotStorage(instances, 'default', deployer, libraryPath);
const files = await fs.readdir(libraryPath);
await CollectionUtil.asyncForEach(files, async (file) => {
await GBUtil.asyncForEach(files, async file => {
if (file.trim().toLowerCase() !== 'default.gbai' && file.charAt(0) !== '_') {
let botId = file.replace(/\.gbai/, '');
await this.syncBotStorage(instances, botId, deployer, libraryPath);
}
});
}
}
private async syncBotStorage(instances: any, botId: any, deployer: GBDeployer, libraryPath: string) {
let instance = instances.find(p => p.botId.toLowerCase().trim() === botId.toLowerCase().trim());
if (process.env.GB_MODE === 'local') {

190
packages/core.gbapp/services/GBDeployer.ts
View file

@ -42,16 +42,14 @@ import urlJoin from 'url-join';
import { Client } from 'minio';
import fs from 'fs/promises';
import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBDeployer, IGBInstance, IGBPackage } from 'botlib';
import { AzureSearch } from 'pragmatismo-io-framework';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBDeployer, IGBInstance, IGBPackage } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import { GBVMService } from '../../basic.gblib/services/GBVMService.js';
import Excel from 'exceljs';
import asyncPromise from 'async-promises';
import { GuaribasInstance, GuaribasPackage } from '../models/GBModel.js';
import { GBAdminService } from './../../admin.gbapp/services/GBAdminService.js';
import { AzureDeployerService } from './../../azuredeployer.gbapp/services/AzureDeployerService.js';
import { KBService } from './../../kb.gbapp/services/KBService.js';
import { GBConfigService } from './GBConfigService.js';
import { GBImporter } from './GBImporterService.js';
@ -60,7 +58,7 @@ import MicrosoftGraph from '@microsoft/microsoft-graph-client';
import { GBLogEx } from './GBLogEx.js';
import { GBUtil } from '../../../src/util.js';
import { HNSWLib } from '@langchain/community/vectorstores/hnswlib';
import { OpenAIEmbeddings } from '@langchain/openai';
import { AzureOpenAIEmbeddings, OpenAIEmbeddings } from '@langchain/openai';
import { GBMinService } from './GBMinService.js';
/**
@ -157,7 +155,7 @@ export class GBDeployer implements IGBDeployer {
const getDirectories = async source =>
(await fs.readdir(source)).map(name => path.join(source, name)).filter(isDirectory);
const dirs = await getDirectories(directory);
await CollectionUtil.asyncForEach(dirs, async element => {
await GBUtil.asyncForEach(dirs, async element => {
// For each folder, checks its extensions looking for valid packages.
if (element === '.') {
@ -190,7 +188,7 @@ export class GBDeployer implements IGBDeployer {
// Start the process of searching.
GBLogEx.info(0, `Deploying Application packages...`);
await CollectionUtil.asyncForEach(paths, async e => {
await GBUtil.asyncForEach(paths, async e => {
GBLogEx.info(0, `Looking in: ${e}...`);
await scanPackageDirectory(e);
});
@ -222,25 +220,6 @@ export class GBDeployer implements IGBDeployer {
const instance = await this.importer.createBotInstance(botId);
const bootInstance = GBServer.globals.bootInstance;
if (GBConfigService.get('GB_MODE') === 'legacy') {
// Gets the access token to perform service operations.
const accessToken = await (GBServer.globals.minBoot.adminService as any)['acquireElevatedToken'](
bootInstance.instanceId,
true
);
// Creates the MSFT application that will be associated to the bot.
const service = await AzureDeployerService.createInstance(this);
const application = await service.createApplication(accessToken, botId);
// Fills new instance base information and get App secret.
instance.marketplaceId = (application as any).appId;
instance.marketplacePassword = await service.createApplicationSecret(accessToken, (application as any).id);
}
instance.adminPass = await GBUtil.hashPassword(GBAdminService.getRndPassword());
instance.title = botId;
instance.activationCode = instance.botId.substring(0, 15);
@ -252,9 +231,6 @@ export class GBDeployer implements IGBDeployer {
// Saves bot information to the store.
await this.core.saveInstance(instance);
if (GBConfigService.get('GB_MODE') === 'legacy') {
await this.deployBotOnAzure(instance, GBServer.globals.publicAddress);
}
// Makes available bot to the channels and .gbui interfaces.
@ -278,61 +254,10 @@ export class GBDeployer implements IGBDeployer {
where: where
})) !== null
);
} else {
const service = await AzureDeployerService.createInstance(this);
return await service.botExists(botId);
return false;
}
}
/**
* Performs all tasks of deploying a new bot on the cloud.
*/
public async deployBotOnAzure(instance: IGBInstance, publicAddress: string): Promise<IGBInstance> {
// Reads base configuration from environent file.
const service = await AzureDeployerService.createInstance(this);
const username = GBConfigService.get('CLOUD_USERNAME');
const password = GBConfigService.get('CLOUD_PASSWORD');
const accessToken = await GBAdminService.getADALTokenFromUsername(username, password);
const group = GBConfigService.get('CLOUD_GROUP') ?? GBConfigService.get('BOT_ID');
const subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID');
// If the bot already exists, just update the endpoint.
if (await service.botExists(instance.botId)) {
await service.updateBot(
instance.botId,
group,
instance.title,
instance.description,
`${publicAddress}/api/messages/${instance.botId}`
);
} else {
// Internally create resources on cloud provider.
instance = await service.internalDeployBot(
instance,
accessToken,
instance.botId,
instance.title,
group,
instance.description,
`${publicAddress}/api/messages/${instance.botId}`,
'global',
instance.nlpAppId,
instance.nlpKey,
instance.marketplaceId,
instance.marketplacePassword,
subscriptionId
);
}
// Saves final instance object and returns it.
return await this.core.saveInstance(instance);
}
public async loadOrCreateEmptyVectorStore(min: GBMinInstance): Promise<HNSWLib> {
let vectorStore: HNSWLib;
@ -390,7 +315,7 @@ export class GBDeployer implements IGBDeployer {
return;
}
embedding = new OpenAIEmbeddings({
embedding = new AzureOpenAIEmbeddings({
maxConcurrency: 5,
azureOpenAIApiKey: azureOpenAIKey,
azureOpenAIApiDeploymentName: azureOpenAIEmbeddingModel,
@ -425,23 +350,14 @@ export class GBDeployer implements IGBDeployer {
* Performs the NLP publishing process on remote service.
*/
public async publishNLP(instance: IGBInstance): Promise<void> {
const service = await AzureDeployerService.createInstance(this);
const res = await service.publishNLP(instance.cloudLocation, instance.nlpAppId, instance.nlpAuthoringKey);
if (res.status !== 200 && res.status !== 201) {
throw res.bodyAsText;
}
// TODO: Azure removed.
}
/**
* Trains NLP on the remote service.
*/
public async trainNLP(instance: IGBInstance): Promise<void> {
const service = await AzureDeployerService.createInstance(this);
const res = await service.trainNLP(instance.cloudLocation, instance.nlpAppId, instance.nlpAuthoringKey);
if (res.status !== 200 && res.status !== 202) {
throw res.bodyAsText;
}
GBUtil.sleep(5000);
// TODO: Azure removed.
}
/**
@ -465,27 +381,13 @@ export class GBDeployer implements IGBDeployer {
* Refreshes NLP entities on the remote service.
*/
public async refreshNLPEntity(instance: IGBInstance, listName, listData): Promise<void> {
const service = await AzureDeployerService.createInstance(this);
const res = await service.refreshEntityList(
instance.cloudLocation,
instance.nlpAppId,
listName,
instance.nlpAuthoringKey,
listData
);
if (res.status !== 200) {
throw res.bodyAsText;
}
// Azure removed.
}
/**
* Deploys a bot to the storage from a .gbot folder.
*/
public async deployBotFromLocalPath(localPath: string, publicAddress: string): Promise<void> {
const packageName = path.basename(localPath);
const instance = await this.importer.importIfNotExistsBotPackage(undefined, packageName, localPath);
await this.deployBotOnAzure(instance, publicAddress);
}
public async deployBotFromLocalPath(localPath: string, publicAddress: string): Promise<void> {}
/**
* Loads all para from tabular file Config.xlsx.
@ -618,7 +520,7 @@ export class GBDeployer implements IGBDeployer {
await fs.mkdir(pathBase, { recursive: true });
}
await CollectionUtil.asyncForEach(parts, async item => {
await GBUtil.asyncForEach(parts, async item => {
pathBase = path.join(pathBase, item);
if (!(await GBUtil.exists(pathBase))) {
await fs.mkdir(pathBase, { recursive: true });
@ -642,7 +544,7 @@ export class GBDeployer implements IGBDeployer {
return null;
}
await CollectionUtil.asyncForEach(documents, async item => {
await GBUtil.asyncForEach(documents, async item => {
const itemPath = path.join(localPath, remotePath, item.name);
if (item.folder) {
@ -680,11 +582,7 @@ export class GBDeployer implements IGBDeployer {
public async undeployBot(botId: string, packageName: string): Promise<void> {
// Deletes Bot registration on cloud.
const service = await AzureDeployerService.createInstance(this);
const group = GBConfigService.get('BOT_ID');
if (await service.botExists(botId)) {
await service.deleteBot(botId, group);
}
// Unbinds resources and listeners.
@ -742,7 +640,7 @@ export class GBDeployer implements IGBDeployer {
// Asks for each .gbapp if it will handle the package publishing.
const _this = this;
await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
await GBUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
// If it will be handled, create a temporary service layer to be
// called by .gbapp and manage the associated package row.
@ -785,7 +683,7 @@ export class GBDeployer implements IGBDeployer {
// Find all tokens in .gbot Config.
const strFind = ' Driver';
const conns = await min.core['findParam'](min.instance, strFind);
await CollectionUtil.asyncForEach(conns, async t => {
await GBUtil.asyncForEach(conns, async t => {
const connectionName = t.replace(strFind, '').trim();
let con = {};
con['name'] = connectionName;
@ -942,61 +840,7 @@ export class GBDeployer implements IGBDeployer {
* Performs automation of the Indexer (Azure Search) and rebuild
* its index based on .gbkb structure.
*/
public async rebuildIndex(instance: IGBInstance, searchSchema: any) {
const key = instance.searchKey ? instance.searchKey : GBServer.globals.minBoot.instance.searchKey;
GBLogEx.info(instance.instanceId, `rebuildIndex running...`);
if (!key) {
return;
}
const searchIndex = instance.searchIndex ? instance.searchIndex : GBServer.globals.minBoot.instance.searchIndex;
const searchIndexer = instance.searchIndexer
? instance.searchIndexer
: GBServer.globals.minBoot.instance.searchIndexer;
const host = instance.searchHost ? instance.searchHost : GBServer.globals.minBoot.instance.searchHost;
// Prepares search.
const search = new AzureSearch(key, host, searchIndex, searchIndexer);
const connectionString = GBDeployer.getConnectionStringFromInstance(GBServer.globals.minBoot.instance);
const dsName = 'gb';
// Removes any previous index.
try {
await search.deleteDataSource(dsName);
} catch (error) {
// If it is a 404 there is nothing to delete as it is the first creation.
if (error.code !== 404) {
throw error;
}
}
// Removes the index.
try {
await search.deleteIndex();
} catch (error) {
// If it is a 404 there is nothing to delete as it is the first creation.
if (error.code !== 404 && error.code !== 'OperationNotAllowed') {
throw error;
}
}
// Creates the data source and index on the cloud.
try {
await search.createDataSource(dsName, dsName, 'GuaribasQuestion', 'azuresql', connectionString);
} catch (error) {
GBLog.error(error);
throw error;
}
await search.createIndex(searchSchema, dsName);
GBLogEx.info(instance.instanceId, `Released rebuildIndex mutex.`);
}
public async rebuildIndex(instance: IGBInstance, searchSchema: any) {}
/**
* Finds a storage package by using package name.
@ -1175,7 +1019,7 @@ export class GBDeployer implements IGBDeployer {
// Loops through all ready to load .gbapp packages.
let appPackagesProcessed = 0;
await CollectionUtil.asyncForEach(gbappPackages, async e => {
await GBUtil.asyncForEach(gbappPackages, async e => {
const filenameOnly = path.basename(e);
// Skips .gbapp inside deploy folder.

2
packages/core.gbapp/services/GBImporterService.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBMinInstance, IGBCoreService, IGBInstance } from 'botlib';
import { GBMinInstance, IGBCoreService, IGBInstance } from 'botlib-legacy';
import { CreateOptions } from 'sequelize/types';
import fs from 'fs/promises';
import urlJoin from 'url-join';

2
packages/core.gbapp/services/GBLogEx.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBLog, IGBInstance } from 'botlib';
import { GBLog, IGBInstance } from 'botlib-legacy';
import { GuaribasLog } from '../models/GBModel.js';
import { GBServer } from '../../../src/app.js';
import { GBConfigService } from './GBConfigService.js';

29
packages/core.gbapp/services/GBMinService.ts
View file

@ -68,7 +68,7 @@ import {
IGBCoreService,
IGBInstance,
IGBPackage
} from 'botlib';
} from 'botlib-legacy';
import cliProgress from 'cli-progress';
import removeRoute from 'express-remove-route';
import fs from 'fs/promises';
@ -76,7 +76,7 @@ import Koa from 'koa';
import mkdirp from 'mkdirp';
import { NlpManager } from 'node-nlp';
import path from 'path';
import { CollectionUtil } from 'pragmatismo-io-framework';
import SwaggerClient from 'swagger-client';
import urlJoin from 'url-join';
import wash from 'washyourmouthoutwithsoap';
@ -179,7 +179,7 @@ export class GBMinService {
let i = 1;
const minInstances = [];
await CollectionUtil.asyncForEach(
await GBUtil.asyncForEach(
instances,
(async instance => {
try {
@ -222,7 +222,7 @@ export class GBMinService {
const steps = process.env.TEST_MESSAGE.split(';');
await CollectionUtil.asyncForEach(steps, async step => {
await GBUtil.asyncForEach(steps, async step => {
client.apis.Conversations.Conversations_PostActivity({
conversationId: conversationId,
activity: {
@ -829,10 +829,7 @@ export class GBMinService {
};
if (GBConfigService.get('GB_MODE') !== 'legacy') {
const url =
process.env.BOT_URL && !process.env.BOT_URL.includes('ngrok')
? process.env.BOT_URL
: `http://localhost:${GBConfigService.get('PORT')}`;
const url = process.env.BOT_URL ? process.env.BOT_URL : `http://localhost:${GBConfigService.get('PORT')}`;
config['domain'] = urlJoin(url, 'directline', botId);
} else {
const webchatTokenContainer = await this.getWebchatToken(instance);
@ -991,7 +988,7 @@ export class GBMinService {
// Creates a hub of services available in .gbapps.
await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
await GBUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
let services: ConcatArray<never>;
if ((services = await e.onExchangeData(min, 'getServices', null))) {
min.gbappServices = { ...min.gbappServices, ...services };
@ -1066,7 +1063,7 @@ export class GBMinService {
private async invokeLoadBot(appPackages: IGBPackage[], sysPackages: IGBPackage[], min: GBMinInstance) {
// Calls loadBot event in all .gbapp packages.
await CollectionUtil.asyncForEach(sysPackages, async p => {
await GBUtil.asyncForEach(sysPackages, async p => {
p.sysPackages = sysPackages;
if (p.getDialogs !== undefined) {
const dialogs = await p.getDialogs(min);
@ -1082,7 +1079,7 @@ export class GBMinService {
// Adds all dialogs from .gbapps into global dialo list for this minimal instance.
await CollectionUtil.asyncForEach(appPackages, async p => {
await GBUtil.asyncForEach(appPackages, async p => {
p.sysPackages = sysPackages;
await p.loadBot(min);
if (p.getDialogs !== undefined) {
@ -1306,7 +1303,7 @@ export class GBMinService {
} else if (context.activity.type === 'conversationUpdate') {
// Calls onNewSession event on each .gbapp package.
await CollectionUtil.asyncForEach(appPackages, async e => {
await GBUtil.asyncForEach(appPackages, async e => {
await e.onNewSession(min, step);
});
@ -1572,7 +1569,7 @@ export class GBMinService {
context.activity.text = context.activity.text.replace(/\<at\>.*\<\/at\>\s/gi, '');
let data = { query: context.activity.text };
await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
await GBUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
await e.onExchangeData(min, 'handleRawInput', data);
});
context.activity.text = data.query;
@ -1784,7 +1781,7 @@ export class GBMinService {
message: message ? message['dataValues'] : null,
user: user ? user.dataValues : null
};
await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
await GBUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
if (!nextDialog) {
nextDialog = await e.onExchangeData(min, 'handleAnswer', data);
}
@ -1826,10 +1823,10 @@ export class GBMinService {
await close();
let proxies = {};
await CollectionUtil.asyncForEach(mins, async min => {
await GBUtil.asyncForEach(mins, async min => {
let dialogs = {};
await CollectionUtil.asyncForEach(Object.values(min.scriptMap), async script => {
await GBUtil.asyncForEach(Object.values(min.scriptMap), async script => {
const api = min.core.getParam(min.instance, 'Server API', null);
if (api) {
dialogs[script] = async data => {

44
packages/core.gbapp/services/GBSSR.ts
View file

@ -41,7 +41,7 @@ import path from 'path';
import fs from 'fs/promises';
import { NextFunction, Request, Response } from 'express';
import urljoin from 'url-join';
import { GBMinInstance } from 'botlib';
import { GBMinInstance } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import { GBLogEx } from './GBLogEx.js';
import urlJoin from 'url-join';
@ -125,11 +125,10 @@ export class GBSSR {
};
}
public static async createBrowser(profilePath): Promise<any> {
const opts = await this.preparePuppeteer(profilePath);
puppeteer.use(hidden());
puppeteer.use(require("puppeteer-extra-plugin-minmax")());
puppeteer.use(require('puppeteer-extra-plugin-minmax')());
const browser = await puppeteer.launch(opts);
return browser;
}
@ -172,9 +171,6 @@ export class GBSSR {
}
});
await page.setExtraHTTPHeaders({
'ngrok-skip-browser-warning': '1'
});
const response = await page.goto(url, {
timeout: 120000,
waitUntil: 'networkidle0'
@ -282,40 +278,35 @@ export class GBSSR {
let onlyChars: any = /\/([A-Za-z0-9\-\_]+)\/*/.exec(req.originalUrl);
onlyChars = onlyChars ? onlyChars[1] : minBoot.botId;
let botId =
req.originalUrl && req.originalUrl === '/' ?
minBoot.botId :
onlyChars;
let botId = req.originalUrl && req.originalUrl === '/' ? minBoot.botId : onlyChars;
let min: GBMinInstance =
req.url === '/'
? minBoot
: GBServer.globals.minInstances.filter(p => p.instance.botId.toLowerCase() === botId.toLowerCase())[0];
if (!min) {
min = req.url === '/'
? minBoot
: GBServer.globals.minInstances.filter(p =>
p.instance.activationCode ? p.instance.activationCode.toLowerCase() === botId.toLowerCase()
: null)[0];
min =
req.url === '/'
? minBoot
: GBServer.globals.minInstances.filter(p =>
p.instance.activationCode ? p.instance.activationCode.toLowerCase() === botId.toLowerCase() : null
)[0];
}
if (!min) {
botId = minBoot.botId;
}
let packagePath = GBUtil.getGBAIPath(botId, `gbui`);
// Checks if the bot has an .gbui published or use default.gbui.
if (!await GBUtil.exists(packagePath)) {
if (!(await GBUtil.exists(packagePath))) {
packagePath = path.join(process.env.PWD, 'packages', `default.gbui`, 'build');
}
let parts = req.url.replace(`/${botId}`, '').split('?');
let url = parts[0];
if (min && req.originalUrl && prerender && exclude && await GBUtil.exists(packagePath)) {
if (min && req.originalUrl && prerender && exclude && (await GBUtil.exists(packagePath))) {
// Reads from static HTML when a bot is crawling.
packagePath = path.join(process.env.PWD, 'work', packagePath, 'index.html');
@ -323,7 +314,6 @@ export class GBSSR {
res.status(200).send(html);
return true;
} else {
// Servers default.gbui web application.
packagePath = path.join(
@ -334,12 +324,12 @@ export class GBSSR {
url === '/' || url === '' ? `index.html` : url
);
if (GBServer.globals.wwwroot && url === '/') {
packagePath = GBServer.globals.wwwroot + "/index.html"; // TODO.
packagePath = GBServer.globals.wwwroot + '/index.html'; // TODO.
}
if (!min && !url.startsWith("/images") && GBServer.globals.wwwroot) {
if (!min && !url.startsWith('/images') && GBServer.globals.wwwroot) {
packagePath = path.join(GBServer.globals.wwwroot, url);
}
if (!min && !url.startsWith("/static") && GBServer.globals.wwwroot) {
if (!min && !url.startsWith('/static') && GBServer.globals.wwwroot) {
packagePath = path.join(GBServer.globals.wwwroot, url);
}
if (await GBUtil.exists(packagePath)) {
@ -348,13 +338,11 @@ export class GBSSR {
html = html.replace(/\{p\}/gi, min.botId);
html = html.replace(/\{botId\}/gi, min.botId);
const theme =
`theme-${await (min.core as any)['getParam'](min.instance, 'Theme Color', 'grey')}`;
const theme = `theme-${await (min.core as any)['getParam'](min.instance, 'Theme Color', 'grey')}`;
html = html.replace(/\{themeColor\}/gi, theme);
html = html.replace(/\{theme\}/gi, min.instance.theme ? min.instance.theme :
'default.gbtheme');
html = html.replace(/\{theme\}/gi, min.instance.theme ? min.instance.theme : 'default.gbtheme');
html = html.replace(/\{title\}/gi, min.instance.title);
res.send(html).end();
} else {

6
packages/customer-satisfaction.gbapp/dialogs/FeedbackDialog.ts
View file

@ -36,7 +36,7 @@
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { GBMinService } from '../../core.gbapp/services/GBMinService.js';
import { AnalyticsService } from '../../analytics.gblib/services/AnalyticsService.js';
import { SecService } from '../../security.gbapp/services/SecService.js';
@ -103,7 +103,6 @@ export class FeedbackDialog extends IGBDialog {
await min.userProfile.set(step.context, profile);
if (agentSystemId.indexOf('@') !== -1) {
// Agent is from Teams or Google Chat.
const agent = await sec.getUserFromSystemId(agentSystemId);
@ -168,8 +167,6 @@ export class FeedbackDialog extends IGBDialog {
await sec.updateHumanAgent(userSystemId, min.instance.instanceId, null);
await sec.updateHumanAgent(manualUser.userSystemId, min.instance.instanceId, null);
} else if (user.agentMode === 'human') {
const agent = await sec.getUserFromSystemId(user.agentSystemId);
@ -198,7 +195,6 @@ export class FeedbackDialog extends IGBDialog {
await sec.updateHumanAgent(user.userSystemId, min.instance.instanceId, null);
await sec.updateHumanAgent(agent.userSystemId, min.instance.instanceId, null);
} else {
if (user.userSystemId.charAt(2) === ':' || userSystemId.indexOf('@') > -1) {
// Agent is from Teams or Google Chat.

2
packages/customer-satisfaction.gbapp/dialogs/QualityDialog.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';

2
packages/customer-satisfaction.gbapp/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import urlJoin from 'url-join';
import { FeedbackDialog } from './dialogs/FeedbackDialog.js';
import { QualityDialog } from './dialogs/QualityDialog.js';

2
packages/google-chat.gblib/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
import { GoogleChatDirectLine } from './services/GoogleChatDirectLine.js';

34
packages/google-chat.gblib/services/GoogleChatDirectLine.ts
View file

@ -31,8 +31,8 @@
import Swagger from 'swagger-client';
import { google } from 'googleapis';
import { PubSub } from '@google-cloud/pubsub';
import fs from 'fs/promises';
import { GBLog, GBMinInstance, GBService } from 'botlib';
import fs from 'fs/promises';
import { GBLog, GBMinInstance, GBService } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import { SecService } from '../../security.gbapp/services/SecService.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
@ -57,7 +57,7 @@ export class GoogleChatDirectLine extends GBService {
GoogleClientPrivateKey: any;
GoogleProjectId: any;
constructor (
constructor(
min: GBMinInstance,
botId,
directLineSecret,
@ -84,14 +84,14 @@ export class GoogleChatDirectLine extends GBService {
});
}
public static async asyncForEach (array, callback) {
public static async asyncForEach(array, callback) {
for (let index = 0; index < array.length; index++) {
await callback(array[index], index, array);
}
}
public async setup (setUrl) {
this.directLineClient = await GBUtil.getDirectLineClient(this.min);
public async setup(setUrl) {
this.directLineClient = await GBUtil.getDirectLineClient(this.min);
const client = await this.directLineClient;
client.clientAuthorizations.add(
@ -109,15 +109,15 @@ export class GoogleChatDirectLine extends GBService {
}
}
public async resetConversationId (key) {
public async resetConversationId(key) {
GoogleChatDirectLine.conversationIds[key] = undefined;
}
public async check () {
public async check() {
GBLogEx.info(0, `GBGoogleChat: Checking server...`);
}
public async receiver (message) {
public async receiver(message) {
const event = JSON.parse(Buffer.from(message.data, 'binary').toString());
let from = '';
@ -158,7 +158,7 @@ export class GoogleChatDirectLine extends GBService {
}
}
public inputMessage (client, conversationId, threadName, text, from, fromName) {
public inputMessage(client, conversationId, threadName, text, from, fromName) {
return client.Conversations.Conversations_PostActivity({
conversationId: conversationId,
activity: {
@ -175,7 +175,7 @@ export class GoogleChatDirectLine extends GBService {
});
}
public pollMessages (client, conversationId, threadName, from, fromName) {
public pollMessages(client, conversationId, threadName, from, fromName) {
GBLogEx.info(0, `GBGoogleChat: Starting message polling(${from}, ${conversationId}).`);
let watermark: any;
@ -195,7 +195,7 @@ export class GoogleChatDirectLine extends GBService {
setInterval(worker, this.pollInterval);
}
public async printMessages (activities, conversationId, threadName, from, fromName) {
public async printMessages(activities, conversationId, threadName, from, fromName) {
if (activities && activities.length) {
// Ignore own messages.
@ -211,7 +211,7 @@ export class GoogleChatDirectLine extends GBService {
}
}
public async printMessage (activity, conversationId, threadName, from, fromName) {
public async printMessage(activity, conversationId, threadName, from, fromName) {
let output = '';
if (activity.text) {
@ -235,14 +235,14 @@ export class GoogleChatDirectLine extends GBService {
await this.sendToDevice(from, conversationId, threadName, output);
}
public async sendToDevice (from: string, conversationId: string, threadName, msg: string) {
public async sendToDevice(from: string, conversationId: string, threadName, msg: string) {
try {
let threadParts = threadName.split('/');
let spaces = threadParts[1];
let threadKey = threadParts[3];
const scopes = ['https://www.googleapis.com/auth/chat.bot'];
const jwtClient = new google.auth.JWT(this.GoogleClientEmail, null, this.GoogleClientPrivateKey, scopes, null);
const jwtClient = new google.auth.JWT(this.GoogleClientEmail);
await jwtClient.authorize();
const chat = google.chat({ version: 'v1', auth: jwtClient });
@ -253,14 +253,14 @@ export class GoogleChatDirectLine extends GBService {
text: msg
}
});
GBLogEx.info(0, `Message [${msg}] sent to ${from}: `);
} catch (error) {
GBLog.error(`Error sending message to GoogleChat provider ${error.message}`);
}
}
public async sendToDeviceEx (to, conversationId, threadName, text, locale) {
public async sendToDeviceEx(to, conversationId, threadName, text, locale) {
const minBoot = GBServer.globals.minBoot as any;
text = await minBoot.conversationalService.translate(minBoot, text, locale);

68
packages/hubspot.gblib/index.ts
View file

@ -1,68 +0,0 @@
/*****************************************************************************\
| █████ █████ ██ █ █████ █████ ████ ██ ████ █████ █████ ███ ® |
| ██ █ ███ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| ██ ███ ████ █ ██ █ ████ █████ ██████ ██ ████ █ █ █ ██ |
| ██ ██ █ █ ██ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| █████ █████ █ ███ █████ ██ ██ ██ ██ █████ ████ █████ █ ███ |
| |
| General Bots Copyright (c) pragmatismo.com.br. All rights reserved. |
| Licensed under the AGPL-3.0. |
| |
| According to our dual licensing model, this program can be used either |
| under the terms of the GNU Affero General Public License, version 3, |
| or under a proprietary license. |
| |
| The texts of the GNU Affero General Public License with an additional |
| permission and of our proprietary license can be found at and |
| in the LICENSE file you have received along with this program. |
| |
| This program is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY, without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| GNU Affero General Public License for more details. |
| |
| "General Bots" is a registered trademark of pragmatismo.com.br. |
| The licensing of the program under the AGPLv3 does not imply a |
| trademark license. Therefore any rights, title and interest in |
| our trademarks remain entirely with us. |
| |
\*****************************************************************************/
/**
* @fileoverview General Bots server core.
*/
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { Sequelize } from 'sequelize-typescript';
/**
* Package for GoogleChat.gblib
*/
export class GBHubSpotPackage implements IGBPackage {
public sysPackages: IGBPackage[];
public async loadBot(min: GBMinInstance): Promise<void> {
GBLog.verbose(`loadBot called.`);
}
public async getDialogs(min: GBMinInstance) {
GBLog.verbose(`getDialogs called.`);
}
public async loadPackage(core: IGBCoreService, sequelize: Sequelize): Promise<void> {
GBLog.verbose(`loadPackage called.`);
}
public async unloadPackage(core: IGBCoreService): Promise<void> {
GBLog.verbose(`unloadPackage called.`);
}
public async unloadBot(min: GBMinInstance): Promise<void> {
GBLog.verbose(`unloadBot called.`);
}
public async onNewSession(min: GBMinInstance, step: GBDialogStep): Promise<void> {
GBLog.verbose(`onNewSession called.`);
}
public async onExchangeData(min: GBMinInstance, kind: string, data: any) {
GBLog.verbose(`onExchangeData called.`);
}
}

39
packages/hubspot.gblib/services/HubSpotServices.ts
View file

@ -1,39 +0,0 @@
/*****************************************************************************\
| █████ █████ ██ █ █████ █████ ████ ██ ████ █████ █████ ███ ® |
| ██ █ ███ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| ██ ███ ████ █ ██ █ ████ █████ ██████ ██ ████ █ █ █ ██ |
| ██ ██ █ █ ██ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| █████ █████ █ ███ █████ ██ ██ ██ ██ █████ ████ █████ █ ███ |
| |
| General Bots Copyright (c) pragmatismo.com.br. All rights reserved. |
| Licensed under the AGPL-3.0. |
| |
| According to our dual licensing model, this program can be used either |
| under the terms of the GNU Affero General Public License, version 3, |
| or under a proprietary license. |
| |
| The texts of the GNU Affero General Public License with an additional |
| permission and of our proprietary license can be found at and |
| in the LICENSE file you have received along with this program. |
| |
| This program is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY, without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| GNU Affero General Public License for more details. |
| |
| "General Bots" is a registered trademark of pragmatismo.com.br. |
| The licensing of the program under the AGPLv3 does not imply a |
| trademark license. Therefore any rights, title and interest in |
| our trademarks remain entirely with us. |
| |
\*****************************************************************************/
import { GBLog, GBMinInstance, GBService } from 'botlib';
import { promisify } from 'util';
import Swagger from 'swagger-client';
import * as hubspot from '@hubspot/api-client';
/**
* Support for Hub Spot XRM.
*/
export class HubSpotServices extends GBService { }

4
packages/hubspot.gblib/strings.ts
View file

@ -1,4 +0,0 @@
export const Messages = {
'en-US': {},
'pt-BR': {}
};

11
packages/kb.gbapp/dialogs/AskDialog.ts
View file

@ -37,12 +37,12 @@
import { GBServer } from '../../../src/app.js';
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBLog, GBMinInstance, IGBDialog, IGBPackage } from 'botlib';
import { GBLog, GBMinInstance, IGBDialog, IGBPackage } from 'botlib-legacy';
import { Messages } from '../strings.js';
import { KBService } from './../services/KBService.js';
import { GuaribasAnswer } from '../models/index.js';
import { SecService } from '../../security.gbapp/services/SecService.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBVMService } from '../../basic.gblib/services/GBVMService.js';
import { GBImporter } from '../../core.gbapp/services/GBImporterService.js';
import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
@ -140,7 +140,7 @@ export class AskDialog extends IGBDialog {
message: text,
user: user ? user['dataValues'] : null
};
await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
await GBUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
if ((nextDialog = await e.onExchangeData(min, 'handleAnswer', data))) {
handled = true;
}
@ -174,10 +174,7 @@ export class AskDialog extends IGBDialog {
}
},
async step => {
min = GBServer.globals.minInstances.find(p=> p.botId === min.botId);
min = GBServer.globals.minInstances.find(p => p.botId === min.botId);
let answer;
const member = step.context.activity.from;

2
packages/kb.gbapp/dialogs/FaqDialog.ts
View file

@ -36,7 +36,7 @@
import { BotAdapter } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { GBConversationalService } from '../../core.gbapp/services/GBConversationalService.js';
import { Messages } from '../strings.js';
import { KBService } from './../services/KBService.js';

2
packages/kb.gbapp/dialogs/MenuDialog.ts
View file

@ -38,7 +38,7 @@ import urlJoin from 'url-join';
import { BotAdapter, CardFactory, MessageFactory } from 'botbuilder';
import { WaterfallDialog } from 'botbuilder-dialogs';
import { GBMinInstance, IGBDialog } from 'botlib';
import { GBMinInstance, IGBDialog } from 'botlib-legacy';
import { GBConversationalService } from '../../core.gbapp/services/GBConversationalService.js';
import { GuaribasSubject } from '../models/index.js';
import { KBService } from '../services/KBService.js';

2
packages/kb.gbapp/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
import { AskDialog } from './dialogs/AskDialog.js';
import { FaqDialog } from './dialogs/FaqDialog.js';

170
packages/kb.gbapp/services/KBService.ts
View file

@ -47,7 +47,7 @@ import { CSVLoader } from '@langchain/community/document_loaders/fs/csv';
import { DocxLoader } from '@langchain/community/document_loaders/fs/docx';
import { EPubLoader } from '@langchain/community/document_loaders/fs/epub';
import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf';
import svg2img from 'svg2img';
import isICO from 'icojs';
import getColors from 'get-image-colors';
import { Document } from 'langchain/document';
@ -62,18 +62,15 @@ import {
IGBCoreService,
IGBInstance,
IGBKBService
} from 'botlib';
} from 'botlib-legacy';
import mammoth from 'mammoth';
import { parse } from 'node-html-parser';
import pdf from 'pdf-extraction';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { Op } from 'sequelize';
import { Sequelize } from 'sequelize-typescript';
import textract from 'textract';
import { GBUtil } from '../../../src/util.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { AzureDeployerService } from '../../azuredeployer.gbapp/services/AzureDeployerService.js';
import webp from 'webp-converter';
import os from 'os';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
import { GBVMService } from '../../basic.gblib/services/GBVMService.js';
@ -88,6 +85,29 @@ import { GuaribasAnswer, GuaribasQuestion, GuaribasSubject } from '../models/ind
import { GBConfigService } from './../../core.gbapp/services/GBConfigService.js';
import { exec } from 'child_process';
let webp: any = null;
try {
webp = await import('webp-converter');
} catch {}
let svg2img: any = null;
try {
const svg2imgModule = await import('svg2img');
svg2img = svg2imgModule.default || svg2imgModule;
} catch {}
let getColors: any = null;
try {
const getColorsModule = await import('get-image-colors');
getColors = getColorsModule.default || getColorsModule;
} catch {}
let textract: any = null;
try {
const textractModule = await import('textract');
textract = textractModule.default || textractModule;
} catch {}
/**
* Result for quey on KB data.
*/
@ -624,7 +644,7 @@ export class KBService implements IGBKBService {
const answersCreated = await GuaribasAnswer.bulkCreate(answers);
let i = 0;
await CollectionUtil.asyncForEach(questions, async question => {
await GBUtil.asyncForEach(questions, async question => {
question.answerId = answersCreated[i++].answerId;
});
@ -656,22 +676,19 @@ export class KBService implements IGBKBService {
} else if (answer.endsWith('.ogg') && process.env.AUDIO_DISABLED !== 'true') {
await this.playAudio(min, answer, channel, step, min.conversationalService);
} else if (answer.startsWith('![')) {
// Checks for text after the image markdown, after the element 4, there are text blocks.
const removeMarkdownImages = (text: string) => {
return text.replace(/!\[[^\]]*\](?:\([^)]*\)|\[[^\]]*\])/g, '').trim();
}
};
if (removeMarkdownImages(answer)) {
await min.conversationalService.sendText(min, step, answer);
}
else {
} else {
const urlMatch = answer.match(/!?\[.*?\]\((.*?)\)/);
const url = urlMatch ? urlMatch[1] : null;
await this.showImage(min, min.conversationalService, step, url, channel)
await this.showImage(min, min.conversationalService, step, url, channel);
}
} else {
await min.conversationalService.sendText(min, step, answer);
}
@ -713,7 +730,6 @@ export class KBService implements IGBKBService {
packageStorage: GuaribasPackage,
instance: IGBInstance
): Promise<any> {
// Imports subjects tree into database and return it.
const subjectFile = urlJoin(localPath, 'subjects.json');
@ -750,7 +766,7 @@ export class KBService implements IGBKBService {
const files = await walkPromise(urlJoin(localPath, 'articles'));
const data = { questions: [], answers: [] };
await CollectionUtil.asyncForEach(files, async file => {
await GBUtil.asyncForEach(files, async file => {
if (file !== null && file.name.endsWith('.md')) {
let content = await this.getAnswerTextByMediaName(instance.instanceId, file.name);
@ -885,7 +901,7 @@ export class KBService implements IGBKBService {
const answersCreated = await GuaribasAnswer.bulkCreate(data.answers);
let i = 0;
await CollectionUtil.asyncForEach(data.questions, async question => {
await GBUtil.asyncForEach(data.questions, async question => {
question.answerId = answersCreated[i++].answerId;
});
return await GuaribasQuestion.bulkCreate(data.questions);
@ -899,11 +915,12 @@ export class KBService implements IGBKBService {
depth: number,
maxDepth: number,
page: Page,
websiteIgnoreUrls, maxDocuments: number
websiteIgnoreUrls,
maxDocuments: number
): Promise<string[]> {
try {
if (
(maxDocuments < visited.size) ||
maxDocuments < visited.size ||
(depth > maxDepth && !url.endsWith('pdf')) ||
visited.has(url) ||
url.endsWith('.jpg') ||
@ -970,8 +987,7 @@ export class KBService implements IGBKBService {
const childLinks = [];
for (const link of filteredLinks) {
const links = await this.crawl(min, link,
visited, depth + 1, maxDepth, page, websiteIgnoreUrls, maxDocuments);
const links = await this.crawl(min, link, visited, depth + 1, maxDepth, page, websiteIgnoreUrls, maxDocuments);
if (links) {
childLinks.push(...links);
}
@ -1037,7 +1053,6 @@ export class KBService implements IGBKBService {
executablePath: process.env.CHROME_PATH ? process.env.CHROME_PATH : executablePath(),
args
});
}
const page = await browser.newPage();
try {
@ -1068,7 +1083,10 @@ export class KBService implements IGBKBService {
const MAX_DOCUMENTS = 15;
const maxDocuments = min.core.getParam<number>(min.instance, 'Website Max Documents', MAX_DOCUMENTS);
const websiteIgnoreUrls = min.core.getParam<[]>(min.instance, 'Website Ignore URLs', null);
GBLogEx.info(min, `Website: ${website}, Max Depth: ${maxDepth}, Website Max Documents: ${maxDocuments}, Ignore URLs: ${websiteIgnoreUrls}`);
GBLogEx.info(
min,
`Website: ${website}, Max Depth: ${maxDepth}, Website Max Documents: ${maxDocuments}, Ignore URLs: ${websiteIgnoreUrls}`
);
let shouldSave = false;
@ -1104,8 +1122,7 @@ export class KBService implements IGBKBService {
const baseUrl = page.url().split('/').slice(0, 3).join('/');
logo = logo.startsWith('//') ? 'https:' + logo : (
logo.startsWith('https') ? logo : urlJoin(baseUrl, logo));
logo = logo.startsWith('//') ? 'https:' + logo : logo.startsWith('https') ? logo : urlJoin(baseUrl, logo);
const logoBinary = await page.goto(logo);
let buffer = await logoBinary.buffer();
@ -1126,7 +1143,7 @@ export class KBService implements IGBKBService {
await fs.writeFile(tempAvif, buffer);
await new Promise((resolve, reject) => {
exec(`node_modules/ffmpeg-static/ffmpeg -i "${tempAvif}" "${tempPng}"`, (error) => {
exec(`node_modules/ffmpeg-static/ffmpeg -i "${tempAvif}" "${tempPng}"`, error => {
if (error) reject(error);
else resolve(null);
});
@ -1134,40 +1151,42 @@ export class KBService implements IGBKBService {
buffer = await fs.readFile(tempPng);
// Clean up temp files
await fs.unlink(tempAvif).catch(() => { });
await fs.unlink(tempPng).catch(() => { });
} else if (buffer.slice(0, 4).toString('hex') === '52494646' &&
buffer.slice(8, 12).toString('hex') === '57454250') {
await fs.unlink(tempAvif).catch(() => {});
await fs.unlink(tempPng).catch(() => {});
} else if (
buffer.slice(0, 4).toString('hex') === '52494646' &&
buffer.slice(8, 12).toString('hex') === '57454250'
) {
// Convert WebP to PNG using temporary files
const tempWebP = path.join(os.tmpdir(), `temp-${Date.now()}.webp`);
const tempPNG = path.join(os.tmpdir(), `temp-${Date.now()}.png`);
await fs.writeFile(tempWebP, buffer);
await webp.dwebp(tempWebP, tempPNG, "-o");
await webp.dwebp(tempWebP, tempPNG, '-o');
buffer = await fs.readFile(tempPNG);
} else if (buffer.toString().includes('<svg')) {
// For SVG files, convert using svg2img
buffer = await new Promise((resolve, reject) => {
svg2img(buffer, {
resvg: {
fitTo: {
mode: 'width', // or height
value: 48,
},
svg2img(
buffer,
{
resvg: {
fitTo: {
mode: 'width', // or height
value: 48
}
}
},
(error: any, buffer: Buffer) => {
if (error) {
reject(error);
} else {
resolve(buffer);
}
}
}, (error: any, buffer: Buffer) => {
if (error) {
reject(error);
} else {
resolve(buffer);
}
});
);
});
}
@ -1179,11 +1198,10 @@ export class KBService implements IGBKBService {
const logoPath = path.join(packagePath, 'cache', logoFilename);
await (image as any).write(logoPath);
await min.core['setConfig'](min, 'Logo', logoFilename);
}
// Extract dominant colors from the screenshot
page = await this.getFreshPage(browser, website);
page = await this.getFreshPage(browser, website);
await page.screenshot({ path: 'screenshot.png' });
const colors = await getColors('screenshot.png');
@ -1208,25 +1226,22 @@ export class KBService implements IGBKBService {
page.setCacheEnabled(false);
const visited = new Set<string>();
page = await this.getFreshPage(browser, website);
page = await this.getFreshPage(browser, website);
files = files.concat(await this.crawl(min, website, visited, 0, maxDepth, page, websiteIgnoreUrls, maxDocuments));
await browser.close();
GBLogEx.info(min, `Vectorizing ${files.length} file(s)...`);
if (await GBUtil.exists(min['vectorStorePath'])) {
GBLogEx.info(min, `Cleaning vector store: ${min['vectorStorePath']}...`)
GBLogEx.info(min, `Cleaning vector store: ${min['vectorStorePath']}...`);
const gbkbPath = GBUtil.getGBAIPath(min.botId, 'gbkb');
min['vectorStorePath'] = path.join('work', gbkbPath, 'docs-vectorized');
min['vectorStore'] = await min.deployService['loadOrCreateEmptyVectorStore'](min);
}
await CollectionUtil.asyncForEach(files, async file => {
await GBUtil.asyncForEach(files, async file => {
let content = null;
shouldSave = true;
@ -1238,16 +1253,13 @@ export class KBService implements IGBKBService {
GBLogEx.info(min, `Ignore processing of ${file}. ${GBUtil.toYAML(error)}`);
}
});
}
GBLogEx.info(min, `Added ${files.length} from site...`);
files = await walkPromise(urlJoin(localPath, 'docs'));
GBLogEx.info(min, `Add ${files.length} files being processed...`);
GBLogEx.info(min, `Add ${files.length} files being processed...`);
// const gbdrive = path.join(process.env.PWD, 'work', GBUtil.getGBAIPath(min.botId, 'gbdrive'));
// files = files.concat(await walkPromise(gbdrive));
@ -1256,26 +1268,26 @@ export class KBService implements IGBKBService {
files = files.concat(await walkPromise(gbdata));
if (files[0]) {
files = files.filter(p => { return p });
files = files.filter(p => {
return p;
});
shouldSave = true;
GBLogEx.info(min, `Add embeddings from packages, ${files.length} files being processed...`);
await CollectionUtil.asyncForEach(files, async file => {
await GBUtil.asyncForEach(files, async file => {
let filePath = typeof file === 'string' ? file : path.join(file.root, file.name);
if (filePath) {
if (filePath) {
let content = null;
let filePath = path.join(file.root, file.name);
try {
if (file.name.endsWith('.csv') || file.name.endsWith('.md')
|| file.name.endsWith('.pdf') || file.name.endsWith('.docx') ||
file.name.endsWith('.epub') || file.name.endsWith('.txt')
if (
file.name.endsWith('.csv') ||
file.name.endsWith('.md') ||
file.name.endsWith('.pdf') ||
file.name.endsWith('.docx') ||
file.name.endsWith('.epub') ||
file.name.endsWith('.txt')
) {
if (file.name.endsWith('.csv')) {
// Read first 1000 lines of CSV file
const csvContent = await fs.readFile(filePath, 'utf8');
@ -1301,8 +1313,6 @@ export class KBService implements IGBKBService {
}
}
defaultRecursiveCharacterTextSplitter = new RecursiveCharacterTextSplitter({
chunkSize: 700,
chunkOverlap: 50
@ -1363,7 +1373,7 @@ export class KBService implements IGBKBService {
public async importKbTabularDirectory(localPath: string, min: GBMinInstance, packageId: number): Promise<any> {
const files = await walkPromise(localPath);
await CollectionUtil.asyncForEach(files, async file => {
await GBUtil.asyncForEach(files, async file => {
if (file !== null && (file.name.endsWith('.xlsx') || file.name.endsWith('.csv'))) {
return await this.importKbTabularFile(urlJoin(file.root, file.name), min, packageId);
}
@ -1494,7 +1504,7 @@ export class KBService implements IGBKBService {
GBConfigService.get('DEFAULT_CONTENT_LANGUAGE')
);
await CollectionUtil.asyncForEach(questions, async question => {
await GBUtil.asyncForEach(questions, async question => {
const text = question.content;
const categoryReg = /.*\((.*)\).*/gi.exec(text);
@ -1529,11 +1539,6 @@ export class KBService implements IGBKBService {
await this.importKbPackage(min, localPath, p, instance);
GBDeployer.mountGBKBAssets(packageName, min.botId, localPath);
if (GBConfigService.get('GB_MODE') === 'legacy') {
const service = await AzureDeployerService.createInstance(deployer);
const searchIndex = instance.searchIndex ? instance.searchIndex : GBServer.globals.minBoot.instance.searchIndex;
await deployer.rebuildIndex(instance, service.getKBSearchSchema(searchIndex));
}
min['groupCache'] = await KBService.getGroupReplies(instance.instanceId);
await KBService.RefreshNER(min);
@ -1669,7 +1674,6 @@ export class KBService implements IGBKBService {
return filePath; // Return the saved file path
} else {
const parsedUrl = new URL(url);
// Get the last part of the URL path or default to 'index' if empty
@ -1688,7 +1692,7 @@ export class KBService implements IGBKBService {
});
} else {
request.abort().catch(() => {
// Ignore errors from requests that were already handled
// Ignore errors from requests that were already handled
});
}
});

2
packages/llm.gblib/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { GuaribasSchedule } from '../core.gbapp/models/GBModel.js';
import { Sequelize } from 'sequelize-typescript';

18
packages/llm.gblib/services/ChatServices.ts
View file

@ -47,15 +47,15 @@ import {
import { RunnableSequence } from '@langchain/core/runnables';
import { DynamicStructuredTool } from '@langchain/core/tools';
import { convertToOpenAITool } from '@langchain/core/utils/function_calling';
import { ChatOpenAI } from '@langchain/openai';
import { AzureOpenAI, ChatOpenAI } from '@langchain/openai';
import { SqlDatabase } from 'langchain/sql_db';
import { DataSource } from 'typeorm';
import { GBMinInstance } from 'botlib';
import { GBMinInstance } from 'botlib-legacy';
import fs from 'fs/promises';
import { BufferWindowMemory } from 'langchain/memory';
import path from 'path';
import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
import { GBVMService } from '../../basic.gblib/services/GBVMService.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
@ -137,7 +137,7 @@ export class GBLLMOutputParser extends BaseLLMOutputParser<ExpectedOutput> {
if (!sources) {
GBLogEx.verbose(this.min, `LLM JSON output sources is NULL.`);
} else {
await CollectionUtil.asyncForEach(sources, async source => {
await GBUtil.asyncForEach(sources, async source => {
let found = false;
if (securityEnabled) {
@ -281,12 +281,6 @@ export class ChatServices {
model = new ChatOpenAI({
model: process.env.LOCAL_LLM_MODEL,
apiKey: 'empty',
azureOpenAIApiDeploymentName: 'v1',
azureOpenAIApiInstanceName: 'v1',
azureOpenAIApiKey: 'empty',
azureOpenAIApiVersion: 'empty',
azureOpenAIBasePath: process.env.LOCAL_LLM_ENDPOINT,
openAIApiKey: 'empty',
configuration: {
baseURL: process.env.LOCAL_LLM_ENDPOINT
}
@ -298,7 +292,7 @@ export class ChatServices {
const azureOpenAIApiInstanceName = process.env.AZURE_OPEN_AI_INSTANCE;
const azureOpenAIEndPoint = process.env.AZURE_OPEN_AI_ENDPOINT;
model = new ChatOpenAI({
model = new AzureOpenAI({
azureOpenAIApiKey: azureOpenAIKey,
azureOpenAIApiInstanceName: azureOpenAIApiInstanceName,
azureOpenAIApiDeploymentName: azureOpenAILLMModel,
@ -756,7 +750,7 @@ export class ChatServices {
let functions = [];
// Adds .gbdialog as functions if any to LLM Functions.
await CollectionUtil.asyncForEach(Object.keys(min.scriptMap), async script => {
await GBUtil.asyncForEach(Object.keys(min.scriptMap), async script => {
const packagePath = GBUtil.getGBAIPath(min.botId, 'gbdialog', null);
const jsonFile = path.join('work', packagePath, `${script}.json`);

23
packages/saas.gbapp/dialog/NewUserDialog.ts
View file

@ -1,10 +1,11 @@
// BotServer/packages/saas.gbapp/dialog/NewUserDialog.ts
import { IGBDialog, GBMinInstance } from 'botlib';
import { IGBDialog, GBMinInstance } from 'botlib-legacy';
import { Messages } from '../strings.js';
import { MainService } from '../service/MainService.js';
import { SaaSPackage } from '../index.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBOService } from '../service/GBOService.js';
import { GBUtil } from '../../../src/util.js';
export class NewUserDialog extends IGBDialog {
static getPlanSelectionDialog(min: GBMinInstance) {
@ -80,7 +81,7 @@ export class NewUserDialog extends IGBDialog {
const list = await gboService.listTemplates(min);
let templateMessage = undefined;
await CollectionUtil.asyncForEach(list, async item => {
await GBUtil.asyncForEach(list, async item => {
if (item.name !== 'Shared.gbai') {
templateMessage = templateMessage ? `${templateMessage}\n- ${item.name}` : `- ${item.name}`;
}
@ -94,7 +95,7 @@ export class NewUserDialog extends IGBDialog {
const list = step.activeDialog.state.options.templateList;
let template = null;
let gboService = new GBOService();
await CollectionUtil.asyncForEach(list, async item => {
await GBUtil.asyncForEach(list, async item => {
if (gboService.kmpSearch(step.context.activity.originalText, item.name) != -1) {
template = item.name;
}
@ -105,7 +106,7 @@ export class NewUserDialog extends IGBDialog {
return await step.replaceDialog('/welcome_saas_bottemplate', step.activeDialog.state.options);
} else {
step.activeDialog.state.options.templateName = template;
const service = new MainService();
const result: any = await service.startSubscriptionProcess(
min,
@ -123,14 +124,10 @@ export class NewUserDialog extends IGBDialog {
} else {
await step.context.sendActivity(`Please complete your payment here: ${result.paymentUrl}`);
await step.context.sendActivity('I will check for payment completion every few seconds...');
try {
const finalResult = await service.waitForPaymentCompletion(
min,
result.subscriptionId,
template
);
const finalResult = await service.waitForPaymentCompletion(min, result.subscriptionId, template);
await step.context.sendActivity(`Payment verified and bot created successfully!`);
await step.context.sendActivity(`Access your bot here: ${finalResult.botUrl}`);
return await step.replaceDialog('/ask', { isReturning: true });
@ -182,4 +179,4 @@ export class NewUserDialog extends IGBDialog {
]
};
}
}
}

58
packages/saas.gbapp/index.ts
View file

@ -28,29 +28,28 @@
| |
\*****************************************************************************/
"use strict"
'use strict';
import { IGBPackage, GBMinInstance, IGBCoreService, GBLog, IGBAdminService, GBDialogStep } from 'botlib'
import { Sequelize } from 'sequelize-typescript'
import { GBOnlineSubscription } from './model/MainModel.js'
import { IGBPackage, GBMinInstance, IGBCoreService, GBLog, IGBAdminService, GBDialogStep } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
import { GBOnlineSubscription } from './model/MainModel.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { NewUserDialog } from './dialog/NewUserDialog.js'
import { GBOService } from './service/GBOService.js'
import { NewUserDialog } from './dialog/NewUserDialog.js';
import { GBOService } from './service/GBOService.js';
export class SaaSPackage implements IGBPackage {
sysPackages: IGBPackage[]
sysPackages: IGBPackage[];
adminService: IGBAdminService;
public static welcomes = {};
instanceId: any
instanceId: any;
public getDialogs(min: GBMinInstance) {
return [NewUserDialog.getDialog(min),
NewUserDialog.getBotNameDialog(min),
NewUserDialog.getBotTemplateDialog(min),
NewUserDialog.getPlanSelectionDialog(min),
return [
NewUserDialog.getDialog(min),
NewUserDialog.getBotNameDialog(min),
NewUserDialog.getBotTemplateDialog(min),
NewUserDialog.getPlanSelectionDialog(min)
];
}
@ -59,7 +58,6 @@ export class SaaSPackage implements IGBPackage {
if (process.env.DISABLE_SAAS_WELCOME !== 'true') {
core.setEntryPointDialog('/welcome_saas');
}
}
/**
@ -74,24 +72,17 @@ export class SaaSPackage implements IGBPackage {
this.adminService = min.adminService;
this.instanceId = min.instanceId;
}
/**
* Called by scheduler to send notification message to phones.
* @param sendToDevice The function used to notify.
*/
private async notifyJob(sendToDevice) {
* Called by scheduler to send notification message to phones.
* @param sendToDevice The function used to notify.
*/
private async notifyJob(sendToDevice) {}
}
async unloadPackage(core: IGBCoreService): Promise<void> {
}
async unloadPackage(core: IGBCoreService): Promise<void> {}
async loadBot(min: GBMinInstance): Promise<void> {
let gboService = new GBOService();
// Gets the sendToDevice method of whatsapp.gblib and setups scheduler.
@ -103,19 +94,13 @@ export class SaaSPackage implements IGBPackage {
}
}
async unloadBot(min: GBMinInstance): Promise<void> {
async unloadBot(min: GBMinInstance): Promise<void> {}
}
async onNewSession(min: GBMinInstance, step: GBDialogStep): Promise<void> {
}
async onNewSession(min: GBMinInstance, step: GBDialogStep): Promise<void> {}
public async onExchangeData(min: GBMinInstance, kind: string, data: any) {
switch (kind) {
case "whatsappMessage":
case 'whatsappMessage':
const from = data.from;
const fromName = data.fromName;
SaaSPackage.welcomes[from] = fromName;
@ -126,5 +111,4 @@ export class SaaSPackage implements IGBPackage {
break;
}
}
}

258
packages/saas.gbapp/service/GBOService.ts
View file

@ -1,27 +1,26 @@
// General Bots Copyright (c) pragmatismo.com.br. All rights reserved. Licensed under the AGPL-3.0.
"use strict"
'use strict';
import { GBMinInstance, GBLog } from "botlib";
import { CollectionUtil } from 'pragmatismo-io-framework';
import MicrosoftGraph from "@microsoft/microsoft-graph-client";
import { GBMinInstance, GBLog } from 'botlib-legacy';
import MicrosoftGraph from '@microsoft/microsoft-graph-client';
import { promises as fs } from 'fs';
import { existsSync } from 'fs';
import { GBConfigService } from "../../core.gbapp/services/GBConfigService.js";
import path from "path";
import { Client } from "minio";
import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js';
import path from 'path';
import { Client } from 'minio';
import { GBUtil } from '../../../src/util.js';
export class GBOService {
public async listTemplates(min: GBMinInstance) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
let templateLibraryId = process.env.SAAS_TEMPLATE_LIBRARY;
let siteId = process.env.STORAGE_SITE_ID;
let token =
await (min.adminService as any).acquireElevatedToken(min.instance.instanceId, true);
let token = await (min.adminService as any).acquireElevatedToken(min.instance.instanceId, true);
let client = MicrosoftGraph.Client.init({
authProvider: done => {
@ -29,14 +28,12 @@ export class GBOService {
}
});
const packagePath = `/`;
let res = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/root/children`)
let res = await client
.api(`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/root/children`)
.get();
return res.value;
}
else {
} else {
const templatesDir = path.join(process.env.PWD, 'templates');
const gbaiDirectories = [];
@ -52,7 +49,13 @@ export class GBOService {
return gbaiDirectories;
}
}
public async copyTemplates(min: GBMinInstance, gbaiDest: any, templateName: string, kind: string, botName: string): Promise<void> {
public async copyTemplates(
min: GBMinInstance,
gbaiDest: any,
templateName: string,
kind: string,
botName: string
): Promise<void> {
const storageMode = process.env.GB_MODE;
if (storageMode === 'legacy') {
@ -63,7 +66,7 @@ export class GBOService {
const libraryId = process.env.STORAGE_LIBRARY;
const client = MicrosoftGraph.Client.init({
authProvider: (done) => done(null, token)
authProvider: done => done(null, token)
});
const packageName = `${templateName.split('.')[0]}.${kind}`;
@ -71,69 +74,70 @@ export class GBOService {
try {
// Try direct copy first
const src = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/root:/${templateName}/${packageName}`
).get();
const src = await client
.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/root:/${templateName}/${packageName}`
)
.get();
await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/items/${src.id}/copy`
).post({
parentReference: {
driveId: gbaiDest.parentReference.driveId,
id: gbaiDest.id
},
name: destinationName
});
await client
.api(`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/items/${src.id}/copy`)
.post({
parentReference: {
driveId: gbaiDest.parentReference.driveId,
id: gbaiDest.id
},
name: destinationName
});
} catch (error) {
if (error.code === "nameAlreadyExists") {
if (error.code === 'nameAlreadyExists') {
// Handle existing destination by copying contents individually
const srcItems = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/root:/${templateName}/${packageName}:/children`
).get();
const srcItems = await client
.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/root:/${templateName}/${packageName}:/children`
)
.get();
const dstPath = `${botName}.gbai/${destinationName}`;
const dst = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/root:/${dstPath}`
).get();
const dst = await client
.api(`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/root:/${dstPath}`)
.get();
await CollectionUtil.asyncForEach(srcItems.value, async (item) => {
await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/items/${item.id}/copy`
).post({
parentReference: {
driveId: dst.parentReference.driveId,
id: dst.id
}
});
await GBUtil.asyncForEach(srcItems.value, async item => {
await client
.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/items/${item.id}/copy`
)
.post({
parentReference: {
driveId: dst.parentReference.driveId,
id: dst.id
}
});
});
} else {
GBLog.error(`Failed to copy templates: ${error.message}`);
throw error;
}
}
}
else if (storageMode === 'gbcluster') {
} else if (storageMode === 'gbcluster') {
// MinIO Implementation
const minioClient = new Client({
endPoint: process.env.DRIVE_SERVER,
port: parseInt(process.env.DRIVE_PORT),
useSSL: process.env.DRIVE_USE_SSL === 'true',
accessKey: process.env.DRIVE_ACCESSKEY,
secretKey: process.env.DRIVE_SECRET,
secretKey: process.env.DRIVE_SECRET
});
const bucketName = `${process.env.DRIVE_ORG_PREFIX}${botName}.gbai`.toLowerCase();
const packageName = `${templateName.split('.')[0]}.${kind}`;
const localTemplatePath = path.join(process.env.PWD, 'templates', templateName, packageName);
const minioDestinationPath = `${botName}.${kind}`;
const uploadDirectory = async (localPath: string, minioPath: string = '') => {
// Ensure the bucket exists in local file system
if (existsSync(localPath)) {
const entries = await fs.readdir(localPath, { withFileTypes: true });
for (const entry of entries) {
@ -148,8 +152,7 @@ export class GBOService {
GBLog.info(`Uploaded ${objectName} to MinIO bucket ${bucketName}`);
}
}
}
else {
} else {
GBLog.verbose(`Package ${localPath} does not exist on templates.`);
}
};
@ -159,9 +162,7 @@ export class GBOService {
}
public async createExcelFile(min: GBMinInstance, destinationFolder: any, name: string) {
let token =
await (min.adminService.acquireElevatedToken as any)(min.instance.instanceId, true);
let token = await (min.adminService.acquireElevatedToken as any)(min.instance.instanceId, true);
let siteId = process.env.STORAGE_SITE_ID;
let templateLibraryId = process.env.SAAS_TEMPLATE_LIBRARY;
@ -172,107 +173,97 @@ export class GBOService {
}
});
const body =
{
"parentReference": { driveId: destinationFolder.parentReference.driveId, id: destinationFolder.id },
"name": name
}
const body = {
parentReference: { driveId: destinationFolder.parentReference.driveId, id: destinationFolder.id },
name: name
};
try {
const src = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/root:/System.gbdata/blank.xlsx`)
const src = await client
.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/root:/System.gbdata/blank.xlsx`
)
.get();
return await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/items/${src.id}/copy`)
return await client
.api(`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${templateLibraryId}/drive/items/${src.id}/copy`)
.post(body);
} catch (error) {
GBLog.error(error);
throw error;
}
}
public async shareWithEmail(bucketName, folder, expiresInHours = 24 * 365) {
public async shareWithEmail(bucketName, folder, expiresInHours = 24 * 365) {
const minioClient = new Client({
endPoint: process.env.DRIVE_SERVER || 'localhost',
port: parseInt(process.env.DRIVE_PORT || '9000', 10),
useSSL: process.env.DRIVE_USE_SSL === 'true',
accessKey: process.env.DRIVE_ACCESSKEY,
secretKey: process.env.DRIVE_SECRET,
secretKey: process.env.DRIVE_SECRET
});
// Generate a time-limited access link (default: 24 hours)
const presignedUrl = await minioClient.presignedGetObject(
bucketName,
folder,
expiresInHours * 60 * 60 // Convert hours to seconds
);
return presignedUrl;
// Generate a time-limited access link (default: 24 hours)
const presignedUrl = await minioClient.presignedGetObject(
bucketName,
folder,
expiresInHours * 60 * 60 // Convert hours to seconds
);
return presignedUrl;
}
public async shareFolder(token: string, driveId: string, itemId: string, email: string) {
return new Promise<string>((resolve, reject) => {
let client = MicrosoftGraph.Client.init({
authProvider: done => {
done(null, token);
}
});
const body =
{
"recipients": [
{
"email": email
}
],
"message": "General Bots Online - Packages folder",
"requireSignIn": true,
"sendInvitation": true,
"roles": ["write"]
};
client.api(`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/invite`)
.post(body, (err, res) => {
if (err) {
GBLog.error('Sharing: ' + err);
reject(err)
}
else {
resolve(res);
}
});
return new Promise<string>((resolve, reject) => {
let client = MicrosoftGraph.Client.init({
authProvider: done => {
done(null, token);
}
});
const body = {
recipients: [
{
email: email
}
],
message: 'General Bots Online - Packages folder',
requireSignIn: true,
sendInvitation: true,
roles: ['write']
};
client.api(`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/invite`).post(body, (err, res) => {
if (err) {
GBLog.error('Sharing: ' + err);
reject(err);
} else {
resolve(res);
}
});
});
}
public kmpSearch(pattern, text) {
pattern = pattern.toLowerCase();
text = text.toLowerCase();
if (pattern.length == 0)
return 0; // Immediate match
if (pattern.length == 0) return 0; // Immediate match
// Compute longest suffix-prefix table
var lsp = [0]; // Base case
for (var i = 1; i < pattern.length; i++) {
var j = lsp[i - 1]; // Start by assuming we're extending the previous LSP
while (j > 0 && pattern.charAt(i) != pattern.charAt(j))
j = lsp[j - 1];
if (pattern.charAt(i) == pattern.charAt(j))
j++;
while (j > 0 && pattern.charAt(i) != pattern.charAt(j)) j = lsp[j - 1];
if (pattern.charAt(i) == pattern.charAt(j)) j++;
lsp.push(j);
}
// Walk through text string
var j = 0; // Number of chars matched in pattern
for (var i = 0; i < text.length; i++) {
while (j > 0 && text.charAt(i) != pattern.charAt(j))
j = lsp[j - 1]; // Fall back in the pattern
while (j > 0 && text.charAt(i) != pattern.charAt(j)) j = lsp[j - 1]; // Fall back in the pattern
if (text.charAt(i) == pattern.charAt(j)) {
j++; // Next char matched, increment position
if (j == pattern.length)
return i - (j - 1);
if (j == pattern.length) return i - (j - 1);
}
}
return -1; // Not found
@ -303,9 +294,7 @@ export class GBOService {
* Retrives a document from the drive, given a path and filename.
*/
private async internalGetDocument(client: any, baseUrl: any, path: string, file: string) {
let res = await client
.api(`${baseUrl}/drive/root:${path}:/children`)
.get();
let res = await client.api(`${baseUrl}/drive/root:${path}:/children`).get();
let documents = res.value.filter(m => {
return m.name.toLowerCase() === file.toLowerCase();
@ -318,8 +307,7 @@ export class GBOService {
return documents[0];
}
public async createRootFolder(token: string, name: string,
siteId: string, libraryId: string) {
public async createRootFolder(token: string, name: string, siteId: string, libraryId: string) {
const storageMode = process.env.GB_MODE;
if (storageMode === 'gbcluster') {
@ -329,10 +317,9 @@ export class GBOService {
port: parseInt(process.env.DRIVE_PORT),
useSSL: process.env.DRIVE_USE_SSL === 'true',
accessKey: process.env.DRIVE_ACCESSKEY,
secretKey: process.env.DRIVE_SECRET,
secretKey: process.env.DRIVE_SECRET
});
// Ensure bucket exists
name = `${process.env.DRIVE_ORG_PREFIX}${name}`.toLowerCase();
@ -341,7 +328,6 @@ export class GBOService {
await minioClient.makeBucket(name);
}
return { name: name, folder: {} }; // Return similar structure to MS Graph
} else {
// Original MS Graph implementation
return new Promise<any>((resolve, reject) => {
@ -351,22 +337,20 @@ export class GBOService {
}
});
const body = {
"name": name,
"folder": {},
"@microsoft.graph.conflictBehavior": "rename"
}
client.api(`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/root/children`)
name: name,
folder: {},
'@microsoft.graph.conflictBehavior': 'rename'
};
client
.api(`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/root/children`)
.post(body, (err, res) => {
if (err) {
reject(err)
}
else {
reject(err);
} else {
resolve(res);
}
});
});
}
}
}

4
packages/saas.gbapp/service/MainService.ts
View file

@ -1,7 +1,7 @@
// BotServer/packages/saas.gbapp/service/MainService.ts
import { GBOnlineSubscription } from '../model/MainModel.js';
import { GBMinInstance, GBLog } from 'botlib';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBMinInstance, GBLog } from 'botlib-legacy';
import urlJoin from 'url-join';
import { GBOService } from './GBOService.js';
import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js';

2
packages/security.gbapp/dialogs/OAuthDialog.ts
View file

@ -35,7 +35,7 @@
'use strict';
import { TokenResponse } from 'botbuilder';
import { GBLog, GBMinInstance, IGBDialog } from 'botlib';
import { GBLog, GBMinInstance, IGBDialog } from 'botlib-legacy';
import { Messages } from '../strings.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';

2
packages/security.gbapp/dialogs/ProfileDialog.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBLog, GBMinInstance, IGBDialog } from 'botlib';
import { GBLog, GBMinInstance, IGBDialog } from 'botlib-legacy';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { Messages } from '../strings.js';
import libphonenumber from 'google-libphonenumber';

2
packages/security.gbapp/dialogs/SMSAuthDialog.ts
View file

@ -35,7 +35,7 @@
'use strict';
import { TokenResponse } from 'botbuilder';
import { GBLog, GBMinInstance, IGBDialog } from 'botlib';
import { GBLog, GBMinInstance, IGBDialog } from 'botlib-legacy';
import { Messages } from '../strings.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { SecService } from '../services/SecService.js';

2
packages/security.gbapp/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
import { OAuthDialog } from './dialogs/OAuthDialog.js';
import { ProfileDialog } from './dialogs/ProfileDialog.js';

46
packages/security.gbapp/services/SecService.ts
View file

@ -1,17 +1,16 @@
import { ConversationReference } from 'botbuilder';
import { GBLog, GBMinInstance, GBService, IGBInstance } from 'botlib';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBLog, GBMinInstance, GBService, IGBInstance } from 'botlib-legacy';
import { GuaribasUser } from '../models/index.js';
import { FindOptions } from 'sequelize';
import { DialogKeywords } from '../../../packages/basic.gblib/services/DialogKeywords.js';
import fs from 'fs/promises';
import fs from 'fs/promises';
import mkdirp from 'mkdirp';
import urlJoin from 'url-join';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { GBServer } from '../../../src/app.js';
import { GBUtil } from '../../../src/util.js';
/**
* Security service layer.
*/
@ -25,11 +24,10 @@ export class SecService extends GBService {
displayName: string,
email: string
): Promise<GuaribasUser> {
const gbaiPath = GBUtil.getGBAIPath(min.botId);
const dir = urlJoin ('work',gbaiPath, 'users', userSystemId);
const dir = urlJoin('work', gbaiPath, 'users', userSystemId);
if (!await GBUtil.exists(dir)) {
if (!(await GBUtil.exists(dir))) {
mkdirp.sync(dir);
}
@ -45,7 +43,7 @@ export class SecService extends GBService {
const systemPromptFile = urlJoin(dir, 'systemPrompt.txt');
if (await GBUtil.exists(systemPromptFile)) {
user[ 'systemPrompt'] = await fs.readFile(systemPromptFile);
user['systemPrompt'] = await fs.readFile(systemPromptFile);
}
user.instanceId = min.instance.instanceId;
@ -54,8 +52,8 @@ export class SecService extends GBService {
user.displayName = displayName;
user.email = email;
user.defaultChannel = channelName;
GBServer.globals.users [user.userId] = user;
if(user.changed()){
GBServer.globals.users[user.userId] = user;
if (user.changed()) {
await user.save();
}
return user;
@ -79,7 +77,7 @@ export class SecService extends GBService {
const user = await GuaribasUser.findOne(options);
user.conversationReference = conversationReference;
GBServer.globals.users [user.userId] = user;
GBServer.globals.users[user.userId] = user;
return await user.save();
}
@ -88,7 +86,7 @@ export class SecService extends GBService {
const user = await GuaribasUser.findOne(options);
user.conversationReference = conversationReference;
GBServer.globals.users [user.userId] = user;
GBServer.globals.users[user.userId] = user;
return await user.save();
}
@ -99,7 +97,7 @@ export class SecService extends GBService {
}
});
user.locale = locale;
GBServer.globals.users [user.userId] = user;
GBServer.globals.users[user.userId] = user;
return await user.save();
}
@ -110,7 +108,7 @@ export class SecService extends GBService {
}
});
user.hearOnDialog = dialogName;
GBServer.globals.users [user.userId] = user;
GBServer.globals.users[user.userId] = user;
return await user.save();
}
@ -121,7 +119,7 @@ export class SecService extends GBService {
}
});
user.instanceId = instanceId;
GBServer.globals.users [user.userId] = user;
GBServer.globals.users[user.userId] = user;
return await user.save();
}
@ -167,11 +165,11 @@ export class SecService extends GBService {
agent.instanceId = user.instanceId;
agent.agentMode = 'self';
agent.agentSystemId = null;
GBServer.globals.users [agent.userId] = user;
GBServer.globals.users[agent.userId] = user;
await agent.save();
}
GBServer.globals.users [user.userId] = user;
GBServer.globals.users[user.userId] = user;
await user.save();
return user;
@ -203,13 +201,8 @@ export class SecService extends GBService {
list = list.split(';');
}
await CollectionUtil.asyncForEach(list, async item => {
if (
item !== undefined &&
!agentSystemId &&
item !== userSystemId &&
!(await this.isAgentSystemId(item))
) {
await GBUtil.asyncForEach(list, async item => {
if (item !== undefined && !agentSystemId && item !== userSystemId && !(await this.isAgentSystemId(item))) {
agentSystemId = item;
}
});
@ -263,7 +256,6 @@ export class SecService extends GBService {
});
}
/**
* Get a dynamic param from user. Dynamic params are defined in .gbdialog SET
* variables and other semantics during conversation.
@ -315,7 +307,7 @@ export class SecService extends GBService {
}
obj[name] = value;
user.params = JSON.stringify(obj);
GBServer.globals.users [userId] = user;
GBServer.globals.users[userId] = user;
return await user.save();
}
}
}

2
packages/sharepoint.gblib/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
/**

2
packages/teams.gblib/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
/**

2
packages/teams.gblib/services/TeamsService.ts
View file

@ -28,7 +28,7 @@
| |
\*****************************************************************************/
import { GBService } from 'botlib';
import { GBService } from 'botlib-legacy';
import fs from 'fs/promises';
import AdmZip from 'adm-zip';

2
packages/whatsapp.gblib/index.ts
View file

@ -34,7 +34,7 @@
'use strict';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib-legacy';
import { Sequelize } from 'sequelize-typescript';
import { WhatsappDirectLine } from './services/WhatsappDirectLine.js';

14
packages/whatsapp.gblib/services/WhatsappDirectLine.ts
View file

@ -33,8 +33,8 @@ import urlJoin from 'url-join';
import path from 'path';
import fs from 'fs/promises';
import Fs from 'fs';
import { GBLog, GBMinInstance, GBService, IGBPackage } from 'botlib';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBLog, GBMinInstance, GBService, IGBPackage } from 'botlib-legacy';
import { GBServer } from '../../../src/app.js';
import { GBConversationalService } from '../../core.gbapp/services/GBConversationalService.js';
import { SecService } from '../../security.gbapp/services/SecService.js';
@ -220,7 +220,7 @@ export class WhatsappDirectLine extends GBService {
// TODO: await client.pupPage['minimize']();
// Keeps the chat list cleaned.
const chats = await client.getChats();
await CollectionUtil.asyncForEach(chats, async chat => {
await GBUtil.asyncForEach(chats, async chat => {
const wait = Math.floor(Math.random() * 5000) + 1000;
await GBUtil.sleep(wait);
if (chat.isGroup) {
@ -442,7 +442,7 @@ export class WhatsappDirectLine extends GBService {
// Processes .gbapp message interception.
await CollectionUtil.asyncForEach(this.min.appPackages, async (e: IGBPackage) => {
await GBUtil.asyncForEach(this.min.appPackages, async (e: IGBPackage) => {
await e.onExchangeData(this.min, 'whatsappMessage', { from, fromName });
});
@ -692,7 +692,7 @@ export class WhatsappDirectLine extends GBService {
}
if (activity.attachments) {
await CollectionUtil.asyncForEach(activity.attachments, async attachment => {
await GBUtil.asyncForEach(activity.attachments, async attachment => {
switch (attachment.contentType) {
case 'application/vnd.microsoft.card.hero':
output += `\n${this.renderHeroCard(attachment)}`;
@ -1003,7 +1003,7 @@ export class WhatsappDirectLine extends GBService {
} else {
messages = msg.match(/(.|[\r\n]){1,4096}/g);
await CollectionUtil.asyncForEach(messages, async msg => {
await GBUtil.asyncForEach(messages, async msg => {
await this.sendTextMessage(to, msg);
@ -1021,7 +1021,7 @@ export class WhatsappDirectLine extends GBService {
messages = msg.match(/(.|[\r\n]){1,1000}/g);
await CollectionUtil.asyncForEach(messages, async msg => {
await GBUtil.asyncForEach(messages, async msg => {
await GBUtil.sleep(3000);
await this.customClient.messages.create({
body: msg,

2
src/RootData.ts
View file

@ -32,7 +32,7 @@
* @fileoverview General Bots server core.
*/
'use strict';
import { GBMinInstance, IGBInstance } from 'botlib';
import { GBMinInstance, IGBInstance } from 'botlib-legacy';
import { GBMinService } from '../packages/core.gbapp/services/GBMinService.js';
/**

39
src/app.ts
View file

@ -37,7 +37,7 @@
import { Mutex } from 'async-mutex';
import auth from 'basic-auth';
import bodyParser from 'body-parser';
import { GBLog, GBMinInstance, IGBCoreService, IGBInstance } from 'botlib';
import { GBLog, GBMinInstance, IGBCoreService, IGBInstance } from 'botlib-legacy';
import child_process from 'child_process';
import express from 'express';
import fs from 'fs/promises';
@ -48,7 +48,6 @@ import mkdirp from 'mkdirp';
import { default as Path, default as path } from 'path';
import swaggerUI from 'swagger-ui-dist';
import { GBAdminService } from '../packages/admin.gbapp/services/GBAdminService.js';
import { AzureDeployerService } from '../packages/azuredeployer.gbapp/services/AzureDeployerService.js';
import { GBConfigService } from '../packages/core.gbapp/services/GBConfigService.js';
import { GBConversationalService } from '../packages/core.gbapp/services/GBConversationalService.js';
import { GBCoreService } from '../packages/core.gbapp/services/GBCoreService.js';
@ -165,7 +164,6 @@ export class GBServer {
const core: IGBCoreService = new GBCoreService();
const importer: GBImporter = new GBImporter(core);
const deployer: GBDeployer = new GBDeployer(core, importer);
let azureDeployer: AzureDeployerService;
// Ensure that local proxy is setup.
@ -186,27 +184,14 @@ export class GBServer {
// Creates a boot instance or load it from storage.
if (GBConfigService.get('GB_MODE') === 'legacy') {
await core.initStorage();
// [GBServer.globals.bootInstance, azureDeployer] = await core['createBootInstanceEx'](
// core,
// null,
// GBServer.globals.publicAddress,
// deployer,
// GBConfigService.get('FREE_TIER')
// );
// await core.saveInstance(GBServer.globals.bootInstance);
}
else {
await core.initStorage();
}
await core.initStorage();
// Deploys system and user packages.
GBLogEx.info(0, `Deploying System packages...`);
GBServer.globals.sysPackages = await core.loadSysPackages(core);
GBLogEx.info(0, `Connecting to Bot Storage...`);
await core.checkStorage(azureDeployer);
await core.checkStorage(null);
await deployer.deployPackages(core, server, GBServer.globals.appPackages);
await core.syncDatabaseStructure();
@ -217,11 +202,7 @@ export class GBServer {
}
GBLogEx.info(0, `Publishing instances...`);
const instances: IGBInstance[] = await core.loadAllInstances(
core,
azureDeployer,
GBServer.globals.publicAddress
);
const instances: IGBInstance[] = await core.loadAllInstances(core, null, GBServer.globals.publicAddress);
if (instances.length === 0) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
@ -235,11 +216,6 @@ export class GBServer {
instances.push(instance);
GBServer.globals.minBoot.instance = instances[0];
GBServer.globals.bootInstance = instances[0];
await deployer.deployBotOnAzure(instance, GBServer.globals.publicAddress);
// Runs the search even with empty content to create structure.
await azureDeployer['runSearch'](instance);
}
}
@ -250,8 +226,7 @@ export class GBServer {
// Just sync if not using LOAD_ONLY.
if (GBConfigService.get('GB_MODE') !== 'legacy'
&& !process.env.LOAD_ONLY) {
if (GBConfigService.get('GB_MODE') !== 'legacy' && !process.env.LOAD_ONLY) {
await core['ensureFolders'](instances, deployer);
}
GBServer.globals.bootInstance = instances[0];
@ -289,7 +264,6 @@ export class GBServer {
return proxy.web(req, res, { target: 'http://localhost:1111' }); // Express server
} else if (host === process.env.ROUTER_1) {
return proxy.web(req, res, { target: `http://localhost:${process.env.ROUTER_1_PORT}` });
} else if (host === process.env.ROUTER_2) {
return proxy.web(req, res, { target: `http://localhost:${process.env.ROUTER_2_PORT}` });
} else {
@ -378,10 +352,7 @@ export class GBServer {
}
function shutdown() {
GBLogEx.info(0, 'General Bots server is now shutdown.');
process.exit(0);
}

66
src/util.ts
View file

@ -1,4 +1,3 @@
/**
* @fileoverview General Bots local utility.
* This file contains utility functions used across the General Bots project.
@ -29,8 +28,6 @@ import { QueryTypes } from '@sequelize/core';
* Utility class containing various helper functions for the General Bots project.
*/
export class GBUtil {
// When creating/updating a user (hashing before saving to DB)
public static async hashPassword(password) {
try {
@ -42,6 +39,12 @@ export class GBUtil {
}
}
public static async asyncForEach(array, callback) {
for (let index = 0; index < array.length; index++) {
await callback(array[index], index, array);
}
}
// When comparing passwords (like during login)
public static async comparePassword(inputPassword, hashedPassword) {
try {
@ -116,8 +119,7 @@ export class GBUtil {
};
config.spec['host'] = `127.0.0.1:${GBConfigService.getServerPort()}`;
config.spec['basePath'] = `/api/messages/${min.botId}`;
config.spec['schemes'] = ["http"];
config.spec['schemes'] = ['http'];
} else {
config = {
spec: JSON.parse(await fs.readFile('directline-v2.json', 'utf8')),
@ -150,10 +152,8 @@ export class GBUtil {
styles: { '!!null': 'canonical' } // Optional: Customize null display
} as any);
//yamlString = yamlString.slice(0, 256); // Truncate to 1024 bytes
return yamlString;
}
@ -242,20 +242,33 @@ export class GBUtil {
const destEntry = path.join(dest, entry);
// Recursively copy each entry
await this.copyIfNewerRecursive(srcEntry, destEntry ,onlyTextFiles);
await this.copyIfNewerRecursive(srcEntry, destEntry, onlyTextFiles);
}
} else {
let skip = false;
if (onlyTextFiles && !(
src.endsWith('.txt') || src.endsWith('.json')
|| src.endsWith('.csv') || src.endsWith('.xlsx') || src.endsWith('.xls')
|| src.endsWith('.xlsm') || src.endsWith('.xlsb') || src.endsWith('.xml')
|| src.endsWith('.html') || src.endsWith('.htm') || src.endsWith('.md')
|| src.endsWith('.docx') || src.endsWith('.pdf')
|| src.endsWith('.doc') || src.endsWith('.pptx') || src.endsWith('.ppt'))) {
skip = true;
if (
onlyTextFiles &&
!(
src.endsWith('.txt') ||
src.endsWith('.json') ||
src.endsWith('.csv') ||
src.endsWith('.xlsx') ||
src.endsWith('.xls') ||
src.endsWith('.xlsm') ||
src.endsWith('.xlsb') ||
src.endsWith('.xml') ||
src.endsWith('.html') ||
src.endsWith('.htm') ||
src.endsWith('.md') ||
src.endsWith('.docx') ||
src.endsWith('.pdf') ||
src.endsWith('.doc') ||
src.endsWith('.pptx') ||
src.endsWith('.ppt')
)
) {
skip = true;
}
if (!skip) {
@ -272,7 +285,6 @@ export class GBUtil {
await fs.cp(src, dest, { force: true });
}
}
}
}
@ -403,10 +415,7 @@ export class GBUtil {
public static isContentPage(text: string): boolean {
// Common patterns that indicate non-content pages
const nonContentPatterns = [
/^index$/i,
/^table of contents$/i,
];
const nonContentPatterns = [/^index$/i, /^table of contents$/i];
// Check if page is mostly dots, numbers or blank
const isDotLeaderPage = text.replace(/\s+/g, '').match(/\.{10,}/);
@ -418,21 +427,12 @@ export class GBUtil {
const hasMinimalContent = wordCount > 10;
// Check if page matches any non-content patterns
const isNonContent = nonContentPatterns.some(pattern =>
pattern.test(text.trim())
);
const isNonContent = nonContentPatterns.some(pattern => pattern.test(text.trim()));
// Page is valid content if:
// - Not mostly dots/numbers/blank
// - Has minimal word count
// - Doesn't match non-content patterns
return !isDotLeaderPage &&
!isNumbersPage &&
!isBlankPage &&
hasMinimalContent &&
!isNonContent;
return !isDotLeaderPage && !isNumbersPage && !isBlankPage && hasMinimalContent && !isNonContent;
}
}

28
src/webapp.ts
View file

@ -1,28 +0,0 @@
import { GBConfigService } from '../packages/core.gbapp/services/GBConfigService.js';
const {app} = (await import('electron')).default;
import path from 'path';
import url from 'url';
export function runUI() {
// Create the browser window.
const win = null;// new BrowserWindow({ width: 800, height: 600, title: 'General Bots Studio' });
import('./app.js').then(gb => {
gb.GBServer.run();
// and load the index.html of the app.
win.loadURL(
url.format({
pathname: path.join(__dirname, `http://localhost:${GBConfigService.get('PORT')}`),
protocol: 'file:',
slashes: true
})
);
});
}
export class GBUI {
static run() {
app.on('ready', runUI);
}
}

38
tsconfig.json
View file

@ -1,5 +1,6 @@
{
"compilerOptions": {
"noImplicitAny": false,
"rootDir": "./",
"strict": false,
"allowJs": true,
@ -17,36 +18,15 @@
"resolveJsonModule": true,
"outDir": "./dist",
"paths": {
"*": [
"types/*"
],
"botlib/*": [
"node_modules/botlib/*"
],
"pragmatismo-io-framework/*": [
"node_modules/pragmatismo-io-framework/*"
]
"*": ["types/*"],
"botlib/*": ["node_modules/botlib/*"],
"pragmatismo-io-framework/*": ["node_modules/pragmatismo-io-framework/*"]
},
"types": [
"node",
"lodash",
"node-fetch"
],
"types": ["node", "lodash", "node-fetch"],
"sourceMap": true,
"target": "ESNext",
"typeRoots": [
"node_modules/@types"
]
"typeRoots": ["node_modules/@types"]
},
"include": [
"src/**/*",
"packages/*.gbapp/**/*",
"packages/*.gblib/**/*",
"packages/*.gbtheme/**/*"
],
"exclude": [
"dist",
"node_modules",
"**/*.test.ts"
]
}
"include": ["src/**/*", "packages/*.gbapp/**/*", "packages/*.gblib/**/*", "packages/*.gbtheme/**/*"],
"exclude": ["dist", "node_modules", "**/*.test.ts"]
}

10
install-addons.sh → unsecure-addons.cmd
View file

@ -1,9 +1,9 @@
npm --save install \
npm install \
c3-chart-maker@0.2.8 \
ms-rest-azure@3.0.2 \
open-docxtemplater-image-module@1.0.3 \
svg2img@1.0.0-beta.2 \
get-image-colors@4.0.1 \
pragmatismo-framework \
botlib \
textract \
webp-converter \
final-stream