GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix (all): path and fs normalization.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2024-09-06 15:30:03 -03:00
parent c1ac3f183b
commit 698df5a830
27 changed files with 326 additions and 332 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
boot.mjs
View file

@ -2,9 +2,9 @@
process.stdout.write(`General Bots. BotServer@${pjson.version}, botlib@${pjson.dependencies.botlib}, node@${process.version.replace('v', '')}, ${process.platform} ${process.arch} `);
import Fs from 'fs';
import fs from 'fs';
import os from 'node:os';
import Path from 'path';
import path from 'path';
import { exec } from 'child_process';
import pjson from './package.json' assert { type: 'json' };
@ -22,10 +22,10 @@ try {
});
};
var processDist = () => {
if (!Fs.existsSync('dist')) {
if (!fs.existsSync('dist')) {
console.log(`\n`);
console.log(`Generall Bots: Compiling...`);
exec(Path.join(__dirname, 'node_modules/.bin/tsc'), (err, stdout, stderr) => {
exec(path.join(__dirname, 'node_modules/.bin/tsc'), (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
@ -39,7 +39,7 @@ try {
// Installing modules if it has not been done yet.
if (!Fs.existsSync('node_modules')) {
if (!fs.existsSync('node_modules')) {
console.log(`\n`);
console.log(`Generall Bots: Installing modules for the first time, please wait...`);
exec('npm install', (err, stdout, stderr) => {

2
package.json
View file

@ -21,7 +21,7 @@
"preferGlobal": true,
"private": false,
"bin": {
"gbot": "./boot.cjs"
"gbot": "./boot.mjs"
},
"readme": "README.md",
"repository": {

18
packages/admin.gbapp/services/GBAdminService.ts
View file

@ -46,10 +46,10 @@ import { GBImporter } from '../../core.gbapp/services/GBImporterService.js';
import { GBSharePointService } from '../../sharepoint.gblib/services/SharePointService.js';
import { GuaribasAdmin } from '../models/AdminModel.js';
import msRestAzure from 'ms-rest-azure';
import Path from 'path';
import path from 'path';
import { caseSensitive_Numbs_SpecialCharacters_PW, lowercase_PW } from 'super-strong-password-generator';
import crypto from 'crypto';
import Fs from 'fs';
import fs from 'fs';
import { GBServer } from '../../../src/app.js';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
@ -77,7 +77,7 @@ export class GBAdminService implements IGBAdminService {
public static getNodeVersion() {
const packageJson = urlJoin(process.cwd(), 'package.json');
const pkg = JSON.parse(Fs.readFileSync(packageJson, 'utf8'));
const pkg = JSON.parse(fs.readFileSync(packageJson, 'utf8'));
return pkg.engines.node.replace('=', '');
}
@ -153,7 +153,7 @@ export class GBAdminService implements IGBAdminService {
const importer = new GBImporter(min.core);
const deployer = new GBDeployer(min.core, importer);
const path = GBUtil.getGBAIPath(min.botId, null, packageName);
const localFolder = Path.join('work', path);
const localFolder = path.join('work', path);
await deployer.undeployPackageFromLocalPath(min.instance, localFolder);
}
@ -169,9 +169,9 @@ export class GBAdminService implements IGBAdminService {
const packageName = text.split(' ')[1];
const folderName = text.split(' ')[2];
const packageType = Path.extname(folderName).substr(1);
const packageType = path.extname(folderName).substr(1);
const gbaiPath = GBUtil.getGBAIPath(min.instance.botId, packageType, null);
const localFolder = Path.join('work', gbaiPath);
const localFolder = path.join('work', gbaiPath);
// .gbot packages are handled using storage API, so no download
// of local resources is required.
@ -182,10 +182,10 @@ export class GBAdminService implements IGBAdminService {
}
if (!GBConfigService.get('STORAGE_NAME')) {
const path = Path.join(GBConfigService.get('STORAGE_LIBRARY'), gbaiPath);
GBUtil.copyIfNewerRecursive(path, localFolder);
const filePath = path.join(GBConfigService.get('STORAGE_LIBRARY'), gbaiPath);
GBUtil.copyIfNewerRecursive(filePath, localFolder);
} else {
await deployer['downloadFolder'](min, Path.join('work', `${gbai}`), Path.basename(localFolder));
await deployer['downloadFolder'](min, path.join('work', `${gbai}`), path.basename(localFolder));
}
await deployer['deployPackage2'](min, user, localFolder);

4
packages/azuredeployer.gbapp/dialogs/StartDialog.ts
View file

@ -35,7 +35,7 @@
'use strict';
import { GBLog, IGBInstallationDeployer, IGBInstance } from 'botlib';
import * as Fs from 'fs';
import fs from 'fs';
import { GBAdminService } from '../../../packages/admin.gbapp/services/GBAdminService.js';
import { GBConfigService } from '../../../packages/core.gbapp/services/GBConfigService.js';
import scanf from 'scanf';
@ -49,7 +49,7 @@ export class StartDialog {
public static async createBaseInstance (deployer, freeTier) {
// No .env so asks for cloud credentials to start a new farm.
if (!Fs.existsSync(`.env`)) {
if (!fs.existsSync(`.env`)) {
process.stdout.write(
'A empty enviroment is detected. To start automatic deploy, please enter some information:\n'
);

2
packages/basic.gblib/services/DebuggerService.ts
View file

@ -32,7 +32,7 @@
import { GBLog, GBMinInstance } from 'botlib';
import { GBServer } from '../../../src/app.js';
import Fs from 'fs';
import fs from 'fs';
import SwaggerClient from 'swagger-client';
import { spawn } from 'child_process';
import { CodeServices } from '../../llm.gblib/services/CodeServices.js';

46
packages/basic.gblib/services/DialogKeywords.ts
View file

@ -40,18 +40,17 @@ import { SecService } from '../../security.gbapp/services/SecService.js';
import { SystemKeywords } from './SystemKeywords.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { Messages } from '../strings.js';
import * as Fs from 'fs';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBConversationalService } from '../../core.gbapp/services/GBConversationalService.js';
import fs from 'fs';
import libphonenumber from 'google-libphonenumber';
import * as df from 'date-diff';
import tesseract from 'node-tesseract-ocr';
import Path from 'path';
import path from 'path';
import sgMail from '@sendgrid/mail';
import mammoth from 'mammoth';
import qrcode from 'qrcode';
import { WebAutomationServices } from './WebAutomationServices.js';
import urljoin from 'url-join';
import QrScanner from 'qr-scanner';
import pkg from 'whatsapp-web.js';
import { ActivityTypes } from 'botbuilder';
@ -59,7 +58,6 @@ const { List, Buttons } = pkg;
import mime from 'mime-types';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { GBUtil } from '../../../src/util.js';
import SwaggerClient from 'swagger-client';
import { GBVMService } from './GBVMService.js';
import { ChatServices } from '../../../packages/llm.gblib/services/ChatServices.js';
import puppeteer from 'puppeteer';
@ -109,10 +107,10 @@ export class DialogKeywords {
const content = await page.$('.bb');
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = Path.join('work', gbaiName, 'cache', `chart${GBAdminService.getRndReadableIdentifier()}.jpg`);
const localName = path.join('work', gbaiName, 'cache', `chart${GBAdminService.getRndReadableIdentifier()}.jpg`);
await content.screenshot({ path: localName, omitBackground: true });
await browser.close();
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
GBLogEx.info(min, `Visualization: Chart generated at ${url}.`);
return { localName, url };
@ -183,11 +181,11 @@ export class DialogKeywords {
}
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.jpg`);
const localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.jpg`);
await ChartServices.screenshot(definition, localName);
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
GBLogEx.info(min, `Visualization: Chart generated at ${url}.`);
@ -1448,11 +1446,11 @@ export class DialogKeywords {
// Web automation.
if (element) {
const localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.jpg`);
nameOnly = Path.basename(localName);
const localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.jpg`);
nameOnly = path.basename(localName);
await element.screenshot({ path: localName, fullPage: true });
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
GBLogEx.info(min, `WebAutomation: Sending ${url} to ${mobile} (${channel}).`);
}
@ -1460,7 +1458,7 @@ export class DialogKeywords {
// GBFILE object.
else if (filename.url) {
url = filename.url;
nameOnly = Path.basename(filename.localName);
nameOnly = path.basename(filename.localName);
GBLogEx.info(min, `Sending the GBFILE ${url} to ${mobile} (${channel}).`);
}
@ -1479,7 +1477,7 @@ export class DialogKeywords {
// .gbdrive direct sending.
else {
const ext = Path.extname(filename);
const ext = path.extname(filename);
const gbaiName = GBUtil.getGBAIPath(min.botId);
let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min);
@ -1496,27 +1494,27 @@ export class DialogKeywords {
const driveUrl = template['@microsoft.graph.downloadUrl'];
const res = await fetch(driveUrl);
let buf: any = Buffer.from(await res.arrayBuffer());
let localName1 = Path.join(
let localName1 = path.join(
'work',
gbaiName,
'cache',
`${fileOnly.replace(/\s/gi, '')}-${GBAdminService.getNumberIdentifier()}.${ext}`
);
Fs.writeFileSync(localName1, buf, { encoding: null });
fs.writeFileSync(localName1, buf, { encoding: null });
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName1));
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName1));
}
if (!url) {
const ext = Path.extname(filename.localName);
const ext = path.extname(filename.localName);
// Prepare a cache to be referenced by Bot Framework.
const buf = Fs.readFileSync(filename);
const buf = fs.readFileSync(filename);
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = Path.join('work', gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.${ext}`);
Fs.writeFileSync(localName, buf, { encoding: null });
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
const localName = path.join('work', gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.${ext}`);
fs.writeFileSync(localName, buf, { encoding: null });
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
}
const contentType = mime.lookup(url);
@ -1547,9 +1545,9 @@ export class DialogKeywords {
const buf = Buffer.from(data, 'base64');
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = Path.join('work', gbaiName, 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png`);
Fs.writeFileSync(localName, buf, { encoding: null });
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
const localName = path.join('work', gbaiName, 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png`);
fs.writeFileSync(localName, buf, { encoding: null });
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
return { data: data, localName: localName, url: url };
}

56
packages/basic.gblib/services/GBVMService.ts
View file

@ -31,7 +31,7 @@
'use strict';
import { GBMinInstance, GBService, IGBCoreService, GBLog } from 'botlib';
import * as Fs from 'fs';
import fs from 'fs';
import * as ji from 'just-indent';
import { GBServer } from '../../../src/app.js';
import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
@ -44,7 +44,7 @@ import { createVm2Pool } from './vm2-process/index.js';
import textract from 'textract';
import walkPromise from 'walk-promise';
import child_process from 'child_process';
import Path from 'path';
import path from 'path';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { DialogKeywords } from './DialogKeywords.js';
import { KeywordsExpressions } from './KeywordsExpressions.js';
@ -69,7 +69,7 @@ export class GBVMService extends GBService {
public static API_PORT = 1111;
public async loadDialogPackage(folder: string, min: GBMinInstance, core: IGBCoreService, deployer: GBDeployer) {
const ignore = Path.join('work', GBUtil.getGBAIPath(min.botId, 'gbdialog'), 'node_modules');
const ignore = path.join('work', GBUtil.getGBAIPath(min.botId, 'gbdialog'), 'node_modules');
const files = await walkPromise(folder, { ignore: [ignore] });
await CollectionUtil.asyncForEach(files, async file => {
@ -120,7 +120,7 @@ export class GBVMService extends GBService {
const wordFile = filename;
const vbsFile = isWord ? filename.substr(0, filename.indexOf('docx')) + 'vbs' : filename;
const fullVbsFile = urlJoin(folder, vbsFile);
const docxStat = Fs.statSync(urlJoin(folder, wordFile));
const docxStat = fs.statSync(urlJoin(folder, wordFile));
const interval = 3000; // If compiled is older 30 seconds, then recompile.
let writeVBS = true;
@ -139,8 +139,8 @@ export class GBVMService extends GBService {
// await client.api('/subscriptions')
// .post(subscription);
if (Fs.existsSync(fullVbsFile)) {
const vbsStat = Fs.statSync(fullVbsFile);
if (fs.existsSync(fullVbsFile)) {
const vbsStat = fs.statSync(fullVbsFile);
if (docxStat['mtimeMs'] < vbsStat['mtimeMs'] + interval) {
writeVBS = false;
}
@ -154,7 +154,7 @@ export class GBVMService extends GBService {
// Write VBS file without pragma keywords.
Fs.writeFileSync(urlJoin(folder, vbsFile), text);
fs.writeFileSync(urlJoin(folder, vbsFile), text);
}
// Process node_modules install.
@ -165,18 +165,18 @@ export class GBVMService extends GBService {
const fullFilename = urlJoin(folder, filename);
if (process.env.DEV_HOTSWAP) {
Fs.watchFile(fullFilename, async () => {
fs.watchFile(fullFilename, async () => {
await this.translateBASIC(mainName, fullFilename, min);
const parsedCode: string = Fs.readFileSync(jsfile, 'utf8');
const parsedCode: string = fs.readFileSync(jsfile, 'utf8');
min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode;
});
}
const compiledAt = Fs.statSync(fullFilename);
const compiledAt = fs.statSync(fullFilename);
const jsfile = urlJoin(folder, `${filename}.js`);
if (Fs.existsSync(jsfile)) {
const jsStat = Fs.statSync(jsfile);
if (fs.existsSync(jsfile)) {
const jsStat = fs.statSync(jsfile);
const interval = 1000; // If compiled is older 1 seconds, then recompile.
if (compiledAt.isFile() && compiledAt['mtimeMs'] > jsStat['mtimeMs'] + interval) {
await this.translateBASIC(mainName, fullFilename, min);
@ -189,13 +189,13 @@ export class GBVMService extends GBService {
this.syncStorageFromTABLE(folder, filename, min, mainName);
const parsedCode: string = Fs.readFileSync(jsfile, 'utf8');
const parsedCode: string = fs.readFileSync(jsfile, 'utf8');
min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode;
return filename;
}
private processNodeModules(folder: string, min: GBMinInstance) {
const node_modules = urlJoin(process.env.PWD, folder, 'node_modules');
if (!Fs.existsSync(node_modules)) {
if (!fs.existsSync(node_modules)) {
const packageJson = `
{
"name": "${min.botId}.gbdialog",
@ -214,7 +214,7 @@ export class GBVMService extends GBService {
"async-retry": "1.3.3"
}
}`;
Fs.writeFileSync(urlJoin(folder, 'package.json'), packageJson);
fs.writeFileSync(urlJoin(folder, 'package.json'), packageJson);
GBLogEx.info(min, `Installing .gbdialog node_modules for ${min.botId}...`);
const npmPath = urlJoin(process.env.PWD, 'node_modules', '.bin', 'npm');
@ -225,10 +225,10 @@ export class GBVMService extends GBService {
public static async loadConnections(min) {
// Loads storage custom connections.
const path = GBUtil.getGBAIPath(min.botId, null);
const filePath = Path.join('work', path, 'connections.json');
const filePath = path.join('work', path, 'connections.json');
let connections = [];
if (Fs.existsSync(filePath)) {
connections = JSON.parse(Fs.readFileSync(filePath, 'utf8'));
if (fs.existsSync(filePath)) {
connections = JSON.parse(fs.readFileSync(filePath, 'utf8'));
}
connections.forEach(async con => {
@ -290,10 +290,10 @@ export class GBVMService extends GBService {
const tablesFile = urlJoin(folder, `${filename}.tables.json`);
let sync = false;
if (Fs.existsSync(tablesFile)) {
if (fs.existsSync(tablesFile)) {
const minBoot = GBServer.globals.minBoot;
const tableDef = JSON.parse(Fs.readFileSync(tablesFile, 'utf8')) as any;
const tableDef = JSON.parse(fs.readFileSync(tablesFile, 'utf8')) as any;
const getTypeBasedOnCondition = (t, size) => {
if (1) {
@ -453,7 +453,7 @@ export class GBVMService extends GBService {
public async translateBASIC(mainName, filename: any, min: GBMinInstance) {
// Converts General Bots BASIC into regular VBS
let basicCode: string = Fs.readFileSync(filename, 'utf8');
let basicCode: string = fs.readFileSync(filename, 'utf8');
basicCode = GBVMService.normalizeQuotes(basicCode);
// Pre process SET SCHEDULE calls.
@ -481,13 +481,13 @@ export class GBVMService extends GBService {
if (include) {
let includeName = include[1].trim();
includeName = Path.join(Path.dirname(filename), includeName);
includeName = path.join(path.dirname(filename), includeName);
includeName = includeName.substr(0, includeName.lastIndexOf('.')) + '.vbs';
// To use include, two /publish will be necessary (for now)
// because of alphabet order may raise not found errors.
let includeCode: string = Fs.readFileSync(includeName, 'utf8');
let includeCode: string = fs.readFileSync(includeName, 'utf8');
basicCode = basicCode.replace(/^include\b.*$/gim, includeCode);
}
} while (include);
@ -497,10 +497,10 @@ export class GBVMService extends GBService {
// Generates function JSON metadata to be used later.
const jsonFile = `${filename}.json`;
Fs.writeFileSync(jsonFile, JSON.stringify(metadata));
fs.writeFileSync(jsonFile, JSON.stringify(metadata));
const mapFile = `${filename}.map`;
Fs.writeFileSync(mapFile, JSON.stringify(map));
fs.writeFileSync(mapFile, JSON.stringify(map));
// Execute off-line code tasks
@ -710,7 +710,7 @@ export class GBVMService extends GBService {
code = ji.default(code, ' ');
Fs.writeFileSync(jsfile, code);
fs.writeFileSync(jsfile, code);
GBLogEx.info(min, `[GBVMService] Finished loading of ${filename}, JavaScript from Word: \n ${code}`);
}
@ -722,7 +722,7 @@ export class GBVMService extends GBService {
// Creates an empty object that will receive Sequelize fields.
const tablesFile = `${task.file}.tables.json`;
Fs.writeFileSync(tablesFile, JSON.stringify(task.tables));
fs.writeFileSync(tablesFile, JSON.stringify(task.tables));
}
}
}
@ -760,7 +760,7 @@ export class GBVMService extends GBService {
textract.fromFileWithPath(path, { preserveLineBreaks: true }, (error, text) => {
if (error) {
if (error.message.startsWith('File not correctly recognized as zip file')) {
text = Fs.readFileSync(path, 'utf8');
text = fs.readFileSync(path, 'utf8');
} else {
reject(error);
}

6
packages/basic.gblib/services/ImageProcessingServices.ts
View file

@ -30,7 +30,7 @@
'use strict';
import Path from 'path';
import path from 'path';
import { GBLog, GBMinInstance } from 'botlib';
import { DialogKeywords } from './DialogKeywords.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
@ -77,8 +77,8 @@ export class ImageProcessingServices {
const botId = min.instance.botId;
const path = GBUtil.getGBAIPath(min.botId);
// TODO: const img = await joinImages(paths);
const localName = Path.join('work', path, 'cache', `img-mrg${GBAdminService.getRndReadableIdentifier()}.png`);
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
const localName = path.join('work', path, 'cache', `img-mrg${GBAdminService.getRndReadableIdentifier()}.png`);
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
// img.toFile(localName);
return { localName: localName, url: url, data: null };

4
packages/basic.gblib/services/KeywordsExpressions.ts
View file

@ -33,7 +33,7 @@
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { GBVMService } from './GBVMService.js';
import Path from 'path';
import path from 'path';
/**
* Image processing services of conversation to be called by BASIC.
@ -297,7 +297,7 @@ export class KeywordsExpressions {
if (kind === 'AS' && KeywordsExpressions.isNumber(sessionName)) {
const jParams = JSON.parse(`{${params}}`);
const filename = `${Path.basename(jParams.url, 'txt')}xlsx`;
const filename = `${path.basename(jParams.url, 'txt')}xlsx`;
return `files[${sessionName}] = "${filename}"`;
} else {
sessionName = `"${sessionName}"`;

107
packages/basic.gblib/services/SystemKeywords.ts
View file

@ -34,7 +34,6 @@ import { runInNewContext } from 'vm';
import { IgApiClient } from 'instagram-private-api';
import { readFileSync } from 'fs';
import { resolve } from 'path';
import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
import { GBLog, GBMinInstance } from 'botlib';
import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js';
import { CollectionUtil } from 'pragmatismo-io-framework';
@ -43,13 +42,13 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import { DialogKeywords } from './DialogKeywords.js';
import { GBServer } from '../../../src/app.js';
import { GBVMService } from './GBVMService.js';
import Fs from 'fs';
import fs from 'fs';
import { GBSSR } from '../../core.gbapp/services/GBSSR.js';
import urlJoin from 'url-join';
import Excel from 'exceljs';
import { BufferWindowMemory } from 'langchain/memory';
import csvdb from 'csv-database';
import Path from 'path';
import path from 'path';
import ComputerVisionClient from '@azure/cognitiveservices-computervision';
import ApiKeyCredentials from '@azure/ms-rest-js';
import alasql from 'alasql';
@ -69,7 +68,7 @@ import { SecService } from '../../security.gbapp/services/SecService.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import retry from 'async-retry';
import { BlobServiceClient, BlockBlobClient, StorageSharedKeyCredential } from '@azure/storage-blob';
import { FacebookAdsApi, Page } from 'facebook-nodejs-business-sdk';
import { Page } from 'facebook-nodejs-business-sdk';
import { md5 } from 'js-md5';
import { GBUtil } from '../../../src/util.js';
@ -322,17 +321,17 @@ export class SystemKeywords {
let url;
let localName;
if (renderImage) {
localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`);
localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`);
await page.screenshot({ path: localName, fullPage: true });
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
GBLogEx.info(min, `Table image generated at ${url} .`);
}
// Handles PDF generation.
if (renderPDF) {
localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.pdf`);
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.pdf`);
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
let pdf = await page.pdf({ format: 'A4' });
GBLogEx.info(min, `Table PDF generated at ${url} .`);
}
@ -411,12 +410,12 @@ export class SystemKeywords {
// Prepare an image on cache and return the GBFILE information.
const localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`);
const localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`);
if (pngPages.length > 0) {
const buffer = pngPages[0].content;
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
Fs.writeFileSync(localName, buffer, { encoding: null });
fs.writeFileSync(localName, buffer, { encoding: null });
return { localName: localName, url: url, data: buffer };
}
@ -702,14 +701,14 @@ export class SystemKeywords {
// It is an SharePoint object that needs to be downloaded.
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = Path.join('work', gbaiName, 'cache', `${GBAdminService.getRndReadableIdentifier()}.tmp`);
const localName = path.join('work', gbaiName, 'cache', `${GBAdminService.getRndReadableIdentifier()}.tmp`);
const url = file['url'];
const response = await fetch(url);
// Writes it to disk and calculate hash.
const data = await response.arrayBuffer();
Fs.writeFileSync(localName, Buffer.from(data), { encoding: null });
fs.writeFileSync(localName, Buffer.from(data), { encoding: null });
const hash = new Uint8Array(md5.array(data));
// Performs uploading passing local hash.
@ -725,7 +724,7 @@ export class SystemKeywords {
// If upload is OK including hash check, removes the temporary file.
if (res._response.status === 201 && new Uint8Array(res.contentMD5).toString() === hash.toString()) {
Fs.rmSync(localName);
fs.rmSync(localName);
file['md5'] = hash.toString();
@ -886,8 +885,8 @@ export class SystemKeywords {
if (e.cause === 404) {
// Creates the file.
const blank = Path.join(process.env.PWD, 'blank.xlsx');
const data = Fs.readFileSync(blank);
const blank = path.join(process.env.PWD, 'blank.xlsx');
const data = fs.readFileSync(blank);
await client.api(`${baseUrl}/drive/root:/${path}/${file}:/content`).put(data);
// Tries to open again.
@ -1151,10 +1150,10 @@ export class SystemKeywords {
result = null;
} else if (file['cTag']) {
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = Path.join('work', gbaiName, 'cache', `csv${GBAdminService.getRndReadableIdentifier()}.csv`);
const localName = path.join('work', gbaiName, 'cache', `csv${GBAdminService.getRndReadableIdentifier()}.csv`);
const url = file['@microsoft.graph.downloadUrl'];
const response = await fetch(url);
Fs.writeFileSync(localName, Buffer.from(await response.arrayBuffer()), { encoding: null });
fs.writeFileSync(localName, Buffer.from(await response.arrayBuffer()), { encoding: null });
var workbook = new Excel.Workbook();
let worksheet = await workbook.csv.readFile(localName);
@ -1200,9 +1199,9 @@ export class SystemKeywords {
rows = results.text;
} else if (file.indexOf('.csv') !== -1) {
let res;
let path = GBUtil.getGBAIPath(min.botId, `gbdata`);
const csvFile = Path.join(GBConfigService.get('STORAGE_LIBRARY'), path, file);
const firstLine = Fs.readFileSync(csvFile, 'utf8').split('\n')[0];
let packagePath = GBUtil.getGBAIPath(min.botId, `gbdata`);
const csvFile = path.join(GBConfigService.get('STORAGE_LIBRARY'), packagePath, file);
const firstLine = fs.readFileSync(csvFile, 'utf8').split('\n')[0];
const headers = firstLine.split(',');
const db = await csvdb(csvFile, headers, ',');
if (args[0]) {
@ -1522,7 +1521,7 @@ export class SystemKeywords {
const path = GBUtil.getGBAIPath(min.botId);
const systemPromptFile = urlJoin(process.cwd(), 'work', path, 'users', user.userSystemId, 'systemPrompt.txt');
Fs.writeFileSync(systemPromptFile, text);
fs.writeFileSync(systemPromptFile, text);
}
}
@ -1611,8 +1610,8 @@ export class SystemKeywords {
// Templates a blank {content} tag inside the blank.docx.
const blank = Path.join(process.env.PWD, 'blank.docx');
let buf = Fs.readFileSync(blank);
const blank = path.join(process.env.PWD, 'blank.docx');
let buf = fs.readFileSync(blank);
let zip = new PizZip(buf);
let doc = new Docxtemplater();
doc.setOptions({ linebreaks: true });
@ -1660,7 +1659,7 @@ export class SystemKeywords {
let folder;
if (dest.indexOf('/') !== -1) {
const pathOnly = Path.dirname(dest);
const pathOnly = path.dirname(dest);
folder = await this.createFolder({ pid, name: pathOnly });
} else {
folder = await client.api(`${baseUrl}/drive/root:/${root}`).get();
@ -1673,7 +1672,7 @@ export class SystemKeywords {
const srcFile = await client.api(`${baseUrl}/drive/root:/${srcPath}`).get();
const destFile = {
parentReference: { driveId: folder.parentReference.driveId, id: folder.id },
name: `${Path.basename(dest)}`
name: `${path.basename(dest)}`
};
const file = await client.api(`${baseUrl}/drive/items/${srcFile.id}/copy`).post(destFile);
GBLogEx.info(min, `FINISHED COPY '${src}' to '${dest}'`);
@ -1721,7 +1720,7 @@ export class SystemKeywords {
let folder;
if (dest.indexOf('/') !== -1) {
const pathOnly = Path.dirname(dest);
const pathOnly = path.dirname(dest);
folder = await this.createFolder({ pid, name: pathOnly });
} else {
folder = await client.api(`${baseUrl}/drive/root:/${root}`).get();
@ -2007,20 +2006,20 @@ export class SystemKeywords {
// Downloads template from .gbdrive.
let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min);
let path = '/' + urlJoin(gbaiName, `${botId}.gbdrive`);
let template = await this.internalGetDocument(client, baseUrl, path, templateName);
let packagePath = '/' + urlJoin(gbaiName, `${botId}.gbdrive`);
let template = await this.internalGetDocument(client, baseUrl, packagePath, templateName);
let url = template['@microsoft.graph.downloadUrl'];
const res = await fetch(url);
let buf: any = Buffer.from(await res.arrayBuffer());
localName = Path.join('work', gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.docx`);
Fs.writeFileSync(localName, buf, { encoding: null });
localName = path.join('work', gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.docx`);
fs.writeFileSync(localName, buf, { encoding: null });
// Replace image path on all elements of data.
const images = [];
let index = 0;
path = Path.join(gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.docx`);
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
packagePath = path.join(gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.docx`);
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
const traverseDataToInjectImageUrl = async o => {
for (var i in o) {
@ -2035,15 +2034,15 @@ export class SystemKeywords {
if (value.endsWith && value.endsWith(`.${kind}`)) {
const { baseUrl, client } = await GBDeployer.internalGetDriveClient(min);
path = urlJoin(gbaiName, `${botId}.gbdrive`);
packagePath = urlJoin(gbaiName, `${botId}.gbdrive`);
if (value.indexOf('/') !== -1) {
path = '/' + urlJoin(path, Path.dirname(value));
value = Path.basename(value);
packagePath = '/' + urlJoin(packagePath, path.dirname(value));
value = path.basename(value);
}
const ref = await this.internalGetDocument(client, baseUrl, path, value);
const ref = await this.internalGetDocument(client, baseUrl, packagePath, value);
let url = ref['@microsoft.graph.downloadUrl'];
const imageName = Path.join(
const imageName = path.join(
'work',
gbaiName,
'cache',
@ -2051,7 +2050,7 @@ export class SystemKeywords {
);
const response = await fetch(url);
const buf = Buffer.from(await response.arrayBuffer());
Fs.writeFileSync(imageName, buf, { encoding: null });
fs.writeFileSync(imageName, buf, { encoding: null });
const getNormalSize = ({ width, height, orientation }) => {
return (orientation || 0) >= 5 ? [height, width] : [width, height];
@ -2063,7 +2062,7 @@ export class SystemKeywords {
height: 400,
orientation: '0'
});
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(imageName));
url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(imageName));
images[index++] = { url: url, size: size, buf: buf };
}
}
@ -2100,7 +2099,7 @@ export class SystemKeywords {
doc.setData(data).render();
buf = doc.getZip().generate({ type: 'nodebuffer', compression: 'DEFLATE' });
Fs.writeFileSync(localName, buf, { encoding: null });
fs.writeFileSync(localName, buf, { encoding: null });
return { localName: localName, url: url, data: buf };
}
@ -2557,14 +2556,14 @@ export class SystemKeywords {
// Prepare an image on cache and return the GBFILE information.
const buf = Buffer.from(data.Payment.QrCodeBase64Image, 'base64');
const localName = Path.join('work', gbaiName, 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png`);
Fs.writeFileSync(localName, buf, { encoding: null });
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
const localName = path.join('work', gbaiName, 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png`);
fs.writeFileSync(localName, buf, { encoding: null });
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
GBLogEx.info(min, `GBPay: ${data.MerchantOrderId} OK: ${url}.`);
return {
name: Path.basename(localName),
name: path.basename(localName),
localName: localName,
url: url,
data: buf,
@ -2590,7 +2589,7 @@ export class SystemKeywords {
const path = GBUtil.getGBAIPath(min.botId, `gbdrive`);
const fileName = file.url ? file.url : file.name;
const contentType = mime.lookup(fileName);
const ext = Path.extname(fileName).substring(1);
const ext = path.extname(fileName).substring(1);
const kind = await this.getExtensionInfo(ext);
let d = new Date(),
@ -2638,7 +2637,7 @@ export class SystemKeywords {
const gbaiPath = GBUtil.getGBAIPath(min.botId);
const fileName = file.name;
const contentType = mime.lookup(fileName);
const ext = Path.extname(fileName).substring(1);
const ext = path.extname(fileName).substring(1);
const kind = await this.getExtensionInfo(ext);
await client.api(`${baseUrl}/drive/root:/${gbaiPath}/${file.path}`).delete();
@ -2702,7 +2701,7 @@ export class SystemKeywords {
obj['name'] = item.name;
obj['size'] = item.size;
obj['hash'] = item.file?.hashes?.quickXorHash;
obj['path'] = Path.join(remotePath, item.name);
obj['path'] = path.join(remotePath, item.name);
obj['url'] = item['@microsoft.graph.downloadUrl'];
array.push(obj);
@ -2735,8 +2734,8 @@ export class SystemKeywords {
}
else {
let path = GBUtil.getGBAIPath(min.botId, `gbdrive`);
let filePath = Path.join(GBConfigService.get('STORAGE_LIBRARY'), path, file);
data = Fs.readFileSync(filePath, 'utf8');
let filePath = path.join(GBConfigService.get('STORAGE_LIBRARY'), path, file);
data = fs.readFileSync(filePath, 'utf8');
data = new Uint8Array(Buffer.from(data, 'utf8'));
}
return await GBUtil.getPdfText(data);
@ -2781,11 +2780,11 @@ export class SystemKeywords {
const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
// Leitura do arquivo de imagem
const imageBuffer = Fs.readFileSync(Path.resolve(imagePath));
const imageBuffer = fs.readFileSync(path.resolve(imagePath));
// Criação de um arquivo temporário para enviar
const tempFilePath = Path.resolve('temp_image.jpg');
Fs.writeFileSync(tempFilePath, imageBuffer);
const tempFilePath = path.resolve('temp_image.jpg');
fs.writeFileSync(tempFilePath, imageBuffer);
// Publicação da imagem
const page = new Page(pageId);
@ -2802,7 +2801,7 @@ export class SystemKeywords {
GBLogEx.info(min, `Imagem publicada no Facebook: ${JSON.stringify(response)}`);
// Limpeza do arquivo temporário
Fs.unlinkSync(tempFilePath);
fs.unlinkSync(tempFilePath);
}

19
packages/basic.gblib/services/WebAutomationServices.ts
View file

@ -31,10 +31,9 @@
'use strict';
import urlJoin from 'url-join';
import Fs from 'fs';
import Path from 'path';
import fs from 'fs';
import path from 'path';
import url from 'url';
import { GBLog } from 'botlib';
import { GBServer } from '../../../src/app.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
@ -350,11 +349,11 @@ export class WebAutomationServices {
GBLogEx.info(min, `Web Automation SCREENSHOT ${selector}.`);
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = Path.join('work', gbaiName, 'cache', `screen-${GBAdminService.getRndReadableIdentifier()}.jpg`);
const localName = path.join('work', gbaiName, 'cache', `screen-${GBAdminService.getRndReadableIdentifier()}.jpg`);
await page.screenshot({ path: localName });
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
GBLogEx.info(min, `WebAutomation: Screenshot captured at ${url}.`);
return { data: null, localName: localName, url: url };
@ -416,18 +415,18 @@ export class WebAutomationServices {
let filename;
if (options.uri.indexOf('file://') != -1) {
local = url.fileURLToPath(options.uri);
filename = Path.basename(local);
filename = path.basename(local);
} else {
const getBasenameFormUrl = urlStr => {
const url = new URL(urlStr);
return Path.basename(url.pathname);
return path.basename(url.pathname);
};
filename = getBasenameFormUrl(options.uri);
}
let result: Buffer;
if (local) {
result = Fs.readFileSync(local);
result = fs.readFileSync(local);
} else {
const res = await fetch(options.uri, options);
result = Buffer.from(await res.arrayBuffer());
@ -440,8 +439,8 @@ export class WebAutomationServices {
folder = folder.replace(/\\/gi, '/');
// Determines full path at source and destination.
const path = GBUtil.getGBAIPath(min.botId, `gbdrive`);
const root = path;
const packagePath = GBUtil.getGBAIPath(min.botId, `gbdrive`);
const root = packagePath;
const dstPath = urlJoin(root, folder, filename);
// Checks if the destination contains subfolders that

14
packages/core.gbapp/services/GBConversationalService.ts
View file

@ -48,11 +48,11 @@ import { CollectionUtil, AzureText } from 'pragmatismo-io-framework';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
import { GBMinService } from './GBMinService.js';
import urlJoin from 'url-join';
import Fs from 'fs';
import fs from 'fs';
import twilio from 'twilio';
import Nexmo from 'nexmo';
import { join } from 'path';
import Path from 'path';
import path from 'path';
import shell from 'any-shell-escape';
import { exec } from 'child_process';
import prism from 'prism-media';
@ -451,15 +451,15 @@ export class GBConversationalService {
const waveFilename = `work/tmp${name}.pcm`;
let audio = await textToSpeech.repairWavHeaderStream(res.result as any);
Fs.writeFileSync(waveFilename, audio);
fs.writeFileSync(waveFilename, audio);
const oggFilenameOnly = `tmp${name}.ogg`;
const oggFilename = `work/${oggFilenameOnly}`;
const output = Fs.createWriteStream(oggFilename);
const output = fs.createWriteStream(oggFilename);
const transcoder = new prism.FFmpeg({
args: ['-analyzeduration', '0', '-loglevel', '0', '-f', 'opus', '-ar', '16000', '-ac', '1']
});
Fs.createReadStream(waveFilename).pipe(transcoder).pipe(output);
fs.createReadStream(waveFilename).pipe(transcoder).pipe(output);
let url = urlJoin(GBServer.globals.publicAddress, 'audios', oggFilenameOnly);
resolve(url);
@ -481,7 +481,7 @@ export class GBConversationalService {
const dest = `work/tmp${name}.wav`;
const src = `work/tmp${name}.ogg`;
Fs.writeFileSync(src, oggFile.read());
fs.writeFileSync(src, oggFile.read());
const makeMp3 = shell([
'node_modules/ffmpeg-static/ffmpeg', // TODO: .exe on MSWin.
@ -504,7 +504,7 @@ export class GBConversationalService {
GBLog.error(error);
return Promise.reject(error);
} else {
let data = Fs.readFileSync(dest);
let data = fs.readFileSync(dest);
const speechToText = new SpeechToTextV1({
authenticator: new IamAuthenticator({ apikey: process.env.WATSON_STT_KEY }),

40
packages/core.gbapp/services/GBCoreService.ts
View file

@ -35,7 +35,7 @@
'use strict';
import { GBLog, GBMinInstance, IGBCoreService, IGBInstallationDeployer, IGBInstance, IGBPackage } from 'botlib';
import * as Fs from 'fs';
import fs from 'fs';
import { Sequelize, SequelizeOptions } from 'sequelize-typescript';
import { Op, Dialect } from 'sequelize';
import { GBServer } from '../../../src/app.js';
@ -60,7 +60,7 @@ import { GBGoogleChatPackage } from '../../google-chat.gblib/index.js';
import { GBHubSpotPackage } from '../../hubspot.gblib/index.js';
import open from 'open';
import ngrok from 'ngrok';
import Path from 'path';
import path from 'path';
import { GBUtil } from '../../../src/util.js';
import { GBLogEx } from './GBLogEx.js';
import { GBDeployer } from './GBDeployer.js';
@ -135,7 +135,7 @@ export class GBCoreService implements IGBCoreService {
} else if (this.dialect === 'sqlite') {
storage = GBConfigService.get('STORAGE_FILE');
if (!Fs.existsSync(storage)) {
if (!fs.existsSync(storage)) {
process.env.STORAGE_SYNC = 'true';
}
} else {
@ -313,7 +313,7 @@ STORAGE_SYNC_ALTER=true
ENDPOINT_UPDATE=true
`;
Fs.writeFileSync('.env', env);
fs.writeFileSync('.env', env);
}
/**
@ -323,7 +323,7 @@ ENDPOINT_UPDATE=true
*/
public async ensureProxy(port): Promise<string> {
try {
if (Fs.existsSync('node_modules/ngrok/bin/ngrok.exe') || Fs.existsSync('node_modules/.bin/ngrok')) {
if (fs.existsSync('node_modules/ngrok/bin/ngrok.exe') || fs.existsSync('node_modules/.bin/ngrok')) {
return await ngrok.connect({ port: port });
} else {
GBLog.warn('ngrok executable not found. Check installation or node_modules folder.');
@ -410,7 +410,7 @@ ENDPOINT_UPDATE=true
let matchingAppPackages = [];
await CollectionUtil.asyncForEach(appPackages, async appPackage => {
const filenameOnly = Path.basename(appPackage.name);
const filenameOnly = path.basename(appPackage.name);
const matchedApp = apps.find(app => app.name === filenameOnly);
if (matchedApp || filenameOnly.endsWith('.gblib')) {
matchingAppPackages.push(appPackage);
@ -717,7 +717,7 @@ ENDPOINT_UPDATE=true
.patch(body);
} else {
let path = GBUtil.getGBAIPath(min.botId, `gbot`);
const config = Path.join(GBConfigService.get('STORAGE_LIBRARY'), path, 'config.csv');
const config = path.join(GBConfigService.get('STORAGE_LIBRARY'), path, 'config.csv');
const db = await csvdb(config, ['name', 'value'], ',');
if (await db.get({ name: name })) {
@ -825,13 +825,13 @@ ENDPOINT_UPDATE=true
public async ensureFolders(instances, deployer: GBDeployer) {
let libraryPath = GBConfigService.get('STORAGE_LIBRARY');
if (!Fs.existsSync(libraryPath)) {
if (!fs.existsSync(libraryPath)) {
mkdirp.sync(libraryPath);
}
await this.syncBotStorage(instances, 'default', deployer, libraryPath);
const files = Fs.readdirSync(libraryPath);
const files = fs.readdirSync(libraryPath);
await CollectionUtil.asyncForEach(files, async file => {
if (file.trim().toLowerCase() !== 'default.gbai') {
let botId = file.replace(/\.gbai/, '');
@ -853,39 +853,39 @@ ENDPOINT_UPDATE=true
email = null;
instance = await deployer.deployBlankBot(botId, mobile, email);
const gbaiPath = Path.join(libraryPath, `${botId}.gbai`);
const gbaiPath = path.join(libraryPath, `${botId}.gbai`);
if (!Fs.existsSync(gbaiPath)) {
Fs.mkdirSync(gbaiPath, { recursive: true });
if (!fs.existsSync(gbaiPath)) {
fs.mkdirSync(gbaiPath, { recursive: true });
const base = Path.join(process.env.PWD, 'templates', 'default.gbai');
const base = path.join(process.env.PWD, 'templates', 'default.gbai');
Fs.cpSync(Path.join(base, `default.gbkb`), Path.join(gbaiPath, `default.gbkb`), {
fs.cpSync(path.join(base, `default.gbkb`), path.join(gbaiPath, `default.gbkb`), {
errorOnExist: false,
force: true,
recursive: true
});
Fs.cpSync(Path.join(base, `default.gbot`), Path.join(gbaiPath, `default.gbot`), {
fs.cpSync(path.join(base, `default.gbot`), path.join(gbaiPath, `default.gbot`), {
errorOnExist: false,
force: true,
recursive: true
});
Fs.cpSync(Path.join(base, `default.gbtheme`), Path.join(gbaiPath, `default.gbtheme`), {
fs.cpSync(path.join(base, `default.gbtheme`), path.join(gbaiPath, `default.gbtheme`), {
errorOnExist: false,
force: true,
recursive: true
});
Fs.cpSync(Path.join(base, `default.gbdata`), Path.join(gbaiPath, `default.gbdata`), {
fs.cpSync(path.join(base, `default.gbdata`), path.join(gbaiPath, `default.gbdata`), {
errorOnExist: false,
force: true,
recursive: true
});
Fs.cpSync(Path.join(base, `default.gbdialog`), Path.join(gbaiPath, `default.gbdialog`), {
fs.cpSync(path.join(base, `default.gbdialog`), path.join(gbaiPath, `default.gbdialog`), {
errorOnExist: false,
force: true,
recursive: true
});
Fs.cpSync(Path.join(base, `default.gbdrive`), Path.join(gbaiPath, `default.gbdrive`), {
fs.cpSync(path.join(base, `default.gbdrive`), path.join(gbaiPath, `default.gbdrive`), {
errorOnExist: false,
force: true,
recursive: true
@ -915,7 +915,7 @@ ENDPOINT_UPDATE=true
const virtualPath = '/' + min.botId;
let path = GBUtil.getGBAIPath(min.botId, null);
const gbaiRoot = Path.join(GBConfigService.get('STORAGE_LIBRARY'), path);
const gbaiRoot = path.join(GBConfigService.get('STORAGE_LIBRARY'), path);
server.setFileSystem(virtualPath, new webdav.PhysicalFileSystem(gbaiRoot), successed => {
GBLogEx.info(min.instance.instanceId, `WebDav online for ${min.botId}...`);

86
packages/core.gbapp/services/GBDeployer.ts
View file

@ -34,12 +34,12 @@
'use strict';
import Path from 'path';
import path from 'path';
import express from 'express';
import child_process from 'child_process';
import { rimraf } from 'rimraf';
import urlJoin from 'url-join';
import Fs from 'fs';
import fs from 'fs';
import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBDeployer, IGBInstance, IGBPackage } from 'botlib';
import { AzureSearch } from 'pragmatismo-io-framework';
import { CollectionUtil } from 'pragmatismo-io-framework';
@ -151,10 +151,10 @@ export class GBDeployer implements IGBDeployer {
async function scanPackageDirectory(path) {
// Gets all directories.
const isDirectory = source => Fs.lstatSync(source).isDirectory();
const isDirectory = source => fs.lstatSync(source).isDirectory();
const getDirectories = source =>
Fs.readdirSync(source)
.map(name => Path.join(source, name))
fs.readdirSync(source)
.map(name => path.join(source, name))
.filter(isDirectory);
const dirs = getDirectories(path);
await CollectionUtil.asyncForEach(dirs, async element => {
@ -163,7 +163,7 @@ export class GBDeployer implements IGBDeployer {
if (element === '.') {
GBLogEx.info(0, `Ignoring ${element}...`);
} else {
const name = Path.basename(element);
const name = path.basename(element);
// Skips what does not need to be loaded.
@ -423,7 +423,7 @@ export class GBDeployer implements IGBDeployer {
* Deploys a bot to the storage from a .gbot folder.
*/
public async deployBotFromLocalPath(localPath: string, publicAddress: string): Promise<void> {
const packageName = Path.basename(localPath);
const packageName = path.basename(localPath);
const instance = await this.importer.importIfNotExistsBotPackage(undefined, packageName, localPath);
await this.deployBotOnAzure(instance, publicAddress);
}
@ -433,15 +433,15 @@ export class GBDeployer implements IGBDeployer {
*/
public async loadParamsFromTabular(min: GBMinInstance, filePath: string): Promise<any> {
const xls = Path.join(filePath, 'Config.xlsx');
const csv = Path.join(filePath, 'config.csv');
const xls = path.join(filePath, 'Config.xlsx');
const csv = path.join(filePath, 'config.csv');
let rows: any[] = [];
let obj: any = {};
const workbook = new Excel.Workbook();
if (Fs.existsSync(xls)) {
if (fs.existsSync(xls)) {
await workbook.xlsx.readFile(xls);
let worksheet: any;
for (let t = 0; t < workbook.worksheets.length; t++) {
@ -456,7 +456,7 @@ export class GBDeployer implements IGBDeployer {
for (let index = 0; index < 6; index++) {
rows.shift();
}
} else if (Fs.existsSync(csv)) {
} else if (fs.existsSync(csv)) {
await workbook.csv.readFile(csv);
let worksheet = workbook.worksheets[0]; // Assuming the CSV file has only one sheet
rows = worksheet.getSheetValues();
@ -498,14 +498,14 @@ export class GBDeployer implements IGBDeployer {
// Creates each subfolder.
let pathBase = localPath;
if (!Fs.existsSync(pathBase)) {
Fs.mkdirSync(pathBase);
if (!fs.existsSync(pathBase)) {
fs.mkdirSync(pathBase);
}
await CollectionUtil.asyncForEach(parts, async item => {
pathBase = Path.join(pathBase, item);
if (!Fs.existsSync(pathBase)) {
Fs.mkdirSync(pathBase);
pathBase = path.join(pathBase, item);
if (!fs.existsSync(pathBase)) {
fs.mkdirSync(pathBase);
}
});
@ -527,19 +527,19 @@ export class GBDeployer implements IGBDeployer {
// Download files or navigate to directory to recurse.
await CollectionUtil.asyncForEach(documents, async item => {
const itemPath = Path.join(localPath, remotePath, item.name);
const itemPath = path.join(localPath, remotePath, item.name);
if (item.folder) {
if (!Fs.existsSync(itemPath)) {
Fs.mkdirSync(itemPath);
if (!fs.existsSync(itemPath)) {
fs.mkdirSync(itemPath);
}
const nextFolder = urlJoin(remotePath, item.name);
await this.downloadFolder(min, localPath, nextFolder);
} else {
let download = true;
if (Fs.existsSync(itemPath)) {
const dt = Fs.statSync(itemPath);
if (fs.existsSync(itemPath)) {
const dt = fs.statSync(itemPath);
if (new Date(dt.mtime) >= new Date(item.lastModifiedDateTime)) {
download = false;
}
@ -550,8 +550,8 @@ export class GBDeployer implements IGBDeployer {
const url = item['@microsoft.graph.downloadUrl'];
const response = await fetch(url);
Fs.writeFileSync(itemPath, Buffer.from(await response.arrayBuffer()), { encoding: null });
Fs.utimesSync(itemPath, new Date(), new Date(item.lastModifiedDateTime));
fs.writeFileSync(itemPath, Buffer.from(await response.arrayBuffer()), { encoding: null });
fs.utimesSync(itemPath, new Date(), new Date(item.lastModifiedDateTime));
} else {
GBLogEx.info(min, `Local is up to date: ${itemPath}...`);
}
@ -596,7 +596,7 @@ export class GBDeployer implements IGBDeployer {
* Deploys a folder into the bot storage.
*/
public async deployPackage2(min: GBMinInstance, user, localPath: string) {
const packageType = Path.extname(localPath);
const packageType = path.extname(localPath);
let handled = false;
let pck = null;
@ -659,7 +659,7 @@ export class GBDeployer implements IGBDeployer {
con['storagePassword'] = min.core.getParam<string>(min.instance, `${connectionName} Password`, null);
} else if (file) {
const path = GBUtil.getGBAIPath(min.botId, 'gbdata');
con['storageFile'] = Path.join(GBConfigService.get('STORAGE_LIBRARY'), path, file);
con['storageFile'] = path.join(GBConfigService.get('STORAGE_LIBRARY'), path, file);
} else {
GBLogEx.debug(min, `No storage information found for ${connectionName}, missing storage name or file.`);
}
@ -667,8 +667,8 @@ export class GBDeployer implements IGBDeployer {
});
const path = GBUtil.getGBAIPath(min.botId, null);
const localFolder = Path.join('work', path, 'connections.json');
Fs.writeFileSync(localFolder, JSON.stringify(connections), { encoding: null });
const localFolder = path.join('work', path, 'connections.json');
fs.writeFileSync(localFolder, JSON.stringify(connections), { encoding: null });
// Updates instance object.
@ -699,7 +699,7 @@ export class GBDeployer implements IGBDeployer {
case '.gbtheme':
// Updates server listeners to serve theme files in .gbtheme.
const packageName = Path.basename(localPath);
const packageName = path.basename(localPath);
GBServer.globals.server.use(`/themes/${packageName}`, express.static(localPath));
GBLogEx.verbose(min, `Theme (.gbtheme) assets accessible at: /themes/${packageName}.`);
@ -727,7 +727,7 @@ export class GBDeployer implements IGBDeployer {
*/
public async cleanupPackage(instance: IGBInstance, packageName: string) {
const path = GBUtil.getGBAIPath(instance.botId, null, packageName);
const localFolder = Path.join('work', path);
const localFolder = path.join('work', path);
rimraf.sync(localFolder);
}
@ -737,11 +737,11 @@ export class GBDeployer implements IGBDeployer {
public async undeployPackageFromPackageName(instance: IGBInstance, packageName: string) {
// Gets information about the package.
const packageType = Path.extname(packageName);
const p = await this.getStoragePackageByName(instance.instanceId, packageName);
const path = GBUtil.getGBAIPath(instance.botId, null, packageName);
const localFolder = Path.join('work', path);
const packagePath = GBUtil.getGBAIPath(instance.botId, null, packageName);
const localFolder = path.join('work', packagePath);
return await this.undeployPackageFromLocalPath(instance, localFolder);
}
@ -752,15 +752,15 @@ export class GBDeployer implements IGBDeployer {
public async undeployPackageFromLocalPath(instance: IGBInstance, localPath: string) {
// Gets information about the package.
const packageType = Path.extname(localPath);
const packageName = Path.basename(localPath);
const packageType = path.extname(localPath);
const packageName = path.basename(localPath);
const p = await this.getStoragePackageByName(instance.instanceId, packageName);
// Removes objects from storage, cloud resources and local files if any.
switch (packageType) {
case '.gbot':
const packageObject = JSON.parse(Fs.readFileSync(urlJoin(localPath, 'package.json'), 'utf8'));
const packageObject = JSON.parse(fs.readFileSync(urlJoin(localPath, 'package.json'), 'utf8'));
await this.undeployBot(packageObject.botId, packageName);
break;
@ -879,10 +879,10 @@ export class GBDeployer implements IGBDeployer {
// Checks if .gbapp compiliation is enabled.
if (!Fs.existsSync(`${root}/build`) && process.env.DISABLE_WEB !== 'true') {
if (!fs.existsSync(`${root}/build`) && process.env.DISABLE_WEB !== 'true') {
// Write a .env required to fix some bungs in create-react-app tool.
Fs.writeFileSync(`${root}/.env`, 'SKIP_PREFLIGHT_CHECK=true');
fs.writeFileSync(`${root}/.env`, 'SKIP_PREFLIGHT_CHECK=true');
// Install modules and compiles the web app.
@ -954,22 +954,22 @@ export class GBDeployer implements IGBDeployer {
) {
// Runs `npm install` for the package.
GBLogEx.info(0, `Deploying General Bots Application (.gbapp) or Library (.gblib): ${Path.basename(gbappPath)}...`);
let folder = Path.join(gbappPath, 'node_modules');
GBLogEx.info(0, `Deploying General Bots Application (.gbapp) or Library (.gblib): ${path.basename(gbappPath)}...`);
let folder = path.join(gbappPath, 'node_modules');
if (process.env.GBAPP_DISABLE_COMPILE !== 'true') {
if (!Fs.existsSync(folder)) {
if (!fs.existsSync(folder)) {
GBLogEx.info(0, `Installing modules for ${gbappPath}...`);
child_process.execSync('npm install', { cwd: gbappPath });
}
}
folder = Path.join(gbappPath, 'dist');
folder = path.join(gbappPath, 'dist');
try {
// Runs TSC in .gbapp folder.
if (process.env.GBAPP_DISABLE_COMPILE !== 'true') {
GBLogEx.info(0, `Compiling: ${gbappPath}.`);
child_process.execSync(Path.join(process.env.PWD, 'node_modules/.bin/tsc'), { cwd: gbappPath });
child_process.execSync(path.join(process.env.PWD, 'node_modules/.bin/tsc'), { cwd: gbappPath });
}
// After compiled, adds the .gbapp to the current server VM context.
@ -1035,7 +1035,7 @@ export class GBDeployer implements IGBDeployer {
let appPackagesProcessed = 0;
await CollectionUtil.asyncForEach(gbappPackages, async e => {
const filenameOnly = Path.basename(e);
const filenameOnly = path.basename(e);
// Skips .gbapp inside deploy folder.

6
packages/core.gbapp/services/GBImporterService.ts
View file

@ -36,7 +36,7 @@
import { GBMinInstance, IGBCoreService, IGBInstance } from 'botlib';
import { CreateOptions } from 'sequelize/types';
import Fs from 'fs';
import fs from 'fs';
import urlJoin from 'url-join';
import { GBServer } from '../../../src/app.js';
import { GuaribasInstance } from '../models/GBModel.js';
@ -61,9 +61,9 @@ export class GBImporter {
const file = urlJoin(localPath, 'settings.json');
let settingsJson = {botId: botId};
if (Fs.existsSync(file)){
if (fs.existsSync(file)){
settingsJson = JSON.parse(Fs.readFileSync(file, 'utf8'));
settingsJson = JSON.parse(fs.readFileSync(file, 'utf8'));
if (botId === undefined) {
botId = settingsJson.botId;
}

50
packages/core.gbapp/services/GBMinService.ts
View file

@ -40,7 +40,7 @@ import removeRoute from 'express-remove-route';
import AuthenticationContext from 'adal-node';
import { FacebookAdapter } from 'botbuilder-adapter-facebook';
import mkdirp from 'mkdirp';
import Fs from 'fs';
import fs from 'fs';
import arrayBufferToBuffer from 'arraybuffer-to-buffer';
import { NlpManager } from 'node-nlp';
import Koa from 'koa';
@ -85,7 +85,7 @@ import { GBDeployer } from './GBDeployer.js';
import urlJoin from 'url-join';
import { GoogleChatDirectLine } from '../../google-chat.gblib/services/GoogleChatDirectLine.js';
import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js';
import Path from 'path';
import path from 'path';
import { GBSSR } from './GBSSR.js';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
import { GBLogEx } from './GBLogEx.js';
@ -244,7 +244,7 @@ export class GBMinService {
// TODO: https://github.com/GeneralBots/BotServer/issues/321
const options = {
passphrase: process.env.CERTIFICATE2_PASSPHRASE,
pfx: Fs.readFileSync(process.env.CERTIFICATE2_PFX)
pfx: fs.readFileSync(process.env.CERTIFICATE2_PFX)
};
const domain = min.core.getParam(min.instance, 'Domain', null);
@ -294,19 +294,19 @@ export class GBMinService {
// Install per bot deployed packages.
let packagePath = urlJoin(`work`, GBUtil.getGBAIPath(min.botId, 'gbdialog'));
if (Fs.existsSync(packagePath)) {
if (fs.existsSync(packagePath)) {
await this.deployer['deployPackage2'](min, user, packagePath);
}
packagePath = urlJoin(`work`, GBUtil.getGBAIPath(min.botId, 'gbapp'));
if (Fs.existsSync(packagePath)) {
if (fs.existsSync(packagePath)) {
await this.deployer['deployPackage2'](min, user, packagePath);
}
packagePath = urlJoin(`work`, GBUtil.getGBAIPath(min.botId, 'gbtheme'));
if (Fs.existsSync(packagePath)) {
if (fs.existsSync(packagePath)) {
await this.deployer['deployPackage2'](min, user, packagePath);
}
packagePath = urlJoin(`work`, GBUtil.getGBAIPath(min.botId, `gblib`));
if (Fs.existsSync(packagePath)) {
if (fs.existsSync(packagePath)) {
await this.deployer['deployPackage2'](min, user, packagePath);
}
@ -314,39 +314,39 @@ export class GBMinService {
let dir = `work/${gbai}/cache`;
const botId = gbai.replace(/\.[^/.]+$/, '');
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
dir = `work/${gbai}/profile`;
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
dir = `work/${gbai}/uploads`;
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
dir = `work/${gbai}/${botId}.gbkb`;
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
dir = `work/${gbai}/${botId}.gbkb/docs-vectorized`;
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
dir = `work/${gbai}/${botId}.gbdialog`;
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
dir = `work/${gbai}/${botId}.gbot`;
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
dir = `work/${gbai}/${botId}.gbui`;
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
dir = `work/${gbai}/users`;
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
@ -388,10 +388,10 @@ export class GBMinService {
const manifest = `${instance.botId}-Teams.zip`;
const packageTeams = urlJoin(`work`, GBUtil.getGBAIPath(instance.botId), manifest);
if (!Fs.existsSync(packageTeams)) {
if (!fs.existsSync(packageTeams)) {
GBLogEx.info(min, 'Generating MS Teams manifest....');
const data = await this.deployer.getBotManifest(instance);
Fs.writeFileSync(packageTeams, data);
fs.writeFileSync(packageTeams, data);
}
// Serves individual URL for each bot user interface.
@ -833,7 +833,7 @@ export class GBMinService {
min['conversationWelcomed'] = {};
if (await min.core.getParam(min.instance, 'Answer Mode', null)) {
const gbkbPath = GBUtil.getGBAIPath(min.botId, 'gbkb');
min['vectorStorePath'] = Path.join('work', gbkbPath, 'docs-vectorized');
min['vectorStorePath'] = path.join('work', gbkbPath, 'docs-vectorized');
min['vectorStore'] = await this.deployer.loadOrCreateEmptyVectorStore(min);
}
min['apiConversations'] = {};
@ -1099,7 +1099,7 @@ export class GBMinService {
const folder = `work/${path}/cache`;
const filename = `${GBAdminService.generateUuid()}.png`;
Fs.writeFileSync(urlJoin(folder, filename), data);
fs.writeFileSync(urlJoin(folder, filename), data);
step.context.activity.text = urlJoin(
GBServer.globals.publicAddress,
`${min.instance.botId}`,
@ -1272,7 +1272,7 @@ export class GBMinService {
const url = attachment.contentUrl;
const localFolder = 'work';
const path = GBUtil.getGBAIPath(this['min'].botId);
const localFileName = Path.join(localFolder, path, 'uploads', attachment.name);
const localFileName = path.join(localFolder, path, 'uploads', attachment.name);
let buffer;
if (url.startsWith('data:')) {
@ -1287,7 +1287,7 @@ export class GBMinService {
buffer = arrayBufferToBuffer(await res.arrayBuffer());
}
Fs.writeFileSync(localFileName, buffer);
fs.writeFileSync(localFileName, buffer);
return {
fileName: attachment.name,
@ -1324,12 +1324,12 @@ export class GBMinService {
const t = new SystemKeywords();
GBLogEx.info(min, `BASIC (${min.botId}): Upload done for ${attachmentData.fileName}.`);
const handle = WebAutomationServices.cyrb53({ pid: 0, str: min.botId + attachmentData.fileName });
let data = Fs.readFileSync(attachmentData.localPath);
let data = fs.readFileSync(attachmentData.localPath);
const gbfile = {
filename: attachmentData.localPath,
data: data,
name: Path.basename(attachmentData.fileName)
name: path.basename(attachmentData.fileName)
};
GBServer.globals.files[handle] = gbfile;
@ -1360,7 +1360,7 @@ export class GBMinService {
const results = successfulSaves.reduce((accum: GBFile[], item) => {
const result: GBFile = {
data: Fs.readFileSync(successfulSaves[0]['localPath']),
data: fs.readFileSync(successfulSaves[0]['localPath']),
filename: successfulSaves[0]['fileName']
};
accum.push(result);

26
packages/core.gbapp/services/GBSSR.ts
View file

@ -37,8 +37,8 @@
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
import Path from 'path';
import Fs from 'fs';
import path from 'path';
import fs from 'fs';
import { NextFunction, Request, Response } from 'express';
import urljoin from 'url-join';
import { GBMinInstance } from 'botlib';
@ -106,11 +106,11 @@ export class GBSSR {
args.push(`--user-data-dir=${profilePath}`);
const preferences = urljoin(profilePath, 'Default', 'Preferences');
if (Fs.existsSync(preferences)) {
const file = Fs.readFileSync(preferences, 'utf8');
if (fs.existsSync(preferences)) {
const file = fs.readFileSync(preferences, 'utf8');
const data = JSON.parse(file);
data['profile']['exit_type'] = 'none';
Fs.writeFileSync(preferences, JSON.stringify(data));
fs.writeFileSync(preferences, JSON.stringify(data));
}
}
@ -307,25 +307,25 @@ export class GBSSR {
// Checks if the bot has an .gbui published or use default.gbui.
if (!Fs.existsSync(path)) {
if (!fs.existsSync(path)) {
path = GBUtil.getGBAIPath(minBoot.botId, `gbui`);
}
let parts = req.url.replace(`/${botId}`, '').split('?');
let url = parts[0];
if (min && req.originalUrl && prerender && exclude && Fs.existsSync(path)) {
if (min && req.originalUrl && prerender && exclude && fs.existsSync(path)) {
// Reads from static HTML when a bot is crawling.
path = Path.join(process.env.PWD, 'work', path, 'index.html');
const html = Fs.readFileSync(path, 'utf8');
path = path.join(process.env.PWD, 'work', path, 'index.html');
const html = fs.readFileSync(path, 'utf8');
res.status(200).send(html);
return true;
} else {
// Servers default.gbui web application.
path = Path.join(
path = path.join(
process.env.PWD,
GBDeployer.deployFolder,
GBMinService.uiPackage,
@ -336,11 +336,11 @@ export class GBSSR {
path = GBServer.globals.wwwroot + "/index.html"; // TODO.
}
if (!min && !url.startsWith("/static") && GBServer.globals.wwwroot) {
path = Path.join(GBServer.globals.wwwroot, url);
path = path.join(GBServer.globals.wwwroot, url);
}
if (Fs.existsSync(path)) {
if (fs.existsSync(path)) {
if (min) {
let html = Fs.readFileSync(path, 'utf8');
let html = fs.readFileSync(path, 'utf8');
html = html.replace(/\{p\}/gi, min.botId);
html = html.replace(/\{botId\}/gi, min.botId);
html = html.replace(/\{theme\}/gi, min.instance.theme ? min.instance.theme :

2
packages/google-chat.gblib/services/GoogleChatDirectLine.ts
View file

@ -31,7 +31,7 @@
import Swagger from 'swagger-client';
import { google } from 'googleapis';
import { PubSub } from '@google-cloud/pubsub';
import Fs from 'fs';
import fs from 'fs';
import { GBLog, GBMinInstance, GBService } from 'botlib';
import { GBServer } from '../../../src/app.js';
import { SecService } from '../../security.gbapp/services/SecService.js';

4
packages/kb.gbapp/dialogs/AskDialog.ts
View file

@ -49,7 +49,7 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import urlJoin from 'url-join';
import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
import Path from 'path';
import path from 'path';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { GBUtil } from '../../../src/util.js';
@ -247,7 +247,7 @@ export class AskDialog extends IGBDialog {
for (const key in results.sources) {
const source = results.sources[key];
const path = GBUtil.getGBAIPath(min.botId, `gbkb`);
let url = urlJoin('kb', path, 'docs', Path.basename(source.file));
let url = urlJoin('kb', path, 'docs', path.basename(source.file));
url = `${url}#page=${source.page}&toolbar=0&messages=0&statusbar=0&navpanes=0`;
urls.push({ url: url });
}

58
packages/kb.gbapp/services/KBService.ts
View file

@ -31,10 +31,8 @@
/**
* @fileoverview Knowledge base services and logic.
*/
import html2md from 'html-to-md';
import Path from 'path';
import Fs from 'fs';
import path from 'path';
import fs from 'fs';
import urlJoin from 'url-join';
import asyncPromise from 'async-promises';
import walkPromise from 'walk-promise';
@ -48,7 +46,7 @@ import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf';
import { DocxLoader } from '@langchain/community/document_loaders/fs/docx';
import { EPubLoader } from '@langchain/community/document_loaders/fs/epub';
import { CSVLoader } from '@langchain/community/document_loaders/fs/csv';
import path from 'path';
import puppeteer, { Page } from 'puppeteer';
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
import { Document } from 'langchain/document';
@ -503,7 +501,7 @@ export class KBService implements IGBKBService {
'Existe um problema na base de conhecimento. Fui treinado para entender sua pergunta, avise a quem me criou que a resposta não foi informada para esta pergunta.';
} else if (answer.indexOf('.md') > -1 || answer.indexOf('.docx') > -1) {
const mediaFilename = urlJoin(path.dirname(filePath), '..', 'articles', answer);
if (Fs.existsSync(mediaFilename)) {
if (fs.existsSync(mediaFilename)) {
// Tries to load .docx file from Articles folder.
if (answer.indexOf('.docx') > -1) {
@ -511,7 +509,7 @@ export class KBService implements IGBKBService {
} else {
// Loads normally markdown file.
answer = Fs.readFileSync(mediaFilename, 'utf8');
answer = fs.readFileSync(mediaFilename, 'utf8');
}
format = '.md';
media = path.basename(mediaFilename);
@ -559,12 +557,12 @@ export class KBService implements IGBKBService {
const code = isBasic ? answer.substr(6) : answer;
const path = GBUtil.getGBAIPath(min.botId, `gbdialog`);
const scriptName = `tmp${GBAdminService.getRndReadableIdentifier()}.docx`;
const localName = Path.join('work', path, `${scriptName}`);
Fs.writeFileSync(localName, code, { encoding: null });
const localName = path.join('work', path, `${scriptName}`);
fs.writeFileSync(localName, code, { encoding: null });
answer = scriptName;
const vm = new GBVMService();
await vm.loadDialog(Path.basename(localName), Path.dirname(localName), min);
await vm.loadDialog(path.basename(localName), path.dirname(localName), min);
}
// Now with all the data ready, creates entities in the store.
@ -692,7 +690,7 @@ export class KBService implements IGBKBService {
// Imports menu.xlsx if any.
if (Fs.existsSync(subjectFile) || Fs.existsSync(menuFile)) {
if (fs.existsSync(subjectFile) || fs.existsSync(menuFile)) {
await this.importSubjectFile(packageStorage.packageId, subjectFile, menuFile, instance);
}
@ -727,7 +725,7 @@ export class KBService implements IGBKBService {
if (content === null) {
const fullFilename = urlJoin(file.root, file.name);
content = Fs.readFileSync(fullFilename, 'utf-8');
content = fs.readFileSync(fullFilename, 'utf-8');
await GuaribasAnswer.create(<GuaribasAnswer>{
instanceId: instance.instanceId,
@ -740,7 +738,7 @@ export class KBService implements IGBKBService {
}
} else if (file !== null && file.name.endsWith('.docx')) {
let path = GBUtil.getGBAIPath(instance.botId, `gbkb`);
const localName = Path.join('work', path, 'articles', file.name);
const localName = path.join('work', path, 'articles', file.name);
let loader = new DocxLoader(localName);
let doc = await loader.load();
let content = doc[0].pageContent;
@ -761,12 +759,12 @@ export class KBService implements IGBKBService {
data.answers.push(answer);
} else if (file !== null && file.name.endsWith('.toc.docx')) {
const path = GBUtil.getGBAIPath(instance.botId, `gbkb`);
const localName = Path.join('work', path, 'articles', file.name);
const buffer = Fs.readFileSync(localName, { encoding: null });
const localName = path.join('work', path, 'articles', file.name);
const buffer = fs.readFileSync(localName, { encoding: null });
var options = {
buffer: buffer,
convertImage: async image => {
const localName = Path.join(
const localName = path.join(
'work',
GBUtil.getGBAIPath(instance.botId),
'cache',
@ -776,10 +774,10 @@ export class KBService implements IGBKBService {
GBServer.globals.publicAddress,
GBUtil.getGBAIPath(instance.botId).replace(/\.[^/.]+$/, ''),
'cache',
Path.basename(localName)
path.basename(localName)
);
const buffer = await image.read();
Fs.writeFileSync(localName, buffer, { encoding: null });
fs.writeFileSync(localName, buffer, { encoding: null });
return { src: url };
}
};
@ -1026,8 +1024,8 @@ export class KBService implements IGBKBService {
website.endsWith('/') ? website.substring(0, website.length - 1) : website;
let path = GBUtil.getGBAIPath(min.botId, `gbot`);
const directoryPath = Path.join(process.env.PWD, 'work', path, 'Website');
Fs.rmSync(directoryPath, { recursive: true, force: true });
const directoryPath = path.join(process.env.PWD, 'work', path, 'Website');
fs.rmSync(directoryPath, { recursive: true, force: true });
let browser = await puppeteer.launch({ headless: false });
const page = await this.getFreshPage(browser, website);
@ -1042,7 +1040,7 @@ export class KBService implements IGBKBService {
try {
const logoBinary = await page.goto(logo);
const buffer = await logoBinary.buffer();
const logoFilename = Path.basename(logo);
const logoFilename = path.basename(logo);
// TODO: sharp(buffer)
// .resize({
// width: 48,
@ -1050,7 +1048,7 @@ export class KBService implements IGBKBService {
// fit: 'inside', // Resize the image to fit within the specified dimensions
// withoutEnlargement: true // Don't enlarge the image if its dimensions are already smaller
// })
// .toFile(Path.join(logoPath, logoFilename));
// .toFile(path.join(logoPath, logoFilename));
await min.core['setConfig'](min, 'Logo', logoFilename);
} catch (error) {
GBLogEx.debug(min, error);
@ -1106,7 +1104,7 @@ export class KBService implements IGBKBService {
} else {
await CollectionUtil.asyncForEach(files, async file => {
let content = null;
let filePath = Path.join(file.root, file.name);
let filePath = path.join(file.root, file.name);
const document = await this.loadAndSplitFile(filePath);
const flattenedDocuments = document.reduce((acc, val) => acc.concat(val), []);
@ -1191,7 +1189,7 @@ export class KBService implements IGBKBService {
instance: IGBInstance
): Promise<any> {
let subjectsLoaded;
if (Fs.existsSync(menuFile)) {
if (fs.existsSync(menuFile)) {
// Loads menu.xlsx and finds worksheet.
const workbook = new Excel.Workbook();
@ -1260,7 +1258,7 @@ export class KBService implements IGBKBService {
subjectsLoaded = subjects;
} else {
subjectsLoaded = JSON.parse(Fs.readFileSync(filename, 'utf8'));
subjectsLoaded = JSON.parse(fs.readFileSync(filename, 'utf8'));
}
const doIt = async (subjects: GuaribasSubject[], parentSubjectId: number) => {
@ -1335,7 +1333,7 @@ export class KBService implements IGBKBService {
* @param localPath Path to the .gbkb folder.
*/
public async deployKb(core: IGBCoreService, deployer: GBDeployer, localPath: string, min: GBMinInstance) {
const packageName = Path.basename(localPath);
const packageName = path.basename(localPath);
const instance = await core.loadInstanceByBotId(min.botId);
GBLogEx.info(min, `[GBDeployer] Importing: ${localPath}`);
@ -1353,10 +1351,10 @@ export class KBService implements IGBKBService {
GBLogEx.info(min, `[GBDeployer] Start Bot Server Side Rendering... ${localPath}`);
const html = await GBSSR.getHTML(min);
let path = GBUtil.getGBAIPath(min.botId, `gbui`);
path = Path.join(process.env.PWD, 'work', path, 'index.html');
GBLogEx.info(min, `[GBDeployer] Saving SSR HTML in ${path}.`);
Fs.writeFileSync(path, html, 'utf8');
let packagePath = GBUtil.getGBAIPath(min.botId, `gbui`);
packagePath = path.join(process.env.PWD, 'work', packagePath, 'index.html');
GBLogEx.info(min, `[GBDeployer] Saving SSR HTML in ${packagePath}.`);
fs.writeFileSync(packagePath, html, 'utf8');
GBLogEx.info(min, `[GBDeployer] Finished import of ${localPath}`);
}

22
packages/llm.gblib/services/ChatServices.ts
View file

@ -51,10 +51,10 @@ import { SqlDatabaseChain } from 'langchain/chains/sql_db';
import { SqlDatabase } from 'langchain/sql_db';
import { DataSource } from 'typeorm';
import { GBMinInstance } from 'botlib';
import * as Fs from 'fs';
import fs from 'fs';
import { jsonSchemaToZod } from 'json-schema-to-zod';
import { BufferWindowMemory } from 'langchain/memory';
import Path from 'path';
import path from 'path';
import { PngPageOutput, pdfToPng } from 'pdf-to-png-converter';
import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
import { CollectionUtil } from 'pragmatismo-io-framework';
@ -147,7 +147,7 @@ export class GBLLMOutputParser extends BaseLLMOutputParser<ExpectedOutput> {
let found = false;
if (source && source.file.endsWith('.pdf')) {
const gbaiName = GBUtil.getGBAIPath(this.min.botId, 'gbkb');
const localName = Path.join(process.env.PWD, 'work', gbaiName, 'docs', source.file);
const localName = path.join(process.env.PWD, 'work', gbaiName, 'docs', source.file);
if (localName) {
const { url } = await ChatServices.pdfPageAsImage(this.min, localName, source.page);
@ -186,9 +186,9 @@ export class ChatServices {
if (pngPages.length > 0) {
const buffer = pngPages[0].content;
const gbaiName = GBUtil.getGBAIPath(min.botId, null);
const localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`);
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName));
Fs.writeFileSync(localName, buffer, { encoding: null });
const localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`);
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
fs.writeFileSync(localName, buffer, { encoding: null });
return { localName: localName, url: url, data: buffer };
}
}
@ -216,7 +216,7 @@ export class ChatServices {
for (const filePaths of Object.keys(uniqueDocuments)) {
const doc = uniqueDocuments[filePaths];
const metadata = doc.metadata;
const filename = Path.basename(metadata.source);
const filename = path.basename(metadata.source);
let page = 0;
if (metadata.source.endsWith('.pdf')) {
page = await ChatServices.findPageForText(metadata.source, doc.pageContent);
@ -234,7 +234,7 @@ export class ChatServices {
}
private static async findPageForText(pdfPath, searchText) {
const data = new Uint8Array(Fs.readFileSync(pdfPath));
const data = new Uint8Array(fs.readFileSync(pdfPath));
const pdf = await getDocument({ data }).promise;
searchText = searchText.replace(/\s/g, '');
@ -709,10 +709,10 @@ export class ChatServices {
// Adds .gbdialog as functions if any to LLM Functions.
await CollectionUtil.asyncForEach(Object.keys(min.scriptMap), async script => {
const path = GBUtil.getGBAIPath(min.botId, 'gbdialog', null);
const jsonFile = Path.join('work', path, `${script}.json`);
const jsonFile = path.join('work', path, `${script}.json`);
if (Fs.existsSync(jsonFile) && script.toLowerCase() !== 'start.vbs') {
const funcJSON = JSON.parse(Fs.readFileSync(jsonFile, 'utf8'));
if (fs.existsSync(jsonFile) && script.toLowerCase() !== 'start.vbs') {
const funcJSON = JSON.parse(fs.readFileSync(jsonFile, 'utf8'));
const funcObj = funcJSON?.function;
if (funcObj) {

8
packages/llm.gblib/services/ImageServices.ts
View file

@ -35,9 +35,9 @@ import OpenAI from 'openai';
import { AzureKeyCredential } from '@azure/core-auth';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords';
import Path from 'path';
import path from 'path';
import { GBServer } from '../../../src/app.js';
import Fs from 'fs';
import fs from 'fs';
import urlJoin from 'url-join';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx';
@ -68,12 +68,12 @@ export class ImageServices {
});
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = Path.join('work', gbaiName, 'cache', `DALL-E${GBAdminService.getRndReadableIdentifier()}.png`);
const localName = path.join('work', gbaiName, 'cache', `DALL-E${GBAdminService.getRndReadableIdentifier()}.png`);
const url = response.data[0].url;
const res = await fetch(url);
let buf: any = Buffer.from(await res.arrayBuffer());
Fs.writeFileSync(localName, buf, { encoding: null });
fs.writeFileSync(localName, buf, { encoding: null });
GBLogEx.info(min, `DALL-E image generated at ${url}.`);

8
packages/security.gbapp/services/SecService.ts
View file

@ -4,7 +4,7 @@ import { CollectionUtil } from 'pragmatismo-io-framework';
import { GuaribasUser } from '../models/index.js';
import { FindOptions } from 'sequelize';
import { DialogKeywords } from '../../../packages/basic.gblib/services/DialogKeywords.js';
import * as Fs from 'fs';
import fs from 'fs';
import mkdirp from 'mkdirp';
import urlJoin from 'url-join';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
@ -29,7 +29,7 @@ export class SecService extends GBService {
const gbaiPath = GBUtil.getGBAIPath(min.botId);
const dir = urlJoin ('work',gbaiPath, 'users', userSystemId);
if (!Fs.existsSync(dir)) {
if (!fs.existsSync(dir)) {
mkdirp.sync(dir);
}
@ -44,8 +44,8 @@ export class SecService extends GBService {
}
const systemPromptFile = urlJoin(dir, 'systemPrompt.txt');
if (Fs.existsSync(systemPromptFile)) {
user[ 'systemPrompt'] = Fs.readFileSync(systemPromptFile);
if (fs.existsSync(systemPromptFile)) {
user[ 'systemPrompt'] = fs.readFileSync(systemPromptFile);
}
user.instanceId = min.instance.instanceId;

4
packages/teams.gblib/services/TeamsService.ts
View file

@ -29,7 +29,7 @@
\*****************************************************************************/
import { GBService } from 'botlib';
import Fs from 'fs';
import fs from 'fs';
import AdmZip from 'adm-zip';
/**
@ -45,7 +45,7 @@ export class TeamsService extends GBService {
}
public async getManifest(marketplaceId, botName, botDescription, id, packageName, yourName) {
let content = Fs.readFileSync('teams-manifest.json', 'utf8');
let content = fs.readFileSync('teams-manifest.json', 'utf8');
content = content.replace(/\@\@marketplaceId/gi, marketplaceId);
content = content.replace(/\@\@botName/gi, botName);

26
packages/whatsapp.gblib/services/WhatsappDirectLine.ts
View file

@ -30,8 +30,8 @@
import mime from 'mime-types';
import urlJoin from 'url-join';
import Path from 'path';
import Fs from 'fs';
import path from 'path';
import fs from 'fs';
import { GBLog, GBMinInstance, GBService, IGBPackage } from 'botlib';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBServer } from '../../../src/app.js';
@ -56,7 +56,7 @@ import { GBVMService } from '../../basic.gblib/services/GBVMService.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { createBot } from 'whatsapp-cloud-api';
import { promisify } from 'util';
const stat = promisify(Fs.stat);
const stat = promisify(fs.stat);
/**
* Support for Whatsapp.
@ -152,7 +152,7 @@ export class WhatsappDirectLine extends GBService {
// Initialize the browser using a local profile for each bot.
const gbaiPath = GBUtil.getGBAIPath(this.min.botId);
const webVersion = '2.2412.51';
const localName = Path.join('work', gbaiPath, 'profile');
const localName = path.join('work', gbaiPath, 'profile');
const createClient = () => {
const client = (this.customClient = new Client({
puppeteer: GBSSR.preparePuppeteer(localName),
@ -181,20 +181,20 @@ export class WhatsappDirectLine extends GBService {
const s = new DialogKeywords();
const qrBuf = await s.getQRCode({ pid, text: qr });
const localName = Path.join(
const localName = path.join(
'work',
gbaiPath,
'cache',
`qr${GBAdminService.getRndReadableIdentifier()}.png`
);
Fs.writeFileSync(localName, qrBuf.data);
const url = urlJoin(GBServer.globals.publicAddress, this.min.botId, 'cache', Path.basename(localName));
fs.writeFileSync(localName, qrBuf.data);
const url = urlJoin(GBServer.globals.publicAddress, this.min.botId, 'cache', path.basename(localName));
if (adminNumber) {
await GBServer.globals.minBoot.whatsAppDirectLine.sendFileToDevice(
adminNumber,
url,
Path.basename(localName),
path.basename(localName),
msg
);
}
@ -324,14 +324,14 @@ export class WhatsappDirectLine extends GBService {
let buf: any = Buffer.from(base64Image.data, 'base64');
const gbaiName = GBUtil.getGBAIPath(this.min.botId);
const localName = Path.join(
const localName = path.join(
'work',
gbaiName,
'cache',
`tmp${GBAdminService.getRndReadableIdentifier()}.docx`
);
Fs.writeFileSync(localName, buf, { encoding: null });
const url = urlJoin(GBServer.globals.publicAddress, this.min.botId, 'cache', Path.basename(localName));
fs.writeFileSync(localName, buf, { encoding: null });
const url = urlJoin(GBServer.globals.publicAddress, this.min.botId, 'cache', path.basename(localName));
attachments = [];
attachments.push({
@ -764,7 +764,7 @@ export class WhatsappDirectLine extends GBService {
// Set folder based on media type
let folder = mediaType === 'video' ? 'videos' : 'images';
let path = GBUtil.getGBAIPath(min.botId, `gbkb`);
path = Path.join(process.env.PWD, 'work', path, folder, mediaFile);
path = path.join(process.env.PWD, 'work', path, folder, mediaFile);
text = text.substring(mediaFile.length + 1).trim();
text = text.replace(/\n/g, '\\n');
@ -1231,7 +1231,7 @@ export class WhatsappDirectLine extends GBService {
while (startOffset < fileSize) {
const endOffset = Math.min(startOffset + CHUNK_SIZE, fileSize);
const fileStream = Fs.createReadStream(filePath, { start: startOffset, end: endOffset - 1 });
const fileStream = fs.createReadStream(filePath, { start: startOffset, end: endOffset - 1 });
const chunkSize = endOffset - startOffset;
const uploadResponse = await fetch(`https://graph.facebook.com/v20.0/upload:${uploadSessionId}`, {

2
src/app.ts
View file

@ -139,7 +139,7 @@ export class GBServer {
// Creates working directory.
process.env.PWD = process.cwd();
const workDir = Path.join(process.env.PWD, 'work');
const workDir = path.join(process.env.PWD, 'work');
if (!fs.existsSync(workDir)) {
mkdirp.sync(workDir);
}

28
src/util.ts
View file

@ -35,7 +35,7 @@
'use strict';
import * as YAML from 'yaml';
import SwaggerClient from 'swagger-client';
import Fs from 'fs';
import fs from 'fs';
import { GBConfigService } from '../packages/core.gbapp/services/GBConfigService.js';
import path from 'path';
import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
@ -74,7 +74,7 @@ export class GBUtil {
public static async getDirectLineClient(min) {
let config = {
spec: JSON.parse(Fs.readFileSync('directline-3.0.json', 'utf8')),
spec: JSON.parse(fs.readFileSync('directline-3.0.json', 'utf8')),
requestInterceptor: req => {
req.headers['Authorization'] = `Bearer ${min.instance.webchatKey}`;
}
@ -131,20 +131,20 @@ export class GBUtil {
}
public static copyIfNewerRecursive(src, dest) {
if (!Fs.existsSync(src)) {
if (!fs.existsSync(src)) {
console.error(`Source path "${src}" does not exist.`);
return;
}
// Check if the source is a directory
if (Fs.statSync(src).isDirectory()) {
if (fs.statSync(src).isDirectory()) {
// Create the destination directory if it doesn't exist
if (!Fs.existsSync(dest)) {
Fs.mkdirSync(dest, { recursive: true });
if (!fs.existsSync(dest)) {
fs.mkdirSync(dest, { recursive: true });
}
// Read all files and directories in the source directory
const entries = Fs.readdirSync(src);
const entries = fs.readdirSync(src);
for (let entry of entries) {
const srcEntry = path.join(src, entry);
@ -155,17 +155,17 @@ export class GBUtil {
}
} else {
// Source is a file, check if we need to copy it
if (Fs.existsSync(dest)) {
const srcStat = Fs.statSync(src);
const destStat = Fs.statSync(dest);
if (fs.existsSync(dest)) {
const srcStat = fs.statSync(src);
const destStat = fs.statSync(dest);
// Copy only if the source file is newer than the destination file
if (srcStat.mtime > destStat.mtime) {
Fs.cpSync(src, dest, { force: true });
fs.cpSync(src, dest, { force: true });
}
} else {
// Destination file doesn't exist, so copy it
Fs.cpSync(src, dest, { force: true });
fs.cpSync(src, dest, { force: true });
}
}
}
@ -227,7 +227,7 @@ export class GBUtil {
const urlPath = urlObj.pathname.endsWith('/') ? urlObj.pathname.slice(0, -1) : urlObj.pathname;
let filename = urlPath.split('/').pop() || 'index';
Fs.mkdirSync(directoryPath, { recursive: true });
fs.mkdirSync(directoryPath, { recursive: true });
const extensionMap = {
'text/html': 'html',
@ -258,7 +258,7 @@ export class GBUtil {
fileContent = await response.buffer();
}
Fs.writeFileSync(filePath, fileContent);
fs.writeFileSync(filePath, fileContent);
return filePath;
}