From fb348599cf78199d4c802a3ef2526afc57cbb2db Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Sat, 7 Sep 2024 18:13:36 -0300 Subject: [PATCH] fix (all): path and fs normalization. --- boot.mjs | 28 +-- .../admin.gbapp/services/GBAdminService.ts | 6 +- .../dialogs/StartDialog.ts | 5 +- .../services/AzureDeployerService.ts | 4 +- .../basic.gblib/services/DebuggerService.ts | 2 +- .../basic.gblib/services/DialogKeywords.ts | 10 +- packages/basic.gblib/services/GBVMService.ts | 61 +++---- .../basic.gblib/services/SystemKeywords.ts | 95 +++++----- .../services/WebAutomationServices.ts | 4 +- .../services/GBConversationalService.ts | 15 +- packages/core.gbapp/services/GBCoreService.ts | 28 +-- packages/core.gbapp/services/GBDeployer.ts | 44 ++--- .../core.gbapp/services/GBImporterService.ts | 7 +- packages/core.gbapp/services/GBMinService.ts | 44 ++--- packages/core.gbapp/services/GBSSR.ts | 22 +-- .../services/GoogleChatDirectLine.ts | 2 +- packages/kb.gbapp/services/KBService.ts | 27 ++- packages/llm.gblib/services/ChatServices.ts | 10 +- packages/llm.gblib/services/ImageServices.ts | 4 +- .../saas.gbapp/service/JunoSubscription.ts | 4 +- .../security.gbapp/services/SecService.ts | 8 +- packages/teams.gblib/services/TeamsService.ts | 4 +- .../services/WhatsappDirectLine.ts | 15 +- src/app.ts | 18 +- src/util.ts | 167 ++++++++++-------- 25 files changed, 331 insertions(+), 303 deletions(-) diff --git a/boot.mjs b/boot.mjs index 28a39f58..9a74f96c 100644 --- a/boot.mjs +++ b/boot.mjs @@ -2,12 +2,12 @@ process.stdout.write(`General Bots. BotServer@${pjson.version}, botlib@${pjson.dependencies.botlib}, node@${process.version.replace('v', '')}, ${process.platform} ${process.arch} `); -import fs from 'fs'; +import fs from 'fs/promises'; import os from 'node:os'; import path from 'path'; import { exec } from 'child_process'; import pjson from './package.json' assert { type: 'json' }; - +import {GBUtil} from './dist/src/util.js' // Displays version of Node JS being used at runtime and others attributes. @@ -15,42 +15,42 @@ console.log(`\nLoading virtual machine source code files...`); var __dirname = process.env.PWD || process.cwd(); try { - var run = () => { + var run = async () => { - import('./dist/src/app.js').then((gb)=> { - gb.GBServer.run() + import('./dist/src/app.js').then(async (gb)=> { + await gb.GBServer.run() }); }; - var processDist = () => { - if (!fs.existsSync('dist')) { + var processDist = async () => { + if (!await GBUtil.exists('dist')) { console.log(`\n`); console.log(`Generall Bots: Compiling...`); - exec(path.join(__dirname, 'node_modules/.bin/tsc'), (err, stdout, stderr) => { + exec(path.join(__dirname, 'node_modules/.bin/tsc'), async (err, stdout, stderr) => { if (err) { console.error(err); return; } - run(); + await run(); }); } else { - run(); + await run(); } }; // Installing modules if it has not been done yet. - if (!fs.existsSync('node_modules')) { + if (!await GBUtil.exists('node_modules')) { console.log(`\n`); console.log(`Generall Bots: Installing modules for the first time, please wait...`); - exec('npm install', (err, stdout, stderr) => { + exec('npm install', async (err, stdout, stderr) => { if (err) { console.error(err); return; } - processDist(); + await processDist(); }); } else { - processDist(); + await processDist(); } } catch (e) { console.log(e); diff --git a/packages/admin.gbapp/services/GBAdminService.ts b/packages/admin.gbapp/services/GBAdminService.ts index 33bf24bd..7a85c61b 100644 --- a/packages/admin.gbapp/services/GBAdminService.ts +++ b/packages/admin.gbapp/services/GBAdminService.ts @@ -49,7 +49,7 @@ import msRestAzure from 'ms-rest-azure'; import path from 'path'; import { caseSensitive_Numbs_SpecialCharacters_PW, lowercase_PW } from 'super-strong-password-generator'; import crypto from 'crypto'; -import fs from 'fs'; +import fs from 'fs/promises'; import { GBServer } from '../../../src/app.js'; import { GuaribasUser } from '../../security.gbapp/models/index.js'; import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js'; @@ -75,9 +75,9 @@ export class GBAdminService implements IGBAdminService { return crypto.randomUUID(); } - public static getNodeVersion() { + public static async getNodeVersion() { const packageJson = urlJoin(process.cwd(), 'package.json'); - const pkg = JSON.parse(fs.readFileSync(packageJson, 'utf8')); + const pkg = JSON.parse(await fs.readFile(packageJson, 'utf8')); return pkg.engines.node.replace('=', ''); } diff --git a/packages/azuredeployer.gbapp/dialogs/StartDialog.ts b/packages/azuredeployer.gbapp/dialogs/StartDialog.ts index fd2d8500..4a184d4d 100644 --- a/packages/azuredeployer.gbapp/dialogs/StartDialog.ts +++ b/packages/azuredeployer.gbapp/dialogs/StartDialog.ts @@ -35,12 +35,13 @@ 'use strict'; import { GBLog, IGBInstallationDeployer, IGBInstance } from 'botlib'; -import fs from 'fs'; +import fs from 'fs/promises'; import { GBAdminService } from '../../../packages/admin.gbapp/services/GBAdminService.js'; import { GBConfigService } from '../../../packages/core.gbapp/services/GBConfigService.js'; import scanf from 'scanf'; import { AzureDeployerService } from '../services/AzureDeployerService.js'; import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js'; +import { GBUtil } from '../../../src/util.js'; /** * Handles command-line dialog for getting info for Boot Bot. @@ -49,7 +50,7 @@ export class StartDialog { public static async createBaseInstance (deployer, freeTier) { // No .env so asks for cloud credentials to start a new farm. - if (!fs.existsSync(`.env`)) { + if (!await GBUtil.exists(`.env`)) { process.stdout.write( 'A empty enviroment is detected. To start automatic deploy, please enter some information:\n' ); diff --git a/packages/azuredeployer.gbapp/services/AzureDeployerService.ts b/packages/azuredeployer.gbapp/services/AzureDeployerService.ts index db8103a1..84e2c4a0 100644 --- a/packages/azuredeployer.gbapp/services/AzureDeployerService.ts +++ b/packages/azuredeployer.gbapp/services/AzureDeployerService.ts @@ -936,7 +936,7 @@ export class AzureDeployerService implements IGBInstallationDeployer { serverFarmId: farmId, siteConfig: { - nodeVersion: GBAdminService.getNodeVersion(), + nodeVersion: await GBAdminService.getNodeVersion(), detailedErrorLoggingEnabled: true, requestTracingEnabled: true } @@ -985,7 +985,7 @@ export class AzureDeployerService implements IGBInstallationDeployer { siteConfig: { appSettings: [ { name: 'WEBSITES_CONTAINER_START_TIME_LIMIT', value: `${WebSiteResponseTimeout}` }, - { name: 'WEBSITE_NODE_DEFAULT_VERSION', value: GBAdminService.getNodeVersion() }, + { name: 'WEBSITE_NODE_DEFAULT_VERSION', value: await GBAdminService.getNodeVersion() }, { name: 'ADMIN_PASS', value: `${instance.adminPass}` }, { name: 'BOT_ID', value: `${instance.botId}` }, { name: 'CLOUD_SUBSCRIPTIONID', value: `${instance.cloudSubscriptionId}` }, diff --git a/packages/basic.gblib/services/DebuggerService.ts b/packages/basic.gblib/services/DebuggerService.ts index 354c688b..e0925cd2 100644 --- a/packages/basic.gblib/services/DebuggerService.ts +++ b/packages/basic.gblib/services/DebuggerService.ts @@ -32,7 +32,7 @@ import { GBLog, GBMinInstance } from 'botlib'; import { GBServer } from '../../../src/app.js'; -import fs from 'fs'; +import fs from 'fs/promises'; import SwaggerClient from 'swagger-client'; import { spawn } from 'child_process'; import { CodeServices } from '../../llm.gblib/services/CodeServices.js'; diff --git a/packages/basic.gblib/services/DialogKeywords.ts b/packages/basic.gblib/services/DialogKeywords.ts index dc016a71..3e7bc200 100644 --- a/packages/basic.gblib/services/DialogKeywords.ts +++ b/packages/basic.gblib/services/DialogKeywords.ts @@ -42,7 +42,7 @@ import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; import { Messages } from '../strings.js'; import { CollectionUtil } from 'pragmatismo-io-framework'; import { GBConversationalService } from '../../core.gbapp/services/GBConversationalService.js'; -import fs from 'fs'; +import fs from 'fs/promises'; import libphonenumber from 'google-libphonenumber'; import * as df from 'date-diff'; import tesseract from 'node-tesseract-ocr'; @@ -1500,7 +1500,7 @@ export class DialogKeywords { 'cache', `${fileOnly.replace(/\s/gi, '')}-${GBAdminService.getNumberIdentifier()}.${ext}` ); - fs.writeFileSync(localName1, buf, { encoding: null }); + fs.writeFile(localName1, buf, { encoding: null }); url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName1)); } @@ -1510,10 +1510,10 @@ export class DialogKeywords { // Prepare a cache to be referenced by Bot Framework. - const buf = fs.readFileSync(filename); + const buf = await fs.readFile(filename); const gbaiName = GBUtil.getGBAIPath(min.botId); const localName = path.join('work', gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.${ext}`); - fs.writeFileSync(localName, buf, { encoding: null }); + fs.writeFile(localName, buf, { encoding: null }); url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName)); } @@ -1546,7 +1546,7 @@ export class DialogKeywords { const gbaiName = GBUtil.getGBAIPath(min.botId); const localName = path.join('work', gbaiName, 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png`); - fs.writeFileSync(localName, buf, { encoding: null }); + fs.writeFile(localName, buf, { encoding: null }); const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName)); return { data: data, localName: localName, url: url }; diff --git a/packages/basic.gblib/services/GBVMService.ts b/packages/basic.gblib/services/GBVMService.ts index e7b35946..e01b4363 100644 --- a/packages/basic.gblib/services/GBVMService.ts +++ b/packages/basic.gblib/services/GBVMService.ts @@ -31,7 +31,7 @@ 'use strict'; import { GBMinInstance, GBService, IGBCoreService, GBLog } from 'botlib'; -import fs from 'fs'; +import fs from 'fs/promises'; import * as ji from 'just-indent'; import { GBServer } from '../../../src/app.js'; import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; @@ -41,6 +41,7 @@ import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js'; import urlJoin from 'url-join'; import { NodeVM, VMScript } from 'vm2'; import { createVm2Pool } from './vm2-process/index.js'; +import { watch } from 'fs'; import textract from 'textract'; import walkPromise from 'walk-promise'; import child_process from 'child_process'; @@ -120,7 +121,7 @@ export class GBVMService extends GBService { const wordFile = filename; const vbsFile = isWord ? filename.substr(0, filename.indexOf('docx')) + 'vbs' : filename; const fullVbsFile = urlJoin(folder, vbsFile); - const docxStat = fs.statSync(urlJoin(folder, wordFile)); + const docxStat =await fs.stat(urlJoin(folder, wordFile)); const interval = 3000; // If compiled is older 30 seconds, then recompile. let writeVBS = true; @@ -139,8 +140,8 @@ export class GBVMService extends GBService { // await client.api('/subscriptions') // .post(subscription); - if (fs.existsSync(fullVbsFile)) { - const vbsStat = fs.statSync(fullVbsFile); + if (await GBUtil.exists(fullVbsFile)) { + const vbsStat =await fs.stat(fullVbsFile); if (docxStat['mtimeMs'] < vbsStat['mtimeMs'] + interval) { writeVBS = false; } @@ -154,29 +155,29 @@ export class GBVMService extends GBService { // Write VBS file without pragma keywords. - fs.writeFileSync(urlJoin(folder, vbsFile), text); + fs.writeFile(urlJoin(folder, vbsFile), text); } // Process node_modules install. - this.processNodeModules(folder, min); + await this.processNodeModules(folder, min); // Hot swap for .vbs files. const fullFilename = urlJoin(folder, filename); if (process.env.DEV_HOTSWAP) { - fs.watchFile(fullFilename, async () => { + watch(fullFilename, async () => { await this.translateBASIC(mainName, fullFilename, min); - const parsedCode: string = fs.readFileSync(jsfile, 'utf8'); + const parsedCode: string = await fs.readFile(jsfile, 'utf8'); min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode; }); } - const compiledAt = fs.statSync(fullFilename); + const compiledAt =await fs.stat(fullFilename); const jsfile = urlJoin(folder, `${filename}.js`); - if (fs.existsSync(jsfile)) { - const jsStat = fs.statSync(jsfile); + if (await GBUtil.exists(jsfile)) { + const jsStat =await fs.stat(jsfile); const interval = 1000; // If compiled is older 1 seconds, then recompile. if (compiledAt.isFile() && compiledAt['mtimeMs'] > jsStat['mtimeMs'] + interval) { await this.translateBASIC(mainName, fullFilename, min); @@ -187,15 +188,15 @@ export class GBVMService extends GBService { // Syncronizes Database Objects with the ones returned from "Word". - this.syncStorageFromTABLE(folder, filename, min, mainName); + await this.syncStorageFromTABLE(folder, filename, min, mainName); - const parsedCode: string = fs.readFileSync(jsfile, 'utf8'); + const parsedCode: string = await fs.readFile(jsfile, 'utf8'); min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode; return filename; } - private processNodeModules(folder: string, min: GBMinInstance) { + private async processNodeModules(folder: string, min: GBMinInstance) { const node_modules = urlJoin(process.env.PWD, folder, 'node_modules'); - if (!fs.existsSync(node_modules)) { + if (!await GBUtil.exists(node_modules)) { const packageJson = ` { "name": "${min.botId}.gbdialog", @@ -214,11 +215,11 @@ export class GBVMService extends GBService { "async-retry": "1.3.3" } }`; - fs.writeFileSync(urlJoin(folder, 'package.json'), packageJson); + fs.writeFile(urlJoin(folder, 'package.json'), packageJson); GBLogEx.info(min, `Installing .gbdialog node_modules for ${min.botId}...`); const npmPath = urlJoin(process.env.PWD, 'node_modules', '.bin', 'npm'); - child_process.execSync(`${npmPath} install`, { cwd: folder }); + child_process.exec(`${npmPath} install`, { cwd: folder }); } } @@ -227,8 +228,8 @@ export class GBVMService extends GBService { const packagePath = GBUtil.getGBAIPath(min.botId, null); const filePath = path.join('work', packagePath, 'connections.json'); let connections = []; - if (fs.existsSync(filePath)) { - connections = JSON.parse(fs.readFileSync(filePath, 'utf8')); + if (await GBUtil.exists(filePath)) { + connections = JSON.parse(await fs.readFile(filePath, 'utf8')); } connections.forEach(async con => { @@ -286,14 +287,14 @@ export class GBVMService extends GBService { }); } - private syncStorageFromTABLE(folder: string, filename: string, min: GBMinInstance, mainName: string) { + private async syncStorageFromTABLE(folder: string, filename: string, min: GBMinInstance, mainName: string) { const tablesFile = urlJoin(folder, `${filename}.tables.json`); let sync = false; - if (fs.existsSync(tablesFile)) { + if (await GBUtil.exists(tablesFile)) { const minBoot = GBServer.globals.minBoot; - const tableDef = JSON.parse(fs.readFileSync(tablesFile, 'utf8')) as any; + const tableDef = JSON.parse(await fs.readFile(tablesFile, 'utf8')) as any; const getTypeBasedOnCondition = (t, size) => { if (1) { @@ -453,7 +454,7 @@ export class GBVMService extends GBService { public async translateBASIC(mainName, filename: any, min: GBMinInstance) { // Converts General Bots BASIC into regular VBS - let basicCode: string = fs.readFileSync(filename, 'utf8'); + let basicCode: string = await fs.readFile(filename, 'utf8'); basicCode = GBVMService.normalizeQuotes(basicCode); // Pre process SET SCHEDULE calls. @@ -487,7 +488,7 @@ export class GBVMService extends GBService { // To use include, two /publish will be necessary (for now) // because of alphabet order may raise not found errors. - let includeCode: string = fs.readFileSync(includeName, 'utf8'); + let includeCode: string = await fs.readFile(includeName, 'utf8'); basicCode = basicCode.replace(/^include\b.*$/gim, includeCode); } } while (include); @@ -497,10 +498,10 @@ export class GBVMService extends GBService { // Generates function JSON metadata to be used later. const jsonFile = `${filename}.json`; - fs.writeFileSync(jsonFile, JSON.stringify(metadata)); + fs.writeFile(jsonFile, JSON.stringify(metadata)); const mapFile = `${filename}.map`; - fs.writeFileSync(mapFile, JSON.stringify(map)); + fs.writeFile(mapFile, JSON.stringify(map)); // Execute off-line code tasks @@ -710,7 +711,7 @@ export class GBVMService extends GBService { code = ji.default(code, ' '); - fs.writeFileSync(jsfile, code); + fs.writeFile(jsfile, code); GBLogEx.info(min, `[GBVMService] Finished loading of ${filename}, JavaScript from Word: \n ${code}`); } @@ -722,7 +723,7 @@ export class GBVMService extends GBService { // Creates an empty object that will receive Sequelize fields. const tablesFile = `${task.file}.tables.json`; - fs.writeFileSync(tablesFile, JSON.stringify(task.tables)); + fs.writeFile(tablesFile, JSON.stringify(task.tables)); } } } @@ -757,10 +758,10 @@ export class GBVMService extends GBService { private async getTextFromWord(folder: string, filename: string) { return new Promise(async (resolve, reject) => { const filePath = urlJoin(folder, filename); - textract.fromFileWithPath(filePath, { preserveLineBreaks: true }, (error, text) => { + textract.fromFileWithPath(filePath, { preserveLineBreaks: true }, async (error, text) => { if (error) { if (error.message.startsWith('File not correctly recognized as zip file')) { - text = fs.readFileSync(filePath, 'utf8'); + text = await fs.readFile(filePath, 'utf8'); } else { reject(error); } diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index d4a4a0b3..fcd3bb78 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -32,7 +32,7 @@ import { setFlagsFromString } from 'v8'; import { runInNewContext } from 'vm'; import { IgApiClient } from 'instagram-private-api'; -import { readFileSync } from 'fs'; +import { readFile } from 'fs'; import path, { resolve } from 'path'; import { GBLog, GBMinInstance } from 'botlib'; import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js'; @@ -42,7 +42,7 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; import { DialogKeywords } from './DialogKeywords.js'; import { GBServer } from '../../../src/app.js'; import { GBVMService } from './GBVMService.js'; -import fs from 'fs'; +import fs from 'fs/promises'; import { GBSSR } from '../../core.gbapp/services/GBSSR.js'; import urlJoin from 'url-join'; import Excel from 'exceljs'; @@ -172,22 +172,22 @@ export class SystemKeywords { if (date) { return array ? array.sort((a, b) => { - const c = new Date(a[memberName]); - const d = new Date(b[memberName]); - return c.getTime() - d.getTime(); - }) + const c = new Date(a[memberName]); + const d = new Date(b[memberName]); + return c.getTime() - d.getTime(); + }) : null; } else { return array ? array.sort((a, b) => { - if (a[memberName] < b[memberName]) { - return -1; - } - if (a[memberName] > b[memberName]) { - return 1; - } - return 0; - }) + if (a[memberName] < b[memberName]) { + return -1; + } + if (a[memberName] > b[memberName]) { + return 1; + } + return 0; + }) : array; } } @@ -346,7 +346,6 @@ export class SystemKeywords { delete this.cachedMerge[pid]; - // Capture memory usage before GC GBLogEx.info(min, ``); @@ -415,7 +414,7 @@ export class SystemKeywords { const buffer = pngPages[0].content; const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName)); - fs.writeFileSync(localName, buffer, { encoding: null }); + fs.writeFile(localName, buffer, { encoding: null }); return { localName: localName, url: url, data: buffer }; } @@ -708,7 +707,7 @@ export class SystemKeywords { // Writes it to disk and calculate hash. const data = await response.arrayBuffer(); - fs.writeFileSync(localName, Buffer.from(data), { encoding: null }); + fs.writeFile(localName, Buffer.from(data), { encoding: null }); const hash = new Uint8Array(md5.array(data)); // Performs uploading passing local hash. @@ -724,7 +723,7 @@ export class SystemKeywords { // If upload is OK including hash check, removes the temporary file. if (res._response.status === 201 && new Uint8Array(res.contentMD5).toString() === hash.toString()) { - fs.rmSync(localName); + fs.rm(localName); file['md5'] = hash.toString(); @@ -759,7 +758,6 @@ export class SystemKeywords { let rowsDest = []; rows.forEach(row => { - if (GBUtil.hasSubObject(row)) { row = this.flattenJSON(row); } @@ -886,7 +884,7 @@ export class SystemKeywords { // Creates the file. const blank = path.join(process.env.PWD, 'blank.xlsx'); - const data = fs.readFileSync(blank); + const data = await fs.readFile(blank); await client.api(`${baseUrl}/drive/root:/${packagePath}/${file}:/content`).put(data); // Tries to open again. @@ -1043,7 +1041,7 @@ export class SystemKeywords { public static async getFilter(text) { let filter; - const operators = [/\<\=/, /\<\>/, /\>\=/, /\/,/\blike\b/, /\bnot in\b/, /\bin\b/, /\=/]; + const operators = [/\<\=/, /\<\>/, /\>\=/, /\/, /\blike\b/, /\bnot in\b/, /\bin\b/, /\=/]; let done = false; await CollectionUtil.asyncForEach(operators, async op => { var re = new RegExp(op, 'gi'); @@ -1153,7 +1151,7 @@ export class SystemKeywords { const localName = path.join('work', gbaiName, 'cache', `csv${GBAdminService.getRndReadableIdentifier()}.csv`); const url = file['@microsoft.graph.downloadUrl']; const response = await fetch(url); - fs.writeFileSync(localName, Buffer.from(await response.arrayBuffer()), { encoding: null }); + fs.writeFile(localName, Buffer.from(await response.arrayBuffer()), { encoding: null }); var workbook = new Excel.Workbook(); let worksheet = await workbook.csv.readFile(localName); @@ -1201,7 +1199,8 @@ export class SystemKeywords { let res; let packagePath = GBUtil.getGBAIPath(min.botId, `gbdata`); const csvFile = path.join(GBConfigService.get('STORAGE_LIBRARY'), packagePath, file); - const firstLine = fs.readFileSync(csvFile, 'utf8').split('\n')[0]; + const data = await fs.readFile(csvFile, 'utf8'); + const firstLine = data.split('\n')[0]; const headers = firstLine.split(','); const db = await csvdb(csvFile, headers, ','); if (args[0]) { @@ -1520,8 +1519,15 @@ export class SystemKeywords { ChatServices.userSystemPrompt[user.userSystemId] = text; const packagePath = GBUtil.getGBAIPath(min.botId); - const systemPromptFile = urlJoin(process.cwd(), 'work', packagePath, 'users', user.userSystemId, 'systemPrompt.txt'); - fs.writeFileSync(systemPromptFile, text); + const systemPromptFile = urlJoin( + process.cwd(), + 'work', + packagePath, + 'users', + user.userSystemId, + 'systemPrompt.txt' + ); + fs.writeFile(systemPromptFile, text); } } @@ -1611,7 +1617,7 @@ export class SystemKeywords { // Templates a blank {content} tag inside the blank.docx. const blank = path.join(process.env.PWD, 'blank.docx'); - let buf = fs.readFileSync(blank); + let buf = await fs.readFile(blank); let zip = new PizZip(buf); let doc = new Docxtemplater(); doc.setOptions({ linebreaks: true }); @@ -2012,7 +2018,7 @@ export class SystemKeywords { const res = await fetch(url); let buf: any = Buffer.from(await res.arrayBuffer()); localName = path.join('work', gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.docx`); - fs.writeFileSync(localName, buf, { encoding: null }); + fs.writeFile(localName, buf, { encoding: null }); // Replace image path on all elements of data. @@ -2050,7 +2056,7 @@ export class SystemKeywords { ); const response = await fetch(url); const buf = Buffer.from(await response.arrayBuffer()); - fs.writeFileSync(imageName, buf, { encoding: null }); + fs.writeFile(imageName, buf, { encoding: null }); const getNormalSize = ({ width, height, orientation }) => { return (orientation || 0) >= 5 ? [height, width] : [width, height]; @@ -2099,7 +2105,7 @@ export class SystemKeywords { doc.setData(data).render(); buf = doc.getZip().generate({ type: 'nodebuffer', compression: 'DEFLATE' }); - fs.writeFileSync(localName, buf, { encoding: null }); + fs.writeFile(localName, buf, { encoding: null }); return { localName: localName, url: url, data: buf }; } @@ -2557,7 +2563,7 @@ export class SystemKeywords { const buf = Buffer.from(data.Payment.QrCodeBase64Image, 'base64'); const localName = path.join('work', gbaiName, 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png`); - fs.writeFileSync(localName, buf, { encoding: null }); + fs.writeFile(localName, buf, { encoding: null }); const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName)); GBLogEx.info(min, `GBPay: ${data.MerchantOrderId} OK: ${url}.`); @@ -2731,11 +2737,10 @@ export class SystemKeywords { const res = await fetch(url); let buf: any = Buffer.from(await res.arrayBuffer()); data = new Uint8Array(buf); - } - else { + } else { let packagePath = GBUtil.getGBAIPath(min.botId, `gbdrive`); let filePath = path.join(GBConfigService.get('STORAGE_LIBRARY'), packagePath, file); - data = fs.readFileSync(filePath, 'utf8'); + data = await fs.readFile(filePath, 'utf8'); data = new Uint8Array(Buffer.from(data, 'utf8')); } return await GBUtil.getPdfText(data); @@ -2780,11 +2785,11 @@ export class SystemKeywords { const { min, user, params } = await DialogKeywords.getProcessInfo(pid); // Leitura do arquivo de imagem - const imageBuffer = fs.readFileSync(path.resolve(imagePath)); + const imageBuffer = await fs.readFile(path.resolve(imagePath)); // Criação de um arquivo temporário para enviar const tempFilePath = path.resolve('temp_image.jpg'); - fs.writeFileSync(tempFilePath, imageBuffer); + fs.writeFile(tempFilePath, imageBuffer); // Publicação da imagem const page = new Page(pageId); @@ -2792,30 +2797,27 @@ export class SystemKeywords { message: caption, attached_media: [ { - media_fbid: tempFilePath, - }, - ], + media_fbid: tempFilePath + } + ] }); // Log do resultado GBLogEx.info(min, `Imagem publicada no Facebook: ${JSON.stringify(response)}`); // Limpeza do arquivo temporário - fs.unlinkSync(tempFilePath); + fs.unlink(tempFilePath); } - public async postToInstagram({ pid, username, password, imagePath, caption }) { const { min, user, params } = await DialogKeywords.getProcessInfo(pid); const ig = new IgApiClient(); ig.state.generateDevice(username); await ig.account.login(username, password); - const imageBuffer = readFileSync(resolve(imagePath)); - const publishResult = await ig.publish.photo({ - file: imageBuffer, - caption - }); + const imageBuffer = await fs.readFile(imagePath); + const publishResult = await ig.publish.photo( + { file: imageBuffer, caption }); GBLogEx.info(min, `Image posted on IG: ${publishResult}`); } @@ -2823,9 +2825,8 @@ export class SystemKeywords { public async setAnswerMode({ pid, mode }) { const { min, user, params } = await DialogKeywords.getProcessInfo(pid); - ChatServices.usersMode[user.userSystemId] = mode; + ChatServices.usersMode[user.userSystemId] = mode; GBLogEx.info(min, `LLM Mode (user.userSystemId) : ${mode}`); } - } diff --git a/packages/basic.gblib/services/WebAutomationServices.ts b/packages/basic.gblib/services/WebAutomationServices.ts index 9dfb70cd..a35e730d 100644 --- a/packages/basic.gblib/services/WebAutomationServices.ts +++ b/packages/basic.gblib/services/WebAutomationServices.ts @@ -31,7 +31,7 @@ 'use strict'; import urlJoin from 'url-join'; -import fs from 'fs'; +import fs from 'fs/promises'; import path from 'path'; import url from 'url'; import { GBLog } from 'botlib'; @@ -426,7 +426,7 @@ export class WebAutomationServices { let result: Buffer; if (local) { - result = fs.readFileSync(local); + result = await fs.readFile(local); } else { const res = await fetch(options.uri, options); result = Buffer.from(await res.arrayBuffer()); diff --git a/packages/core.gbapp/services/GBConversationalService.ts b/packages/core.gbapp/services/GBConversationalService.ts index 020dfb56..4a32b0b6 100644 --- a/packages/core.gbapp/services/GBConversationalService.ts +++ b/packages/core.gbapp/services/GBConversationalService.ts @@ -48,7 +48,8 @@ import { CollectionUtil, AzureText } from 'pragmatismo-io-framework'; import { GuaribasUser } from '../../security.gbapp/models/index.js'; import { GBMinService } from './GBMinService.js'; import urlJoin from 'url-join'; -import fs from 'fs'; +import {createWriteStream, createReadStream} from 'fs'; +import fs from 'fs/promises'; import twilio from 'twilio'; import Nexmo from 'nexmo'; import { join } from 'path'; @@ -451,15 +452,15 @@ export class GBConversationalService { const waveFilename = `work/tmp${name}.pcm`; let audio = await textToSpeech.repairWavHeaderStream(res.result as any); - fs.writeFileSync(waveFilename, audio); + fs.writeFile(waveFilename, audio); const oggFilenameOnly = `tmp${name}.ogg`; const oggFilename = `work/${oggFilenameOnly}`; - const output = fs.createWriteStream(oggFilename); + const output = createWriteStream(oggFilename); const transcoder = new prism.FFmpeg({ args: ['-analyzeduration', '0', '-loglevel', '0', '-f', 'opus', '-ar', '16000', '-ac', '1'] }); - fs.createReadStream(waveFilename).pipe(transcoder).pipe(output); + createReadStream(waveFilename).pipe(transcoder).pipe(output); let url = urlJoin(GBServer.globals.publicAddress, 'audios', oggFilenameOnly); resolve(url); @@ -481,7 +482,7 @@ export class GBConversationalService { const dest = `work/tmp${name}.wav`; const src = `work/tmp${name}.ogg`; - fs.writeFileSync(src, oggFile.read()); + fs.writeFile(src, oggFile.read()); const makeMp3 = shell([ 'node_modules/ffmpeg-static/ffmpeg', // TODO: .exe on MSWin. @@ -499,12 +500,12 @@ export class GBConversationalService { join(process.cwd(), dest) ]); - exec(makeMp3, error => { + exec(makeMp3, async error => { if (error) { GBLog.error(error); return Promise.reject(error); } else { - let data = fs.readFileSync(dest); + let data = await fs.readFile(dest); const speechToText = new SpeechToTextV1({ authenticator: new IamAuthenticator({ apikey: process.env.WATSON_STT_KEY }), diff --git a/packages/core.gbapp/services/GBCoreService.ts b/packages/core.gbapp/services/GBCoreService.ts index 1258a916..3232d932 100644 --- a/packages/core.gbapp/services/GBCoreService.ts +++ b/packages/core.gbapp/services/GBCoreService.ts @@ -35,7 +35,7 @@ 'use strict'; import { GBLog, GBMinInstance, IGBCoreService, IGBInstallationDeployer, IGBInstance, IGBPackage } from 'botlib'; -import fs from 'fs'; +import fs from 'fs/promises'; import { Sequelize, SequelizeOptions } from 'sequelize-typescript'; import { Op, Dialect } from 'sequelize'; import { GBServer } from '../../../src/app.js'; @@ -135,7 +135,7 @@ export class GBCoreService implements IGBCoreService { } else if (this.dialect === 'sqlite') { storage = GBConfigService.get('STORAGE_FILE'); - if (!fs.existsSync(storage)) { + if (!await GBUtil.exists(storage)) { process.env.STORAGE_SYNC = 'true'; } } else { @@ -313,7 +313,7 @@ STORAGE_SYNC_ALTER=true ENDPOINT_UPDATE=true `; - fs.writeFileSync('.env', env); + fs.writeFile('.env', env); } /** @@ -323,7 +323,7 @@ ENDPOINT_UPDATE=true */ public async ensureProxy(port): Promise { try { - if (fs.existsSync('node_modules/ngrok/bin/ngrok.exe') || fs.existsSync('node_modules/.bin/ngrok')) { + if (await GBUtil.exists('node_modules/ngrok/bin/ngrok.exe') || await GBUtil.exists('node_modules/.bin/ngrok')) { return await ngrok.connect({ port: port }); } else { GBLog.warn('ngrok executable not found. Check installation or node_modules folder.'); @@ -825,13 +825,13 @@ ENDPOINT_UPDATE=true public async ensureFolders(instances, deployer: GBDeployer) { let libraryPath = GBConfigService.get('STORAGE_LIBRARY'); - if (!fs.existsSync(libraryPath)) { + if (!await GBUtil.exists(libraryPath)) { mkdirp.sync(libraryPath); } await this.syncBotStorage(instances, 'default', deployer, libraryPath); - const files = fs.readdirSync(libraryPath); + const files = fs.readdir(libraryPath); await CollectionUtil.asyncForEach(files, async file => { if (file.trim().toLowerCase() !== 'default.gbai') { let botId = file.replace(/\.gbai/, ''); @@ -855,37 +855,37 @@ ENDPOINT_UPDATE=true instance = await deployer.deployBlankBot(botId, mobile, email); const gbaiPath = path.join(libraryPath, `${botId}.gbai`); - if (!fs.existsSync(gbaiPath)) { - fs.mkdirSync(gbaiPath, { recursive: true }); + if (!await GBUtil.exists(gbaiPath)) { + fs.mkdir(gbaiPath, { recursive: true }); const base = path.join(process.env.PWD, 'templates', 'default.gbai'); - fs.cpSync(path.join(base, `default.gbkb`), path.join(gbaiPath, `default.gbkb`), { + fs.cp(path.join(base, `default.gbkb`), path.join(gbaiPath, `default.gbkb`), { errorOnExist: false, force: true, recursive: true }); - fs.cpSync(path.join(base, `default.gbot`), path.join(gbaiPath, `default.gbot`), { + fs.cp(path.join(base, `default.gbot`), path.join(gbaiPath, `default.gbot`), { errorOnExist: false, force: true, recursive: true }); - fs.cpSync(path.join(base, `default.gbtheme`), path.join(gbaiPath, `default.gbtheme`), { + fs.cp(path.join(base, `default.gbtheme`), path.join(gbaiPath, `default.gbtheme`), { errorOnExist: false, force: true, recursive: true }); - fs.cpSync(path.join(base, `default.gbdata`), path.join(gbaiPath, `default.gbdata`), { + fs.cp(path.join(base, `default.gbdata`), path.join(gbaiPath, `default.gbdata`), { errorOnExist: false, force: true, recursive: true }); - fs.cpSync(path.join(base, `default.gbdialog`), path.join(gbaiPath, `default.gbdialog`), { + fs.cp(path.join(base, `default.gbdialog`), path.join(gbaiPath, `default.gbdialog`), { errorOnExist: false, force: true, recursive: true }); - fs.cpSync(path.join(base, `default.gbdrive`), path.join(gbaiPath, `default.gbdrive`), { + fs.cp(path.join(base, `default.gbdrive`), path.join(gbaiPath, `default.gbdrive`), { errorOnExist: false, force: true, recursive: true diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index 76e90e35..bc1af018 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -39,7 +39,7 @@ import express from 'express'; import child_process from 'child_process'; import { rimraf } from 'rimraf'; import urlJoin from 'url-join'; -import fs from 'fs'; +import fs from 'fs/promises'; import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBDeployer, IGBInstance, IGBPackage } from 'botlib'; import { AzureSearch } from 'pragmatismo-io-framework'; import { CollectionUtil } from 'pragmatismo-io-framework'; @@ -150,9 +150,9 @@ export class GBDeployer implements IGBDeployer { async function scanPackageDirectory(directory) { // Gets all directories. - const isDirectory = source => fs.lstatSync(source).isDirectory(); - const getDirectories = source => - fs.readdirSync(source) + const isDirectory = async source => (await fs.lstat(source)).isDirectory(); + const getDirectories = async source => + (await fs.readdir(source)) .map(name => path.join(source, name)) .filter(isDirectory); const dirs = getDirectories(directory); @@ -440,7 +440,7 @@ export class GBDeployer implements IGBDeployer { const workbook = new Excel.Workbook(); - if (fs.existsSync(xls)) { + if (await GBUtil.exists(xls)) { await workbook.xlsx.readFile(xls); let worksheet: any; for (let t = 0; t < workbook.worksheets.length; t++) { @@ -455,7 +455,7 @@ export class GBDeployer implements IGBDeployer { for (let index = 0; index < 6; index++) { rows.shift(); } - } else if (fs.existsSync(csv)) { + } else if (await GBUtil.exists(csv)) { await workbook.csv.readFile(csv); let worksheet = workbook.worksheets[0]; // Assuming the CSV file has only one sheet rows = worksheet.getSheetValues(); @@ -497,14 +497,14 @@ export class GBDeployer implements IGBDeployer { // Creates each subfolder. let pathBase = localPath; - if (!fs.existsSync(pathBase)) { - fs.mkdirSync(pathBase); + if (!await GBUtil.exists(pathBase)) { + fs.mkdir(pathBase); } await CollectionUtil.asyncForEach(parts, async item => { pathBase = packagePath.join(pathBase, item); - if (!fs.existsSync(pathBase)) { - fs.mkdirSync(pathBase); + if (!await GBUtil.exists(pathBase)) { + fs.mkdir(pathBase); } }); @@ -529,16 +529,16 @@ export class GBDeployer implements IGBDeployer { const itemPath = packagePath.join(localPath, remotePath, item.name); if (item.folder) { - if (!fs.existsSync(itemPath)) { - fs.mkdirSync(itemPath); + if (!await GBUtil.exists(itemPath)) { + fs.mkdir(itemPath); } const nextFolder = urlJoin(remotePath, item.name); await this.downloadFolder(min, localPath, nextFolder); } else { let download = true; - if (fs.existsSync(itemPath)) { - const dt = fs.statSync(itemPath); + if (await GBUtil.exists(itemPath)) { + const dt =await fs.stat(itemPath); if (new Date(dt.mtime) >= new Date(item.lastModifiedDateTime)) { download = false; } @@ -549,8 +549,8 @@ export class GBDeployer implements IGBDeployer { const url = item['@microsoft.graph.downloadUrl']; const response = await fetch(url); - fs.writeFileSync(itemPath, Buffer.from(await response.arrayBuffer()), { encoding: null }); - fs.utimesSync(itemPath, new Date(), new Date(item.lastModifiedDateTime)); + fs.writeFile(itemPath, Buffer.from(await response.arrayBuffer()), { encoding: null }); + fs.utimes(itemPath, new Date(), new Date(item.lastModifiedDateTime)); } else { GBLogEx.info(min, `Local is up to date: ${itemPath}...`); } @@ -667,7 +667,7 @@ export class GBDeployer implements IGBDeployer { const packagePath = GBUtil.getGBAIPath(min.botId, null); const localFolder = path.join('work', packagePath, 'connections.json'); - fs.writeFileSync(localFolder, JSON.stringify(connections), { encoding: null }); + fs.writeFile(localFolder, JSON.stringify(connections), { encoding: null }); // Updates instance object. @@ -759,7 +759,7 @@ export class GBDeployer implements IGBDeployer { switch (packageType) { case '.gbot': - const packageObject = JSON.parse(fs.readFileSync(urlJoin(localPath, 'package.json'), 'utf8')); + const packageObject = JSON.parse(await fs.readFile(urlJoin(localPath, 'package.json'), 'utf8')); await this.undeployBot(packageObject.botId, packageName); break; @@ -870,7 +870,7 @@ export class GBDeployer implements IGBDeployer { * Prepares the React application inside default.gbui folder and * makes this web application available as default web front-end. */ - public setupDefaultGBUI() { + public async setupDefaultGBUI() { // Setups paths. const root = 'packages/default.gbui'; @@ -878,10 +878,10 @@ export class GBDeployer implements IGBDeployer { // Checks if .gbapp compiliation is enabled. - if (!fs.existsSync(`${root}/build`) && process.env.DISABLE_WEB !== 'true') { + if (!(await GBUtil.exists(`${root}/build`)) && process.env.DISABLE_WEB !== 'true') { // Write a .env required to fix some bungs in create-react-app tool. - fs.writeFileSync(`${root}/.env`, 'SKIP_PREFLIGHT_CHECK=true'); + fs.writeFile(`${root}/.env`, 'SKIP_PREFLIGHT_CHECK=true'); // Install modules and compiles the web app. @@ -956,7 +956,7 @@ export class GBDeployer implements IGBDeployer { GBLogEx.info(0, `Deploying General Bots Application (.gbapp) or Library (.gblib): ${path.basename(gbappPath)}...`); let folder = path.join(gbappPath, 'node_modules'); if (process.env.GBAPP_DISABLE_COMPILE !== 'true') { - if (!fs.existsSync(folder)) { + if (!await GBUtil.exists(folder)) { GBLogEx.info(0, `Installing modules for ${gbappPath}...`); child_process.execSync('npm install', { cwd: gbappPath }); } diff --git a/packages/core.gbapp/services/GBImporterService.ts b/packages/core.gbapp/services/GBImporterService.ts index 5f6f4ace..34cda633 100644 --- a/packages/core.gbapp/services/GBImporterService.ts +++ b/packages/core.gbapp/services/GBImporterService.ts @@ -36,11 +36,12 @@ import { GBMinInstance, IGBCoreService, IGBInstance } from 'botlib'; import { CreateOptions } from 'sequelize/types'; -import fs from 'fs'; +import fs from 'fs/promises'; import urlJoin from 'url-join'; import { GBServer } from '../../../src/app.js'; import { GuaribasInstance } from '../models/GBModel.js'; import { GBConfigService } from './GBConfigService.js'; +import { GBUtil } from '../../../src/util.js'; /** * Handles the importing of packages. @@ -61,9 +62,9 @@ export class GBImporter { const file = urlJoin(localPath, 'settings.json'); let settingsJson = {botId: botId}; - if (fs.existsSync(file)){ + if (await GBUtil.exists(file)){ - settingsJson = JSON.parse(fs.readFileSync(file, 'utf8')); + settingsJson = JSON.parse(await fs.readFile(file, 'utf8')); if (botId === undefined) { botId = settingsJson.botId; } diff --git a/packages/core.gbapp/services/GBMinService.ts b/packages/core.gbapp/services/GBMinService.ts index bfb7749e..d5a88c2c 100644 --- a/packages/core.gbapp/services/GBMinService.ts +++ b/packages/core.gbapp/services/GBMinService.ts @@ -40,7 +40,7 @@ import removeRoute from 'express-remove-route'; import AuthenticationContext from 'adal-node'; import { FacebookAdapter } from 'botbuilder-adapter-facebook'; import mkdirp from 'mkdirp'; -import fs from 'fs'; +import fs from 'fs/promises'; import arrayBufferToBuffer from 'arraybuffer-to-buffer'; import { NlpManager } from 'node-nlp'; import Koa from 'koa'; @@ -244,7 +244,7 @@ export class GBMinService { // TODO: https://github.com/GeneralBots/BotServer/issues/321 const options = { passphrase: process.env.CERTIFICATE2_PASSPHRASE, - pfx: fs.readFileSync(process.env.CERTIFICATE2_PFX) + pfx: await fs.readFile(process.env.CERTIFICATE2_PFX) }; const domain = min.core.getParam(min.instance, 'Domain', null); @@ -294,19 +294,19 @@ export class GBMinService { // Install per bot deployed packages. let packagePath = urlJoin(`work`, GBUtil.getGBAIPath(min.botId, 'gbdialog')); - if (fs.existsSync(packagePath)) { + if (await GBUtil.exists(packagePath)) { await this.deployer['deployPackage2'](min, user, packagePath); } packagePath = urlJoin(`work`, GBUtil.getGBAIPath(min.botId, 'gbapp')); - if (fs.existsSync(packagePath)) { + if (await GBUtil.exists(packagePath)) { await this.deployer['deployPackage2'](min, user, packagePath); } packagePath = urlJoin(`work`, GBUtil.getGBAIPath(min.botId, 'gbtheme')); - if (fs.existsSync(packagePath)) { + if (await GBUtil.exists(packagePath)) { await this.deployer['deployPackage2'](min, user, packagePath); } packagePath = urlJoin(`work`, GBUtil.getGBAIPath(min.botId, `gblib`)); - if (fs.existsSync(packagePath)) { + if (await GBUtil.exists(packagePath)) { await this.deployer['deployPackage2'](min, user, packagePath); } @@ -314,39 +314,39 @@ export class GBMinService { let dir = `work/${gbai}/cache`; const botId = gbai.replace(/\.[^/.]+$/, ''); - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } dir = `work/${gbai}/profile`; - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } dir = `work/${gbai}/uploads`; - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } dir = `work/${gbai}/${botId}.gbkb`; - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } dir = `work/${gbai}/${botId}.gbkb/docs-vectorized`; - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } dir = `work/${gbai}/${botId}.gbdialog`; - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } dir = `work/${gbai}/${botId}.gbot`; - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } dir = `work/${gbai}/${botId}.gbui`; - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } dir = `work/${gbai}/users`; - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } @@ -388,10 +388,10 @@ export class GBMinService { const manifest = `${instance.botId}-Teams.zip`; const packageTeams = urlJoin(`work`, GBUtil.getGBAIPath(instance.botId), manifest); - if (!fs.existsSync(packageTeams)) { + if (!await GBUtil.exists(packageTeams)) { GBLogEx.info(min, 'Generating MS Teams manifest....'); const data = await this.deployer.getBotManifest(instance); - fs.writeFileSync(packageTeams, data); + fs.writeFile(packageTeams, data); } // Serves individual URL for each bot user interface. @@ -1099,7 +1099,7 @@ export class GBMinService { const folder = `work/${path}/cache`; const filename = `${GBAdminService.generateUuid()}.png`; - fs.writeFileSync(urlJoin(folder, filename), data); + fs.writeFile(urlJoin(folder, filename), data); step.context.activity.text = urlJoin( GBServer.globals.publicAddress, `${min.instance.botId}`, @@ -1287,7 +1287,7 @@ export class GBMinService { buffer = arrayBufferToBuffer(await res.arrayBuffer()); } - fs.writeFileSync(localFileName, buffer); + fs.writeFile(localFileName, buffer); return { fileName: attachment.name, @@ -1324,7 +1324,7 @@ export class GBMinService { const t = new SystemKeywords(); GBLogEx.info(min, `BASIC (${min.botId}): Upload done for ${attachmentData.fileName}.`); const handle = WebAutomationServices.cyrb53({ pid: 0, str: min.botId + attachmentData.fileName }); - let data = fs.readFileSync(attachmentData.localPath); + let data = await fs.readFile(attachmentData.localPath); const gbfile = { filename: attachmentData.localPath, @@ -1358,9 +1358,9 @@ export class GBMinService { filename: string; } - const results = successfulSaves.reduce((accum: GBFile[], item) => { + const results = successfulSaves.reduce(async (accum: GBFile[], item) => { const result: GBFile = { - data: fs.readFileSync(successfulSaves[0]['localPath']), + data: await fs.readFile(successfulSaves[0]['localPath']), filename: successfulSaves[0]['fileName'] }; accum.push(result); diff --git a/packages/core.gbapp/services/GBSSR.ts b/packages/core.gbapp/services/GBSSR.ts index 825ec27b..c4a08b7f 100644 --- a/packages/core.gbapp/services/GBSSR.ts +++ b/packages/core.gbapp/services/GBSSR.ts @@ -38,7 +38,7 @@ import { createRequire } from 'module'; const require = createRequire(import.meta.url); import path from 'path'; -import fs from 'fs'; +import fs from 'fs/promises'; import { NextFunction, Request, Response } from 'express'; import urljoin from 'url-join'; import { GBMinInstance } from 'botlib'; @@ -91,7 +91,7 @@ export class GBSSR { 'tiqcdn' ]; - public static preparePuppeteer(profilePath) { + public static async preparePuppeteer(profilePath) { let args = [ '--check-for-update-interval=2592000', '--disable-accelerated-2d-canvas', @@ -106,11 +106,11 @@ export class GBSSR { args.push(`--user-data-dir=${profilePath}`); const preferences = urljoin(profilePath, 'Default', 'Preferences'); - if (fs.existsSync(preferences)) { - const file = fs.readFileSync(preferences, 'utf8'); + if (await GBUtil.exists(preferences)) { + const file = await fs.readFile(preferences, 'utf8'); const data = JSON.parse(file); data['profile']['exit_type'] = 'none'; - fs.writeFileSync(preferences, JSON.stringify(data)); + fs.writeFile(preferences, JSON.stringify(data)); } } @@ -126,7 +126,7 @@ export class GBSSR { public static async createBrowser(profilePath): Promise { - const opts = this.preparePuppeteer(profilePath); + const opts = await this.preparePuppeteer(profilePath); puppeteer.use(hidden()); puppeteer.use(require("puppeteer-extra-plugin-minmax")()); const browser = await puppeteer.launch(opts); @@ -307,18 +307,18 @@ export class GBSSR { // Checks if the bot has an .gbui published or use default.gbui. - if (!fs.existsSync(packagePath)) { + if (!await GBUtil.exists(packagePath)) { packagePath = GBUtil.getGBAIPath(minBoot.botId, `gbui`); } let parts = req.url.replace(`/${botId}`, '').split('?'); let url = parts[0]; - if (min && req.originalUrl && prerender && exclude && fs.existsSync(packagePath)) { + if (min && req.originalUrl && prerender && exclude && await GBUtil.exists(packagePath)) { // Reads from static HTML when a bot is crawling. packagePath = path.join(process.env.PWD, 'work', packagePath, 'index.html'); - const html = fs.readFileSync(packagePath, 'utf8'); + const html = await fs.readFile(packagePath, 'utf8'); res.status(200).send(html); return true; } else { @@ -338,9 +338,9 @@ export class GBSSR { if (!min && !url.startsWith("/static") && GBServer.globals.wwwroot) { packagePath = packagePath.join(GBServer.globals.wwwroot, url); } - if (fs.existsSync(packagePath)) { + if (await GBUtil.exists(packagePath)) { if (min) { - let html = fs.readFileSync(packagePath, 'utf8'); + let html = await fs.readFile(packagePath, 'utf8'); html = html.replace(/\{p\}/gi, min.botId); html = html.replace(/\{botId\}/gi, min.botId); html = html.replace(/\{theme\}/gi, min.instance.theme ? min.instance.theme : diff --git a/packages/google-chat.gblib/services/GoogleChatDirectLine.ts b/packages/google-chat.gblib/services/GoogleChatDirectLine.ts index 081ba057..5ac58329 100644 --- a/packages/google-chat.gblib/services/GoogleChatDirectLine.ts +++ b/packages/google-chat.gblib/services/GoogleChatDirectLine.ts @@ -31,7 +31,7 @@ import Swagger from 'swagger-client'; import { google } from 'googleapis'; import { PubSub } from '@google-cloud/pubsub'; -import fs from 'fs'; +import fs from 'fs/promises'; import { GBLog, GBMinInstance, GBService } from 'botlib'; import { GBServer } from '../../../src/app.js'; import { SecService } from '../../security.gbapp/services/SecService.js'; diff --git a/packages/kb.gbapp/services/KBService.ts b/packages/kb.gbapp/services/KBService.ts index 3aea1de6..a731d104 100644 --- a/packages/kb.gbapp/services/KBService.ts +++ b/packages/kb.gbapp/services/KBService.ts @@ -32,7 +32,7 @@ * @fileoverview Knowledge base services and logic. */ import path from 'path'; -import fs from 'fs'; +import fs from 'fs/promises'; import urlJoin from 'url-join'; import asyncPromise from 'async-promises'; import walkPromise from 'walk-promise'; @@ -501,7 +501,7 @@ export class KBService implements IGBKBService { 'Existe um problema na base de conhecimento. Fui treinado para entender sua pergunta, avise a quem me criou que a resposta não foi informada para esta pergunta.'; } else if (answer.indexOf('.md') > -1 || answer.indexOf('.docx') > -1) { const mediaFilename = urlJoin(path.dirname(filePath), '..', 'articles', answer); - if (fs.existsSync(mediaFilename)) { + if (await GBUtil.exists(mediaFilename)) { // Tries to load .docx file from Articles folder. if (answer.indexOf('.docx') > -1) { @@ -509,7 +509,7 @@ export class KBService implements IGBKBService { } else { // Loads normally markdown file. - answer = fs.readFileSync(mediaFilename, 'utf8'); + answer = await fs.readFile(mediaFilename, 'utf8'); } format = '.md'; media = path.basename(mediaFilename); @@ -558,7 +558,7 @@ export class KBService implements IGBKBService { const packagePath = GBUtil.getGBAIPath(min.botId, `gbdialog`); const scriptName = `tmp${GBAdminService.getRndReadableIdentifier()}.docx`; const localName = path.join('work', packagePath, `${scriptName}`); - fs.writeFileSync(localName, code, { encoding: null }); + fs.writeFile(localName, code, { encoding: null }); answer = scriptName; const vm = new GBVMService(); @@ -690,7 +690,7 @@ export class KBService implements IGBKBService { // Imports menu.xlsx if any. - if (fs.existsSync(subjectFile) || fs.existsSync(menuFile)) { + if (await GBUtil.exists(subjectFile) || await GBUtil.exists(menuFile)) { await this.importSubjectFile(packageStorage.packageId, subjectFile, menuFile, instance); } @@ -725,7 +725,7 @@ export class KBService implements IGBKBService { if (content === null) { const fullFilename = urlJoin(file.root, file.name); - content = fs.readFileSync(fullFilename, 'utf-8'); + content = await fs.readFile(fullFilename, 'utf-8'); await GuaribasAnswer.create({ instanceId: instance.instanceId, @@ -760,7 +760,7 @@ export class KBService implements IGBKBService { } else if (file !== null && file.name.endsWith('.toc.docx')) { const packagePath = GBUtil.getGBAIPath(instance.botId, `gbkb`); const localName = path.join('work', packagePath, 'articles', file.name); - const buffer = fs.readFileSync(localName, { encoding: null }); + const buffer = await fs.readFile(localName, { encoding: null }); var options = { buffer: buffer, convertImage: async image => { @@ -777,7 +777,7 @@ export class KBService implements IGBKBService { path.basename(localName) ); const buffer = await image.read(); - fs.writeFileSync(localName, buffer, { encoding: null }); + fs.writeFile(localName, buffer, { encoding: null }); return { src: url }; } }; @@ -875,7 +875,6 @@ export class KBService implements IGBKBService { depth > maxDepth || visited.has(url) || url.endsWith('.jpg') || - url.endsWith('.jpg') || url.endsWith('.png') || url.endsWith('.mp4') ) { @@ -1025,7 +1024,7 @@ export class KBService implements IGBKBService { let packagePath = GBUtil.getGBAIPath(min.botId, `gbot`); const directoryPath = path.join(process.env.PWD, 'work', packagePath, 'Website'); - fs.rmSync(directoryPath, { recursive: true, force: true }); + fs.rm(directoryPath, { recursive: true, force: true }); let browser = await puppeteer.launch({ headless: false }); const page = await this.getFreshPage(browser, website); @@ -1040,7 +1039,7 @@ export class KBService implements IGBKBService { try { const logoBinary = await page.goto(logo); const buffer = await logoBinary.buffer(); - const logoFilename = packagePath.basename(logo); + const logoFilename = path.basename(logo); // TODO: sharp(buffer) // .resize({ // width: 48, @@ -1189,7 +1188,7 @@ export class KBService implements IGBKBService { instance: IGBInstance ): Promise { let subjectsLoaded; - if (fs.existsSync(menuFile)) { + if (await GBUtil.exists(menuFile)) { // Loads menu.xlsx and finds worksheet. const workbook = new Excel.Workbook(); @@ -1258,7 +1257,7 @@ export class KBService implements IGBKBService { subjectsLoaded = subjects; } else { - subjectsLoaded = JSON.parse(fs.readFileSync(filename, 'utf8')); + subjectsLoaded = JSON.parse(await fs.readFile(filename, 'utf8')); } const doIt = async (subjects: GuaribasSubject[], parentSubjectId: number) => { @@ -1354,7 +1353,7 @@ export class KBService implements IGBKBService { let packagePath = GBUtil.getGBAIPath(min.botId, `gbui`); packagePath = path.join(process.env.PWD, 'work', packagePath, 'index.html'); GBLogEx.info(min, `[GBDeployer] Saving SSR HTML in ${packagePath}.`); - fs.writeFileSync(packagePath, html, 'utf8'); + fs.writeFile(packagePath, html, 'utf8'); GBLogEx.info(min, `[GBDeployer] Finished import of ${localPath}`); } diff --git a/packages/llm.gblib/services/ChatServices.ts b/packages/llm.gblib/services/ChatServices.ts index 84e58025..18796c64 100644 --- a/packages/llm.gblib/services/ChatServices.ts +++ b/packages/llm.gblib/services/ChatServices.ts @@ -51,7 +51,7 @@ import { SqlDatabaseChain } from 'langchain/chains/sql_db'; import { SqlDatabase } from 'langchain/sql_db'; import { DataSource } from 'typeorm'; import { GBMinInstance } from 'botlib'; -import fs from 'fs'; +import fs from 'fs/promises'; import { jsonSchemaToZod } from 'json-schema-to-zod'; import { BufferWindowMemory } from 'langchain/memory'; import path from 'path'; @@ -188,7 +188,7 @@ export class ChatServices { const gbaiName = GBUtil.getGBAIPath(min.botId, null); const localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`); const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName)); - fs.writeFileSync(localName, buffer, { encoding: null }); + fs.writeFile(localName, buffer, { encoding: null }); return { localName: localName, url: url, data: buffer }; } } @@ -234,7 +234,7 @@ export class ChatServices { } private static async findPageForText(pdfPath, searchText) { - const data = new Uint8Array(fs.readFileSync(pdfPath)); + const data = new Uint8Array(await fs.readFile(pdfPath)); const pdf = await getDocument({ data }).promise; searchText = searchText.replace(/\s/g, ''); @@ -712,8 +712,8 @@ export class ChatServices { const packagePath = GBUtil.getGBAIPath(min.botId, 'gbdialog', null); const jsonFile = path.join('work', packagePath, `${script}.json`); - if (fs.existsSync(jsonFile) && script.toLowerCase() !== 'start.vbs') { - const funcJSON = JSON.parse(fs.readFileSync(jsonFile, 'utf8')); + if (await GBUtil.exists(jsonFile) && script.toLowerCase() !== 'start.vbs') { + const funcJSON = JSON.parse(await fs.readFile(jsonFile, 'utf8')); const funcObj = funcJSON?.function; if (funcObj) { diff --git a/packages/llm.gblib/services/ImageServices.ts b/packages/llm.gblib/services/ImageServices.ts index 7a8518f8..09823b3d 100644 --- a/packages/llm.gblib/services/ImageServices.ts +++ b/packages/llm.gblib/services/ImageServices.ts @@ -37,7 +37,7 @@ import { AzureKeyCredential } from '@azure/core-auth'; import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords'; import path from 'path'; import { GBServer } from '../../../src/app.js'; -import fs from 'fs'; +import fs from 'fs/promises'; import urlJoin from 'url-join'; import { GBAdminService } from '../../admin.gbapp/services/GBAdminService'; import { GBLogEx } from '../../core.gbapp/services/GBLogEx'; @@ -73,7 +73,7 @@ export class ImageServices { const url = response.data[0].url; const res = await fetch(url); let buf: any = Buffer.from(await res.arrayBuffer()); - fs.writeFileSync(localName, buf, { encoding: null }); + fs.writeFile(localName, buf, { encoding: null }); GBLogEx.info(min, `DALL-E image generated at ${url}.`); diff --git a/packages/saas.gbapp/service/JunoSubscription.ts b/packages/saas.gbapp/service/JunoSubscription.ts index fbc0c91a..857cdb76 100755 --- a/packages/saas.gbapp/service/JunoSubscription.ts +++ b/packages/saas.gbapp/service/JunoSubscription.ts @@ -29,7 +29,7 @@ \*****************************************************************************/ 'use strict'; -import fs from 'fs'; +import fs from 'fs/promises'; import { HttpMethods, HttpOperationResponse, ServiceClient, WebResource } from '@azure/ms-rest-js'; import { GBLog } from 'botlib'; import urlJoin from 'url-join'; @@ -420,7 +420,7 @@ export class JunoSubscription { const httpClient = new ServiceClient(); const url = urlJoin(JunoSubscription.getResourceUrl(), `/documents/${id}/files`); var form = new FormData(); - form.append('file', fs.readFileSync(file)); + form.append('file', await fs.readFile(file)); const req = JunoSubscription.createRequestObject( token, diff --git a/packages/security.gbapp/services/SecService.ts b/packages/security.gbapp/services/SecService.ts index 1b9a6d1f..82e658c6 100644 --- a/packages/security.gbapp/services/SecService.ts +++ b/packages/security.gbapp/services/SecService.ts @@ -4,7 +4,7 @@ import { CollectionUtil } from 'pragmatismo-io-framework'; import { GuaribasUser } from '../models/index.js'; import { FindOptions } from 'sequelize'; import { DialogKeywords } from '../../../packages/basic.gblib/services/DialogKeywords.js'; -import fs from 'fs'; +import fs from 'fs/promises'; import mkdirp from 'mkdirp'; import urlJoin from 'url-join'; import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js'; @@ -29,7 +29,7 @@ export class SecService extends GBService { const gbaiPath = GBUtil.getGBAIPath(min.botId); const dir = urlJoin ('work',gbaiPath, 'users', userSystemId); - if (!fs.existsSync(dir)) { + if (!await GBUtil.exists(dir)) { mkdirp.sync(dir); } @@ -44,8 +44,8 @@ export class SecService extends GBService { } const systemPromptFile = urlJoin(dir, 'systemPrompt.txt'); - if (fs.existsSync(systemPromptFile)) { - user[ 'systemPrompt'] = fs.readFileSync(systemPromptFile); + if (await GBUtil.exists(systemPromptFile)) { + user[ 'systemPrompt'] = await fs.readFile(systemPromptFile); } user.instanceId = min.instance.instanceId; diff --git a/packages/teams.gblib/services/TeamsService.ts b/packages/teams.gblib/services/TeamsService.ts index bed4ff39..e37e7822 100644 --- a/packages/teams.gblib/services/TeamsService.ts +++ b/packages/teams.gblib/services/TeamsService.ts @@ -29,7 +29,7 @@ \*****************************************************************************/ import { GBService } from 'botlib'; -import fs from 'fs'; +import fs from 'fs/promises'; import AdmZip from 'adm-zip'; /** @@ -45,7 +45,7 @@ export class TeamsService extends GBService { } public async getManifest(marketplaceId, botName, botDescription, id, packageName, yourName) { - let content = fs.readFileSync('teams-manifest.json', 'utf8'); + let content = await fs.readFile('teams-manifest.json', 'utf8'); content = content.replace(/\@\@marketplaceId/gi, marketplaceId); content = content.replace(/\@\@botName/gi, botName); diff --git a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts index e93b6912..c822bfa5 100644 --- a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts +++ b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts @@ -31,7 +31,7 @@ import mime from 'mime-types'; import urlJoin from 'url-join'; import path from 'path'; -import fs from 'fs'; +import fs from 'fs/promises'; import { GBLog, GBMinInstance, GBService, IGBPackage } from 'botlib'; import { CollectionUtil } from 'pragmatismo-io-framework'; import { GBServer } from '../../../src/app.js'; @@ -57,6 +57,7 @@ import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js'; import { createBot } from 'whatsapp-cloud-api'; import { promisify } from 'util'; const stat = promisify(fs.stat); +import {createReadStream} from 'fs'; /** * Support for Whatsapp. @@ -153,9 +154,9 @@ export class WhatsappDirectLine extends GBService { const gbaiPath = GBUtil.getGBAIPath(this.min.botId); const webVersion = '2.2412.51'; const localName = path.join('work', gbaiPath, 'profile'); - const createClient = () => { + const createClient = async () => { const client = (this.customClient = new Client({ - puppeteer: GBSSR.preparePuppeteer(localName), + puppeteer: await GBSSR.preparePuppeteer(localName), webVersionCache: { type: 'remote', remotePath: `https://raw.githubusercontent.com/wppconnect-team/wa-version/main/html/${webVersion}.html` @@ -187,7 +188,7 @@ export class WhatsappDirectLine extends GBService { 'cache', `qr${GBAdminService.getRndReadableIdentifier()}.png` ); - fs.writeFileSync(localName, qrBuf.data); + fs.writeFile(localName, qrBuf.data); const url = urlJoin(GBServer.globals.publicAddress, this.min.botId, 'cache', path.basename(localName)); if (adminNumber) { @@ -330,7 +331,7 @@ export class WhatsappDirectLine extends GBService { 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.docx` ); - fs.writeFileSync(localName, buf, { encoding: null }); + fs.writeFile(localName, buf, { encoding: null }); const url = urlJoin(GBServer.globals.publicAddress, this.min.botId, 'cache', path.basename(localName)); attachments = []; @@ -1200,7 +1201,7 @@ export class WhatsappDirectLine extends GBService { public async uploadLargeFile(min, filePath) { const CHUNK_SIZE = 4 * 1024 * 1024; // 4MB chunks let uploadSessionId; - const fileSize = (await stat(filePath)).size; + const fileSize = (await fs.stat(filePath)).size; const fileName = filePath.split('/').pop(); const fileType = mime.lookup(filePath); const appId = this.whatsappFBAppId; @@ -1231,7 +1232,7 @@ export class WhatsappDirectLine extends GBService { while (startOffset < fileSize) { const endOffset = Math.min(startOffset + CHUNK_SIZE, fileSize); - const fileStream = fs.createReadStream(filePath, { start: startOffset, end: endOffset - 1 }); + const fileStream = createReadStream(filePath, { start: startOffset, end: endOffset - 1 }); const chunkSize = endOffset - startOffset; const uploadResponse = await fetch(`https://graph.facebook.com/v20.0/upload:${uploadSessionId}`, { diff --git a/src/app.ts b/src/app.ts index 6e489a7e..32b0b4c1 100644 --- a/src/app.ts +++ b/src/app.ts @@ -40,7 +40,7 @@ import bodyParser from 'body-parser'; import { GBLog, GBMinInstance, IGBCoreService, IGBInstance } from 'botlib'; import child_process from 'child_process'; import express from 'express'; -import fs from 'fs'; +import fs from 'fs/promises'; import http from 'http'; import httpProxy from 'http-proxy'; import https from 'https'; @@ -70,7 +70,7 @@ export class GBServer { * Program entry-point. */ - public static run() { + public static async run() { GBLogEx.info(0, `The Bot Server is in STARTING mode...`); GBServer.globals = new RootData(); GBConfigService.init(); @@ -140,7 +140,7 @@ export class GBServer { process.env.PWD = process.cwd(); const workDir = path.join(process.env.PWD, 'work'); - if (!fs.existsSync(workDir)) { + if (!await GBUtil.exists(workDir)) { mkdirp.sync(workDir); } @@ -205,7 +205,7 @@ export class GBServer { // Deployment of local applications for the first time. if (GBConfigService.get('DISABLE_WEB') !== 'true') { - deployer.setupDefaultGBUI(); + await deployer.setupDefaultGBUI(); } GBLogEx.info(0, `Publishing instances...`); @@ -301,7 +301,7 @@ export class GBServer { }; if (process.env.CERTIFICATE_PFX) { - const server1 = http.createServer((req, res) => { + const server1 = http.createServer(async (req, res) => { const host = req.headers.host.startsWith('www.') ? req.headers.host.substring(4) : req.headers.host; res @@ -314,8 +314,8 @@ export class GBServer { const options1 = { passphrase: process.env.CERTIFICATE_PASSPHRASE, - pfx: fs.readFileSync(process.env.CERTIFICATE_PFX), - ca: fs.existsSync(process.env.CERTIFICATE_CA) ? fs.readFileSync(process.env.CERTIFICATE_CA) : null + pfx: await fs.readFile(process.env.CERTIFICATE_PFX), + ca: await GBUtil.exists(process.env.CERTIFICATE_CA) ? await fs.readFile(process.env.CERTIFICATE_CA) : null }; const httpsServer = https.createServer(options1, server).listen(port, mainCallback); @@ -329,7 +329,7 @@ export class GBServer { if (process.env[certPfxEnv] && process.env[certPassphraseEnv] && process.env[certDomainEnv]) { const options = { passphrase: process.env[certPassphraseEnv], - pfx: fs.readFileSync(process.env[certPfxEnv]) + pfx: await fs.readFile(process.env[certPfxEnv]) }; httpsServer.addContext(process.env[certDomainEnv], options); } else { @@ -348,7 +348,7 @@ export class GBServer { // A workaround for swagger-ui-dist not being able to set custom swagger URL const indexContent = fs - .readFileSync(path.join(swaggerUiAssetPath, 'swagger-initializer.js')) + .readFile(path.join(swaggerUiAssetPath, 'swagger-initializer.js')) .toString() .replace('https://petstore.swagger.io/v2/swagger.json', `/${SWAGGER_FILE_NAME}`); app.get(`${ENDPOINT}/swagger-initializer.js`, (req, res) => res.send(indexContent)); diff --git a/src/util.ts b/src/util.ts index 3d5c7ddc..ab9130fd 100644 --- a/src/util.ts +++ b/src/util.ts @@ -35,10 +35,13 @@ 'use strict'; import * as YAML from 'yaml'; import SwaggerClient from 'swagger-client'; -import fs from 'fs'; +import fs from 'fs/promises'; import { GBConfigService } from '../packages/core.gbapp/services/GBConfigService.js'; import path from 'path'; -import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs'; +import { VerbosityLevel, getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs'; +VerbosityLevel.ERRORS=0; +VerbosityLevel.WARNINGS=0; +VerbosityLevel.INFOS=0; import { Page } from 'puppeteer'; import urljoin from 'url-join'; import html2md from 'html-to-md'; @@ -74,7 +77,7 @@ export class GBUtil { public static async getDirectLineClient(min) { let config = { - spec: JSON.parse(fs.readFileSync('directline-3.0.json', 'utf8')), + spec: JSON.parse(await fs.readFile('directline-3.0.json', 'utf8')), requestInterceptor: req => { req.headers['Authorization'] = `Bearer ${min.instance.webchatKey}`; } @@ -130,21 +133,93 @@ export class GBUtil { } } - public static copyIfNewerRecursive(src, dest) { - if (!fs.existsSync(src)) { + public static async exists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; // File exists + } catch (error) { + return false; // File does not exist + } + } + + public static async savePage(url: string, page: Page, directoryPath: string): Promise { + try { + // Check if the directory exists, create it if not + const directoryExists = await this.fileExists(directoryPath); + if (!directoryExists) { + await fs.mkdir(directoryPath, { recursive: true }); // Create directory if it doesn't exist + } + + // Check if the URL is for a downloadable file (e.g., .pdf) + if (url.endsWith('.pdf')) { + const response = await fetch(url); + + if (!response.ok) { + throw new Error('Failed to download the file'); + } + + const buffer = await response.arrayBuffer(); // Convert response to array buffer + const fileName = path.basename(url); // Extract file name from URL + const filePath = path.join(directoryPath, fileName); // Create file path + + const data = new Uint8Array(buffer); + const text = await GBUtil.getPdfText(data); + + // Write the buffer to the file asynchronously + await fs.writeFile(filePath, text); + + return filePath; // Return the saved file path + } else { + // Use Puppeteer for non-downloadable pages + + const parsedUrl = new URL(url); + + // Get the last part of the URL path or default to 'index' if empty + const pathParts = parsedUrl.pathname.split('/').filter(Boolean); // Remove empty parts + const lastPath = pathParts.length > 0 ? pathParts[pathParts.length - 1] : 'index'; + const flatLastPath = lastPath.replace(/\W+/g, '-'); // Flatten the last part of the path + + const fileName = `${flatLastPath}.html`; + const filePath = path.join(directoryPath, fileName); + + const htmlContent = await page.content(); + + // Write HTML content asynchronously + await fs.writeFile(filePath, htmlContent); + + return filePath; + } + } catch (error) { + console.error('Error saving page:', error); + return null; + } + } + + public static async fileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch (error) { + return false; + } + } + + + public static async copyIfNewerRecursive(src, dest) { + if (!await GBUtil.exists(src)) { console.error(`Source path "${src}" does not exist.`); return; } // Check if the source is a directory - if (fs.statSync(src).isDirectory()) { + if ((await fs.stat(src)).isDirectory()) { // Create the destination directory if it doesn't exist - if (!fs.existsSync(dest)) { - fs.mkdirSync(dest, { recursive: true }); + if (!await GBUtil.exists(dest)) { + fs.mkdir(dest, { recursive: true }); } // Read all files and directories in the source directory - const entries = fs.readdirSync(src); + const entries =await fs.readdir(src); for (let entry of entries) { const srcEntry = path.join(src, entry); @@ -155,17 +230,17 @@ export class GBUtil { } } else { // Source is a file, check if we need to copy it - if (fs.existsSync(dest)) { - const srcStat = fs.statSync(src); - const destStat = fs.statSync(dest); + if (await GBUtil.exists(dest)) { + const srcStat =await fs.stat(src); + const destStat =await fs.stat(dest); // Copy only if the source file is newer than the destination file if (srcStat.mtime > destStat.mtime) { - fs.cpSync(src, dest, { force: true }); + fs.cp(src, dest, { force: true }); } } else { // Destination file doesn't exist, so copy it - fs.cpSync(src, dest, { force: true }); + fs.cp(src, dest, { force: true }); } } } @@ -179,8 +254,9 @@ export class GBUtil { return false; } - public static async getPdfText(data: Buffer): Promise { - const pdf = await getDocument({ data }).promise; + public static async getPdfText(data): Promise { + + const pdf = await getDocument({data}).promise; let pages = []; for (let i = 1; i <= pdf.numPages; i++) { @@ -188,12 +264,12 @@ export class GBUtil { const textContent = await page.getTextContent(); const text = textContent.items .map(item => item['str']) - .join('') - .replace(/\s/g, ''); // Optionally remove extra spaces + .join(' ') + .replace(/\s+/g, ' '); // Optionally remove extra spaces pages.push(text); } - return pages.join(''); + return pages.join(' '); } static getGBAIPath(botId, packageType = null, packageName = null) { @@ -211,58 +287,5 @@ export class GBUtil { } } - public static async savePage(url: string, page: Page, directoryPath: string): Promise { - - let response = await page.goto(url); - if (!response) { - response = await page.waitForResponse(() => true); - } - - if (response && response.headers && response.status() === 200) { - const contentType = response.headers()['content-type']; - - if (contentType) { - const urlObj = new URL(url); - const urlPath = urlObj.pathname.endsWith('/') ? urlObj.pathname.slice(0, -1) : urlObj.pathname; - let filename = urlPath.split('/').pop() || 'index'; - - fs.mkdirSync(directoryPath, { recursive: true }); - - const extensionMap = { - 'text/html': 'html', - 'application/pdf': 'pdf', - 'text/plain': 'txt', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx', - 'application/json': 'json', - 'application/xml': 'xml', - 'text/csv': 'csv', - 'application/x-httpd-php': 'php', - 'application/javascript': 'js', - 'text/javascript': 'js', - 'text/css': 'css', - 'text/xml': 'xml' - }; - - const extension = Object.keys(extensionMap).find(key => contentType.includes(key)) || 'bin'; - filename = `${filename}.${extension}`; - const filePath = path.join(directoryPath, filename); - - let fileContent; - if (extension === 'html') { - fileContent = html2md(await response.text()); - } else if (extension === 'pdf') { - const pdfBuffer = await response.buffer(); - fileContent = await GBUtil.getPdfText(pdfBuffer); // Extract text from the PDF - } else { - fileContent = await response.buffer(); - } - - fs.writeFileSync(filePath, fileContent); - - return filePath; - } - } - return null; - } }