From 93c6c51f3a4508cbe7fd16068aa3f3fa60cd358a Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Mon, 9 Dec 2024 08:07:34 -0300 Subject: [PATCH] fix(llm.gblib): Fix in doc. publishing. --- package.json | 1 + .../basic.gblib/services/SystemKeywords.ts | 71 +++++++++++-------- packages/kb.gbapp/services/KBService.ts | 22 +++++- 3 files changed, 61 insertions(+), 33 deletions(-) diff --git a/package.json b/package.json index 723ddf3e..f9aad051 100644 --- a/package.json +++ b/package.json @@ -237,6 +237,7 @@ "walk-promise": "0.2.0", "washyourmouthoutwithsoap": "1.0.2", "webdav-server": "2.6.2", + "webp-converter": "^2.3.3", "whatsapp-cloud-api": "0.3.1", "whatsapp-web.js": "1.26.1-alpha.1", "winston": "3.14.2", diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index e9d694db..47f9fd6e 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -780,10 +780,19 @@ export class SystemKeywords { }); GBLogEx.info(min, `SAVE '${table}': ${rows.length} row(s).`); + // Capture the values we need for retries + const tableName = table; + const minRef = min; + await retry( - async bail => { - await t.bulkCreate(rowsDest); - rowsDest = null; + async (bail) => { + const t = this.getTableFromName(tableName, minRef); + try { + await t.bulkCreate(rowsDest); + rowsDest = null; + } catch (error) { + throw error; + } }, { retries: 5, @@ -1690,7 +1699,7 @@ export class SystemKeywords { const dstPath = urlJoin(packagePath, dest); - if (path.extname(srcPath) === 'ai'){ + if (path.extname(srcPath) === 'ai') { // TODO: To be done. @@ -1733,7 +1742,7 @@ export class SystemKeywords { } throw error; } - } + } } /** @@ -2897,31 +2906,31 @@ export class SystemKeywords { await this.setMemoryContext({ pid, erase: true }); } - + public async convertAI2HTML(aiFilePath) { - - // Convert the AI file to HTML and assets - const result = await ai2html.convertFile(aiFilePath, { - outputFormat: 'html', - outputWriteMethod: 'write-file', - outputPath: path.dirname(aiFilePath), - useDocumentSettings: true, - }); - - // Get the generated HTML file path - const htmlFilePath = result.outputFiles.find((file) => file.endsWith('.html')).filePath; - - // Read the HTML content - const htmlContent = await fs.readFile(htmlFilePath, 'utf8'); - - // Save the HTML and assets to a cache directory - const cacheDir = path.join('work', 'cache'); - await fs.mkdir(cacheDir, { recursive: true }); - const cacheFilePath = path.join(cacheDir, path.basename(htmlFilePath)); - await fs.writeFile(cacheFilePath, htmlContent); - - return cacheFilePath; - + + // Convert the AI file to HTML and assets + const result = await ai2html.convertFile(aiFilePath, { + outputFormat: 'html', + outputWriteMethod: 'write-file', + outputPath: path.dirname(aiFilePath), + useDocumentSettings: true, + }); + + // Get the generated HTML file path + const htmlFilePath = result.outputFiles.find((file) => file.endsWith('.html')).filePath; + + // Read the HTML content + const htmlContent = await fs.readFile(htmlFilePath, 'utf8'); + + // Save the HTML and assets to a cache directory + const cacheDir = path.join('work', 'cache'); + await fs.mkdir(cacheDir, { recursive: true }); + const cacheFilePath = path.join(cacheDir, path.basename(htmlFilePath)); + await fs.writeFile(cacheFilePath, htmlContent); + + return cacheFilePath; + } @@ -2997,7 +3006,7 @@ export class SystemKeywords { }; // If the column is named 'id' or 'Id', set it as the primary key - if (! pkAdded && (col.toLowerCase() === 'id' || col.toLowerCase() === 'internal_id')) { + if (!pkAdded && (col.toLowerCase() === 'id' || col.toLowerCase() === 'internal_id')) { schema[col].primaryKey = true; pkAdded = true; } @@ -3041,5 +3050,5 @@ export class SystemKeywords { // Close SQLite connection await sqlite.close(); -} + } } diff --git a/packages/kb.gbapp/services/KBService.ts b/packages/kb.gbapp/services/KBService.ts index 8e5c2145..a3af4e86 100644 --- a/packages/kb.gbapp/services/KBService.ts +++ b/packages/kb.gbapp/services/KBService.ts @@ -73,6 +73,8 @@ import textract from 'textract'; import { GBUtil } from '../../../src/util.js'; import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; import { AzureDeployerService } from '../../azuredeployer.gbapp/services/AzureDeployerService.js'; +import webp from 'webp-converter'; +import os from 'os'; import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js'; import { GBVMService } from '../../basic.gblib/services/GBVMService.js'; import { GuaribasPackage } from '../../core.gbapp/models/GBModel.js'; @@ -1080,6 +1082,18 @@ export class KBService implements IGBKBService { throw new Error('Failed to parse ICO file'); } buffer = Buffer.from(images[0].buffer); + } else if (buffer.slice(0, 4).toString('hex') === '52494646' && + buffer.slice(8, 12).toString('hex') === '57454250') { + + // Convert WebP to PNG using temporary files + const tempWebP = path.join(os.tmpdir(), `temp-${Date.now()}.webp`); + const tempPNG = path.join(os.tmpdir(), `temp-${Date.now()}.png`); + + await fs.writeFile(tempWebP, buffer); + await webp.dwebp(tempWebP, tempPNG, "-o"); + + buffer = await fs.readFile(tempPNG); + } else if (buffer.toString().includes(' request.abort()); + const parsedUrl = new URL(url);