diff --git a/packages/basic.gblib/services/DialogKeywords.ts b/packages/basic.gblib/services/DialogKeywords.ts index 8b889eed..0bed426b 100644 --- a/packages/basic.gblib/services/DialogKeywords.ts +++ b/packages/basic.gblib/services/DialogKeywords.ts @@ -37,7 +37,7 @@ import urlJoin from 'url-join'; import { GBServer } from '../../../src/app.js'; import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; import { SecService } from '../../security.gbapp/services/SecService.js'; -import {Jimp} from 'jimp'; +import { Jimp } from 'jimp'; import jsQR from 'jsqr'; import { SystemKeywords } from './SystemKeywords.js'; import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; @@ -254,28 +254,28 @@ export class DialogKeywords { * * @example EXIT */ - public async exit({}) {} + public async exit({ }) { } /** * Get active tasks. * * @example list = ACTIVE TASKS */ - public async getActiveTasks({ pid }) {} + public async getActiveTasks({ pid }) { } /** * Creates a new deal. * * @example CREATE DEAL dealname,contato,empresa,amount */ - public async createDeal({ pid, dealName, contact, company, amount }) {} + public async createDeal({ pid, dealName, contact, company, amount }) { } /** * Finds contacts in XRM. * * @example list = FIND CONTACT "Sandra" */ - public async fndContact({ pid, name }) {} + public async fndContact({ pid, name }) { } public getContentLocaleWithCulture(contentLocale) { switch (contentLocale) { @@ -936,7 +936,7 @@ export class DialogKeywords { * @example MENU * */ - public async showMenu({}) { + public async showMenu({ }) { // https://github.com/GeneralBots/BotServer/issues/237 // return await beginDialog('/menu'); } @@ -1215,20 +1215,20 @@ export class DialogKeywords { const handle = WebAutomationServices.cyrb53({ pid, str: min.botId + answer.filename }); GBServer.globals.files[handle] = answer; - // Load the image with Jimp - const image = await Jimp.read(answer.data); + // Load the image with Jimp + const image = await Jimp.read(answer.data); - // Get the image data - const imageData = { - data: new Uint8ClampedArray(image.bitmap.data), - width: image.bitmap.width, - height: image.bitmap.height, - }; + // Get the image data + const imageData = { + data: new Uint8ClampedArray(image.bitmap.data), + width: image.bitmap.width, + height: image.bitmap.height, + }; - // Use jsQR to decode the QR code - const decodedQR = jsQR(imageData.data, imageData.width, imageData.height); + // Use jsQR to decode the QR code + const decodedQR = jsQR(imageData.data, imageData.width, imageData.height); - result = decodedQR.data; + result = decodedQR.data; } else if (kind === 'zipcode') { const extractEntity = (text: string) => { @@ -1455,6 +1455,36 @@ export class DialogKeywords { let nameOnly; const gbaiName = GBUtil.getGBAIPath(min.botId); + if (filename.endsWith('.pdf')) { + const gbdriveName = GBUtil.getGBAIPath(min.botId, 'gbdrive'); + const pdf = path.join(GBConfigService.get('STORAGE_LIBRARY'), gbdriveName, filename); + + const pngs = await GBUtil.pdfPageAsImage(min, pdf, undefined); + + await CollectionUtil.asyncForEach(pngs, async png => { + + + // Prepare a cache to be referenced by Bot Framework. + + url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(png.localName)); + + const contentType = mime.lookup(url); + const reply = { type: ActivityTypes.Message, text: caption }; + reply['attachments'] = []; + reply['attachments'].push({ + name: nameOnly, + contentType: contentType, + contentUrl: url + }); + + if (channel === 'omnichannel' || !user) { + await min.conversationalService.sendFile(min, null, mobile, url, caption); + } else { + await min.conversationalService['sendOnConversation'](min, user, reply); + } + }); + } + // Web automation. if (element) { @@ -1489,32 +1519,37 @@ export class DialogKeywords { // .gbdrive direct sending. else { - const ext = path.extname(filename); - const gbaiName = GBUtil.getGBAIPath(min.botId); - let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min); - const fileUrl = urlJoin('/', gbaiName, `${min.botId}.gbdrive`, filename); - GBLogEx.info(min, `Direct send from .gbdrive: ${fileUrl} to ${mobile}.`); - const sys = new SystemKeywords(); + if (GBConfigService.get('STORAGE_NAME')) { - const pathOnly = fileUrl.substring(0, fileUrl.lastIndexOf('/')); - const fileOnly = fileUrl.substring(fileUrl.lastIndexOf('/') + 1); + const ext = path.extname(filename); + const gbaiName = GBUtil.getGBAIPath(min.botId); - let template = await sys.internalGetDocument(client, baseUrl, pathOnly, fileOnly); + let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min); + const fileUrl = urlJoin('/', gbaiName, `${min.botId}.gbdrive`, filename); + GBLogEx.info(min, `Direct send from .gbdrive: ${fileUrl} to ${mobile}.`); - const driveUrl = template['@microsoft.graph.downloadUrl']; - const res = await fetch(driveUrl); - let buf: any = Buffer.from(await res.arrayBuffer()); - let localName1 = path.join( - 'work', - gbaiName, - 'cache', - `${fileOnly.replace(/\s/gi, '')}-${GBAdminService.getNumberIdentifier()}.${ext}` - ); - await fs.writeFile(localName1, buf, { encoding: null }); + const sys = new SystemKeywords(); - url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName1)); + const pathOnly = fileUrl.substring(0, fileUrl.lastIndexOf('/')); + const fileOnly = fileUrl.substring(fileUrl.lastIndexOf('/') + 1); + + let template = await sys.internalGetDocument(client, baseUrl, pathOnly, fileOnly); + + const driveUrl = template['@microsoft.graph.downloadUrl']; + const res = await fetch(driveUrl); + let buf: any = Buffer.from(await res.arrayBuffer()); + let localName1 = path.join( + 'work', + gbaiName, + 'cache', + `${fileOnly.replace(/\s/gi, '')}-${GBAdminService.getNumberIdentifier()}.${ext}` + ); + await fs.writeFile(localName1, buf, { encoding: null }); + + url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName1)); + } } if (!url) { diff --git a/packages/basic.gblib/services/KeywordsExpressions.ts b/packages/basic.gblib/services/KeywordsExpressions.ts index f12090de..1336e4d5 100644 --- a/packages/basic.gblib/services/KeywordsExpressions.ts +++ b/packages/basic.gblib/services/KeywordsExpressions.ts @@ -775,14 +775,14 @@ export class KeywordsExpressions { // Handles the GET http version. else { - const value = $2.replace(/\`/gi, ''); + return ` - if (value.endsWith('.pdf') && !value.startsWith('https')) { - return `${$1} = await sys.getPdf({pid: pid, file: ${$2}});`; + if (${$2}.endsWith('.pdf') && !${$2}.startsWith('https')) { + ${$1} = await sys.getPdf({pid: pid, file: ${$2}}); } else { - return ` + let __${$1} = null - await retry( + await retry( async (bail) => { await ensureTokens(); @@ -790,11 +790,10 @@ export class KeywordsExpressions { },{ retries: 5}); ${$1} = __${$1} - __${$1} = null - + __${$1} = null + } `; } - } } ]; diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index a55347a8..8c75b4b9 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -952,7 +952,7 @@ export class SystemKeywords { GBLogEx.info(min, `GET '${addressOrHeaders}' in '${file}'.`); let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min); const botId = min.instance.botId; - (''); + const packagePath = GBUtil.getGBAIPath(botId, 'gbdata'); let document = await this.internalGetDocument(client, baseUrl, packagePath, file); diff --git a/packages/llm.gblib/services/ChatServices.ts b/packages/llm.gblib/services/ChatServices.ts index 31960626..70eee696 100644 --- a/packages/llm.gblib/services/ChatServices.ts +++ b/packages/llm.gblib/services/ChatServices.ts @@ -149,8 +149,8 @@ export class GBLLMOutputParser extends BaseLLMOutputParser { const localName = path.join(process.env.PWD, 'work', gbaiName, 'docs', source.file); if (localName) { - const { url } = await ChatServices.pdfPageAsImage(this.min, localName, source.page); - text = `![alt text](${url}) + const pngs = await GBUtil.pdfPageAsImage(this.min, localName, source.page); + text = `![alt text](${pngs[0].url}) ${text}`; found = true; source.file = localName; @@ -167,30 +167,7 @@ export class GBLLMOutputParser extends BaseLLMOutputParser { } export class ChatServices { - public static async pdfPageAsImage(min, filename, pageNumber) { - // Converts the PDF to PNG. - GBLogEx.info(min, `Converting ${filename}, page: ${pageNumber}...`); - const pngPages: PngPageOutput[] = await pdfToPng(filename, { - disableFontFace: true, - useSystemFonts: true, - viewportScale: 2.0, - pagesToProcess: [pageNumber], - strictPagesToProcess: false, - verbosityLevel: 0 - }); - - // Prepare an image on cache and return the GBFILE information. - - if (pngPages.length > 0) { - const buffer = pngPages[0].content; - const gbaiName = GBUtil.getGBAIPath(min.botId, null); - const localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`); - const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName)); - await fs.writeFile(localName, buffer, { encoding: null }); - return { localName: localName, url: url, data: buffer }; - } - } private static async getRelevantContext( vectorStore: HNSWLib, diff --git a/src/util.ts b/src/util.ts index 64a95942..fe3f666b 100644 --- a/src/util.ts +++ b/src/util.ts @@ -42,8 +42,12 @@ import { VerbosityLevel, getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs'; VerbosityLevel.ERRORS = 0; VerbosityLevel.WARNINGS = 0; VerbosityLevel.INFOS = 0; -import { Page } from 'puppeteer'; import urljoin from 'url-join'; +import { GBAdminService } from '../packages/admin.gbapp/services/GBAdminService.js'; +import { GBLogEx } from '../packages/core.gbapp/services/GBLogEx.js'; +import { PngPageOutput, pdfToPng } from 'pdf-to-png-converter'; +import urlJoin from 'url-join'; +import { GBServer } from './app.js'; export class GBUtil { public static repeat(chr, count) { @@ -244,4 +248,37 @@ export class GBUtil { return urljoin(gbai, packageName ? packageName : `${botId}.${packageType}`); } } + + public static async pdfPageAsImage(min, filename, pageNumber) { + // Converts the PDF to PNG. + + GBLogEx.info(min, `Converting ${filename}, page: ${pageNumber ?? 'all'}...`); + + const options = { + disableFontFace: true, + useSystemFonts: true, + viewportScale: 2.0, + pagesToProcess: pageNumber !== undefined ? [pageNumber] : undefined, + strictPagesToProcess: false, + verbosityLevel: 0 + }; + + const pngPages: PngPageOutput[] = await pdfToPng(filename, options); + + const generatedFiles = []; + + for (const pngPage of pngPages) { + const buffer = pngPage.content; + const gbaiName = GBUtil.getGBAIPath(min.botId, null); + const localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`); + const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName)); + + await fs.writeFile(localName, buffer, { encoding: null }); + + generatedFiles.push({ localName: localName, url: url, data: buffer }); + } + + return generatedFiles.length > 0 ? generatedFiles : null; + } + } diff --git a/templates/ai-search.gbai/ai-search.gbdialog/qr.bas b/templates/ai-search.gbai/ai-search.gbdialog/qr.bas index 95090140..b3770af1 100644 --- a/templates/ai-search.gbai/ai-search.gbdialog/qr.bas +++ b/templates/ai-search.gbai/ai-search.gbdialog/qr.bas @@ -1,6 +1,7 @@ TALK "Please, take a photo of the QR Code and send to me." HEAR doc as QRCODE -text = GET "doc-" + doc + ".pdf" +TALK "Reading document " + doc + "..." +text = GET doc IF text THEN @@ -8,7 +9,10 @@ IF text THEN SET CONTEXT text SET ANSWER MODE "document" TALK "Document ${doc} loaded. You can ask me anything about it." + TALK "I will also send it to you..." + SEND FILE doc ELSE TALK "Document was not found, please try again." -END IF \ No newline at end of file +END IF + diff --git a/templates/ai-search.gbai/ai-search.gbdialog/start.bas b/templates/ai-search.gbai/ai-search.gbdialog/start.bas index 54280876..f9d5075d 100644 --- a/templates/ai-search.gbai/ai-search.gbdialog/start.bas +++ b/templates/ai-search.gbai/ai-search.gbdialog/start.bas @@ -1,3 +1,14 @@ +BEGIN TALK + Welcome to the General Bots AI Search Template! + + We are here to assist you! To get started, please choose one of the options below: + + 1 - Scan a QR Code: Send a photo of the QR Code you would like to scan typping 'QR'. + + 2 - Find a Procedure: If you need information about a specific procedure, just let me know what it is, and I will help you! + + Let’s get started! +END TALK BEGIN SYSTEM PROMPT diff --git a/templates/ai-search.gbai/ai-search.gbdrive/42LB5800.pdf b/templates/ai-search.gbai/ai-search.gbdrive/42LB5800.pdf new file mode 100644 index 00000000..314a490c Binary files /dev/null and b/templates/ai-search.gbai/ai-search.gbdrive/42LB5800.pdf differ diff --git a/templates/ai-search.gbai/ai-search.gbdrive/doc-hello.pdf b/templates/ai-search.gbai/ai-search.gbdrive/doc-hello.pdf deleted file mode 100644 index dae30a53..00000000 Binary files a/templates/ai-search.gbai/ai-search.gbdrive/doc-hello.pdf and /dev/null differ