diff --git a/package.json b/package.json index 19988d6fd..13e30636d 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "5.0.0", "description": "General Bot Community Edition open-core server.", "main": "./boot.mjs", - "type":"module", + "type": "module", "bugs": "https://github.com/pragmatismo-io/BotServer/issues", "homepage": "https://github.com/pragmatismo-io/BotServer/#readme", "contributors": [ @@ -14,6 +14,9 @@ "Dário Vieira ", "Alan Perdomo " ], + "opencv4nodejs": { + "disableAutoBuild": "1" + }, "engines": { "node": "=22.9.0" }, @@ -99,6 +102,7 @@ "@sequelize/core": "7.0.0-alpha.37", "@types/node": "22.5.2", "@types/validator": "13.12.1", + "@u4/opencv4nodejs": "7.1.2", "adm-zip": "0.5.16", "alasql": "4.5.1", "any-shell-escape": "0.1.1", @@ -136,7 +140,9 @@ "express-remove-route": "1.0.0", "facebook-nodejs-business-sdk": "^20.0.2", "ffmpeg-static": "5.2.0", + "formidable": "^3.5.1", "get-image-colors": "4.0.1", + "glob": "^11.0.0", "google-libphonenumber": "3.2.38", "googleapis": "143.0.0", "hnswlib-node": "3.0.0", @@ -146,8 +152,10 @@ "instagram-private-api": "1.46.1", "iso-639-1": "3.1.3", "isomorphic-fetch": "3.0.0", + "jimp": "^1.6.0", "js-md5": "0.8.3", "json-schema-to-zod": "2.4.0", + "jsqr": "^1.4.0", "just-indent": "0.0.1", "keyv": "5.0.1", "koa": "2.15.3", @@ -194,6 +202,7 @@ "puppeteer-extra-plugin-stealth": "2.11.2", "qr-scanner": "1.4.2", "qrcode": "1.5.4", + "qrcode-reader": "^1.0.4", "qrcode-terminal": "0.12.0", "readline": "1.3.0", "reflect-metadata": "0.2.2", diff --git a/packages/basic.gblib/services/DialogKeywords.ts b/packages/basic.gblib/services/DialogKeywords.ts index b13ac16ed..8b889eed4 100644 --- a/packages/basic.gblib/services/DialogKeywords.ts +++ b/packages/basic.gblib/services/DialogKeywords.ts @@ -37,6 +37,8 @@ import urlJoin from 'url-join'; import { GBServer } from '../../../src/app.js'; import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; import { SecService } from '../../security.gbapp/services/SecService.js'; +import {Jimp} from 'jimp'; +import jsQR from 'jsqr'; import { SystemKeywords } from './SystemKeywords.js'; import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; import { Messages } from '../strings.js'; @@ -61,6 +63,7 @@ import { GBUtil } from '../../../src/util.js'; import { GBVMService } from './GBVMService.js'; import { ChatServices } from '../../../packages/llm.gblib/services/ChatServices.js'; import puppeteer from 'puppeteer'; +import QRCodeProcessor from './QRCodeServices.js'; /** * Default check interval for user replay @@ -1206,14 +1209,27 @@ export class DialogKeywords { } result = phoneNumber; - } else if (kind === 'qr-scanner') { + } else if (kind === 'qrcode') { //https://github.com/GeneralBots/BotServer/issues/171 - GBLogEx.info(min, `BASIC (${min.botId}): Upload done for ${answer.filename}.`); + GBLogEx.info(min, `BASIC (${min.botId}): QRCode for ${answer.filename}.`); const handle = WebAutomationServices.cyrb53({ pid, str: min.botId + answer.filename }); GBServer.globals.files[handle] = answer; - QrScanner.scanImage(GBServer.globals.files[handle]) - .then(result => console.log(result)) - .catch(error => console.log(error || 'no QR code found.')); + + // Load the image with Jimp + const image = await Jimp.read(answer.data); + + // Get the image data + const imageData = { + data: new Uint8ClampedArray(image.bitmap.data), + width: image.bitmap.width, + height: image.bitmap.height, + }; + + // Use jsQR to decode the QR code + const decodedQR = jsQR(imageData.data, imageData.width, imageData.height); + + result = decodedQR.data; + } else if (kind === 'zipcode') { const extractEntity = (text: string) => { text = text.replace(/\-/gi, ''); diff --git a/packages/basic.gblib/services/QRCodeServices.ts b/packages/basic.gblib/services/QRCodeServices.ts new file mode 100644 index 000000000..54f36b655 --- /dev/null +++ b/packages/basic.gblib/services/QRCodeServices.ts @@ -0,0 +1,99 @@ +import cv from '@u4/opencv4nodejs' +import QRCodeReader from 'qrcode-reader'; + +class QRCodeProcessor { + async decodeQRCode(imagePath) { + const image = cv.imread(imagePath, cv.IMREAD_COLOR); + const grayImage = image.bgrToGray(); + const blurredImage = grayImage.gaussianBlur(new cv.Size(5, 5), 0); + const edges = blurredImage.canny(50, 150); + + const contour = this.findQRCodeContour(edges); + if (!contour) { + throw new Error('QR Code não encontrado.'); + } + + const transformedImage = this.getPerspectiveTransform(image, contour); + return await this.readQRCode(transformedImage); + } + + findQRCodeContour(edges) { + const contours = edges.findContours(cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE); + let maxContour = null; + let maxArea = 0; + + contours.forEach(contour => { + const area = contour.area; + if (area > maxArea) { + maxArea = area; + maxContour = contour; + } + }); + + return maxContour; + } + + getPerspectiveTransform(image, contour) { + // Ensure the contour has at least 4 points + const points = contour.getPoints(); + if (points.length < 4) { + throw new Error("Contour must have at least 4 points."); + } + + // Get the first four points + const srcPoints = points.slice(0, 4).map(point => new cv.Point2(point.x, point.y)); + + // Define destination points for the perspective transform + const dst = [ + new cv.Point2(0, 0), + new cv.Point2(300, 0), + new cv.Point2(300, 300), + new cv.Point2(0, 300) + ]; + + // Get the perspective transform matrix + const M = cv.getPerspectiveTransform(srcPoints, dst); + + // Create a new Mat for the transformed image + const transformedImage = new cv.Mat(300, 300, cv.CV_8UC3); + + // Manually apply the perspective transformation + for (let y = 0; y < transformedImage.rows; y++) { + for (let x = 0; x < transformedImage.cols; x++) { + const srcPoint = this.applyPerspective(M, x, y); + const srcX = Math.round(srcPoint.x); + const srcY = Math.round(srcPoint.y); + + // Check if the mapped source point is within the bounds of the source image + if (srcX >= 0 && srcX < image.cols && srcY >= 0 && srcY < image.rows) { + const pixelValue = image.atVector(srcY, srcX); // Use atVector to get pixel values + transformedImage.set(y, x, pixelValue); + } + } + } + + return transformedImage; + } + + applyPerspective(M, x, y) { + const a = M.getData(); // Get the matrix data + const denominator = a[6] * x + a[7] * y + 1; // Calculate the denominator + const newX = (a[0] * x + a[1] * y + a[2]) / denominator; + const newY = (a[3] * x + a[4] * y + a[5]) / denominator; + return new cv.Point2(newX, newY); + } + + async readQRCode(image) { + return new Promise((resolve, reject) => { + const qrCodeReader = new QRCodeReader(); + qrCodeReader.decode(image.getData(), (err, result) => { + if (err) { + return reject(err); + } + resolve(result.result); + }); + }); + } +} + +export default QRCodeProcessor; diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index c8a0af071..e6be90302 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -465,7 +465,7 @@ export class GBDeployer implements IGBDeployer { } else { return []; } - GBLogEx.error(0, GBUtil.toYAML(rows)); + await asyncPromise.eachSeries(rows, async (line: any) => { if (line && line.length > 0) { const key = line[1]; diff --git a/packages/core.gbapp/services/GBMinService.ts b/packages/core.gbapp/services/GBMinService.ts index 43cb1f208..2baf555d1 100644 --- a/packages/core.gbapp/services/GBMinService.ts +++ b/packages/core.gbapp/services/GBMinService.ts @@ -135,6 +135,7 @@ export class GBMinService { public deployer: GBDeployer; bar1; + static pidsConversation = {}; /** * Static initialization of minimal instance. @@ -455,7 +456,7 @@ export class GBMinService { const status = req.body?.entry?.[0]?.changes?.[0]?.value?.statuses?.[0]; if (status) { - GBLogEx.info(min, `WhatsApp: ${status.recipient_id} ${status.status}`); + GBLogEx.verbose(min, `WhatsApp: ${status.recipient_id} ${status.status}`); return; } @@ -1040,7 +1041,7 @@ export class GBMinService { const step = await min.dialogs.createContext(context); step.context.activity.locale = 'pt-BR'; - + const sec = new SecService(); const member = context.activity.recipient ? context.activity.recipient : context.activity.from; let user = await sec.ensureUser(min, member.id, member.name, '', 'web', member.name, null); @@ -1094,8 +1095,27 @@ export class GBMinService { }); } } - - + + let conversationId = step.context.activity.conversation.id; + + let pid = GBMinService.pidsConversation[conversationId]; + + if (!pid) { + + pid = step.context.activity['pid']; + if (!pid) { + pid = WhatsappDirectLine.pidByNumber[context.activity.from.id]; + if (!pid) { + pid = GBVMService.createProcessInfo(user, min, step.context.activity.channelId, null, step); + } + } + } + GBMinService.pidsConversation[conversationId] = pid; + step.context.activity['pid'] = pid; + + const notes = min.core.getParam(min.instance, 'Notes', null); + await this.handleUploads(min, step, user, params, notes != null); + // Required for MSTEAMS handling of persisted conversations. if (step.context.activity.channelId === 'msteams') { @@ -1137,7 +1157,7 @@ export class GBMinService { min, `Auto start (teams) dialog is now being called: ${startDialog} for ${min.instance.botId}...` ); - + await GBVMService.callVM(startDialog.toLowerCase(), min, step, 0); } } @@ -1178,7 +1198,7 @@ export class GBMinService { const pid = GBVMService.createProcessInfo(user, min, step.context.activity.channelId, null, step); step.context.activity['pid'] = pid; - + min['conversationWelcomed'][step.context.activity.conversation.id] = true; GBLogEx.info( @@ -1195,16 +1215,16 @@ export class GBMinService { } } else if (context.activity.type === 'message') { - let pid = WhatsappDirectLine.pidByNumber[context.activity.from.id]; - + + // Required for F0 handling of persisted conversations. - + GBLogEx.info( min, `Human: pid:${pid} ${context.activity.from.id} ${GBUtil.toYAML(WhatsappDirectLine.pidByNumber)} ${context.activity.text} (type: ${context.activity.type}, name: ${context.activity.name}, channelId: ${context.activity.channelId})` ); - - + + // Processes messages activities. await this.processMessageActivity(context, min, step, pid); @@ -1302,26 +1322,28 @@ export class GBMinService { const url = attachment.contentUrl; const localFolder = 'work'; const packagePath = GBUtil.getGBAIPath(this['min'].botId); - const localFileName = path.join(localFolder, packagePath, 'uploads', attachment.name); + const localFileName = path.join(localFolder, packagePath, 'cache', attachment.name); let buffer; - if (url.startsWith('data:')) { - const base64Data = url.split(';base64,')[1]; - buffer = Buffer.from(base64Data, 'base64'); - } else { - const options = { - method: 'GET', - encoding: 'binary' - }; - const res = await fetch(url, options); - buffer = arrayBufferToBuffer(await res.arrayBuffer()); - } - - await fs.writeFile(localFileName, buffer); + // if (url.startsWith('data:')) { + // const base64Data = url.split(';base64,')[1]; + // buffer = Buffer.from(base64Data, 'base64'); + // } else { + // const options = { + // method: 'GET', + // encoding: 'binary' + // }; + // const res = await fetch(url, options); + // buffer = arrayBufferToBuffer(await res.arrayBuffer()); + // } + //write + buffer = await fs.readFile(localFileName); return { - fileName: attachment.name, - localPath: localFileName + name: attachment.name, + filename: localFileName, + url: url, + data: buffer }; } @@ -1343,7 +1365,7 @@ export class GBMinService { step.context.activity.attachments[0].contentType != 'text/html' ) { const promises = step.context.activity.attachments.map( - GBMinService.downloadAttachmentAndWrite.bind({ min, user, params }) + GBMinService.downloadAttachmentAndWrite.bind({ min, user, params }) ); const successfulSaves = await Promise.all(promises); async function replyForReceivedAttachments(attachmentData) { @@ -1352,14 +1374,15 @@ export class GBMinService { // a upload with no Dialog, so run Auto Save to .gbdrive. const t = new SystemKeywords(); - GBLogEx.info(min, `BASIC (${min.botId}): Upload done for ${attachmentData.fileName}.`); - const handle = WebAutomationServices.cyrb53({ pid: 0, str: min.botId + attachmentData.fileName }); - let data = await fs.readFile(attachmentData.localPath); + GBLogEx.info(min, `BASIC (${min.botId}): Upload done for ${attachmentData.filename}.`); + const handle = WebAutomationServices.cyrb53({ pid: 0, str: min.botId + attachmentData.filename }); + let data = await fs.readFile(attachmentData.filename); const gbfile = { - filename: attachmentData.localPath, + filename: path.join(process.env.PWD, attachmentData.filename), data: data, - name: path.basename(attachmentData.fileName) + url: attachmentData.url, + name: path.basename(attachmentData.filename) }; GBServer.globals.files[handle] = gbfile; @@ -1386,12 +1409,16 @@ export class GBMinService { class GBFile { data: Buffer; filename: string; + url: string; + name: string; } - const results = successfulSaves.reduce(async (accum: GBFile[], item) => { + const results = await successfulSaves.reduce(async (accum: GBFile[], item) => { const result: GBFile = { - data: await fs.readFile(successfulSaves[0]['localPath']), - filename: successfulSaves[0]['fileName'] + data: await fs.readFile(successfulSaves[0]['filename']), + filename: successfulSaves[0]['filename'], + name: successfulSaves[0]['name'], + url: successfulSaves[0]['url'], }; accum.push(result); return accum; @@ -1515,9 +1542,6 @@ export class GBMinService { } } - const notes = min.core.getParam(min.instance, 'Notes', null); - await this.handleUploads(min, step, user, params, notes != null); - // Files in .gbdialog can be called directly by typing its name normalized into JS . const isVMCall = Object.keys(min.scriptMap).find(key => min.scriptMap[key] === context.activity.text) !== undefined; @@ -1572,13 +1596,6 @@ export class GBMinService { step.context.activity['originalText']; step.context.activity['text'] = text; - if (notes && text && text !== '') { - const sys = new SystemKeywords(); - await sys.note({ pid, text }); - await step.context.sendActivity('OK.'); - return; - } - // Checks for bad words on input text. const hasBadWord = wash.check(step.context.activity.locale, text); diff --git a/packages/core.gbapp/services/router/bridge.ts b/packages/core.gbapp/services/router/bridge.ts index 38cf236c7..17582a508 100644 --- a/packages/core.gbapp/services/router/bridge.ts +++ b/packages/core.gbapp/services/router/bridge.ts @@ -1,3 +1,6 @@ +import fs from 'fs/promises'; +import formidable from 'formidable'; +import path from 'path'; import bodyParser from 'body-parser'; import express from 'express'; import fetch from 'isomorphic-fetch'; @@ -5,6 +8,9 @@ import moment from 'moment'; import * as uuidv4 from 'uuid'; import { IActivity, IBotData, IConversation, IConversationUpdateActivity, IMessageActivity } from './types'; import { GBConfigService } from '../GBConfigService.js'; +import { GBUtil } from '../../../../src/util.js'; +import urlJoin from 'url-join'; +import { GBServer } from '../../../../src/app.js'; const expiresIn = 1800; const conversationsCleanupInterval = 10000; @@ -38,6 +44,7 @@ export const getRouter = ( // Creates a conversation const reqs = (req, res) => { + const conversationId: string = uuidv4.v4().toString(); conversations[conversationId] = { conversationId, @@ -45,9 +52,11 @@ export const getRouter = ( }; console.log('Created conversation with conversationId: ' + conversationId); - const userId = req.query?.userSystemId ? req.query?.userSystemId : req.body.user.id; + let userId = req.query?.userSystemId ? req.query?.userSystemId : req.body?.user?.id; + userId = userId ? userId : req.query.userId; const activity = createConversationUpdateActivity(serviceUrl, conversationId, userId); + fetch(botUrl, { method: 'POST', body: JSON.stringify(activity), @@ -162,9 +171,82 @@ export const getRouter = ( } }; + // import { createMessageActivity, getConversation } from './yourModule'; // Update this import as needed + + const resupload = async (req, res) => { + // Extract botId from the URL using the pathname + const urlParts = req.url.split('/'); + const botId = urlParts[2]; // Assuming the URL is structured like /directline/{botId}/conversations/:conversationId/upload + const conversationId = req.params.conversationId; // Extract conversationId from parameters + + const uploadDir = path.join(process.cwd(), 'work', `${botId}.gbai`, 'cache'); // Create upload directory path + + // Create the uploads directory if it doesn't exist + + await fs.mkdir(uploadDir, { recursive: true }); + + const form = formidable({ + uploadDir, // Use the constructed upload directory + keepExtensions: true, // Keep file extensions + }); + + form.parse(req, async (err, fields, files) => { + if (err) { + console.log(`Error parsing the file: ${GBUtil.toYAML(err)}.`); + return res.status(400).send('Error parsing the file.'); + } + + const incomingActivity = fields; // Get incoming activity data + const file = files.file[0]; // Access the uploaded file + + const fileName = file['newFilename']; + const fileUrl = urlJoin(GBServer.globals.publicAddress, `${botId}.gbai`,'cache', fileName); + + // Create the activity message + let userId = req.query?.userSystemId ? req.query?.userSystemId : req.body?.user?.id; + userId = userId ? userId : req.query?.userId; + + const activity = createMessageActivity(incomingActivity, serviceUrl, conversationId, req.params['pid']); + activity.from = { id: userId, name: 'webbot' }; + activity.attachments = [{ + contentType: 'application/octet-stream', // Adjust as necessary + contentUrl: fileUrl, + name: fileName, // Original filename + }]; + const conversation = getConversation(conversationId, conversationInitRequired); + + if (conversation) { + // Add the uploaded file info to the activity + activity['fileUrl'] = fileUrl; // Set the file URL + + conversation.history.push(activity); + + try { + const response = await fetch(botUrl, { + method: 'POST', + body: JSON.stringify(activity), + headers: { + 'Content-Type': 'application/json' + } + }); + + res.status(response.status).json({ id: activity.id }); + } catch (fetchError) { + console.error('Error fetching bot:', fetchError); + res.status(500).send('Error processing request.'); + } + } else { + // Conversation was never initialized + res.status(400).send('Conversation not initialized.'); + } + }); + }; + router.post(`/api/messages/${botId}/v3/directline/conversations/:conversationId/activities`, res2); router.post(`/directline/${botId}/conversations/:conversationId/activities`, res2); + router.post(`/directline/${botId}/conversations/:conversationId/upload`, resupload); + router.post('/v3/directline/conversations/:conversationId/upload', (req, res) => { console.warn('/v3/directline/conversations/:conversationId/upload not implemented'); }); diff --git a/templates/crawler.gbai/crawler.gbot/config.csv b/templates/crawler.gbai/crawler.gbot/config.csv index d77f5bccf..7fd0bc968 100644 --- a/templates/crawler.gbai/crawler.gbot/config.csv +++ b/templates/crawler.gbai/crawler.gbot/config.csv @@ -1,4 +1,4 @@ name,value -Website,https://pragmatismo.cloud +Website,https://www.dgti.uerj.br/ Answer Mode,document -Theme Color,purple +Theme Color,purple \ No newline at end of file diff --git a/templates/edu.gbai/edu.gbdata/enrollments.csv b/templates/edu.gbai/edu.gbdata/enrollments.csv index 5edeb69c2..1c32f6941 100644 --- a/templates/edu.gbai/edu.gbdata/enrollments.csv +++ b/templates/edu.gbai/edu.gbdata/enrollments.csv @@ -1,2 +1,3 @@ Id,Name,Birthday,Email,Personalid,Address -lwkerderv,John Godf,12/12/2001,johng@fool.com.tg,12381239923,"Boulevard Street, 329" \ No newline at end of file +lwkerderv,John Godf,12/12/2001,johng@fool.com.tg,12381239923,"Boulevard Street, 329" +ekelwbctw,Jorge Santos,12/01/1978,jorge@uol.com.br,1239892998,"Rua Teodoro, 39"