new(basic.gblib): HEAR AS QRCODE.

package.json

@@ -3,7 +3,7 @@
   "version": "5.0.0",
   "description": "General Bot Community Edition open-core server.",
   "main": "./boot.mjs",
-  "type":"module",
+  "type": "module",
   "bugs": "https://github.com/pragmatismo-io/BotServer/issues",
   "homepage": "https://github.com/pragmatismo-io/BotServer/#readme",
   "contributors": [
@@ -14,6 +14,9 @@
     "Dário Vieira <dario.junior3@gmail.com>",
     "Alan Perdomo <alanperdomo@hotmail.com>"
   ],
+  "opencv4nodejs": {
+    "disableAutoBuild": "1"
+  },
   "engines": {
     "node": "=22.9.0"
   },
@@ -99,6 +102,7 @@
     "@sequelize/core": "7.0.0-alpha.37",
     "@types/node": "22.5.2",
     "@types/validator": "13.12.1",
+    "@u4/opencv4nodejs": "7.1.2",
     "adm-zip": "0.5.16",
     "alasql": "4.5.1",
     "any-shell-escape": "0.1.1",
@@ -136,7 +140,9 @@
     "express-remove-route": "1.0.0",
     "facebook-nodejs-business-sdk": "^20.0.2",
     "ffmpeg-static": "5.2.0",
+    "formidable": "^3.5.1",
     "get-image-colors": "4.0.1",
+    "glob": "^11.0.0",
     "google-libphonenumber": "3.2.38",
     "googleapis": "143.0.0",
     "hnswlib-node": "3.0.0",
@@ -146,8 +152,10 @@
     "instagram-private-api": "1.46.1",
     "iso-639-1": "3.1.3",
     "isomorphic-fetch": "3.0.0",
+    "jimp": "^1.6.0",
     "js-md5": "0.8.3",
     "json-schema-to-zod": "2.4.0",
+    "jsqr": "^1.4.0",
     "just-indent": "0.0.1",
     "keyv": "5.0.1",
     "koa": "2.15.3",
@@ -194,6 +202,7 @@
     "puppeteer-extra-plugin-stealth": "2.11.2",
     "qr-scanner": "1.4.2",
     "qrcode": "1.5.4",
+    "qrcode-reader": "^1.0.4",
     "qrcode-terminal": "0.12.0",
     "readline": "1.3.0",
     "reflect-metadata": "0.2.2",

packages/basic.gblib/services/DialogKeywords.ts

@@ -37,6 +37,8 @@ import urlJoin from 'url-join';
 import { GBServer } from '../../../src/app.js';
 import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
 import { SecService } from '../../security.gbapp/services/SecService.js';
+import {Jimp} from 'jimp';
+import jsQR from 'jsqr';
 import { SystemKeywords } from './SystemKeywords.js';
 import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
 import { Messages } from '../strings.js';
@@ -61,6 +63,7 @@ import { GBUtil } from '../../../src/util.js';
 import { GBVMService } from './GBVMService.js';
 import { ChatServices } from '../../../packages/llm.gblib/services/ChatServices.js';
 import puppeteer from 'puppeteer';
+import QRCodeProcessor from './QRCodeServices.js';
 
 /**
  * Default check interval for user replay
@@ -1206,14 +1209,27 @@ export class DialogKeywords {
         }
 
         result = phoneNumber;
-      } else if (kind === 'qr-scanner') {
+      } else if (kind === 'qrcode') {
         //https://github.com/GeneralBots/BotServer/issues/171
-        GBLogEx.info(min, `BASIC (${min.botId}): Upload done for ${answer.filename}.`);
+        GBLogEx.info(min, `BASIC (${min.botId}): QRCode for ${answer.filename}.`);
         const handle = WebAutomationServices.cyrb53({ pid, str: min.botId + answer.filename });
         GBServer.globals.files[handle] = answer;
-        QrScanner.scanImage(GBServer.globals.files[handle])
+
-          .then(result => console.log(result))
+          // Load the image with Jimp
-          .catch(error => console.log(error || 'no QR code found.'));
+          const image = await Jimp.read(answer.data);
+
+          // Get the image data
+          const imageData = {
+              data: new Uint8ClampedArray(image.bitmap.data),
+              width: image.bitmap.width,
+              height: image.bitmap.height,
+          };
+
+          // Use jsQR to decode the QR code
+          const decodedQR = jsQR(imageData.data, imageData.width, imageData.height);
+
+          result = decodedQR.data;
+
       } else if (kind === 'zipcode') {
         const extractEntity = (text: string) => {
           text = text.replace(/\-/gi, '');

packages/basic.gblib/services/QRCodeServices.ts

@@ -0,0 +1,99 @@
+import cv from '@u4/opencv4nodejs'
+import QRCodeReader from 'qrcode-reader';
+
+class QRCodeProcessor {
+    async decodeQRCode(imagePath) {
+        const image = cv.imread(imagePath, cv.IMREAD_COLOR);
+        const grayImage = image.bgrToGray();
+        const blurredImage = grayImage.gaussianBlur(new cv.Size(5, 5), 0);
+        const edges = blurredImage.canny(50, 150);
+
+        const contour = this.findQRCodeContour(edges);
+        if (!contour) {
+            throw new Error('QR Code não encontrado.');
+        }
+
+        const transformedImage = this.getPerspectiveTransform(image, contour);
+        return await this.readQRCode(transformedImage);
+    }
+
+    findQRCodeContour(edges) {
+        const contours = edges.findContours(cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE);
+        let maxContour = null;
+        let maxArea = 0;
+
+        contours.forEach(contour => {
+            const area = contour.area;
+            if (area > maxArea) {
+                maxArea = area;
+                maxContour = contour;
+            }
+        });
+
+        return maxContour;
+    }
+
+    getPerspectiveTransform(image, contour) {
+        // Ensure the contour has at least 4 points
+        const points = contour.getPoints();
+        if (points.length < 4) {
+            throw new Error("Contour must have at least 4 points.");
+        }
+
+        // Get the first four points
+        const srcPoints = points.slice(0, 4).map(point => new cv.Point2(point.x, point.y));
+
+        // Define destination points for the perspective transform
+        const dst = [
+            new cv.Point2(0, 0),
+            new cv.Point2(300, 0),
+            new cv.Point2(300, 300),
+            new cv.Point2(0, 300)
+        ];
+
+        // Get the perspective transform matrix
+        const M = cv.getPerspectiveTransform(srcPoints, dst);
+
+        // Create a new Mat for the transformed image
+        const transformedImage = new cv.Mat(300, 300, cv.CV_8UC3);
+
+        // Manually apply the perspective transformation
+        for (let y = 0; y < transformedImage.rows; y++) {
+            for (let x = 0; x < transformedImage.cols; x++) {
+                const srcPoint = this.applyPerspective(M, x, y);
+                const srcX = Math.round(srcPoint.x);
+                const srcY = Math.round(srcPoint.y);
+
+                // Check if the mapped source point is within the bounds of the source image
+                if (srcX >= 0 && srcX < image.cols && srcY >= 0 && srcY < image.rows) {
+                    const pixelValue = image.atVector(srcY, srcX); // Use atVector to get pixel values
+                    transformedImage.set(y, x, pixelValue);
+                }
+            }
+        }
+
+        return transformedImage;
+    }
+
+    applyPerspective(M, x, y) {
+        const a = M.getData(); // Get the matrix data
+        const denominator = a[6] * x + a[7] * y + 1; // Calculate the denominator
+        const newX = (a[0] * x + a[1] * y + a[2]) / denominator;
+        const newY = (a[3] * x + a[4] * y + a[5]) / denominator;
+        return new cv.Point2(newX, newY);
+    }
+
+    async readQRCode(image) {
+        return new Promise((resolve, reject) => {
+            const qrCodeReader = new QRCodeReader();
+            qrCodeReader.decode(image.getData(), (err, result) => {
+                if (err) {
+                    return reject(err);
+                }
+                resolve(result.result);
+            });
+        });
+    }
+}
+
+export default QRCodeProcessor;

packages/core.gbapp/services/GBDeployer.ts

@@ -465,7 +465,7 @@ export class GBDeployer implements IGBDeployer {
     } else {
       return [];
     }
-    GBLogEx.error(0, GBUtil.toYAML(rows));
+    
     await asyncPromise.eachSeries(rows, async (line: any) => {
       if (line && line.length > 0) {
         const key = line[1];

packages/core.gbapp/services/GBMinService.ts

@@ -135,6 +135,7 @@ export class GBMinService {
   public deployer: GBDeployer;
 
   bar1;
+  static pidsConversation = {};
 
   /**
    * Static initialization of minimal instance.
@@ -455,7 +456,7 @@ export class GBMinService {
         const status = req.body?.entry?.[0]?.changes?.[0]?.value?.statuses?.[0];
 
         if (status) {
-          GBLogEx.info(min, `WhatsApp: ${status.recipient_id} ${status.status}`);
+          GBLogEx.verbose(min, `WhatsApp: ${status.recipient_id} ${status.status}`);
           return;
         }
 
@@ -1040,7 +1041,7 @@ export class GBMinService {
       const step = await min.dialogs.createContext(context);
       step.context.activity.locale = 'pt-BR';
 
-      
+
       const sec = new SecService();
       const member = context.activity.recipient ? context.activity.recipient : context.activity.from;
       let user = await sec.ensureUser(min, member.id, member.name, '', 'web', member.name, null);
@@ -1094,8 +1095,27 @@ export class GBMinService {
             });
           }
         }
-       
+
-        
+        let conversationId = step.context.activity.conversation.id;
+
+        let pid = GBMinService.pidsConversation[conversationId];
+
+        if (!pid) {
+
+          pid = step.context.activity['pid'];
+          if (!pid) {
+            pid = WhatsappDirectLine.pidByNumber[context.activity.from.id];
+            if (!pid) {
+              pid = GBVMService.createProcessInfo(user, min, step.context.activity.channelId, null, step);
+            }
+          }
+        }
+        GBMinService.pidsConversation[conversationId] = pid; 
+        step.context.activity['pid'] = pid;
+
+        const notes = min.core.getParam(min.instance, 'Notes', null);
+        await this.handleUploads(min, step, user, params, notes != null);
+    
         // Required for MSTEAMS handling of persisted conversations.
 
         if (step.context.activity.channelId === 'msteams') {
@@ -1137,7 +1157,7 @@ export class GBMinService {
                 min,
                 `Auto start (teams) dialog is now being called: ${startDialog} for ${min.instance.botId}...`
               );
-              
+
               await GBVMService.callVM(startDialog.toLowerCase(), min, step, 0);
             }
           }
@@ -1178,7 +1198,7 @@ export class GBMinService {
 
                 const pid = GBVMService.createProcessInfo(user, min, step.context.activity.channelId, null, step);
                 step.context.activity['pid'] = pid;
-      
+
                 min['conversationWelcomed'][step.context.activity.conversation.id] = true;
 
                 GBLogEx.info(
@@ -1195,16 +1215,16 @@ export class GBMinService {
           }
         } else if (context.activity.type === 'message') {
 
-          let pid = WhatsappDirectLine.pidByNumber[context.activity.from.id];
+
-  
+
           // Required for F0 handling of persisted conversations.
-  
+
           GBLogEx.info(
             min,
             `Human: pid:${pid} ${context.activity.from.id} ${GBUtil.toYAML(WhatsappDirectLine.pidByNumber)} ${context.activity.text} (type: ${context.activity.type}, name: ${context.activity.name}, channelId: ${context.activity.channelId})`
           );
-          
+
-  
+
           // Processes messages activities.
 
           await this.processMessageActivity(context, min, step, pid);
@@ -1302,26 +1322,28 @@ export class GBMinService {
     const url = attachment.contentUrl;
     const localFolder = 'work';
     const packagePath = GBUtil.getGBAIPath(this['min'].botId);
-    const localFileName = path.join(localFolder, packagePath, 'uploads', attachment.name);
+    const localFileName = path.join(localFolder, packagePath, 'cache', attachment.name);
 
     let buffer;
-    if (url.startsWith('data:')) {
+    // if (url.startsWith('data:')) {
-      const base64Data = url.split(';base64,')[1];
+    //   const base64Data = url.split(';base64,')[1];
-      buffer = Buffer.from(base64Data, 'base64');
+    //   buffer = Buffer.from(base64Data, 'base64');
-    } else {
+    // } else {
-      const options = {
+    //   const options = {
-        method: 'GET',
+    //     method: 'GET',
-        encoding: 'binary'
+    //     encoding: 'binary'
-      };
+    //   };
-      const res = await fetch(url, options);
+    //   const res = await fetch(url, options);
-      buffer = arrayBufferToBuffer(await res.arrayBuffer());
+    //   buffer = arrayBufferToBuffer(await res.arrayBuffer());
-    }
+    // }
-
+    //write
-    await fs.writeFile(localFileName, buffer);
+    buffer = await fs.readFile(localFileName);
 
     return {
-      fileName: attachment.name,
+      name: attachment.name,
-      localPath: localFileName
+      filename: localFileName,
+      url: url,
+      data: buffer
     };
   }
 
@@ -1343,7 +1365,7 @@ export class GBMinService {
       step.context.activity.attachments[0].contentType != 'text/html'
     ) {
       const promises = step.context.activity.attachments.map(
-        GBMinService.downloadAttachmentAndWrite.bind({ min, user, params })
+         GBMinService.downloadAttachmentAndWrite.bind({ min, user, params })
       );
       const successfulSaves = await Promise.all(promises);
       async function replyForReceivedAttachments(attachmentData) {
@@ -1352,14 +1374,15 @@ export class GBMinService {
           // a upload with no Dialog, so run Auto Save to .gbdrive.
 
           const t = new SystemKeywords();
-          GBLogEx.info(min, `BASIC (${min.botId}): Upload done for ${attachmentData.fileName}.`);
+          GBLogEx.info(min, `BASIC (${min.botId}): Upload done for ${attachmentData.filename}.`);
-          const handle = WebAutomationServices.cyrb53({ pid: 0, str: min.botId + attachmentData.fileName });
+          const handle = WebAutomationServices.cyrb53({ pid: 0, str: min.botId + attachmentData.filename });
-          let data = await fs.readFile(attachmentData.localPath);
+          let data = await fs.readFile(attachmentData.filename);
 
           const gbfile = {
-            filename: attachmentData.localPath,
+            filename: path.join(process.env.PWD, attachmentData.filename),
             data: data,
-            name: path.basename(attachmentData.fileName)
+            url: attachmentData.url,
+            name: path.basename(attachmentData.filename)
           };
 
           GBServer.globals.files[handle] = gbfile;
@@ -1386,12 +1409,16 @@ export class GBMinService {
         class GBFile {
           data: Buffer;
           filename: string;
+          url: string;
+          name: string;
         }
 
-        const results = successfulSaves.reduce(async (accum: GBFile[], item) => {
+        const results = await successfulSaves.reduce(async (accum: GBFile[], item) => {
           const result: GBFile = {
-            data: await fs.readFile(successfulSaves[0]['localPath']),
+            data: await fs.readFile(successfulSaves[0]['filename']),
-            filename: successfulSaves[0]['fileName']
+            filename: successfulSaves[0]['filename'],
+            name: successfulSaves[0]['name'],
+            url: successfulSaves[0]['url'],
           };
           accum.push(result);
           return accum;
@@ -1515,9 +1542,6 @@ export class GBMinService {
       }
     }
 
-    const notes = min.core.getParam(min.instance, 'Notes', null);
-    await this.handleUploads(min, step, user, params, notes != null);
-
     // Files in .gbdialog can be called directly by typing its name normalized into JS .
 
     const isVMCall = Object.keys(min.scriptMap).find(key => min.scriptMap[key] === context.activity.text) !== undefined;
@@ -1572,13 +1596,6 @@ export class GBMinService {
       step.context.activity['originalText'];
       step.context.activity['text'] = text;
 
-      if (notes && text && text !== '') {
-        const sys = new SystemKeywords();
-        await sys.note({ pid, text });
-        await step.context.sendActivity('OK.');
-        return;
-      }
-
       // Checks for bad words on input text.
 
       const hasBadWord = wash.check(step.context.activity.locale, text);

packages/core.gbapp/services/router/bridge.ts

@@ -1,3 +1,6 @@
+import fs from 'fs/promises';
+import formidable from 'formidable';
+import path from 'path';
 import bodyParser from 'body-parser';
 import express from 'express';
 import fetch from 'isomorphic-fetch';
@@ -5,6 +8,9 @@ import moment from 'moment';
 import * as uuidv4 from 'uuid';
 import { IActivity, IBotData, IConversation, IConversationUpdateActivity, IMessageActivity } from './types';
 import { GBConfigService } from '../GBConfigService.js';
+import { GBUtil } from '../../../../src/util.js';
+import urlJoin from 'url-join';
+import { GBServer } from '../../../../src/app.js';
 
 const expiresIn = 1800;
 const conversationsCleanupInterval = 10000;
@@ -38,6 +44,7 @@ export const getRouter = (
 
   // Creates a conversation
   const reqs = (req, res) => {
+
     const conversationId: string = uuidv4.v4().toString();
     conversations[conversationId] = {
       conversationId,
@@ -45,9 +52,11 @@ export const getRouter = (
     };
     console.log('Created conversation with conversationId: ' + conversationId);
 
-    const userId = req.query?.userSystemId ? req.query?.userSystemId : req.body.user.id;
+    let userId = req.query?.userSystemId ? req.query?.userSystemId : req.body?.user?.id;
+    userId = userId ? userId : req.query.userId;
 
     const activity = createConversationUpdateActivity(serviceUrl, conversationId, userId);
+
     fetch(botUrl, {
       method: 'POST',
       body: JSON.stringify(activity),
@@ -162,9 +171,82 @@ export const getRouter = (
     }
   };
 
+  // import { createMessageActivity, getConversation } from './yourModule'; // Update this import as needed
+
+  const resupload = async (req, res) => {
+    // Extract botId from the URL using the pathname
+    const urlParts = req.url.split('/');
+    const botId = urlParts[2]; // Assuming the URL is structured like /directline/{botId}/conversations/:conversationId/upload
+    const conversationId = req.params.conversationId; // Extract conversationId from parameters
+
+    const uploadDir = path.join(process.cwd(), 'work', `${botId}.gbai`, 'cache'); // Create upload directory path
+
+    // Create the uploads directory if it doesn't exist
+
+    await fs.mkdir(uploadDir, { recursive: true });
+
+    const form = formidable({
+      uploadDir, // Use the constructed upload directory
+      keepExtensions: true,  // Keep file extensions
+    });
+
+    form.parse(req, async (err, fields, files) => {
+      if (err) {
+        console.log(`Error parsing the file: ${GBUtil.toYAML(err)}.`);
+        return res.status(400).send('Error parsing the file.');
+      }
+
+      const incomingActivity = fields; // Get incoming activity data
+      const file = files.file[0]; // Access the uploaded file
+
+      const fileName = file['newFilename'];
+      const fileUrl = urlJoin(GBServer.globals.publicAddress, `${botId}.gbai`,'cache', fileName);
+
+      // Create the activity message
+      let userId = req.query?.userSystemId ? req.query?.userSystemId : req.body?.user?.id;
+      userId = userId ? userId : req.query?.userId;
+
+      const activity = createMessageActivity(incomingActivity, serviceUrl, conversationId, req.params['pid']);
+      activity.from = { id: userId, name: 'webbot' };
+      activity.attachments = [{
+        contentType: 'application/octet-stream', // Adjust as necessary
+        contentUrl: fileUrl,
+        name: fileName, // Original filename
+      }];
+      const conversation = getConversation(conversationId, conversationInitRequired);
+
+      if (conversation) {
+        // Add the uploaded file info to the activity
+        activity['fileUrl'] = fileUrl; // Set the file URL
+
+        conversation.history.push(activity);
+
+        try {
+          const response = await fetch(botUrl, {
+            method: 'POST',
+            body: JSON.stringify(activity),
+            headers: {
+              'Content-Type': 'application/json'
+            }
+          });
+
+          res.status(response.status).json({ id: activity.id });
+        } catch (fetchError) {
+          console.error('Error fetching bot:', fetchError);
+          res.status(500).send('Error processing request.');
+        }
+      } else {
+        // Conversation was never initialized
+        res.status(400).send('Conversation not initialized.');
+      }
+    });
+  };
+
   router.post(`/api/messages/${botId}/v3/directline/conversations/:conversationId/activities`, res2);
   router.post(`/directline/${botId}/conversations/:conversationId/activities`, res2);
 
+  router.post(`/directline/${botId}/conversations/:conversationId/upload`, resupload);
+
   router.post('/v3/directline/conversations/:conversationId/upload', (req, res) => {
     console.warn('/v3/directline/conversations/:conversationId/upload not implemented');
   });

templates/ai-search.gbai/ai-search.gbdialog/qr.bas

@@ -0,0 +1,14 @@
+TALK "Please, take a photo of the QR Code and send to me."
+HEAR doc as QRCODE
+text = GET "doc-"  + doc + ".pdf"
+
+IF text THEN
+
+    text = "Based on this document, answer the person's questions:\n\n" + text
+    SET CONTEXT text 
+    SET ANSWER MODE "document"
+    TALK "Document ${doc} loaded. You can ask me anything about it."
+
+ELSE
+    TALK "Document was not found, please try again."
+END IF

templates/ai-search.gbai/ai-search.gbot/config.csv

@@ -1,4 +1,4 @@
 name,value
 Answer Mode,document-ref
 Theme Color,green
-Start Dialog, start
+Start Dialog,start

templates/crawler.gbai/crawler.gbot/config.csv

@@ -1,4 +1,4 @@
 name,value
-Website,https://pragmatismo.cloud
+Website,https://www.dgti.uerj.br/
 Answer Mode,document
-Theme Color,purple    
+Theme Color,purple    

templates/edu.gbai/edu.gbdata/enrollments.csv

@@ -1,2 +1,3 @@
 Id,Name,Birthday,Email,Personalid,Address
-lwkerderv,John Godf,12/12/2001,johng@fool.com.tg,12381239923,"Boulevard Street, 329"
+lwkerderv,John Godf,12/12/2001,johng@fool.com.tg,12381239923,"Boulevard Street, 329"
+ekelwbctw,Jorge Santos,12/01/1978,jorge@uol.com.br,1239892998,"Rua Teodoro, 39"