From 4342c6d3e5dce238b4f4cb1f479086fea7e31bb2 Mon Sep 17 00:00:00 2001
From: Rodrigo Rodriguez <me@rodrigorodriguez.com>
Date: Sun, 14 Apr 2024 23:17:37 -0300
Subject: [PATCH] new(gpt.gblib): PDF opener.

---
 packages/default.gbui/src/GBUIApp.js          | 13 ++-
 .../src/players/GBMultiUrlPlayer.js           | 91 +++++++++++++++++++
 packages/gpt.gblib/services/ChatServices.ts   | 70 +++++++++-----
 packages/kb.gbapp/dialogs/AskDialog.ts        | 31 ++++---
 4 files changed, 171 insertions(+), 34 deletions(-)
 create mode 100644 packages/default.gbui/src/players/GBMultiUrlPlayer.js

diff --git a/packages/default.gbui/src/GBUIApp.js b/packages/default.gbui/src/GBUIApp.js
index 24a395a6..77b2aafd 100644
--- a/packages/default.gbui/src/GBUIApp.js
+++ b/packages/default.gbui/src/GBUIApp.js
@@ -33,6 +33,7 @@ import GBMarkdownPlayer from './players/GBMarkdownPlayer.js';
 import GBImagePlayer from './players/GBImagePlayer.js';
 import GBVideoPlayer from './players/GBVideoPlayer.js';
 import GBUrlPlayer from './players/GBUrlPlayer.js';
+import GBMultiUrlPlayer from './players/GBMultiUrlPlayer.js';
 import GBLoginPlayer from './players/GBLoginPlayer.js';
 import GBBulletPlayer from './players/GBBulletPlayer.js';
 import SidebarMenu from './components/SidebarMenu.js';
@@ -255,7 +256,17 @@ class GBUIApp extends React.Component {
             />
           );
           break;
-        case 'image':
+        case 'multiurl':
+            playerComponent = (
+              <GBMultiUrlPlayer
+                app={this}
+                ref={player => {
+                  this.player = player;
+                }}
+              />
+            );
+            break;
+          case 'image':
           playerComponent = (
             <GBImagePlayer
               app={this}
diff --git a/packages/default.gbui/src/players/GBMultiUrlPlayer.js b/packages/default.gbui/src/players/GBMultiUrlPlayer.js
new file mode 100644
index 00000000..a6312002
--- /dev/null
+++ b/packages/default.gbui/src/players/GBMultiUrlPlayer.js
@@ -0,0 +1,91 @@
+/*****************************************************************************\
+|  █████  █████ ██    █ █████ █████   ████  ██      ████   █████ █████  ███ ® |
+| ██      █     ███   █ █     ██  ██ ██  ██ ██      ██  █ ██   ██  █   █      |
+| ██  ███ ████  █ ██  █ ████  █████  ██████ ██      ████   █   █   █    ██    |
+| ██   ██ █     █  ██ █ █     ██  ██ ██  ██ ██      ██  █ ██   ██  █      █   |
+|  █████  █████ █   ███ █████ ██  ██ ██  ██ █████   ████   █████   █   ███    |
+|                                                                             |
+| General Bots Copyright (c) pragmatismo.com.br. All rights reserved.         |
+| Licensed under the AGPL-3.0.                                                |
+|                                                                             |
+| According to our dual licensing model, this program can be used either      |
+| under the terms of the GNU Affero General Public License, version 3,        |
+| or under a proprietary license.                                             |
+|                                                                             |
+| The texts of the GNU Affero General Public License with an additional       |
+| permission and of our proprietary license can be found at and               |
+| in the LICENSE file you have received along with this program.              |
+|                                                                             |
+| This program is distributed in the hope that it will be useful,             |
+| but WITHOUT ANY WARRANTY; without even the implied warranty of              |
+| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                |
+| GNU Affero General Public License for more details.                         |
+|                                                                             |
+| "General Bots" is a registered trademark of pragmatismo.com.br.             |
+| The licensing of the program under the AGPLv3 does not imply a              |
+| trademark license. Therefore any rights, title and interest in              |
+| our trademarks remain entirely with us.                                     |
+|                                                                             |
+\*****************************************************************************/
+
+import React, { Component } from 'react';
+
+class RenderItem extends Component {
+  send(item) {
+    setTimeout(() => {
+      window.botConnection.postActivity({
+        type: 'event',
+        name: 'answerEvent',
+        data: item.questionId,
+        locale: 'en-us',
+        textFormat: 'plain',
+        timestamp: new Date().toISOString(),
+        from: window.user
+      });
+    }, 400);
+  }
+
+  render() {
+    return (
+      <div className="gb-video-player-wrapper">
+        {this.props.list.map(item => (
+          <iframe
+            title="Video"
+            ref="video"
+            className="gb-video-react-player"
+            src={item.url}
+            width="100%"
+            height="100%"
+          />
+        ))}
+      </div>
+    );
+  }
+}
+
+class GBMultiUrlPlayer extends Component {
+  constructor() {
+    super();
+    this.state = {
+      list: []
+    };
+  }
+
+  play(data) {
+    this.setState({ list: data });
+  }
+
+  stop() {
+    this.setState({ list: [] });
+  }
+
+  render() {
+    return (
+      <div className="gb-bullet-player" ref={i => (this.playerText = i)}>
+        <RenderItem app={this.props.app} list={this.state.list} ref={i => (this.playerList = i)} />
+      </div>
+    );
+  }
+}
+
+export default GBMultiUrlPlayer;
diff --git a/packages/gpt.gblib/services/ChatServices.ts b/packages/gpt.gblib/services/ChatServices.ts
index 4d4a534d..0e4088fd 100644
--- a/packages/gpt.gblib/services/ChatServices.ts
+++ b/packages/gpt.gblib/services/ChatServices.ts
@@ -130,18 +130,37 @@ export class GBLLMOutputParser extends
 
     let res;
     try {
+      GBLogEx.info(this.min, result);
       result = result.replace(/\\n/g, '');
       res = JSON.parse(result);
     } catch {
       return result;
     }
 
-    let { file, page, text } = res;
-    const { url } = await ChatServices.pdfPageAsImage(this.min, file, page);
-    text = `![alt text](${url})
-      ${text}`;
+    let { sources, text } = res;
+    
+    await CollectionUtil.asyncForEach(sources, async (source) => {
+      let found = false;
+      if (source) {
+        
+        const gbaiName = DialogKeywords.getGBAIPath(this.min.botId, 'gbkb');
+        const localName = Path.join('work', gbaiName, 'docs', source.file);
 
-    return {text, file, page};
+        if (localName) {
+          const { url } = await ChatServices.pdfPageAsImage(this.min, localName, source.page);
+          text = `![alt text](${url})
+          ${text}`;
+          found = true;
+          source.file = localName;
+        }
+      }
+
+      if (found) {
+        GBLogEx.info(this.min, `File not found referenced in other .pdf: ${source.file}`);
+      }
+    });
+
+    return { text, sources };
   }
 }
 
@@ -149,13 +168,10 @@ export class ChatServices {
 
   public static async pdfPageAsImage(min, filename, pageNumber) {
 
-    const gbaiName = DialogKeywords.getGBAIPath(min.botId, 'gbkb');
-    const localName = Path.join('work', gbaiName, 'docs', filename);
-
     // Converts the PDF to PNG.
 
     GBLogEx.info(min, `Converting ${filename}, page: ${pageNumber}...`);
-    const pngPages: PngPageOutput[] = await pdfToPng(localName, {
+    const pngPages: PngPageOutput[] = await pdfToPng(filename, {
       disableFontFace: true,
       useSystemFonts: true,
       viewportScale: 2.0,
@@ -187,19 +203,27 @@ export class ChatServices {
       return '';
     }
 
-    const documents = await vectorStore.similaritySearch(sanitizedQuestion, numDocuments);
+    let documents = await vectorStore.similaritySearch(sanitizedQuestion, numDocuments);
+    const uniqueDocuments = {};
+
+    for (const document of documents) {
+      if (!uniqueDocuments[document.metadata.source]) {
+        uniqueDocuments[document.metadata.source] = document;
+      }
+    }
+
     let output = '';
 
-    await CollectionUtil.asyncForEach(documents, async (doc) => {
-
+    for(const filePaths of Object.keys(uniqueDocuments)) {
+      const doc = uniqueDocuments[filePaths];
       const metadata = doc.metadata;
       const filename = Path.basename(metadata.source);
-      const page = await ChatServices.findPageForText(doc.metadata.source,
+      const page = await ChatServices.findPageForText(metadata.source,
         doc.pageContent);
 
       output = `${output}\n\n\n\nThe following context is coming from ${filename} at page: ${page}, 
       memorize this block among document information and return when you are refering this part of content:\n\n\n\n ${doc.pageContent} \n\n\n\n.`;
-    });
+    }
     return output;
   }
 
@@ -217,7 +241,7 @@ export class ChatServices {
       if (text.includes(searchText)) return i;
     }
 
-    return -1; 
+    return -1;
   }
 
   /**
@@ -314,12 +338,13 @@ export class ChatServices {
         \n\n{context}\n\n
         
         And based on \n\n{chat_history}\n\n
-        rephrase the response to the user using the aforementioned context. If you're unsure of the answer, utilize any relevant context provided to answer the question effectively. Don´t output MD images tags url previously shown.
+        rephrase the response to the user using the aforementioned context. If you're unsure of the answer, 
+        utilize any relevant context provided to answer the question effectively. Don´t output MD images tags url previously shown.
 
         VERY IMPORTANT: ALWAYS return VALID standard JSON with the folowing structure: 'text' as answer, 
-          'file' indicating the PDF filename and 'page' indicating the page number. 
+          sources as an array of ('file' indicating the PDF filename and 'page' indicating the page number) listing all segmented context. 
         Example JSON format: "text": "this is the answer, anything LLM output as text answer shoud be here.", 
-          "file": "filename.pdf", "page": 3,
+          "sources": [{{"file": "filename.pdf", "page": 3}}, {{"file": "filename2.pdf", "page": 1}}],
          return valid JSON with brackets. Avoid explaining the context directly
           to the user; instead, refer to the document source. 
           
@@ -384,7 +409,7 @@ export class ChatServices {
       new StringOutputParser()
     ]);
 
-    let result;
+    let result, sources;
     let text, file, page;
 
 
@@ -401,8 +426,9 @@ export class ChatServices {
     }
     else if (LLMMode === "document") {
 
-      const {text, file, page} = await combineDocumentsChain.invoke(question);
-      result = text;
+      const res = await combineDocumentsChain.invoke(question);
+      result = res.text;
+      sources = res.sources;
 
     } else if (LLMMode === "function") {
 
@@ -429,7 +455,7 @@ export class ChatServices {
     );
 
     GBLog.info(`GPT Result: ${result.toString()}`);
-    return { answer: result.toString(), file, questionId: 0, page };
+    return { answer: result.toString(), sources, questionId: 0, page };
   }
 
   private static getToolsAsText(tools) {
diff --git a/packages/kb.gbapp/dialogs/AskDialog.ts b/packages/kb.gbapp/dialogs/AskDialog.ts
index eadc4576..b00d11b3 100644
--- a/packages/kb.gbapp/dialogs/AskDialog.ts
+++ b/packages/kb.gbapp/dialogs/AskDialog.ts
@@ -49,6 +49,7 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
 import urlJoin from 'url-join';
 import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js';
 import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
+import Path from 'path';
 
 /**
  * Dialog arguments.
@@ -233,21 +234,29 @@ export class AskDialog extends IGBDialog {
           return;
         }
 
-        const results:any = await service.ask(min, user, step, step.context.activity['pid'], text, searchScore, null /* user.subjects */);
+        const results: any = await service.ask(min, user, step, step.context.activity['pid'], text, searchScore, null /* user.subjects */);
 
         // If there is some result, answer immediately.
 
         if (results !== undefined && results.answer !== undefined) {
+          let urls = [];
+          if (results.sources) {
 
-          if (results.file){
-            const path = DialogKeywords.getGBAIPath(min.botId, `gbkb`);
-            const url = urlJoin('kb', path, 'docs', results.file);
-      
-            await min.conversationalService.sendEvent(
-              min, step,  'play', {
-                playerType: 'url',
-                data: `${url}#page=${results.page}&toolbar=0&messages=0&statusbar=0&navpanes=0`
+            for (const key in results.sources) {
+              const source = results.sources[key];
+              const path = DialogKeywords.getGBAIPath(min.botId, `gbkb`);
+              let url = urlJoin('kb', path, 'docs', Path.basename(source.file));
+              url = `${url}#page=${source.page}&toolbar=0&messages=0&statusbar=0&navpanes=0`;
+              urls.push({ url: url });
+            }
+
+            if (urls.length > 0) {
+              await min.conversationalService.sendEvent(
+                min, step, 'play', {
+                playerType: 'multiurl',
+                data: urls
               });
+            }
           }
 
           // Sends the answer to all outputs, including projector.
@@ -266,7 +275,7 @@ export class AskDialog extends IGBDialog {
         const message = min.core.getParam<string>(min.instance, 'Not Found Message', Messages[locale].did_not_find);
 
         await min.conversationalService.sendText(min, step, message);
-        
+
         return await step.replaceDialog('/ask', { isReturning: true });
       }
     ];
@@ -287,7 +296,7 @@ export class AskDialog extends IGBDialog {
       return await step.replaceDialog('/ask', { isReturning: true });
     }
   }
-  
+
   private static getChannel(step): string {
     return !isNaN(step.context.activity['mobile']) ? 'whatsapp' : step.context.activity.channelId;
   }