GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

new(gpt.gblib): PDF opener.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2024-04-14 23:17:37 -03:00
parent 51107fcd76
commit 4342c6d3e5
4 changed files with 171 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
packages/default.gbui/src/GBUIApp.js
View file

@ -33,6 +33,7 @@ import GBMarkdownPlayer from './players/GBMarkdownPlayer.js';
import GBImagePlayer from './players/GBImagePlayer.js';
import GBVideoPlayer from './players/GBVideoPlayer.js';
import GBUrlPlayer from './players/GBUrlPlayer.js';
import GBMultiUrlPlayer from './players/GBMultiUrlPlayer.js';
import GBLoginPlayer from './players/GBLoginPlayer.js';
import GBBulletPlayer from './players/GBBulletPlayer.js';
import SidebarMenu from './components/SidebarMenu.js';
@ -255,7 +256,17 @@ class GBUIApp extends React.Component {
/>
);
break;
case 'image':
case 'multiurl':
playerComponent = (
<GBMultiUrlPlayer
app={this}
ref={player => {
this.player = player;
}}
/>
);
break;
case 'image':
playerComponent = (
<GBImagePlayer
app={this}

91
packages/default.gbui/src/players/GBMultiUrlPlayer.js Normal file
View file

@ -0,0 +1,91 @@
/*****************************************************************************\
| █████ █████ ██ █ █████ █████ ████ ██ ████ █████ █████ ███ ® |
| ██ █ ███ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| ██ ███ ████ █ ██ █ ████ █████ ██████ ██ ████ █ █ █ ██ |
| ██ ██ █ █ ██ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ |
| █████ █████ █ ███ █████ ██ ██ ██ ██ █████ ████ █████ █ ███ |
| |
| General Bots Copyright (c) pragmatismo.com.br. All rights reserved. |
| Licensed under the AGPL-3.0. |
| |
| According to our dual licensing model, this program can be used either |
| under the terms of the GNU Affero General Public License, version 3, |
| or under a proprietary license. |
| |
| The texts of the GNU Affero General Public License with an additional |
| permission and of our proprietary license can be found at and |
| in the LICENSE file you have received along with this program. |
| |
| This program is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| GNU Affero General Public License for more details. |
| |
| "General Bots" is a registered trademark of pragmatismo.com.br. |
| The licensing of the program under the AGPLv3 does not imply a |
| trademark license. Therefore any rights, title and interest in |
| our trademarks remain entirely with us. |
| |
\*****************************************************************************/
import React, { Component } from 'react';
class RenderItem extends Component {
send(item) {
setTimeout(() => {
window.botConnection.postActivity({
type: 'event',
name: 'answerEvent',
data: item.questionId,
locale: 'en-us',
textFormat: 'plain',
timestamp: new Date().toISOString(),
from: window.user
});
}, 400);
}
render() {
return (
<div className="gb-video-player-wrapper">
{this.props.list.map(item => (
<iframe
title="Video"
ref="video"
className="gb-video-react-player"
src={item.url}
width="100%"
height="100%"
/>
))}
</div>
);
}
}
class GBMultiUrlPlayer extends Component {
constructor() {
super();
this.state = {
list: []
};
}
play(data) {
this.setState({ list: data });
}
stop() {
this.setState({ list: [] });
}
render() {
return (
<div className="gb-bullet-player" ref={i => (this.playerText = i)}>
<RenderItem app={this.props.app} list={this.state.list} ref={i => (this.playerList = i)} />
</div>
);
}
}
export default GBMultiUrlPlayer;

70
packages/gpt.gblib/services/ChatServices.ts
View file

@ -130,18 +130,37 @@ export class GBLLMOutputParser extends
let res;
try {
GBLogEx.info(this.min, result);
result = result.replace(/\\n/g, '');
res = JSON.parse(result);
} catch {
return result;
}
let { file, page, text } = res;
const { url } = await ChatServices.pdfPageAsImage(this.min, file, page);
text = `![alt text](${url})
${text}`;
let { sources, text } = res;
await CollectionUtil.asyncForEach(sources, async (source) => {
let found = false;
if (source) {
const gbaiName = DialogKeywords.getGBAIPath(this.min.botId, 'gbkb');
const localName = Path.join('work', gbaiName, 'docs', source.file);
return {text, file, page};
if (localName) {
const { url } = await ChatServices.pdfPageAsImage(this.min, localName, source.page);
text = `![alt text](${url})
${text}`;
found = true;
source.file = localName;
}
}
if (found) {
GBLogEx.info(this.min, `File not found referenced in other .pdf: ${source.file}`);
}
});
return { text, sources };
}
}
@ -149,13 +168,10 @@ export class ChatServices {
public static async pdfPageAsImage(min, filename, pageNumber) {
const gbaiName = DialogKeywords.getGBAIPath(min.botId, 'gbkb');
const localName = Path.join('work', gbaiName, 'docs', filename);
// Converts the PDF to PNG.
GBLogEx.info(min, `Converting ${filename}, page: ${pageNumber}...`);
const pngPages: PngPageOutput[] = await pdfToPng(localName, {
const pngPages: PngPageOutput[] = await pdfToPng(filename, {
disableFontFace: true,
useSystemFonts: true,
viewportScale: 2.0,
@ -187,19 +203,27 @@ export class ChatServices {
return '';
}
const documents = await vectorStore.similaritySearch(sanitizedQuestion, numDocuments);
let documents = await vectorStore.similaritySearch(sanitizedQuestion, numDocuments);
const uniqueDocuments = {};
for (const document of documents) {
if (!uniqueDocuments[document.metadata.source]) {
uniqueDocuments[document.metadata.source] = document;
}
}
let output = '';
await CollectionUtil.asyncForEach(documents, async (doc) => {
for(const filePaths of Object.keys(uniqueDocuments)) {
const doc = uniqueDocuments[filePaths];
const metadata = doc.metadata;
const filename = Path.basename(metadata.source);
const page = await ChatServices.findPageForText(doc.metadata.source,
const page = await ChatServices.findPageForText(metadata.source,
doc.pageContent);
output = `${output}\n\n\n\nThe following context is coming from ${filename} at page: ${page},
memorize this block among document information and return when you are refering this part of content:\n\n\n\n ${doc.pageContent} \n\n\n\n.`;
});
}
return output;
}
@ -217,7 +241,7 @@ export class ChatServices {
if (text.includes(searchText)) return i;
}
return -1;
return -1;
}
/**
@ -314,12 +338,13 @@ export class ChatServices {
\n\n{context}\n\n
And based on \n\n{chat_history}\n\n
rephrase the response to the user using the aforementioned context. If you're unsure of the answer, utilize any relevant context provided to answer the question effectively. Don´t output MD images tags url previously shown.
rephrase the response to the user using the aforementioned context. If you're unsure of the answer,
utilize any relevant context provided to answer the question effectively. Don´t output MD images tags url previously shown.
VERY IMPORTANT: ALWAYS return VALID standard JSON with the folowing structure: 'text' as answer,
'file' indicating the PDF filename and 'page' indicating the page number.
sources as an array of ('file' indicating the PDF filename and 'page' indicating the page number) listing all segmented context.
Example JSON format: "text": "this is the answer, anything LLM output as text answer shoud be here.",
"file": "filename.pdf", "page": 3,
"sources": [{{"file": "filename.pdf", "page": 3}}, {{"file": "filename2.pdf", "page": 1}}],
return valid JSON with brackets. Avoid explaining the context directly
to the user; instead, refer to the document source.
@ -384,7 +409,7 @@ export class ChatServices {
new StringOutputParser()
]);
let result;
let result, sources;
let text, file, page;
@ -401,8 +426,9 @@ export class ChatServices {
}
else if (LLMMode === "document") {
const {text, file, page} = await combineDocumentsChain.invoke(question);
result = text;
const res = await combineDocumentsChain.invoke(question);
result = res.text;
sources = res.sources;
} else if (LLMMode === "function") {
@ -429,7 +455,7 @@ export class ChatServices {
);
GBLog.info(`GPT Result: ${result.toString()}`);
return { answer: result.toString(), file, questionId: 0, page };
return { answer: result.toString(), sources, questionId: 0, page };
}
private static getToolsAsText(tools) {

31
packages/kb.gbapp/dialogs/AskDialog.ts
View file

@ -49,6 +49,7 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import urlJoin from 'url-join';
import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords.js';
import Path from 'path';
/**
* Dialog arguments.
@ -233,21 +234,29 @@ export class AskDialog extends IGBDialog {
return;
}
const results:any = await service.ask(min, user, step, step.context.activity['pid'], text, searchScore, null /* user.subjects */);
const results: any = await service.ask(min, user, step, step.context.activity['pid'], text, searchScore, null /* user.subjects */);
// If there is some result, answer immediately.
if (results !== undefined && results.answer !== undefined) {
let urls = [];
if (results.sources) {
if (results.file){
const path = DialogKeywords.getGBAIPath(min.botId, `gbkb`);
const url = urlJoin('kb', path, 'docs', results.file);
await min.conversationalService.sendEvent(
min, step, 'play', {
playerType: 'url',
data: `${url}#page=${results.page}&toolbar=0&messages=0&statusbar=0&navpanes=0`
for (const key in results.sources) {
const source = results.sources[key];
const path = DialogKeywords.getGBAIPath(min.botId, `gbkb`);
let url = urlJoin('kb', path, 'docs', Path.basename(source.file));
url = `${url}#page=${source.page}&toolbar=0&messages=0&statusbar=0&navpanes=0`;
urls.push({ url: url });
}
if (urls.length > 0) {
await min.conversationalService.sendEvent(
min, step, 'play', {
playerType: 'multiurl',
data: urls
});
}
}
// Sends the answer to all outputs, including projector.
@ -266,7 +275,7 @@ export class AskDialog extends IGBDialog {
const message = min.core.getParam<string>(min.instance, 'Not Found Message', Messages[locale].did_not_find);
await min.conversationalService.sendText(min, step, message);
return await step.replaceDialog('/ask', { isReturning: true });
}
];
@ -287,7 +296,7 @@ export class AskDialog extends IGBDialog {
return await step.replaceDialog('/ask', { isReturning: true });
}
}
private static getChannel(step): string {
return !isNaN(step.context.activity['mobile']) ? 'whatsapp' : step.context.activity.channelId;
}