GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix(llm.gblib): Talk to data local db use fix.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2024-12-12 09:46:31 -03:00
parent 173d473166
commit 9cebd81a37
1 changed files with 24 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
packages/kb.gbapp/services/KBService.ts
View file

@ -1571,6 +1571,15 @@ export class KBService implements IGBKBService {
return filePath; // Return the saved file path
} else {
// Get the last part of the URL path or default to 'index' if empty
const pathParts = parsedUrl.pathname.split('/').filter(Boolean); // Remove empty parts
const lastPath = pathParts.length > 0 ? pathParts[pathParts.length - 1] : 'index';
const flatLastPath = lastPath.replace(/\W+/g, '-'); // Flatten the last part of the path
const fileName = `${flatLastPath}.html`;
const filePath = path.join(directoryPath, fileName);
// Configure request interception before navigation
await page.setRequestInterception(true);
page.on('request', request => {
@ -1583,21 +1592,18 @@ export class KBService implements IGBKBService {
});
// Navigate with strict timeout and wait for content
// Navigate and get content even if page fails to load fully
let content = '';
try {
await page.goto(url, {
waitUntil: 'networkidle0', // Wait until network is idle
timeout: 30000 // 30 second timeout
});
} catch (err) {
// Ignore timeout/navigation errors
}
const parsedUrl = new URL(url);
// Get the last part of the URL path or default to 'index' if empty
const pathParts = parsedUrl.pathname.split('/').filter(Boolean); // Remove empty parts
const lastPath = pathParts.length > 0 ? pathParts[pathParts.length - 1] : 'index';
const flatLastPath = lastPath.replace(/\W+/g, '-'); // Flatten the last part of the path
const fileName = `${flatLastPath}.html`;
const filePath = path.join(directoryPath, fileName);
// Get whatever HTML content was loaded
const htmlContent = await page.content();
// Convert HTML to Markdown using html2md