GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix(kb.gbapp): HTML crawler improved.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2024-12-15 16:27:40 -03:00
parent 5d0a58ee6d
commit d694038532
1 changed files with 13 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
packages/kb.gbapp/services/KBService.ts
View file

@ -1019,17 +1019,20 @@ export class KBService implements IGBKBService {
}
async getFreshPage(browser, url) {
try {
if (!browser || browser.isConnected() === false) {
browser = await puppeteer.launch({ headless: false }); // Change headless to true if you don't want to see the browser window
}
const page = await browser.newPage();
await page.goto(url);
return page;
} catch (error) {
console.error('An error occurred while getting fresh page:', error);
throw error;
if (!browser || browser.isConnected() === false) {
browser = await puppeteer.launch({ headless: false });
}
const page = await browser.newPage();
try {
await page.goto(url, {
waitUntil: 'networkidle0', // Wait until network is idle
timeout: 30000 // 30 second timeout
});
} catch (err) {
// Ignore timeout/navigation errors
}
return page;
}
/**