GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix(kb.gbapp): HTML crawler improved.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2024-12-15 16:27:40 -03:00
parent 5d0a58ee6d
commit d694038532
1 changed files with 13 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
packages/kb.gbapp/services/KBService.ts
View file

@ -1019,17 +1019,20 @@ export class KBService implements IGBKBService {
} }
async getFreshPage(browser, url) { async getFreshPage(browser, url) {
try {
if (!browser || browser.isConnected() === false) { if (!browser || browser.isConnected() === false) {
browser = await puppeteer.launch({ headless: false }); // Change headless to true if you don't want to see the browser window browser = await puppeteer.launch({ headless: false });
} }
const page = await browser.newPage(); const page = await browser.newPage();
await page.goto(url); try {
return page; await page.goto(url, {
} catch (error) { waitUntil: 'networkidle0', // Wait until network is idle
console.error('An error occurred while getting fresh page:', error); timeout: 30000 // 30 second timeout
throw error; });
} catch (err) {
// Ignore timeout/navigation errors
} }
return page;
} }
/** /**