GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

new(all): Auto import for logo, colors and website content.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2024-05-23 14:42:04 -03:00
parent fc1bfa8e18
commit 5d32eedba9
1 changed files with 11 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
packages/kb.gbapp/services/KBService.ts
View file

@ -872,7 +872,7 @@ export class KBService implements IGBKBService {
Fs.mkdirSync(directoryPath, { recursive: true }); // Create directory recursively if it doesn't exist Fs.mkdirSync(directoryPath, { recursive: true }); // Create directory recursively if it doesn't exist
Fs.writeFileSync(filePath, buffer); Fs.writeFileSync(filePath, buffer);
await GBUtil.sleep(400);
return filePath; return filePath;
} }
} }
@ -911,7 +911,7 @@ export class KBService implements IGBKBService {
// If the URL doesn't represent an HTML page, skip crawling its links // If the URL doesn't represent an HTML page, skip crawling its links
return []; return [];
} }
const currentDomain = new URL(page.url()).hostname; const currentDomain = new URL(page.url()).hostname.toLocaleLowerCase();
let links = await page.evaluate( let links = await page.evaluate(
({ currentDomain, websiteIgnoreUrls }) => { ({ currentDomain, websiteIgnoreUrls }) => {
@ -920,8 +920,10 @@ export class KBService implements IGBKBService {
// Check if urlToCheck contains any of the ignored URLs // Check if urlToCheck contains any of the ignored URLs
const isIgnored = websiteIgnoreUrls.split(';').some(ignoredUrl => p.href.includes(ignoredUrl)); const isIgnored = websiteIgnoreUrls.split(';').some(ignoredUrl => p.href.includes(ignoredUrl));
console.log(currentDomain);
console.log(new URL(p.href).hostname);
return !isIgnored && currentDomain == new URL(p.href).hostname; return !isIgnored && currentDomain == new URL(p.href).hostname.toLocaleLowerCase();
} catch (err) { } catch (err) {
return false; return false;
} }
@ -1023,10 +1025,15 @@ export class KBService implements IGBKBService {
): Promise<any> { ): Promise<any> {
let files = []; let files = [];
const website = min.core.getParam<string>(min.instance, 'Website', null); let website = min.core.getParam<string>(min.instance, 'Website', null);
const websiteIgnoreUrls = min.core.getParam<string>(min.instance, 'Website Ignore URLs', null); const websiteIgnoreUrls = min.core.getParam<string>(min.instance, 'Website Ignore URLs', null);
if (website) { if (website) {
// Removes last slash if any.
website =website.replace(/\/(?=[^\/]*$)/, "");
Fs.rmSync(min['vectorStorePath'], { recursive: true, force: true }); Fs.rmSync(min['vectorStorePath'], { recursive: true, force: true });
let path = DialogKeywords.getGBAIPath(min.botId, `gbot`); let path = DialogKeywords.getGBAIPath(min.botId, `gbot`);
const directoryPath = Path.join(process.env.PWD, 'work', path, 'Website'); const directoryPath = Path.join(process.env.PWD, 'work', path, 'Website');
@ -1075,11 +1082,9 @@ export class KBService implements IGBKBService {
}); });
page.on('dialog', async dialog => { page.on('dialog', async dialog => {
console.log(dialog.message());
await dialog.dismiss(); await dialog.dismiss();
}); });
page.setDefaultTimeout(15000);
page.setCacheEnabled(false); page.setCacheEnabled(false);
const maxDepth = 1; // Maximum depth of recursion const maxDepth = 1; // Maximum depth of recursion