GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

new(all): Auto import for logo, colors and website content.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2024-05-23 14:11:33 -03:00
parent f42e42c5e2
commit fc1bfa8e18
1 changed files with 35 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
packages/kb.gbapp/services/KBService.ts
View file

@ -879,7 +879,15 @@ export class KBService implements IGBKBService {
return null;
}
async crawl(min, url: string, visited: Set<string>, depth: number, maxDepth: number, page: Page, websiteIgnoreUrls): Promise<string[]> {
async crawl(
min,
url: string,
visited: Set<string>,
depth: number,
maxDepth: number,
page: Page,
websiteIgnoreUrls
): Promise<string[]> {
try {
if (
depth > maxDepth ||
@ -905,17 +913,15 @@ export class KBService implements IGBKBService {
}
const currentDomain = new URL(page.url()).hostname;
let links = await page.evaluate(({currentDomain, websiteIgnoreUrls}) => {
let links = await page.evaluate(
({ currentDomain, websiteIgnoreUrls }) => {
const anchors = Array.from(document.querySelectorAll('a')).filter(p => {
try {
// Check if urlToCheck contains any of the ignored URLs
const isIgnored = websiteIgnoreUrls.split(";").some(ignoredUrl => p.href.includes(ignoredUrl));
const isIgnored = websiteIgnoreUrls.split(';').some(ignoredUrl => p.href.includes(ignoredUrl));
return !isIgnored && currentDomain == new URL(p.href).hostname;
} catch (err) {
return false;
}
@ -924,7 +930,9 @@ export class KBService implements IGBKBService {
return anchors.map(anchor => {
return anchor.href.replace(/#.*/, '');
});
}, {currentDomain, websiteIgnoreUrls});
},
{ currentDomain, websiteIgnoreUrls }
);
if (!Array.isArray(links)) {
links = [];
@ -1015,12 +1023,10 @@ export class KBService implements IGBKBService {
): Promise<any> {
let files = [];
const website = min.core.getParam<string>(min.instance, 'Website', null);
const websiteIgnoreUrls = min.core.getParam<string>(min.instance, 'Website Ignore URLs', null);
if (website) {
Fs.rmSync(min['vectorStorePath'], { recursive: true, force: true });
let path = DialogKeywords.getGBAIPath(min.botId, `gbot`);
const directoryPath = Path.join(process.env.PWD, 'work', path, 'Website');
@ -1034,7 +1040,8 @@ export class KBService implements IGBKBService {
path = DialogKeywords.getGBAIPath(min.botId);
const logoPath = Path.join(process.env.PWD, 'work', path, 'cache');
const baseUrl = page.url().split('/').slice(0, 3).join('/');
const logoBinary = await page.goto(urlJoin(baseUrl, logo));
logo = logo.startsWith('https') ? logo : urlJoin(baseUrl, logo);
const logoBinary = await page.goto(logo);
const buffer = await logoBinary.buffer();
const logoFilename = Path.basename(logo);
sharp(buffer)