GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix(KBService): skip processing of files in gbdata path
All checks were successful
GBCI / build (push) Successful in 47s
Details

Browse source
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-05-13 22:33:29 -03:00
parent 5e492d590e
commit 0f0f8d610e
1 changed files with 8 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
packages/kb.gbapp/services/KBService.ts
View file

@ -1223,21 +1223,29 @@ export class KBService implements IGBKBService {
await CollectionUtil.asyncForEach(files, async file => { await CollectionUtil.asyncForEach(files, async file => {
let content = null; let content = null;
let filePath = path.join(file.root, file.name); let filePath = path.join(file.root, file.name);
let skip = false;
try { try {
if (file.root.endsWith('.gbdata'))
{
skip = true;
if (file.name.endsWith('.csv')) { if (file.name.endsWith('.csv')) {
skip = false;
// Read first 1000 lines of CSV file // Read first 1000 lines of CSV file
const csvContent = await fs.readFile(filePath, 'utf8'); const csvContent = await fs.readFile(filePath, 'utf8');
const lines = csvContent.split('\n').slice(0, 200).join('\n'); const lines = csvContent.split('\n').slice(0, 200).join('\n');
await fs.writeFile(filePath, lines, 'utf8'); await fs.writeFile(filePath, lines, 'utf8');
content = lines; content = lines;
} }
}
if (!skip){
const document = await this.loadAndSplitFile(filePath); const document = await this.loadAndSplitFile(filePath);
// TODO: Add full filename. // TODO: Add full filename.
const flattenedDocuments = document.reduce((acc, val) => acc.concat(val), []); const flattenedDocuments = document.reduce((acc, val) => acc.concat(val), []);
await min['vectorStore'].addDocuments(flattenedDocuments); await min['vectorStore'].addDocuments(flattenedDocuments);
GBLogEx.info(min, `Added ${filePath} to vector store.`); GBLogEx.info(min, `Added ${filePath} to vector store.`);
}
} catch (error) { } catch (error) {
GBLogEx.info(min, `Ignore processing of ${file}. ${GBUtil.toYAML(error)}`); GBLogEx.info(min, `Ignore processing of ${file}. ${GBUtil.toYAML(error)}`);
} }