diff --git a/packages/kb.gbapp/services/KBService.ts b/packages/kb.gbapp/services/KBService.ts index b81cc0b7f..fb0016149 100644 --- a/packages/kb.gbapp/services/KBService.ts +++ b/packages/kb.gbapp/services/KBService.ts @@ -1223,21 +1223,29 @@ export class KBService implements IGBKBService { await CollectionUtil.asyncForEach(files, async file => { let content = null; let filePath = path.join(file.root, file.name); + let skip = false; try { + if (file.root.endsWith('.gbdata')) + { + skip = true; if (file.name.endsWith('.csv')) { + skip = false; // Read first 1000 lines of CSV file const csvContent = await fs.readFile(filePath, 'utf8'); const lines = csvContent.split('\n').slice(0, 200).join('\n'); await fs.writeFile(filePath, lines, 'utf8'); content = lines; } + } + if (!skip){ const document = await this.loadAndSplitFile(filePath); // TODO: Add full filename. const flattenedDocuments = document.reduce((acc, val) => acc.concat(val), []); await min['vectorStore'].addDocuments(flattenedDocuments); GBLogEx.info(min, `Added ${filePath} to vector store.`); + } } catch (error) { GBLogEx.info(min, `Ignore processing of ${file}. ${GBUtil.toYAML(error)}`); }