fix(KBService): skip processing of files in gbdata path
All checks were successful
GBCI / build (push) Successful in 47s

This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-05-13 22:33:29 -03:00
parent 5e492d590e
commit 0f0f8d610e

View file

@ -1223,21 +1223,29 @@ export class KBService implements IGBKBService {
await CollectionUtil.asyncForEach(files, async file => {
let content = null;
let filePath = path.join(file.root, file.name);
let skip = false;
try {
if (file.root.endsWith('.gbdata'))
{
skip = true;
if (file.name.endsWith('.csv')) {
skip = false;
// Read first 1000 lines of CSV file
const csvContent = await fs.readFile(filePath, 'utf8');
const lines = csvContent.split('\n').slice(0, 200).join('\n');
await fs.writeFile(filePath, lines, 'utf8');
content = lines;
}
}
if (!skip){
const document = await this.loadAndSplitFile(filePath);
// TODO: Add full filename.
const flattenedDocuments = document.reduce((acc, val) => acc.concat(val), []);
await min['vectorStore'].addDocuments(flattenedDocuments);
GBLogEx.info(min, `Added ${filePath} to vector store.`);
}
} catch (error) {
GBLogEx.info(min, `Ignore processing of ${file}. ${GBUtil.toYAML(error)}`);
}