fix(KBService): skip processing of files in gbdata path
All checks were successful
GBCI / build (push) Successful in 47s
All checks were successful
GBCI / build (push) Successful in 47s
This commit is contained in:
parent
5e492d590e
commit
0f0f8d610e
1 changed files with 8 additions and 0 deletions
|
@ -1223,21 +1223,29 @@ export class KBService implements IGBKBService {
|
|||
await CollectionUtil.asyncForEach(files, async file => {
|
||||
let content = null;
|
||||
let filePath = path.join(file.root, file.name);
|
||||
let skip = false;
|
||||
try {
|
||||
|
||||
if (file.root.endsWith('.gbdata'))
|
||||
{
|
||||
skip = true;
|
||||
if (file.name.endsWith('.csv')) {
|
||||
skip = false;
|
||||
// Read first 1000 lines of CSV file
|
||||
const csvContent = await fs.readFile(filePath, 'utf8');
|
||||
const lines = csvContent.split('\n').slice(0, 200).join('\n');
|
||||
await fs.writeFile(filePath, lines, 'utf8');
|
||||
content = lines;
|
||||
}
|
||||
}
|
||||
if (!skip){
|
||||
|
||||
const document = await this.loadAndSplitFile(filePath);
|
||||
// TODO: Add full filename.
|
||||
const flattenedDocuments = document.reduce((acc, val) => acc.concat(val), []);
|
||||
await min['vectorStore'].addDocuments(flattenedDocuments);
|
||||
GBLogEx.info(min, `Added ${filePath} to vector store.`);
|
||||
}
|
||||
} catch (error) {
|
||||
GBLogEx.info(min, `Ignore processing of ${file}. ${GBUtil.toYAML(error)}`);
|
||||
}
|
||||
|
|
Loading…
Add table
Reference in a new issue