fix(llm.gblib): Talk to data local db use fix.

This commit is contained in:
Rodrigo Rodriguez 2024-11-24 12:43:58 -03:00
parent ba320489f6
commit 5b69a12d74
3 changed files with 35 additions and 21 deletions

View file

@ -656,9 +656,7 @@ export class KBService implements IGBKBService {
// Checks for text after the image markdown, after the element 4, there are text blocks.
const removeMarkdownImages = (text: string) => {
// Remove both inline images ![alt](url) and reference images ![alt][ref]
return text.replace(/!\[[^\]]*\](?:\([^)]*\)|\[[^\]]*\])/g, '').trim();
}
@ -1046,7 +1044,8 @@ export class KBService implements IGBKBService {
const websiteIgnoreUrls = min.core.getParam<[]>(min.instance, 'Website Ignore URLs', null);
GBLogEx.info(min, `Website: ${website}, Max Depth: ${maxDepth}, Ignore URLs: ${websiteIgnoreUrls}`);
let shouldSave = false;
if (website) {
// Removes last slash if any.
@ -1128,6 +1127,7 @@ export class KBService implements IGBKBService {
await CollectionUtil.asyncForEach(files, async file => {
let content = null;
shouldSave = true;
try {
const document = await this.loadAndSplitFile(file);
@ -1144,6 +1144,7 @@ export class KBService implements IGBKBService {
files = await walkPromise(urlJoin(localPath, 'docs'));
if (files[0]) {
shouldSave = true;
GBLogEx.info(min, `Add embeddings from .gbkb: ${files.length}}...`);
await CollectionUtil.asyncForEach(files, async file => {
let content = null;
@ -1154,7 +1155,7 @@ export class KBService implements IGBKBService {
await min['vectorStore'].addDocuments(flattenedDocuments);
});
}
if (min['vectorStore']) {
if (shouldSave && min['vectorStore']) {
await min['vectorStore'].save(min['vectorStorePath']);
}
}

View file

@ -538,22 +538,36 @@ export class ChatServices {
logging: true
});
} else {
const host = con['storageServer'];
const port = con['storagePort'];
const storageName = con['storageName'];
const username = con['storageUsername'];
const password = con['storagePassword'];
dataSource = new DataSource({
type: dialect as any,
host: host,
port: port,
database: storageName,
username: username,
password: password,
synchronize: false,
logging: true
});
if (dialect === 'sqlite') {
const storageFile = con['storageFile'];
dataSource = new DataSource({
type: 'sqlite',
database: storageFile,
synchronize: false,
logging: true
});
}
else {
const host = con['storageServer'];
const port = con['storagePort'];
const storageName = con['storageName'];
const username = con['storageUsername'];
const password = con['storagePassword'];
dataSource = new DataSource({
type: dialect as any,
host: host,
port: port,
database: storageName,
username: username,
password: password,
synchronize: false,
logging: true
});
}
}
const db = await SqlDatabase.fromDataSourceParams({

View file

@ -1,7 +1,6 @@
name,value
Admin, 5521999998888
Answer Mode,sql
Answer Source,cache
Answer Mode,chart
llm File,northwind.db
llm Driver,sqlite
Theme Color,red
1 name value
2 Admin 5521999998888
3 Answer Mode sql chart
Answer Source cache
4 llm File northwind.db
5 llm Driver sqlite
6 Theme Color red