fix(llm.gblib): Talk to data local db use fix.
This commit is contained in:
parent
ba320489f6
commit
5b69a12d74
3 changed files with 35 additions and 21 deletions
|
@ -656,9 +656,7 @@ export class KBService implements IGBKBService {
|
||||||
|
|
||||||
// Checks for text after the image markdown, after the element 4, there are text blocks.
|
// Checks for text after the image markdown, after the element 4, there are text blocks.
|
||||||
|
|
||||||
|
|
||||||
const removeMarkdownImages = (text: string) => {
|
const removeMarkdownImages = (text: string) => {
|
||||||
// Remove both inline images  and reference images ![alt][ref]
|
|
||||||
return text.replace(/!\[[^\]]*\](?:\([^)]*\)|\[[^\]]*\])/g, '').trim();
|
return text.replace(/!\[[^\]]*\](?:\([^)]*\)|\[[^\]]*\])/g, '').trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1046,6 +1044,7 @@ export class KBService implements IGBKBService {
|
||||||
const websiteIgnoreUrls = min.core.getParam<[]>(min.instance, 'Website Ignore URLs', null);
|
const websiteIgnoreUrls = min.core.getParam<[]>(min.instance, 'Website Ignore URLs', null);
|
||||||
GBLogEx.info(min, `Website: ${website}, Max Depth: ${maxDepth}, Ignore URLs: ${websiteIgnoreUrls}`);
|
GBLogEx.info(min, `Website: ${website}, Max Depth: ${maxDepth}, Ignore URLs: ${websiteIgnoreUrls}`);
|
||||||
|
|
||||||
|
let shouldSave = false;
|
||||||
|
|
||||||
if (website) {
|
if (website) {
|
||||||
// Removes last slash if any.
|
// Removes last slash if any.
|
||||||
|
@ -1128,6 +1127,7 @@ export class KBService implements IGBKBService {
|
||||||
|
|
||||||
await CollectionUtil.asyncForEach(files, async file => {
|
await CollectionUtil.asyncForEach(files, async file => {
|
||||||
let content = null;
|
let content = null;
|
||||||
|
shouldSave = true;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const document = await this.loadAndSplitFile(file);
|
const document = await this.loadAndSplitFile(file);
|
||||||
|
@ -1144,6 +1144,7 @@ export class KBService implements IGBKBService {
|
||||||
files = await walkPromise(urlJoin(localPath, 'docs'));
|
files = await walkPromise(urlJoin(localPath, 'docs'));
|
||||||
|
|
||||||
if (files[0]) {
|
if (files[0]) {
|
||||||
|
shouldSave = true;
|
||||||
GBLogEx.info(min, `Add embeddings from .gbkb: ${files.length}}...`);
|
GBLogEx.info(min, `Add embeddings from .gbkb: ${files.length}}...`);
|
||||||
await CollectionUtil.asyncForEach(files, async file => {
|
await CollectionUtil.asyncForEach(files, async file => {
|
||||||
let content = null;
|
let content = null;
|
||||||
|
@ -1154,7 +1155,7 @@ export class KBService implements IGBKBService {
|
||||||
await min['vectorStore'].addDocuments(flattenedDocuments);
|
await min['vectorStore'].addDocuments(flattenedDocuments);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (min['vectorStore']) {
|
if (shouldSave && min['vectorStore']) {
|
||||||
await min['vectorStore'].save(min['vectorStorePath']);
|
await min['vectorStore'].save(min['vectorStorePath']);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -538,22 +538,36 @@ export class ChatServices {
|
||||||
logging: true
|
logging: true
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
const host = con['storageServer'];
|
|
||||||
const port = con['storagePort'];
|
|
||||||
const storageName = con['storageName'];
|
|
||||||
const username = con['storageUsername'];
|
|
||||||
const password = con['storagePassword'];
|
|
||||||
|
|
||||||
dataSource = new DataSource({
|
|
||||||
type: dialect as any,
|
if (dialect === 'sqlite') {
|
||||||
host: host,
|
const storageFile = con['storageFile'];
|
||||||
port: port,
|
dataSource = new DataSource({
|
||||||
database: storageName,
|
type: 'sqlite',
|
||||||
username: username,
|
database: storageFile,
|
||||||
password: password,
|
synchronize: false,
|
||||||
synchronize: false,
|
logging: true
|
||||||
logging: true
|
});
|
||||||
});
|
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
const host = con['storageServer'];
|
||||||
|
const port = con['storagePort'];
|
||||||
|
const storageName = con['storageName'];
|
||||||
|
const username = con['storageUsername'];
|
||||||
|
const password = con['storagePassword'];
|
||||||
|
|
||||||
|
dataSource = new DataSource({
|
||||||
|
type: dialect as any,
|
||||||
|
host: host,
|
||||||
|
port: port,
|
||||||
|
database: storageName,
|
||||||
|
username: username,
|
||||||
|
password: password,
|
||||||
|
synchronize: false,
|
||||||
|
logging: true
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const db = await SqlDatabase.fromDataSourceParams({
|
const db = await SqlDatabase.fromDataSourceParams({
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
name,value
|
name,value
|
||||||
Admin, 5521999998888
|
Admin, 5521999998888
|
||||||
Answer Mode,sql
|
Answer Mode,chart
|
||||||
Answer Source,cache
|
|
||||||
llm File,northwind.db
|
llm File,northwind.db
|
||||||
llm Driver,sqlite
|
llm Driver,sqlite
|
||||||
Theme Color,red
|
Theme Color,red
|
|
Loading…
Add table
Reference in a new issue