GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix(core.gbapp):Fix in tabular blank lines and other fixes.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2020-04-02 19:03:57 -03:00
parent 3c717c3f0c
commit 0645c310c3
6 changed files with 157 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
package.json
View file

@ -8,8 +8,9 @@
"contributors": [
"Rodrigo Rodriguez <me@rodrigorodriguez.com>",
"João Ferreira <joao.parana@gmail.com>",
"Jorge Ramos <jramos@pobox.com>"
],
"Jorge Ramos <jramos@pobox.com>",
"PH <ph.an@outlook.com>",
"Dário Vieira <dario.junior3@gmail.com>" ],
"engines": {
"node": "=10.15.2"
},
@ -63,7 +64,7 @@
"botbuilder-ai": "4.7.0",
"botbuilder-dialogs": "4.7.0",
"botframework-connector": "4.7.0",
"botlib": "1.4.4",
"botlib": "1.4.5",
"chai": "4.2.0",
"cli-spinner": "0.2.10",
"csv-parse": "4.8.3",

2
packages/admin.gbapp/services/GBAdminService.ts
View file

@ -73,8 +73,6 @@ export class GBAdminService implements IGBAdminService {
}
GBLog.info('Now, *deploying* package...');
await GBAdminService.deployPackageCommand(min, `deployPackage ${packageName}`, min.deployService);
GBLog.info('Package deployed. Just need to rebuild the index... Doing it right now.');
await GBAdminService.rebuildIndexPackageCommand(min, min.deployService);
GBLog.info('Finished importing of that .gbkb package.');
}

2
packages/core.gbapp/dialogs/WelcomeDialog.ts
View file

@ -66,7 +66,7 @@ export class WelcomeDialog extends IGBDialog {
const user = await min.userProfile.get(step.context, {});
const locale = step.context.activity.locale;
if (!user.once) {
if (!user.once && step.context.activity.channelId === "webchat") {
user.once = true;
await min.userProfile.set(step.context, user);
const a = new Date();

22
packages/core.gbapp/services/GBConversationalService.ts
View file

@ -149,7 +149,8 @@ export class GBConversationalService implements IGBConversationalService {
InEmbedBegin,
InEmbedEnd,
InEmbedAddressBegin,
InEmbedAddressEnd
InEmbedAddressEnd,
InLineBreak,
};
let state = State.InText;
let currentImage = '';
@ -163,16 +164,35 @@ export class GBConversationalService implements IGBConversationalService {
switch (state) {
case State.InText:
if (c === '!') {
state = State.InImageBegin;
}
else if (c === '[') {
state = State.InEmbedBegin;
}
else if (c === '\n') {
state = State.InLineBreak;
}
else {
currentText = currentText.concat(c);
}
break;
case State.InLineBreak:
if (c === '\n') {
if (currentText !== '') {
if (mobile === null) {
await step.context.sendActivity(currentText);
}
else {
this.sendToMobile(min, mobile, currentText);
}
await sleep(3000);
}
currentText = '';
state = State.InText;
}
break;
case State.InEmbedBegin:
if (c === '=') {
if (currentText !== '') {

12
packages/core.gbapp/services/GBDeployer.ts
View file

@ -63,7 +63,7 @@ import { CollectionUtil } from 'pragmatismo-io-framework';
* Deployer service for bots, themes, ai and more.
*/
export class GBDeployer implements IGBDeployer{
export class GBDeployer implements IGBDeployer {
public static deployFolder = 'packages';
public static workFolder = 'work';
public core: IGBCoreService;
@ -368,6 +368,7 @@ export class GBDeployer implements IGBDeployer{
instance.searchIndexer
);
const connectionString = GBDeployer.getConnectionStringFromInstance(instance);
const dsName = 'gb';
@ -380,7 +381,13 @@ export class GBDeployer implements IGBDeployer{
}
}
await search.createDataSource(dsName, dsName, 'GuaribasQuestion', 'azuresql', connectionString);
try {
await search.createDataSource(dsName, dsName, 'GuaribasQuestion', 'azuresql', connectionString);
} catch (err) {
GBLog.error(err);
throw err;
}
try {
await search.deleteIndex();
@ -390,6 +397,7 @@ export class GBDeployer implements IGBDeployer{
throw err;
}
}
await search.createIndex(searchSchema, dsName);
}

197
packages/kb.gbapp/services/KBService.ts
View file

@ -274,92 +274,102 @@ export class KBService implements IGBKBService {
let rows = data._worksheets[1]._rows;
return asyncPromise.eachSeries(rows, async line => {
// Extracts values from columns in the current line.
const subjectsText = line._cells[0].value;
const from = line._cells[1].value;
const to = line._cells[2].value;
const question = line._cells[3].value;
let answer = line._cells[4].value;
// Skips the first line.
if (!(subjectsText === 'subjects' && from === 'from')) {
let format = '.txt';
if (line._cells[0] !== undefined &&
line._cells[1] !== undefined &&
line._cells[2] !== undefined &&
line._cells[3] !== undefined &&
line._cells[4] !== undefined) {
// Extracts values from columns in the current line.
// Extracts answer from external media if any.
const subjectsText = line._cells[0].value;
const from = line._cells[1].value;
const to = line._cells[2].value;
const question = line._cells[3].value;
let answer = line._cells[4].value;
let media = null;
if (answer.indexOf('.md') > -1) {
const mediaFilename = urlJoin(path.dirname(filePath), '..', 'articles', answer);
if (Fs.existsSync(mediaFilename)) {
answer = Fs.readFileSync(mediaFilename, 'utf8');
format = '.md';
media = path.basename(mediaFilename);
} else {
GBLog.info(`[GBImporter] File not found: ${mediaFilename}.`);
answer = '';
if (!(subjectsText === 'subjects' && from === 'from')
&& (answer !== null && question !== null)) {
let format = '.txt';
// Extracts answer from external media if any.
let media = null;
if (answer.indexOf('.md') > -1) {
const mediaFilename = urlJoin(path.dirname(filePath), '..', 'articles', answer);
if (Fs.existsSync(mediaFilename)) {
answer = Fs.readFileSync(mediaFilename, 'utf8');
format = '.md';
media = path.basename(mediaFilename);
} else {
GBLog.info(`[GBImporter] File not found: ${mediaFilename}.`);
answer = '';
}
}
}
// Processes subjects hierarchy splitting by dots.
// Processes subjects hierarchy splitting by dots.
const subjectArray = subjectsText.split('.');
let subject1: string;
let subject2: string;
let subject3: string;
let subject4: string;
let indexer = 0;
const subjectArray = subjectsText.split('.');
let subject1: string;
let subject2: string;
let subject3: string;
let subject4: string;
let indexer = 0;
subjectArray.forEach(element => {
if (indexer === 0) {
subject1 = subjectArray[indexer].substring(0, 63);
} else if (indexer === 1) {
subject2 = subjectArray[indexer].substring(0, 63);
} else if (indexer === 2) {
subject3 = subjectArray[indexer].substring(0, 63);
} else if (indexer === 3) {
subject4 = subjectArray[indexer].substring(0, 63);
subjectArray.forEach(element => {
if (indexer === 0) {
subject1 = subjectArray[indexer].substring(0, 63);
} else if (indexer === 1) {
subject2 = subjectArray[indexer].substring(0, 63);
} else if (indexer === 2) {
subject3 = subjectArray[indexer].substring(0, 63);
} else if (indexer === 3) {
subject4 = subjectArray[indexer].substring(0, 63);
}
indexer++;
});
// Now with all the data ready, creates entities in the store.
const answer1 = await GuaribasAnswer.create({
instanceId: instanceId,
content: answer,
format: format,
media: media,
packageId: packageId,
prevId: lastQuestionId !== null ? lastQuestionId : 0
});
const question1 = await GuaribasQuestion.create({
from: from,
to: to,
subject1: subject1,
subject2: subject2,
subject3: subject3,
subject4: subject4,
content: question,
instanceId: instanceId,
answerId: answer1.answerId,
packageId: packageId
});
if (lastAnswer !== undefined && lastQuestionId !== 0) {
await lastAnswer.update({ nextId: lastQuestionId });
}
indexer++;
});
lastAnswer = answer1;
lastQuestionId = question1.questionId;
// Now with all the data ready, creates entities in the store.
return Promise.resolve(question1.questionId);
} else {
// Skips the header.
const answer1 = await GuaribasAnswer.create({
instanceId: instanceId,
content: answer,
format: format,
media: media,
packageId: packageId,
prevId: lastQuestionId !== null ? lastQuestionId : 0
});
const question1 = await GuaribasQuestion.create({
from: from,
to: to,
subject1: subject1,
subject2: subject2,
subject3: subject3,
subject4: subject4,
content: question,
instanceId: instanceId,
answerId: answer1.answerId,
packageId: packageId
});
if (lastAnswer !== undefined && lastQuestionId !== 0) {
await lastAnswer.update({ nextId: lastQuestionId });
return Promise.resolve(undefined);
}
lastAnswer = answer1;
lastQuestionId = question1.questionId;
return Promise.resolve(question1.questionId);
} else {
// Skips the header.
return Promise.resolve(undefined);
}
});
}
@ -439,22 +449,56 @@ export class KBService implements IGBKBService {
packageStorage: GuaribasPackage,
instance: IGBInstance
): Promise<any> {
// Imports subjects tree into database and return it.
await this.importSubjectFile(packageStorage.packageId, urlJoin(localPath, 'subjects.json'), instance);
// Import all .tsv files in the tabular directory.
// Import tabular files in the tabular directory.
return this.importKbTabularDirectory(localPath, instance, packageStorage.packageId);
await this.importKbTabularDirectory(localPath, instance, packageStorage.packageId);
// Import remaining .md files in articles directory.
return await this.importRemainingArticles(localPath, instance, packageStorage.packageId);
}
/**
* Import all .md files in artcles folder that has not been referenced by tabular files.
*/
public async importRemainingArticles(localPath: string, instance: IGBInstance, packageId: number): Promise<any> {
const files = await walkPromise(urlJoin(localPath, 'articles'));
return Promise.all(
files.map(async file => {
if (file.name.endsWith('.md')) {
let content = await this.getAnswerTextByMediaName(instance.instanceId, file.name);
if (content === null) {
const fullFilename = urlJoin(file.root, file.name);
content = Fs.readFileSync(fullFilename, 'utf-8');
await GuaribasAnswer.create({
instanceId: instance.instanceId,
content: content,
format: ".md",
media: file.name,
packageId: packageId,
prevId: 0 // TODO: Calculate total rows and increment.
});
}
}
}));
}
public async importKbTabularDirectory(localPath: string, instance: IGBInstance, packageId: number): Promise<any> {
const files = await walkPromise(urlJoin(localPath, 'tabular'));
return Promise.all(
files.map(async file => {
if (file.name.endsWith('.xlsx')) {
return this.importKbTabularFile(urlJoin(file.root, file.name), instance.instanceId, packageId);
return await this.importKbTabularFile(urlJoin(file.root, file.name), instance.instanceId, packageId);
}
})
);
@ -501,7 +545,8 @@ export class KBService implements IGBKBService {
where: { instanceId: instance.instanceId, packageId: packageId }
});
await deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex));
GBLog.info("Remember to call rebuild index manually after package removal.");
}
/**