GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix(core.gbapp): Fix in NLP handling of acents.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2021-01-03 18:29:41 -03:00
parent e7d7a1a4b2
commit 2a5ecd487f
8 changed files with 70 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
packages/basic.gblib/strings.ts
View file

@ -1,34 +1,9 @@
export const Messages = {
global_quit: /^(sair|sai|chega|exit|quit|finish|end|ausfahrt|verlassen)/i,
'en-US': {
show_video: 'I will show you a video, please wait...',
good_morning: 'good morning',
good_evening: 'good evening',
good_night: 'good night',
hi: (msg) => `Hello, ${msg}.`,
very_sorry_about_error: `I'm sorry to inform that there was an error which was recorded to be solved.`,
canceled: 'Canceled. If I can be useful, let me know how',
whats_email: 'What\'s your E-mail address?',
which_language: 'Please, type the language name you would like to talk through.',
validation_enter_valid_email: 'Please enter a valid e-mail.' ,
language_chosen: 'Very good, so let\'s go...' ,
affirmative_sentences: /^(sim|s|positivo|afirmativo|claro|evidente|sem dúvida|confirmo|confirmar|confirmado|uhum|si|y|yes|sure)/i
},
'pt-BR': {
show_video: 'Vou te mostrar um vídeo. Por favor, aguarde...',
good_morning: 'bom dia',
good_evening: 'boa tarde',
good_night: 'boa noite',
hi: (msg) => `Oi, ${msg}.`,
very_sorry_about_error: `Lamento, ocorreu um erro que já foi registrado para ser tratado.`,
canceled: 'Cancelado, avise como posso ser útil novamente.',
whats_email: 'Qual seu e-mail?',
which_language: 'Por favor, digite o idioma que você gostaria de usar para conversarmos.',
validation_enter_valid_email: 'Por favor digite um email válido.',
language_chosen: 'Muito bem, então vamos lá...' ,
affirmative_sentences: /^(sim|s|positivo|afirmativo|claro|evidente|sem dúvida|confirmo|confirmar|confirmado|uhum|si|y|yes|sure)/i
}
};

16
packages/core.gbapp/services/GBConversationalService.ts
View file

@ -46,10 +46,10 @@ import { SecService } from '../../security.gbapp/services/SecService';
import { AnalyticsService } from '../../analytics.gblib/services/AnalyticsService';
import { MicrosoftAppCredentials } from 'botframework-connector';
import { GBConfigService } from './GBConfigService';
import { CollectionUtil, AzureText } from 'pragmatismo-io-framework';
const urlJoin = require('url-join');
const PasswordGenerator = require('strict-password-generator').default;
const Nexmo = require('nexmo');
import { CollectionUtil, AzureText } from 'pragmatismo-io-framework';
const { join } = require('path');
const shell = require('any-shell-escape');
const { exec } = require('child_process');
@ -60,23 +60,25 @@ const fs = require('fs');
const SpeechToTextV1 = require('ibm-watson/speech-to-text/v1');
const { IamAuthenticator } = require('ibm-watson/auth');
export interface LanguagePickerSettings {
defaultLocale?: string;
supportedLocales?: string[];
}
/**
* Provides basic services for handling messages and dispatching to back-end
* services like NLP or Search.
*/
export class GBConversationalService {
/**
* Reference to the core service.
*/
public coreService: IGBCoreService;
/**
*
* @param coreService
*/
constructor(coreService: IGBCoreService) {
this.coreService = coreService;
}
static defaultDiacriticsRemovalMap = [
{ 'base': 'A', 'letters': '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F' },
{ 'base': 'AA', 'letters': '\uA732' },

2
packages/core.gbapp/services/GBCoreService.ts
View file

@ -100,7 +100,7 @@ export class GBCoreService implements IGBCoreService {
private dialect: string;
/**
* Constructor retrieves default values.
*
*/
constructor() {
this.adminService = new GBAdminService(this);

1
packages/core.gbapp/services/GBDeployer.ts
View file

@ -800,6 +800,7 @@ export class GBDeployer implements IGBDeployer {
'whatsapp.gblib',
'sharepoint.gblib',
'core.gbapp',
'basic.gblib',
'admin.gbapp',
'azuredeployer.gbapp',
'customer-satisfaction.gbapp',

38
packages/core.gbapp/services/GBMinService.ts
View file

@ -62,7 +62,7 @@ import {
IGBInstance,
IGBPackage
} from 'botlib';
import { AzureText, CollectionUtil } from 'pragmatismo-io-framework';
import { CollectionUtil } from 'pragmatismo-io-framework';
import { MicrosoftAppCredentials } from 'botframework-connector';
import fs = require('fs');
@ -112,8 +112,6 @@ export class GBMinService {
/**
* Static initialization of minimal instance.
*
* @param core Basic database services to identify instance, for example.
*/
constructor(
core: IGBCoreService,
@ -582,7 +580,7 @@ export class GBMinService {
const userState = new UserState(storage);
adapter.use(new AutoSaveStateMiddleware(conversationState, userState));
MicrosoftAppCredentials.trustServiceUrl('https://directline.botframework.com',
new Date(new Date().setFullYear(new Date().getFullYear() + 10))
new Date(new Date().setFullYear(new Date().getFullYear() + 10))
);
// The minimal bot is built here.
@ -612,7 +610,7 @@ export class GBMinService {
await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
let services: ConcatArray<never>;
if ((services = await e.onExchangeData(min, 'getServices', null))) {
min.gbappServices = {...min.gbappServices, ...services};
min.gbappServices = { ...min.gbappServices, ...services };
}
});
@ -958,7 +956,7 @@ export class GBMinService {
process.env.GBKB_ENABLE_AUTO_PUBLISH === 'true'
) {
await min.conversationalService.sendText(min, step,
`Oi, ainda não possuo pacotes de conhecimento publicados. Por favor, aguarde alguns segundos enquanto eu auto-publico alguns pacotes.`
`Oi, ainda não possuo pacotes de conhecimento publicados. Por favor, aguarde alguns segundos enquanto eu auto-publico alguns pacotes.`
);
await step.beginDialog('/publish', { confirm: true, firstTime: true });
} else {
@ -984,18 +982,36 @@ export class GBMinService {
keepTextList = keepTextList.concat(result);
}
});
let textProcessed = GBConversationalService.removeDiacritics(text);
const getNormalizedRegExp = (value) => {
var chars = [
{ letter: 'a', reg: '[aáàãäâ]' },
{ letter: 'e', reg: '[eéèëê]' },
{ letter: 'i', reg: '[iíìïî]' },
{ letter: 'o', reg: '[oóòõöô]' },
{ letter: 'u', reg: '[uúùüû]' },
{ letter: 'c', reg: '[cç]' }
];
for (var i in chars) {
value = value.replace(new RegExp(chars[i].letter, 'gi'), chars[i].reg);
};
return value;
};
let textProcessed = text;
if (keepTextList) {
keepTextList = keepTextList.filter(p => p.trim() !== '');
let i = 0;
await CollectionUtil.asyncForEach(keepTextList, item => {
const it = GBConversationalService.removeDiacritics(item);
const noAccentText = GBConversationalService.removeDiacritics(textProcessed);
if (textProcessed.toLowerCase().indexOf(it.toLowerCase()) != -1) {
if (noAccentText.toLowerCase().indexOf(it.toLowerCase()) != -1) {
const replacementToken = 'X' + GBAdminService.getNumberIdentifier().substr(0, 4);
replacements[i] = { text: item, replacementToken: replacementToken };
i++;
textProcessed = textProcessed.replace(new RegExp(`\\b${it.trim()}\\b`, 'gi'), `${replacementToken}`);
textProcessed = textProcessed.replace(new RegExp(`\\b${getNormalizedRegExp(it.trim())}\\b`, 'gi'), `${replacementToken}`);
}
});
}
@ -1008,10 +1024,10 @@ export class GBMinService {
// Detects user typed language and updates their locale profile if applies.
let locale = min.core.getParam<string>(min.instance, 'Default User Language',
GBConfigService.get('DEFAULT_USER_LANGUAGE')
GBConfigService.get('DEFAULT_USER_LANGUAGE')
);
const detectLanguage = min.core.getParam<boolean>(min.instance, 'Language Detector',
GBConfigService.getBoolean('LANGUAGE_DETECTOR')
GBConfigService.getBoolean('LANGUAGE_DETECTOR')
) === 'true';
const systemUser = user.systemUser;
locale = systemUser.locale;

24
packages/core.gbapp/strings.ts
View file

@ -2,9 +2,33 @@
export const Messages = {
global_quit: /^(sair|sai|chega|exit|quit|finish|end|ausfahrt|verlassen)/i,
'en-US': {
show_video: 'I will show you a video, please wait...',
good_morning: 'good morning',
good_evening: 'good evening',
good_night: 'good night',
hi: (msg) => `Hello, ${msg}.`,
very_sorry_about_error: `I'm sorry to inform that there was an error which was recorded to be solved.`,
canceled: 'Canceled. If I can be useful, let me know how',
whats_email: 'What\'s your E-mail address?',
which_language: 'Please, type the language name you would like to talk through.',
validation_enter_valid_email: 'Please enter a valid e-mail.',
language_chosen: 'Very good, so let\'s go...',
affirmative_sentences: /^(sim|s|positivo|afirmativo|claro|evidente|sem dúvida|confirmo|confirmar|confirmado|uhum|si|y|yes|sure)/i
},
'pt-BR': {
show_video: 'Vou te mostrar um vídeo. Por favor, aguarde...',
good_morning: 'bom dia',
good_evening: 'boa tarde',
good_night: 'boa noite',
hi: (msg) => `Oi, ${msg}.`,
very_sorry_about_error: `Lamento, ocorreu um erro que já foi registrado para ser tratado.`,
canceled: 'Cancelado, avise como posso ser útil novamente.',
whats_email: 'Qual seu e-mail?',
which_language: 'Por favor, digite o idioma que você gostaria de usar para conversarmos.',
validation_enter_valid_email: 'Por favor digite um email válido.',
language_chosen: 'Muito bem, então vamos lá...',
affirmative_sentences: /^(sim|s|positivo|afirmativo|claro|evidente|sem dúvida|confirmo|confirmar|confirmado|uhum|si|y|yes|sure)/i
}
};

BIN
packages/default.gbtest/first-test.xlsx Normal file
View file

Binary file not shown.

16
packages/kb.gbapp/dialogs/AskDialog.ts
View file

@ -116,9 +116,11 @@ export class AskDialog extends IGBDialog {
const user = await sec.ensureUser(min.instance.instanceId, member.id, member.name, '', 'web', member.name);
let handled = false;
let nextDialog = null;
await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
if (
await e.onExchangeData(min, 'handleAnswer', {
nextDialog = await e.onExchangeData(min, 'handleAnswer', {
query: text,
step: step,
message: text,
@ -129,13 +131,11 @@ export class AskDialog extends IGBDialog {
}
});
if (!handled) {
return await step.beginDialog('/answer', {
query: text
});
} else {
return await step.next();
}
await step.beginDialog(nextDialog ? nextDialog : '/answer', {
query: text,
user: user ? user['dataValues'] : null,
message: text
});
} else {
return await step.next();
}