From 2bbbe502e64ba540de15cab3f8e51f1dbeec21cd Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Fri, 16 Feb 2024 21:26:11 -0300 Subject: [PATCH] fix(all): Translator and spelling and audio. --- .../basic.gblib/services/DialogKeywords.ts | 9 ++++++++- .../services/GBConversationalService.ts | 20 +++++++++---------- packages/core.gbapp/services/GBCoreService.ts | 2 +- packages/kb.gbapp/dialogs/AskDialog.ts | 11 +++++----- packages/kb.gbapp/services/KBService.ts | 15 ++++++++++++++ .../services/WhatsappDirectLine.ts | 12 +++-------- 6 files changed, 42 insertions(+), 27 deletions(-) diff --git a/packages/basic.gblib/services/DialogKeywords.ts b/packages/basic.gblib/services/DialogKeywords.ts index b9fd03e9..1ff98c60 100644 --- a/packages/basic.gblib/services/DialogKeywords.ts +++ b/packages/basic.gblib/services/DialogKeywords.ts @@ -1261,7 +1261,14 @@ export class DialogKeywords { if (user) { // TODO: const translate = user ? user.basicOptions.translatorOn : false; - + text = await min.conversationalService.translate( + min, + text, + user.locale ? user.locale : min.core.getParam(min.instance, 'Locale', + GBConfigService.get('LOCALE')) + ); + GBLog.verbose(`Translated text(playMarkdown): ${text}.`); + await min.conversationalService['sendOnConversation'](min, user, text); } return { status: 0 }; diff --git a/packages/core.gbapp/services/GBConversationalService.ts b/packages/core.gbapp/services/GBConversationalService.ts index cb295bfd..a8f5e9cb 100644 --- a/packages/core.gbapp/services/GBConversationalService.ts +++ b/packages/core.gbapp/services/GBConversationalService.ts @@ -472,7 +472,7 @@ export class GBConversationalService { Fs.writeFileSync(src, oggFile.read()); const makeMp3 = shell([ - 'node_modules/ffmpeg-static/ffmpeg.exe', + 'node_modules/ffmpeg-static/ffmpeg', // TODO: .exe on MSWin. '-y', '-v', 'error', @@ -985,7 +985,12 @@ export class GBConversationalService { let results = await fetch(url, options); results = await results.json(); - return results[0].translations[0].text; + if (results[0]){ + return results[0].translations[0].text; + } + else{ + return text; + } } catch (error) { const msg = `Error calling MSFT Translator service layer. Error is: ${error}.`; @@ -1082,8 +1087,8 @@ export class GBConversationalService { 'Language Detector', false) != false; - locale = user.locale; - if (text != '' && detectLanguage && !locale) { + + if (text != '' && detectLanguage) { locale = await min.conversationalService.getLanguage(min, text); if (user.locale != locale) { @@ -1095,12 +1100,7 @@ export class GBConversationalService { // Translates text into content language, keeping // reserved tokens specified in Config. - const contentLocale = min.core.getParam( - min.instance, - 'Default Content Language', - GBConfigService.get('DEFAULT_CONTENT_LANGUAGE') - ); - text = await min.conversationalService.translate(min, text, contentLocale); + text = await min.conversationalService.translate(min, text, locale); GBLog.verbose(`Translated text (processMessageActivity): ${text}.`); // Restores all token text back after spell checking and translation. diff --git a/packages/core.gbapp/services/GBCoreService.ts b/packages/core.gbapp/services/GBCoreService.ts index bf4f52c2..637a3d9d 100644 --- a/packages/core.gbapp/services/GBCoreService.ts +++ b/packages/core.gbapp/services/GBCoreService.ts @@ -700,7 +700,7 @@ ENDPOINT_UPDATE=true value = instance['dataValues'][name]; if (value === null) { const minBoot = GBServer.globals.minBoot as any; - params = GBUtil.caseInsensitive(minBoot.instance); + params = GBUtil.caseInsensitive(minBoot.instance.dataValues); value = params[name]; } } diff --git a/packages/kb.gbapp/dialogs/AskDialog.ts b/packages/kb.gbapp/dialogs/AskDialog.ts index 63d50a0e..f7b9080c 100644 --- a/packages/kb.gbapp/dialogs/AskDialog.ts +++ b/packages/kb.gbapp/dialogs/AskDialog.ts @@ -106,11 +106,10 @@ export class AskDialog extends IGBDialog { text = Messages[locale].ask_first_time; } else if (step.options && step.options.isReturning && !step.context.activity.group) { const askForMore = min.core.getParam(min.instance, 'Ask For More', null); - if (askForMore){ - text = askForMore ; + if (askForMore) { + text = askForMore; } - else - { + else { return await step.endDialog(null); } @@ -340,7 +339,7 @@ export class AskDialog extends IGBDialog { } private static async handleAnswer(service: KBService, min: GBMinInstance, step: any, user, answer: GuaribasAnswer) { - const text = typeof(answer)==='string'? answer:answer.content; + let text = typeof (answer) === 'string' ? answer : answer.content; if (text.endsWith('.docx')) { const mainName = GBVMService.getMethodNameFromVBSFilename(text); await step.endDialog(); @@ -348,7 +347,7 @@ export class AskDialog extends IGBDialog { } else if (text.startsWith('/')) { return await step.replaceDialog(text, { answer: answer }); } else { - await service.sendAnswer(min, AskDialog.getChannel(step), step, answer); + await service.sendAnswer(min, AskDialog.getChannel(step), step, text); return await step.replaceDialog('/ask', { isReturning: true }); } } diff --git a/packages/kb.gbapp/services/KBService.ts b/packages/kb.gbapp/services/KBService.ts index 9bb9e18a..44b1cba1 100644 --- a/packages/kb.gbapp/services/KBService.ts +++ b/packages/kb.gbapp/services/KBService.ts @@ -285,6 +285,20 @@ export class KBService implements IGBKBService { const instance = min.instance; + const contentLocale = min.core.getParam( + min.instance, + 'Default Content Language', + GBConfigService.get('DEFAULT_CONTENT_LANGUAGE') + ); + + query = await min.conversationalService.translate( + min, + query, + contentLocale + ); + + GBLog.info(`Translated query (prompt): ${query}.`); + // Try simple search first. const data = await this.getAnswerByText(instance.instanceId, query.trim()); @@ -639,6 +653,7 @@ export class KBService implements IGBKBService { } else if (answer.endsWith('.ogg') && process.env.AUDIO_DISABLED !== 'true') { await this.playAudio(min, answer, channel, step, min.conversationalService); } else { + await min.conversationalService.sendText(min, step, answer); await min.conversationalService.sendEvent(min, step, 'stop', undefined); } diff --git a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts index 8835aa56..51d7365c 100644 --- a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts +++ b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts @@ -482,16 +482,10 @@ export class WhatsappDirectLine extends GBService { } if (message.type === 'ptt') { - let url = provider ? message.body : message.text; if (process.env.AUDIO_DISABLED !== 'true') { - const options = { - url: url, - method: 'GET', - encoding: 'binary' - }; - - const res = await fetch(url, options); - const buf = Buffer.from(await res.arrayBuffer()); + const media = await message.downloadMedia(); + const buf = Buffer.from(media.data, 'base64'); + text = await GBConversationalService.getTextFromAudioBuffer( this.min.instance.speechKey, this.min.instance.cloudLocation,