GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

new(core.gbapp): Added support for STT (IBM Watson).

Browse source
This commit is contained in:
Rodrigo Rodriguez 2020-08-07 18:36:42 -03:00
parent 73b711663d
commit 42b4ad546d
3 changed files with 1125 additions and 186 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

970
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

4
package.json
View file

@ -78,14 +78,16 @@
"express-remove-route": "1.0.0", "express-remove-route": "1.0.0",
"ffmpeg-static": "4.2.5", "ffmpeg-static": "4.2.5",
"google-libphonenumber": "3.2.10", "google-libphonenumber": "3.2.10",
"ibm-watson": "^5.6.2",
"js-beautify": "1.11.0", "js-beautify": "1.11.0",
"marked": "1.1.0", "marked": "1.1.0",
"microsoft-cognitiveservices-speech-sdk": "1.12.1", "microsoft-cognitiveservices-speech-sdk": "^1.13.1",
"ms-rest-azure": "3.0.0", "ms-rest-azure": "3.0.0",
"nexmo": "2.8.0", "nexmo": "2.8.0",
"ngrok": "3.2.7", "ngrok": "3.2.7",
"npm": "6.14.6", "npm": "6.14.6",
"opn": "6.0.0", "opn": "6.0.0",
"phone": "^2.4.14",
"pragmatismo-io-framework": "1.0.20", "pragmatismo-io-framework": "1.0.20",
"prism-media": "1.2.2", "prism-media": "1.2.2",
"public-ip": "4.0.2", "public-ip": "4.0.2",

277
packages/core.gbapp/services/GBConversationalService.ts
View file

@ -40,22 +40,24 @@ import { MessageFactory, RecognizerResult } from 'botbuilder';
import { LuisRecognizer } from 'botbuilder-ai'; import { LuisRecognizer } from 'botbuilder-ai';
import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService } from 'botlib'; import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService } from 'botlib';
import { GBServer } from '../../../src/app'; import { GBServer } from '../../../src/app';
import { Readable } from 'stream' import { Readable } from 'stream';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService'; import { GBAdminService } from '../../admin.gbapp/services/GBAdminService';
import { SecService } from '../../security.gbapp/services/SecService'; import { SecService } from '../../security.gbapp/services/SecService';
import { AnalyticsService } from '../../analytics.gblib/services/AnalyticsService'; import { AnalyticsService } from '../../analytics.gblib/services/AnalyticsService';
const urlJoin = require('url-join'); const urlJoin = require('url-join');
const PasswordGenerator = require("strict-password-generator").default; const PasswordGenerator = require('strict-password-generator').default;
const Nexmo = require('nexmo'); const Nexmo = require('nexmo');
const { join } = require('path') const { join } = require('path');
const shell = require('any-shell-escape') const shell = require('any-shell-escape');
const { exec } = require('child_process') const { exec } = require('child_process');
const fs = require('fs') const prism = require('prism-media');
const prism = require('prism-media') const sdk = require('microsoft-cognitiveservices-speech-sdk');
const sdk = require("microsoft-cognitiveservices-speech-sdk");
sdk.Recognizer.enableTelemetry(false); sdk.Recognizer.enableTelemetry(false);
const uuidv4 = require('uuid/v4'); const uuidv4 = require('uuid/v4');
const request = require('request-promise-native'); const request = require('request-promise-native');
const fs = require('fs');
const SpeechToTextV1 = require('ibm-watson/speech-to-text/v1');
const { IamAuthenticator } = require('ibm-watson/auth');
export interface LanguagePickerSettings { export interface LanguagePickerSettings {
defaultLocale?: string; defaultLocale?: string;
@ -91,24 +93,27 @@ export class GBConversationalService {
return step.context.activity.locale; return step.context.activity.locale;
} }
public async sendFile(
min: GBMinInstance,
public async sendFile(min: GBMinInstance, step: GBDialogStep, mobile: string, url: string, caption: string): Promise<any> { step: GBDialogStep,
mobile: string,
url: string,
caption: string
): Promise<any> {
if (step !== null) { if (step !== null) {
if (!isNaN(step.context.activity.from.id as any)) { if (!isNaN(step.context.activity.from.id as any)) {
mobile = step.context.activity.from.id; mobile = step.context.activity.from.id;
GBLog.info(`Sending file ${url} to ${mobile}...`) GBLog.info(`Sending file ${url} to ${mobile}...`);
const filename = url.substring(url.lastIndexOf('/') + 1); const filename = url.substring(url.lastIndexOf('/') + 1);
await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename, caption); await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename, caption);
} } else {
else { GBLog.info(
GBLog.info(`Sending ${url} as file attachment not available in this channel ${step.context.activity.from.id}...`); `Sending ${url} as file attachment not available in this channel ${step.context.activity.from.id}...`
);
await min.conversationalService.sendText(min, step, url); await min.conversationalService.sendText(min, step, url);
} }
} } else {
else { GBLog.info(`Sending file ${url} to ${mobile}...`);
GBLog.info(`Sending file ${url} to ${mobile}...`)
const filename = url.substring(url.lastIndexOf('/') + 1); const filename = url.substring(url.lastIndexOf('/') + 1);
await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename, caption); await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename, caption);
} }
@ -132,8 +137,7 @@ export class GBConversationalService {
// tslint:disable:no-unsafe-any due to Nexmo. // tslint:disable:no-unsafe-any due to Nexmo.
public async sendSms(min: GBMinInstance, mobile: string, text: string): Promise<any> { public async sendSms(min: GBMinInstance, mobile: string, text: string): Promise<any> {
return new Promise( return new Promise((resolve: any, reject: any): any => {
(resolve: any, reject: any): any => {
const nexmo = new Nexmo({ const nexmo = new Nexmo({
apiKey: min.instance.smsKey, apiKey: min.instance.smsKey,
apiSecret: min.instance.smsSecret apiSecret: min.instance.smsSecret
@ -146,8 +150,7 @@ export class GBConversationalService {
resolve(data); resolve(data);
} }
}); });
} });
);
} }
public async sendToMobile(min: GBMinInstance, mobile: string, message: string) { public async sendToMobile(min: GBMinInstance, mobile: string, message: string) {
@ -167,38 +170,27 @@ export class GBConversationalService {
try { try {
speechConfig.speechSynthesisLanguage = locale; speechConfig.speechSynthesisLanguage = locale;
speechConfig.speechSynthesisVoiceName = "pt-BR-FranciscaNeural"; speechConfig.speechSynthesisVoiceName = 'pt-BR-FranciscaNeural';
synthesizer.speakTextAsync(text, synthesizer.speakTextAsync(text, result => {
(result) => {
if (result.reason === sdk.ResultReason.SynthesizingAudioCompleted) { if (result.reason === sdk.ResultReason.SynthesizingAudioCompleted) {
let raw = Buffer.from(result.audioData); let raw = Buffer.from(result.audioData);
fs.writeFileSync(waveFilename, raw); fs.writeFileSync(waveFilename, raw);
GBLog.info(`Audio data byte size: ${result.audioData.byteLength}.`) GBLog.info(`Audio data byte size: ${result.audioData.byteLength}.`);
const oggFilenameOnly = `tmp${name}.ogg`; const oggFilenameOnly = `tmp${name}.ogg`;
const oggFilename = `work/${oggFilenameOnly}`; const oggFilename = `work/${oggFilenameOnly}`;
const output = fs.createWriteStream(oggFilename); const output = fs.createWriteStream(oggFilename);
const transcoder = new prism.FFmpeg({ const transcoder = new prism.FFmpeg({
args: [ args: ['-analyzeduration', '0', '-loglevel', '0', '-f', 'opus', '-ar', '16000', '-ac', '1']
'-analyzeduration', '0',
'-loglevel', '0',
'-f', 'opus',
'-ar', '16000',
'-ac', '1',
],
}); });
fs.createReadStream(waveFilename) fs.createReadStream(waveFilename).pipe(transcoder).pipe(output);
.pipe(transcoder)
.pipe(output);
let url = urlJoin(GBServer.globals.publicAddress, 'audios', oggFilenameOnly); let url = urlJoin(GBServer.globals.publicAddress, 'audios', oggFilenameOnly);
resolve(url); resolve(url);
} else { } else {
const error = "Speech synthesis canceled, " + result.errorDetails; const error = 'Speech synthesis canceled, ' + result.errorDetails;
reject(error); reject(error);
} }
synthesizer.close(); synthesizer.close();
@ -217,7 +209,7 @@ export class GBConversationalService {
let serviceRegion = cloudRegion; let serviceRegion = cloudRegion;
const oggFile = new Readable(); const oggFile = new Readable();
oggFile._read = () => { } // _read is required but you can noop it oggFile._read = () => {}; // _read is required but you can noop it
oggFile.push(buffer); oggFile.push(buffer);
oggFile.push(null); oggFile.push(null);
@ -228,48 +220,78 @@ export class GBConversationalService {
fs.writeFileSync(src, oggFile.read()); fs.writeFileSync(src, oggFile.read());
const makeMp3 = shell([ const makeMp3 = shell([
'node_modules/ffmpeg-static/ffmpeg.exe', '-y', '-v', 'error', 'node_modules/ffmpeg-static/ffmpeg.exe',
'-i', join(process.cwd(), src), '-y',
'-ar', '16000', '-v',
'-ac', '1', 'error',
'-acodec', 'pcm_s16le', '-i',
join(process.cwd(), src),
'-ar',
'44100',
'-ac',
'1',
'-acodec',
'pcm_s16le',
join(process.cwd(), dest) join(process.cwd(), dest)
]) ]);
exec(makeMp3, (error) => { exec(makeMp3, error => {
if (error) { if (error) {
GBLog.error(error); GBLog.error(error);
return Promise.reject(error); return Promise.reject(error);
} else { } else {
let data = fs.readFileSync(dest); let data = fs.readFileSync(dest);
let pushStream = sdk.AudioInputStream.createPushStream(); const speechToText = new SpeechToTextV1({
pushStream.write(data); authenticator: new IamAuthenticator({ apikey: process.env.WATSON_STT_KEY }),
pushStream.close(); url: process.env.WATSON_STT_URL
let audioConfig = sdk.AudioConfig.fromStreamInput(pushStream);
let speechConfig = sdk.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion);
speechConfig.speechRecognitionLanguage = locale;
let recognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig);
recognizer.recognizeOnceAsync(
(result) => {
resolve(result.text ? result.text : 'Speech to Text failed: Audio not converted');
recognizer.close();
recognizer = undefined;
},
(err) => {
reject(err);
recognizer.close();
recognizer = undefined;
}); });
const params = {
audio: data,
contentType: 'audio/l16; rate=44100',
model: "pt-BR_BroadbandModel",
keywords: ['azul', 'céu', 'sol'],
keywordsThreshold: 0.5
};
speechToText
.recognize(params)
.then(response => {
if (response.result.results.length > 0) {
resolve(response.result.results[0].alternatives[0].transcript);
} }
}) })
.catch(error => {
GBLog.error(error);
return Promise.reject(error);
});
// let pushStream = sdk.AudioInputStream.createPushStream();
// pushStream.write(data);
// pushStream.close();
// let audioConfig = sdk.AudioConfig.fromStreamInput(pushStream);
// let speechConfig = sdk.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion);
// speechConfig.speechRecognitionLanguage = locale;
// let recognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig);
// recognizer.recognizeOnceAsync(
// (result) => {
// resolve(result.text ? result.text : 'Speech to Text failed: Audio not converted');
// recognizer.close();
// recognizer = undefined;
// },
// (err) => {
// reject(err);
// recognizer.close();
// recognizer = undefined;
// });
}
});
} catch (error) { } catch (error) {
GBLog.error(error); GBLog.error(error);
return Promise.reject(error); return Promise.reject(error);
@ -277,16 +299,14 @@ export class GBConversationalService {
}); });
} }
// tslint:enable:no-unsafe-any // tslint:enable:no-unsafe-any
public async sendMarkdownToMobile(min: GBMinInstance, step: GBDialogStep, mobile: string, text: string) { public async sendMarkdownToMobile(min: GBMinInstance, step: GBDialogStep, mobile: string, text: string) {
let sleep = ms => {
let sleep = (ms) => {
return new Promise(resolve => { return new Promise(resolve => {
setTimeout(resolve, ms) setTimeout(resolve, ms);
}) });
} };
enum State { enum State {
InText, InText,
InImage, InImage,
@ -300,8 +320,8 @@ export class GBConversationalService {
InEmbedAddressEnd, InEmbedAddressEnd,
InLineBreak, InLineBreak,
InLineBreak1, InLineBreak1,
InLineBreak2, InLineBreak2
}; }
let state = State.InText; let state = State.InText;
let currentImage = ''; let currentImage = '';
let currentText = ''; let currentText = '';
@ -314,17 +334,13 @@ export class GBConversationalService {
switch (state) { switch (state) {
case State.InText: case State.InText:
if (c === '!') { if (c === '!') {
state = State.InImageBegin; state = State.InImageBegin;
} } else if (c === '[') {
else if (c === '[') {
state = State.InEmbedBegin; state = State.InEmbedBegin;
} } else if (c === '\n') {
else if (c === '\n') {
state = State.InLineBreak; state = State.InLineBreak;
} } else {
else {
state = State.InText; state = State.InText;
currentText = currentText.concat(c); currentText = currentText.concat(c);
} }
@ -332,11 +348,9 @@ export class GBConversationalService {
case State.InLineBreak: case State.InLineBreak:
if (c === '\n') { if (c === '\n') {
state = State.InLineBreak1; state = State.InLineBreak1;
} } else if (c === '!') {
else if (c === '!') {
state = State.InImageBegin; state = State.InImageBegin;
} } else if (c === '[') {
else if (c === '[') {
state = State.InEmbedBegin; state = State.InEmbedBegin;
} else { } else {
currentText = currentText.concat('\n', c); currentText = currentText.concat('\n', c);
@ -347,18 +361,15 @@ export class GBConversationalService {
if (c === '\n') { if (c === '\n') {
if (mobile === null) { if (mobile === null) {
await step.context.sendActivity(currentText); await step.context.sendActivity(currentText);
} } else {
else {
await this.sendToMobile(min, mobile, currentText); await this.sendToMobile(min, mobile, currentText);
} }
await sleep(3000); await sleep(3000);
currentText = ''; currentText = '';
state = State.InText; state = State.InText;
} } else if (c === '!') {
else if (c === '!') {
state = State.InImageBegin; state = State.InImageBegin;
} } else if (c === '[') {
else if (c === '[') {
state = State.InEmbedBegin; state = State.InEmbedBegin;
} else { } else {
currentText = currentText.concat('\n', '\n', c); currentText = currentText.concat('\n', '\n', c);
@ -370,8 +381,7 @@ export class GBConversationalService {
if (currentText !== '') { if (currentText !== '') {
if (mobile === null) { if (mobile === null) {
await step.context.sendActivity(currentText); await step.context.sendActivity(currentText);
} } else {
else {
await this.sendToMobile(min, mobile, currentText); await this.sendToMobile(min, mobile, currentText);
} }
await sleep(3000); await sleep(3000);
@ -388,8 +398,7 @@ export class GBConversationalService {
await this.sendFile(min, step, mobile, url, null); await this.sendFile(min, step, mobile, url, null);
await sleep(5000); await sleep(5000);
currentEmbedUrl = ''; currentEmbedUrl = '';
} } else {
else {
currentEmbedUrl = currentEmbedUrl.concat(c); currentEmbedUrl = currentEmbedUrl.concat(c);
} }
break; break;
@ -403,16 +412,14 @@ export class GBConversationalService {
if (currentText !== '') { if (currentText !== '') {
if (mobile === null) { if (mobile === null) {
await step.context.sendActivity(currentText); await step.context.sendActivity(currentText);
} } else {
else {
await this.sendToMobile(min, mobile, currentText); await this.sendToMobile(min, mobile, currentText);
} }
await sleep(2900); await sleep(2900);
} }
currentText = ''; currentText = '';
state = State.InImageCaption; state = State.InImageCaption;
} } else {
else {
state = State.InText; state = State.InText;
currentText = currentText.concat('!').concat(c); currentText = currentText.concat('!').concat(c);
} }
@ -420,8 +427,7 @@ export class GBConversationalService {
case State.InImageCaption: case State.InImageCaption:
if (c === ']') { if (c === ']') {
state = State.InImageAddressBegin; state = State.InImageAddressBegin;
} } else {
else {
currentCaption = currentCaption.concat(c); currentCaption = currentCaption.concat(c);
} }
break; break;
@ -438,27 +444,22 @@ export class GBConversationalService {
currentCaption = ''; currentCaption = '';
await sleep(4500); await sleep(4500);
currentImage = ''; currentImage = '';
} } else {
else {
currentImage = currentImage.concat(c); currentImage = currentImage.concat(c);
} }
break; break;
} }
} }
if (currentText !== '') { if (currentText !== '') {
if (mobile === null) { if (mobile === null) {
await step.context.sendActivity(currentText); await step.context.sendActivity(currentText);
} else {
}
else {
await this.sendToMobile(min, mobile, currentText); await this.sendToMobile(min, mobile, currentText);
} }
} }
} }
public async routeNLP(step: GBDialogStep, min: GBMinInstance, text: string): Promise<boolean> { public async routeNLP(step: GBDialogStep, min: GBMinInstance, text: string): Promise<boolean> {
if (min.instance.nlpAppId === null || min.instance.nlpAppId === undefined) { if (min.instance.nlpAppId === null || min.instance.nlpAppId === undefined) {
return false; return false;
} }
@ -490,7 +491,7 @@ export class GBConversationalService {
let nlpActive = false; let nlpActive = false;
Object.keys(nlp.intents).forEach((name) => { Object.keys(nlp.intents).forEach(name => {
const score = nlp.intents[name].score; const score = nlp.intents[name].score;
if (score > min.instance.nlpScore) { if (score > min.instance.nlpScore) {
nlpActive = true; nlpActive = true;
@ -501,7 +502,6 @@ export class GBConversationalService {
const topIntent = LuisRecognizer.topIntent(nlp); const topIntent = LuisRecognizer.topIntent(nlp);
if (topIntent !== undefined && nlpActive) { if (topIntent !== undefined && nlpActive) {
const intent = topIntent; const intent = topIntent;
// tslint:disable:no-unsafe-any // tslint:disable:no-unsafe-any
const firstEntity = nlp.entities && nlp.entities.length > 0 ? nlp.entities[0].entity.toUpperCase() : undefined; const firstEntity = nlp.entities && nlp.entities.length > 0 ? nlp.entities[0].entity.toUpperCase() : undefined;
@ -527,23 +527,16 @@ export class GBConversationalService {
return Promise.resolve(false); return Promise.resolve(false);
} }
async translate(min: GBMinInstance, key: string, endPoint: string, text: string, language: string): Promise<string> {
async translate(min: GBMinInstance,
key: string,
endPoint: string,
text: string,
language: string
): Promise<string> {
const translatorEnabled = () => { const translatorEnabled = () => {
if (min.instance.params) { if (min.instance.params) {
const params = JSON.parse(min.instance.params); const params = JSON.parse(min.instance.params);
return params ? params['Enable Worldwide Translator'] === "TRUE" : false; return params ? params['Enable Worldwide Translator'] === 'TRUE' : false;
} }
return false; return false;
} // TODO: Encapsulate. }; // TODO: Encapsulate.
if (endPoint === null || (!translatorEnabled() || process.env.TRANSLATOR_DISABLED === "true")) { if (endPoint === null || !translatorEnabled() || process.env.TRANSLATOR_DISABLED === 'true') {
return text; return text;
} }
@ -558,7 +551,7 @@ export class GBConversationalService {
url: 'translate', url: 'translate',
qs: { qs: {
'api-version': '3.0', 'api-version': '3.0',
'to': [language] to: [language]
}, },
headers: { headers: {
'Ocp-Apim-Subscription-Key': key, 'Ocp-Apim-Subscription-Key': key,
@ -566,11 +559,13 @@ export class GBConversationalService {
'Content-type': 'application/json', 'Content-type': 'application/json',
'X-ClientTraceId': uuidv4().toString() 'X-ClientTraceId': uuidv4().toString()
}, },
body: [{ body: [
'text': text {
}], text: text
json: true,
} }
],
json: true
};
try { try {
const results = await request(options); const results = await request(options);
@ -588,10 +583,10 @@ export class GBConversationalService {
let sec = new SecService(); let sec = new SecService();
const member = step.context.activity.from; const member = step.context.activity.from;
const user = await sec.ensureUser(min.instance.instanceId, member.id, const user = await sec.ensureUser(min.instance.instanceId, member.id, member.name, '', 'web', member.name);
member.name, "", "web", member.name);
if (text !== null) { if (text !== null) {
text = await min.conversationalService.translate(min, text = await min.conversationalService.translate(
min,
min.instance.translatorKey ? min.instance.translatorKey : minBoot.instance.translatorKey, min.instance.translatorKey ? min.instance.translatorKey : minBoot.instance.translatorKey,
min.instance.translatorEndpoint ? min.instance.translatorEndpoint : minBoot.instance.translatorEndpoint, min.instance.translatorEndpoint ? min.instance.translatorEndpoint : minBoot.instance.translatorEndpoint,
text, text,
@ -599,18 +594,18 @@ export class GBConversationalService {
); );
} }
return await step.prompt("textPrompt", text ? text : {}); return await step.prompt('textPrompt', text ? text : {});
} }
public async sendText(min, step, text) { public async sendText(min, step, text) {
let sec = new SecService(); let sec = new SecService();
const member = step.context.activity.from; const member = step.context.activity.from;
const user = await sec.ensureUser(min.instance.instanceId, member.id, const user = await sec.ensureUser(min.instance.instanceId, member.id, member.name, '', 'web', member.name);
member.name, "", "web", member.name);
if (user) { if (user) {
const minBoot = GBServer.globals.minBoot as any; const minBoot = GBServer.globals.minBoot as any;
text = await min.conversationalService.translate(min, text = await min.conversationalService.translate(
min,
min.instance.translatorKey ? min.instance.translatorKey : minBoot.instance.translatorKey, min.instance.translatorKey ? min.instance.translatorKey : minBoot.instance.translatorKey,
min.instance.translatorEndpoint ? min.instance.translatorEndpoint : minBoot.instance.translatorEndpoint, min.instance.translatorEndpoint ? min.instance.translatorEndpoint : minBoot.instance.translatorEndpoint,
text, text,
@ -618,17 +613,13 @@ export class GBConversationalService {
); );
const analytics = new AnalyticsService(); const analytics = new AnalyticsService();
const userProfile = await min.userProfile.get(step.context, {}); const userProfile = await min.userProfile.get(step.context, {});
analytics.createMessage(min.instance.instanceId, analytics.createMessage(min.instance.instanceId, userProfile.conversation, null, text);
userProfile.conversation, null,
text);
if (!isNaN(member.id)) { if (!isNaN(member.id)) {
await min.whatsAppDirectLine.sendToDevice(member.id, text); await min.whatsAppDirectLine.sendToDevice(member.id, text);
} } else {
else {
await step.context.sendActivity(text); await step.context.sendActivity(text);
} }
} }
} }
} }