From 6442ebf9834d520bb1c0662ac69e529ac9bd5620 Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Thu, 30 Apr 2020 21:14:22 -0300 Subject: [PATCH] fix(whatsapp.gblib): Speech is now full duplex. --- package-lock.json | 142 +----------------- package.json | 10 +- .../services/GBConversationalService.ts | 98 ++++++------ packages/core.gbapp/services/GBCoreService.ts | 2 +- src/app.ts | 2 +- 5 files changed, 60 insertions(+), 194 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d369f0d..7b25a35c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2150,6 +2150,11 @@ "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" }, + "any-shell-escape": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/any-shell-escape/-/any-shell-escape-0.1.1.tgz", + "integrity": "sha1-1Vq5ciRMcaml4asIefML8RCAaVk=" + }, "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", @@ -2908,11 +2913,6 @@ "resolved": "https://registry.npmjs.org/binary-search-bounds/-/binary-search-bounds-2.0.3.tgz", "integrity": "sha1-X/hhbW3SylOIvIWy1iZuK52lAtw=" }, - "bindings": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", - "integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE=" - }, "bl": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz", @@ -3420,20 +3420,6 @@ "ieee754": "^1.1.4" } }, - "buffer-alloc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", - "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", - "requires": { - "buffer-alloc-unsafe": "^1.1.0", - "buffer-fill": "^1.0.0" - } - }, - "buffer-alloc-unsafe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", - "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" - }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -3444,11 +3430,6 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, - "buffer-fill": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", - "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=" - }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -9597,11 +9578,6 @@ "integrity": "sha512-ISDqGcspVu6U3VKqtJZG1uR55SmNNF9uK0EMq1IvNVVZOui6MW6VR0+pIZhqz85ORAGp+4zW+5fJ/SE7bwEibA==", "dev": true }, - "nan": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", - "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==" - }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -14055,48 +14031,6 @@ "integrity": "sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ==", "dev": true }, - "ogg": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/ogg/-/ogg-1.2.6.tgz", - "integrity": "sha512-5TXXQJbECkxkwfJkXSUdl09gbFdv/K/fTS0YqrWGWiHVXiaCoiUZ3EB8RtBYyXSWc0eKkEbUAh4Wic8st7MbFA==", - "requires": { - "bindings": "~1.2.0", - "debug": "2", - "nan": "2", - "readable-stream": "1.0" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "readable-stream": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", - "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" - } - } - } - }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -14839,7 +14773,8 @@ "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "dev": true }, "prepend-http": { "version": "2.0.0", @@ -18787,69 +18722,6 @@ } } }, - "vorbis": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/vorbis/-/vorbis-0.2.2.tgz", - "integrity": "sha512-sGyyyzlWLankqp3NPUsPbeQ01AFRGgNyptiLxFW+4UCEQbfHqlNTIcFQCKtfBY5kqkbNOoqpkRW4JUnB/ehJ1A==", - "requires": { - "bindings": "^1.2.0", - "buffer-alloc": "^1.1.0", - "debug": "^2.2.0", - "nan": "^2.10.0", - "ogg": "^1.2.5", - "readable-stream": "^2.3.6" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - } - } - }, - "vorbis-encoder-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/vorbis-encoder-js/-/vorbis-encoder-js-1.0.2.tgz", - "integrity": "sha1-eBuIEv7w8bei0VwQMeXnsR6E8Xw=", - "requires": { - "prelude-ls": "^1.1.2" - } - }, "wait-until": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/wait-until/-/wait-until-0.0.2.tgz", diff --git a/package.json b/package.json index 269310f9..aa981e4a 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "Dário Vieira " ], "engines": { - "node": "=10.15.2" + "node": "=10.15.2" }, "license": "AGPL-3.0", "preferGlobal": true, @@ -49,10 +49,10 @@ }, "dependencies": { "@azure/ms-rest-js": "2.0.4", - "@discordjs/opus": "0.1.0", "@microsoft/microsoft-graph-client": "2.0.0", "@types/validator": "12.0.1", "adal-node": "0.2.1", + "any-shell-escape": "^0.1.1", "async-promises": "0.2.2", "azure-arm-cognitiveservices": "3.0.0", "azure-arm-resource": "7.3.0", @@ -84,11 +84,10 @@ "nexmo": "2.5.2", "ngrok": "3.2.7", "npm": "6.13.4", - + "prism-media": "1.2.1", "opn": "6.0.0", "pragmatismo-io-framework": "1.0.20", - "prism-media": "1.2.1", - "public-ip": "4.0.0", + "public-ip": "4.0.0", "readline": "1.3.0", "reflect-metadata": "0.1.13", "request-promise": "4.2.5", @@ -107,7 +106,6 @@ "typescript": "3.7.4", "url-join": "4.0.1", "vbscript-to-typescript": "1.0.8", - "wait-until": "0.0.2", "walk-promise": "0.2.0", "washyourmouthoutwithsoap": "1.0.2" diff --git a/packages/core.gbapp/services/GBConversationalService.ts b/packages/core.gbapp/services/GBConversationalService.ts index 2c9fbc7d..e879993b 100644 --- a/packages/core.gbapp/services/GBConversationalService.ts +++ b/packages/core.gbapp/services/GBConversationalService.ts @@ -42,14 +42,16 @@ import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService } from 'botlib'; import { AzureText } from 'pragmatismo-io-framework'; import { Messages } from '../strings'; import { GBServer } from '../../../src/app'; +import { Readable } from 'stream' +import { GBAdminService } from '../../admin.gbapp/services/GBAdminService'; const urlJoin = require('url-join'); const PasswordGenerator = require("strict-password-generator").default; const Nexmo = require('nexmo'); -let sdk = require("microsoft-cognitiveservices-speech-sdk"); -var fs = require('fs') -import { Readable } from 'stream' -import { GBAdminService } from '../../admin.gbapp/services/GBAdminService'; -const prism = require('prism-media'); +const { join } = require('path') +const shell = require('any-shell-escape') +const { exec } = require('child_process') +const fs = require('fs') +const sdk = require("microsoft-cognitiveservices-speech-sdk"); export interface LanguagePickerSettings { defaultLocale?: string; @@ -147,7 +149,7 @@ export class GBConversationalService { try { speechConfig.speechSynthesisLanguage = locale; - speechConfig.speechSynthesisVoiceName = "pt-BR-HeloisaRUS"; + speechConfig.speechSynthesisVoiceName = "pt-BR-FranciscaNeural"; synthesizer.speakTextAsync(text, (result) => { @@ -158,15 +160,6 @@ export class GBConversationalService { const oggFilenameOnly = `tmp${name}.ogg`; const oggFilename = `work/${oggFilenameOnly}`; - const ffmpeg = prism.FFmpeg.getInfo(); - - console.log(`Using FFmpeg version ${ffmpeg.version}`); - - if (ffmpeg.output.includes('--enable-libopus')) { - console.log('libopus is available!'); - } else { - console.log('libopus is unavailable!'); - } const output = fs.createWriteStream(oggFilename); const transcoder = new prism.FFmpeg({ @@ -183,7 +176,6 @@ export class GBConversationalService { .pipe(transcoder) .pipe(output); - console.log("synthesis finished."); let url = urlJoin(GBServer.globals.publicAddress, 'audios', oggFilenameOnly); resolve(url); @@ -206,11 +198,6 @@ export class GBConversationalService { let subscriptionKey = speechKey; let serviceRegion = cloudRegion; - var samplingRate = 16000; - var frameDuration = 20; - var channels = 1; - var frameSize = samplingRate * frameDuration / 1000; - const oggFile = new Readable(); oggFile._read = () => { } // _read is required but you can noop it oggFile.push(buffer); @@ -218,44 +205,53 @@ export class GBConversationalService { const name = GBAdminService.getRndReadableIdentifier(); - fs.writeFileSync(`work/tmp${name}.ogg`, buffer); + const dest = `work/tmp${name}.wav`; + const src = `work/tmp${name}.ogg`; + fs.writeFileSync(src, oggFile.read()); - let wr = fs.createWriteStream(`work/tmp${name}.pcm`); - wr.on('finish', () => { - let data = fs.readFileSync(`work/tmp${name}.pcm`); + const makeMp3 = shell([ + 'node_modules/ffmpeg-static/ffmpeg.exe', '-y', '-v', 'error', + '-i', join(process.cwd(), src), + '-ar', '16000', + '-ac', '1', + '-acodec', 'pcm_s16le', + join(process.cwd(), dest) + ]) - let pushStream = sdk.AudioInputStream.createPushStream(); - pushStream.write(data); - pushStream.close(); + exec(makeMp3, (error) => { + if (error) { + GBLog.error(error); + return Promise.reject(error); + } else { + let data = fs.readFileSync(dest); - let audioConfig = sdk.AudioConfig.fromStreamInput(pushStream); - let speechConfig = sdk.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion); - speechConfig.speechRecognitionLanguage = locale; - let recognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig); + let pushStream = sdk.AudioInputStream.createPushStream(); + pushStream.write(data); + pushStream.close(); - recognizer.recognizeOnceAsync( - (result) => { + let audioConfig = sdk.AudioConfig.fromStreamInput(pushStream); + let speechConfig = sdk.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion); + speechConfig.speechRecognitionLanguage = locale; + let recognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig); - resolve(result.text ? result.text : 'Speech to Text failed: Audio not converted'); + recognizer.recognizeOnceAsync( + (result) => { - recognizer.close(); - recognizer = undefined; - }, - (err) => { - reject(err); + resolve(result.text ? result.text : 'Speech to Text failed: Audio not converted'); - recognizer.close(); - recognizer = undefined; - }); - }); + recognizer.close(); + recognizer = undefined; + }, + (err) => { + reject(err); + + recognizer.close(); + recognizer = undefined; + }); + + } + }) - fs.createReadStream(`work/tmp${name}.ogg`) - .pipe(new prism.opus.OggDemuxer()) - .pipe(new prism.opus.Decoder({ - rate: samplingRate, - channels: channels, frameSize: frameSize - })) - .pipe(wr); } catch (error) { GBLog.error(error); return Promise.reject(error); diff --git a/packages/core.gbapp/services/GBCoreService.ts b/packages/core.gbapp/services/GBCoreService.ts index cb9ffbdc..e6b22cc4 100644 --- a/packages/core.gbapp/services/GBCoreService.ts +++ b/packages/core.gbapp/services/GBCoreService.ts @@ -351,7 +351,7 @@ STORAGE_SYNC=true } else { - throw new Error(`Error updating bot proxy, details: ${error.message}.`); + throw new Error(`Error updating bot proxy, details: ${error}.`); } } }); diff --git a/src/app.ts b/src/app.ts index f10187ab..33c90dcf 100644 --- a/src/app.ts +++ b/src/app.ts @@ -82,7 +82,7 @@ export class GBServer { */ public static run() { - + GBLog.info(`The Bot Server is in STARTING mode...`); GBServer.globals = new RootData(); GBConfigService.init();