From 4452a311b18db74dd2b32492a14cd87881a610de Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Tue, 28 Dec 2021 15:51:14 -0300 Subject: [PATCH] fix(all): Added MSFT cognitive stack again to test if it is working now on Azure. --- package-lock.json | 210 ++++++++++++++++++ package.json | 1 + .../services/GBConversationalService.ts | 47 ++++ .../services/WhatsappDirectLine.ts | 13 +- 4 files changed, 264 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 05417af1..bd36bfef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3818,6 +3818,33 @@ "safer-buffer": "~2.1.0" } }, + "asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "requires": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + } + }, + "asn1.js-rfc2560": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/asn1.js-rfc2560/-/asn1.js-rfc2560-5.0.1.tgz", + "integrity": "sha512-1PrVg6kuBziDN3PGFmRk3QrjpKvP9h/Hv5yMrFZvC1kpzP6dQRzf5BpKstANqHBkaOUmTpakJWhicTATOA/SbA==", + "requires": { + "asn1.js-rfc5280": "^3.0.0" + } + }, + "asn1.js-rfc5280": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/asn1.js-rfc5280/-/asn1.js-rfc5280-3.0.0.tgz", + "integrity": "sha512-Y2LZPOWeZ6qehv698ZgOGGCZXBQShObWnGthTrIFlIQjuV1gg2B8QOhWFRExq/MR1VnPpIIe7P9vX2vElxv+Pg==", + "requires": { + "asn1.js": "^5.0.0" + } + }, "assert": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/assert/-/assert-1.5.0.tgz", @@ -3846,6 +3873,30 @@ "lodash": "^4.17.14" } }, + "async-disk-cache": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/async-disk-cache/-/async-disk-cache-2.1.0.tgz", + "integrity": "sha512-iH+boep2xivfD9wMaZWkywYIURSmsL96d6MoqrC94BnGSvXE4Quf8hnJiHGFYhw/nLeIa1XyRaf4vvcvkwAefg==", + "requires": { + "debug": "^4.1.1", + "heimdalljs": "^0.2.3", + "istextorbinary": "^2.5.1", + "mkdirp": "^0.5.0", + "rimraf": "^3.0.0", + "rsvp": "^4.8.5", + "username-sync": "^1.0.2" + }, + "dependencies": { + "debug": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "requires": { + "ms": "2.1.2" + } + } + } + }, "async-promises": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/async-promises/-/async-promises-0.2.3.tgz", @@ -4692,6 +4743,23 @@ "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==", "dev": true }, + "bent": { + "version": "7.3.12", + "resolved": "https://registry.npmjs.org/bent/-/bent-7.3.12.tgz", + "integrity": "sha512-T3yrKnVGB63zRuoco/7Ybl7BwwGZR0lceoVG5XmQyMIH9s19SV5m+a8qam4if0zQuAmOQTyPTPmsQBdAorGK3w==", + "requires": { + "bytesish": "^0.4.1", + "caseless": "~0.12.0", + "is-stream": "^2.0.0" + }, + "dependencies": { + "is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" + } + } + }, "big-integer": { "version": "1.6.48", "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", @@ -4716,6 +4784,11 @@ "resolved": "https://registry.npmjs.org/binary-search-bounds/-/binary-search-bounds-2.0.3.tgz", "integrity": "sha1-X/hhbW3SylOIvIWy1iZuK52lAtw=" }, + "binaryextensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binaryextensions/-/binaryextensions-2.3.0.tgz", + "integrity": "sha512-nAihlQsYGyc5Bwq6+EsubvANYGExeJKHDO3RjnvwU042fawQTQfM3Kxn7IHUXQOz4bzfwsGYYHGSvXyW4zOGLg==" + }, "bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -4743,6 +4816,11 @@ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, "body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", @@ -5582,6 +5660,11 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, + "bytesish": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/bytesish/-/bytesish-0.4.4.tgz", + "integrity": "sha512-i4uu6M4zuMUiyfZN4RU2+i9+peJh//pXhd9x1oSe1LBkZ3LEbCoygu8W0bXTukU1Jme2txKuotpCZRaC3FLxcQ==" + }, "cacheable-lookup": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", @@ -8163,6 +8246,22 @@ "safe-buffer": "^5.0.1" } }, + "editions": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/editions/-/editions-2.3.1.tgz", + "integrity": "sha512-ptGvkwTvGdGfC0hfhKg0MT+TRLRKGtUiWGBInxOm5pz7ssADezahjCUaYuZ8Dr+C05FW0AECIIPt4WBxVINEhA==", + "requires": { + "errlop": "^2.0.0", + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, "editorconfig": { "version": "0.15.3", "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", @@ -8350,6 +8449,11 @@ } } }, + "errlop": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/errlop/-/errlop-2.2.0.tgz", + "integrity": "sha512-e64Qj9+4aZzjzzFpZC7p5kmm/ccCrbLhAJplhsDXQFs87XTsXwOpH4s1Io2s90Tau/8r2j9f4l/thhDevRjzxw==" + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -10115,6 +10219,21 @@ } } }, + "heimdalljs": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/heimdalljs/-/heimdalljs-0.2.6.tgz", + "integrity": "sha512-o9bd30+5vLBvBtzCPwwGqpry2+n0Hi6H1+qwt6y+0kwRHGGF8TFIhJPmnuM0xO97zaKrDZMwO/V56fAnn8m/tA==", + "requires": { + "rsvp": "~3.2.1" + }, + "dependencies": { + "rsvp": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-3.2.1.tgz", + "integrity": "sha1-B8tKXfJa3Z6Cbrxn3Mn9idsn2Eo=" + } + } + }, "hoek": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", @@ -11154,6 +11273,16 @@ "lodash.uniqby": "^4.7.0" } }, + "istextorbinary": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/istextorbinary/-/istextorbinary-2.6.0.tgz", + "integrity": "sha512-+XRlFseT8B3L9KyjxxLjfXSLMuErKDsd8DBNrsaxoViABMEZlOSCstwmw0qpoFX3+U6yWU1yhLudAe6/lETGGA==", + "requires": { + "binaryextensions": "^2.1.2", + "editions": "^2.2.0", + "textextensions": "^2.5.0" + } + }, "iterall": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/iterall/-/iterall-1.3.0.tgz", @@ -12795,6 +12924,62 @@ } } }, + "microsoft-cognitiveservices-speech-sdk": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.19.0.tgz", + "integrity": "sha512-FYyXdJcTLX8Iunnn9XHzTfR+43YdEZn1q89KX6Vzqtaa4hz/LemdC9Zni50VIfkXMNtq2sJFYf5wKXvxm1wQHA==", + "requires": { + "agent-base": "^6.0.1", + "asn1.js-rfc2560": "^5.0.1", + "asn1.js-rfc5280": "^3.0.0", + "async-disk-cache": "^2.1.0", + "bent": "^7.3.12", + "https-proxy-agent": "^4.0.0", + "simple-lru-cache": "0.0.2", + "url-parse": "^1.4.7", + "uuid": "^8.3.0", + "ws": "^7.3.1" + }, + "dependencies": { + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + } + }, + "debug": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "requires": { + "ms": "2.1.2" + } + }, + "https-proxy-agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", + "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", + "requires": { + "agent-base": "5", + "debug": "4" + }, + "dependencies": { + "agent-base": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", + "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" + } + } + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + } + } + }, "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", @@ -12829,6 +13014,11 @@ "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", "dev": true }, + "minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -17783,6 +17973,11 @@ "resolved": "https://registry.npmjs.org/rsa-pem-from-mod-exp/-/rsa-pem-from-mod-exp-0.8.4.tgz", "integrity": "sha1-NipCxtMEBW1JOz8SvOq7LGV2ptQ=" }, + "rsvp": { + "version": "4.8.5", + "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz", + "integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==" + }, "run-async": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", @@ -18719,6 +18914,11 @@ "integrity": "sha512-w/3FEDN94r4JQ9WoYrIr8RqDIPZdyNkdpbK9glFady1CAEyD97XWCv8HFetQO21w81e7h7Nh59iYTyG1mUJftg==", "dev": true }, + "simple-lru-cache": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz", + "integrity": "sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0=" + }, "simple-swizzle": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", @@ -19624,6 +19824,11 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "textextensions": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/textextensions/-/textextensions-2.6.0.tgz", + "integrity": "sha512-49WtAWS+tcsy93dRt6P0P3AMD2m5PvXRhuEA0kaXos5ZLlujtYmpmFsB+QvWUSxE1ZsstmYXfQ7L40+EcQgpAQ==" + }, "textract": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/textract/-/textract-2.5.0.tgz", @@ -20634,6 +20839,11 @@ "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", "integrity": "sha1-/FZaPMy/93MMd19WQflVV5FDnyE=" }, + "username-sync": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/username-sync/-/username-sync-1.0.3.tgz", + "integrity": "sha512-m/7/FSqjJNAzF2La448c/aEom0gJy7HY7Y509h6l0ePvEkFictAGptwWaj1msWJ38JbfEDOUoE8kqFee9EHKdA==" + }, "utf-8-validate": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.5.tgz", diff --git a/package.json b/package.json index f95a3453..76ae2f1d 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "js-beautify": "1.13.13", "luxon": "2.0.2", "marked": "2.0.7", + "microsoft-cognitiveservices-speech-sdk": "^1.19.0", "momentjs": "2.0.0", "ms-rest-azure": "3.0.0", "nexmo": "2.9.1", diff --git a/packages/core.gbapp/services/GBConversationalService.ts b/packages/core.gbapp/services/GBConversationalService.ts index 28a16828..68464305 100644 --- a/packages/core.gbapp/services/GBConversationalService.ts +++ b/packages/core.gbapp/services/GBConversationalService.ts @@ -292,6 +292,53 @@ export class GBConversationalService { await min.whatsAppDirectLine.sendToDevice(mobile, message, conversationId); } + public static async getAudioBufferFromText(speechKey, cloudRegion, text, locale): Promise { + return new Promise(async (resolve, reject) => { + const name = GBAdminService.getRndReadableIdentifier(); + + const waveFilename = `work/tmp${name}.pcm`; + const sdk = require('microsoft-cognitiveservices-speech-sdk'); + sdk.Recognizer.enableTelemetry(false); + + var audioConfig = sdk.AudioConfig.fromAudioFileOutput(waveFilename); + var speechConfig = sdk.SpeechConfig.fromSubscription(speechKey, cloudRegion); + + var synthesizer = new sdk.SpeechSynthesizer(speechConfig, audioConfig); + + try { + speechConfig.speechSynthesisLanguage = locale; + speechConfig.speechSynthesisVoiceName = 'pt-BR-FranciscaNeural'; + + synthesizer.speakTextAsync(text, result => { + if (result.reason === sdk.ResultReason.SynthesizingAudioCompleted) { + let raw = Buffer.from(result.audioData); + fs.writeFileSync(waveFilename, raw); + GBLog.info(`Audio data byte size: ${result.audioData.byteLength}.`); + const oggFilenameOnly = `tmp${name}.ogg`; + const oggFilename = `work/${oggFilenameOnly}`; + + const output = fs.createWriteStream(oggFilename); + const transcoder = new prism.FFmpeg({ + args: ['-analyzeduration', '0', '-loglevel', '0', '-f', 'opus', '-ar', '16000', '-ac', '1'] + }); + + fs.createReadStream(waveFilename).pipe(transcoder).pipe(output); + + let url = urlJoin(GBServer.globals.publicAddress, 'audios', oggFilenameOnly); + resolve(url); + } else { + const error = 'Speech synthesis canceled, ' + result.errorDetails; + reject(error); + } + synthesizer.close(); + synthesizer = undefined; + }); + } catch (error) { + reject(error); + } + }); + } + public static async getTextFromAudioBuffer(speechKey, cloudRegion, buffer, locale): Promise { return new Promise(async (resolve, reject) => { try { diff --git a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts index 6c11f153..7c1e0c51 100644 --- a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts +++ b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts @@ -523,13 +523,12 @@ export class WhatsappDirectLine extends GBService { public async sendTextAsAudioToDevice(to, msg) { - // DISABLED: - // const url = await GBConversationalService.getAudioBufferFromText( - // this.min.instance.speechKey, - // this.min.instance.cloudLocation, - // msg, this.locale - // ); - const url = 'https://'; + const url = await GBConversationalService.getAudioBufferFromText( + this.min.instance.speechKey, + this.min.instance.cloudLocation, + msg, this.locale + ); + await this.sendFileToDevice(to, url, 'Audio', msg); }