diff --git a/package-lock.json b/package-lock.json index c14dda8f..7d369f0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2908,6 +2908,11 @@ "resolved": "https://registry.npmjs.org/binary-search-bounds/-/binary-search-bounds-2.0.3.tgz", "integrity": "sha1-X/hhbW3SylOIvIWy1iZuK52lAtw=" }, + "bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE=" + }, "bl": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz", @@ -3415,6 +3420,20 @@ "ieee754": "^1.1.4" } }, + "buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "requires": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" + }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -3425,6 +3444,11 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, + "buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=" + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -9573,6 +9597,11 @@ "integrity": "sha512-ISDqGcspVu6U3VKqtJZG1uR55SmNNF9uK0EMq1IvNVVZOui6MW6VR0+pIZhqz85ORAGp+4zW+5fJ/SE7bwEibA==", "dev": true }, + "nan": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", + "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==" + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -14026,6 +14055,48 @@ "integrity": "sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ==", "dev": true }, + "ogg": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/ogg/-/ogg-1.2.6.tgz", + "integrity": "sha512-5TXXQJbECkxkwfJkXSUdl09gbFdv/K/fTS0YqrWGWiHVXiaCoiUZ3EB8RtBYyXSWc0eKkEbUAh4Wic8st7MbFA==", + "requires": { + "bindings": "~1.2.0", + "debug": "2", + "nan": "2", + "readable-stream": "1.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + } + } + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -14768,8 +14839,7 @@ "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", - "dev": true + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" }, "prepend-http": { "version": "2.0.0", @@ -15893,6 +15963,11 @@ "once": "^1.3.0" } }, + "readline": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/readline/-/readline-1.3.0.tgz", + "integrity": "sha1-xYDXfvLPyHUrEySYBg3JeTp6wBw=" + }, "rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", @@ -18712,6 +18787,69 @@ } } }, + "vorbis": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/vorbis/-/vorbis-0.2.2.tgz", + "integrity": "sha512-sGyyyzlWLankqp3NPUsPbeQ01AFRGgNyptiLxFW+4UCEQbfHqlNTIcFQCKtfBY5kqkbNOoqpkRW4JUnB/ehJ1A==", + "requires": { + "bindings": "^1.2.0", + "buffer-alloc": "^1.1.0", + "debug": "^2.2.0", + "nan": "^2.10.0", + "ogg": "^1.2.5", + "readable-stream": "^2.3.6" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "vorbis-encoder-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/vorbis-encoder-js/-/vorbis-encoder-js-1.0.2.tgz", + "integrity": "sha1-eBuIEv7w8bei0VwQMeXnsR6E8Xw=", + "requires": { + "prelude-ls": "^1.1.2" + } + }, "wait-until": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/wait-until/-/wait-until-0.0.2.tgz", diff --git a/package.json b/package.json index 603240e1..c12a52f7 100644 --- a/package.json +++ b/package.json @@ -85,10 +85,12 @@ "nexmo": "2.5.2", "ngrok": "3.2.7", "npm": "6.13.4", + "opn": "6.0.0", "pragmatismo-io-framework": "1.0.20", "prism-media": "1.2.1", "public-ip": "4.0.0", + "readline": "1.3.0", "reflect-metadata": "0.1.13", "request-promise": "4.2.5", "request-promise-native": "1.0.8", @@ -106,6 +108,7 @@ "typescript": "3.7.4", "url-join": "4.0.1", "vbscript-to-typescript": "1.0.8", + "wait-until": "0.0.2", "walk-promise": "0.2.0", "washyourmouthoutwithsoap": "1.0.2" @@ -218,4 +221,4 @@ "post-merge": [] } } -} \ No newline at end of file +} diff --git a/packages/core.gbapp/services/GBConversationalService.ts b/packages/core.gbapp/services/GBConversationalService.ts index 8b9a7e4e..3ff08358 100644 --- a/packages/core.gbapp/services/GBConversationalService.ts +++ b/packages/core.gbapp/services/GBConversationalService.ts @@ -38,7 +38,7 @@ import { MessageFactory, RecognizerResult } from 'botbuilder'; import { LuisRecognizer } from 'botbuilder-ai'; -import { GBDialogStep, GBLog, GBMinInstance, IGBConversationalService, IGBCoreService } from 'botlib'; +import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService } from 'botlib'; import { AzureText } from 'pragmatismo-io-framework'; import { Messages } from '../strings'; import { GBServer } from '../../../src/app'; @@ -60,7 +60,7 @@ export interface LanguagePickerSettings { * Provides basic services for handling messages and dispatching to back-end * services like NLP or Search. */ -export class GBConversationalService implements IGBConversationalService { +export class GBConversationalService { public coreService: IGBCoreService; constructor(coreService: IGBCoreService) { @@ -134,6 +134,72 @@ export class GBConversationalService implements IGBConversationalService { min.whatsAppDirectLine.sendToDevice(mobile, message); } + public static async getAudioBufferFromText(speechKey, cloudRegion, text, locale): Promise { + return new Promise(async (resolve, reject) => { + const name = GBAdminService.getRndReadableIdentifier(); + + const waveFilename = `work/tmp${name}.pcm`; + + var audioConfig = sdk.AudioConfig.fromAudioFileOutput(waveFilename); + var speechConfig = sdk.SpeechConfig.fromSubscription(speechKey, cloudRegion); + + var synthesizer = new sdk.SpeechSynthesizer(speechConfig, audioConfig); + + try { + speechConfig.speechSynthesisLanguage = locale; + speechConfig.speechSynthesisVoiceName = "pt-BR-Daniel-Apollo" // pt-BR-HeloisaRUS; + + synthesizer.speakTextAsync(text, + (result) => { + if (result.reason === sdk.ResultReason.SynthesizingAudioCompleted) { + let raw = Buffer.from(result.audioData); + fs.writeFileSync(waveFilename, raw); + GBLog.info(`Audio data byte size: ${result.audioData.byteLength}.`) + const oggFilenameOnly = `tmp${name}.ogg`; + const oggFilename = `work/${oggFilenameOnly}`; + + const ffmpeg = prism.FFmpeg.getInfo(); + + console.log(`Using FFmpeg version ${ffmpeg.version}`); + + if (ffmpeg.output.includes('--enable-libopus')) { + console.log('libopus is available!'); + } else { + console.log('libopus is unavailable!'); + } + + const output = fs.createWriteStream(oggFilename); + const transcoder = new prism.FFmpeg({ + args: [ + '-analyzeduration', '0', + '-loglevel', '0', + '-f', 'opus', + '-ar', '16000', + '-ac', '1', + ], + }); + + fs.createReadStream(waveFilename) + .pipe(transcoder) + .pipe(output); + + console.log("synthesis finished."); + + let url = urlJoin(GBServer.globals.publicAddress, 'audios', oggFilenameOnly); + resolve(url); + } else { + const error = "Speech synthesis canceled, " + result.errorDetails; + reject(error); + } + synthesizer.close(); + synthesizer = undefined; + }); + } catch (error) { + reject(error); + } + }); + } + public static async getTextFromAudioBuffer(speechKey, cloudRegion, buffer, locale): Promise { return new Promise(async (resolve, reject) => { try { @@ -182,7 +248,7 @@ export class GBConversationalService implements IGBConversationalService { recognizer = undefined; }); }); - + fs.createReadStream(`work/tmp${name}.ogg`) .pipe(new prism.opus.OggDemuxer()) .pipe(new prism.opus.Decoder({ diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index 14b397fe..7294b3bf 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -48,7 +48,7 @@ const rimraf = require('rimraf'); import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBInstance, IGBPackage, IGBDeployer } from 'botlib'; import { AzureSearch } from 'pragmatismo-io-framework'; import { GBServer } from '../../../src/app'; -import { GuaribasPackage, GuaribasInstance } from '../models/GBModel'; +import { GuaribasPackage} from '../models/GBModel'; import { GBAdminService } from './../../admin.gbapp/services/GBAdminService'; import { AzureDeployerService } from './../../azuredeployer.gbapp/services/AzureDeployerService'; import { KBService } from './../../kb.gbapp/services/KBService'; diff --git a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts index 9b60c66d..37487ec7 100644 --- a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts +++ b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts @@ -104,6 +104,9 @@ export class WhatsappDirectLine extends GBService { }; if (setUrl) { + const express = require('express'); + GBServer.globals.server.use(`/audios`, express.static('work')); + try { let res = await request.post(options); GBLog.info(res); @@ -210,7 +213,7 @@ export class WhatsappDirectLine extends GBService { watermark = response.obj.watermark; await this.printMessages(response.obj.activities, conversationId, from, fromName); } catch (err) { - GBLog.error(`Error calling printMessages on Whatsapp channel ${err.data}`); + GBLog.error(`Error calling printMessages on Whatsapp channel ${err.data === undefined ? err : err.data}`); } }; setInterval(worker, this.pollInterval); @@ -296,7 +299,7 @@ export class WhatsappDirectLine extends GBService { qs: { token: this.whatsappServiceKey, phone: to, - audio: url + body: url }, headers: { 'cache-control': 'no-cache' @@ -312,28 +315,49 @@ export class WhatsappDirectLine extends GBService { } } + public async sendTextAsAudioToDevice(to, msg) { + + let url = await GBConversationalService.getAudioBufferFromText( + this.min.instance.speechKey, + this.min.instance.cloudLocation, + msg, 'pt-br' + ); + + await this.sendFileToDevice(to, url, 'Audio', msg); + + } + public async sendToDevice(to, msg) { - const options = { - method: 'POST', - url: urlJoin(this.whatsappServiceUrl, 'message'), - qs: { - token: this.whatsappServiceKey, - phone: to, - body: msg - }, - headers: { - 'cache-control': 'no-cache' + + const cmd = '/audio '; + if (msg.startsWith (cmd)) { + msg = msg.substr(cmd.length); + return await this.sendTextAsAudioToDevice(to, msg); + } + else { + + const options = { + method: 'POST', + url: urlJoin(this.whatsappServiceUrl, 'message'), + qs: { + token: this.whatsappServiceKey, + phone: to, + body: msg + }, + headers: { + 'cache-control': 'no-cache' + } + }; + + try { + // tslint:disable-next-line: await-promise + const result = await request.post(options); + GBLog.info(result); + } catch (error) { + GBLog.error(`Error sending message to Whatsapp provider ${error.message}`); + + // TODO: Handle Error: socket hang up and retry. } - }; - - try { - // tslint:disable-next-line: await-promise - const result = await request.post(options); - GBLog.info(result); - } catch (error) { - GBLog.error(`Error sending message to Whatsapp provider ${error.message}`); - - // TODO: Handle Error: socket hang up and retry. } } } diff --git a/src/app.ts b/src/app.ts index eaa40b62..f10187ab 100644 --- a/src/app.ts +++ b/src/app.ts @@ -67,7 +67,7 @@ export class RootData { public minBoot: GBMinInstance; public wwwroot: string; // .gbui or a static webapp. public entryPointDialog: string; // To replace default welcome dialog. - + } /** @@ -82,6 +82,7 @@ export class GBServer { */ public static run() { + GBLog.info(`The Bot Server is in STARTING mode...`); GBServer.globals = new RootData(); GBConfigService.init(); @@ -127,8 +128,8 @@ export class GBServer { GBLog.info(`Establishing a development local proxy (ngrok)...`); GBServer.globals.publicAddress = await core.ensureProxy(port); - process.env.BOT_URL = GBServer.globals.publicAddress; } + process.env.BOT_URL = GBServer.globals.publicAddress; } else { const serverAddress = process.env.BOT_URL; GBLog.info(`Defining server address at ${serverAddress}...`);