new(all): TTS/STT on Whatsapp channel.

2020-04-28 20:54:04 -03:00 · 2020-04-28 20:54:04 -03:00 · ec317fbd6d
commit ec317fbd6d
parent 7b878a3311
6 changed files with 263 additions and 31 deletions
--- a/package-lock.json
+++ b/package-lock.json
@ -2908,6 +2908,11 @@
      "resolved": "https://registry.npmjs.org/binary-search-bounds/-/binary-search-bounds-2.0.3.tgz",
      "integrity": "sha1-X/hhbW3SylOIvIWy1iZuK52lAtw="
    },
+    "bindings": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz",
+      "integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE="
+    },
    "bl": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz",
@ -3415,6 +3420,20 @@
        "ieee754": "^1.1.4"
      }
    },
+    "buffer-alloc": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
+      "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
+      "requires": {
+        "buffer-alloc-unsafe": "^1.1.0",
+        "buffer-fill": "^1.0.0"
+      }
+    },
+    "buffer-alloc-unsafe": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
+      "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg=="
+    },
    "buffer-crc32": {
      "version": "0.2.13",
      "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@ -3425,6 +3444,11 @@
      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
      "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
    },
+    "buffer-fill": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
+      "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw="
+    },
    "buffer-from": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
@ -9573,6 +9597,11 @@
      "integrity": "sha512-ISDqGcspVu6U3VKqtJZG1uR55SmNNF9uK0EMq1IvNVVZOui6MW6VR0+pIZhqz85ORAGp+4zW+5fJ/SE7bwEibA==",
      "dev": true
    },
+    "nan": {
+      "version": "2.14.1",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
+      "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw=="
+    },
    "nanomatch": {
      "version": "1.2.13",
      "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
@ -14026,6 +14055,48 @@
      "integrity": "sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ==",
      "dev": true
    },
+    "ogg": {
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/ogg/-/ogg-1.2.6.tgz",
+      "integrity": "sha512-5TXXQJbECkxkwfJkXSUdl09gbFdv/K/fTS0YqrWGWiHVXiaCoiUZ3EB8RtBYyXSWc0eKkEbUAh4Wic8st7MbFA==",
+      "requires": {
+        "bindings": "~1.2.0",
+        "debug": "2",
+        "nan": "2",
+        "readable-stream": "1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "isarray": {
+          "version": "0.0.1",
+          "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
+          "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        },
+        "readable-stream": {
+          "version": "1.0.34",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
+          "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.1",
+            "isarray": "0.0.1",
+            "string_decoder": "~0.10.x"
+          }
+        }
+      }
+    },
    "on-finished": {
      "version": "2.3.0",
      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@ -14768,8 +14839,7 @@
    "prelude-ls": {
      "version": "1.1.2",
      "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
-      "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=",
-      "dev": true
+      "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ="
    },
    "prepend-http": {
      "version": "2.0.0",
@ -15893,6 +15963,11 @@
        "once": "^1.3.0"
      }
    },
+    "readline": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/readline/-/readline-1.3.0.tgz",
+      "integrity": "sha1-xYDXfvLPyHUrEySYBg3JeTp6wBw="
+    },
    "rechoir": {
      "version": "0.6.2",
      "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
@ -18712,6 +18787,69 @@
        }
      }
    },
+    "vorbis": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/vorbis/-/vorbis-0.2.2.tgz",
+      "integrity": "sha512-sGyyyzlWLankqp3NPUsPbeQ01AFRGgNyptiLxFW+4UCEQbfHqlNTIcFQCKtfBY5kqkbNOoqpkRW4JUnB/ehJ1A==",
+      "requires": {
+        "bindings": "^1.2.0",
+        "buffer-alloc": "^1.1.0",
+        "debug": "^2.2.0",
+        "nan": "^2.10.0",
+        "ogg": "^1.2.5",
+        "readable-stream": "^2.3.6"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        },
+        "process-nextick-args": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+          "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+        },
+        "readable-stream": {
+          "version": "2.3.7",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        }
+      }
+    },
+    "vorbis-encoder-js": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/vorbis-encoder-js/-/vorbis-encoder-js-1.0.2.tgz",
+      "integrity": "sha1-eBuIEv7w8bei0VwQMeXnsR6E8Xw=",
+      "requires": {
+        "prelude-ls": "^1.1.2"
+      }
+    },
    "wait-until": {
      "version": "0.0.2",
      "resolved": "https://registry.npmjs.org/wait-until/-/wait-until-0.0.2.tgz",
--- a/package.json
+++ b/package.json
@ -85,10 +85,12 @@
    "nexmo": "2.5.2",
    "ngrok": "3.2.7",
    "npm": "6.13.4",
+    
    "opn": "6.0.0",
    "pragmatismo-io-framework": "1.0.20",
    "prism-media": "1.2.1",
    "public-ip": "4.0.0",
+    "readline": "1.3.0",
    "reflect-metadata": "0.1.13",
    "request-promise": "4.2.5",
    "request-promise-native": "1.0.8",
@ -106,6 +108,7 @@
    "typescript": "3.7.4",
    "url-join": "4.0.1",
    "vbscript-to-typescript": "1.0.8",
+    
    "wait-until": "0.0.2",
    "walk-promise": "0.2.0",
    "washyourmouthoutwithsoap": "1.0.2"
@ -218,4 +221,4 @@
      "post-merge": []
    }
  }
-}
+}
--- a/packages/core.gbapp/services/GBConversationalService.ts
+++ b/packages/core.gbapp/services/GBConversationalService.ts
@ -38,7 +38,7 @@

 import { MessageFactory, RecognizerResult } from 'botbuilder';
 import { LuisRecognizer } from 'botbuilder-ai';
-import { GBDialogStep, GBLog, GBMinInstance, IGBConversationalService, IGBCoreService } from 'botlib';
+import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService } from 'botlib';
 import { AzureText } from 'pragmatismo-io-framework';
 import { Messages } from '../strings';
 import { GBServer } from '../../../src/app';
@ -60,7 +60,7 @@ export interface LanguagePickerSettings {
 * Provides basic services for handling messages and dispatching to back-end
 * services like NLP or Search.
 */
-export class GBConversationalService implements IGBConversationalService {
+export class GBConversationalService {
  public coreService: IGBCoreService;

  constructor(coreService: IGBCoreService) {
@ -134,6 +134,72 @@ export class GBConversationalService implements IGBConversationalService {
    min.whatsAppDirectLine.sendToDevice(mobile, message);
  }

+  public static async getAudioBufferFromText(speechKey, cloudRegion, text, locale): Promise<string> {
+    return new Promise<string>(async (resolve, reject) => {
+      const name = GBAdminService.getRndReadableIdentifier();
+
+      const waveFilename = `work/tmp${name}.pcm`;
+
+      var audioConfig = sdk.AudioConfig.fromAudioFileOutput(waveFilename);
+      var speechConfig = sdk.SpeechConfig.fromSubscription(speechKey, cloudRegion);
+
+      var synthesizer = new sdk.SpeechSynthesizer(speechConfig, audioConfig);
+
+      try {
+        speechConfig.speechSynthesisLanguage = locale;
+        speechConfig.speechSynthesisVoiceName = "pt-BR-Daniel-Apollo" // pt-BR-HeloisaRUS;
+
+        synthesizer.speakTextAsync(text,
+          (result) => {
+            if (result.reason === sdk.ResultReason.SynthesizingAudioCompleted) {
+              let raw = Buffer.from(result.audioData);
+              fs.writeFileSync(waveFilename, raw);
+              GBLog.info(`Audio data byte size: ${result.audioData.byteLength}.`)
+              const oggFilenameOnly = `tmp${name}.ogg`;
+              const oggFilename = `work/${oggFilenameOnly}`;
+
+              const ffmpeg = prism.FFmpeg.getInfo();
+
+              console.log(`Using FFmpeg version ${ffmpeg.version}`);
+
+              if (ffmpeg.output.includes('--enable-libopus')) {
+                console.log('libopus is available!');
+              } else {
+                console.log('libopus is unavailable!');
+              }
+
+              const output = fs.createWriteStream(oggFilename);
+              const transcoder = new prism.FFmpeg({
+                args: [
+                  '-analyzeduration', '0',
+                  '-loglevel', '0',
+                  '-f', 'opus',
+                  '-ar', '16000',
+                  '-ac', '1',
+                ],
+              });
+
+              fs.createReadStream(waveFilename)
+                .pipe(transcoder)
+                .pipe(output);
+
+              console.log("synthesis finished.");
+
+              let url = urlJoin(GBServer.globals.publicAddress, 'audios', oggFilenameOnly);
+              resolve(url);
+            } else {
+              const error = "Speech synthesis canceled, " + result.errorDetails;
+              reject(error);
+            }
+            synthesizer.close();
+            synthesizer = undefined;
+          });
+      } catch (error) {
+        reject(error);
+      }
+    });
+  }
+
  public static async  getTextFromAudioBuffer(speechKey, cloudRegion, buffer, locale): Promise<string> {
    return new Promise<string>(async (resolve, reject) => {
      try {
@ -182,7 +248,7 @@ export class GBConversationalService implements IGBConversationalService {
              recognizer = undefined;
            });
        });
-        
+
        fs.createReadStream(`work/tmp${name}.ogg`)
          .pipe(new prism.opus.OggDemuxer())
          .pipe(new prism.opus.Decoder({
--- a/packages/core.gbapp/services/GBDeployer.ts
+++ b/packages/core.gbapp/services/GBDeployer.ts
@ -48,7 +48,7 @@ const rimraf = require('rimraf');
 import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBInstance, IGBPackage, IGBDeployer } from 'botlib';
 import { AzureSearch } from 'pragmatismo-io-framework';
 import { GBServer } from '../../../src/app';
-import { GuaribasPackage, GuaribasInstance } from '../models/GBModel';
+import { GuaribasPackage} from '../models/GBModel';
 import { GBAdminService } from './../../admin.gbapp/services/GBAdminService';
 import { AzureDeployerService } from './../../azuredeployer.gbapp/services/AzureDeployerService';
 import { KBService } from './../../kb.gbapp/services/KBService';
--- a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts
+++ b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts
@ -104,6 +104,9 @@ export class WhatsappDirectLine extends GBService {
    };

    if (setUrl) {
+      const express = require('express');
+      GBServer.globals.server.use(`/audios`, express.static('work'));
+
      try {
        let res = await request.post(options);
        GBLog.info(res);
@ -210,7 +213,7 @@ export class WhatsappDirectLine extends GBService {
        watermark = response.obj.watermark;
        await this.printMessages(response.obj.activities, conversationId, from, fromName);
      } catch (err) {
-        GBLog.error(`Error calling printMessages on Whatsapp channel ${err.data}`);
+        GBLog.error(`Error calling printMessages on Whatsapp channel ${err.data === undefined ? err : err.data}`);
      }
    };
    setInterval(worker, this.pollInterval);
@ -296,7 +299,7 @@ export class WhatsappDirectLine extends GBService {
      qs: {
        token: this.whatsappServiceKey,
        phone: to,
-        audio: url
+        body: url
      },
      headers: {
        'cache-control': 'no-cache'
@ -312,28 +315,49 @@ export class WhatsappDirectLine extends GBService {
    }
  }

+  public async sendTextAsAudioToDevice(to, msg) {
+
+    let url = await GBConversationalService.getAudioBufferFromText(
+      this.min.instance.speechKey,
+      this.min.instance.cloudLocation,
+      msg, 'pt-br'
+    );
+
+    await this.sendFileToDevice(to, url, 'Audio', msg);
+
+  }
+
  public async sendToDevice(to, msg) {
-    const options = {
-      method: 'POST',
-      url: urlJoin(this.whatsappServiceUrl, 'message'),
-      qs: {
-        token: this.whatsappServiceKey,
-        phone: to,
-        body: msg
-      },
-      headers: {
-        'cache-control': 'no-cache'
+
+    const cmd = '/audio ';
+    if (msg.startsWith (cmd)) {
+      msg = msg.substr(cmd.length);
+      return await this.sendTextAsAudioToDevice(to, msg);
+    }
+    else {
+
+      const options = {
+        method: 'POST',
+        url: urlJoin(this.whatsappServiceUrl, 'message'),
+        qs: {
+          token: this.whatsappServiceKey,
+          phone: to,
+          body: msg
+        },
+        headers: {
+          'cache-control': 'no-cache'
+        }
+      };
+
+      try {
+        // tslint:disable-next-line: await-promise
+        const result = await request.post(options);
+        GBLog.info(result);
+      } catch (error) {
+        GBLog.error(`Error sending message to Whatsapp provider ${error.message}`);
+
+        // TODO: Handle Error: socket hang up and retry.
      }
-    };
-
-    try {
-      // tslint:disable-next-line: await-promise
-      const result = await request.post(options);
-      GBLog.info(result);
-    } catch (error) {
-      GBLog.error(`Error sending message to Whatsapp provider ${error.message}`);
-
-      // TODO: Handle Error: socket hang up and retry.
    }
  }
 }
--- a/src/app.ts
+++ b/src/app.ts
@ -67,7 +67,7 @@ export class RootData {
  public minBoot: GBMinInstance;
  public wwwroot: string; // .gbui or a static webapp.
  public entryPointDialog: string; // To replace default welcome dialog.
-  
+
 }

 /**
@ -82,6 +82,7 @@ export class GBServer {
   */

  public static run() {
+ 
    GBLog.info(`The Bot Server is in STARTING mode...`);
    GBServer.globals = new RootData();
    GBConfigService.init();
@ -127,8 +128,8 @@ export class GBServer {

              GBLog.info(`Establishing a development local proxy (ngrok)...`);
              GBServer.globals.publicAddress = await core.ensureProxy(port);
-              process.env.BOT_URL = GBServer.globals.publicAddress;
            }
+            process.env.BOT_URL = GBServer.globals.publicAddress;
          } else {
            const serverAddress = process.env.BOT_URL;
            GBLog.info(`Defining server address at ${serverAddress}...`);