fix(whatsapp.gblib): Speech is now full duplex.
This commit is contained in:
Rodrigo Rodriguez
parent
2a41d9c5e1
commit
6442ebf983
5 changed files with 60 additions and 194 deletions
142
package-lock.json
generated
142
package-lock.json
generated
|
|
@ -2150,6 +2150,11 @@
|
|||
"resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
|
||||
"integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8="
|
||||
},
|
||||
"any-shell-escape": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/any-shell-escape/-/any-shell-escape-0.1.1.tgz",
|
||||
"integrity": "sha1-1Vq5ciRMcaml4asIefML8RCAaVk="
|
||||
},
|
||||
"aproba": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
|
||||
|
|
@ -2908,11 +2913,6 @@
|
|||
"resolved": "https://registry.npmjs.org/binary-search-bounds/-/binary-search-bounds-2.0.3.tgz",
|
||||
"integrity": "sha1-X/hhbW3SylOIvIWy1iZuK52lAtw="
|
||||
},
|
||||
"bindings": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz",
|
||||
"integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE="
|
||||
},
|
||||
"bl": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz",
|
||||
|
|
@ -3420,20 +3420,6 @@
|
|||
"ieee754": "^1.1.4"
|
||||
}
|
||||
},
|
||||
"buffer-alloc": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
|
||||
"integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
|
||||
"requires": {
|
||||
"buffer-alloc-unsafe": "^1.1.0",
|
||||
"buffer-fill": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"buffer-alloc-unsafe": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
|
||||
"integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg=="
|
||||
},
|
||||
"buffer-crc32": {
|
||||
"version": "0.2.13",
|
||||
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
|
||||
|
|
@ -3444,11 +3430,6 @@
|
|||
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
|
||||
"integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
|
||||
},
|
||||
"buffer-fill": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
|
||||
"integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw="
|
||||
},
|
||||
"buffer-from": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
|
||||
|
|
@ -9597,11 +9578,6 @@
|
|||
"integrity": "sha512-ISDqGcspVu6U3VKqtJZG1uR55SmNNF9uK0EMq1IvNVVZOui6MW6VR0+pIZhqz85ORAGp+4zW+5fJ/SE7bwEibA==",
|
||||
"dev": true
|
||||
},
|
||||
"nan": {
|
||||
"version": "2.14.1",
|
||||
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
|
||||
"integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw=="
|
||||
},
|
||||
"nanomatch": {
|
||||
"version": "1.2.13",
|
||||
"resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
|
||||
|
|
@ -14055,48 +14031,6 @@
|
|||
"integrity": "sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ==",
|
||||
"dev": true
|
||||
},
|
||||
"ogg": {
|
||||
"version": "1.2.6",
|
||||
"resolved": "https://registry.npmjs.org/ogg/-/ogg-1.2.6.tgz",
|
||||
"integrity": "sha512-5TXXQJbECkxkwfJkXSUdl09gbFdv/K/fTS0YqrWGWiHVXiaCoiUZ3EB8RtBYyXSWc0eKkEbUAh4Wic8st7MbFA==",
|
||||
"requires": {
|
||||
"bindings": "~1.2.0",
|
||||
"debug": "2",
|
||||
"nan": "2",
|
||||
"readable-stream": "1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "2.6.9",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
|
||||
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
|
||||
"requires": {
|
||||
"ms": "2.0.0"
|
||||
}
|
||||
},
|
||||
"isarray": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
|
||||
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
|
||||
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "1.0.34",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
|
||||
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
|
||||
"requires": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.1",
|
||||
"isarray": "0.0.1",
|
||||
"string_decoder": "~0.10.x"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"on-finished": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
|
||||
|
|
@ -14839,7 +14773,8 @@
|
|||
"prelude-ls": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
|
||||
"integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ="
|
||||
"integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=",
|
||||
"dev": true
|
||||
},
|
||||
"prepend-http": {
|
||||
"version": "2.0.0",
|
||||
|
|
@ -18787,69 +18722,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"vorbis": {
|
||||
"version": "0.2.2",
|
||||
"resolved": "https://registry.npmjs.org/vorbis/-/vorbis-0.2.2.tgz",
|
||||
"integrity": "sha512-sGyyyzlWLankqp3NPUsPbeQ01AFRGgNyptiLxFW+4UCEQbfHqlNTIcFQCKtfBY5kqkbNOoqpkRW4JUnB/ehJ1A==",
|
||||
"requires": {
|
||||
"bindings": "^1.2.0",
|
||||
"buffer-alloc": "^1.1.0",
|
||||
"debug": "^2.2.0",
|
||||
"nan": "^2.10.0",
|
||||
"ogg": "^1.2.5",
|
||||
"readable-stream": "^2.3.6"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "2.6.9",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
|
||||
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
|
||||
"requires": {
|
||||
"ms": "2.0.0"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
|
||||
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
|
||||
},
|
||||
"process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "2.3.7",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
|
||||
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
|
||||
"requires": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~2.0.0",
|
||||
"safe-buffer": "~5.1.1",
|
||||
"string_decoder": "~1.1.1",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"requires": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"vorbis-encoder-js": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/vorbis-encoder-js/-/vorbis-encoder-js-1.0.2.tgz",
|
||||
"integrity": "sha1-eBuIEv7w8bei0VwQMeXnsR6E8Xw=",
|
||||
"requires": {
|
||||
"prelude-ls": "^1.1.2"
|
||||
}
|
||||
},
|
||||
"wait-until": {
|
||||
"version": "0.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wait-until/-/wait-until-0.0.2.tgz",
|
||||
|
|
|
|||
10
package.json
10
package.json
|
|
@ -13,7 +13,7 @@
|
|||
"Dário Vieira <dario.junior3@gmail.com>"
|
||||
],
|
||||
"engines": {
|
||||
"node": "=10.15.2"
|
||||
"node": "=10.15.2"
|
||||
},
|
||||
"license": "AGPL-3.0",
|
||||
"preferGlobal": true,
|
||||
|
|
@ -49,10 +49,10 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@azure/ms-rest-js": "2.0.4",
|
||||
"@discordjs/opus": "0.1.0",
|
||||
"@microsoft/microsoft-graph-client": "2.0.0",
|
||||
"@types/validator": "12.0.1",
|
||||
"adal-node": "0.2.1",
|
||||
"any-shell-escape": "^0.1.1",
|
||||
"async-promises": "0.2.2",
|
||||
"azure-arm-cognitiveservices": "3.0.0",
|
||||
"azure-arm-resource": "7.3.0",
|
||||
|
|
@ -84,11 +84,10 @@
|
|||
"nexmo": "2.5.2",
|
||||
"ngrok": "3.2.7",
|
||||
"npm": "6.13.4",
|
||||
|
||||
"prism-media": "1.2.1",
|
||||
"opn": "6.0.0",
|
||||
"pragmatismo-io-framework": "1.0.20",
|
||||
"prism-media": "1.2.1",
|
||||
"public-ip": "4.0.0",
|
||||
"public-ip": "4.0.0",
|
||||
"readline": "1.3.0",
|
||||
"reflect-metadata": "0.1.13",
|
||||
"request-promise": "4.2.5",
|
||||
|
|
@ -107,7 +106,6 @@
|
|||
"typescript": "3.7.4",
|
||||
"url-join": "4.0.1",
|
||||
"vbscript-to-typescript": "1.0.8",
|
||||
|
||||
"wait-until": "0.0.2",
|
||||
"walk-promise": "0.2.0",
|
||||
"washyourmouthoutwithsoap": "1.0.2"
|
||||
|
|
|
|||
|
|
@ -42,14 +42,16 @@ import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService } from 'botlib';
|
|||
import { AzureText } from 'pragmatismo-io-framework';
|
||||
import { Messages } from '../strings';
|
||||
import { GBServer } from '../../../src/app';
|
||||
import { Readable } from 'stream'
|
||||
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService';
|
||||
const urlJoin = require('url-join');
|
||||
const PasswordGenerator = require("strict-password-generator").default;
|
||||
const Nexmo = require('nexmo');
|
||||
let sdk = require("microsoft-cognitiveservices-speech-sdk");
|
||||
var fs = require('fs')
|
||||
import { Readable } from 'stream'
|
||||
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService';
|
||||
const prism = require('prism-media');
|
||||
const { join } = require('path')
|
||||
const shell = require('any-shell-escape')
|
||||
const { exec } = require('child_process')
|
||||
const fs = require('fs')
|
||||
const sdk = require("microsoft-cognitiveservices-speech-sdk");
|
||||
|
||||
export interface LanguagePickerSettings {
|
||||
defaultLocale?: string;
|
||||
|
|
@ -147,7 +149,7 @@ export class GBConversationalService {
|
|||
|
||||
try {
|
||||
speechConfig.speechSynthesisLanguage = locale;
|
||||
speechConfig.speechSynthesisVoiceName = "pt-BR-HeloisaRUS";
|
||||
speechConfig.speechSynthesisVoiceName = "pt-BR-FranciscaNeural";
|
||||
|
||||
synthesizer.speakTextAsync(text,
|
||||
(result) => {
|
||||
|
|
@ -158,15 +160,6 @@ export class GBConversationalService {
|
|||
const oggFilenameOnly = `tmp${name}.ogg`;
|
||||
const oggFilename = `work/${oggFilenameOnly}`;
|
||||
|
||||
const ffmpeg = prism.FFmpeg.getInfo();
|
||||
|
||||
console.log(`Using FFmpeg version ${ffmpeg.version}`);
|
||||
|
||||
if (ffmpeg.output.includes('--enable-libopus')) {
|
||||
console.log('libopus is available!');
|
||||
} else {
|
||||
console.log('libopus is unavailable!');
|
||||
}
|
||||
|
||||
const output = fs.createWriteStream(oggFilename);
|
||||
const transcoder = new prism.FFmpeg({
|
||||
|
|
@ -183,7 +176,6 @@ export class GBConversationalService {
|
|||
.pipe(transcoder)
|
||||
.pipe(output);
|
||||
|
||||
console.log("synthesis finished.");
|
||||
|
||||
let url = urlJoin(GBServer.globals.publicAddress, 'audios', oggFilenameOnly);
|
||||
resolve(url);
|
||||
|
|
@ -206,11 +198,6 @@ export class GBConversationalService {
|
|||
let subscriptionKey = speechKey;
|
||||
let serviceRegion = cloudRegion;
|
||||
|
||||
var samplingRate = 16000;
|
||||
var frameDuration = 20;
|
||||
var channels = 1;
|
||||
var frameSize = samplingRate * frameDuration / 1000;
|
||||
|
||||
const oggFile = new Readable();
|
||||
oggFile._read = () => { } // _read is required but you can noop it
|
||||
oggFile.push(buffer);
|
||||
|
|
@ -218,44 +205,53 @@ export class GBConversationalService {
|
|||
|
||||
const name = GBAdminService.getRndReadableIdentifier();
|
||||
|
||||
fs.writeFileSync(`work/tmp${name}.ogg`, buffer);
|
||||
const dest = `work/tmp${name}.wav`;
|
||||
const src = `work/tmp${name}.ogg`;
|
||||
fs.writeFileSync(src, oggFile.read());
|
||||
|
||||
let wr = fs.createWriteStream(`work/tmp${name}.pcm`);
|
||||
wr.on('finish', () => {
|
||||
let data = fs.readFileSync(`work/tmp${name}.pcm`);
|
||||
const makeMp3 = shell([
|
||||
'node_modules/ffmpeg-static/ffmpeg.exe', '-y', '-v', 'error',
|
||||
'-i', join(process.cwd(), src),
|
||||
'-ar', '16000',
|
||||
'-ac', '1',
|
||||
'-acodec', 'pcm_s16le',
|
||||
join(process.cwd(), dest)
|
||||
])
|
||||
|
||||
let pushStream = sdk.AudioInputStream.createPushStream();
|
||||
pushStream.write(data);
|
||||
pushStream.close();
|
||||
exec(makeMp3, (error) => {
|
||||
if (error) {
|
||||
GBLog.error(error);
|
||||
return Promise.reject(error);
|
||||
} else {
|
||||
let data = fs.readFileSync(dest);
|
||||
|
||||
let audioConfig = sdk.AudioConfig.fromStreamInput(pushStream);
|
||||
let speechConfig = sdk.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion);
|
||||
speechConfig.speechRecognitionLanguage = locale;
|
||||
let recognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig);
|
||||
let pushStream = sdk.AudioInputStream.createPushStream();
|
||||
pushStream.write(data);
|
||||
pushStream.close();
|
||||
|
||||
recognizer.recognizeOnceAsync(
|
||||
(result) => {
|
||||
let audioConfig = sdk.AudioConfig.fromStreamInput(pushStream);
|
||||
let speechConfig = sdk.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion);
|
||||
speechConfig.speechRecognitionLanguage = locale;
|
||||
let recognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig);
|
||||
|
||||
resolve(result.text ? result.text : 'Speech to Text failed: Audio not converted');
|
||||
recognizer.recognizeOnceAsync(
|
||||
(result) => {
|
||||
|
||||
recognizer.close();
|
||||
recognizer = undefined;
|
||||
},
|
||||
(err) => {
|
||||
reject(err);
|
||||
resolve(result.text ? result.text : 'Speech to Text failed: Audio not converted');
|
||||
|
||||
recognizer.close();
|
||||
recognizer = undefined;
|
||||
});
|
||||
});
|
||||
recognizer.close();
|
||||
recognizer = undefined;
|
||||
},
|
||||
(err) => {
|
||||
reject(err);
|
||||
|
||||
recognizer.close();
|
||||
recognizer = undefined;
|
||||
});
|
||||
|
||||
}
|
||||
})
|
||||
|
||||
fs.createReadStream(`work/tmp${name}.ogg`)
|
||||
.pipe(new prism.opus.OggDemuxer())
|
||||
.pipe(new prism.opus.Decoder({
|
||||
rate: samplingRate,
|
||||
channels: channels, frameSize: frameSize
|
||||
}))
|
||||
.pipe(wr);
|
||||
} catch (error) {
|
||||
GBLog.error(error);
|
||||
return Promise.reject(error);
|
||||
|
|
|
|||
|
|
@ -351,7 +351,7 @@ STORAGE_SYNC=true
|
|||
}
|
||||
else {
|
||||
|
||||
throw new Error(`Error updating bot proxy, details: ${error.message}.`);
|
||||
throw new Error(`Error updating bot proxy, details: ${error}.`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ export class GBServer {
|
|||
*/
|
||||
|
||||
public static run() {
|
||||
|
||||
|
||||
GBLog.info(`The Bot Server is in STARTING mode...`);
|
||||
GBServer.globals = new RootData();
|
||||
GBConfigService.init();
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue