From 09555998555e4ca9fc09bd0960bfc0a68df42bff Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Fri, 1 Jun 2018 16:11:52 -0300 Subject: [PATCH] NEW: Added STT and TTS capabilities to default.gbui. --- VERSION.md | 4 ++ deploy/core.gbapp/models/GBModel.ts | 2 + deploy/core.gbapp/services/GBMinService.ts | 32 ++++++++++---- deploy/default.gbui/src/GBUIApp.js | 49 +++++++++++++++++++--- package.json | 4 +- 5 files changed, 75 insertions(+), 16 deletions(-) diff --git a/VERSION.md b/VERSION.md index 6ab553ec..12d91cd4 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1,5 +1,9 @@ # Release History +## Version 0.0.29 + +- NEW: Added STT and TTS capabilities to default.gbui. + ## Version 0.0.28 - FIX: gbui packages updated. diff --git a/deploy/core.gbapp/models/GBModel.ts b/deploy/core.gbapp/models/GBModel.ts index 1c3ed0f6..a32feb19 100644 --- a/deploy/core.gbapp/models/GBModel.ts +++ b/deploy/core.gbapp/models/GBModel.ts @@ -101,6 +101,8 @@ export class GuaribasInstance extends Model implements IGBInst @Column whatsappServiceWebhookUrl: string; + @Column speechKey: string; + @Column spellcheckerKey: string; @Column theme: string; diff --git a/deploy/core.gbapp/services/GBMinService.ts b/deploy/core.gbapp/services/GBMinService.ts index 0c74c59b..3fb96975 100644 --- a/deploy/core.gbapp/services/GBMinService.ts +++ b/deploy/core.gbapp/services/GBMinService.ts @@ -131,15 +131,29 @@ export class GBMinService { botId, (instance: IGBInstance, err) => { if (instance) { - res.send( - JSON.stringify({ - instanceId: instance.instanceId, - botId: botId, - theme: instance.theme, - secret: instance.webchatKey, // TODO: Use token. - conversationId: responseObject.conversationId - }) - ); + + let options = { + url: + "https://api.cognitive.microsoft.com/sts/v1.0/issueToken", + method: "POST", + headers: { + "Ocp-Apim-Subscription-Key": instance.speechKey + } + }; + request(options).then((response: + string) => { + + res.send( + JSON.stringify({ + instanceId: instance.instanceId, + botId: botId, + theme: instance.theme, + secret: instance.webchatKey, // TODO: Use token. + speechToken: response, + conversationId: responseObject.conversationId + }) + ); + }); } else { let error = `Instance not found: ${botId}.`; res.send(error); diff --git a/deploy/default.gbui/src/GBUIApp.js b/deploy/default.gbui/src/GBUIApp.js index 2b699411..f8d7186b 100644 --- a/deploy/default.gbui/src/GBUIApp.js +++ b/deploy/default.gbui/src/GBUIApp.js @@ -39,7 +39,12 @@ import SidebarMenu from "./components/SidebarMenu.js"; import GBCss from "./components/GBCss.js"; import { DirectLine } from "botframework-directlinejs"; import { ConnectionStatus } from "botframework-directlinejs"; +import { SpeechRecognizer } from "botframework-webchat/CognitiveServices"; +import { SpeechSynthesizer } from "botframework-webchat/CognitiveServices"; +import { SynthesisGender } from "botframework-webchat/CognitiveServices"; import { Chat } from "botframework-webchat"; +import { BotChat } from "botframework-webchat"; +import { Speech } from "botframework-webchat/botchat"; import GBPowerBIPlayer from "./players/GBPowerBIPlayer.js"; class GBUIApp extends React.Component { @@ -49,7 +54,8 @@ class GBUIApp extends React.Component { this.state = { botConnection: null, instance: null, - token: null + token: null, + instanceClient: null }; } @@ -97,7 +103,8 @@ class GBUIApp extends React.Component { .then(res => res.json()) .then( result => { - this.setupBotConnection(result.secret); + this.setState({instanceClient:result}); + this.setupBotConnection(); }, error => { this.setState({ @@ -108,12 +115,12 @@ class GBUIApp extends React.Component { ); } - setupBotConnection(secret) { + setupBotConnection() { let _this_ = this; window["botchatDebug"] = true; const botConnection = new DirectLine({ - secret: secret + secret: this.state.instanceClient.secret }); botConnection.connectionStatus$.subscribe(connectionStatus => { @@ -224,25 +231,57 @@ class GBUIApp extends React.Component { ); break; } + + } + let speechOptions; + let sideBar = (
); - if (this.state.botConnection) { + + if (this.state.botConnection && this.state.instance) { + let token = this.state.instanceClient.speechToken; + + function getToken() { + return new Promise((resolve, reject) => { + resolve(token); + }); + } + + + speechOptions = { + speechRecognizer: new SpeechRecognizer({ + locale: "pt-br", + fetchCallback: (authFetchEventId) => getToken(), + fetchOnExpiryCallback: (authFetchEventId) => getToken() + }), + speechSynthesizer: new SpeechSynthesizer({ + fetchCallback: (authFetchEventId) => getToken(), + fetchOnExpiryCallback: (authFetchEventId) => getToken(), + gender: SynthesisGender.Male, + voiceName: 'Microsoft Server Speech Text to Speech Voice (pt-BR, Daniel, Apollo)' + }) + }; + chat = ( { this.chat = chat; }} + locale={'pt-br'} botConnection={this.state.botConnection} user={this.getUser()} bot={{ id: "bot@gb", name: "Bot" }} + speechOptions={speechOptions} /> ); + + } if (!this.state.instance) { diff --git a/package.json b/package.json index 91d10960..a98cf89e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "botserver", - "version": "0.0.28", + "version": "0.0.29", "description": "General Bot Community Edition open-core server.", "main": "./src/app.ts", "homepage": "http://www.generalbot.com", @@ -33,7 +33,7 @@ "async": "^2.6.1", "body-parser": "^1.18.3", "botbuilder": "^3.15.0", - "botlib": "^0.0.25", + "botlib": "^0.0.28", "chokidar": "^2.0.3", "csv-parse": "^2.4.0", "dotenv-extended": "^2.0.2",