From faa5ec710c6d5703c0a7a689dbb3afadd27548e1 Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Sat, 24 Aug 2019 18:46:04 -0300 Subject: [PATCH] feat(whatsapp.gblib): Now Whatsapp will display markdown from .gbkb including images. --- packages/core.gbapp/dialogs/SwitchBot.ts | 1 - .../services/GBConversationalService.ts | 10 +- packages/core.gbapp/services/GBDeployer.ts | 15 +- packages/core.gbapp/services/GBMinService.ts | 17 +- packages/kb.gbapp/dialogs/AskDialog.ts | 8 +- packages/kb.gbapp/services/KBService.ts | 233 ++++++++++++------ .../services/WhatsappDirectLine.ts | 27 +- src/app.ts | 3 +- 8 files changed, 218 insertions(+), 96 deletions(-) diff --git a/packages/core.gbapp/dialogs/SwitchBot.ts b/packages/core.gbapp/dialogs/SwitchBot.ts index 863caaba..bfb0e40b 100644 --- a/packages/core.gbapp/dialogs/SwitchBot.ts +++ b/packages/core.gbapp/dialogs/SwitchBot.ts @@ -63,7 +63,6 @@ export class SwitchBotDialog extends IGBDialog { async step => { let sec = new SecService(); let from = step.context.activity.from.id; - const minBoot = GBServer.globals.minInstances[0]; await sec.updateCurrentBotId(from, step.result); await step.context.sendActivity(`Opa, vamos lá!`); diff --git a/packages/core.gbapp/services/GBConversationalService.ts b/packages/core.gbapp/services/GBConversationalService.ts index 5bc8caaf..25fd0c03 100644 --- a/packages/core.gbapp/services/GBConversationalService.ts +++ b/packages/core.gbapp/services/GBConversationalService.ts @@ -64,11 +64,17 @@ export class GBConversationalService implements IGBConversationalService { } public async sendFile(min: GBMinInstance, step: GBDialogStep, url: string): Promise { - let mobile = step.context.activity.from.id; - min.whatsAppDirectLine.sendFile(mobile, url); + const mobile = step.context.activity.from.id; + const filename = url.substring(url.lastIndexOf('/')+1); + await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename); } + public async sendAudio(min: GBMinInstance, step: GBDialogStep, url: string): Promise { + const mobile = step.context.activity.from.id; + await min.whatsAppDirectLine.sendAudioToDevice(mobile, url); + } + public async sendEvent(step: GBDialogStep, name: string, value: Object): Promise { if (step.context.activity.channelId === 'webchat') { const msg = MessageFactory.text(''); diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index 36379937..85714c56 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -203,6 +203,10 @@ export class GBDeployer { instance.whatsappServiceNumber = bootInstance.whatsappServiceNumber; instance.whatsappServiceUrl = bootInstance.whatsappServiceUrl; instance.whatsappServiceWebhookUrl = bootInstance.whatsappServiceWebhookUrl; + instance.storageServer = bootInstance.storageServer; + instance.storageName = bootInstance.storageName; + instance.storageUsername = bootInstance.storageUsername; + instance.storagePassword = bootInstance.storagePassword; instance = await service.internalDeployBot( instance, @@ -422,9 +426,7 @@ export class GBDeployer { server.use(`/themes/${filenameOnly}`, express.static(filename)); GBLog.info(`Theme (.gbtheme) assets accessible at: /themes/${filenameOnly}.`); } else if (Path.extname(filename) === '.gbkb') { - server.use(`/kb/${filenameOnly}/subjects`, express.static(urlJoin(filename, 'subjects'))); - server.use(`/kb/${filenameOnly}/images`, express.static(urlJoin(filename, 'images'))); - GBLog.info(`KB (.gbkb) assets accessible at: /kb/${filenameOnly}.`); + this.mountGBKBAssets( filenameOnly, filename); } else if (Path.extname(filename) === '.gbui') { // Already Handled } else if (Path.extname(filename) === '.gbdialog') { @@ -458,6 +460,13 @@ export class GBDeployer { return { generalPackages, totalPackages }; } + private mountGBKBAssets(packageName: any, filename: string) { + GBServer.globals.server.use(`/kb/${packageName}/subjects`, express.static(urlJoin(filename, 'subjects'))); + GBServer.globals.server.use(`/kb/${packageName}/images`, express.static(urlJoin(filename, 'images'))); + GBServer.globals.server.use(`/kb/${packageName}/audios`, express.static(urlJoin(filename, 'audios'))); + GBLog.info(`KB (.gbkb) assets accessible at: /kb/${packageName}.`); + } + private isSystemPackage(name: string): Boolean { const names = ['core.gbapp', 'admin.gbapp', 'azuredeployer.gbapp', 'customer-satisfaction.gbapp', 'kb.gbapp']; diff --git a/packages/core.gbapp/services/GBMinService.ts b/packages/core.gbapp/services/GBMinService.ts index a5d9c3da..cfee847e 100644 --- a/packages/core.gbapp/services/GBMinService.ts +++ b/packages/core.gbapp/services/GBMinService.ts @@ -138,9 +138,8 @@ export class GBMinService { return; // Exit here. } - const minBoot = GBServer.globals.bootInstance; const toSwitchMin = GBServer.globals.minInstances.filter(p => p.botId === text)[0]; - let activeMin = toSwitchMin ? toSwitchMin : minBoot; + let activeMin = toSwitchMin ? toSwitchMin : GBServer.globals.minBoot; let sec = new SecService(); let user = await sec.getUserFromPhone(id); @@ -190,6 +189,10 @@ export class GBMinService { // Build bot adapter. const { min, adapter, conversationState } = await this.buildBotAdapter(instance, GBServer.globals.publicAddress, GBServer.globals.sysPackages); + + if (GBServer.globals.minInstances.length === 0) { + GBServer.globals.minBoot = min; + } GBServer.globals.minInstances.push(min); // Install default VBA module. @@ -453,11 +456,13 @@ export class GBMinService { user.cb = undefined; await min.userProfile.set(step.context, user); - let sec = new SecService(); - const member = context.activity.membersAdded[0]; + if (context.activity.membersAdded !== undefined) { + let sec = new SecService(); + const member = context.activity.membersAdded[0]; - await sec.ensureUser(instance.instanceId, member.id, - min.botId, member.id, "", "web", member.name, member.id); + await sec.ensureUser(instance.instanceId, member.id, + min.botId, member.id, "", "web", member.name, member.id); + } } GBLog.info( diff --git a/packages/kb.gbapp/dialogs/AskDialog.ts b/packages/kb.gbapp/dialogs/AskDialog.ts index f6391afe..212fa427 100644 --- a/packages/kb.gbapp/dialogs/AskDialog.ts +++ b/packages/kb.gbapp/dialogs/AskDialog.ts @@ -144,7 +144,7 @@ export class AskDialog extends IGBDialog { // Sends the answer to all outputs, including projector. - await service.sendAnswer(AskDialog.getChannel(step), min.conversationalService, step, resultsA.answer); + await service.sendAnswer(min, AskDialog.getChannel(step), step, resultsA.answer); // Goes to ask loop, again. return await step.replaceDialog('/ask', { isReturning: true }); @@ -164,7 +164,7 @@ export class AskDialog extends IGBDialog { await step.context.sendActivity(Messages[locale].wider_answer); } // Sends the answer to all outputs, including projector. - await service.sendAnswer(AskDialog.getChannel(step), min.conversationalService, step, resultsB.answer); + await service.sendAnswer(min, AskDialog.getChannel(step), step, resultsB.answer); return await step.replaceDialog('/ask', { isReturning: true }); } else { @@ -180,7 +180,7 @@ export class AskDialog extends IGBDialog { } private static getChannel(step): string { - return Number.isInteger(step.context.activity.from.id) ? 'whatsapp' : step.context.activity.channelId; + return !isNaN(step.context.activity.from.id) ? 'whatsapp' : step.context.activity.channelId; } private static getAnswerEventDialog(service: KBService, min: GBMinInstance) { @@ -191,7 +191,7 @@ export class AskDialog extends IGBDialog { const question = await service.getQuestionById(min.instance.instanceId, data.questionId); const answer = await service.getAnswerById(min.instance.instanceId, question.answerId); // Sends the answer to all outputs, including projector. - await service.sendAnswer(AskDialog.getChannel(step), min.conversationalService, step, answer); + await service.sendAnswer(min, AskDialog.getChannel(step), step, answer); await step.replaceDialog('/ask', { isReturning: true }); } diff --git a/packages/kb.gbapp/services/KBService.ts b/packages/kb.gbapp/services/KBService.ts index 79326f04..8d1947dd 100644 --- a/packages/kb.gbapp/services/KBService.ts +++ b/packages/kb.gbapp/services/KBService.ts @@ -46,7 +46,7 @@ const walkPromise = require('walk-promise'); const parse = require('bluebird').promisify(require('csv-parse')); const { SearchService } = require('azure-search-client'); -import { GBDialogStep, GBLog, IGBConversationalService, IGBCoreService, IGBInstance } from 'botlib'; +import { GBDialogStep, GBLog, IGBConversationalService, IGBCoreService, IGBInstance, GBMinInstance } from 'botlib'; import { Sequelize } from 'sequelize-typescript'; import { AzureDeployerService } from '../../azuredeployer.gbapp/services/AzureDeployerService'; import { GuaribasPackage } from '../../core.gbapp/models/GBModel'; @@ -54,6 +54,8 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer'; import { GuaribasAnswer, GuaribasQuestion, GuaribasSubject } from '../models'; import { Messages } from '../strings'; import { GBConfigService } from './../../core.gbapp/services/GBConfigService'; +import { GBServer } from '../../../src/app'; + /** * Result for quey on KB data. @@ -349,21 +351,28 @@ export class KBService { }); } - public async sendAnswer(channel: string, conversationalService: IGBConversationalService, step: GBDialogStep, answer: GuaribasAnswer) { + public async sendAnswer(min: GBMinInstance, channel: string, step: GBDialogStep, answer: GuaribasAnswer) { if (answer.content.endsWith('.mp4')) { - await this.playVideo(conversationalService, step, answer); + await this.playVideo(min.conversationalService, step, answer); } else if (answer.format === '.md') { - await this.playMarkdown(answer, channel, step, conversationalService); + await this.playMarkdown(min, answer, channel, step, min.conversationalService); + } else if (answer.content.endsWith('.ogg')) { + + await this.playAudio(min, answer, channel, step, min.conversationalService); } else { await step.context.sendActivity(answer.content); - await conversationalService.sendEvent(step, 'stop', undefined); + await min.conversationalService.sendEvent(step, 'stop', undefined); } - } + } - private async playMarkdown(answer: GuaribasAnswer, channel: string, step: GBDialogStep, conversationalService: IGBConversationalService) { + private async playAudio(min: GBMinInstance, answer: GuaribasAnswer, channel: string, step: GBDialogStep, conversationalService: IGBConversationalService) { + conversationalService.sendAudio(min, step, answer.content); + } + + private async playMarkdown(min: GBMinInstance, answer: GuaribasAnswer, channel: string, step: GBDialogStep, conversationalService: IGBConversationalService) { let html = answer.content; marked.setOptions({ renderer: new marked.Renderer(), @@ -392,12 +401,82 @@ export class KBService { }); } else if (channel === 'whatsapp') { - let from = step.context.activity.from.id; - //conversationalService.sendFile(min, from, answer.content); + await this.sendMarkdownToMobile(step, answer, conversationalService, min); } } -private async playVideo(conversationalService: IGBConversationalService, step: GBDialogStep, answer: GuaribasAnswer) { + private async sendMarkdownToMobile(step: GBDialogStep, answer: GuaribasAnswer, conversationalService: IGBConversationalService, min: GBMinInstance) { + + let text = answer.content; + + enum State { + InText, + InImage, + InImageBegin, + InImageCaption, + InImageAddressBegin, + InImageAddressBody + }; + let state = State.InText; + let currentImage = ''; + let currentText = ''; + + //![General Bots](/instance/images/gb.png) + for (var i = 0; i < text.length; i++) { + const c = text.charAt(i); + + switch (state) { + case State.InText: + if (c === '!') { + state = State.InImageBegin; + } + else { + currentText = currentText.concat(c); + } + break; + case State.InImageBegin: + if (c === '[') { + if (currentText !== '') { + await step.context.sendActivity(currentText); + } + + currentText = ''; + state = State.InImageCaption; + } + else { + state = State.InText; + currentText = currentText.concat('!').concat(c); + } + break; + case State.InImageCaption: + if (c === ']') { + state = State.InImageAddressBegin; + } + break; + case State.InImageAddressBegin: + if (c === '(') { + state = State.InImageAddressBody; + } + break; + case State.InImageAddressBody: + if (c === ')') { + state = State.InText; + let url = urlJoin(GBServer.globals.publicAddress, currentImage); + await conversationalService.sendFile(min, step, url); + } + else { + currentImage = currentImage.concat(c); + } + break; + } + + } + if (currentText !== '') { + await step.context.sendActivity(currentText); + } + } + + private async playVideo(conversationalService: IGBConversationalService, step: GBDialogStep, answer: GuaribasAnswer) { await conversationalService.sendEvent(step, 'play', { playerType: 'video', data: answer.content @@ -405,74 +484,74 @@ private async playVideo(conversationalService: IGBConversationalService, step: G } public async importKbPackage( - localPath: string, - packageStorage: GuaribasPackage, - instance: IGBInstance -): Promise < any > { - // Imports subjects tree into database and return it. + localPath: string, + packageStorage: GuaribasPackage, + instance: IGBInstance + ): Promise { + // Imports subjects tree into database and return it. - await this.importSubjectFile(packageStorage.packageId, urlJoin(localPath, 'subjects.json'), instance); + await this.importSubjectFile(packageStorage.packageId, urlJoin(localPath, 'subjects.json'), instance); - // Import all .tsv files in the tabular directory. + // Import all .tsv files in the tabular directory. - return this.importKbTabularDirectory(localPath, instance, packageStorage.packageId); -} + return this.importKbTabularDirectory(localPath, instance, packageStorage.packageId); + } - public async importKbTabularDirectory(localPath: string, instance: IGBInstance, packageId: number): Promise < any > { - const files = await walkPromise(urlJoin(localPath, 'tabular')); + public async importKbTabularDirectory(localPath: string, instance: IGBInstance, packageId: number): Promise { + const files = await walkPromise(urlJoin(localPath, 'tabular')); - return Promise.all( - files.map(async file => { - if (file.name.endsWith('.xlsx')) { - return this.importKbTabularFile(urlJoin(file.root, file.name), instance.instanceId, packageId); - } - }) - ); -} + return Promise.all( + files.map(async file => { + if (file.name.endsWith('.xlsx')) { + return this.importKbTabularFile(urlJoin(file.root, file.name), instance.instanceId, packageId); + } + }) + ); + } - public async importSubjectFile(packageId: number, filename: string, instance: IGBInstance): Promise < any > { - const subjectsLoaded = JSON.parse(Fs.readFileSync(filename, 'utf8')); + public async importSubjectFile(packageId: number, filename: string, instance: IGBInstance): Promise { + const subjectsLoaded = JSON.parse(Fs.readFileSync(filename, 'utf8')); - const doIt = async (subjects: GuaribasSubject[], parentSubjectId: number) => { - return asyncPromise.eachSeries(subjects, async item => { - const value = await GuaribasSubject.create({ - internalId: item.id, - parentSubjectId: parentSubjectId, - instanceId: instance.instanceId, - from: item.from, - to: item.to, - title: item.title, - description: item.description, - packageId: packageId + const doIt = async (subjects: GuaribasSubject[], parentSubjectId: number) => { + return asyncPromise.eachSeries(subjects, async item => { + const value = await GuaribasSubject.create({ + internalId: item.id, + parentSubjectId: parentSubjectId, + instanceId: instance.instanceId, + from: item.from, + to: item.to, + title: item.title, + description: item.description, + packageId: packageId + }); + + if (item.children) { + return Promise.resolve(doIt(item.children, value.subjectId)); + } else { + return Promise.resolve(item); + } }); + }; - if (item.children) { - return Promise.resolve(doIt(item.children, value.subjectId)); - } else { - return Promise.resolve(item); - } - }); - }; - - return doIt(subjectsLoaded.children, undefined); -} + return doIt(subjectsLoaded.children, undefined); + } public async undeployKbFromStorage(instance: IGBInstance, deployer: GBDeployer, packageId: number) { - await GuaribasQuestion.destroy({ - where: { instanceId: instance.instanceId, packageId: packageId } - }); - await GuaribasAnswer.destroy({ - where: { instanceId: instance.instanceId, packageId: packageId } - }); - await GuaribasSubject.destroy({ - where: { instanceId: instance.instanceId, packageId: packageId } - }); - await GuaribasPackage.destroy({ - where: { instanceId: instance.instanceId, packageId: packageId } - }); + await GuaribasQuestion.destroy({ + where: { instanceId: instance.instanceId, packageId: packageId } + }); + await GuaribasAnswer.destroy({ + where: { instanceId: instance.instanceId, packageId: packageId } + }); + await GuaribasSubject.destroy({ + where: { instanceId: instance.instanceId, packageId: packageId } + }); + await GuaribasPackage.destroy({ + where: { instanceId: instance.instanceId, packageId: packageId } + }); - await deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex)); -} + await deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex)); + } /** * Deploys a knowledge base to the storage using the .gbkb format. @@ -480,17 +559,17 @@ private async playVideo(conversationalService: IGBConversationalService, step: G * @param localPath Path to the .gbkb folder. */ public async deployKb(core: IGBCoreService, deployer: GBDeployer, localPath: string) { - const packageType = Path.extname(localPath); - const packageName = Path.basename(localPath); - GBLog.info(`[GBDeployer] Opening package: ${localPath}`); - const packageObject = JSON.parse(Fs.readFileSync(urlJoin(localPath, 'package.json'), 'utf8')); + const packageType = Path.extname(localPath); + const packageName = Path.basename(localPath); + GBLog.info(`[GBDeployer] Opening package: ${localPath}`); + const packageObject = JSON.parse(Fs.readFileSync(urlJoin(localPath, 'package.json'), 'utf8')); - const instance = await core.loadInstance(packageObject.botId); - GBLog.info(`[GBDeployer] Importing: ${localPath}`); - const p = await deployer.deployPackageToStorage(instance.instanceId, packageName); - await this.importKbPackage(localPath, p, instance); + const instance = await core.loadInstance(packageObject.botId); + GBLog.info(`[GBDeployer] Importing: ${localPath}`); + const p = await deployer.deployPackageToStorage(instance.instanceId, packageName); + await this.importKbPackage(localPath, p, instance); - deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex)); - GBLog.info(`[GBDeployer] Finished import of ${localPath}`); -} + deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex)); + GBLog.info(`[GBDeployer] Finished import of ${localPath}`); + } } diff --git a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts index c1782ee2..43eb63cc 100644 --- a/packages/whatsapp.gblib/services/WhatsappDirectLine.ts +++ b/packages/whatsapp.gblib/services/WhatsappDirectLine.ts @@ -231,14 +231,15 @@ export class WhatsappDirectLine extends GBService { return `${attachment.content.title} - ${attachment.content.text}`; } - public async sendFileToDevice(to, url) { + public async sendFileToDevice(to, url, filename) { const options = { method: 'POST', url: urlJoin(this.whatsappServiceUrl, 'sendFile'), qs: { token: this.whatsappServiceKey, phone: to, - body: url + body: url, + filename: filename }, headers: { 'cache-control': 'no-cache' @@ -254,6 +255,28 @@ export class WhatsappDirectLine extends GBService { } } + public async sendAudioToDevice(to, url) { + const options = { + method: 'POST', + url: urlJoin(this.whatsappServiceUrl, 'sendPTT'), + qs: { + token: this.whatsappServiceKey, + phone: to, + audio:url + }, + headers: { + 'cache-control': 'no-cache' + } + }; + + try { + // tslint:disable-next-line: await-promise + const result = await request.post(options); + GBLog.info(result); + } catch (error) { + GBLog.error(`Error sending message to Whatsapp provider ${error.message}`); + } + } public async sendToDevice(to, msg) { const options = { diff --git a/src/app.ts b/src/app.ts index 97a1e401..1893d68c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -39,7 +39,7 @@ const express = require('express'); const bodyParser = require('body-parser'); -import { GBLog, IGBCoreService, IGBInstance, IGBPackage } from 'botlib'; +import { GBLog, IGBCoreService, IGBInstance, IGBPackage, GBMinInstance } from 'botlib'; import { GBAdminService } from '../packages/admin.gbapp/services/GBAdminService'; import { AzureDeployerService } from '../packages/azuredeployer.gbapp/services/AzureDeployerService'; import { GBConfigService } from '../packages/core.gbapp/services/GBConfigService'; @@ -60,6 +60,7 @@ export class RootData { minService: GBMinService; bootInstance: IGBInstance; public minInstances: any[]; + minBoot: GBMinInstance; } /**