feat(whatsapp.gblib): Now Whatsapp will display markdown from .gbkb including images.

packages/core.gbapp/dialogs/SwitchBot.ts

@@ -63,7 +63,6 @@ export class SwitchBotDialog extends IGBDialog {
       async step => {
         let sec = new SecService();
         let from = step.context.activity.from.id;
-        const minBoot = GBServer.globals.minInstances[0];
         await sec.updateCurrentBotId(from, step.result);
         await step.context.sendActivity(`Opa, vamos lá!`);
                 

packages/core.gbapp/services/GBConversationalService.ts

@@ -64,11 +64,17 @@ export class GBConversationalService implements IGBConversationalService {
   }
 
   public async sendFile(min: GBMinInstance, step: GBDialogStep, url: string): Promise<any> {
-    let mobile = step.context.activity.from.id;
+    const mobile = step.context.activity.from.id;
-    min.whatsAppDirectLine.sendFile(mobile, url);
+    const filename = url.substring(url.lastIndexOf('/')+1);
+    await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename);
 
   }
 
+  public async sendAudio(min: GBMinInstance, step: GBDialogStep, url: string): Promise<any> {
+    const mobile = step.context.activity.from.id;
+    await min.whatsAppDirectLine.sendAudioToDevice(mobile, url);
+  }
+
   public async sendEvent(step: GBDialogStep, name: string, value: Object): Promise<any> {
     if (step.context.activity.channelId === 'webchat') {
       const msg = MessageFactory.text('');

packages/core.gbapp/services/GBDeployer.ts

@@ -203,6 +203,10 @@ export class GBDeployer {
       instance.whatsappServiceNumber = bootInstance.whatsappServiceNumber;
       instance.whatsappServiceUrl = bootInstance.whatsappServiceUrl;
       instance.whatsappServiceWebhookUrl = bootInstance.whatsappServiceWebhookUrl;
+      instance.storageServer = bootInstance.storageServer;
+      instance.storageName = bootInstance.storageName;
+      instance.storageUsername = bootInstance.storageUsername;
+      instance.storagePassword = bootInstance.storagePassword;
 
       instance = await service.internalDeployBot(
         instance,
@@ -422,9 +426,7 @@ export class GBDeployer {
         server.use(`/themes/${filenameOnly}`, express.static(filename));
         GBLog.info(`Theme (.gbtheme) assets accessible at: /themes/${filenameOnly}.`);
       } else if (Path.extname(filename) === '.gbkb') {
-        server.use(`/kb/${filenameOnly}/subjects`, express.static(urlJoin(filename, 'subjects')));
+        this.mountGBKBAssets( filenameOnly, filename);
-        server.use(`/kb/${filenameOnly}/images`, express.static(urlJoin(filename, 'images')));
-        GBLog.info(`KB (.gbkb) assets accessible at: /kb/${filenameOnly}.`);
       } else if (Path.extname(filename) === '.gbui') {
         // Already Handled
       } else if (Path.extname(filename) === '.gbdialog') {
@@ -458,6 +460,13 @@ export class GBDeployer {
     return { generalPackages, totalPackages };
   }
 
+  private mountGBKBAssets(packageName: any, filename: string) {
+    GBServer.globals.server.use(`/kb/${packageName}/subjects`, express.static(urlJoin(filename, 'subjects')));
+    GBServer.globals.server.use(`/kb/${packageName}/images`, express.static(urlJoin(filename, 'images')));
+    GBServer.globals.server.use(`/kb/${packageName}/audios`, express.static(urlJoin(filename, 'audios')));
+    GBLog.info(`KB (.gbkb) assets accessible at: /kb/${packageName}.`);
+  }
+
   private isSystemPackage(name: string): Boolean {
     const names = ['core.gbapp', 'admin.gbapp', 'azuredeployer.gbapp', 'customer-satisfaction.gbapp', 'kb.gbapp'];
 

packages/core.gbapp/services/GBMinService.ts

@@ -138,9 +138,8 @@ export class GBMinService {
           return; // Exit here.
         }
 
-        const minBoot = GBServer.globals.bootInstance;
         const toSwitchMin = GBServer.globals.minInstances.filter(p => p.botId === text)[0];
-        let activeMin = toSwitchMin ? toSwitchMin : minBoot;
+        let activeMin = toSwitchMin ? toSwitchMin : GBServer.globals.minBoot;
 
         let sec = new SecService();
         let user = await sec.getUserFromPhone(id);
@@ -190,6 +189,10 @@ export class GBMinService {
 
     // Build bot adapter.
     const { min, adapter, conversationState } = await this.buildBotAdapter(instance, GBServer.globals.publicAddress, GBServer.globals.sysPackages);
+
+    if (GBServer.globals.minInstances.length === 0) {
+      GBServer.globals.minBoot = min;
+    }
     GBServer.globals.minInstances.push(min);
 
     // Install default VBA module.
@@ -453,11 +456,13 @@ export class GBMinService {
           user.cb = undefined;
           await min.userProfile.set(step.context, user);
 
-          let sec = new SecService();
+          if (context.activity.membersAdded !== undefined) {
-          const member = context.activity.membersAdded[0];
+            let sec = new SecService();
+            const member = context.activity.membersAdded[0];
 
-          await sec.ensureUser(instance.instanceId, member.id,
+            await sec.ensureUser(instance.instanceId, member.id,
-            min.botId, member.id, "", "web", member.name, member.id);
+              min.botId, member.id, "", "web", member.name, member.id);
+          }
         }
 
         GBLog.info(

packages/kb.gbapp/dialogs/AskDialog.ts

@@ -144,7 +144,7 @@ export class AskDialog extends IGBDialog {
 
           // Sends the answer to all outputs, including projector.
 
-          await service.sendAnswer(AskDialog.getChannel(step), min.conversationalService, step, resultsA.answer);
+          await service.sendAnswer(min, AskDialog.getChannel(step), step, resultsA.answer);
 
           // Goes to ask loop, again.
           return await step.replaceDialog('/ask', { isReturning: true });
@@ -164,7 +164,7 @@ export class AskDialog extends IGBDialog {
               await step.context.sendActivity(Messages[locale].wider_answer);
             }
             // Sends the answer to all outputs, including projector.
-            await service.sendAnswer(AskDialog.getChannel(step), min.conversationalService, step, resultsB.answer);
+            await service.sendAnswer(min, AskDialog.getChannel(step), step, resultsB.answer);
 
             return await step.replaceDialog('/ask', { isReturning: true });
           } else {
@@ -180,7 +180,7 @@ export class AskDialog extends IGBDialog {
   }
 
   private static getChannel(step): string {
-    return Number.isInteger(step.context.activity.from.id) ? 'whatsapp' : step.context.activity.channelId;
+    return !isNaN(step.context.activity.from.id) ? 'whatsapp' : step.context.activity.channelId;
   }
 
   private static getAnswerEventDialog(service: KBService, min: GBMinInstance) {
@@ -191,7 +191,7 @@ export class AskDialog extends IGBDialog {
           const question = await service.getQuestionById(min.instance.instanceId, data.questionId);
           const answer = await service.getAnswerById(min.instance.instanceId, question.answerId);
           // Sends the answer to all outputs, including projector.
-          await service.sendAnswer(AskDialog.getChannel(step), min.conversationalService, step, answer);
+          await service.sendAnswer(min, AskDialog.getChannel(step), step, answer);
           await step.replaceDialog('/ask', { isReturning: true });
         }
 

packages/kb.gbapp/services/KBService.ts

@@ -46,7 +46,7 @@ const walkPromise = require('walk-promise');
 const parse = require('bluebird').promisify(require('csv-parse'));
 const { SearchService } = require('azure-search-client');
 
-import { GBDialogStep, GBLog, IGBConversationalService, IGBCoreService, IGBInstance } from 'botlib';
+import { GBDialogStep, GBLog, IGBConversationalService, IGBCoreService, IGBInstance, GBMinInstance } from 'botlib';
 import { Sequelize } from 'sequelize-typescript';
 import { AzureDeployerService } from '../../azuredeployer.gbapp/services/AzureDeployerService';
 import { GuaribasPackage } from '../../core.gbapp/models/GBModel';
@@ -54,6 +54,8 @@ import { GBDeployer } from '../../core.gbapp/services/GBDeployer';
 import { GuaribasAnswer, GuaribasQuestion, GuaribasSubject } from '../models';
 import { Messages } from '../strings';
 import { GBConfigService } from './../../core.gbapp/services/GBConfigService';
+import { GBServer } from '../../../src/app';
+
 
 /**
  * Result for quey on KB data.
@@ -349,21 +351,28 @@ export class KBService {
     });
   }
 
-  public async sendAnswer(channel: string, conversationalService: IGBConversationalService, step: GBDialogStep, answer: GuaribasAnswer) {
+  public async sendAnswer(min: GBMinInstance, channel: string, step: GBDialogStep, answer: GuaribasAnswer) {
     if (answer.content.endsWith('.mp4')) {
-      await this.playVideo(conversationalService, step, answer);
+      await this.playVideo(min.conversationalService, step, answer);
     }
     else if (answer.format === '.md') {
 
-      await this.playMarkdown(answer, channel, step, conversationalService);
+      await this.playMarkdown(min, answer, channel, step, min.conversationalService);
 
+    } else if (answer.content.endsWith('.ogg')) {
+
+      await this.playAudio(min, answer, channel, step, min.conversationalService);
     } else {
       await step.context.sendActivity(answer.content);
-      await conversationalService.sendEvent(step, 'stop', undefined);
+      await min.conversationalService.sendEvent(step, 'stop', undefined);
     }
-  } 
+  }
 
-  private async playMarkdown(answer: GuaribasAnswer, channel: string, step: GBDialogStep, conversationalService: IGBConversationalService) {
+  private async playAudio(min: GBMinInstance, answer: GuaribasAnswer, channel: string, step: GBDialogStep, conversationalService: IGBConversationalService) {
+    conversationalService.sendAudio(min, step, answer.content);
+  }
+
+  private async playMarkdown(min: GBMinInstance, answer: GuaribasAnswer, channel: string, step: GBDialogStep, conversationalService: IGBConversationalService) {
     let html = answer.content;
     marked.setOptions({
       renderer: new marked.Renderer(),
@@ -392,12 +401,82 @@ export class KBService {
       });
     }
     else if (channel === 'whatsapp') {
-      let from = step.context.activity.from.id;
+      await this.sendMarkdownToMobile(step, answer, conversationalService, min);
-      //conversationalService.sendFile(min, from, answer.content);
     }
   }
 
-private async playVideo(conversationalService: IGBConversationalService, step: GBDialogStep, answer: GuaribasAnswer) {
+  private async sendMarkdownToMobile(step: GBDialogStep, answer: GuaribasAnswer, conversationalService: IGBConversationalService, min: GBMinInstance) {
+
+    let text = answer.content;
+
+    enum State {
+      InText,
+      InImage,
+      InImageBegin,
+      InImageCaption,
+      InImageAddressBegin,
+      InImageAddressBody
+    };
+    let state = State.InText;
+    let currentImage = '';
+    let currentText = '';
+
+    //![General Bots](/instance/images/gb.png)
+    for (var i = 0; i < text.length; i++) {
+      const c = text.charAt(i);
+
+      switch (state) {
+        case State.InText:
+          if (c === '!') {
+            state = State.InImageBegin;
+          }
+          else {
+            currentText = currentText.concat(c);
+          }
+          break;
+        case State.InImageBegin:
+          if (c === '[') {
+            if (currentText !== '') {
+              await step.context.sendActivity(currentText);
+            }
+       
+            currentText = '';
+            state = State.InImageCaption;
+          }
+          else {
+            state = State.InText;
+            currentText = currentText.concat('!').concat(c);
+          }
+          break;
+        case State.InImageCaption:
+          if (c === ']') {
+            state = State.InImageAddressBegin;
+          }
+          break;
+        case State.InImageAddressBegin:
+          if (c === '(') {
+            state = State.InImageAddressBody;
+          }
+          break;
+        case State.InImageAddressBody:
+          if (c === ')') {
+            state = State.InText;
+            let url = urlJoin(GBServer.globals.publicAddress, currentImage);
+            await conversationalService.sendFile(min, step, url);
+          }
+          else {
+            currentImage = currentImage.concat(c);
+          }
+          break;
+      }
+
+    }
+    if (currentText !== '') {
+      await step.context.sendActivity(currentText);
+    }
+  }
+
+  private async playVideo(conversationalService: IGBConversationalService, step: GBDialogStep, answer: GuaribasAnswer) {
     await conversationalService.sendEvent(step, 'play', {
       playerType: 'video',
       data: answer.content
@@ -405,74 +484,74 @@ private async playVideo(conversationalService: IGBConversationalService, step: G
   }
 
   public async importKbPackage(
-  localPath: string,
+    localPath: string,
-  packageStorage: GuaribasPackage,
+    packageStorage: GuaribasPackage,
-  instance: IGBInstance
+    instance: IGBInstance
-): Promise < any > {
+  ): Promise<any> {
-  // Imports subjects tree into database and return it.
+    // Imports subjects tree into database and return it.
 
-  await this.importSubjectFile(packageStorage.packageId, urlJoin(localPath, 'subjects.json'), instance);
+    await this.importSubjectFile(packageStorage.packageId, urlJoin(localPath, 'subjects.json'), instance);
 
-  // Import all .tsv files in the tabular directory.
+    // Import all .tsv files in the tabular directory.
 
-  return this.importKbTabularDirectory(localPath, instance, packageStorage.packageId);
+    return this.importKbTabularDirectory(localPath, instance, packageStorage.packageId);
-}
+  }
 
-  public async importKbTabularDirectory(localPath: string, instance: IGBInstance, packageId: number): Promise < any > {
+  public async importKbTabularDirectory(localPath: string, instance: IGBInstance, packageId: number): Promise<any> {
-  const files = await walkPromise(urlJoin(localPath, 'tabular'));
+    const files = await walkPromise(urlJoin(localPath, 'tabular'));
 
-  return Promise.all(
+    return Promise.all(
-    files.map(async file => {
+      files.map(async file => {
-      if (file.name.endsWith('.xlsx')) {
+        if (file.name.endsWith('.xlsx')) {
-        return this.importKbTabularFile(urlJoin(file.root, file.name), instance.instanceId, packageId);
+          return this.importKbTabularFile(urlJoin(file.root, file.name), instance.instanceId, packageId);
-      }
+        }
-    })
+      })
-  );
+    );
-}
+  }
 
-  public async importSubjectFile(packageId: number, filename: string, instance: IGBInstance): Promise < any > {
+  public async importSubjectFile(packageId: number, filename: string, instance: IGBInstance): Promise<any> {
-  const subjectsLoaded = JSON.parse(Fs.readFileSync(filename, 'utf8'));
+    const subjectsLoaded = JSON.parse(Fs.readFileSync(filename, 'utf8'));
 
-  const doIt = async (subjects: GuaribasSubject[], parentSubjectId: number) => {
+    const doIt = async (subjects: GuaribasSubject[], parentSubjectId: number) => {
-    return asyncPromise.eachSeries(subjects, async item => {
+      return asyncPromise.eachSeries(subjects, async item => {
-      const value = await GuaribasSubject.create({
+        const value = await GuaribasSubject.create({
-        internalId: item.id,
+          internalId: item.id,
-        parentSubjectId: parentSubjectId,
+          parentSubjectId: parentSubjectId,
-        instanceId: instance.instanceId,
+          instanceId: instance.instanceId,
-        from: item.from,
+          from: item.from,
-        to: item.to,
+          to: item.to,
-        title: item.title,
+          title: item.title,
-        description: item.description,
+          description: item.description,
-        packageId: packageId
+          packageId: packageId
+        });
+
+        if (item.children) {
+          return Promise.resolve(doIt(item.children, value.subjectId));
+        } else {
+          return Promise.resolve(item);
+        }
       });
+    };
 
-      if (item.children) {
+    return doIt(subjectsLoaded.children, undefined);
-        return Promise.resolve(doIt(item.children, value.subjectId));
+  }
-      } else {
-        return Promise.resolve(item);
-      }
-    });
-  };
-
-  return doIt(subjectsLoaded.children, undefined);
-}
 
   public async undeployKbFromStorage(instance: IGBInstance, deployer: GBDeployer, packageId: number) {
-  await GuaribasQuestion.destroy({
+    await GuaribasQuestion.destroy({
-    where: { instanceId: instance.instanceId, packageId: packageId }
+      where: { instanceId: instance.instanceId, packageId: packageId }
-  });
+    });
-  await GuaribasAnswer.destroy({
+    await GuaribasAnswer.destroy({
-    where: { instanceId: instance.instanceId, packageId: packageId }
+      where: { instanceId: instance.instanceId, packageId: packageId }
-  });
+    });
-  await GuaribasSubject.destroy({
+    await GuaribasSubject.destroy({
-    where: { instanceId: instance.instanceId, packageId: packageId }
+      where: { instanceId: instance.instanceId, packageId: packageId }
-  });
+    });
-  await GuaribasPackage.destroy({
+    await GuaribasPackage.destroy({
-    where: { instanceId: instance.instanceId, packageId: packageId }
+      where: { instanceId: instance.instanceId, packageId: packageId }
-  });
+    });
 
-  await deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex));
+    await deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex));
-}
+  }
 
   /**
    * Deploys a knowledge base to the storage using the .gbkb format.
@@ -480,17 +559,17 @@ private async playVideo(conversationalService: IGBConversationalService, step: G
    * @param localPath Path to the .gbkb folder.
    */
   public async deployKb(core: IGBCoreService, deployer: GBDeployer, localPath: string) {
-  const packageType = Path.extname(localPath);
+    const packageType = Path.extname(localPath);
-  const packageName = Path.basename(localPath);
+    const packageName = Path.basename(localPath);
-  GBLog.info(`[GBDeployer] Opening package: ${localPath}`);
+    GBLog.info(`[GBDeployer] Opening package: ${localPath}`);
-  const packageObject = JSON.parse(Fs.readFileSync(urlJoin(localPath, 'package.json'), 'utf8'));
+    const packageObject = JSON.parse(Fs.readFileSync(urlJoin(localPath, 'package.json'), 'utf8'));
 
-  const instance = await core.loadInstance(packageObject.botId);
+    const instance = await core.loadInstance(packageObject.botId);
-  GBLog.info(`[GBDeployer] Importing: ${localPath}`);
+    GBLog.info(`[GBDeployer] Importing: ${localPath}`);
-  const p = await deployer.deployPackageToStorage(instance.instanceId, packageName);
+    const p = await deployer.deployPackageToStorage(instance.instanceId, packageName);
-  await this.importKbPackage(localPath, p, instance);
+    await this.importKbPackage(localPath, p, instance);
 
-  deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex));
+    deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex));
-  GBLog.info(`[GBDeployer] Finished import of ${localPath}`);
+    GBLog.info(`[GBDeployer] Finished import of ${localPath}`);
-}
+  }
 }

packages/whatsapp.gblib/services/WhatsappDirectLine.ts

@@ -231,14 +231,15 @@ export class WhatsappDirectLine extends GBService {
     return `${attachment.content.title} - ${attachment.content.text}`;
   }
 
-  public async sendFileToDevice(to, url) {
+  public async sendFileToDevice(to, url, filename) {
     const options = {
       method: 'POST',
       url: urlJoin(this.whatsappServiceUrl, 'sendFile'),
       qs: {
         token: this.whatsappServiceKey,
         phone: to,
-        body: url
+        body: url,
+        filename: filename
       },
       headers: {
         'cache-control': 'no-cache'
@@ -254,6 +255,28 @@ export class WhatsappDirectLine extends GBService {
     }
   }
 
+  public async sendAudioToDevice(to, url) {
+    const options = {
+      method: 'POST',
+      url: urlJoin(this.whatsappServiceUrl, 'sendPTT'),
+      qs: {
+        token: this.whatsappServiceKey,
+        phone: to,
+        audio:url
+      },
+      headers: {
+        'cache-control': 'no-cache'
+      }
+    };
+
+    try {
+      // tslint:disable-next-line: await-promise
+      const result = await request.post(options);
+      GBLog.info(result);
+    } catch (error) {
+      GBLog.error(`Error sending message to Whatsapp provider ${error.message}`);
+    }
+  }
 
   public async sendToDevice(to, msg) {
     const options = {

src/app.ts

@@ -39,7 +39,7 @@
 const express = require('express');
 const bodyParser = require('body-parser');
 
-import { GBLog, IGBCoreService, IGBInstance, IGBPackage } from 'botlib';
+import { GBLog, IGBCoreService, IGBInstance, IGBPackage, GBMinInstance } from 'botlib';
 import { GBAdminService } from '../packages/admin.gbapp/services/GBAdminService';
 import { AzureDeployerService } from '../packages/azuredeployer.gbapp/services/AzureDeployerService';
 import { GBConfigService } from '../packages/core.gbapp/services/GBConfigService';
@@ -60,6 +60,7 @@ export class RootData {
   minService: GBMinService;
   bootInstance: IGBInstance;
   public minInstances: any[];
+  minBoot: GBMinInstance;
 }
 
 /**