new(basic.gblib): NER (NLP) added for .gbdialog https://github.com/GeneralBots/BotServer/issues/217

package.json

@@ -99,6 +99,7 @@
     "ms-rest-azure": "3.0.0",
     "nexmo": "2.9.1",
     "node-cron": "3.0.0",
+    "node-nlp": "^3.10.2",
     "node-tesseract-ocr": "^2.2.1",
     "npm": "7.21.0",
     "opn": "6.0.0",

packages/basic.gblib/services/GBVMService.ts

@@ -41,6 +41,7 @@ const urlJoin = require('url-join');
 import { DialogKeywords } from './DialogKeywords';
 import { ScheduleServices } from './ScheduleServices';
 import { HearDialog } from '../dialogs/HearDialog';
+import { GBConfigService } from '../../core.gbapp/services/GBConfigService';
 //tslint:disable-next-line:no-submodule-imports
 const vm = require('vm');
 const vb2ts = require('./vbscript-to-typescript');
@@ -497,12 +498,12 @@ export class GBVMService extends GBService {
     let include = null;
     do {
       include = /^include\b(.*)$/gmi.exec(basicCode);
-      
+
       if (include) {
         let includeName = include[1].trim();
         includeName = Path.join(Path.dirname(filename), includeName);
         includeName = includeName.substr(0, includeName.lastIndexOf(".")) + ".vbs";
-        
+
         // To use include, two /publish will be necessary (for now)
         // because of alphabet order may raise not found errors.
 
@@ -721,8 +722,25 @@ export class GBVMService extends GBService {
     // in BASIC.
 
     const user = step ? await min.userProfile.get(step.context, {}) : null;
+
     const sandbox: DialogKeywords = new DialogKeywords(min, deployer, step, user);
 
+    const contentLocale = min.core.getParam<string>(
+      min.instance,
+      'Default Content Language',
+      GBConfigService.get('DEFAULT_CONTENT_LANGUAGE')
+    );
+
+    const entities = await min["nerEngine"].findEntities(
+      step.context.activity['originalText'],
+      contentLocale);
+
+    for (let i = 0; i < entities.length; i++) {
+      const v = entities[i];
+      const variableName = `${v.entity}`;
+      sandbox[variableName] = v.option;
+    }
+
     // Injects the .gbdialog generated code into the VM.
 
     const context = vm.createContext(sandbox);

packages/core.gbapp/services/GBConversationalService.ts

@@ -662,6 +662,7 @@ export class GBConversationalService {
     }
   }
 
+
   // TODO: Update botlib.
   public async routeNLP(step: GBDialogStep, min: GBMinInstance, text: string): Promise<boolean> {
     return false;

packages/core.gbapp/services/GBMinService.ts

@@ -44,6 +44,7 @@ const AuthenticationContext = require('adal-node').AuthenticationContext;
 const wash = require('washyourmouthoutwithsoap');
 const { FacebookAdapter } = require('botbuilder-adapter-facebook');
 const path = require('path');
+const { NerManager } = require('node-nlp');
 import {
   AutoSaveStateMiddleware,
   BotFrameworkAdapter,
@@ -246,6 +247,12 @@ export class GBMinService {
       await this.deployer.deployPackage(min, packagePath);
     }
 
+    // Loads Named Entity data for this bot.
+
+    await KBService.RefreshNER(min);
+
+    // Loads schedules.
+
     const service = new ScheduleServices();
     await service.loadSchedules(min);
 
@@ -705,6 +712,7 @@ export class GBMinService {
     min.sandBoxMap = {};
     min["scheduleMap"] = {};
     min["conversationWelcomed"] = {};
+    min["nerEngine"] = new NerManager();;
     min.packages = sysPackages;
     min.appPackages = appPackages;
 
@@ -730,6 +738,8 @@ export class GBMinService {
       }
     });
 
+    
+    
     if (min.instance.googlePrivateKey) {
       min['googleDirectLine'] = new GoogleChatDirectLine(
         min,

packages/kb.gbapp/dialogs/AskDialog.ts

@@ -301,6 +301,8 @@ export class AskDialog extends IGBDialog {
   private static async handleAnswer(service: KBService, min: GBMinInstance, step: any, answer: GuaribasAnswer) {
     const text = answer.content;
     if (text.endsWith('.docx')) {
+
+
       const mainName = GBVMService.getMethodNameFromVBSFilename(text);
       return await GBVMService.callVM(mainName, min, step, this.deployer);
     } else {
@@ -328,7 +330,7 @@ export class AskDialog extends IGBDialog {
         const data = step.options as AskDialogArgs;
         if (data !== undefined && data.questionId !== undefined) {
           const question = await service.getQuestionById(min.instance.instanceId, data.questionId);
-          const answer = await service.getAnswerById(min.instance.instanceId, question.answerId);
+          const answer = await service.getAnswerById(min.instance.instanceId, question.answerId );
           // Sends the answer to all outputs, including projector.
           await service.sendAnswer(min, AskDialog.getChannel(step), step, answer);
           await step.replaceDialog('/ask', { isReturning: true });

packages/kb.gbapp/services/KBService.ts

@@ -166,6 +166,19 @@ export class KBService implements IGBKBService {
 
     return question;
   }
+  public static async getQuestionsNER(instanceId: number) {
+    const where = {
+      instanceId: instanceId,
+      content: { [Op.like]: `%(%` }
+    };
+
+    const questions = await GuaribasQuestion.findAll({
+      where: where
+    });
+
+    return questions;
+
+  }
 
   public async getQuestionsSEO(instanceId: number) {
 
@@ -204,14 +217,13 @@ export class KBService implements IGBKBService {
     let question = await service.getQuestionFromAlternateText(instanceId, text);
 
     if (!question) {
-      const where={
+      const where = {
         instanceId: instanceId,
         content: { [Op.like]: `%[^a-z]${text}[^a-z]%` }
       };
 
-      if (from)
+      if (from) {
-      {
+        where['from'] = from;
-        where['from']= from;
       }
       question = await GuaribasQuestion.findOne({
         where: where
@@ -241,7 +253,7 @@ export class KBService implements IGBKBService {
     return undefined;
   }
 
-  
+
 
 
   public async addAnswer(obj: GuaribasAnswer): Promise<GuaribasAnswer> {
@@ -392,9 +404,9 @@ export class KBService implements IGBKBService {
   }
 
   public static async getGroupReplies(instanceId: number): Promise<GuaribasQuestion[]> {
-      return await GuaribasQuestion.findAll({
+    return await GuaribasQuestion.findAll({
-        where: { from: 'group', instanceId: instanceId }
+      where: { from: 'group', instanceId: instanceId }
-      });
+    });
   }
 
   public async importKbTabularFile(
@@ -712,6 +724,28 @@ export class KBService implements IGBKBService {
     await this.undeployPackageFromStorage(instance, packageId);
   }
 
+  public static  async RefreshNER(min: GBMinInstance) {
+    const questions = await KBService.getQuestionsNER(min.instance.instanceId);
+    const contentLocale = min.core.getParam<string>(
+      min.instance,
+      'Default Content Language',
+      GBConfigService.get('DEFAULT_CONTENT_LANGUAGE')
+    );
+
+    await CollectionUtil.asyncForEach(questions, async question => {
+      const text = question.content;
+
+      let category = /.*\((.*)\).*/gi.exec(text)[1];
+      let name =/(\w+)\(.*\).*/gi.exec(text)[1];
+
+      min["nerEngine"].addNamedEntityText(category, name,
+        [contentLocale], [name]);
+
+    });
+
+
+  }
+
   /**
    * Deploys a knowledge base to the storage using the .gbkb format.
    *
@@ -730,6 +764,7 @@ export class KBService implements IGBKBService {
     await deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex));
 
     min['groupCache'] = await KBService.getGroupReplies(instance.instanceId);
+    await KBService.RefreshNER(min);
 
     GBLog.info(`[GBDeployer] Finished import of ${localPath}`);
   }