new(all) General Bots Auto Tester 1.0; Unlimited conditionals after-code BASIC 3.0;

package.json

@@ -99,6 +99,9 @@
     "ibm-watson": "6.1.1",
     "js-beautify": "1.13.13",
     "keyv": "^4.5.0",
+    "koa": "^2.13.4",
+    "koa-body": "^5.0.0",
+    "koa-router": "^12.0.0",
     "lodash": "^4.17.21",
     "luxon": "2.0.2",
     "mammoth": "^1.4.21",
@@ -119,6 +122,7 @@
     "pragmatismo-io-framework": "1.0.20",
     "prism-media": "1.3.1",
     "public-ip": "4.0.4",
+    "punycode": "^2.1.1",
     "puppeteer": "13.7.0",
     "puppeteer-extra": "^3.3.4",
     "puppeteer-extra-plugin-stealth": "2.4.5",
@@ -145,13 +149,15 @@
     "textract": "2.5.0",
     "twitter-api-v2": "1.12.7",
     "typescript": "3.6.4",
+    "typescript-rest-rpc": "^1.0.10",
     "url-join": "4.0.1",
     "vbscript-to-typescript": "1.0.8",
     "vhost": "^3.0.2",
     "vm2": "^3.9.11",
+    "vm2-process": "^2.1.1",
     "walk-promise": "0.2.0",
     "washyourmouthoutwithsoap": "1.0.2",
-    "whatsapp-web.js": "github:pedroslopez/whatsapp-web.js#fix-buttons-list",
+    "whatsapp-web.js": "1.18.0",
     "winston": "^2.4.2",
     "winston-logs-display": "^1.0.0",
     "yarn": "^1.22.19"

packages/azuredeployer.gbapp/services/AzureDeployerService.ts

@@ -53,8 +53,6 @@ const MicrosoftGraph = require("@microsoft/microsoft-graph-client");
 
 const Spinner = require('cli-spinner').Spinner;
 // tslint:disable-next-line: no-submodule-imports
-import * as simplegit from 'simple-git/promise';
-const git = simplegit();
 
 // tslint:disable-next-line:no-submodule-imports
 import { CognitiveServicesAccount } from 'azure-arm-cognitiveservices/lib/models';

packages/basic.gblib/index.ts

@@ -39,16 +39,25 @@
 import { GBDialogStep, GBLog, GBMinInstance, IGBCoreService, IGBPackage } from 'botlib';
 import { GuaribasSchedule } from '../core.gbapp/models/GBModel';
 import { Sequelize } from 'sequelize-typescript';
-
+import { createServerRouter } from "typescript-rest-rpc/lib/server"
+import { DialogKeywords } from './services/DialogKeywords';
+const Koa = require('koa');
+import * as koaBody from "koa-body"
+const app = new Koa()
 
 /**
  * Package for core.gbapp.
  */
+ 
 export class GBBasicPackage implements IGBPackage {
   public sysPackages: IGBPackage[];
   public CurrentEngineName = "guaribas-1.0.0";
+
+
   public async loadPackage(core: IGBCoreService, sequelize: Sequelize): Promise<void> {
     core.sequelize.addModels([GuaribasSchedule]);
+    app.use(koaBody({ multipart: true }));
+    app.listen(1111);
   }
 
   public async getDialogs(min: GBMinInstance) {
@@ -67,6 +76,8 @@ export class GBBasicPackage implements IGBPackage {
     GBLog.verbose(`onExchangeData called.`);
   }
   public async loadBot(min: GBMinInstance): Promise<void> {
-
+    const backendRouter = createServerRouter(`/api/v2/${min.botId}/dialog`, new DialogKeywords(min, null, null, null));
+    app.use(backendRouter.routes());
+    
   }
 }

packages/basic.gblib/services/DialogKeywords.ts

@@ -437,7 +437,7 @@ export class DialogKeywords {
    *
    * @example x = TODAY
    */
-  public async getToday(step) {
+  public async getToday() {
     let d = new Date(),
       month = '' + (d.getMonth() + 1),
       day = '' + d.getDate(),

packages/basic.gblib/services/GBVMService.ts

@@ -42,8 +42,8 @@ import { DialogKeywords } from './DialogKeywords';
 import { ScheduleServices } from './ScheduleServices';
 import { GBConfigService } from '../../core.gbapp/services/GBConfigService';
 //tslint:disable-next-line:no-submodule-imports
-const { VM } = require('vm2');
-
+const { NodeVM, VMScript } = require('vm2');
+const { createVm2Pool } = require('./vm2-process/index');
 
 const vb2ts = require('./vbscript-to-typescript');
 const beautify = require('js-beautify').js;
@@ -776,27 +776,70 @@ export class GBVMService extends GBService {
       }
     }
 
-    // Injects the .gbdialog generated code into the VM.
+    const botId = min.botId;
+    const gbdialogPath = urlJoin(process.cwd(), 'work', `${botId}.gbai`, `${botId}.gbdialog`);
+    const scriptPath = urlJoin(gbdialogPath, `${text}.js`);
 
     let code = min.sandBoxMap[text];
 
-    code = `(async () => {${code}})();`;
-    sandbox['this'] = sandbox;
-    const vm = new VM({
-      timeout: 600000,
-      allowAsync: true,
-      sandbox: { dk: sandbox }
-    });
+    if (process.env.VM3) {
+      try {
 
-    // Calls the function.
+        const vm1 = new NodeVM({
+          allowAsync: true,
+          sandbox: {},
+          console: 'inherit',
+          wrapper: 'commonjs',
+          require: {
+            builtin: ['stream', 'http', 'https', 'url', 'zlib'],
+            root: ['./'],
+            external: true,
+            context: 'sandbox'
+          },
+        });
+        const s = new VMScript(code, { filename: scriptPath });
+        let x = vm1.run(s);
+        return x;
+      } catch (error) {
+        throw new Error(`BASIC RUNTIME ERR: ${error.message ? error.message : error}\n Stack:${error.stack}`);
+      }
 
-    let ret = null;
-    try {
-      vm.run(code)
-    } catch (error) {
-      throw new Error(`BASIC RUNTIME ERR: ${error.message ? error.message : error}\n Stack:${error.stack}`);
+    } else {
+
+      // {
+      //   "name": "dev-rodriguez.gbdialog",
+      //   "version": "1.0.0",
+      //   "description": "",
+      //    "author": "",
+      //   "license": "ISC",
+      //   "dependencies": {
+      //     "encoding": "^0.1.13",
+      //     "isomorphic-fetch": "^3.0.0",
+      //     "punycode": "^2.1.1",
+      //     "typescript-rest-rpc": "^1.0.10",
+      //     "vm2": "^3.9.11"
+      //   }
+      // }
+
+      const runnerPath = urlJoin(process.cwd(), 'dist', 'packages', 'basic.gblib', 'services', 'vm2-process', 'vm2ProcessRunner.js');
+
+      try {
+        const { run, drain } = createVm2Pool({
+          min: 1,
+          max: 1,
+          cpu: 100,
+          memory: 120000,
+          time: 5520000,
+          cwd: gbdialogPath,
+          script: runnerPath
+        });
+
+        const result = await run(code, { filename: scriptPath });
+
+        drain();
+      } catch (error) {
+        throw new Error(`BASIC RUNTIME ERR: ${error.message ? error.message : error}\n Stack:${error.stack}`);
+      }
     }
-
-    return ret;
   }
 }

packages/basic.gblib/services/vm2-process/index.ts

@@ -0,0 +1,119 @@
+const crypto2 = require('crypto');
+const Fs = require('fs');
+const Path = require('path');
+const { spawn } = require('child_process');
+
+const {  } = require('child_process');
+const { dirname } = require('path');
+const { fileURLToPath } = require('url');
+const net = require('net');
+
+const genericPool = require('generic-pool');
+const finalStream = require('final-stream');
+
+const waitUntil = (condition) => {
+  if (condition()) {
+    return Promise.resolve();
+  }
+
+  return new Promise((resolve) => {
+    const interval = setInterval(() => {
+      if (!condition()) {
+        return;
+      }
+
+      clearInterval(interval);
+      resolve(0);
+    }, 0);
+  });
+};
+
+const createVm2Pool = ({ min, max, ...limits }) => {
+  limits = Object.assign({
+    cpu: 100,
+    memory: 2000,
+    time: 4000
+  }, limits);
+
+  let limitError = null;
+
+  const ref = crypto2.randomBytes(20).toString('hex');
+
+  const kill = (x) => {
+    spawn('sh', ['-c', `pkill -9 -f ${ref}`]);
+  };
+
+  let stderrCache = '';
+  const factory = {
+    create: function () {
+      
+      const runner = spawn('cpulimit', [
+        '-ql', limits.cpu,
+        '--',
+        'node', `--experimental-fetch`, `--max-old-space-size=${limits.memory}`, 
+          limits.script
+        , ref
+      ], { cwd: limits.cwd, shell: false });
+
+      runner.stdout.on('data', (data) => {
+        runner.socket = runner.socket || data.toString().trim();
+      });
+
+      runner.stderr.on('data', (data) => {
+        stderrCache = stderrCache + data.toString();
+        if (stderrCache.includes('FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory')) {
+          limitError = 'code execution exceeed allowed memory';
+        }
+      });
+
+      return runner;
+    },
+
+    destroy: function (childProcess) {
+      kill(childProcess);
+    }
+  };
+
+  const pool = genericPool.createPool(factory, { min, max });
+
+  const run = async (code, scope) => {
+    const childProcess = await pool.acquire();
+
+    await waitUntil(() => childProcess.socket);
+
+    const socket = net.createConnection(childProcess.socket);
+
+    const timer = setTimeout(() => {
+      limitError = 'code execution took too long and was killed';
+      kill(childProcess);
+    }, limits.time);
+
+    socket.write(JSON.stringify({ code, scope }) + '\n');
+
+    try {
+      let data = await finalStream(socket);
+      
+      data = JSON.parse(data)
+
+      if (data.error) {
+        throw new Error(data.error);
+      }
+
+      return data.result;
+    } catch (error) {
+      throw new Error(limitError || error);
+    } finally {
+      clearTimeout(timer);
+      pool.destroy(childProcess);
+    }
+  };
+
+  return {
+    run,
+    drain: () => {
+      pool.drain().then(() => pool.clear());
+    }
+  };
+};
+
+exports.createVm2Pool = createVm2Pool;

packages/basic.gblib/services/vm2-process/vm2ProcessRunner.ts

@@ -0,0 +1,57 @@
+const { VMScript, NodeVM } = require('vm2');
+const crypto1 = require('crypto');
+const net1 = require('net');
+
+const evaluate = (script, scope) => {
+  const vm = new NodeVM({
+    allowAsync: true,
+    sandbox: {},
+    console: 'inherit',
+    wrapper: 'none',
+    require: {
+      builtin:['stream', 'http' , 'https', 'url', 'buffer', 'zlib', 'isomorphic-fetch', 'punycode', 'encoding'],
+      root: ['./'],
+      external: true,
+      context: 'sandbox'
+    },
+  });
+  
+  const s = new VMScript(script, scope);
+  return vm.run(script, scope);
+};
+
+const socketName = crypto1.randomBytes(20).toString('hex');
+
+const server = net1.createServer((socket) => {
+  const buffer = [];
+
+  const sync = () => {
+    const request = buffer.join('').toString();
+    if (request.includes('\n')) {
+      try {
+        const { code, scope } = JSON.parse(request);
+        const result = evaluate(code, {
+          ...scope,
+          module: null
+        });
+
+        socket.write(JSON.stringify({ result }) + '\n');
+        socket.end();
+      } catch (error) {
+        socket.write(JSON.stringify({ error: error.message }) + '\n');
+        socket.end();
+      }
+    }
+  };
+
+  socket.on('data', data => {
+    buffer.push(data);
+    sync();
+  });
+});
+
+server.on('listening', () => {
+  console.log(`/tmp/vm2-${socketName}.sock`);
+});
+
+server.listen(`/tmp/vm2-${socketName}.sock`);

packages/core.gbapp/services/GBMinService.ts

@@ -220,11 +220,11 @@ export class GBMinService {
       this.bar1.stop();
     }
 
-    // Loads schedules.
-    GBLog.info(`Preparing SET SCHEDULE dialog calls...`);
+    // // Loads schedules.
+    // GBLog.info(`Preparing SET SCHEDULE dialog calls...`);
 
-    const service = new ScheduleServices();
-    await service.scheduleAll();
+    // const service = new ScheduleServices();
+    // await service.scheduleAll();
 
     GBLog.info(`All Bot instances loaded.`);
   }
@@ -803,6 +803,12 @@ export class GBMinService {
   // TODO: Unify in util.
   public static userMobile(step) {
     let mobile = WhatsappDirectLine.mobiles[step.context.activity.conversation.id]
+
+    if (!mobile && step)
+    {
+      return step.context.activity.from.id;
+    }
+
     return mobile;
 
   }

packages/whatsapp.gblib/services/WhatsappDirectLine.ts

@@ -290,7 +290,7 @@ export class WhatsappDirectLine extends GBService {
         break;
     }
 
-    if (setUrl && options) {
+    if (setUrl && options&& this.whatsappServiceUrl) {
       const express = require('express');
       GBServer.globals.server.use(`/audios`, express.static('work'));
 

src/app.ts

@@ -82,13 +82,16 @@ export class GBServer {
    */
 
   public static run() {
-    
+
 
     GBLog.info(`The Bot Server is in STARTING mode...`);
     GBServer.globals = new RootData();
     GBConfigService.init();
     const port = GBConfigService.getServerPort();
+
+
     const server = express();
+
     GBServer.globals.server = server;
     GBServer.globals.appPackages = [];
     GBServer.globals.sysPackages = [];
@@ -207,15 +210,16 @@ export class GBServer {
 
             // ... some not authenticated middlewares
 
-            server.use((req, res, next) => {
+            server.use(async (req, res, next) => {
               if (req.originalUrl.startsWith('/logs')) {
                 var user = auth(req);
                 if (!user || !admins[user.name] || admins[user.name].password !== user.pass) {
                   res.set('WWW-Authenticate', 'Basic realm="example"');
                   return res.status(401).send();
                 }
+              } else {
+                return next();
               }
-              return next();
             });
 
             // If global log enabled, reorders transports adding web logging.
@@ -226,7 +230,7 @@ export class GBServer {
 
 
           GBLog.info(`The Bot Server is in RUNNING mode...`);
-          
+
           // Opens Navigator.
 
           // TODO: Config: core.openBrowserInDevelopment();

tsconfig.json

@@ -8,9 +8,9 @@
     "experimentalDecorators": true,
     "skipLibCheck": true,
     "mapRoot": "./dist/",
-    "module": "commonjs",
-    "moduleResolution": "node",
-    "resolveJsonModule": true,
+    "module": "CommonJS",
+    "moduleResolution": "Node",
+    "resolveJsonModule": false,
     "outDir": "./dist",
     "paths": {
       "*": [