new(core.gbapp): FIND BASIC keyword is now much more faster.

This commit is contained in:
Rodrigo Rodriguez 2020-05-15 14:07:30 -03:00
parent daeaf8a8e5
commit 230a9e3cbc
11 changed files with 1121 additions and 1134 deletions

1872
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -67,9 +67,7 @@
"botbuilder-dialogs": "4.7.0",
"botframework-connector": "4.7.0",
"botlib": "1.5.1",
"chai": "4.2.0",
"cli-spinner": "0.2.10",
"csv-parse": "4.8.3",
"dotenv-extended": "2.7.1",
"exceljs": "3.5.0",
"express": "4.17.1",
@ -79,12 +77,10 @@
"js-beautify": "1.10.2",
"marked": "0.8.0",
"microsoft-cognitiveservices-speech-sdk": "1.11.0",
"mocha": "6.2.2",
"ms-rest-azure": "3.0.0",
"nexmo": "2.5.2",
"ngrok": "3.2.7",
"npm": "6.13.4",
"officeparser": "^2.1.1",
"opn": "6.0.0",
"pragmatismo-io-framework": "1.0.20",
"prism-media": "1.2.1",
@ -94,27 +90,24 @@
"request-promise": "4.2.5",
"request-promise-native": "1.0.8",
"rimraf": "3.0.0",
"safe-buffer": "^5.2.0",
"safe-buffer": "5.2.0",
"scanf": "1.1.1",
"sequelize": "5.21.5",
"sequelize-typescript": "1.1.0",
"shx": "0.3.2",
"simple-git": "1.129.0",
"sppull": "2.5.1",
"strict-password-generator": "1.1.2",
"swagger-client": "2.1.18",
"tedious": "6.6.5",
"typedoc": "0.15.6",
"textract": "2.5.0",
"typescript": "3.7.4",
"url-join": "4.0.1",
"vbscript-to-typescript": "1.0.8",
"wait-until": "0.0.2",
"walk-promise": "0.2.0",
"washyourmouthoutwithsoap": "1.0.2"
},
"devDependencies": {
"@types/chai": "4.2.7",
"@types/mocha": "5.2.7",
"typedoc": "0.15.6",
"@types/url-join": "4.0.0",
"@types/winston": "2.4.4",
"ban-sensitive-files": "1.9.2",

View file

@ -154,21 +154,22 @@ export class AdminDialog extends IGBDialog {
min.dialogs.add(
new WaterfallDialog('/publish', [
async step => {
const botId = min.instance.botId;
const locale = step.context.activity.locale;
await step.context.sendActivity(Messages[locale].working('Publishing'));
let unknownCommand = false;
step.activeDialog.state.options.args = (step.options as any).args;
let args = step.activeDialog.state.options.args.split(' ');
let filename = args[0];
const packages = [];
if (filename === null) {
await step.context.sendActivity(`Starting publishing for all bot packages...`);
packages.push(`${min.instance.botId}.gbkb`);
packages.push(`${min.instance.botId}.gbdialog`);
packages.push(`${min.instance.botId}.gbot`);
packages.push(`${min.instance.botId}.gbtheme`);
packages.push(`${min.instance.botId}.gbapp`);
packages.push(`${min.instance.botId}.gblib`);
packages.push(`${botId}.gbkb`);
packages.push(`${botId}.gbdialog`);
packages.push(`${botId}.gbot`);
packages.push(`${botId}.gbtheme`);
packages.push(`${botId}.gbapp`);
packages.push(`${botId}.gblib`);
} else {
await step.context.sendActivity(`Starting publishing for ${filename}...`);
packages.push(filename);
@ -178,7 +179,7 @@ export class AdminDialog extends IGBDialog {
await CollectionUtil.asyncForEach(packages, async packageName => {
const cmd1 = `deployPackage ${process.env.STORAGE_SITE} /${process.env.STORAGE_LIBRARY}/${min.instance.botId}.gbai/${packageName}`;
const cmd1 = `deployPackage ${process.env.STORAGE_SITE} /${process.env.STORAGE_LIBRARY}/${botId}.gbai/${packageName}`;
if (await (deployer as any).getStoragePackageByName(min.instance.instanceId,
packageName) !== null) { // TODO: Move to interface.
@ -193,7 +194,7 @@ export class AdminDialog extends IGBDialog {
await step.context.sendActivity(`Finished publishing ${packageName}.`);
});
return await step.replaceDialog('/ask', { firstRun: false });
return await step.replaceDialog('/ask', { isReturning: true });
} catch (error) {
await step.context.sendActivity(error.message);

View file

@ -42,7 +42,7 @@ import { AzureDeployerService } from '../../azuredeployer.gbapp/services/AzureDe
import { GBDeployer } from './GBDeployer';
const MicrosoftGraph = require("@microsoft/microsoft-graph-client");
import { Messages } from "../strings";
import { sys } from 'typescript';
import { GBServer } from '../../../src/app';
const request = require('request-promise-native');
/**
@ -160,69 +160,64 @@ class SysClass {
const path = `/${botId}.gbai/${botId}.gbdata`;
let res = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/root:${path}:/children`)
.get();
try {
// Performs validation.
let document = res.value.filter(m => {
return m.name === file
});
let res = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/root:${path}:/children`)
.get();
if (document === undefined) {
throw `File '${file}' specified on save GBasic command FIND not found. Check the .gbdata or the .gbdialog associated.`;
}
if (args.length > 1) {
throw `File '${file}' has a FIND call with more than 1 arguments. Check the .gbdialog associated.`;
}
// Performs validation.
// Creates workbook session that will be discarded.
let document = res.value.filter(m => {
return m.name === file
});
const columnName = "CPF";
const value = "99988877766";
let body = { "persistChanges": false };
const session = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/items/${document[0].id}/workbook/createSession`)
.post(body);
// Applies filtering.
const bodyFilter = {
criteria:
{
filterOn: "Custom",
criterion1: `=${value}`
if (document === undefined) {
throw `File '${file}' specified on save GBasic command FIND not found. Check the .gbdata or the .gbdialog associated.`;
}
if (args.length > 1) {
throw `File '${file}' has a FIND call with more than 1 arguments. Check the .gbdialog associated.`;
}
};
await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/items/${document[0].id}/workbook/worksheets('Sheet1')/tables(id='Table1')/columns('${columnName}')/filter/apply`)
.headers("workbook-session-id", session.id)
.post(bodyFilter);
// Creates workbook session that will be discarded.
// Get the filtered values.
const filter = args[0].split('=');
const columnName = filter[0];
const value = filter[1];
let results = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/items/${document[0].id}/workbook/worksheets('Sheet1')/range(address='A1:Z100')`)
.get();
let results = await client.api(
`https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}/drive/items/${document[0].id}/workbook/worksheets('Sheet1')/tables('Table1')/range/visibleView/rows?$select=values`)
.headers("workbook-session-id", session.id)
.get();
let columnIndex = 0;
const header = results.text[0];
for (; columnIndex < header.length; columnIndex++) {
if (header[columnIndex] === columnName) {
break;
}
}
// Translate an array into a readable BASIC object.
let output = {};
results.value[1].values[0]
const firstRow = results.value[0];
for (let index = 0; index < firstRow.values[0].length; index++) {
output[firstRow.values[0][index]] = results.value[1].values[0][index];
}
return output;
let foundIndex = 0;
for (; foundIndex < results.text.length; foundIndex++) {
if (results.text[foundIndex][columnIndex] === value) {
break;
}
}
if (foundIndex === results.text.length) {
return null;
}
else {
let output = {};
const row = results.text[foundIndex];
for (let colIndex = 0; colIndex < row.length; colIndex++) {
output[header[colIndex]] = row[colIndex];
}
return output;
}
} catch (error) {
GBLog.error(error);
}
}
public generatePassword() {
@ -351,6 +346,12 @@ export class DialogClass {
}
}
public async sendFile(step, filename, caption) {
let url = urlJoin(GBServer.globals.publicAddress, 'kb', this.min.botId + '.gbkb', 'assets', filename);
await this.min.conversationalService.sendFile(this.min, step,
null, url, caption);
}
public async getFrom(step) {
return step.context.activity.from.id;
}

View file

@ -91,10 +91,14 @@ export class GBConversationalService {
public async sendFile(min: GBMinInstance, step: GBDialogStep, mobile: string, url: string, caption: string): Promise<any> {
if (step !== null) {
mobile = step.context.activity.from.id;
if (step.context.activity.channelId === 'whatsapp') {
const filename = url.substring(url.lastIndexOf('/') + 1);
await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename, caption);
}
else {
await step.context.sendActivity(url);
}
}
const filename = url.substring(url.lastIndexOf('/') + 1);
await min.whatsAppDirectLine.sendFileToDevice(mobile, url, filename, caption);
}
public async sendAudio(min: GBMinInstance, step: GBDialogStep, url: string): Promise<any> {

View file

@ -39,7 +39,6 @@
const Path = require('path');
import urlJoin = require('url-join');
const Fs = require('fs');
const WaitUntil = require('wait-until');
const express = require('express');
const child_process = require('child_process');
const graph = require('@microsoft/microsoft-graph-client');
@ -92,67 +91,63 @@ export class GBDeployer implements IGBDeployer {
public async deployPackages(core: IGBCoreService, server: any, appPackages: IGBPackage[]) {
const _this = this;
return new Promise(
async (resolve: any, reject: any) => {
GBLog.info(`PWD ${process.env.PWD}...`);
let totalPackages = 0;
let paths = [urlJoin(process.env.PWD, GBDeployer.deployFolder), urlJoin(process.env.PWD, GBDeployer.workFolder)];
const additionalPath = GBConfigService.get('ADDITIONAL_DEPLOY_PATH');
if (additionalPath !== undefined && additionalPath !== '') {
paths = paths.concat(additionalPath.toLowerCase().split(';'));
GBLog.info(`PWD ${process.env.PWD}...`);
let totalPackages = 0;
let paths = [urlJoin(process.env.PWD, GBDeployer.deployFolder), urlJoin(process.env.PWD, GBDeployer.workFolder)];
const additionalPath = GBConfigService.get('ADDITIONAL_DEPLOY_PATH');
if (additionalPath !== undefined && additionalPath !== '') {
paths = paths.concat(additionalPath.toLowerCase().split(';'));
}
const botPackages: string[] = [];
const gbappPackages: string[] = [];
let generalPackages: string[] = [];
async function scanPackageDirectory(path) {
const isDirectory = source => Fs.lstatSync(source).isDirectory();
const getDirectories = source =>
Fs.readdirSync(source)
.map(name => Path.join(source, name))
.filter(isDirectory);
const dirs = getDirectories(path);
await CollectionUtil.asyncForEach(dirs, async element => {
if (element === '.') {
GBLog.info(`Ignoring ${element}...`);
} else {
if (element.endsWith('.gbot')) {
botPackages.push(element);
} else if (element.endsWith('.gbapp')) {
gbappPackages.push(element);
} else {
generalPackages.push(element);
}
}
const botPackages: string[] = [];
const gbappPackages: string[] = [];
let generalPackages: string[] = [];
});
}
async function scanPackageDirectory(path) {
const isDirectory = source => Fs.lstatSync(source).isDirectory();
const getDirectories = source =>
Fs.readdirSync(source)
.map(name => Path.join(source, name))
.filter(isDirectory);
GBLog.info(`Starting looking for packages (.gbot, .gbtheme, .gbkb, .gbapp)...`);
await CollectionUtil.asyncForEach(paths, async e => {
GBLog.info(`Looking in: ${e}...`);
await scanPackageDirectory(e);
});
const dirs = getDirectories(path);
await CollectionUtil.asyncForEach(dirs, async element => {
if (element === '.') {
GBLog.info(`Ignoring ${element}...`);
} else {
if (element.endsWith('.gbot')) {
botPackages.push(element);
} else if (element.endsWith('.gbapp')) {
gbappPackages.push(element);
} else {
generalPackages.push(element);
}
}
});
}
// Deploys all .gbapp files first.
GBLog.info(`Starting looking for packages (.gbot, .gbtheme, .gbkb, .gbapp)...`);
await CollectionUtil.asyncForEach(paths, async e => {
GBLog.info(`Looking in: ${e}...`);
await scanPackageDirectory(e);
});
await this.deployAppPackages(gbappPackages, core, appPackages);
// Deploys all .gbapp files first.
GBLog.info(`App Package deployment done.`);
const appPackagesProcessed = await this.deployAppPackages(gbappPackages, core, appPackages);
({ generalPackages, totalPackages } = await this.deployDataPackages(
GBLog.info(`App Package deployment done.`);
core,
botPackages,
_this,
generalPackages,
server,
totalPackages
));
({ generalPackages, totalPackages } = await this.deployDataPackages(
core,
botPackages,
_this,
generalPackages,
server,
reject,
totalPackages,
resolve
));
}
);
}
public async deployBlankBot(botId: string) {
@ -317,7 +312,7 @@ export class GBDeployer implements IGBDeployer {
case '.gbtheme':
const packageName = Path.basename(localPath);
GBServer.globals.server.use(`/themes/${packageName}`, express.static(packageName));
GBServer.globals.server.use(`/themes/${packageName}`, express.static(localPath));
GBLog.info(`Theme (.gbtheme) assets accessible at: /themes/${packageName}.`);
break;
@ -446,9 +441,8 @@ export class GBDeployer implements IGBDeployer {
_this: this,
generalPackages: string[],
server: any,
reject: any,
totalPackages: number,
resolve: any
totalPackages: number
) {
try {
await core.syncDatabaseStructure();
@ -489,27 +483,13 @@ export class GBDeployer implements IGBDeployer {
} else {
// Unknown package format.
const err = new Error(`Package type not handled: ${filename}.`);
reject(err);
throw err;
}
totalPackages++;
});
WaitUntil()
.interval(100)
.times(5)
.condition(cb => {
GBLog.info(`Waiting for package deployment...`);
cb(totalPackages === generalPackages.length);
})
.done(() => {
if (botPackages.length === 0) {
GBLog.info('Use ADDITIONAL_DEPLOY_PATH to point to a .gbai package folder (no external packages).');
} else {
GBLog.info(`Package deployment done.`);
}
resolve();
});
GBLog.info(`Package deployment done.`);
return { generalPackages, totalPackages };
}

View file

@ -210,7 +210,8 @@ export class GBMinService {
const { min, adapter, conversationState } = await this.buildBotAdapter(instance, GBServer.globals.sysPackages);
GBServer.globals.minInstances.push(min);
this.deployer.deployPackage(min, 'packages/default.gbdialog');
await this.deployer.deployPackage(min, 'packages/default.gbdialog');
await this.deployer.deployPackage(min, 'packages/default.gbtheme');
// Install per bot deployed packages.

View file

@ -43,10 +43,11 @@ const walkPromise = require('walk-promise');
const vm = require('vm');
import urlJoin = require('url-join');
import { DialogClass } from './GBAPIService';
import { Messages } from '../strings';
//tslint:disable-next-line:no-submodule-imports
const vb2ts = require('vbscript-to-typescript/dist/converter');
const beautify = require('js-beautify').js;
const parseOffice = require('bluebird').promisify(require('officeparser').parseOffice);
var textract = require('textract');
/**
* @fileoverview Virtualization services for emulation of BASIC.
@ -70,7 +71,7 @@ export class GBVMService extends GBService {
return Promise.all(
files.map(async file => {
let filename:string = file.name;
let filename: string = file.name;
if (
filename.endsWith('.vbs') ||
@ -81,21 +82,19 @@ export class GBVMService extends GBService {
) {
if (filename.endsWith('.docx')) {
const text = await parseOffice(filename);
filename = filename.substr(0, file.indexOf('docx')) + '.vbs';
let text = await this.getTextFromWord(folder, filename);
filename = filename.substr(0, filename.indexOf('docx')) + 'vbs';
fs.writeFileSync(urlJoin(folder, filename), text);
}
const mainName = filename.replace(/\-|\./g, '');
const mainName = filename.replace(/\s|\-/g, '').split('.')[0];
min.scriptMap[filename] = mainName;
const fullFilename = urlJoin(folder, filename);
fs.watchFile(fullFilename, async () => {
await this.run(fullFilename, min, deployer, mainName);
});
// TODO: Implement in development mode, how swap for .vbs files
// fs.watchFile(fullFilename, async () => {
// await this.run(fullFilename, min, deployer, mainName);
// });
await this.run(fullFilename, min, deployer, mainName);
}
@ -103,6 +102,20 @@ export class GBVMService extends GBService {
);
}
private async getTextFromWord(folder: string, filename: string) {
return new Promise<string>(async (resolve, reject) => {
textract.fromFileWithPath(urlJoin(folder, filename), { preserveLineBreaks: true },
(error, text) => {
if (error) {
reject(error);
}
else {
resolve(text);
}
});
});
}
/**
* Converts General Bots BASIC
*
@ -152,6 +165,9 @@ export class GBVMService extends GBService {
code = code.replace(/(talk)(\s)(.*)/g, ($0, $1, $2, $3) => {
return `talk (step, ${$3})\n`;
});
code = code.replace(/(send file)(\s*)(.*)/g, ($0, $1, $2, $3) => {
return `sendFile (step, ${$3})\n`;
});
code = code.replace(/(save)(\s)(.*)/g, ($0, $1, $2, $3) => {
return `sys().save(${$3})\n`;
@ -282,6 +298,10 @@ export class GBVMService extends GBService {
code = code.replace(/("[^"]*"|'[^']*')|\baskEmail\b/g, ($0, $1) => {
return $1 === undefined ? 'this.askEmail' : $1;
});
code = code.replace(/("[^"]*"|'[^']*')|\bsendFile\b/g, ($0, $1) => {
return $1 === undefined ? 'this.sendFile' : $1;
});
// await insertion.
@ -305,9 +325,17 @@ export class GBVMService extends GBService {
const promise = min.cbMap[cbId].promise;
delete min.cbMap[cbId];
const opts = await promise(step, step.result);
try {
const opts = await promise(step, step.result);
return await step.replaceDialog('/hear', opts);
return await step.replaceDialog('/hear', opts);
} catch (error) {
GBLog.error(`Error running BASIC code: ${error}`);
const locale = step.context.activity.locale;
step.context.sendActivity(Messages[locale].very_sorry_about_error);
return await step.replaceDialog('/ask', { isReturning: true });
}
}
])
);

View file

@ -79,10 +79,15 @@ export class TSCompiler {
const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n');
if (diagnostic.file !== undefined) {
const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
GBLog.error(`${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`);
if (
diagnostic.file.fileName.indexOf('readable-stream') == -1 &&
diagnostic.file.fileName.indexOf('request-promise') == -1
) {
const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
GBLog.warn(`BASIC error: ${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`);
}
} else {
GBLog.error(`${message}`);
GBLog.warn(`BASIC error: ${message}`);
}
}
});

View file

@ -1,4 +1,30 @@
cpf = "999888777-66"
Linha = find "Cadastro.xlsx", "CPF=" + cpf
talk Linha.Nome
talk "Para conseguir te ajudar, você quer informar o CPF ou código?"
hear consulta
if consulta = "cpf" then
talk "Qual seu CPF?"
hear cpf
talk "Aguarde alguns instantes que eu localizo seu cadastro..."
row = find "Cadastro.xlsx", "CPF=" + cpf
if row != null then
talk "Oi, " + row.Nome + "Tudo bem? "
talk "Seu código de cliente é " + row.Cod
talk "Vamos te enviar o pedido para seu endereço em: " + row.Endereço
send file "boleta.pdf", "Pague já e evite multas!"
else
talk "Tente novamente."
end if
else
talk "Qual seria seu código?"
hear cod
talk "Aguarde alguns instantes que eu localizo seu cadastro..."
row = find "Cadastro.xlsx", "Cod=" + cod
if row != null then
talk "Oi, " + row.Nome + "Tudo bem? "
talk "Seu CPF é " + row.CPF
talk "Vamos te enviar o pedido para seu endereço em: " + row.Endereço
send file "boleta.pdf", "Pague já e evite multas!"
else
talk "Tente novamente."
end if
end if

View file

@ -34,7 +34,6 @@
* @fileoverview Knowledge base services and logic.
*/
var Excel = require('exceljs');
const Path = require('path');
const Fs = require('fs');
const urlJoin = require('url-join');
@ -43,9 +42,8 @@ const path = require('path');
const asyncPromise = require('async-promises');
const walkPromise = require('walk-promise');
// tslint:disable-next-line:newline-per-chained-call
const parse = require('bluebird').promisify(require('csv-parse'));
const { SearchService } = require('azure-search-client');
var Excel = require('exceljs');
import { IGBKBService, GBDialogStep, GBLog, IGBConversationalService, IGBCoreService, IGBInstance, GBMinInstance } from 'botlib';
import { Op } from 'sequelize';
import { Sequelize } from 'sequelize-typescript';