fix(core.gbapp): Update of all packages and nodejs to 19.1.0.

This commit is contained in:
rodrigorodriguez 2022-11-19 19:50:19 -03:00
parent a315d21cfe
commit f8d2cd895a
6 changed files with 8216 additions and 25936 deletions

View file

@ -26,7 +26,7 @@ try {
var run = () => {
console.log(`[GB Runtime] Initializing General Bots (BotServer)...`);
console.log(`[GB Runtime] ${now()} - Running '${GBServer.name}' on '${import.meta.url}'`);
console.log(`[GB Runtime] ${now()} - Running on '${import.meta.url}'`);
GBServer.GBServer.run();
}
var processDist = () => {

33851
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -51,7 +51,9 @@
},
"dependencies": {
"@azure/cognitiveservices-computervision": "8.2.0",
"@azure/keyvault-keys": "4.6.0",
"@azure/ms-rest-js": "2.6.2",
"@azure/search-documents": "^11.3.1",
"@google-cloud/pubsub": "3.2.1",
"@google-cloud/translate": "7.0.4",
"@hubspot/api-client": "7.1.2",
@ -67,12 +69,6 @@
"alasql": "2.1.6",
"any-shell-escape": "0.1.1",
"async-promises": "0.2.3",
"azure-arm-cognitiveservices": "3.0.0",
"azure-arm-resource": "7.4.0",
"azure-arm-search": "1.3.0-preview",
"azure-arm-sql": "5.7.0",
"azure-arm-website": "5.7.0",
"azure-search-client": "3.1.5",
"basic-auth": "2.0.1",
"billboard.js": "3.6.3",
"bluebird": "3.7.2",
@ -110,14 +106,14 @@
"luxon": "3.1.0",
"mammoth": "1.5.1",
"marked": "4.2.2",
"momentjs": "2.0.0",
"moment": "^1.3.0",
"ms-rest-azure": "3.0.2",
"nexmo": "2.9.1",
"node-cron": "3.0.2",
"node-nlp": "3.10.2",
"node-nlp": "^4.24.0",
"node-tesseract-ocr": "2.2.1",
"npm": "9.1.2",
"opn": "6.0.0",
"open": "8.4.0",
"pdf-extraction": "1.0.2",
"pdfkit": "0.13.0",
"phone": "3.1.30",
@ -134,26 +130,23 @@
"qrcode-terminal": "0.12.0",
"readline": "1.3.0",
"reflect-metadata": "0.1.13",
"request-promise": "4.2.5",
"request-promise-native": "1.0.8",
"rimraf": "3.0.2",
"safe-buffer": "5.2.1",
"scanf": "1.1.2",
"sequelize": "6.25.6",
"sequelize": "6.25.7",
"sequelize-cli": "6.5.2",
"sequelize-typescript": "2.1.5",
"simple-git": "3.15.0",
"speakingurl": "14.0.1",
"sppull": "3.0.1",
"ssr-for-bots": "1.0.1-c",
"strict-password-generator": "1.1.2",
"swagger-client": "2.1.18",
"swagger-client": "^3.18.5",
"tabulator-tables": "5.4.2",
"tedious": "15.1.2",
"textract": "2.5.0",
"textract": "^0.20.0",
"twitter-api-v2": "1.12.9",
"typescript": "4.9.3",
"typescript-rest-rpc": "1.0.10",
"typescript-rest-rpc": "^1.0.7",
"url-join": "5.0.0",
"vbscript-to-typescript": "1.0.8",
"vhost": "3.0.2",
@ -167,23 +160,21 @@
"yarn": "1.22.19"
},
"devDependencies": {
"@types/puppeteer": "5.4.7",
"@types/puppeteer": "7.0.4",
"@types/url-join": "4.0.1",
"@types/winston": "2.4.4",
"ban-sensitive-files": "1.9.18",
"ban-sensitive-files": "^1.3.0",
"commitizen": "4.2.5",
"cz-conventional-changelog": "3.3.0",
"dependency-check": "4.1.0",
"git-issues": "1.3.1",
"git-issues": "^1.0.0",
"license-checker": "25.0.1",
"ngrok": "4.3.3",
"nsp": "3.2.1",
"prettier-standard": "16.4.1",
"prettier-standard": "^15.0.1",
"semantic-release": "19.0.5",
"simple-commit-message": "4.1.3",
"travis-deploy-once": "5.0.11",
"simple-commit-message": "^1.1.0",
"travis-deploy-once": "^3.0.0",
"ts-node": "10.9.1",
"tslint": "6.1.2"
"tslint": "6.1.3"
},
"eslintConfig": {
"env": {

View file

@ -38,18 +38,19 @@
import urlJoin from 'url-join';
import { HttpMethods, ServiceClient, WebResource } from '@azure/ms-rest-js';
import { CognitiveServicesManagementClient } from 'azure-arm-cognitiveservices';
import { ResourceManagementClient, SubscriptionClient } from 'azure-arm-resource';
import { SearchManagementClient } from 'azure-arm-search';
import { SqlManagementClient } from 'azure-arm-sql';
import { WebSiteManagementClient } from 'azure-arm-website';
import { AppServicePlan, Site, SiteLogsConfig, SiteSourceControl } from 'azure-arm-website/lib/models/index.js';
import { CognitiveServicesManagementClient } from '@azure/arm-cognitiveservices';
import { ResourceManagementClient } from '@azure/arm-resources';
import { SubscriptionClient } from '@azure/arm-subscriptions';
import { SearchManagementClient } from '@azure/arm-search';
import { SqlManagementClient } from '@azure/arm-sql';
import { WebSiteManagementClient } from '@azure/arm-appservice';
import { AppServicePlan, Site, SiteLogsConfig, SiteSourceControl } from '@azure/arm-appservice';
import { GBLog, IGBInstallationDeployer, IGBInstance, IGBDeployer, IGBCoreService } from 'botlib';
import { GBAdminService } from '../../../packages/admin.gbapp/services/GBAdminService.js';
import { GBCorePackage } from '../../../packages/core.gbapp/index.js';
import { GBConfigService } from '../../../packages/core.gbapp/services/GBConfigService.js';
import { GBDeployer } from '../../../packages/core.gbapp/services/GBDeployer.js';
import { CognitiveServicesAccount } from 'azure-arm-cognitiveservices/lib/models/index.js';
import { Account } from '@azure/arm-cognitiveservices';
import MicrosoftGraph from "@microsoft/microsoft-graph-client";
import Spinner from 'cli-spinner';
import * as publicIp from 'public-ip';
@ -64,13 +65,13 @@ export class AzureDeployerService implements IGBInstallationDeployer {
public apiVersion = '2017-12-01';
public defaultEndPoint = 'http://localhost:4242';
public instance: IGBInstance;
public cloud: ResourceManagementClient.ResourceManagementClient;
public cloud: ResourceManagementClient;
public webSiteClient: WebSiteManagementClient;
public storageClient: SqlManagementClient;
public cognitiveClient: CognitiveServicesManagementClient;
public searchClient: SearchManagementClient;
public provider = 'Microsoft.BotService';
public subscriptionClient: SubscriptionClient.SubscriptionClient;
public subscriptionClient: SubscriptionClient;
public accessToken: string;
public location: string;
public subscriptionId: string;
@ -115,7 +116,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
}
public async getSubscriptions(credentials) {
const subscriptionClient = new SubscriptionClient.default(credentials);
const subscriptionClient = new SubscriptionClient(credentials);
return subscriptionClient.subscriptions.list();
}
@ -341,7 +342,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
const subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID');
const credentials = await GBAdminService.getADALCredentialsFromUsername(username, password);
const storageClient = new SqlManagementClient(credentials, subscriptionId);
const storageClient = new SqlManagementClient(credentials as any, subscriptionId);
const ip = await publicIp.publicIpv4();
let params = {
@ -407,21 +408,21 @@ export class AzureDeployerService implements IGBInstallationDeployer {
GBLog.info(`Deploying Speech...`);
const speech = await this.createSpeech(name, `${name}speech`, instance.cloudLocation);
keys = await this.cognitiveClient.accounts.listKeys(name, speech.name);
instance.speechEndpoint = speech.endpoint;
instance.speechEndpoint = speech.properties.endpoint;
instance.speechKey = keys.key1;
GBLog.info(`Deploying Text Analytics...`);
const textAnalytics = await this.createTextAnalytics(name, `${name}-textanalytics`, instance.cloudLocation);
instance.textAnalyticsEndpoint = textAnalytics.endpoint.replace(`/text/analytics/v2.0`, '');
instance.textAnalyticsEndpoint = textAnalytics.properties.endpoint.replace(`/text/analytics/v2.0`, '');
GBLog.info(`Deploying SpellChecker...`);
const spellChecker = await this.createSpellChecker(name, `${name}-spellchecker`);
instance.spellcheckerEndpoint = spellChecker.endpoint;
instance.spellcheckerEndpoint = spellChecker.properties.endpoint;
GBLog.info(`Deploying NLP...`);
const nlp = await this.createNLP(name, `${name}-nlp`, instance.cloudLocation);
const nlpa = await this.createNLPAuthoring(name, `${name}-nlpa`, instance.cloudLocation);
instance.nlpEndpoint = nlp.endpoint;
instance.nlpEndpoint = nlp.properties.endpoint;
const sleep = ms => {
return new Promise(resolve => {
@ -575,7 +576,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
}
public initServices(credentials: any, subscriptionId: string) {
this.cloud = new ResourceManagementClient.default(credentials, subscriptionId);
this.cloud = new ResourceManagementClient(credentials, subscriptionId);
this.webSiteClient = new WebSiteManagementClient(credentials, subscriptionId);
this.storageClient = new SqlManagementClient(credentials, subscriptionId);
this.cognitiveClient = new CognitiveServicesManagementClient(credentials, subscriptionId);
@ -591,7 +592,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
fullyQualifiedDomainName: serverName
};
const database = await this.storageClient.servers.createOrUpdate(group, name, params);
const database = await this.storageClient.servers.beginCreateOrUpdateAndWait(group, name, params);
// AllowAllWindowsAzureIps must be created that way, so the Azure Search can
// access SQL Database to index its contents.
@ -790,7 +791,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
location: location
};
return await this.searchClient.services.createOrUpdate(group, name, params);
return await this.searchClient.services.beginCreateOrUpdateAndWait(group, name, params as any);
}
private async createStorage(group, serverName, name, location) {
@ -800,10 +801,10 @@ export class AzureDeployerService implements IGBInstallationDeployer {
location: location
};
return await this.storageClient.databases.createOrUpdate(group, serverName, name, params);
return await this.storageClient.databases.beginCreateOrUpdateAndWait(group, serverName, name, params);
}
private async createCognitiveServices(group, name, location, kind): Promise<CognitiveServicesAccount> {
private async createCognitiveServices(group, name, location, kind): Promise<Account> {
const params = {
sku: {
name: name
@ -826,26 +827,26 @@ export class AzureDeployerService implements IGBInstallationDeployer {
params.sku.name = this.freeTier ? 'F0' : 'S0';
}
return await this.cognitiveClient.accounts.create(group, name, params);
return await this.cognitiveClient.accounts.beginCreateAndWait(group, name, params);
}
private async createSpeech(group, name, location): Promise<CognitiveServicesAccount> {
private async createSpeech(group, name, location): Promise<Account> {
return await this.createCognitiveServices(group, name, location, 'SpeechServices');
}
private async createNLP(group, name, location): Promise<CognitiveServicesAccount> {
private async createNLP(group, name, location): Promise<Account> {
return await this.createCognitiveServices(group, name, location, 'LUIS');
}
private async createNLPAuthoring(group, name, location): Promise<CognitiveServicesAccount> {
private async createNLPAuthoring(group, name, location): Promise<Account> {
return await this.createCognitiveServices(group, name, location, 'LUIS.Authoring');
}
private async createSpellChecker(group, name): Promise<CognitiveServicesAccount> {
private async createSpellChecker(group, name): Promise<Account> {
return await this.createCognitiveServices(group, name, 'westus', 'CognitiveServices');
}
private async createTextAnalytics(group, name, location): Promise<CognitiveServicesAccount> {
private async createTextAnalytics(group, name, location): Promise<Account> {
return await this.createCognitiveServices(group, name, location, 'TextAnalytics');
}
@ -873,7 +874,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
}
};
return await this.webSiteClient.appServicePlans.createOrUpdate(group, name, params);
return await this.webSiteClient.appServicePlans.beginCreateOrUpdateAndWait(group, name, params);
}
private async createServer(farmId, group, name, location) {
@ -891,7 +892,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
requestTracingEnabled: true
}
};
const server = await this.webSiteClient.webApps.createOrUpdate(group, name, parameters);
const server = await this.webSiteClient.webApps.beginCreateOrUpdateAndWait(group, name, parameters);
const siteLogsConfig: SiteLogsConfig = {
applicationLogs: {
@ -908,7 +909,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
deploymentRollbackEnabled: false
};
await this.webSiteClient.webApps.createOrUpdateSourceControl(group, name, souceControlConfig);
await this.webSiteClient.webApps.beginCreateOrUpdateSourceControlAndWait(group, name, souceControlConfig);
return server;
};
@ -957,7 +958,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
}
};
return await this.webSiteClient.webApps.createOrUpdate(group, name, parameters);
return await this.webSiteClient.webApps.beginCreateOrUpdateAndWait(group, name, parameters);
}
}

View file

@ -58,7 +58,7 @@ import { CollectionUtil } from 'pragmatismo-io-framework';
import { GBBasicPackage } from '../../basic.gblib/index.js';
import { GBGoogleChatPackage } from '../../google-chat.gblib/index.js';
import { GBHubSpotPackage } from '../../hubspot.gblib/index.js';
import opn from 'opn';
import open from 'open';
import ngrok from 'ngrok';
/**
@ -524,7 +524,7 @@ ENDPOINT_UPDATE=true
*/
public openBrowserInDevelopment() {
if (process.env.NODE_ENV === 'development') {
opn('http://localhost:4242');
open('http://localhost:4242');
}
}

View file

@ -40,7 +40,7 @@ import urlJoin from 'url-join';
import path from 'path';
import asyncPromise from 'async-promises';
import walkPromise from 'walk-promise';
import { SearchService } from 'azure-search-client';
import { SearchClient } from '@azure/search-documents';
import Excel from 'exceljs';
import getSlug from 'speakingurl';
import { GBServer } from '../../../src/app.js';
@ -64,7 +64,7 @@ import { GuaribasAnswer, GuaribasQuestion, GuaribasSubject } from '../models/ind
import { GBConfigService } from './../../core.gbapp/services/GBConfigService.js';
import request from 'request-promise-native';
import textract from 'textract';
import pdf from "pdf-extraction";
import pdf from 'pdf-extraction';
/**
* Result for quey on KB data.
@ -141,7 +141,6 @@ export class KBService implements IGBKBService {
* Returns a question object given a SEO friendly URL.
*/
public async getQuestionIdFromURL(core: IGBCoreService, url: string) {
// Extracts questionId from URL.
const id = url.substr(url.lastIndexOf('-') + 1);
@ -176,11 +175,9 @@ export class KBService implements IGBKBService {
});
return questions;
}
public async getQuestionsSEO(instanceId: number) {
const questions = await GuaribasQuestion.findAll({
where: {
instanceId: instanceId
@ -199,14 +196,12 @@ export class KBService implements IGBKBService {
}
public async getDocs(instanceId: number) {
return await GuaribasAnswer.findAll({
where: {
instanceId: instanceId,
format: '.docx'
}
});
}
public async getAnswerByText(instanceId: number, text: string, from: string = null): Promise<any> {
@ -232,7 +227,7 @@ export class KBService implements IGBKBService {
let where = {
instanceId: instanceId,
content: { [Op.eq]: `${text}` }
}
};
question = await GuaribasQuestion.findOne({
where: where
});
@ -252,9 +247,6 @@ export class KBService implements IGBKBService {
return undefined;
}
public async addAnswer(obj: GuaribasAnswer): Promise<GuaribasAnswer> {
return await GuaribasAnswer.create(obj);
}
@ -265,8 +257,6 @@ export class KBService implements IGBKBService {
searchScore: number,
subjects: GuaribasSubject[]
): Promise<KBServiceSearchResults> {
// Builds search query.
query = query.toLowerCase();
@ -292,43 +282,51 @@ export class KBService implements IGBKBService {
}
}
// No direct match found, so Search is used.
if (instance.searchKey !== null && GBConfigService.get('STORAGE_DIALECT') === 'mssql') {
const client = new SearchService(instance.searchHost.split('.')[0], instance.searchKey);
interface SearchResults {
instanceId: number;
questionId: number;
answerId: number;
content: string;
subject1: string;
subject2: string;
subject: string;
subject4: string;
}
const results = await client.indexes.use('azuresql-index').search({
count: true,
const client = new SearchClient<SearchResults>(instance.searchHost.split('.')[0], 'azuresql-index', {
key: instance.searchKey
} as any);
const results = await client.search(query, {
filter: `instanceId eq ${instance.instanceId} and skipIndex eq false`,
search: query,
searchFields: 'content, subject1, subject2, subject3, subject4',
select: 'instanceId, questionId, answerId',
searchFields: ['content', 'subject1', 'subject2', 'subject', 'subject4'],
select: ['instanceId', 'questionId', 'answerId'],
skip: 0,
top: 1,
top: 1
});
const values = results.result.value;
const values = results.results; // TODO: See.
let returnedScore = 0;
// Searches via Search (Azure Search).
if (values && values.length > 0) {
returnedScore = values[0]['@search.score'];
let found = false;
for await (const result of values) {
found = true;
returnedScore = result['@search.score'];
if (returnedScore >= searchScore) {
const value = await this.getAnswerById(instance.instanceId, values[0].answerId);
const value = await this.getAnswerById(instance.instanceId, result.document.answerId);
if (value !== null) {
GBLog.info(
`SEARCH WILL BE USED with score: ${returnedScore} > required (searchScore): ${searchScore}`
);
GBLog.info(`SEARCH WILL BE USED with score: ${returnedScore} > required (searchScore): ${searchScore}`);
return { answer: value, questionId: values[0].questionId };
return { answer: value, questionId: result.document.questionId };
} else {
GBLog.info(
`SEARCH WILL NOT be used as answerId ${values[0].answerId} was not found in database,
`SEARCH WILL NOT be used as answerId ${result.document.answerId} was not found in database,
returnedScore: ${returnedScore} < required (searchScore): ${searchScore}`
);
@ -342,15 +340,15 @@ export class KBService implements IGBKBService {
return { answer: undefined, questionId: 0 };
}
} else {
GBLog.info(
`SEARCH called but NO answer could be found (zero results).`
);
}
if (!found) {
GBLog.info(`SEARCH called but NO answer could be found (zero results).`);
}
return { answer: undefined, questionId: 0 };
}
}
}
public async getSubjectItems(instanceId: number, parentId: number): Promise<GuaribasSubject[]> {
const where = { parentSubjectId: parentId, instanceId: instanceId };
@ -455,8 +453,13 @@ export class KBService implements IGBKBService {
const question = line._cells[3].text.trim();
let answer = line._cells[4].text.trim();
if (!(subjectsText === 'subjects' && from === 'from') && answer !== null && question !== null &&
answer !== '' && question !== '') {
if (
!(subjectsText === 'subjects' && from === 'from') &&
answer !== null &&
question !== null &&
answer !== '' &&
question !== ''
) {
let format = '.txt';
// Extracts answer from external media if any.
@ -468,16 +471,13 @@ export class KBService implements IGBKBService {
answer =
'Existe um problema na base de conhecimento. Fui treinado para entender sua pergunta, avise a quem me criou que a resposta não foi informada para esta pergunta.';
} else if (answer.indexOf('.md') > -1 || answer.indexOf('.docx') > -1) {
const mediaFilename = urlJoin(path.dirname(filePath), '..', 'articles', answer);
if (Fs.existsSync(mediaFilename)) {
// Tries to load .docx file from Articles folder.
if (answer.indexOf('.docx') > -1) {
answer = await this.getTextFromFile(filePath);
}
else {
} else {
// Loads normally markdown file.
answer = Fs.readFileSync(mediaFilename, 'utf8');
@ -542,7 +542,7 @@ export class KBService implements IGBKBService {
subject4: subject4,
content: question.replace(/["]+/g, ''),
instanceId: instanceId,
skipIndex: (question.charAt(0) === "\""),
skipIndex: question.charAt(0) === '"',
packageId: packageId
};
questions.push(question1);
@ -582,16 +582,19 @@ export class KBService implements IGBKBService {
answer.content.endsWith('.docx') ||
answer.content.endsWith('.xls') ||
answer.content.endsWith('.xlsx')
) {
const doc = urlJoin(GBServer.globals.publicAddress, 'kb', `${min.instance.botId}.gbai`,
`${min.instance.botId}.gbkb`, 'assets', answer.content)
const doc = urlJoin(
GBServer.globals.publicAddress,
'kb',
`${min.instance.botId}.gbai`,
`${min.instance.botId}.gbkb`,
'assets',
answer.content
);
const url = `http://view.officeapps.live.com/op/view.aspx?src=${doc}`;
await this.playUrl(min, min.conversationalService, step, url, channel);
} else if (answer.content.endsWith('.pdf')) {
const url = urlJoin('kb', `${min.instance.botId}.gbai`,
`${min.instance.botId}.gbkb`, 'assets', answer.content);
const url = urlJoin('kb', `${min.instance.botId}.gbai`, `${min.instance.botId}.gbkb`, 'assets', answer.content);
await this.playUrl(min, min.conversationalService, step, url, channel);
} else if (answer.format === '.md') {
await min.conversationalService['playMarkdown'](min, answer.content, channel, step, min.conversationalService);
@ -660,12 +663,18 @@ export class KBService implements IGBKBService {
/**
* Import all .docx files in reading comprehension folder.
*/
public async importDocs(min: GBMinInstance, localPath: string, instance: IGBInstance, packageId: number): Promise<any> {
public async importDocs(
min: GBMinInstance,
localPath: string,
instance: IGBInstance,
packageId: number
): Promise<any> {
const files = await walkPromise(urlJoin(localPath, 'docs'));
if (!files[0]) {
GBLog.info(`[GBDeployer] docs folder not created yet in .gbkb. To use Reading Comprehension, create this folder at root and put a document to get read by the.`);
}
else {
GBLog.info(
`[GBDeployer] docs folder not created yet in .gbkb. To use Reading Comprehension, create this folder at root and put a document to get read by the.`
);
} else {
await CollectionUtil.asyncForEach(files, async file => {
let content = null;
let filePath = Path.join(file.root, file.name);
@ -688,7 +697,6 @@ export class KBService implements IGBKBService {
packageId: packageId
});
}
});
}
}
@ -760,12 +768,9 @@ export class KBService implements IGBKBService {
if (categoryReg && nameReg) {
let category = categoryReg[1];
let name = nameReg[1];
min["nerEngine"].addNamedEntityText(category, name,
[contentLocale], [name]);
min['nerEngine'].addNamedEntityText(category, name, [contentLocale], [name]);
}
});
}
/**
@ -865,5 +870,3 @@ export class KBService implements IGBKBService {
});
}
}