From 6b1bc00e41cf57578a44a8dc1b4ab1d01e0cb934 Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Mon, 29 Mar 2021 18:50:27 -0300 Subject: [PATCH] new(admin.gbapp): MSGraph now used to download .gbkb artifacts. --- .../admin.gbapp/services/GBAdminService.ts | 14 +-- .../services/AzureDeployerService.ts | 2 +- .../basic.gblib/services/SystemKeywords.ts | 79 +++++++---------- packages/core.gbapp/services/GBDeployer.ts | 86 ++++++++++++++++++- packages/core.gbapp/services/GBMinService.ts | 2 +- src/app.ts | 2 +- 6 files changed, 120 insertions(+), 65 deletions(-) diff --git a/packages/admin.gbapp/services/GBAdminService.ts b/packages/admin.gbapp/services/GBAdminService.ts index cea0a105..90dd7034 100644 --- a/packages/admin.gbapp/services/GBAdminService.ts +++ b/packages/admin.gbapp/services/GBAdminService.ts @@ -192,17 +192,9 @@ export class GBAdminService implements IGBAdminService { // .gbot packages are handled using storage API, so no download // of local resources is required. - - if (!localFolder.endsWith('.gbot') && !process.env.DONT_DOWNLOAD) { - GBLog.warn(`${GBConfigService.get('CLOUD_USERNAME')} must be authorized on SharePoint related site to download to: ${localFolder}`); - await s.downloadFolder( - localFolder, - siteName, - folderName, - GBConfigService.get('CLOUD_USERNAME'), - GBConfigService.get('CLOUD_PASSWORD') - ); - } + await deployer['downloadFolder'](min, + Path.join('work', `${min.instance.botId}.gbai`), + Path.basename(folderName)); await deployer.deployPackage(min, localFolder); } } diff --git a/packages/azuredeployer.gbapp/services/AzureDeployerService.ts b/packages/azuredeployer.gbapp/services/AzureDeployerService.ts index 01571d77..09a22681 100644 --- a/packages/azuredeployer.gbapp/services/AzureDeployerService.ts +++ b/packages/azuredeployer.gbapp/services/AzureDeployerService.ts @@ -285,7 +285,7 @@ export class AzureDeployerService implements IGBInstallationDeployer { const parameters = { properties: { - description: description, + description: `${description}`, displayName: name, endpoint: endpoint, iconUrl: iconUrl diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index fdc5553d..0a303a35 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -70,23 +70,6 @@ export class SystemKeywords { this.deployer = deployer; } - /** - * Retrives token and initialize drive client API. - */ - private async internalGetDriveClient() { - let token = await this.min.adminService.acquireElevatedToken(this.min.instance.instanceId); - let siteId = process.env.STORAGE_SITE_ID; - let libraryId = process.env.STORAGE_LIBRARY; - - let client = MicrosoftGraph.Client.init({ - authProvider: done => { - done(null, token); - } - }); - const baseUrl = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}`; - return [baseUrl, client]; - } - /** * Retrives the content of a given URL. */ @@ -175,7 +158,7 @@ export class SystemKeywords { public async set(file: string, address: string, value: any): Promise { GBLog.info(`BASIC: Defining '${address}' in '${file}' to '${value}' (SET). `); - let [baseUrl, client] = await this.internalGetDriveClient(); + let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min); const botId = this.min.instance.botId; const path = `/${botId}.gbai/${botId}.gbdata`; @@ -223,7 +206,7 @@ export class SystemKeywords { */ public async save(file: string, ...args): Promise { GBLog.info(`BASIC: Saving '${file}' (SAVE). Args: ${args.join(',')}.`); - let [baseUrl, client] = await this.internalGetDriveClient(); + let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min); const botId = this.min.instance.botId; const path = `/${botId}.gbai/${botId}.gbdata`; @@ -257,7 +240,7 @@ export class SystemKeywords { */ public async get(file: string, address: string): Promise { GBLog.info(`BASIC: GET '${address}' in '${file}'.`); - let [baseUrl, client] = await this.internalGetDriveClient(); + let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min); const botId = this.min.instance.botId; const path = `/${botId}.gbai/${botId}.gbdata`; @@ -293,7 +276,7 @@ export class SystemKeywords { */ public async find(file: string, ...args): Promise { GBLog.info(`BASIC: FIND running on ${file} and args: ${JSON.stringify(args)}...`); - let [baseUrl, client] = await this.internalGetDriveClient(); + let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min); const botId = this.min.instance.botId; const path = `/${botId}.gbai/${botId}.gbdata`; @@ -313,7 +296,7 @@ export class SystemKeywords { .get(); let results = await client - .api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A1:Z100')`) + .api(`${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A1:Z2000')`) .get(); // Increments columnIndex by looping until find a column match. @@ -380,7 +363,7 @@ export class SystemKeywords { */ public async createFolder(name: string) { - let [baseUrl, client] = await this.internalGetDriveClient(); + let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min); const botId = this.min.instance.botId; let path = `/${botId}.gbai/${botId}.gbdata`; @@ -436,7 +419,7 @@ export class SystemKeywords { * */ public async shareFolder(folderReference, email: string, message: string) { - let [, client] = await this.internalGetDriveClient(); + let [, client] = await GBDeployer.internalGetDriveClient(this.min); const driveId = folderReference.parentReference.driveId; const itemId = folderReference.id; const body = { @@ -462,7 +445,7 @@ export class SystemKeywords { */ public async copyFile(src, dest) { GBLog.info(`BASIC: BEGINING COPY '${src}' to '${dest}'`); - let [baseUrl, client] = await this.internalGetDriveClient(); + let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min); const botId = this.min.instance.botId; // Normalizes all slashes. @@ -531,7 +514,7 @@ export class SystemKeywords { */ public async convert(src, dest) { GBLog.info(`BASIC: CONVERT '${src}' to '${dest}'`); - let [baseUrl, client] = await this.internalGetDriveClient(); + let [baseUrl, client] = await GBDeployer.internalGetDriveClient(this.min); const botId = this.min.instance.botId; // Normalizes all slashes. @@ -616,9 +599,9 @@ export class SystemKeywords { * */ public async sendEmail(to, subject, body) { - + // tslint:disable-next-line:no-console - + GBLog.info(`[E-mail]: to:${to}, subject: ${subject}, body: ${body}.`); const emailToken = process.env.EMAIL_API_KEY; @@ -626,7 +609,7 @@ export class SystemKeywords { sgMail.setApiKey(emailToken); const msg = { to: to, - from: process.env.EMAIL_FROM, + from: process.env.EMAIL_FROM, subject: subject, text: body, html: body @@ -642,8 +625,6 @@ export class SystemKeywords { }); } -} - /** * Calls any REST API by using GET HTTP method. * @@ -651,14 +632,14 @@ export class SystemKeywords { * */ public async getByHttp(url: string) { - const options = { - uri: url - }; + const options = { + uri: url + }; - let result = await request.get(options); - GBLog.info(`[GET]: ${url} : ${result}`); - return JSON.parse(result); -} + let result = await request.get(options); + GBLog.info(`[GET]: ${url} : ${result}`); + return JSON.parse(result); + } /** * Calls any REST API by using POST HTTP method. @@ -670,18 +651,18 @@ export class SystemKeywords { * */ public async postByHttp(url: string, data) { - const options = { - uri: url, - json: true, - body: data - }; + const options = { + uri: url, + json: true, + body: data + }; - let result = await request.post(options); - GBLog.info(`[POST]: ${url} (${data}): ${result}`); - return JSON.parse(result); -} + let result = await request.post(options); + GBLog.info(`[POST]: ${url} (${data}): ${result}`); + return JSON.parse(result); + } public async numberOnly(text: string) { - return text.replace(/\D/gi, ''); -} + return text.replace(/\D/gi, ''); + } } diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index 83fb26e4..cc52a928 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -41,8 +41,8 @@ import urlJoin = require('url-join'); const Fs = require('fs'); const express = require('express'); const child_process = require('child_process'); -const graph = require('@microsoft/microsoft-graph-client'); const rimraf = require('rimraf'); +const request = require('request-promise-native'); import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBDeployer, IGBInstance, IGBPackage } from 'botlib'; import { AzureSearch } from 'pragmatismo-io-framework'; @@ -98,6 +98,23 @@ export class GBDeployer implements IGBDeployer { return `Server=tcp:${instance.storageServer},1433;Database=${instance.storageName};User ID=${instance.storageUsername};Password=${instance.storagePassword};Trusted_Connection=False;Encrypt=True;Connection Timeout=30;`; } + /** + * Retrives token and initialize drive client API. + */ + public static async internalGetDriveClient(min: GBMinInstance) { + let token = await min.adminService.acquireElevatedToken(min.instance.instanceId); + let siteId = process.env.STORAGE_SITE_ID; + let libraryId = process.env.STORAGE_LIBRARY; + + let client = MicrosoftGraph.Client.init({ + authProvider: done => { + done(null, token); + } + }); + const baseUrl = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${libraryId}`; + return [baseUrl, client]; + } + /** * Performs package deployment in all .gbai or default. */ @@ -422,6 +439,71 @@ export class GBDeployer implements IGBDeployer { return obj; } + /** + * Loads all para from tabular file Config.xlsx. + */ + public async downloadFolder(min: GBMinInstance, localPath: string, remotePath: string, + baseUrl: string = null, client = null): Promise { + if (!baseUrl) { + [baseUrl, client] = await GBDeployer.internalGetDriveClient(min); + + remotePath = remotePath.replace(/\\/gi, '/'); + const parts = remotePath.split('/'); + + // Creates each subfolder. + + let pathBase = localPath; + if (!Fs.existsSync(pathBase)) { + Fs.mkdirSync(pathBase); + } + + await CollectionUtil.asyncForEach(parts, async item => { + pathBase = Path.join(pathBase, item); + if (!Fs.existsSync(pathBase)) { + Fs.mkdirSync(pathBase); + } + }); + + // Retrieves all files in remote folder. + + const botId = min.instance.botId; + const path = urlJoin(`/${botId}.gbai`, remotePath); + let url = `${baseUrl}/drive/root:${path}:/children`; + + const res = await client + .api(url) + .get(); + const documents = res.value; + if (documents === undefined || documents.length === 0) { + GBLog.info(`${remotePath} is an empty folder.`); + return null; + } + + // Download files or navigate to directory to recurse. + + await CollectionUtil.asyncForEach(documents, async item => { + + const itemPath = Path.join(localPath, remotePath, item.name); + + if (item.folder) { + if (!Fs.existsSync(itemPath)) { + Fs.mkdirSync(itemPath); + } + const nextFolder = urlJoin(remotePath, item.name); + await this.downloadFolder(min, localPath, nextFolder); + } else { + GBLog.info(`Downloading ${itemPath}...`); + const url = item['@microsoft.graph.downloadUrl']; + const options = { + uri: url, + encoding: null + }; + const response = await request({ uri: url, encoding: null }); + Fs.writeFileSync(itemPath, response, { encoding: null }); + } + }); + } + } /** * UndDeploys a bot to the storage. */ @@ -561,7 +643,7 @@ export class GBDeployer implements IGBDeployer { break; default: - + const err = GBError.create(`Unhandled package type: ${packageType}.`); Promise.reject(err); break; diff --git a/packages/core.gbapp/services/GBMinService.ts b/packages/core.gbapp/services/GBMinService.ts index 7d992b4b..0ca877ed 100644 --- a/packages/core.gbapp/services/GBMinService.ts +++ b/packages/core.gbapp/services/GBMinService.ts @@ -157,7 +157,7 @@ export class GBMinService { try { await this.mountBot(instance); } catch (error) { - GBLog.error(`Error mounting bot ${instance.botId}: ${error.message}`); + GBLog.error(`Error mounting bot ${instance.botId}: ${error.message}\n${error.stack}`); } }); } diff --git a/src/app.ts b/src/app.ts index 9c9e2635..e8f0f7a2 100644 --- a/src/app.ts +++ b/src/app.ts @@ -186,7 +186,7 @@ export class GBServer { } GBLog.info(`The Bot Server is in RUNNING mode...`); - + // Opens Navigator. // TODO: Config: core.openBrowserInDevelopment();