/*****************************************************************************\ | █████ █████ ██ █ █████ █████ ████ ██ ████ █████ █████ ███ ® | | ██ █ ███ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ | | ██ ███ ████ █ ██ █ ████ █████ ██████ ██ ████ █ █ █ ██ | | ██ ██ █ █ ██ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ | | █████ █████ █ ███ █████ ██ ██ ██ ██ █████ ████ █████ █ ███ | | | | General Bots Copyright (c) pragmatismo.cloud. All rights reserved. | | Licensed under the AGPL-3.0. | | | | According to our dual licensing model, this program can be used either | | under the terms of the GNU Affero General Public License, version 3, | | or under a proprietary license. | | | | The texts of the GNU Affero General Public License with an additional | | permission and of our proprietary license can be found at and | | in the LICENSE file you have received along with this program. | | | | This program is distributed in the hope that it will be useful, | | but WITHOUT ANY WARRANTY, without even the implied warranty of | | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | | GNU Affero General Public License for more details. | | | | "General Bots" is a registered trademark of pragmatismo.cloud. | | The licensing of the program under the AGPLv3 does not imply a | | trademark license. Therefore any rights, title and interest in | | our trademarks remain entirely with us. | | | \*****************************************************************************/ /** * @fileoverview General Bots local utility. */ 'use strict'; import * as YAML from 'yaml'; import SwaggerClient from 'swagger-client'; import fs from 'fs/promises'; import { GBConfigService } from '../packages/core.gbapp/services/GBConfigService.js'; import path from 'path'; import { VerbosityLevel, getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs'; VerbosityLevel.ERRORS=0; VerbosityLevel.WARNINGS=0; VerbosityLevel.INFOS=0; import { Page } from 'puppeteer'; import urljoin from 'url-join'; import html2md from 'html-to-md'; export class GBUtil { public static repeat(chr, count) { let str = ''; for (let x = 0; x < count; x++) { str += chr; } return str; } public static padL(value, width, pad) { if (!width || width < 1) return value; if (!pad) pad = ' '; const length = width - value.length; if (length < 1) return value.substr(0, width); return (GBUtil.repeat(pad, length) + value).substr(0, width); } public static padR(value, width, pad) { if (!width || width < 1) return value; if (!pad) pad = ' '; const length = width - value.length; if (length < 1) value.substr(0, width); return (value + GBUtil.repeat(pad, length)).substr(0, width); } public static async getDirectLineClient(min) { let config = { spec: JSON.parse(await fs.readFile('directline-3.0.json', 'utf8')), requestInterceptor: req => { req.headers['Authorization'] = `Bearer ${min.instance.webchatKey}`; } }; if (!GBConfigService.get('STORAGE_NAME')) { (config['spec'].url = `http://127.0.0.1:${GBConfigService.getServerPort()}/api/messages/${min.botId}`), (config['spec'].servers = [ { url: `http://127.0.0.1:${GBConfigService.getServerPort()}/api/messages/${min.botId}` } ]); config['spec'].openapi = '3.0.0'; delete config['spec'].host; delete config['spec'].swagger; } return await new SwaggerClient(config); } public static toYAML(data) { const extractProps = obj => { return Object.getOwnPropertyNames(obj).reduce((acc, key) => { const value = obj[key]; acc[key] = value && typeof value === 'object' && !Array.isArray(value) ? extractProps(value) : value; return acc; }, {}); }; const extractedError = extractProps(data); return YAML.stringify(extractedError); } public static sleep(ms) { return new Promise(resolve => { setTimeout(resolve, ms); }); } public static caseInsensitive(listOrRow) { if (!listOrRow || typeof listOrRow !== 'object') { return listOrRow; } const lowercase = oldKey => (typeof oldKey === 'string' ? oldKey.toLowerCase() : oldKey); const createCaseInsensitiveProxy = obj => { const propertiesMap = new Map(Object.keys(obj).map(propKey => [lowercase(propKey), obj[propKey]])); const caseInsensitiveGetHandler = { get: (target, property) => propertiesMap.get(lowercase(property)) }; return new Proxy(obj, caseInsensitiveGetHandler); }; if (Array.isArray(listOrRow)) { return listOrRow.map(row => createCaseInsensitiveProxy(row)); } else { return createCaseInsensitiveProxy(listOrRow); } } public static async exists(filePath: string): Promise { try { await fs.access(filePath); return true; // File exists } catch (error) { return false; // File does not exist } } public static async savePage(url: string, page: Page, directoryPath: string): Promise { try { // Check if the directory exists, create it if not const directoryExists = await this.fileExists(directoryPath); if (!directoryExists) { await fs.mkdir(directoryPath, { recursive: true }); // Create directory if it doesn't exist } // Check if the URL is for a downloadable file (e.g., .pdf) if (url.endsWith('.pdf')) { const response = await fetch(url); if (!response.ok) { throw new Error('Failed to download the file'); } const buffer = await response.arrayBuffer(); // Convert response to array buffer const fileName = path.basename(url); // Extract file name from URL const filePath = path.join(directoryPath, fileName); // Create file path const data = new Uint8Array(buffer); const text = await GBUtil.getPdfText(data); // Write the buffer to the file asynchronously await fs.writeFile(filePath, text); return filePath; // Return the saved file path } else { // Use Puppeteer for non-downloadable pages const parsedUrl = new URL(url); // Get the last part of the URL path or default to 'index' if empty const pathParts = parsedUrl.pathname.split('/').filter(Boolean); // Remove empty parts const lastPath = pathParts.length > 0 ? pathParts[pathParts.length - 1] : 'index'; const flatLastPath = lastPath.replace(/\W+/g, '-'); // Flatten the last part of the path const fileName = `${flatLastPath}.html`; const filePath = path.join(directoryPath, fileName); const htmlContent = await page.content(); // Write HTML content asynchronously await fs.writeFile(filePath, htmlContent); return filePath; } } catch (error) { console.error('Error saving page:', error); return null; } } public static async fileExists(filePath: string): Promise { try { await fs.access(filePath); return true; } catch (error) { return false; } } public static async copyIfNewerRecursive(src, dest) { if (!await GBUtil.exists(src)) { console.error(`Source path "${src}" does not exist.`); return; } // Check if the source is a directory if ((await fs.stat(src)).isDirectory()) { // Create the destination directory if it doesn't exist if (!await GBUtil.exists(dest)) { fs.mkdir(dest, { recursive: true }); } // Read all files and directories in the source directory const entries =await fs.readdir(src); for (let entry of entries) { const srcEntry = path.join(src, entry); const destEntry = path.join(dest, entry); // Recursively copy each entry this.copyIfNewerRecursive(srcEntry, destEntry); } } else { // Source is a file, check if we need to copy it if (await GBUtil.exists(dest)) { const srcStat =await fs.stat(src); const destStat =await fs.stat(dest); // Copy only if the source file is newer than the destination file if (srcStat.mtime > destStat.mtime) { fs.cp(src, dest, { force: true }); } } else { // Destination file doesn't exist, so copy it fs.cp(src, dest, { force: true }); } } } // Check if is a tree or flat object. public static hasSubObject(t) { for (var key in t) { if (!t.hasOwnProperty(key)) continue; if (typeof t[key] === 'object') return true; } return false; } public static async getPdfText(data): Promise { const pdf = await getDocument({data}).promise; let pages = []; for (let i = 1; i <= pdf.numPages; i++) { const page = await pdf.getPage(i); const textContent = await page.getTextContent(); const text = textContent.items .map(item => item['str']) .join(' ') .replace(/\s+/g, ' '); // Optionally remove extra spaces pages.push(text); } return pages.join(' '); } static getGBAIPath(botId, packageType = null, packageName = null) { let gbai = `${botId}.gbai`; if (!packageType && !packageName) { return GBConfigService.get('DEV_GBAI') ? GBConfigService.get('DEV_GBAI') : gbai; } if (GBConfigService.get('DEV_GBAI')) { gbai = GBConfigService.get('DEV_GBAI'); botId = gbai.replace(/\.[^/.]+$/, ''); return urljoin(GBConfigService.get('DEV_GBAI'), packageName ? packageName : `${botId}.${packageType}`); } else { return urljoin(gbai, packageName ? packageName : `${botId}.${packageType}`); } } }