From 0832f9d769382243629c7ae04d9d64c825b9518b Mon Sep 17 00:00:00 2001 From: "me@rodrigorodriguez.com" Date: Tue, 15 Oct 2024 15:05:43 -0300 Subject: [PATCH] new(core.gbapp): Timestamp fields are now default. --- .../basic.gblib/services/DialogKeywords.ts | 54 +++++------ packages/basic.gblib/services/GBVMService.ts | 15 +--- .../services/KeywordsExpressions.ts | 7 ++ .../basic.gblib/services/SystemKeywords.ts | 89 ++++++++++++++++++- src/util.ts | 57 ++++++++---- .../whatsapp.gbdialog/create-task.bas | 1 + .../notify-latest-orders.bas | 7 ++ .../talk-to-data.gbot/config.csv | 1 + 8 files changed, 172 insertions(+), 59 deletions(-) create mode 100644 templates/_drafts/whatsapp.gbai/whatsapp.gbdialog/create-task.bas create mode 100644 templates/talk-to-data.gbai/talk-to-data.gbdialog/notify-latest-orders.bas diff --git a/packages/basic.gblib/services/DialogKeywords.ts b/packages/basic.gblib/services/DialogKeywords.ts index 5d1f4b70..4294e617 100644 --- a/packages/basic.gblib/services/DialogKeywords.ts +++ b/packages/basic.gblib/services/DialogKeywords.ts @@ -30,39 +30,38 @@ 'use strict'; +import sgMail from '@sendgrid/mail'; +import { ActivityTypes } from 'botbuilder'; import { GBLog, GBMinInstance } from 'botlib'; -import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js'; -import { ChartServices } from './ChartServices.js'; -import urlJoin from 'url-join'; -import { GBServer } from '../../../src/app.js'; -import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; -import { SecService } from '../../security.gbapp/services/SecService.js'; -import { Jimp } from 'jimp'; -import jsQR from 'jsqr'; -import { SystemKeywords } from './SystemKeywords.js'; -import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; -import { Messages } from '../strings.js'; -import { CollectionUtil } from 'pragmatismo-io-framework'; -import { GBConversationalService } from '../../core.gbapp/services/GBConversationalService.js'; +import * as df from 'date-diff'; import fs from 'fs/promises'; import libphonenumber from 'google-libphonenumber'; -import * as df from 'date-diff'; +import { Jimp } from 'jimp'; +import jsQR from 'jsqr'; +import mammoth from 'mammoth'; +import mime from 'mime-types'; import tesseract from 'node-tesseract-ocr'; import path from 'path'; -import sgMail from '@sendgrid/mail'; -import mammoth from 'mammoth'; -import qrcode from 'qrcode'; -import { WebAutomationServices } from './WebAutomationServices.js'; -import QrScanner from 'qr-scanner'; -import pkg from 'whatsapp-web.js'; -import { ActivityTypes } from 'botbuilder'; -const { List, Buttons } = pkg; -import mime from 'mime-types'; -import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js'; -import { GBUtil } from '../../../src/util.js'; -import { GBVMService } from './GBVMService.js'; -import { ChatServices } from '../../../packages/llm.gblib/services/ChatServices.js'; +import { CollectionUtil } from 'pragmatismo-io-framework'; import puppeteer from 'puppeteer'; +import qrcode from 'qrcode'; +import urlJoin from 'url-join'; +import pkg from 'whatsapp-web.js'; +import { ChatServices } from '../../../packages/llm.gblib/services/ChatServices.js'; +import { GBServer } from '../../../src/app.js'; +import { GBUtil } from '../../../src/util.js'; +import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; +import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js'; +import { GBConversationalService } from '../../core.gbapp/services/GBConversationalService.js'; +import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; +import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js'; +import { SecService } from '../../security.gbapp/services/SecService.js'; +import { Messages } from '../strings.js'; +import { ChartServices } from './ChartServices.js'; +import { GBVMService } from './GBVMService.js'; +import { SystemKeywords } from './SystemKeywords.js'; +import { WebAutomationServices } from './WebAutomationServices.js'; +const { List, Buttons } = pkg; /** @@ -384,6 +383,7 @@ export class DialogKeywords { var year = date.getFullYear(); format = format.replace('MM', GBUtil.padL(month.toString(), 2, '0')); + format = format.toLowerCase(); if (format.indexOf('yyyy') > -1) format = format.replace('yyyy', year.toString()); else if (format.indexOf('yy') > -1) format = format.replace('yy', year.toString().substr(2, 2)); diff --git a/packages/basic.gblib/services/GBVMService.ts b/packages/basic.gblib/services/GBVMService.ts index 974ff288..5e71eaf4 100644 --- a/packages/basic.gblib/services/GBVMService.ts +++ b/packages/basic.gblib/services/GBVMService.ts @@ -405,19 +405,7 @@ export class GBVMService extends GBService { let tables; const dialect = con.dialect.name; - if (dialect === 'mssql') { - tables = await seq.query( - `SELECT table_name, table_schema - FROM information_schema.tables - WHERE table_type = 'BASE TABLE' - ORDER BY table_name ASC`, - { - type: QueryTypes.RAW - } - )[0]; - } else if (dialect === 'mariadb') { - tables = await seq.getQueryInterface().showAllTables(); - } + tables = await GBUtil.listTables(dialect, seq); let found = false; tables.forEach(storageTable => { @@ -457,6 +445,7 @@ export class GBVMService extends GBService { } public async translateBASIC(mainName, filename: any, min: GBMinInstance) { + // Converts General Bots BASIC into regular VBS let basicCode: string = await fs.readFile(filename, 'utf8'); diff --git a/packages/basic.gblib/services/KeywordsExpressions.ts b/packages/basic.gblib/services/KeywordsExpressions.ts index 64ad55f6..88c5f736 100644 --- a/packages/basic.gblib/services/KeywordsExpressions.ts +++ b/packages/basic.gblib/services/KeywordsExpressions.ts @@ -1308,6 +1308,13 @@ export class KeywordsExpressions { } ]; + keywords[i++] = [ + /^\s*refresh\s*(.*)/gim, + ($0, $1, $2, $3) => { + return `await sys.refreshDataSourceCache({pid: pid, connectionName: ${$2}})`; + } + ]; + keywords[i++] = [ /^\s*(save)(\s*)(.*)(.*)/gim, ($0, $1, $2, $3, $4) => { diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index f59dce57..711e5649 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -48,6 +48,7 @@ import urlJoin from 'url-join'; import Excel from 'exceljs'; import { BufferWindowMemory } from 'langchain/memory'; import csvdb from 'csv-database'; +import { Sequelize, QueryTypes, DataTypes } from '@sequelize/core'; import packagePath from 'path'; import ComputerVisionClient from '@azure/cognitiveservices-computervision'; import ApiKeyCredentials from '@azure/ms-rest-js'; @@ -752,7 +753,7 @@ export class SystemKeywords { if (typeof rows === 'object' && rows !== null) { rows = [rows]; } - + if (rows.length === 0) { return; } @@ -2877,4 +2878,90 @@ export class SystemKeywords { await this.setMemoryContext({ pid, erase: true }); } + + + public async refreshDataSourceCache({ pid, connectionName }) { + const { min, user, params, step } = await DialogKeywords.getProcessInfo(pid); + + let sqliteFilePath =path.join('work', GBUtil.getGBAIPath(min.botId), `${connectionName}.sqlite`); + + // Step 1: Clean the SQLite file if it already exists + if (await GBUtil.exists(sqliteFilePath)) { + await fs.unlink(sqliteFilePath); // Remove the file + GBLogEx.info(min, `${sqliteFilePath} has been cleaned.`); + } + + // Step 2: Connect to SQLite (Local) + const sqlite = new Sequelize({ + dialect: 'sqlite', + storage: sqliteFilePath + }); + + // Get the connection details from the min object + let con = min[connectionName]; + const dialect = con.dialect.name; + + // Step 3: Get the list of all tables from the source database + const tables = await GBUtil.listTables(dialect, min.core.sequelize); + const tableNames = tables.map(table => Object.values(table)[0]); + + // Function to map source database datatypes to SQLite-compatible datatypes + const mapToSQLiteType = (columnType) => { + const typeMapping = { + 'VARCHAR': DataTypes.STRING, + 'CHAR': DataTypes.STRING, + 'TEXT': DataTypes.TEXT, + 'TINYINT': DataTypes.INTEGER, + 'SMALLINT': DataTypes.INTEGER, + 'MEDIUMINT': DataTypes.INTEGER, + 'INT': DataTypes.INTEGER, + 'INTEGER': DataTypes.INTEGER, + 'BIGINT': DataTypes.BIGINT, + 'FLOAT': DataTypes.FLOAT, + 'DOUBLE': DataTypes.DOUBLE, + 'DECIMAL': DataTypes.DECIMAL, + 'DATE': DataTypes.DATE, + 'DATETIME': DataTypes.DATE, + 'TIMESTAMP': DataTypes.DATE, + 'BLOB': DataTypes.BLOB, + 'BOOLEAN': DataTypes.BOOLEAN, + // Add more mappings as needed + }; + + // Return mapped type or fallback to STRING if not mapped + return typeMapping[columnType.toUpperCase()] || DataTypes.STRING; + }; + + // Step 4: Retrieve and export data for each table + for (const table of tableNames) { + // Retrieve rows from the source table + const [rows] = await min.core.sequelize.query(`SELECT * FROM ${table}`); + + if (rows.length === 0) continue; // Skip if the table has no data + + // Get the schema for the current table from the source database + const columns = await min.core.sequelize.queryInterface.describeTable(table); + + // Create a schema object for SQLite + const schema = {}; + Object.keys(columns).forEach(col => { + const columnType = columns[col].type; + schema[col] = mapToSQLiteType(columnType); // Map source type to SQLite type + }); + + // Define the model dynamically for each table in SQLite + const Model = sqlite.define(table, schema, { timestamps: false }); + + // Sync the model (create table) + await Model.sync({ force: true }); + + // Bulk insert rows into the SQLite table + await Model.bulkCreate(rows); + } + + GBLogEx.info(min, `All tables have been successfully exported to ${sqliteFilePath}`); + + // Close SQLite connection + await sqlite.close(); + } } diff --git a/src/util.ts b/src/util.ts index 7c3678ef..137a574a 100644 --- a/src/util.ts +++ b/src/util.ts @@ -31,7 +31,7 @@ /** * @fileoverview General Bots local utility. */ - + 'use strict'; import * as YAML from 'yaml'; import SwaggerClient from 'swagger-client'; @@ -48,6 +48,7 @@ import { GBLogEx } from '../packages/core.gbapp/services/GBLogEx.js'; import { PngPageOutput, pdfToPng } from 'pdf-to-png-converter'; import urlJoin from 'url-join'; import { GBServer } from './app.js'; +import { QueryTypes } from '@sequelize/core'; export class GBUtil { public static repeat(chr, count) { @@ -110,21 +111,21 @@ export class GBUtil { return acc; }, {}); }; - + const extractedError = extractProps(data); let yamlString = YAML.stringify(extractedError, { indent: 2, // Defines the indentation flowLevel: -1, // Forces inline formatting styles: { '!!null': 'canonical' } // Optional: Customize null display } as any); - - - //yamlString = yamlString.slice(0, 256); // Truncate to 1024 bytes - - + + + //yamlString = yamlString.slice(0, 256); // Truncate to 1024 bytes + + return yamlString; } - + public static sleep(ms) { return new Promise(resolve => { setTimeout(resolve, ms); @@ -207,6 +208,26 @@ export class GBUtil { } } } + + public static async listTables(dialect: any, seq: any) { + let tables; + if (dialect === 'mssql') { + tables = await seq.query( + `SELECT table_name, table_schema + FROM information_schema.tables + WHERE table_type = 'BASE TABLE' + ORDER BY table_name ASC`, + { + type: QueryTypes.RAW + } + )[0]; + } else if (dialect === 'mariadb') { + tables = await seq.getQueryInterface().showAllTables(); + } + return tables; + } + + // Check if is a tree or flat object. public static hasSubObject(t) { @@ -251,9 +272,9 @@ export class GBUtil { public static async pdfPageAsImage(min, filename, pageNumber) { // Converts the PDF to PNG. - + GBLogEx.info(min, `Converting ${filename}, page: ${pageNumber ?? 'all'}...`); - + const options = { disableFontFace: true, useSystemFonts: true, @@ -262,27 +283,27 @@ export class GBUtil { strictPagesToProcess: false, verbosityLevel: 0 }; - + const pngPages: PngPageOutput[] = await pdfToPng(filename, options); - + const generatedFiles = []; - + for (const pngPage of pngPages) { const buffer = pngPage.content; const gbaiName = GBUtil.getGBAIPath(min.botId, null); const localName = path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`); const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName)); - + await fs.writeFile(localName, buffer, { encoding: null }); - + generatedFiles.push({ localName: localName, url: url, data: buffer }); } - + return generatedFiles.length > 0 ? generatedFiles : null; } public static async sleepRandom(min = 1, max = 5) { - const randomDelay = Math.floor(Math.random() * (max - min + 1) + min) * 1000; + const randomDelay = Math.floor(Math.random() * (max - min + 1) + min) * 1000; await new Promise(resolve => setTimeout(resolve, randomDelay)); - } + } } diff --git a/templates/_drafts/whatsapp.gbai/whatsapp.gbdialog/create-task.bas b/templates/_drafts/whatsapp.gbai/whatsapp.gbdialog/create-task.bas new file mode 100644 index 00000000..2453b1c6 --- /dev/null +++ b/templates/_drafts/whatsapp.gbai/whatsapp.gbdialog/create-task.bas @@ -0,0 +1 @@ +REM Use in groups. \ No newline at end of file diff --git a/templates/talk-to-data.gbai/talk-to-data.gbdialog/notify-latest-orders.bas b/templates/talk-to-data.gbai/talk-to-data.gbdialog/notify-latest-orders.bas new file mode 100644 index 00000000..b897c129 --- /dev/null +++ b/templates/talk-to-data.gbai/talk-to-data.gbdialog/notify-latest-orders.bas @@ -0,0 +1,7 @@ +REM SET SCHEDULE + +REFRESH "llm" + +list = REWRITE "A list of latest 10 orders made." + +TALK TO admin \ No newline at end of file diff --git a/templates/talk-to-data.gbai/talk-to-data.gbot/config.csv b/templates/talk-to-data.gbai/talk-to-data.gbot/config.csv index dbcf8940..11fdc433 100644 --- a/templates/talk-to-data.gbai/talk-to-data.gbot/config.csv +++ b/templates/talk-to-data.gbai/talk-to-data.gbot/config.csv @@ -1,4 +1,5 @@ name,value +Admin, 5521999998888 Answer Mode,sql llm File,northwind.db llm Driver,sqlite