From 789fd79777aa120a17845df8c99302d4376fc5e0 Mon Sep 17 00:00:00 2001 From: "me@rodrigorodriguez.com" Date: Wed, 16 Oct 2024 02:50:24 -0300 Subject: [PATCH] new(core.gbapp): LLM alerts for data. --- .../services/KeywordsExpressions.ts | 8 + .../basic.gblib/services/SystemKeywords.ts | 173 +++++++++--------- src/util.ts | 2 +- .../notify-latest-orders.bas | 4 +- 4 files changed, 102 insertions(+), 85 deletions(-) diff --git a/packages/basic.gblib/services/KeywordsExpressions.ts b/packages/basic.gblib/services/KeywordsExpressions.ts index 0a31e111..45397139 100644 --- a/packages/basic.gblib/services/KeywordsExpressions.ts +++ b/packages/basic.gblib/services/KeywordsExpressions.ts @@ -481,6 +481,14 @@ export class KeywordsExpressions { } ]; + keywords[i++] = [ + /^\s*(.*)\=\s*(ANSWER)(\s*)(.*)/gim, + ($0, $1, $2, $3, $4) => { + const params = this.getParams($4, ['text']); + return `${$1} = await sys.answer ({pid: pid, ${params}})`; + } + ]; + keywords[i++] = [ /^\s*(.*)\=\s*(GET IMAGE)(\s*)(.*)/gim, ($0, $1, $2, $3, $4) => { diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index 711e5649..d0c39e96 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -2460,6 +2460,16 @@ export class SystemKeywords { GBLogEx.info(min, `BlueSky Automation: ${text}.`); } + + /** + */ + public async answer({ pid, text }) { + const { min, user } = await DialogKeywords.getProcessInfo(pid); + const answer = await ChatServices.answerByLLM(pid, min, user, text) + GBLogEx.info(min, `ANSWER ${text} TO ${answer}`); + return answer.answer; + } + /** * HEAR description * text = REWRITE description @@ -2879,89 +2889,88 @@ export class SystemKeywords { await this.setMemoryContext({ pid, erase: true }); } - + public async refreshDataSourceCache({ pid, connectionName }) { - const { min, user, params, step } = await DialogKeywords.getProcessInfo(pid); - - let sqliteFilePath =path.join('work', GBUtil.getGBAIPath(min.botId), `${connectionName}.sqlite`); - - // Step 1: Clean the SQLite file if it already exists - if (await GBUtil.exists(sqliteFilePath)) { - await fs.unlink(sqliteFilePath); // Remove the file - GBLogEx.info(min, `${sqliteFilePath} has been cleaned.`); - } - - // Step 2: Connect to SQLite (Local) - const sqlite = new Sequelize({ - dialect: 'sqlite', - storage: sqliteFilePath - }); - - // Get the connection details from the min object - let con = min[connectionName]; - const dialect = con.dialect.name; - - // Step 3: Get the list of all tables from the source database - const tables = await GBUtil.listTables(dialect, min.core.sequelize); - const tableNames = tables.map(table => Object.values(table)[0]); - - // Function to map source database datatypes to SQLite-compatible datatypes - const mapToSQLiteType = (columnType) => { - const typeMapping = { - 'VARCHAR': DataTypes.STRING, - 'CHAR': DataTypes.STRING, - 'TEXT': DataTypes.TEXT, - 'TINYINT': DataTypes.INTEGER, - 'SMALLINT': DataTypes.INTEGER, - 'MEDIUMINT': DataTypes.INTEGER, - 'INT': DataTypes.INTEGER, - 'INTEGER': DataTypes.INTEGER, - 'BIGINT': DataTypes.BIGINT, - 'FLOAT': DataTypes.FLOAT, - 'DOUBLE': DataTypes.DOUBLE, - 'DECIMAL': DataTypes.DECIMAL, - 'DATE': DataTypes.DATE, - 'DATETIME': DataTypes.DATE, - 'TIMESTAMP': DataTypes.DATE, - 'BLOB': DataTypes.BLOB, - 'BOOLEAN': DataTypes.BOOLEAN, - // Add more mappings as needed - }; - - // Return mapped type or fallback to STRING if not mapped - return typeMapping[columnType.toUpperCase()] || DataTypes.STRING; + const { min, user, params, step } = await DialogKeywords.getProcessInfo(pid); + + let sqliteFilePath = path.join('work', GBUtil.getGBAIPath(min.botId), `${connectionName}.sqlite`); + + // Step 1: Clean the SQLite file if it already exists + if (await GBUtil.exists(sqliteFilePath)) { + await fs.unlink(sqliteFilePath); // Remove the file + GBLogEx.info(min, `${sqliteFilePath} has been cleaned.`); + } + + // Step 2: Connect to SQLite (Local) + const sqlite = new Sequelize({ + dialect: 'sqlite', + storage: sqliteFilePath + }); + + // Get the connection details from the min object + let con = min[connectionName]; + const dialect = con.dialect.name; + + // Step 3: Get the list of all tables from the source database + const tables = await GBUtil.listTables(dialect, min.core.sequelize); + + // Function to map source database datatypes to SQLite-compatible datatypes + const mapToSQLiteType = (columnType) => { + const typeMapping = { + 'VARCHAR': DataTypes.STRING, + 'CHAR': DataTypes.STRING, + 'TEXT': DataTypes.TEXT, + 'TINYINT': DataTypes.INTEGER, + 'SMALLINT': DataTypes.INTEGER, + 'MEDIUMINT': DataTypes.INTEGER, + 'INT': DataTypes.INTEGER, + 'INTEGER': DataTypes.INTEGER, + 'BIGINT': DataTypes.BIGINT, + 'FLOAT': DataTypes.FLOAT, + 'DOUBLE': DataTypes.DOUBLE, + 'DECIMAL': DataTypes.DECIMAL, + 'DATE': DataTypes.DATE, + 'DATETIME': DataTypes.DATE, + 'TIMESTAMP': DataTypes.DATE, + 'BLOB': DataTypes.BLOB, + 'BOOLEAN': DataTypes.BOOLEAN, + // Add more mappings as needed }; - - // Step 4: Retrieve and export data for each table - for (const table of tableNames) { - // Retrieve rows from the source table - const [rows] = await min.core.sequelize.query(`SELECT * FROM ${table}`); - - if (rows.length === 0) continue; // Skip if the table has no data - - // Get the schema for the current table from the source database - const columns = await min.core.sequelize.queryInterface.describeTable(table); - - // Create a schema object for SQLite - const schema = {}; - Object.keys(columns).forEach(col => { - const columnType = columns[col].type; - schema[col] = mapToSQLiteType(columnType); // Map source type to SQLite type - }); - - // Define the model dynamically for each table in SQLite - const Model = sqlite.define(table, schema, { timestamps: false }); - - // Sync the model (create table) - await Model.sync({ force: true }); - - // Bulk insert rows into the SQLite table - await Model.bulkCreate(rows); - } - - GBLogEx.info(min, `All tables have been successfully exported to ${sqliteFilePath}`); - - // Close SQLite connection - await sqlite.close(); + + // Return mapped type or fallback to STRING if not mapped + return typeMapping[columnType.toUpperCase()] || DataTypes.STRING; + }; + + // Step 4: Retrieve and export data for each table + for (const table of tables) { + // Retrieve rows from the source table + const [rows] = await min.core.sequelize.query(`SELECT * FROM ${table}`); + + if (rows.length === 0) continue; // Skip if the table has no data + + // Get the schema for the current table from the source database + const columns = await min.core.sequelize.queryInterface.describeTable(table); + + // Create a schema object for SQLite + const schema = {}; + Object.keys(columns).forEach(col => { + const columnType = columns[col].type; + schema[col] = mapToSQLiteType(columnType); // Map source type to SQLite type + }); + + // Define the model dynamically for each table in SQLite + const Model = sqlite.define(table, schema, { timestamps: false }); + + // Sync the model (create table) + await Model.sync({ force: true }); + + // Bulk insert rows into the SQLite table + await Model.bulkCreate(rows); + } + + GBLogEx.info(min, `All tables have been successfully exported to ${sqliteFilePath}`); + + // Close SQLite connection + await sqlite.close(); } } diff --git a/src/util.ts b/src/util.ts index 137a574a..07226600 100644 --- a/src/util.ts +++ b/src/util.ts @@ -221,7 +221,7 @@ export class GBUtil { type: QueryTypes.RAW } )[0]; - } else if (dialect === 'mariadb') { + } else { tables = await seq.getQueryInterface().showAllTables(); } return tables; diff --git a/templates/talk-to-data.gbai/talk-to-data.gbdialog/notify-latest-orders.bas b/templates/talk-to-data.gbai/talk-to-data.gbdialog/notify-latest-orders.bas index 2315ba62..05105ae8 100644 --- a/templates/talk-to-data.gbai/talk-to-data.gbdialog/notify-latest-orders.bas +++ b/templates/talk-to-data.gbai/talk-to-data.gbdialog/notify-latest-orders.bas @@ -2,6 +2,6 @@ REM SET SCHEDULE REFRESH "llm" -list = REWRITE "A list of latest 10 orders made." +list = ANSWER "A list of latest 10 orders made." -TALK TO admin \ No newline at end of file +TALK "The Report is: \n" + list \ No newline at end of file