new(core.gbapp): Timestamp fields are now default.

packages/basic.gblib/services/DialogKeywords.ts

@@ -30,39 +30,38 @@
 
 'use strict';
 
+import sgMail from '@sendgrid/mail';
+import { ActivityTypes } from 'botbuilder';
 import { GBLog, GBMinInstance } from 'botlib';
-import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js';
+import * as df from 'date-diff';
-import { ChartServices } from './ChartServices.js';
-import urlJoin from 'url-join';
-import { GBServer } from '../../../src/app.js';
-import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
-import { SecService } from '../../security.gbapp/services/SecService.js';
-import { Jimp } from 'jimp';
-import jsQR from 'jsqr';
-import { SystemKeywords } from './SystemKeywords.js';
-import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
-import { Messages } from '../strings.js';
-import { CollectionUtil } from 'pragmatismo-io-framework';
-import { GBConversationalService } from '../../core.gbapp/services/GBConversationalService.js';
 import fs from 'fs/promises';
 import libphonenumber from 'google-libphonenumber';
-import * as df from 'date-diff';
+import { Jimp } from 'jimp';
+import jsQR from 'jsqr';
+import mammoth from 'mammoth';
+import mime from 'mime-types';
 import tesseract from 'node-tesseract-ocr';
 import path from 'path';
-import sgMail from '@sendgrid/mail';
+import { CollectionUtil } from 'pragmatismo-io-framework';
-import mammoth from 'mammoth';
-import qrcode from 'qrcode';
-import { WebAutomationServices } from './WebAutomationServices.js';
-import QrScanner from 'qr-scanner';
-import pkg from 'whatsapp-web.js';
-import { ActivityTypes } from 'botbuilder';
-const { List, Buttons } = pkg;
-import mime from 'mime-types';
-import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
-import { GBUtil } from '../../../src/util.js';
-import { GBVMService } from './GBVMService.js';
-import { ChatServices } from '../../../packages/llm.gblib/services/ChatServices.js';
 import puppeteer from 'puppeteer';
+import qrcode from 'qrcode';
+import urlJoin from 'url-join';
+import pkg from 'whatsapp-web.js';
+import { ChatServices } from '../../../packages/llm.gblib/services/ChatServices.js';
+import { GBServer } from '../../../src/app.js';
+import { GBUtil } from '../../../src/util.js';
+import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
+import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js';
+import { GBConversationalService } from '../../core.gbapp/services/GBConversationalService.js';
+import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
+import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
+import { SecService } from '../../security.gbapp/services/SecService.js';
+import { Messages } from '../strings.js';
+import { ChartServices } from './ChartServices.js';
+import { GBVMService } from './GBVMService.js';
+import { SystemKeywords } from './SystemKeywords.js';
+import { WebAutomationServices } from './WebAutomationServices.js';
+const { List, Buttons } = pkg;
 
 
 /**
@@ -384,6 +383,7 @@ export class DialogKeywords {
     var year = date.getFullYear();
 
     format = format.replace('MM', GBUtil.padL(month.toString(), 2, '0'));
+    format = format.toLowerCase();
 
     if (format.indexOf('yyyy') > -1) format = format.replace('yyyy', year.toString());
     else if (format.indexOf('yy') > -1) format = format.replace('yy', year.toString().substr(2, 2));

packages/basic.gblib/services/GBVMService.ts

@@ -405,19 +405,7 @@ export class GBVMService extends GBService {
             let tables;
             const dialect = con.dialect.name;
 
-            if (dialect === 'mssql') {
+            tables = await GBUtil.listTables(dialect, seq);
-              tables = await seq.query(
-                `SELECT table_name, table_schema
-          FROM information_schema.tables
-          WHERE table_type = 'BASE TABLE'
-          ORDER BY table_name ASC`,
-                {
-                  type: QueryTypes.RAW
-                }
-              )[0];
-            } else if (dialect === 'mariadb') {
-              tables = await seq.getQueryInterface().showAllTables();
-            }
 
             let found = false;
             tables.forEach(storageTable => {
@@ -457,6 +445,7 @@ export class GBVMService extends GBService {
   }
 
   public async translateBASIC(mainName, filename: any, min: GBMinInstance) {
+
     // Converts General Bots BASIC into regular VBS
 
     let basicCode: string = await fs.readFile(filename, 'utf8');

packages/basic.gblib/services/KeywordsExpressions.ts

@@ -1308,6 +1308,13 @@ export class KeywordsExpressions {
       }
     ];
 
+    keywords[i++] = [
+      /^\s*refresh\s*(.*)/gim,
+      ($0, $1, $2, $3) => {
+        return `await sys.refreshDataSourceCache({pid: pid, connectionName: ${$2}})`;
+      }
+    ];
+  
     keywords[i++] = [
       /^\s*(save)(\s*)(.*)(.*)/gim,
       ($0, $1, $2, $3, $4) => {

packages/basic.gblib/services/SystemKeywords.ts

@@ -48,6 +48,7 @@ import urlJoin from 'url-join';
 import Excel from 'exceljs';
 import { BufferWindowMemory } from 'langchain/memory';
 import csvdb from 'csv-database';
+import { Sequelize, QueryTypes, DataTypes } from '@sequelize/core';
 import packagePath from 'path';
 import ComputerVisionClient from '@azure/cognitiveservices-computervision';
 import ApiKeyCredentials from '@azure/ms-rest-js';
@@ -2877,4 +2878,90 @@ export class SystemKeywords {
 
     await this.setMemoryContext({ pid, erase: true });
   }
+
+  
+  public async refreshDataSourceCache({ pid, connectionName }) {
+      const { min, user, params, step } = await DialogKeywords.getProcessInfo(pid);
+  
+      let sqliteFilePath =path.join('work', GBUtil.getGBAIPath(min.botId), `${connectionName}.sqlite`);
+    
+      // Step 1: Clean the SQLite file if it already exists
+      if (await GBUtil.exists(sqliteFilePath)) {
+          await fs.unlink(sqliteFilePath); // Remove the file
+          GBLogEx.info(min, `${sqliteFilePath} has been cleaned.`);
+      }
+  
+      // Step 2: Connect to SQLite (Local)
+      const sqlite = new Sequelize({
+          dialect: 'sqlite',
+          storage: sqliteFilePath
+      });
+  
+      // Get the connection details from the min object
+      let con = min[connectionName];
+      const dialect = con.dialect.name;
+  
+      // Step 3: Get the list of all tables from the source database
+      const tables = await GBUtil.listTables(dialect, min.core.sequelize);
+      const tableNames = tables.map(table => Object.values(table)[0]);
+  
+      // Function to map source database datatypes to SQLite-compatible datatypes
+      const mapToSQLiteType = (columnType) => {
+          const typeMapping = {
+              'VARCHAR': DataTypes.STRING,
+              'CHAR': DataTypes.STRING,
+              'TEXT': DataTypes.TEXT,
+              'TINYINT': DataTypes.INTEGER,
+              'SMALLINT': DataTypes.INTEGER,
+              'MEDIUMINT': DataTypes.INTEGER,
+              'INT': DataTypes.INTEGER,
+              'INTEGER': DataTypes.INTEGER,
+              'BIGINT': DataTypes.BIGINT,
+              'FLOAT': DataTypes.FLOAT,
+              'DOUBLE': DataTypes.DOUBLE,
+              'DECIMAL': DataTypes.DECIMAL,
+              'DATE': DataTypes.DATE,
+              'DATETIME': DataTypes.DATE,
+              'TIMESTAMP': DataTypes.DATE,
+              'BLOB': DataTypes.BLOB,
+              'BOOLEAN': DataTypes.BOOLEAN,
+              // Add more mappings as needed
+          };
+  
+          // Return mapped type or fallback to STRING if not mapped
+          return typeMapping[columnType.toUpperCase()] || DataTypes.STRING;
+      };
+  
+      // Step 4: Retrieve and export data for each table
+      for (const table of tableNames) {
+          // Retrieve rows from the source table
+          const [rows] = await min.core.sequelize.query(`SELECT * FROM ${table}`);
+  
+          if (rows.length === 0) continue; // Skip if the table has no data
+  
+          // Get the schema for the current table from the source database
+          const columns = await min.core.sequelize.queryInterface.describeTable(table);
+  
+          // Create a schema object for SQLite
+          const schema = {};
+          Object.keys(columns).forEach(col => {
+              const columnType = columns[col].type;
+              schema[col] = mapToSQLiteType(columnType); // Map source type to SQLite type
+          });
+  
+          // Define the model dynamically for each table in SQLite
+          const Model = sqlite.define(table, schema, { timestamps: false });
+  
+          // Sync the model (create table)
+          await Model.sync({ force: true });
+  
+          // Bulk insert rows into the SQLite table
+          await Model.bulkCreate(rows);
+      }
+  
+      GBLogEx.info(min, `All tables have been successfully exported to ${sqliteFilePath}`);
+  
+      // Close SQLite connection
+      await sqlite.close();
+  }
 }

src/util.ts

@@ -48,6 +48,7 @@ import { GBLogEx } from '../packages/core.gbapp/services/GBLogEx.js';
 import { PngPageOutput, pdfToPng } from 'pdf-to-png-converter';
 import urlJoin from 'url-join';
 import { GBServer } from './app.js';
+import { QueryTypes } from '@sequelize/core';
 
 export class GBUtil {
   public static repeat(chr, count) {
@@ -119,7 +120,7 @@ export class GBUtil {
     } as any);
 
 
-      //yamlString = yamlString.slice(0, 256); // Truncate to 1024 bytes
+    //yamlString = yamlString.slice(0, 256); // Truncate to 1024 bytes
 
 
     return yamlString;
@@ -207,6 +208,26 @@ export class GBUtil {
       }
     }
   }
+
+  public static async listTables(dialect: any, seq: any) {
+    let tables;
+    if (dialect === 'mssql') {
+      tables = await seq.query(
+        `SELECT table_name, table_schema
+          FROM information_schema.tables
+          WHERE table_type = 'BASE TABLE'
+          ORDER BY table_name ASC`,
+        {
+          type: QueryTypes.RAW
+        }
+      )[0];
+    } else if (dialect === 'mariadb') {
+      tables = await seq.getQueryInterface().showAllTables();
+    }
+    return tables;
+  }
+
+
   // Check if is a tree or flat object.
 
   public static hasSubObject(t) {

templates/_drafts/whatsapp.gbai/whatsapp.gbdialog/create-task.bas

@@ -0,0 +1 @@
+REM Use in groups.

templates/talk-to-data.gbai/talk-to-data.gbdialog/notify-latest-orders.bas

@@ -0,0 +1,7 @@
+REM SET SCHEDULE
+
+REFRESH "llm"
+
+list = REWRITE "A list of latest 10 orders made."
+
+TALK TO admin 

templates/talk-to-data.gbai/talk-to-data.gbot/config.csv

@@ -1,4 +1,5 @@
 name,value
+Admin, 5521999998888
 Answer Mode,sql
 llm File,northwind.db
 llm Driver,sqlite