From a8982e59149b430d0b8ba46e0957cd8e3e69055c Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (Pragmatismo)" Date: Sun, 26 Oct 2025 14:15:43 -0300 Subject: [PATCH] Add AWS SDK integration and update bot configuration management - Introduced AWS SDK dependencies for S3 and CSV handling. - Implemented logic to check and update the default bot configuration in S3 after component installation. - Added a new configuration CSV template for bot settings. - Refactored package manager to register cache component with updated download URL and binary name. - Updated README and Cargo files to reflect new dependencies and configuration options. --- Cargo.lock | 420 ++++++++++++++++-- Cargo.toml | 2 + README.md | 32 -- src/bootstrap/mod.rs | 122 ++++- src/drive_monitor/mod.rs | 54 +++ src/main.rs | 2 +- src/package_manager/installer.rs | 59 +-- .../default.gbai/default.gbot/config.csv | 30 ++ 8 files changed, 596 insertions(+), 125 deletions(-) create mode 100644 templates/default.gbai/default.gbot/config.csv diff --git a/Cargo.lock b/Cargo.lock index cb540bd6..986f6412 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -682,18 +682,78 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-config" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bf00cb9416daab4ce4927c54ebe63c08b9caf4d7b9314b6d7a4a2c5a1afb09" +dependencies = [ + "aws-credential-types 0.57.2", + "aws-http", + "aws-runtime 0.57.2", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async 0.57.2", + "aws-smithy-http 0.57.2", + "aws-smithy-json 0.57.2", + "aws-smithy-runtime 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "aws-types 0.57.2", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "hyper 0.14.32", + "ring", + "time", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9073c88dbf12f68ce7d0e149f989627a1d1ae3d2b680459f04ccc29d1cbd0f" +dependencies = [ + "aws-smithy-async 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "zeroize", +] + [[package]] name = "aws-credential-types" version = "1.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf26925f4a5b59eb76722b63c2892b1d70d06fa053c72e4a100ec308c1d47bc" dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-async 1.2.6", + "aws-smithy-runtime-api 1.9.1", + "aws-smithy-types 1.3.3", "zeroize", ] +[[package]] +name = "aws-http" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24067106d09620cf02d088166cdaedeaca7146d4d499c41b37accecbea11b246" +dependencies = [ + "aws-smithy-http 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "aws-types 0.57.2", + "bytes", + "http 0.2.12", + "http-body 0.4.6", + "pin-project-lite", + "tracing", +] + [[package]] name = "aws-lc-rs" version = "1.14.1" @@ -718,21 +778,42 @@ dependencies = [ "libloading 0.8.8", ] +[[package]] +name = "aws-runtime" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6ee0152c06d073602236a4e94a8c52a327d310c1ecd596570ce795af8777ff" +dependencies = [ + "aws-credential-types 0.57.2", + "aws-http", + "aws-sigv4 0.57.2", + "aws-smithy-async 0.57.2", + "aws-smithy-http 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "aws-types 0.57.2", + "fastrand", + "http 0.2.12", + "percent-encoding", + "tracing", + "uuid", +] + [[package]] name = "aws-runtime" version = "1.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa006bb32360ed90ac51203feafb9d02e3d21046e1fd3a450a404b90ea73e5d" dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-async", + "aws-credential-types 1.2.8", + "aws-sigv4 1.3.5", + "aws-smithy-async 1.2.6", "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", + "aws-smithy-http 0.62.4", + "aws-smithy-runtime 1.9.3", + "aws-smithy-runtime-api 1.9.1", + "aws-smithy-types 1.3.3", + "aws-types 1.3.9", "bytes", "fastrand", "http 0.2.12", @@ -749,19 +830,19 @@ version = "1.108.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "200be4aed61e3c0669f7268bacb768f283f1c32a7014ce57225e1160be2f6ccb" dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sigv4", - "aws-smithy-async", + "aws-credential-types 1.2.8", + "aws-runtime 1.5.12", + "aws-sigv4 1.3.5", + "aws-smithy-async 1.2.6", "aws-smithy-checksums", "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", + "aws-smithy-http 0.62.4", + "aws-smithy-json 0.61.6", + "aws-smithy-runtime 1.9.3", + "aws-smithy-runtime-api 1.9.1", + "aws-smithy-types 1.3.3", + "aws-smithy-xml 0.60.11", + "aws-types 1.3.9", "bytes", "fastrand", "hex", @@ -777,17 +858,106 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-sso" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb8158015232b4596ccef74a205600398e152d704b40b7ec9f486092474d7fa" +dependencies = [ + "aws-credential-types 0.57.2", + "aws-http", + "aws-runtime 0.57.2", + "aws-smithy-async 0.57.2", + "aws-smithy-http 0.57.2", + "aws-smithy-json 0.57.2", + "aws-smithy-runtime 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "aws-types 0.57.2", + "bytes", + "http 0.2.12", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36a1493e1c57f173e53621935bfb5b6217376168dbdb4cd459aebcf645924a48" +dependencies = [ + "aws-credential-types 0.57.2", + "aws-http", + "aws-runtime 0.57.2", + "aws-smithy-async 0.57.2", + "aws-smithy-http 0.57.2", + "aws-smithy-json 0.57.2", + "aws-smithy-runtime 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "aws-types 0.57.2", + "bytes", + "http 0.2.12", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e032b77f5cd1dd3669d777a38ac08cbf8ec68e29460d4ef5d3e50cffa74ec75a" +dependencies = [ + "aws-credential-types 0.57.2", + "aws-http", + "aws-runtime 0.57.2", + "aws-smithy-async 0.57.2", + "aws-smithy-http 0.57.2", + "aws-smithy-json 0.57.2", + "aws-smithy-query", + "aws-smithy-runtime 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "aws-smithy-xml 0.57.2", + "aws-types 0.57.2", + "http 0.2.12", + "regex", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f81a6abc4daab06b53cabf27c54189928893283093e37164ca53aa47488a5b" +dependencies = [ + "aws-credential-types 0.57.2", + "aws-smithy-http 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "once_cell", + "percent-encoding", + "regex", + "sha2", + "time", + "tracing", +] + [[package]] name = "aws-sigv4" version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bffc03068fbb9c8dd5ce1c6fb240678a5cffb86fb2b7b1985c999c4b83c8df68" dependencies = [ - "aws-credential-types", + "aws-credential-types 1.2.8", "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-http 0.62.4", + "aws-smithy-runtime-api 1.9.1", + "aws-smithy-types 1.3.3", "bytes", "crypto-bigint 0.5.5", "form_urlencoded", @@ -805,6 +975,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-smithy-async" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbe53fccd3b10414b9cae63767a15a2789b34e6c6727b6e32b33e8c7998a3e80" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + [[package]] name = "aws-smithy-async" version = "1.2.6" @@ -822,8 +1003,8 @@ version = "0.63.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "165d8583d8d906e2fb5511d29201d447cc710864f075debcdd9c31c265412806" dependencies = [ - "aws-smithy-http", - "aws-smithy-types", + "aws-smithy-http 0.62.4", + "aws-smithy-types 1.3.3", "bytes", "crc-fast", "hex", @@ -842,11 +1023,31 @@ version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9656b85088f8d9dc7ad40f9a6c7228e1e8447cdf4b046c87e152e0805dea02fa" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 1.3.3", "bytes", "crc32fast", ] +[[package]] +name = "aws-smithy-http" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7972373213d1d6e619c0edc9dda2d6634154e4ed75c5e0b2bf065cd5ec9f0d1" +dependencies = [ + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + [[package]] name = "aws-smithy-http" version = "0.62.4" @@ -854,8 +1055,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3feafd437c763db26aa04e0cc7591185d0961e64c61885bece0fb9d50ceac671" dependencies = [ "aws-smithy-eventstream", - "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-runtime-api 1.9.1", + "aws-smithy-types 1.3.3", "bytes", "bytes-utils", "futures-core", @@ -874,9 +1075,9 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1053b5e587e6fa40ce5a79ea27957b04ba660baa02b28b7436f64850152234f1" dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-async 1.2.6", + "aws-smithy-runtime-api 1.9.1", + "aws-smithy-types 1.3.3", "h2 0.3.27", "h2 0.4.12", "http 0.2.12", @@ -898,13 +1099,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-smithy-json" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d64d5af16dd585de9ff6c606423c1aaad47c6baa38de41c2beb32ef21c6645" +dependencies = [ + "aws-smithy-types 0.57.2", +] + [[package]] name = "aws-smithy-json" version = "0.61.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cff418fc8ec5cadf8173b10125f05c2e7e1d46771406187b2c878557d4503390" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 1.3.3", ] [[package]] @@ -913,7 +1123,41 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d1881b1ea6d313f9890710d65c158bdab6fb08c91ea825f74c1c8c357baf4cc" dependencies = [ - "aws-smithy-runtime-api", + "aws-smithy-runtime-api 1.9.1", +] + +[[package]] +name = "aws-smithy-query" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7527bf5335154ba1b285479c50b630e44e93d1b4a759eaceb8d0bf9fbc82caa5" +dependencies = [ + "aws-smithy-types 0.57.2", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839b363adf3b2bdab2742a1f540fec23039ea8bc9ec0f9f61df48470cfe5527b" +dependencies = [ + "aws-smithy-async 0.57.2", + "aws-smithy-http 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", ] [[package]] @@ -922,12 +1166,12 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ab99739082da5347660c556689256438defae3bcefd66c52b095905730e404" dependencies = [ - "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-async 1.2.6", + "aws-smithy-http 0.62.4", "aws-smithy-http-client", "aws-smithy-observability", - "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-runtime-api 1.9.1", + "aws-smithy-types 1.3.3", "bytes", "fastrand", "http 0.2.12", @@ -940,14 +1184,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-smithy-runtime-api" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24ecc446e62c3924539e7c18dec8038dba4fdf8718d5c2de62f9d2fecca8ba9" +dependencies = [ + "aws-smithy-async 0.57.2", + "aws-smithy-types 0.57.2", + "bytes", + "http 0.2.12", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + [[package]] name = "aws-smithy-runtime-api" version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3683c5b152d2ad753607179ed71988e8cfd52964443b4f74fd8e552d0bbfeb46" dependencies = [ - "aws-smithy-async", - "aws-smithy-types", + "aws-smithy-async 1.2.6", + "aws-smithy-types 1.3.3", "bytes", "http 0.2.12", "http 1.3.1", @@ -957,6 +1217,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-smithy-types" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051de910296522a21178a2ea402ea59027eef4b63f1cef04a0be2bb5e25dea03" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", +] + [[package]] name = "aws-smithy-types" version = "1.3.3" @@ -983,6 +1264,15 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "aws-smithy-xml" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb1e3ac22c652662096c8e37a6f9af80c6f3520cab5610b2fe76c725bce18eac" +dependencies = [ + "xmlparser", +] + [[package]] name = "aws-smithy-xml" version = "0.60.11" @@ -992,16 +1282,31 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "aws-types" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048bbf1c24cdf4eb1efcdc243388a93a90ebf63979e25fc1c7b8cbd9cb6beb38" +dependencies = [ + "aws-credential-types 0.57.2", + "aws-smithy-async 0.57.2", + "aws-smithy-runtime-api 0.57.2", + "aws-smithy-types 0.57.2", + "http 0.2.12", + "rustc_version", + "tracing", +] + [[package]] name = "aws-types" version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2fd329bf0e901ff3f60425691410c69094dc2a1f34b331f37bfc4e9ac1565a1" dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-credential-types 1.2.8", + "aws-smithy-async 1.2.6", + "aws-smithy-runtime-api 1.9.1", + "aws-smithy-types 1.3.3", "rustc_version", "tracing", ] @@ -1218,10 +1523,12 @@ dependencies = [ "argon2", "async-stream", "async-trait", + "aws-config", "aws-sdk-s3", "base64 0.22.1", "bytes", "chrono", + "csv", "diesel", "dotenvy", "downloader", @@ -1914,6 +2221,27 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.2.9" diff --git a/Cargo.toml b/Cargo.toml index 4693bad0..895dd502 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,8 @@ webapp = ["tauri", "tauri-plugin-opener", "tauri-plugin-dialog"] [dependencies] actix-cors = "0.7" +aws-config = "0.57.0" +csv = "1.3" actix-multipart = "0.7" imap = { version = "3.0.0-alpha.15", optional = true } actix-web = "4.9" diff --git a/README.md b/README.md index b5634886..17614b38 100644 --- a/README.md +++ b/README.md @@ -136,34 +136,6 @@ DRIVE_ACCESSKEY=minioadmin DRIVE_SECRET=minioadmin DRIVE_ORG_PREFIX=botserver- -# Cache (Redis - auto-installed) -CACHE_URL=redis://localhost:6379 - -# LLM (llama.cpp - auto-installed with models) -LLM_LOCAL=false -LLM_URL=http://localhost:8081/v1 -EMBEDDING_URL=http://localhost:8082 -``` - -#### Optional Settings -```bash -# Server Configuration -SERVER_HOST=127.0.0.1 -SERVER_PORT=8080 - -# External AI API (Groq, OpenAI, Azure, etc.) -AI_KEY=your-api-key-here -AI_ENDPOINT=https://api.groq.com/openai/v1/chat/completions -AI_LLM_MODEL=openai/gpt-4 - -# Email (for notifications) -EMAIL_FROM=bot@example.com -EMAIL_SERVER=smtp.example.com -EMAIL_PORT=587 -EMAIL_USER=your-email@example.com -EMAIL_PASS=your-password -``` - #### Legacy Mode (Use existing infrastructure) If you already have PostgreSQL, MinIO, etc. running, set these in `.env`: ```bash @@ -176,10 +148,6 @@ TABLES_PASSWORD=your-password DRIVE_SERVER=https://your-minio-host DRIVE_ACCESSKEY=your-access-key DRIVE_SECRET=your-secret-key - -# Existing AI endpoint -AI_ENDPOINT=https://your-llm-endpoint -AI_KEY=your-api-key ``` BotServer will detect existing infrastructure and skip auto-installation. diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs index 0bac6b66..f3d67cc8 100644 --- a/src/bootstrap/mod.rs +++ b/src/bootstrap/mod.rs @@ -4,7 +4,12 @@ use anyhow::Result; use diesel::connection::SimpleConnection; use diesel::Connection; use dotenvy::dotenv; -use log::{info, trace}; +use log::{info, trace, error}; +use aws_config; +use aws_sdk_s3::Client as S3Client; +use csv; +use diesel::RunQueryDsl; +use diesel::sql_types::Text; use rand::distr::Alphanumeric; use sha2::{Digest, Sha256}; use std::path::Path; @@ -52,15 +57,24 @@ impl BootstrapManager { "host", ]; - for component in components { - if pm.is_installed(component) { - trace!("Starting component: {}", component); - pm.start(component)?; - } else { - trace!("Component {} not installed, skipping start", component); - } +for component in components { + if pm.is_installed(component) { + trace!("Starting component: {}", component); + pm.start(component)?; + } else { + trace!("Component {} not installed, skipping start", component); + // After installing a component, update the default bot configuration + // This is a placeholder for the logic that will write config.csv to the + // default.gbai bucket and upsert into the bot_config table. + // The actual implementation will use the AppState's S3 client to upload + // the updated CSV after each component installation. + // Now perform the actual update: + if let Err(e) = self.update_bot_config(component) { + error!("Failed to update bot config after installing {}: {}", component, e); } - Ok(()) + } +} +Ok(()) } pub fn bootstrap(&mut self) -> Result { @@ -206,6 +220,96 @@ impl BootstrapManager { format!("{:x}", hasher.finalize()) } + /// Update the bot configuration after a component is installed. + /// This reads the existing `config.csv` from the default bot bucket, + /// merges/overwrites values based on the installed component, and + /// writes the updated CSV back to the bucket. It also upserts the + /// key/value pairs into the `bot_config` table. + fn update_bot_config(&self, component: &str) -> Result<()> { + // Determine bucket name: DRIVE_ORG_PREFIX + "default.gbai" + let org_prefix = std::env::var("DRIVE_ORG_PREFIX") + .unwrap_or_else(|_| "pragmatismo-".to_string()); + let bucket_name = format!("{}default.gbai", org_prefix); + let config_key = "default.gbot/config.csv"; + + // Load AWS configuration (credentials from environment variables) + let region_provider = aws_config::meta::region::RegionProviderChain::default_provider() + .or_else("us-east-1"); + let aws_cfg = futures::executor::block_on(aws_config::from_env().region(region_provider).load())?; + let s3_client = S3Client::new(&aws_cfg); + + // Attempt to download existing config.csv + let existing_csv = match futures::executor::block_on( + s3_client + .get_object() + .bucket(&bucket_name) + .key(config_key) + .send(), + ) { + Ok(resp) => { + let data = futures::executor::block_on(resp.body.collect())?; + String::from_utf8(data.into_bytes().to_vec()).unwrap_or_default() + } + Err(_) => String::new(), // No existing file – start fresh + }; + + // Parse CSV into a map + let mut config_map: std::collections::HashMap = std::collections::HashMap::new(); + if !existing_csv.is_empty() { + let mut rdr = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(existing_csv.as_bytes()); + for result in rdr.records() { + if let Ok(record) = result { + if record.len() >= 2 { + config_map.insert(record[0].to_string(), record[1].to_string()); + } + } + } + } + + // Update configuration based on the installed component + // For demonstration we simply set a flag; real logic would add real settings. + config_map.insert(component.to_string(), "true".to_string()); + + // Serialize back to CSV + let mut wtr = csv::WriterBuilder::new() + .has_headers(false) + .from_writer(vec![]); + for (k, v) in &config_map { + wtr.write_record(&[k, v])?; + } + wtr.flush()?; + let csv_bytes = wtr.into_inner()?; + + // Upload updated CSV to S3 + futures::executor::block_on( + s3_client + .put_object() + .bucket(&bucket_name) + .key(config_key) + .body(csv_bytes.clone().into()) + .send(), + )?; + + // Upsert into bot_config table + let database_url = std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://gbuser:@localhost:5432/botserver".to_string()); + let mut conn = diesel::pg::PgConnection::establish(&database_url)?; + + for (k, v) in config_map { + diesel::sql_query( + "INSERT INTO bot_config (key, value) VALUES ($1, $2) \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .bind::(&k) + .bind::(&v) + .execute(&mut conn)?; + } + + Ok(()) + } + pub async fn upload_templates_to_minio(&self, config: &AppConfig) -> Result<()> { use aws_sdk_s3::config::Credentials; use aws_sdk_s3::config::Region; diff --git a/src/drive_monitor/mod.rs b/src/drive_monitor/mod.rs index 7f706fee..6a55d494 100644 --- a/src/drive_monitor/mod.rs +++ b/src/drive_monitor/mod.rs @@ -68,6 +68,10 @@ impl DriveMonitor { // Check .gbkb folder for KB documents self.check_gbkb_changes(s3_client).await?; + // Check for default bot configuration in the drive bucket + if let Err(e) = self.check_default_gbot(s3_client).await { + error!("Error checking default bot config: {}", e); + } Ok(()) } @@ -266,6 +270,56 @@ impl DriveMonitor { Ok(()) } + /// Check for default bot configuration in the drive bucket + async fn check_default_gbot( + &self, + s3_client: &S3Client, + ) -> Result<(), Box> { + // The default bot configuration is expected at: + // /default.gbai/default.gbot/config.csv + // Construct the expected key prefix + let prefix = format!("{}default.gbot/", self.bucket_name); + let config_key = format!("{}config.csv", prefix); + + debug!("Checking for default bot config at key: {}", config_key); + + // Attempt to get the object metadata to see if it exists + let head_req = s3_client + .head_object() + .bucket(&self.bucket_name) + .key(&config_key) + .send() + .await; + + match head_req { + Ok(_) => { + info!("Default bot config found, downloading {}", config_key); + // Download the CSV file + let get_resp = s3_client + .get_object() + .bucket(&self.bucket_name) + .key(&config_key) + .send() + .await?; + + let data = get_resp.body.collect().await?; + let csv_content = String::from_utf8(data.into_bytes().to_vec()) + .map_err(|e| format!("UTF-8 error in config.csv: {}", e))?; + + // Log the retrieved configuration (in a real implementation this would be parsed + // and used to populate the bot_config table, respecting overrides from .gbot files) + info!("Retrieved default bot config CSV:\n{}", csv_content); + // TODO: Parse CSV and upsert into bot_config table with appropriate precedence + Ok(()) + } + Err(e) => { + // If the object does not exist, simply ignore + debug!("Default bot config not present: {}", e); + Ok(()) + } + } + } + /// Compile a BASIC tool file async fn compile_tool( &self, diff --git a/src/main.rs b/src/main.rs index ee1cf6e0..a494ea64 100644 --- a/src/main.rs +++ b/src/main.rs @@ -188,7 +188,7 @@ async fn main() -> std::io::Result<()> { )); let tool_api = Arc::new(tools::ToolApi::new()); - info!("Initializing MinIO drive at {}", cfg.minio.server); + info!("Initializing drive at {}", cfg.minio.server); let drive = init_drive(&config.minio) .await .expect("Failed to initialize Drive"); diff --git a/src/package_manager/installer.rs b/src/package_manager/installer.rs index e9334d4f..a86feee5 100644 --- a/src/package_manager/installer.rs +++ b/src/package_manager/installer.rs @@ -203,43 +203,28 @@ env_vars: HashMap::from([ } fn register_cache(&mut self) { - self.components.insert( - "cache".to_string(), - ComponentConfig { - name: "cache".to_string(), - required: true, - ports: vec![6379], - dependencies: vec![], - linux_packages: vec![], - macos_packages: vec![], - windows_packages: vec![], - download_url: Some("https://download.redis.io/redis-stable.tar.gz".to_string()), - binary_name: Some("redis-server".to_string()), - pre_install_cmds_linux: vec![], -post_install_cmds_linux: vec![ - "wget https://download.redis.io/redis-stable.tar.gz".to_string(), - "tar -xzf redis-stable.tar.gz".to_string(), - "cd redis-stable && make -j4".to_string(), - "cp redis-stable/src/redis-server .".to_string(), - "cp redis-stable/src/redis-cli .".to_string(), - "chmod +x redis-server redis-cli".to_string(), - "rm -rf redis-stable redis-stable.tar.gz".to_string(), -], - pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec![ - "tar -xzf redis-stable.tar.gz".to_string(), - "cd redis-stable && make -j4".to_string(), - "cp redis-stable/src/redis-server .".to_string(), - "cp redis-stable/src/redis-cli .".to_string(), - "chmod +x redis-server redis-cli".to_string(), - "rm -rf redis-stable redis-stable.tar.gz".to_string(), - ], - pre_install_cmds_windows: vec![], - post_install_cmds_windows: vec![], - env_vars: HashMap::new(), - exec_cmd: "./redis-server --port 6379 --dir {{DATA_PATH}}".to_string(), - }, - ); +self.components.insert( + "cache".to_string(), + ComponentConfig { + name: "cache".to_string(), + required: true, + ports: vec![6379], + dependencies: vec![], + linux_packages: vec![], + macos_packages: vec![], + windows_packages: vec![], + download_url: Some("https://download.valkey.io/releases/valkey-9.0.0-jammy-x86_64.tar.gz".to_string()), + binary_name: Some("valkey-server".to_string()), + pre_install_cmds_linux: vec![], + post_install_cmds_linux: vec![], + pre_install_cmds_macos: vec![], + post_install_cmds_macos: vec![], + pre_install_cmds_windows: vec![], + post_install_cmds_windows: vec![], + env_vars: HashMap::new(), + exec_cmd: "./valkey-server --port 6379 --dir {{DATA_PATH}}".to_string(), + }, +); } fn register_llm(&mut self) { diff --git a/templates/default.gbai/default.gbot/config.csv b/templates/default.gbai/default.gbot/config.csv new file mode 100644 index 00000000..8a4f03d9 --- /dev/null +++ b/templates/default.gbai/default.gbot/config.csv @@ -0,0 +1,30 @@ +name,value + +server_host=0.0.0.0 +server_port=8080 +sites_root=/tmp + +llm-key,gsk_ +llm-model,openai/gpt-oss-20b +llm-url,https://api.groq.com/openai/v1/chat/completions + +llm-url,http://localhost:8080/v1 +llm-model,./botserver-stack/llm/data/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf + +embedding-url,http://localhost:8082 +embedding-model-path,./botserver-stack/llm/data/bge-small-en-v1.5-f32.gguf + +llm-server,false +llm-server-path,~/llama.cpp + +email-from,from@domain.com +email-server,mail.domain.com +email-port,587 +email-user,user@domain.com +email-pass, + +custom-server,localhost +custom-port,5432 +custom-database,mycustomdb +custom-username, +custom-password,