diff --git a/Cargo.lock b/Cargo.lock index ed7fcfe2b..d60a3c952 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1032,6 +1032,7 @@ dependencies = [ "futures-util", "headless_chrome", "imap", + "include_dir", "indicatif", "lettre", "livekit", @@ -2855,6 +2856,25 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "1.9.3" diff --git a/Cargo.toml b/Cargo.toml index 23ae4b5ff..dcad4ee0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,7 @@ futures = "0.3" futures-util = "0.3" lettre = { version = "0.11", features = ["smtp-transport", "builder", "tokio1", "tokio1-native-tls"] } livekit = "0.7" +include_dir = "0.7" log = "0.4" mailparse = "0.15" native-tls = "0.2" diff --git a/fix-errors.sh b/fix-errors.sh index 3a6fbd6f7..e3c276f7f 100755 --- a/fix-errors.sh +++ b/fix-errors.sh @@ -24,7 +24,7 @@ dirs=( #"bot" "bootstrap" #"channels" - #"config" + "config" #"context" #"email" #"file" @@ -53,7 +53,7 @@ cat "$PROJECT_ROOT/src/main.rs" >> "$OUTPUT_FILE" echo "" >> "$OUTPUT_FILE" -echo "Compiling..." +echo "Compiling..." cargo build --message-format=short 2>&1 | grep -E 'error' >> "$OUTPUT_FILE" diff --git a/gbot.sh b/gbot.sh index 681f60a0a..dfca0087c 100755 --- a/gbot.sh +++ b/gbot.sh @@ -1,2 +1,2 @@ -clear && \ - RUST_LOG=trace,hyper_util=off cargo build && clear && cargo run +pkill postgres && rm -rf botserver-stack && clear && \ + RUST_LOG=trace,hyper_util=off cargo run diff --git a/prompts/dev/platform/shared.md b/prompts/dev/platform/shared.md index 15034b848..0ec4301d8 100644 --- a/prompts/dev/platform/shared.md +++ b/prompts/dev/platform/shared.md @@ -1,4 +1,6 @@ MOST IMPORTANT CODE GENERATION RULES: +- Use rustc 1.90.0 (1159e78c4 2025-09-14). +- Check for warnings related to use of mut where is dispensable. - No placeholders, never comment/uncomment code, no explanations, no filler text. - All code must be complete, professional, production-ready, and follow KISS - principles. - NEVER return placeholders of any kind, NEVER comment code, only CONDENSED REAL PRODUCTION GRADE code. @@ -14,7 +16,8 @@ MOST IMPORTANT CODE GENERATION RULES: - Return *only the modified* files as a single `.sh` script using `cat`, so the code can be - restored directly. - Pay attention to shared::utils and shared::models to reuse shared things. - NEVER return a untouched file in output. Just files that need to be updated. - +- Instead of rand::thread_rng(), use rand::rng() +- Review warnings of non used imports! Give me 0 warnings, please. - You MUST return exactly this example format: ```sh #!/bin/bash diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs index f4af8fb37..ae52de2cc 100644 --- a/src/bootstrap/mod.rs +++ b/src/bootstrap/mod.rs @@ -1,9 +1,9 @@ use crate::config::AppConfig; use crate::package_manager::{InstallMode, PackageManager}; use anyhow::Result; -use log::{trace, warn}; +use diesel::{Connection, RunQueryDsl}; +use log::trace; use rand::distr::Alphanumeric; -use rand::rngs::ThreadRng; use rand::Rng; use sha2::{Digest, Sha256}; @@ -63,7 +63,8 @@ impl BootstrapManager { pub fn bootstrap(&mut self) -> Result { let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?; - let required_components = vec!["tables"]; // , "cache", "drive", "llm"]; + let required_components = vec!["tables"]; + let mut config = AppConfig::from_env(); for component in required_components { if !pm.is_installed(component) { @@ -71,46 +72,68 @@ impl BootstrapManager { futures::executor::block_on(pm.install(component))?; trace!("Starting component after install: {}", component); pm.start(component)?; + + if component == "tables" { + let db_password = self.generate_secure_password(16); + let farm_password = self.generate_secure_password(32); + let encrypted_db_password = self.encrypt_password(&db_password, &farm_password); + + let env_contents = format!( + "FARM_PASSWORD={}\nDATABASE_URL=postgres://gbuser:{}@localhost:5432/botserver", + farm_password, db_password + ); + + std::fs::write(".env", env_contents) + .map_err(|e| anyhow::anyhow!("Failed to write .env file: {}", e))?; + + trace!("Waiting 5 seconds for database to start..."); + std::thread::sleep(std::time::Duration::from_secs(5)); + + let migration_dir = include_dir::include_dir!("./migrations"); + let mut migration_files: Vec<_> = migration_dir + .files() + .filter_map(|file| { + let path = file.path(); + if path.extension()? == "sql" { + Some(path.to_path_buf()) + } else { + None + } + }) + .collect(); + + migration_files.sort(); + let mut conn = diesel::PgConnection::establish(&format!( + "postgres://gbuser:{}@localhost:5432/botserver", + db_password + ))?; + + for migration_file in migration_files { + let migration = std::fs::read_to_string(&migration_file)?; + trace!("Executing migration: {}", migration_file.display()); + diesel::sql_query(&migration).execute(&mut conn)?; + } + + self.setup_secure_credentials(&mut conn, &encrypted_db_password)?; + config = AppConfig::from_database(&mut conn); + } } else { trace!("Required component {} already installed", component); } } - let config = match diesel::Connection::establish( - "postgres://botserver:botserver@localhost:5432/botserver", - ) { - Ok(mut conn) => { - self.setup_secure_credentials(&mut conn)?; - AppConfig::from_database(&mut conn) - } - Err(e) => { - warn!("Failed to connect to database for config: {}", e); - AppConfig::from_env() - } - }; - Ok(config) } - fn setup_secure_credentials(&self, conn: &mut diesel::PgConnection) -> Result<()> { + fn setup_secure_credentials( + &self, + conn: &mut diesel::PgConnection, + encrypted_db_password: &str, + ) -> Result<()> { use crate::shared::models::schema::bots::dsl::*; use diesel::prelude::*; use uuid::Uuid; - let farm_password = - std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32)); - let db_password = self.generate_secure_password(16); - - let encrypted_db_password = self.encrypt_password(&db_password, &farm_password); - - let env_contents = format!( - "FARM_PASSWORD={}\nDATABASE_URL=postgres://gbuser:{}@localhost:5432/botserver", - farm_password, db_password - ); - - std::fs::write(".env", env_contents) - .map_err(|e| anyhow::anyhow!("Failed to write .env file: {}", e))?; - let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?; diesel::update(bots) .filter(bot_id.eq(system_bot_id)) @@ -123,7 +146,7 @@ impl BootstrapManager { } fn generate_secure_password(&self, length: usize) -> String { - let rng: ThreadRng = rand::rng(); + let rng = rand::rng(); rng.sample_iter(&Alphanumeric) .take(length) .map(char::from) diff --git a/src/package_manager/facade.rs b/src/package_manager/facade.rs index eed00bd32..f2b2075c9 100644 --- a/src/package_manager/facade.rs +++ b/src/package_manager/facade.rs @@ -132,7 +132,10 @@ impl PackageManager { if !packages.is_empty() { let pkg_list = packages.join(" "); - self.exec_in_container(&container_name, &format!("apt-get update && apt-get install -y {}", pkg_list))?; + self.exec_in_container( + &container_name, + &format!("apt-get update && apt-get install -y {}", pkg_list), + )?; } if let Some(url) = &component.download_url { @@ -266,10 +269,8 @@ impl PackageManager { match self.os_type { OsType::Linux => { - let output = Command::new("apt-get") - .args(&["update"]) - .output()?; - + let output = Command::new("apt-get").args(&["update"]).output()?; + if !output.status.success() { warn!("apt-get update had issues"); } @@ -569,6 +570,7 @@ impl PackageManager { .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()); if target == "local" { + trace!("Executing command: {}", rendered_cmd); let output = Command::new("bash") .current_dir(&bin_path) .args(&["-c", &rendered_cmd]) @@ -750,17 +752,17 @@ impl PackageManager { ]) .output()?; - if !output.status.success() { - warn!("Failed to setup port forwarding for port {}", port); + if !output.status.success() { + warn!("Failed to setup port forwarding for port {}", port); + } + + trace!( + "Port forwarding configured: {} -> container {}", + port, + container + ); } - trace!( - "Port forwarding configured: {} -> container {}", - port, - container - ); + Ok(()) } - - Ok(()) -} } diff --git a/src/package_manager/installer.rs b/src/package_manager/installer.rs index 04a85699f..4435f8946 100644 --- a/src/package_manager/installer.rs +++ b/src/package_manager/installer.rs @@ -4,7 +4,6 @@ use crate::package_manager::{InstallMode, OsType}; use anyhow::Result; use log::trace; use rand::distr::Alphanumeric; -use rand::rngs::ThreadRng; use rand::Rng; use sha2::{Digest, Sha256}; use std::collections::HashMap; @@ -64,8 +63,7 @@ impl PackageManager { fn register_drive(&mut self) { let drive_password = self.generate_secure_password(16); - let farm_password = - std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32)); + let farm_password = std::env::var("FARM_PASSWORD").unwrap(); let encrypted_drive_password = self.encrypt_password(&drive_password, &farm_password); self.components.insert("drive".to_string(), ComponentConfig { @@ -110,10 +108,7 @@ impl PackageManager { use diesel::pg::PgConnection; use diesel::prelude::*; use uuid::Uuid; - - if let Ok(mut conn) = - PgConnection::establish("postgres://botserver:botserver@localhost:5432/botserver") - { + if let Ok(mut conn) = PgConnection::establish(&std::env::var("DATABASE_URL")?) { let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?; diesel::update(bots) .filter(bot_id.eq(system_bot_id)) @@ -148,7 +143,7 @@ impl PackageManager { "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"log_directory = '{{LOGS_PATH}}'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"logging_collector = on\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), "if [ ! -f \"{{CONF_PATH}}/pg_hba.conf\" ]; then echo \"host all all all md5\" > {{CONF_PATH}}/pg_hba.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/pg_ident.conf\" ]; then touch {{CONF_PATH}}/pg_ident.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/pg_ident.conf\" ]; then touch {{CONF_PATH}}/pg_ident.conf; fi ".to_string(), "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start; sleep 5; ./bin/createdb -p 5432 -h localhost botserver; ./bin/createuser -p 5432 -h localhost gbuser; fi".to_string() ], pre_install_cmds_macos: vec![], @@ -193,7 +188,7 @@ impl PackageManager { self.components.insert("llm".to_string(), ComponentConfig { name: "llm".to_string(), required: true, - ports: vec![8081], + ports: vec![8081, 8082], dependencies: vec![], linux_packages: vec!["unzip".to_string()], macos_packages: vec!["unzip".to_string()], @@ -213,7 +208,7 @@ impl PackageManager { pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), - exec_cmd: "{{BIN_PATH}}/llama-server -m {{DATA_PATH}}/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf --port 8081".to_string(), + exec_cmd: "{{BIN_PATH}}/llama-server -m {{DATA_PATH}}/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf --port 8081 & {{BIN_PATH}}/llama-server -m {{DATA_PATH}}/bge-small-en-v1.5-f32.gguf --port 8082 --embedding".to_string(), }); } @@ -656,7 +651,7 @@ impl PackageManager { } fn generate_secure_password(&self, length: usize) -> String { - let rng: ThreadRng = rand::rng(); + let rng = rand::rng(); rng.sample_iter(&Alphanumeric) .take(length) .map(char::from)