From f8d4e8925f4476c0de92e4922d3f43d6ec9c080f Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (Pragmatismo)" Date: Sun, 19 Oct 2025 19:28:08 -0300 Subject: [PATCH] Add indicatif for progress bars and enhance bootstrap ---------------------------------------------------------------- --- .zed/debug.json | 2 +- Cargo.lock | 39 +++ Cargo.toml | 1 + add-req.sh | 3 +- fix-errors.sh | 8 +- src/bootstrap/mod.rs | 113 ++++---- src/main.rs | 5 +- src/package_manager/facade.rs | 146 +++++------ src/package_manager/installer.rs | 243 +++++++++++------- src/shared/models.rs | 7 +- src/shared/utils.rs | 63 +++-- .../announcements.gbdialog/start.bas | 5 +- .../announcements.gbkb/weekly/default.pdf | Bin 0 -> 16894 bytes 13 files changed, 365 insertions(+), 270 deletions(-) create mode 100644 templates/announcements.gbai/announcements.gbkb/weekly/default.pdf diff --git a/.zed/debug.json b/.zed/debug.json index c843649a..77364a0c 100644 --- a/.zed/debug.json +++ b/.zed/debug.json @@ -2,7 +2,7 @@ { "label": "Debug BotServer", "build": { - "command": "rm -rf ./botserver-stack && cargo", + "command": "rm -rf .env ./botserver-stack && cargo", "args": ["build"] }, "program": "$ZED_WORKTREE_ROOT/target/debug/botserver", diff --git a/Cargo.lock b/Cargo.lock index 1512ea03..ed7fcfe2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1032,6 +1032,7 @@ dependencies = [ "futures-util", "headless_chrome", "imap", + "indicatif", "lettre", "livekit", "log", @@ -1336,6 +1337,19 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "console" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1970,6 +1984,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -2855,6 +2875,19 @@ dependencies = [ "hashbrown 0.16.0", ] +[[package]] +name = "indicatif" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "inout" version = "0.1.4" @@ -5626,6 +5659,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unit-prefix" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" + [[package]] name = "universal-hash" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 48344b6d..23ae4b5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,3 +93,4 @@ pdf-extract = "0.10.0" scraper = "0.20" sha2 = "0.10.9" ureq = "3.1.2" +indicatif = "0.18.0" diff --git a/add-req.sh b/add-req.sh index c8949b4a..ed1297f1 100755 --- a/add-req.sh +++ b/add-req.sh @@ -23,10 +23,11 @@ dirs=( #"automation" #"basic" #"bot" + "bootstrap" "package_manager" #"channels" "config" - "context" + #"context" #"email" #"file" #"llm" diff --git a/fix-errors.sh b/fix-errors.sh index d82b6439..3a6fbd6f 100755 --- a/fix-errors.sh +++ b/fix-errors.sh @@ -22,9 +22,9 @@ dirs=( #"automation" #"basic" #"bot" - #"bootstrap" + "bootstrap" #"channels" - "config" + #"config" #"context" #"email" #"file" @@ -33,7 +33,7 @@ dirs=( #"org" "package_manager" #"session" - #"shared" + "shared" #"tests" #"tools" #"web_automation" @@ -53,7 +53,7 @@ cat "$PROJECT_ROOT/src/main.rs" >> "$OUTPUT_FILE" echo "" >> "$OUTPUT_FILE" - +echo "Compiling..." cargo build --message-format=short 2>&1 | grep -E 'error' >> "$OUTPUT_FILE" diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs index beff5512..b110e45a 100644 --- a/src/bootstrap/mod.rs +++ b/src/bootstrap/mod.rs @@ -1,7 +1,11 @@ use crate::config::AppConfig; use crate::package_manager::{InstallMode, PackageManager}; use anyhow::Result; -use log::{debug, info, trace, warn}; +use log::{debug, trace, warn}; +use rand::distr::Alphanumeric; +use rand::rngs::ThreadRng; +use rand::Rng; +use sha2::{Digest, Sha256}; pub struct BootstrapManager { pub install_mode: InstallMode, @@ -10,10 +14,7 @@ pub struct BootstrapManager { impl BootstrapManager { pub fn new(install_mode: InstallMode, tenant: Option) -> Self { - info!( - "Initializing BootstrapManager with mode {:?} and tenant {:?}", - install_mode, tenant - ); + trace!("Initializing BootstrapManager with mode {:?} and tenant {:?}", install_mode, tenant); Self { install_mode, tenant, @@ -21,76 +22,94 @@ impl BootstrapManager { } pub fn start_all(&mut self) -> Result<()> { - info!("Starting all components"); let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?; - let components = vec![ - "tables", - "cache", - "drive", - "llm", - "email", - "proxy", - "directory", - "alm", - "alm_ci", - "dns", - "webmail", - "meeting", - "table_editor", - "doc_editor", - "desktop", - "devtools", - "bot", - "system", - "vector_db", - "host", + "tables", "cache", "drive", "llm", "email", "proxy", "directory", + "alm", "alm_ci", "dns", "webmail", "meeting", "table_editor", + "doc_editor", "desktop", "devtools", "bot", "system", "vector_db", "host" ]; for component in components { - info!("Starting component: {}", component); - pm.start(component)?; - trace!("Successfully started component: {}", component); + if pm.is_installed(component) { + trace!("Starting component: {}", component); + pm.start(component)?; + } else { + trace!("Component {} not installed, skipping start", component); + } } - - info!("All components started successfully"); Ok(()) } pub fn bootstrap(&mut self) -> Result { - info!("Starting bootstrap process"); - let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?; - let required_components = vec!["tables", "cache", "drive", "llm"]; - + for component in required_components { - info!("Checking component: {}", component); if !pm.is_installed(component) { - info!("Installing required component: {}", component); + trace!("Installing required component: {}", component); futures::executor::block_on(pm.install(component))?; - trace!("Successfully installed component: {}", component); + trace!("Starting component after install: {}", component); + pm.start(component)?; } else { - debug!("Component {} already installed", component); + trace!("Required component {} already installed", component); } } - info!("Bootstrap completed successfully"); - - let config = match diesel::Connection::establish( - "postgres://botserver:botserver@localhost:5432/botserver", - ) { + let config = match diesel::Connection::establish("postgres://botserver:botserver@localhost:5432/botserver") { Ok(mut conn) => { - trace!("Connected to database for config loading"); + self.setup_secure_credentials(&mut conn)?; AppConfig::from_database(&mut conn) } Err(e) => { warn!("Failed to connect to database for config: {}", e); - trace!("Falling back to environment configuration"); AppConfig::from_env() } }; Ok(config) } + + fn setup_secure_credentials(&self, conn: &mut diesel::PgConnection) -> Result<()> { + use crate::shared::models::schema::bots::dsl::*; + use diesel::prelude::*; + use uuid::Uuid; + + let farm_password = std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32)); + let db_password = self.generate_secure_password(16); + + let encrypted_db_password = self.encrypt_password(&db_password, &farm_password); + + let env_contents = format!( + "FARM_PASSWORD={}\nDATABASE_URL=postgres://gbuser:{}@localhost:5432/botserver", + farm_password, db_password + ); + + std::fs::write(".env", env_contents) + .map_err(|e| anyhow::anyhow!("Failed to write .env file: {}", e))?; + + let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?; + diesel::update(bots) + .filter(bot_id.eq(system_bot_id)) + .set(config.eq(serde_json::json!({ + "encrypted_db_password": encrypted_db_password, + }))) + .execute(conn)?; + + Ok(()) + } + + fn generate_secure_password(&self, length: usize) -> String { + let mut rng: ThreadRng = rand::thread_rng(); + rng.sample_iter(&Alphanumeric) + .take(length) + .map(char::from) + .collect() + } + + fn encrypt_password(&self, password: &str, key: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(key.as_bytes()); + hasher.update(password.as_bytes()); + format!("{:x}", hasher.finalize()) + } } diff --git a/src/main.rs b/src/main.rs index c80a3e04..c21dc211 100644 --- a/src/main.rs +++ b/src/main.rs @@ -104,7 +104,6 @@ async fn main() -> std::io::Result<()> { }; let mut bootstrap = BootstrapManager::new(install_mode.clone(), tenant.clone()); - let _ = bootstrap.start_all(); let cfg = match bootstrap.bootstrap() { Ok(config) => { info!("Bootstrap completed successfully, configuration loaded from database"); @@ -113,8 +112,7 @@ async fn main() -> std::io::Result<()> { Err(e) => { log::error!("Bootstrap failed: {}", e); info!("Attempting to load configuration from database"); - match diesel::Connection::establish(&format!("postgres://localhost:5432/botserver_db")) - { + match diesel::Connection::establish("postgres://botserver:botserver@localhost:5432/botserver") { Ok(mut conn) => AppConfig::from_database(&mut conn), Err(_) => { info!("Database not available, using environment variables as fallback"); @@ -124,6 +122,7 @@ async fn main() -> std::io::Result<()> { } }; + let _ = bootstrap.start_all(); let config = std::sync::Arc::new(cfg.clone()); info!("Establishing database connection to {}", cfg.database_url()); diff --git a/src/package_manager/facade.rs b/src/package_manager/facade.rs index c83b2458..eed00bd3 100644 --- a/src/package_manager/facade.rs +++ b/src/package_manager/facade.rs @@ -1,17 +1,15 @@ +use crate::package_manager::component::ComponentConfig; +use crate::package_manager::installer::PackageManager; +use crate::package_manager::OsType; +use crate::shared::utils; +use crate::InstallMode; use anyhow::{Context, Result}; -use log::{debug, info, trace, warn}; +use log::{trace, warn}; use reqwest::Client; use std::collections::HashMap; use std::path::PathBuf; use std::process::Command; -use crate::shared::utils; // Adjust based on your actual module structure - -use crate::package_manager::component::ComponentConfig; -use crate::package_manager::installer::PackageManager; -use crate::package_manager::OsType; -use crate::InstallMode; - impl PackageManager { pub async fn install(&self, component_name: &str) -> Result<()> { let component = self @@ -19,9 +17,10 @@ impl PackageManager { .get(component_name) .context(format!("Component '{}' not found", component_name))?; - info!( + trace!( "Starting installation of component '{}' in {:?} mode", - component_name, self.mode + component_name, + self.mode ); for dep in &component.dependencies { @@ -36,16 +35,15 @@ impl PackageManager { InstallMode::Container => self.install_container(component)?, } - info!( + trace!( "Component '{}' installation completed successfully", component_name ); Ok(()) } - // ... rest of the implementation remains exactly the same ... pub async fn install_local(&self, component: &ComponentConfig) -> Result<()> { - info!( + trace!( "Installing component '{}' locally to {}", component.name, self.base_path.display() @@ -75,18 +73,19 @@ impl PackageManager { let url = url.clone(); let name = component.name.clone(); let binary_name = component.binary_name.clone(); - self.download_and_install(&url, &name, binary_name.as_deref()) .await?; - self.run_commands(post_cmds, "local", &component.name)?; } - info!("Local installation of '{}' completed", component.name); + + self.run_commands(post_cmds, "local", &component.name)?; + trace!("Starting component after installation: {}", component.name); + self.start(&component.name)?; + Ok(()) } pub fn install_container(&self, component: &ComponentConfig) -> Result<()> { let container_name = format!("{}-{}", self.tenant, component.name); - info!("Creating LXC container: {}", container_name); let output = Command::new("lxc") .args(&[ @@ -106,7 +105,6 @@ impl PackageManager { } std::thread::sleep(std::time::Duration::from_secs(15)); - self.exec_in_container(&container_name, "mkdir -p /opt/gbo/{bin,data,conf,logs}")?; let (pre_cmds, post_cmds) = match self.os_type { @@ -134,10 +132,7 @@ impl PackageManager { if !packages.is_empty() { let pkg_list = packages.join(" "); - self.exec_in_container( - &container_name, - &format!("apt-get update && apt-get install -y {}", pkg_list), - )?; + self.exec_in_container(&container_name, &format!("apt-get update && apt-get install -y {}", pkg_list))?; } if let Some(url) = &component.download_url { @@ -167,11 +162,12 @@ impl PackageManager { } self.setup_port_forwarding(&container_name, &component.ports)?; - - info!( + trace!( "Container installation of '{}' completed in {}", - component.name, container_name + component.name, + container_name ); + Ok(()) } @@ -181,27 +177,22 @@ impl PackageManager { .get(component_name) .context(format!("Component '{}' not found", component_name))?; - info!("Removing component: {}", component_name); - match self.mode { InstallMode::Local => self.remove_local(component)?, InstallMode::Container => self.remove_container(component)?, } - info!("Component '{}' removed successfully", component_name); Ok(()) } pub fn remove_local(&self, component: &ComponentConfig) -> Result<()> { let bin_path = self.base_path.join("bin").join(&component.name); let _ = std::fs::remove_dir_all(bin_path); - Ok(()) } pub fn remove_container(&self, component: &ComponentConfig) -> Result<()> { let container_name = format!("{}-{}", self.tenant, component.name); - let _ = Command::new("lxc") .args(&["stop", &container_name]) .output(); @@ -252,7 +243,6 @@ impl PackageManager { let path = self.base_path.join(dir).join(component); std::fs::create_dir_all(&path) .context(format!("Failed to create directory: {:?}", path))?; - trace!("Created directory: {:?}", path); } Ok(()) } @@ -268,7 +258,7 @@ impl PackageManager { return Ok(()); } - info!( + trace!( "Installing {} system packages for component '{}'", packages.len(), component.name @@ -276,8 +266,10 @@ impl PackageManager { match self.os_type { OsType::Linux => { - let output = Command::new("apt-get").args(&["update"]).output()?; - + let output = Command::new("apt-get") + .args(&["update"]) + .output()?; + if !output.status.success() { warn!("apt-get update had issues"); } @@ -286,6 +278,7 @@ impl PackageManager { .args(&["install", "-y"]) .args(packages) .output()?; + if !output.status.success() { warn!("Some packages may have failed to install"); } @@ -295,6 +288,7 @@ impl PackageManager { .args(&["install"]) .args(packages) .output()?; + if !output.status.success() { warn!("Homebrew installation had warnings"); } @@ -323,11 +317,8 @@ impl PackageManager { bin_path.join(filename) }; - info!("Downloading from: {} to {:?}", url, temp_file); - self.download_with_reqwest(url, &temp_file, component) .await?; - self.handle_downloaded_file(&temp_file, &bin_path, binary_name)?; Ok(()) @@ -351,25 +342,25 @@ impl PackageManager { for attempt in 0..=MAX_RETRIES { if attempt > 0 { - info!( + trace!( "Retry attempt {}/{} for {}", - attempt, MAX_RETRIES, component + attempt, + MAX_RETRIES, + component ); std::thread::sleep(RETRY_DELAY * attempt); } match self.attempt_reqwest_download(&client, url, temp_file).await { - Ok(size) => { - info!("Downloaded {} bytes for {}", size, component); + Ok(_size) => { if attempt > 0 { - info!("Download succeeded on attempt {}", attempt + 1); + trace!("Download succeeded on retry attempt {}", attempt); } return Ok(()); } Err(e) => { warn!("Download attempt {} failed: {}", attempt + 1, e); last_error = Some(e); - let _ = std::fs::remove_file(temp_file); } } @@ -385,25 +376,17 @@ impl PackageManager { pub async fn attempt_reqwest_download( &self, - _client: &Client, // We won't use this if using shared utils + _client: &Client, url: &str, temp_file: &PathBuf, ) -> Result { - info!("Downloading from: {} to {:?}", url, temp_file); - - // Convert PathBuf to string for the shared function let output_path = temp_file.to_str().context("Invalid temp file path")?; - - // Use the shared download_file utility utils::download_file(url, output_path) .await .map_err(|e| anyhow::anyhow!("Failed to download file using shared utility: {}", e))?; - // Get file size to return let metadata = std::fs::metadata(temp_file).context("Failed to get file metadata")?; let size = metadata.len(); - - info!("Downloaded {} bytes", size); Ok(size) } @@ -418,8 +401,6 @@ impl PackageManager { return Err(anyhow::anyhow!("Downloaded file is empty")); } - info!("Final file size: {} bytes", metadata.len()); - let file_extension = temp_file .extension() .and_then(|ext| ext.to_str()) @@ -447,7 +428,6 @@ impl PackageManager { } pub fn extract_tar_gz(&self, temp_file: &PathBuf, bin_path: &PathBuf) -> Result<()> { - info!("Extracting tar.gz archive to {:?}", bin_path); let output = Command::new("tar") .current_dir(bin_path) .args(&["-xzf", temp_file.to_str().unwrap(), "--strip-components=1"]) @@ -465,8 +445,6 @@ impl PackageManager { } pub fn extract_zip(&self, temp_file: &PathBuf, bin_path: &PathBuf) -> Result<()> { - info!("Extracting zip archive to {:?}", bin_path); - let output = Command::new("unzip") .current_dir(bin_path) .args(&["-o", "-q", temp_file.to_str().unwrap()]) @@ -490,12 +468,8 @@ impl PackageManager { name: &str, ) -> Result<()> { let final_path = bin_path.join(name); - std::fs::rename(temp_file, &final_path)?; - self.make_executable(&final_path)?; - - info!("Installed binary: {:?}", final_path); Ok(()) } @@ -517,7 +491,6 @@ impl PackageManager { env_vars: &HashMap, ) -> Result<()> { let service_path = format!("/etc/systemd/system/{}.service", component); - let bin_path = self.base_path.join("bin").join(component); let data_path = self.base_path.join("data").join(component); let conf_path = self.base_path.join("conf").join(component); @@ -545,12 +518,11 @@ impl PackageManager { } let service_content = format!( - "[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory={}\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", - component, env_section, rendered_cmd, data_path.to_string_lossy() - ); + "[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory={}\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", + component, env_section, rendered_cmd, data_path.to_string_lossy() + ); std::fs::write(&service_path, service_content)?; - Command::new("systemctl") .args(&["daemon-reload"]) .output()?; @@ -561,7 +533,6 @@ impl PackageManager { .args(&["start", &format!("{}.service", component)]) .output()?; - info!("Created and started systemd service: {}.service", component); Ok(()) } @@ -597,13 +568,12 @@ impl PackageManager { .replace("{{CONF_PATH}}", &conf_path.to_string_lossy()) .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()); - trace!("Executing command: {}", rendered_cmd); - if target == "local" { let output = Command::new("bash") .current_dir(&bin_path) .args(&["-c", &rendered_cmd]) .output()?; + if !output.status.success() { warn!( "Command had non-zero exit: {}", @@ -614,11 +584,11 @@ impl PackageManager { self.exec_in_container(target, &rendered_cmd)?; } } + Ok(()) } pub fn exec_in_container(&self, container: &str, command: &str) -> Result<()> { - debug!("Executing in container {}: {}", container, command); let output = Command::new("lxc") .args(&["exec", container, "--", "bash", "-c", command]) .output()?; @@ -629,6 +599,7 @@ impl PackageManager { String::from_utf8_lossy(&output.stderr) ); } + Ok(()) } @@ -696,6 +667,7 @@ impl PackageManager { container ); } + Ok(()) } @@ -723,9 +695,9 @@ impl PackageManager { } let service_content = format!( - "[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory=/opt/gbo/data\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", - component, env_section, rendered_cmd - ); + "[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory=/opt/gbo/data\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", + component, env_section, rendered_cmd + ); let service_file = format!("/tmp/{}.service", component); std::fs::write(&service_file, &service_content)?; @@ -748,11 +720,12 @@ impl PackageManager { self.exec_in_container(container, &format!("systemctl start {}", component))?; std::fs::remove_file(&service_file)?; - - info!( + trace!( "Created and started service in container {}: {}", - container, component + container, + component ); + Ok(()) } @@ -777,16 +750,17 @@ impl PackageManager { ]) .output()?; - if !output.status.success() { - warn!("Failed to setup port forwarding for port {}", port); - } - - trace!( - "Port forwarding configured: {} -> container {}", - port, - container - ); + if !output.status.success() { + warn!("Failed to setup port forwarding for port {}", port); } - Ok(()) + + trace!( + "Port forwarding configured: {} -> container {}", + port, + container + ); } + + Ok(()) +} } diff --git a/src/package_manager/installer.rs b/src/package_manager/installer.rs index 8b600c4f..1841c9ec 100644 --- a/src/package_manager/installer.rs +++ b/src/package_manager/installer.rs @@ -1,11 +1,14 @@ -use anyhow::Result; -use log::info; -use std::collections::HashMap; -use std::path::PathBuf; - use crate::package_manager::component::ComponentConfig; use crate::package_manager::os::detect_os; use crate::package_manager::{InstallMode, OsType}; +use anyhow::Result; +use log::trace; +use rand::distr::Alphanumeric; +use rand::rngs::ThreadRng; +use rand::Rng; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::path::PathBuf; pub struct PackageManager { pub mode: InstallMode, @@ -23,7 +26,6 @@ impl PackageManager { } else { std::env::current_dir()?.join("botserver-stack") }; - let tenant = tenant.unwrap_or_else(|| "default".to_string()); let mut pm = PackageManager { @@ -33,14 +35,7 @@ impl PackageManager { tenant, components: HashMap::new(), }; - pm.register_components(); - info!( - "PackageManager initialized with {} components in {:?} mode for tenant {}", - pm.components.len(), - pm.mode, - pm.tenant - ); Ok(pm) } @@ -68,20 +63,29 @@ impl PackageManager { } fn register_drive(&mut self) { + let drive_password = self.generate_secure_password(16); + let farm_password = + std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32)); + let encrypted_drive_password = self.encrypt_password(&drive_password, &farm_password); + self.components.insert("drive".to_string(), ComponentConfig { name: "drive".to_string(), required: true, ports: vec![9000, 9001], dependencies: vec![], - linux_packages: vec!["wget".to_string()], - macos_packages: vec!["wget".to_string()], + linux_packages: vec![], + macos_packages: vec![], windows_packages: vec![], download_url: Some("https://dl.min.io/server/minio/release/linux-amd64/minio".to_string()), binary_name: Some("minio".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![ "wget https://dl.min.io/client/mc/release/linux-amd64/mc -O {{BIN_PATH}}/mc".to_string(), - "chmod +x {{BIN_PATH}}/mc".to_string() + "chmod +x {{BIN_PATH}}/mc".to_string(), + format!("{{BIN_PATH}}/mc alias set mc http://localhost:9000 gbdriveuser {}", drive_password).to_string(), + "{{BIN_PATH}}/mc mb mc/default.gbai".to_string(), + format!("{{BIN_PATH}}/mc admin user add mc gbdriveuser {}", drive_password).to_string(), + "{{BIN_PATH}}/mc admin policy attach mc readwrite --user=gbdriveuser".to_string() ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![ @@ -91,11 +95,74 @@ impl PackageManager { pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::from([ - ("MINIO_ROOT_USER".to_string(), "minioadmin".to_string()), - ("MINIO_ROOT_PASSWORD".to_string(), "minioadmin".to_string()) + ("MINIO_ROOT_USER".to_string(), "gbdriveuser".to_string()), + ("MINIO_ROOT_PASSWORD".to_string(), drive_password) ]), exec_cmd: "{{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001".to_string(), }); + + self.update_drive_credentials_in_database(&encrypted_drive_password) + .ok(); + } + + fn update_drive_credentials_in_database(&self, encrypted_drive_password: &str) -> Result<()> { + use crate::shared::models::schema::bots::dsl::*; + use diesel::pg::PgConnection; + use diesel::prelude::*; + use uuid::Uuid; + + if let Ok(mut conn) = + PgConnection::establish("postgres://botserver:botserver@localhost:5432/botserver") + { + let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?; + diesel::update(bots) + .filter(bot_id.eq(system_bot_id)) + .set(config.eq(serde_json::json!({ + "encrypted_drive_password": encrypted_drive_password, + }))) + .execute(&mut conn)?; + } + Ok(()) + } + + fn register_tables(&mut self) { + self.components.insert("tables".to_string(), ComponentConfig { + name: "tables".to_string(), + required: true, + ports: vec![5432], + dependencies: vec![], + linux_packages: vec![], + macos_packages: vec![], + windows_packages: vec![], + download_url: Some("https://github.com/theseus-rs/postgresql-binaries/releases/download/18.0.0/postgresql-18.0.0-x86_64-unknown-linux-gnu.tar.gz".to_string()), + binary_name: None, + pre_install_cmds_linux: vec![], + post_install_cmds_linux: vec![ + "tar -xzf postgresql-18.0.0-x86_64-unknown-linux-gnu.tar.gz --strip-components=1".to_string(), + "chmod +x bin/*".to_string(), + "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/initdb -D {{DATA_PATH}}/pgdata -U postgres; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"data_directory = '{{DATA_PATH}}/pgdata'\" > {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"hba_file = '{{CONF_PATH}}/pg_hba.conf'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"ident_file = '{{CONF_PATH}}/pg_ident.conf'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"port = 5432\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"listen_addresses = '*'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"log_directory = '{{LOGS_PATH}}'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"logging_collector = on\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/pg_hba.conf\" ]; then echo \"host all all all md5\" > {{CONF_PATH}}/pg_hba.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/pg_ident.conf\" ]; then touch {{CONF_PATH}}/pg_ident.conf; fi".to_string(), + "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start; sleep 5; ./bin/createdb -p 5432 -h localhost botserver; ./bin/createuser -p 5432 -h localhost gbuser; fi".to_string() + ], + pre_install_cmds_macos: vec![], + post_install_cmds_macos: vec![ + "tar -xzf postgresql-18.0.0-x86_64-unknown-linux-gnu.tar.gz --strip-components=1".to_string(), + "chmod +x bin/*".to_string(), + "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/initdb -D {{DATA_PATH}}/pgdata -U postgres; fi".to_string(), + ], + pre_install_cmds_windows: vec![], + post_install_cmds_windows: vec![], + env_vars: HashMap::new(), + exec_cmd: "./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start".to_string(), + }); } fn register_cache(&mut self) { @@ -104,15 +171,15 @@ impl PackageManager { required: true, ports: vec![6379], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "curl".to_string(), "gnupg".to_string(), "lsb-release".to_string()], + linux_packages: vec!["curl".to_string(), "gnupg".to_string(), "lsb-release".to_string()], macos_packages: vec!["redis".to_string()], windows_packages: vec![], download_url: None, binary_name: Some("valkey-server".to_string()), pre_install_cmds_linux: vec![ - "if [ ! -f /usr/share/keyrings/valkey.gpg ]; then curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/valkey.gpg; fi".to_string(), - "if [ ! -f /etc/apt/sources.list.d/valkey.list ]; then echo 'deb [signed-by=/usr/share/keyrings/valkey.gpg] https://packages.redis.io/deb $(lsb_release -cs) main' | tee /etc/apt/sources.list.d/valkey.list; fi".to_string(), - "apt-get update && apt-get install -y valkey".to_string() + "if [ ! -f /usr/share/keyrings/valkey.gpg ]; then curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/valkey.gpg; fi".to_string(), + "if [ ! -f /etc/apt/sources.list.d/valkey.list ]; then echo 'deb [signed-by=/usr/share/keyrings/valkey.gpg] https://packages.redis.io/deb $(lsb_release -cs) main' | tee /etc/apt/sources.list.d/valkey.list; fi".to_string(), + "apt-get update && apt-get install -y valkey".to_string() ], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], @@ -124,62 +191,26 @@ impl PackageManager { }); } - fn register_tables(&mut self) { - self.components.insert("tables".to_string(), ComponentConfig { - name: "tables".to_string(), - required: true, - ports: vec![5432], - dependencies: vec![], - linux_packages: vec!["wget".to_string()], - macos_packages: vec!["wget".to_string()], - windows_packages: vec![], - download_url: Some("https://github.com/theseus-rs/postgresql-binaries/releases/download/18.0.0/postgresql-18.0.0-x86_64-unknown-linux-gnu.tar.gz".to_string()), - binary_name: Some("postgres".to_string()), - pre_install_cmds_linux: vec![], - post_install_cmds_linux: vec![ - "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/initdb -D {{DATA_PATH}}/pgdata -U postgres; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"data_directory = '{{DATA_PATH}}/pgdata'\" > {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"hba_file = '{{CONF_PATH}}/pg_hba.conf'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"ident_file = '{{CONF_PATH}}/pg_ident.conf'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"port = 5432\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"listen_addresses = '*'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"log_directory = '{{LOGS_PATH}}'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"logging_collector = on\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/pg_hba.conf\" ]; then echo \"host all all all md5\" > {{CONF_PATH}}/pg_hba.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/pg_ident.conf\" ]; then touch {{CONF_PATH}}/pg_ident.conf; fi".to_string(), - "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start; sleep 5; ./bin/psql -p 5432 -d postgres -c \" CREATE USER default WITH PASSWORD 'defaultpass'\"; ./bin/psql -p 5432 -d postgres -c \"CREATE DATABASE default_db OWNER default\"; ./bin/psql -p 5432 -d postgres -c \"GRANT ALL PRIVILEGES ON DATABASE default_db TO default\"; pkill; fi".to_string() - ], - pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec![ - "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/initdb -D {{DATA_PATH}}/pgdata -U postgres; fi".to_string(), - ], - pre_install_cmds_windows: vec![], - post_install_cmds_windows: vec![], - env_vars: HashMap::new(), - exec_cmd: "./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start".to_string(), - }); - } - fn register_llm(&mut self) { self.components.insert("llm".to_string(), ComponentConfig { name: "llm".to_string(), required: true, ports: vec![8081], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "unzip".to_string()], - macos_packages: vec!["wget".to_string(), "unzip".to_string()], + linux_packages: vec!["unzip".to_string()], + macos_packages: vec!["unzip".to_string()], windows_packages: vec![], download_url: Some("https://github.com/ggml-org/llama.cpp/releases/download/b6148/llama-b6148-bin-ubuntu-x64.zip".to_string()), binary_name: Some("llama-server".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![ - "wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), - "wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() + "wget -q https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), + "wget -q https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![ - "wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), - "wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() + "wget -q https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), + "wget -q https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() ], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], @@ -194,7 +225,7 @@ impl PackageManager { required: false, ports: vec![25, 80, 110, 143, 465, 587, 993, 995, 4190], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "libcap2-bin".to_string(), "resolvconf".to_string()], + linux_packages: vec!["libcap2-bin".to_string(), "resolvconf".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/stalwartlabs/stalwart/releases/download/v0.13.1/stalwart-x86_64-unknown-linux-gnu.tar.gz".to_string()), @@ -218,8 +249,8 @@ impl PackageManager { required: false, ports: vec![80, 443], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "libcap2-bin".to_string()], - macos_packages: vec!["wget".to_string()], + linux_packages: vec!["libcap2-bin".to_string()], + macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/caddyserver/caddy/releases/download/v2.10.0-beta.3/caddy_2.10.0-beta.3_linux_amd64.tar.gz".to_string()), binary_name: Some("caddy".to_string()), @@ -244,7 +275,7 @@ impl PackageManager { required: false, ports: vec![8080], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "libcap2-bin".to_string()], + linux_packages: vec!["libcap2-bin".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/zitadel/zitadel/releases/download/v2.71.2/zitadel-linux-amd64.tar.gz".to_string()), @@ -268,7 +299,7 @@ impl PackageManager { required: false, ports: vec![3000], dependencies: vec![], - linux_packages: vec!["git".to_string(), "git-lfs".to_string(), "wget".to_string()], + linux_packages: vec!["git".to_string(), "git-lfs".to_string()], macos_packages: vec!["git".to_string(), "git-lfs".to_string()], windows_packages: vec![], download_url: Some("https://codeberg.org/forgejo/forgejo/releases/download/v10.0.2/forgejo-10.0.2-linux-amd64".to_string()), @@ -293,14 +324,14 @@ impl PackageManager { required: false, ports: vec![], dependencies: vec!["alm".to_string()], - linux_packages: vec!["wget".to_string(), "git".to_string(), "curl".to_string(), "gnupg".to_string(), "ca-certificates".to_string(), "build-essential".to_string()], + linux_packages: vec!["git".to_string(), "curl".to_string(), "gnupg".to_string(), "ca-certificates".to_string(), "build-essential".to_string()], macos_packages: vec!["git".to_string(), "node".to_string()], windows_packages: vec![], download_url: Some("https://code.forgejo.org/forgejo/runner/releases/download/v6.3.1/forgejo-runner-6.3.1-linux-amd64".to_string()), binary_name: Some("forgejo-runner".to_string()), pre_install_cmds_linux: vec![ "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -".to_string(), - "apt-get update && apt-get install -y nodejs".to_string() + "apt-get install -y nodejs".to_string() ], post_install_cmds_linux: vec![ "npm install -g pnpm@latest".to_string() @@ -322,7 +353,7 @@ impl PackageManager { required: false, ports: vec![53], dependencies: vec![], - linux_packages: vec!["wget".to_string()], + linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/coredns/coredns/releases/download/v1.12.4/coredns_1.12.4_linux_amd64.tgz".to_string()), @@ -368,7 +399,7 @@ impl PackageManager { required: false, ports: vec![7880, 3478], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "coturn".to_string()], + linux_packages: vec!["coturn".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/livekit/livekit/releases/download/v1.8.4/livekit_1.8.4_linux_amd64.tar.gz".to_string()), @@ -386,13 +417,13 @@ impl PackageManager { fn register_table_editor(&mut self) { self.components.insert( - "table-editor".to_string(), + "table_editor".to_string(), ComponentConfig { - name: "table-editor".to_string(), + name: "table_editor".to_string(), required: false, ports: vec![5757], dependencies: vec!["tables".to_string()], - linux_packages: vec!["wget".to_string(), "curl".to_string()], + linux_packages: vec!["curl".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: Some("http://get.nocodb.com/linux-x64".to_string()), @@ -411,13 +442,13 @@ impl PackageManager { fn register_doc_editor(&mut self) { self.components.insert( - "doc-editor".to_string(), + "doc_editor".to_string(), ComponentConfig { - name: "doc-editor".to_string(), + name: "doc_editor".to_string(), required: false, ports: vec![9980], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "gnupg".to_string()], + linux_packages: vec!["gnupg".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: None, @@ -504,7 +535,7 @@ impl PackageManager { binary_name: None, pre_install_cmds_linux: vec![ "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -".to_string(), - "apt-get update && apt-get install -y nodejs".to_string(), + "apt-get install -y nodejs".to_string(), ], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], @@ -525,12 +556,7 @@ impl PackageManager { required: false, ports: vec![8000], dependencies: vec![], - linux_packages: vec![ - "wget".to_string(), - "curl".to_string(), - "unzip".to_string(), - "git".to_string(), - ], + linux_packages: vec!["curl".to_string(), "unzip".to_string(), "git".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: None, @@ -548,13 +574,13 @@ impl PackageManager { } fn register_vector_db(&mut self) { - self.components.insert("vector-db".to_string(), ComponentConfig { - name: "vector-db".to_string(), + self.components.insert("vector_db".to_string(), ComponentConfig { + name: "vector_db".to_string(), required: false, ports: vec![6333], dependencies: vec![], - linux_packages: vec!["wget".to_string()], - macos_packages: vec!["wget".to_string()], + linux_packages: vec![], + macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/qdrant/qdrant/releases/latest/download/qdrant-x86_64-unknown-linux-gnu.tar.gz".to_string()), binary_name: Some("qdrant".to_string()), @@ -601,14 +627,47 @@ impl PackageManager { ); } - pub(crate) fn start(&self, component: &str) -> Result { + pub fn start(&self, component: &str) -> Result { if let Some(component) = self.components.get(component) { + let bin_path = self.base_path.join("bin").join(&component.name); + let data_path = self.base_path.join("data").join(&component.name); + let conf_path = self.base_path.join("conf").join(&component.name); + let logs_path = self.base_path.join("logs").join(&component.name); + + let rendered_cmd = component + .exec_cmd + .replace("{{BIN_PATH}}", &bin_path.to_string_lossy()) + .replace("{{DATA_PATH}}", &data_path.to_string_lossy()) + .replace("{{CONF_PATH}}", &conf_path.to_string_lossy()) + .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()); + + trace!( + "Starting component {} with command: {}", + component.name, + rendered_cmd + ); + Ok(std::process::Command::new("sh") .arg("-c") - .arg(&component.exec_cmd) + .arg(&rendered_cmd) .spawn()?) } else { Err(anyhow::anyhow!("Component {} not found", component)) } } + + fn generate_secure_password(&self, length: usize) -> String { + let mut rng: ThreadRng = rand::thread_rng(); + rng.sample_iter(&Alphanumeric) + .take(length) + .map(char::from) + .collect() + } + + fn encrypt_password(&self, password: &str, key: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(key.as_bytes()); + hasher.update(password.as_bytes()); + format!("{:x}", hasher.finalize()) + } } diff --git a/src/shared/models.rs b/src/shared/models.rs index 42ab7ce0..72f7a34e 100644 --- a/src/shared/models.rs +++ b/src/shared/models.rs @@ -219,6 +219,12 @@ pub struct SessionToolAssociation { pub added_at: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SystemCredentials { + pub encrypted_db_password: String, + pub encrypted_drive_password: String, +} + pub mod schema { diesel::table! { organizations (org_id) { @@ -384,5 +390,4 @@ pub mod schema { } } -// Re-export all tables at the module level for backward compatibility pub use schema::*; diff --git a/src/shared/utils.rs b/src/shared/utils.rs index ad09ac61..16a4ae9d 100644 --- a/src/shared/utils.rs +++ b/src/shared/utils.rs @@ -1,4 +1,4 @@ -use log::debug; +use log::{debug, trace}; use rhai::{Array, Dynamic}; use serde_json::Value; use smartstring::SmartString; @@ -7,17 +7,14 @@ use std::fs::File; use std::io::BufReader; use std::path::Path; use tokio::fs::File as TokioFile; - use zip::ZipArchive; - use crate::config::AIConfig; use reqwest::Client; use tokio::io::AsyncWriteExt; +use indicatif::{ProgressBar, ProgressStyle}; +use futures_util::StreamExt; -pub fn extract_zip_recursive( - zip_path: &Path, - destination_path: &Path, -) -> Result<(), Box> { +pub fn extract_zip_recursive(zip_path: &Path, destination_path: &Path) -> Result<(), Box> { let file = File::open(zip_path)?; let buf_reader = BufReader::new(file); let mut archive = ZipArchive::new(buf_reader)?; @@ -38,7 +35,6 @@ pub fn extract_zip_recursive( std::io::copy(&mut file, &mut outfile)?; } } - Ok(()) } @@ -79,21 +75,36 @@ pub fn to_array(value: Dynamic) -> Array { } } -pub async fn download_file( - url: &str, - output_path: &str, -) -> Result<(), Box> { +pub async fn download_file(url: &str, output_path: &str) -> Result<(), Box> { let url = url.to_string(); let output_path = output_path.to_string(); + let download_handle = tokio::spawn(async move { let client = Client::new(); let response = client.get(&url).send().await?; - + if response.status().is_success() { + let total_size = response.content_length().unwrap_or(0); + let pb = ProgressBar::new(total_size); + pb.set_style(ProgressStyle::default_bar() + .template("{msg}\n{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})") + .unwrap() + .progress_chars("#>-")); + pb.set_message(format!("Downloading {}", url)); + let mut file = TokioFile::create(&output_path).await?; - let bytes = response.bytes().await?; - file.write_all(&bytes).await?; - debug!("File downloaded successfully to {}", output_path); + let mut downloaded: u64 = 0; + let mut stream = response.bytes_stream(); + + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result?; + file.write_all(&chunk).await?; + downloaded += chunk.len() as u64; + pb.set_position(downloaded); + } + + pb.finish_with_message(format!("Downloaded {}", output_path)); + trace!("Download completed: {} -> {}", url, output_path); Ok(()) } else { Err(format!("HTTP {}: {}", response.status(), url).into()) @@ -112,20 +123,14 @@ pub fn parse_filter(filter_str: &str) -> Result<(String, Vec), Box Result<(String, Vec), Box> { +pub fn parse_filter_with_offset(filter_str: &str, offset: usize) -> Result<(String, Vec), Box> { let mut clauses = Vec::new(); let mut params = Vec::new(); @@ -138,10 +143,7 @@ pub fn parse_filter_with_offset( let column = parts[0].trim(); let value = parts[1].trim(); - if !column - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_') - { + if !column.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { return Err("Invalid column name".into()); } @@ -152,9 +154,6 @@ pub fn parse_filter_with_offset( Ok((clauses.join(" AND "), params)) } -pub async fn call_llm( - prompt: &str, - _ai_config: &AIConfig, -) -> Result> { +pub async fn call_llm(prompt: &str, _ai_config: &AIConfig) -> Result> { Ok(format!("Generated response for: {}", prompt)) } diff --git a/templates/announcements.gbai/announcements.gbdialog/start.bas b/templates/announcements.gbai/announcements.gbdialog/start.bas index c3890cc8..e55ea9d9 100644 --- a/templates/announcements.gbai/announcements.gbdialog/start.bas +++ b/templates/announcements.gbai/announcements.gbdialog/start.bas @@ -1,7 +1,6 @@ let resume = GET_BOT_MEMORY ("resume") - TALK resume -let text = GET "default.gbdrive/default.pdf" -SET_CONTEXT "Este é o documento que você deve usar para responder dúvidas: " + text +ADD_KB "weekly" + TALK "Olá, pode me perguntar sobre qualquer coisa destas circulares..." diff --git a/templates/announcements.gbai/announcements.gbkb/weekly/default.pdf b/templates/announcements.gbai/announcements.gbkb/weekly/default.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f38959e6d263c2939a11035ba06768ee97cf86f8 GIT binary patch literal 16894 zcmeHu2T)Ymv$vus3?L{72na(C0y8kc0FoqUBdh&t=g)uzS>)L=APTPPj{d0K4+?ar!V7uNofu)PAGt}Be$ce zqoyMRzzqZg5oWdkAt8_=%)ts}4TPX6Y9MJFI11(jl7^e2V3II%gau4Q1b{?2!A$J| zZmG_yN_GKa_|3y*z4e()IYZ&VOWxvmo;qxY<%hs(eGlD2oU<>3UU2;c(t8In2`j?> ztS<|Clbt<9he4E8!OC72u<}_5ShB0eE;j()9%%~Azm&os=%cPZc`@)_ZE`xRapz)* z{g{#<>l-6-^=mVd|Lu zclTo5a0E{hX+Ez6g=@Lr4scfvYztL2-IhR>odz+ z&WJV<6+*Mb&b=M=v2O(oiU+gmY+Bp%He2mjT?x0nzfD-S?%^0m`i3b0U=9{P{f6#g zTn)f@7}J4(xiD|P)1dzm2mb#i4nO0>1?1x4=i&J^DytrzUc{62izk-SAcY#~7t}R@ z#n;=LEKV3xOsMyO@aab{v1VV>T~^THb}`d$kEdl=TAmY&v6Gij-_Nec9UOxvTT%*) z7{#nzsby@Q9aNHluJGCP1ucvXHqYihP)~Ll*I7Ueg@-H3oxYzxL-Y&`2)uolx+Ax95bza3YqC8Zdqb;MSKI8%4cY;p9i?pwp>x97GNSf0 zCP}2nWLwKZXy=YGuyaHvq_P4;{lxzH>5IrH#^Gi1{!^ln*y)0o=dcf;4NArN__41* z>0ULCPwJOK)2#=^5F}q*&3xXN9M)I}PY#)U!~YbRwPEr#GE)7t7H2e)ftdaoc|GDv zutVJLt0lJ=N1unZ%SFX-JV{I+`HaEZ^7XbL>;-CeRSiPTw5@HuDFcC|MT15f2Fr;u z=RS2BG9htQg<8b)a%1bMVGrX;Vr6iX+>N&kA6n{JiJbVb25NZ=i>K2Syt=;6SFd~T zvRuevtZKlD(OY`adxfaLUcP<&Ti?P>GZ?LiK;u%qXE98Z&EhMjPLIEir|ODZM9ulW z@JeYSc(Woy&W5{x-TIZ`%1x>x`T+Z;h5L#ZMLcXOp;u3@ov-3`eKgv^ari1j z(LE+tBT~hMBL{4owuY}XCw%C!y1Pm5S*`?nv!&uAje@jFlzfUkT0y*`;Ew9pyCL*) zE$f?LFR}Ibj;aof^qGw*!nCCWJZ}7QDB%@nQ_iCXh|CmkhG|yoso7mQ5q8u5kZlj0 z{CILm2ACJ3J%!B>i5gObnL;Qny?dk~5(ZGHN_Dd5(ZSk%pCn322kwKSf!j%mwy|5R zhZExgA%jqRJXdFzD#SFE?QB(rbH}MO$KcES?rL9NvyLZ4+9d|6tzUy+&Tlm1Zno@q zg^|W{><+j#K4xcH(1|J=9=GAOz1hPp))&YlnLGwhG66^nE2Ue9l^IyGZP^{hLEZhL ztB7uneO@N@(9>h-C04?_PTNdmJWA!rz(EJw;eDE|$U3Bk1uCX1%4HQ2KdqLIbi7hy zzuo>RT3SU%)<-3MN)7IZ3+ zO2Rc`VZyI)b-d_Du2k09bX}IHIvJ}fv{C1Yzh=K}Ym2IzH#I(KK=~F#XZPv}TIel~I!kl+K4sT>Z zzwi?!n=+oQAdbodc%2|)t|qKWhWwlGJ!c`BCQ-5*n!qjY^J7Wd{G@7|sj<~~P8lM5 zVG-&>oR@pOZifQ{Q_7KTp54s&L3{V>m5&Aa6|($H#9=|p z@xdz5S^N4c&75JR-PP#?AK#5FQp3djV#*YVSp&LOnXZ;#tBHG-e*y^Q^C)fX;i^83 zsYe9zgjgBE=f)SRAFG)QPw&(jCig4ywFN~yDx*@+H)+3gY@bfF)bh#R5VTvz*;JPE z-u+5qb+nk9*~R4sBck1$ds*R^;~uHWLX+x_9Y&Bzig&Zb=UkZC5x5l4UZqPO zl@1BVf43S+Iu6D{N>6EHO)wF5rp;yN|RK)zCvb$GqmBuep;H#Ej zd=Z(oBViLox0uff&Ck8RPv67;&3EM)m^ALexwZ(&kY7Oeu!)-7kYjao(V3yD0-EWzeM} zBaewmZS6SSd1vS&nh#?s&XsKqOVx@8^t(9;OKDd5KB&`?!`ID@pMBaXot>SpZAbcy z3GNcOT75!39M2hB#tSiDd-#cNn+U#guOcP!KtS1QVmeGCu%`XKjyc4M{8YdFq-ih{ zFOXn@SjzDk4pZ185SH5Iqp9m*na^=9&!~{ydx1ko5@MfA-7g-s`22Z8>=iMa@`fSRQn27KX@jUe0e)G7k4rNd=1OL zu{&^%$Y{l@!M8ZaaTduKzD5Zl*2;(smRaoY63L)HxW4O_Z}Fx7a+J>lF#qVWzaVV%RaO;uz02O5!V?`gAx zqo^f?6}M_De|BTnr4uTBg7@Xx6wh)K{RZt2oX3)3qy!Ta0AqP*33!Uu1(!Ul~i!yk%_ zJ1v#edY^+t@18-A=Y$)MOa0AvmAjvwd_;(xb?hvho`)rPpG2;&%mjJDK9N}myIZKu zsQ~c84nN;w8{=14zZu+q5FR*HaWk;pov%oPR>xZ*aaAS0eN~NL8xXIyD4#`rw@02< z6HuYH!4lZ5R>%>Eowt~|fno*AD3}(L#N-jNVpot1CA2e0rS|%+3)Xoa^BVijzi?HW zK3zXvpSf4j_2iLdd9p>1!o)~o>6Lr$x)CLHXG2Q^C10Sf4vk;L*5cCVB z97?E}fDJ#+4{f%my6q&oXSiRouS^W+(yQ%Oi;2wGW>h$sFP=A|0Nni`6vivpG1DC{ z?!+n?FK*3>qp(z;H_}Wp6ECiH`^a5zX1egF?tJuQZ%#gL)Hck>7n&5_{iroe z*Ol=}E#BRHx%G0Z>T1z~HvN74FHKn1rlj!t z*VYH$QgOaWRy-&dPcmTYZcA2_OtBhwEX;}1kuYtK|~T` zWnP&tcy-3q_WJ2GCNwh88qhm&!rLmAb>!C2N+jk`f(rF3#V zL|f#N8M<8Aks-5>FNQPf6^(AOb1)U^R1}flyu-V6b4^X-x2Hozq9S-_Nx@%k?6$eM%?h1c_ZAy(KX;?ipz(RSIgb@7{BZwu zj@mXiWT(ie8&MPZRhfNI{T2sX*qd(agMIzPuU=tXDtRy4;bY`@ho=yT56O;nKst8& zv><%M+8>*5(Y|eZ@iBq&>|y{x#Pnh&ff|Xv6t1$G$-tFfakx&qyVqp33ndEtAL@7@2T*k}psZLo`&P zr&N`Tb7zsf;`#o5xR$wGh4O9O_ox#~NSMNdmb-AvgYc6&{9))@W2~$4Kvk?bMskB) z2i#N+UuKrvlFJk26_Q{>>nyVu``%>}eT zLX!Da<29SKldBtYc=_Sp!EgNd+12IK!*_N`N(klF)6U9W+@^1>-_n$P&J@OjBaB~N z&SeB0yU7_^9`=08_FAWTN%bOXFnc(6^wCR`>Ss!wohgY}aiK7Fp~xc0LLt}6t=(A@ zskaS<>#pmvEH|-RX%5;XwTsqHDFpC@!q$qAfXdrNdMj@n+5#^qUbzseqPFah428+xcy#Atonq+lAY;VYbmCEcE_0=1Jre074ZfQv zYZGt*ER^QL1{_Mr6;jDlRF8{Li;KA!ayj$yjbTGs)H*y{#HxqawKh&6QGFeZb!U}( zXOQ)C)y6OxR9g-ErCEC8TW6DHO6P@)YOR}lJwCf;x_!>0NVktMCdjp^v4*=n=LfZrf-8ub z(aBaw``p=zdUvBz-c|LpNayWa#j|fT>YBGtju13HGl>bi)bqQWV$F3x)oSOCw1JKD zOTjLLG(v7E8Y&NNmvMI2l?APL-HN>@j7U#mH~Mf;Pf*;N-#|xF%E#$447ufHWX$q` z+oPn*wp!V0`o_dwSZ1!{0UxOPS8y6zMcdV?^pq!sgfI(3t%ED@B{KOpbMxmG8IdlS*{L0n_LdwQq(-ojAM1eP;JI zsfXEFl07k9N9}#cjxK%1QRi^@z;$UTTi4*J_Wev&3ti9aRwL#WuWb&c@o4i0P0c+0 zw&^GyXU1%Qu~I1TaS4l*iq%rOx%T9h`s4-gqPJUIa_)2-^d{=LUn0)-f-=rgus7)c z`Fs7loNISlq!2CBTn|l@9vQ2zWba2M9Sdyw@Z$k#hpfpBFRGiY;Grmj%*9QGtzKKx z8Y{HLFVl8QTqS%W^m@Pi(N#P`Dnf0yx`7UC>Y+Wy_0| zoktr*ZbWX2tY-p(YvhrXDh!dkz^k*>GV&dh20nZETeeb%UtH%>4GRZ!b1VpNZG`QW z41b7hbEUovDWnPbILqGm+yaD5ER}t%KpgxoS^3@j_r= z#*x|Ge0#|0@iyx!0b_f{9X|)wOiQUbu108jFo{vTh52Ew2(Sdzb=MHvO`RGXEp~%kW^T*a(GdN45*mF5spdz{Tr5 z?mbT>l**G(R8pTAuz2()XwwD*5e`{9Ptx`qOVeArP8PP}&E|w|5{nqyadbY+iYBq< z*D|)&yo8qpwo4_boxUVkb=7TeD}HD__wI4Xwd07GK-9EBmX@!)g0SJj}GC}6Xvsp8AhfNzE&xoja#aJR-<=@`zI0-?q|Fxp9@9_$~J=ytu;@H1IC<03Km> zRj^N72B+ijSsa!5iJ zqB;(g0Ii|>IxJt`uv7JHIf|Q;wk%JgGGmqblOdWV#H9BD%a<+ta1&A;7jT(V!D|7z z#FET7%8O}H`G)CZgU?#$CyT~1s!OB5+A1cpE>g50V4*QU*cmz?;QYYvFez{EIE*>t z>#0BGr8g}nngrDuUi-E|cL!WFL4ed&jn6Z;Csy`)1VzrKp4dr-^Sb88DVcw)A%+St zIB@bhSync)45vfinnU#mvdX)nK>_V2d<7Ekrl{6!R2zCMgHsu*T{YvR=H5(Rz!|*=4WHSie$DB)vdi>ld%N#T(v%I~K zWZW*Y&omlKr>gSWRzjrc%a#(H*5-bnaLG~S)uHmTXEimf8^>2`wQH+1zWCpx%E7mr zI(GbK@2j~i1BNuPHDqP6JCzV*rf8 zDvauOYthT9qQ?M?d19wD6H@=~Ya+HnAYT6|f z1DRh>`7DFP^E}GkI)04HMOi1%7wf>WcmF*!WBQV%hi${hk6SG#wSO&DjjJ zo&mG{q5zUNfhn0-*?y<2Hx-{R#sG%KrS-6xS=?jBZr@*}_OQD7F+HV6p5=`>%e&gF zxQaH3$gxTgyL(q;%h5nLS;LbeZxw_6ovm}bcPBK)XKyr^3OT!2sEYL#Jg`dc zC9@1&OwP~~cykm`cpj?5FHYr_lH{@6<%wGZT{2`P=n|B>>KRbAcw1aAh?4eBk`;?m zf?j^YXWV5d7k9>MUsGX`rWXb6d%BOP*fl zX_jEFx52^i4D0-UlV6k00{toW=?*z5<-%~}>Ra^@wOxicZE_Qt#~Xx0j=WlOj`N*g zADz**Gr(E#2e;~-wXDSYO5{^$DaMMbbeDnt@+lI@U!K-HUWt`?T9ZuvYBH`SXii!A zjy=cPgTS$gC2#u7ssj@oad~}}H7?NbxnQ;5Nz&L5vvaYQ4O!F;mud!K=CkM9x}Wgm zDMQ*S(2}uJWo#-fZfTcP3FJ=aRcYo1G-}%v$*~^u=9SQD*Hy~vE07i`v)bQID)5l8 zS?3bfKD%f0{*}{5O8Hr_n)iqDcQfX)`-e2eY@J1IKD>RCvDGR}O>8!*p|K&ya7=K# zv!#F=<{6K}tNBJ#97$JU2jnbP!e=J#7}fAGE2KkwthV57C#A|}OlBjSAUM*f`m__? zUnTD8RoCjO;5E4+hH@K}Rq%;Ho|EtsiwO`s`ha7 zY3`wk5bxYu;@jj?!JL$ebd$=5qYLL6G|EEcG}rmMjsf~N{718_*ep~>_?oTNvwXk; zsZ$cL!zYLMJ?|v=b$LkVr$t9H+DA{bJ4UIi$dgN2tpfBe6VNi`NVMdSE=h7n#CT`b zAFjj>>W+zcgrNafE)t&35z?zt3qwU)hkYmxMJqm7(}V zBm9t1gCWROvj$sVj((H5aaVci;oZ! zqHUY7$HFymI|+fUViRVbOI-Fa!pHPPd?BlDaz`P$&2 z`snt&bDS>ha?$J!#@S+j`Dn31i?C*vP#y%0z05OV^QZccD`*3e%+hJQ^#?crgvMS! zQJQ%qU0$bp=~QEah|ETl1b?vDYHYKRtApy_Qj zhQFMfdg{e3wtD;3Ezb~n%9J;%H8qmFnF9=VLSpYnODfMd^Y)nZd8B0y9tI71KC@Tr zE5>D0#s6jz*x_W!d(K zHQ1RAAbwAICnpAM^&soEv@+(RpVT$duiN$mo_BC|)-K_fNa}{gCP}c2IhW!ZS`R9R z5I+)`k^+R{(OD3qVwH6ywERO5*B@N;D(h-FEPeFG0)JDB{;TwdGV#yy#c7;ge2eoz z$7gOxw|){$S=G=T`MjMDZvm&K2$js*%l2Z0henZSq+ZZFJHYmo-TBIw$KN@;0kGa?_(ck2nKpA)|POl$q> z>es7Tgmg3vqWpWlg2SIh$a~9P9`-Br8}5|0Or3~oeQSy}kR0UHGQ>BeYD{b?Y`}r9 z59_9cWsooxF~!E3$F4Jjl$4chdyGU*xep>inhoKgdLuJ~f*Ix8{9V%NSKf`679@>m z(;LGgM6PuA7({A{bW{6OwI5j{pS@oSCavcb<~bswzilb<6ai?tY(FBel|ukcGqW-n zE~#jj*60sWAzO%Zt6fTK`{Ld7VAy|!v*}KjrooVlD9-6sDnd7!x_7=EgoQE3;q%Bt zYL^cqqY?v1&!KHoX>kREP}(xPLicAyQ^v2K?Nz4Prb-Kn8`f6BAArX=8XBTE^6fQC z?FSVM0=%`1!QUz{ARxu%H1wOpYc#GgT*4R+yCH~kgZ{LRW= z-%g!e-l#e$Nn--L?0cItU)r!1-7MxABEOYWG3(aGa(nfL<-ip-{!2225M6mla;)z_7|BfyWm!jBtZ5gc3%1 z_TWTQ-x#|Ck(;CMb9}1nJt93C@$C|+0Y&0}QFnr=D8Uq)@I$!-elI)`_(?<;0BN{8 z!a&jp2b3fXY3^j>h(e$XfY6W1ruHz9_&v;ZN72R%=46VpK{%+xoNO!sm%k1k4raRV`>GHL*H;mdEeGhE<5 z=I~n)6A%CYP{L#wqvYTZCXzl*ZSQxv?wfWYG$85*&W*71LK{M{-6*=7=C@ufemWW$ z(b~c~3~#+93J;btoXF4U(1zBKB}z?Fel(%jDaC#esE@#U7O*|*etWd>HU(us%cmEL z5d7-RZg#wOs3>&M;X0v=u+b71j&bUh$&?hg&n%qYUiJE4urDR*O5RJyBS_mhcs;dl z`6%i+U-hT)vnIILE91q76+n_Fj73#BL9h5Y>bmEEz1b{ex|*vO$js^Nu_r8>aPj!NAL>(dj_zb+6`z62CPBk9o8?ynhH`ru zA1GE^otH-488Vla9$@7DQgC0cWm+^}=u1%ypVz$ZoJSL-s0`aoBl4Vo6Khmp8TZo_@14Zk99p9X1iRYB6aPi# zadG_-q2IF%r0#5n!layrlQZl)RooN_!^HW&Pc|DTBuc{C)CrSBil#sN-24ELwv7eK z8mSKkLbW_x${G#{N=vCylP;!no(p48PTg z2?_m>k6ueZ#NKy8n5)B3`XF>IA4mh{hBExqQuvFX@0(h|K=)0ZFbj+iBLJiZLn54= z%wb3%7e_fAXN8gz2nNC`UmM$ko-A(-p#raIyk%2?z*)z}z5i zZVogD2h!aEW$MP^fTYKW_#xxB2&1rilwEEISidtoZOt~24vCU=QHNxVFiMKzrL41 z*xNfh*r42nFe3kufwu3bR7q(qm=h9hp9s3f^C#CIG|W6e7{h;=$p3b7Xg~k%wLeYH z&D0S+Iq+|jL-Pqrpld|Y{~Q_!1YAVh$p&3TDg^o~{kzS-6Je6*B2y7=FgGs;Sb&3z zO9KoRp@IKK;9pE|3>5=a_rw0`~Ocw{(#odXt^+*D+W0GF92HJ zzhHMiU_U81%pMH^qCr4Y2Mbdt3jpXo5QFle(arl-z#pI`=pGP#bfAJpW8Kks80bFS z)Es7y1_(joaD=%X2Ac#aprJ)BG*J2rSol}0@Xz`r5$4Vq85n(B7}NfwP(lAY3HWD; z-}x}78^0Z7{1B<(1cRv|5ExP4QBce?kn1}X3IM4g;4mdqNA!r_W(d0f)8g;b{HqGg zu%9ZFo$ZnO05DJ=_`{eVzlH$AUncnr`*(O71I8lJTK|H;q8(&ofz^9i8uJm@hzjq2=(RDuJc^Em|Xd34Jf2*GdV++PU$sY}h*3E~u<+o`<1w?*00v!$v z?#Lo3h`~L<0$f~ZLj=Au!fo2A)W1kARYk$ z9tnP^xU?h}lvi9z0>UjJjm~IkhyYlchgX0PvwtB`rcS8uX~V?_;Q|8~8KqRD0sjZI Cm;{di literal 0 HcmV?d00001