diff --git a/.zed/debug.json b/.zed/debug.json index c843649a..77364a0c 100644 --- a/.zed/debug.json +++ b/.zed/debug.json @@ -2,7 +2,7 @@ { "label": "Debug BotServer", "build": { - "command": "rm -rf ./botserver-stack && cargo", + "command": "rm -rf .env ./botserver-stack && cargo", "args": ["build"] }, "program": "$ZED_WORKTREE_ROOT/target/debug/botserver", diff --git a/Cargo.lock b/Cargo.lock index 1512ea03..ed7fcfe2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1032,6 +1032,7 @@ dependencies = [ "futures-util", "headless_chrome", "imap", + "indicatif", "lettre", "livekit", "log", @@ -1336,6 +1337,19 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "console" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1970,6 +1984,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -2855,6 +2875,19 @@ dependencies = [ "hashbrown 0.16.0", ] +[[package]] +name = "indicatif" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "inout" version = "0.1.4" @@ -5626,6 +5659,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unit-prefix" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" + [[package]] name = "universal-hash" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 48344b6d..23ae4b5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,3 +93,4 @@ pdf-extract = "0.10.0" scraper = "0.20" sha2 = "0.10.9" ureq = "3.1.2" +indicatif = "0.18.0" diff --git a/add-req.sh b/add-req.sh index c8949b4a..ed1297f1 100755 --- a/add-req.sh +++ b/add-req.sh @@ -23,10 +23,11 @@ dirs=( #"automation" #"basic" #"bot" + "bootstrap" "package_manager" #"channels" "config" - "context" + #"context" #"email" #"file" #"llm" diff --git a/fix-errors.sh b/fix-errors.sh index d82b6439..3a6fbd6f 100755 --- a/fix-errors.sh +++ b/fix-errors.sh @@ -22,9 +22,9 @@ dirs=( #"automation" #"basic" #"bot" - #"bootstrap" + "bootstrap" #"channels" - "config" + #"config" #"context" #"email" #"file" @@ -33,7 +33,7 @@ dirs=( #"org" "package_manager" #"session" - #"shared" + "shared" #"tests" #"tools" #"web_automation" @@ -53,7 +53,7 @@ cat "$PROJECT_ROOT/src/main.rs" >> "$OUTPUT_FILE" echo "" >> "$OUTPUT_FILE" - +echo "Compiling..." cargo build --message-format=short 2>&1 | grep -E 'error' >> "$OUTPUT_FILE" diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs index beff5512..b110e45a 100644 --- a/src/bootstrap/mod.rs +++ b/src/bootstrap/mod.rs @@ -1,7 +1,11 @@ use crate::config::AppConfig; use crate::package_manager::{InstallMode, PackageManager}; use anyhow::Result; -use log::{debug, info, trace, warn}; +use log::{debug, trace, warn}; +use rand::distr::Alphanumeric; +use rand::rngs::ThreadRng; +use rand::Rng; +use sha2::{Digest, Sha256}; pub struct BootstrapManager { pub install_mode: InstallMode, @@ -10,10 +14,7 @@ pub struct BootstrapManager { impl BootstrapManager { pub fn new(install_mode: InstallMode, tenant: Option) -> Self { - info!( - "Initializing BootstrapManager with mode {:?} and tenant {:?}", - install_mode, tenant - ); + trace!("Initializing BootstrapManager with mode {:?} and tenant {:?}", install_mode, tenant); Self { install_mode, tenant, @@ -21,76 +22,94 @@ impl BootstrapManager { } pub fn start_all(&mut self) -> Result<()> { - info!("Starting all components"); let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?; - let components = vec![ - "tables", - "cache", - "drive", - "llm", - "email", - "proxy", - "directory", - "alm", - "alm_ci", - "dns", - "webmail", - "meeting", - "table_editor", - "doc_editor", - "desktop", - "devtools", - "bot", - "system", - "vector_db", - "host", + "tables", "cache", "drive", "llm", "email", "proxy", "directory", + "alm", "alm_ci", "dns", "webmail", "meeting", "table_editor", + "doc_editor", "desktop", "devtools", "bot", "system", "vector_db", "host" ]; for component in components { - info!("Starting component: {}", component); - pm.start(component)?; - trace!("Successfully started component: {}", component); + if pm.is_installed(component) { + trace!("Starting component: {}", component); + pm.start(component)?; + } else { + trace!("Component {} not installed, skipping start", component); + } } - - info!("All components started successfully"); Ok(()) } pub fn bootstrap(&mut self) -> Result { - info!("Starting bootstrap process"); - let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?; - let required_components = vec!["tables", "cache", "drive", "llm"]; - + for component in required_components { - info!("Checking component: {}", component); if !pm.is_installed(component) { - info!("Installing required component: {}", component); + trace!("Installing required component: {}", component); futures::executor::block_on(pm.install(component))?; - trace!("Successfully installed component: {}", component); + trace!("Starting component after install: {}", component); + pm.start(component)?; } else { - debug!("Component {} already installed", component); + trace!("Required component {} already installed", component); } } - info!("Bootstrap completed successfully"); - - let config = match diesel::Connection::establish( - "postgres://botserver:botserver@localhost:5432/botserver", - ) { + let config = match diesel::Connection::establish("postgres://botserver:botserver@localhost:5432/botserver") { Ok(mut conn) => { - trace!("Connected to database for config loading"); + self.setup_secure_credentials(&mut conn)?; AppConfig::from_database(&mut conn) } Err(e) => { warn!("Failed to connect to database for config: {}", e); - trace!("Falling back to environment configuration"); AppConfig::from_env() } }; Ok(config) } + + fn setup_secure_credentials(&self, conn: &mut diesel::PgConnection) -> Result<()> { + use crate::shared::models::schema::bots::dsl::*; + use diesel::prelude::*; + use uuid::Uuid; + + let farm_password = std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32)); + let db_password = self.generate_secure_password(16); + + let encrypted_db_password = self.encrypt_password(&db_password, &farm_password); + + let env_contents = format!( + "FARM_PASSWORD={}\nDATABASE_URL=postgres://gbuser:{}@localhost:5432/botserver", + farm_password, db_password + ); + + std::fs::write(".env", env_contents) + .map_err(|e| anyhow::anyhow!("Failed to write .env file: {}", e))?; + + let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?; + diesel::update(bots) + .filter(bot_id.eq(system_bot_id)) + .set(config.eq(serde_json::json!({ + "encrypted_db_password": encrypted_db_password, + }))) + .execute(conn)?; + + Ok(()) + } + + fn generate_secure_password(&self, length: usize) -> String { + let mut rng: ThreadRng = rand::thread_rng(); + rng.sample_iter(&Alphanumeric) + .take(length) + .map(char::from) + .collect() + } + + fn encrypt_password(&self, password: &str, key: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(key.as_bytes()); + hasher.update(password.as_bytes()); + format!("{:x}", hasher.finalize()) + } } diff --git a/src/main.rs b/src/main.rs index c80a3e04..c21dc211 100644 --- a/src/main.rs +++ b/src/main.rs @@ -104,7 +104,6 @@ async fn main() -> std::io::Result<()> { }; let mut bootstrap = BootstrapManager::new(install_mode.clone(), tenant.clone()); - let _ = bootstrap.start_all(); let cfg = match bootstrap.bootstrap() { Ok(config) => { info!("Bootstrap completed successfully, configuration loaded from database"); @@ -113,8 +112,7 @@ async fn main() -> std::io::Result<()> { Err(e) => { log::error!("Bootstrap failed: {}", e); info!("Attempting to load configuration from database"); - match diesel::Connection::establish(&format!("postgres://localhost:5432/botserver_db")) - { + match diesel::Connection::establish("postgres://botserver:botserver@localhost:5432/botserver") { Ok(mut conn) => AppConfig::from_database(&mut conn), Err(_) => { info!("Database not available, using environment variables as fallback"); @@ -124,6 +122,7 @@ async fn main() -> std::io::Result<()> { } }; + let _ = bootstrap.start_all(); let config = std::sync::Arc::new(cfg.clone()); info!("Establishing database connection to {}", cfg.database_url()); diff --git a/src/package_manager/facade.rs b/src/package_manager/facade.rs index c83b2458..eed00bd3 100644 --- a/src/package_manager/facade.rs +++ b/src/package_manager/facade.rs @@ -1,17 +1,15 @@ +use crate::package_manager::component::ComponentConfig; +use crate::package_manager::installer::PackageManager; +use crate::package_manager::OsType; +use crate::shared::utils; +use crate::InstallMode; use anyhow::{Context, Result}; -use log::{debug, info, trace, warn}; +use log::{trace, warn}; use reqwest::Client; use std::collections::HashMap; use std::path::PathBuf; use std::process::Command; -use crate::shared::utils; // Adjust based on your actual module structure - -use crate::package_manager::component::ComponentConfig; -use crate::package_manager::installer::PackageManager; -use crate::package_manager::OsType; -use crate::InstallMode; - impl PackageManager { pub async fn install(&self, component_name: &str) -> Result<()> { let component = self @@ -19,9 +17,10 @@ impl PackageManager { .get(component_name) .context(format!("Component '{}' not found", component_name))?; - info!( + trace!( "Starting installation of component '{}' in {:?} mode", - component_name, self.mode + component_name, + self.mode ); for dep in &component.dependencies { @@ -36,16 +35,15 @@ impl PackageManager { InstallMode::Container => self.install_container(component)?, } - info!( + trace!( "Component '{}' installation completed successfully", component_name ); Ok(()) } - // ... rest of the implementation remains exactly the same ... pub async fn install_local(&self, component: &ComponentConfig) -> Result<()> { - info!( + trace!( "Installing component '{}' locally to {}", component.name, self.base_path.display() @@ -75,18 +73,19 @@ impl PackageManager { let url = url.clone(); let name = component.name.clone(); let binary_name = component.binary_name.clone(); - self.download_and_install(&url, &name, binary_name.as_deref()) .await?; - self.run_commands(post_cmds, "local", &component.name)?; } - info!("Local installation of '{}' completed", component.name); + + self.run_commands(post_cmds, "local", &component.name)?; + trace!("Starting component after installation: {}", component.name); + self.start(&component.name)?; + Ok(()) } pub fn install_container(&self, component: &ComponentConfig) -> Result<()> { let container_name = format!("{}-{}", self.tenant, component.name); - info!("Creating LXC container: {}", container_name); let output = Command::new("lxc") .args(&[ @@ -106,7 +105,6 @@ impl PackageManager { } std::thread::sleep(std::time::Duration::from_secs(15)); - self.exec_in_container(&container_name, "mkdir -p /opt/gbo/{bin,data,conf,logs}")?; let (pre_cmds, post_cmds) = match self.os_type { @@ -134,10 +132,7 @@ impl PackageManager { if !packages.is_empty() { let pkg_list = packages.join(" "); - self.exec_in_container( - &container_name, - &format!("apt-get update && apt-get install -y {}", pkg_list), - )?; + self.exec_in_container(&container_name, &format!("apt-get update && apt-get install -y {}", pkg_list))?; } if let Some(url) = &component.download_url { @@ -167,11 +162,12 @@ impl PackageManager { } self.setup_port_forwarding(&container_name, &component.ports)?; - - info!( + trace!( "Container installation of '{}' completed in {}", - component.name, container_name + component.name, + container_name ); + Ok(()) } @@ -181,27 +177,22 @@ impl PackageManager { .get(component_name) .context(format!("Component '{}' not found", component_name))?; - info!("Removing component: {}", component_name); - match self.mode { InstallMode::Local => self.remove_local(component)?, InstallMode::Container => self.remove_container(component)?, } - info!("Component '{}' removed successfully", component_name); Ok(()) } pub fn remove_local(&self, component: &ComponentConfig) -> Result<()> { let bin_path = self.base_path.join("bin").join(&component.name); let _ = std::fs::remove_dir_all(bin_path); - Ok(()) } pub fn remove_container(&self, component: &ComponentConfig) -> Result<()> { let container_name = format!("{}-{}", self.tenant, component.name); - let _ = Command::new("lxc") .args(&["stop", &container_name]) .output(); @@ -252,7 +243,6 @@ impl PackageManager { let path = self.base_path.join(dir).join(component); std::fs::create_dir_all(&path) .context(format!("Failed to create directory: {:?}", path))?; - trace!("Created directory: {:?}", path); } Ok(()) } @@ -268,7 +258,7 @@ impl PackageManager { return Ok(()); } - info!( + trace!( "Installing {} system packages for component '{}'", packages.len(), component.name @@ -276,8 +266,10 @@ impl PackageManager { match self.os_type { OsType::Linux => { - let output = Command::new("apt-get").args(&["update"]).output()?; - + let output = Command::new("apt-get") + .args(&["update"]) + .output()?; + if !output.status.success() { warn!("apt-get update had issues"); } @@ -286,6 +278,7 @@ impl PackageManager { .args(&["install", "-y"]) .args(packages) .output()?; + if !output.status.success() { warn!("Some packages may have failed to install"); } @@ -295,6 +288,7 @@ impl PackageManager { .args(&["install"]) .args(packages) .output()?; + if !output.status.success() { warn!("Homebrew installation had warnings"); } @@ -323,11 +317,8 @@ impl PackageManager { bin_path.join(filename) }; - info!("Downloading from: {} to {:?}", url, temp_file); - self.download_with_reqwest(url, &temp_file, component) .await?; - self.handle_downloaded_file(&temp_file, &bin_path, binary_name)?; Ok(()) @@ -351,25 +342,25 @@ impl PackageManager { for attempt in 0..=MAX_RETRIES { if attempt > 0 { - info!( + trace!( "Retry attempt {}/{} for {}", - attempt, MAX_RETRIES, component + attempt, + MAX_RETRIES, + component ); std::thread::sleep(RETRY_DELAY * attempt); } match self.attempt_reqwest_download(&client, url, temp_file).await { - Ok(size) => { - info!("Downloaded {} bytes for {}", size, component); + Ok(_size) => { if attempt > 0 { - info!("Download succeeded on attempt {}", attempt + 1); + trace!("Download succeeded on retry attempt {}", attempt); } return Ok(()); } Err(e) => { warn!("Download attempt {} failed: {}", attempt + 1, e); last_error = Some(e); - let _ = std::fs::remove_file(temp_file); } } @@ -385,25 +376,17 @@ impl PackageManager { pub async fn attempt_reqwest_download( &self, - _client: &Client, // We won't use this if using shared utils + _client: &Client, url: &str, temp_file: &PathBuf, ) -> Result { - info!("Downloading from: {} to {:?}", url, temp_file); - - // Convert PathBuf to string for the shared function let output_path = temp_file.to_str().context("Invalid temp file path")?; - - // Use the shared download_file utility utils::download_file(url, output_path) .await .map_err(|e| anyhow::anyhow!("Failed to download file using shared utility: {}", e))?; - // Get file size to return let metadata = std::fs::metadata(temp_file).context("Failed to get file metadata")?; let size = metadata.len(); - - info!("Downloaded {} bytes", size); Ok(size) } @@ -418,8 +401,6 @@ impl PackageManager { return Err(anyhow::anyhow!("Downloaded file is empty")); } - info!("Final file size: {} bytes", metadata.len()); - let file_extension = temp_file .extension() .and_then(|ext| ext.to_str()) @@ -447,7 +428,6 @@ impl PackageManager { } pub fn extract_tar_gz(&self, temp_file: &PathBuf, bin_path: &PathBuf) -> Result<()> { - info!("Extracting tar.gz archive to {:?}", bin_path); let output = Command::new("tar") .current_dir(bin_path) .args(&["-xzf", temp_file.to_str().unwrap(), "--strip-components=1"]) @@ -465,8 +445,6 @@ impl PackageManager { } pub fn extract_zip(&self, temp_file: &PathBuf, bin_path: &PathBuf) -> Result<()> { - info!("Extracting zip archive to {:?}", bin_path); - let output = Command::new("unzip") .current_dir(bin_path) .args(&["-o", "-q", temp_file.to_str().unwrap()]) @@ -490,12 +468,8 @@ impl PackageManager { name: &str, ) -> Result<()> { let final_path = bin_path.join(name); - std::fs::rename(temp_file, &final_path)?; - self.make_executable(&final_path)?; - - info!("Installed binary: {:?}", final_path); Ok(()) } @@ -517,7 +491,6 @@ impl PackageManager { env_vars: &HashMap, ) -> Result<()> { let service_path = format!("/etc/systemd/system/{}.service", component); - let bin_path = self.base_path.join("bin").join(component); let data_path = self.base_path.join("data").join(component); let conf_path = self.base_path.join("conf").join(component); @@ -545,12 +518,11 @@ impl PackageManager { } let service_content = format!( - "[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory={}\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", - component, env_section, rendered_cmd, data_path.to_string_lossy() - ); + "[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory={}\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", + component, env_section, rendered_cmd, data_path.to_string_lossy() + ); std::fs::write(&service_path, service_content)?; - Command::new("systemctl") .args(&["daemon-reload"]) .output()?; @@ -561,7 +533,6 @@ impl PackageManager { .args(&["start", &format!("{}.service", component)]) .output()?; - info!("Created and started systemd service: {}.service", component); Ok(()) } @@ -597,13 +568,12 @@ impl PackageManager { .replace("{{CONF_PATH}}", &conf_path.to_string_lossy()) .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()); - trace!("Executing command: {}", rendered_cmd); - if target == "local" { let output = Command::new("bash") .current_dir(&bin_path) .args(&["-c", &rendered_cmd]) .output()?; + if !output.status.success() { warn!( "Command had non-zero exit: {}", @@ -614,11 +584,11 @@ impl PackageManager { self.exec_in_container(target, &rendered_cmd)?; } } + Ok(()) } pub fn exec_in_container(&self, container: &str, command: &str) -> Result<()> { - debug!("Executing in container {}: {}", container, command); let output = Command::new("lxc") .args(&["exec", container, "--", "bash", "-c", command]) .output()?; @@ -629,6 +599,7 @@ impl PackageManager { String::from_utf8_lossy(&output.stderr) ); } + Ok(()) } @@ -696,6 +667,7 @@ impl PackageManager { container ); } + Ok(()) } @@ -723,9 +695,9 @@ impl PackageManager { } let service_content = format!( - "[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory=/opt/gbo/data\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", - component, env_section, rendered_cmd - ); + "[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory=/opt/gbo/data\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", + component, env_section, rendered_cmd + ); let service_file = format!("/tmp/{}.service", component); std::fs::write(&service_file, &service_content)?; @@ -748,11 +720,12 @@ impl PackageManager { self.exec_in_container(container, &format!("systemctl start {}", component))?; std::fs::remove_file(&service_file)?; - - info!( + trace!( "Created and started service in container {}: {}", - container, component + container, + component ); + Ok(()) } @@ -777,16 +750,17 @@ impl PackageManager { ]) .output()?; - if !output.status.success() { - warn!("Failed to setup port forwarding for port {}", port); - } - - trace!( - "Port forwarding configured: {} -> container {}", - port, - container - ); + if !output.status.success() { + warn!("Failed to setup port forwarding for port {}", port); } - Ok(()) + + trace!( + "Port forwarding configured: {} -> container {}", + port, + container + ); } + + Ok(()) +} } diff --git a/src/package_manager/installer.rs b/src/package_manager/installer.rs index 8b600c4f..1841c9ec 100644 --- a/src/package_manager/installer.rs +++ b/src/package_manager/installer.rs @@ -1,11 +1,14 @@ -use anyhow::Result; -use log::info; -use std::collections::HashMap; -use std::path::PathBuf; - use crate::package_manager::component::ComponentConfig; use crate::package_manager::os::detect_os; use crate::package_manager::{InstallMode, OsType}; +use anyhow::Result; +use log::trace; +use rand::distr::Alphanumeric; +use rand::rngs::ThreadRng; +use rand::Rng; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::path::PathBuf; pub struct PackageManager { pub mode: InstallMode, @@ -23,7 +26,6 @@ impl PackageManager { } else { std::env::current_dir()?.join("botserver-stack") }; - let tenant = tenant.unwrap_or_else(|| "default".to_string()); let mut pm = PackageManager { @@ -33,14 +35,7 @@ impl PackageManager { tenant, components: HashMap::new(), }; - pm.register_components(); - info!( - "PackageManager initialized with {} components in {:?} mode for tenant {}", - pm.components.len(), - pm.mode, - pm.tenant - ); Ok(pm) } @@ -68,20 +63,29 @@ impl PackageManager { } fn register_drive(&mut self) { + let drive_password = self.generate_secure_password(16); + let farm_password = + std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32)); + let encrypted_drive_password = self.encrypt_password(&drive_password, &farm_password); + self.components.insert("drive".to_string(), ComponentConfig { name: "drive".to_string(), required: true, ports: vec![9000, 9001], dependencies: vec![], - linux_packages: vec!["wget".to_string()], - macos_packages: vec!["wget".to_string()], + linux_packages: vec![], + macos_packages: vec![], windows_packages: vec![], download_url: Some("https://dl.min.io/server/minio/release/linux-amd64/minio".to_string()), binary_name: Some("minio".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![ "wget https://dl.min.io/client/mc/release/linux-amd64/mc -O {{BIN_PATH}}/mc".to_string(), - "chmod +x {{BIN_PATH}}/mc".to_string() + "chmod +x {{BIN_PATH}}/mc".to_string(), + format!("{{BIN_PATH}}/mc alias set mc http://localhost:9000 gbdriveuser {}", drive_password).to_string(), + "{{BIN_PATH}}/mc mb mc/default.gbai".to_string(), + format!("{{BIN_PATH}}/mc admin user add mc gbdriveuser {}", drive_password).to_string(), + "{{BIN_PATH}}/mc admin policy attach mc readwrite --user=gbdriveuser".to_string() ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![ @@ -91,11 +95,74 @@ impl PackageManager { pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::from([ - ("MINIO_ROOT_USER".to_string(), "minioadmin".to_string()), - ("MINIO_ROOT_PASSWORD".to_string(), "minioadmin".to_string()) + ("MINIO_ROOT_USER".to_string(), "gbdriveuser".to_string()), + ("MINIO_ROOT_PASSWORD".to_string(), drive_password) ]), exec_cmd: "{{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001".to_string(), }); + + self.update_drive_credentials_in_database(&encrypted_drive_password) + .ok(); + } + + fn update_drive_credentials_in_database(&self, encrypted_drive_password: &str) -> Result<()> { + use crate::shared::models::schema::bots::dsl::*; + use diesel::pg::PgConnection; + use diesel::prelude::*; + use uuid::Uuid; + + if let Ok(mut conn) = + PgConnection::establish("postgres://botserver:botserver@localhost:5432/botserver") + { + let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?; + diesel::update(bots) + .filter(bot_id.eq(system_bot_id)) + .set(config.eq(serde_json::json!({ + "encrypted_drive_password": encrypted_drive_password, + }))) + .execute(&mut conn)?; + } + Ok(()) + } + + fn register_tables(&mut self) { + self.components.insert("tables".to_string(), ComponentConfig { + name: "tables".to_string(), + required: true, + ports: vec![5432], + dependencies: vec![], + linux_packages: vec![], + macos_packages: vec![], + windows_packages: vec![], + download_url: Some("https://github.com/theseus-rs/postgresql-binaries/releases/download/18.0.0/postgresql-18.0.0-x86_64-unknown-linux-gnu.tar.gz".to_string()), + binary_name: None, + pre_install_cmds_linux: vec![], + post_install_cmds_linux: vec![ + "tar -xzf postgresql-18.0.0-x86_64-unknown-linux-gnu.tar.gz --strip-components=1".to_string(), + "chmod +x bin/*".to_string(), + "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/initdb -D {{DATA_PATH}}/pgdata -U postgres; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"data_directory = '{{DATA_PATH}}/pgdata'\" > {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"hba_file = '{{CONF_PATH}}/pg_hba.conf'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"ident_file = '{{CONF_PATH}}/pg_ident.conf'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"port = 5432\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"listen_addresses = '*'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"log_directory = '{{LOGS_PATH}}'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"logging_collector = on\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/pg_hba.conf\" ]; then echo \"host all all all md5\" > {{CONF_PATH}}/pg_hba.conf; fi".to_string(), + "if [ ! -f \"{{CONF_PATH}}/pg_ident.conf\" ]; then touch {{CONF_PATH}}/pg_ident.conf; fi".to_string(), + "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start; sleep 5; ./bin/createdb -p 5432 -h localhost botserver; ./bin/createuser -p 5432 -h localhost gbuser; fi".to_string() + ], + pre_install_cmds_macos: vec![], + post_install_cmds_macos: vec![ + "tar -xzf postgresql-18.0.0-x86_64-unknown-linux-gnu.tar.gz --strip-components=1".to_string(), + "chmod +x bin/*".to_string(), + "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/initdb -D {{DATA_PATH}}/pgdata -U postgres; fi".to_string(), + ], + pre_install_cmds_windows: vec![], + post_install_cmds_windows: vec![], + env_vars: HashMap::new(), + exec_cmd: "./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start".to_string(), + }); } fn register_cache(&mut self) { @@ -104,15 +171,15 @@ impl PackageManager { required: true, ports: vec![6379], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "curl".to_string(), "gnupg".to_string(), "lsb-release".to_string()], + linux_packages: vec!["curl".to_string(), "gnupg".to_string(), "lsb-release".to_string()], macos_packages: vec!["redis".to_string()], windows_packages: vec![], download_url: None, binary_name: Some("valkey-server".to_string()), pre_install_cmds_linux: vec![ - "if [ ! -f /usr/share/keyrings/valkey.gpg ]; then curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/valkey.gpg; fi".to_string(), - "if [ ! -f /etc/apt/sources.list.d/valkey.list ]; then echo 'deb [signed-by=/usr/share/keyrings/valkey.gpg] https://packages.redis.io/deb $(lsb_release -cs) main' | tee /etc/apt/sources.list.d/valkey.list; fi".to_string(), - "apt-get update && apt-get install -y valkey".to_string() + "if [ ! -f /usr/share/keyrings/valkey.gpg ]; then curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/valkey.gpg; fi".to_string(), + "if [ ! -f /etc/apt/sources.list.d/valkey.list ]; then echo 'deb [signed-by=/usr/share/keyrings/valkey.gpg] https://packages.redis.io/deb $(lsb_release -cs) main' | tee /etc/apt/sources.list.d/valkey.list; fi".to_string(), + "apt-get update && apt-get install -y valkey".to_string() ], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], @@ -124,62 +191,26 @@ impl PackageManager { }); } - fn register_tables(&mut self) { - self.components.insert("tables".to_string(), ComponentConfig { - name: "tables".to_string(), - required: true, - ports: vec![5432], - dependencies: vec![], - linux_packages: vec!["wget".to_string()], - macos_packages: vec!["wget".to_string()], - windows_packages: vec![], - download_url: Some("https://github.com/theseus-rs/postgresql-binaries/releases/download/18.0.0/postgresql-18.0.0-x86_64-unknown-linux-gnu.tar.gz".to_string()), - binary_name: Some("postgres".to_string()), - pre_install_cmds_linux: vec![], - post_install_cmds_linux: vec![ - "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/initdb -D {{DATA_PATH}}/pgdata -U postgres; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"data_directory = '{{DATA_PATH}}/pgdata'\" > {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"hba_file = '{{CONF_PATH}}/pg_hba.conf'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"ident_file = '{{CONF_PATH}}/pg_ident.conf'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"port = 5432\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"listen_addresses = '*'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"log_directory = '{{LOGS_PATH}}'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"logging_collector = on\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/pg_hba.conf\" ]; then echo \"host all all all md5\" > {{CONF_PATH}}/pg_hba.conf; fi".to_string(), - "if [ ! -f \"{{CONF_PATH}}/pg_ident.conf\" ]; then touch {{CONF_PATH}}/pg_ident.conf; fi".to_string(), - "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start; sleep 5; ./bin/psql -p 5432 -d postgres -c \" CREATE USER default WITH PASSWORD 'defaultpass'\"; ./bin/psql -p 5432 -d postgres -c \"CREATE DATABASE default_db OWNER default\"; ./bin/psql -p 5432 -d postgres -c \"GRANT ALL PRIVILEGES ON DATABASE default_db TO default\"; pkill; fi".to_string() - ], - pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec![ - "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/initdb -D {{DATA_PATH}}/pgdata -U postgres; fi".to_string(), - ], - pre_install_cmds_windows: vec![], - post_install_cmds_windows: vec![], - env_vars: HashMap::new(), - exec_cmd: "./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start".to_string(), - }); - } - fn register_llm(&mut self) { self.components.insert("llm".to_string(), ComponentConfig { name: "llm".to_string(), required: true, ports: vec![8081], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "unzip".to_string()], - macos_packages: vec!["wget".to_string(), "unzip".to_string()], + linux_packages: vec!["unzip".to_string()], + macos_packages: vec!["unzip".to_string()], windows_packages: vec![], download_url: Some("https://github.com/ggml-org/llama.cpp/releases/download/b6148/llama-b6148-bin-ubuntu-x64.zip".to_string()), binary_name: Some("llama-server".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![ - "wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), - "wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() + "wget -q https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), + "wget -q https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![ - "wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), - "wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() + "wget -q https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), + "wget -q https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() ], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], @@ -194,7 +225,7 @@ impl PackageManager { required: false, ports: vec![25, 80, 110, 143, 465, 587, 993, 995, 4190], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "libcap2-bin".to_string(), "resolvconf".to_string()], + linux_packages: vec!["libcap2-bin".to_string(), "resolvconf".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/stalwartlabs/stalwart/releases/download/v0.13.1/stalwart-x86_64-unknown-linux-gnu.tar.gz".to_string()), @@ -218,8 +249,8 @@ impl PackageManager { required: false, ports: vec![80, 443], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "libcap2-bin".to_string()], - macos_packages: vec!["wget".to_string()], + linux_packages: vec!["libcap2-bin".to_string()], + macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/caddyserver/caddy/releases/download/v2.10.0-beta.3/caddy_2.10.0-beta.3_linux_amd64.tar.gz".to_string()), binary_name: Some("caddy".to_string()), @@ -244,7 +275,7 @@ impl PackageManager { required: false, ports: vec![8080], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "libcap2-bin".to_string()], + linux_packages: vec!["libcap2-bin".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/zitadel/zitadel/releases/download/v2.71.2/zitadel-linux-amd64.tar.gz".to_string()), @@ -268,7 +299,7 @@ impl PackageManager { required: false, ports: vec![3000], dependencies: vec![], - linux_packages: vec!["git".to_string(), "git-lfs".to_string(), "wget".to_string()], + linux_packages: vec!["git".to_string(), "git-lfs".to_string()], macos_packages: vec!["git".to_string(), "git-lfs".to_string()], windows_packages: vec![], download_url: Some("https://codeberg.org/forgejo/forgejo/releases/download/v10.0.2/forgejo-10.0.2-linux-amd64".to_string()), @@ -293,14 +324,14 @@ impl PackageManager { required: false, ports: vec![], dependencies: vec!["alm".to_string()], - linux_packages: vec!["wget".to_string(), "git".to_string(), "curl".to_string(), "gnupg".to_string(), "ca-certificates".to_string(), "build-essential".to_string()], + linux_packages: vec!["git".to_string(), "curl".to_string(), "gnupg".to_string(), "ca-certificates".to_string(), "build-essential".to_string()], macos_packages: vec!["git".to_string(), "node".to_string()], windows_packages: vec![], download_url: Some("https://code.forgejo.org/forgejo/runner/releases/download/v6.3.1/forgejo-runner-6.3.1-linux-amd64".to_string()), binary_name: Some("forgejo-runner".to_string()), pre_install_cmds_linux: vec![ "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -".to_string(), - "apt-get update && apt-get install -y nodejs".to_string() + "apt-get install -y nodejs".to_string() ], post_install_cmds_linux: vec![ "npm install -g pnpm@latest".to_string() @@ -322,7 +353,7 @@ impl PackageManager { required: false, ports: vec![53], dependencies: vec![], - linux_packages: vec!["wget".to_string()], + linux_packages: vec![], macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/coredns/coredns/releases/download/v1.12.4/coredns_1.12.4_linux_amd64.tgz".to_string()), @@ -368,7 +399,7 @@ impl PackageManager { required: false, ports: vec![7880, 3478], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "coturn".to_string()], + linux_packages: vec!["coturn".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/livekit/livekit/releases/download/v1.8.4/livekit_1.8.4_linux_amd64.tar.gz".to_string()), @@ -386,13 +417,13 @@ impl PackageManager { fn register_table_editor(&mut self) { self.components.insert( - "table-editor".to_string(), + "table_editor".to_string(), ComponentConfig { - name: "table-editor".to_string(), + name: "table_editor".to_string(), required: false, ports: vec![5757], dependencies: vec!["tables".to_string()], - linux_packages: vec!["wget".to_string(), "curl".to_string()], + linux_packages: vec!["curl".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: Some("http://get.nocodb.com/linux-x64".to_string()), @@ -411,13 +442,13 @@ impl PackageManager { fn register_doc_editor(&mut self) { self.components.insert( - "doc-editor".to_string(), + "doc_editor".to_string(), ComponentConfig { - name: "doc-editor".to_string(), + name: "doc_editor".to_string(), required: false, ports: vec![9980], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "gnupg".to_string()], + linux_packages: vec!["gnupg".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: None, @@ -504,7 +535,7 @@ impl PackageManager { binary_name: None, pre_install_cmds_linux: vec![ "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -".to_string(), - "apt-get update && apt-get install -y nodejs".to_string(), + "apt-get install -y nodejs".to_string(), ], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], @@ -525,12 +556,7 @@ impl PackageManager { required: false, ports: vec![8000], dependencies: vec![], - linux_packages: vec![ - "wget".to_string(), - "curl".to_string(), - "unzip".to_string(), - "git".to_string(), - ], + linux_packages: vec!["curl".to_string(), "unzip".to_string(), "git".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: None, @@ -548,13 +574,13 @@ impl PackageManager { } fn register_vector_db(&mut self) { - self.components.insert("vector-db".to_string(), ComponentConfig { - name: "vector-db".to_string(), + self.components.insert("vector_db".to_string(), ComponentConfig { + name: "vector_db".to_string(), required: false, ports: vec![6333], dependencies: vec![], - linux_packages: vec!["wget".to_string()], - macos_packages: vec!["wget".to_string()], + linux_packages: vec![], + macos_packages: vec![], windows_packages: vec![], download_url: Some("https://github.com/qdrant/qdrant/releases/latest/download/qdrant-x86_64-unknown-linux-gnu.tar.gz".to_string()), binary_name: Some("qdrant".to_string()), @@ -601,14 +627,47 @@ impl PackageManager { ); } - pub(crate) fn start(&self, component: &str) -> Result { + pub fn start(&self, component: &str) -> Result { if let Some(component) = self.components.get(component) { + let bin_path = self.base_path.join("bin").join(&component.name); + let data_path = self.base_path.join("data").join(&component.name); + let conf_path = self.base_path.join("conf").join(&component.name); + let logs_path = self.base_path.join("logs").join(&component.name); + + let rendered_cmd = component + .exec_cmd + .replace("{{BIN_PATH}}", &bin_path.to_string_lossy()) + .replace("{{DATA_PATH}}", &data_path.to_string_lossy()) + .replace("{{CONF_PATH}}", &conf_path.to_string_lossy()) + .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()); + + trace!( + "Starting component {} with command: {}", + component.name, + rendered_cmd + ); + Ok(std::process::Command::new("sh") .arg("-c") - .arg(&component.exec_cmd) + .arg(&rendered_cmd) .spawn()?) } else { Err(anyhow::anyhow!("Component {} not found", component)) } } + + fn generate_secure_password(&self, length: usize) -> String { + let mut rng: ThreadRng = rand::thread_rng(); + rng.sample_iter(&Alphanumeric) + .take(length) + .map(char::from) + .collect() + } + + fn encrypt_password(&self, password: &str, key: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(key.as_bytes()); + hasher.update(password.as_bytes()); + format!("{:x}", hasher.finalize()) + } } diff --git a/src/shared/models.rs b/src/shared/models.rs index 42ab7ce0..72f7a34e 100644 --- a/src/shared/models.rs +++ b/src/shared/models.rs @@ -219,6 +219,12 @@ pub struct SessionToolAssociation { pub added_at: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SystemCredentials { + pub encrypted_db_password: String, + pub encrypted_drive_password: String, +} + pub mod schema { diesel::table! { organizations (org_id) { @@ -384,5 +390,4 @@ pub mod schema { } } -// Re-export all tables at the module level for backward compatibility pub use schema::*; diff --git a/src/shared/utils.rs b/src/shared/utils.rs index ad09ac61..16a4ae9d 100644 --- a/src/shared/utils.rs +++ b/src/shared/utils.rs @@ -1,4 +1,4 @@ -use log::debug; +use log::{debug, trace}; use rhai::{Array, Dynamic}; use serde_json::Value; use smartstring::SmartString; @@ -7,17 +7,14 @@ use std::fs::File; use std::io::BufReader; use std::path::Path; use tokio::fs::File as TokioFile; - use zip::ZipArchive; - use crate::config::AIConfig; use reqwest::Client; use tokio::io::AsyncWriteExt; +use indicatif::{ProgressBar, ProgressStyle}; +use futures_util::StreamExt; -pub fn extract_zip_recursive( - zip_path: &Path, - destination_path: &Path, -) -> Result<(), Box> { +pub fn extract_zip_recursive(zip_path: &Path, destination_path: &Path) -> Result<(), Box> { let file = File::open(zip_path)?; let buf_reader = BufReader::new(file); let mut archive = ZipArchive::new(buf_reader)?; @@ -38,7 +35,6 @@ pub fn extract_zip_recursive( std::io::copy(&mut file, &mut outfile)?; } } - Ok(()) } @@ -79,21 +75,36 @@ pub fn to_array(value: Dynamic) -> Array { } } -pub async fn download_file( - url: &str, - output_path: &str, -) -> Result<(), Box> { +pub async fn download_file(url: &str, output_path: &str) -> Result<(), Box> { let url = url.to_string(); let output_path = output_path.to_string(); + let download_handle = tokio::spawn(async move { let client = Client::new(); let response = client.get(&url).send().await?; - + if response.status().is_success() { + let total_size = response.content_length().unwrap_or(0); + let pb = ProgressBar::new(total_size); + pb.set_style(ProgressStyle::default_bar() + .template("{msg}\n{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})") + .unwrap() + .progress_chars("#>-")); + pb.set_message(format!("Downloading {}", url)); + let mut file = TokioFile::create(&output_path).await?; - let bytes = response.bytes().await?; - file.write_all(&bytes).await?; - debug!("File downloaded successfully to {}", output_path); + let mut downloaded: u64 = 0; + let mut stream = response.bytes_stream(); + + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result?; + file.write_all(&chunk).await?; + downloaded += chunk.len() as u64; + pb.set_position(downloaded); + } + + pb.finish_with_message(format!("Downloaded {}", output_path)); + trace!("Download completed: {} -> {}", url, output_path); Ok(()) } else { Err(format!("HTTP {}: {}", response.status(), url).into()) @@ -112,20 +123,14 @@ pub fn parse_filter(filter_str: &str) -> Result<(String, Vec), Box Result<(String, Vec), Box> { +pub fn parse_filter_with_offset(filter_str: &str, offset: usize) -> Result<(String, Vec), Box> { let mut clauses = Vec::new(); let mut params = Vec::new(); @@ -138,10 +143,7 @@ pub fn parse_filter_with_offset( let column = parts[0].trim(); let value = parts[1].trim(); - if !column - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_') - { + if !column.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { return Err("Invalid column name".into()); } @@ -152,9 +154,6 @@ pub fn parse_filter_with_offset( Ok((clauses.join(" AND "), params)) } -pub async fn call_llm( - prompt: &str, - _ai_config: &AIConfig, -) -> Result> { +pub async fn call_llm(prompt: &str, _ai_config: &AIConfig) -> Result> { Ok(format!("Generated response for: {}", prompt)) } diff --git a/templates/announcements.gbai/announcements.gbdialog/start.bas b/templates/announcements.gbai/announcements.gbdialog/start.bas index c3890cc8..e55ea9d9 100644 --- a/templates/announcements.gbai/announcements.gbdialog/start.bas +++ b/templates/announcements.gbai/announcements.gbdialog/start.bas @@ -1,7 +1,6 @@ let resume = GET_BOT_MEMORY ("resume") - TALK resume -let text = GET "default.gbdrive/default.pdf" -SET_CONTEXT "Este é o documento que você deve usar para responder dúvidas: " + text +ADD_KB "weekly" + TALK "Olá, pode me perguntar sobre qualquer coisa destas circulares..." diff --git a/templates/announcements.gbai/announcements.gbkb/weekly/default.pdf b/templates/announcements.gbai/announcements.gbkb/weekly/default.pdf new file mode 100644 index 00000000..f38959e6 Binary files /dev/null and b/templates/announcements.gbai/announcements.gbkb/weekly/default.pdf differ