diff --git a/src/analytics/goals.rs b/src/analytics/goals.rs index 13f24870a..e80fa00ec 100644 --- a/src/analytics/goals.rs +++ b/src/analytics/goals.rs @@ -620,7 +620,7 @@ pub async fn create_objective( let record = new_objective.clone(); - let _result = tokio::task::spawn_blocking(move || { + tokio::task::spawn_blocking(move || { let mut conn = pool.get().map_err(|e| GoalsError::Database(e.to_string()))?; diesel::insert_into(okr_objectives::table) .values(&new_objective) @@ -718,7 +718,7 @@ pub async fn delete_objective( ) -> Result, GoalsError> { let pool = state.conn.clone(); - let _result = tokio::task::spawn_blocking(move || { + tokio::task::spawn_blocking(move || { let mut conn = pool.get().map_err(|e| GoalsError::Database(e.to_string()))?; let deleted = diesel::delete(okr_objectives::table.find(objective_id)) .execute(&mut conn) @@ -793,7 +793,7 @@ pub async fn create_key_result( let record = new_kr.clone(); - let _result = tokio::task::spawn_blocking(move || { + tokio::task::spawn_blocking(move || { let mut conn = pool.get().map_err(|e| GoalsError::Database(e.to_string()))?; diesel::insert_into(okr_key_results::table) .values(&new_kr) diff --git a/src/auto_task/intent_classifier.rs b/src/auto_task/intent_classifier.rs index 3958eba99..e09a0e66a 100644 --- a/src/auto_task/intent_classifier.rs +++ b/src/auto_task/intent_classifier.rs @@ -930,8 +930,6 @@ END SCHEDULE .clone() .unwrap_or_else(|| "unspecified".to_string()); - let _goal_id = Uuid::new_v4(); - // Goals are more complex - they create a monitoring + action loop let basic_code = format!( r#"' Goal: {goal_name} diff --git a/src/basic/compiler/mod.rs b/src/basic/compiler/mod.rs index 12f01bcb8..30b5a3199 100644 --- a/src/basic/compiler/mod.rs +++ b/src/basic/compiler/mod.rs @@ -155,10 +155,12 @@ impl BasicCompiler { } } if line.starts_with("DESCRIPTION ") { - let desc_start = line.find('"').unwrap_or(0); - let desc_end = line.rfind('"').unwrap_or(line.len()); - if desc_start < desc_end { - description = line[desc_start + 1..desc_end].to_string(); + if let Some(desc_start) = line.find('"') { + if let Some(desc_end) = line.rfind('"') { + if desc_start < desc_end { + description = line[desc_start + 1..desc_end].to_string(); + } + } } } i += 1; @@ -357,9 +359,7 @@ impl BasicCompiler { if parts.len() >= 3 { #[cfg(feature = "tasks")] { - #[allow(unused_variables, unused_mut)] let cron = parts[1]; - #[allow(unused_variables, unused_mut)] let mut conn = self .state .conn @@ -408,7 +408,8 @@ impl BasicCompiler { } if trimmed.to_uppercase().starts_with("USE WEBSITE") { - let re = Regex::new(r#"(?i)USE\s+WEBSITE\s+"([^"]+)"(?:\s+REFRESH\s+"([^"]+)")?"#).unwrap(); + let re = Regex::new(r#"(?i)USE\s+WEBSITE\s+"([^"]+)"(?:\s+REFRESH\s+"([^"]+)")?"#) + .map_err(|e| format!("Failed to compile USE_WEBSITE regex: {e}"))?; if let Some(caps) = re.captures(&normalized) { if let Some(url_match) = caps.get(1) { let url = url_match.as_str(); diff --git a/src/basic/keywords/create_site.rs b/src/basic/keywords/create_site.rs index 322d7245a..e4c2da60a 100644 --- a/src/basic/keywords/create_site.rs +++ b/src/basic/keywords/create_site.rs @@ -16,11 +16,6 @@ use std::path::PathBuf; #[cfg(feature = "llm")] use std::sync::Arc; -// When llm feature is disabled, create a dummy trait for type compatibility -#[cfg(not(feature = "llm"))] -#[allow(dead_code)] -trait LLMProvider: Send + Sync {} - pub fn create_site_keyword(state: &AppState, user: UserSession, engine: &mut Engine) { let state_clone = state.clone(); let user_clone = user; diff --git a/src/basic/keywords/enhanced_memory.rs b/src/basic/keywords/enhanced_memory.rs index 64681f1df..ea517266d 100644 --- a/src/basic/keywords/enhanced_memory.rs +++ b/src/basic/keywords/enhanced_memory.rs @@ -71,14 +71,12 @@ async fn share_bot_memory( let target_bot_uuid = find_bot_by_name(&mut conn, target_bot_name)?; - let memory_value = match bot_memories::table + let memory_value: String = bot_memories::table .filter(bot_memories::bot_id.eq(source_bot_uuid)) .filter(bot_memories::key.eq(memory_key)) .select(bot_memories::value) - .first(&mut conn) { - Ok(value) => value, - Err(_) => String::new(), - }; + .first(&mut conn) + .unwrap_or_default(); let shared_memory = BotSharedMemory { id: Uuid::new_v4(), diff --git a/src/basic/keywords/on_change.rs b/src/basic/keywords/on_change.rs index 18dd9fe08..75ae092ce 100644 --- a/src/basic/keywords/on_change.rs +++ b/src/basic/keywords/on_change.rs @@ -1,7 +1,6 @@ use log::info; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use std::path::Path; use uuid::Uuid; use crate::shared::state::AppState; @@ -174,56 +173,56 @@ pub fn fetch_folder_changes( Ok(events) } -#[allow(dead_code)] -fn apply_filters(events: Vec, filters: &Option) -> Vec { - let Some(filters) = filters else { - return events; - }; - - events - .into_iter() - .filter(|event| { - if let Some(ref extensions) = filters.extensions { - let ext = Path::new(&event.path) - .extension() - .and_then(|e| e.to_str()) - .unwrap_or(""); - if !extensions.iter().any(|e| e.eq_ignore_ascii_case(ext)) { - return false; - } - } - - if let Some(min_size) = filters.min_size { - if event.size.unwrap_or(0) < min_size { - return false; - } - } - - if let Some(max_size) = filters.max_size { - if event.size.unwrap_or(i64::MAX) > max_size { - return false; - } - } - - if let Some(ref pattern) = filters.name_pattern { - let file_name = Path::new(&event.path) - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or(""); - if !file_name.contains(pattern) { - return false; - } - } - - true - }) - .collect() -} - #[cfg(test)] mod tests { use super::*; + fn apply_filters(events: Vec, filters: &Option) -> Vec { + let Some(ref filters) = filters else { + return events; + }; + + events + .into_iter() + .filter(|event| { + if let Some(ref extensions) = filters.extensions { + let ext = Path::new(&event.path) + .extension() + .and_then(|e| e.to_str()) + .unwrap_or(""); + if !extensions.iter().any(|e| e.eq_ignore_ascii_case(ext)) { + return false; + } + } + + if let Some(min_size) = filters.min_size { + if event.size.unwrap_or(0) < min_size { + return false; + } + } + + if let Some(max_size) = filters.max_size { + if event.size.unwrap_or(i64::MAX) > max_size { + return false; + } + } + + if let Some(ref pattern) = filters.name_pattern { + let file_name = Path::new(&event.path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(""); + if !file_name.contains(pattern) { + return false; + } + } + + true + }) + .collect() + } + use super::*; + #[test] fn test_folder_provider_from_str() { assert_eq!( diff --git a/src/basic/keywords/table_definition.rs b/src/basic/keywords/table_definition.rs index 642b0b8ec..5f58dda3e 100644 --- a/src/basic/keywords/table_definition.rs +++ b/src/basic/keywords/table_definition.rs @@ -623,7 +623,7 @@ pub fn process_table_definitions( if table.connection_name == "default" { let create_sql = generate_create_table_sql(table, "postgres"); - info!("Creating table {} on default connection", table.name); + info!("Creating table {} on bot's default database", table.name); trace!("SQL: {}", create_sql); sql_query(&create_sql).execute(&mut conn)?; @@ -646,10 +646,18 @@ pub fn process_table_definitions( } } Err(e) => { - error!( - "Failed to load connection config for {}: {}", - table.connection_name, e + warn!( + "External connection '{}' not configured for bot {}, creating table {} in bot's database instead: {}", + table.connection_name, bot_id, table.name, e ); + + let create_sql = generate_create_table_sql(table, "postgres"); + info!("Creating table {} on bot's database (external DB fallback)", table.name); + trace!("SQL: {}", create_sql); + + if let Err(e) = sql_query(&create_sql).execute(&mut conn) { + error!("Failed to create table {} in bot's database: {}", table.name, e); + } } } } diff --git a/src/basic/mod.rs b/src/basic/mod.rs index 412841269..12817c5e0 100644 --- a/src/basic/mod.rs +++ b/src/basic/mod.rs @@ -7,6 +7,7 @@ use crate::shared::models::UserSession; use crate::shared::state::AppState; use diesel::prelude::*; use log::info; +use regex::Regex; use rhai::{Dynamic, Engine, EvalAltResult, Scope}; use std::collections::HashMap; use std::sync::Arc; @@ -566,8 +567,6 @@ impl ScriptService { } fn normalize_variables_to_lowercase(script: &str) -> String { - use regex::Regex; - let mut result = String::new(); let keywords = [ @@ -799,8 +798,6 @@ impl ScriptService { "MODEL", ]; - let _identifier_re = Regex::new(r"([a-zA-Z_][a-zA-Z0-9_]*)").expect("valid regex"); - for line in script.lines() { let trimmed = line.trim(); @@ -865,8 +862,6 @@ impl ScriptService { /// - "SET BOT MEMORY key AS value" → "SET_BOT_MEMORY(key, value)" /// - "CLEAR SUGGESTIONS" → "CLEAR_SUGGESTIONS()" fn convert_multiword_keywords(script: &str) -> String { - use regex::Regex; - // Known multi-word keywords with their conversion patterns // Format: (keyword_pattern, min_params, max_params, param_names) let multiword_patterns = vec![ @@ -970,9 +965,9 @@ impl ScriptService { let mut current = String::new(); let mut in_quotes = false; let mut quote_char = '"'; - let mut chars = params_str.chars().peekable(); + let chars = params_str.chars().peekable(); - while let Some(c) = chars.next() { + for c in chars { match c { '"' | '\'' if !in_quotes => { in_quotes = true; diff --git a/src/channels/oauth.rs b/src/channels/oauth.rs index 62d7a77ee..d8ba8cf29 100644 --- a/src/channels/oauth.rs +++ b/src/channels/oauth.rs @@ -49,10 +49,7 @@ impl SocialPlatform { } pub fn requires_oauth(&self) -> bool { - match self { - Self::Bluesky | Self::Telegram | Self::Twilio => false, - _ => true, - } + !matches!(self, Self::Bluesky | Self::Telegram | Self::Twilio) } pub fn authorization_url(&self) -> Option<&'static str> { diff --git a/src/compliance/code_scanner.rs b/src/compliance/code_scanner.rs index b81ad01b8..74c486da3 100644 --- a/src/compliance/code_scanner.rs +++ b/src/compliance/code_scanner.rs @@ -139,18 +139,18 @@ pub struct CodeScanner { } impl CodeScanner { - pub fn new(base_path: impl AsRef) -> Self { - let patterns = Self::build_patterns(); - Self { + pub fn new(base_path: impl AsRef) -> Result> { + let patterns = Self::build_patterns()?; + Ok(Self { patterns, base_path: base_path.as_ref().to_path_buf(), - } + }) } - fn build_patterns() -> Vec { - vec![ + fn build_patterns() -> Result, Box> { + Ok(vec![ ScanPattern { - regex: Regex::new(r#"(?i)password\s*=\s*["'][^"']+["']"#).expect("valid regex"), + regex: Regex::new(r#"(?i)password\s*=\s*["'][^"']+["']"#)?, issue_type: IssueType::PasswordInConfig, severity: IssueSeverity::Critical, title: "Hardcoded Password".to_string(), @@ -159,7 +159,7 @@ impl CodeScanner { category: "Security".to_string(), }, ScanPattern { - regex: Regex::new(r#"(?i)(api[_-]?key|apikey|secret[_-]?key|client[_-]?secret)\s*=\s*["'][^"']{8,}["']"#).expect("valid regex"), + regex: Regex::new(r#"(?i)(api[_-]?key|apikey|secret[_-]?key|client[_-]?secret)\s*=\s*["'][^"']{8,}["']"#)?, issue_type: IssueType::HardcodedSecret, severity: IssueSeverity::Critical, title: "Hardcoded API Key/Secret".to_string(), @@ -168,7 +168,7 @@ impl CodeScanner { category: "Security".to_string(), }, ScanPattern { - regex: Regex::new(r#"(?i)token\s*=\s*["'][a-zA-Z0-9_\-]{20,}["']"#).expect("valid regex"), + regex: Regex::new(r#"(?i)token\s*=\s*["'][a-zA-Z0-9_\-]{20,}["']"#)?, issue_type: IssueType::HardcodedSecret, severity: IssueSeverity::High, title: "Hardcoded Token".to_string(), @@ -177,7 +177,7 @@ impl CodeScanner { category: "Security".to_string(), }, ScanPattern { - regex: Regex::new(r"(?i)IF\s+.*\binput\b").expect("valid regex"), + regex: Regex::new(r"(?i)IF\s+.*\binput\b")?, issue_type: IssueType::DeprecatedIfInput, severity: IssueSeverity::Medium, title: "Deprecated IF...input Pattern".to_string(), @@ -189,7 +189,7 @@ impl CodeScanner { category: "Code Quality".to_string(), }, ScanPattern { - regex: Regex::new(r"(?i)\b(GET_BOT_MEMORY|SET_BOT_MEMORY|GET_USER_MEMORY|SET_USER_MEMORY|USE_KB|USE_TOOL|SEND_MAIL|CREATE_TASK)\b").expect("valid regex"), + regex: Regex::new(r"(?i)\b(GET_BOT_MEMORY|SET_BOT_MEMORY|GET_USER_MEMORY|SET_USER_MEMORY|USE_KB|USE_TOOL|SEND_MAIL|CREATE_TASK)\b")?, issue_type: IssueType::UnderscoreInKeyword, severity: IssueSeverity::Low, title: "Underscore in Keyword".to_string(), @@ -198,7 +198,7 @@ impl CodeScanner { category: "Naming Convention".to_string(), }, ScanPattern { - regex: Regex::new(r"(?i)POST\s+TO\s+INSTAGRAM\s+\w+\s*,\s*\w+").expect("valid regex"), + regex: Regex::new(r"(?i)POST\s+TO\s+INSTAGRAM\s+\w+\s*,\s*\w+")?, issue_type: IssueType::InsecurePattern, severity: IssueSeverity::High, title: "Instagram Credentials in Code".to_string(), @@ -209,7 +209,7 @@ impl CodeScanner { category: "Security".to_string(), }, ScanPattern { - regex: Regex::new(r"(?i)(SELECT|INSERT|UPDATE|DELETE)\s+.*(FROM|INTO|SET)\s+").expect("valid regex"), + regex: Regex::new(r"(?i)(SELECT|INSERT|UPDATE|DELETE)\s+.*(FROM|INTO|SET)\s+")?, issue_type: IssueType::FragileCode, severity: IssueSeverity::Medium, title: "Raw SQL Query".to_string(), @@ -221,7 +221,7 @@ impl CodeScanner { category: "Security".to_string(), }, ScanPattern { - regex: Regex::new(r"(?i)\bEVAL\s*\(").expect("valid regex"), + regex: Regex::new(r"(?i)\bEVAL\s*\(")?, issue_type: IssueType::FragileCode, severity: IssueSeverity::High, title: "Dynamic Code Execution".to_string(), @@ -233,7 +233,7 @@ impl CodeScanner { regex: Regex::new( r#"(?i)(password|secret|key|token)\s*=\s*["'][A-Za-z0-9+/=]{40,}["']"#, ) - .expect("valid regex"), + ?, issue_type: IssueType::HardcodedSecret, severity: IssueSeverity::High, title: "Potential Encoded Secret".to_string(), @@ -243,7 +243,7 @@ impl CodeScanner { category: "Security".to_string(), }, ScanPattern { - regex: Regex::new(r"(?i)(AKIA[0-9A-Z]{16})").expect("valid regex"), + regex: Regex::new(r"(?i)(AKIA[0-9A-Z]{16})")?, issue_type: IssueType::HardcodedSecret, severity: IssueSeverity::Critical, title: "AWS Access Key".to_string(), @@ -253,7 +253,7 @@ impl CodeScanner { category: "Security".to_string(), }, ScanPattern { - regex: Regex::new(r"-----BEGIN\s+(RSA\s+)?PRIVATE\s+KEY-----").expect("valid regex"), + regex: Regex::new(r"-----BEGIN\s+(RSA\s+)?PRIVATE\s+KEY-----")?, issue_type: IssueType::HardcodedSecret, severity: IssueSeverity::Critical, title: "Private Key in Code".to_string(), @@ -263,7 +263,7 @@ impl CodeScanner { category: "Security".to_string(), }, ScanPattern { - regex: Regex::new(r"(?i)(postgres|mysql|mongodb|redis)://[^:]+:[^@]+@").expect("valid regex"), + regex: Regex::new(r"(?i)(postgres|mysql|mongodb|redis)://[^:]+:[^@]+@")?, issue_type: IssueType::HardcodedSecret, severity: IssueSeverity::Critical, title: "Database Credentials in Connection String".to_string(), @@ -450,12 +450,12 @@ impl CodeScanner { fn redact_sensitive(line: &str) -> String { let mut result = line.to_string(); - let secret_pattern = Regex::new(r#"(["'])[^"']{8,}(["'])"#).expect("valid regex"); + let secret_pattern = Regex::new(r#"(["'])[^"']{8,}(["'])"#)?; result = secret_pattern .replace_all(&result, "$1***REDACTED***$2") .to_string(); - let aws_pattern = Regex::new(r"AKIA[0-9A-Z]{16}").expect("valid regex"); + let aws_pattern = Regex::new(r"AKIA[0-9A-Z]{16}")?; result = aws_pattern .replace_all(&result, "AKIA***REDACTED***") .to_string(); diff --git a/src/core/bootstrap/mod.rs b/src/core/bootstrap/mod.rs index 7dd698f7d..11615b019 100644 --- a/src/core/bootstrap/mod.rs +++ b/src/core/bootstrap/mod.rs @@ -1233,10 +1233,154 @@ impl BootstrapManager { } } } + + // Install pdftotext for PDF processing (no root required) + // Do this BEFORE template sync so PDFs can be processed + if let Err(e) = self.ensure_pdftotext() { + warn!("Failed to install pdftotext: {} (PDF processing may fall back to Rust library)", e); + } else { + // Add botserver-stack/bin/shared to PATH for current process + let shared_dir = self.stack_dir("bin/shared").display().to_string(); + if let Ok(current_path) = std::env::var("PATH") { + if !current_path.contains(&shared_dir) { + std::env::set_var("PATH", format!("{}:{}", shared_dir, current_path)); + info!("Added {} to PATH for PDF processing", shared_dir); + } + } + } + info!("=== BOOTSTRAP COMPLETED SUCCESSFULLY ==="); Ok(()) } + /// Ensures pdftotext is available for PDF processing. + /// Downloads and extracts the binary from poppler-utils package if not found. + /// Installs to botserver-stack/bin/shared (no root/sudo required). + fn ensure_pdftotext(&self) -> Result<()> { + use std::env; + use std::process::Command; + + let shared_bin = self.stack_dir("bin/shared/pdftotext"); + + // Check if pdftotext is already available + if shared_bin.exists() { + if let Ok(output) = Command::new(&shared_bin).arg("-v").output() { + if output.status.success() { + info!("pdftotext already available at {}", shared_bin.display()); + return Ok(()); + } + } + } + + // Also check PATH + if let Ok(output) = Command::new("pdftotext").arg("-v").output() { + if output.status.success() { + info!("pdftotext already available in PATH"); + return Ok(()); + } + } + + info!("pdftotext not found, installing from poppler-utils package..."); + + // Create bin/shared directory + let bin_dir = shared_bin.parent().unwrap(); + fs::create_dir_all(bin_dir)?; + + // Download the poppler-utils package + let temp_dir = env::temp_dir(); + + info!("Downloading poppler-utils package..."); + let download_cmd = format!( + "cd {} && apt-get download poppler-utils 2>&1 | tail -1", + temp_dir.display() + ); + let download_result = safe_sh_command(&download_cmd); + + let deb_file = match download_result { + Some(output) => { + let stdout = String::from_utf8_lossy(&output.stdout); + // Find the .deb file that was downloaded + let deb_name = stdout.lines().find(|l| l.contains("poppler-utils")) + .and_then(|l| l.split_whitespace().find(|p| p.ends_with(".deb"))); + match deb_name { + Some(name) => temp_dir.join(name.trim()), + None => { + // Try listing files to find the deb + let list_cmd = format!("ls -1 {}/poppler-utils*.deb 2>/dev/null", temp_dir.display()); + if let Some(list_output) = safe_sh_command(&list_cmd) { + let deb_name = String::from_utf8_lossy(&list_output.stdout).trim().to_string(); + if !deb_name.is_empty() { + PathBuf::from(deb_name) + } else { + return Err(anyhow::anyhow!("Failed to find downloaded .deb file")); + } + } else { + return Err(anyhow::anyhow!("Failed to download poppler-utils package")); + } + } + } + } + None => return Err(anyhow::anyhow!("Failed to download poppler-utils")), + }; + + info!("Extracting pdftotext from package..."); + // Extract the .deb package (it's an ar archive) + let extract_ar_cmd = format!( + "cd {} && ar -x '{}' 2>&1", + temp_dir.display(), + deb_file.display() + ); + let _ = safe_sh_command(&extract_ar_cmd); + + // Extract the data.tar.xz + let extract_tar_cmd = format!("cd {} && tar -xf data.tar.xz 2>&1", temp_dir.display()); + let _ = safe_sh_command(&extract_tar_cmd); + + // Copy the pdftotext binary to botserver-stack/bin/shared + let src_binary = temp_dir.join("usr/bin/pdftotext"); + if !src_binary.exists() { + return Err(anyhow::anyhow!("pdftotext not found in extracted package")); + } + + fs::copy(&src_binary, &shared_bin)?; + + // Make it executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(&shared_bin)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(&shared_bin, perms)?; + } + + // Verify it works + if let Ok(output) = Command::new(&shared_bin).arg("-v").output() { + if output.status.success() { + info!("pdftotext installed successfully to {}", shared_bin.display()); + + // Add botserver-stack/bin/shared to PATH for current process + let shared_dir = bin_dir.display().to_string(); + if let Ok(current_path) = env::var("PATH") { + if !current_path.contains(&shared_dir) { + env::set_var("PATH", format!("{}:{}", shared_dir, current_path)); + info!("Added {} to PATH for current session", shared_dir); + } + } + + // Clean up temporary files + let _ = fs::remove_file(&deb_file); + let _ = fs::remove_file(temp_dir.join("data.tar.xz")); + let _ = fs::remove_file(temp_dir.join("control.tar.xz")); + let _ = fs::remove_file(temp_dir.join("debian-binary")); + let _ = fs::remove_dir_all(temp_dir.join("usr")); + + return Ok(()); + } + } + + Err(anyhow::anyhow!("pdftotext installation failed")) + } + fn configure_services_in_directory(&self, db_password: &str) -> Result<()> { info!("Creating Zitadel configuration files..."); @@ -1958,6 +2102,18 @@ VAULT_CACHE_TTL=300 info!(" Generated and stored encryption key"); } + if secret_exists("secret/gbo/jwt") { + info!(" JWT secret already exists - preserving (CRITICAL)"); + } else { + let jwt_secret = Self::generate_secure_password(48); + let jwt_cmd = format!( + "VAULT_ADDR={} VAULT_TOKEN={} VAULT_CACERT={} {} kv put secret/gbo/jwt secret='{}'", + vault_addr, root_token, ca_cert_path, vault_bin, jwt_secret + ); + let _ = safe_sh_command(&jwt_cmd); + info!(" Generated and stored JWT secret"); + } + info!("Vault setup complete!"); info!(" Vault UI: {}/ui", vault_addr); info!(" Root token saved to: {}", vault_init_path.display()); diff --git a/src/core/config_reload.rs b/src/core/config_reload.rs index 9fd536a44..8cbbe0248 100644 --- a/src/core/config_reload.rs +++ b/src/core/config_reload.rs @@ -16,7 +16,7 @@ pub async fn reload_config( let mut conn = conn_arc .get() .map_err(|e| format!("failed to get db connection: {e}"))?; - Ok(crate::bot::get_default_bot(&mut *conn)) + Ok(crate::bot::get_default_bot(&mut conn)) }) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? diff --git a/src/core/kb/kb_indexer.rs b/src/core/kb/kb_indexer.rs index d3d94ddb5..52b7fdc61 100644 --- a/src/core/kb/kb_indexer.rs +++ b/src/core/kb/kb_indexer.rs @@ -172,9 +172,9 @@ impl KbIndexer { let mut batch_docs = Vec::with_capacity(BATCH_SIZE); // Process documents in iterator to avoid keeping all in memory - let mut doc_iter = documents.into_iter(); + let doc_iter = documents.into_iter(); - while let Some((doc_path, chunks)) = doc_iter.next() { + for (doc_path, chunks) in doc_iter { if chunks.is_empty() { debug!("[KB_INDEXER] Skipping document with no chunks: {}", doc_path); continue; @@ -262,9 +262,9 @@ impl KbIndexer { // Process chunks in smaller sub-batches to prevent memory exhaustion const CHUNK_BATCH_SIZE: usize = 20; // Process 20 chunks at a time - let mut chunk_batches = chunks.chunks(CHUNK_BATCH_SIZE); + let chunk_batches = chunks.chunks(CHUNK_BATCH_SIZE); - while let Some(chunk_batch) = chunk_batches.next() { + for chunk_batch in chunk_batches { trace!("[KB_INDEXER] Processing chunk batch of {} chunks", chunk_batch.len()); let embeddings = match self diff --git a/src/core/kb/web_crawler.rs b/src/core/kb/web_crawler.rs index 669dfbcd1..ef220f840 100644 --- a/src/core/kb/web_crawler.rs +++ b/src/core/kb/web_crawler.rs @@ -221,7 +221,7 @@ impl WebCrawler { self.pages.push(page); // Aggressive memory cleanup every 10 pages - if self.pages.len() % 10 == 0 { + if self.pages.len().is_multiple_of(10) { self.pages.shrink_to_fit(); self.visited_urls.shrink_to_fit(); } diff --git a/src/core/kb/website_crawler_service.rs b/src/core/kb/website_crawler_service.rs index 138213210..78e38b566 100644 --- a/src/core/kb/website_crawler_service.rs +++ b/src/core/kb/website_crawler_service.rs @@ -228,7 +228,7 @@ impl WebsiteCrawlerService { let total_pages = pages.len(); for (batch_idx, batch) in pages.chunks(BATCH_SIZE).enumerate() { - info!("Processing batch {} of {} pages", batch_idx + 1, (total_pages + BATCH_SIZE - 1) / BATCH_SIZE); + info!("Processing batch {} of {} pages", batch_idx + 1, total_pages.div_ceil(BATCH_SIZE)); for (idx, page) in batch.iter().enumerate() { let global_idx = batch_idx * BATCH_SIZE + idx; diff --git a/src/core/package_manager/facade.rs b/src/core/package_manager/facade.rs index f047a2197..768a1de63 100644 --- a/src/core/package_manager/facade.rs +++ b/src/core/package_manager/facade.rs @@ -1047,7 +1047,7 @@ Store credentials in Vault: Ok(()) } pub fn run_commands(&self, commands: &[String], target: &str, component: &str) -> Result<()> { - self.run_commands_with_password(commands, target, component, &String::new()) + self.run_commands_with_password(commands, target, component, "") } pub fn run_commands_with_password(&self, commands: &[String], target: &str, component: &str, db_password_override: &str) -> Result<()> { diff --git a/src/core/package_manager/installer.rs b/src/core/package_manager/installer.rs index edfdca533..ead215e6e 100644 --- a/src/core/package_manager/installer.rs +++ b/src/core/package_manager/installer.rs @@ -21,7 +21,9 @@ struct ThirdPartyConfig { static THIRDPARTY_CONFIG: Lazy = Lazy::new(|| { let toml_str = include_str!("../../../3rdparty.toml"); - toml::from_str(toml_str).expect("Failed to parse embedded 3rdparty.toml") + toml::from_str(toml_str).unwrap_or_else(|e| { + panic!("Failed to parse embedded 3rdparty.toml: {e}") + }) }); fn get_component_url(name: &str) -> Option { diff --git a/src/core/secrets/mod.rs b/src/core/secrets/mod.rs index 94380caa2..95f5a3305 100644 --- a/src/core/secrets/mod.rs +++ b/src/core/secrets/mod.rs @@ -20,6 +20,7 @@ impl SecretPaths { pub const EMAIL: &'static str = "gbo/email"; pub const LLM: &'static str = "gbo/llm"; pub const ENCRYPTION: &'static str = "gbo/encryption"; + pub const JWT: &'static str = "gbo/jwt"; pub const MEET: &'static str = "gbo/meet"; pub const ALM: &'static str = "gbo/alm"; pub const VECTORDB: &'static str = "gbo/vectordb"; @@ -270,6 +271,10 @@ impl SecretsManager { self.get_value(SecretPaths::ENCRYPTION, "master_key").await } + pub async fn get_jwt_secret(&self) -> Result { + self.get_value(SecretPaths::JWT, "secret").await + } + pub async fn put_secret(&self, path: &str, data: HashMap) -> Result<()> { let client = self .client diff --git a/src/core/shared/utils.rs b/src/core/shared/utils.rs index 1d9a430cf..7c1fc24ab 100644 --- a/src/core/shared/utils.rs +++ b/src/core/shared/utils.rs @@ -548,13 +548,9 @@ pub fn truncate_text_for_model(text: &str, model: &str, max_tokens: usize) -> St /// Estimates characters per token based on model type fn estimate_chars_per_token(model: &str) -> usize { - if model.contains("gpt") || model.contains("claude") { - 4 // GPT/Claude models: ~4 chars per token - } else if model.contains("llama") || model.contains("mistral") { + if model.contains("llama") || model.contains("mistral") { 3 // Llama/Mistral models: ~3 chars per token - } else if model.contains("bert") || model.contains("mpnet") { - 4 // BERT-based models: ~4 chars per token } else { - 4 // Default conservative estimate + 4 // Default conservative estimate (GPT, Claude, BERT, etc.) } } diff --git a/src/designer/bas_analyzer.rs b/src/designer/bas_analyzer.rs index d6bd96745..e52712390 100644 --- a/src/designer/bas_analyzer.rs +++ b/src/designer/bas_analyzer.rs @@ -1,3 +1,4 @@ +use serde::Serialize; use std::path::Path; use std::fs; @@ -83,7 +84,7 @@ impl BasFileAnalyzer { } } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, Serialize)] pub struct WorkflowMetadata { pub name: String, pub step_count: usize, diff --git a/src/designer/mod.rs b/src/designer/mod.rs index de918706d..e4d1c5ca2 100644 --- a/src/designer/mod.rs +++ b/src/designer/mod.rs @@ -586,7 +586,7 @@ pub async fn handle_export( State(_state): State>, Query(params): Query, ) -> impl IntoResponse { - let _file_id = params.path.unwrap_or_else(|| "dialog".to_string()); + let _ = params.path.unwrap_or_else(|| "dialog".to_string()); Html("".to_string()) } diff --git a/src/designer/workflow_canvas.rs b/src/designer/workflow_canvas.rs index 4b19bf2d6..833fa2286 100644 --- a/src/designer/workflow_canvas.rs +++ b/src/designer/workflow_canvas.rs @@ -189,15 +189,15 @@ pub async fn workflow_designer_page( - + - +
- +
Generated BASIC code will appear here...
@@ -242,7 +242,7 @@ pub async fn workflow_designer_page( content = 'Parallel
Multiple branches'; break; case 'event': - content = 'Event
'; + content = 'Event
'; break; } diff --git a/src/directory/auth_routes.rs b/src/directory/auth_routes.rs index 297a16009..16fe3800d 100644 --- a/src/directory/auth_routes.rs +++ b/src/directory/auth_routes.rs @@ -397,7 +397,7 @@ pub async fn get_current_user( is_anonymous: true, }) } - Some(token) if token.is_empty() => { + Some("") => { info!("get_current_user: empty authorization token - returning anonymous user"); Json(CurrentUserResponse { id: None, diff --git a/src/drive/drive_monitor/mod.rs b/src/drive/drive_monitor/mod.rs index 490b6f6ed..0bb964e94 100644 --- a/src/drive/drive_monitor/mod.rs +++ b/src/drive/drive_monitor/mod.rs @@ -23,7 +23,6 @@ use serde::{Deserialize, Serialize}; use tokio::fs as tokio_fs; #[cfg(any(feature = "research", feature = "llm"))] -#[allow(dead_code)] const KB_INDEXING_TIMEOUT_SECS: u64 = 60; const MAX_BACKOFF_SECS: u64 = 300; const INITIAL_BACKOFF_SECS: u64 = 30; @@ -43,7 +42,6 @@ pub struct DriveMonitor { is_processing: Arc, consecutive_failures: Arc, #[cfg(any(feature = "research", feature = "llm"))] - #[allow(dead_code)] kb_indexing_in_progress: Arc>>, } impl DriveMonitor { diff --git a/src/embedded_ui.rs b/src/embedded_ui.rs index 4a5ac2f5c..24ec0663b 100644 --- a/src/embedded_ui.rs +++ b/src/embedded_ui.rs @@ -13,8 +13,7 @@ use std::path::Path; #[cfg(feature = "embed-ui")] #[derive(RustEmbed)] -#[folder = "ui"] -#[prefix = "suite"] +#[folder = "../botui/ui/suite"] struct EmbeddedUi; #[cfg(feature = "embed-ui")] @@ -61,11 +60,9 @@ async fn serve_embedded_file(req: Request) -> Response { let file_path = if path.is_empty() || path == "/" { "index.html" } else { - path + path.trim_start_matches('/') }; - let file_path = file_path.strip_prefix("suite/").unwrap_or(file_path); - log::trace!("Serving embedded file: {}", file_path); let try_paths = [ @@ -88,8 +85,9 @@ async fn serve_embedded_file(req: Request) -> Response { .unwrap_or_else(|_| { Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) + .header(header::CONTENT_TYPE, "text/plain") .body(Body::from("Internal Server Error")) - .unwrap() + .unwrap_or_else(|_| Body::empty()) }); } } @@ -110,7 +108,13 @@ async fn serve_embedded_file(req: Request) -> Response { "#, )) - .unwrap() + .unwrap_or_else(|_| { + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .header(header::CONTENT_TYPE, "text/plain") + .body(Body::from("Response generation failed")) + .unwrap_or_else(|_| Body::empty()) + }) } #[cfg(feature = "embed-ui")] diff --git a/src/llm/local.rs b/src/llm/local.rs index a2d2c9af2..7e0f5663e 100644 --- a/src/llm/local.rs +++ b/src/llm/local.rs @@ -34,7 +34,7 @@ pub async fn ensure_llama_servers_running( let mut conn = conn_arc .get() .map_err(|e| format!("failed to get db connection: {e}"))?; - Ok(crate::bot::get_default_bot(&mut *conn)) + Ok(crate::bot::get_default_bot(&mut conn)) }) .await??; let config_manager = ConfigManager::new(app_state.conn.clone()); diff --git a/src/main.rs b/src/main.rs index 5117c8db4..7efb50d28 100644 --- a/src/main.rs +++ b/src/main.rs @@ -54,8 +54,6 @@ pub mod research; pub mod search; pub mod security; pub mod settings; -#[cfg(feature = "dashboards")] -pub mod shared; #[cfg(feature = "sheet")] pub mod sheet; #[cfg(feature = "slides")] @@ -379,7 +377,7 @@ async fn run_axum_server( let cors = create_cors_layer(); let auth_config = Arc::new( - AuthConfig::from_env() + AuthConfig::from_vault_blocking() .add_anonymous_path("/health") .add_anonymous_path("/healthz") .add_anonymous_path("/api/health") @@ -898,6 +896,30 @@ async fn main() -> std::io::Result<()> { dotenvy::dotenv().ok(); + // Add botserver-stack/bin/shared to PATH for pdftotext and other utilities + if let Ok(stack_path) = std::env::var("BOTSERVER_STACK_PATH") { + let shared_bin = format!("{}/bin/shared", stack_path); + if std::path::Path::new(&shared_bin).exists() { + if let Ok(current_path) = std::env::var("PATH") { + if !current_path.contains(&shared_bin) { + std::env::set_var("PATH", format!("{}:{}", shared_bin, current_path)); + log::info!("Added {} to PATH", shared_bin); + } + } + } + } else { + // Try default path + let shared_bin = "./botserver-stack/bin/shared"; + if std::path::Path::new(shared_bin).exists() { + if let Ok(current_path) = std::env::var("PATH") { + if !current_path.contains(shared_bin) { + std::env::set_var("PATH", format!("{}:{}", shared_bin, current_path)); + log::info!("Added {} to PATH", shared_bin); + } + } + } + } + let env_path_early = std::path::Path::new("./.env"); let vault_init_path_early = std::path::Path::new("./botserver-stack/conf/vault/init.json"); let bootstrap_ready = env_path_early.exists() && vault_init_path_early.exists() && { diff --git a/src/security/auth.rs b/src/security/auth.rs index 2394e9457..6a97a19cf 100644 --- a/src/security/auth.rs +++ b/src/security/auth.rs @@ -556,10 +556,106 @@ impl AuthConfig { pub fn from_env() -> Self { let mut config = Self::default(); - if let Ok(secret) = std::env::var("JWT_SECRET") { + // Try to load from Vault first (if available) + if let Ok(handle) = tokio::runtime::Handle::try_current() { + if let Ok(secret) = handle.block_on(async { + if let Some(manager) = crate::core::shared::utils::get_secrets_manager().await { + if manager.is_enabled() { + manager.get_jwt_secret().await + } else { + Err(anyhow::anyhow!("Vault not enabled")) + } + } else { + Err(anyhow::anyhow!("SecretsManager not initialized")) + } + }) { + config.jwt_secret = Some(secret); + } + } + + // Fall back to environment variable (for development or if Vault is not available) + if config.jwt_secret.is_none() { + if let Ok(secret) = std::env::var("JWT_SECRET") { + config.jwt_secret = Some(secret); + } + } + + if let Ok(require) = std::env::var("REQUIRE_AUTH") { + config.require_auth = require == "true" || require == "1"; + } + + if let Ok(paths) = std::env::var("ANONYMOUS_PATHS") { + config.allow_anonymous_paths = paths + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + } + + config + } + + pub async fn from_vault() -> Result { + let mut config = Self::default(); + + let manager = crate::core::shared::utils::get_secrets_manager() + .await + .ok_or_else(|| anyhow::anyhow!("SecretsManager not initialized"))?; + + if manager.is_enabled() { + let secret = manager.get_jwt_secret().await?; config.jwt_secret = Some(secret); } + Ok(config) + } + + pub fn from_vault_blocking() -> Self { + let mut config = Self::default(); + + if let Ok(handle) = tokio::runtime::Handle::try_current() { + if let Ok(secret) = handle.block_on(async { + if let Some(manager) = crate::core::shared::utils::get_secrets_manager().await { + if manager.is_enabled() { + manager.get_jwt_secret().await + } else { + Err(anyhow::anyhow!("Vault not enabled")) + } + } else { + Err(anyhow::anyhow!("SecretsManager not initialized")) + } + }) { + config.jwt_secret = Some(secret); + } + } else { + let rt = tokio::runtime::Runtime::new() + .map_err(|e| log::warn!("Failed to create runtime: {}", e)) + .ok(); + + if let Some(rt) = rt { + if let Ok(secret) = rt.block_on(async { + if let Some(manager) = crate::core::shared::utils::get_secrets_manager().await { + if manager.is_enabled() { + manager.get_jwt_secret().await + } else { + Err(anyhow::anyhow!("Vault not enabled")) + } + } else { + Err(anyhow::anyhow!("SecretsManager not initialized")) + } + }) { + config.jwt_secret = Some(secret); + } + } + } + + // Fall back to environment variable if Vault doesn't have it + if config.jwt_secret.is_none() { + if let Ok(secret) = std::env::var("JWT_SECRET") { + config.jwt_secret = Some(secret); + } + } + if let Ok(require) = std::env::var("REQUIRE_AUTH") { config.require_auth = require == "true" || require == "1"; } diff --git a/src/security/command_guard.rs b/src/security/command_guard.rs index 49fb5abd5..9fa9c27e2 100644 --- a/src/security/command_guard.rs +++ b/src/security/command_guard.rs @@ -261,7 +261,23 @@ impl SafeCommand { } cmd.env_clear(); - cmd.env("PATH", "/usr/local/bin:/usr/bin:/bin"); + + // Build PATH with standard locations plus botserver-stack/bin/shared + let mut path_entries = vec![ + "/usr/local/bin".to_string(), + "/usr/bin".to_string(), + "/bin".to_string(), + ]; + + // Add botserver-stack/bin/shared to PATH if it exists + let stack_path = std::env::var("BOTSERVER_STACK_PATH") + .unwrap_or_else(|_| "./botserver-stack".to_string()); + let shared_bin = format!("{}/bin/shared", stack_path); + if std::path::Path::new(&shared_bin).exists() { + path_entries.insert(0, shared_bin); + } + + cmd.env("PATH", path_entries.join(":")); cmd.env("HOME", dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp"))); cmd.env("LANG", "C.UTF-8"); @@ -282,7 +298,23 @@ impl SafeCommand { } cmd.env_clear(); - cmd.env("PATH", "/usr/local/bin:/usr/bin:/bin"); + + // Build PATH with standard locations plus botserver-stack/bin/shared + let mut path_entries = vec![ + "/usr/local/bin".to_string(), + "/usr/bin".to_string(), + "/bin".to_string(), + ]; + + // Add botserver-stack/bin/shared to PATH if it exists + let stack_path = std::env::var("BOTSERVER_STACK_PATH") + .unwrap_or_else(|_| "./botserver-stack".to_string()); + let shared_bin = format!("{}/bin/shared", stack_path); + if std::path::Path::new(&shared_bin).exists() { + path_entries.insert(0, shared_bin); + } + + cmd.env("PATH", path_entries.join(":")); cmd.env("HOME", dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp"))); cmd.env("LANG", "C.UTF-8"); @@ -303,7 +335,23 @@ impl SafeCommand { } cmd.env_clear(); - cmd.env("PATH", "/usr/local/bin:/usr/bin:/bin"); + + // Build PATH with standard locations plus botserver-stack/bin/shared + let mut path_entries = vec![ + "/usr/local/bin".to_string(), + "/usr/bin".to_string(), + "/bin".to_string(), + ]; + + // Add botserver-stack/bin/shared to PATH if it exists + let stack_path = std::env::var("BOTSERVER_STACK_PATH") + .unwrap_or_else(|_| "./botserver-stack".to_string()); + let shared_bin = format!("{}/bin/shared", stack_path); + if std::path::Path::new(&shared_bin).exists() { + path_entries.insert(0, shared_bin); + } + + cmd.env("PATH", path_entries.join(":")); cmd.env("HOME", dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp"))); cmd.env("LANG", "C.UTF-8"); @@ -324,7 +372,23 @@ impl SafeCommand { } cmd.env_clear(); - cmd.env("PATH", "/usr/local/bin:/usr/bin:/bin"); + + // Build PATH with standard locations plus botserver-stack/bin/shared + let mut path_entries = vec![ + "/usr/local/bin".to_string(), + "/usr/bin".to_string(), + "/bin".to_string(), + ]; + + // Add botserver-stack/bin/shared to PATH if it exists + let stack_path = std::env::var("BOTSERVER_STACK_PATH") + .unwrap_or_else(|_| "./botserver-stack".to_string()); + let shared_bin = format!("{}/bin/shared", stack_path); + if std::path::Path::new(&shared_bin).exists() { + path_entries.insert(0, shared_bin); + } + + cmd.env("PATH", path_entries.join(":")); cmd.env("HOME", dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp"))); cmd.env("LANG", "C.UTF-8"); diff --git a/src/security/panic_handler.rs b/src/security/panic_handler.rs index 04e100044..0080a6b7c 100644 --- a/src/security/panic_handler.rs +++ b/src/security/panic_handler.rs @@ -337,8 +337,6 @@ mod tests { fn test_catch_panic_failure() { let result = catch_panic(|| { panic!("test panic"); - #[allow(unreachable_code)] - 42 }); assert!(result.is_err()); assert!(result.unwrap_err().message.contains("test panic")); diff --git a/src/settings/mod.rs b/src/settings/mod.rs index ec05f8534..fe27b1085 100644 --- a/src/settings/mod.rs +++ b/src/settings/mod.rs @@ -1,4 +1,3 @@ -#![cfg_attr(feature = "mail", allow(unused_imports))] pub mod audit_log; pub mod menu_config; pub mod permission_inheritance; @@ -147,13 +146,10 @@ r##"
"## .to_string(), ) } #[derive(Debug, Deserialize)] -#[allow(dead_code)] struct SearchSettingsRequest { enable_fuzzy_search: Option, search_result_limit: Option, enable_ai_suggestions: Option, -index_attachments: Option, -search_sources: Option>, } #[derive(Debug, Serialize)] @@ -183,8 +179,15 @@ Json(SearchSettingsResponse { } +#[derive(Debug, Serialize)] +struct SmtpTestResponse { +success: bool, +message: Option, +error: Option, +} + +#[cfg(feature = "mail")] #[derive(Debug, Deserialize)] -#[allow(dead_code)] struct SmtpTestRequest { host: String, port: i32, @@ -193,11 +196,14 @@ password: Option, use_tls: Option, } -#[derive(Debug, Serialize)] -struct SmtpTestResponse { -success: bool, -message: Option, -error: Option, +#[cfg(not(feature = "mail"))] +#[derive(Debug, Deserialize)] +struct SmtpTestRequest { +_host: String, +_port: i32, +_username: Option, +_password: Option, +_use_tls: Option, } #[cfg(feature = "mail")] diff --git a/src/whatsapp/mod.rs b/src/whatsapp/mod.rs index 74ecaa212..60f4b15ee 100644 --- a/src/whatsapp/mod.rs +++ b/src/whatsapp/mod.rs @@ -344,7 +344,6 @@ async fn process_attendant_command( } async fn check_is_attendant(state: &Arc, phone: &str) -> bool { - let _conn = state.conn.clone(); let phone_clone = phone.to_string(); tokio::task::spawn_blocking(move || {