diff --git a/3rdparty.toml b/3rdparty.toml index b7dcb9407..a5da852f9 100644 --- a/3rdparty.toml +++ b/3rdparty.toml @@ -38,13 +38,12 @@ sha256 = "" [components.cache] name = "Valkey Cache (Redis-compatible)" -# Note: Valkey doesn't provide prebuilt binaries, using source tarball -# You may need to compile from source or use system package manager +# Valkey requires compilation from source - no prebuilt binaries available +# The installer will run 'make' to build valkey-server and valkey-cli +# Requires: gcc, make (usually available on most Linux systems) url = "https://github.com/valkey-io/valkey/archive/refs/tags/8.0.2.tar.gz" filename = "valkey-8.0.2.tar.gz" sha256 = "" -# Alternative: Use Redis from system package or Docker -# For prebuilt, consider: https://download.redis.io/releases/redis-7.2.4.tar.gz [components.llm] name = "Llama.cpp Server" diff --git a/config/directory_config.json b/config/directory_config.json index c9dedf042..492f4b276 100644 --- a/config/directory_config.json +++ b/config/directory_config.json @@ -1,7 +1,7 @@ { "base_url": "http://localhost:8080", "default_org": { - "id": "350192628202995726", + "id": "350217359614541838", "name": "default", "domain": "default.localhost" }, @@ -13,8 +13,8 @@ "first_name": "Admin", "last_name": "User" }, - "admin_token": "D2IMmmxhLcL_DJMUMbmXxMuebowhWz0m8jBBwyCjI80wWz8kMfW2XqSsoXydz3oluL9gcns", + "admin_token": "MPMhGsichldNO5Aw7vdM57CciCeU6Kl8lu736BuLwfMgJ3K4YLFGEUK-5h2MPM3x7ZHxc74", "project_id": "", - "client_id": "350192628773486606", - "client_secret": "lMT0aQarbjRVBRtzFUlheVkqKZlbcVO8j58EHOu0gIl4W65BGJVEc6k7WxJ8v4wr" + "client_id": "350217360168255502", + "client_secret": "6jVliHfRDxcocVeQxGybjMR2E5lnX0q3J7Z7Pfsegg66hrzDlrXCKhYzEyHtZdMF" } \ No newline at end of file diff --git a/src/console/mod.rs b/src/console/mod.rs index 79634a77d..28f07b5f9 100644 --- a/src/console/mod.rs +++ b/src/console/mod.rs @@ -9,7 +9,7 @@ use crossterm::{ use log::LevelFilter; use ratatui::{ backend::CrosstermBackend, - layout::{Constraint, Direction, Layout, Rect}, + layout::{Alignment, Constraint, Direction, Layout, Rect}, style::{Color, Modifier, Style}, text::{Line, Span}, widgets::{Block, Borders, List, ListItem, Paragraph, Wrap}, @@ -42,6 +42,7 @@ pub struct XtreeUI { progress_channel: Option< Arc>>, >, + state_channel: Option>>>>, bootstrap_status: String, } #[derive(Debug, Clone, Copy, PartialEq)] @@ -65,6 +66,7 @@ impl XtreeUI { active_panel: ActivePanel::Logs, should_quit: false, progress_channel: None, + state_channel: None, bootstrap_status: "Initializing...".to_string(), } } @@ -74,6 +76,12 @@ impl XtreeUI { ) { self.progress_channel = Some(rx); } + pub fn set_state_channel( + &mut self, + rx: Arc>>>, + ) { + self.state_channel = Some(rx); + } pub fn set_app_state(&mut self, app_state: Arc) { self.file_tree = Some(FileTree::new(app_state.clone())); self.status_panel = Some(StatusPanel::new(app_state.clone())); @@ -82,6 +90,7 @@ impl XtreeUI { self.active_panel = ActivePanel::FileTree; self.bootstrap_status = "Ready".to_string(); } + pub fn start_ui(&mut self) -> Result<()> { color_eyre::install()?; if !std::io::IsTerminal::is_terminal(&std::io::stdout()) { @@ -92,8 +101,12 @@ impl XtreeUI { execute!(stdout, EnterAlternateScreen)?; let backend = CrosstermBackend::new(stdout); let mut terminal = Terminal::new(backend)?; - init_logger(self.log_panel.clone())?; - log::set_max_level(LevelFilter::Trace); + // Initialize UI logger to capture logs for the log panel + // This works because env_logger is not initialized when console UI is enabled + if let Err(e) = init_logger(self.log_panel.clone()) { + eprintln!("Warning: Could not initialize UI logger: {}", e); + } + log::set_max_level(log::LevelFilter::Trace); let result = self.run_event_loop(&mut terminal); disable_raw_mode()?; execute!(terminal.backend_mut(), LeaveAlternateScreen)?; @@ -110,6 +123,28 @@ impl XtreeUI { let mut last_blink = std::time::Instant::now(); let rt = tokio::runtime::Runtime::new()?; loop { + // Poll for AppState updates from the main thread + if self.app_state.is_none() { + if let Some(ref state_rx) = self.state_channel { + if let Ok(mut rx) = state_rx.try_lock() { + if let Ok(app_state) = rx.try_recv() { + // Initialize all panels with the new state + self.file_tree = Some(FileTree::new(app_state.clone())); + self.status_panel = Some(StatusPanel::new(app_state.clone())); + self.chat_panel = Some(ChatPanel::new(app_state.clone())); + self.app_state = Some(app_state); + self.active_panel = ActivePanel::FileTree; + self.bootstrap_status = "Ready".to_string(); + + // Log that we received the state + if let Ok(mut log_panel) = self.log_panel.lock() { + log_panel.add_log("AppState received - UI fully initialized"); + } + } + } + } + } + if let Some(ref progress_rx) = self.progress_channel { if let Ok(mut rx) = progress_rx.try_lock() { while let Ok(progress) = rx.try_recv() { @@ -358,42 +393,130 @@ impl XtreeUI { title_bg: Color, title_fg: Color, ) { - let chunks = Layout::default() + // Same layout as the real UI - header, content, logs + let main_chunks = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Percentage(40), - Constraint::Percentage(20), - Constraint::Percentage(40), + Constraint::Length(3), + Constraint::Min(0), + Constraint::Length(12), ]) .split(f.area()); - let center = Layout::default() + + // Render header with GENERAL BOTS title + let header_block = Block::default().style(Style::default().bg(title_bg)); + f.render_widget(header_block, main_chunks[0]); + + let title = " GENERAL BOTS "; + let title_len = title.len() as u16; + let centered_x = (main_chunks[0].width.saturating_sub(title_len)) / 2; + let title_span = Span::styled( + title, + Style::default() + .fg(title_fg) + .bg(title_bg) + .add_modifier(Modifier::BOLD), + ); + f.render_widget( + Paragraph::new(Line::from(title_span)), + Rect { + x: main_chunks[0].x + centered_x, + y: main_chunks[0].y + 1, + width: title_len, + height: 1, + }, + ); + + // Content area - same 3 columns as real UI + let content_chunks = Layout::default() .direction(Direction::Horizontal) .constraints([ - Constraint::Percentage(30), + Constraint::Percentage(25), Constraint::Percentage(40), - Constraint::Percentage(30), + Constraint::Percentage(35), ]) - .split(chunks[1])[1]; - let block = Block::default() + .split(main_chunks[1]); + + // Left panel - FILE EXPLORER (loading) + let file_block = Block::default() .title(Span::styled( - " General Bots ", - Style::default() - .fg(title_fg) - .bg(title_bg) - .add_modifier(Modifier::BOLD), + " FILE EXPLORER ", + Style::default().fg(title_fg).bg(title_bg), )) .borders(Borders::ALL) .border_style(Style::default().fg(border)) .style(Style::default().bg(bg)); - let loading_text = format!( -"\n ╔════════════════════════════════╗\n ║ ║\n ║ Initializing System... ║\n ║ ║\n ║ {} ║\n ║ ║\n ╚════════════════════════════════╝\n", -format!("{:^30}", self.bootstrap_status) -); - let paragraph = Paragraph::new(loading_text) - .block(block) + let file_text = Paragraph::new("\n\n Loading files...") + .block(file_block) + .style(Style::default().fg(Color::DarkGray)); + f.render_widget(file_text, content_chunks[0]); + + // Middle panel - STATUS (loading with bootstrap info) + let middle_chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Percentage(50), Constraint::Percentage(50)]) + .split(content_chunks[1]); + + let status_block = Block::default() + .title(Span::styled( + " STATUS ", + Style::default().fg(title_fg).bg(title_bg), + )) + .borders(Borders::ALL) + .border_style(Style::default().fg(border)) + .style(Style::default().bg(bg)); + + let status_text = format!( + "\n ⏳ {}\n\n Components:\n ○ Vault\n ○ Database\n ○ Drive\n ○ Cache\n ○ LLM", + self.bootstrap_status + ); + let status_para = Paragraph::new(status_text) + .block(status_block) + .style(Style::default().fg(text)); + f.render_widget(status_para, middle_chunks[0]); + + // Empty space below status (will be editor later) + let empty_block = Block::default() + .borders(Borders::ALL) + .border_style(Style::default().fg(Color::DarkGray)) + .style(Style::default().bg(bg)); + f.render_widget(empty_block, middle_chunks[1]); + + // Right panel - CHAT (loading) + let chat_block = Block::default() + .title(Span::styled( + " CHAT ", + Style::default().fg(title_fg).bg(title_bg), + )) + .borders(Borders::ALL) + .border_style(Style::default().fg(border)) + .style(Style::default().bg(bg)); + let chat_text = Paragraph::new("\n\n Connecting...") + .block(chat_block) + .style(Style::default().fg(Color::DarkGray)); + f.render_widget(chat_text, content_chunks[2]); + + // Bottom panel - LOGS (showing bootstrap progress) + let logs_block = Block::default() + .title(Span::styled( + " SYSTEM LOGS ", + Style::default().fg(title_fg).bg(title_bg), + )) + .borders(Borders::ALL) + .border_style(Style::default().fg(border)) + .style(Style::default().bg(bg)); + + let logs_content = if let Ok(panel) = self.log_panel.lock() { + panel.render() + } else { + String::from(" Waiting for logs...") + }; + + let logs_para = Paragraph::new(logs_content) + .block(logs_block) .style(Style::default().fg(text)) .wrap(Wrap { trim: false }); - f.render_widget(paragraph, center); + f.render_widget(logs_para, main_chunks[2]); } fn render_file_tree( &self, diff --git a/src/core/bootstrap/mod.rs b/src/core/bootstrap/mod.rs index cc5ca4e6a..1e7de6336 100644 --- a/src/core/bootstrap/mod.rs +++ b/src/core/bootstrap/mod.rs @@ -111,31 +111,47 @@ impl BootstrapManager { // VAULT MUST START FIRST - all other services depend on it for secrets if pm.is_installed("vault") { - info!("Starting Vault secrets service..."); - match pm.start("vault") { - Ok(_child) => { - info!("Vault process started, waiting for initialization..."); - } - Err(e) => { - warn!("Vault might already be running: {}", e); - } - } + // Check if Vault is already running before trying to start + let vault_already_running = Command::new("sh") + .arg("-c") + .arg("curl -f -s http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 >/dev/null 2>&1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false); - // Wait for Vault to be ready (up to 10 seconds) - for i in 0..10 { - let vault_ready = Command::new("sh") - .arg("-c") - .arg("curl -f -s http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 >/dev/null 2>&1") - .status() - .map(|s| s.success()) - .unwrap_or(false); - - if vault_ready { - info!("Vault is responding"); - break; + if vault_already_running { + info!("Vault is already running"); + } else { + info!("Starting Vault secrets service..."); + match pm.start("vault") { + Ok(_child) => { + info!("Vault process started, waiting for initialization..."); + } + Err(e) => { + warn!("Vault might already be running: {}", e); + } } - if i < 9 { - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + + // Wait for Vault to be ready (up to 10 seconds) + for i in 0..10 { + let vault_ready = Command::new("sh") + .arg("-c") + .arg("curl -f -s http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 >/dev/null 2>&1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false); + + if vault_ready { + info!("Vault is responding"); + break; + } + if i < 9 { + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } } } @@ -251,6 +267,8 @@ impl BootstrapManager { let vault_running = Command::new("sh") .arg("-c") .arg("curl -f -s http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 >/dev/null 2>&1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) .status() .map(|s| s.success()) .unwrap_or(false); @@ -380,9 +398,11 @@ impl BootstrapManager { let status_output = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>&1", + "VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>/dev/null", vault_addr )) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) .output()?; let status_str = String::from_utf8_lossy(&status_output.stdout); @@ -406,9 +426,11 @@ impl BootstrapManager { let unseal_output = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator unseal {}", + "VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator unseal {} >/dev/null 2>&1", vault_addr, unseal_key )) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) .output()?; if !unseal_output.status.success() { @@ -421,9 +443,11 @@ impl BootstrapManager { let verify_output = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>&1", + "VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>/dev/null", vault_addr )) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) .output()?; let verify_str = String::from_utf8_lossy(&verify_output.stdout); @@ -1317,15 +1341,41 @@ VAULT_CACHE_TTL=300 } else { format!("{}/", config.drive.server) }; + + // Get credentials from config, or fetch from Vault if empty + let (access_key, secret_key) = + if config.drive.access_key.is_empty() || config.drive.secret_key.is_empty() { + // Try to get from Vault using the global SecretsManager + match crate::shared::utils::get_secrets_manager().await { + Some(manager) if manager.is_enabled() => { + match manager.get_drive_credentials().await { + Ok((ak, sk)) => (ak, sk), + Err(e) => { + warn!("Failed to get drive credentials from Vault: {}", e); + ( + config.drive.access_key.clone(), + config.drive.secret_key.clone(), + ) + } + } + } + _ => ( + config.drive.access_key.clone(), + config.drive.secret_key.clone(), + ), + } + } else { + ( + config.drive.access_key.clone(), + config.drive.secret_key.clone(), + ) + }; + let base_config = aws_config::defaults(BehaviorVersion::latest()) .endpoint_url(endpoint) .region("auto") .credentials_provider(aws_sdk_s3::config::Credentials::new( - config.drive.access_key.clone(), - config.drive.secret_key.clone(), - None, - None, - "static", + access_key, secret_key, None, None, "static", )) .load() .await; diff --git a/src/core/package_manager/installer.rs b/src/core/package_manager/installer.rs index 5ca967776..53783d9b6 100644 --- a/src/core/package_manager/installer.rs +++ b/src/core/package_manager/installer.rs @@ -107,8 +107,8 @@ impl PackageManager { ("MINIO_ROOT_PASSWORD".to_string(), "$DRIVE_SECRET".to_string()), ]), data_download_list: Vec::new(), - exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 --certs-dir {{CONF_PATH}}/system/certificates/drive > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(), - check_cmd: "ps -ef | grep minio | grep -v grep | grep {{BIN_PATH}}".to_string(), + exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(), + check_cmd: "ps -ef | grep minio | grep -v grep | grep {{BIN_PATH}} >/dev/null 2>&1".to_string(), }, ); } @@ -165,8 +165,8 @@ impl PackageManager { } fn register_cache(&mut self) { - // Using Valkey - the Redis-compatible fork with pre-built binaries - // Valkey is maintained by the Linux Foundation and provides direct binary downloads + // Using Valkey - the Redis-compatible fork + // Source tarball - requires compilation with make self.components.insert( "cache".to_string(), ComponentConfig { @@ -177,7 +177,7 @@ impl PackageManager { macos_packages: vec![], windows_packages: vec![], download_url: Some( - "https://github.com/valkey-io/valkey/releases/download/9.0.0/valkey-9.0.0-linux-x86_64.tar.gz".to_string(), + "https://github.com/valkey-io/valkey/archive/refs/tags/8.0.2.tar.gz".to_string(), ), binary_name: Some("valkey-server".to_string()), pre_install_cmds_linux: vec![], @@ -411,7 +411,7 @@ impl PackageManager { }, data_download_list: Vec::new(), exec_cmd: "{{BIN_PATH}}/forgejo-runner daemon --config {{CONF_PATH}}/alm-ci/config.yaml".to_string(), - check_cmd: "ps -ef | grep forgejo-runner | grep -v grep | grep {{BIN_PATH}}".to_string(), + check_cmd: "ps -ef | grep forgejo-runner | grep -v grep | grep {{BIN_PATH}} >/dev/null 2>&1".to_string(), }, ); } @@ -856,13 +856,15 @@ impl PackageManager { .replace("{{CONF_PATH}}", &conf_path.to_string_lossy()) .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()); - let check_status = std::process::Command::new("sh") + let check_output = std::process::Command::new("sh") .current_dir(&bin_path) .arg("-c") .arg(&check_cmd) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) .status(); - if check_status.is_ok() && check_status.unwrap().success() { + if check_output.is_ok() && check_output.unwrap().success() { trace!("Component {} is already running", component.name); return Ok(std::process::Command::new("sh") .arg("-c") @@ -884,12 +886,21 @@ impl PackageManager { rendered_cmd ); + // Fetch credentials from Vault for special placeholders + let vault_credentials = Self::fetch_vault_credentials(); + // Create new env vars map with evaluated $VAR references let mut evaluated_envs = HashMap::new(); for (k, v) in &component.env_vars { if v.starts_with('$') { let var_name = &v[1..]; - evaluated_envs.insert(k.clone(), std::env::var(var_name).unwrap_or_default()); + // First check Vault credentials, then fall back to env vars + let value = vault_credentials + .get(var_name) + .cloned() + .or_else(|| std::env::var(var_name).ok()) + .unwrap_or_default(); + evaluated_envs.insert(k.clone(), value); } else { evaluated_envs.insert(k.clone(), v.clone()); } @@ -923,7 +934,73 @@ impl PackageManager { } } } else { - Err(anyhow::anyhow!("Component {} not found", component)) + Err(anyhow::anyhow!("Component not found: {}", component)) } } + + /// Fetch credentials from Vault for component env var placeholders + /// Returns a HashMap with keys like DRIVE_ACCESSKEY, DRIVE_SECRET, etc. + fn fetch_vault_credentials() -> HashMap { + let mut credentials = HashMap::new(); + + // Try to fetch drive credentials from Vault using vault CLI + let vault_addr = + std::env::var("VAULT_ADDR").unwrap_or_else(|_| "http://localhost:8200".to_string()); + let vault_token = std::env::var("VAULT_TOKEN").unwrap_or_default(); + + if vault_token.is_empty() { + trace!("VAULT_TOKEN not set, skipping Vault credential fetch"); + return credentials; + } + + // Fetch drive credentials + if let Ok(output) = std::process::Command::new("sh") + .arg("-c") + .arg(format!( + "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv get -format=json secret/gbo/drive 2>/dev/null", + vault_addr, vault_token + )) + .output() + { + if output.status.success() { + if let Ok(json_str) = String::from_utf8(output.stdout) { + if let Ok(json) = serde_json::from_str::(&json_str) { + if let Some(data) = json.get("data").and_then(|d| d.get("data")) { + if let Some(accesskey) = data.get("accesskey").and_then(|v| v.as_str()) { + credentials.insert("DRIVE_ACCESSKEY".to_string(), accesskey.to_string()); + } + if let Some(secret) = data.get("secret").and_then(|v| v.as_str()) { + credentials.insert("DRIVE_SECRET".to_string(), secret.to_string()); + } + } + } + } + } + } + + // Fetch cache credentials + if let Ok(output) = std::process::Command::new("sh") + .arg("-c") + .arg(format!( + "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv get -format=json secret/gbo/cache 2>/dev/null", + vault_addr, vault_token + )) + .output() + { + if output.status.success() { + if let Ok(json_str) = String::from_utf8(output.stdout) { + if let Ok(json) = serde_json::from_str::(&json_str) { + if let Some(data) = json.get("data").and_then(|d| d.get("data")) { + if let Some(password) = data.get("password").and_then(|v| v.as_str()) { + credentials.insert("CACHE_PASSWORD".to_string(), password.to_string()); + } + } + } + } + } + } + + trace!("Fetched {} credentials from Vault", credentials.len()); + credentials + } } diff --git a/src/core/shared/utils.rs b/src/core/shared/utils.rs index c8fe5142e..6960a978c 100644 --- a/src/core/shared/utils.rs +++ b/src/core/shared/utils.rs @@ -23,7 +23,7 @@ use tokio::io::AsyncWriteExt; use tokio::sync::RwLock; /// Global SecretsManager instance - initialized once, used everywhere -static SECRETS_MANAGER: Lazy>>> = +static SECRETS_MANAGER: Lazy>>> = Lazy::new(|| Arc::new(RwLock::new(None))); /// Initialize the global secrets manager (call once at startup) @@ -43,7 +43,9 @@ pub async fn get_database_url() -> Result { } } // NO FALLBACK - Vault is mandatory - Err(anyhow::anyhow!("Vault not configured. Set VAULT_ADDR and VAULT_TOKEN in .env")) + Err(anyhow::anyhow!( + "Vault not configured. Set VAULT_ADDR and VAULT_TOKEN in .env" + )) } /// Get database URL synchronously (blocking) for diesel connections - NO FALLBACK @@ -51,21 +53,23 @@ pub fn get_database_url_sync() -> Result { // Check if we're in an async runtime context if let Ok(handle) = tokio::runtime::Handle::try_current() { // We're inside a tokio runtime - use block_in_place to avoid nesting - let result = tokio::task::block_in_place(|| { - handle.block_on(async { get_database_url().await }) - }); + let result = + tokio::task::block_in_place(|| handle.block_on(async { get_database_url().await })); if let Ok(url) = result { return Ok(url); } } else { // Not in a runtime - create a new one - let rt = tokio::runtime::Runtime::new().map_err(|e| anyhow::anyhow!("Failed to create runtime: {}", e))?; + let rt = tokio::runtime::Runtime::new() + .map_err(|e| anyhow::anyhow!("Failed to create runtime: {}", e))?; if let Ok(url) = rt.block_on(async { get_database_url().await }) { return Ok(url); } } // NO FALLBACK - Vault is mandatory - Err(anyhow::anyhow!("Vault not configured. Set VAULT_ADDR and VAULT_TOKEN in .env")) + Err(anyhow::anyhow!( + "Vault not configured. Set VAULT_ADDR and VAULT_TOKEN in .env" + )) } /// Get the global SecretsManager instance @@ -82,15 +86,35 @@ pub async fn create_s3_operator( } else { config.server.clone() }; + + // Get credentials from config, or fetch from Vault if empty + let (access_key, secret_key) = if config.access_key.is_empty() || config.secret_key.is_empty() { + // Try to get from Vault + let guard = SECRETS_MANAGER.read().await; + if let Some(ref manager) = *guard { + if manager.is_enabled() { + match manager.get_drive_credentials().await { + Ok((ak, sk)) => (ak, sk), + Err(e) => { + log::warn!("Failed to get drive credentials from Vault: {}", e); + (config.access_key.clone(), config.secret_key.clone()) + } + } + } else { + (config.access_key.clone(), config.secret_key.clone()) + } + } else { + (config.access_key.clone(), config.secret_key.clone()) + } + } else { + (config.access_key.clone(), config.secret_key.clone()) + }; + let base_config = aws_config::defaults(BehaviorVersion::latest()) .endpoint_url(endpoint) .region("auto") .credentials_provider(aws_sdk_s3::config::Credentials::new( - config.access_key.clone(), - config.secret_key.clone(), - None, - None, - "static", + access_key, secret_key, None, None, "static", )) .load() .await; @@ -247,7 +271,8 @@ pub fn create_conn() -> Result { /// Create database connection pool using SecretsManager (async version) pub async fn create_conn_async() -> Result { - let database_url = get_database_url().await + let database_url = get_database_url() + .await .expect("Vault not configured. Set VAULT_ADDR and VAULT_TOKEN in .env"); let manager = ConnectionManager::::new(database_url); Pool::builder().build(manager) diff --git a/src/core/urls.rs b/src/core/urls.rs index 0486fdd80..a97d64e0f 100644 --- a/src/core/urls.rs +++ b/src/core/urls.rs @@ -153,16 +153,16 @@ impl ApiUrls { pub struct InternalUrls; impl InternalUrls { - pub const DIRECTORY_BASE: &'static str = "https://localhost:8080"; + pub const DIRECTORY_BASE: &'static str = "http://localhost:8080"; pub const DATABASE: &'static str = "postgres://localhost:5432"; - pub const CACHE: &'static str = "rediss://localhost:6379"; - pub const DRIVE: &'static str = "https://localhost:9000"; - pub const EMAIL: &'static str = "https://localhost:8025"; - pub const LLM: &'static str = "https://localhost:8081"; - pub const EMBEDDING: &'static str = "https://localhost:8082"; - pub const QDRANT: &'static str = "https://localhost:6334"; - pub const FORGEJO: &'static str = "https://localhost:3000"; - pub const LIVEKIT: &'static str = "https://localhost:7880"; + pub const CACHE: &'static str = "redis://localhost:6379"; + pub const DRIVE: &'static str = "http://localhost:9000"; + pub const EMAIL: &'static str = "http://localhost:8025"; + pub const LLM: &'static str = "http://localhost:8081"; + pub const EMBEDDING: &'static str = "http://localhost:8082"; + pub const QDRANT: &'static str = "http://localhost:6334"; + pub const FORGEJO: &'static str = "http://localhost:3000"; + pub const LIVEKIT: &'static str = "http://localhost:7880"; } /// Helper functions for URL construction diff --git a/src/main.rs b/src/main.rs index 83a114e95..df92f4dbc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -291,6 +291,11 @@ async fn run_axum_server( #[tokio::main] async fn main() -> std::io::Result<()> { + // Parse args early to check for --noconsole/--noui + let args: Vec = std::env::args().collect(); + let no_ui = args.contains(&"--noui".to_string()); + let no_console = args.contains(&"--noconsole".to_string()); + // Install rustls crypto provider (ring) before any TLS operations // This must be done before any code that might use rustls let _ = rustls::crypto::ring::default_provider().install_default(); @@ -340,22 +345,22 @@ async fn main() -> std::io::Result<()> { // Set the RUST_LOG env var if not already set std::env::set_var("RUST_LOG", &rust_log); - env_logger::Builder::from_env(env_logger::Env::default()) - .write_style(env_logger::WriteStyle::Always) - .init(); - - println!( - "Starting {} {}...", - "General Bots".to_string(), - env!("CARGO_PKG_VERSION") - ); - use crate::llm::local::ensure_llama_servers_running; use botserver::config::ConfigManager; - let args: Vec = std::env::args().collect(); - let no_ui = args.contains(&"--noui".to_string()); - let no_console = args.contains(&"--noconsole".to_string()); + // Only initialize env_logger if console UI is disabled + // When console is enabled, the UI will set up its own logger to capture logs + if no_console || no_ui { + env_logger::Builder::from_env(env_logger::Env::default()) + .write_style(env_logger::WriteStyle::Always) + .init(); + + println!( + "Starting {} {}...", + "General Bots".to_string(), + env!("CARGO_PKG_VERSION") + ); + } // Configuration comes from Directory service, not .env files @@ -382,6 +387,7 @@ async fn main() -> std::io::Result<()> { } // Start UI thread if console is enabled (default) and not disabled by --noconsole or --noui + // Start UI IMMEDIATELY - empty shell first, data fills in later via channel let ui_handle: Option> = if !no_console && !no_ui { #[cfg(feature = "console")] { @@ -394,28 +400,10 @@ async fn main() -> std::io::Result<()> { .spawn(move || { let mut ui = botserver::console::XtreeUI::new(); ui.set_progress_channel(progress_rx.clone()); + ui.set_state_channel(state_rx.clone()); - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("Failed to create UI runtime"); - - rt.block_on(async { - tokio::select! { - result = async { - let mut rx = state_rx.lock().await; - rx.recv().await - } => { - if let Some(app_state) = result { - ui.set_app_state(app_state); - } - } - _ = tokio::time::sleep(tokio::time::Duration::from_secs(300)) => { - eprintln!("UI initialization timeout"); - } - } - }); - + // Start UI right away - shows empty loading state + // UI will poll for state updates internally if let Err(e) = ui.start_ui() { eprintln!("UI error: {}", e); } @@ -625,7 +613,7 @@ async fn main() -> std::io::Result<()> { config.server.host, config.server.port ); - let cache_url = "rediss://localhost:6379".to_string(); + let cache_url = "redis://localhost:6379".to_string(); let redis_client = match redis::Client::open(cache_url.as_str()) { Ok(client) => Some(Arc::new(client)), Err(e) => {