From 89e92a4739010347a7e9d2a1da45bf62253b6532 Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (Pragmatismo)" Date: Mon, 8 Dec 2025 14:08:49 -0300 Subject: [PATCH] feat: add offline installer cache and health endpoints - Add /health and /api/health endpoints for botui connectivity - Create 3rdparty.toml with all download URLs for offline bundles - Add botserver-installers/ cache directory for downloaded files - Implement DownloadCache module with: - Automatic cache lookup before downloading - Support for pre-populated offline bundles - SHA256 checksum verification (optional) - Cache management utilities (list, clear, size) - Update download_and_install to use cache system - Data files (models) also cached for reuse Cache behavior: - First run: downloads to botserver-installers/ - Subsequent runs: uses cached files - Delete botserver-stack/ without losing downloads - Pre-populate cache for fully offline installation --- .gitignore | 2 + 3rdparty.toml | 154 ++++++++ config/directory_config.json | 8 +- src/core/package_manager/cache.rs | 592 +++++++++++++++++++++++++++++ src/core/package_manager/facade.rs | 117 ++++-- src/core/package_manager/mod.rs | 2 + src/main.rs | 66 +++- 7 files changed, 905 insertions(+), 36 deletions(-) create mode 100644 3rdparty.toml create mode 100644 src/core/package_manager/cache.rs diff --git a/.gitignore b/.gitignore index 58f4eaaf..23339eb7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ botserver-stack *-log* docs/book *.rdb +botserver-installers/* +!botserver-installers/.gitkeep diff --git a/3rdparty.toml b/3rdparty.toml new file mode 100644 index 00000000..b7dcb940 --- /dev/null +++ b/3rdparty.toml @@ -0,0 +1,154 @@ +# Third-Party Dependencies Configuration +# ====================================== +# This file lists all external downloads required by botserver. +# +# Caching Behavior: +# - On first run, files are downloaded from the URLs below +# - Downloaded files are cached in ./botserver-installers/ +# - On subsequent runs, cached files are used instead of downloading +# - To force re-download, delete the cached file +# +# Offline Installation: +# - Pre-download all files to ./botserver-installers/ +# - The installer will use cached files automatically +# - You can safely delete ./botserver-stack/ without losing downloads + +[cache_settings] +# Directory where downloaded files are cached (relative to botserver root) +cache_dir = "botserver-installers" + +# Components +# ========== +# Each component has: +# - url: Download URL +# - filename: Local filename in cache +# - sha256: Optional checksum for verification (empty = skip verification) + +[components.drive] +name = "MinIO Object Storage" +url = "https://dl.min.io/server/minio/release/linux-amd64/minio" +filename = "minio" +sha256 = "" + +[components.tables] +name = "PostgreSQL Database" +url = "https://github.com/theseus-rs/postgresql-binaries/releases/download/17.2.0/postgresql-17.2.0-x86_64-unknown-linux-gnu.tar.gz" +filename = "postgresql-17.2.0-x86_64-unknown-linux-gnu.tar.gz" +sha256 = "" + +[components.cache] +name = "Valkey Cache (Redis-compatible)" +# Note: Valkey doesn't provide prebuilt binaries, using source tarball +# You may need to compile from source or use system package manager +url = "https://github.com/valkey-io/valkey/archive/refs/tags/8.0.2.tar.gz" +filename = "valkey-8.0.2.tar.gz" +sha256 = "" +# Alternative: Use Redis from system package or Docker +# For prebuilt, consider: https://download.redis.io/releases/redis-7.2.4.tar.gz + +[components.llm] +name = "Llama.cpp Server" +url = "https://github.com/ggml-org/llama.cpp/releases/download/b4547/llama-b4547-bin-ubuntu-x64.zip" +filename = "llama-b4547-bin-ubuntu-x64.zip" +sha256 = "" + +[components.email] +name = "Stalwart Mail Server" +url = "https://github.com/stalwartlabs/mail-server/releases/download/v0.10.7/stalwart-mail-x86_64-linux.tar.gz" +filename = "stalwart-mail-x86_64-linux.tar.gz" +sha256 = "" + +[components.proxy] +name = "Caddy Web Server" +url = "https://github.com/caddyserver/caddy/releases/download/v2.9.1/caddy_2.9.1_linux_amd64.tar.gz" +filename = "caddy_2.9.1_linux_amd64.tar.gz" +sha256 = "" + +[components.directory] +name = "Zitadel Identity Provider" +url = "https://github.com/zitadel/zitadel/releases/download/v2.70.4/zitadel-linux-amd64.tar.gz" +filename = "zitadel-linux-amd64.tar.gz" +sha256 = "" + +[components.alm] +name = "Forgejo Git Server" +url = "https://codeberg.org/forgejo/forgejo/releases/download/v10.0.2/forgejo-10.0.2-linux-amd64" +filename = "forgejo-10.0.2-linux-amd64" +sha256 = "" + +[components.alm_ci] +name = "Forgejo Actions Runner" +url = "https://code.forgejo.org/forgejo/runner/releases/download/v6.3.1/forgejo-runner-6.3.1-linux-amd64" +filename = "forgejo-runner-6.3.1-linux-amd64" +sha256 = "" + +[components.dns] +name = "CoreDNS Server" +url = "https://github.com/coredns/coredns/releases/download/v1.11.1/coredns_1.11.1_linux_amd64.tgz" +filename = "coredns_1.11.1_linux_amd64.tgz" +sha256 = "" + +[components.webmail] +name = "Roundcube Webmail" +url = "https://github.com/roundcube/roundcubemail/releases/download/1.6.6/roundcubemail-1.6.6-complete.tar.gz" +filename = "roundcubemail-1.6.6-complete.tar.gz" +sha256 = "" + +[components.meet] +name = "LiveKit Media Server" +url = "https://github.com/livekit/livekit/releases/download/v2.8.2/livekit_2.8.2_linux_amd64.tar.gz" +filename = "livekit_2.8.2_linux_amd64.tar.gz" +sha256 = "" + +[components.table_editor] +name = "NocoDB" +url = "http://get.nocodb.com/linux-x64" +filename = "nocodb-linux-x64" +sha256 = "" + +[components.vector_db] +name = "Qdrant Vector Database" +url = "https://github.com/qdrant/qdrant/releases/latest/download/qdrant-x86_64-unknown-linux-gnu.tar.gz" +filename = "qdrant-x86_64-unknown-linux-gnu.tar.gz" +sha256 = "" + +[components.timeseries_db] +name = "InfluxDB Time Series Database" +url = "https://download.influxdata.com/influxdb/releases/influxdb2-2.7.5-linux-amd64.tar.gz" +filename = "influxdb2-2.7.5-linux-amd64.tar.gz" +sha256 = "" + +[components.vault] +name = "HashiCorp Vault" +url = "https://releases.hashicorp.com/vault/1.15.4/vault_1.15.4_linux_amd64.zip" +filename = "vault_1.15.4_linux_amd64.zip" +sha256 = "" + +[components.observability] +name = "Vector Log Aggregator" +url = "https://packages.timber.io/vector/0.35.0/vector-0.35.0-x86_64-unknown-linux-gnu.tar.gz" +filename = "vector-0.35.0-x86_64-unknown-linux-gnu.tar.gz" +sha256 = "" + +# LLM Models +# ========== +# Large model files for AI/ML functionality + +[models.deepseek_small] +name = "DeepSeek R1 Distill Qwen 1.5B (Q3_K_M)" +url = "https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf" +filename = "DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf" +sha256 = "" + +[models.bge_embedding] +name = "BGE Small EN v1.5 Embedding Model" +url = "https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf" +filename = "bge-small-en-v1.5-f32.gguf" +sha256 = "" + +# Optional larger models (uncomment to include) +# [models.gpt_oss_20b] +# name = "GPT-OSS 20B F16 (requires 16GB+ VRAM)" +# url = "https://huggingface.co/unsloth/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-F16.gguf" +# filename = "gpt-oss-20b-F16.gguf" +# sha256 = "" diff --git a/config/directory_config.json b/config/directory_config.json index 6fd537bd..3a0e3e00 100644 --- a/config/directory_config.json +++ b/config/directory_config.json @@ -1,7 +1,7 @@ { "base_url": "http://localhost:8080", "default_org": { - "id": "350139980594544654", + "id": "350166510557200398", "name": "default", "domain": "default.localhost" }, @@ -13,8 +13,8 @@ "first_name": "Admin", "last_name": "User" }, - "admin_token": "WFe8gHNf6oPO6B9S1sPlRyIgrAz1hsrao4k1NwuGOXb0GyGb4U2ZKysQb4jab0YLDMQKZ4o", + "admin_token": "ljVnxOognfMMqF_EOrkBse3YPEV7pUJDn78z9RFzIgGdzejdC35UfiYK7TSljt2mVaV6p40", "project_id": "", - "client_id": "350139982339440654", - "client_secret": "iQ3yc8eQpjJtttpOvzZs2pcUp2tkRwnhqvzC13dbj2tKGrfaeksjRg5wMFO0pAKP" + "client_id": "350166511245131790", + "client_secret": "JovJLn0DFlriDW4mtaDCTZ7cZPWPWCkLQgL7dVoOmhRqD3zMEkEGPTmLb8rISKCO" } \ No newline at end of file diff --git a/src/core/package_manager/cache.rs b/src/core/package_manager/cache.rs new file mode 100644 index 00000000..89281028 --- /dev/null +++ b/src/core/package_manager/cache.rs @@ -0,0 +1,592 @@ +//! Download Cache Module +//! +//! Provides caching functionality for third-party downloads. +//! Files are cached in `botserver-installers/` directory and reused +//! on subsequent runs, allowing offline installation. +//! +//! Configuration is read from `3rdparty.toml` at the botserver root. + +use anyhow::{Context, Result}; +use log::{debug, info, trace, warn}; +use serde::Deserialize; +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +/// Default cache directory relative to botserver root +const DEFAULT_CACHE_DIR: &str = "botserver-installers"; + +/// Configuration file name +const CONFIG_FILE: &str = "3rdparty.toml"; + +/// Third-party dependencies configuration +#[derive(Debug, Deserialize, Default)] +pub struct ThirdPartyConfig { + #[serde(default)] + pub cache_settings: CacheSettings, + #[serde(default)] + pub components: HashMap, + #[serde(default)] + pub models: HashMap, +} + +/// Cache settings +#[derive(Debug, Deserialize)] +pub struct CacheSettings { + #[serde(default = "default_cache_dir")] + pub cache_dir: String, +} + +impl Default for CacheSettings { + fn default() -> Self { + Self { + cache_dir: default_cache_dir(), + } + } +} + +fn default_cache_dir() -> String { + DEFAULT_CACHE_DIR.to_string() +} + +/// Component download configuration +#[derive(Debug, Deserialize, Clone)] +pub struct ComponentDownload { + pub name: String, + pub url: String, + pub filename: String, + #[serde(default)] + pub sha256: String, +} + +/// Download cache manager +#[derive(Debug)] +pub struct DownloadCache { + /// Base path for the botserver (where 3rdparty.toml lives) + base_path: PathBuf, + /// Cache directory path + cache_dir: PathBuf, + /// Loaded configuration + config: ThirdPartyConfig, +} + +impl DownloadCache { + /// Create a new download cache manager + /// + /// # Arguments + /// * `base_path` - Base path for botserver (typically current directory or botserver root) + pub fn new(base_path: impl AsRef) -> Result { + let base_path = base_path.as_ref().to_path_buf(); + let config = Self::load_config(&base_path)?; + let cache_dir = base_path.join(&config.cache_settings.cache_dir); + + // Ensure cache directory exists + if !cache_dir.exists() { + fs::create_dir_all(&cache_dir) + .with_context(|| format!("Failed to create cache directory: {:?}", cache_dir))?; + info!("Created cache directory: {:?}", cache_dir); + } + + Ok(Self { + base_path, + cache_dir, + config, + }) + } + + /// Load configuration from 3rdparty.toml + fn load_config(base_path: &Path) -> Result { + let config_path = base_path.join(CONFIG_FILE); + + if !config_path.exists() { + debug!( + "No {} found at {:?}, using defaults", + CONFIG_FILE, config_path + ); + return Ok(ThirdPartyConfig::default()); + } + + let content = fs::read_to_string(&config_path) + .with_context(|| format!("Failed to read config file: {:?}", config_path))?; + + let config: ThirdPartyConfig = toml::from_str(&content) + .with_context(|| format!("Failed to parse config file: {:?}", config_path))?; + + debug!( + "Loaded {} with {} components and {} models", + CONFIG_FILE, + config.components.len(), + config.models.len() + ); + + Ok(config) + } + + /// Get the cache directory path + pub fn cache_dir(&self) -> &Path { + &self.cache_dir + } + + /// Get the base path + pub fn base_path(&self) -> &Path { + &self.base_path + } + + /// Check if a file is cached + /// + /// # Arguments + /// * `filename` - The filename to check in the cache + pub fn is_cached(&self, filename: &str) -> bool { + let cached_path = self.cache_dir.join(filename); + if cached_path.exists() { + // Also check that file is not empty + if let Ok(metadata) = fs::metadata(&cached_path) { + return metadata.len() > 0; + } + } + false + } + + /// Get the cached file path if it exists + /// + /// # Arguments + /// * `filename` - The filename to get from cache + pub fn get_cached_path(&self, filename: &str) -> Option { + let cached_path = self.cache_dir.join(filename); + if self.is_cached(filename) { + Some(cached_path) + } else { + None + } + } + + /// Get the path where a file should be cached + /// + /// # Arguments + /// * `filename` - The filename + pub fn get_cache_path(&self, filename: &str) -> PathBuf { + self.cache_dir.join(filename) + } + + /// Look up component download info by component name + /// + /// # Arguments + /// * `component` - Component name (e.g., "drive", "tables", "llm") + pub fn get_component(&self, component: &str) -> Option<&ComponentDownload> { + self.config.components.get(component) + } + + /// Look up model download info by model name + /// + /// # Arguments + /// * `model` - Model name (e.g., "deepseek_small", "bge_embedding") + pub fn get_model(&self, model: &str) -> Option<&ComponentDownload> { + self.config.models.get(model) + } + + /// Get all component downloads + pub fn all_components(&self) -> &HashMap { + &self.config.components + } + + /// Get all model downloads + pub fn all_models(&self) -> &HashMap { + &self.config.models + } + + /// Resolve a URL to either a cached file path or the original URL + /// + /// This is the main method to use when downloading. It will: + /// 1. Extract filename from URL + /// 2. Check if file exists in cache + /// 3. Return cached path if available, otherwise return original URL + /// + /// # Arguments + /// * `url` - The download URL + /// + /// # Returns + /// * `CacheResult` - Either a cached file path or the URL to download from + pub fn resolve_url(&self, url: &str) -> CacheResult { + let filename = Self::extract_filename(url); + + if let Some(cached_path) = self.get_cached_path(&filename) { + info!("Using cached file: {:?}", cached_path); + CacheResult::Cached(cached_path) + } else { + trace!("File not in cache, will download: {}", url); + CacheResult::Download { + url: url.to_string(), + cache_path: self.get_cache_path(&filename), + } + } + } + + /// Resolve a URL for a specific component + /// + /// Uses the filename from config if available, otherwise extracts from URL + /// + /// # Arguments + /// * `component` - Component name + /// * `url` - Fallback URL if component not in config + pub fn resolve_component_url(&self, component: &str, url: &str) -> CacheResult { + // Check if we have config for this component + if let Some(comp) = self.get_component(component) { + let cached_path = self.cache_dir.join(&comp.filename); + if cached_path.exists() + && fs::metadata(&cached_path) + .map(|m| m.len() > 0) + .unwrap_or(false) + { + info!("Using cached {} from: {:?}", comp.name, cached_path); + return CacheResult::Cached(cached_path); + } + // Use URL from config + trace!("Will download {} from config URL", comp.name); + return CacheResult::Download { + url: comp.url.clone(), + cache_path: self.cache_dir.join(&comp.filename), + }; + } + + // Fall back to URL-based resolution + self.resolve_url(url) + } + + /// Save a downloaded file to the cache + /// + /// # Arguments + /// * `source` - Path to the downloaded file + /// * `filename` - Filename to use in the cache + pub fn save_to_cache(&self, source: &Path, filename: &str) -> Result { + let cache_path = self.cache_dir.join(filename); + + // If source is already in the cache directory, just return it + if source == cache_path { + return Ok(cache_path); + } + + // Copy to cache + fs::copy(source, &cache_path) + .with_context(|| format!("Failed to copy {:?} to cache at {:?}", source, cache_path))?; + + info!("Cached file: {:?}", cache_path); + Ok(cache_path) + } + + /// Extract filename from a URL + pub fn extract_filename(url: &str) -> String { + url.split('/') + .last() + .unwrap_or("download.tmp") + .split('?') + .next() + .unwrap_or("download.tmp") + .to_string() + } + + /// Verify a cached file's checksum if sha256 is provided + /// + /// # Arguments + /// * `filename` - The cached filename + /// * `expected_sha256` - Expected SHA256 hash (empty string to skip) + pub fn verify_checksum(&self, filename: &str, expected_sha256: &str) -> Result { + if expected_sha256.is_empty() { + return Ok(true); // Skip verification if no hash provided + } + + let cached_path = self.cache_dir.join(filename); + if !cached_path.exists() { + return Ok(false); + } + + let content = fs::read(&cached_path)?; + let computed = sha256_hex(&content); + + if computed == expected_sha256.to_lowercase() { + trace!("Checksum verified for {}", filename); + Ok(true) + } else { + warn!( + "Checksum mismatch for {}: expected {}, got {}", + filename, expected_sha256, computed + ); + Ok(false) + } + } + + /// List all cached files + pub fn list_cached(&self) -> Result> { + let mut files = Vec::new(); + + if self.cache_dir.exists() { + for entry in fs::read_dir(&self.cache_dir)? { + let entry = entry?; + if entry.file_type()?.is_file() { + if let Some(name) = entry.file_name().to_str() { + files.push(name.to_string()); + } + } + } + } + + files.sort(); + Ok(files) + } + + /// Get total size of cached files in bytes + pub fn cache_size(&self) -> Result { + let mut total = 0u64; + + if self.cache_dir.exists() { + for entry in fs::read_dir(&self.cache_dir)? { + let entry = entry?; + if entry.file_type()?.is_file() { + total += entry.metadata()?.len(); + } + } + } + + Ok(total) + } + + /// Clear all cached files + pub fn clear_cache(&self) -> Result<()> { + if self.cache_dir.exists() { + for entry in fs::read_dir(&self.cache_dir)? { + let entry = entry?; + if entry.file_type()?.is_file() { + fs::remove_file(entry.path())?; + } + } + info!("Cleared cache directory: {:?}", self.cache_dir); + } + Ok(()) + } +} + +/// Result of resolving a URL through the cache +#[derive(Debug)] +pub enum CacheResult { + /// File was found in cache + Cached(PathBuf), + /// File needs to be downloaded + Download { + /// URL to download from + url: String, + /// Path where file should be cached + cache_path: PathBuf, + }, +} + +impl CacheResult { + /// Check if result is a cached file + pub fn is_cached(&self) -> bool { + matches!(self, CacheResult::Cached(_)) + } + + /// Get the path (either cached or target cache path) + pub fn path(&self) -> &Path { + match self { + CacheResult::Cached(p) => p, + CacheResult::Download { cache_path, .. } => cache_path, + } + } + + /// Get the URL if this is a download result + pub fn url(&self) -> Option<&str> { + match self { + CacheResult::Cached(_) => None, + CacheResult::Download { url, .. } => Some(url), + } + } +} + +/// Compute SHA256 hash of data and return as lowercase hex string +fn sha256_hex(data: &[u8]) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(data); + let result = hasher.finalize(); + hex::encode(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::TempDir; + + fn create_test_config(dir: &Path) -> Result<()> { + let config = r#" +[cache_settings] +cache_dir = "test-cache" + +[components.test] +name = "Test Component" +url = "https://example.com/test.tar.gz" +filename = "test.tar.gz" +sha256 = "" + +[models.test_model] +name = "Test Model" +url = "https://example.com/model.gguf" +filename = "model.gguf" +sha256 = "" +"#; + let config_path = dir.join(CONFIG_FILE); + fs::write(config_path, config)?; + Ok(()) + } + + #[test] + fn test_extract_filename() { + assert_eq!( + DownloadCache::extract_filename("https://example.com/path/file.tar.gz"), + "file.tar.gz" + ); + assert_eq!( + DownloadCache::extract_filename("https://example.com/file.zip?token=abc"), + "file.zip" + ); + assert_eq!(DownloadCache::extract_filename("https://example.com/"), ""); + } + + #[test] + fn test_cache_creation() -> Result<()> { + let temp_dir = TempDir::new()?; + create_test_config(temp_dir.path())?; + + let cache = DownloadCache::new(temp_dir.path())?; + + assert!(cache.cache_dir().exists()); + assert_eq!(cache.cache_dir().file_name().unwrap(), "test-cache"); + + Ok(()) + } + + #[test] + fn test_is_cached() -> Result<()> { + let temp_dir = TempDir::new()?; + create_test_config(temp_dir.path())?; + + let cache = DownloadCache::new(temp_dir.path())?; + + // Initially not cached + assert!(!cache.is_cached("test.tar.gz")); + + // Create a cached file + let cache_path = cache.get_cache_path("test.tar.gz"); + let mut file = fs::File::create(&cache_path)?; + file.write_all(b"test content")?; + + // Now it should be cached + assert!(cache.is_cached("test.tar.gz")); + + // Empty file should not count as cached + let empty_path = cache.get_cache_path("empty.tar.gz"); + fs::File::create(&empty_path)?; + assert!(!cache.is_cached("empty.tar.gz")); + + Ok(()) + } + + #[test] + fn test_resolve_url() -> Result<()> { + let temp_dir = TempDir::new()?; + create_test_config(temp_dir.path())?; + + let cache = DownloadCache::new(temp_dir.path())?; + + // Test with uncached URL + let result = cache.resolve_url("https://example.com/newfile.tar.gz"); + assert!(!result.is_cached()); + assert_eq!(result.url(), Some("https://example.com/newfile.tar.gz")); + + // Create cached file + let cache_path = cache.get_cache_path("newfile.tar.gz"); + let mut file = fs::File::create(&cache_path)?; + file.write_all(b"cached content")?; + + // Now it should resolve to cached + let result = cache.resolve_url("https://example.com/newfile.tar.gz"); + assert!(result.is_cached()); + assert!(result.url().is_none()); + + Ok(()) + } + + #[test] + fn test_get_component() -> Result<()> { + let temp_dir = TempDir::new()?; + create_test_config(temp_dir.path())?; + + let cache = DownloadCache::new(temp_dir.path())?; + + let component = cache.get_component("test"); + assert!(component.is_some()); + assert_eq!(component.unwrap().name, "Test Component"); + + let missing = cache.get_component("nonexistent"); + assert!(missing.is_none()); + + Ok(()) + } + + #[test] + fn test_list_cached() -> Result<()> { + let temp_dir = TempDir::new()?; + create_test_config(temp_dir.path())?; + + let cache = DownloadCache::new(temp_dir.path())?; + + // Create some cached files + fs::write(cache.get_cache_path("file1.tar.gz"), "content1")?; + fs::write(cache.get_cache_path("file2.zip"), "content2")?; + + let files = cache.list_cached()?; + assert_eq!(files.len(), 2); + assert!(files.contains(&"file1.tar.gz".to_string())); + assert!(files.contains(&"file2.zip".to_string())); + + Ok(()) + } + + #[test] + fn test_cache_size() -> Result<()> { + let temp_dir = TempDir::new()?; + create_test_config(temp_dir.path())?; + + let cache = DownloadCache::new(temp_dir.path())?; + + // Initially empty + assert_eq!(cache.cache_size()?, 0); + + // Add files + fs::write(cache.get_cache_path("file1.txt"), "12345")?; // 5 bytes + fs::write(cache.get_cache_path("file2.txt"), "1234567890")?; // 10 bytes + + assert_eq!(cache.cache_size()?, 15); + + Ok(()) + } + + #[test] + fn test_clear_cache() -> Result<()> { + let temp_dir = TempDir::new()?; + create_test_config(temp_dir.path())?; + + let cache = DownloadCache::new(temp_dir.path())?; + + // Create some cached files + fs::write(cache.get_cache_path("file1.tar.gz"), "content1")?; + fs::write(cache.get_cache_path("file2.zip"), "content2")?; + + assert_eq!(cache.list_cached()?.len(), 2); + + cache.clear_cache()?; + + assert_eq!(cache.list_cached()?.len(), 0); + + Ok(()) + } +} diff --git a/src/core/package_manager/facade.rs b/src/core/package_manager/facade.rs index f0116544..f027c203 100644 --- a/src/core/package_manager/facade.rs +++ b/src/core/package_manager/facade.rs @@ -1,10 +1,11 @@ +use crate::package_manager::cache::{CacheResult, DownloadCache}; use crate::package_manager::component::ComponentConfig; use crate::package_manager::installer::PackageManager; use crate::package_manager::InstallMode; use crate::package_manager::OsType; use crate::shared::utils::{self, get_database_url_sync, parse_database_url}; use anyhow::{Context, Result}; -use log::{error, trace, warn}; +use log::{error, info, trace, warn}; use reqwest::Client; use std::collections::HashMap; use std::path::PathBuf; @@ -67,14 +68,54 @@ impl PackageManager { .await?; } if !component.data_download_list.is_empty() { + // Initialize cache for data files (models, etc.) + let cache_base = self.base_path.parent().unwrap_or(&self.base_path); + let cache = DownloadCache::new(cache_base).ok(); + for url in &component.data_download_list { - let filename = url.split('/').last().unwrap_or("download.tmp"); + let filename = DownloadCache::extract_filename(url); let output_path = self .base_path .join("data") .join(&component.name) - .join(filename); - utils::download_file(url, output_path.to_str().unwrap()).await?; + .join(&filename); + + // Check if already exists at destination + if output_path.exists() { + info!("Data file already exists: {:?}", output_path); + continue; + } + + // Ensure data directory exists + if let Some(parent) = output_path.parent() { + std::fs::create_dir_all(parent)?; + } + + // Check cache first + if let Some(ref c) = cache { + if let Some(cached_path) = c.get_cached_path(&filename) { + info!("Using cached data file: {:?}", cached_path); + std::fs::copy(&cached_path, &output_path)?; + continue; + } + } + + // Download to cache if available, otherwise directly to destination + let download_target = if let Some(ref c) = cache { + c.get_cache_path(&filename) + } else { + output_path.clone() + }; + + info!("Downloading data file: {}", url); + println!("Downloading {}", url); + utils::download_file(url, download_target.to_str().unwrap()).await?; + + // Copy from cache to destination if we downloaded to cache + if cache.is_some() && download_target != output_path { + std::fs::copy(&download_target, &output_path)?; + info!("Copied cached file to: {:?}", output_path); + } } } self.run_commands(post_cmds, "local", &component.name)?; @@ -121,10 +162,7 @@ impl PackageManager { }; if !packages.is_empty() { let pkg_list = packages.join(" "); - self.exec_in_container( - &container_name, - &format!("apt-get install -y {}", pkg_list), - )?; + self.exec_in_container(&container_name, &format!("apt-get install -y {}", pkg_list))?; } if let Some(url) = &component.download_url { self.download_in_container( @@ -156,7 +194,7 @@ impl PackageManager { ); Ok(()) } - + pub fn remove(&self, component_name: &str) -> Result<()> { let component = self .components @@ -272,25 +310,57 @@ impl PackageManager { ) -> Result<()> { let bin_path = self.base_path.join("bin").join(component); std::fs::create_dir_all(&bin_path)?; - let filename = url.split('/').last().unwrap_or("download.tmp"); - let temp_file = if filename.starts_with('/') { - PathBuf::from(filename) - } else { - bin_path.join(filename) + + // Initialize cache - use parent of base_path (botserver root) for cache + let cache_base = self.base_path.parent().unwrap_or(&self.base_path); + let cache = DownloadCache::new(cache_base).unwrap_or_else(|e| { + warn!("Failed to initialize download cache: {}", e); + // Create a fallback cache in base_path + DownloadCache::new(&self.base_path).expect("Failed to create fallback cache") + }); + + // Check cache first + let cache_result = cache.resolve_component_url(component, url); + + let source_file = match cache_result { + CacheResult::Cached(cached_path) => { + info!("Using cached file for {}: {:?}", component, cached_path); + cached_path + } + CacheResult::Download { + url: download_url, + cache_path, + } => { + info!("Downloading {} from {}", component, download_url); + println!("Downloading {}", download_url); + + // Download to cache directory + self.download_with_reqwest(&download_url, &cache_path, component) + .await?; + + info!("Cached {} to {:?}", component, cache_path); + cache_path + } }; - self.download_with_reqwest(url, &temp_file, component) - .await?; - self.handle_downloaded_file(&temp_file, &bin_path, binary_name)?; + + // Now extract/install from the source file (either cached or freshly downloaded) + self.handle_downloaded_file(&source_file, &bin_path, binary_name)?; Ok(()) } pub async fn download_with_reqwest( &self, url: &str, - temp_file: &PathBuf, + target_file: &PathBuf, component: &str, ) -> Result<()> { const MAX_RETRIES: u32 = 3; const RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(2); + + // Ensure parent directory exists + if let Some(parent) = target_file.parent() { + std::fs::create_dir_all(parent)?; + } + let client = Client::builder() .timeout(std::time::Duration::from_secs(30)) .user_agent("botserver-package-manager/1.0") @@ -306,7 +376,10 @@ impl PackageManager { ); std::thread::sleep(RETRY_DELAY * attempt); } - match self.attempt_reqwest_download(&client, url, temp_file).await { + match self + .attempt_reqwest_download(&client, url, target_file) + .await + { Ok(_size) => { if attempt > 0 { trace!("Download succeeded on retry attempt {}", attempt); @@ -316,7 +389,7 @@ impl PackageManager { Err(e) => { warn!("Download attempt {} failed: {}", attempt + 1, e); last_error = Some(e); - let _ = std::fs::remove_file(temp_file); + let _ = std::fs::remove_file(target_file); } } } @@ -446,7 +519,7 @@ impl PackageManager { } else { PathBuf::from("/opt/gbo/logs") }; - + // Get DB password from Vault for commands that need it (e.g., PostgreSQL initdb) let db_password = match get_database_url_sync() { Ok(url) => { @@ -460,7 +533,7 @@ impl PackageManager { String::new() } }; - + for cmd in commands { let rendered_cmd = cmd .replace("{{BIN_PATH}}", &bin_path.to_string_lossy()) diff --git a/src/core/package_manager/mod.rs b/src/core/package_manager/mod.rs index d655a148..3a4dbe98 100644 --- a/src/core/package_manager/mod.rs +++ b/src/core/package_manager/mod.rs @@ -1,7 +1,9 @@ +pub mod cache; pub mod component; pub mod installer; pub mod os; pub mod setup; +pub use cache::{CacheResult, DownloadCache}; pub use installer::PackageManager; pub mod cli; pub mod facade; diff --git a/src/main.rs b/src/main.rs index c88e43b5..83a114e9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,6 @@ -use axum::extract::Extension; +use axum::extract::{Extension, State}; +use axum::http::StatusCode; +use axum::Json; use axum::{ routing::{get, post}, Router, @@ -95,6 +97,42 @@ use crate::shared::utils::create_s3_operator; // Use BootstrapProgress from lib.rs use botserver::BootstrapProgress; +/// Health check endpoint handler +/// Returns server health status for monitoring and load balancers +async fn health_check(State(state): State>) -> (StatusCode, Json) { + // Check database connectivity + let db_ok = state.conn.get().is_ok(); + + let status = if db_ok { "healthy" } else { "degraded" }; + let code = if db_ok { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + + ( + code, + Json(serde_json::json!({ + "status": status, + "service": "botserver", + "version": env!("CARGO_PKG_VERSION"), + "database": db_ok + })), + ) +} + +/// Simple health check without state (for basic liveness probes) +async fn health_check_simple() -> (StatusCode, Json) { + ( + StatusCode::OK, + Json(serde_json::json!({ + "status": "ok", + "service": "botserver", + "version": env!("CARGO_PKG_VERSION") + })), + ) +} + async fn run_axum_server( app_state: Arc, port: u16, @@ -111,6 +149,9 @@ async fn run_axum_server( // Build API router with module-specific routes let mut api_router = Router::new() + // Health check endpoints - both /health and /api/health for compatibility + .route("/health", get(health_check_simple)) + .route(ApiUrls::HEALTH, get(health_check)) .route(ApiUrls::SESSIONS, post(create_session)) .route(ApiUrls::SESSIONS, get(get_sessions)) .route( @@ -213,8 +254,13 @@ async fn run_axum_server( // Bind to address let addr = SocketAddr::from(([0, 0, 0, 0], port)); - // Check if certificates exist - if cert_path.exists() && key_path.exists() { + // Check if TLS is disabled via environment variable (for local development) + let disable_tls = std::env::var("BOTSERVER_DISABLE_TLS") + .map(|v| v == "true" || v == "1") + .unwrap_or(false); + + // Check if certificates exist and TLS is not disabled + if !disable_tls && cert_path.exists() && key_path.exists() { // Use HTTPS with existing certificates let tls_config = axum_server::tls_rustls::RustlsConfig::from_pem_file(cert_path, key_path) .await @@ -228,15 +274,15 @@ async fn run_axum_server( .serve(app.into_make_service()) .await } else { - // Generate self-signed certificate if not present - warn!("TLS certificates not found, generating self-signed certificate..."); + // Use HTTP - either TLS is disabled or certificates don't exist + if disable_tls { + info!("TLS disabled via BOTSERVER_DISABLE_TLS environment variable"); + } else { + warn!("TLS certificates not found, using HTTP"); + } - // Fall back to HTTP temporarily (bootstrap will generate certs) let listener = tokio::net::TcpListener::bind(addr).await?; - info!( - "HTTP server listening on {} (certificates will be generated on next restart)", - addr - ); + info!("HTTP server listening on {}", addr); axum::serve(listener, app.into_make_service()) .await .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))