From 88a52f172e52b3e64b34bc4f44826046f89cb9e8 Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (Pragmatismo)" Date: Sat, 18 Oct 2025 19:08:00 -0300 Subject: [PATCH] - New bootstrap engine. --- Cargo.lock | 2 +- Cargo.toml | 3 +- add-req.sh | 18 +- src/bootstrap/mod.rs | 495 +++++++++++++++++++++++ src/config/mod.rs | 2 +- src/main.rs | 208 ++++------ src/package_manager/mod.rs | 790 +++++++++++++++---------------------- 7 files changed, 896 insertions(+), 622 deletions(-) create mode 100644 src/bootstrap/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 4230015e..ba716e98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1009,7 +1009,7 @@ dependencies = [ [[package]] name = "botserver" -version = "6.0.1" +version = "6.0.4" dependencies = [ "actix-cors", "actix-multipart", diff --git a/Cargo.toml b/Cargo.toml index 4a3eda43..219aee32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,6 @@ vectordb = ["qdrant-client"] email = ["imap"] web_automation = ["headless_chrome"] - [dependencies] actix-cors = "0.7" actix-multipart = "0.7" @@ -67,6 +66,8 @@ livekit = "0.7" log = "0.4" mailparse = "0.15" native-tls = "0.2" + + num-format = "0.4" qdrant-client = { version = "1.12", optional = true } rhai = { git = "https://github.com/therealprof/rhai.git", branch = "features/use-web-time" } diff --git a/add-req.sh b/add-req.sh index 01842829..e645e196 100755 --- a/add-req.sh +++ b/add-req.sh @@ -8,7 +8,7 @@ rm -f "$OUTPUT_FILE" echo "Consolidated LLM Context" > "$OUTPUT_FILE" prompts=( - "./prompts/dev/shared.md" + "./prompts/dev/platform/shared.md" "./Cargo.toml" ) @@ -23,19 +23,20 @@ dirs=( #"auth" #"automation" #"basic" - "bot" + #"bot" + "package_manager" #"channels" "config" "context" #"email" - "file" - "llm" + #"file" + #"llm" #"llm_legacy" #"org" - "session" + #"session" "shared" #"tests" - "tools" + #"tools" #"web_automation" #"whatsapp" ) @@ -58,10 +59,7 @@ done # Additional specific files files=( "$PROJECT_ROOT/src/main.rs" - "$PROJECT_ROOT/src/basic/keywords/mod.rs" - "$PROJECT_ROOT/src/basic/keywords/get.rs" - "$PROJECT_ROOT/src/basic/keywords/find.rs" - "$PROJECT_ROOT/src/basic/keywords/hear_talk.rs" + #"$PROJECT_ROOT/src/basic/keywords/mod.rs" ) diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs new file mode 100644 index 00000000..af272284 --- /dev/null +++ b/src/bootstrap/mod.rs @@ -0,0 +1,495 @@ +use crate::config::AppConfig; +use crate::package_manager::{InstallMode, PackageManager}; +use anyhow::{Context, Result}; +use diesel::prelude::*; +use log::{info, warn}; +use std::collections::HashMap; +use std::net::TcpListener; +use std::path::PathBuf; +use std::process::Command; +use std::thread; +use std::time::Duration; + +pub struct BootstrapManager { + mode: InstallMode, + tenant: String, + base_path: PathBuf, + config_values: HashMap, +} + +impl BootstrapManager { + pub fn new(mode: InstallMode, tenant: Option) -> Self { + let tenant = tenant.unwrap_or_else(|| "default".to_string()); + let base_path = if mode == InstallMode::Container { + PathBuf::from("/opt/gbo") + } else { + PathBuf::from("./botserver-stack") + }; + + Self { + mode, + tenant, + base_path, + config_values: HashMap::new(), + } + } + + pub fn bootstrap(&mut self) -> Result { + info!( + "Starting bootstrap process in {:?} mode for tenant {}", + self.mode, self.tenant + ); + + std::fs::create_dir_all(&self.base_path).context("Failed to create base directory")?; + + let pm = PackageManager::new(self.mode.clone(), Some(self.tenant.clone()))?; + + info!("Installing core infrastructure components"); + self.install_and_configure_tables(&pm)?; + self.install_and_configure_drive(&pm)?; + self.install_and_configure_cache(&pm)?; + self.install_and_configure_llm(&pm)?; + + info!("Creating database schema and storing configuration"); + let config = self.build_config()?; + self.initialize_database(&config)?; + self.store_configuration_in_db(&config)?; + + info!("Bootstrap completed successfully"); + Ok(config) + } + + fn install_and_configure_tables(&mut self, pm: &PackageManager) -> Result<()> { + info!("Installing PostgreSQL tables component"); + pm.install("tables")?; + + let tables_port = self.find_available_port(5432); + let tables_password = self.generate_password(); + + self.config_values + .insert("TABLES_USERNAME".to_string(), self.tenant.clone()); + self.config_values + .insert("TABLES_PASSWORD".to_string(), tables_password.clone()); + self.config_values + .insert("TABLES_SERVER".to_string(), self.get_service_host("tables")); + self.config_values + .insert("TABLES_PORT".to_string(), tables_port.to_string()); + self.config_values + .insert("TABLES_DATABASE".to_string(), format!("{}_db", self.tenant)); + + self.wait_for_service(&self.get_service_host("tables"), tables_port, 30)?; + + info!( + "PostgreSQL configured: {}:{}", + self.get_service_host("tables"), + tables_port + ); + Ok(()) + } + + fn install_and_configure_drive(&mut self, pm: &PackageManager) -> Result<()> { + info!("Installing MinIO drive component"); + pm.install("drive")?; + + let drive_port = self.find_available_port(9000); + let _drive_console_port = self.find_available_port(9001); + let drive_user = "minioadmin".to_string(); + let drive_password = self.generate_password(); + + self.config_values.insert( + "DRIVE_SERVER".to_string(), + format!("{}:{}", self.get_service_host("drive"), drive_port), + ); + self.config_values + .insert("DRIVE_ACCESSKEY".to_string(), drive_user.clone()); + self.config_values + .insert("DRIVE_SECRET".to_string(), drive_password.clone()); + self.config_values + .insert("DRIVE_USE_SSL".to_string(), "false".to_string()); + self.config_values + .insert("DRIVE_ORG_PREFIX".to_string(), self.tenant.clone()); + self.config_values.insert( + "DRIVE_BUCKET".to_string(), + format!("{}default.gbai", self.tenant), + ); + + self.wait_for_service(&self.get_service_host("drive"), drive_port, 30)?; + + info!( + "MinIO configured: {}:{}", + self.get_service_host("drive"), + drive_port + ); + Ok(()) + } + + fn install_and_configure_cache(&mut self, pm: &PackageManager) -> Result<()> { + info!("Installing Redis cache component"); + pm.install("cache")?; + + let cache_port = self.find_available_port(6379); + + self.config_values.insert( + "CACHE_URL".to_string(), + format!("redis://{}:{}/", self.get_service_host("cache"), cache_port), + ); + + self.wait_for_service(&self.get_service_host("cache"), cache_port, 30)?; + + info!( + "Redis configured: {}:{}", + self.get_service_host("cache"), + cache_port + ); + Ok(()) + } + + fn install_and_configure_llm(&mut self, pm: &PackageManager) -> Result<()> { + info!("Installing LLM server component"); + pm.install("llm")?; + + let llm_port = self.find_available_port(8081); + + self.config_values.insert( + "LLM_URL".to_string(), + format!("http://{}:{}", self.get_service_host("llm"), llm_port), + ); + self.config_values.insert( + "AI_ENDPOINT".to_string(), + format!("http://{}:{}", self.get_service_host("llm"), llm_port), + ); + self.config_values + .insert("AI_KEY".to_string(), "empty".to_string()); + self.config_values + .insert("AI_INSTANCE".to_string(), "llama-local".to_string()); + self.config_values + .insert("AI_VERSION".to_string(), "1.0".to_string()); + + self.wait_for_service(&self.get_service_host("llm"), llm_port, 60)?; + + info!( + "LLM server configured: {}:{}", + self.get_service_host("llm"), + llm_port + ); + Ok(()) + } + + fn build_config(&self) -> Result { + info!("Building application configuration from discovered services"); + + let get_str = |key: &str, default: &str| -> String { + self.config_values + .get(key) + .cloned() + .unwrap_or_else(|| default.to_string()) + }; + + let get_u32 = |key: &str, default: u32| -> u32 { + self.config_values + .get(key) + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + }; + + let get_u16 = |key: &str, default: u16| -> u16 { + self.config_values + .get(key) + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + }; + + let get_bool = |key: &str, default: bool| -> bool { + self.config_values + .get(key) + .map(|v| v.to_lowercase() == "true") + .unwrap_or(default) + }; + + let stack_path = self.base_path.clone(); + + let database = crate::config::DatabaseConfig { + username: get_str("TABLES_USERNAME", "botserver"), + password: get_str("TABLES_PASSWORD", "botserver"), + server: get_str("TABLES_SERVER", "localhost"), + port: get_u32("TABLES_PORT", 5432), + database: get_str("TABLES_DATABASE", "botserver_db"), + }; + + let database_custom = database.clone(); + + let minio = crate::config::DriveConfig { + server: get_str("DRIVE_SERVER", "localhost:9000"), + access_key: get_str("DRIVE_ACCESSKEY", "minioadmin"), + secret_key: get_str("DRIVE_SECRET", "minioadmin"), + use_ssl: get_bool("DRIVE_USE_SSL", false), + org_prefix: get_str("DRIVE_ORG_PREFIX", "botserver"), + }; + + let email = crate::config::EmailConfig { + from: get_str("EMAIL_FROM", "noreply@example.com"), + server: get_str("EMAIL_SERVER", "smtp.example.com"), + port: get_u16("EMAIL_PORT", 587), + username: get_str("EMAIL_USER", "user"), + password: get_str("EMAIL_PASS", "pass"), + }; + + let ai = crate::config::AIConfig { + instance: get_str("AI_INSTANCE", "llama-local"), + key: get_str("AI_KEY", "empty"), + version: get_str("AI_VERSION", "1.0"), + endpoint: get_str("AI_ENDPOINT", "http://localhost:8081"), + }; + + let server_host = if self.mode == InstallMode::Container { + "0.0.0.0".to_string() + } else { + "127.0.0.1".to_string() + }; + + Ok(AppConfig { + minio, + server: crate::config::ServerConfig { + host: server_host, + port: self.find_available_port(8080), + }, + database, + database_custom, + email, + ai, + s3_bucket: get_str("DRIVE_BUCKET", "default.gbai"), + site_path: format!("{}/sites", stack_path.display()), + stack_path, + db_conn: None, + }) + } + + fn initialize_database(&self, config: &AppConfig) -> Result<()> { + use diesel::pg::PgConnection; + + info!("Initializing database schema at {}", config.database_url()); + + // Attempt to establish a PostgreSQL connection with retries. + let mut retries = 5; + let mut conn = loop { + match PgConnection::establish(&config.database_url()) { + Ok(c) => break c, + Err(e) if retries > 0 => { + warn!("Database connection failed, retrying in 2s: {}", e); + thread::sleep(Duration::from_secs(2)); + retries -= 1; + } + Err(e) => { + return Err(anyhow::anyhow!( + "Failed to connect to database after retries: {}", + e + )) + } + } + }; + + // Create the server_configuration table. + diesel::sql_query( + "CREATE TABLE IF NOT EXISTS server_configuration ( + id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, + config_key TEXT NOT NULL UNIQUE, + config_value TEXT NOT NULL, + config_type TEXT NOT NULL DEFAULT 'string', + is_encrypted BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + )", + ) + .execute(&mut conn) + .context("Failed to create server_configuration table")?; + + // Create the bot_configuration table. + diesel::sql_query( + "CREATE TABLE IF NOT EXISTS bot_configuration ( + id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, + bot_id UUID NOT NULL, + config_key TEXT NOT NULL, + config_value TEXT NOT NULL, + config_type TEXT NOT NULL DEFAULT 'string', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(bot_id, config_key) + )", + ) + .execute(&mut conn) + .context("Failed to create bot_configuration table")?; + + // Create the gbot_config_sync table. + diesel::sql_query( + "CREATE TABLE IF NOT EXISTS gbot_config_sync ( + id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, + bot_id UUID NOT NULL UNIQUE, + config_file_path TEXT NOT NULL, + file_hash TEXT NOT NULL, + last_sync_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + sync_count INTEGER NOT NULL DEFAULT 0 + )", + ) + .execute(&mut conn) + .context("Failed to create gbot_config_sync table")?; + + info!("Database schema initialized successfully"); + Ok(()) + } + + fn store_configuration_in_db(&self, config: &AppConfig) -> Result<()> { + use diesel::pg::PgConnection; + + info!("Storing configuration in database"); + + // Establish a PostgreSQL connection explicitly. + let mut conn = PgConnection::establish(&config.database_url()) + .context("Failed to establish database connection for storing configuration")?; + + // Store dynamic configuration values. + for (key, value) in &self.config_values { + diesel::sql_query( + "INSERT INTO server_configuration (config_key, config_value, config_type) + VALUES ($1, $2, 'string') + ON CONFLICT (config_key) + DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()", + ) + .bind::(key) + .bind::(value) + .execute(&mut conn) + .with_context(|| format!("Failed to store config key: {}", key))?; + } + + // Store static configuration entries. + diesel::sql_query( + "INSERT INTO server_configuration (config_key, config_value, config_type) + VALUES ('SERVER_HOST', $1, 'string') + ON CONFLICT (config_key) + DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()", + ) + .bind::(&config.server.host) + .execute(&mut conn) + .context("Failed to store SERVER_HOST")?; + + diesel::sql_query( + "INSERT INTO server_configuration (config_key, config_value, config_type) + VALUES ('SERVER_PORT', $1, 'string') + ON CONFLICT (config_key) + DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()", + ) + .bind::(&config.server.port.to_string()) + .execute(&mut conn) + .context("Failed to store SERVER_PORT")?; + + diesel::sql_query( + "INSERT INTO server_configuration (config_key, config_value, config_type) + VALUES ('STACK_PATH', $1, 'string') + ON CONFLICT (config_key) + DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()", + ) + .bind::(&config.stack_path.display().to_string()) + .execute(&mut conn) + .context("Failed to store STACK_PATH")?; + + diesel::sql_query( + "INSERT INTO server_configuration (config_key, config_value, config_type) + VALUES ('SITES_ROOT', $1, 'string') + ON CONFLICT (config_key) + DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()", + ) + .bind::(&config.site_path) + .execute(&mut conn) + .context("Failed to store SITES_ROOT")?; + + info!( + "Configuration stored in database successfully with {} entries", + self.config_values.len() + 4 + ); + Ok(()) + } + + fn find_available_port(&self, preferred: u16) -> u16 { + if self.mode == InstallMode::Container { + return preferred; + } + + for port in preferred..preferred + 100 { + if TcpListener::bind(("127.0.0.1", port)).is_ok() { + return port; + } + } + preferred + } + + fn generate_password(&self) -> String { + use rand::Rng; + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + let mut rng = rand::rng(); + (0..16) + .map(|_| { + let idx = rng.random_range(0..CHARSET.len()); + CHARSET[idx] as char + }) + .collect() + } + + fn get_service_host(&self, component: &str) -> String { + match self.mode { + InstallMode::Container => { + let container_name = format!("{}-{}", self.tenant, component); + self.get_container_ip(&container_name) + .unwrap_or_else(|_| "127.0.0.1".to_string()) + } + InstallMode::Local => "127.0.0.1".to_string(), + } + } + + fn get_container_ip(&self, container_name: &str) -> Result { + let output = Command::new("lxc") + .args(&["list", container_name, "--format=json"]) + .output()?; + + if !output.status.success() { + return Err(anyhow::anyhow!("Failed to get container info")); + } + + let json: serde_json::Value = serde_json::from_slice(&output.stdout)?; + + if let Some(ip) = json + .get(0) + .and_then(|c| c.get("state")) + .and_then(|s| s.get("network")) + .and_then(|n| n.get("eth0")) + .and_then(|e| e.get("addresses")) + .and_then(|a| a.get(0)) + .and_then(|a| a.get("address")) + .and_then(|a| a.as_str()) + { + Ok(ip.to_string()) + } else { + Err(anyhow::anyhow!("Could not extract container IP")) + } + } + + fn wait_for_service(&self, host: &str, port: u16, timeout_secs: u64) -> Result<()> { + info!( + "Waiting for service at {}:{} (timeout: {}s)", + host, port, timeout_secs + ); + + let start = std::time::Instant::now(); + while start.elapsed().as_secs() < timeout_secs { + if TcpListener::bind((host, port)).is_err() { + info!("Service {}:{} is ready", host, port); + return Ok(()); + } + thread::sleep(Duration::from_secs(1)); + } + + Err(anyhow::anyhow!( + "Timeout waiting for service at {}:{}", + host, + port + )) + } +} diff --git a/src/config/mod.rs b/src/config/mod.rs index dd0626e3..60152d6d 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -18,7 +18,7 @@ pub struct AppConfig { pub site_path: String, pub s3_bucket: String, pub stack_path: PathBuf, - db_conn: Option>>, + pub(crate) db_conn: Option>>, } #[derive(Clone)] diff --git a/src/main.rs b/src/main.rs index ddbe9ade..826d8ffe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,9 +6,11 @@ use dotenvy::dotenv; use log::info; use std::collections::HashMap; use std::sync::{Arc, Mutex}; + mod auth; mod automation; mod basic; +mod bootstrap; mod bot; mod channels; mod config; @@ -30,21 +32,20 @@ mod tools; mod web_automation; mod web_server; mod whatsapp; + use crate::auth::auth_handler; use crate::automation::AutomationService; use crate::bot::{start_session, websocket_handler}; +use crate::bootstrap::BootstrapManager; use crate::channels::{VoiceAdapter, WebChannelAdapter}; use crate::config::AppConfig; use crate::drive_monitor::DriveMonitor; #[cfg(feature = "email")] -use crate::email::{ - get_emails, get_latest_email_from, list_emails, save_click, save_draft, send_email, -}; +use crate::email::{get_emails, get_latest_email_from, list_emails, save_click, save_draft, send_email}; use crate::file::{init_drive, upload_file}; -use crate::llm_legacy::llm_local::{ - chat_completions_local, embeddings_local, ensure_llama_servers_running, -}; +use crate::llm_legacy::llm_local::{chat_completions_local, embeddings_local, ensure_llama_servers_running}; use crate::meet::{voice_start, voice_stop}; +use crate::package_manager::InstallMode; use crate::session::{create_session, get_session_history, get_sessions}; use crate::shared::state::AppState; use crate::web_server::{index, static_files}; @@ -53,154 +54,104 @@ use crate::whatsapp::WhatsAppAdapter; #[actix_web::main] async fn main() -> std::io::Result<()> { - // ---------------------------------------------------------------------- - // CLI handling - must be first to intercept package manager commands - // ---------------------------------------------------------------------- let args: Vec = std::env::args().collect(); - - // Check if a CLI command was provided (anything beyond just the program name) + if args.len() > 1 { let command = &args[1]; - // Check if it's a recognized CLI command match command.as_str() { "install" | "remove" | "list" | "status" | "--help" | "-h" => { - // Run the CLI and exit (don't start the server) match package_manager::cli::run() { Ok(_) => return Ok(()), Err(e) => { eprintln!("CLI error: {}", e); - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("CLI command failed: {}", e), - )); + return Err(std::io::Error::new(std::io::ErrorKind::Other, format!("CLI command failed: {}", e))); } } } _ => { - // Unknown command - print error and exit eprintln!("Unknown command: {}", command); eprintln!("Run 'botserver --help' for usage information"); - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!("Unknown command: {}", command), - )); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("Unknown command: {}", command))); } } } - // No CLI commands - proceed with normal server startup - // ---------------------------------------------------------------------- - - // Load environment variables from a .env file, if present. dotenv().ok(); - let llama_url = - std::env::var("LLM_URL").unwrap_or_else(|_| "http://localhost:8081".to_string()); - - // Initialise logger with environment‑based log level (default to "info"). env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); - // Load application configuration. - let cfg = AppConfig::from_env(); - let config = std::sync::Arc::new(cfg.clone()); + info!("Starting BotServer bootstrap process"); + + let install_mode = if args.contains(&"--container".to_string()) { + InstallMode::Container + } else { + InstallMode::Local + }; + + let tenant = if let Some(idx) = args.iter().position(|a| a == "--tenant") { + args.get(idx + 1).cloned() + } else { + None + }; - let db_pool = match diesel::Connection::establish(&cfg.database_url()) { - Ok(conn) => { - info!("Connected to main database successfully"); - Arc::new(Mutex::new(conn)) + let mut bootstrap = BootstrapManager::new(install_mode.clone(), tenant.clone()); + let cfg = match bootstrap.bootstrap() { + Ok(config) => { + info!("Bootstrap completed successfully, configuration loaded from database"); + config } Err(e) => { - log::error!("Failed to connect to main database: {}", e); - return Err(std::io::Error::new( - std::io::ErrorKind::ConnectionRefused, - format!("Database connection failed: {}", e), - )); + log::error!("Bootstrap failed: {}", e); + info!("Attempting to load configuration from database"); + match diesel::Connection::establish(&format!("postgres://localhost:5432/botserver_db")) { + Ok(mut conn) => AppConfig::from_database(&mut conn), + Err(_) => { + info!("Database not available, using environment variables as fallback"); + AppConfig::from_env() + } + } + } + }; + + let config = std::sync::Arc::new(cfg.clone()); + + info!("Establishing database connection to {}", cfg.database_url()); + let db_pool = match diesel::Connection::establish(&cfg.database_url()) { + Ok(conn) => Arc::new(Mutex::new(conn)), + Err(e) => { + log::error!("Failed to connect to main database: {}", e); + return Err(std::io::Error::new(std::io::ErrorKind::ConnectionRefused, format!("Database connection failed: {}", e))); } }; - // Placeholder for a second/custom database – currently just re‑using the main pool. - let _custom_db_url = format!( - "postgres://{}:{}@{}:{}/{}", - cfg.database_custom.username, - cfg.database_custom.password, - cfg.database_custom.server, - cfg.database_custom.port, - cfg.database_custom.database - ); let db_custom_pool = db_pool.clone(); - // ---------------------------------------------------------------------- - // LLM local server initialisation - // ---------------------------------------------------------------------- - ensure_llama_servers_running() - .await - .expect("Failed to initialize LLM local server."); - - // ---------------------------------------------------------------------- - // Redis client (optional) - // ---------------------------------------------------------------------- - let cache_url = std::env::var("CACHE_URL").unwrap_or_else(|_| "redis://127.0.0.1/".to_string()); + info!("Initializing LLM server at {}", cfg.ai.endpoint); + ensure_llama_servers_running().await.expect("Failed to initialize LLM local server"); + let cache_url = cfg.config_path("cache").join("redis.conf").display().to_string(); let redis_client = match redis::Client::open(cache_url.as_str()) { - Ok(client) => { - info!("Connected to Redis successfully"); - Some(Arc::new(client)) - } + Ok(client) => Some(Arc::new(client)), Err(e) => { log::warn!("Failed to connect to Redis: {}", e); None } }; - // ---------------------------------------------------------------------- - // Tooling and LLM provider - // ---------------------------------------------------------------------- let tool_manager = Arc::new(tools::ToolManager::new()); - let llm_provider = Arc::new(crate::llm::OpenAIClient::new( - "empty".to_string(), - Some(llama_url.clone()), - )); - - // ---------------------------------------------------------------------- - // Channel adapters - // ---------------------------------------------------------------------- + let llm_provider = Arc::new(crate::llm::OpenAIClient::new("empty".to_string(), Some(cfg.ai.endpoint.clone()))); + let web_adapter = Arc::new(WebChannelAdapter::new()); - let voice_adapter = Arc::new(VoiceAdapter::new( - "https://livekit.example.com".to_string(), - "api_key".to_string(), - "api_secret".to_string(), - )); - let whatsapp_adapter = Arc::new(WhatsAppAdapter::new( - "whatsapp_token".to_string(), - "phone_number_id".to_string(), - "verify_token".to_string(), - )); + let voice_adapter = Arc::new(VoiceAdapter::new("https://livekit.example.com".to_string(), "api_key".to_string(), "api_secret".to_string())); + let whatsapp_adapter = Arc::new(WhatsAppAdapter::new("whatsapp_token".to_string(), "phone_number_id".to_string(), "verify_token".to_string())); let tool_api = Arc::new(tools::ToolApi::new()); - // ---------------------------------------------------------------------- - // S3 / MinIO client - // ---------------------------------------------------------------------- - let drive = init_drive(&config.minio) - .await - .expect("Failed to initialize Drive"); + info!("Initializing MinIO drive at {}", cfg.minio.server); + let drive = init_drive(&config.minio).await.expect("Failed to initialize Drive"); - // ---------------------------------------------------------------------- - // Session and authentication services - // ---------------------------------------------------------------------- - let session_manager = Arc::new(tokio::sync::Mutex::new(session::SessionManager::new( - diesel::Connection::establish(&cfg.database_url()).unwrap(), - redis_client.clone(), - ))); + let session_manager = Arc::new(tokio::sync::Mutex::new(session::SessionManager::new(diesel::Connection::establish(&cfg.database_url()).unwrap(), redis_client.clone()))); + let auth_service = Arc::new(tokio::sync::Mutex::new(auth::AuthService::new(diesel::Connection::establish(&cfg.database_url()).unwrap(), redis_client.clone()))); - let auth_service = Arc::new(tokio::sync::Mutex::new(auth::AuthService::new( - diesel::Connection::establish(&cfg.database_url()).unwrap(), - redis_client.clone(), - ))); - - // ---------------------------------------------------------------------- - // Global application state - // ---------------------------------------------------------------------- let app_state = Arc::new(AppState { - // `s3_client` expects an `Option`. s3_client: Some(drive.clone()), config: Some(cfg.clone()), conn: db_pool.clone(), @@ -212,10 +163,7 @@ async fn main() -> std::io::Result<()> { auth_service: auth_service.clone(), channels: Arc::new(Mutex::new({ let mut map = HashMap::new(); - map.insert( - "web".to_string(), - web_adapter.clone() as Arc, - ); + map.insert("web".to_string(), web_adapter.clone() as Arc); map })), response_channels: Arc::new(tokio::sync::Mutex::new(HashMap::new())), @@ -225,51 +173,29 @@ async fn main() -> std::io::Result<()> { tool_api: tool_api.clone(), }); - // ---------------------------------------------------------------------- - // Start HTTP server (multithreaded) - // ---------------------------------------------------------------------- - info!( - "Starting server on {}:{}", - config.server.host, config.server.port - ); + info!("Starting HTTP server on {}:{}", config.server.host, config.server.port); - // Determine the number of worker threads – default to the number of logical CPUs, - // fallback to 4 if the information cannot be retrieved. - let worker_count = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4); + let worker_count = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4); - // Start automation service in background let automation_state = app_state.clone(); - - let automation = AutomationService::new( - automation_state, - "templates/announcements.gbai/announcements.gbdialog", - ); + let automation = AutomationService::new(automation_state, "templates/announcements.gbai/announcements.gbdialog"); let _automation_handle = automation.spawn(); - // Start Drive Monitor service in background let drive_state = app_state.clone(); let bucket_name = format!("{}default.gbai", cfg.minio.org_prefix); let drive_monitor = Arc::new(DriveMonitor::new(drive_state, bucket_name)); let _drive_handle = drive_monitor.spawn(); HttpServer::new(move || { - // CORS configuration – allow any origin/method/header (adjust for production). - let cors = Cors::default() - .allow_any_origin() - .allow_any_method() - .allow_any_header() - .max_age(3600); - + let cors = Cors::default().allow_any_origin().allow_any_method().allow_any_header().max_age(3600); let app_state_clone = app_state.clone(); + let mut app = App::new() .wrap(cors) .wrap(Logger::default()) .wrap(Logger::new("HTTP REQUEST: %a %{User-Agent}i")) .app_data(web::Data::from(app_state_clone)); - // Register all route handlers / services. app = app .service(upload_file) .service(index) @@ -299,7 +225,7 @@ async fn main() -> std::io::Result<()> { app }) - .workers(worker_count) // Enable multithreaded handling + .workers(worker_count) .bind((config.server.host.clone(), config.server.port))? .run() .await diff --git a/src/package_manager/mod.rs b/src/package_manager/mod.rs index 3db92331..a97b9a5b 100644 --- a/src/package_manager/mod.rs +++ b/src/package_manager/mod.rs @@ -48,16 +48,15 @@ pub struct PackageManager { impl PackageManager { pub fn new(mode: InstallMode, tenant: Option) -> Result { - info!("Initializing PackageManager with mode: {:?}", mode); let os_type = Self::detect_os(); - debug!("Detected OS type: {:?}", os_type); let base_path = if mode == InstallMode::Container { PathBuf::from("/opt/gbo") } else { - PathBuf::from(".") + PathBuf::from("./botserver-stack") }; + let tenant = tenant.unwrap_or_else(|| "default".to_string()); - trace!("Using tenant: {}, base_path: {:?}", tenant, base_path); + let mut pm = PackageManager { mode, os_type, @@ -65,11 +64,9 @@ impl PackageManager { tenant, components: HashMap::new(), }; + pm.register_components(); - info!( - "PackageManager initialized with {} components", - pm.components.len() - ); + info!("PackageManager initialized with {} components in {:?} mode for tenant {}", pm.components.len(), pm.mode, pm.tenant); Ok(pm) } @@ -86,7 +83,6 @@ impl PackageManager { } fn register_components(&mut self) { - trace!("Registering all components"); self.register_drive(); self.register_cache(); self.register_tables(); @@ -107,7 +103,6 @@ impl PackageManager { self.register_system(); self.register_vector_db(); self.register_host(); - debug!("Component registration complete"); } fn register_drive(&mut self) { @@ -122,12 +117,21 @@ impl PackageManager { download_url: Some("https://dl.min.io/server/minio/release/linux-amd64/minio".to_string()), binary_name: Some("minio".to_string()), pre_install_cmds_linux: vec![], - post_install_cmds_linux: vec!["wget https://dl.min.io/client/mc/release/linux-amd64/mc -O {{BIN_PATH}}/mc".to_string(), "chmod +x {{BIN_PATH}}/mc".to_string()], + post_install_cmds_linux: vec![ + "wget https://dl.min.io/client/mc/release/linux-amd64/mc -O {{BIN_PATH}}/mc".to_string(), + "chmod +x {{BIN_PATH}}/mc".to_string() + ], pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec!["wget https://dl.min.io/client/mc/release/darwin-amd64/mc -O {{BIN_PATH}}/mc".to_string(), "chmod +x {{BIN_PATH}}/mc".to_string()], + post_install_cmds_macos: vec![ + "wget https://dl.min.io/client/mc/release/darwin-amd64/mc -O {{BIN_PATH}}/mc".to_string(), + "chmod +x {{BIN_PATH}}/mc".to_string() + ], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], - env_vars: HashMap::from([("MINIO_ROOT_USER".to_string(), "${PARAM_DRIVE_USER}".to_string()), ("MINIO_ROOT_PASSWORD".to_string(), "${PARAM_DRIVE_PASSWORD}".to_string())]), + env_vars: HashMap::from([ + ("MINIO_ROOT_USER".to_string(), "minioadmin".to_string()), + ("MINIO_ROOT_PASSWORD".to_string(), "minioadmin".to_string()) + ]), exec_cmd: "{{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001".to_string(), }); } @@ -143,7 +147,11 @@ impl PackageManager { windows_packages: vec![], download_url: None, binary_name: Some("valkey-server".to_string()), - pre_install_cmds_linux: vec!["curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/valkey.gpg".to_string(), "echo 'deb [signed-by=/usr/share/keyrings/valkey.gpg] https://packages.redis.io/deb $(lsb_release -cs) main' | tee /etc/apt/sources.list.d/valkey.list".to_string(), "apt-get update && apt-get install -y valkey".to_string()], + pre_install_cmds_linux: vec![ + "curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/valkey.gpg".to_string(), + "echo 'deb [signed-by=/usr/share/keyrings/valkey.gpg] https://packages.redis.io/deb $(lsb_release -cs) main' | tee /etc/apt/sources.list.d/valkey.list".to_string(), + "apt-get update && apt-get install -y valkey".to_string() + ], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], @@ -165,14 +173,26 @@ impl PackageManager { windows_packages: vec![], download_url: None, binary_name: Some("postgres".to_string()), - pre_install_cmds_linux: vec!["/usr/share/postgresql-common/pgdg/apt.postgresql.org.sh".to_string(), "apt-get update && apt-get install -y postgresql-16".to_string()], - post_install_cmds_linux: vec!["until sudo -u postgres psql -p ${PARAM_TABLES_PORT} -c '\\q' 2>/dev/null; do sleep 3; done".to_string(), "sudo -u postgres psql -p ${PARAM_TABLES_PORT} -c \"CREATE USER ${PARAM_TENANT} WITH PASSWORD '${PARAM_TABLES_PASSWORD}'\"".to_string(), "sudo -u postgres psql -p ${PARAM_TABLES_PORT} -c \"CREATE DATABASE ${PARAM_TENANT}_db OWNER ${PARAM_TENANT}\"".to_string(), "sudo -u postgres psql -p ${PARAM_TABLES_PORT} -c \"GRANT ALL PRIVILEGES ON DATABASE ${PARAM_TENANT}_db TO ${PARAM_TENANT}\"".to_string()], + pre_install_cmds_linux: vec![ + "/usr/share/postgresql-common/pgdg/apt.postgresql.org.sh".to_string(), + "apt-get update && apt-get install -y postgresql-16".to_string() + ], + post_install_cmds_linux: vec![ + "sudo -u postgres psql -p 5432 -c \"CREATE USER default WITH PASSWORD 'defaultpass'\" || true".to_string(), + "sudo -u postgres psql -p 5432 -c \"CREATE DATABASE default_db OWNER default\" || true".to_string(), + "sudo -u postgres psql -p 5432 -c \"GRANT ALL PRIVILEGES ON DATABASE default_db TO default\" || true".to_string() + ], pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec!["initdb -D {{DATA_PATH}}/pgdata".to_string(), "sleep 5".to_string(), "psql -p ${PARAM_TABLES_PORT} -d postgres -c \"CREATE USER ${PARAM_TENANT} WITH PASSWORD '${PARAM_TABLES_PASSWORD}'\"".to_string(), "psql -p ${PARAM_TABLES_PORT} -d postgres -c \"CREATE DATABASE ${PARAM_TENANT}_db OWNER ${PARAM_TENANT}\"".to_string()], + post_install_cmds_macos: vec![ + "initdb -D {{DATA_PATH}}/pgdata".to_string(), + "sleep 5".to_string(), + "psql -p 5432 -d postgres -c \"CREATE USER default WITH PASSWORD 'defaultpass'\" || true".to_string(), + "psql -p 5432 -d postgres -c \"CREATE DATABASE default_db OWNER default\" || true".to_string() + ], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), - exec_cmd: "postgres -D {{DATA_PATH}}/pgdata -p ${PARAM_TABLES_PORT}".to_string(), + exec_cmd: "postgres -D {{DATA_PATH}}/pgdata -p 5432".to_string(), }); } @@ -188,9 +208,15 @@ impl PackageManager { download_url: Some("https://github.com/ggml-org/llama.cpp/releases/download/b6148/llama-b6148-bin-ubuntu-x64.zip".to_string()), binary_name: Some("llama-server".to_string()), pre_install_cmds_linux: vec![], - post_install_cmds_linux: vec!["wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), "wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string()], + post_install_cmds_linux: vec![ + "wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), + "wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() + ], pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec!["wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), "wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string()], + post_install_cmds_macos: vec![ + "wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf -P {{DATA_PATH}}".to_string(), + "wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf -P {{DATA_PATH}}".to_string() + ], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), @@ -209,8 +235,10 @@ impl PackageManager { windows_packages: vec![], download_url: Some("https://github.com/stalwartlabs/stalwart/releases/download/v0.13.1/stalwart-x86_64-unknown-linux-gnu.tar.gz".to_string()), binary_name: Some("stalwart".to_string()), - pre_install_cmds_linux: vec!["echo 'nameserver ${PARAM_DNS_INTERNAL_IP}' > /etc/resolv.conf".to_string()], - post_install_cmds_linux: vec!["setcap 'cap_net_bind_service=+ep' {{BIN_PATH}}/stalwart".to_string()], + pre_install_cmds_linux: vec![], + post_install_cmds_linux: vec![ + "setcap 'cap_net_bind_service=+ep' {{BIN_PATH}}/stalwart".to_string() + ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], @@ -232,12 +260,16 @@ impl PackageManager { download_url: Some("https://github.com/caddyserver/caddy/releases/download/v2.10.0-beta.3/caddy_2.10.0-beta.3_linux_amd64.tar.gz".to_string()), binary_name: Some("caddy".to_string()), pre_install_cmds_linux: vec![], - post_install_cmds_linux: vec!["setcap 'cap_net_bind_service=+ep' {{BIN_PATH}}/caddy".to_string()], + post_install_cmds_linux: vec![ + "setcap 'cap_net_bind_service=+ep' {{BIN_PATH}}/caddy".to_string() + ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], - env_vars: HashMap::from([("XDG_DATA_HOME".to_string(), "{{DATA_PATH}}".to_string())]), + env_vars: HashMap::from([ + ("XDG_DATA_HOME".to_string(), "{{DATA_PATH}}".to_string()) + ]), exec_cmd: "{{BIN_PATH}}/caddy run --config {{CONF_PATH}}/Caddyfile".to_string(), }); } @@ -254,7 +286,9 @@ impl PackageManager { download_url: Some("https://github.com/zitadel/zitadel/releases/download/v2.71.2/zitadel-linux-amd64.tar.gz".to_string()), binary_name: Some("zitadel".to_string()), pre_install_cmds_linux: vec![], - post_install_cmds_linux: vec!["setcap 'cap_net_bind_service=+ep' {{BIN_PATH}}/zitadel".to_string()], + post_install_cmds_linux: vec![ + "setcap 'cap_net_bind_service=+ep' {{BIN_PATH}}/zitadel".to_string() + ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], @@ -281,7 +315,10 @@ impl PackageManager { post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], - env_vars: HashMap::from([("USER".to_string(), "alm".to_string()), ("HOME".to_string(), "{{DATA_PATH}}".to_string())]), + env_vars: HashMap::from([ + ("USER".to_string(), "alm".to_string()), + ("HOME".to_string(), "{{DATA_PATH}}".to_string()) + ]), exec_cmd: "{{BIN_PATH}}/forgejo web --work-path {{DATA_PATH}}".to_string(), }); } @@ -292,18 +329,25 @@ impl PackageManager { required: false, ports: vec![], dependencies: vec!["alm".to_string()], - linux_packages: vec!["wget".to_string(), "git".to_string(), "curl".to_string(), "gnupg".to_string(), "ca-certificates".to_string(), "build-essential".to_string(), "cmake".to_string(), "pkg-config".to_string(), "libjpeg-dev".to_string(), "libtiff-dev".to_string(), "libpng-dev".to_string(), "libavcodec-dev".to_string(), "libavformat-dev".to_string(), "libswscale-dev".to_string(), "libv4l-dev".to_string(), "libatlas-base-dev".to_string(), "gfortran".to_string(), "python3-dev".to_string(), "cpulimit".to_string(), "expect".to_string(), "libxtst-dev".to_string(), "libcairo2-dev".to_string(), "libpango1.0-dev".to_string(), "libgif-dev".to_string(), "librsvg2-dev".to_string(), "xvfb".to_string(), "libnss3".to_string(), "libatk1.0-0".to_string(), "libatk-bridge2.0-0".to_string(), "libcups2".to_string(), "libdrm2".to_string(), "libxkbcommon0".to_string(), "libxcomposite1".to_string(), "libxdamage1".to_string(), "libxfixes3".to_string(), "libxrandr2".to_string(), "libgbm1".to_string(), "libasound2".to_string(), "libpangocairo-1.0-0".to_string(), "libssl-dev".to_string(), "lxd-client".to_string()], + linux_packages: vec!["wget".to_string(), "git".to_string(), "curl".to_string(), "gnupg".to_string(), "ca-certificates".to_string(), "build-essential".to_string()], macos_packages: vec!["git".to_string(), "node".to_string()], windows_packages: vec![], download_url: Some("https://code.forgejo.org/forgejo/runner/releases/download/v6.3.1/forgejo-runner-6.3.1-linux-amd64".to_string()), binary_name: Some("forgejo-runner".to_string()), - pre_install_cmds_linux: vec!["curl -fsSL https://deb.nodesource.com/setup_22.x | bash -".to_string(), "apt-get update && apt-get install -y nodejs".to_string(), "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain 1.85.1 -y".to_string()], - post_install_cmds_linux: vec!["npm install -g pnpm@latest".to_string(), "source ~/.cargo/env && rustc --version".to_string(), "{{BIN_PATH}}/forgejo-runner register --no-interactive --name CI --instance ${PARAM_ALM_CI_INSTANCE} --token ${PARAM_ALM_CI_TOKEN} --labels gbo".to_string()], + pre_install_cmds_linux: vec![ + "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -".to_string(), + "apt-get update && apt-get install -y nodejs".to_string() + ], + post_install_cmds_linux: vec![ + "npm install -g pnpm@latest".to_string() + ], pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec!["npm install -g pnpm@latest".to_string(), "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain 1.85.1 -y".to_string(), "source ~/.cargo/env && rustc --version".to_string()], + post_install_cmds_macos: vec![ + "npm install -g pnpm@latest".to_string() + ], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], - env_vars: HashMap::from([("OPENCV4NODEJS_DISABLE_AUTOBUILD".to_string(), "1".to_string()), ("OPENCV_LIB_DIR".to_string(), "/usr/lib/x86_64-linux-gnu".to_string())]), + env_vars: HashMap::new(), exec_cmd: "{{BIN_PATH}}/forgejo-runner daemon --config {{CONF_PATH}}/config.yaml".to_string(), }); } @@ -319,8 +363,10 @@ impl PackageManager { windows_packages: vec![], download_url: Some("https://github.com/coredns/coredns/releases/download/v1.12.4/coredns_1.12.4_linux_amd64.tgz".to_string()), binary_name: Some("coredns".to_string()), - pre_install_cmds_linux: vec!["echo 'nameserver 8.8.8.8' > /etc/resolv.conf".to_string()], - post_install_cmds_linux: vec!["setcap cap_net_bind_service=+ep {{BIN_PATH}}/coredns".to_string()], + pre_install_cmds_linux: vec![], + post_install_cmds_linux: vec![ + "setcap cap_net_bind_service=+ep {{BIN_PATH}}/coredns".to_string() + ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], @@ -336,13 +382,13 @@ impl PackageManager { required: false, ports: vec![8080], dependencies: vec!["email".to_string()], - linux_packages: vec!["ca-certificates".to_string(), "apt-transport-https".to_string(), "lsb-release".to_string(), "gnupg".to_string(), "wget".to_string(), "php8.1".to_string(), "php8.1-fpm".to_string(), "php8.1-imap".to_string(), "php8.1-pgsql".to_string(), "php8.1-mbstring".to_string(), "php8.1-xml".to_string(), "php8.1-curl".to_string(), "php8.1-zip".to_string(), "php8.1-cli".to_string(), "php8.1-intl".to_string(), "php8.1-dom".to_string()], + linux_packages: vec!["ca-certificates".to_string(), "apt-transport-https".to_string(), "php8.1".to_string(), "php8.1-fpm".to_string()], macos_packages: vec!["php".to_string()], windows_packages: vec![], download_url: Some("https://github.com/roundcube/roundcubemail/releases/download/1.6.6/roundcubemail-1.6.6-complete.tar.gz".to_string()), binary_name: None, - pre_install_cmds_linux: vec!["wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg".to_string(), "echo 'deb https://packages.sury.org/php/ $(lsb_release -sc) main' > /etc/apt/sources.list.d/php.list".to_string(), "apt-get update && apt-get install -y php8.1 php8.1-fpm php8.1-imap php8.1-pgsql php8.1-mbstring php8.1-xml php8.1-curl php8.1-zip php8.1-cli php8.1-intl php8.1-dom".to_string()], - post_install_cmds_linux: vec!["systemctl restart php8.1-fpm".to_string()], + pre_install_cmds_linux: vec![], + post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], @@ -369,7 +415,7 @@ impl PackageManager { post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], - env_vars: HashMap::from([("TURN_PORT".to_string(), "${PARAM_MEETING_TURN_PORT}".to_string())]), + env_vars: HashMap::new(), exec_cmd: "{{BIN_PATH}}/livekit-server --config {{CONF_PATH}}/config.yaml".to_string(), }); } @@ -391,7 +437,7 @@ impl PackageManager { post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], - env_vars: HashMap::from([("PORT".to_string(), "${PARAM_TABLE_EDITOR_PORT}".to_string()), ("DATABASE_URL".to_string(), "postgres://${PARAM_TABLES_USER}:${PARAM_TABLES_PASSWORD}@${PARAM_TABLES_HOST}:${PARAM_TABLES_PORT}/${PARAM_TABLE_EDITOR_DATABASE}".to_string())]), + env_vars: HashMap::new(), exec_cmd: "{{BIN_PATH}}/nocodb".to_string(), }); } @@ -407,7 +453,7 @@ impl PackageManager { windows_packages: vec![], download_url: None, binary_name: Some("coolwsd".to_string()), - pre_install_cmds_linux: vec!["wget https://collaboraoffice.com/downloads/gpg/collaboraonline-release-keyring.gpg -P /usr/share/keyrings".to_string(), "echo 'Types: deb\nURIs: https://www.collaboraoffice.com/repos/CollaboraOnline/24.04/customer-deb-${customer_hash}\nSuites: ./\nSigned-By: /usr/share/keyrings/collaboraonline-release-keyring.gpg' > /etc/apt/sources.list.d/collaboraonline.sources".to_string(), "apt-get update && apt-get install -y coolwsd".to_string()], + pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], @@ -424,13 +470,13 @@ impl PackageManager { required: false, ports: vec![3389], dependencies: vec![], - linux_packages: vec!["xvfb".to_string(), "xrdp".to_string(), "xfce4".to_string(), "xfce4-goodies".to_string(), "curl".to_string(), "apt-transport-https".to_string(), "gnupg".to_string(), "gnome-tweaks".to_string()], + linux_packages: vec!["xvfb".to_string(), "xrdp".to_string(), "xfce4".to_string()], macos_packages: vec![], windows_packages: vec![], download_url: None, binary_name: None, - pre_install_cmds_linux: vec!["curl -s https://brave-browser-apt-release.s3.brave.com/brave-core.asc | gpg --dearmor > /usr/share/keyrings/brave-browser-archive-keyring.gpg".to_string(), "echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/brave-browser-archive-keyring.gpg] https://brave-browser-apt-release.s3.brave.com/ stable main' > /etc/apt/sources.list.d/brave-browser-release.list".to_string(), "apt-get update && apt-get install -y brave-browser".to_string()], - post_install_cmds_linux: vec!["echo 'exec startxfce4' > /root/.xsession".to_string(), "chmod +x /root/.xsession".to_string(), "echo 'GTK_IM_MODULE=cedilla\nQT_IM_MODULE=cedilla' >> /etc/environment".to_string(), "systemctl restart xrdp".to_string(), "systemctl enable xrdp".to_string()], + pre_install_cmds_linux: vec![], + post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], @@ -441,28 +487,25 @@ impl PackageManager { } fn register_devtools(&mut self) { - self.components.insert( - "devtools".to_string(), - ComponentConfig { - name: "devtools".to_string(), - required: false, - ports: vec![], - dependencies: vec![], - linux_packages: vec!["xclip".to_string(), "git".to_string(), "curl".to_string()], - macos_packages: vec!["git".to_string()], - windows_packages: vec![], - download_url: None, - binary_name: None, - pre_install_cmds_linux: vec![], - post_install_cmds_linux: vec![], - pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec![], - pre_install_cmds_windows: vec![], - post_install_cmds_windows: vec![], - env_vars: HashMap::new(), - exec_cmd: "".to_string(), - }, - ); + self.components.insert("devtools".to_string(), ComponentConfig { + name: "devtools".to_string(), + required: false, + ports: vec![], + dependencies: vec![], + linux_packages: vec!["xclip".to_string(), "git".to_string(), "curl".to_string()], + macos_packages: vec!["git".to_string()], + windows_packages: vec![], + download_url: None, + binary_name: None, + pre_install_cmds_linux: vec![], + post_install_cmds_linux: vec![], + pre_install_cmds_macos: vec![], + post_install_cmds_macos: vec![], + pre_install_cmds_windows: vec![], + post_install_cmds_windows: vec![], + env_vars: HashMap::new(), + exec_cmd: "".to_string(), + }); } fn register_bot(&mut self) { @@ -471,19 +514,24 @@ impl PackageManager { required: false, ports: vec![3000], dependencies: vec![], - linux_packages: vec!["curl".to_string(), "gnupg".to_string(), "ca-certificates".to_string(), "git".to_string(), "build-essential".to_string(), "cmake".to_string(), "pkg-config".to_string(), "libjpeg-dev".to_string(), "libtiff-dev".to_string(), "libpng-dev".to_string(), "libavcodec-dev".to_string(), "libavformat-dev".to_string(), "libswscale-dev".to_string(), "libv4l-dev".to_string(), "libatlas-base-dev".to_string(), "gfortran".to_string(), "python3-dev".to_string(), "cpulimit".to_string(), "expect".to_string(), "libxtst-dev".to_string(), "libcairo2-dev".to_string(), "libpango1.0-dev".to_string(), "libgif-dev".to_string(), "librsvg2-dev".to_string(), "xvfb".to_string(), "libnss3".to_string(), "libatk1.0-0".to_string(), "libatk-bridge2.0-0".to_string(), "libcups2".to_string(), "libdrm2".to_string(), "libxkbcommon0".to_string(), "libxcomposite1".to_string(), "libxdamage1".to_string(), "libxfixes3".to_string(), "libxrandr2".to_string(), "libgbm1".to_string(), "libasound2".to_string(), "libpangocairo-1.0-0".to_string(), "libgbm-dev".to_string()], + linux_packages: vec!["curl".to_string(), "gnupg".to_string(), "ca-certificates".to_string(), "git".to_string()], macos_packages: vec!["node".to_string()], windows_packages: vec![], download_url: None, binary_name: None, - pre_install_cmds_linux: vec!["curl -fsSL https://deb.nodesource.com/setup_22.x | bash -".to_string(), "apt-get update && apt-get install -y nodejs".to_string(), "wget https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_128.0.6613.119-1_amd64.deb".to_string(), "dpkg -i google-chrome-stable_128.0.6613.119-1_amd64.deb || apt-get install -f -y".to_string()], - post_install_cmds_linux: vec!["cd {{DATA_PATH}} && git clone https://alm.pragmatismo.com.br/generalbots/botserver.git".to_string(), "cd {{DATA_PATH}}/botserver && npm install && ./node_modules/.bin/tsc".to_string(), "cd {{DATA_PATH}}/botserver/packages/default.gbui && npm install && npm run build".to_string()], + pre_install_cmds_linux: vec![ + "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -".to_string(), + "apt-get update && apt-get install -y nodejs".to_string() + ], + post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec!["cd {{DATA_PATH}} && git clone https://alm.pragmatismo.com.br/generalbots/botserver.git".to_string(), "cd {{DATA_PATH}}/botserver && npm install && ./node_modules/.bin/tsc".to_string(), "cd {{DATA_PATH}}/botserver/packages/default.gbui && npm install && npm run build".to_string()], + post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], - env_vars: HashMap::from([("DISPLAY".to_string(), ":99".to_string()), ("OPENCV4NODEJS_DISABLE_AUTOBUILD".to_string(), "1".to_string()), ("OPENCV_LIB_DIR".to_string(), "/usr/lib/x86_64-linux-gnu".to_string())]), - exec_cmd: "Xvfb :99 -screen 0 1920x1080x24 & cd {{DATA_PATH}}/botserver && node ./dist/packages/core.gbapp/index.js".to_string(), + env_vars: HashMap::from([ + ("DISPLAY".to_string(), ":99".to_string()) + ]), + exec_cmd: "".to_string(), }); } @@ -493,19 +541,19 @@ impl PackageManager { required: false, ports: vec![8000], dependencies: vec![], - linux_packages: vec!["wget".to_string(), "curl".to_string(), "unzip".to_string(), "git".to_string(), "build-essential".to_string(), "pkg-config".to_string(), "libssl-dev".to_string(), "gcc-multilib".to_string(), "g++-multilib".to_string(), "clang".to_string(), "lld".to_string(), "binutils-dev".to_string(), "libudev-dev".to_string(), "libdbus-1-dev".to_string(), "libpq-dev".to_string()], + linux_packages: vec!["wget".to_string(), "curl".to_string(), "unzip".to_string(), "git".to_string()], macos_packages: vec![], windows_packages: vec![], - download_url: Some("https://github.com/ggml-org/llama.cpp/releases/download/b6148/llama-b6148-bin-ubuntu-x64.zip".to_string()), - binary_name: Some("llama-server".to_string()), - pre_install_cmds_linux: vec!["curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y".to_string(), "curl -fsSLo /usr/share/keyrings/brave-browser-beta-archive-keyring.gpg https://brave-browser-apt-beta.s3.brave.com/brave-browser-beta-archive-keyring.gpg".to_string(), "curl -fsSLo /etc/apt/sources.list.d/brave-browser-beta.sources https://brave-browser-apt-beta.s3.brave.com/brave-browser.sources".to_string(), "apt-get update && apt-get install -y brave-browser-beta".to_string()], - post_install_cmds_linux: vec!["source ~/.cargo/env".to_string(), "git clone https://alm.pragmatismo.com.br/generalbots/gbserver {{DATA_PATH}}/gbserver".to_string(), "cd {{DATA_PATH}}/gbserver && cargo build --release".to_string(), "cp {{DATA_PATH}}/gbserver/target/release/gbserver {{BIN_PATH}}/gbserver".to_string()], - pre_install_cmds_macos: vec!["curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y".to_string()], - post_install_cmds_macos: vec!["source ~/.cargo/env".to_string(), "git clone https://alm.pragmatismo.com.br/generalbots/gbserver {{DATA_PATH}}/gbserver".to_string(), "cd {{DATA_PATH}}/gbserver && cargo build --release".to_string(), "cp {{DATA_PATH}}/gbserver/target/release/gbserver {{BIN_PATH}}/gbserver".to_string()], + download_url: None, + binary_name: None, + pre_install_cmds_linux: vec![], + post_install_cmds_linux: vec![], + pre_install_cmds_macos: vec![], + post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), - exec_cmd: "{{BIN_PATH}}/gbserver".to_string(), + exec_cmd: "".to_string(), }); } @@ -532,235 +580,184 @@ impl PackageManager { } fn register_host(&mut self) { - self.components.insert( - "host".to_string(), - ComponentConfig { - name: "host".to_string(), - required: false, - ports: vec![], - dependencies: vec![], - linux_packages: vec!["sshfs".to_string(), "bridge-utils".to_string()], - macos_packages: vec![], - windows_packages: vec![], - download_url: None, - binary_name: None, - pre_install_cmds_linux: vec![ - "echo 'net.ipv4.ip_forward=1' | tee -a /etc/sysctl.conf".to_string(), - "sysctl -p".to_string(), - ], - post_install_cmds_linux: vec![ - "lxd init --auto".to_string(), - "lxc storage create default dir".to_string(), - "lxc profile device add default root disk path=/ pool=default".to_string(), - ], - pre_install_cmds_macos: vec![], - post_install_cmds_macos: vec![], - pre_install_cmds_windows: vec![], - post_install_cmds_windows: vec![], - env_vars: HashMap::new(), - exec_cmd: "".to_string(), - }, - ); + self.components.insert("host".to_string(), ComponentConfig { + name: "host".to_string(), + required: false, + ports: vec![], + dependencies: vec![], + linux_packages: vec!["sshfs".to_string(), "bridge-utils".to_string()], + macos_packages: vec![], + windows_packages: vec![], + download_url: None, + binary_name: None, + pre_install_cmds_linux: vec![ + "echo 'net.ipv4.ip_forward=1' | tee -a /etc/sysctl.conf".to_string(), + "sysctl -p".to_string(), + ], + post_install_cmds_linux: vec![ + "lxd init --auto".to_string(), + "lxc storage create default dir".to_string(), + "lxc profile device add default root disk path=/ pool=default".to_string(), + ], + pre_install_cmds_macos: vec![], + post_install_cmds_macos: vec![], + pre_install_cmds_windows: vec![], + post_install_cmds_windows: vec![], + env_vars: HashMap::new(), + exec_cmd: "".to_string(), + }); } pub fn install(&self, component_name: &str) -> Result<()> { - let component = self - .components - .get(component_name) - .context(format!("Component '{}' not found", component_name))?; - info!( - "Starting installation process for component: {}", - component_name - ); + let component = self.components.get(component_name).context(format!("Component '{}' not found", component_name))?; + + info!("Starting installation of component '{}' in {:?} mode", component_name, self.mode); + for dep in &component.dependencies { if !self.is_installed(dep) { - warn!("Dependency '{}' missing, installing now", dep); + warn!("Installing missing dependency: {}", dep); self.install(dep)?; } } + match self.mode { InstallMode::Local => self.install_local(component)?, InstallMode::Container => self.install_container(component)?, } - info!( - "Component '{}' installation completed successfully", - component_name - ); + + info!("Component '{}' installation completed successfully", component_name); Ok(()) } fn install_local(&self, component: &ComponentConfig) -> Result<()> { - trace!( - "Starting local installation for component: {}", - component.name - ); + info!("Installing component '{}' locally to {}", component.name, self.base_path.display()); + self.create_directories(&component.name)?; + let (pre_cmds, post_cmds) = match self.os_type { - OsType::Linux => ( - &component.pre_install_cmds_linux, - &component.post_install_cmds_linux, - ), - OsType::MacOS => ( - &component.pre_install_cmds_macos, - &component.post_install_cmds_macos, - ), - OsType::Windows => ( - &component.pre_install_cmds_windows, - &component.post_install_cmds_windows, - ), + OsType::Linux => (&component.pre_install_cmds_linux, &component.post_install_cmds_linux), + OsType::MacOS => (&component.pre_install_cmds_macos, &component.post_install_cmds_macos), + OsType::Windows => (&component.pre_install_cmds_windows, &component.post_install_cmds_windows), }; + self.run_commands(pre_cmds, "local", &component.name)?; self.install_system_packages(component)?; + if let Some(url) = &component.download_url { self.download_and_install(url, &component.name, component.binary_name.as_deref())?; } + self.run_commands(post_cmds, "local", &component.name)?; + if self.os_type == OsType::Linux && !component.exec_cmd.is_empty() { self.create_service_file(&component.name, &component.exec_cmd, &component.env_vars)?; } - debug!( - "Local installation completed for component: {}", - component.name - ); + + info!("Local installation of '{}' completed", component.name); Ok(()) } fn install_container(&self, component: &ComponentConfig) -> Result<()> { let container_name = format!("{}-{}", self.tenant, component.name); info!("Creating LXC container: {}", container_name); - let output = Command::new("lxc") - .args(&[ - "launch", - "images:debian/12", - &container_name, - "-c", - "security.privileged=true", - ]) - .output()?; + + let output = Command::new("lxc").args(&["launch", "images:debian/12", &container_name, "-c", "security.privileged=true"]).output()?; + if !output.status.success() { - return Err(anyhow::anyhow!( - "LXC container creation failed: {}", - String::from_utf8_lossy(&output.stderr) - )); + return Err(anyhow::anyhow!("LXC container creation failed: {}", String::from_utf8_lossy(&output.stderr))); } - trace!("Waiting for container to initialize"); + std::thread::sleep(std::time::Duration::from_secs(15)); + self.exec_in_container(&container_name, "mkdir -p /opt/gbo/{bin,data,conf,logs}")?; + let (pre_cmds, post_cmds) = match self.os_type { - OsType::Linux => ( - &component.pre_install_cmds_linux, - &component.post_install_cmds_linux, - ), - OsType::MacOS => ( - &component.pre_install_cmds_macos, - &component.post_install_cmds_macos, - ), - OsType::Windows => ( - &component.pre_install_cmds_windows, - &component.post_install_cmds_windows, - ), + OsType::Linux => (&component.pre_install_cmds_linux, &component.post_install_cmds_linux), + OsType::MacOS => (&component.pre_install_cmds_macos, &component.post_install_cmds_macos), + OsType::Windows => (&component.pre_install_cmds_windows, &component.post_install_cmds_windows), }; + self.run_commands(pre_cmds, &container_name, &component.name)?; + let packages = match self.os_type { OsType::Linux => &component.linux_packages, OsType::MacOS => &component.macos_packages, OsType::Windows => &component.windows_packages, }; + if !packages.is_empty() { let pkg_list = packages.join(" "); - debug!("Installing packages in container: {}", pkg_list); - self.exec_in_container( - &container_name, - &format!("apt-get update && apt-get install -y {}", pkg_list), - )?; + self.exec_in_container(&container_name, &format!("apt-get update && apt-get install -y {}", pkg_list))?; } + if let Some(url) = &component.download_url { - self.download_in_container( - &container_name, - url, - &component.name, - component.binary_name.as_deref(), - )?; + self.download_in_container(&container_name, url, &component.name, component.binary_name.as_deref())?; } + self.run_commands(post_cmds, &container_name, &component.name)?; - self.exec_in_container( - &container_name, - "useradd --system --no-create-home --shell /bin/false gbuser", - )?; + + self.exec_in_container(&container_name, "useradd --system --no-create-home --shell /bin/false gbuser")?; self.mount_container_directories(&container_name, &component.name)?; + if !component.exec_cmd.is_empty() { - self.create_container_service( - &container_name, - &component.name, - &component.exec_cmd, - &component.env_vars, - )?; + self.create_container_service(&container_name, &component.name, &component.exec_cmd, &component.env_vars)?; } + self.setup_port_forwarding(&container_name, &component.ports)?; - info!("Container installation complete for: {}", container_name); + + info!("Container installation of '{}' completed in {}", component.name, container_name); Ok(()) } pub fn remove(&self, component_name: &str) -> Result<()> { - let component = self - .components - .get(component_name) - .context(format!("Component '{}' not found", component_name))?; - info!( - "Beginning removal process for component: {}", - component_name - ); + let component = self.components.get(component_name).context(format!("Component '{}' not found", component_name))?; + + info!("Removing component: {}", component_name); + match self.mode { InstallMode::Local => self.remove_local(component)?, InstallMode::Container => self.remove_container(component)?, } + info!("Component '{}' removed successfully", component_name); Ok(()) } fn remove_local(&self, component: &ComponentConfig) -> Result<()> { - trace!("Removing local component: {}", component.name); if self.os_type == OsType::Linux { - let _ = Command::new("systemctl") - .args(&["stop", &format!("{}.service", component.name)]) - .output(); - let _ = Command::new("systemctl") - .args(&["disable", &format!("{}.service", component.name)]) - .output(); + let _ = Command::new("systemctl").args(&["stop", &format!("{}.service", component.name)]).output(); + let _ = Command::new("systemctl").args(&["disable", &format!("{}.service", component.name)]).output(); let service_path = format!("/etc/systemd/system/{}.service", component.name); let _ = std::fs::remove_file(service_path); let _ = Command::new("systemctl").args(&["daemon-reload"]).output(); } + let bin_path = self.base_path.join("bin").join(&component.name); let data_path = self.base_path.join("data").join(&component.name); let conf_path = self.base_path.join("conf").join(&component.name); let logs_path = self.base_path.join("logs").join(&component.name); + let _ = std::fs::remove_dir_all(bin_path); let _ = std::fs::remove_dir_all(data_path); let _ = std::fs::remove_dir_all(conf_path); let _ = std::fs::remove_dir_all(logs_path); - debug!("Local component directories removed: {}", component.name); + Ok(()) } fn remove_container(&self, component: &ComponentConfig) -> Result<()> { let container_name = format!("{}-{}", self.tenant, component.name); - info!("Removing LXC container: {}", container_name); - let _ = Command::new("lxc") - .args(&["stop", &container_name]) - .output(); - let output = Command::new("lxc") - .args(&["delete", &container_name]) - .output()?; + + let _ = Command::new("lxc").args(&["stop", &container_name]).output(); + let output = Command::new("lxc").args(&["delete", &container_name]).output()?; + if !output.status.success() { - warn!( - "Container deletion encountered issues: {}", - String::from_utf8_lossy(&output.stderr) - ); + warn!("Container deletion had issues: {}", String::from_utf8_lossy(&output.stderr)); } + let host_base = format!("/opt/gbo/tenants/{}/{}", self.tenant, component.name); let _ = std::fs::remove_dir_all(host_base); - trace!("Container and host directories cleaned up"); + Ok(()) } @@ -776,24 +773,18 @@ impl PackageManager { } InstallMode::Container => { let container_name = format!("{}-{}", self.tenant, component_name); - Command::new("lxc") - .args(&["list", &container_name, "--format=json"]) - .output() - .map(|o| o.status.success()) - .unwrap_or(false) + Command::new("lxc").args(&["list", &container_name, "--format=json"]).output().map(|o| o.status.success()).unwrap_or(false) } } } fn create_directories(&self, component: &str) -> Result<()> { - trace!("Creating directory structure for component: {}", component); let dirs = ["bin", "data", "conf", "logs"]; for dir in &dirs { let path = self.base_path.join(dir).join(component); - std::fs::create_dir_all(&path) - .context(format!("Failed to create directory: {:?}", path))?; + std::fs::create_dir_all(&path).context(format!("Failed to create directory: {:?}", path))?; + trace!("Created directory: {:?}", path); } - debug!("Directories created successfully for: {}", component); Ok(()) } @@ -803,75 +794,52 @@ impl PackageManager { OsType::MacOS => &component.macos_packages, OsType::Windows => &component.windows_packages, }; + if packages.is_empty() { return Ok(()); } - info!( - "Installing system packages for component: {}", - component.name - ); + + info!("Installing {} system packages for component '{}'", packages.len(), component.name); + match self.os_type { OsType::Linux => { - let output = Command::new("apt-get") - .args(&["install", "-y"]) - .args(packages) - .output()?; + let output = Command::new("apt-get").args(&["install", "-y"]).args(packages).output()?; if !output.status.success() { - warn!("Some package installations may have encountered issues"); + warn!("Some packages may have failed to install"); } } OsType::MacOS => { - let output = Command::new("brew") - .args(&["install"]) - .args(packages) - .output()?; + let output = Command::new("brew").args(&["install"]).args(packages).output()?; if !output.status.success() { - warn!("Homebrew package installation had warnings"); + warn!("Homebrew installation had warnings"); } } OsType::Windows => { - warn!("Windows package installation not implemented yet"); + warn!("Windows package installation not implemented"); } } - debug!("System packages installed for: {}", component.name); + Ok(()) } - fn download_and_install( - &self, - url: &str, - component: &str, - binary_name: Option<&str>, - ) -> Result<()> { - info!("Downloading component binary from: {}", url); + fn download_and_install(&self, url: &str, component: &str, binary_name: Option<&str>) -> Result<()> { let bin_path = self.base_path.join("bin").join(component); let temp_file = bin_path.join("download.tmp"); - let output = Command::new("wget") - .args(&["-O", temp_file.to_str().unwrap(), url]) - .output()?; + + info!("Downloading from: {}", url); + let output = Command::new("wget").args(&["-O", temp_file.to_str().unwrap(), url]).output()?; + if !output.status.success() { return Err(anyhow::anyhow!("Download failed from URL: {}", url)); } + if url.ends_with(".tar.gz") || url.ends_with(".tgz") { - trace!("Extracting tar.gz archive"); - Command::new("tar") - .args(&[ - "-xzf", - temp_file.to_str().unwrap(), - "-C", - bin_path.to_str().unwrap(), - ]) - .output()?; + trace!("Extracting tar.gz archive to {:?}", bin_path); + Command::new("tar").args(&["-xzf", temp_file.to_str().unwrap(), "-C", bin_path.to_str().unwrap()]).output()?; std::fs::remove_file(&temp_file)?; } else if url.ends_with(".zip") { - trace!("Extracting zip archive"); - Command::new("unzip") - .args(&[ - temp_file.to_str().unwrap(), - "-d", - bin_path.to_str().unwrap(), - ]) - .output()?; + trace!("Extracting zip archive to {:?}", bin_path); + Command::new("unzip").args(&[temp_file.to_str().unwrap(), "-d", bin_path.to_str().unwrap()]).output()?; std::fs::remove_file(&temp_file)?; } else if let Some(name) = binary_name { let final_path = bin_path.join(name); @@ -883,116 +851,81 @@ impl PackageManager { perms.set_mode(0o755); std::fs::set_permissions(&final_path, perms)?; } - debug!("Binary installed as: {}", name); } + Ok(()) } - fn create_service_file( - &self, - component: &str, - exec_cmd: &str, - env_vars: &HashMap, - ) -> Result<()> { - trace!("Creating systemd service file for: {}", component); + fn create_service_file(&self, component: &str, exec_cmd: &str, env_vars: &HashMap) -> Result<()> { let service_path = format!("/etc/systemd/system/{}.service", component); - let bin_path = self - .base_path - .join("bin") - .join(component) - .to_string_lossy() - .to_string(); - let data_path = self - .base_path - .join("data") - .join(component) - .to_string_lossy() - .to_string(); - let conf_path = self - .base_path - .join("conf") - .join(component) - .to_string_lossy() - .to_string(); - let logs_path = self - .base_path - .join("logs") - .join(component) - .to_string_lossy() - .to_string(); + + let bin_path = self.base_path.join("bin").join(component).to_string_lossy().to_string(); + let data_path = self.base_path.join("data").join(component).to_string_lossy().to_string(); + let conf_path = self.base_path.join("conf").join(component).to_string_lossy().to_string(); + let logs_path = self.base_path.join("logs").join(component).to_string_lossy().to_string(); + let rendered_cmd = exec_cmd .replace("{{BIN_PATH}}", &bin_path) .replace("{{DATA_PATH}}", &data_path) .replace("{{CONF_PATH}}", &conf_path) .replace("{{LOGS_PATH}}", &logs_path); + let mut env_section = String::new(); for (key, value) in env_vars { let rendered_value = value.replace("{{DATA_PATH}}", &data_path); env_section.push_str(&format!("Environment=\"{}={}\"\n", key, rendered_value)); } + let service_content = format!("[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory={}\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", component, env_section, rendered_cmd, data_path); + std::fs::write(&service_path, service_content)?; - Command::new("systemctl") - .args(&["daemon-reload"]) - .output()?; - Command::new("systemctl") - .args(&["enable", &format!("{}.service", component)]) - .output()?; - info!("Service file created and enabled for: {}", component); + + Command::new("systemctl").args(&["daemon-reload"]).output()?; + Command::new("systemctl").args(&["enable", &format!("{}.service", component)]).output()?; + Command::new("systemctl").args(&["start", &format!("{}.service", component)]).output()?; + + info!("Created and started systemd service: {}.service", component); Ok(()) } fn run_commands(&self, commands: &[String], target: &str, component: &str) -> Result<()> { for cmd in commands { let bin_path = if target == "local" { - self.base_path - .join("bin") - .join(component) - .to_string_lossy() - .to_string() + self.base_path.join("bin").join(component).to_string_lossy().to_string() } else { "/opt/gbo/bin".to_string() }; + let data_path = if target == "local" { - self.base_path - .join("data") - .join(component) - .to_string_lossy() - .to_string() + self.base_path.join("data").join(component).to_string_lossy().to_string() } else { "/opt/gbo/data".to_string() }; + let conf_path = if target == "local" { - self.base_path - .join("conf") - .join(component) - .to_string_lossy() - .to_string() + self.base_path.join("conf").join(component).to_string_lossy().to_string() } else { "/opt/gbo/conf".to_string() }; + let logs_path = if target == "local" { - self.base_path - .join("logs") - .join(component) - .to_string_lossy() - .to_string() + self.base_path.join("logs").join(component).to_string_lossy().to_string() } else { "/opt/gbo/logs".to_string() }; + let rendered_cmd = cmd .replace("{{BIN_PATH}}", &bin_path) .replace("{{DATA_PATH}}", &data_path) .replace("{{CONF_PATH}}", &conf_path) .replace("{{LOGS_PATH}}", &logs_path); - info!("Executing command: {}", rendered_cmd); + + trace!("Executing command: {}", rendered_cmd); + if target == "local" { let output = Command::new("bash").args(&["-c", &rendered_cmd]).output()?; if !output.status.success() { - warn!( - "Command execution had non-zero exit: {}", - String::from_utf8_lossy(&output.stderr) - ); + warn!("Command had non-zero exit: {}", String::from_utf8_lossy(&output.stderr)); } } else { self.exec_in_container(target, &rendered_cmd)?; @@ -1002,132 +935,90 @@ impl PackageManager { } fn exec_in_container(&self, container: &str, command: &str) -> Result<()> { - trace!("Executing in container {}: {}", container, command); - let output = Command::new("lxc") - .args(&["exec", container, "--", "bash", "-c", command]) - .output()?; + debug!("Executing in container {}: {}", container, command); + let output = Command::new("lxc").args(&["exec", container, "--", "bash", "-c", command]).output()?; + if !output.status.success() { - warn!( - "Container command failed: {}", - String::from_utf8_lossy(&output.stderr) - ); + warn!("Container command failed: {}", String::from_utf8_lossy(&output.stderr)); } Ok(()) } - fn download_in_container( - &self, - container: &str, - url: &str, - _component: &str, - binary_name: Option<&str>, - ) -> Result<()> { - debug!("Downloading in container from URL: {}", url); + fn download_in_container(&self, container: &str, url: &str, _component: &str, binary_name: Option<&str>) -> Result<()> { let download_cmd = format!("wget -O /tmp/download.tmp {}", url); self.exec_in_container(container, &download_cmd)?; + if url.ends_with(".tar.gz") || url.ends_with(".tgz") { self.exec_in_container(container, "tar -xzf /tmp/download.tmp -C /opt/gbo/bin")?; } else if url.ends_with(".zip") { self.exec_in_container(container, "unzip /tmp/download.tmp -d /opt/gbo/bin")?; } else if let Some(name) = binary_name { - let mv_cmd = format!( - "mv /tmp/download.tmp /opt/gbo/bin/{} && chmod +x /opt/gbo/bin/{}", - name, name - ); + let mv_cmd = format!("mv /tmp/download.tmp /opt/gbo/bin/{} && chmod +x /opt/gbo/bin/{}", name, name); self.exec_in_container(container, &mv_cmd)?; } + self.exec_in_container(container, "rm -f /tmp/download.tmp")?; - trace!("Download and extraction complete in container"); Ok(()) } fn mount_container_directories(&self, container: &str, component: &str) -> Result<()> { - debug!("Mounting host directories into container: {}", container); let host_base = format!("/opt/gbo/tenants/{}/{}", self.tenant, component); + for dir in &["data", "conf", "logs"] { let host_path = format!("{}/{}", host_base, dir); std::fs::create_dir_all(&host_path)?; + let device_name = format!("{}{}", component, dir); let container_path = format!("/opt/gbo/{}", dir); - let _ = Command::new("lxc") - .args(&["config", "device", "remove", container, &device_name]) - .output(); - Command::new("lxc") - .args(&[ - "config", - "device", - "add", - container, - &device_name, - "disk", - &format!("source={}", host_path), - &format!("path={}", container_path), - ]) - .output()?; + + let _ = Command::new("lxc").args(&["config", "device", "remove", container, &device_name]).output(); + + Command::new("lxc").args(&["config", "device", "add", container, &device_name, "disk", &format!("source={}", host_path), &format!("path={}", container_path)]).output()?; + + trace!("Mounted {} to {} in container {}", host_path, container_path, container); } - trace!("Container directory mounts configured"); Ok(()) } - fn create_container_service( - &self, - container: &str, - component: &str, - exec_cmd: &str, - env_vars: &HashMap, - ) -> Result<()> { - info!("Creating service inside container: {}", container); + fn create_container_service(&self, container: &str, component: &str, exec_cmd: &str, env_vars: &HashMap) -> Result<()> { let rendered_cmd = exec_cmd .replace("{{BIN_PATH}}", "/opt/gbo/bin") .replace("{{DATA_PATH}}", "/opt/gbo/data") .replace("{{CONF_PATH}}", "/opt/gbo/conf") .replace("{{LOGS_PATH}}", "/opt/gbo/logs"); + let mut env_section = String::new(); for (key, value) in env_vars { let rendered_value = value.replace("{{DATA_PATH}}", "/opt/gbo/data"); env_section.push_str(&format!("Environment=\"{}={}\"\n", key, rendered_value)); } + let service_content = format!("[Unit]\nDescription={} Service\nAfter=network.target\n\n[Service]\nType=simple\n{}ExecStart={}\nWorkingDirectory=/opt/gbo/data\nRestart=always\nRestartSec=10\nUser=root\n\n[Install]\nWantedBy=multi-user.target\n", component, env_section, rendered_cmd); + let service_file = format!("/tmp/{}.service", component); std::fs::write(&service_file, &service_content)?; - Command::new("lxc") - .args(&[ - "file", - "push", - &service_file, - &format!("{}/etc/systemd/system/{}.service", container, component), - ]) - .output()?; + + Command::new("lxc").args(&["file", "push", &service_file, &format!("{}/etc/systemd/system/{}.service", container, component)]).output()?; + self.exec_in_container(container, "systemctl daemon-reload")?; self.exec_in_container(container, &format!("systemctl enable {}", component))?; self.exec_in_container(container, &format!("systemctl start {}", component))?; + std::fs::remove_file(&service_file)?; - debug!("Service created and started in container: {}", component); + + info!("Created and started service in container {}: {}", container, component); Ok(()) } fn setup_port_forwarding(&self, container: &str, ports: &[u16]) -> Result<()> { for port in ports { let device_name = format!("port-{}", port); - let _ = Command::new("lxc") - .args(&["config", "device", "remove", container, &device_name]) - .output(); - Command::new("lxc") - .args(&[ - "config", - "device", - "add", - container, - &device_name, - "proxy", - &format!("listen=tcp:0.0.0.0:{}", port), - &format!("connect=tcp:127.0.0.1:{}", port), - ]) - .output()?; - info!( - "Port forwarding configured: {} -> container {}", - port, container - ); + + let _ = Command::new("lxc").args(&["config", "device", "remove", container, &device_name]).output(); + + Command::new("lxc").args(&["config", "device", "add", container, &device_name, "proxy", &format!("listen=tcp:0.0.0.0:{}", port), &format!("connect=tcp:127.0.0.1:{}", port)]).output()?; + + trace!("Port forwarding configured: {} -> container {}", port, container); } Ok(()) } @@ -1140,102 +1031,64 @@ pub mod cli { pub fn run() -> Result<()> { env_logger::init(); let args: Vec = env::args().collect(); - trace!("CLI invoked with arguments: {:?}", args); + if args.len() < 2 { print_usage(); return Ok(()); } + let command = &args[1]; - debug!("Processing command: {}", command); + match command.as_str() { "install" => { if args.len() < 3 { - eprintln!( - "Usage: botserver install [--container] [--tenant ]" - ); + eprintln!("Usage: botserver install [--container] [--tenant ]"); return Ok(()); } + let component = &args[2]; - let mode = if args.contains(&"--container".to_string()) { - InstallMode::Container - } else { - InstallMode::Local - }; - let tenant = if let Some(idx) = args.iter().position(|a| a == "--tenant") { - args.get(idx + 1).cloned() - } else { - None - }; - info!("Installing component '{}' in {:?} mode", component, mode); + let mode = if args.contains(&"--container".to_string()) { InstallMode::Container } else { InstallMode::Local }; + let tenant = if let Some(idx) = args.iter().position(|a| a == "--tenant") { args.get(idx + 1).cloned() } else { None }; + let pm = PackageManager::new(mode, tenant)?; pm.install(component)?; println!("✓ Component '{}' installed successfully", component); } "remove" => { if args.len() < 3 { - eprintln!( - "Usage: botserver remove [--container] [--tenant ]" - ); + eprintln!("Usage: botserver remove [--container] [--tenant ]"); return Ok(()); } + let component = &args[2]; - let mode = if args.contains(&"--container".to_string()) { - InstallMode::Container - } else { - InstallMode::Local - }; - let tenant = if let Some(idx) = args.iter().position(|a| a == "--tenant") { - args.get(idx + 1).cloned() - } else { - None - }; - info!("Removing component '{}' from {:?} mode", component, mode); + let mode = if args.contains(&"--container".to_string()) { InstallMode::Container } else { InstallMode::Local }; + let tenant = if let Some(idx) = args.iter().position(|a| a == "--tenant") { args.get(idx + 1).cloned() } else { None }; + let pm = PackageManager::new(mode, tenant)?; pm.remove(component)?; println!("✓ Component '{}' removed successfully", component); } "list" => { - let mode = if args.contains(&"--container".to_string()) { - InstallMode::Container - } else { - InstallMode::Local - }; - let tenant = if let Some(idx) = args.iter().position(|a| a == "--tenant") { - args.get(idx + 1).cloned() - } else { - None - }; - debug!("Listing components for {:?} mode", mode); + let mode = if args.contains(&"--container".to_string()) { InstallMode::Container } else { InstallMode::Local }; + let tenant = if let Some(idx) = args.iter().position(|a| a == "--tenant") { args.get(idx + 1).cloned() } else { None }; + let pm = PackageManager::new(mode, tenant)?; println!("Available components:"); for component in pm.list() { - let status = if pm.is_installed(&component) { - "✓ installed" - } else { - " available" - }; - println!(" {} {}", status, component); + let status = if pm.is_installed(&component) { "✓ installed" } else { " available" }; + println!(" {} {}", status, component); } } "status" => { if args.len() < 3 { - eprintln!( - "Usage: botserver status [--container] [--tenant ]" - ); + eprintln!("Usage: botserver status [--container] [--tenant ]"); return Ok(()); } + let component = &args[2]; - let mode = if args.contains(&"--container".to_string()) { - InstallMode::Container - } else { - InstallMode::Local - }; - let tenant = if let Some(idx) = args.iter().position(|a| a == "--tenant") { - args.get(idx + 1).cloned() - } else { - None - }; - trace!("Checking status for component: {}", component); + let mode = if args.contains(&"--container".to_string()) { InstallMode::Container } else { InstallMode::Local }; + let tenant = if let Some(idx) = args.iter().position(|a| a == "--tenant") { args.get(idx + 1).cloned() } else { None }; + let pm = PackageManager::new(mode, tenant)?; if pm.is_installed(component) { println!("✓ Component '{}' is installed", component); @@ -1251,10 +1104,11 @@ pub mod cli { print_usage(); } } + Ok(()) } fn print_usage() { - println!("BotServer Package Manager\n\nUSAGE:\n botserver [options]\n\nCOMMANDS:\n install Install component\n remove Remove component\n list List all components\n status Check component status\n\nOPTIONS:\n --container Use container mode (LXC)\n --tenant Specify tenant (default: 'default')\n\nCOMPONENTS:\n Required: drive cache tables llm\n Optional: email proxy directory alm alm-ci dns webmail meeting table-editor doc-editor desktop devtools bot system vector-db host\n\nEXAMPLES:\n botserver install email\n botserver install email --container --tenant myorg\n botserver remove email\n botserver list"); + println!("BotServer Package Manager\n\nUSAGE:\n botserver [options]\n\nCOMMANDS:\n install Install component\n remove Remove component\n list List all components\n status Check component status\n\nOPTIONS:\n --container Use container mode (LXC)\n --tenant Specify tenant (default: 'default')\n\nCOMPONENTS:\n Required: drive cache tables llm\n Optional: email proxy directory alm alm-ci dns webmail meeting table-editor doc-editor desktop devtools bot system vector-db host\n\nEXAMPLES:\n botserver install email\n botserver install email --container --tenant myorg\n botserver remove email\n botserver list"); } }