botserver/src/core/bootstrap/mod.rs

use crate::config::AppConfig;
use crate::package_manager::setup::{DirectorySetup, EmailSetup};
use crate::package_manager::{InstallMode, PackageManager};
use crate::shared::utils::establish_pg_connection;
use anyhow::Result;
use aws_config::BehaviorVersion;
use aws_sdk_s3::Client;
use log::{error, info, trace, warn};
use rand::distr::Alphanumeric;
use rcgen::{
    BasicConstraints, CertificateParams, DistinguishedName, DnType, IsCa, Issuer, KeyPair,
};
use std::fs;
use std::io::{self, Write};
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::process::Command;
#[derive(Debug)]
pub struct ComponentInfo {
    pub name: &'static str,
}
#[derive(Debug)]
pub struct BootstrapManager {
    pub install_mode: InstallMode,
    pub tenant: Option<String>,
}
impl BootstrapManager {
    pub async fn new(mode: InstallMode, tenant: Option<String>) -> Self {
        trace!(
            "Initializing BootstrapManager with mode {:?} and tenant {:?}",
            mode,
            tenant
        );
        Self {
            install_mode: mode,
            tenant,
        }
    }
    pub fn start_all(&mut self) -> Result<()> {
        let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?;
        let components = vec![
            ComponentInfo { name: "vault" },
            ComponentInfo { name: "tables" },
            ComponentInfo { name: "cache" },
            ComponentInfo { name: "drive" },
            ComponentInfo { name: "llm" },
            ComponentInfo { name: "email" },
            ComponentInfo { name: "proxy" },
            ComponentInfo { name: "directory" },
            ComponentInfo { name: "alm" },
            ComponentInfo { name: "alm_ci" },
            ComponentInfo { name: "dns" },
            ComponentInfo { name: "meeting" },
            ComponentInfo {
                name: "remote_terminal",
            },
            ComponentInfo { name: "vector_db" },
            ComponentInfo { name: "host" },
        ];
        for component in components {
            if pm.is_installed(component.name) {
                match pm.start(component.name) {
                    Ok(_child) => {
                        trace!("Started component: {}", component.name);
                    }
                    Err(e) => {
                        warn!(
                            "Component {} might already be running: {}",
                            component.name, e
                        );
                    }
                }
            }
        }
        Ok(())
    }

    fn generate_secure_password(&self, length: usize) -> String {
        let mut rng = rand::rng();
        (0..length)
            .map(|_| {
                let byte = rand::Rng::sample(&mut rng, Alphanumeric);
                char::from(byte)
            })
            .collect()
    }

    /// Ensure critical services are running - Vault MUST be first
    /// Order: vault -> tables -> drive
    pub async fn ensure_services_running(&mut self) -> Result<()> {
        info!("Ensuring critical services are running...");

        let installer = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?;

        // VAULT MUST BE FIRST - it provides all secrets
        if installer.is_installed("vault") {
            info!("Starting Vault secrets service...");
            match installer.start("vault") {
                Ok(_child) => {
                    info!("Vault started successfully");
                    // Give Vault time to initialize
                    tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
                    
                    // Check if Vault needs to be unsealed
                    if let Err(e) = self.ensure_vault_unsealed().await {
                        warn!("Failed to unseal Vault: {}", e);
                    }
                }
                Err(e) => {
                    warn!("Vault might already be running or failed to start: {}", e);
                }
            }
        } else {
            // Vault not installed - cannot proceed, need to run bootstrap
            warn!("Vault (secrets) component not installed - run bootstrap first");
            return Err(anyhow::anyhow!("Vault not installed. Run bootstrap command first."));
        }

        // Check and start PostgreSQL (after Vault is running)
        if installer.is_installed("tables") {
            info!("Starting PostgreSQL database service...");
            match installer.start("tables") {
                Ok(_child) => {
                    info!("PostgreSQL started successfully");
                    // Give PostgreSQL time to initialize
                    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
                }
                Err(e) => {
                    warn!(
                        "PostgreSQL might already be running or failed to start: {}",
                        e
                    );
                }
            }
        } else {
            warn!("PostgreSQL (tables) component not installed");
        }

        // Check and start MinIO
        if installer.is_installed("drive") {
            info!("Starting MinIO drive service...");
            match installer.start("drive") {
                Ok(_child) => {
                    info!("MinIO started successfully");
                    // Give MinIO time to initialize
                    tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
                }
                Err(e) => {
                    warn!("MinIO might already be running or failed to start: {}", e);
                }
            }
        } else {
            warn!("MinIO (drive) component not installed");
        }

        Ok(())
    }

    /// Ensure Vault is unsealed (required after restart)
    async fn ensure_vault_unsealed(&self) -> Result<()> {
        let vault_init_path = PathBuf::from("./botserver-stack/conf/vault/init.json");
        
        if !vault_init_path.exists() {
            return Err(anyhow::anyhow!("Vault init.json not found"));
        }

        // Read unseal key from init.json
        let init_json = fs::read_to_string(&vault_init_path)?;
        let init_data: serde_json::Value = serde_json::from_str(&init_json)?;
        
        let unseal_key = init_data["unseal_keys_b64"]
            .as_array()
            .and_then(|arr| arr.first())
            .and_then(|v| v.as_str())
            .unwrap_or("")
            .to_string();

        let root_token = init_data["root_token"]
            .as_str()
            .unwrap_or("")
            .to_string();

        if unseal_key.is_empty() || root_token.is_empty() {
            return Err(anyhow::anyhow!("Invalid Vault init.json"));
        }

        let vault_addr = "https://localhost:8200";
        
        // Check if Vault is sealed
        let status_output = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true ./botserver-stack/bin/vault/vault status -format=json 2>/dev/null || echo '{{\"sealed\":true}}'",
                vault_addr
            ))
            .output()?;
        
        let status_json = String::from_utf8_lossy(&status_output.stdout);
        if let Ok(status) = serde_json::from_str::<serde_json::Value>(&status_json) {
            if status["sealed"].as_bool().unwrap_or(true) {
                info!("Unsealing Vault...");
                let _ = std::process::Command::new("sh")
                    .arg("-c")
                    .arg(format!(
                        "VAULT_ADDR={} VAULT_SKIP_VERIFY=true ./botserver-stack/bin/vault/vault operator unseal {}",
                        vault_addr, unseal_key
                    ))
                    .output()?;
            }
        }

        // Set environment variables for other components
        std::env::set_var("VAULT_ADDR", vault_addr);
        std::env::set_var("VAULT_TOKEN", &root_token);
        std::env::set_var("VAULT_SKIP_VERIFY", "true");

        Ok(())
    }

    pub async fn bootstrap(&mut self) -> Result<()> {
        // Generate certificates first (including for Vault)
        info!("🔒 Generating TLS certificates...");
        if let Err(e) = self.generate_certificates().await {
            error!("Failed to generate certificates: {}", e);
        }

        // Create Vault configuration with mTLS
        info!("📝 Creating Vault configuration...");
        if let Err(e) = self.create_vault_config().await {
            error!("Failed to create Vault config: {}", e);
        }

        // Generate secure passwords for all services - these are ONLY used during bootstrap
        // and immediately stored in Vault. NO LEGACY ENV VARS.
        let db_password = self.generate_secure_password(24);
        let drive_accesskey = self.generate_secure_password(20);
        let drive_secret = self.generate_secure_password(40);
        let cache_password = self.generate_secure_password(24);

        // Configuration is stored in Vault, not .env files
        info!("Configuring services through Vault...");

        let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone()).unwrap();
        
        // Vault MUST be installed first - it stores all secrets
        // Order: vault -> tables -> directory -> drive -> cache -> llm
        let required_components = vec![
            "vault",     // Secrets management - MUST BE FIRST
            "tables",    // Database - required by Directory
            "directory", // Identity service - manages users
            "drive",     // S3 storage - credentials in Vault
            "cache",     // Redis cache
            "llm",       // LLM service
        ];
        for component in required_components {
            if !pm.is_installed(component) {
                let termination_cmd = pm
                    .components
                    .get(component)
                    .and_then(|cfg| cfg.binary_name.clone())
                    .unwrap_or_else(|| component.to_string());
                if !termination_cmd.is_empty() {
                    let check = Command::new("pgrep")
                        .arg("-f")
                        .arg(&termination_cmd)
                        .output();
                    if let Ok(output) = check {
                        if !output.stdout.is_empty() {
                            println!("Component '{}' appears to be already running from a previous install.", component);
                            println!("Do you want to terminate it? (y/n)");
                            let mut input = String::new();
                            io::stdout().flush().unwrap();
                            io::stdin().read_line(&mut input).unwrap();
                            if input.trim().eq_ignore_ascii_case("y") {
                                let _ = Command::new("pkill")
                                    .arg("-f")
                                    .arg(&termination_cmd)
                                    .status();
                                println!("Terminated existing '{}' process.", component);
                            } else {
                                println!(
                                    "Skipping start of '{}' as it is already running.",
                                    component
                                );
                                continue;
                            }
                        }
                    }
                }
                _ = pm.install(component).await;

                // After tables is installed, create Zitadel config files before installing directory
                if component == "tables" {
                    info!("🔧 Creating Directory configuration files...");
                    if let Err(e) = self.configure_services_in_directory(&db_password).await {
                        error!("Failed to create Directory config files: {}", e);
                    }
                }

                // Directory configuration - setup happens after install starts Zitadel
                if component == "directory" {
                    info!("🔧 Waiting for Directory to be ready...");
                    if let Err(e) = self.setup_directory().await {
                        // Don't fail completely - Zitadel may still be usable with first instance setup
                        warn!("Directory additional setup had issues: {}", e);
                    }
                }

                // After Vault is installed and running, store all secrets
                if component == "vault" {
                    info!("🔐 Initializing Vault with secrets...");
                    if let Err(e) = self.setup_vault(&db_password, &drive_accesskey, &drive_secret, &cache_password).await {
                        error!("Failed to setup Vault: {}", e);
                    }
                }

                if component == "tables" {
                    let mut conn = establish_pg_connection().unwrap();
                    self.apply_migrations(&mut conn)?;
                }

                if component == "email" {
                    info!("🔧 Auto-configuring Email (Stalwart)...");
                    if let Err(e) = self.setup_email().await {
                        error!("Failed to setup Email: {}", e);
                    }
                }

                if component == "proxy" {
                    info!("🔧 Configuring Caddy reverse proxy...");
                    if let Err(e) = self.setup_caddy_proxy().await {
                        error!("Failed to setup Caddy: {}", e);
                    }
                }

                if component == "dns" {
                    info!("🔧 Configuring CoreDNS for dynamic DNS...");
                    if let Err(e) = self.setup_coredns().await {
                        error!("Failed to setup CoreDNS: {}", e);
                    }
                }
            }
        }
        Ok(())
    }

    /// Configure database and drive credentials in Directory
    /// This creates the Zitadel config files BEFORE Zitadel is installed
    /// db_password is passed directly from bootstrap - NO ENV VARS
    async fn configure_services_in_directory(&self, db_password: &str) -> Result<()> {
        info!("Creating Zitadel configuration files...");

        let zitadel_config_path = PathBuf::from("./botserver-stack/conf/directory/zitadel.yaml");
        let steps_config_path = PathBuf::from("./botserver-stack/conf/directory/steps.yaml");
        let pat_path = PathBuf::from("./botserver-stack/conf/directory/admin-pat.txt");
        
        fs::create_dir_all(zitadel_config_path.parent().unwrap())?;

        // Generate Zitadel database password
        let zitadel_db_password = self.generate_secure_password(24);

        // Create zitadel.yaml - main configuration
        let zitadel_config = format!(
            r#"Log:
  Level: info
  Formatter:
    Format: text

Database:
  postgres:
    Host: localhost
    Port: 5432
    Database: zitadel
    User: zitadel
    Password: "{}"
    SSL:
      Mode: disable
    Admin:
      Username: gbuser
      Password: "{}"
      SSL:
        Mode: disable

Machine:
  Identification:
    Hostname:
      Enabled: true

ExternalSecure: false
ExternalDomain: localhost
ExternalPort: 8080

DefaultInstance:
  OIDCSettings:
    AccessTokenLifetime: 12h
    IdTokenLifetime: 12h
    RefreshTokenIdleExpiration: 720h
    RefreshTokenExpiration: 2160h
"#,
            zitadel_db_password,
            db_password,  // Use the password passed directly from bootstrap
        );

        fs::write(&zitadel_config_path, zitadel_config)?;
        info!("Created zitadel.yaml configuration");

        // Create steps.yaml - first instance setup that generates admin PAT
        let steps_config = format!(
            r#"FirstInstance:
  Org:
    Name: "BotServer"
    Human:
      UserName: "admin"
      FirstName: "Admin"
      LastName: "User"
      Email:
        Address: "admin@localhost"
        Verified: true
      Password: "{}"
  PatPath: "{}"
  InstanceName: "BotServer"
  DefaultLanguage: "en"
"#,
            self.generate_secure_password(16),
            pat_path.to_string_lossy(),
        );

        fs::write(&steps_config_path, steps_config)?;
        info!("Created steps.yaml for first instance setup");

        // Create zitadel database in PostgreSQL
        info!("Creating zitadel database...");
        let create_db_result = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "PGPASSWORD='{}' psql -h localhost -p 5432 -U gbuser -d postgres -c \"CREATE DATABASE zitadel\" 2>&1 || true",
                db_password
            ))
            .output();
        
        if let Ok(output) = create_db_result {
            let stdout = String::from_utf8_lossy(&output.stdout);
            if !stdout.contains("already exists") {
                info!("Created zitadel database");
            }
        }

        // Create zitadel user
        let create_user_result = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "PGPASSWORD='{}' psql -h localhost -p 5432 -U gbuser -d postgres -c \"CREATE USER zitadel WITH PASSWORD '{}' SUPERUSER\" 2>&1 || true",
                db_password,
                zitadel_db_password
            ))
            .output();
        
        if let Ok(output) = create_user_result {
            let stdout = String::from_utf8_lossy(&output.stdout);
            if !stdout.contains("already exists") {
                info!("Created zitadel database user");
            }
        }

        info!("Zitadel configuration files created");
        Ok(())
    }

    /// Setup Caddy as reverse proxy for all services
    async fn setup_caddy_proxy(&self) -> Result<()> {
        let caddy_config = PathBuf::from("./botserver-stack/conf/proxy/Caddyfile");
        fs::create_dir_all(caddy_config.parent().unwrap())?;

        let config = format!(
            r#"{{
    admin off
    auto_https disable_redirects
}}

# Main API
api.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}

# Directory/Auth service
auth.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}

# LLM service
llm.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}

# Mail service
mail.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}

# Meet service
meet.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}
"#,
            crate::core::urls::InternalUrls::DIRECTORY_BASE.replace("https://", ""),
            crate::core::urls::InternalUrls::DIRECTORY_BASE.replace("https://", ""),
            crate::core::urls::InternalUrls::LLM.replace("https://", ""),
            crate::core::urls::InternalUrls::EMAIL.replace("https://", ""),
            crate::core::urls::InternalUrls::LIVEKIT.replace("https://", "")
        );

        fs::write(caddy_config, config)?;
        info!("Caddy proxy configured");
        Ok(())
    }

    /// Setup CoreDNS for dynamic DNS service
    async fn setup_coredns(&self) -> Result<()> {
        let dns_config = PathBuf::from("./botserver-stack/conf/dns/Corefile");
        fs::create_dir_all(dns_config.parent().unwrap())?;

        let zone_file = PathBuf::from("./botserver-stack/conf/dns/botserver.local.zone");

        // Create Corefile
        let corefile = r#"botserver.local:53 {
    file /botserver-stack/conf/dns/botserver.local.zone
    reload 10s
    log
}

.:53 {
    forward . 8.8.8.8 8.8.4.4
    cache 30
    log
}
"#;

        fs::write(dns_config, corefile)?;

        // Create initial zone file
        let zone = r#"$ORIGIN botserver.local.
$TTL 60
@       IN      SOA     ns1.botserver.local. admin.botserver.local. (
                        2024010101      ; Serial
                        3600            ; Refresh
                        1800            ; Retry
                        604800          ; Expire
                        60              ; Minimum TTL
)
        IN      NS      ns1.botserver.local.
ns1     IN      A       127.0.0.1

; Static entries
api     IN      A       127.0.0.1
auth    IN      A       127.0.0.1
llm     IN      A       127.0.0.1
mail    IN      A       127.0.0.1
meet    IN      A       127.0.0.1

; Dynamic entries will be added below
"#;

        fs::write(zone_file, zone)?;
        info!("CoreDNS configured for dynamic DNS");
        Ok(())
    }

    /// Setup Directory (Zitadel) with default organization and user
    async fn setup_directory(&self) -> Result<()> {
        let config_path = PathBuf::from("./config/directory_config.json");
        let pat_path = PathBuf::from("./botserver-stack/conf/directory/admin-pat.txt");

        // Ensure config directory exists
        tokio::fs::create_dir_all("./config").await?;

        // Wait for Directory to be ready and check for PAT file
        info!("Waiting for Zitadel to be ready...");
        let mut attempts = 0;
        let max_attempts = 60; // 60 seconds max wait
        
        while attempts < max_attempts {
            // Check if Zitadel is healthy
            let health_check = std::process::Command::new("curl")
                .args(["-f", "-s", "http://localhost:8080/healthz"])
                .output();
            
            if let Ok(output) = health_check {
                if output.status.success() {
                    info!("Zitadel is healthy");
                    break;
                }
            }
            
            attempts += 1;
            tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
        }

        if attempts >= max_attempts {
            warn!("Zitadel health check timed out, continuing anyway...");
        }

        // Wait a bit more for PAT file to be generated
        tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;

        // Read the admin PAT generated by Zitadel first instance setup
        let admin_token = if pat_path.exists() {
            let token = fs::read_to_string(&pat_path)?;
            let token = token.trim().to_string();
            info!("Loaded admin PAT from {}", pat_path.display());
            Some(token)
        } else {
            warn!("Admin PAT file not found at {}", pat_path.display());
            warn!("Zitadel first instance setup may not have completed");
            None
        };

        let mut setup = DirectorySetup::new(
            "http://localhost:8080".to_string(), // Use HTTP since TLS is disabled
            config_path,
        );

        // Set the admin token if we have it
        if let Some(token) = admin_token {
            setup.set_admin_token(token);
        } else {
            // If no PAT, we can't proceed with API calls
            info!("Directory setup skipped - no admin token available");
            info!("First instance setup created initial admin user via steps.yaml");
            return Ok(());
        }

        // Wait a bit more for Zitadel to be fully ready
        tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;

        // Try to create additional organization for bot users
        let org_name = "default";
        match setup.create_organization(org_name, "Default Organization").await {
            Ok(org_id) => {
                info!("Created default organization: {}", org_name);

                // Generate secure passwords
                let user_password = self.generate_secure_password(16);

                // Create user@default account for regular bot usage
                match setup.create_user(
                    &org_id,
                    "user",
                    "user@default",
                    &user_password,
                    "User",
                    "Default",
                    false,
                ).await {
                    Ok(regular_user) => {
                        info!("Created regular user: user@default");
                        info!("   Regular user ID: {}", regular_user.id);
                    }
                    Err(e) => {
                        warn!("Failed to create regular user: {}", e);
                    }
                }

                // Create OAuth2 application for BotServer
                match setup.create_oauth_application(&org_id).await {
                    Ok((project_id, client_id, client_secret)) => {
                        info!("Created OAuth2 application in project: {}", project_id);
                        
                        // Save configuration
                        let admin_user = crate::package_manager::setup::DefaultUser {
                            id: "admin".to_string(),
                            username: "admin".to_string(),
                            email: "admin@localhost".to_string(),
                            password: "".to_string(), // Don't store password
                            first_name: "Admin".to_string(),
                            last_name: "User".to_string(),
                        };

                        if let Ok(config) = setup.save_config(
                            org_id.clone(),
                            org_name.to_string(),
                            admin_user,
                            client_id.clone(),
                            client_secret,
                        ).await {
                            info!("Directory initialized successfully!");
                            info!("   Organization: default");
                            info!("   Client ID: {}", client_id);
                            info!("   Login URL: {}", config.base_url);
                        }
                    }
                    Err(e) => {
                        warn!("Failed to create OAuth2 application: {}", e);
                    }
                }
            }
            Err(e) => {
                warn!("Failed to create organization: {}", e);
                info!("Using Zitadel's default organization from first instance setup");
            }
        }

        info!("Directory setup complete");
        Ok(())
    }

    /// Setup Vault with all service secrets and write .env file with VAULT_* variables
    async fn setup_vault(&self, db_password: &str, drive_accesskey: &str, drive_secret: &str, cache_password: &str) -> Result<()> {
        let vault_conf_path = PathBuf::from("./botserver-stack/conf/vault");
        let vault_init_path = vault_conf_path.join("init.json");
        let env_file_path = PathBuf::from("./.env");
        
        // Wait for Vault to be ready
        info!("Waiting for Vault to be ready...");
        let mut attempts = 0;
        let max_attempts = 30;
        
        while attempts < max_attempts {
            let health_check = std::process::Command::new("curl")
                .args(["-f", "-s", "-k", "https://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200"])
                .output();
            
            if let Ok(output) = health_check {
                if output.status.success() {
                    info!("Vault is responding");
                    break;
                }
            }
            
            attempts += 1;
            tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
        }

        if attempts >= max_attempts {
            warn!("Vault health check timed out");
            return Err(anyhow::anyhow!("Vault not ready after {} seconds", max_attempts));
        }

        // Check if Vault is already initialized
        let vault_addr = "https://localhost:8200";
        std::env::set_var("VAULT_ADDR", vault_addr);
        std::env::set_var("VAULT_SKIP_VERIFY", "true");

        // Read init.json if it exists (from post_install_cmds)
        let (unseal_key, root_token) = if vault_init_path.exists() {
            info!("Reading Vault initialization from init.json...");
            let init_json = fs::read_to_string(&vault_init_path)?;
            let init_data: serde_json::Value = serde_json::from_str(&init_json)?;
            
            let unseal_key = init_data["unseal_keys_b64"]
                .as_array()
                .and_then(|arr| arr.first())
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();
            
            let root_token = init_data["root_token"]
                .as_str()
                .unwrap_or("")
                .to_string();
            
            (unseal_key, root_token)
        } else {
            // Initialize Vault if not already done
            info!("Initializing Vault...");
            let init_output = std::process::Command::new("sh")
                .arg("-c")
                .arg(format!(
                    "VAULT_ADDR={} VAULT_SKIP_VERIFY=true ./botserver-stack/bin/vault/vault operator init -key-shares=1 -key-threshold=1 -format=json",
                    vault_addr
                ))
                .output()?;
            
            if !init_output.status.success() {
                let stderr = String::from_utf8_lossy(&init_output.stderr);
                if stderr.contains("already initialized") {
                    warn!("Vault already initialized but init.json not found");
                    return Err(anyhow::anyhow!("Vault initialized but credentials lost"));
                }
                return Err(anyhow::anyhow!("Vault init failed: {}", stderr));
            }
            
            let init_json = String::from_utf8_lossy(&init_output.stdout);
            fs::write(&vault_init_path, init_json.as_ref())?;
            fs::set_permissions(&vault_init_path, std::fs::Permissions::from_mode(0o600))?;
            
            let init_data: serde_json::Value = serde_json::from_str(&init_json)?;
            let unseal_key = init_data["unseal_keys_b64"]
                .as_array()
                .and_then(|arr| arr.first())
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();
            let root_token = init_data["root_token"]
                .as_str()
                .unwrap_or("")
                .to_string();
            
            (unseal_key, root_token)
        };

        if root_token.is_empty() {
            return Err(anyhow::anyhow!("Failed to get Vault root token"));
        }

        // Unseal Vault
        info!("Unsealing Vault...");
        let unseal_output = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true ./botserver-stack/bin/vault/vault operator unseal {}",
                vault_addr, unseal_key
            ))
            .output()?;
        
        if !unseal_output.status.success() {
            let stderr = String::from_utf8_lossy(&unseal_output.stderr);
            if !stderr.contains("already unsealed") {
                warn!("Vault unseal warning: {}", stderr);
            }
        }

        // Set VAULT_TOKEN for subsequent commands
        std::env::set_var("VAULT_TOKEN", &root_token);

        // Enable KV secrets engine at gbo/ path
        info!("Enabling KV secrets engine...");
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault secrets enable -path=secret kv-v2 2>&1 || true",
                vault_addr, root_token
            ))
            .output();

        // Store all secrets in Vault
        info!("Storing secrets in Vault...");

        // Database credentials
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/tables host=localhost port=5432 database=botserver username=gbuser password='{}'",
                vault_addr, root_token, db_password
            ))
            .output()?;
        info!("  ✓ Stored database credentials");

        // Drive credentials
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/drive accesskey='{}' secret='{}'",
                vault_addr, root_token, drive_accesskey, drive_secret
            ))
            .output()?;
        info!("  ✓ Stored drive credentials");

        // Cache credentials
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/cache password='{}'",
                vault_addr, root_token, cache_password
            ))
            .output()?;
        info!("  ✓ Stored cache credentials");

        // Directory placeholder (will be updated after Zitadel setup)
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/directory url=https://localhost:8080 project_id= client_id= client_secret=",
                vault_addr, root_token
            ))
            .output()?;
        info!("  ✓ Created directory placeholder");

        // LLM placeholder
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/llm openai_key= anthropic_key= groq_key=",
                vault_addr, root_token
            ))
            .output()?;
        info!("  ✓ Created LLM placeholder");

        // Email placeholder
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/email username= password=",
                vault_addr, root_token
            ))
            .output()?;
        info!("  ✓ Created email placeholder");

        // Encryption key
        let encryption_key = self.generate_secure_password(32);
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/encryption master_key='{}'",
                vault_addr, root_token, encryption_key
            ))
            .output()?;
        info!("  ✓ Generated and stored encryption key");

        // Write .env file with ONLY Vault variables - NO LEGACY FALLBACK
        info!("Writing .env file with Vault configuration...");
        let env_content = format!(
r#"# BotServer Environment Configuration
# Generated by bootstrap - DO NOT ADD OTHER SECRETS HERE
# All secrets are stored in Vault at the paths below:
#   - gbo/tables     - PostgreSQL credentials
#   - gbo/drive      - MinIO/S3 credentials
#   - gbo/cache      - Redis credentials
#   - gbo/directory  - Zitadel credentials
#   - gbo/email      - Email credentials
#   - gbo/llm        - LLM API keys
#   - gbo/encryption - Encryption keys

# Vault Configuration - THESE ARE THE ONLY ALLOWED ENV VARS
VAULT_ADDR={}
VAULT_TOKEN={}

# mTLS Configuration for Vault
# Set VAULT_SKIP_VERIFY=false in production with proper CA cert
VAULT_SKIP_VERIFY=true
VAULT_CACERT=./botserver-stack/conf/system/certificates/ca/ca.crt
VAULT_CLIENT_CERT=./botserver-stack/conf/system/certificates/botserver/client.crt
VAULT_CLIENT_KEY=./botserver-stack/conf/system/certificates/botserver/client.key

# Cache TTL for secrets (seconds)
VAULT_CACHE_TTL=300
"#,
            vault_addr, root_token
        );

        fs::write(&env_file_path, env_content)?;
        info!("  ✓ Created .env file with Vault configuration");

        info!("🔐 Vault setup complete!");
        info!("   Vault UI: {}/ui", vault_addr);
        info!("   Root token saved to: {}", vault_init_path.display());
        
        Ok(())
    }

    /// Setup Email (Stalwart) with Directory integration
    pub async fn setup_email(&self) -> Result<()> {
        let config_path = PathBuf::from("./config/email_config.json");
        let directory_config_path = PathBuf::from("./config/directory_config.json");

        let mut setup = EmailSetup::new(
            crate::core::urls::InternalUrls::DIRECTORY_BASE.to_string(),
            config_path,
        );

        // Try to integrate with Directory if it exists
        let directory_config = if directory_config_path.exists() {
            Some(directory_config_path)
        } else {
            None
        };

        let config = setup.initialize(directory_config).await?;

        info!("Email server initialized successfully!");
        info!("   SMTP: {}:{}", config.smtp_host, config.smtp_port);
        info!("   IMAP: {}:{}", config.imap_host, config.imap_port);
        info!("   Admin: {} / {}", config.admin_user, config.admin_pass);
        if config.directory_integration {
            info!("   🔗 Integrated with Directory for authentication");
        }

        Ok(())
    }

    async fn get_drive_client(config: &AppConfig) -> Client {
        let endpoint = if config.drive.server.ends_with('/') {
            config.drive.server.clone()
        } else {
            format!("{}/", config.drive.server)
        };
        let base_config = aws_config::defaults(BehaviorVersion::latest())
            .endpoint_url(endpoint)
            .region("auto")
            .credentials_provider(aws_sdk_s3::config::Credentials::new(
                config.drive.access_key.clone(),
                config.drive.secret_key.clone(),
                None,
                None,
                "static",
            ))
            .load()
            .await;
        let s3_config = aws_sdk_s3::config::Builder::from(&base_config)
            .force_path_style(true)
            .build();
        aws_sdk_s3::Client::from_conf(s3_config)
    }

    pub async fn upload_templates_to_drive(&self, _config: &AppConfig) -> Result<()> {
        let mut conn = establish_pg_connection()?;
        self.create_bots_from_templates(&mut conn)?;
        let templates_dir = Path::new("templates");
        if !templates_dir.exists() {
            return Ok(());
        }
        let client = Self::get_drive_client(_config).await;
        let mut read_dir = tokio::fs::read_dir(templates_dir).await?;
        while let Some(entry) = read_dir.next_entry().await? {
            let path = entry.path();
            if path.is_dir()
                && path
                    .file_name()
                    .unwrap()
                    .to_string_lossy()
                    .ends_with(".gbai")
            {
                let bot_name = path.file_name().unwrap().to_string_lossy().to_string();
                let bucket = bot_name.trim_start_matches('/').to_string();
                if client.head_bucket().bucket(&bucket).send().await.is_err() {
                    match client.create_bucket().bucket(&bucket).send().await {
                        Ok(_) => {
                            self.upload_directory_recursive(&client, &path, &bucket, "/")
                                .await?;
                        }
                        Err(e) => {
                            error!("Failed to create bucket {}: {:?}", bucket, e);
                            return Err(anyhow::anyhow!("Failed to create bucket {}: {}. Check S3 credentials and endpoint configuration", bucket, e));
                        }
                    }
                } else {
                    trace!("Bucket {} already exists", bucket);
                }
            }
        }
        Ok(())
    }
    fn create_bots_from_templates(&self, conn: &mut diesel::PgConnection) -> Result<()> {
        use crate::shared::models::schema::bots;
        use diesel::prelude::*;
        let templates_dir = Path::new("templates");
        if !templates_dir.exists() {
            return Ok(());
        }
        for entry in std::fs::read_dir(templates_dir)? {
            let entry = entry?;
            let path = entry.path();
            if path.is_dir() && path.extension().map(|e| e == "gbai").unwrap_or(false) {
                let bot_folder = path.file_name().unwrap().to_string_lossy().to_string();
                let bot_name = bot_folder.trim_end_matches(".gbai");
                let existing: Option<String> = bots::table
                    .filter(bots::name.eq(&bot_name))
                    .select(bots::name)
                    .first(conn)
                    .optional()?;
                if existing.is_none() {
                    diesel::sql_query("INSERT INTO bots (id, name, description, llm_provider, llm_config, context_provider, context_config, is_active) VALUES (gen_random_uuid(), $1, $2, 'openai', '{\"model\": \"gpt-4\", \"temperature\": 0.7}', 'database', '{}', true)").bind::<diesel::sql_types::Text, _>(&bot_name).bind::<diesel::sql_types::Text, _>(format!("Bot for {} template", bot_name)).execute(conn)?;
                } else {
                    trace!("Bot {} already exists", bot_name);
                }
            }
        }
        Ok(())
    }
    fn upload_directory_recursive<'a>(
        &'a self,
        client: &'a Client,
        local_path: &'a Path,
        bucket: &'a str,
        prefix: &'a str,
    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {
        Box::pin(async move {
            let _normalized_path = if local_path.to_string_lossy().ends_with('/') {
                local_path.to_string_lossy().to_string()
            } else {
                format!("{}/", local_path.display())
            };
            let mut read_dir = tokio::fs::read_dir(local_path).await?;
            while let Some(entry) = read_dir.next_entry().await? {
                let path = entry.path();
                let file_name = path.file_name().unwrap().to_string_lossy().to_string();
                let mut key = prefix.trim_matches('/').to_string();
                if !key.is_empty() {
                    key.push('/');
                }
                key.push_str(&file_name);
                if path.is_file() {
                    trace!(
                        "Uploading file {} to bucket {} with key {}",
                        path.display(),
                        bucket,
                        key
                    );
                    let content = tokio::fs::read(&path).await?;
                    client
                        .put_object()
                        .bucket(bucket)
                        .key(&key)
                        .body(content.into())
                        .send()
                        .await?;
                } else if path.is_dir() {
                    self.upload_directory_recursive(client, &path, bucket, &key)
                        .await?;
                }
            }
            Ok(())
        })
    }
    pub fn apply_migrations(&self, conn: &mut diesel::PgConnection) -> Result<()> {
        use diesel_migrations::HarnessWithOutput;
        use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};

        const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");

        let mut harness = HarnessWithOutput::write_to_stdout(conn);
        if let Err(e) = harness.run_pending_migrations(MIGRATIONS) {
            error!("Failed to apply migrations: {}", e);
            return Err(anyhow::anyhow!("Migration error: {}", e));
        }

        Ok(())
    }

    /// Create Vault configuration with mTLS settings
    async fn create_vault_config(&self) -> Result<()> {
        let vault_conf_dir = PathBuf::from("./botserver-stack/conf/vault");
        let config_path = vault_conf_dir.join("config.hcl");
        
        fs::create_dir_all(&vault_conf_dir)?;
        
        // Vault config with mTLS - requires client certificate verification
        let config = r#"# Vault Configuration with mTLS
# Generated by BotServer bootstrap

# Storage backend - file-based for single instance
storage "file" {
  path = "./botserver-stack/data/vault"
}

# Listener with mTLS enabled
listener "tcp" {
  address       = "0.0.0.0:8200"
  tls_disable   = false
  
  # Server TLS certificate
  tls_cert_file = "./botserver-stack/conf/system/certificates/vault/server.crt"
  tls_key_file  = "./botserver-stack/conf/system/certificates/vault/server.key"
  
  # mTLS - require client certificate
  tls_require_and_verify_client_cert = true
  tls_client_ca_file = "./botserver-stack/conf/system/certificates/ca/ca.crt"
  
  # TLS settings
  tls_min_version = "tls12"
}

# API settings
api_addr = "https://localhost:8200"
cluster_addr = "https://localhost:8201"

# UI enabled for administration
ui = true

# Disable memory locking (for development - enable in production)
disable_mlock = true

# Telemetry
telemetry {
  disable_hostname = true
}

# Log level
log_level = "info"
"#;
        
        fs::write(&config_path, config)?;
        
        // Create data directory for Vault storage
        fs::create_dir_all("./botserver-stack/data/vault")?;
        
        info!("Created Vault config with mTLS at {}", config_path.display());
        Ok(())
    }

    /// Generate TLS certificates for all services
    async fn generate_certificates(&self) -> Result<()> {
        let cert_dir = PathBuf::from("./botserver-stack/conf/system/certificates");

        // Create certificate directory structure
        fs::create_dir_all(&cert_dir)?;
        fs::create_dir_all(cert_dir.join("ca"))?;

        // Check if CA already exists
        let ca_cert_path = cert_dir.join("ca/ca.crt");
        let ca_key_path = cert_dir.join("ca/ca.key");

        // CA params for issuer creation
        let mut ca_params = CertificateParams::default();
        ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);

        let mut dn = DistinguishedName::new();
        dn.push(DnType::CountryName, "BR");
        dn.push(DnType::OrganizationName, "BotServer");
        dn.push(DnType::CommonName, "BotServer CA");
        ca_params.distinguished_name = dn;

        ca_params.not_before = time::OffsetDateTime::now_utc();
        ca_params.not_after = time::OffsetDateTime::now_utc() + time::Duration::days(3650);

        let ca_key_pair: KeyPair = if ca_cert_path.exists() && ca_key_path.exists() {
            info!("Using existing CA certificate");
            // Load existing CA key
            let key_pem = fs::read_to_string(&ca_key_path)?;
            KeyPair::from_pem(&key_pem)?
        } else {
            info!("Generating new CA certificate");
            let key_pair = KeyPair::generate()?;
            let cert = ca_params.self_signed(&key_pair)?;

            // Save CA certificate and key
            fs::write(&ca_cert_path, cert.pem())?;
            fs::write(&ca_key_path, key_pair.serialize_pem())?;

            key_pair
        };

        // Create issuer from CA params and key
        let ca_issuer = Issuer::from_params(&ca_params, &ca_key_pair);

        // Generate client certificate for botserver (for mTLS to all services)
        let botserver_dir = cert_dir.join("botserver");
        fs::create_dir_all(&botserver_dir)?;
        
        let client_cert_path = botserver_dir.join("client.crt");
        let client_key_path = botserver_dir.join("client.key");
        
        if !client_cert_path.exists() || !client_key_path.exists() {
            info!("Generating mTLS client certificate for botserver");
            
            let mut client_params = CertificateParams::default();
            client_params.not_before = time::OffsetDateTime::now_utc();
            client_params.not_after = time::OffsetDateTime::now_utc() + time::Duration::days(365);
            
            let mut client_dn = DistinguishedName::new();
            client_dn.push(DnType::CountryName, "BR");
            client_dn.push(DnType::OrganizationName, "BotServer");
            client_dn.push(DnType::CommonName, "botserver-client");
            client_params.distinguished_name = client_dn;
            
            // Add client auth extended key usage
            client_params.subject_alt_names.push(rcgen::SanType::DnsName("botserver".to_string().try_into()?));
            
            let client_key = KeyPair::generate()?;
            let client_cert = client_params.signed_by(&client_key, &ca_issuer)?;
            
            fs::write(&client_cert_path, client_cert.pem())?;
            fs::write(&client_key_path, client_key.serialize_pem())?;
            fs::copy(&ca_cert_path, botserver_dir.join("ca.crt"))?;
            
            info!("Generated mTLS client certificate at {}", client_cert_path.display());
        }

        // Services that need certificates - Vault FIRST
        let services = vec![
            ("vault", vec!["localhost", "127.0.0.1", "vault.botserver.local"]),
            ("api", vec!["localhost", "127.0.0.1", "api.botserver.local"]),
            ("llm", vec!["localhost", "127.0.0.1", "llm.botserver.local"]),
            (
                "embedding",
                vec!["localhost", "127.0.0.1", "embedding.botserver.local"],
            ),
            (
                "qdrant",
                vec!["localhost", "127.0.0.1", "qdrant.botserver.local"],
            ),
            (
                "postgres",
                vec!["localhost", "127.0.0.1", "postgres.botserver.local"],
            ),
            (
                "redis",
                vec!["localhost", "127.0.0.1", "redis.botserver.local"],
            ),
            (
                "minio",
                vec!["localhost", "127.0.0.1", "minio.botserver.local"],
            ),
            (
                "directory",
                vec![
                    "localhost",
                    "127.0.0.1",
                    "directory.botserver.local",
                    "auth.botserver.local",
                ],
            ),
            (
                "email",
                vec![
                    "localhost",
                    "127.0.0.1",
                    "mail.botserver.local",
                    "smtp.botserver.local",
                    "imap.botserver.local",
                ],
            ),
            (
                "meet",
                vec![
                    "localhost",
                    "127.0.0.1",
                    "meet.botserver.local",
                    "turn.botserver.local",
                ],
            ),
            (
                "caddy",
                vec![
                    "localhost",
                    "127.0.0.1",
                    "*.botserver.local",
                    "botserver.local",
                ],
            ),
        ];

        for (service, sans) in services {
            let service_dir = cert_dir.join(service);
            fs::create_dir_all(&service_dir)?;

            let cert_path = service_dir.join("server.crt");
            let key_path = service_dir.join("server.key");

            // Skip if certificate already exists
            if cert_path.exists() && key_path.exists() {
                trace!("Certificate for {} already exists", service);
                continue;
            }

            info!("Generating certificate for {}", service);

            // Generate service certificate
            let mut params = CertificateParams::default();
            params.not_before = time::OffsetDateTime::now_utc();
            params.not_after = time::OffsetDateTime::now_utc() + time::Duration::days(365);

            let mut dn = DistinguishedName::new();
            dn.push(DnType::CountryName, "BR");
            dn.push(DnType::OrganizationName, "BotServer");
            dn.push(DnType::CommonName, &format!("{}.botserver.local", service));
            params.distinguished_name = dn;

            // Add SANs
            for san in sans {
                params
                    .subject_alt_names
                    .push(rcgen::SanType::DnsName(san.to_string().try_into()?));
            }

            let key_pair = KeyPair::generate()?;
            let cert = params.signed_by(&key_pair, &ca_issuer)?;

            // Save certificate and key
            fs::write(cert_path, cert.pem())?;
            fs::write(key_path, key_pair.serialize_pem())?;

            // Copy CA cert to service directory for easy access
            fs::copy(&ca_cert_path, service_dir.join("ca.crt"))?;
        }

        info!("TLS certificates generated successfully");
        Ok(())
    }
}