botserver/src/core/bootstrap/mod.rs

use crate::config::AppConfig;
use crate::package_manager::setup::{DirectorySetup, EmailSetup};
use crate::package_manager::{InstallMode, PackageManager};
use crate::shared::utils::{establish_pg_connection, init_secrets_manager};
use anyhow::Result;
use aws_config::BehaviorVersion;
use aws_sdk_s3::Client;
use diesel::{Connection, RunQueryDsl};
use log::debug;
use log::{error, info, trace, warn};
use rand::distr::Alphanumeric;
use rcgen::{
    BasicConstraints, CertificateParams, DistinguishedName, DnType, IsCa, Issuer, KeyPair,
};
use std::fs;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::process::Command;
#[derive(Debug)]
pub struct ComponentInfo {
    pub name: &'static str,
}
#[derive(Debug)]
pub struct BootstrapManager {
    pub install_mode: InstallMode,
    pub tenant: Option<String>,
}
impl BootstrapManager {
    pub async fn new(mode: InstallMode, tenant: Option<String>) -> Self {
        trace!(
            "Initializing BootstrapManager with mode {:?} and tenant {:?}",
            mode,
            tenant
        );
        Self {
            install_mode: mode,
            tenant,
        }
    }

    /// Kill all processes running from the botserver-stack directory
    /// This ensures a clean startup when bootstrapping fresh
    pub fn kill_stack_processes() {
        info!("Killing any existing stack processes...");

        // Kill processes by pattern matching on botserver-stack path
        let patterns = vec![
            "botserver-stack/bin/vault",
            "botserver-stack/bin/tables",
            "botserver-stack/bin/drive",
            "botserver-stack/bin/cache",
            "botserver-stack/bin/directory",
            "botserver-stack/bin/llm",
            "botserver-stack/bin/email",
            "botserver-stack/bin/proxy",
            "botserver-stack/bin/dns",
            "botserver-stack/bin/meeting",
            "botserver-stack/bin/vector_db",
        ];

        for pattern in patterns {
            let _ = Command::new("pkill").args(["-9", "-f", pattern]).output();
        }

        // Also kill by specific process names
        let process_names = vec![
            "vault",
            "postgres",
            "minio",
            "redis-server",
            "zitadel",
            "ollama",
            "stalwart",
            "caddy",
            "coredns",
            "livekit",
            "qdrant",
        ];

        for name in process_names {
            let _ = Command::new("pkill").args(["-9", "-x", name]).output();
        }

        // Give processes time to die
        std::thread::sleep(std::time::Duration::from_millis(500));
        info!("Stack processes terminated");
    }

    /// Clean up the entire stack directory for a fresh bootstrap
    pub fn clean_stack_directory() -> Result<()> {
        let stack_dir = PathBuf::from("./botserver-stack");
        let env_file = PathBuf::from("./.env");

        if stack_dir.exists() {
            info!("Removing existing stack directory...");
            fs::remove_dir_all(&stack_dir)?;
            info!("Stack directory removed");
        }

        if env_file.exists() {
            info!("Removing existing .env file...");
            fs::remove_file(&env_file)?;
            info!(".env file removed");
        }

        Ok(())
    }
    pub async fn start_all(&mut self) -> Result<()> {
        let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?;

        // VAULT MUST START FIRST - all other services depend on it for secrets
        if pm.is_installed("vault") {
            // Check if Vault is already running before trying to start
            let vault_already_running = Command::new("sh")
                .arg("-c")
                .arg("curl -f -s http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 >/dev/null 2>&1")
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .status()
                .map(|s| s.success())
                .unwrap_or(false);

            if vault_already_running {
                info!("Vault is already running");
            } else {
                info!("Starting Vault secrets service...");
                match pm.start("vault") {
                    Ok(_child) => {
                        info!("Vault process started, waiting for initialization...");
                    }
                    Err(e) => {
                        warn!("Vault might already be running: {}", e);
                    }
                }

                // Wait for Vault to be ready (up to 10 seconds)
                for i in 0..10 {
                    let vault_ready = Command::new("sh")
                        .arg("-c")
                        .arg("curl -f -s http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 >/dev/null 2>&1")
                        .stdout(std::process::Stdio::null())
                        .stderr(std::process::Stdio::null())
                        .status()
                        .map(|s| s.success())
                        .unwrap_or(false);

                    if vault_ready {
                        info!("Vault is responding");
                        break;
                    }
                    if i < 9 {
                        tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
                    }
                }
            }

            // Try to unseal Vault
            if let Err(e) = self.ensure_vault_unsealed().await {
                warn!("Vault unseal check: {}", e);
            }

            // Initialize SecretsManager so other code can use Vault
            info!("Initializing SecretsManager...");
            match init_secrets_manager().await {
                Ok(_) => info!("SecretsManager initialized successfully"),
                Err(e) => {
                    warn!("Failed to initialize SecretsManager: {}", e);
                }
            }
        }

        // Start tables (PostgreSQL) - needed for database operations
        if pm.is_installed("tables") {
            info!("Starting PostgreSQL database...");
            match pm.start("tables") {
                Ok(_child) => {
                    // Give PostgreSQL time to initialize
                    tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
                    info!("PostgreSQL started");
                }
                Err(e) => {
                    warn!("PostgreSQL might already be running: {}", e);
                }
            }
        }

        // Start other components (order matters less for these)
        let other_components = vec![
            ComponentInfo { name: "cache" },
            ComponentInfo { name: "drive" },
            ComponentInfo { name: "llm" },
            ComponentInfo { name: "email" },
            ComponentInfo { name: "proxy" },
            ComponentInfo { name: "directory" },
            ComponentInfo { name: "alm" },
            ComponentInfo { name: "alm_ci" },
            ComponentInfo { name: "dns" },
            ComponentInfo { name: "meeting" },
            ComponentInfo {
                name: "remote_terminal",
            },
            ComponentInfo { name: "vector_db" },
            ComponentInfo { name: "host" },
        ];

        for component in other_components {
            if pm.is_installed(component.name) {
                match pm.start(component.name) {
                    Ok(_child) => {
                        trace!("Started component: {}", component.name);
                    }
                    Err(e) => {
                        trace!(
                            "Component {} might already be running: {}",
                            component.name,
                            e
                        );
                    }
                }
            }
        }

        Ok(())
    }

    fn generate_secure_password(&self, length: usize) -> String {
        let mut rng = rand::rng();
        let base: String = (0..length.saturating_sub(4))
            .map(|_| {
                let byte = rand::Rng::sample(&mut rng, Alphanumeric);
                char::from(byte)
            })
            .collect();
        // Add required symbols/complexity for Zitadel password policy
        // Use ! instead of @ to avoid breaking database connection strings
        format!("{}!1Aa", base)
    }

    /// Ensure critical services are running - Vault MUST be first
    /// Order: vault -> tables -> drive
    /// If fresh_start is true, kills existing processes first
    pub async fn ensure_services_running(&mut self) -> Result<()> {
        info!("Ensuring critical services are running...");

        let installer = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?;

        // Check if we need to bootstrap first
        let vault_installed = installer.is_installed("vault");
        let vault_initialized = PathBuf::from("./botserver-stack/conf/vault/init.json").exists();

        if !vault_installed || !vault_initialized {
            info!("Stack not fully bootstrapped, running bootstrap first...");
            // Kill any leftover processes
            Self::kill_stack_processes();

            // Run bootstrap - this will start all services
            self.bootstrap().await?;

            // After bootstrap, services are already running, just ensure Vault is unsealed and env vars set
            info!("Bootstrap complete, verifying Vault is ready...");
            tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;

            if let Err(e) = self.ensure_vault_unsealed().await {
                warn!("Failed to unseal Vault after bootstrap: {}", e);
            }

            // Services were started by bootstrap, no need to restart them
            return Ok(());
        }

        // If we get here, bootstrap was already done previously - just start services
        // VAULT MUST BE FIRST - it provides all secrets
        if installer.is_installed("vault") {
            // Check if Vault is already running
            let vault_running = Command::new("sh")
                .arg("-c")
                .arg("curl -f -s http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 >/dev/null 2>&1")
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .status()
                .map(|s| s.success())
                .unwrap_or(false);

            if !vault_running {
                info!("Starting Vault secrets service...");
                match installer.start("vault") {
                    Ok(_child) => {
                        info!("Vault started successfully");
                        // Give Vault time to initialize
                        tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
                    }
                    Err(e) => {
                        warn!("Vault might already be running or failed to start: {}", e);
                    }
                }
            } else {
                info!("Vault is already running");
            }

            // Always try to unseal Vault (it may have restarted)
            // If unseal fails, Vault may need re-initialization (data deleted)
            if let Err(e) = self.ensure_vault_unsealed().await {
                warn!("Vault unseal failed: {} - running re-bootstrap", e);

                // Kill all processes and run fresh bootstrap
                Self::kill_stack_processes();
                Self::clean_stack_directory()?;

                // Run bootstrap from scratch
                self.bootstrap().await?;

                // After bootstrap, services are already running
                info!("Re-bootstrap complete, verifying Vault is ready...");
                tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;

                if let Err(e) = self.ensure_vault_unsealed().await {
                    return Err(anyhow::anyhow!(
                        "Failed to configure Vault after re-bootstrap: {}",
                        e
                    ));
                }

                // Services were started by bootstrap, no need to restart them
                return Ok(());
            }

            // Initialize SecretsManager so other code can use Vault
            info!("Initializing SecretsManager...");
            match init_secrets_manager().await {
                Ok(_) => info!("SecretsManager initialized successfully"),
                Err(e) => {
                    error!("Failed to initialize SecretsManager: {}", e);
                    return Err(anyhow::anyhow!(
                        "SecretsManager initialization failed: {}",
                        e
                    ));
                }
            }
        } else {
            // Vault not installed - cannot proceed, need to run bootstrap
            warn!("Vault (secrets) component not installed - run bootstrap first");
            return Err(anyhow::anyhow!(
                "Vault not installed. Run bootstrap command first."
            ));
        }

        // Check and start PostgreSQL (after Vault is running)
        if installer.is_installed("tables") {
            info!("Starting PostgreSQL database service...");
            match installer.start("tables") {
                Ok(_child) => {
                    info!("PostgreSQL started successfully");
                    // Give PostgreSQL time to initialize
                    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
                }
                Err(e) => {
                    warn!(
                        "PostgreSQL might already be running or failed to start: {}",
                        e
                    );
                }
            }
        } else {
            warn!("PostgreSQL (tables) component not installed");
        }

        // Check and start MinIO
        if installer.is_installed("drive") {
            info!("Starting MinIO drive service...");
            match installer.start("drive") {
                Ok(_child) => {
                    info!("MinIO started successfully");
                    // Give MinIO time to initialize
                    tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
                }
                Err(e) => {
                    warn!("MinIO might already be running or failed to start: {}", e);
                }
            }
        } else {
            warn!("MinIO (drive) component not installed");
        }

        Ok(())
    }

    /// Ensure Vault is unsealed (required after restart)
    /// Returns Ok(()) if Vault is ready, Err if it needs re-initialization
    async fn ensure_vault_unsealed(&self) -> Result<()> {
        let vault_init_path = PathBuf::from("./botserver-stack/conf/vault/init.json");
        let vault_addr = "http://localhost:8200";

        if !vault_init_path.exists() {
            return Err(anyhow::anyhow!(
                "Vault init.json not found - needs re-initialization"
            ));
        }

        // Read unseal key from init.json
        let init_json = fs::read_to_string(&vault_init_path)?;
        let init_data: serde_json::Value = serde_json::from_str(&init_json)?;

        let unseal_key = init_data["unseal_keys_b64"]
            .as_array()
            .and_then(|arr| arr.first())
            .and_then(|v| v.as_str())
            .unwrap_or("")
            .to_string();

        let root_token = init_data["root_token"].as_str().unwrap_or("").to_string();

        if unseal_key.is_empty() || root_token.is_empty() {
            return Err(anyhow::anyhow!(
                "Invalid Vault init.json - needs re-initialization"
            ));
        }

        // First check if Vault is initialized (not just running)
        let status_output = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>/dev/null",
                vault_addr
            ))
            .stdout(std::process::Stdio::piped())
            .stderr(std::process::Stdio::null())
            .output()?;

        let status_str = String::from_utf8_lossy(&status_output.stdout);

        // Parse status - handle both success and error cases
        if let Ok(status) = serde_json::from_str::<serde_json::Value>(&status_str) {
            let initialized = status["initialized"].as_bool().unwrap_or(false);
            let sealed = status["sealed"].as_bool().unwrap_or(true);

            if !initialized {
                // Vault is running but not initialized - this means data was deleted
                // We need to re-run bootstrap
                warn!("Vault is running but not initialized - data may have been deleted");
                return Err(anyhow::anyhow!(
                    "Vault not initialized - needs re-bootstrap"
                ));
            }

            if sealed {
                info!("Unsealing Vault...");
                let unseal_output = std::process::Command::new("sh")
                    .arg("-c")
                    .arg(format!(
                        "VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator unseal {} >/dev/null 2>&1",
                        vault_addr, unseal_key
                    ))
                    .stdout(std::process::Stdio::null())
                    .stderr(std::process::Stdio::null())
                    .output()?;

                if !unseal_output.status.success() {
                    let stderr = String::from_utf8_lossy(&unseal_output.stderr);
                    warn!("Vault unseal may have failed: {}", stderr);
                }

                // Verify unseal succeeded
                tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
                let verify_output = std::process::Command::new("sh")
                    .arg("-c")
                    .arg(format!(
                        "VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>/dev/null",
                        vault_addr
                    ))
                    .stdout(std::process::Stdio::piped())
                    .stderr(std::process::Stdio::null())
                    .output()?;

                let verify_str = String::from_utf8_lossy(&verify_output.stdout);
                if let Ok(verify_status) = serde_json::from_str::<serde_json::Value>(&verify_str) {
                    if verify_status["sealed"].as_bool().unwrap_or(true) {
                        return Err(anyhow::anyhow!(
                            "Failed to unseal Vault - may need re-initialization"
                        ));
                    }
                }
                info!("Vault unsealed successfully");
            }
        } else {
            // Could not parse status - Vault might not be responding properly
            warn!("Could not get Vault status: {}", status_str);
            return Err(anyhow::anyhow!("Vault not responding properly"));
        }

        // Set environment variables for other components
        std::env::set_var("VAULT_ADDR", vault_addr);
        std::env::set_var("VAULT_TOKEN", &root_token);
        std::env::set_var("VAULT_SKIP_VERIFY", "true");

        // Also set mTLS cert paths
        std::env::set_var(
            "VAULT_CACERT",
            "./botserver-stack/conf/system/certificates/ca/ca.crt",
        );
        std::env::set_var(
            "VAULT_CLIENT_CERT",
            "./botserver-stack/conf/system/certificates/botserver/client.crt",
        );
        std::env::set_var(
            "VAULT_CLIENT_KEY",
            "./botserver-stack/conf/system/certificates/botserver/client.key",
        );

        info!("Vault environment configured");
        Ok(())
    }

    pub async fn bootstrap(&mut self) -> Result<()> {
        // Generate certificates first (including for Vault)
        info!("Generating TLS certificates...");
        if let Err(e) = self.generate_certificates().await {
            error!("Failed to generate certificates: {}", e);
        }

        // Create Vault configuration with mTLS
        info!("Creating Vault configuration...");
        if let Err(e) = self.create_vault_config().await {
            error!("Failed to create Vault config: {}", e);
        }

        // Generate secure passwords for all services - these are ONLY used during bootstrap
        // and immediately stored in Vault. NO LEGACY ENV VARS.
        let db_password = self.generate_secure_password(24);
        let drive_accesskey = self.generate_secure_password(20);
        let drive_secret = self.generate_secure_password(40);
        let cache_password = self.generate_secure_password(24);

        // Configuration is stored in Vault, not .env files
        info!("Configuring services through Vault...");

        let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone()).unwrap();

        // Vault MUST be installed first - it stores all secrets
        // Order: vault -> tables -> directory -> drive -> cache -> llm
        let required_components = vec![
            "vault",     // Secrets management - MUST BE FIRST
            "tables",    // Database - required by Directory
            "directory", // Identity service - manages users
            "drive",     // S3 storage - credentials in Vault
            "cache",     // Redis cache
            "llm",       // LLM service
        ];

        // Special check: Vault needs setup even if binary exists but not initialized
        let vault_needs_setup = !PathBuf::from("./botserver-stack/conf/vault/init.json").exists();

        for component in required_components {
            // For vault, also check if it needs initialization
            let needs_install = if component == "vault" {
                !pm.is_installed(component) || vault_needs_setup
            } else {
                !pm.is_installed(component)
            };

            if needs_install {
                // Quick check if component might be running - don't hang on this
                let bin_path = pm.base_path.join("bin").join(component);
                let binary_name = pm
                    .components
                    .get(component)
                    .and_then(|cfg| cfg.binary_name.clone())
                    .unwrap_or_else(|| component.to_string());

                // Only terminate for services that are known to conflict
                // Use simple, fast commands with timeout
                if component == "vault" || component == "tables" || component == "directory" {
                    let _ = Command::new("sh")
                        .arg("-c")
                        .arg(format!(
                            "pkill -9 -f '{}/{}' 2>/dev/null; true",
                            bin_path.display(),
                            binary_name
                        ))
                        .status();
                    std::thread::sleep(std::time::Duration::from_millis(200));
                }
                _ = pm.install(component).await;

                // After tables is installed, START PostgreSQL and create Zitadel config files before installing directory
                if component == "tables" {
                    info!("Starting PostgreSQL database...");
                    match pm.start("tables") {
                        Ok(_) => {
                            info!("PostgreSQL started successfully");
                            // Give PostgreSQL time to initialize
                            tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
                        }
                        Err(e) => {
                            warn!("Failed to start PostgreSQL: {}", e);
                        }
                    }

                    // Run migrations using direct connection (Vault not set up yet)
                    info!("Running database migrations...");
                    let database_url =
                        format!("postgres://gbuser:{}@localhost:5432/botserver", db_password);
                    match diesel::PgConnection::establish(&database_url) {
                        Ok(mut conn) => {
                            if let Err(e) = self.apply_migrations(&mut conn) {
                                error!("Failed to apply migrations: {}", e);
                            } else {
                                info!("Database migrations applied");
                            }
                        }
                        Err(e) => {
                            error!("Failed to connect to database for migrations: {}", e);
                        }
                    }

                    info!("Creating Directory configuration files...");
                    if let Err(e) = self.configure_services_in_directory(&db_password).await {
                        error!("Failed to create Directory config files: {}", e);
                    }
                }

                // Directory configuration - setup happens after install starts Zitadel
                if component == "directory" {
                    info!("Waiting for Directory to be ready...");
                    if let Err(e) = self.setup_directory().await {
                        // Don't fail completely - Zitadel may still be usable with first instance setup
                        warn!("Directory additional setup had issues: {}", e);
                    }
                }

                // After Vault is installed, START the server then initialize it
                if component == "vault" {
                    info!("Starting Vault server...");
                    match pm.start("vault") {
                        Ok(_) => {
                            info!("Vault server started");
                            // Give Vault time to start
                            tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
                        }
                        Err(e) => {
                            warn!("Failed to start Vault server: {}", e);
                        }
                    }

                    info!("Initializing Vault with secrets...");
                    if let Err(e) = self
                        .setup_vault(
                            &db_password,
                            &drive_accesskey,
                            &drive_secret,
                            &cache_password,
                        )
                        .await
                    {
                        error!("Failed to setup Vault: {}", e);
                    }

                    // Initialize the global SecretsManager so other components can use Vault
                    info!("Initializing SecretsManager...");
                    debug!(
                        "VAULT_ADDR={:?}, VAULT_TOKEN set={}",
                        std::env::var("VAULT_ADDR").ok(),
                        std::env::var("VAULT_TOKEN").is_ok()
                    );
                    match init_secrets_manager().await {
                        Ok(_) => info!("SecretsManager initialized successfully"),
                        Err(e) => {
                            error!("Failed to initialize SecretsManager: {}", e);
                            // Don't continue if SecretsManager fails - it's required for DB connection
                            return Err(anyhow::anyhow!(
                                "SecretsManager initialization failed: {}",
                                e
                            ));
                        }
                    }
                }

                if component == "email" {
                    info!("Auto-configuring Email (Stalwart)...");
                    if let Err(e) = self.setup_email().await {
                        error!("Failed to setup Email: {}", e);
                    }
                }

                if component == "proxy" {
                    info!("Configuring Caddy reverse proxy...");
                    if let Err(e) = self.setup_caddy_proxy().await {
                        error!("Failed to setup Caddy: {}", e);
                    }
                }

                if component == "dns" {
                    info!("Configuring CoreDNS for dynamic DNS...");
                    if let Err(e) = self.setup_coredns().await {
                        error!("Failed to setup CoreDNS: {}", e);
                    }
                }
            }
        }
        Ok(())
    }

    /// Configure database and drive credentials in Directory
    /// This creates the Zitadel config files BEFORE Zitadel is installed
    /// db_password is passed directly from bootstrap - NO ENV VARS
    async fn configure_services_in_directory(&self, db_password: &str) -> Result<()> {
        info!("Creating Zitadel configuration files...");

        let zitadel_config_path = PathBuf::from("./botserver-stack/conf/directory/zitadel.yaml");
        let steps_config_path = PathBuf::from("./botserver-stack/conf/directory/steps.yaml");
        // Use absolute path for PAT file since zitadel runs from bin/directory/
        let pat_path =
            std::env::current_dir()?.join("botserver-stack/conf/directory/admin-pat.txt");

        fs::create_dir_all(zitadel_config_path.parent().unwrap())?;

        // Generate Zitadel database password
        let zitadel_db_password = self.generate_secure_password(24);

        // Create zitadel.yaml - main configuration
        // Note: Zitadel uses lowercase 'postgres' and nested User/Admin with Username field
        let zitadel_config = format!(
            r#"Log:
  Level: info
  Formatter:
    Format: text

Database:
  postgres:
    Host: localhost
    Port: 5432
    Database: zitadel
    User:
      Username: zitadel
      Password: "{}"
      SSL:
        Mode: disable
    Admin:
      Username: gbuser
      Password: "{}"
      SSL:
        Mode: disable

Machine:
  Identification:
    Hostname:
      Enabled: true

ExternalSecure: false
ExternalDomain: localhost
ExternalPort: 8080

DefaultInstance:
  OIDCSettings:
    AccessTokenLifetime: 12h
    IdTokenLifetime: 12h
    RefreshTokenIdleExpiration: 720h
    RefreshTokenExpiration: 2160h
"#,
            zitadel_db_password,
            db_password, // Use the password passed directly from bootstrap
        );

        fs::write(&zitadel_config_path, zitadel_config)?;
        info!("Created zitadel.yaml configuration");

        // Create steps.yaml - first instance setup that generates admin PAT
        // Use Machine user with PAT for API access (Human users don't generate PAT files)
        let steps_config = format!(
            r#"FirstInstance:
  InstanceName: "BotServer"
  DefaultLanguage: "en"
  PatPath: "{}"
  Org:
    Name: "BotServer"
    Machine:
      Machine:
        Username: "admin-sa"
        Name: "Admin Service Account"
      Pat:
        ExpirationDate: "2099-12-31T23:59:59Z"
    Human:
      UserName: "admin"
      FirstName: "Admin"
      LastName: "User"
      Email:
        Address: "admin@localhost"
        Verified: true
      Password: "{}"
      PasswordChangeRequired: false
"#,
            pat_path.to_string_lossy(),
            self.generate_secure_password(16),
        );

        fs::write(&steps_config_path, steps_config)?;
        info!("Created steps.yaml for first instance setup");

        // Create zitadel database in PostgreSQL
        info!("Creating zitadel database...");
        let create_db_result = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "PGPASSWORD='{}' psql -h localhost -p 5432 -U gbuser -d postgres -c \"CREATE DATABASE zitadel\" 2>&1 || true",
                db_password
            ))
            .output();

        if let Ok(output) = create_db_result {
            let stdout = String::from_utf8_lossy(&output.stdout);
            if !stdout.contains("already exists") {
                info!("Created zitadel database");
            }
        }

        // Create zitadel user
        let create_user_result = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "PGPASSWORD='{}' psql -h localhost -p 5432 -U gbuser -d postgres -c \"CREATE USER zitadel WITH PASSWORD '{}' SUPERUSER\" 2>&1 || true",
                db_password,
                zitadel_db_password
            ))
            .output();

        if let Ok(output) = create_user_result {
            let stdout = String::from_utf8_lossy(&output.stdout);
            if !stdout.contains("already exists") {
                info!("Created zitadel database user");
            }
        }

        info!("Zitadel configuration files created");
        Ok(())
    }

    /// Setup Caddy as reverse proxy for all services
    async fn setup_caddy_proxy(&self) -> Result<()> {
        let caddy_config = PathBuf::from("./botserver-stack/conf/proxy/Caddyfile");
        fs::create_dir_all(caddy_config.parent().unwrap())?;

        let config = format!(
            r#"{{
    admin off
    auto_https disable_redirects
}}

# Main API
api.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}

# Directory/Auth service
auth.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}

# LLM service
llm.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}

# Mail service
mail.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}

# Meet service
meet.botserver.local {{
    tls /botserver-stack/conf/system/certificates/caddy/server.crt /botserver-stack/conf/system/certificates/caddy/server.key
    reverse_proxy {}
}}
"#,
            crate::core::urls::InternalUrls::DIRECTORY_BASE.replace("https://", ""),
            crate::core::urls::InternalUrls::DIRECTORY_BASE.replace("https://", ""),
            crate::core::urls::InternalUrls::LLM.replace("https://", ""),
            crate::core::urls::InternalUrls::EMAIL.replace("https://", ""),
            crate::core::urls::InternalUrls::LIVEKIT.replace("https://", "")
        );

        fs::write(caddy_config, config)?;
        info!("Caddy proxy configured");
        Ok(())
    }

    /// Setup CoreDNS for dynamic DNS service
    async fn setup_coredns(&self) -> Result<()> {
        let dns_config = PathBuf::from("./botserver-stack/conf/dns/Corefile");
        fs::create_dir_all(dns_config.parent().unwrap())?;

        let zone_file = PathBuf::from("./botserver-stack/conf/dns/botserver.local.zone");

        // Create Corefile
        let corefile = r#"botserver.local:53 {
    file /botserver-stack/conf/dns/botserver.local.zone
    reload 10s
    log
}

.:53 {
    forward . 8.8.8.8 8.8.4.4
    cache 30
    log
}
"#;

        fs::write(dns_config, corefile)?;

        // Create initial zone file with component names
        let zone = r#"$ORIGIN botserver.local.
$TTL 60
@       IN      SOA     ns1.botserver.local. admin.botserver.local. (
                        2024010101      ; Serial
                        3600            ; Refresh
                        1800            ; Retry
                        604800          ; Expire
                        60              ; Minimum TTL
)
        IN      NS      ns1.botserver.local.
ns1     IN      A       127.0.0.1

; Core services
api         IN      A       127.0.0.1
tables      IN      A       127.0.0.1
drive       IN      A       127.0.0.1
cache       IN      A       127.0.0.1
vectordb    IN      A       127.0.0.1
vault       IN      A       127.0.0.1

; Application services
llm         IN      A       127.0.0.1
embedding   IN      A       127.0.0.1
directory   IN      A       127.0.0.1
auth        IN      A       127.0.0.1
email       IN      A       127.0.0.1
meet        IN      A       127.0.0.1

; Dynamic entries will be added below
"#;

        fs::write(zone_file, zone)?;
        info!("CoreDNS configured for dynamic DNS");
        Ok(())
    }

    /// Setup Directory (Zitadel) with default organization and user
    async fn setup_directory(&self) -> Result<()> {
        let config_path = PathBuf::from("./config/directory_config.json");
        let pat_path = PathBuf::from("./botserver-stack/conf/directory/admin-pat.txt");

        // Ensure config directory exists
        tokio::fs::create_dir_all("./config").await?;

        // Wait for Directory to be ready and check for PAT file
        info!("Waiting for Zitadel to be ready...");
        let mut attempts = 0;
        let max_attempts = 60; // 60 seconds max wait

        while attempts < max_attempts {
            // Check if Zitadel is healthy
            let health_check = std::process::Command::new("curl")
                .args(["-f", "-s", "http://localhost:8080/healthz"])
                .output();

            if let Ok(output) = health_check {
                if output.status.success() {
                    info!("Zitadel is healthy");
                    break;
                }
            }

            attempts += 1;
            tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
        }

        if attempts >= max_attempts {
            warn!("Zitadel health check timed out, continuing anyway...");
        }

        // Wait a bit more for PAT file to be generated
        tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;

        // Read the admin PAT generated by Zitadel first instance setup
        let admin_token = if pat_path.exists() {
            let token = fs::read_to_string(&pat_path)?;
            let token = token.trim().to_string();
            info!("Loaded admin PAT from {}", pat_path.display());
            Some(token)
        } else {
            warn!("Admin PAT file not found at {}", pat_path.display());
            warn!("Zitadel first instance setup may not have completed");
            None
        };

        let mut setup = DirectorySetup::new(
            "http://localhost:8080".to_string(), // Use HTTP since TLS is disabled
            config_path,
        );

        // Set the admin token if we have it
        if let Some(token) = admin_token {
            setup.set_admin_token(token);
        } else {
            // If no PAT, we can't proceed with API calls
            info!("Directory setup skipped - no admin token available");
            info!("First instance setup created initial admin user via steps.yaml");
            return Ok(());
        }

        // Wait a bit more for Zitadel to be fully ready
        tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;

        // Try to create additional organization for bot users
        let org_name = "default";
        match setup
            .create_organization(org_name, "Default Organization")
            .await
        {
            Ok(org_id) => {
                info!("Created default organization: {}", org_name);

                // Generate secure passwords
                let user_password = self.generate_secure_password(16);

                // Create user@default account for regular bot usage
                match setup
                    .create_user(
                        &org_id,
                        "user",
                        "user@default",
                        &user_password,
                        "User",
                        "Default",
                        false,
                    )
                    .await
                {
                    Ok(regular_user) => {
                        info!("Created regular user: user@default");
                        info!("   Regular user ID: {}", regular_user.id);
                    }
                    Err(e) => {
                        warn!("Failed to create regular user: {}", e);
                    }
                }

                // Create OAuth2 application for BotServer
                match setup.create_oauth_application(&org_id).await {
                    Ok((project_id, client_id, client_secret)) => {
                        info!("Created OAuth2 application in project: {}", project_id);

                        // Save configuration
                        let admin_user = crate::package_manager::setup::DefaultUser {
                            id: "admin".to_string(),
                            username: "admin".to_string(),
                            email: "admin@localhost".to_string(),
                            password: "".to_string(), // Don't store password
                            first_name: "Admin".to_string(),
                            last_name: "User".to_string(),
                        };

                        if let Ok(config) = setup
                            .save_config(
                                org_id.clone(),
                                org_name.to_string(),
                                admin_user,
                                client_id.clone(),
                                client_secret,
                            )
                            .await
                        {
                            info!("Directory initialized successfully!");
                            info!("   Organization: default");
                            info!("   Client ID: {}", client_id);
                            info!("   Login URL: {}", config.base_url);
                        }
                    }
                    Err(e) => {
                        warn!("Failed to create OAuth2 application: {}", e);
                    }
                }
            }
            Err(e) => {
                warn!("Failed to create organization: {}", e);
                info!("Using Zitadel's default organization from first instance setup");
            }
        }

        info!("Directory setup complete");
        Ok(())
    }

    /// Setup Vault with all service secrets and write .env file with VAULT_* variables
    async fn setup_vault(
        &self,
        db_password: &str,
        drive_accesskey: &str,
        drive_secret: &str,
        cache_password: &str,
    ) -> Result<()> {
        let vault_conf_path = PathBuf::from("./botserver-stack/conf/vault");
        let vault_init_path = vault_conf_path.join("init.json");
        let env_file_path = PathBuf::from("./.env");

        // Wait for Vault to be ready
        info!("Waiting for Vault to be ready...");
        let mut attempts = 0;
        let max_attempts = 30;

        while attempts < max_attempts {
            let health_check = std::process::Command::new("curl")
                .args(["-f", "-s", "http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200"])
                .output();

            if let Ok(output) = health_check {
                if output.status.success() {
                    info!("Vault is responding");
                    break;
                }
            }

            attempts += 1;
            tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
        }

        if attempts >= max_attempts {
            warn!("Vault health check timed out");
            return Err(anyhow::anyhow!(
                "Vault not ready after {} seconds",
                max_attempts
            ));
        }

        // Check if Vault is already initialized
        let vault_addr = "http://localhost:8200";
        std::env::set_var("VAULT_ADDR", vault_addr);
        std::env::set_var("VAULT_SKIP_VERIFY", "true");

        // Read init.json if it exists (from post_install_cmds)
        let (unseal_key, root_token) = if vault_init_path.exists() {
            info!("Reading Vault initialization from init.json...");
            let init_json = fs::read_to_string(&vault_init_path)?;
            let init_data: serde_json::Value = serde_json::from_str(&init_json)?;

            let unseal_key = init_data["unseal_keys_b64"]
                .as_array()
                .and_then(|arr| arr.first())
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();

            let root_token = init_data["root_token"].as_str().unwrap_or("").to_string();

            (unseal_key, root_token)
        } else {
            // Initialize Vault if not already done
            info!("Initializing Vault...");
            // Clear any mTLS env vars that might interfere with CLI
            let init_output = std::process::Command::new("sh")
                .arg("-c")
                .arg(format!(
                    "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator init -key-shares=1 -key-threshold=1 -format=json",
                    vault_addr
                ))
                .output()?;

            if !init_output.status.success() {
                let stderr = String::from_utf8_lossy(&init_output.stderr);
                if stderr.contains("already initialized") {
                    warn!("Vault already initialized but init.json not found");
                    return Err(anyhow::anyhow!("Vault initialized but credentials lost"));
                }
                return Err(anyhow::anyhow!("Vault init failed: {}", stderr));
            }

            let init_json = String::from_utf8_lossy(&init_output.stdout);
            fs::write(&vault_init_path, init_json.as_ref())?;
            fs::set_permissions(&vault_init_path, std::fs::Permissions::from_mode(0o600))?;

            let init_data: serde_json::Value = serde_json::from_str(&init_json)?;
            let unseal_key = init_data["unseal_keys_b64"]
                .as_array()
                .and_then(|arr| arr.first())
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();
            let root_token = init_data["root_token"].as_str().unwrap_or("").to_string();

            (unseal_key, root_token)
        };

        if root_token.is_empty() {
            return Err(anyhow::anyhow!("Failed to get Vault root token"));
        }

        // Unseal Vault
        info!("Unsealing Vault...");
        // Clear any mTLS env vars that might interfere with CLI
        let unseal_output = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator unseal {}",
                vault_addr, unseal_key
            ))
            .output()?;

        if !unseal_output.status.success() {
            let stderr = String::from_utf8_lossy(&unseal_output.stderr);
            if !stderr.contains("already unsealed") {
                warn!("Vault unseal warning: {}", stderr);
            }
        }

        // Set VAULT_TOKEN for subsequent commands
        std::env::set_var("VAULT_TOKEN", &root_token);

        // Enable KV secrets engine at gbo/ path
        info!("Enabling KV secrets engine...");
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault secrets enable -path=secret kv-v2 2>&1 || true",
                vault_addr, root_token
            ))
            .output();

        // Store all secrets in Vault
        info!("Storing secrets in Vault...");

        // Database credentials
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/tables host=localhost port=5432 database=botserver username=gbuser password='{}'",
                vault_addr, root_token, db_password
            ))
            .output()?;
        info!("  Stored database credentials");

        // Drive credentials
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/drive accesskey='{}' secret='{}'",
                vault_addr, root_token, drive_accesskey, drive_secret
            ))
            .output()?;
        info!("  Stored drive credentials");

        // Cache credentials
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/cache password='{}'",
                vault_addr, root_token, cache_password
            ))
            .output()?;
        info!("  Stored cache credentials");

        // Directory placeholder (will be updated after Zitadel setup)
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/directory url=https://localhost:8080 project_id= client_id= client_secret=",
                vault_addr, root_token
            ))
            .output()?;
        info!("  Created directory placeholder");

        // LLM placeholder
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/llm openai_key= anthropic_key= groq_key=",
                vault_addr, root_token
            ))
            .output()?;
        info!("  Created LLM placeholder");

        // Email placeholder
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/email username= password=",
                vault_addr, root_token
            ))
            .output()?;
        info!("  Created email placeholder");

        // Encryption key
        let encryption_key = self.generate_secure_password(32);
        let _ = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/encryption master_key='{}'",
                vault_addr, root_token, encryption_key
            ))
            .output()?;
        info!("  Generated and stored encryption key");

        // Write .env file with ONLY Vault variables - NO LEGACY FALLBACK
        info!("Writing .env file with Vault configuration...");
        let env_content = format!(
            r#"# BotServer Environment Configuration
# Generated by bootstrap - DO NOT ADD OTHER SECRETS HERE
# All secrets are stored in Vault at the paths below:
#   - gbo/tables     - PostgreSQL credentials
#   - gbo/drive      - MinIO/S3 credentials
#   - gbo/cache      - Redis credentials
#   - gbo/directory  - Zitadel credentials
#   - gbo/email      - Email credentials
#   - gbo/llm        - LLM API keys
#   - gbo/encryption - Encryption keys

# Vault Configuration - THESE ARE THE ONLY ALLOWED ENV VARS
VAULT_ADDR={}
VAULT_TOKEN={}

# Vault uses HTTP for local development (TLS disabled in config.hcl)
# In production, enable TLS and set VAULT_CACERT, VAULT_CLIENT_CERT, VAULT_CLIENT_KEY

# Cache TTL for secrets (seconds)
VAULT_CACHE_TTL=300
"#,
            vault_addr, root_token
        );

        fs::write(&env_file_path, env_content)?;
        info!("  Created .env file with Vault configuration");

        info!("Vault setup complete!");
        info!("   Vault UI: {}/ui", vault_addr);
        info!("   Root token saved to: {}", vault_init_path.display());

        Ok(())
    }

    /// Setup Email (Stalwart) with Directory integration
    pub async fn setup_email(&self) -> Result<()> {
        let config_path = PathBuf::from("./config/email_config.json");
        let directory_config_path = PathBuf::from("./config/directory_config.json");

        let mut setup = EmailSetup::new(
            crate::core::urls::InternalUrls::DIRECTORY_BASE.to_string(),
            config_path,
        );

        // Try to integrate with Directory if it exists
        let directory_config = if directory_config_path.exists() {
            Some(directory_config_path)
        } else {
            None
        };

        let config = setup.initialize(directory_config).await?;

        info!("Email server initialized successfully!");
        info!("   SMTP: {}:{}", config.smtp_host, config.smtp_port);
        info!("   IMAP: {}:{}", config.imap_host, config.imap_port);
        info!("   Admin: {} / {}", config.admin_user, config.admin_pass);
        if config.directory_integration {
            info!("    Integrated with Directory for authentication");
        }

        Ok(())
    }

    async fn get_drive_client(config: &AppConfig) -> Client {
        let endpoint = if config.drive.server.ends_with('/') {
            config.drive.server.clone()
        } else {
            format!("{}/", config.drive.server)
        };

        // Get credentials from config, or fetch from Vault if empty
        let (access_key, secret_key) =
            if config.drive.access_key.is_empty() || config.drive.secret_key.is_empty() {
                // Try to get from Vault using the global SecretsManager
                match crate::shared::utils::get_secrets_manager().await {
                    Some(manager) if manager.is_enabled() => {
                        match manager.get_drive_credentials().await {
                            Ok((ak, sk)) => (ak, sk),
                            Err(e) => {
                                warn!("Failed to get drive credentials from Vault: {}", e);
                                (
                                    config.drive.access_key.clone(),
                                    config.drive.secret_key.clone(),
                                )
                            }
                        }
                    }
                    _ => (
                        config.drive.access_key.clone(),
                        config.drive.secret_key.clone(),
                    ),
                }
            } else {
                (
                    config.drive.access_key.clone(),
                    config.drive.secret_key.clone(),
                )
            };

        let base_config = aws_config::defaults(BehaviorVersion::latest())
            .endpoint_url(endpoint)
            .region("auto")
            .credentials_provider(aws_sdk_s3::config::Credentials::new(
                access_key, secret_key, None, None, "static",
            ))
            .load()
            .await;
        let s3_config = aws_sdk_s3::config::Builder::from(&base_config)
            .force_path_style(true)
            .build();
        aws_sdk_s3::Client::from_conf(s3_config)
    }

    /// Sync bot configurations from template config.csv files to database
    /// This is separate from drive upload and does not require S3 connection
    pub fn sync_templates_to_database(&self) -> Result<()> {
        let mut conn = establish_pg_connection()?;
        self.create_bots_from_templates(&mut conn)?;
        Ok(())
    }

    pub async fn upload_templates_to_drive(&self, _config: &AppConfig) -> Result<()> {
        let templates_dir = Path::new("templates");
        if !templates_dir.exists() {
            return Ok(());
        }
        let client = Self::get_drive_client(_config).await;
        let mut read_dir = tokio::fs::read_dir(templates_dir).await?;
        while let Some(entry) = read_dir.next_entry().await? {
            let path = entry.path();
            if path.is_dir()
                && path
                    .file_name()
                    .unwrap()
                    .to_string_lossy()
                    .ends_with(".gbai")
            {
                let bot_name = path.file_name().unwrap().to_string_lossy().to_string();
                let bucket = bot_name.trim_start_matches('/').to_string();
                if client.head_bucket().bucket(&bucket).send().await.is_err() {
                    match client.create_bucket().bucket(&bucket).send().await {
                        Ok(_) => {
                            self.upload_directory_recursive(&client, &path, &bucket, "/")
                                .await?;
                        }
                        Err(e) => {
                            error!("Failed to create bucket {}: {:?}", bucket, e);
                            return Err(anyhow::anyhow!("Failed to create bucket {}: {}. Check S3 credentials and endpoint configuration", bucket, e));
                        }
                    }
                } else {
                    trace!("Bucket {} already exists", bucket);
                }
            }
        }
        Ok(())
    }
    fn create_bots_from_templates(&self, conn: &mut diesel::PgConnection) -> Result<()> {
        use crate::shared::models::schema::bots;
        use diesel::prelude::*;

        let templates_dir = Path::new("templates");
        if !templates_dir.exists() {
            warn!("Templates directory does not exist");
            return Ok(());
        }

        // Get the default bot (created by migrations) - we'll sync all template configs to it
        let default_bot: Option<(uuid::Uuid, String)> = bots::table
            .filter(bots::is_active.eq(true))
            .select((bots::id, bots::name))
            .first(conn)
            .optional()?;

        let (default_bot_id, default_bot_name) = match default_bot {
            Some((id, name)) => (id, name),
            None => {
                error!("No active bot found in database - cannot sync template configs");
                return Ok(());
            }
        };

        info!(
            "Syncing template configs to bot '{}' ({})",
            default_bot_name, default_bot_id
        );

        // Only sync the default.gbai template config (main config for the system)
        let default_template = templates_dir.join("default.gbai");
        if default_template.exists() {
            let config_path = default_template.join("default.gbot").join("config.csv");

            if config_path.exists() {
                match std::fs::read_to_string(&config_path) {
                    Ok(csv_content) => {
                        info!("Syncing config.csv from {:?}", config_path);
                        if let Err(e) =
                            self.sync_config_csv_to_db(conn, &default_bot_id, &csv_content)
                        {
                            error!("Failed to sync config.csv: {}", e);
                        }
                    }
                    Err(e) => {
                        warn!("Could not read config.csv: {}", e);
                    }
                }
            } else {
                warn!("No config.csv found at {:?}", config_path);
            }
        } else {
            warn!("default.gbai template not found");
        }

        Ok(())
    }

    /// Sync config.csv content to the bot_configuration table
    /// This is critical for loading LLM settings on fresh starts
    fn sync_config_csv_to_db(
        &self,
        conn: &mut diesel::PgConnection,
        bot_id: &uuid::Uuid,
        content: &str,
    ) -> Result<()> {
        let mut synced = 0;
        let mut skipped = 0;
        let lines: Vec<&str> = content.lines().collect();

        debug!(
            "Parsing config.csv with {} lines for bot {}",
            lines.len(),
            bot_id
        );

        for (line_num, line) in lines.iter().enumerate().skip(1) {
            // Skip header line (name,value)
            let line = line.trim();
            if line.is_empty() || line.starts_with('#') {
                continue;
            }

            let parts: Vec<&str> = line.splitn(2, ',').collect();
            if parts.len() >= 2 {
                let key = parts[0].trim();
                let value = parts[1].trim();

                if key.is_empty() {
                    skipped += 1;
                    continue;
                }

                // Use UUID type since migration 6.1.1 converted column to UUID
                let new_id = uuid::Uuid::new_v4();

                match diesel::sql_query(
                    "INSERT INTO bot_configuration (id, bot_id, config_key, config_value, config_type, created_at, updated_at) \
                     VALUES ($1, $2, $3, $4, 'string', NOW(), NOW()) \
                     ON CONFLICT (bot_id, config_key) DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()"
                )
                .bind::<diesel::sql_types::Uuid, _>(new_id)
                .bind::<diesel::sql_types::Uuid, _>(bot_id)
                .bind::<diesel::sql_types::Text, _>(key)
                .bind::<diesel::sql_types::Text, _>(value)
                .execute(conn) {
                    Ok(_) => {
                        trace!("  Synced config: {} = {}", key, if key.contains("pass") || key.contains("secret") || key.contains("key") { "***" } else { value });
                        synced += 1;
                    }
                    Err(e) => {
                        error!("Failed to sync config key '{}' at line {}: {}", key, line_num + 1, e);
                        // Continue with other keys instead of failing completely
                    }
                }
            }
        }

        if synced > 0 {
            info!(
                "Synced {} config values for bot {} (skipped {} empty lines)",
                synced, bot_id, skipped
            );
        } else {
            warn!(
                "No config values synced for bot {} - check config.csv format",
                bot_id
            );
        }
        Ok(())
    }
    fn upload_directory_recursive<'a>(
        &'a self,
        client: &'a Client,
        local_path: &'a Path,
        bucket: &'a str,
        prefix: &'a str,
    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {
        Box::pin(async move {
            let _normalized_path = if local_path.to_string_lossy().ends_with('/') {
                local_path.to_string_lossy().to_string()
            } else {
                format!("{}/", local_path.display())
            };
            let mut read_dir = tokio::fs::read_dir(local_path).await?;
            while let Some(entry) = read_dir.next_entry().await? {
                let path = entry.path();
                let file_name = path.file_name().unwrap().to_string_lossy().to_string();
                let mut key = prefix.trim_matches('/').to_string();
                if !key.is_empty() {
                    key.push('/');
                }
                key.push_str(&file_name);
                if path.is_file() {
                    trace!(
                        "Uploading file {} to bucket {} with key {}",
                        path.display(),
                        bucket,
                        key
                    );
                    let content = tokio::fs::read(&path).await?;
                    client
                        .put_object()
                        .bucket(bucket)
                        .key(&key)
                        .body(content.into())
                        .send()
                        .await?;
                } else if path.is_dir() {
                    self.upload_directory_recursive(client, &path, bucket, &key)
                        .await?;
                }
            }
            Ok(())
        })
    }
    pub fn apply_migrations(&self, conn: &mut diesel::PgConnection) -> Result<()> {
        use diesel_migrations::HarnessWithOutput;
        use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};

        const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");

        let mut harness = HarnessWithOutput::write_to_stdout(conn);
        if let Err(e) = harness.run_pending_migrations(MIGRATIONS) {
            error!("Failed to apply migrations: {}", e);
            return Err(anyhow::anyhow!("Migration error: {}", e));
        }

        Ok(())
    }

    /// Create Vault configuration with mTLS settings
    async fn create_vault_config(&self) -> Result<()> {
        let vault_conf_dir = PathBuf::from("./botserver-stack/conf/vault");
        let config_path = vault_conf_dir.join("config.hcl");

        fs::create_dir_all(&vault_conf_dir)?;

        // Vault is started from botserver-stack/bin/vault/, so paths must be relative to that
        // From bin/vault/ to conf/ is ../../conf/
        // From bin/vault/ to data/ is ../../data/
        let config = r#"# Vault Configuration
# Generated by BotServer bootstrap
# Note: Paths are relative to botserver-stack/bin/vault/ (Vault's working directory)

# Storage backend - file-based for single instance
storage "file" {
  path = "../../data/vault"
}

# Listener with TLS DISABLED for local development
# In production, enable TLS with proper certificates
listener "tcp" {
  address     = "0.0.0.0:8200"
  tls_disable = true
}

# API settings - use HTTP for local dev
api_addr = "http://localhost:8200"
cluster_addr = "http://localhost:8201"

# UI enabled for administration
ui = true

# Disable memory locking (for development - enable in production)
disable_mlock = true

# Telemetry
telemetry {
  disable_hostname = true
}

# Log level
log_level = "info"
"#;

        fs::write(&config_path, config)?;

        // Create data directory for Vault storage
        fs::create_dir_all("./botserver-stack/data/vault")?;

        info!(
            "Created Vault config with mTLS at {}",
            config_path.display()
        );
        Ok(())
    }

    /// Generate TLS certificates for all services
    async fn generate_certificates(&self) -> Result<()> {
        let cert_dir = PathBuf::from("./botserver-stack/conf/system/certificates");

        // Create certificate directory structure
        fs::create_dir_all(&cert_dir)?;
        fs::create_dir_all(cert_dir.join("ca"))?;

        // Check if CA already exists
        let ca_cert_path = cert_dir.join("ca/ca.crt");
        let ca_key_path = cert_dir.join("ca/ca.key");

        // CA params for issuer creation
        let mut ca_params = CertificateParams::default();
        ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);

        let mut dn = DistinguishedName::new();
        dn.push(DnType::CountryName, "BR");
        dn.push(DnType::OrganizationName, "BotServer");
        dn.push(DnType::CommonName, "BotServer CA");
        ca_params.distinguished_name = dn;

        ca_params.not_before = time::OffsetDateTime::now_utc();
        ca_params.not_after = time::OffsetDateTime::now_utc() + time::Duration::days(3650);

        let ca_key_pair: KeyPair = if ca_cert_path.exists() && ca_key_path.exists() {
            info!("Using existing CA certificate");
            // Load existing CA key
            let key_pem = fs::read_to_string(&ca_key_path)?;
            KeyPair::from_pem(&key_pem)?
        } else {
            info!("Generating new CA certificate");
            let key_pair = KeyPair::generate()?;
            let cert = ca_params.self_signed(&key_pair)?;

            // Save CA certificate and key
            fs::write(&ca_cert_path, cert.pem())?;
            fs::write(&ca_key_path, key_pair.serialize_pem())?;

            key_pair
        };

        // Create issuer from CA params and key
        let ca_issuer = Issuer::from_params(&ca_params, &ca_key_pair);

        // Generate client certificate for botserver (for mTLS to all services)
        let botserver_dir = cert_dir.join("botserver");
        fs::create_dir_all(&botserver_dir)?;

        let client_cert_path = botserver_dir.join("client.crt");
        let client_key_path = botserver_dir.join("client.key");

        if !client_cert_path.exists() || !client_key_path.exists() {
            info!("Generating mTLS client certificate for botserver");

            let mut client_params = CertificateParams::default();
            client_params.not_before = time::OffsetDateTime::now_utc();
            client_params.not_after = time::OffsetDateTime::now_utc() + time::Duration::days(365);

            let mut client_dn = DistinguishedName::new();
            client_dn.push(DnType::CountryName, "BR");
            client_dn.push(DnType::OrganizationName, "BotServer");
            client_dn.push(DnType::CommonName, "botserver-client");
            client_params.distinguished_name = client_dn;

            // Add client auth extended key usage
            client_params
                .subject_alt_names
                .push(rcgen::SanType::DnsName("botserver".to_string().try_into()?));

            let client_key = KeyPair::generate()?;
            let client_cert = client_params.signed_by(&client_key, &ca_issuer)?;

            fs::write(&client_cert_path, client_cert.pem())?;
            fs::write(&client_key_path, client_key.serialize_pem())?;
            fs::copy(&ca_cert_path, botserver_dir.join("ca.crt"))?;

            info!(
                "Generated mTLS client certificate at {}",
                client_cert_path.display()
            );
        }

        // Services that need certificates - Vault FIRST
        // Using component names: tables (postgres), drive (minio), cache (redis), vectordb (qdrant)
        let services = vec![
            (
                "vault",
                vec!["localhost", "127.0.0.1", "vault.botserver.local"],
            ),
            ("api", vec!["localhost", "127.0.0.1", "api.botserver.local"]),
            ("llm", vec!["localhost", "127.0.0.1", "llm.botserver.local"]),
            (
                "embedding",
                vec!["localhost", "127.0.0.1", "embedding.botserver.local"],
            ),
            (
                "vectordb",
                vec!["localhost", "127.0.0.1", "vectordb.botserver.local"],
            ),
            (
                "tables",
                vec!["localhost", "127.0.0.1", "tables.botserver.local"],
            ),
            (
                "cache",
                vec!["localhost", "127.0.0.1", "cache.botserver.local"],
            ),
            (
                "drive",
                vec!["localhost", "127.0.0.1", "drive.botserver.local"],
            ),
            (
                "directory",
                vec![
                    "localhost",
                    "127.0.0.1",
                    "directory.botserver.local",
                    "auth.botserver.local",
                ],
            ),
            (
                "email",
                vec![
                    "localhost",
                    "127.0.0.1",
                    "email.botserver.local",
                    "smtp.botserver.local",
                    "imap.botserver.local",
                ],
            ),
            (
                "meet",
                vec![
                    "localhost",
                    "127.0.0.1",
                    "meet.botserver.local",
                    "turn.botserver.local",
                ],
            ),
            (
                "caddy",
                vec![
                    "localhost",
                    "127.0.0.1",
                    "*.botserver.local",
                    "botserver.local",
                ],
            ),
        ];

        for (service, sans) in services {
            let service_dir = cert_dir.join(service);
            fs::create_dir_all(&service_dir)?;

            let cert_path = service_dir.join("server.crt");
            let key_path = service_dir.join("server.key");

            // Skip if certificate already exists
            if cert_path.exists() && key_path.exists() {
                trace!("Certificate for {} already exists", service);
                continue;
            }

            info!("Generating certificate for {}", service);

            // Generate service certificate
            let mut params = CertificateParams::default();
            params.not_before = time::OffsetDateTime::now_utc();
            params.not_after = time::OffsetDateTime::now_utc() + time::Duration::days(365);

            let mut dn = DistinguishedName::new();
            dn.push(DnType::CountryName, "BR");
            dn.push(DnType::OrganizationName, "BotServer");
            dn.push(DnType::CommonName, &format!("{}.botserver.local", service));
            params.distinguished_name = dn;

            // Add SANs
            for san in sans {
                params
                    .subject_alt_names
                    .push(rcgen::SanType::DnsName(san.to_string().try_into()?));
            }

            let key_pair = KeyPair::generate()?;
            let cert = params.signed_by(&key_pair, &ca_issuer)?;

            // Save certificate and key
            fs::write(cert_path, cert.pem())?;
            fs::write(key_path, key_pair.serialize_pem())?;

            // Copy CA cert to service directory for easy access
            fs::copy(&ca_cert_path, service_dir.join("ca.crt"))?;
        }

        info!("TLS certificates generated successfully");
        Ok(())
    }
}