GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests Projects Releases 323 Packages Wiki Activity Actions

feat: configurable stack paths and local installer support

Browse source 
- Add BOTSERVER_INSTALLERS_PATH env var to use local installers without downloading
- Replace hardcoded ./botserver-stack paths with configurable stack_path
- Add stack_dir() and vault_bin() helper methods in BootstrapManager
- Add Port: 8300 to Zitadel config to fix port binding issue
- Start Directory service before setup_directory() call
- Add SKIP_LLM_SERVER env var to skip local LLM in tests
- Update template loading to check ../bottemplates and botserver-templates paths
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-12-14 15:58:54 -03:00
parent 7029e49f80
commit 17618f692c
4 changed files with 220 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
config/directory_config.json
View file

@ -1,7 +1,7 @@
{ {
"base_url": "http://localhost:8080", "base_url": "http://localhost:8300",
"default_org": { "default_org": {
"id": "350493764617240590", "id": "351049887434932238",
"name": "default", "name": "default",
"domain": "default.localhost" "domain": "default.localhost"
}, },
@ -13,8 +13,8 @@
"first_name": "Admin", "first_name": "Admin",
"last_name": "User" "last_name": "User"
}, },
"admin_token": "PQV0tJlkYiGwiXCbaGDszHZZEJ5zNeCppXq4C8Ryi4A_astZuM_aYZUEq1PnNbn6g7Mcc_A", "admin_token": "yR_pDxClepmQw-7neHEcRa6lEMyFB2ECoMEVfBCZGZW7F-TdUvG2W-dWGhEhGYqGDYApbCM",
"project_id": "", "project_id": "",
"client_id": "350493765221285902", "client_id": "351049888072531982",
"client_secret": "KHKXlgoRPdVLhwjnjoy0DvCSREdGv5ukRA2ZhTeSyCjRZrSkZKEc6aQ2pG351nlM" "client_secret": "LojyVztS8EpcnM6qyhCfjtSkeohUy2rO0oi36lKZmtyF5OpNUX88bruNdgqOQWEQ"
} }

282
src/core/bootstrap/mod.rs
View file

@ -24,15 +24,35 @@ pub struct ComponentInfo {
pub struct BootstrapManager { pub struct BootstrapManager {
pub install_mode: InstallMode, pub install_mode: InstallMode,
pub tenant: Option<String>, pub tenant: Option<String>,
pub stack_path: PathBuf,
} }
impl BootstrapManager { impl BootstrapManager {
pub async fn new(mode: InstallMode, tenant: Option<String>) -> Self { pub async fn new(mode: InstallMode, tenant: Option<String>) -> Self {
// Get stack path from env var or use default
let stack_path = std::env::var("BOTSERVER_STACK_PATH")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("./botserver-stack"));
Self { Self {
install_mode: mode, install_mode: mode,
tenant, tenant,
stack_path,
} }
} }
/// Get a path relative to the stack directory
fn stack_dir(&self, subpath: &str) -> PathBuf {
self.stack_path.join(subpath)
}
/// Get the vault binary path as a string for shell commands
fn vault_bin(&self) -> String {
self.stack_dir("bin/vault/vault")
.to_str()
.unwrap_or("./botserver-stack/bin/vault/vault")
.to_string()
}
/// Kill all processes running from the botserver-stack directory /// Kill all processes running from the botserver-stack directory
/// This ensures a clean startup when bootstrapping fresh /// This ensures a clean startup when bootstrapping fresh
pub fn kill_stack_processes() { pub fn kill_stack_processes() {
@ -105,7 +125,9 @@ impl BootstrapManager {
/// Check if another botserver process is already running on this stack /// Check if another botserver process is already running on this stack
pub fn check_single_instance() -> Result<bool> { pub fn check_single_instance() -> Result<bool> {
let lock_file = PathBuf::from("./botserver-stack/.lock"); let stack_path = std::env::var("BOTSERVER_STACK_PATH")
.unwrap_or_else(|_| "./botserver-stack".to_string());
let lock_file = PathBuf::from(&stack_path).join(".lock");
if lock_file.exists() { if lock_file.exists() {
// Check if the PID in the lock file is still running // Check if the PID in the lock file is still running
if let Ok(pid_str) = fs::read_to_string(&lock_file) { if let Ok(pid_str) = fs::read_to_string(&lock_file) {
@ -131,7 +153,9 @@ impl BootstrapManager {
/// Release the instance lock on shutdown /// Release the instance lock on shutdown
pub fn release_instance_lock() { pub fn release_instance_lock() {
let lock_file = PathBuf::from("./botserver-stack/.lock"); let stack_path = std::env::var("BOTSERVER_STACK_PATH")
.unwrap_or_else(|_| "./botserver-stack".to_string());
let lock_file = PathBuf::from(&stack_path).join(".lock");
if lock_file.exists() { if lock_file.exists() {
fs::remove_file(&lock_file).ok(); fs::remove_file(&lock_file).ok();
} }
@ -140,19 +164,21 @@ impl BootstrapManager {
/// Check if botserver-stack has installed components (indicating a working installation) /// Check if botserver-stack has installed components (indicating a working installation)
/// This is used to prevent accidental re-initialization of existing installations /// This is used to prevent accidental re-initialization of existing installations
fn has_installed_stack() -> bool { fn has_installed_stack() -> bool {
let stack_dir = PathBuf::from("./botserver-stack"); let stack_path = std::env::var("BOTSERVER_STACK_PATH")
.unwrap_or_else(|_| "./botserver-stack".to_string());
let stack_dir = PathBuf::from(&stack_path);
if !stack_dir.exists() { if !stack_dir.exists() {
return false; return false;
} }
// Check for key indicators of an installed stack // Check for key indicators of an installed stack
let indicators = vec![ let indicators = vec![
"./botserver-stack/bin/vault/vault", stack_dir.join("bin/vault/vault"),
"./botserver-stack/data/vault", stack_dir.join("data/vault"),
"./botserver-stack/conf/vault/config.hcl", stack_dir.join("conf/vault/config.hcl"),
]; ];
indicators.iter().any(|path| PathBuf::from(path).exists()) indicators.iter().any(|path| path.exists())
} }
/// Reset only Vault credentials (when re-initialization is needed) /// Reset only Vault credentials (when re-initialization is needed)
@ -168,7 +194,9 @@ impl BootstrapManager {
)); ));
} }
let vault_init = PathBuf::from("./botserver-stack/conf/vault/init.json"); let stack_path = std::env::var("BOTSERVER_STACK_PATH")
.unwrap_or_else(|_| "./botserver-stack".to_string());
let vault_init = PathBuf::from(&stack_path).join("conf/vault/init.json");
let env_file = PathBuf::from("./.env"); let env_file = PathBuf::from("./.env");
// Only remove vault init.json and .env - NEVER touch data/ // Only remove vault init.json and .env - NEVER touch data/
@ -242,21 +270,21 @@ impl BootstrapManager {
if Self::has_installed_stack() { if Self::has_installed_stack() {
error!("Vault failed to unseal but stack is installed - NOT re-initializing"); error!("Vault failed to unseal but stack is installed - NOT re-initializing");
error!("Try manually restarting Vault or check ./botserver-stack/logs/vault/vault.log"); error!("Try manually restarting Vault or check ./botserver-stack/logs/vault/vault.log");
// Kill only Vault process and try to restart // Kill only Vault process and try to restart
let _ = Command::new("pkill") let _ = Command::new("pkill")
.args(["-9", "-f", "botserver-stack/bin/vault"]) .args(["-9", "-f", "botserver-stack/bin/vault"])
.output(); .output();
tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
// Try to restart Vault // Try to restart Vault
if let Err(e) = pm.start("vault") { if let Err(e) = pm.start("vault") {
warn!("Failed to restart Vault: {}", e); warn!("Failed to restart Vault: {}", e);
} }
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
// Final attempt to unseal // Final attempt to unseal
if let Err(e) = self.ensure_vault_unsealed().await { if let Err(e) = self.ensure_vault_unsealed().await {
return Err(anyhow::anyhow!( return Err(anyhow::anyhow!(
@ -343,8 +371,7 @@ impl BootstrapManager {
Err(e) => { Err(e) => {
debug!( debug!(
"Component {} might already be running: {}", "Component {} might already be running: {}",
component.name, component.name, e
e
); );
} }
} }
@ -377,7 +404,7 @@ impl BootstrapManager {
// Check if we need to bootstrap first // Check if we need to bootstrap first
let vault_installed = installer.is_installed("vault"); let vault_installed = installer.is_installed("vault");
let vault_initialized = PathBuf::from("./botserver-stack/conf/vault/init.json").exists(); let vault_initialized = self.stack_dir("conf/vault/init.json").exists();
if !vault_installed || !vault_initialized { if !vault_installed || !vault_initialized {
info!("Stack not fully bootstrapped, running bootstrap first..."); info!("Stack not fully bootstrapped, running bootstrap first...");
@ -553,7 +580,7 @@ impl BootstrapManager {
/// Ensure Vault is unsealed (required after restart) /// Ensure Vault is unsealed (required after restart)
/// Returns Ok(()) if Vault is ready, Err if it needs re-initialization /// Returns Ok(()) if Vault is ready, Err if it needs re-initialization
async fn ensure_vault_unsealed(&self) -> Result<()> { async fn ensure_vault_unsealed(&self) -> Result<()> {
let vault_init_path = PathBuf::from("./botserver-stack/conf/vault/init.json"); let vault_init_path = self.stack_dir("conf/vault/init.json");
let vault_addr = "http://localhost:8200"; let vault_addr = "http://localhost:8200";
if !vault_init_path.exists() { if !vault_init_path.exists() {
@ -582,11 +609,12 @@ impl BootstrapManager {
} }
// First check if Vault is initialized (not just running) // First check if Vault is initialized (not just running)
let vault_bin = self.vault_bin();
let status_output = std::process::Command::new("sh") let status_output = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>/dev/null", "VAULT_ADDR={} {} status -format=json 2>/dev/null",
vault_addr vault_addr, vault_bin
)) ))
.stdout(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null()) .stderr(std::process::Stdio::null())
@ -613,8 +641,8 @@ impl BootstrapManager {
let unseal_output = std::process::Command::new("sh") let unseal_output = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator unseal {} >/dev/null 2>&1", "VAULT_ADDR={} {} operator unseal {} >/dev/null 2>&1",
vault_addr, unseal_key vault_addr, vault_bin, unseal_key
)) ))
.stdout(std::process::Stdio::null()) .stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null()) .stderr(std::process::Stdio::null())
@ -630,8 +658,8 @@ impl BootstrapManager {
let verify_output = std::process::Command::new("sh") let verify_output = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>/dev/null", "VAULT_ADDR={} {} status -format=json 2>/dev/null",
vault_addr vault_addr, vault_bin
)) ))
.stdout(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null()) .stderr(std::process::Stdio::null())
@ -661,15 +689,21 @@ impl BootstrapManager {
// Also set mTLS cert paths // Also set mTLS cert paths
std::env::set_var( std::env::set_var(
"VAULT_CACERT", "VAULT_CACERT",
"./botserver-stack/conf/system/certificates/ca/ca.crt", self.stack_dir("conf/system/certificates/ca/ca.crt")
.to_str()
.unwrap_or(""),
); );
std::env::set_var( std::env::set_var(
"VAULT_CLIENT_CERT", "VAULT_CLIENT_CERT",
"./botserver-stack/conf/system/certificates/botserver/client.crt", self.stack_dir("conf/system/certificates/botserver/client.crt")
.to_str()
.unwrap_or(""),
); );
std::env::set_var( std::env::set_var(
"VAULT_CLIENT_KEY", "VAULT_CLIENT_KEY",
"./botserver-stack/conf/system/certificates/botserver/client.key", self.stack_dir("conf/system/certificates/botserver/client.key")
.to_str()
.unwrap_or(""),
); );
info!("Vault environment configured"); info!("Vault environment configured");
@ -720,7 +754,7 @@ impl BootstrapManager {
]; ];
// Special check: Vault needs setup even if binary exists but not initialized // Special check: Vault needs setup even if binary exists but not initialized
let vault_needs_setup = !PathBuf::from("./botserver-stack/conf/vault/init.json").exists(); let vault_needs_setup = !self.stack_dir("conf/vault/init.json").exists();
for component in required_components { for component in required_components {
// For vault, also check if it needs initialization // For vault, also check if it needs initialization
@ -809,6 +843,18 @@ impl BootstrapManager {
// Directory configuration - setup happens after install starts Zitadel // Directory configuration - setup happens after install starts Zitadel
if component == "directory" { if component == "directory" {
info!("Starting Directory (Zitadel) service...");
match pm.start("directory") {
Ok(_) => {
info!("Directory service started successfully");
// Give Zitadel time to initialize before health checks
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
}
Err(e) => {
warn!("Failed to start Directory service: {}", e);
}
}
info!("Waiting for Directory to be ready..."); info!("Waiting for Directory to be ready...");
if let Err(e) = self.setup_directory().await { if let Err(e) = self.setup_directory().await {
// Don't fail completely - Zitadel may still be usable with first instance setup // Don't fail completely - Zitadel may still be usable with first instance setup
@ -821,7 +867,7 @@ impl BootstrapManager {
info!("Setting up Vault secrets service..."); info!("Setting up Vault secrets service...");
// Verify vault binary exists and is executable // Verify vault binary exists and is executable
let vault_bin = PathBuf::from("./botserver-stack/bin/vault/vault"); let vault_bin = self.stack_dir("bin/vault/vault");
if !vault_bin.exists() { if !vault_bin.exists() {
error!("Vault binary not found at {:?}", vault_bin); error!("Vault binary not found at {:?}", vault_bin);
return Err(anyhow::anyhow!("Vault binary not found after installation")); return Err(anyhow::anyhow!("Vault binary not found after installation"));
@ -829,7 +875,7 @@ impl BootstrapManager {
info!("Vault binary verified at {:?}", vault_bin); info!("Vault binary verified at {:?}", vault_bin);
// Ensure logs directory exists // Ensure logs directory exists
let vault_log_path = PathBuf::from("./botserver-stack/logs/vault/vault.log"); let vault_log_path = self.stack_dir("logs/vault/vault.log");
if let Some(parent) = vault_log_path.parent() { if let Some(parent) = vault_log_path.parent() {
if let Err(e) = fs::create_dir_all(parent) { if let Err(e) = fs::create_dir_all(parent) {
error!("Failed to create vault logs directory: {}", e); error!("Failed to create vault logs directory: {}", e);
@ -837,7 +883,7 @@ impl BootstrapManager {
} }
// Ensure data directory exists // Ensure data directory exists
let vault_data_path = PathBuf::from("./botserver-stack/data/vault"); let vault_data_path = self.stack_dir("data/vault");
if let Err(e) = fs::create_dir_all(&vault_data_path) { if let Err(e) = fs::create_dir_all(&vault_data_path) {
error!("Failed to create vault data directory: {}", e); error!("Failed to create vault data directory: {}", e);
} }
@ -845,9 +891,14 @@ impl BootstrapManager {
info!("Starting Vault server..."); info!("Starting Vault server...");
// Try starting vault directly first // Try starting vault directly first
let vault_bin_dir = self.stack_dir("bin/vault");
let vault_start_cmd = format!(
"cd {} && nohup ./vault server -config=../../conf/vault/config.hcl > ../../logs/vault/vault.log 2>&1 &",
vault_bin_dir.display()
);
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg("cd ./botserver-stack/bin/vault && nohup ./vault server -config=../../conf/vault/config.hcl > ../../logs/vault/vault.log 2>&1 &") .arg(&vault_start_cmd)
.status(); .status();
std::thread::sleep(std::time::Duration::from_secs(2)); std::thread::sleep(std::time::Duration::from_secs(2));
@ -968,11 +1019,14 @@ impl BootstrapManager {
async fn configure_services_in_directory(&self, db_password: &str) -> Result<()> { async fn configure_services_in_directory(&self, db_password: &str) -> Result<()> {
info!("Creating Zitadel configuration files..."); info!("Creating Zitadel configuration files...");
let zitadel_config_path = PathBuf::from("./botserver-stack/conf/directory/zitadel.yaml"); let zitadel_config_path = self.stack_dir("conf/directory/zitadel.yaml");
let steps_config_path = PathBuf::from("./botserver-stack/conf/directory/steps.yaml"); let steps_config_path = self.stack_dir("conf/directory/steps.yaml");
// Use absolute path for PAT file since zitadel runs from bin/directory/ // Use absolute path for PAT file since zitadel runs from bin/directory/
let pat_path = let pat_path = if self.stack_path.is_absolute() {
std::env::current_dir()?.join("botserver-stack/conf/directory/admin-pat.txt"); self.stack_dir("conf/directory/admin-pat.txt")
} else {
std::env::current_dir()?.join(self.stack_dir("conf/directory/admin-pat.txt"))
};
fs::create_dir_all(zitadel_config_path.parent().unwrap())?; fs::create_dir_all(zitadel_config_path.parent().unwrap())?;
@ -987,6 +1041,8 @@ impl BootstrapManager {
Formatter: Formatter:
Format: text Format: text
Port: 8300
Database: Database:
postgres: postgres:
Host: localhost Host: localhost
@ -1098,7 +1154,7 @@ DefaultInstance:
/// Setup Caddy as reverse proxy for all services /// Setup Caddy as reverse proxy for all services
async fn setup_caddy_proxy(&self) -> Result<()> { async fn setup_caddy_proxy(&self) -> Result<()> {
let caddy_config = PathBuf::from("./botserver-stack/conf/proxy/Caddyfile"); let caddy_config = self.stack_dir("conf/proxy/Caddyfile");
fs::create_dir_all(caddy_config.parent().unwrap())?; fs::create_dir_all(caddy_config.parent().unwrap())?;
let config = format!( let config = format!(
@ -1151,10 +1207,10 @@ meet.botserver.local {{
/// Setup CoreDNS for dynamic DNS service /// Setup CoreDNS for dynamic DNS service
async fn setup_coredns(&self) -> Result<()> { async fn setup_coredns(&self) -> Result<()> {
let dns_config = PathBuf::from("./botserver-stack/conf/dns/Corefile"); let dns_config = self.stack_dir("conf/dns/Corefile");
fs::create_dir_all(dns_config.parent().unwrap())?; fs::create_dir_all(dns_config.parent().unwrap())?;
let zone_file = PathBuf::from("./botserver-stack/conf/dns/botserver.local.zone"); let zone_file = self.stack_dir("conf/dns/botserver.local.zone");
// Create Corefile // Create Corefile
let corefile = r#"botserver.local:53 { let corefile = r#"botserver.local:53 {
@ -1212,7 +1268,7 @@ meet IN A 127.0.0.1
/// Setup Directory (Zitadel) with default organization and user /// Setup Directory (Zitadel) with default organization and user
async fn setup_directory(&self) -> Result<()> { async fn setup_directory(&self) -> Result<()> {
let config_path = PathBuf::from("./config/directory_config.json"); let config_path = PathBuf::from("./config/directory_config.json");
let pat_path = PathBuf::from("./botserver-stack/conf/directory/admin-pat.txt"); let pat_path = self.stack_dir("conf/directory/admin-pat.txt");
// Ensure config directory exists // Ensure config directory exists
tokio::fs::create_dir_all("./config").await?; tokio::fs::create_dir_all("./config").await?;
@ -1364,7 +1420,7 @@ meet IN A 127.0.0.1
drive_secret: &str, drive_secret: &str,
cache_password: &str, cache_password: &str,
) -> Result<()> { ) -> Result<()> {
let vault_conf_path = PathBuf::from("./botserver-stack/conf/vault"); let vault_conf_path = self.stack_dir("conf/vault");
let vault_init_path = vault_conf_path.join("init.json"); let vault_init_path = vault_conf_path.join("init.json");
let env_file_path = PathBuf::from("./.env"); let env_file_path = PathBuf::from("./.env");
@ -1385,7 +1441,7 @@ meet IN A 127.0.0.1
if ps_result.contains("NOT_RUNNING") { if ps_result.contains("NOT_RUNNING") {
warn!("Vault process is not running (attempt {})", attempts + 1); warn!("Vault process is not running (attempt {})", attempts + 1);
// Check vault.log for crash info // Check vault.log for crash info
let vault_log_path = PathBuf::from("./botserver-stack/logs/vault/vault.log"); let vault_log_path = self.stack_dir("logs/vault/vault.log");
if vault_log_path.exists() { if vault_log_path.exists() {
if let Ok(log_content) = fs::read_to_string(&vault_log_path) { if let Ok(log_content) = fs::read_to_string(&vault_log_path) {
let last_lines: Vec<&str> = let last_lines: Vec<&str> =
@ -1428,7 +1484,7 @@ meet IN A 127.0.0.1
max_attempts max_attempts
); );
// Final check of vault.log // Final check of vault.log
let vault_log_path = PathBuf::from("./botserver-stack/logs/vault/vault.log"); let vault_log_path = self.stack_dir("logs/vault/vault.log");
if vault_log_path.exists() { if vault_log_path.exists() {
if let Ok(log_content) = fs::read_to_string(&vault_log_path) { if let Ok(log_content) = fs::read_to_string(&vault_log_path) {
let last_lines: Vec<&str> = log_content.lines().rev().take(20).collect(); let last_lines: Vec<&str> = log_content.lines().rev().take(20).collect();
@ -1471,7 +1527,8 @@ meet IN A 127.0.0.1
// Check if .env exists with VAULT_TOKEN - try to recover from that // Check if .env exists with VAULT_TOKEN - try to recover from that
let env_token = if env_file_path.exists() { let env_token = if env_file_path.exists() {
if let Ok(env_content) = fs::read_to_string(&env_file_path) { if let Ok(env_content) = fs::read_to_string(&env_file_path) {
env_content.lines() env_content
.lines()
.find(|line| line.starts_with("VAULT_TOKEN=")) .find(|line| line.starts_with("VAULT_TOKEN="))
.map(|line| line.trim_start_matches("VAULT_TOKEN=").to_string()) .map(|line| line.trim_start_matches("VAULT_TOKEN=").to_string())
} else { } else {
@ -1483,12 +1540,13 @@ meet IN A 127.0.0.1
// Initialize Vault if not already done // Initialize Vault if not already done
info!("Initializing Vault..."); info!("Initializing Vault...");
let vault_bin = self.vault_bin();
// Clear any mTLS env vars that might interfere with CLI // Clear any mTLS env vars that might interfere with CLI
let init_output = std::process::Command::new("sh") let init_output = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator init -key-shares=1 -key-threshold=1 -format=json", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} {} operator init -key-shares=1 -key-threshold=1 -format=json",
vault_addr vault_addr, vault_bin
)) ))
.output()?; .output()?;
@ -1496,48 +1554,52 @@ meet IN A 127.0.0.1
let stderr = String::from_utf8_lossy(&init_output.stderr); let stderr = String::from_utf8_lossy(&init_output.stderr);
if stderr.contains("already initialized") { if stderr.contains("already initialized") {
warn!("Vault already initialized but init.json not found"); warn!("Vault already initialized but init.json not found");
// If we have a token from .env, check if Vault is already unsealed // If we have a token from .env, check if Vault is already unsealed
// and we can continue (maybe it was manually unsealed) // and we can continue (maybe it was manually unsealed)
if let Some(_token) = env_token { if let Some(_token) = env_token {
info!("Found VAULT_TOKEN in .env, checking if Vault is unsealed..."); info!("Found VAULT_TOKEN in .env, checking if Vault is unsealed...");
// Check Vault status // Check Vault status
let status_check = std::process::Command::new("sh") let status_check = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>/dev/null", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} {} status -format=json 2>/dev/null",
vault_addr vault_addr, vault_bin
)) ))
.output(); .output();
if let Ok(status_output) = status_check { if let Ok(status_output) = status_check {
let status_str = String::from_utf8_lossy(&status_output.stdout); let status_str = String::from_utf8_lossy(&status_output.stdout);
if let Ok(status) = serde_json::from_str::<serde_json::Value>(&status_str) { if let Ok(status) =
serde_json::from_str::<serde_json::Value>(&status_str)
{
let sealed = status["sealed"].as_bool().unwrap_or(true); let sealed = status["sealed"].as_bool().unwrap_or(true);
if !sealed { if !sealed {
// Vault is unsealed! We can continue with the token from .env // Vault is unsealed! We can continue with the token from .env
warn!("Vault is already unsealed - continuing with existing token"); warn!("Vault is already unsealed - continuing with existing token");
warn!("NOTE: Unseal key is lost - Vault will need manual unseal after restart"); warn!("NOTE: Unseal key is lost - Vault will need manual unseal after restart");
return Ok(()); // Skip rest of setup, Vault is already working return Ok(()); // Skip rest of setup, Vault is already working
} }
} }
} }
// Vault is sealed but we don't have unseal key // Vault is sealed but we don't have unseal key
error!("Vault is sealed and unseal key is lost (init.json missing)"); error!("Vault is sealed and unseal key is lost (init.json missing)");
error!("Options:"); error!("Options:");
error!(" 1. If you have a backup of init.json, restore it to ./botserver-stack/conf/vault/init.json"); error!(" 1. If you have a backup of init.json, restore it to ./botserver-stack/conf/vault/init.json");
error!(" 2. To start fresh, delete ./botserver-stack/data/vault/ and restart"); error!(
" 2. To start fresh, delete ./botserver-stack/data/vault/ and restart"
);
return Err(anyhow::anyhow!( return Err(anyhow::anyhow!(
"Vault is sealed but unseal key is lost. See error messages above for recovery options." "Vault is sealed but unseal key is lost. See error messages above for recovery options."
)); ));
} }
// No token in .env either // No token in .env either
error!("Vault already initialized but credentials are lost"); error!("Vault already initialized but credentials are lost");
error!("Options:"); error!("Options:");
error!(" 1. If you have a backup of init.json, restore it to ./botserver-stack/conf/vault/init.json"); error!(" 1. If you have a backup of init.json, restore it to ./botserver-stack/conf/vault/init.json");
error!(" 2. To start fresh, delete ./botserver-stack/data/vault/ and ./botserver-stack/conf/vault/init.json and restart"); error!(" 2. To start fresh, delete ./botserver-stack/data/vault/ and ./botserver-stack/conf/vault/init.json and restart");
return Err(anyhow::anyhow!( return Err(anyhow::anyhow!(
"Vault initialized but credentials lost. See error messages above for recovery options." "Vault initialized but credentials lost. See error messages above for recovery options."
@ -1568,12 +1630,13 @@ meet IN A 127.0.0.1
// Unseal Vault // Unseal Vault
info!("Unsealing Vault..."); info!("Unsealing Vault...");
let vault_bin = self.vault_bin();
// Clear any mTLS env vars that might interfere with CLI // Clear any mTLS env vars that might interfere with CLI
let unseal_output = std::process::Command::new("sh") let unseal_output = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator unseal {}", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} {} operator unseal {}",
vault_addr, unseal_key vault_addr, vault_bin, unseal_key
)) ))
.output()?; .output()?;
@ -1628,8 +1691,8 @@ VAULT_CACHE_TTL=300
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault secrets enable -path=secret kv-v2 2>&1 || true", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} {} secrets enable -path=secret kv-v2 2>&1 || true",
vault_addr, root_token vault_addr, root_token, vault_bin
)) ))
.output(); .output();
@ -1638,12 +1701,13 @@ VAULT_CACHE_TTL=300
info!("Storing secrets in Vault (only if not existing)..."); info!("Storing secrets in Vault (only if not existing)...");
// Helper to check if a secret path exists // Helper to check if a secret path exists
let vault_bin_clone = vault_bin.clone();
let secret_exists = |path: &str| -> bool { let secret_exists = |path: &str| -> bool {
let output = std::process::Command::new("sh") let output = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv get {} 2>/dev/null", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} {} kv get {} 2>/dev/null",
vault_addr, root_token, path vault_addr, root_token, vault_bin_clone, path
)) ))
.output(); .output();
output.map(|o| o.status.success()).unwrap_or(false) output.map(|o| o.status.success()).unwrap_or(false)
@ -1654,8 +1718,8 @@ VAULT_CACHE_TTL=300
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/tables host=localhost port=5432 database=botserver username=gbuser password='{}'", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} {} kv put secret/gbo/tables host=localhost port=5432 database=botserver username=gbuser password='{}'",
vault_addr, root_token, db_password vault_addr, root_token, vault_bin, db_password
)) ))
.output()?; .output()?;
info!(" Stored database credentials"); info!(" Stored database credentials");
@ -1668,8 +1732,8 @@ VAULT_CACHE_TTL=300
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/drive accesskey='{}' secret='{}'", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} {} kv put secret/gbo/drive accesskey='{}' secret='{}'",
vault_addr, root_token, drive_accesskey, drive_secret vault_addr, root_token, vault_bin, drive_accesskey, drive_secret
)) ))
.output()?; .output()?;
info!(" Stored drive credentials"); info!(" Stored drive credentials");
@ -1682,8 +1746,8 @@ VAULT_CACHE_TTL=300
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/cache password='{}'", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} {} kv put secret/gbo/cache password='{}'",
vault_addr, root_token, cache_password vault_addr, root_token, vault_bin, cache_password
)) ))
.output()?; .output()?;
info!(" Stored cache credentials"); info!(" Stored cache credentials");
@ -1696,8 +1760,8 @@ VAULT_CACHE_TTL=300
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/directory url=https://localhost:8300 project_id= client_id= client_secret=", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} {} kv put secret/gbo/directory url=https://localhost:8300 project_id= client_id= client_secret=",
vault_addr, root_token vault_addr, root_token, vault_bin
)) ))
.output()?; .output()?;
info!(" Created directory placeholder"); info!(" Created directory placeholder");
@ -1710,8 +1774,8 @@ VAULT_CACHE_TTL=300
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/llm openai_key= anthropic_key= groq_key=", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} {} kv put secret/gbo/llm openai_key= anthropic_key= groq_key=",
vault_addr, root_token vault_addr, root_token, vault_bin
)) ))
.output()?; .output()?;
info!(" Created LLM placeholder"); info!(" Created LLM placeholder");
@ -1724,8 +1788,8 @@ VAULT_CACHE_TTL=300
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/email username= password=", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} {} kv put secret/gbo/email username= password=",
vault_addr, root_token vault_addr, root_token, vault_bin
)) ))
.output()?; .output()?;
info!(" Created email placeholder"); info!(" Created email placeholder");
@ -1739,8 +1803,8 @@ VAULT_CACHE_TTL=300
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/encryption master_key='{}'", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} {} kv put secret/gbo/encryption master_key='{}'",
vault_addr, root_token, encryption_key vault_addr, root_token, vault_bin, encryption_key
)) ))
.output()?; .output()?;
info!(" Generated and stored encryption key"); info!(" Generated and stored encryption key");
@ -1844,10 +1908,26 @@ VAULT_CACHE_TTL=300
} }
pub async fn upload_templates_to_drive(&self, _config: &AppConfig) -> Result<()> { pub async fn upload_templates_to_drive(&self, _config: &AppConfig) -> Result<()> {
let templates_dir = Path::new("templates"); // Check multiple possible template locations
if !templates_dir.exists() { let possible_paths = [
return Ok(()); "../bottemplates", // Development: sibling directory
} "bottemplates", // In current directory
"botserver-templates", // Installation bundle subdirectory
"templates", // Legacy path
];
let templates_dir = possible_paths.iter().map(Path::new).find(|p| p.exists());
let templates_dir = match templates_dir {
Some(dir) => {
info!("Using templates from: {:?}", dir);
dir
}
None => {
info!("No templates directory found, skipping template upload");
return Ok(());
}
};
let client = Self::get_drive_client(_config).await; let client = Self::get_drive_client(_config).await;
let mut read_dir = tokio::fs::read_dir(templates_dir).await?; let mut read_dir = tokio::fs::read_dir(templates_dir).await?;
while let Some(entry) = read_dir.next_entry().await? { while let Some(entry) = read_dir.next_entry().await? {
@ -1881,11 +1961,32 @@ VAULT_CACHE_TTL=300
use crate::shared::models::schema::bots; use crate::shared::models::schema::bots;
use diesel::prelude::*; use diesel::prelude::*;
let templates_dir = Path::new("templates"); // Check multiple possible template locations
if !templates_dir.exists() { let possible_paths = [
warn!("Templates directory does not exist"); "../bottemplates", // Development: sibling directory
return Ok(()); "bottemplates", // In current directory
} "botserver-templates", // Installation bundle subdirectory
"templates", // Legacy path
];
let templates_dir = possible_paths
.iter()
.map(|p| PathBuf::from(p))
.find(|p| p.exists());
let templates_dir = match templates_dir {
Some(dir) => {
info!("Loading templates from: {:?}", dir);
dir
}
None => {
warn!(
"Templates directory does not exist (checked: {:?})",
possible_paths
);
return Ok(());
}
};
// Get the default bot (created by migrations) - we'll sync all template configs to it // Get the default bot (created by migrations) - we'll sync all template configs to it
let default_bot: Option<(uuid::Uuid, String)> = bots::table let default_bot: Option<(uuid::Uuid, String)> = bots::table
@ -1909,6 +2010,7 @@ VAULT_CACHE_TTL=300
// Only sync the default.gbai template config (main config for the system) // Only sync the default.gbai template config (main config for the system)
let default_template = templates_dir.join("default.gbai"); let default_template = templates_dir.join("default.gbai");
info!("Looking for default template at: {:?}", default_template);
if default_template.exists() { if default_template.exists() {
let config_path = default_template.join("default.gbot").join("config.csv"); let config_path = default_template.join("default.gbot").join("config.csv");
@ -2063,7 +2165,7 @@ VAULT_CACHE_TTL=300
/// Create Vault configuration with mTLS settings /// Create Vault configuration with mTLS settings
async fn create_vault_config(&self) -> Result<()> { async fn create_vault_config(&self) -> Result<()> {
let vault_conf_dir = PathBuf::from("./botserver-stack/conf/vault"); let vault_conf_dir = self.stack_dir("conf/vault");
let config_path = vault_conf_dir.join("config.hcl"); let config_path = vault_conf_dir.join("config.hcl");
fs::create_dir_all(&vault_conf_dir)?; fs::create_dir_all(&vault_conf_dir)?;
@ -2109,7 +2211,7 @@ log_level = "info"
fs::write(&config_path, config)?; fs::write(&config_path, config)?;
// Create data directory for Vault storage // Create data directory for Vault storage
fs::create_dir_all("./botserver-stack/data/vault")?; fs::create_dir_all(self.stack_dir("data/vault"))?;
info!( info!(
"Created Vault config with mTLS at {}", "Created Vault config with mTLS at {}",
@ -2120,7 +2222,7 @@ log_level = "info"
/// Generate TLS certificates for all services /// Generate TLS certificates for all services
async fn generate_certificates(&self) -> Result<()> { async fn generate_certificates(&self) -> Result<()> {
let cert_dir = PathBuf::from("./botserver-stack/conf/system/certificates"); let cert_dir = self.stack_dir("conf/system/certificates");
// Create certificate directory structure // Create certificate directory structure
fs::create_dir_all(&cert_dir)?; fs::create_dir_all(&cert_dir)?;

18
src/core/package_manager/cache.rs
View file

@ -75,10 +75,26 @@ impl DownloadCache {
/// ///
/// # Arguments /// # Arguments
/// * `base_path` - Base path for botserver (typically current directory or botserver root) /// * `base_path` - Base path for botserver (typically current directory or botserver root)
///
/// # Environment Variables
/// * `BOTSERVER_INSTALLERS_PATH` - Override path to pre-downloaded installers directory
pub fn new(base_path: impl AsRef<Path>) -> Result<Self> { pub fn new(base_path: impl AsRef<Path>) -> Result<Self> {
let base_path = base_path.as_ref().to_path_buf(); let base_path = base_path.as_ref().to_path_buf();
let config = Self::load_config(&base_path)?; let config = Self::load_config(&base_path)?;
let cache_dir = base_path.join(&config.cache_settings.cache_dir);
// Check for BOTSERVER_INSTALLERS_PATH env var first (for testing/offline installs)
let cache_dir = if let Ok(installers_path) = std::env::var("BOTSERVER_INSTALLERS_PATH") {
let path = PathBuf::from(&installers_path);
if path.exists() {
info!("Using installers from BOTSERVER_INSTALLERS_PATH: {:?}", path);
path
} else {
warn!("BOTSERVER_INSTALLERS_PATH set but path doesn't exist: {:?}", path);
base_path.join(&config.cache_settings.cache_dir)
}
} else {
base_path.join(&config.cache_settings.cache_dir)
};
// Ensure cache directory exists // Ensure cache directory exists
if !cache_dir.exists() { if !cache_dir.exists() {

6
src/llm/local.rs
View file

@ -10,6 +10,12 @@ use tokio;
pub async fn ensure_llama_servers_running( pub async fn ensure_llama_servers_running(
app_state: Arc<AppState>, app_state: Arc<AppState>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Skip LLM server startup if SKIP_LLM_SERVER is set (for testing with mock LLM)
if std::env::var("SKIP_LLM_SERVER").is_ok() {
info!("SKIP_LLM_SERVER set - skipping local LLM server startup (using mock/external LLM)");
return Ok(());
}
let config_values = { let config_values = {
let conn_arc = app_state.conn.clone(); let conn_arc = app_state.conn.clone();
let default_bot_id = tokio::task::spawn_blocking(move || { let default_bot_id = tokio::task::spawn_blocking(move || {