GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests Projects Releases 323 Packages Wiki Activity Actions

fix(bootstrap): NEVER delete user data, suppress migration output, protect secrets

Browse source 
Critical safety improvements:

1. REMOVED clean_stack_directory() - NEVER delete botserver-stack
   - Contains user data, configs, databases
   - Only reset_vault_only() for Vault re-initialization

2. Single instance check
   - check_single_instance() uses .lock file with PID
   - Prevents multiple botserver processes on same stack

3. Protect existing Vault secrets
   - Check if secret exists before writing
   - Never overwrite customer credentials in distributed environments
   - Especially critical for encryption key

4. Suppress migration output
   - Use MigrationHarness directly instead of HarnessWithOutput
   - Prevents console UI corruption from migration messages
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-12-09 09:04:56 -03:00
parent a2b091914f
commit 0b9ad6c80d
2 changed files with 179 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
config/directory_config.json
View file

@ -1,7 +1,7 @@
{ {
"base_url": "http://localhost:8080", "base_url": "http://localhost:8080",
"default_org": { "default_org": {
"id": "350233021212786702", "id": "350284283375517710",
"name": "default", "name": "default",
"domain": "default.localhost" "domain": "default.localhost"
}, },
@ -13,8 +13,8 @@
"first_name": "Admin", "first_name": "Admin",
"last_name": "User" "last_name": "User"
}, },
"admin_token": "OPPlIuGyjrDYtM2D5SlIWiGOlt9QSjLzQre-QYaRO8jwkQdZa3f3zvsZihoCkq-cBwQ0gio", "admin_token": "mp1N0PI5mP7VNbj-g-d1e-LFFxV22l6pHuCdPvcbQtS0U35e_jLFIY1GsgREaMOqvrtAu3E",
"project_id": "", "project_id": "",
"client_id": "350233021783277582", "client_id": "350284283929231374",
"client_secret": "zQbganbJqEnqjt0y565aL3TeF02WHGmjX2EYQfssja4SoKLBiOFwfcKUM7kWpbp8" "client_secret": "sBttWgX1v1ENGDyqBxtPRLItMf8Y4oQHk2hAoBStW6BMPuYQIY6xV6dkaSxsjSoe"
} }

268
src/core/bootstrap/mod.rs
View file

@ -87,21 +87,55 @@ impl BootstrapManager {
info!("Stack processes terminated"); info!("Stack processes terminated");
} }
/// Clean up the entire stack directory for a fresh bootstrap /// Check if another botserver process is already running on this stack
pub fn clean_stack_directory() -> Result<()> { pub fn check_single_instance() -> Result<bool> {
let stack_dir = PathBuf::from("./botserver-stack"); let lock_file = PathBuf::from("./botserver-stack/.lock");
if lock_file.exists() {
// Check if the PID in the lock file is still running
if let Ok(pid_str) = fs::read_to_string(&lock_file) {
if let Ok(pid) = pid_str.trim().parse::<i32>() {
let check = Command::new("kill").args(["-0", &pid.to_string()]).output();
if let Ok(output) = check {
if output.status.success() {
warn!("Another botserver process (PID {}) is already running on this stack", pid);
return Ok(false);
}
}
}
}
}
// Write our PID to the lock file
let pid = std::process::id();
if let Some(parent) = lock_file.parent() {
fs::create_dir_all(parent).ok();
}
fs::write(&lock_file, pid.to_string()).ok();
Ok(true)
}
/// Release the instance lock on shutdown
pub fn release_instance_lock() {
let lock_file = PathBuf::from("./botserver-stack/.lock");
if lock_file.exists() {
fs::remove_file(&lock_file).ok();
}
}
/// Reset only Vault credentials (when re-initialization is needed)
/// NEVER deletes user data in botserver-stack
fn reset_vault_only() -> Result<()> {
let vault_init = PathBuf::from("./botserver-stack/conf/vault/init.json");
let env_file = PathBuf::from("./.env"); let env_file = PathBuf::from("./.env");
if stack_dir.exists() { // Only remove vault init.json and .env - NEVER touch data/
info!("Removing existing stack directory..."); if vault_init.exists() {
fs::remove_dir_all(&stack_dir)?; info!("Removing vault init.json for re-initialization...");
info!("Stack directory removed"); fs::remove_file(&vault_init)?;
} }
if env_file.exists() { if env_file.exists() {
info!("Removing existing .env file..."); info!("Removing .env file for re-initialization...");
fs::remove_file(&env_file)?; fs::remove_file(&env_file)?;
info!(".env file removed");
} }
Ok(()) Ok(())
@ -155,21 +189,25 @@ impl BootstrapManager {
} }
} }
// Try to unseal Vault - if this fails, we need to re-bootstrap // Try to unseal Vault - if this fails, we need to re-initialize Vault only
if let Err(e) = self.ensure_vault_unsealed().await { if let Err(e) = self.ensure_vault_unsealed().await {
warn!("Vault unseal failed: {} - running re-bootstrap", e); warn!("Vault unseal failed: {} - re-initializing Vault only", e);
// Kill all processes and run fresh bootstrap // Kill only Vault process, reset only Vault credentials
Self::kill_stack_processes(); // NEVER delete user data in botserver-stack
if let Err(e) = Self::clean_stack_directory() { let _ = Command::new("pkill")
error!("Failed to clean stack directory: {}", e); .args(["-9", "-f", "botserver-stack/bin/vault"])
.output();
if let Err(e) = Self::reset_vault_only() {
error!("Failed to reset Vault: {}", e);
} }
// Run bootstrap from scratch // Run bootstrap to re-initialize Vault
self.bootstrap().await?; self.bootstrap().await?;
// After bootstrap, services are already running // After bootstrap, services are already running
info!("Re-bootstrap complete from start_all"); info!("Vault re-initialization complete");
return Ok(()); return Ok(());
} }
@ -318,22 +356,26 @@ impl BootstrapManager {
// Always try to unseal Vault (it may have restarted) // Always try to unseal Vault (it may have restarted)
// If unseal fails, Vault may need re-initialization (data deleted) // If unseal fails, Vault may need re-initialization (data deleted)
if let Err(e) = self.ensure_vault_unsealed().await { if let Err(e) = self.ensure_vault_unsealed().await {
warn!("Vault unseal failed: {} - running re-bootstrap", e); warn!("Vault unseal failed: {} - re-initializing Vault only", e);
// Kill all processes and run fresh bootstrap // Kill only Vault process - NEVER delete user data
Self::kill_stack_processes(); let _ = Command::new("pkill")
Self::clean_stack_directory()?; .args(["-9", "-f", "botserver-stack/bin/vault"])
.output();
// Run bootstrap from scratch // Reset only Vault credentials, preserve everything else
Self::reset_vault_only()?;
// Run bootstrap to re-initialize Vault
self.bootstrap().await?; self.bootstrap().await?;
// After bootstrap, services are already running // After bootstrap, services are already running
info!("Re-bootstrap complete, verifying Vault is ready..."); info!("Vault re-initialization complete, verifying...");
tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
if let Err(e) = self.ensure_vault_unsealed().await { if let Err(e) = self.ensure_vault_unsealed().await {
return Err(anyhow::anyhow!( return Err(anyhow::anyhow!(
"Failed to configure Vault after re-bootstrap: {}", "Failed to configure Vault after re-initialization: {}",
e e
)); ));
} }
@ -1292,79 +1334,120 @@ VAULT_CACHE_TTL=300
)) ))
.output(); .output();
// Store all secrets in Vault // Store secrets in Vault - ONLY if they don't already exist
info!("Storing secrets in Vault..."); // This protects existing customer data in distributed environments
info!("Storing secrets in Vault (only if not existing)...");
// Database credentials // Helper to check if a secret path exists
let _ = std::process::Command::new("sh") let secret_exists = |path: &str| -> bool {
.arg("-c") let output = std::process::Command::new("sh")
.arg(format!( .arg("-c")
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/tables host=localhost port=5432 database=botserver username=gbuser password='{}'", .arg(format!(
vault_addr, root_token, db_password "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv get {} 2>/dev/null",
)) vault_addr, root_token, path
.output()?; ))
info!(" Stored database credentials"); .output();
output.map(|o| o.status.success()).unwrap_or(false)
};
// Drive credentials // Database credentials - only create if not existing
let _ = std::process::Command::new("sh") if !secret_exists("secret/gbo/tables") {
.arg("-c") let _ = std::process::Command::new("sh")
.arg(format!( .arg("-c")
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/drive accesskey='{}' secret='{}'", .arg(format!(
vault_addr, root_token, drive_accesskey, drive_secret "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/tables host=localhost port=5432 database=botserver username=gbuser password='{}'",
)) vault_addr, root_token, db_password
.output()?; ))
info!(" Stored drive credentials"); .output()?;
info!(" Stored database credentials");
} else {
info!(" Database credentials already exist - preserving");
}
// Cache credentials // Drive credentials - only create if not existing
let _ = std::process::Command::new("sh") if !secret_exists("secret/gbo/drive") {
.arg("-c") let _ = std::process::Command::new("sh")
.arg(format!( .arg("-c")
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/cache password='{}'", .arg(format!(
vault_addr, root_token, cache_password "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/drive accesskey='{}' secret='{}'",
)) vault_addr, root_token, drive_accesskey, drive_secret
.output()?; ))
info!(" Stored cache credentials"); .output()?;
info!(" Stored drive credentials");
} else {
info!(" Drive credentials already exist - preserving");
}
// Directory placeholder (will be updated after Zitadel setup) // Cache credentials - only create if not existing
let _ = std::process::Command::new("sh") if !secret_exists("secret/gbo/cache") {
.arg("-c") let _ = std::process::Command::new("sh")
.arg(format!( .arg("-c")
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/directory url=https://localhost:8080 project_id= client_id= client_secret=", .arg(format!(
vault_addr, root_token "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/cache password='{}'",
)) vault_addr, root_token, cache_password
.output()?; ))
info!(" Created directory placeholder"); .output()?;
info!(" Stored cache credentials");
} else {
info!(" Cache credentials already exist - preserving");
}
// LLM placeholder // Directory placeholder - only create if not existing
let _ = std::process::Command::new("sh") if !secret_exists("secret/gbo/directory") {
.arg("-c") let _ = std::process::Command::new("sh")
.arg(format!( .arg("-c")
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/llm openai_key= anthropic_key= groq_key=", .arg(format!(
vault_addr, root_token "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/directory url=https://localhost:8080 project_id= client_id= client_secret=",
)) vault_addr, root_token
.output()?; ))
info!(" Created LLM placeholder"); .output()?;
info!(" Created directory placeholder");
} else {
info!(" Directory credentials already exist - preserving");
}
// Email placeholder // LLM placeholder - only create if not existing
let _ = std::process::Command::new("sh") if !secret_exists("secret/gbo/llm") {
.arg("-c") let _ = std::process::Command::new("sh")
.arg(format!( .arg("-c")
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/email username= password=", .arg(format!(
vault_addr, root_token "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/llm openai_key= anthropic_key= groq_key=",
)) vault_addr, root_token
.output()?; ))
info!(" Created email placeholder"); .output()?;
info!(" Created LLM placeholder");
} else {
info!(" LLM credentials already exist - preserving");
}
// Encryption key // Email placeholder - only create if not existing
let encryption_key = self.generate_secure_password(32); if !secret_exists("secret/gbo/email") {
let _ = std::process::Command::new("sh") let _ = std::process::Command::new("sh")
.arg("-c") .arg("-c")
.arg(format!( .arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/encryption master_key='{}'", "unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/email username= password=",
vault_addr, root_token, encryption_key vault_addr, root_token
)) ))
.output()?; .output()?;
info!(" Generated and stored encryption key"); info!(" Created email placeholder");
} else {
info!(" Email credentials already exist - preserving");
}
// Encryption key - only create if not existing (CRITICAL - never overwrite!)
if !secret_exists("secret/gbo/encryption") {
let encryption_key = self.generate_secure_password(32);
let _ = std::process::Command::new("sh")
.arg("-c")
.arg(format!(
"unset VAULT_CLIENT_CERT VAULT_CLIENT_KEY VAULT_CACERT; VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/encryption master_key='{}'",
vault_addr, root_token, encryption_key
))
.output()?;
info!(" Generated and stored encryption key");
} else {
info!(" Encryption key already exists - preserving (CRITICAL)");
}
info!("Vault setup complete!"); info!("Vault setup complete!");
info!(" Vault UI: {}/ui", vault_addr); info!(" Vault UI: {}/ui", vault_addr);
@ -1675,13 +1758,12 @@ VAULT_CACHE_TTL=300
}) })
} }
pub fn apply_migrations(&self, conn: &mut diesel::PgConnection) -> Result<()> { pub fn apply_migrations(&self, conn: &mut diesel::PgConnection) -> Result<()> {
use diesel_migrations::HarnessWithOutput;
use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};
const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");
let mut harness = HarnessWithOutput::write_to_stdout(conn); // Run migrations silently - don't output to console
if let Err(e) = harness.run_pending_migrations(MIGRATIONS) { if let Err(e) = conn.run_pending_migrations(MIGRATIONS) {
error!("Failed to apply migrations: {}", e); error!("Failed to apply migrations: {}", e);
return Err(anyhow::anyhow!("Migration error: {}", e)); return Err(anyhow::anyhow!("Migration error: {}", e));
} }