diff --git a/Cargo.toml b/Cargo.toml index fc727da3..b7e250df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ repository = "https://github.com/GeneralBots/BotServer" [features] # ===== DEFAULT FEATURE SET ===== -default = ["ui-server", "chat", "automation", "tasks", "drive", "llm", "redis-cache", "progress-bars", "directory"] +default = ["ui-server", "console", "chat", "automation", "tasks", "drive", "llm", "redis-cache", "progress-bars", "directory"] # ===== UI FEATURES ===== desktop = ["dep:tauri", "dep:tauri-plugin-dialog", "dep:tauri-plugin-opener", "ui-server"] diff --git a/README.md b/README.md index 3f978881..d9541210 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,34 @@ General Bots is a **self-hosted AI automation platform** that provides: - ✅ **Git-like Version Control** - Full history with rollback capabilities - ✅ **Contract Analysis** - Legal document review and summary +## 🎮 Command-Line Options + +```bash +# Run with default settings (console UI enabled) +cargo run + +# Run without console UI +cargo run -- --noconsole + +# Run in desktop mode (Tauri) +cargo run -- --desktop + +# Run without any UI +cargo run -- --noui + +# Specify tenant +cargo run -- --tenant + +# Container mode +cargo run -- --container +``` + +### Default Behavior +- **Console UI is enabled by default** - Shows real-time system status, logs, and file browser +- Use `--noconsole` to disable the terminal UI and run as a background service +- The HTTP server always runs on port 8080 unless in desktop mode + + ## 🏆 Key Features ### 4 Essential Keywords diff --git a/docs/QUICK_START.md b/docs/QUICK_START.md new file mode 100644 index 00000000..c3a4d409 --- /dev/null +++ b/docs/QUICK_START.md @@ -0,0 +1,264 @@ +# Quick Start Guide + +## Prerequisites + +- Rust 1.75+ and Cargo +- PostgreSQL 14+ (or Docker) +- Optional: MinIO for S3-compatible storage +- Optional: Redis/Valkey for caching + +## Installation + +### 1. Clone the Repository + +```bash +git clone https://github.com/GeneralBots/BotServer.git +cd BotServer +``` + +### 2. Build the Project + +```bash +# Build with default features (includes console UI) +cargo build + +# Build with all features +cargo build --all-features + +# Build for release +cargo build --release +``` + +## Running BotServer + +### Default Mode (with Console UI) + +```bash +# Run with console UI showing real-time status, logs, and file browser +cargo run + +# The console UI provides: +# - System metrics (CPU, Memory, GPU if available) +# - Service status monitoring +# - Real-time logs +# - File browser for drive storage +# - Database status +``` + +### Background Service Mode + +```bash +# Run without console UI (background service) +cargo run -- --noconsole + +# Run without any UI +cargo run -- --noui +``` + +### Desktop Mode + +```bash +# Run with Tauri desktop application +cargo run -- --desktop +``` + +### Advanced Options + +```bash +# Specify a tenant +cargo run -- --tenant my-organization + +# Container deployment mode +cargo run -- --container + +# Combine options +cargo run -- --noconsole --tenant production +``` + +## First-Time Setup + +When you run BotServer for the first time, it will: + +1. **Automatically start required services:** + - PostgreSQL database (if not running) + - MinIO S3-compatible storage (if configured) + - Redis cache (if configured) + +2. **Run database migrations automatically:** + - Creates all required tables and indexes + - Sets up initial schema + +3. **Bootstrap initial configuration:** + - Creates `.env` file with defaults + - Sets up bot templates + - Configures service endpoints + +## Configuration + +### Environment Variables + +Copy the example environment file and customize: + +```bash +cp .env.example .env +``` + +Key configuration options in `.env`: + +```bash +# Database +DATABASE_URL=postgres://postgres:postgres@localhost:5432/botserver + +# Server +SERVER_HOST=127.0.0.1 +SERVER_PORT=8080 + +# Drive (MinIO) +DRIVE_SERVER=http://localhost:9000 +DRIVE_ACCESSKEY=minioadmin +DRIVE_SECRET=minioadmin + +# LLM Configuration +LLM_SERVER=http://localhost:8081 +LLM_MODEL=llama2 + +# Logging (automatically configured) +# All external library traces are suppressed by default +# Use RUST_LOG=botserver=trace for detailed debugging +RUST_LOG=info +``` + +## Accessing the Application + +### Web Interface + +Once running, access the web interface at: +``` +http://localhost:8080 +``` + +### API Endpoints + +- Health Check: `GET http://localhost:8080/api/health` +- Chat: `POST http://localhost:8080/api/chat` +- Tasks: `GET/POST http://localhost:8080/api/tasks` +- Drive: `GET http://localhost:8080/api/drive/files` + +## Console UI Controls + +When running with the console UI (default): + +### Keyboard Shortcuts + +- `Tab` - Switch between panels +- `↑/↓` - Navigate lists +- `Enter` - Select/Open +- `Esc` - Go back/Cancel +- `q` - Quit application +- `l` - View logs +- `f` - File browser +- `s` - System status +- `h` - Help + +### Panels + +1. **Status Panel** - System metrics and service health +2. **Logs Panel** - Real-time application logs +3. **File Browser** - Navigate drive storage +4. **Database Panel** - Connection status and stats + +## Troubleshooting + +### Database Connection Issues + +```bash +# Check if PostgreSQL is running +ps aux | grep postgres + +# Start PostgreSQL manually if needed +sudo systemctl start postgresql + +# Verify connection +psql -U postgres -h localhost +``` + +### Drive/MinIO Issues + +```bash +# Check if MinIO is running +ps aux | grep minio + +# Start MinIO manually +./botserver-stack/bin/drive/minio server ./botserver-stack/data/drive +``` + +### Console UI Not Showing + +```bash +# Ensure console feature is compiled +cargo build --features console + +# Check terminal compatibility +echo $TERM # Should be xterm-256color or similar +``` + +### High CPU/Memory Usage + +```bash +# Run without console for lower resource usage +cargo run -- --noconsole + +# Check running services +htop +``` + +## Development + +### Running Tests + +```bash +# Run all tests +cargo test + +# Run with specific features +cargo test --features console + +# Run integration tests +cargo test --test '*' +``` + +### Enable Detailed Logging + +```bash +# Trace level for botserver only +RUST_LOG=botserver=trace cargo run + +# Debug level +RUST_LOG=botserver=debug cargo run + +# Info level (default) +RUST_LOG=info cargo run +``` + +### Building Documentation + +```bash +# Build and open Rust documentation +cargo doc --open + +# Build book documentation +cd docs && mdbook build +``` + +## Next Steps + +1. **Configure your first bot** - See [Bot Configuration Guide](./BOT_CONFIGURATION.md) +2. **Set up integrations** - See [Integration Guide](./05-INTEGRATION_STATUS.md) +3. **Deploy to production** - See [Deployment Guide](./DEPLOYMENT.md) +4. **Explore the API** - See [API Documentation](./API.md) + +## Getting Help + +- **Documentation**: [Complete Docs](./INDEX.md) +- **Issues**: [GitHub Issues](https://github.com/GeneralBots/BotServer/issues) +- **Community**: [Discussions](https://github.com/GeneralBots/BotServer/discussions) \ No newline at end of file diff --git a/src/core/bootstrap/mod.rs b/src/core/bootstrap/mod.rs index 9bdcec92..4dc72c21 100644 --- a/src/core/bootstrap/mod.rs +++ b/src/core/bootstrap/mod.rs @@ -6,7 +6,7 @@ use anyhow::Result; use aws_config::BehaviorVersion; use aws_sdk_s3::Client; use dotenvy::dotenv; -use log::{error, info, trace}; +use log::{error, info, trace, warn}; use rand::distr::Alphanumeric; use std::io::{self, Write}; use std::path::{Path, PathBuf}; @@ -21,14 +21,14 @@ pub struct BootstrapManager { pub tenant: Option, } impl BootstrapManager { - pub async fn new(install_mode: InstallMode, tenant: Option) -> Self { + pub async fn new(mode: InstallMode, tenant: Option) -> Self { trace!( "Initializing BootstrapManager with mode {:?} and tenant {:?}", - install_mode, + mode, tenant ); Self { - install_mode, + install_mode: mode, tenant, } } @@ -60,7 +60,17 @@ impl BootstrapManager { ]; for component in components { if pm.is_installed(component.name) { - pm.start(component.name)?; + match pm.start(component.name) { + Ok(_child) => { + trace!("Started component: {}", component.name); + } + Err(e) => { + warn!( + "Component {} might already be running: {}", + component.name, e + ); + } + } } } Ok(()) @@ -76,6 +86,54 @@ impl BootstrapManager { .collect() } + /// Ensure critical services (tables and drive) are running + pub async fn ensure_services_running(&mut self) -> Result<()> { + info!("Ensuring critical services are running..."); + + let installer = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?; + + // Check and start PostgreSQL + if installer.is_installed("tables") { + info!("Starting PostgreSQL database service..."); + match installer.start("tables") { + Ok(_child) => { + info!("PostgreSQL started successfully"); + // Give PostgreSQL time to initialize + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + } + Err(e) => { + // Check if it's already running (start might fail if already running) + warn!( + "PostgreSQL might already be running or failed to start: {}", + e + ); + } + } + } else { + warn!("PostgreSQL (tables) component not installed"); + } + + // Check and start MinIO + if installer.is_installed("drive") { + info!("Starting MinIO drive service..."); + match installer.start("drive") { + Ok(_child) => { + info!("MinIO started successfully"); + // Give MinIO time to initialize + tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + } + Err(e) => { + // MinIO is not critical, just log + warn!("MinIO might already be running or failed to start: {}", e); + } + } + } else { + warn!("MinIO (drive) component not installed"); + } + + Ok(()) + } + pub async fn bootstrap(&mut self) -> Result<()> { let env_path = std::env::current_dir().unwrap().join(".env"); let db_password = self.generate_secure_password(32); diff --git a/src/core/shared/utils.rs b/src/core/shared/utils.rs index d5c54383..bf95740e 100644 --- a/src/core/shared/utils.rs +++ b/src/core/shared/utils.rs @@ -1,4 +1,7 @@ +use crate::config::DriveConfig; use anyhow::{Context, Result}; +use aws_config::BehaviorVersion; +use aws_sdk_s3::{config::Builder as S3ConfigBuilder, Client as S3Client}; use diesel::Connection; use diesel::{ r2d2::{ConnectionManager, Pool}, @@ -13,10 +16,9 @@ use smartstring::SmartString; use std::error::Error; use tokio::fs::File as TokioFile; use tokio::io::AsyncWriteExt; -use aws_sdk_s3::{Client as S3Client, config::Builder as S3ConfigBuilder}; -use aws_config::BehaviorVersion; -use crate::config::DriveConfig; -pub async fn create_s3_operator(config: &DriveConfig) -> Result> { +pub async fn create_s3_operator( + config: &DriveConfig, +) -> Result> { let endpoint = if !config.server.ends_with('/') { format!("{}/", config.server) } else { @@ -25,15 +27,13 @@ pub async fn create_s3_operator(config: &DriveConfig) -> Result Result { } pub type DbPool = Pool>; pub fn create_conn() -> Result { - let database_url = std::env::var("DATABASE_URL") - .unwrap(); + let database_url = std::env::var("DATABASE_URL").unwrap(); let manager = ConnectionManager::::new(database_url); Pool::builder().build(manager) } @@ -160,5 +159,29 @@ pub fn parse_database_url(url: &str) -> (String, String, String, u32, String) { } } } - ("".to_string(), "".to_string(), "".to_string(), 5432, "".to_string()) + ( + "".to_string(), + "".to_string(), + "".to_string(), + 5432, + "".to_string(), + ) +} + +/// Run database migrations +pub fn run_migrations(pool: &DbPool) -> Result<(), Box> { + use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; + + const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); + + let mut conn = pool.get()?; + conn.run_pending_migrations(MIGRATIONS).map_err( + |e| -> Box { + Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Migration error: {}", e), + )) + }, + )?; + Ok(()) } diff --git a/src/main.rs b/src/main.rs index 689fe1c7..97a93368 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ use axum::{ Router, }; use dotenvy::dotenv; -use log::{error, info, trace}; +use log::{error, info, trace, warn}; use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; @@ -212,6 +212,7 @@ async fn main() -> std::io::Result<()> { // Initialize logger early to capture all logs with filters for noisy libraries let rust_log = std::env::var("RUST_LOG").unwrap_or_else(|_| { // Default log level for botserver and suppress all other crates + // Note: r2d2 is set to warn to see database connection pool warnings "info,botserver=info,\ aws_sigv4=off,aws_smithy_checksums=off,aws_runtime=off,aws_smithy_http_client=off,\ aws_smithy_runtime=off,aws_smithy_runtime_api=off,aws_sdk_s3=off,aws_config=off,\ @@ -220,7 +221,7 @@ async fn main() -> std::io::Result<()> { reqwest=off,hyper=off,hyper_util=off,h2=off,\ rustls=off,rustls_pemfile=off,tokio_rustls=off,\ tracing=off,tracing_core=off,tracing_subscriber=off,\ - diesel=off,diesel_migrations=off,r2d2=off,\ + diesel=off,diesel_migrations=off,r2d2=warn,\ serde=off,serde_json=off,\ axum=off,axum_core=off,\ tonic=off,prost=off,\ @@ -255,7 +256,7 @@ async fn main() -> std::io::Result<()> { let args: Vec = std::env::args().collect(); let no_ui = args.contains(&"--noui".to_string()); let desktop_mode = args.contains(&"--desktop".to_string()); - let console_mode = args.contains(&"--console".to_string()); + let no_console = args.contains(&"--noconsole".to_string()); dotenv().ok(); @@ -281,10 +282,8 @@ async fn main() -> std::io::Result<()> { } } - // Start UI thread if console mode is explicitly requested or if not in no-ui mode and not in desktop mode - let ui_handle: Option> = if console_mode - || (!no_ui && !desktop_mode) - { + // Start UI thread if console is enabled (default) and not disabled by --noconsole or desktop mode + let ui_handle: Option> = if !no_console && !desktop_mode && !no_ui { #[cfg(feature = "console")] { let progress_rx = Arc::new(tokio::sync::Mutex::new(_progress_rx)); @@ -327,10 +326,8 @@ async fn main() -> std::io::Result<()> { } #[cfg(not(feature = "console"))] { - if console_mode { - eprintln!("Console mode requested but console feature not enabled. Rebuild with --features console"); - } else { - eprintln!("Console feature not enabled"); + if !no_console { + eprintln!("Console feature not compiled. Rebuild with --features console or use --noconsole to suppress this message"); } None } @@ -364,13 +361,20 @@ async fn main() -> std::io::Result<()> { trace!("Checking for .env file at: {:?}", env_path); let cfg = if env_path.exists() { - trace!(".env file exists, starting all services..."); + trace!(".env file exists, ensuring all services are running..."); + info!("Ensuring database and drive services are running..."); progress_tx_clone .send(BootstrapProgress::StartingComponent( "all services".to_string(), )) .ok(); trace!("Calling bootstrap.start_all()..."); + + // Ensure critical services are started + if let Err(e) = bootstrap.ensure_services_running().await { + warn!("Some services might not be running: {}", e); + } + bootstrap .start_all() .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; @@ -449,7 +453,19 @@ async fn main() -> std::io::Result<()> { progress_tx.send(BootstrapProgress::ConnectingDatabase).ok(); let pool = match create_conn() { - Ok(pool) => pool, + Ok(pool) => { + // Run automatic migrations + trace!("Running database migrations..."); + info!("Running database migrations..."); + if let Err(e) = crate::shared::utils::run_migrations(&pool) { + error!("Failed to run migrations: {}", e); + // Continue anyway as some migrations might have already been applied + warn!("Continuing despite migration errors - database might be partially migrated"); + } else { + info!("Database migrations completed successfully"); + } + pool + } Err(e) => { error!("Failed to create database pool: {}", e); progress_tx