diff --git a/docs/src/assets/architecture.svg b/docs/src/assets/architecture.svg index bd8b2564d..05358ed21 100644 --- a/docs/src/assets/architecture.svg +++ b/docs/src/assets/architecture.svg @@ -1,96 +1,226 @@ - + + + - - + + + + + + + + + - General Bots Architecture + General Bots Architecture + Single binary with everything included - no external dependencies - - - - Web Server + + Interface Layer + Core Runtime + Processing - - - BASIC Interpreter + + + + + + Web Server - - - LLM Integration + + + Console UI - - - Package Manager + + + BASIC Interpreter - - - Console UI + + + LLM Integration - - - Session Manager (Tokio Async Runtime) + + + Package Manager - - - - PostgreSQL + + + BotModels (AI) - - - Valkey Cache + + + Session Manager (Tokio Async Runtime) - - - Qdrant Vectors + + + Channels - - - Object Storage + + + External API - - - Channels + + + + PostgreSQL - - - External API + + + Valkey Cache - - - - - - - - + + + Qdrant Vectors - - - - - - + + + SeaweedFS - - + + + .gbkb (Docs) | .gbdialog (Scripts) | .gbot (Config) + Templates | User Assets + + + + Zitadel (IAM) - - - - Storage Contents: - .gbkb (Documents) - .gbdialog (Scripts) - .gbot (Configs) - Templates - User Assets + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Request + Web/Console + + Process + BASIC + LLM + + Decide + AI Routing + + Execute + Tools + APIs + + Respond + Multi-Channel + + + + + Component Types: + + + Interface / Routing + + + Processing / Scripts + + + AI / ML / Decision + + + Execution / Storage + + + Output / Response - - Single binary with everything included - no external dependencies + + Rust-powered single binary serving web UI, BASIC scripting, LLM orchestration, and multi-channel messaging + + + Auto-installed infrastructure: PostgreSQL, Valkey, Qdrant, SeaweedFS, Zitadel - zero external dependencies diff --git a/docs/src/assets/botmodels-architecture.svg b/docs/src/assets/botmodels-architecture.svg new file mode 100644 index 000000000..c6146a656 --- /dev/null +++ b/docs/src/assets/botmodels-architecture.svg @@ -0,0 +1,182 @@ + + + + + + + + + + + + + + + + BotServer ↔ BotModels Integration + Rust backend calling Python AI services over HTTPS + + + + + + botserver (Rust) + + + + BASIC Interpreter + + + LLM Integration + + + + BASIC Keywords + • IMAGE prompt + • VIDEO prompt + • AUDIO text + • SEE image + • HEAR AS AUDIO + + + + config.csv + botmodels-url, botmodels-enabled + + + + botmodels (Python) + + + + Image Service + Stable Diffusion + + + Video Service + Zeroscope + + + Speech Service + TTS / Whisper + + + Vision Service + BLIP2 / QRCode + + + + FastAPI Endpoints + /api/v1/image/generate + /api/v1/video/generate + /api/v1/speech/to-text + /api/v1/vision/describe + + + + + HTTPS + JSON / Binary + + + + outputs/ (Generated Files) + Images, Videos, Audio files served via /outputs + + + + + + + + + + + + + + + + + + + + + + + + BASIC Keyword + HTTP Request + AI Processing + Generate + Return URL + + + + + + Rust Backend + + + Python AI + + + Processing + + + Output + + + Config/API + + + + + BotModels runs as a separate Python service for GPU-accelerated AI inference + + + Enable with: botmodels-enabled=true and botmodels-url=http://localhost:8001 + + diff --git a/docs/src/assets/conversation-flow.svg b/docs/src/assets/conversation-flow.svg index 19aa20366..dc4f518a0 100644 --- a/docs/src/assets/conversation-flow.svg +++ b/docs/src/assets/conversation-flow.svg @@ -1,73 +1,216 @@ - + + + - - + + + + + + + + + - - The Flow + + Conversation Flow + How General Bots processes user messages through the AI pipeline - - + + + + Input + Script + AI Decision + Execution + Response + + - - User Input + + User Input + Web / WhatsApp / Teams - - BASIC Script + + BASIC Script + start.bas / tools - - LLM Decision + + LLM Decision + Intent / Tool Selection - - Bot Executor + + Bot Executor + Run Keywords - - Bot Response - + + Bot Response + TALK / Send - - - - - Search Knowledge + + Parallel Operations - - - Call API + + + Vector Search + Qdrant / USE KB + + + + External APIs + GET / POST / GraphQL + + + + Database + TABLE / SAVE + + + + Knowledge Base + .gbkb Documents - + - - - - + + + + - - - + + + + - - - + + - - + + + + + + + - - - The AI handles everything else - understanding intent, collecting information, executing tools, + + + + + + + + + + + + + + + + + + + + Receive + Parse + Decide + Execute + Respond + + + + + Legend: + + + Main Flow + + + Parallel / Optional + + + Feedback Loop + + + + + The AI handles everything automatically - understanding intent, searching knowledge, - - answering from documents. Zero configuration. + + executing tools, and generating responses. Zero configuration required. + + + + + Multi-Channel + + + BASIC Scripting + + + LLM Powered + + + Tool Execution + + + RAG Search + + + Real-time + diff --git a/docs/src/assets/infrastructure-architecture.svg b/docs/src/assets/infrastructure-architecture.svg new file mode 100644 index 000000000..190845067 --- /dev/null +++ b/docs/src/assets/infrastructure-architecture.svg @@ -0,0 +1,248 @@ + + + + + + + + + + + + + + + + + Infrastructure Architecture + Load Balancing, Multi-Tenant Isolation, and Auto-Scaling + + + + + Clients + + + + Web + + + Mobile + + + WhatsApp + + + Teams + + + API + + + + Load Balancer (Caddy) + TLS Termination | Rate Limiting | Health Checks + + + + + + + + + + + + BotServer Instances (Auto-Scaled) + + + BotServer 1 + LXC Container + + + BotServer 2 + LXC Container + + + BotServer 3 + LXC Container + + + BotServer N + Auto-Scale + + + + + + + + + + + Shared Data Layer + + + + PostgreSQL + Relational Data + + + Valkey + Cache + Sessions + + + Qdrant + Vector Search + + + SeaweedFS + Object Storage + + + Zitadel + IAM / Auth + + + LiveKit + Real-time Meet + + + Stalwart + Email Server + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Request + Multi-Channel + + Route + Load Balance + + Process + BotServer + + Store + Data Layer + + + + + Component Types: + + + Client / Interface + + + Load Balancer + + + Compute Instance + + + Data Storage + + + Services + + + + + + Auto-Scaling + + + TLS Everywhere + + + Multi-Tenant + + + Health Checks + + + Zero Downtime + + + LXC Containers + + + + + Production-ready infrastructure with automatic scaling, load balancing, and multi-tenant isolation + + diff --git a/docs/src/assets/observability-flow.svg b/docs/src/assets/observability-flow.svg new file mode 100644 index 000000000..525fd0aae --- /dev/null +++ b/docs/src/assets/observability-flow.svg @@ -0,0 +1,255 @@ + + + + + + + + + + + + + + + + + Observability Flow + Vector Agent collects logs from BotServer without code changes + + + + + Application + Collection + Processing + Outputs + + + + BotServer Application + + + log::trace!() + log::debug!() + log::info!() + log::warn!() + log::error!() + + + ↓ writes to + + ./botserver-stack/logs/ + + + + Vector Agent + (Collects from log files) + + + + Sources + + + Transforms + + + Sinks + + + (Files) + (Parse) + (Output) + + + + + + + + Processing + + + + Filter Errors + + + Filter Warnings + + + Log to Metrics + + + Enrich Data + + + + + InfluxDB + (Metrics) + + + + Grafana + (Dashboard) + + + + Alerts + (Webhook) + + + + + + + + + + + + + + + + + + + + Log Directory Structure + logs/system/ → BotServer logs + logs/drive/ → SeaweedFS logs + logs/tables/ → PostgreSQL logs + logs/cache/ → Valkey logs + logs/vectordb/ → Qdrant + logs/email/ → Stalwart + logs/directory/ → Zitadel + logs/meet/ → LiveKit + + + + + + Vector Configuration + [sources.botserver_logs] + type = "file" + include = ["logs/system/*.log"] + [transforms.parse_botserver] + type = "remap" + + + + + + Key Benefits + ✓ Zero code changes required + ✓ Works with existing logging + ✓ Add/remove without recompile + ✓ Scales independently + + + + + + + + + + + + + + + + + + + Generate + log::*!() + + Collect + Vector Agent + + Process + Parse & Route + + Store/Alert + InfluxDB/Grafana + + + + + Automatic Metrics: + + + log_events_total + + + errors_total + + + warnings_total + + + llm_latency_seconds + + + sessions_active + + + + + Keep using standard Rust log macros - Vector handles collection, parsing, and routing + + + Install with: ./botserver install observability + + diff --git a/docs/src/assets/package-system-flow.svg b/docs/src/assets/package-system-flow.svg index 4541da085..b305f2c48 100644 --- a/docs/src/assets/package-system-flow.svg +++ b/docs/src/assets/package-system-flow.svg @@ -1,93 +1,205 @@ - + + + - - + + + + + + + + + - Package System Flow + Package System Flow + How .gbai packages orchestrate bot behavior - - - - User Request + + + + Input + Script + Intelligence + Execution + Output - - - start.bas + + + + User Request - - - LLM Engine + + + start.bas - - - Bot Response + + + LLM Engine - - - - Vector Search + + + Tool Executor - - - .gbkb docs + + + Bot Response + + + + + Vector Search + + + + .gbkb Docs + + + + External APIs + + + + + + + + + + + + + + + + + + + + + + + + Query + Context + + + + + + Package Structure + my-bot.gbai/ + ├── .gbdialog/ + └── start.bas, tools.bas + ├── .gbkb/ + └── docs, PDFs, data + └── .gbot/config.csv + - - - BASIC Commands - USE KB "docs" - answer = HEAR - result = LLM() - TALK result + + + BASIC Commands + USE KB "knowledge" + question = HEAR + answer = LLM(question) + TALK answer + // Zero configuration AI - - - - Package Structure - my-bot.gbai/ - ├─ .gbdialog/ - ├─ .gbkb/ - └─ .gbot/ + + + + Key Features + ✓ Auto tool registration + ✓ RAG from any document + ✓ Multi-channel deploy + ✓ Hot reload packages + ✓ Session persistence - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - Commands - Results - Query - Context + + + + + + + + + + + + + + + + + + + Receive + Parse Script + LLM Process + Execute Tools + Respond + - + BASIC scripts orchestrate LLM decisions, vector search, and responses with zero configuration + + Drop files in .gbkb → Write scripts in .gbdialog → Deploy instantly + diff --git a/docs/src/assets/script-execution-flow.svg b/docs/src/assets/script-execution-flow.svg new file mode 100644 index 000000000..7552c2157 --- /dev/null +++ b/docs/src/assets/script-execution-flow.svg @@ -0,0 +1,239 @@ + + + + + + + + + + + + + + + + + + + Script Execution Flow + BASIC Compilation Pipeline - From Source to Execution + + + + + Entry Points + + + + start.bas + (Bot Start) + + + + SET SCHEDULE + (Cron Jobs) + + + + WEBHOOK + (HTTP POST) + + + + BASIC Compiler & Runtime + + + + 1. Load Source + + + 2. Load Config + + + 3. Preprocess + + + 4. Compile + + + Read .bas file + param-* vars + Transform + Rhai AST + + + + 5. Execute with Injected Scope + Keywords call registered handlers → LLM Tools → API Calls + + + + + + + + + + + + + + + + + Execution Outputs + + + + LLM Tools + (ADD TOOL) + + + + ON Events + (Triggers) + + + + API Calls + (External) + + + + + + + + + + + + + + + Package Structure (mybot.gbai) + + mybot.gbai/ + ├── mybot.gbdialog/ + ├── start.bas ' Entry point + ├── tables.bas ' TABLE definitions + ├── create-order.bas ' Tool script + └── sync-data.bas ' Scheduled script + └── mybot.gbot/config.csv + + + + + + config.csv (Injected Variables) + param-api-key,sk-xxxxx + param-webhook-url,https://... + param-max-retries,3 + → Available as variables in BASIC + + + + + + Preprocessing Steps + • Case normalization (TALK → talk) + • Multi-word keyword transform + • FOR EACH block handling + • Variable scope injection + + + + + + Keyword Execution + • TALK → Send message to channel + • HEAR → Wait for user input + • LLM → Call language model + • GET/POST → HTTP requests + + + + + + + + + + + + + + + + + + + + + + + Load + Config + Preprocess + Compile + Execute + Respond + + + + + BASIC scripts compile to Rhai AST and execute with registered keyword handlers + + + Scripts in .gbdialog/ are hot-reloaded on change - no restart required + + diff --git a/docs/src/chapter-06-gbdialog/keyword-kb-collection-stats.md b/docs/src/chapter-06-gbdialog/keyword-kb-collection-stats.md new file mode 100644 index 000000000..247d368bb --- /dev/null +++ b/docs/src/chapter-06-gbdialog/keyword-kb-collection-stats.md @@ -0,0 +1 @@ +# KB COLLECTION STATS diff --git a/docs/src/chapter-06-gbdialog/keyword-kb-documents-added-since.md b/docs/src/chapter-06-gbdialog/keyword-kb-documents-added-since.md new file mode 100644 index 000000000..3ab894b2d --- /dev/null +++ b/docs/src/chapter-06-gbdialog/keyword-kb-documents-added-since.md @@ -0,0 +1 @@ +# KB DOCUMENTS ADDED SINCE diff --git a/docs/src/chapter-06-gbdialog/keyword-kb-documents-count.md b/docs/src/chapter-06-gbdialog/keyword-kb-documents-count.md new file mode 100644 index 000000000..8d068d985 --- /dev/null +++ b/docs/src/chapter-06-gbdialog/keyword-kb-documents-count.md @@ -0,0 +1 @@ +# KB DOCUMENTS COUNT diff --git a/docs/src/chapter-06-gbdialog/keyword-kb-list-collections.md b/docs/src/chapter-06-gbdialog/keyword-kb-list-collections.md new file mode 100644 index 000000000..f4960d7f3 --- /dev/null +++ b/docs/src/chapter-06-gbdialog/keyword-kb-list-collections.md @@ -0,0 +1 @@ +# KB LIST COLLECTIONS diff --git a/docs/src/chapter-06-gbdialog/keyword-kb-statistics.md b/docs/src/chapter-06-gbdialog/keyword-kb-statistics.md new file mode 100644 index 000000000..3cb12d5fc --- /dev/null +++ b/docs/src/chapter-06-gbdialog/keyword-kb-statistics.md @@ -0,0 +1 @@ +# KB STATISTICS diff --git a/docs/src/chapter-06-gbdialog/keyword-kb-storage-size.md b/docs/src/chapter-06-gbdialog/keyword-kb-storage-size.md new file mode 100644 index 000000000..7612e4be9 --- /dev/null +++ b/docs/src/chapter-06-gbdialog/keyword-kb-storage-size.md @@ -0,0 +1 @@ +# KB STORAGE SIZE diff --git a/docs/src/chapter-06-gbdialog/script-execution-flow.md b/docs/src/chapter-06-gbdialog/script-execution-flow.md index 1546a65e1..7ede5963a 100644 --- a/docs/src/chapter-06-gbdialog/script-execution-flow.md +++ b/docs/src/chapter-06-gbdialog/script-execution-flow.md @@ -6,33 +6,9 @@ Understanding how General Bots BASIC scripts are loaded, compiled, and executed Scripts in General Bots can be triggered through several entry points: -``` -┌─────────────────────────────────────────────────────────────────┐ -│ SCRIPT ENTRY POINTS │ -├─────────────────────────────────────────────────────────────────┤ -│ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ start.bas │ │ SET SCHEDULE│ │ WEBHOOK │ │ -│ │ (Bot Start) │ │ (Cron Jobs) │ │ (HTTP POST) │ │ -│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌─────────────────────────────────────────────────────────┐ │ -│ │ BASIC COMPILER & RUNTIME │ │ -│ │ 1. Load config.csv param-* variables │ │ -│ │ 2. Preprocess (case normalization, syntax transform) │ │ -│ │ 3. Compile to AST │ │ -│ │ 4. Execute with injected scope │ │ -│ └─────────────────────────────────────────────────────────┘ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ LLM Tools │ │ ON Events │ │ API Calls │ │ -│ │ (ADD TOOL) │ │ (Triggers) │ │ (External) │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────────┘ -``` +![Script Execution Flow](../assets/script-execution-flow.svg) + +*BASIC scripts compile to Rhai AST and execute with registered keyword handlers. Scripts in .gbdialog/ are hot-reloaded on change.* ### 1. Bot Startup (`start.bas`) diff --git a/docs/src/chapter-07-gbapp/infrastructure.md b/docs/src/chapter-07-gbapp/infrastructure.md index ec17c7128..1eeb56110 100644 --- a/docs/src/chapter-07-gbapp/infrastructure.md +++ b/docs/src/chapter-07-gbapp/infrastructure.md @@ -12,36 +12,11 @@ General Bots uses a modular architecture where each component runs in isolated L - **Portability**: Move containers between hosts easily ## Component Diagram +## High Availability Architecture -``` -┌──────────────────────────────────────────────────────────────────────────────┐ -│ Load Balancer (Caddy) │ -│ Rate Limiting │ TLS Termination │ -└─────────────────────────────────┬────────────────────────────────────────────┘ - │ - ┌────────────────────────┼────────────────────────┐ - │ │ │ - ▼ ▼ ▼ -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ BotServer 1 │ │ BotServer 2 │ │ BotServer N │ -│ (LXC/Auto) │ │ (LXC/Auto) │ │ (LXC/Auto) │ -└────────┬────────┘ └────────┬────────┘ └────────┬────────┘ - │ │ │ - └──────────────────────┼──────────────────────┘ - │ - ┌───────────────────────────┼───────────────────────────┐ - │ │ │ - ▼ ▼ ▼ -┌─────────┐ ┌─────────────┐ ┌──────────┐ -│ Secrets │ │ Data Layer │ │ Services │ -│ (Vault) │ │ │ │ │ -└─────────┘ │ PostgreSQL │ │ Zitadel │ - │ Redis │ │ LiveKit │ - │ Qdrant │ │ Stalwart │ - │ InfluxDB │ │ MinIO │ - │ MinIO │ │ Forgejo │ - └─────────────┘ └──────────┘ -``` +![Infrastructure Architecture](../assets/infrastructure-architecture.svg) + +*Production-ready infrastructure with automatic scaling, load balancing, and multi-tenant isolation.* ## Encryption at Rest @@ -232,23 +207,15 @@ messaging-retention-hours,24 ### Option 1: Tenant-Based Sharding (Recommended) Each tenant/organization gets isolated databases: +## Multi-Tenant Architecture -``` -┌─────────────────────────────────────────────────────────────────┐ -│ Router │ -│ (tenant_id → database mapping) │ -└─────────────────────────────┬───────────────────────────────────┘ - │ - ┌─────────────────────┼─────────────────────┐ - │ │ │ - ▼ ▼ ▼ -┌───────────────┐ ┌───────────────┐ ┌───────────────┐ -│ tenant_001 │ │ tenant_002 │ │ tenant_003 │ -│ PostgreSQL │ │ PostgreSQL │ │ PostgreSQL │ -│ Redis │ │ Redis │ │ Redis │ -│ Qdrant │ │ Qdrant │ │ Qdrant │ -└───────────────┘ └───────────────┘ └───────────────┘ -``` +Each tenant gets isolated resources with dedicated database schemas, cache namespaces, and vector collections. The router maps tenant IDs to their respective data stores automatically. + +**Key isolation features:** +- Database-per-tenant or schema-per-tenant options +- Namespace isolation in Valkey cache +- Collection isolation in Qdrant vectors +- Bucket isolation in SeaweedFS storage Configuration: @@ -332,21 +299,17 @@ shard-regions,us-east,eu-west,ap-south shard-default,us-east shard-detection,ip ``` +## Geographic Distribution -``` -┌─────────────────────────────────────────────────────────────┐ -│ Global Router │ -│ (GeoIP → Region mapping) │ -└─────────────────────────────┬───────────────────────────────┘ - │ - ┌─────────────────────┼─────────────────────┐ - │ │ │ - ▼ ▼ ▼ -┌───────────────┐ ┌───────────────┐ ┌───────────────┐ -│ US-East │ │ EU-West │ │ AP-South │ -│ Cluster │ │ Cluster │ │ Cluster │ -└───────────────┘ └───────────────┘ └───────────────┘ -``` +Global router uses GeoIP to direct users to the nearest regional cluster: + +| Region | Location | Services | +|--------|----------|----------| +| US-East | Virginia | Full cluster | +| EU-West | Frankfurt | Full cluster | +| AP-South | Singapore | Full cluster | + +Each regional cluster runs independently with data replication between regions for disaster recovery. ## Auto-Scaling with LXC @@ -394,20 +357,19 @@ RestartSec=10 WantedBy=multi-user.target ``` -### Container Lifecycle +## Container Lifecycle -``` -┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ -│ Create │ ──▶ │Configure │ ──▶ │ Start │ ──▶ │ Ready │ -│Container │ │Resources │ │BotServer │ │(In Pool) │ -└──────────┘ └──────────┘ └──────────┘ └──────────┘ - │ - ▼ -┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ -│ Delete │ ◀── │ Stop │ ◀── │ Drain │ ◀── │ Active │ -│Container │ │BotServer │ │ Conns │ │(Serving) │ -└──────────┘ └──────────┘ └──────────┘ └──────────┘ -``` +**Startup Flow:** +1. **Create** → LXC container created from template +2. **Configure** → Resources allocated (CPU, memory, storage) +3. **Start** → BotServer binary launched +4. **Ready** → Added to load balancer pool + +**Shutdown Flow:** +1. **Active** → Container serving requests +2. **Drain** → Stop accepting new connections +3. **Stop** → Graceful BotServer shutdown +4. **Delete** → Container removed (or returned to pool) ## Load Balancing @@ -502,20 +464,17 @@ States: ### Database Failover PostgreSQL with streaming replication: +## Database Replication -``` -┌──────────────┐ ┌──────────────┐ -│ Primary │ ──────▶ │ Replica │ -│ PostgreSQL │ (sync) │ PostgreSQL │ -└──────────────┘ └──────────────┘ - │ │ - └────────┬───────────────┘ - │ - ┌──────┴──────┐ - │ Patroni │ - │ (Failover) │ - └─────────────┘ -``` +PostgreSQL replication is managed by Patroni for automatic failover: + +| Component | Role | Description | +|-----------|------|-------------| +| Primary | Write leader | Handles all write operations | +| Replica | Read replica | Synchronous replication from primary | +| Patroni | Failover manager | Automatic leader election on failure | + +Failover happens automatically within seconds, with clients redirected via connection pooler. ### Graceful Degradation diff --git a/docs/src/chapter-07-gbapp/observability.md b/docs/src/chapter-07-gbapp/observability.md index ea3c4f6dc..a7c463f1b 100644 --- a/docs/src/chapter-07-gbapp/observability.md +++ b/docs/src/chapter-07-gbapp/observability.md @@ -4,38 +4,9 @@ General Bots uses a comprehensive observability stack for monitoring, logging, a ## Architecture Overview -``` -┌─────────────────────────────────────────────────────────────────────────────┐ -│ BotServer Application │ -│ │ -│ log::trace!() ──┐ │ -│ log::debug!() ──┼──▶ Log Files (./botserver-stack/logs/) │ -│ log::info!() ──┤ │ -│ log::warn!() ──┤ │ -│ log::error!() ──┘ │ -│ │ -└─────────────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ Vector Agent │ -│ (Collects from log files) │ -│ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Sources │ ──▶ │ Transforms │ ──▶ │ Sinks │ │ -│ │ (Files) │ │ (Parse) │ │ (Outputs) │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────────────────────┘ - │ - ┌─────────────────┼─────────────────┐ - │ │ │ - ▼ ▼ ▼ - ┌───────────┐ ┌───────────┐ ┌───────────┐ - │ InfluxDB │ │ Grafana │ │ Alerts │ - │ (Metrics) │ │(Dashboard)│ │(Webhook) │ - └───────────┘ └───────────┘ └───────────┘ -``` +![Observability Flow](../assets/observability-flow.svg) + +*Vector Agent collects logs from BotServer without requiring any code changes.* ## No Code Changes Required diff --git a/src/drive/mod.rs b/src/drive/mod.rs index 3329d38e6..79ef3c25f 100644 --- a/src/drive/mod.rs +++ b/src/drive/mod.rs @@ -9,6 +9,8 @@ //! - POST /files/write - Write file content //! - POST /files/delete - Delete file/folder //! - POST /files/create-folder - Create new folder +//! - GET /files/versions - List file versions +//! - POST /files/restore - Restore file to specific version #[cfg(feature = "console")] use crate::console::file_tree::FileTree; @@ -147,6 +149,44 @@ pub struct SyncStatus { pub bytes_synced: i64, } +// ===== File Versioning Structures ===== + +#[derive(Debug, Deserialize)] +pub struct VersionsQuery { + pub bucket: Option, + pub path: String, +} + +#[derive(Debug, Serialize)] +pub struct FileVersion { + pub version_id: String, + pub modified: String, + pub size: i64, + pub is_latest: bool, + pub etag: Option, +} + +#[derive(Debug, Serialize)] +pub struct VersionsResponse { + pub path: String, + pub versions: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct RestoreRequest { + pub bucket: Option, + pub path: String, + pub version_id: String, +} + +#[derive(Debug, Serialize)] +pub struct RestoreResponse { + pub success: bool, + pub message: String, + pub restored_version: String, + pub new_version_id: Option, +} + // ===== API Configuration ===== /// Configure drive API routes @@ -182,6 +222,9 @@ pub fn configure() -> Router> { .route("/files/sync/status", get(sync_status)) .route("/files/sync/start", post(start_sync)) .route("/files/sync/stop", post(stop_sync)) + // File versioning + .route("/files/versions", get(list_versions)) + .route("/files/restore", post(restore_version)) // Document processing .route("/docs/merge", post(document_processing::merge_documents)) .route("/docs/convert", post(document_processing::convert_document)) @@ -914,3 +957,122 @@ pub async fn stop_sync( message: Some("Sync stopped".to_string()), })) } + +// ===== File Versioning API ===== + +/// GET /files/versions - List all versions of a file +/// +/// SeaweedFS/S3 supports object versioning. This endpoint lists all versions +/// of a specific file, allowing users to restore previous versions. +/// +/// Query parameters: +/// - path: The file path to get versions for +/// - bucket: Optional bucket name (defaults to bot's bucket) +pub async fn list_versions( + State(state): State>, + Query(params): Query, +) -> Result, (StatusCode, Json)> { + let bucket = params.bucket.unwrap_or_else(|| "default".to_string()); + let path = params.path; + + // Get S3 client from state + let s3_client = state.s3_client.as_ref().ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ "error": "S3 storage not configured" })), + ) + })?; + + // List object versions using S3 API + let versions_result = s3_client + .list_object_versions() + .bucket(&bucket) + .prefix(&path) + .send() + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({ "error": format!("Failed to list versions: {}", e) })), + ) + })?; + + let mut versions: Vec = Vec::new(); + + // Process version list + for version in versions_result.versions() { + if version.key().unwrap_or_default() == path { + versions.push(FileVersion { + version_id: version.version_id().unwrap_or("null").to_string(), + modified: version + .last_modified() + .map(|t| t.to_string()) + .unwrap_or_else(|| chrono::Utc::now().to_rfc3339()), + size: version.size().unwrap_or(0), + is_latest: version.is_latest().unwrap_or(false), + etag: version.e_tag().map(|s| s.to_string()), + }); + } + } + + // Sort by modified date, newest first + versions.sort_by(|a, b| b.modified.cmp(&a.modified)); + + Ok(Json(VersionsResponse { + path: path.clone(), + versions, + })) +} + +/// POST /files/restore - Restore a file to a specific version +/// +/// Restores a file to a previous version by copying the old version +/// to become the new current version. The previous versions are preserved. +/// +/// Request body: +/// - path: The file path to restore +/// - version_id: The version ID to restore to +/// - bucket: Optional bucket name (defaults to bot's bucket) +pub async fn restore_version( + State(state): State>, + Json(payload): Json, +) -> Result, (StatusCode, Json)> { + let bucket = payload.bucket.unwrap_or_else(|| "default".to_string()); + let path = payload.path; + let version_id = payload.version_id; + + // Get S3 client from state + let s3_client = state.s3_client.as_ref().ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ "error": "S3 storage not configured" })), + ) + })?; + + // Copy the specific version to itself, making it the latest version + // S3 copy with version-id copies that version as a new object + let copy_source = format!("{}/{}?versionId={}", bucket, path, version_id); + + let copy_result = s3_client + .copy_object() + .bucket(&bucket) + .key(&path) + .copy_source(©_source) + .send() + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({ "error": format!("Failed to restore version: {}", e) })), + ) + })?; + + let new_version_id = copy_result.version_id().map(|s| s.to_string()); + + Ok(Json(RestoreResponse { + success: true, + message: format!("Successfully restored {} to version {}", path, version_id), + restored_version: version_id, + new_version_id, + })) +} diff --git a/templates/drive.html b/templates/drive.html new file mode 100644 index 000000000..c3a691abe --- /dev/null +++ b/templates/drive.html @@ -0,0 +1,527 @@ + + + + + + Drive - General Bots + + + + + + + + + +
+ + + + +
+ +
+
+ My Drive +
+ +
+ + + + + +
+
+ + +
+ + + + + +
+
+
+
+ + + +
+ + + + + + + + + + + + + + + +