Compare commits

...

12 commits

Author SHA1 Message Date
5fb4c889b7 fix(llm-config): Fix ConfigManager fallback logic for LLM configuration
Some checks failed
GBCI / build (push) Failing after 12m26s
- Fix ConfigManager to treat 'none', 'null', 'n/a', and empty values as placeholders
  and fall back to default bot's configuration instead of using these as literal values

- Fix ConfigManager to detect local file paths (e.g., .gguf, .bin, ../) and fall back
  to default bot's model when using remote API, allowing bots to keep local model
  config for local LLM server while automatically using remote model for API calls

- Fix get_default_bot() to return the bot actually named 'default' instead of
  the first active bot by ID, ensuring consistent fallback behavior

- Add comprehensive debug logging to trace LLM configuration from database to API call

This fixes the issue where bots with incomplete or local LLM configuration would
fail with 401/400 errors when trying to use remote API, instead of automatically
falling back to the default bot's configuration from config.csv.

Closes: #llm-config-fallback
2026-02-02 19:20:37 -03:00
39c4dba838 feat: Add template validation system with .valid file
- Modify bootstrap to read .valid file and validate templates before loading
- Templates not in .valid file are skipped during bootstrap
- Backward compatible: if .valid file missing, all templates are loaded
- Enables controlled template loading during bootstrap
2026-02-01 14:20:35 -03:00
748fceff5d Fix issues: remove unused import, fix ownership error, reduce crawler interval 2026-01-30 12:21:30 -03:00
94fede7cc4 feat: Add search_enabled and menu_launcher_enabled directives to .product file
- Add search_enabled field to ProductConfig to control omnibox visibility (defaults to false)
- Add menu_launcher_enabled field to ProductConfig to control apps menu button visibility (defaults to false)
- Update .product file to set both directives to false by default
- Update get_product_config_json to include new fields in API response
- Parse search_enabled and menu_launcher_enabled from .product file with support for true/false, 1/0, yes/no values

This allows disabling the suite search mechanism and hiding the menu launcher when empty,
providing a cleaner UI for deployments that don't need these features.
2026-01-29 23:55:50 -03:00
1f7cdfa9cf Fix conditional compilation for Windows-specific security methods
- Wrapped Windows security configuration code blocks in #[cfg(windows)] attributes
- Removed nested cfg attributes that were causing compilation errors
- Properly separated Windows and Linux code paths using compile-time attributes
- Fixed calls to configure_windows_security() and update_windows_signatures()
2026-01-28 20:11:18 -03:00
26963f2caf Fix bot_id: Use bot_id from URL path instead of client message
- Extract bot_name from WebSocket query parameters
- Look up bot_id from bot_name using database
- Pass bot_id to WebSocket message handler
- Use session's bot_id for LLM configuration instead of client-provided bot_id
- Fixes issue where client sends 'default' bot_id when accessing /edu
2026-01-28 17:18:22 -03:00
51c8a53a90 Enable LLM feature by default and fix compilation errors
- Add llm to default features in Cargo.toml
- Fix duplicate smart_router module declaration
- Remove unused LLMProvider import and fix unused variable warnings
- Fix move error in enhanced_llm.rs by cloning state separately for each closure
- Improve code formatting and consistency
2026-01-28 16:58:14 -03:00
a51e3a0758 chore: Set default RUST_LOG to info and revert logger init to info 2026-01-27 18:45:41 -03:00
0ccf7f8971 chore: Set default internal log level to trace 2026-01-27 18:39:48 -03:00
e7fa5bf72c Auto-enable noconsole mode when console feature is disabled 2026-01-27 16:28:36 -03:00
5568ef5802 Fix migration 6.0.0: Comment out sent_email_tracking 2026-01-27 14:14:20 -03:00
b103c07248 Fix migration errors and reorganize migration files
- Fixed 'relation session_kb_associations does not exist' error in core consolidated migration.
- Renamed migration directories from timestamp-based to version-based (6.0.x, 6.1.x, 6.2.x).
- Reorganized migrations into dedicated feature folders (products, dashboards, learn, video).
- Updated migration execution order in core/shared/utils.rs.
- Moves legacy migrations to 6.0.x/6.1.x and workflow to 6.2.0.
2026-01-27 13:45:54 -03:00
94 changed files with 3124 additions and 1002 deletions

View file

@ -14,6 +14,17 @@ name=General Bots
# Only listed apps will be visible in the UI and have their APIs enabled.
apps=chat,drive,tasks,sources,settings
# Search mechanism enabled
# Controls whether the omnibox/search toolbar is displayed in the suite
# Set to false to disable the search mechanism
search_enabled=false
# Menu launcher enabled
# Controls whether the apps menu launcher is displayed in the suite
# Set to false to hide the menu launcher button
# When the menu is empty (no apps to show), it will be automatically hidden
menu_launcher_enabled=false
# Default theme
# Available themes: dark, light, blue, purple, green, orange, sentient, cyberpunk,
# retrowave, vapordream, y2kglow, arcadeflash, discofever, grungeera,

View file

@ -2,15 +2,15 @@
{
"label": "Debug BotServer",
"build": {
"command": "rm -rf .env ./botserver-stack && cargo",
"args": ["build"]
"command": "cargo",
"args": ["build"],
},
"program": "$ZED_WORKTREE_ROOT/target/debug/botserver",
"env": {
"RUST_LOG": "trace"
"RUST_LOG": "trace",
},
"sourceLanguages": ["rust"],
"request": "launch",
"adapter": "CodeLLDB"
}
"adapter": "CodeLLDB",
},
]

View file

@ -10,7 +10,7 @@ features = ["database", "i18n"]
[features]
# ===== DEFAULT =====
default = ["chat", "automation", "drive", "tasks", "cache", "directory"]
default = ["chat", "automation", "drive", "tasks", "cache", "directory", "llm"]
# ===== CORE INFRASTRUCTURE (Can be used standalone) =====
scripting = ["dep:rhai"]

259
PROMPT.md
View file

@ -1,259 +0,0 @@
# botserver Development Guide
**Version:** 6.2.0
**Purpose:** Main API server for General Bots (Axum + Diesel + Rhai BASIC + HTMX in botui)
---
## ZERO TOLERANCE POLICY
**EVERY SINGLE WARNING MUST BE FIXED. NO EXCEPTIONS.**
---
## ❌ ABSOLUTE PROHIBITIONS
```
❌ NEVER use #![allow()] or #[allow()] in source code
❌ NEVER use .unwrap() - use ? or proper error handling
❌ NEVER use .expect() - use ? or proper error handling
❌ NEVER use panic!() or unreachable!()
❌ NEVER use todo!() or unimplemented!()
❌ NEVER leave unused imports or dead code
❌ NEVER use approximate constants - use std::f64::consts
❌ NEVER use CDN links - all assets must be local
❌ NEVER add comments - code must be self-documenting
❌ NEVER build SQL queries with format! - use parameterized queries
❌ NEVER pass user input to Command::new() without validation
❌ NEVER log passwords, tokens, API keys, or PII
```
---
## 🔐 SECURITY REQUIREMENTS
### Error Handling
```rust
// ❌ WRONG
let value = something.unwrap();
let value = something.expect("msg");
// ✅ CORRECT
let value = something?;
let value = something.ok_or_else(|| Error::NotFound)?;
let value = something.unwrap_or_default();
```
### Rhai Syntax Registration
```rust
// ❌ WRONG
engine.register_custom_syntax([...], false, |...| {...}).unwrap();
// ✅ CORRECT
if let Err(e) = engine.register_custom_syntax([...], false, |...| {...}) {
log::warn!("Failed to register syntax: {e}");
}
```
### Regex Patterns
```rust
// ❌ WRONG
let re = Regex::new(r"pattern").unwrap();
// ✅ CORRECT
static RE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"pattern").expect("invalid regex")
});
```
### Tokio Runtime
```rust
// ❌ WRONG
let rt = tokio::runtime::Runtime::new().unwrap();
// ✅ CORRECT
let Ok(rt) = tokio::runtime::Runtime::new() else {
return Err("Failed to create runtime".into());
};
```
### SQL Injection Prevention
```rust
// ❌ WRONG
let query = format!("SELECT * FROM {}", table_name);
// ✅ CORRECT - whitelist validation
const ALLOWED_TABLES: &[&str] = &["users", "sessions"];
if !ALLOWED_TABLES.contains(&table_name) {
return Err(Error::InvalidTable);
}
```
### Command Injection Prevention
```rust
// ❌ WRONG
Command::new("tool").arg(user_input).output()?;
// ✅ CORRECT
fn validate_input(s: &str) -> Result<&str, Error> {
if s.chars().all(|c| c.is_alphanumeric() || c == '.') {
Ok(s)
} else {
Err(Error::InvalidInput)
}
}
let safe = validate_input(user_input)?;
Command::new("/usr/bin/tool").arg(safe).output()?;
```
---
## ✅ CODE PATTERNS
### Format Strings - Inline Variables
```rust
// ❌ WRONG
format!("Hello {}", name)
// ✅ CORRECT
format!("Hello {name}")
```
### Self Usage in Impl Blocks
```rust
// ❌ WRONG
impl MyStruct {
fn new() -> MyStruct { MyStruct { } }
}
// ✅ CORRECT
impl MyStruct {
fn new() -> Self { Self { } }
}
```
### Derive Eq with PartialEq
```rust
// ❌ WRONG
#[derive(PartialEq)]
struct MyStruct { }
// ✅ CORRECT
#[derive(PartialEq, Eq)]
struct MyStruct { }
```
### Option Handling
```rust
// ✅ CORRECT
opt.unwrap_or(default)
opt.unwrap_or_else(|| compute_default())
opt.map_or(default, |x| transform(x))
```
### Chrono DateTime
```rust
// ❌ WRONG
date.with_hour(9).unwrap().with_minute(0).unwrap()
// ✅ CORRECT
date.with_hour(9).and_then(|d| d.with_minute(0)).unwrap_or(date)
```
---
## 📁 KEY DIRECTORIES
```
src/
├── core/ # Bootstrap, config, routes
├── basic/ # Rhai BASIC interpreter
│ └── keywords/ # BASIC keyword implementations
├── security/ # Security modules
├── shared/ # Shared types, models
├── tasks/ # AutoTask system
└── auto_task/ # App generator
```
---
## 🗄️ DATABASE STANDARDS
- **TABLES AND INDEXES ONLY** (no views, triggers, functions)
- **JSON columns:** use TEXT with `_json` suffix
- **ORM:** Use diesel - no sqlx
- **Migrations:** Located in `botserver/migrations/`
---
## 🎨 FRONTEND RULES
- **Use HTMX** - minimize JavaScript
- **NO external CDN** - all assets local
- **Server-side rendering** with Askama templates
---
## 📦 KEY DEPENDENCIES
| Library | Version | Purpose |
|---------|---------|---------|
| axum | 0.7.5 | Web framework |
| diesel | 2.1 | PostgreSQL ORM |
| tokio | 1.41 | Async runtime |
| rhai | git | BASIC scripting |
| reqwest | 0.12 | HTTP client |
| serde | 1.0 | Serialization |
| askama | 0.12 | HTML Templates |
---
## 🚀 CI/CD WORKFLOW
When configuring CI/CD pipelines (e.g., Forgejo Actions):
- **Minimal Checkout**: Clone only the root `gb` and the `botlib` submodule. Do NOT recursively clone everything.
- **BotServer Context**: Replace the empty `botserver` directory with the current set of files being tested.
**Example Step:**
```yaml
- name: Setup Workspace
run: |
# 1. Clone only the root workspace configuration
git clone --depth 1 <your-git-repo-url> workspace
# 2. Setup only the necessary dependencies (botlib)
cd workspace
git submodule update --init --depth 1 botlib
cd ..
# 3. Inject current BotServer code
rm -rf workspace/botserver
mv botserver workspace/botserver
```
---
## 🔑 REMEMBER
- **ZERO WARNINGS** - fix every clippy warning
- **ZERO COMMENTS** - no comments, no doc comments
- **NO ALLOW IN CODE** - configure exceptions in Cargo.toml only
- **NO DEAD CODE** - delete unused code
- **NO UNWRAP/EXPECT** - use ? or combinators
- **PARAMETERIZED SQL** - never format! for queries
- **VALIDATE COMMANDS** - never pass raw user input
- **INLINE FORMAT ARGS** - `format!("{name}")` not `format!("{}", name)`
- **USE SELF** - in impl blocks, use Self not type name
- **Version 6.2.0** - do not change without approval

512
README.md
View file

@ -1,69 +1,79 @@
# General Bots - Enterprise-Grade LLM Orchestrator
**Version:** 6.2.0
**Purpose:** Main API server for General Bots (Axum + Diesel + Rhai BASIC)
---
![General Bot Logo](https://github.com/GeneralBots/botserver/blob/main/logo.png?raw=true)
**A strongly-typed LLM conversational platform focused on convention over configuration and code-less approaches.**
## Overview
## Quick Links
General Bots is a **self-hosted AI automation platform** and strongly-typed LLM conversational platform focused on convention over configuration and code-less approaches. It serves as the core API server handling LLM orchestration, business logic, database operations, and multi-channel communication.
- **[Getting Started](docs/guides/getting-started.md)** - Installation and first bot
- **[API Reference](docs/api/README.md)** - REST and WebSocket endpoints
- **[BASIC Language](docs/reference/basic-language.md)** - Dialog scripting reference
For comprehensive documentation, see **[docs.pragmatismo.com.br](https://docs.pragmatismo.com.br)** or the **[BotBook](../botbook)** for detailed guides, API references, and tutorials.
## What is General Bots?
---
General Bots is a **self-hosted AI automation platform** that provides:
- **Multi-Vendor LLM API** - Unified interface for OpenAI, Groq, Claude, Anthropic
- **MCP + LLM Tools Generation** - Instant tool creation from code/functions
- **Semantic Caching** - Intelligent response caching (70% cost reduction)
- **Web Automation Engine** - Browser automation + AI intelligence
- **Enterprise Data Connectors** - CRM, ERP, database native integrations
- **Git-like Version Control** - Full history with rollback capabilities
## Quick Start
## 🚀 Quick Start
### Prerequisites
- **Rust** (1.75+) - [Install from rustup.rs](https://rustup.rs/)
- **Git** - [Download from git-scm.com](https://git-scm.com/downloads)
- Mold sudo apt-get install mold
- **Mold** - `sudo apt-get install mold`
### Installation
```bash
git clone https://github.com/GeneralBots/botserver
cd botserver
cargo run
```
cargo install sccache
sudo apt-get install mold # or build from source
On first run, botserver automatically sets up PostgreSQL, S3 storage, Redis cache, and downloads AI models.
The server will be available at `http://localhost:8080`.
## Documentation
```
docs/
├── api/ # API documentation
│ ├── README.md # API overview
│ ├── rest-endpoints.md # HTTP endpoints
│ └── websocket.md # Real-time communication
├── guides/ # How-to guides
│ ├── getting-started.md # Quick start
│ ├── deployment.md # Production setup
│ └── templates.md # Using templates
└── reference/ # Technical reference
├── basic-language.md # BASIC keywords
├── configuration.md # Config options
└── architecture.md # System design
cargo run
```
## Key Features
On first run, botserver automatically:
- Installs required components (PostgreSQL, S3 storage, Redis cache, LLM)
- Sets up database with migrations
- Downloads AI models
- Starts HTTP server at `http://localhost:8088`
### 4 Essential Keywords
### Command-Line Options
```bash
cargo run # Default: console UI + web server
cargo run -- --noconsole # Background service mode
cargo run -- --desktop # Desktop application (Tauri)
cargo run -- --tenant <name> # Specify tenant
cargo run -- --container # LXC container mode
```
---
## ✨ Key Features
### Multi-Vendor LLM API
Unified interface for OpenAI, Groq, Claude, Anthropic, and local models.
### MCP + LLM Tools Generation
Instant tool creation from code and functions - no complex configurations.
### Semantic Caching
Intelligent response caching achieving **70% cost reduction** on LLM calls.
### Web Automation Engine
Browser automation combined with AI intelligence for complex workflows.
### Enterprise Data Connectors
Native integrations with CRM, ERP, databases, and external services.
### Git-like Version Control
Full history with rollback capabilities for all configurations and data.
---
## 🎯 4 Essential Keywords
```basic
USE KB "kb-name" ' Load knowledge base into vector database
@ -85,66 +95,347 @@ SET CONTEXT "support" AS "You are a helpful customer support agent."
TALK "Welcome! How can I help you today?"
```
## Command-Line Options
---
```bash
cargo run # Default: console UI + web server
cargo run -- --noconsole # Background service mode
cargo run -- --desktop # Desktop application (Tauri)
cargo run -- --tenant <name> # Specify tenant
cargo run -- --container # LXC container mode
## 📁 Project Structure
```
src/
├── core/ # Bootstrap, config, routes
├── basic/ # Rhai BASIC interpreter
│ └── keywords/ # BASIC keyword implementations
├── security/ # Security modules
│ ├── command_guard.rs # Safe command execution
│ ├── error_sanitizer.rs # Error message sanitization
│ └── sql_guard.rs # SQL injection prevention
├── shared/ # Shared types, models
├── tasks/ # AutoTask system (2651 lines - NEEDS REFACTORING)
├── auto_task/ # App generator (2981 lines - NEEDS REFACTORING)
├── drive/ # File operations (1522 lines - NEEDS REFACTORING)
├── learn/ # Learning system (2306 lines - NEEDS REFACTORING)
└── attendance/ # LLM assistance (2053 lines - NEEDS REFACTORING)
migrations/ # Database migrations
botserver-stack/ # Stack deployment files
```
## Environment Variables
---
Only directory service variables are required:
## ✅ ZERO TOLERANCE POLICY
| Variable | Purpose |
|----------|---------|
| `DIRECTORY_URL` | Zitadel instance URL |
| `DIRECTORY_CLIENT_ID` | OAuth client ID |
| `DIRECTORY_CLIENT_SECRET` | OAuth client secret |
**EVERY SINGLE WARNING MUST BE FIXED. NO EXCEPTIONS.**
All service credentials are managed automatically. See [Configuration](docs/reference/configuration.md) for details.
### Absolute Prohibitions
## Current Status
```
❌ NEVER use #![allow()] or #[allow()] in source code
❌ NEVER use .unwrap() - use ? or proper error handling
❌ NEVER use .expect() - use ? or proper error handling
❌ NEVER use panic!() or unreachable!()
❌ NEVER use todo!() or unimplemented!()
❌ NEVER leave unused imports or dead code
❌ NEVER add comments - code must be self-documenting
❌ NEVER use CDN links - all assets must be local
❌ NEVER build SQL queries with format! - use parameterized queries
❌ NEVER pass user input to Command::new() without validation
❌ NEVER log passwords, tokens, API keys, or PII
```
**Version:** 6.0.8
**Build Status:** SUCCESS
**Production Ready:** YES
---
## Deployment
## 🔐 Security Requirements
See [Deployment Guide](docs/guides/deployment.md) for:
### Error Handling - CRITICAL DEBT
- Single server setup
- Docker Compose
- LXC containers
- Kubernetes
- Reverse proxy configuration
**Current Status**: 955 instances of `unwrap()`/`expect()` found in codebase
**Target**: 0 instances in production code (tests excluded)
## Contributing
```rust
// ❌ WRONG - Found 955 times in codebase
let value = something.unwrap();
let value = something.expect("msg");
// ✅ CORRECT - Required replacements
let value = something?;
let value = something.ok_or_else(|| Error::NotFound)?;
let value = something.unwrap_or_default();
let value = something.unwrap_or_else(|e| {
log::error!("Operation failed: {e}");
default_value
});
```
### Performance Issues - CRITICAL DEBT
**Current Status**: 12,973 excessive `clone()`/`to_string()` calls
**Target**: Minimize allocations, use references where possible
```rust
// ❌ WRONG - Excessive allocations
let name = user.name.clone();
let msg = format!("Hello {}", name.to_string());
// ✅ CORRECT - Minimize allocations
let name = &user.name;
let msg = format!("Hello {name}");
// ✅ CORRECT - Use Cow for conditional ownership
use std::borrow::Cow;
fn process_name(name: Cow<str>) -> String {
match name {
Cow::Borrowed(s) => s.to_uppercase(),
Cow::Owned(s) => s.to_uppercase(),
}
}
```
### SQL Injection Prevention
```rust
// ❌ WRONG
let query = format!("SELECT * FROM {}", table_name);
// ✅ CORRECT - whitelist validation
const ALLOWED_TABLES: &[&str] = &["users", "sessions"];
if !ALLOWED_TABLES.contains(&table_name) {
return Err(Error::InvalidTable);
}
```
### Command Injection Prevention
```rust
// ❌ WRONG
Command::new("tool").arg(user_input).output()?;
// ✅ CORRECT - Use SafeCommand
use crate::security::command_guard::SafeCommand;
SafeCommand::new("allowed_command")?
.arg("safe_arg")?
.execute()
```
### Error Responses - Use ErrorSanitizer
```rust
// ❌ WRONG
Json(json!({ "error": e.to_string() }))
format!("Database error: {}", e)
// ✅ CORRECT
use crate::security::error_sanitizer::log_and_sanitize;
let sanitized = log_and_sanitize(&e, "context", None);
(StatusCode::INTERNAL_SERVER_ERROR, sanitized)
```
---
## ✅ Mandatory Code Patterns
### Format Strings - Inline Variables
```rust
// ❌ WRONG
format!("Hello {}", name)
// ✅ CORRECT
format!("Hello {name}")
```
### Self Usage in Impl Blocks
```rust
// ❌ WRONG
impl MyStruct {
fn new() -> MyStruct { MyStruct { } }
}
// ✅ CORRECT
impl MyStruct {
fn new() -> Self { Self { } }
}
```
### Derive Eq with PartialEq
```rust
// ❌ WRONG
#[derive(PartialEq)]
struct MyStruct { }
// ✅ CORRECT
#[derive(PartialEq, Eq)]
struct MyStruct { }
```
### Option Handling
```rust
// ✅ CORRECT
opt.unwrap_or(default)
opt.unwrap_or_else(|| compute_default())
opt.map_or(default, |x| transform(x))
```
### Chrono DateTime
```rust
// ❌ WRONG
date.with_hour(9).unwrap().with_minute(0).unwrap()
// ✅ CORRECT
date.with_hour(9).and_then(|d| d.with_minute(0)).unwrap_or(date)
```
---
## 📏 File Size Limits - MANDATORY
### Maximum 450 Lines Per File
When a file grows beyond this limit:
1. **Identify logical groups** - Find related functions
2. **Create subdirectory module** - e.g., `handlers/`
3. **Split by responsibility:**
- `types.rs` - Structs, enums, type definitions
- `handlers.rs` - HTTP handlers and routes
- `operations.rs` - Core business logic
- `utils.rs` - Helper functions
- `mod.rs` - Re-exports and configuration
4. **Keep files focused** - Single responsibility
5. **Update mod.rs** - Re-export all public items
**NEVER let a single file exceed 450 lines - split proactively at 350 lines**
### Files Requiring Immediate Refactoring
| File | Lines | Target Split |
|------|-------|--------------|
| `auto_task/app_generator.rs` | 2981 | → 7 files |
| `tasks/mod.rs` | 2651 | → 6 files |
| `learn/mod.rs` | 2306 | → 5 files |
| `attendance/llm_assist.rs` | 2053 | → 5 files |
| `drive/mod.rs` | 1522 | → 4 files |
**See `TODO-refactor1.md` for detailed refactoring plans**
---
## 🗄️ Database Standards
- **TABLES AND INDEXES ONLY** (no views, triggers, functions)
- **JSON columns:** use TEXT with `_json` suffix
- **ORM:** Use diesel - no sqlx
- **Migrations:** Located in `botserver/migrations/`
---
## 🎨 Frontend Rules
- **Use HTMX** - minimize JavaScript
- **NO external CDN** - all assets local
- **Server-side rendering** with Askama templates
---
## 📦 Key Dependencies
| Library | Version | Purpose |
|---------|---------|---------|
| axum | 0.7.5 | Web framework |
| diesel | 2.1 | PostgreSQL ORM |
| tokio | 1.41 | Async runtime |
| rhai | git | BASIC scripting |
| reqwest | 0.12 | HTTP client |
| serde | 1.0 | Serialization |
| askama | 0.12 | HTML Templates |
---
## 🚀 CI/CD Workflow
When configuring CI/CD pipelines (e.g., Forgejo Actions):
- **Minimal Checkout**: Clone only the root `gb` and the `botlib` submodule. Do NOT recursively clone everything.
- **BotServer Context**: Replace the empty `botserver` directory with the current set of files being tested.
**Example Step:**
```yaml
- name: Setup Workspace
run: |
# 1. Clone only the root workspace configuration
git clone --depth 1 <your-git-repo-url> workspace
# 2. Setup only the necessary dependencies (botlib)
cd workspace
git submodule update --init --depth 1 botlib
cd ..
# 3. Inject current BotServer code
rm -rf workspace/botserver
mv botserver workspace/botserver
```
---
## 📚 Documentation
### Documentation Structure
```
docs/
├── api/ # API documentation
│ ├── README.md # API overview
│ ├── rest-endpoints.md # HTTP endpoints
│ └── websocket.md # Real-time communication
├── guides/ # How-to guides
│ ├── getting-started.md # Quick start
│ ├── deployment.md # Production setup
│ └── templates.md # Using templates
└── reference/ # Technical reference
├── basic-language.md # BASIC keywords
├── configuration.md # Config options
└── architecture.md # System design
```
### Additional Resources
- **[docs.pragmatismo.com.br](https://docs.pragmatismo.com.br)** - Full online documentation
- **[BotBook](../botbook)** - Local comprehensive guide with tutorials and examples
- **[API Reference](docs/api/README.md)** - REST and WebSocket endpoints
- **[BASIC Language](docs/reference/basic-language.md)** - Dialog scripting reference
---
## 🔗 Related Projects
| Project | Description |
|---------|-------------|
| [botui](https://github.com/GeneralBots/botui) | Pure web UI (HTMX-based) |
| [botapp](https://github.com/GeneralBots/botapp) | Tauri desktop wrapper |
| [botlib](https://github.com/GeneralBots/botlib) | Shared Rust library |
| [botbook](https://github.com/GeneralBots/botbook) | Documentation |
| [bottemplates](https://github.com/GeneralBots/bottemplates) | Templates and examples |
---
## 🛡️ Security
- **AGPL-3.0 License** - True open source with contribution requirements
- **Self-hosted** - Your data stays on your infrastructure
- **Enterprise-grade** - 5+ years of stability
- **No vendor lock-in** - Open protocols and standards
Report security issues to: **security@pragmatismo.com.br**
---
## 🤝 Contributing
We welcome contributions! Please read our contributing guidelines before submitting PRs.
## Security
Security issues should be reported to: **security@pragmatismo.com.br**
## License
General Bot Copyright (c) pragmatismo.com.br. All rights reserved.
Licensed under the **AGPL-3.0**.
According to our dual licensing model, this program can be used either under the terms of the GNU Affero General Public License, version 3, or under a proprietary license.
## Support
- **GitHub Issues:** [github.com/GeneralBots/botserver/issues](https://github.com/GeneralBots/botserver/issues)
- **Stack Overflow:** Tag questions with `generalbots`
- **Video Tutorial:** [7 AI General Bots LLM Templates](https://www.youtube.com/watch?v=KJgvUPXi3Fw)
## Contributors
### Contributors
<a href="https://github.com/generalbots/botserver/graphs/contributors">
<img src="https://contrib.rocks/image?repo=generalbots/botserver" />
@ -152,6 +443,53 @@ According to our dual licensing model, this program can be used either under the
---
## 🔑 Remember
- **ZERO WARNINGS** - Fix every clippy warning
- **ZERO COMMENTS** - No comments, no doc comments
- **NO ALLOW IN CODE** - Configure exceptions in Cargo.toml only
- **NO DEAD CODE** - Delete unused code
- **NO UNWRAP/EXPECT** - Use ? or combinators (955 instances to fix)
- **MINIMIZE CLONES** - Avoid excessive allocations (12,973 instances to optimize)
- **PARAMETERIZED SQL** - Never format! for queries
- **VALIDATE COMMANDS** - Never pass raw user input
- **INLINE FORMAT ARGS** - `format!("{name}")` not `format!("{}", name)`
- **USE SELF** - In impl blocks, use Self not type name
- **FILE SIZE LIMIT** - Max 450 lines per file, refactor at 350 lines
- **Version 6.2.0** - Do not change without approval
- **GIT WORKFLOW** - ALWAYS push to ALL repositories (github, pragmatismo)
---
## 🚨 Immediate Action Required
1. **Replace 955 unwrap()/expect() calls** with proper error handling
2. **Optimize 12,973 clone()/to_string() calls** for performance
3. **Refactor 5 large files** following TODO-refactor1.md
4. **Add missing error handling** in critical paths
5. **Implement proper logging** instead of panicking
---
## 📄 License
General Bot Copyright (c) pragmatismo.com.br. All rights reserved.
Licensed under the **AGPL-3.0**.
According to our dual licensing model, this program can be used either under the terms of the GNU Affero General Public License, version 3, or under a proprietary license.
---
## 🔗 Links
- **Website:** [pragmatismo.com.br](https://pragmatismo.com.br)
- **Documentation:** [docs.pragmatismo.com.br](https://docs.pragmatismo.com.br)
- **GitHub:** [github.com/GeneralBots/botserver](https://github.com/GeneralBots/botserver)
- **Stack Overflow:** Tag questions with `generalbots`
- **Video Tutorial:** [7 AI General Bots LLM Templates](https://www.youtube.com/watch?v=KJgvUPXi3Fw)
---
**General Bots Code Name:** [Guaribas](https://en.wikipedia.org/wiki/Guaribas)
> "No one should have to do work that can be done by a machine." - Roberto Mangabeira Unger

View file

@ -713,12 +713,12 @@ CREATE INDEX idx_user_login_tokens_expires ON public.user_login_tokens USING btr
-- Session KB Associations moved to migrations/research
-- Comments
COMMENT ON TABLE session_kb_associations IS 'Tracks which Knowledge Base collections are active in each conversation session';
COMMENT ON COLUMN session_kb_associations.kb_name IS 'Name of the KB folder (e.g., "circular", "comunicado", "geral")';
COMMENT ON COLUMN session_kb_associations.kb_folder_path IS 'Full path to KB folder: work/{bot}/{bot}.gbkb/{kb_name}';
COMMENT ON COLUMN session_kb_associations.qdrant_collection IS 'Qdrant collection name for this KB';
COMMENT ON COLUMN session_kb_associations.added_by_tool IS 'Name of the .bas tool that added this KB (e.g., "change-subject.bas")';
COMMENT ON COLUMN session_kb_associations.is_active IS 'Whether this KB is currently active in the session';
-- COMMENT ON TABLE session_kb_associations IS 'Tracks which Knowledge Base collections are active in each conversation session';
-- COMMENT ON COLUMN session_kb_associations.kb_name IS 'Name of the KB folder (e.g., "circular", "comunicado", "geral")';
-- COMMENT ON COLUMN session_kb_associations.kb_folder_path IS 'Full path to KB folder: work/{bot}/{bot}.gbkb/{kb_name}';
-- COMMENT ON COLUMN session_kb_associations.qdrant_collection IS 'Qdrant collection name for this KB';
-- COMMENT ON COLUMN session_kb_associations.added_by_tool IS 'Name of the .bas tool that added this KB (e.g., "change-subject.bas")';
-- COMMENT ON COLUMN session_kb_associations.is_active IS 'Whether this KB is currently active in the session';
-- Add organization relationship to bots
ALTER TABLE public.bots
ADD COLUMN IF NOT EXISTS org_id UUID,
@ -2029,12 +2029,12 @@ CREATE INDEX IF NOT EXISTS idx_source_templates_category ON source_templates(cat
-- Email tracking moved to migrations/mail
-- Add comment for documentation
COMMENT ON TABLE sent_email_tracking IS 'Tracks sent emails for read receipt functionality via tracking pixel';
COMMENT ON COLUMN sent_email_tracking.tracking_id IS 'Unique ID embedded in tracking pixel URL';
COMMENT ON COLUMN sent_email_tracking.is_read IS 'Whether the email has been opened (pixel loaded)';
COMMENT ON COLUMN sent_email_tracking.read_count IS 'Number of times the email was opened';
COMMENT ON COLUMN sent_email_tracking.first_read_ip IS 'IP address of first email open';
COMMENT ON COLUMN sent_email_tracking.last_read_ip IS 'IP address of most recent email open';
-- COMMENT ON TABLE sent_email_tracking IS 'Tracks sent emails for read receipt functionality via tracking pixel';
-- COMMENT ON COLUMN sent_email_tracking.tracking_id IS 'Unique ID embedded in tracking pixel URL';
-- COMMENT ON COLUMN sent_email_tracking.is_read IS 'Whether the email has been opened (pixel loaded)';
-- COMMENT ON COLUMN sent_email_tracking.read_count IS 'Number of times the email was opened';
-- COMMENT ON COLUMN sent_email_tracking.first_read_ip IS 'IP address of first email open';
-- COMMENT ON COLUMN sent_email_tracking.last_read_ip IS 'IP address of most recent email open';
-- ============================================
-- TABLE KEYWORD SUPPORT (from 6.1.0_table_keyword)
-- ============================================

View file

@ -0,0 +1,3 @@
-- Remove the refresh_policy column from website_crawls table
ALTER TABLE website_crawls
DROP COLUMN IF EXISTS refresh_policy;

View file

@ -0,0 +1,13 @@
-- Add refresh_policy column to website_crawls table
-- This column stores the user-configured refresh interval (e.g., "1d", "1w", "1m", "1y")
ALTER TABLE website_crawls
ADD COLUMN IF NOT EXISTS refresh_policy VARCHAR(20);
-- Update existing records to have a default refresh policy (1 month)
UPDATE website_crawls
SET refresh_policy = '1m'
WHERE refresh_policy IS NULL;
-- Add comment for documentation
COMMENT ON COLUMN website_crawls.refresh_policy IS 'User-configured refresh interval (e.g., "1d", "1w", "1m", "1y") - shortest interval is used when duplicates exist';

View file

@ -6,24 +6,26 @@ declare -A container_limits=(
["*tables*"]="4096MB:100ms/100ms"
["*postgre*"]="4096MB:100ms/100ms" # PostgreSQL alternative
["*dns*"]="2048MB:100ms/100ms"
["*table-editor*"]="2048MB:25s/100ms"
["*oppbot*"]="4048MB:100ms/100ms"
["*table-editor*"]="2048MB:25ms/100ms"
["*proxy*"]="2048MB:100ms/100ms"
["*directory*"]="1024MB:50ms/100ms"
["*drive*"]="4096MB:100ms/100ms"
["*minio*"]="4096MB:100ms/100ms" # MinIO alternative
["*email*"]="4096MB:100ms/100ms"
["*webmail*"]="2096MB:100ms/100ms"
["*bot*"]="2048MB:5ms/100ms"
["*webmail*"]="4096MB:100ms/100ms"
["*bot*"]="2048MB:25ms/100ms"
["*oppbot*"]="4096MB:50ms/100ms"
["*meeting*"]="4096MB:100ms/100ms"
["*alm*"]="512MB:50ms/100ms"
["*vault*"]="512MB:50ms/100ms"
["*alm*"]="2048MB:50ms/100ms"
["*vault*"]="2048MB:50ms/100ms"
["*alm-ci*"]="8192MB:200ms/100ms" # CHANGED: 100ms → 200ms (HIGHEST PRIORITY)
["*system*"]="4096MB:50ms/100ms"
["*mailer*"]="2096MB:25ms/100ms"
)
# Default values (for containers that don't match any pattern)
DEFAULT_MEMORY="512MB"
DEFAULT_MEMORY="2048MB"
DEFAULT_CPU_ALLOWANCE="15ms/100ms"
DEFAULT_CPU_COUNT=1

View file

@ -1,5 +1,5 @@
lxc config device override $CONTAINER_NAME root
lxc config device set $CONTAINER_NAME root size 6GB
lxc config device set $CONTAINER_NAME root size 12GB
zpool set autoexpand=on default
zpool online -e default /var/snap/lxd/common/lxd/disks/default.img

View file

@ -34,9 +34,16 @@ pub async fn serve_suite_js_file(
}
if file_path.starts_with("vendor/") || file_path.starts_with("vendor\\") {
return serve_vendor_file(State(state), Path(VendorFilePath {
file_path: file_path.strip_prefix("vendor/").unwrap_or(&file_path).to_string()
})).await;
return serve_vendor_file(
State(state),
Path(VendorFilePath {
file_path: file_path
.strip_prefix("vendor/")
.unwrap_or(&file_path)
.to_string(),
}),
)
.await;
}
if !file_path.ends_with(".js") {
@ -45,7 +52,15 @@ pub async fn serve_suite_js_file(
let content_type = get_content_type(&file_path);
let ui_path = std::env::var("BOTUI_PATH").unwrap_or_else(|_| "./botui/ui/suite".to_string());
let ui_path = std::env::var("BOTUI_PATH").unwrap_or_else(|_| {
if std::path::Path::new("./botui/ui/suite").exists() {
"./botui/ui/suite".to_string()
} else if std::path::Path::new("../botui/ui/suite").exists() {
"../botui/ui/suite".to_string()
} else {
"./botui/ui/suite".to_string()
}
});
let local_path = format!("{}/js/{}", ui_path, file_path);
match tokio::fs::read(&local_path).await {
@ -57,7 +72,10 @@ pub async fn serve_suite_js_file(
.header(header::CACHE_CONTROL, "public, max-age=3600")
.body(Body::from(content))
.unwrap_or_else(|_| {
(StatusCode::INTERNAL_SERVER_ERROR, "Failed to build response")
(
StatusCode::INTERNAL_SERVER_ERROR,
"Failed to build response",
)
.into_response()
})
}
@ -82,6 +100,7 @@ pub async fn serve_vendor_file(
let local_paths = [
format!("./botui/ui/suite/js/vendor/{}", file_path),
format!("../botui/ui/suite/js/vendor/{}", file_path),
format!("./botserver-stack/static/js/vendor/{}", file_path),
];
@ -94,32 +113,30 @@ pub async fn serve_vendor_file(
.header(header::CACHE_CONTROL, "public, max-age=86400")
.body(Body::from(content))
.unwrap_or_else(|_| {
(StatusCode::INTERNAL_SERVER_ERROR, "Failed to build response")
(
StatusCode::INTERNAL_SERVER_ERROR,
"Failed to build response",
)
.into_response()
});
}
}
let bot_name = state.bucket_name
.trim_end_matches(".gbai")
.to_string();
let bot_name = state.bucket_name.trim_end_matches(".gbai").to_string();
let sanitized_bot_name = bot_name.to_lowercase().replace([' ', '_'], "-");
let bucket = format!("{}.gbai", sanitized_bot_name);
let key = format!("{}.gblib/vendor/{}", sanitized_bot_name, file_path);
trace!("Trying MinIO for vendor file: bucket={}, key={}", bucket, key);
trace!(
"Trying MinIO for vendor file: bucket={}, key={}",
bucket,
key
);
if let Some(ref drive) = state.drive {
match drive
.get_object()
.bucket(&bucket)
.key(&key)
.send()
.await
{
Ok(response) => {
match response.body.collect().await {
match drive.get_object().bucket(&bucket).key(&key).send().await {
Ok(response) => match response.body.collect().await {
Ok(body) => {
let content = body.into_bytes();
@ -129,15 +146,17 @@ pub async fn serve_vendor_file(
.header(header::CACHE_CONTROL, "public, max-age=86400")
.body(Body::from(content.to_vec()))
.unwrap_or_else(|_| {
(StatusCode::INTERNAL_SERVER_ERROR, "Failed to build response")
(
StatusCode::INTERNAL_SERVER_ERROR,
"Failed to build response",
)
.into_response()
});
}
Err(e) => {
error!("Failed to read MinIO response body: {}", e);
}
}
}
},
Err(e) => {
warn!("MinIO get_object failed for {}/{}: {}", bucket, key, e);
}
@ -150,17 +169,47 @@ pub async fn serve_vendor_file(
fn rewrite_cdn_urls(html: &str) -> String {
html
// HTMX from various CDNs
.replace("https://unpkg.com/htmx.org@1.9.10", "/js/vendor/htmx.min.js")
.replace("https://unpkg.com/htmx.org@1.9.10/dist/htmx.min.js", "/js/vendor/htmx.min.js")
.replace("https://unpkg.com/htmx.org@1.9.11", "/js/vendor/htmx.min.js")
.replace("https://unpkg.com/htmx.org@1.9.11/dist/htmx.min.js", "/js/vendor/htmx.min.js")
.replace("https://unpkg.com/htmx.org@1.9.12", "/js/vendor/htmx.min.js")
.replace("https://unpkg.com/htmx.org@1.9.12/dist/htmx.min.js", "/js/vendor/htmx.min.js")
.replace(
"https://unpkg.com/htmx.org@1.9.10",
"/js/vendor/htmx.min.js",
)
.replace(
"https://unpkg.com/htmx.org@1.9.10/dist/htmx.min.js",
"/js/vendor/htmx.min.js",
)
.replace(
"https://unpkg.com/htmx.org@1.9.11",
"/js/vendor/htmx.min.js",
)
.replace(
"https://unpkg.com/htmx.org@1.9.11/dist/htmx.min.js",
"/js/vendor/htmx.min.js",
)
.replace(
"https://unpkg.com/htmx.org@1.9.12",
"/js/vendor/htmx.min.js",
)
.replace(
"https://unpkg.com/htmx.org@1.9.12/dist/htmx.min.js",
"/js/vendor/htmx.min.js",
)
.replace("https://unpkg.com/htmx.org", "/js/vendor/htmx.min.js")
.replace("https://cdn.jsdelivr.net/npm/htmx.org", "/js/vendor/htmx.min.js")
.replace("https://cdnjs.cloudflare.com/ajax/libs/htmx/1.9.10/htmx.min.js", "/js/vendor/htmx.min.js")
.replace("https://cdnjs.cloudflare.com/ajax/libs/htmx/1.9.11/htmx.min.js", "/js/vendor/htmx.min.js")
.replace("https://cdnjs.cloudflare.com/ajax/libs/htmx/1.9.12/htmx.min.js", "/js/vendor/htmx.min.js")
.replace(
"https://cdn.jsdelivr.net/npm/htmx.org",
"/js/vendor/htmx.min.js",
)
.replace(
"https://cdnjs.cloudflare.com/ajax/libs/htmx/1.9.10/htmx.min.js",
"/js/vendor/htmx.min.js",
)
.replace(
"https://cdnjs.cloudflare.com/ajax/libs/htmx/1.9.11/htmx.min.js",
"/js/vendor/htmx.min.js",
)
.replace(
"https://cdnjs.cloudflare.com/ajax/libs/htmx/1.9.12/htmx.min.js",
"/js/vendor/htmx.min.js",
)
}
pub fn configure_app_server_routes() -> Router<Arc<AppState>> {
@ -240,26 +289,24 @@ async fn serve_app_file_internal(state: &AppState, app_name: &str, file_path: &s
}
// Get bot name from bucket_name config (default to "default")
let bot_name = state.bucket_name
.trim_end_matches(".gbai")
.to_string();
let bot_name = state.bucket_name.trim_end_matches(".gbai").to_string();
let sanitized_bot_name = bot_name.to_lowercase().replace([' ', '_'], "-");
// MinIO bucket and path: botname.gbai / botname.gbapp/appname/file
let bucket = format!("{}.gbai", sanitized_bot_name);
let key = format!("{}.gbapp/{}/{}", sanitized_bot_name, sanitized_app_name, sanitized_file_path);
let key = format!(
"{}.gbapp/{}/{}",
sanitized_bot_name, sanitized_app_name, sanitized_file_path
);
info!("Serving app file from MinIO: bucket={}, key={}", bucket, key);
info!(
"Serving app file from MinIO: bucket={}, key={}",
bucket, key
);
// Try to serve from MinIO
if let Some(ref drive) = state.drive {
match drive
.get_object()
.bucket(&bucket)
.key(&key)
.send()
.await
{
match drive.get_object().bucket(&bucket).key(&key).send().await {
Ok(response) => {
match response.body.collect().await {
Ok(body) => {
@ -281,7 +328,10 @@ async fn serve_app_file_internal(state: &AppState, app_name: &str, file_path: &s
.header(header::CACHE_CONTROL, "public, max-age=3600")
.body(Body::from(final_content))
.unwrap_or_else(|_| {
(StatusCode::INTERNAL_SERVER_ERROR, "Failed to build response")
(
StatusCode::INTERNAL_SERVER_ERROR,
"Failed to build response",
)
.into_response()
});
}
@ -390,8 +440,6 @@ pub async fn list_all_apps(State(state): State<Arc<AppState>>) -> impl IntoRespo
.into_response()
}
#[cfg(test)]
mod tests {
use super::*;
@ -431,9 +479,15 @@ mod tests {
fn test_sanitize_file_path() {
assert_eq!(sanitize_file_path("styles.css"), "styles.css");
assert_eq!(sanitize_file_path("css/styles.css"), "css/styles.css");
assert_eq!(sanitize_file_path("assets/img/logo.png"), "assets/img/logo.png");
assert_eq!(
sanitize_file_path("assets/img/logo.png"),
"assets/img/logo.png"
);
assert_eq!(sanitize_file_path("../../../etc/passwd"), "etc/passwd");
assert_eq!(sanitize_file_path("./styles.css"), "styles.css");
assert_eq!(sanitize_file_path("path//double//slash.js"), "path/double/slash.js");
assert_eq!(
sanitize_file_path("path//double//slash.js"),
"path/double/slash.js"
);
}
}

View file

@ -1,16 +1,17 @@
#[cfg(feature = "llm")]
use crate::llm::smart_router::{SmartLLMRouter, OptimizationGoal};
use crate::core::shared::state::AppState;
use crate::basic::UserSession;
use crate::core::shared::state::AppState;
#[cfg(feature = "llm")]
use rhai::{Dynamic, Engine};
use crate::llm::smart_router::{OptimizationGoal, SmartLLMRouter};
#[cfg(not(feature = "llm"))]
use rhai::Engine;
#[cfg(feature = "llm")]
use rhai::{Dynamic, Engine};
use std::sync::Arc;
#[cfg(feature = "llm")]
pub fn register_enhanced_llm_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engine) {
let state_clone = Arc::clone(&state);
let state_clone1 = Arc::clone(&state);
let state_clone2 = Arc::clone(&state);
let user_clone = user;
if let Err(e) = engine.register_custom_syntax(
@ -20,14 +21,18 @@ pub fn register_enhanced_llm_keyword(state: Arc<AppState>, user: UserSession, en
let prompt = context.eval_expression_tree(&inputs[0])?.to_string();
let optimization = context.eval_expression_tree(&inputs[1])?.to_string();
let state_for_spawn = Arc::clone(&state_clone);
let state_for_spawn = Arc::clone(&state_clone1);
let _user_clone_spawn = user_clone.clone();
tokio::spawn(async move {
let router = SmartLLMRouter::new(state_for_spawn);
let goal = OptimizationGoal::from_str(&optimization);
match crate::llm::smart_router::enhanced_llm_call(&router, &prompt, goal, None, None).await {
match crate::llm::smart_router::enhanced_llm_call(
&router, &prompt, goal, None, None,
)
.await
{
Ok(_response) => {
log::info!("LLM response generated with {} optimization", optimization);
}
@ -44,14 +49,22 @@ pub fn register_enhanced_llm_keyword(state: Arc<AppState>, user: UserSession, en
}
if let Err(e) = engine.register_custom_syntax(
["LLM", "$string$", "WITH", "MAX_COST", "$float$", "MAX_LATENCY", "$int$"],
[
"LLM",
"$string$",
"WITH",
"MAX_COST",
"$float$",
"MAX_LATENCY",
"$int$",
],
false,
move |context, inputs| {
let prompt = context.eval_expression_tree(&inputs[0])?.to_string();
let max_cost = context.eval_expression_tree(&inputs[1])?.as_float()?;
let max_latency = context.eval_expression_tree(&inputs[2])?.as_int()? as u64;
let state_for_spawn = Arc::clone(&state_clone);
let state_for_spawn = Arc::clone(&state_clone2);
tokio::spawn(async move {
let router = SmartLLMRouter::new(state_for_spawn);
@ -61,10 +74,16 @@ pub fn register_enhanced_llm_keyword(state: Arc<AppState>, user: UserSession, en
&prompt,
OptimizationGoal::Balanced,
Some(max_cost),
Some(max_latency)
).await {
Some(max_latency),
)
.await
{
Ok(_response) => {
log::info!("LLM response with constraints: cost<={}, latency<={}", max_cost, max_latency);
log::info!(
"LLM response with constraints: cost<={}, latency<={}",
max_cost,
max_latency
);
}
Err(e) => {
log::error!("Constrained LLM call failed: {}", e);
@ -80,6 +99,10 @@ pub fn register_enhanced_llm_keyword(state: Arc<AppState>, user: UserSession, en
}
#[cfg(not(feature = "llm"))]
pub fn register_enhanced_llm_keyword(_state: Arc<AppState>, _user: UserSession, _engine: &mut Engine) {
pub fn register_enhanced_llm_keyword(
_state: Arc<AppState>,
_user: UserSession,
_engine: &mut Engine,
) {
// No-op when LLM feature is disabled
}

View file

@ -6,21 +6,64 @@ use rhai::{Dynamic, Engine};
use std::sync::Arc;
use uuid::Uuid;
/// Parse refresh interval string (e.g., "1d", "1w", "1m", "1y") into days
/// Returns the number of days for the refresh interval
fn parse_refresh_interval(interval: &str) -> Result<i32, String> {
let interval_lower = interval.trim().to_lowercase();
// Match patterns like "1d", "7d", "2w", "1m", "1y", etc.
if interval_lower.ends_with('d') {
let days: i32 = interval_lower[..interval_lower.len()-1]
.parse()
.map_err(|_| format!("Invalid days format: {}", interval))?;
Ok(days)
} else if interval_lower.ends_with('w') {
let weeks: i32 = interval_lower[..interval_lower.len()-1]
.parse()
.map_err(|_| format!("Invalid weeks format: {}", interval))?;
Ok(weeks * 7)
} else if interval_lower.ends_with('m') {
let months: i32 = interval_lower[..interval_lower.len()-1]
.parse()
.map_err(|_| format!("Invalid months format: {}", interval))?;
Ok(months * 30) // Approximate month as 30 days
} else if interval_lower.ends_with('y') {
let years: i32 = interval_lower[..interval_lower.len()-1]
.parse()
.map_err(|_| format!("Invalid years format: {}", interval))?;
Ok(years * 365) // Approximate year as 365 days
} else {
// Try to parse as plain number (assume days)
interval.parse()
.map_err(|_| format!("Invalid refresh interval format: {}. Use format like '1d', '1w', '1m', '1y'", interval))
}
}
/// Convert days to expires_policy string format
fn days_to_expires_policy(days: i32) -> String {
format!("{}d", days)
}
pub fn use_website_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engine) {
let state_clone = Arc::clone(&state);
let user_clone = user;
let user_clone = user.clone();
// Register syntax for USE WEBSITE "url" REFRESH "interval"
engine
.register_custom_syntax(
["USE", "WEBSITE", "$expr$"],
["USE", "WEBSITE", "$expr$", "REFRESH", "$expr$"],
false,
move |context, inputs| {
let url = context.eval_expression_tree(&inputs[0])?;
let url_str = url.to_string().trim_matches('"').to_string();
let refresh = context.eval_expression_tree(&inputs[1])?;
let refresh_str = refresh.to_string().trim_matches('"').to_string();
trace!(
"USE WEBSITE command executed: {} for session: {}",
"USE WEBSITE command executed: {} REFRESH {} for session: {}",
url_str,
refresh_str,
user_clone.id
);
@ -35,6 +78,83 @@ pub fn use_website_keyword(state: Arc<AppState>, user: UserSession, engine: &mut
let state_for_task = Arc::clone(&state_clone);
let user_for_task = user_clone.clone();
let url_for_task = url_str;
let refresh_for_task = refresh_str;
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build();
let send_err = if let Ok(_rt) = rt {
let result = associate_website_with_session_refresh(
&state_for_task,
&user_for_task,
&url_for_task,
&refresh_for_task,
);
tx.send(result).err()
} else {
tx.send(Err("Failed to build tokio runtime".to_string()))
.err()
};
if send_err.is_some() {
error!("Failed to send result from thread");
}
});
match rx.recv_timeout(std::time::Duration::from_secs(10)) {
Ok(Ok(message)) => Ok(Dynamic::from(message)),
Ok(Err(e)) => Err(Box::new(rhai::EvalAltResult::ErrorRuntime(
e.into(),
rhai::Position::NONE,
))),
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
Err(Box::new(rhai::EvalAltResult::ErrorRuntime(
"USE WEBSITE timed out".into(),
rhai::Position::NONE,
)))
}
Err(e) => Err(Box::new(rhai::EvalAltResult::ErrorRuntime(
format!("USE WEBSITE failed: {}", e).into(),
rhai::Position::NONE,
))),
}
},
)
.expect("valid syntax registration");
// Register syntax for USE WEBSITE "url" (without REFRESH)
let state_clone2 = Arc::clone(&state);
let user_clone2 = user.clone();
engine
.register_custom_syntax(
["USE", "WEBSITE", "$expr$"],
false,
move |context, inputs| {
let url = context.eval_expression_tree(&inputs[0])?;
let url_str = url.to_string().trim_matches('"').to_string();
trace!(
"USE WEBSITE command executed: {} for session: {}",
url_str,
user_clone2.id
);
let is_valid = url_str.starts_with("http://") || url_str.starts_with("https://");
if !is_valid {
return Err(Box::new(rhai::EvalAltResult::ErrorRuntime(
"Invalid URL format. Must start with http:// or https://".into(),
rhai::Position::NONE,
)));
}
let state_for_task = Arc::clone(&state_clone2);
let user_for_task = user_clone2.clone();
let url_for_task = url_str;
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
@ -87,7 +207,16 @@ fn associate_website_with_session(
user: &UserSession,
url: &str,
) -> Result<String, String> {
info!("Associating website {} with session {}", url, user.id);
associate_website_with_session_refresh(state, user, url, "1m") // Default: 1 month
}
fn associate_website_with_session_refresh(
state: &AppState,
user: &UserSession,
url: &str,
refresh_interval: &str,
) -> Result<String, String> {
info!("Associating website {} with session {} (refresh: {})", url, user.id, refresh_interval);
let mut conn = state.conn.get().map_err(|e| format!("DB error: {}", e))?;
@ -97,16 +226,25 @@ fn associate_website_with_session(
match website_status {
WebsiteCrawlStatus::NotRegistered => {
return Err(format!(
"Website {} has not been registered for crawling. It should be added to the script for preprocessing.",
url
// Auto-register website for crawling instead of failing
info!("Website {} not registered, auto-registering for crawling with refresh: {}", url, refresh_interval);
register_website_for_crawling_with_refresh(&mut conn, &user.bot_id, url, refresh_interval)
.map_err(|e| format!("Failed to register website: {}", e))?;
return Ok(format!(
"Website {} has been registered for crawling (refresh: {}). It will be available once crawling completes.",
url, refresh_interval
));
}
WebsiteCrawlStatus::Pending => {
info!("Website {} is pending crawl, associating anyway", url);
// Update refresh policy if needed
update_refresh_policy_if_shorter(&mut conn, &user.bot_id, url, refresh_interval)?;
}
WebsiteCrawlStatus::Crawled => {
info!("Website {} is already crawled and ready", url);
// Update refresh policy if needed
update_refresh_policy_if_shorter(&mut conn, &user.bot_id, url, refresh_interval)?;
}
WebsiteCrawlStatus::Failed => {
return Err(format!(
@ -165,26 +303,96 @@ pub fn register_website_for_crawling(
bot_id: &Uuid,
url: &str,
) -> Result<(), String> {
let expires_policy = "1d";
register_website_for_crawling_with_refresh(conn, bot_id, url, "1m") // Default: 1 month
}
pub fn register_website_for_crawling_with_refresh(
conn: &mut PgConnection,
bot_id: &Uuid,
url: &str,
refresh_interval: &str,
) -> Result<(), String> {
let days = parse_refresh_interval(refresh_interval)
.map_err(|e| format!("Invalid refresh interval: {}", e))?;
let expires_policy = days_to_expires_policy(days);
let query = diesel::sql_query(
"INSERT INTO website_crawls (id, bot_id, url, expires_policy, crawl_status, next_crawl)
VALUES (gen_random_uuid(), $1, $2, $3, 0, NOW())
ON CONFLICT (bot_id, url) DO UPDATE SET next_crawl =
CASE
"INSERT INTO website_crawls (id, bot_id, url, expires_policy, crawl_status, next_crawl, refresh_policy)
VALUES (gen_random_uuid(), $1, $2, $3, 0, NOW(), $4)
ON CONFLICT (bot_id, url) DO UPDATE SET
next_crawl = CASE
WHEN website_crawls.crawl_status = 2 THEN NOW() -- Failed, retry now
ELSE website_crawls.next_crawl -- Keep existing schedule
END,
refresh_policy = CASE
WHEN website_crawls.refresh_policy IS NULL THEN $4
ELSE LEAST(website_crawls.refresh_policy, $4) -- Use shorter interval
END",
)
.bind::<diesel::sql_types::Uuid, _>(bot_id)
.bind::<diesel::sql_types::Text, _>(url)
.bind::<diesel::sql_types::Text, _>(expires_policy);
.bind::<diesel::sql_types::Text, _>(expires_policy)
.bind::<diesel::sql_types::Text, _>(refresh_interval);
query
.execute(conn)
.map_err(|e| format!("Failed to register website for crawling: {}", e))?;
info!("Website {} registered for crawling for bot {}", url, bot_id);
info!("Website {} registered for crawling for bot {} with refresh policy: {}", url, bot_id, refresh_interval);
Ok(())
}
/// Update refresh policy if the new interval is shorter than the existing one
fn update_refresh_policy_if_shorter(
conn: &mut PgConnection,
bot_id: &Uuid,
url: &str,
refresh_interval: &str,
) -> Result<(), String> {
// Get current record to compare in Rust (no SQL business logic!)
#[derive(QueryableByName)]
struct CurrentRefresh {
#[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Text>)]
refresh_policy: Option<String>,
}
let current = diesel::sql_query(
"SELECT refresh_policy FROM website_crawls WHERE bot_id = $1 AND url = $2"
)
.bind::<diesel::sql_types::Uuid, _>(bot_id)
.bind::<diesel::sql_types::Text, _>(url)
.get_result::<CurrentRefresh>(conn)
.ok();
let new_days = parse_refresh_interval(refresh_interval)
.map_err(|e| format!("Invalid refresh interval: {}", e))?;
// Check if we should update (no policy exists or new interval is shorter)
let should_update = match &current {
Some(c) if c.refresh_policy.is_some() => {
let existing_days = parse_refresh_interval(c.refresh_policy.as_ref().unwrap())
.unwrap_or(i32::MAX);
new_days < existing_days
}
_ => true, // No existing policy, so update
};
if should_update {
let expires_policy = days_to_expires_policy(new_days);
diesel::sql_query(
"UPDATE website_crawls SET refresh_policy = $3, expires_policy = $4
WHERE bot_id = $1 AND url = $2"
)
.bind::<diesel::sql_types::Uuid, _>(bot_id)
.bind::<diesel::sql_types::Text, _>(url)
.bind::<diesel::sql_types::Text, _>(refresh_interval)
.bind::<diesel::sql_types::Text, _>(expires_policy)
.execute(conn)
.map_err(|e| format!("Failed to update refresh policy: {}", e))?;
}
Ok(())
}
@ -193,7 +401,16 @@ pub fn execute_use_website_preprocessing(
url: &str,
bot_id: Uuid,
) -> Result<serde_json::Value, Box<dyn std::error::Error>> {
trace!("Preprocessing USE_WEBSITE: {}, bot_id: {:?}", url, bot_id);
execute_use_website_preprocessing_with_refresh(conn, url, bot_id, "1m") // Default: 1 month
}
pub fn execute_use_website_preprocessing_with_refresh(
conn: &mut PgConnection,
url: &str,
bot_id: Uuid,
refresh_interval: &str,
) -> Result<serde_json::Value, Box<dyn std::error::Error>> {
trace!("Preprocessing USE_WEBSITE: {}, bot_id: {:?}, refresh: {}", url, bot_id, refresh_interval);
if !url.starts_with("http://") && !url.starts_with("https://") {
return Err(format!(
@ -203,12 +420,13 @@ pub fn execute_use_website_preprocessing(
.into());
}
register_website_for_crawling(conn, &bot_id, url)?;
register_website_for_crawling_with_refresh(conn, &bot_id, url, refresh_interval)?;
Ok(serde_json::json!({
"command": "use_website",
"url": url,
"bot_id": bot_id.to_string(),
"refresh_policy": refresh_interval,
"status": "registered_for_crawling"
}))
}

View file

@ -4,6 +4,7 @@ use crate::package_manager::{InstallMode, PackageManager};
use crate::security::command_guard::SafeCommand;
use crate::shared::utils::{establish_pg_connection, init_secrets_manager};
use anyhow::Result;
use uuid::Uuid;
#[cfg(feature = "drive")]
use aws_sdk_s3::Client;
@ -18,6 +19,13 @@ use std::fs;
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
#[derive(diesel::QueryableByName)]
#[diesel(check_for_backend(diesel::pg::Pg))]
struct BotExistsResult {
#[diesel(sql_type = diesel::sql_types::Bool)]
exists: bool,
}
fn safe_pkill(args: &[&str]) {
if let Ok(cmd) = SafeCommand::new("pkill").and_then(|c| c.args(args)) {
let _ = cmd.execute();
@ -1971,6 +1979,68 @@ VAULT_CACHE_TTL=300
debug!("Drive feature disabled, skipping template upload");
Ok(())
}
fn create_bot_from_template(conn: &mut diesel::PgConnection, bot_name: &str) -> Result<Uuid> {
use diesel::sql_query;
info!("Creating bot '{}' from template", bot_name);
let bot_id = Uuid::new_v4();
let db_name = format!("bot_{}", bot_name.replace(['-', ' '], "_").to_lowercase());
sql_query(
"INSERT INTO bots (id, name, description, is_active, database_name, created_at, updated_at, llm_provider, llm_config, context_provider, context_config)
VALUES ($1, $2, $3, true, $4, NOW(), NOW(), $5, $6, $7, $8)",
)
.bind::<diesel::sql_types::Uuid, _>(bot_id)
.bind::<diesel::sql_types::Text, _>(bot_name)
.bind::<diesel::sql_types::Text, _>(format!("Bot agent: {}", bot_name))
.bind::<diesel::sql_types::Text, _>(&db_name)
.bind::<diesel::sql_types::Text, _>("local")
.bind::<diesel::sql_types::Json, _>(serde_json::json!({}))
.bind::<diesel::sql_types::Text, _>("postgres")
.bind::<diesel::sql_types::Json, _>(serde_json::json!({}))
.execute(conn)
.map_err(|e| anyhow::anyhow!("Failed to create bot '{}': {}", bot_name, e))?;
// Create the bot database
let safe_db_name: String = db_name
.chars()
.filter(|c| c.is_alphanumeric() || *c == '_')
.collect();
if !safe_db_name.is_empty() && safe_db_name.len() <= 63 {
let create_query = format!("CREATE DATABASE {}", safe_db_name);
if let Err(e) = sql_query(&create_query).execute(conn) {
let err_str = e.to_string();
if !err_str.contains("already exists") {
warn!("Failed to create database for bot '{}': {}", bot_name, e);
}
}
info!("Created database '{}' for bot '{}'", safe_db_name, bot_name);
}
Ok(bot_id)
}
fn read_valid_templates(templates_dir: &Path) -> std::collections::HashSet<String> {
let valid_file = templates_dir.join(".valid");
let mut valid_set = std::collections::HashSet::new();
if let Ok(content) = std::fs::read_to_string(&valid_file) {
for line in content.lines() {
let line = line.trim();
if !line.is_empty() && !line.starts_with('#') {
valid_set.insert(line.to_string());
}
}
info!("Loaded {} valid templates from .valid file", valid_set.len());
} else {
info!("No .valid file found, will load all templates");
}
valid_set
}
fn create_bots_from_templates(conn: &mut diesel::PgConnection) -> Result<()> {
use crate::shared::models::schema::bots;
use diesel::prelude::*;
@ -2001,15 +2071,23 @@ VAULT_CACHE_TTL=300
}
};
let valid_templates = Self::read_valid_templates(&templates_dir);
let load_all = valid_templates.is_empty();
let default_bot: Option<(uuid::Uuid, String)> = bots::table
.filter(bots::is_active.eq(true))
.select((bots::id, bots::name))
.first(conn)
.optional()?;
let Some((default_bot_id, default_bot_name)) = default_bot else {
error!("No active bot found in database - cannot sync template configs");
return Ok(());
let (default_bot_id, default_bot_name) = match default_bot {
Some(bot) => bot,
None => {
// Create default bot if it doesn't exist
info!("No active bot found, creating 'default' bot from template");
let bot_id = Self::create_bot_from_template(conn, "default")?;
(bot_id, "default".to_string())
}
};
info!(
@ -2017,6 +2095,55 @@ VAULT_CACHE_TTL=300
default_bot_name, default_bot_id
);
// Scan for .gbai template files and create bots if they don't exist
let entries = std::fs::read_dir(&templates_dir)
.map_err(|e| anyhow::anyhow!("Failed to read templates directory: {}", e))?;
for entry in entries.flatten() {
let file_name = entry.file_name();
let file_name_str = match file_name.to_str() {
Some(name) => name,
None => continue,
};
if !file_name_str.ends_with(".gbai") {
continue;
}
if !load_all && !valid_templates.contains(file_name_str) {
debug!("Skipping template '{}' (not in .valid file)", file_name_str);
continue;
}
let bot_name = file_name_str.trim_end_matches(".gbai");
// Check if bot already exists
let bot_exists: bool =
diesel::sql_query("SELECT EXISTS(SELECT 1 FROM bots WHERE name = $1) as exists")
.bind::<diesel::sql_types::Text, _>(bot_name)
.get_result::<BotExistsResult>(conn)
.map(|r| r.exists)
.unwrap_or(false);
if bot_exists {
info!("Bot '{}' already exists, skipping creation", bot_name);
continue;
}
// Create bot from template
match Self::create_bot_from_template(conn, bot_name) {
Ok(bot_id) => {
info!(
"Successfully created bot '{}' ({}) from template",
bot_name, bot_id
);
}
Err(e) => {
error!("Failed to create bot '{}' from template: {:#}", bot_name, e);
}
}
}
let default_template = templates_dir.join("default.gbai");
info!(
"Looking for default template at: {}",

View file

@ -1,5 +1,7 @@
#[cfg(any(feature = "research", feature = "llm"))]
pub mod kb_context;
#[cfg(any(feature = "research", feature = "llm"))]
use kb_context::inject_kb_context;
#[cfg(feature = "llm")]
use crate::core::config::ConfigManager;
@ -20,11 +22,14 @@ use axum::{
http::StatusCode,
response::{IntoResponse, Json},
};
use diesel::ExpressionMethods;
use diesel::PgConnection;
use diesel::QueryDsl;
use diesel::RunQueryDsl;
use futures::{sink::SinkExt, stream::StreamExt};
use log::{error, info, warn};
#[cfg(feature = "llm")]
use log::trace;
use log::{error, info, warn};
use serde_json;
use std::collections::HashMap;
use std::sync::Arc;
@ -39,6 +44,18 @@ pub fn get_default_bot(conn: &mut PgConnection) -> (Uuid, String) {
use crate::shared::models::schema::bots::dsl::*;
use diesel::prelude::*;
// First try to get the bot named "default"
match bots
.filter(name.eq("default"))
.filter(is_active.eq(true))
.select((id, name))
.first::<(Uuid, String)>(conn)
.optional()
{
Ok(Some((bot_id, bot_name))) => (bot_id, bot_name),
Ok(None) => {
warn!("Bot named 'default' not found, falling back to first active bot");
// Fall back to first active bot
match bots
.filter(is_active.eq(true))
.select((id, name))
@ -50,6 +67,12 @@ pub fn get_default_bot(conn: &mut PgConnection) -> (Uuid, String) {
warn!("No active bots found, using nil UUID");
(Uuid::nil(), "default".to_string())
}
Err(e) => {
error!("Failed to query fallback bot: {}", e);
(Uuid::nil(), "default".to_string())
}
}
}
Err(e) => {
error!("Failed to query default bot: {}", e);
(Uuid::nil(), "default".to_string())
@ -72,10 +95,116 @@ impl BotOrchestrator {
}
pub fn mount_all_bots(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!("mount_all_bots called");
info!("Scanning drive for .gbai files to mount bots...");
let mut bots_mounted = 0;
let mut bots_created = 0;
let directories_to_scan: Vec<std::path::PathBuf> = vec![
self.state
.config
.as_ref()
.map(|c| c.site_path.clone())
.unwrap_or_else(|| "./botserver-stack/sites".to_string())
.into(),
"./templates".into(),
"../bottemplates".into(),
];
for dir_path in directories_to_scan {
info!("Checking directory for bots: {}", dir_path.display());
if !dir_path.exists() {
info!("Directory does not exist, skipping: {}", dir_path.display());
continue;
}
match self.scan_directory(&dir_path, &mut bots_mounted, &mut bots_created) {
Ok(()) => {}
Err(e) => {
error!("Failed to scan directory {}: {}", dir_path.display(), e);
}
}
}
info!(
"Bot mounting complete: {} bots processed ({} created, {} already existed)",
bots_mounted,
bots_created,
bots_mounted - bots_created
);
Ok(())
}
fn scan_directory(
&self,
dir_path: &std::path::Path,
bots_mounted: &mut i32,
_bots_created: &mut i32,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let entries =
std::fs::read_dir(dir_path).map_err(|e| format!("Failed to read directory: {}", e))?;
for entry in entries.flatten() {
let name = entry.file_name();
let bot_name = match name.to_str() {
Some(n) if n.ends_with(".gbai") => n.trim_end_matches(".gbai"),
_ => continue,
};
info!("Found .gbai file: {}", bot_name);
match self.ensure_bot_exists(bot_name) {
Ok(true) => {
info!("Bot '{}' already exists in database, mounting", bot_name);
*bots_mounted += 1;
}
Ok(false) => {
info!(
"Bot '{}' does not exist in database, skipping (run import to create)",
bot_name
);
}
Err(e) => {
error!("Failed to check if bot '{}' exists: {}", bot_name, e);
}
}
}
Ok(())
}
fn ensure_bot_exists(
&self,
bot_name: &str,
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
use diesel::sql_query;
let mut conn = self
.state
.conn
.get()
.map_err(|e| format!("Failed to get database connection: {e}"))?;
#[derive(diesel::QueryableByName)]
#[diesel(check_for_backend(diesel::pg::Pg))]
struct BotExistsResult {
#[diesel(sql_type = diesel::sql_types::Bool)]
exists: bool,
}
let exists: BotExistsResult = sql_query(
"SELECT EXISTS(SELECT 1 FROM bots WHERE name = $1 AND is_active = true) as exists",
)
.bind::<diesel::sql_types::Text, _>(bot_name)
.get_result(&mut conn)
.map_err(|e| format!("Failed to check if bot exists: {e}"))?;
Ok(exists.exists)
}
#[cfg(feature = "llm")]
pub async fn stream_response(
&self,
@ -90,7 +219,7 @@ impl BotOrchestrator {
let user_id = Uuid::parse_str(&message.user_id)?;
let session_id = Uuid::parse_str(&message.session_id)?;
let bot_id = Uuid::parse_str(&message.bot_id).unwrap_or_default();
let message_content = message.content.clone();
let (session, context_data, history, model, key) = {
let state_clone = self.state.clone();
@ -118,13 +247,24 @@ impl BotOrchestrator {
};
let config_manager = ConfigManager::new(state_clone.conn.clone());
// DEBUG: Log which bot we're getting config for
info!("[CONFIG_TRACE] Getting LLM config for bot_id: {}", session.bot_id);
let model = config_manager
.get_config(&bot_id, "llm-model", Some("gpt-3.5-turbo"))
.get_config(&session.bot_id, "llm-model", Some("gpt-3.5-turbo"))
.unwrap_or_else(|_| "gpt-3.5-turbo".to_string());
let key = config_manager
.get_config(&bot_id, "llm-key", Some(""))
.get_config(&session.bot_id, "llm-key", Some(""))
.unwrap_or_default();
// DEBUG: Log the exact config values retrieved
info!("[CONFIG_TRACE] Model: '{}'", model);
info!("[CONFIG_TRACE] API Key: '{}' ({} chars)", key, key.len());
info!("[CONFIG_TRACE] API Key first 10 chars: '{}'", &key.chars().take(10).collect::<String>());
info!("[CONFIG_TRACE] API Key last 10 chars: '{}'", &key.chars().rev().take(10).collect::<String>());
Ok((session, context_data, history, model, key))
},
)
@ -132,7 +272,39 @@ impl BotOrchestrator {
};
let system_prompt = "You are a helpful assistant.".to_string();
let messages = OpenAIClient::build_messages(&system_prompt, &context_data, &history);
let mut messages = OpenAIClient::build_messages(&system_prompt, &context_data, &history);
#[cfg(any(feature = "research", feature = "llm"))]
{
if let Some(kb_manager) = self.state.kb_manager.as_ref() {
let bot_name_for_kb = {
let conn = self.state.conn.get().ok();
if let Some(mut db_conn) = conn {
use crate::shared::models::schema::bots::dsl::*;
bots.filter(id.eq(session.bot_id))
.select(name)
.first::<String>(&mut db_conn)
.unwrap_or_else(|_| "default".to_string())
} else {
"default".to_string()
}
};
if let Err(e) = inject_kb_context(
kb_manager.clone(),
self.state.conn.clone(),
session_id,
&bot_name_for_kb,
&message_content,
&mut messages,
8000,
)
.await
{
error!("Failed to inject KB context: {}", e);
}
}
}
let (stream_tx, mut stream_rx) = mpsc::channel::<String>(100);
let llm = self.state.llm_provider.clone();
@ -140,6 +312,16 @@ impl BotOrchestrator {
let model_clone = model.clone();
let key_clone = key.clone();
let messages_clone = messages.clone();
// DEBUG: Log exact values being passed to LLM
info!("[LLM_CALL] Calling generate_stream with:");
info!("[LLM_CALL] Model: '{}'", model_clone);
info!("[LLM_CALL] Key length: {} chars", key_clone.len());
info!("[LLM_CALL] Key preview: '{}...{}'",
&key_clone.chars().take(8).collect::<String>(),
&key_clone.chars().rev().take(8).collect::<String>()
);
tokio::spawn(async move {
if let Err(e) = llm
.generate_stream("", &messages_clone, stream_tx, &model_clone, &key_clone)
@ -161,7 +343,7 @@ impl BotOrchestrator {
let initial_tokens = crate::shared::utils::estimate_token_count(&context_data);
let config_manager = ConfigManager::new(self.state.conn.clone());
let max_context_size = config_manager
.get_config(&bot_id, "llm-server-ctx-size", None)
.get_config(&session.bot_id, "llm-server-ctx-size", None)
.unwrap_or_default()
.parse::<usize>()
.unwrap_or(0);
@ -368,6 +550,10 @@ pub async fn websocket_handler(
.get("session_id")
.and_then(|s| Uuid::parse_str(s).ok());
let user_id = params.get("user_id").and_then(|s| Uuid::parse_str(s).ok());
let bot_name = params
.get("bot_name")
.cloned()
.unwrap_or_else(|| "default".to_string());
if session_id.is_none() || user_id.is_none() {
return (
@ -380,9 +566,34 @@ pub async fn websocket_handler(
let session_id = session_id.unwrap_or_default();
let user_id = user_id.unwrap_or_default();
ws.on_upgrade(move |socket| {
handle_websocket(socket, state, session_id, user_id)
// Look up bot_id from bot_name
let bot_id = {
let conn = state.conn.get().ok();
if let Some(mut db_conn) = conn {
use crate::shared::models::schema::bots::dsl::*;
// Try to parse as UUID first, if that fails treat as bot name
let result: Result<Uuid, _> = if let Ok(uuid) = Uuid::parse_str(&bot_name) {
// Parameter is a UUID, look up by id
bots.filter(id.eq(uuid)).select(id).first(&mut db_conn)
} else {
// Parameter is a bot name, look up by name
bots.filter(name.eq(&bot_name))
.select(id)
.first(&mut db_conn)
};
result.unwrap_or_else(|_| {
log::warn!("Bot not found: {}, using nil bot_id", bot_name);
Uuid::nil()
})
} else {
log::warn!("Could not get database connection, using nil bot_id");
Uuid::nil()
}
};
ws.on_upgrade(move |socket| handle_websocket(socket, state, session_id, user_id, bot_id))
.into_response()
}
@ -391,6 +602,7 @@ async fn handle_websocket(
state: Arc<AppState>,
session_id: Uuid,
user_id: Uuid,
bot_id: Uuid,
) {
let (mut sender, mut receiver) = socket.split();
let (tx, mut rx) = mpsc::channel::<BotResponse>(100);
@ -406,14 +618,15 @@ async fn handle_websocket(
}
info!(
"WebSocket connected for session: {}, user: {}",
session_id, user_id
"WebSocket connected for session: {}, user: {}, bot: {}",
session_id, user_id, bot_id
);
let welcome = serde_json::json!({
"type": "connected",
"session_id": session_id,
"user_id": user_id,
"bot_id": bot_id,
"message": "Connected to bot server"
});
@ -423,6 +636,89 @@ async fn handle_websocket(
}
}
// Execute start.bas automatically on connection (similar to auth.ast pattern)
{
let bot_name_result = {
let conn = state.conn.get().ok();
if let Some(mut db_conn) = conn {
use crate::shared::models::schema::bots::dsl::*;
bots.filter(id.eq(bot_id))
.select(name)
.first::<String>(&mut db_conn)
.ok()
} else {
None
}
};
// DEBUG: Log start script execution attempt
info!(
"Checking for start.bas: bot_id={}, bot_name_result={:?}",
bot_id,
bot_name_result
);
if let Some(bot_name) = bot_name_result {
let start_script_path = format!("./work/{}.gbai/{}.gbdialog/start.bas", bot_name, bot_name);
info!("Looking for start.bas at: {}", start_script_path);
if let Ok(metadata) = tokio::fs::metadata(&start_script_path).await {
if metadata.is_file() {
info!("Found start.bas file, reading contents...");
if let Ok(start_script) = tokio::fs::read_to_string(&start_script_path).await {
info!(
"Executing start.bas for bot {} on session {}",
bot_name, session_id
);
let state_for_start = state.clone();
let _tx_for_start = tx.clone();
tokio::spawn(async move {
let session_result = {
let mut sm = state_for_start.session_manager.lock().await;
sm.get_session_by_id(session_id)
};
if let Ok(Some(session)) = session_result {
info!("Executing start.bas for bot {} on session {}", bot_name, session_id);
let result = tokio::task::spawn_blocking(move || {
let mut script_service = crate::basic::ScriptService::new(
state_for_start.clone(),
session.clone()
);
script_service.load_bot_config_params(&state_for_start, bot_id);
match script_service.compile(&start_script) {
Ok(ast) => match script_service.run(&ast) {
Ok(_) => Ok(()),
Err(e) => Err(format!("Script execution error: {}", e)),
},
Err(e) => Err(format!("Script compilation error: {}", e)),
}
}).await;
match result {
Ok(Ok(())) => {
info!("start.bas executed successfully for bot {}", bot_name);
}
Ok(Err(e)) => {
error!("start.bas error for bot {}: {}", bot_name, e);
}
Err(e) => {
error!("start.bas task error for bot {}: {}", bot_name, e);
}
}
}
});
}
}
}
}
}
let mut send_task = tokio::spawn(async move {
while let Some(response) = rx.recv().await {
if let Ok(json_str) = serde_json::to_string(&response) {
@ -447,8 +743,13 @@ async fn handle_websocket(
.await
.get(&session_id.to_string())
{
// Use bot_id from WebSocket connection instead of from message
let corrected_msg = UserMessage {
bot_id: bot_id.to_string(),
..user_msg
};
if let Err(e) = orchestrator
.stream_response(user_msg, tx_clone.clone())
.stream_response(corrected_msg, tx_clone.clone())
.await
{
error!("Failed to stream response: {}", e);

View file

@ -362,14 +362,55 @@ impl ConfigManager {
use crate::shared::models::schema::bot_configuration::dsl::*;
let mut conn = self.get_conn()?;
let fallback_str = fallback.unwrap_or("");
// Helper function to check if a value should be treated as "not configured"
fn is_placeholder_value(value: &str) -> bool {
let trimmed = value.trim().to_lowercase();
trimmed.is_empty() || trimmed == "none" || trimmed == "null" || trimmed == "n/a"
}
// Helper function to check if a value is a local file path (for local LLM server)
// These should fall back to default bot's config when using remote API
fn is_local_file_path(value: &str) -> bool {
let value = value.trim();
// Check for file path patterns
value.starts_with("../") ||
value.starts_with("./") ||
value.starts_with('/') ||
value.starts_with("~") ||
value.contains(".gguf") ||
value.contains(".bin") ||
value.contains(".safetensors") ||
value.starts_with("data/") ||
value.starts_with("../../") ||
value.starts_with("models/")
}
// Try to get value for the specific bot
let result = bot_configuration
.filter(bot_id.eq(code_bot_id))
.filter(config_key.eq(key))
.select(config_value)
.first::<String>(&mut conn);
let value = match result {
Ok(v) => v,
Ok(v) => {
// Check if it's a placeholder value or local file path - if so, fall back to default bot
// Local file paths are valid for local LLM server but NOT for remote APIs
if is_placeholder_value(&v) || is_local_file_path(&v) {
let (default_bot_id, _default_bot_name) = crate::bot::get_default_bot(&mut conn);
bot_configuration
.filter(bot_id.eq(default_bot_id))
.filter(config_key.eq(key))
.select(config_value)
.first::<String>(&mut conn)
.unwrap_or_else(|_| fallback_str.to_string())
} else {
v
}
}
Err(_) => {
// Value not found, fall back to default bot
let (default_bot_id, _default_bot_name) = crate::bot::get_default_bot(&mut conn);
bot_configuration
.filter(bot_id.eq(default_bot_id))
@ -379,7 +420,15 @@ impl ConfigManager {
.unwrap_or_else(|_| fallback_str.to_string())
}
};
Ok(value)
// Final check: if the result is still a placeholder value, use the fallback_str
let final_value = if is_placeholder_value(&value) {
fallback_str.to_string()
} else {
value
};
Ok(final_value)
}
pub fn get_bot_config_value(

374
src/core/incus/cloud.rs Normal file
View file

@ -0,0 +1,374 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::process::Command;
use tokio::process::Command as AsyncCommand;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IncusCloudConfig {
pub cluster_name: String,
pub nodes: Vec<IncusNode>,
pub storage_pools: Vec<StoragePool>,
pub networks: Vec<NetworkConfig>,
pub profiles: Vec<ProfileConfig>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IncusNode {
pub name: String,
pub address: String,
pub role: NodeRole,
pub resources: NodeResources,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum NodeRole {
Controller,
Worker,
Storage,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NodeResources {
pub cpu_cores: u32,
pub memory_gb: u32,
pub storage_gb: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StoragePool {
pub name: String,
pub driver: String,
pub config: HashMap<String, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NetworkConfig {
pub name: String,
pub type_: String,
pub config: HashMap<String, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProfileConfig {
pub name: String,
pub devices: HashMap<String, HashMap<String, String>>,
pub config: HashMap<String, String>,
}
pub struct IncusCloudManager {
config: IncusCloudConfig,
}
impl IncusCloudManager {
pub fn new(config: IncusCloudConfig) -> Self {
Self { config }
}
pub async fn bootstrap_cluster(&self) -> Result<()> {
self.init_first_node().await?;
self.setup_storage_pools().await?;
self.setup_networks().await?;
self.setup_profiles().await?;
self.join_additional_nodes().await?;
Ok(())
}
async fn init_first_node(&self) -> Result<()> {
let first_node = &self.config.nodes[0];
let output = AsyncCommand::new("incus")
.args(&["admin", "init", "--auto"])
.output()
.await?;
if !output.status.success() {
return Err(anyhow::anyhow!("Failed to initialize Incus: {}",
String::from_utf8_lossy(&output.stderr)));
}
AsyncCommand::new("incus")
.args(&["config", "set", "cluster.https_address", &first_node.address])
.output()
.await?;
AsyncCommand::new("incus")
.args(&["config", "set", "core.https_address", &first_node.address])
.output()
.await?;
Ok(())
}
async fn setup_storage_pools(&self) -> Result<()> {
for pool in &self.config.storage_pools {
let mut args = vec!["storage", "create", &pool.name, &pool.driver];
for (key, value) in &pool.config {
args.push(key);
args.push(value);
}
AsyncCommand::new("incus")
.args(&args)
.output()
.await?;
}
Ok(())
}
async fn setup_networks(&self) -> Result<()> {
for network in &self.config.networks {
let mut args = vec!["network", "create", &network.name, "--type", &network.type_];
for (key, value) in &network.config {
args.push("--config");
args.push(&format!("{}={}", key, value));
}
AsyncCommand::new("incus")
.args(&args)
.output()
.await?;
}
Ok(())
}
async fn setup_profiles(&self) -> Result<()> {
for profile in &self.config.profiles {
AsyncCommand::new("incus")
.args(&["profile", "create", &profile.name])
.output()
.await?;
for (key, value) in &profile.config {
AsyncCommand::new("incus")
.args(&["profile", "set", &profile.name, key, value])
.output()
.await?;
}
for (device_name, device_config) in &profile.devices {
let mut args = vec!["profile", "device", "add", &profile.name, device_name];
for (key, value) in device_config {
args.push(key);
args.push(value);
}
AsyncCommand::new("incus")
.args(&args)
.output()
.await?;
}
}
Ok(())
}
async fn join_additional_nodes(&self) -> Result<()> {
if self.config.nodes.len() <= 1 {
return Ok(());
}
let token_output = AsyncCommand::new("incus")
.args(&["cluster", "add", "new-node"])
.output()
.await?;
let token = String::from_utf8_lossy(&token_output.stdout).trim().to_string();
for node in &self.config.nodes[1..] {
self.join_node_to_cluster(&node.address, &token).await?;
}
Ok(())
}
async fn join_node_to_cluster(&self, node_address: &str, token: &str) -> Result<()> {
AsyncCommand::new("ssh")
.args(&[
node_address,
&format!("incus admin init --join-token {}", token)
])
.output()
.await?;
Ok(())
}
pub async fn deploy_component(&self, component_name: &str, node_name: Option<&str>) -> Result<String> {
let instance_name = format!("gb-{}-{}", component_name, uuid::Uuid::new_v4().to_string()[..8].to_string());
let mut args = vec!["launch", "ubuntu:24.04", &instance_name, "--profile", "gbo"];
if let Some(node) = node_name {
args.extend(&["--target", node]);
}
let output = AsyncCommand::new("incus")
.args(&args)
.output()
.await?;
if !output.status.success() {
return Err(anyhow::anyhow!("Failed to launch instance: {}",
String::from_utf8_lossy(&output.stderr)));
}
AsyncCommand::new("incus")
.args(&["exec", &instance_name, "--", "cloud-init", "status", "--wait"])
.output()
.await?;
self.setup_component_in_instance(&instance_name, component_name).await?;
Ok(instance_name)
}
async fn setup_component_in_instance(&self, instance_name: &str, component_name: &str) -> Result<()> {
let setup_script = format!(r#"
#!/bin/bash
set -e
# Update system
apt-get update -qq
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq wget curl unzip ca-certificates
# Create gbo directories
mkdir -p /opt/gbo/{{bin,data,conf,logs}}
# Create gbo user
useradd --system --no-create-home --shell /bin/false gbuser
chown -R gbuser:gbuser /opt/gbo
# Install component: {}
echo "Component {} setup complete"
"#, component_name, component_name);
AsyncCommand::new("incus")
.args(&["exec", instance_name, "--", "bash", "-c", &setup_script])
.output()
.await?;
Ok(())
}
pub async fn create_vm(&self, vm_name: &str, template: &str) -> Result<String> {
let output = AsyncCommand::new("incus")
.args(&["launch", template, vm_name, "--vm", "--profile", "gbo-vm"])
.output()
.await?;
if !output.status.success() {
return Err(anyhow::anyhow!("Failed to create VM: {}",
String::from_utf8_lossy(&output.stderr)));
}
Ok(vm_name.to_string())
}
pub async fn get_cluster_status(&self) -> Result<serde_json::Value> {
let output = AsyncCommand::new("incus")
.args(&["cluster", "list", "--format", "json"])
.output()
.await?;
let status: serde_json::Value = serde_json::from_slice(&output.stdout)?;
Ok(status)
}
pub async fn get_instances(&self) -> Result<serde_json::Value> {
let output = AsyncCommand::new("incus")
.args(&["list", "--format", "json"])
.output()
.await?;
let instances: serde_json::Value = serde_json::from_slice(&output.stdout)?;
Ok(instances)
}
pub async fn get_metrics(&self) -> Result<serde_json::Value> {
let output = AsyncCommand::new("incus")
.args(&["query", "/1.0/metrics"])
.output()
.await?;
let metrics: serde_json::Value = serde_json::from_slice(&output.stdout)?;
Ok(metrics)
}
}
pub fn create_default_cloud_config() -> IncusCloudConfig {
IncusCloudConfig {
cluster_name: "gbo-cloud".to_string(),
nodes: vec![
IncusNode {
name: "controller-1".to_string(),
address: "10.0.0.10:8443".to_string(),
role: NodeRole::Controller,
resources: NodeResources {
cpu_cores: 8,
memory_gb: 16,
storage_gb: 500,
},
}
],
storage_pools: vec![
StoragePool {
name: "gbo-pool".to_string(),
driver: "zfs".to_string(),
config: HashMap::from([
("size".to_string(), "100GB".to_string()),
]),
}
],
networks: vec![
NetworkConfig {
name: "gbo-net".to_string(),
type_: "bridge".to_string(),
config: HashMap::from([
("ipv4.address".to_string(), "10.10.10.1/24".to_string()),
("ipv4.nat".to_string(), "true".to_string()),
]),
}
],
profiles: vec![
ProfileConfig {
name: "gbo".to_string(),
devices: HashMap::from([
("eth0".to_string(), HashMap::from([
("type".to_string(), "nic".to_string()),
("network".to_string(), "gbo-net".to_string()),
])),
("root".to_string(), HashMap::from([
("type".to_string(), "disk".to_string()),
("pool".to_string(), "gbo-pool".to_string()),
("path".to_string(), "/".to_string()),
])),
]),
config: HashMap::from([
("security.privileged".to_string(), "true".to_string()),
("limits.cpu".to_string(), "2".to_string()),
("limits.memory".to_string(), "4GB".to_string()),
]),
},
ProfileConfig {
name: "gbo-vm".to_string(),
devices: HashMap::from([
("eth0".to_string(), HashMap::from([
("type".to_string(), "nic".to_string()),
("network".to_string(), "gbo-net".to_string()),
])),
("root".to_string(), HashMap::from([
("type".to_string(), "disk".to_string()),
("pool".to_string(), "gbo-pool".to_string()),
("path".to_string(), "/".to_string()),
("size".to_string(), "20GB".to_string()),
])),
]),
config: HashMap::from([
("limits.cpu".to_string(), "4".to_string()),
("limits.memory".to_string(), "8GB".to_string()),
]),
}
],
}
}

View file

@ -5,6 +5,7 @@ use crate::shared::state::AppState;
use crate::shared::utils::DbPool;
use diesel::prelude::*;
use log::{error, info, warn};
use regex;
use std::sync::Arc;
use tokio::time::{interval, Duration};
use uuid::Uuid;
@ -22,7 +23,7 @@ impl WebsiteCrawlerService {
Self {
db_pool,
kb_manager,
check_interval: Duration::from_secs(3600),
check_interval: Duration::from_secs(60),
running: Arc::new(tokio::sync::RwLock::new(false)),
}
}
@ -57,10 +58,13 @@ impl WebsiteCrawlerService {
fn check_and_crawl_websites(&self) -> Result<(), Box<dyn std::error::Error>> {
info!("Checking for websites that need recrawling");
// First, scan for new USE WEBSITE commands in .bas files
self.scan_and_register_websites_from_scripts()?;
let mut conn = self.db_pool.get()?;
let websites = diesel::sql_query(
"SELECT id, bot_id, url, expires_policy, max_depth, max_pages
"SELECT id, bot_id, url, expires_policy, refresh_policy, max_depth, max_pages
FROM website_crawls
WHERE next_crawl <= NOW()
AND crawl_status != 2
@ -116,6 +120,7 @@ impl WebsiteCrawlerService {
max_pages: website_max_pages,
crawl_delay_ms: 500,
expires_policy: website.expires_policy.clone(),
refresh_policy: website.refresh_policy.clone(),
last_crawled: None,
next_crawl: None,
};
@ -207,6 +212,103 @@ impl WebsiteCrawlerService {
Ok(())
}
fn scan_and_register_websites_from_scripts(&self) -> Result<(), Box<dyn std::error::Error>> {
info!("Scanning .bas files for USE WEBSITE commands");
let work_dir = std::path::Path::new("work");
if !work_dir.exists() {
return Ok(());
}
let mut conn = self.db_pool.get()?;
for entry in std::fs::read_dir(work_dir)? {
let entry = entry?;
let path = entry.path();
if path.is_dir() && path.file_name().unwrap().to_string_lossy().ends_with(".gbai") {
let bot_name = path.file_name().unwrap().to_string_lossy().replace(".gbai", "");
// Get bot_id from database
#[derive(QueryableByName)]
struct BotIdResult {
#[diesel(sql_type = diesel::sql_types::Uuid)]
id: uuid::Uuid,
}
let bot_id_result: Result<BotIdResult, _> = diesel::sql_query("SELECT id FROM bots WHERE name = $1")
.bind::<diesel::sql_types::Text, _>(&bot_name)
.get_result(&mut conn);
let bot_id = match bot_id_result {
Ok(result) => result.id,
Err(_) => continue, // Skip if bot not found
};
// Scan .gbdialog directory for .bas files
let dialog_dir = path.join(format!("{}.gbdialog", bot_name));
if dialog_dir.exists() {
self.scan_directory_for_websites(&dialog_dir, bot_id, &mut conn)?;
}
}
}
Ok(())
}
fn scan_directory_for_websites(
&self,
dir: &std::path::Path,
bot_id: uuid::Uuid,
conn: &mut diesel::PgConnection,
) -> Result<(), Box<dyn std::error::Error>> {
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if path.extension().map_or(false, |ext| ext == "bas") {
let content = std::fs::read_to_string(&path)?;
// Regex to find USE WEBSITE commands with optional REFRESH parameter
let re = regex::Regex::new(r#"USE\s+WEBSITE\s+"([^"]+)"(?:\s+REFRESH\s+"([^"]+)")?"#)?;
for cap in re.captures_iter(&content) {
if let Some(url) = cap.get(1) {
let url_str = url.as_str();
let refresh_str = cap.get(2).map(|m| m.as_str()).unwrap_or("1m");
// Check if already registered
let exists = diesel::sql_query(
"SELECT COUNT(*) as count FROM website_crawls WHERE bot_id = $1 AND url = $2"
)
.bind::<diesel::sql_types::Uuid, _>(&bot_id)
.bind::<diesel::sql_types::Text, _>(url_str)
.get_result::<CountResult>(conn)
.map(|r| r.count)
.unwrap_or(0);
if exists == 0 {
info!("Auto-registering website {} for bot {} with refresh: {}", url_str, bot_id, refresh_str);
// Register website for crawling with refresh policy
crate::basic::keywords::use_website::register_website_for_crawling_with_refresh(
conn, &bot_id, url_str, refresh_str
)?;
}
}
}
}
}
Ok(())
}
}
#[derive(QueryableByName)]
struct CountResult {
#[diesel(sql_type = diesel::sql_types::BigInt)]
count: i64,
}
#[derive(QueryableByName, Debug)]
@ -219,6 +321,8 @@ struct WebsiteCrawlRecord {
url: String,
#[diesel(sql_type = diesel::sql_types::Text)]
expires_policy: String,
#[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Text>)]
refresh_policy: Option<String>,
#[diesel(sql_type = diesel::sql_types::Integer)]
max_depth: i32,
#[diesel(sql_type = diesel::sql_types::Integer)]

View file

@ -12,8 +12,6 @@ use std::path::PathBuf;
#[derive(Deserialize, Debug)]
struct ComponentEntry {
url: String,
filename: String,
sha256: String,
}
#[derive(Deserialize, Debug)]
@ -33,6 +31,7 @@ fn get_component_url(name: &str) -> Option<String> {
.map(|c| c.url.clone())
}
#[cfg(target_os = "windows")]
fn safe_nvcc_version() -> Option<std::process::Output> {
SafeCommand::new("nvcc")
.and_then(|c| c.arg("--version"))

View file

@ -44,6 +44,14 @@ pub struct ProductConfig {
/// Copyright text (optional)
pub copyright: Option<String>,
/// Search mechanism enabled (optional)
/// Controls whether the omnibox/search toolbar is displayed in the suite
pub search_enabled: Option<bool>,
/// Menu launcher enabled (optional)
/// Controls whether the apps menu launcher is displayed in the suite
pub menu_launcher_enabled: Option<bool>,
}
impl Default for ProductConfig {
@ -81,6 +89,8 @@ impl Default for ProductConfig {
support_email: None,
docs_url: None,
copyright: None,
search_enabled: Some(false),
menu_launcher_enabled: Some(false),
}
}
}
@ -179,6 +189,18 @@ impl ProductConfig {
config.copyright = Some(value.to_string());
}
}
"search_enabled" => {
let enabled = value.eq_ignore_ascii_case("true")
|| value == "1"
|| value.eq_ignore_ascii_case("yes");
config.search_enabled = Some(enabled);
}
"menu_launcher_enabled" => {
let enabled = value.eq_ignore_ascii_case("true")
|| value == "1"
|| value.eq_ignore_ascii_case("yes");
config.menu_launcher_enabled = Some(enabled);
}
_ => {
warn!("Unknown product configuration key: {}", key);
}
@ -331,6 +353,8 @@ pub fn get_product_config_json() -> serde_json::Value {
"primary_color": c.primary_color,
"docs_url": c.docs_url,
"copyright": c.get_copyright(),
"search_enabled": c.search_enabled.unwrap_or(false),
"menu_launcher_enabled": c.menu_launcher_enabled.unwrap_or(false),
}),
None => serde_json::json!({
"name": "General Bots",
@ -338,6 +362,8 @@ pub fn get_product_config_json() -> serde_json::Value {
"compiled_features": compiled,
"version": env!("CARGO_PKG_VERSION"),
"theme": "sentient",
"search_enabled": false,
"menu_launcher_enabled": false,
}),
}
}

View file

@ -2,12 +2,12 @@ use crate::core::config::DriveConfig;
use crate::core::secrets::SecretsManager;
use anyhow::{Context, Result};
#[cfg(feature = "drive")]
use aws_config::BehaviorVersion;
#[cfg(feature = "drive")]
use aws_config::retry::RetryConfig;
#[cfg(feature = "drive")]
use aws_config::timeout::TimeoutConfig;
#[cfg(feature = "drive")]
use aws_config::BehaviorVersion;
#[cfg(feature = "drive")]
use aws_sdk_s3::{config::Builder as S3ConfigBuilder, Client as S3Client};
use diesel::Connection;
use diesel::{
@ -112,7 +112,10 @@ pub async fn create_s3_operator(
if std::path::Path::new(CA_CERT_PATH).exists() {
std::env::set_var("AWS_CA_BUNDLE", CA_CERT_PATH);
std::env::set_var("SSL_CERT_FILE", CA_CERT_PATH);
debug!("Set AWS_CA_BUNDLE and SSL_CERT_FILE to {} for S3 client", CA_CERT_PATH);
debug!(
"Set AWS_CA_BUNDLE and SSL_CERT_FILE to {} for S3 client",
CA_CERT_PATH
);
}
// Configure timeouts to prevent memory leaks on connection failures
@ -124,8 +127,7 @@ pub async fn create_s3_operator(
.build();
// Limit retries to prevent 100% CPU on connection failures
let retry_config = RetryConfig::standard()
.with_max_attempts(2);
let retry_config = RetryConfig::standard().with_max_attempts(2);
let base_config = aws_config::defaults(BehaviorVersion::latest())
.endpoint_url(endpoint)
@ -330,145 +332,316 @@ pub fn run_migrations(pool: &DbPool) -> Result<(), Box<dyn std::error::Error + S
run_migrations_on_conn(&mut conn)
}
pub fn run_migrations_on_conn(conn: &mut diesel::PgConnection) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
pub fn run_migrations_on_conn(
conn: &mut diesel::PgConnection,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};
// Core migrations (Always run)
const CORE_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/core");
conn.run_pending_migrations(CORE_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Core migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(CORE_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!(
"Core migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
// Calendar
#[cfg(feature = "calendar")]
{
const CALENDAR_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/calendar");
conn.run_pending_migrations(CALENDAR_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Calendar migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(CALENDAR_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Calendar migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// People (CRM)
#[cfg(feature = "people")]
{
const PEOPLE_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/people");
conn.run_pending_migrations(PEOPLE_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("People migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(PEOPLE_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"People migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Mail
#[cfg(feature = "mail")]
{
const MAIL_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/mail");
conn.run_pending_migrations(MAIL_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Mail migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(MAIL_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!(
"Mail migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Tasks
#[cfg(feature = "tasks")]
{
const TASKS_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/tasks");
conn.run_pending_migrations(TASKS_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Tasks migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(TASKS_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!(
"Tasks migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Drive
#[cfg(feature = "drive")]
{
const DRIVE_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/drive");
conn.run_pending_migrations(DRIVE_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Drive migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(DRIVE_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!(
"Drive migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Automation
#[cfg(feature = "automation")]
{
const AUTOMATION_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/automation");
conn.run_pending_migrations(AUTOMATION_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Automation migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
const AUTOMATION_MIGRATIONS: EmbeddedMigrations =
embed_migrations!("migrations/automation");
conn.run_pending_migrations(AUTOMATION_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Automation migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Paper
#[cfg(feature = "paper")]
{
const PAPER_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/paper");
conn.run_pending_migrations(PAPER_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Paper migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(PAPER_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!(
"Paper migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Designer
#[cfg(feature = "designer")]
{
const DESIGNER_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/designer");
conn.run_pending_migrations(DESIGNER_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Designer migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(DESIGNER_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Designer migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Learn
#[cfg(feature = "learn")]
{
const LEARN_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/learn");
conn.run_pending_migrations(LEARN_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!(
"Learn migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Video
#[cfg(feature = "video")]
{
const VIDEO_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/video");
conn.run_pending_migrations(VIDEO_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!(
"Video migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// LLM
#[cfg(feature = "llm")]
{
const LLM_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/llm");
conn.run_pending_migrations(LLM_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("LLM migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(LLM_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!("LLM migration error: {}", e)))
as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Products
#[cfg(feature = "billing")]
{
const PRODUCTS_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/products");
conn.run_pending_migrations(PRODUCTS_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Products migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Billing
const BILLING_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/billing");
conn.run_pending_migrations(BILLING_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Billing migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(BILLING_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Billing migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
// Attendant
#[cfg(feature = "attendant")]
{
const ATTENDANT_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/attendant");
conn.run_pending_migrations(ATTENDANT_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Attendant migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(ATTENDANT_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Attendant migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Analytics
#[cfg(feature = "analytics")]
{
const ANALYTICS_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/analytics");
conn.run_pending_migrations(ANALYTICS_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Analytics migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(ANALYTICS_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Analytics migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Dashboards
#[cfg(feature = "dashboards")]
{
const DASHBOARDS_MIGRATIONS: EmbeddedMigrations =
embed_migrations!("migrations/dashboards");
conn.run_pending_migrations(DASHBOARDS_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Dashboards migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Meet
#[cfg(feature = "meet")]
{
const MEET_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/meet");
conn.run_pending_migrations(MEET_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Meet migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(MEET_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!(
"Meet migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Tickets (Feedback)
const TICKETS_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/tickets");
conn.run_pending_migrations(TICKETS_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Tickets migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(TICKETS_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Tickets migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
// Compliance
#[cfg(feature = "compliance")]
{
const COMPLIANCE_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/compliance");
conn.run_pending_migrations(COMPLIANCE_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Compliance migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
const COMPLIANCE_MIGRATIONS: EmbeddedMigrations =
embed_migrations!("migrations/compliance");
conn.run_pending_migrations(COMPLIANCE_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Compliance migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Canvas
#[cfg(feature = "canvas")]
{
const CANVAS_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/canvas");
conn.run_pending_migrations(CANVAS_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Canvas migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(CANVAS_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Canvas migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Social
#[cfg(feature = "social")]
{
const SOCIAL_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/social");
conn.run_pending_migrations(SOCIAL_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Social migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(SOCIAL_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Social migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Workspaces
#[cfg(feature = "workspaces")]
{
const WORKSPACE_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/workspaces");
conn.run_pending_migrations(WORKSPACE_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Workspace migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(WORKSPACE_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Workspace migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Goals
#[cfg(feature = "goals")]
{
const GOALS_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/goals");
conn.run_pending_migrations(GOALS_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Goals migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(GOALS_MIGRATIONS).map_err(|e| {
Box::new(std::io::Error::other(format!(
"Goals migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
// Research
#[cfg(feature = "research")]
{
const RESEARCH_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/research");
conn.run_pending_migrations(RESEARCH_MIGRATIONS).map_err(|e| Box::new(std::io::Error::other(format!("Research migration error: {}", e))) as Box<dyn std::error::Error + Send + Sync>)?;
conn.run_pending_migrations(RESEARCH_MIGRATIONS)
.map_err(|e| {
Box::new(std::io::Error::other(format!(
"Research migration error: {}",
e
))) as Box<dyn std::error::Error + Send + Sync>
})?;
}
Ok(())
@ -487,7 +660,13 @@ pub fn sanitize_path_component(component: &str) -> String {
pub fn sanitize_path_for_filename(path: &str) -> String {
path.chars()
.map(|c| if c.is_alphanumeric() || c == '_' || c == '-' { c } else { '_' })
.map(|c| {
if c.is_alphanumeric() || c == '_' || c == '-' {
c
} else {
'_'
}
})
.collect()
}
@ -569,23 +748,30 @@ pub fn create_tls_client_with_ca(ca_cert_path: &str, timeout_secs: Option<u64>)
// If it doesn't exist, we use system CA store (production with public certs)
if std::path::Path::new(ca_cert_path).exists() {
match std::fs::read(ca_cert_path) {
Ok(ca_cert_pem) => {
match Certificate::from_pem(&ca_cert_pem) {
Ok(ca_cert_pem) => match Certificate::from_pem(&ca_cert_pem) {
Ok(ca_cert) => {
builder = builder.add_root_certificate(ca_cert);
debug!("Using local CA certificate from {} (dev stack mode)", ca_cert_path);
debug!(
"Using local CA certificate from {} (dev stack mode)",
ca_cert_path
);
}
Err(e) => {
warn!("Failed to parse CA certificate from {}: {}", ca_cert_path, e);
}
}
warn!(
"Failed to parse CA certificate from {}: {}",
ca_cert_path, e
);
}
},
Err(e) => {
warn!("Failed to read CA certificate from {}: {}", ca_cert_path, e);
}
}
} else {
debug!("Local CA cert not found at {}, using system CA store (production mode)", ca_cert_path);
debug!(
"Local CA cert not found at {}, using system CA store (production mode)",
ca_cert_path
);
}
builder.build().unwrap_or_else(|e| {
@ -606,7 +792,13 @@ pub fn format_timestamp_vtt(ms: i64) -> String {
let mins = secs / 60;
let hours = mins / 60;
let millis = ms % 1000;
format!("{:02}:{:02}:{:02}.{:03}", hours, mins % 60, secs % 60, millis)
format!(
"{:02}:{:02}:{:02}.{:03}",
hours,
mins % 60,
secs % 60,
millis
)
}
pub fn format_timestamp_srt(ms: i64) -> String {
@ -614,7 +806,13 @@ pub fn format_timestamp_srt(ms: i64) -> String {
let mins = secs / 60;
let hours = mins / 60;
let millis = ms % 1000;
format!("{:02}:{:02}:{:02},{:03}", hours, mins % 60, secs % 60, millis)
format!(
"{:02}:{:02}:{:02},{:03}",
hours,
mins % 60,
secs % 60,
millis
)
}
pub fn parse_hex_color(hex: &str) -> Option<(u8, u8, u8)> {

View file

@ -1,6 +1,6 @@
use async_trait::async_trait;
use futures::StreamExt;
use log::{info, trace};
use log::{error, info};
use serde_json::Value;
use std::sync::Arc;
use tokio::sync::{mpsc, RwLock};
@ -8,7 +8,6 @@ use tokio::sync::{mpsc, RwLock};
pub mod cache;
pub mod claude;
pub mod episodic_memory;
pub mod smart_router;
pub mod llm_models;
pub mod local;
pub mod smart_router;
@ -45,13 +44,116 @@ pub trait LLMProvider: Send + Sync {
pub struct OpenAIClient {
client: reqwest::Client,
base_url: String,
endpoint_path: String,
}
impl OpenAIClient {
pub fn new(_api_key: String, base_url: Option<String>) -> Self {
/// Estimates token count for a text string (roughly 4 characters per token for English)
fn estimate_tokens(text: &str) -> usize {
// Rough estimate: ~4 characters per token for English text
// This is a heuristic and may not be accurate for all languages
text.len().div_ceil(4)
}
/// Estimates total tokens for a messages array
fn estimate_messages_tokens(messages: &Value) -> usize {
if let Some(msg_array) = messages.as_array() {
msg_array
.iter()
.map(|msg| {
if let Some(content) = msg.get("content").and_then(|c| c.as_str()) {
Self::estimate_tokens(content)
} else {
0
}
})
.sum()
} else {
0
}
}
/// Truncates messages to fit within the max_tokens limit
/// Keeps system messages and the most recent user/assistant messages
fn truncate_messages(messages: &Value, max_tokens: usize) -> Value {
let mut result = Vec::new();
let mut token_count = 0;
if let Some(msg_array) = messages.as_array() {
// First pass: keep all system messages
for msg in msg_array {
if let Some(role) = msg.get("role").and_then(|r| r.as_str()) {
if role == "system" {
if let Some(content) = msg.get("content").and_then(|c| c.as_str()) {
let msg_tokens = Self::estimate_tokens(content);
if token_count + msg_tokens <= max_tokens {
result.push(msg.clone());
token_count += msg_tokens;
}
}
}
}
}
// Second pass: add user/assistant messages from newest to oldest
let mut recent_messages: Vec<&Value> = msg_array
.iter()
.filter(|msg| msg.get("role").and_then(|r| r.as_str()) != Some("system"))
.collect();
// Reverse to get newest first
recent_messages.reverse();
for msg in recent_messages {
if let Some(content) = msg.get("content").and_then(|c| c.as_str()) {
let msg_tokens = Self::estimate_tokens(content);
if token_count + msg_tokens <= max_tokens {
result.push(msg.clone());
token_count += msg_tokens;
} else {
break;
}
}
}
// Reverse back to chronological order for non-system messages
// But keep system messages at the beginning
let system_count = result.len()
- result
.iter()
.filter(|m| m.get("role").and_then(|r| r.as_str()) != Some("system"))
.count();
let mut user_messages: Vec<Value> = result.drain(system_count..).collect();
user_messages.reverse();
result.extend(user_messages);
}
serde_json::Value::Array(result)
}
/// Ensures messages fit within model's context limit
fn ensure_token_limit(messages: &Value, model_context_limit: usize) -> Value {
let estimated_tokens = Self::estimate_messages_tokens(messages);
// Use 90% of context limit to leave room for response
let safe_limit = (model_context_limit as f64 * 0.9) as usize;
if estimated_tokens > safe_limit {
log::warn!(
"Messages exceed token limit ({} > {}), truncating...",
estimated_tokens,
safe_limit
);
Self::truncate_messages(messages, safe_limit)
} else {
messages.clone()
}
}
pub fn new(_api_key: String, base_url: Option<String>, endpoint_path: Option<String>) -> Self {
Self {
client: reqwest::Client::new(),
base_url: base_url.unwrap_or_else(|| "https://api.openai.com".to_string()),
endpoint_path: endpoint_path.unwrap_or_else(|| "/v1/chat/completions".to_string()),
}
}
@ -93,21 +195,64 @@ impl LLMProvider for OpenAIClient {
key: &str,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let default_messages = serde_json::json!([{"role": "user", "content": prompt}]);
let response = self
.client
.post(format!("{}/v1/chat/completions", self.base_url))
.header("Authorization", format!("Bearer {}", key))
.json(&serde_json::json!({
"model": model,
"messages": if messages.is_array() && !messages.as_array().unwrap_or(&vec![]).is_empty() {
// Get the messages to use
let raw_messages =
if messages.is_array() && !messages.as_array().unwrap_or(&vec![]).is_empty() {
messages
} else {
&default_messages
};
// Ensure messages fit within model's context limit
// GLM-4.7 has 202750 tokens, other models vary
let context_limit = if model.contains("glm-4") || model.contains("GLM-4") {
202750
} else if model.contains("gpt-4") {
128000
} else if model.contains("gpt-3.5") {
16385
} else if model.starts_with("http://localhost:808") || model == "local" {
768 // Local llama.cpp server context limit
} else {
4096 // Default conservative limit
};
let messages = OpenAIClient::ensure_token_limit(raw_messages, context_limit);
let full_url = format!("{}{}", self.base_url, self.endpoint_path);
let auth_header = format!("Bearer {}", key);
// Debug logging to help troubleshoot 401 errors
info!("LLM Request Details:");
info!(" URL: {}", full_url);
info!(" Authorization: Bearer <{} chars>", key.len());
info!(" Model: {}", model);
if let Some(msg_array) = messages.as_array() {
info!(" Messages: {} messages", msg_array.len());
}
info!(" API Key First 8 chars: '{}...'", &key.chars().take(8).collect::<String>());
info!(" API Key Last 8 chars: '...{}'", &key.chars().rev().take(8).collect::<String>());
let response = self
.client
.post(&full_url)
.header("Authorization", &auth_header)
.json(&serde_json::json!({
"model": model,
"messages": messages,
"stream": true
}))
.send()
.await?;
let status = response.status();
if status != reqwest::StatusCode::OK {
let error_text = response.text().await.unwrap_or_default();
error!("LLM generate error: {}", error_text);
return Err(format!("LLM request failed with status: {}", status).into());
}
let result: Value = response.json().await?;
let raw_content = result["choices"][0]["message"]["content"]
.as_str()
@ -128,18 +273,51 @@ impl LLMProvider for OpenAIClient {
key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let default_messages = serde_json::json!([{"role": "user", "content": prompt}]);
let response = self
.client
.post(format!("{}/v1/chat/completions", self.base_url))
.header("Authorization", format!("Bearer {}", key))
.json(&serde_json::json!({
"model": model,
"messages": if messages.is_array() && !messages.as_array().unwrap_or(&vec![]).is_empty() {
// Get the messages to use
let raw_messages =
if messages.is_array() && !messages.as_array().unwrap_or(&vec![]).is_empty() {
info!("Using provided messages: {:?}", messages);
messages
} else {
&default_messages
},
};
// Ensure messages fit within model's context limit
// GLM-4.7 has 202750 tokens, other models vary
let context_limit = if model.contains("glm-4") || model.contains("GLM-4") {
202750
} else if model.contains("gpt-4") {
128000
} else if model.contains("gpt-3.5") {
16385
} else if model.starts_with("http://localhost:808") || model == "local" {
768 // Local llama.cpp server context limit
} else {
4096 // Default conservative limit
};
let messages = OpenAIClient::ensure_token_limit(raw_messages, context_limit);
let full_url = format!("{}{}", self.base_url, self.endpoint_path);
let auth_header = format!("Bearer {}", key);
// Debug logging to help troubleshoot 401 errors
info!("LLM Request Details:");
info!(" URL: {}", full_url);
info!(" Authorization: Bearer <{} chars>", key.len());
info!(" Model: {}", model);
if let Some(msg_array) = messages.as_array() {
info!(" Messages: {} messages", msg_array.len());
}
let response = self
.client
.post(&full_url)
.header("Authorization", &auth_header)
.json(&serde_json::json!({
"model": model,
"messages": messages,
"stream": true
}))
.send()
@ -148,7 +326,7 @@ impl LLMProvider for OpenAIClient {
let status = response.status();
if status != reqwest::StatusCode::OK {
let error_text = response.text().await.unwrap_or_default();
trace!("LLM generate_stream error: {}", error_text);
error!("LLM generate_stream error: {}", error_text);
return Err(format!("LLM request failed with status: {}", status).into());
}
@ -214,11 +392,16 @@ pub fn create_llm_provider(
provider_type: LLMProviderType,
base_url: String,
deployment_name: Option<String>,
endpoint_path: Option<String>,
) -> std::sync::Arc<dyn LLMProvider> {
match provider_type {
LLMProviderType::OpenAI => {
info!("Creating OpenAI LLM provider with URL: {}", base_url);
std::sync::Arc::new(OpenAIClient::new("empty".to_string(), Some(base_url)))
std::sync::Arc::new(OpenAIClient::new(
"empty".to_string(),
Some(base_url),
endpoint_path,
))
}
LLMProviderType::Claude => {
info!("Creating Claude LLM provider with URL: {}", base_url);
@ -235,9 +418,13 @@ pub fn create_llm_provider(
}
}
pub fn create_llm_provider_from_url(url: &str, model: Option<String>) -> std::sync::Arc<dyn LLMProvider> {
pub fn create_llm_provider_from_url(
url: &str,
model: Option<String>,
endpoint_path: Option<String>,
) -> std::sync::Arc<dyn LLMProvider> {
let provider_type = LLMProviderType::from(url);
create_llm_provider(provider_type, url.to_string(), model)
create_llm_provider(provider_type, url.to_string(), model, endpoint_path)
}
pub struct DynamicLLMProvider {
@ -257,8 +444,13 @@ impl DynamicLLMProvider {
info!("LLM provider updated dynamically");
}
pub async fn update_from_config(&self, url: &str, model: Option<String>) {
let new_provider = create_llm_provider_from_url(url, model);
pub async fn update_from_config(
&self,
url: &str,
model: Option<String>,
endpoint_path: Option<String>,
) {
let new_provider = create_llm_provider_from_url(url, model, endpoint_path);
self.update_provider(new_provider).await;
}
@ -276,7 +468,10 @@ impl LLMProvider for DynamicLLMProvider {
model: &str,
key: &str,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
self.get_provider().await.generate(prompt, config, model, key).await
self.get_provider()
.await
.generate(prompt, config, model, key)
.await
}
async fn generate_stream(
@ -287,7 +482,10 @@ impl LLMProvider for DynamicLLMProvider {
model: &str,
key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.get_provider().await.generate_stream(prompt, config, tx, model, key).await
self.get_provider()
.await
.generate_stream(prompt, config, tx, model, key)
.await
}
async fn cancel_job(
@ -482,7 +680,7 @@ mod tests {
#[test]
fn test_openai_client_new_default_url() {
let client = OpenAIClient::new("test_key".to_string(), None);
let client = OpenAIClient::new("test_key".to_string(), None, None);
assert_eq!(client.base_url, "https://api.openai.com");
}
@ -491,6 +689,7 @@ mod tests {
let client = OpenAIClient::new(
"test_key".to_string(),
Some("http://localhost:8080".to_string()),
None,
);
assert_eq!(client.base_url, "http://localhost:8080");
}

View file

@ -1,5 +1,4 @@
use crate::core::shared::state::AppState;
use crate::llm::LLMProvider;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
@ -49,7 +48,7 @@ impl SmartLLMRouter {
pub async fn select_optimal_model(
&self,
task_type: &str,
_task_type: &str,
optimization_goal: OptimizationGoal,
max_cost: Option<f64>,
max_latency: Option<u64>,
@ -73,30 +72,32 @@ impl SmartLLMRouter {
// Select based on optimization goal
let selected = match optimization_goal {
OptimizationGoal::Speed => {
candidates.iter().min_by_key(|p| p.avg_latency_ms)
}
OptimizationGoal::Cost => {
candidates.iter().min_by(|a, b| a.avg_cost_per_token.partial_cmp(&b.avg_cost_per_token).unwrap())
}
OptimizationGoal::Quality => {
candidates.iter().max_by(|a, b| a.success_rate.partial_cmp(&b.success_rate).unwrap())
}
OptimizationGoal::Speed => candidates.iter().min_by_key(|p| p.avg_latency_ms),
OptimizationGoal::Cost => candidates.iter().min_by(|a, b| {
a.avg_cost_per_token
.partial_cmp(&b.avg_cost_per_token)
.unwrap()
}),
OptimizationGoal::Quality => candidates
.iter()
.max_by(|a, b| a.success_rate.partial_cmp(&b.success_rate).unwrap()),
OptimizationGoal::Balanced => {
// Weighted score: 40% success rate, 30% speed, 30% cost
candidates.iter().max_by(|a, b| {
let score_a = (a.success_rate * 0.4) +
((1000.0 / a.avg_latency_ms as f64) * 0.3) +
((1.0 / (a.avg_cost_per_token + 0.001)) * 0.3);
let score_b = (b.success_rate * 0.4) +
((1000.0 / b.avg_latency_ms as f64) * 0.3) +
((1.0 / (b.avg_cost_per_token + 0.001)) * 0.3);
let score_a = (a.success_rate * 0.4)
+ ((1000.0 / a.avg_latency_ms as f64) * 0.3)
+ ((1.0 / (a.avg_cost_per_token + 0.001)) * 0.3);
let score_b = (b.success_rate * 0.4)
+ ((1000.0 / b.avg_latency_ms as f64) * 0.3)
+ ((1.0 / (b.avg_cost_per_token + 0.001)) * 0.3);
score_a.partial_cmp(&score_b).unwrap()
})
}
};
Ok(selected.map(|p| p.model_name.clone()).unwrap_or_else(|| "gpt-4o-mini".to_string()))
Ok(selected
.map(|p| p.model_name.clone())
.unwrap_or_else(|| "gpt-4o-mini".to_string()))
}
pub async fn track_performance(
@ -108,21 +109,24 @@ impl SmartLLMRouter {
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let mut performance_data = self.performance_cache.write().await;
let performance = performance_data.entry(model_name.to_string()).or_insert_with(|| {
ModelPerformance {
let performance = performance_data
.entry(model_name.to_string())
.or_insert_with(|| ModelPerformance {
model_name: model_name.to_string(),
avg_latency_ms: latency_ms,
avg_cost_per_token: cost_per_token,
success_rate: if success { 1.0 } else { 0.0 },
total_requests: 0,
last_updated: chrono::Utc::now(),
}
});
// Update running averages
let total = performance.total_requests as f64;
performance.avg_latency_ms = ((performance.avg_latency_ms as f64 * total) + latency_ms as f64) as u64 / (total + 1.0) as u64;
performance.avg_cost_per_token = (performance.avg_cost_per_token * total + cost_per_token) / (total + 1.0);
performance.avg_latency_ms = ((performance.avg_latency_ms as f64 * total)
+ latency_ms as f64) as u64
/ (total + 1.0) as u64;
performance.avg_cost_per_token =
(performance.avg_cost_per_token * total + cost_per_token) / (total + 1.0);
let success_count = (performance.success_rate * total) + if success { 1.0 } else { 0.0 };
performance.success_rate = success_count / (total + 1.0);
@ -149,7 +153,9 @@ pub async fn enhanced_llm_call(
let start_time = Instant::now();
// Select optimal model
let model = router.select_optimal_model("general", optimization_goal, max_cost, max_latency).await?;
let model = router
.select_optimal_model("general", optimization_goal, max_cost, max_latency)
.await?;
// Make LLM call (simplified - would use actual LLM provider)
let response = format!("Response from {} for: {}", model, prompt);
@ -163,7 +169,9 @@ pub async fn enhanced_llm_call(
_ => 0.01,
};
router.track_performance(&model, latency, cost_per_token, true).await?;
router
.track_performance(&model, latency, cost_per_token, true)
.await?;
Ok(response)
}

View file

@ -9,47 +9,53 @@ use tikv_jemallocator::Jemalloc;
static GLOBAL: Jemalloc = Jemalloc;
// Module declarations
#[cfg(feature = "analytics")]
pub mod analytics;
#[cfg(feature = "attendant")]
pub mod attendant;
#[cfg(feature = "automation")]
pub mod auto_task;
#[cfg(feature = "scripting")]
pub mod basic;
#[cfg(feature = "billing")]
pub mod billing;
pub mod botmodels;
#[cfg(feature = "canvas")]
pub mod canvas;
pub mod channels;
#[cfg(feature = "people")]
pub mod contacts;
pub mod core;
#[cfg(feature = "dashboards")]
pub mod shared;
pub mod embedded_ui;
pub mod maintenance;
pub mod multimodal;
#[cfg(feature = "player")]
pub mod player;
#[cfg(feature = "people")]
pub mod people;
#[cfg(feature = "billing")]
pub mod products;
pub mod search;
pub mod security;
#[cfg(feature = "tickets")]
pub mod tickets;
#[cfg(feature = "attendant")]
pub mod attendant;
#[cfg(feature = "analytics")]
pub mod analytics;
#[cfg(feature = "designer")]
pub mod designer;
#[cfg(feature = "docs")]
pub mod docs;
pub mod embedded_ui;
#[cfg(feature = "learn")]
pub mod learn;
#[cfg(feature = "compliance")]
pub mod legal;
pub mod maintenance;
#[cfg(feature = "monitoring")]
pub mod monitoring;
pub mod multimodal;
#[cfg(feature = "paper")]
pub mod paper;
#[cfg(feature = "people")]
pub mod people;
#[cfg(feature = "player")]
pub mod player;
#[cfg(feature = "billing")]
pub mod products;
#[cfg(feature = "project")]
pub mod project;
#[cfg(feature = "research")]
pub mod research;
pub mod search;
pub mod security;
pub mod settings;
#[cfg(feature = "dashboards")]
pub mod shared;
#[cfg(feature = "sheet")]
pub mod sheet;
#[cfg(feature = "slides")]
@ -58,18 +64,12 @@ pub mod slides;
pub mod social;
#[cfg(feature = "sources")]
pub mod sources;
#[cfg(feature = "tickets")]
pub mod tickets;
#[cfg(feature = "video")]
pub mod video;
#[cfg(feature = "monitoring")]
pub mod monitoring;
#[cfg(feature = "project")]
pub mod project;
#[cfg(feature = "workspaces")]
pub mod workspaces;
pub mod botmodels;
#[cfg(feature = "compliance")]
pub mod legal;
pub mod settings;
#[cfg(feature = "attendant")]
pub mod attendance;
@ -174,9 +174,7 @@ async fn ensure_vendor_files_in_minio(drive: &aws_sdk_s3::Client) {
"../botui/ui/suite/js/vendor/htmx.min.js",
];
let htmx_content = htmx_paths
.iter()
.find_map(|path| std::fs::read(path).ok());
let htmx_content = htmx_paths.iter().find_map(|path| std::fs::read(path).ok());
let Some(content) = htmx_content else {
warn!("Could not find htmx.min.js in botui, skipping MinIO upload");
@ -201,18 +199,16 @@ async fn ensure_vendor_files_in_minio(drive: &aws_sdk_s3::Client) {
}
use crate::security::{
create_cors_layer, create_rate_limit_layer, create_security_headers_layer,
request_id_middleware, security_headers_middleware, set_cors_allowed_origins,
set_global_panic_hook, AuthConfig, HttpRateLimitConfig, PanicHandlerConfig,
SecurityHeadersConfig, AuthProviderBuilder, ApiKeyAuthProvider, JwtConfig, JwtKey,
JwtManager, RbacManager, RbacConfig, AuthMiddlewareState,
build_default_route_permissions,
build_default_route_permissions, create_cors_layer, create_rate_limit_layer,
create_security_headers_layer, request_id_middleware, security_headers_middleware,
set_cors_allowed_origins, set_global_panic_hook, ApiKeyAuthProvider, AuthConfig,
AuthMiddlewareState, AuthProviderBuilder, HttpRateLimitConfig, JwtConfig, JwtKey, JwtManager,
PanicHandlerConfig, RbacConfig, RbacManager, SecurityHeadersConfig,
};
use botlib::SystemLimits;
use crate::core::shared::memory_monitor::{
start_memory_monitor, log_process_memory, MemoryStats,
register_thread, record_thread_activity
log_process_memory, record_thread_activity, register_thread, start_memory_monitor, MemoryStats,
};
#[cfg(feature = "automation")]
@ -222,22 +218,21 @@ use crate::core::bot;
use crate::core::package_manager;
use crate::core::session;
#[cfg(feature = "automation")]
use automation::AutomationService;
use bootstrap::BootstrapManager;
use crate::core::bot::channels::{VoiceAdapter, WebChannelAdapter};
use crate::core::bot::websocket_handler;
use crate::core::bot::BotOrchestrator;
use crate::core::bot_database::BotDatabaseManager;
use crate::core::config::AppConfig;
#[cfg(feature = "automation")]
use automation::AutomationService;
use bootstrap::BootstrapManager;
use package_manager::InstallMode;
use session::{create_session, get_session_history, get_sessions, start_session};
use crate::shared::state::AppState;
use crate::shared::utils::create_conn;
#[cfg(feature = "drive")]
use crate::shared::utils::create_s3_operator;
use package_manager::InstallMode;
use session::{create_session, get_session_history, get_sessions, start_session};
async fn health_check(State(state): State<Arc<AppState>>) -> (StatusCode, Json<serde_json::Value>) {
let db_ok = state.conn.get().is_ok();
@ -341,7 +336,8 @@ async fn run_axum_server(
let cors = create_cors_layer();
let auth_config = Arc::new(AuthConfig::from_env()
let auth_config = Arc::new(
AuthConfig::from_env()
.add_anonymous_path("/health")
.add_anonymous_path("/healthz")
.add_anonymous_path("/api/health")
@ -356,10 +352,10 @@ async fn run_axum_server(
.add_public_path("/static")
.add_public_path("/favicon.ico")
.add_public_path("/suite")
.add_public_path("/themes"));
.add_public_path("/themes"),
);
let jwt_secret = std::env::var("JWT_SECRET")
.unwrap_or_else(|_| {
let jwt_secret = std::env::var("JWT_SECRET").unwrap_or_else(|_| {
warn!("JWT_SECRET not set, using default development secret - DO NOT USE IN PRODUCTION");
"dev-secret-key-change-in-production-minimum-32-chars".to_string()
});
@ -382,8 +378,10 @@ async fn run_axum_server(
let default_permissions = build_default_route_permissions();
rbac_manager.register_routes(default_permissions).await;
info!("RBAC Manager initialized with {} default route permissions",
rbac_manager.config().cache_ttl_seconds);
info!(
"RBAC Manager initialized with {} default route permissions",
rbac_manager.config().cache_ttl_seconds
);
let auth_provider_registry = {
let mut builder = AuthProviderBuilder::new()
@ -401,25 +399,32 @@ async fn run_axum_server(
info!("Zitadel environment variables detected - external IdP authentication available");
}
Arc::new(builder.build().await)
};
info!("Auth provider registry initialized with {} providers",
auth_provider_registry.provider_count().await);
info!(
"Auth provider registry initialized with {} providers",
auth_provider_registry.provider_count().await
);
let auth_middleware_state = AuthMiddlewareState::new(
Arc::clone(&auth_config),
Arc::clone(&auth_provider_registry),
);
use crate::core::product::{get_product_config_json, PRODUCT_CONFIG};
use crate::core::urls::ApiUrls;
use crate::core::product::{PRODUCT_CONFIG, get_product_config_json};
{
let config = PRODUCT_CONFIG.read().expect("Failed to read product config");
info!("Product: {} | Theme: {} | Apps: {:?}",
config.name, config.theme, config.get_enabled_apps());
let config = PRODUCT_CONFIG
.read()
.expect("Failed to read product config");
info!(
"Product: {} | Theme: {} | Apps: {:?}",
config.name,
config.theme,
config.get_enabled_apps()
);
}
async fn get_product_config() -> Json<serde_json::Value> {
@ -467,8 +472,9 @@ async fn run_axum_server(
#[cfg(all(feature = "calendar", feature = "scripting"))]
{
let calendar_engine =
Arc::new(crate::basic::keywords::book::CalendarEngine::new(app_state.conn.clone()));
let calendar_engine = Arc::new(crate::basic::keywords::book::CalendarEngine::new(
app_state.conn.clone(),
));
api_router = api_router.merge(crate::calendar::caldav::create_caldav_router(
calendar_engine,
@ -491,22 +497,22 @@ async fn run_axum_server(
api_router = api_router.merge(crate::analytics::configure_analytics_routes());
}
api_router = api_router.merge(crate::core::i18n::configure_i18n_routes());
#[cfg(feature = "docs")]
{
api_router = api_router.merge(crate::docs::configure_docs_routes());
}
#[cfg(feature = "paper")]
{
api_router = api_router.merge(crate::paper::configure_paper_routes());
}
#[cfg(feature = "sheet")]
{
api_router = api_router.merge(crate::sheet::configure_sheet_routes());
}
#[cfg(feature = "slides")]
{
api_router = api_router.merge(crate::slides::configure_slides_routes());
}
#[cfg(feature = "docs")]
{
api_router = api_router.merge(crate::docs::configure_docs_routes());
}
#[cfg(feature = "paper")]
{
api_router = api_router.merge(crate::paper::configure_paper_routes());
}
#[cfg(feature = "sheet")]
{
api_router = api_router.merge(crate::sheet::configure_sheet_routes());
}
#[cfg(feature = "slides")]
{
api_router = api_router.merge(crate::slides::configure_slides_routes());
}
#[cfg(feature = "video")]
{
api_router = api_router.merge(crate::video::configure_video_routes());
@ -590,14 +596,14 @@ api_router = api_router.merge(crate::slides::configure_slides_routes());
{
api_router = api_router.merge(crate::learn::ui::configure_learn_ui_routes());
}
#[cfg(feature = "mail")]
{
api_router = api_router.merge(crate::email::ui::configure_email_ui_routes());
}
#[cfg(feature = "meet")]
{
api_router = api_router.merge(crate::meet::ui::configure_meet_ui_routes());
}
#[cfg(feature = "mail")]
{
api_router = api_router.merge(crate::email::ui::configure_email_ui_routes());
}
#[cfg(feature = "meet")]
{
api_router = api_router.merge(crate::meet::ui::configure_meet_ui_routes());
}
#[cfg(feature = "people")]
{
api_router = api_router.merge(crate::contacts::crm_ui::configure_crm_routes());
@ -654,7 +660,8 @@ api_router = api_router.merge(crate::meet::ui::configure_meet_ui_routes());
// Create rate limiter integrating with botlib's RateLimiter
let http_rate_config = HttpRateLimitConfig::api();
let system_limits = SystemLimits::default();
let (rate_limit_extension, _rate_limiter) = create_rate_limit_layer(http_rate_config, system_limits);
let (rate_limit_extension, _rate_limiter) =
create_rate_limit_layer(http_rate_config, system_limits);
// Create security headers layer
let security_headers_config = SecurityHeadersConfig::default();
@ -673,18 +680,32 @@ api_router = api_router.merge(crate::meet::ui::configure_meet_ui_routes());
info!("Security middleware enabled: rate limiting, security headers, panic handler, request ID tracking, authentication");
// Path to UI files (botui) - use external folder or fallback to embedded
let ui_path = std::env::var("BOTUI_PATH").unwrap_or_else(|_| "./botui/ui/suite".to_string());
let ui_path = std::env::var("BOTUI_PATH").unwrap_or_else(|_| {
if std::path::Path::new("./botui/ui/suite").exists() {
"./botui/ui/suite".to_string()
} else if std::path::Path::new("../botui/ui/suite").exists() {
"../botui/ui/suite".to_string()
} else {
"./botui/ui/suite".to_string()
}
});
let ui_path_exists = std::path::Path::new(&ui_path).exists();
let use_embedded_ui = !ui_path_exists && embedded_ui::has_embedded_ui();
if ui_path_exists {
info!("Serving UI from external folder: {}", ui_path);
} else if use_embedded_ui {
info!("External UI folder not found at '{}', using embedded UI", ui_path);
info!(
"External UI folder not found at '{}', using embedded UI",
ui_path
);
let file_count = embedded_ui::list_embedded_files().len();
info!("Embedded UI contains {} files", file_count);
} else {
warn!("No UI available: folder '{}' not found and no embedded UI", ui_path);
warn!(
"No UI available: folder '{}' not found and no embedded UI",
ui_path
);
}
// Update app_state with auth components
@ -707,8 +728,7 @@ api_router = api_router.merge(crate::meet::ui::configure_meet_ui_routes());
.nest_service("/themes", ServeDir::new(format!("{}/../themes", ui_path)))
.fallback_service(ServeDir::new(&ui_path))
} else if use_embedded_ui {
base_router
.merge(embedded_ui::embedded_ui_router())
base_router.merge(embedded_ui::embedded_ui_router())
} else {
base_router
};
@ -716,7 +736,8 @@ api_router = api_router.merge(crate::meet::ui::configure_meet_ui_routes());
// Clone rbac_manager for use in middleware
let rbac_manager_for_middleware = Arc::clone(&rbac_manager);
let app = app_with_ui
let app =
app_with_ui
// Security middleware stack (order matters - last added is outermost/runs first)
.layer(middleware::from_fn(security_headers_middleware))
.layer(security_headers_extension)
@ -726,19 +747,21 @@ api_router = api_router.merge(crate::meet::ui::configure_meet_ui_routes());
// RBAC middleware - checks permissions AFTER authentication
// NOTE: In Axum, layers run in reverse order (last added = first to run)
// So RBAC is added BEFORE auth, meaning auth runs first, then RBAC
.layer(middleware::from_fn(move |req: axum::http::Request<axum::body::Body>, next: axum::middleware::Next| {
.layer(middleware::from_fn(
move |req: axum::http::Request<axum::body::Body>, next: axum::middleware::Next| {
let rbac = Arc::clone(&rbac_manager_for_middleware);
async move {
crate::security::rbac_middleware_fn(req, next, rbac).await
}
}))
async move { crate::security::rbac_middleware_fn(req, next, rbac).await }
},
))
// Authentication middleware - MUST run before RBAC (so added after)
.layer(middleware::from_fn(move |req: axum::http::Request<axum::body::Body>, next: axum::middleware::Next| {
.layer(middleware::from_fn(
move |req: axum::http::Request<axum::body::Body>, next: axum::middleware::Next| {
let state = auth_middleware_state.clone();
async move {
crate::security::auth_middleware_with_providers(req, next, state).await
}
}))
},
))
// Panic handler catches panics and returns safe 500 responses
.layer(middleware::from_fn(move |req, next| {
let config = panic_config.clone();
@ -794,7 +817,10 @@ api_router = api_router.merge(crate::meet::ui::configure_meet_ui_routes());
let listener = match tokio::net::TcpListener::bind(addr).await {
Ok(l) => l,
Err(e) => {
error!("Failed to bind to {}: {} - is another instance running?", addr, e);
error!(
"Failed to bind to {}: {} - is another instance running?",
addr, e
);
return Err(e);
}
};
@ -813,8 +839,13 @@ async fn main() -> std::io::Result<()> {
let args: Vec<String> = std::env::args().collect();
let no_ui = args.contains(&"--noui".to_string());
#[cfg(feature = "console")]
let no_console = args.contains(&"--noconsole".to_string());
#[cfg(not(feature = "console"))]
let no_console = true;
let _ = rustls::crypto::ring::default_provider().install_default();
dotenvy::dotenv().ok();
@ -840,8 +871,7 @@ async fn main() -> std::io::Result<()> {
trace!("Bootstrap not complete - skipping early SecretsManager init");
}
let noise_filters =
"vaultrs=off,rustify=off,rustify_derive=off,\
let noise_filters = "vaultrs=off,rustify=off,rustify_derive=off,\
aws_sigv4=off,aws_smithy_checksums=off,aws_runtime=off,aws_smithy_http_client=off,\
aws_smithy_runtime=off,aws_smithy_runtime_api=off,aws_sdk_s3=off,aws_config=off,\
aws_credential_types=off,aws_http=off,aws_sig_auth=off,aws_types=off,\
@ -865,14 +895,14 @@ async fn main() -> std::io::Result<()> {
let rust_log = match std::env::var("RUST_LOG") {
Ok(existing) if !existing.is_empty() => format!("{},{}", existing, noise_filters),
_ => noise_filters.to_string(),
_ => format!("info,{}", noise_filters),
};
std::env::set_var("RUST_LOG", &rust_log);
#[cfg(feature = "llm")]
use crate::llm::local::ensure_llama_servers_running;
use crate::core::config::ConfigManager;
use crate::core::config::ConfigManager;
#[cfg(feature = "llm")]
use crate::llm::local::ensure_llama_servers_running;
if no_console || no_ui {
botlib::logging::init_compact_logger_with_style("info");
@ -889,9 +919,16 @@ use crate::core::config::ConfigManager;
"./locales"
};
if let Err(e) = crate::core::i18n::init_i18n(locales_path) {
warn!("Failed to initialize i18n from {}: {}. Translations will show keys.", locales_path, e);
warn!(
"Failed to initialize i18n from {}: {}. Translations will show keys.",
locales_path, e
);
} else {
info!("i18n initialized from {} with locales: {:?}", locales_path, crate::core::i18n::available_locales());
info!(
"i18n initialized from {} with locales: {:?}",
locales_path,
crate::core::i18n::available_locales()
);
}
let (progress_tx, _progress_rx) = tokio::sync::mpsc::unbounded_channel::<BootstrapProgress>();
@ -922,7 +959,7 @@ use crate::core::config::ConfigManager;
std::thread::Builder::new()
.name("ui-thread".to_string())
.spawn(move || {
let mut ui =crate::console::XtreeUI::new();
let mut ui = crate::console::XtreeUI::new();
ui.set_progress_channel(progress_rx);
ui.set_state_channel(state_rx);
@ -930,7 +967,9 @@ use crate::core::config::ConfigManager;
eprintln!("UI error: {e}");
}
})
.map_err(|e| std::io::Error::other(format!("Failed to spawn UI thread: {}", e)))?,
.map_err(|e| {
std::io::Error::other(format!("Failed to spawn UI thread: {}", e))
})?,
)
}
#[cfg(not(feature = "console"))]
@ -1169,7 +1208,9 @@ use crate::core::config::ConfigManager;
ensure_vendor_files_in_minio(&drive).await;
let session_manager = Arc::new(tokio::sync::Mutex::new(session::SessionManager::new(
pool.get().map_err(|e| std::io::Error::other(format!("Failed to get database connection: {}", e)))?,
pool.get().map_err(|e| {
std::io::Error::other(format!("Failed to get database connection: {}", e))
})?,
#[cfg(feature = "cache")]
redis_client.clone(),
)));
@ -1180,17 +1221,20 @@ use crate::core::config::ConfigManager;
let config_path = "./config/directory_config.json";
if let Ok(content) = std::fs::read_to_string(config_path) {
if let Ok(json) = serde_json::from_str::<serde_json::Value>(&content) {
let base_url = json.get("base_url")
let base_url = json
.get("base_url")
.and_then(|v| v.as_str())
.unwrap_or("http://localhost:8300");
let client_id = json.get("client_id")
.and_then(|v| v.as_str())
.unwrap_or("");
let client_secret = json.get("client_secret")
let client_id = json.get("client_id").and_then(|v| v.as_str()).unwrap_or("");
let client_secret = json
.get("client_secret")
.and_then(|v| v.as_str())
.unwrap_or("");
info!("Loaded Zitadel config from {}: url={}", config_path, base_url);
info!(
"Loaded Zitadel config from {}: url={}",
config_path, base_url
);
crate::directory::ZitadelConfig {
issuer_url: base_url.to_string(),
@ -1231,7 +1275,8 @@ use crate::core::config::ConfigManager;
};
#[cfg(feature = "directory")]
let auth_service = Arc::new(tokio::sync::Mutex::new(
crate::directory::AuthService::new(zitadel_config.clone()).map_err(|e| std::io::Error::other(format!("Failed to create auth service: {}", e)))?,
crate::directory::AuthService::new(zitadel_config.clone())
.map_err(|e| std::io::Error::other(format!("Failed to create auth service: {}", e)))?,
));
#[cfg(feature = "directory")]
@ -1243,18 +1288,28 @@ use crate::core::config::ConfigManager;
let pat_token = pat_token.trim().to_string();
info!("Using admin PAT token for bootstrap authentication");
crate::directory::ZitadelClient::with_pat_token(zitadel_config, pat_token)
.map_err(|e| std::io::Error::other(format!("Failed to create bootstrap client with PAT: {}", e)))?
.map_err(|e| {
std::io::Error::other(format!(
"Failed to create bootstrap client with PAT: {}",
e
))
})?
}
Err(e) => {
warn!("Failed to read admin PAT token: {}, falling back to OAuth2", e);
crate::directory::ZitadelClient::new(zitadel_config)
.map_err(|e| std::io::Error::other(format!("Failed to create bootstrap client: {}", e)))?
warn!(
"Failed to read admin PAT token: {}, falling back to OAuth2",
e
);
crate::directory::ZitadelClient::new(zitadel_config).map_err(|e| {
std::io::Error::other(format!("Failed to create bootstrap client: {}", e))
})?
}
}
} else {
info!("Admin PAT not found, using OAuth2 client credentials for bootstrap");
crate::directory::ZitadelClient::new(zitadel_config)
.map_err(|e| std::io::Error::other(format!("Failed to create bootstrap client: {}", e)))?
crate::directory::ZitadelClient::new(zitadel_config).map_err(|e| {
std::io::Error::other(format!("Failed to create bootstrap client: {}", e))
})?
};
match crate::directory::bootstrap::check_and_bootstrap_admin(&bootstrap_client).await {
@ -1271,7 +1326,9 @@ use crate::core::config::ConfigManager;
}
let config_manager = ConfigManager::new(pool.clone());
let mut bot_conn = pool.get().map_err(|e| std::io::Error::other(format!("Failed to get database connection: {}", e)))?;
let mut bot_conn = pool
.get()
.map_err(|e| std::io::Error::other(format!("Failed to get database connection: {}", e)))?;
let (default_bot_id, default_bot_name) = crate::bot::get_default_bot(&mut bot_conn);
info!(
"Using default bot: {} (id: {})",
@ -1297,7 +1354,11 @@ use crate::core::config::ConfigManager;
#[cfg(feature = "llm")]
let base_llm_provider = crate::llm::create_llm_provider_from_url(
&llm_url,
if llm_model.is_empty() { None } else { Some(llm_model.clone()) },
if llm_model.is_empty() {
None
} else {
Some(llm_model.clone())
},
);
#[cfg(feature = "llm")]
@ -1321,8 +1382,7 @@ use crate::core::config::ConfigManager;
let embedding_service = Some(Arc::new(crate::llm::cache::LocalEmbeddingService::new(
embedding_url,
embedding_model,
))
as Arc<dyn crate::llm::cache::EmbeddingService>);
)) as Arc<dyn crate::llm::cache::EmbeddingService>);
let cache_config = crate::llm::cache::CacheConfig {
ttl: 3600,
@ -1349,18 +1409,16 @@ use crate::core::config::ConfigManager;
#[cfg(feature = "tasks")]
let task_engine = Arc::new(crate::tasks::TaskEngine::new(pool.clone()));
let metrics_collector =crate::core::shared::analytics::MetricsCollector::new();
let metrics_collector = crate::core::shared::analytics::MetricsCollector::new();
#[cfg(feature = "tasks")]
let task_scheduler = None;
let (attendant_tx, _attendant_rx) = tokio::sync::broadcast::channel::<
crate::core::shared::state::AttendantNotification,
>(1000);
let (attendant_tx, _attendant_rx) =
tokio::sync::broadcast::channel::<crate::core::shared::state::AttendantNotification>(1000);
let (task_progress_tx, _task_progress_rx) = tokio::sync::broadcast::channel::<
crate::core::shared::state::TaskProgressEvent,
>(1000);
let (task_progress_tx, _task_progress_rx) =
tokio::sync::broadcast::channel::<crate::core::shared::state::TaskProgressEvent>(1000);
// Initialize BotDatabaseManager for per-bot database support
let database_url = crate::shared::utils::get_database_url_sync().unwrap_or_default();
@ -1431,7 +1489,9 @@ use crate::core::config::ConfigManager;
billing_alert_broadcast: None,
task_manifests: Arc::new(std::sync::RwLock::new(HashMap::new())),
#[cfg(feature = "project")]
project_service: Arc::new(tokio::sync::RwLock::new(crate::project::ProjectService::new())),
project_service: Arc::new(tokio::sync::RwLock::new(
crate::project::ProjectService::new(),
)),
#[cfg(feature = "compliance")]
legal_service: Arc::new(tokio::sync::RwLock::new(crate::legal::LegalService::new())),
jwt_manager: None,
@ -1440,7 +1500,9 @@ use crate::core::config::ConfigManager;
});
// Resume workflows after server restart
if let Err(e) = crate::basic::keywords::orchestration::resume_workflows_on_startup(app_state.clone()).await {
if let Err(e) =
crate::basic::keywords::orchestration::resume_workflows_on_startup(app_state.clone()).await
{
log::warn!("Failed to resume workflows on startup: {}", e);
}
@ -1453,7 +1515,7 @@ use crate::core::config::ConfigManager;
task_scheduler.start();
#[cfg(any(feature = "research", feature = "llm"))]
if let Err(e) =crate::core::kb::ensure_crawler_service_running(app_state.clone()).await {
if let Err(e) = crate::core::kb::ensure_crawler_service_running(app_state.clone()).await {
log::warn!("Failed to start website crawler service: {}", e);
}
@ -1487,11 +1549,8 @@ use crate::core::config::ConfigManager;
tokio::spawn(async move {
register_thread("drive-monitor", "drive");
trace!("DriveMonitor::new starting...");
let monitor =crate::DriveMonitor::new(
drive_monitor_state,
bucket_name.clone(),
monitor_bot_id,
);
let monitor =
crate::DriveMonitor::new(drive_monitor_state, bucket_name.clone(), monitor_bot_id);
trace!("DriveMonitor::new done, calling start_monitoring...");
info!("Starting DriveMonitor for bucket: {}", bucket_name);
if let Err(e) = monitor.start_monitoring().await {
@ -1507,8 +1566,10 @@ use crate::core::config::ConfigManager;
tokio::spawn(async move {
register_thread("automation-service", "automation");
let automation = AutomationService::new(automation_state);
trace!("[TASK] AutomationService starting, RSS={}",
MemoryStats::format_bytes(MemoryStats::current().rss_bytes));
trace!(
"[TASK] AutomationService starting, RSS={}",
MemoryStats::format_bytes(MemoryStats::current().rss_bytes)
);
loop {
record_thread_activity("automation-service");
if let Err(e) = automation.check_scheduled_tasks().await {

View file

@ -1,13 +1,14 @@
use anyhow::{Context, Result};
use std::fs;
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use std::process::Command;
use tracing::{error, info, warn};
use crate::security::command_guard::SafeCommand;
#[cfg(not(windows))]
const SUDOERS_FILE: &str = "/etc/sudoers.d/gb-protection";
const SUDOERS_CONTENT: &str = r#"# General Bots Security Protection Tools
# This file is managed by botserver install protection
# DO NOT EDIT MANUALLY
@ -53,6 +54,7 @@ const SUDOERS_CONTENT: &str = r#"# General Bots Security Protection Tools
{user} ALL=(ALL) NOPASSWD: /usr/local/sbin/maldet --update-ver
"#;
#[cfg(not(windows))]
const PACKAGES: &[&str] = &[
"lynis",
"rkhunter",
@ -62,19 +64,43 @@ const PACKAGES: &[&str] = &[
"clamav-daemon",
];
#[cfg(windows)]
const WINDOWS_TOOLS: &[(&str, &str)] = &[
("Windows Defender", "MpCmdRun"),
("PowerShell", "powershell"),
("Windows Firewall", "netsh"),
];
pub struct ProtectionInstaller {
user: String,
}
impl ProtectionInstaller {
pub fn new() -> Result<Self> {
let user = std::env::var("SUDO_USER")
.or_else(|_| std::env::var("USER"))
.unwrap_or_else(|_| "root".to_string());
let user = std::env::var("USER").unwrap_or_else(|_| "unknown".to_string());
Ok(Self { user })
}
#[cfg(windows)]
pub fn check_admin() -> bool {
let result = Command::new("powershell")
.args([
"-Command",
"([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)"
])
.output();
match result {
Ok(output) => {
let stdout = String::from_utf8_lossy(&output.stdout);
stdout.trim() == "True"
}
Err(_) => false,
}
}
#[cfg(not(windows))]
pub fn check_root() -> bool {
Command::new("id")
.arg("-u")
@ -84,16 +110,67 @@ impl ProtectionInstaller {
}
pub fn install(&self) -> Result<InstallResult> {
#[cfg(windows)]
{
if !Self::check_admin() {
return Err(anyhow::anyhow!(
"This command requires administrator privileges. Run as Administrator."
));
}
}
#[cfg(not(windows))]
{
if !Self::check_root() {
return Err(anyhow::anyhow!(
"This command requires root privileges. Run with: sudo botserver install protection"
));
}
}
info!("Starting security protection installation for user: {}", self.user);
info!(
"Starting security protection installation for user: {}",
self.user
);
let mut result = InstallResult::default();
#[cfg(windows)]
{
match self.configure_windows_security() {
Ok(()) => {
result
.packages_installed
.push("Windows Defender".to_string());
result
.packages_installed
.push("Windows Firewall".to_string());
info!("Windows security configured successfully");
}
Err(e) => {
warn!("Windows security configuration had issues: {e}");
result
.warnings
.push(format!("Windows security configuration: {e}"));
}
}
match self.update_windows_signatures() {
Ok(()) => {
result.databases_updated = true;
info!("Windows security signatures updated");
}
Err(e) => {
warn!("Windows signature update failed: {e}");
result
.warnings
.push(format!("Windows signature update: {e}"));
}
}
}
#[cfg(not(windows))]
{
match self.install_packages() {
Ok(installed) => {
result.packages_installed = installed;
@ -101,7 +178,9 @@ impl ProtectionInstaller {
}
Err(e) => {
error!("Failed to install packages: {e}");
result.errors.push(format!("Package installation failed: {e}"));
result
.errors
.push(format!("Package installation failed: {e}"));
}
}
@ -125,7 +204,9 @@ impl ProtectionInstaller {
}
Err(e) => {
warn!("LMD installation skipped: {e}");
result.warnings.push(format!("LMD installation skipped: {e}"));
result
.warnings
.push(format!("LMD installation skipped: {e}"));
}
}
@ -139,11 +220,13 @@ impl ProtectionInstaller {
result.warnings.push(format!("Database update failed: {e}"));
}
}
}
result.success = result.errors.is_empty();
Ok(result)
}
#[cfg(not(windows))]
fn install_packages(&self) -> Result<Vec<String>> {
info!("Updating package lists...");
@ -181,13 +264,44 @@ impl ProtectionInstaller {
Ok(installed)
}
#[cfg(windows)]
fn configure_windows_security(&self) -> Result<()> {
info!("Configuring Windows security settings...");
// Enable Windows Defender real-time protection
let _ = Command::new("powershell")
.args([
"-Command",
"Set-MpPreference -DisableRealtimeMonitoring $false; Set-MpPreference -DisableIOAVProtection $false; Set-MpPreference -DisableScriptScanning $false"
])
.output();
// Enable Windows Firewall
let _ = Command::new("netsh")
.args(["advfirewall", "set", "allprofiles", "state", "on"])
.output();
// Enable Windows Defender scanning for mapped drives
let _ = Command::new("powershell")
.args([
"-Command",
"Set-MpPreference -DisableRemovableDriveScanning $false -DisableScanningMappedNetworkDrivesForFullScan $false"
])
.output();
info!("Windows security configuration completed");
Ok(())
}
#[cfg(not(windows))]
fn create_sudoers(&self) -> Result<()> {
use std::os::unix::fs::PermissionsExt;
let content = SUDOERS_CONTENT.replace("{user}", &self.user);
info!("Creating sudoers file at {SUDOERS_FILE}");
fs::write(SUDOERS_FILE, &content)
.context("Failed to write sudoers file")?;
fs::write(SUDOERS_FILE, &content).context("Failed to write sudoers file")?;
let permissions = fs::Permissions::from_mode(0o440);
fs::set_permissions(SUDOERS_FILE, permissions)
@ -199,6 +313,8 @@ impl ProtectionInstaller {
Ok(())
}
#[cfg(not(windows))]
#[cfg(not(windows))]
fn validate_sudoers(&self) -> Result<()> {
let output = std::process::Command::new("visudo")
.args(["-c", "-f", SUDOERS_FILE])
@ -214,6 +330,8 @@ impl ProtectionInstaller {
Ok(())
}
#[cfg(not(windows))]
#[cfg(not(windows))]
fn install_lmd(&self) -> Result<bool> {
let maldet_path = Path::new("/usr/local/sbin/maldet");
if maldet_path.exists() {
@ -250,13 +368,18 @@ impl ProtectionInstaller {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() && path.file_name().is_some_and(|n| n.to_string_lossy().starts_with("maldetect")) {
if path.is_dir()
&& path
.file_name()
.is_some_and(|n| n.to_string_lossy().starts_with("maldetect"))
{
install_dir = Some(path);
break;
}
}
let install_dir = install_dir.ok_or_else(|| anyhow::anyhow!("LMD install directory not found"))?;
let install_dir =
install_dir.ok_or_else(|| anyhow::anyhow!("LMD install directory not found"))?;
let install_script = install_dir.join("install.sh");
if !install_script.exists() {
@ -275,14 +398,14 @@ impl ProtectionInstaller {
Ok(true)
}
#[cfg(not(windows))]
#[cfg(not(windows))]
fn update_databases(&self) -> Result<()> {
info!("Updating security tool databases...");
if Path::new("/usr/bin/rkhunter").exists() {
info!("Updating RKHunter database...");
let result = SafeCommand::new("rkhunter")?
.arg("--update")?
.execute();
let result = SafeCommand::new("rkhunter")?.arg("--update")?.execute();
if let Err(e) = result {
warn!("RKHunter update failed: {e}");
}
@ -290,8 +413,7 @@ impl ProtectionInstaller {
if Path::new("/usr/bin/freshclam").exists() {
info!("Updating ClamAV signatures...");
let result = SafeCommand::new("freshclam")?
.execute();
let result = SafeCommand::new("freshclam")?.execute();
if let Err(e) = result {
warn!("ClamAV update failed: {e}");
}
@ -299,8 +421,7 @@ impl ProtectionInstaller {
if Path::new("/usr/bin/suricata-update").exists() {
info!("Updating Suricata rules...");
let result = SafeCommand::new("suricata-update")?
.execute();
let result = SafeCommand::new("suricata-update")?.execute();
if let Err(e) = result {
warn!("Suricata update failed: {e}");
}
@ -308,9 +429,7 @@ impl ProtectionInstaller {
if Path::new("/usr/local/sbin/maldet").exists() {
info!("Updating LMD signatures...");
let result = SafeCommand::new("maldet")?
.arg("--update-sigs")?
.execute();
let result = SafeCommand::new("maldet")?.arg("--update-sigs")?.execute();
if let Err(e) = result {
warn!("LMD update failed: {e}");
}
@ -319,17 +438,58 @@ impl ProtectionInstaller {
Ok(())
}
#[cfg(windows)]
fn update_windows_signatures(&self) -> Result<()> {
info!("Updating Windows Defender signatures...");
let result = Command::new("powershell")
.args([
"-Command",
"Update-MpSignature; Write-Host 'Windows Defender signatures updated'",
])
.output();
match result {
Ok(output) => {
if output.status.success() {
info!("Windows Defender signatures updated successfully");
} else {
warn!("Windows Defender signature update had issues");
}
}
Err(e) => {
warn!("Failed to update Windows Defender signatures: {e}");
}
}
Ok(())
}
pub fn uninstall(&self) -> Result<UninstallResult> {
#[cfg(windows)]
{
if !Self::check_admin() {
return Err(anyhow::anyhow!(
"This command requires administrator privileges. Run as Administrator."
));
}
}
#[cfg(not(windows))]
{
if !Self::check_root() {
return Err(anyhow::anyhow!(
"This command requires root privileges. Run with: sudo botserver remove protection"
));
}
}
info!("Removing security protection components...");
let mut result = UninstallResult::default();
#[cfg(not(windows))]
{
if Path::new(SUDOERS_FILE).exists() {
match fs::remove_file(SUDOERS_FILE) {
Ok(()) => {
@ -342,15 +502,23 @@ impl ProtectionInstaller {
}
}
result.success = result.errors.is_empty();
result.message = "Protection sudoers removed. Packages were NOT uninstalled - remove manually if needed.".to_string();
}
#[cfg(windows)]
{
result.message = "Windows protection uninstalled. Windows Defender and Firewall settings were not modified - remove manually if needed.".to_string();
}
result.success = result.errors.is_empty();
Ok(result)
}
pub fn verify(&self) -> VerifyResult {
let mut result = VerifyResult::default();
#[cfg(not(windows))]
{
for package in PACKAGES {
let binary = match *package {
"clamav" | "clamav-daemon" => "clamscan",
@ -381,15 +549,48 @@ impl ProtectionInstaller {
if result.sudoers_exists {
if let Ok(content) = fs::read_to_string(SUDOERS_FILE) {
for tool in &mut result.tools {
tool.sudo_configured = content.contains(&tool.name) ||
(tool.name == "clamav" && content.contains("clamav-daemon")) ||
(tool.name == "clamav-daemon" && content.contains("clamav-daemon"));
tool.sudo_configured = content.contains(&tool.name)
|| (tool.name == "clamav" && content.contains("clamav-daemon"))
|| (tool.name == "clamav-daemon" && content.contains("clamav-daemon"));
}
}
}
result.all_installed = result.tools.iter().filter(|t| t.name != "clamav-daemon").all(|t| t.installed);
result.all_configured = result.sudoers_exists && result.tools.iter().all(|t| t.sudo_configured || !t.installed);
result.all_installed = result
.tools
.iter()
.filter(|t| t.name != "clamav-daemon")
.all(|t| t.installed);
result.all_configured = result.sudoers_exists
&& result
.tools
.iter()
.all(|t| t.sudo_configured || !t.installed);
}
#[cfg(windows)]
{
for (tool_name, tool_cmd) in WINDOWS_TOOLS {
let check = Command::new(tool_cmd)
.arg("--version")
.or_else(|_| {
Command::new("powershell")
.args(["-Command", &format!("Get-Command {}", tool_cmd)])
})
.output();
let installed = check.map(|o| o.status.success()).unwrap_or(false);
result.tools.push(ToolVerification {
name: tool_name.to_string(),
installed,
sudo_configured: true, // Windows tools are typically pre-configured
});
}
result.sudoers_exists = false; // No sudoers on Windows
result.all_installed = result.tools.iter().all(|t| t.installed);
result.all_configured = true; // Windows tools are pre-configured
}
result
}
@ -397,11 +598,12 @@ impl ProtectionInstaller {
impl Default for ProtectionInstaller {
fn default() -> Self {
Self::new().unwrap_or(Self { user: "root".to_string() })
Self::new().unwrap_or(Self {
user: "unknown".to_string(),
})
}
}
#[derive(Debug, Default)]
pub struct InstallResult {
pub success: bool,
pub packages_installed: Vec<String>,
@ -411,60 +613,70 @@ pub struct InstallResult {
pub warnings: Vec<String>,
}
impl Default for InstallResult {
fn default() -> Self {
Self {
success: false,
packages_installed: Vec::new(),
sudoers_created: false,
databases_updated: false,
errors: Vec::new(),
warnings: Vec::new(),
}
}
}
impl InstallResult {
pub fn print(&self) {
println!();
println!("\n=== Security Protection Installation Result ===");
println!(
"Status: {}",
if self.success {
println!("✓ Security Protection installed successfully!");
"✓ SUCCESS"
} else {
println!("✗ Security Protection installation completed with errors");
"✗ FAILED"
}
println!();
);
if !self.packages_installed.is_empty() {
println!("Packages installed:");
println!("\nInstalled Packages:");
for pkg in &self.packages_installed {
println!(" {pkg}");
println!(" - {pkg}");
}
println!();
}
if self.sudoers_created {
println!("✓ Sudoers configuration created at {SUDOERS_FILE}");
#[cfg(not(windows))]
{
println!("\nSudoers Configuration:");
println!(
" - File created: {}",
if self.sudoers_created { "" } else { "" }
);
}
if self.databases_updated {
println!("✓ Security databases updated");
}
println!(
"\nDatabases Updated: {}",
if self.databases_updated { "" } else { "" }
);
if !self.warnings.is_empty() {
println!();
println!("Warnings:");
for warn in &self.warnings {
println!("{warn}");
println!("\nWarnings:");
for warning in &self.warnings {
println!(" ! {warning}");
}
}
if !self.errors.is_empty() {
println!();
println!("Errors:");
for err in &self.errors {
println!("{err}");
println!("\nErrors:");
for error in &self.errors {
println!("{error}");
}
}
println!();
println!("The following commands are now available via the UI:");
println!(" - Lynis security audits");
println!(" - RKHunter rootkit scans");
println!(" - Chkrootkit scans");
println!(" - Suricata IDS management");
println!(" - ClamAV antivirus scans");
println!(" - LMD malware detection");
println!("\n");
}
}
#[derive(Debug, Default)]
pub struct UninstallResult {
pub success: bool,
pub sudoers_removed: bool,
@ -472,21 +684,51 @@ pub struct UninstallResult {
pub errors: Vec<String>,
}
impl UninstallResult {
pub fn print(&self) {
println!();
if self.success {
println!("{}", self.message);
} else {
println!("✗ Uninstall completed with errors");
for err in &self.errors {
println!("{err}");
}
impl Default for UninstallResult {
fn default() -> Self {
Self {
success: false,
sudoers_removed: false,
message: String::new(),
errors: Vec::new(),
}
}
}
#[derive(Debug, Default)]
impl UninstallResult {
pub fn print(&self) {
println!("\n=== Security Protection Uninstallation Result ===");
println!(
"Status: {}",
if self.success {
"✓ SUCCESS"
} else {
"✗ FAILED"
}
);
#[cfg(not(windows))]
{
println!(
"Sudoers removed: {}",
if self.sudoers_removed { "" } else { "" }
);
}
println!("\nMessage:");
println!(" {}", self.message);
if !self.errors.is_empty() {
println!("\nErrors:");
for error in &self.errors {
println!("{error}");
}
}
println!("\n");
}
}
pub struct VerifyResult {
pub all_installed: bool,
pub all_configured: bool,
@ -494,7 +736,17 @@ pub struct VerifyResult {
pub tools: Vec<ToolVerification>,
}
#[derive(Debug, Default)]
impl Default for VerifyResult {
fn default() -> Self {
Self {
all_installed: false,
all_configured: false,
sudoers_exists: false,
tools: Vec::new(),
}
}
}
pub struct ToolVerification {
pub name: String,
pub installed: bool,
@ -503,33 +755,44 @@ pub struct ToolVerification {
impl VerifyResult {
pub fn print(&self) {
println!();
println!("Security Protection Status:");
println!();
println!("\n=== Security Protection Verification ===");
println!(
"All tools installed: {}",
if self.all_installed { "" } else { "" }
);
println!(
"All tools configured: {}",
if self.all_configured { "" } else { "" }
);
println!("Tools:");
for tool in &self.tools {
let installed_mark = if tool.installed { "" } else { "" };
let sudo_mark = if tool.sudo_configured { "" } else { "" };
println!(" {} {} (installed: {}, sudo: {})",
if tool.installed && tool.sudo_configured { "" } else { "" },
tool.name,
installed_mark,
sudo_mark
#[cfg(not(windows))]
{
println!(
"Sudoers file exists: {}",
if self.sudoers_exists { "" } else { "" }
);
}
println!();
println!("Sudoers file: {}", if self.sudoers_exists { "✓ exists" } else { "✗ missing" });
println!();
if self.all_installed && self.all_configured {
println!("✓ All protection tools are properly configured");
} else if !self.all_installed {
println!("⚠ Some tools are not installed. Run: sudo botserver install protection");
println!("\nTool Status:");
for tool in &self.tools {
println!(
" {} {}: {} {}",
if tool.installed { "" } else { "" },
tool.name,
if tool.installed {
"installed"
} else {
println!("⚠ Sudoers not configured. Run: sudo botserver install protection");
"not installed"
},
if tool.sudo_configured {
"(configured)"
} else {
"(not configured)"
}
);
}
println!("\n");
}
}
@ -542,7 +805,7 @@ mod tests {
let result = InstallResult::default();
assert!(!result.success);
assert!(result.packages_installed.is_empty());
assert!(!result.sudoers_created);
assert!(result.errors.is_empty());
}
#[test]
@ -553,30 +816,13 @@ mod tests {
assert!(result.tools.is_empty());
}
#[test]
fn test_sudoers_content_has_placeholder() {
assert!(SUDOERS_CONTENT.contains("{user}"));
}
#[test]
fn test_sudoers_content_no_wildcards() {
assert!(!SUDOERS_CONTENT.contains(" * "));
assert!(!SUDOERS_CONTENT.lines().any(|l| l.trim().ends_with('*')));
}
#[test]
fn test_packages_list() {
assert!(PACKAGES.contains(&"lynis"));
assert!(PACKAGES.contains(&"rkhunter"));
assert!(PACKAGES.contains(&"chkrootkit"));
assert!(PACKAGES.contains(&"suricata"));
assert!(PACKAGES.contains(&"clamav"));
}
#[test]
fn test_tool_verification_default() {
let tool = ToolVerification::default();
assert!(tool.name.is_empty());
let tool = ToolVerification {
name: "test".to_string(),
installed: false,
sudo_configured: false,
};
assert!(!tool.installed);
assert!(!tool.sudo_configured);
}
@ -585,8 +831,7 @@ mod tests {
fn test_uninstall_result_default() {
let result = UninstallResult::default();
assert!(!result.success);
assert!(!result.sudoers_removed);
assert!(result.message.is_empty());
assert!(result.errors.is_empty());
}
#[test]
@ -594,4 +839,32 @@ mod tests {
let installer = ProtectionInstaller::default();
assert!(!installer.user.is_empty());
}
#[cfg(not(windows))]
#[test]
fn test_sudoers_content_has_placeholder() {
assert!(SUDOERS_CONTENT.contains("{user}"));
}
#[cfg(not(windows))]
#[test]
fn test_sudoers_content_no_wildcards() {
// Ensure no dangerous wildcards in sudoers
assert!(!SUDOERS_CONTENT.contains(" ALL=(ALL) ALL"));
}
#[cfg(not(windows))]
#[test]
fn test_packages_list() {
assert!(PACKAGES.len() > 0);
assert!(PACKAGES.contains(&"lynis"));
assert!(PACKAGES.contains(&"rkhunter"));
}
#[cfg(windows)]
#[test]
fn test_windows_tools_list() {
assert!(WINDOWS_TOOLS.len() > 0);
assert!(WINDOWS_TOOLS.iter().any(|t| t.0 == "Windows Defender"));
}
}