Update workspace configuration and submodules

This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2026-01-30 23:25:02 -03:00
parent 0dfabc46b3
commit 43b168a4b5
11 changed files with 658 additions and 13 deletions

198
.kiro/settings/lsp.json Normal file
View file

@ -0,0 +1,198 @@
{
"languages": {
"typescript": {
"name": "typescript-language-server",
"command": "typescript-language-server",
"args": [
"--stdio"
],
"file_extensions": [
"ts",
"js",
"tsx",
"jsx"
],
"project_patterns": [
"package.json",
"tsconfig.json"
],
"exclude_patterns": [
"**/node_modules/**",
"**/dist/**"
],
"multi_workspace": false,
"initialization_options": {
"preferences": {
"disableSuggestions": false
}
},
"request_timeout_secs": 60
},
"python": {
"name": "pyright",
"command": "pyright-langserver",
"args": [
"--stdio"
],
"file_extensions": [
"py"
],
"project_patterns": [
"pyproject.toml",
"setup.py",
"requirements.txt",
"pyrightconfig.json"
],
"exclude_patterns": [
"**/__pycache__/**",
"**/venv/**",
"**/.venv/**",
"**/.pytest_cache/**"
],
"multi_workspace": false,
"initialization_options": {},
"request_timeout_secs": 60
},
"rust": {
"name": "rust-analyzer",
"command": "rust-analyzer",
"args": [],
"file_extensions": [
"rs"
],
"project_patterns": [
"Cargo.toml"
],
"exclude_patterns": [
"**/target/**"
],
"multi_workspace": false,
"initialization_options": {
"cargo": {
"buildScripts": {
"enable": true
}
},
"diagnostics": {
"enable": true,
"enableExperimental": true
},
"workspace": {
"symbol": {
"search": {
"scope": "workspace"
}
}
}
},
"request_timeout_secs": 60
},
"java": {
"name": "jdtls",
"command": "jdtls",
"args": [],
"file_extensions": [
"java"
],
"project_patterns": [
"pom.xml",
"build.gradle",
"build.gradle.kts",
".project"
],
"exclude_patterns": [
"**/target/**",
"**/build/**",
"**/.gradle/**"
],
"multi_workspace": false,
"initialization_options": {
"settings": {
"java": {
"compile": {
"nullAnalysis": {
"mode": "automatic"
}
},
"configuration": {
"annotationProcessing": {
"enabled": true
}
}
}
}
},
"request_timeout_secs": 60
},
"ruby": {
"name": "solargraph",
"command": "solargraph",
"args": [
"stdio"
],
"file_extensions": [
"rb"
],
"project_patterns": [
"Gemfile",
"Rakefile"
],
"exclude_patterns": [
"**/vendor/**",
"**/tmp/**"
],
"multi_workspace": false,
"initialization_options": {},
"request_timeout_secs": 60
},
"go": {
"name": "gopls",
"command": "gopls",
"args": [],
"file_extensions": [
"go"
],
"project_patterns": [
"go.mod",
"go.sum"
],
"exclude_patterns": [
"**/vendor/**"
],
"multi_workspace": false,
"initialization_options": {
"usePlaceholders": true,
"completeUnimported": true
},
"request_timeout_secs": 60
},
"cpp": {
"name": "clangd",
"command": "clangd",
"args": [
"--background-index"
],
"file_extensions": [
"cpp",
"cc",
"cxx",
"c",
"h",
"hpp",
"hxx"
],
"project_patterns": [
"CMakeLists.txt",
"compile_commands.json",
"Makefile"
],
"exclude_patterns": [
"**/build/**",
"**/cmake-build-**/**"
],
"multi_workspace": false,
"initialization_options": {},
"request_timeout_secs": 60
}
}
}

View file

@ -1,5 +1,7 @@
{
"diagnostics": {
"enabled": false
}
"languages": {
"Rust": {
"enable_language_server": false,
},
},
}

406
README.md
View file

@ -31,14 +31,263 @@ For comprehensive documentation, see **[docs.pragmatismo.com.br](https://docs.pr
- **Binary:** `target/debug/botserver`
- **Run from:** `botserver/` directory
- **Env file:** `botserver/.env`
- **Stack:** `botserver/botserver-stack/`
- **Stack:** `botserver-stack/`
- **UI Files:** `botui/ui/suite/`
---
## 🏗️ Component Dependency Graph
```
┌─────────────────────────────────────────────────────────────────┐
│ Client Layer │
├─────────────────────────────────────────────────────────────────┤
│ botui (Web UI) │ botapp (Desktop) │ botplugin (Ext) │
│ HTMX + Axum │ Tauri 2 Wrapper │ Browser Extension │
└─────────┬───────────────────┬──────────────────┬─────────────────┘
│ │ │
└───────────────────┼──────────────────┘
┌─────────▼─────────┐
│ botlib │
│ (Shared Types) │
└─────────┬─────────┘
┌───────────────────┼───────────────────┐
│ │ │
┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐
│ botserver │ │ bottest │ │ botdevice │
│ API Core │ │ Tests │ │ IoT/Device │
└───────────┘ └───────────┘ └───────────┘
```
### Dependency Rules
| Crate | Depends On | Why |
|-------|-----------|-----|
| **botserver** | botlib | Shared types, error handling, models |
| **botui** | botlib | Common data structures, API client |
| **botapp** | botlib | Shared types, desktop-specific utilities |
| **bottest** | botserver, botlib | Integration testing with real components |
| **botdevice** | botlib | Device types, communication protocols |
| **botplugin** | - | Standalone browser extension (JS) |
**Key Principle:** `botlib` contains ONLY shared types and utilities. No business logic. All business logic lives in botserver or specialized crates.
## 📦 Module Responsibility Matrix
### botserver/src/ Module Structure
| Module | Responsibility | Key Types | Dependencies |
|--------|---------------|-----------|--------------|
| **core/bot/** | WebSocket handling, bot orchestration | BotOrchestrator, UserMessage | basic, shared |
| **core/session/** | Session management, conversation history | SessionManager, UserSession | shared, database |
| **basic/** | Rhai BASIC scripting engine | ScriptService, Engine | rhai, keywords |
| **basic/keywords/** | BASIC keyword implementations (TALK, HEAR, etc.) | Keyword functions | basic, state |
| **llm/** | Multi-vendor LLM API integration | LLMClient, ModelConfig | reqwest, shared |
| **drive/** | S3 file storage and monitoring | DriveMonitor, compile_tool | s3, basic |
| **security/** | Security guards (command, SQL, error) | SafeCommand, ErrorSanitizer | state |
| **shared/** | Database models, schema definitions | Bot, Session, Message | diesel |
| **tasks/** | AutoTask execution system | TaskRunner, TaskScheduler | core/basic |
| **auto_task/** | LLM-powered app generation | AppGenerator, template engine | llm, tasks |
| **learn/** | Knowledge base management | KBManager, vector storage | database, drive |
| **attendance/** | LLM-assisted customer service | AttendantManager, queue | core/bot |
### Data Flow Patterns
```
1. User Request Flow:
Client → WebSocket → botserver/src/core/bot/mod.rs
BotOrchestrator::stream_response()
┌───────────┴───────────┐
│ │
LLM API Call Script Execution
(llm/mod.rs) (basic/mod.rs)
│ │
└───────────┬───────────┘
Response → WebSocket → Client
2. File Sync Flow:
S3 Drive → drive_monitor/src/drive_monitor/mod.rs
Download .bas files
compile_file() → Generate .ast
Store in ./work/{bot_name}.gbai/
3. Script Execution Flow:
.bas file → ScriptService::compile()
preprocess_basic_script()
engine.compile() → AST
ScriptService::run() → Execute
TALK commands → WebSocket messages
```
### Common Architectural Patterns
| Pattern | Where Used | Purpose |
|---------|-----------|---------|
| **State via Arc<AppState>** | All handlers | Shared async state (DB, cache, config) |
| **Extension(state) extractor** | Axum handlers | Inject Arc<AppState> into route handlers |
| **tokio::spawn_blocking** | CPU-intensive tasks | Offload blocking work from async runtime |
| **WebSocket with split()** | Real-time comms | Separate sender/receiver for WS streams |
| **ErrorSanitizer for responses** | All HTTP errors | Prevent leaking sensitive info in errors |
| **SafeCommand for execution** | Command running | Whitelist-based command validation |
| **Rhai for scripting** | BASIC interpreter | Embeddable scripting language |
| **Diesel ORM** | Database access | Type-safe SQL queries |
| **Redis for cache** | Session data | Fast key-value storage |
| **S3 for storage** | File system | Scalable object storage |
---
## Quick Start
### Start Both Servers
### MAIN INSTRUCTION: When someone asks to "FOLLOW readme.md" or mentions a project name, read ALL relevant README.md files from involved projects (botserver, botui, etc.) and run `./restart.sh` until no more errors. IF THERE ARE ERRORS, FIX THEM AND RUN `./restart.sh` AGAIN UNTIL 0 ERRORS! THEN LOOP: CONTINUOUSLY MONITOR SERVER/UI OUTPUT FOR ERRORS DURING LOADING - WAIT FOR FULL LOAD AND WATCH FOR ANY ERRORS/WARNINGS IN REAL-TIME OUTPUT! JavaScript errors from browser automatically appear in server logs with CLIENT: prefix.
### 🚀 ULTIMATE ERROR HUNTING MODE
**STEP 1: RESTART & START BACKGROUND SERVERS**
```bash
./restart.sh
pkill -f botserver; pkill -f botui
cd botserver && cargo run -- --noconsole > ../botserver.log 2>&1 &
cd botui && BOTSERVER_URL="http://localhost:8088" cargo run > ../botui.log 2>&1 &
```
**STEP 2: CONTINUOUS ERROR HUNTING LOOP**
```bash
while true; do
echo "=== ERROR SCAN $(date) ==="
echo "🔥 SERVER ERRORS:"
grep " E " botserver.log | tail -5
echo "⚠️ SERVER WARNINGS:"
grep " W " botserver.log | tail -3
echo "💥 CLIENT ERRORS:"
grep "CLIENT:" botserver.log | tail -3
echo "🚨 UI ERRORS:"
grep -i "error\|fail" botui.log | tail-3
echo "================================"
# IF ERRORS FOUND: INTERRUPT AND FIX IMMEDIATELY
if grep -q " E " botserver.log || grep -q "CLIENT:" botserver.log || grep -qi "error\|fail" botui.log; then
echo "⚠️ ERRORS DETECTED - STOPPING FOR FIXES"
break
fi
sleep 10
done
```
**STEP 3: SINGLE SCAN (for quick check)**
```bash
echo "=== ERROR SCAN $(date) ===" && echo "🔥 SERVER ERRORS:" && grep " E " botserver.log | tail -5 && echo "⚠️ SERVER WARNINGS:" && grep " W " botserver.log | tail -3 && echo "💥 CLIENT ERRORS:" && grep "CLIENT:" botserver.log | tail-3 && echo "🚨 UI ERRORS:" && grep -i "error\|fail" botui.log | tail-3 && echo "================================"
```
**STEP 4: ERROR FIXING PROTOCOL**
When errors are detected:
1. **STOP SERVERS IMMEDIATELY**: `pkill -f botserver; pkill -f botui`
2. **ANALYZE ERROR PATTERNS**: Group by module/file
3. **FIX ALL ERRORS OFFLINE**: Use diagnostics, don't compile during fixes
4. **RESTART AND VERIFY**: `./restart.sh` then repeat scan
5. **LOOP UNTIL ZERO ERRORS**: Continue until clean scan
### 🎯 Comprehensive Startup Strategy
**Phase 1: Initial Validation**
```bash
# 1. Run restart script first
./restart.sh
# 2. Verify compilation success
cd botserver && cargo check
cd ../botui && cargo check
```
**Phase 2: Controlled Startup with Monitoring**
```bash
# 3. Kill any existing processes
pkill -f botserver; pkill -f botui
# 4. Start with logging for monitoring
cd botserver && nohup cargo run -- --noconsole > ../botserver.log 2>&1 &
cd botui && nohup bash -c 'BOTSERVER_URL="http://localhost:8088" cargo run' > ../botui.log 2>&1 &
```
**Phase 3: Real-time Monitoring Loop**
```bash
# 5. Monitor startup logs continuously
tail -f botserver.log &
tail -f botui.log &
# 6. Check for specific startup milestones:
# - botserver: "SecretsManager initialized successfully"
# - botserver: "PostgreSQL ready"
# - botserver: "API server listening on"
# - botui: "UI server listening on http://0.0.0.0:3000"
```
**Phase 4: Error Detection & Response**
- Watch for ERROR/WARN messages in logs
- Monitor PostgreSQL initialization (common bottleneck)
- Check Vault unsealing process
- Verify both servers reach "listening" state
- Test connectivity: `curl http://localhost:3000` and `curl http://localhost:8088`
**Phase 5: Error Pattern Scanning**
```bash
# Scan for errors in logs (E prefix indicates errors)
grep " E " botserver.log | tail -10
grep " W " botserver.log | tail -10 # Warnings
grep -i "error\|fail" botserver.log | tail -10
grep -i "error\|fail" botui.log | tail-10
# Common error patterns to watch for:
# - "E compiler:" - BASIC script compilation errors
# - "E table_definition:" - Database table creation issues
# - "Failed to" - General operation failures
# - "Could not parse" - Parsing/syntax errors
# - "CLIENT:" - JavaScript errors from browser (appear in server logs)
```
**Phase 6: CONTINUOUS MONITORING**
```bash
# Start servers in background and monitor logs continuously
cd botserver && cargo run -- --noconsole > ../botserver.log 2>&1 &
cd botui && BOTSERVER_URL="http://localhost:8088" cargo run > ../botui.log 2>&1 &
# CONTINUOUS LOOP: Watch for ALL errors in real-time
while true; do
echo "=== SCANNING FOR ERRORS $(date) ==="
echo "--- SERVER ERRORS ---"
grep " E " botserver.log | tail -5
echo "--- SERVER WARNINGS ---"
grep " W " botserver.log | tail -3
echo "--- CLIENT ERRORS ---"
grep "CLIENT:" botserver.log | tail -3
echo "--- UI ERRORS ---"
grep -i "error\|fail" botui.log | tail -3
echo "=========================="
sleep 10
done
```
**⚠️ IMPORTANT: Server References**
All servers (PostgreSQL, Vault, Redis, etc.) are managed through `botserver-stack/` directory, NOT global system installations. The system uses local binaries in `botserver-stack/bin/` and configurations in `botserver-stack/conf/`. Do not install or reference global PostgreSQL, Redis, or other services.
### Start Both Servers (Automated)
```bash
# Use restart script (RECOMMENDED)
./restart.sh
```
### Start Both Servers (Manual)
```bash
# Terminal 1: botserver
cd botserver && cargo run -- --noconsole
@ -61,6 +310,156 @@ cargo test -p bottest
---
## 🧭 LLM Navigation Guide
### Quick Context Jump
- [Primary Purpose](#overview) - Unified workspace for AI automation platform
- [Crate Structure](#-workspace-structure) - 9 independent crates with shared libraries
- [Dependencies](#-component-dependency-graph) - How crates depend on each other
- [Quick Start](#quick-start) - Get running in 2 commands
- [Error Patterns](#common-error-patterns) - Fix compilation errors efficiently
- [Security Rules](#-security-directives---mandatory) - MUST-FOLLOW security patterns
- [Code Patterns](#-mandatory-code-patterns) - Required coding conventions
- [Testing](#testing-strategy) - How to test changes
- [Debugging](#debugging-guide) - Troubleshoot common issues
### Reading This Workspace
**For LLMs analyzing this codebase:**
1. Start with [Component Dependency Graph](#-component-dependency-graph) to understand relationships
2. Review [Module Responsibility Matrix](#-module-responsibility-matrix) for what each module does
3. Study [Data Flow Patterns](#-data-flow-patterns) to understand execution flow
4. Reference [Common Architectural Patterns](#-common-architectural-patterns) before making changes
5. Check [Security Rules](#-security-directives---mandatory) - violations are blocking issues
6. Follow [Code Patterns](#-mandatory-code-patterns) - consistency is mandatory
**For Humans working on this codebase:**
1. Follow [Error Fixing Workflow](#-error-fixing-workflow) for compilation errors
2. Observe [File Size Limits](#-file-size-limits---mandatory) - max 450 lines per file
3. Run [Weekly Maintenance Tasks](#-weekly-maintenance-tasks) to keep codebase healthy
4. Read project-specific READMEs in [Project-Specific Guidelines](#-project-specific-guidelines)
## 🧪 Testing Strategy
### Unit Tests
- **Location**: Each crate has `tests/` directory or inline `#[cfg(test)]` modules
- **Naming**: Test functions use `test_` prefix or describe what they test
- **Running**: `cargo test -p <crate_name>` or `cargo test` for all
### Integration Tests
- **Location**: `bottest/` crate contains integration tests
- **Scope**: Tests full workflows across multiple crates
- **Running**: `cargo test -p bottest`
- **Database**: Uses test database, automatically migrates on first run
### Test Utilities Available
- **TestAppStateBuilder** (`bottest/src/harness.rs`) - Build test state with mocked components
- **TestBot** (`bottest/src/bot/mod.rs`) - Mock bot for testing
- **Test Database**: Auto-created, migrations run automatically
### Coverage Goals
- **Critical paths**: 80%+ coverage required
- **Error handling**: ALL error paths must have tests
- **Security**: All security guards must have tests
## 🐛 Debugging Guide
### Log Locations
| Component | Log File | What's Logged |
|-----------|----------|---------------|
| **botserver** | `botserver/logs/botserver.log` | API requests, errors, script execution |
| **botui** | `botui/logs/botui.log` | UI rendering, WebSocket connections |
| **drive_monitor** | In botserver logs with `[drive_monitor]` prefix | File sync, compilation |
| **script execution** | In botserver logs with `[ScriptService]` prefix | BASIC compilation, runtime errors |
### Common Error Messages
| Error | Meaning | Fix |
|-------|---------|-----|
| `Session not found` | Invalid session_id in request | Check auth flow, verify session exists in DB |
| `Bot not found` | Invalid bot_name or bot_id | Verify bot exists in `bots` table |
| `Script compilation error` | BASIC syntax error in .bas file | Check .bas file syntax, look for typos |
| `Failed to send TALK message` | WebSocket disconnected | Check client connection, verify web_adapter running |
| `Drive sync failed` | S3 connection or permission issue | Verify S3 credentials, check bucket exists |
| `unwrap() called on None value` | Panic in production code | MUST FIX - replace with proper error handling |
### Useful Debugging Commands
```bash
# Check if botserver is running
ps aux | grep botserver
# View botserver logs in real-time
tail -f botserver/logs/botserver.log
# Check work directory structure
ls -la ./work/*.gbai/*/
# Test database connection
cd botserver && cargo run --bin botserver -- --test-db
# Run specific test with output
cargo test -p botserver test_name -- --nocapture
# Check for memory leaks during compilation
CARGO_BUILD_JOBS=1 cargo check -p botserver 2>&1 | grep -i error
```
### Troubleshooting Workflows
**Problem: Script not executing**
1. Check if .bas file exists in `./work/{bot_name}.gbai/{bot_name}.gbdialog/`
2. Verify file has correct syntax (compile with ScriptService)
3. Check logs for compilation errors
4. Verify drive_monitor is running and syncing files
**Problem: WebSocket messages not received**
1. Check browser console for WebSocket errors
2. Verify session_id is valid in database
3. Check web_adapter is registered for session
4. Look for TALK execution in botserver logs
**Problem: LLM not responding**
1. Check LLM API credentials in config
2. Verify API key has available quota
3. Check network connectivity to LLM provider
4. Review request/response logs for API errors
### Performance Profiling
```bash
# Profile compilation time
cargo build --release --timings
# Profile runtime performance
cargo flamegraph --bin botserver
# Check binary size
ls -lh target/release/botserver
# Memory usage
valgrind --leak-check=full target/release/botserver
```
## 📖 Glossary
| Term | Definition | Usage |
|------|-----------|-------|
| **Bot** | AI agent with configuration, scripts, and knowledge bases | Primary entity in system |
| **Session** | Single conversation instance between user and bot | Stored in `sessions` table |
| **Dialog** | Collection of BASIC scripts (.bas files) for bot logic | Stored in `{bot_name}.gbdialog/` |
| **Tool** | Reusable function callable by LLM | Defined in .bas files, compiled to .ast |
| **Knowledge Base (KB)** | Vector database of documents for semantic search | Managed in `learn/` module |
| **Scheduler** | Time-triggered task execution | Cron-like scheduling in BASIC scripts |
| **Drive** | S3-compatible storage for files | Abstracted in `drive/` module |
| **Rhai** | Embedded scripting language for BASIC dialect | Rhai engine in `basic/` module |
| **WebSocket Adapter** | Component that sends messages to connected clients | `web_adapter` in state |
| **AutoTask** | LLM-generated task automation system | In `auto_task/` and `tasks/` modules |
| **Orchestrator** | Coordinates LLM, tools, KBs, and user input | `BotOrchestrator` in `core/bot/` |
---
## 🔥 Error Fixing Workflow
### Mode 1: OFFLINE Batch Fix (PREFERRED)
@ -258,6 +657,7 @@ validate_table_name(&safe_table)?;
❌ NEVER add comments - code must be self-documenting
❌ NEVER modify Cargo.toml lints section!
❌ NEVER use CDN links - all assets must be local
❌ NEVER use cargo clean - causes 30min rebuilds, use ./reset.sh for database issues
```
---
@ -445,7 +845,7 @@ Subprojects (botapp, botserver, etc.) are **not** git submodules - they are inde
## Main Directive
**LOOP AND COMPACT UNTIL 0 WARNINGS - MAXIMUM YOLO**
**LOOP AND COMPACT UNTIL 0 WARNINGS - MAXIMUM PRECISION**
- 0 warnings
- 0 errors

@ -1 +1 @@
Subproject commit c8d39c0e6232ca17118a6f106182c38b7159ccac
Subproject commit 6df9f7dce536291b35661be975852077698b3980

@ -1 +1 @@
Subproject commit 94fede7cc4eed8dd94dbe691b7b37fe90ddf0976
Subproject commit 748fceff5dc054a4032d2880d73d3d9836c601fe

View file

@ -0,0 +1,20 @@
{
"base_url": "http://localhost:8300",
"default_org": {
"id": "357908715812159502",
"name": "default",
"domain": "default.localhost"
},
"default_user": {
"id": "admin",
"username": "admin",
"email": "admin@localhost",
"password": "",
"first_name": "Admin",
"last_name": "User"
},
"admin_token": "wlzEVkJmVyI4NLL7DRWQ7QKOuJyQeZb-oeuRVV2eN_rOYIlJXMDwLKYW9SfzYTLxlgf-xog",
"project_id": "",
"client_id": "357908716382650382",
"client_secret": "SenhyqTLxOHziSqFTcVi4CcfDeljMiQ7IdIxdlGNVJ6qPyj6kQZ0z6T7nWDaKjkl"
}

1
reset.sh Executable file
View file

@ -0,0 +1 @@
rm -rf botserver-stack/ botserver.log botserver/work/

View file

@ -2,11 +2,11 @@ clear
pkill rustc -9
pkill botserver -9
pkill botui -9
rm botserver.log
cd botui
cargo build &
cd ../botserver
cargo build
cargo run -- --noconsole &
cd ../botui
cargo run &
cd ..
cargo run -p botserver -- --noconsole &
cargo run -p botui &

View file

@ -22,7 +22,7 @@ ADD SUGGESTION "python" AS "How do I use Python dictionaries?"
ADD SUGGESTION "javascript" AS "Explain JavaScript async/await"
ADD SUGGESTION "web" AS "What is the DOM in web development?"
REM Initial greeting and exit - no interaction loop
REM Initial greeting
TALK intro
TALK "I have access to Python documentation, MDN web docs, and Stack Overflow."
TALK "Ask me any programming question!"

4
stop.sh Normal file
View file

@ -0,0 +1,4 @@
pkill botui
pkill botserver
clear

20
test_cron.rs Normal file
View file

@ -0,0 +1,20 @@
use cron::Schedule;
use std::str::FromStr;
fn main() {
let test_schedules = vec![
"59 * * * *",
"0 * * * *",
"0 11 * * *",
"0 0 */2 * *",
"0 30 23 * * *",
];
for schedule in test_schedules {
println!("Testing: '{}'", schedule);
match Schedule::from_str(schedule) {
Ok(_) => println!(" ✅ Valid"),
Err(e) => println!(" ❌ Error: {}", e),
}
}
}