Add toml dependency for Askama custom filters config

Cargo.lock

@@ -1487,6 +1487,7 @@ dependencies = [
  "tokio",
  "tokio-rustls 0.24.1",
  "tokio-stream",
+ "toml 0.8.2",
  "tonic 0.14.2",
  "tower 0.4.13",
  "tower-cookies",

Cargo.toml

@@ -129,6 +129,7 @@ regex = "1.11"
 reqwest = { version = "0.12", features = ["json", "stream", "multipart", "rustls-tls", "rustls-tls-native-roots"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+toml = "0.8"
 sha2 = "0.10.9"
 tokio = { version = "1.41", features = ["full"] }
 tokio-stream = "0.1"

askama.toml

@@ -7,3 +7,8 @@ syntax = [{ name = "html", ext = ["html"] }]
 
 # Escape HTML by default for security
 escape = "html"
+
+# Custom filters module path
+[custom]
+# Register custom filters from the web::filters module
+filters = "crate::web::filters"

docs/src/README.md

@@ -150,7 +150,7 @@ General Bots is a **monolithic Rust application** (single crate) with the follow
 - **Language**: Rust 2021 edition
 - **Web**: Axum + Tower + Tokio
 - **Database**: Diesel ORM + PostgreSQL
-- **Cache**: Valkey (Redis-compatible)
+- **Cache**: Redis-compatible cache component
 - **Storage**: AWS SDK S3 (drive component)
 - **Vector DB**: Qdrant (optional)
 - **Scripting**: Rhai engine

docs/src/SUMMARY.md

@@ -22,7 +22,6 @@
   - [.gbtheme UI Theming](./chapter-02/gbtheme.md)
   - [.gbdrive File Storage](./chapter-02/gbdrive.md)
   - [Bot Templates](./chapter-02/templates.md)
-  - [Template: CRM Contacts](./chapter-02/template-crm-contacts.md)
   - [Template Samples & Conversations](./chapter-02/template-samples.md)
 
 # Part III - Knowledge Base

docs/src/appendix-env-vars/README.md

@@ -1,84 +1,81 @@
 # Appendix C: Environment Variables
 
-General Bots uses a minimal set of environment variables. All service configuration is managed through the Directory service (Zitadel), and application settings are stored in `config.csv` files within each bot's `.gbot` folder.
+General Bots uses a minimal set of environment variables. All configuration is managed through `config.csv` files within each bot's `.gbot` folder, with secrets stored securely in Vault.
 
 ## Required Environment Variables
 
-Only one set of environment variables is used by General Bots:
+Only Vault-related environment variables are used by General Bots:
 
-### DIRECTORY_* Variables
+### VAULT_* Variables
 
-**Purpose**: Directory service (Zitadel) configuration for identity and service management.
+**Purpose**: HashiCorp Vault integration for secure secrets management.
 
 | Variable | Description | Example |
 |----------|-------------|---------|
-| `DIRECTORY_URL` | Zitadel instance URL | `http://localhost:8080` |
+| `VAULT_ADDR` | Vault server URL | `http://localhost:8200` |
-| `DIRECTORY_CLIENT_ID` | OAuth client ID | Auto-generated during bootstrap |
+| `VAULT_TOKEN` | Authentication token | Auto-generated during bootstrap |
-| `DIRECTORY_CLIENT_SECRET` | OAuth client secret | Auto-generated during bootstrap |
+| `VAULT_NAMESPACE` | Vault namespace (optional) | `admin` |
 
 **Example**:
 ```bash
-DIRECTORY_URL=http://localhost:8080
+VAULT_ADDR=http://localhost:8200
-DIRECTORY_CLIENT_ID=your-client-id
+VAULT_TOKEN=hvs.your-vault-token
-DIRECTORY_CLIENT_SECRET=your-client-secret
 ```
 
 ## Auto-Managed Services
 
-The following services are automatically configured through the Directory service:
+The following services are automatically configured through Vault:
 
 | Service | Management |
 |---------|------------|
-| PostgreSQL | Connection managed via Directory |
+| PostgreSQL | Connection credentials in Vault |
-| S3-Compatible Storage | Credentials managed via Directory |
+| S3-Compatible Storage | Access keys in Vault |
-| Cache (Valkey) | Connection managed via Directory |
+| Cache | Connection managed via Vault |
-| Email (Stalwart) | Accounts managed via Directory |
+| Email (Stalwart) | Credentials in Vault |
+| LLM API Keys | Stored in Vault |
 
-You do **not** need to set environment variables for these services. The Directory service handles credential distribution and rotation automatically.
+You do **not** need to set environment variables for these services. Vault handles credential distribution and rotation automatically.
 
 ## What NOT to Use Environment Variables For
 
-**Do NOT use environment variables for**:
+**All application configuration belongs in `config.csv`**:
 
 | Configuration | Where to Configure |
 |--------------|-------------------|
-| Database connection | Managed by Directory service |
+| Database connection | Managed by Vault |
-| Storage credentials | Managed by Directory service |
+| Storage credentials | Managed by Vault |
-| LLM API keys | `config.csv`: `llm-api-key` |
+| LLM API keys | Managed by Vault |
-| LLM provider | `config.csv`: `llm-provider` |
+| LLM provider | `config.csv`: `llm-url` |
-| Email settings | Managed by Directory service |
+| Email settings | `config.csv`: `email-*` |
-| Channel tokens | `config.csv`: `whatsapp-api-key`, etc. |
+| Channel tokens | `config.csv`: `whatsapp-*`, etc. |
 | Bot settings | `config.csv`: all bot-specific settings |
-| Weather API | `config.csv`: `weather-api-key` |
+| Feature flags | `config.csv`: various keys |
-| Feature flags | `config.csv`: `enable-*` keys |
 
 ## Configuration Philosophy
 
 General Bots follows these principles:
 
-1. **Directory-First**: Infrastructure credentials are managed by the Directory service
+1. **Vault-First**: All secrets are managed by Vault
-2. **Minimal Environment**: Only identity provider settings use environment variables
+2. **Minimal Environment**: Only Vault address and token use environment variables
-3. **Database-Stored**: All application configuration is stored in the database via `config.csv` sync
+3. **config.csv for Settings**: All application configuration is in `config.csv`
 4. **Per-Bot Configuration**: Each bot has its own `config.csv` in its `.gbot` folder
-5. **No Hardcoded Defaults**: Configuration must be explicitly provided
+5. **No Hardcoded Secrets**: Never store secrets in code or config files
 
 ## Setting Environment Variables
 
 ### Linux/macOS
 
 ```bash
-export DIRECTORY_URL=http://localhost:8080
+export VAULT_ADDR=http://localhost:8200
-export DIRECTORY_CLIENT_ID=your-client-id
+export VAULT_TOKEN=hvs.your-vault-token
-export DIRECTORY_CLIENT_SECRET=your-client-secret
 ```
 
 ### Systemd Service
 
 ```ini
 [Service]
-Environment="DIRECTORY_URL=http://localhost:8080"
+Environment="VAULT_ADDR=http://localhost:8200"
-Environment="DIRECTORY_CLIENT_ID=your-client-id"
+Environment="VAULT_TOKEN=hvs.your-vault-token"
-Environment="DIRECTORY_CLIENT_SECRET=your-client-secret"
 ```
 
 ### LXC Container
@@ -86,54 +83,54 @@ Environment="DIRECTORY_CLIENT_SECRET=your-client-secret"
 When using LXC deployment, environment variables are set in the container configuration:
 
 ```bash
-lxc config set container-name environment.DIRECTORY_URL="http://localhost:8080"
+lxc config set container-name environment.VAULT_ADDR="http://localhost:8200"
+lxc config set container-name environment.VAULT_TOKEN="hvs.your-vault-token"
 ```
 
 ## Security Notes
 
-1. **Never commit credentials**: Use `.env` files (gitignored) or secrets management
+1. **Never commit tokens**: Use `.env` files (gitignored) or secrets management
-2. **Rotate regularly**: The Directory service can rotate credentials automatically
+2. **Rotate regularly**: Vault tokens should be rotated periodically
 3. **Limit access**: Only the botserver process needs these variables
-4. **Use TLS**: Always use HTTPS for the Directory URL in production
+4. **Use TLS**: Always use HTTPS for Vault in production
-
-## Troubleshooting
-
-### Directory Connection Failed
-
-```
-Error: Failed to connect to Directory service
-```
-
-Verify:
-- `DIRECTORY_URL` is set correctly
-- Zitadel is running and accessible
-- Network allows connection to Directory host
-- Client credentials are valid
-
-### Service Not Available
-
-If a managed service (database, storage, cache) is unavailable:
-
-1. Check the Directory service is running
-2. Verify service registration in Zitadel
-3. Check service container/process status
-4. Review logs for connection errors
 
 ## Bootstrap Process
 
 During bootstrap, General Bots:
 
-1. Connects to the Directory service using `DIRECTORY_*` variables
+1. Connects to Vault using `VAULT_*` variables
-2. Registers itself as an application
+2. Retrieves credentials for all managed services
-3. Retrieves credentials for managed services
+3. Configures database, storage, cache, and other services
-4. Starts services with provided credentials
+4. Stores service endpoints securely
-5. Stores service endpoints in the database
 
 This eliminates the need for manual credential management.
 
+## Troubleshooting
+
+### Vault Connection Failed
+
+```
+Error: Failed to connect to Vault
+```
+
+Verify:
+- `VAULT_ADDR` is set correctly
+- Vault server is running and accessible
+- `VAULT_TOKEN` is valid and not expired
+- Network allows connection to Vault host
+
+### Service Not Available
+
+If a managed service (database, storage, cache) is unavailable:
+
+1. Check Vault is running and unsealed
+2. Verify secrets exist in Vault
+3. Check service container/process status
+4. Review logs for connection errors
+
 ## See Also
 
 - [config.csv Format](../chapter-08-config/config-csv.md) - Bot configuration
-- [External Services](../appendix-external-services/README.md) - Service configuration
+- [Secrets Management](../chapter-08-config/secrets-management.md) - Vault integration details
 - [Drive Integration](../chapter-08-config/drive.md) - Storage setup
-- [Authentication](../chapter-12-auth/README.md) - Directory service integration
+- [Authentication](../chapter-12-auth/README.md) - Security features

docs/src/appendix-external-services/README.md

@@ -15,12 +15,12 @@ Infrastructure services (database, storage, cache) are automatically managed by
 | LLM Providers | OpenAI, Groq, Anthropic, Azure OpenAI | `config.csv` |
 | Weather | OpenWeatherMap | `config.csv` |
 | Messaging Channels | WhatsApp, Teams, Instagram, Telegram | `config.csv` |
-| Storage | S3-Compatible (MinIO, etc.) | Directory service (automatic) |
+| Storage | S3-Compatible (MinIO, etc.) | Vault (automatic) |
-| Directory | Zitadel | `DIRECTORY_*` environment variables |
+| Directory | Zitadel | `VAULT_*` environment variables |
-| Email | Stalwart / IMAP/SMTP | Directory service (automatic) |
+| Email | Stalwart / IMAP/SMTP | Vault (automatic) |
 | Calendar | CalDAV servers | `config.csv` |
-| Database | PostgreSQL | Directory service (automatic) |
+| Database | PostgreSQL | Vault (automatic) |
-| Cache | Valkey | Directory service (automatic) |
+| Cache | Redis-compatible | Vault (automatic) |
 
 ## Quick Reference
 
@@ -70,7 +70,7 @@ The following services are automatically configured by the Directory service (Zi
 |---------|----------------|
 | PostgreSQL | Connection credentials, database creation |
 | S3-Compatible Storage | Access keys, bucket policies |
-| Valkey Cache | Connection credentials |
+| Cache | Connection credentials |
 | Stalwart Email | User accounts, SMTP/IMAP access |
 
 You do **not** need to configure these services manually. The Directory service handles credential provisioning and rotation.

docs/src/appendix-external-services/catalog.md

@@ -235,10 +235,10 @@ These services are deployed locally as part of the General Bots stack. All are m
 
 | Service | Default Port | Purpose | Management |
 |---------|-------------|---------|------------|
-| PostgreSQL | 5432 | Primary database | Directory service |
+| PostgreSQL | 5432 | Primary database | Vault |
-| Qdrant | 6333 | Vector storage for KB | Directory service |
+| Qdrant | 6333 | Vector storage for KB | Vault |
-| Valkey | 6379 | Caching | Directory service |
+| Cache | 6379 | Caching | Vault |
-| Stalwart | 25, 993 | Email server (optional) | Directory service |
+| Stalwart | 25, 993 | Email server (optional) | Vault |
 | BotModels | 5000 | AI model inference | config.csv |
 
 ---

docs/src/chapter-01/installation.md

@@ -33,7 +33,7 @@ BotServer handles all dependencies automatically:
 The bootstrap process automatically downloads everything to `botserver-stack/`:
 - PostgreSQL database  
 - Drive (S3-compatible object storage)  
-- Valkey cache
+- Cache component
 - LLM server and models
 - All required dependencies
 
@@ -335,7 +335,7 @@ Default ports used:
 | Drive Console | 9001 | N/A |
 | LLM Server | 8081 | config.csv: `llm-server-port` |
 | Embedding Server | 8082 | config.csv: `embedding-url` |
-| Valkey Cache | 6379 | Internal |
+| Cache | 6379 | Internal |
 
 ### Memory Issues
 

docs/src/chapter-01/quick-start.md

@@ -24,7 +24,7 @@ You'll see:
 📦 Installing Drive...
    ✓ Object storage ready
    ✓ Buckets created
-📦 Installing Valkey...
+📦 Installing Cache...
    ✓ Cache server running
 🤖 Creating bots from templates...
    ✓ default.gbai → Default bot
@@ -53,7 +53,7 @@ The **automatic bootstrap** process:
 1. ✅ Detected your OS (Linux/macOS/Windows)
 2. ✅ Downloaded PostgreSQL database to botserver-stack/
 3. ✅ Downloaded drive (S3-compatible storage) to botserver-stack/
-4. ✅ Downloaded Valkey cache to botserver-stack/
+4. ✅ Downloaded cache component to botserver-stack/
 5. ✅ Generated secure credentials → `.env` (from blank environment)
 6. ✅ Created database schema
 7. ✅ Deployed default bots to object storage
@@ -281,7 +281,7 @@ rm .env
 ```bash
 ./botserver status tables    # PostgreSQL
 ./botserver status drive     # Drive storage
-./botserver status cache     # Valkey cache
+./botserver status cache     # Cache component
 ```
 
 ---

docs/src/chapter-01/sessions.md

@@ -20,7 +20,7 @@ Think of it like a phone call that can pause and resume anytime.
 2. Browser gets a session token (UUID)
 3. Token stored in localStorage
 4. Session created in PostgreSQL
-5. Cached in Valkey for speed
+5. Cached for speed
 
 ### API Access
 ```bash
@@ -71,7 +71,7 @@ session_state:
   context: current knowledge/tools
 ```
 
-### In Valkey Cache (Fast Access)
+### In Cache (Fast Access)
 ```
 session:uuid:messages     # Recent messages
 session:uuid:variables    # Current state

docs/src/chapter-02/template-crm-contacts.md

@@ -9,7 +9,6 @@ The CRM Contacts template provides a complete contact management solution with n
 | Template ID | `crm/contacts.gbai` |
 | Category | CRM |
 | Complexity | Intermediate |
-| Dependencies | Database, Email (optional) |
 
 ## Features
 
@@ -18,19 +17,10 @@ The CRM Contacts template provides a complete contact management solution with n
 - Update contact information
 - Tag and categorize contacts
 - Export contacts to CSV
-- Integration with email campaigns
 
 ## Installation
 
-```basic
+Copy the template folder to your work directory or use the Sources app to create from template.
-DEPLOY TEMPLATE "crm/contacts.gbai"
-```
-
-Or copy the template folder to your work directory:
-
-```bash
-cp -r templates/crm/contacts.gbai work/
-```
 
 ## Configuration
 
@@ -43,399 +33,271 @@ Add these settings to your `config.csv`:
 | `allow-duplicates` | `false` | Allow duplicate entries |
 | `default-tags` | `lead` | Default tags for new contacts |
 
-## Sample Conversation
+---
 
-<div class="chat-container">
+## Sample Conversations
 
-<div class="chat-message user">
+### Adding a New Contact
-<div class="chat-bubble">
+
-<div class="chat-sender">You</div>
+<div class="wa-chat">
-<div class="chat-text">Hi, I need to add a new contact</div>
+  <div class="wa-message bot">
-<div class="chat-time">10:32 AM</div>
+    <div class="wa-bubble">
-</div>
+      <p>I'd be happy to help you add a new contact! 📇</p>
+      <p>What's the contact's full name?</p>
+      <div class="wa-time">10:32</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Maria Santos</p>
+      <div class="wa-time">10:32</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>Got it! Maria Santos.</p>
+      <p>What's Maria's email address?</p>
+      <div class="wa-time">10:32</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>maria.santos@company.com</p>
+      <div class="wa-time">10:33</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>Perfect! And what's the phone number?</p>
+      <p>(You can skip this by typing "skip")</p>
+      <div class="wa-time">10:33</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>+55 11 98765-4321</p>
+      <div class="wa-time">10:33</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>Which company does Maria work for?</p>
+      <div class="wa-time">10:33</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Tech Solutions Ltd</p>
+      <div class="wa-time">10:34</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>Great! Here's the contact I'm about to save:</p>
+      <p>📋 <strong>Contact Details</strong></p>
+      <p>━━━━━━━━━━━━━━━━━</p>
+      <p>👤 <strong>Name:</strong> Maria Santos</p>
+      <p>📧 <strong>Email:</strong> maria.santos@company.com</p>
+      <p>📱 <strong>Phone:</strong> +55 11 98765-4321</p>
+      <p>🏢 <strong>Company:</strong> Tech Solutions Ltd</p>
+      <p>🏷️ <strong>Tags:</strong> lead</p>
+      <p>Should I save this contact?</p>
+      <div class="wa-time">10:34</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Yes, save it</p>
+      <div class="wa-time">10:34</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>✅ Contact saved successfully!</p>
+      <p>Maria Santos has been added to your contacts with ID #1247.</p>
+      <p>What would you like to do next?</p>
+      <p>• Add another contact</p>
+      <p>• Search contacts</p>
+      <p>• View recent contacts</p>
+      <div class="wa-time">10:34</div>
+    </div>
+  </div>
 </div>
 
-<div class="chat-message bot">
+---
-<div class="chat-bubble">
-<div class="chat-sender">CRM Bot</div>
-<div class="chat-text">I'd be happy to help you add a new contact! 📇
 
-What's the contact's full name?</div>
+### Searching Contacts
-<div class="chat-time">10:32 AM</div>
+
-</div>
+<div class="wa-chat">
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Search for contacts at Tech Solutions</p>
+      <div class="wa-time">10:35</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>🔍 Found 3 contacts at "Tech Solutions":</p>
+      <p></p>
+      <p>1. <strong>Maria Santos</strong> - maria.santos@company.com</p>
+      <p>   📱 +55 11 98765-4321 | 🏷️ lead</p>
+      <p></p>
+      <p>2. <strong>João Silva</strong> - joao.silva@techsolutions.com</p>
+      <p>   📱 +55 11 91234-5678 | 🏷️ customer</p>
+      <p></p>
+      <p>3. <strong>Ana Costa</strong> - ana.costa@techsolutions.com</p>
+      <p>   📱 +55 11 99876-5432 | 🏷️ prospect</p>
+      <p></p>
+      <p>Type a number to see full details, or tell me what else you need.</p>
+      <div class="wa-time">10:35</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>2</p>
+      <div class="wa-time">10:35</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>📋 <strong>Contact Details</strong></p>
+      <p>━━━━━━━━━━━━━━━━━</p>
+      <p>👤 <strong>Name:</strong> João Silva</p>
+      <p>📧 <strong>Email:</strong> joao.silva@techsolutions.com</p>
+      <p>📱 <strong>Phone:</strong> +55 11 91234-5678</p>
+      <p>🏢 <strong>Company:</strong> Tech Solutions Ltd</p>
+      <p>🏷️ <strong>Tags:</strong> customer</p>
+      <p>📅 <strong>Added:</strong> Jan 15, 2025</p>
+      <p></p>
+      <p>What would you like to do?</p>
+      <p>• Edit this contact</p>
+      <p>• Delete this contact</p>
+      <p>• Send email</p>
+      <div class="wa-time">10:35</div>
+    </div>
+  </div>
 </div>
 
-<div class="chat-message user">
+---
-<div class="chat-bubble">
+
-<div class="chat-sender">You</div>
+### Updating a Contact
-<div class="chat-text">Maria Santos</div>
+
-<div class="chat-time">10:32 AM</div>
+<div class="wa-chat">
-</div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Update Maria's phone number</p>
+      <div class="wa-time">10:40</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>I found Maria Santos in your contacts.</p>
+      <p>Current phone: +55 11 98765-4321</p>
+      <p>What's the new phone number?</p>
+      <div class="wa-time">10:40</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>+55 11 99999-8888</p>
+      <div class="wa-time">10:40</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>✅ Updated!</p>
+      <p>Maria Santos's phone is now +55 11 99999-8888</p>
+      <div class="wa-time">10:40</div>
+    </div>
+  </div>
 </div>
 
-<div class="chat-message bot">
+---
-<div class="chat-bubble">
-<div class="chat-sender">CRM Bot</div>
-<div class="chat-text">Got it! Maria Santos.
 
-What's Maria's email address?</div>
+### Exporting Contacts
-<div class="chat-time">10:32 AM</div>
+
-</div>
+<div class="wa-chat">
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Export all contacts to CSV</p>
+      <div class="wa-time">11:00</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>📊 Exporting contacts...</p>
+      <p></p>
+      <p>✅ Export complete!</p>
+      <p>📁 <strong>contacts-export-2025-05-15.csv</strong></p>
+      <p>• 47 contacts exported</p>
+      <p>• File saved to your Drive</p>
+      <p></p>
+      <p>Would you like me to email this file to you?</p>
+      <div class="wa-time">11:00</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Yes please</p>
+      <div class="wa-time">11:00</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>📧 Sent! Check your inbox for the export file.</p>
+      <div class="wa-time">11:00</div>
+    </div>
+  </div>
 </div>
 
-<div class="chat-message user">
+---
-<div class="chat-bubble">
-<div class="chat-sender">You</div>
-<div class="chat-text">maria.santos@company.com</div>
-<div class="chat-time">10:33 AM</div>
-</div>
-</div>
 
-<div class="chat-message bot">
+## What It Can Do
-<div class="chat-bubble">
-<div class="chat-sender">CRM Bot</div>
-<div class="chat-text">Perfect! And what's the phone number? (You can skip this by typing "skip")</div>
-<div class="chat-time">10:33 AM</div>
-</div>
-</div>
 
-<div class="chat-message user">
+| Capability | Description |
-<div class="chat-bubble">
+|------------|-------------|
-<div class="chat-sender">You</div>
+| Add contacts | Collect name, email, phone, company through conversation |
-<div class="chat-text">+55 11 98765-4321</div>
+| Search | Find by any field - name, email, company, phone |
-<div class="chat-time">10:33 AM</div>
+| Update | Modify any contact field naturally |
-</div>
+| Delete | Remove contacts with confirmation |
-</div>
+| Tags | Categorize contacts (lead, customer, prospect) |
+| Export | Generate CSV files for external use |
+| Bulk import | Upload CSV to add multiple contacts |
 
-<div class="chat-message bot">
+---
-<div class="chat-bubble">
-<div class="chat-sender">CRM Bot</div>
-<div class="chat-text">Which company does Maria work for?</div>
-<div class="chat-time">10:33 AM</div>
-</div>
-</div>
 
-<div class="chat-message user">
+## Customization Ideas
-<div class="chat-bubble">
-<div class="chat-sender">You</div>
-<div class="chat-text">Tech Solutions Ltd</div>
-<div class="chat-time">10:34 AM</div>
-</div>
-</div>
 
-<div class="chat-message bot">
+### Add Custom Fields
-<div class="chat-bubble">
-<div class="chat-sender">CRM Bot</div>
-<div class="chat-text">Great! Here's the contact I'm about to save:
 
-📋 **Contact Details**
+Configure additional fields like LinkedIn profile, job title, or lead source in your bot's `config.csv`.
-━━━━━━━━━━━━━━━━━
-👤 **Name:** Maria Santos
-📧 **Email:** maria.santos@company.com
-📱 **Phone:** +55 11 98765-4321
-🏢 **Company:** Tech Solutions Ltd
-🏷️ **Tags:** lead
 
-Should I save this contact?</div>
+### Add Validation
-<div class="chat-time">10:34 AM</div>
-</div>
-</div>
 
-<div class="chat-message user">
+The bot validates email formats and phone numbers automatically. Configure stricter rules as needed.
-<div class="chat-bubble">
-<div class="chat-sender">You</div>
-<div class="chat-text">Yes, save it</div>
-<div class="chat-time">10:34 AM</div>
-</div>
-</div>
 
-<div class="chat-message bot">
+### Connect to External CRM
-<div class="chat-bubble">
-<div class="chat-sender">CRM Bot</div>
-<div class="chat-text">✅ Contact saved successfully!
 
-Maria Santos has been added to your contacts with ID #1247.
+Use the `POST` and `GET` keywords to sync contacts with Salesforce, HubSpot, or other CRM systems.
 
-What would you like to do next?
+---
-• Add another contact
-• Search contacts
-• View recent contacts</div>
-<div class="chat-time">10:34 AM</div>
-</div>
-</div>
-
-<div class="chat-message user">
-<div class="chat-bubble">
-<div class="chat-sender">You</div>
-<div class="chat-text">Search for contacts at Tech Solutions</div>
-<div class="chat-time">10:35 AM</div>
-</div>
-</div>
-
-<div class="chat-message bot">
-<div class="chat-bubble">
-<div class="chat-sender">CRM Bot</div>
-<div class="chat-text">🔍 Found 3 contacts at "Tech Solutions":
-
-1. **Maria Santos** - maria.santos@company.com
-   📱 +55 11 98765-4321 | 🏷️ lead
-
-2. **João Silva** - joao.silva@techsolutions.com
-   📱 +55 11 91234-5678 | 🏷️ customer
-
-3. **Ana Costa** - ana.costa@techsolutions.com
-   📱 +55 11 99876-5432 | 🏷️ prospect
-
-Type a number to see full details, or tell me what else you need.</div>
-<div class="chat-time">10:35 AM</div>
-</div>
-</div>
-
-</div>
-
-## Dialog Files
-
-### start.bas
-
-The main entry point for the CRM contacts bot:
-
-```basic
-' CRM Contacts - Main Dialog
-' Handles contact management through conversation
-
-SET CONTEXT "You are a helpful CRM assistant that manages contacts. 
-Be professional and efficient. Always confirm before saving data."
-
-ADD SUGGESTION "add" AS "Add new contact"
-ADD SUGGESTION "search" AS "Search contacts"
-ADD SUGGESTION "recent" AS "View recent"
-ADD SUGGESTION "export" AS "Export to CSV"
-
-BEGIN TALK
-Welcome to Contact Manager! 📇
-
-I can help you:
-• Add new contacts
-• Search existing contacts  
-• Update contact information
-• Export your contact list
-
-What would you like to do?
-END TALK
-```
-
-### add-contact.bas
-
-Handles adding new contacts:
-
-```basic
-' Add Contact Dialog
-
-TALK "I'd be happy to help you add a new contact! 📇"
-TALK "What's the contact's full name?"
-
-HEAR contact_name AS STRING
-
-TALK "Got it! " + contact_name + "."
-TALK "What's " + contact_name + "'s email address?"
-
-HEAR contact_email AS EMAIL
-
-TALK "Perfect! And what's the phone number? (You can skip this by typing \"skip\")"
-
-HEAR contact_phone AS STRING
-
-IF contact_phone = "skip" THEN
-    contact_phone = ""
-END IF
-
-TALK "Which company does " + contact_name + " work for?"
-
-HEAR contact_company AS STRING
-
-' Build confirmation message
-WITH contact_data
-    name = contact_name
-    email = contact_email
-    phone = contact_phone
-    company = contact_company
-    tags = "lead"
-    created_at = NOW()
-END WITH
-
-TALK "Great! Here's the contact I'm about to save:"
-TALK ""
-TALK "📋 **Contact Details**"
-TALK "━━━━━━━━━━━━━━━━━"
-TALK "👤 **Name:** " + contact_name
-TALK "📧 **Email:** " + contact_email
-TALK "📱 **Phone:** " + contact_phone
-TALK "🏢 **Company:** " + contact_company
-TALK "🏷️ **Tags:** lead"
-TALK ""
-TALK "Should I save this contact?"
-
-HEAR confirmation AS STRING
-
-IF INSTR(LOWER(confirmation), "yes") > 0 OR INSTR(LOWER(confirmation), "save") > 0 THEN
-    SAVE "contacts", contact_data
-    
-    contact_id = LAST("contacts", "id")
-    
-    TALK "✅ Contact saved successfully!"
-    TALK ""
-    TALK contact_name + " has been added to your contacts with ID #" + contact_id + "."
-ELSE
-    TALK "No problem! The contact was not saved."
-END IF
-
-TALK ""
-TALK "What would you like to do next?"
-TALK "• Add another contact"
-TALK "• Search contacts"
-TALK "• View recent contacts"
-```
-
-### search-contact.bas
-
-Handles contact search:
-
-```basic
-' Search Contact Dialog
-
-TALK "🔍 What would you like to search for?"
-TALK "You can search by name, email, company, or phone number."
-
-HEAR search_term AS STRING
-
-' Search across multiple fields
-results = FIND "contacts" WHERE 
-    name LIKE "%" + search_term + "%" OR
-    email LIKE "%" + search_term + "%" OR
-    company LIKE "%" + search_term + "%" OR
-    phone LIKE "%" + search_term + "%"
-
-result_count = COUNT(results)
-
-IF result_count = 0 THEN
-    TALK "No contacts found matching \"" + search_term + "\"."
-    TALK ""
-    TALK "Would you like to:"
-    TALK "• Try a different search"
-    TALK "• Add a new contact"
-ELSE
-    TALK "🔍 Found " + result_count + " contact(s) matching \"" + search_term + "\":"
-    TALK ""
-    
-    counter = 1
-    FOR EACH contact IN results
-        TALK counter + ". **" + contact.name + "** - " + contact.email
-        TALK "   📱 " + contact.phone + " | 🏷️ " + contact.tags
-        TALK ""
-        counter = counter + 1
-    NEXT
-    
-    TALK "Type a number to see full details, or tell me what else you need."
-END IF
-```
-
-## Database Schema
-
-The template creates this table structure:
-
-```sql
-CREATE TABLE contacts (
-    id SERIAL PRIMARY KEY,
-    name VARCHAR(255) NOT NULL,
-    email VARCHAR(255) UNIQUE,
-    phone VARCHAR(50),
-    company VARCHAR(255),
-    tags VARCHAR(255) DEFAULT 'lead',
-    notes TEXT,
-    created_at TIMESTAMP DEFAULT NOW(),
-    updated_at TIMESTAMP DEFAULT NOW(),
-    created_by UUID REFERENCES users(id)
-);
-
-CREATE INDEX idx_contacts_email ON contacts(email);
-CREATE INDEX idx_contacts_company ON contacts(company);
-CREATE INDEX idx_contacts_tags ON contacts(tags);
-```
-
-## API Endpoints
-
-The template exposes these REST endpoints:
-
-| Method | Endpoint | Description |
-|--------|----------|-------------|
-| GET | `/api/v1/contacts` | List all contacts |
-| GET | `/api/v1/contacts/:id` | Get single contact |
-| POST | `/api/v1/contacts` | Create contact |
-| PUT | `/api/v1/contacts/:id` | Update contact |
-| DELETE | `/api/v1/contacts/:id` | Delete contact |
-| GET | `/api/v1/contacts/search?q=` | Search contacts |
-| GET | `/api/v1/contacts/export` | Export to CSV |
-
-## Customization
-
-### Adding Custom Fields
-
-Edit `tables.bas` to add custom fields:
-
-```basic
-TABLE contacts
-    FIELD id AS INTEGER PRIMARY KEY
-    FIELD name AS STRING(255) REQUIRED
-    FIELD email AS EMAIL UNIQUE
-    FIELD phone AS PHONE
-    FIELD company AS STRING(255)
-    FIELD tags AS STRING(255)
-    FIELD notes AS TEXT
-    ' Add your custom fields below
-    FIELD linkedin AS STRING(255)
-    FIELD job_title AS STRING(255)
-    FIELD lead_source AS STRING(100)
-    FIELD lead_score AS INTEGER DEFAULT 0
-END TABLE
-```
-
-### Changing Default Tags
-
-Update `config.csv`:
-
-```csv
-key,value
-default-tags,"prospect,website"
-```
-
-### Adding Validation
-
-Edit `add-contact.bas` to add custom validation:
-
-```basic
-' Validate email domain
-IF NOT INSTR(contact_email, "@company.com") THEN
-    TALK "⚠️ Warning: This email is not from your company domain."
-END IF
-
-' Check for duplicates
-existing = FIND "contacts" WHERE email = contact_email
-IF COUNT(existing) > 0 THEN
-    TALK "⚠️ A contact with this email already exists!"
-    TALK "Would you like to update the existing contact instead?"
-    ' Handle duplicate logic
-END IF
-```
 
 ## Related Templates
 
-- [Sales Pipeline](./template-sales-pipeline.md) - Track deals and opportunities
+- [Sales Pipeline](./templates.md) - Track deals and opportunities
-- [Marketing Campaigns](./template-marketing.md) - Email campaigns and automation
+- [Customer Support](./templates.md) - Support ticket management
-- [Customer Support](./template-helpdesk.md) - Support ticket management
+- [Template Samples](./template-samples.md) - More conversation examples
 
-## Support
+---
 
-For issues with this template:
+<style>
-- Check the [troubleshooting guide](../chapter-13-community/README.md)
+.wa-chat{background-color:#e5ddd5;border-radius:8px;padding:20px 15px;margin:20px 0;max-width:500px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;font-size:14px}
-- Open an issue on [GitHub](https://github.com/GeneralBots/BotServer/issues)
+.wa-message{margin-bottom:10px}
-- Join the [community chat](https://discord.gg/generalbots)
+.wa-message.user{text-align:right}
+.wa-message.user .wa-bubble{background-color:#dcf8c6;display:inline-block;text-align:left}
+.wa-message.bot .wa-bubble{background-color:#fff;display:inline-block}
+.wa-bubble{padding:8px 12px;border-radius:8px;box-shadow:0 1px .5px rgba(0,0,0,.13);max-width:85%}
+.wa-bubble p{margin:0 0 4px 0;line-height:1.4;color:#303030}
+.wa-bubble p:last-child{margin-bottom:0}
+.wa-time{font-size:11px;color:#8696a0;text-align:right;margin-top:4px}
+</style>

docs/src/chapter-03/caching.md

@@ -1,6 +1,6 @@
 # Caching
 
-BotServer includes automatic caching to improve response times and reduce redundant processing, including semantic caching for LLM responses using Valkey (a Redis-compatible in-memory database).
+BotServer includes automatic caching to improve response times and reduce redundant processing, including semantic caching for LLM responses using an in-memory cache component.
 
 ## Features
 
@@ -66,11 +66,11 @@ prompt-compact,4    # Compact conversation after N exchanges
 - **prompt-history**: Keeps the last 2 exchanges in the conversation context
 - **prompt-compact**: After 4 exchanges, older messages are summarized or removed to save tokens
 
-## Cache Storage with Valkey
+## Cache Storage
 
 ### Architecture
 
-<img src="./assets/caching-architecture.svg" alt="Caching Architecture Flow" style="max-height: 400px; width: 100%; object-fit: contain;">
+The caching system uses a multi-level approach for optimal performance.
 
 ### Cache Key Structure
 
@@ -78,9 +78,9 @@ The cache uses a multi-level key structure:
 - **Exact match**: Hash of the exact prompt
 - **Semantic match**: Embedding vector stored with semantic index
 
-### Valkey Integration
+### Cache Component Features
 
-Valkey provides:
+The cache component provides:
 - **Fast in-memory storage**: Sub-millisecond response times
 - **Automatic expiration**: TTL-based cache invalidation
 - **Distributed caching**: Share cache across multiple bot instances
@@ -116,7 +116,7 @@ USE TOOL "weather-api"
 
 ## Cache Management
 
-The cache operates automatically based on your configuration settings. Cache entries are managed through TTL expiration and Valkey's memory policies.
+The cache operates automatically based on your configuration settings. Cache entries are managed through TTL expiration and memory policies.
 
 ## Best Practices
 
@@ -148,9 +148,9 @@ Disable caching for:
 
 ### Memory Management
 
-Valkey automatically manages memory through:
+The cache component automatically manages memory through:
 - **Eviction policies**: LRU (Least Recently Used) by default
-- **Max memory limits**: Configure in Valkey settings
+- **Max memory limits**: Configurable memory settings
 - **Key expiration**: Automatic cleanup of expired entries
 
 ## Performance Impact
@@ -169,40 +169,14 @@ Typical performance improvements with caching enabled:
 ### Cache Not Working
 
 Check:
-1. Valkey is running: `ps aux | grep valkey`
+1. Cache service is running
 2. Cache enabled in config: `llm-cache,true`
 3. TTL not expired
 4. Similarity threshold not too high
 
 ### Clear Cache
 
-To clear the cache manually:
+Cache is managed automatically. To clear cache manually, restart the cache component or use the admin API endpoint `/api/admin/cache/clear`.
-
-```bash
-# Connect to Valkey
-valkey-cli
-
-# Clear all bot cache
-FLUSHDB
-
-# Clear specific bot cache
-DEL bot:cache:*
-```
-
-### Clear Cache
-
-To clear the cache manually:
-
-```bash
-# Connect to Valkey
-valkey-cli
-
-# Clear all bot cache
-FLUSHDB
-
-# Clear specific bot cache
-DEL bot:cache:*
-```
 
 ## Summary
 
@@ -210,6 +184,6 @@ The semantic caching system in BotServer provides intelligent response caching t
 - Reduces response latency by 10-100x
 - Cuts API costs by 90%+
 - Maintains response quality through semantic matching
-- Scales automatically with Valkey
+- Scales automatically with the cache component
 
 Configure caching based on your bot's needs, monitor performance metrics, and tune parameters for optimal results.

docs/src/chapter-03/summary.md

@@ -8,7 +8,7 @@ This chapter explains how GeneralBots manages knowledge‑base collections, inde
 | **Caching** | [caching.md](caching.md) | Optional in‑memory and persistent SQLite caching to speed up frequent `FIND` queries. |
 | **Context Compaction** | [context-compaction.md](context-compaction.md) | Techniques to keep the LLM context window within limits (summarization, memory pruning, sliding window). |
 | **Indexing** | [indexing.md](indexing.md) | Process of extracting, chunking, embedding, and storing document vectors in the VectorDB. |
-| **Semantic Caching** | [caching.md](caching.md) | Intelligent caching with Valkey for LLM responses, including semantic similarity matching. |
+| **Semantic Caching** | [caching.md](caching.md) | Intelligent caching for LLM responses, including semantic similarity matching. |
 | **Semantic Search** | [semantic-search.md](semantic-search.md) | How the `FIND` keyword performs meaning‑based retrieval using the VectorDB. |
 | **Vector Collections** | [vector-collections.md](vector-collections.md) | Definition and management of vector collections, including creation, document addition, and usage in dialogs. |
 

docs/src/chapter-04-gbui/console-mode.md

@@ -23,7 +23,7 @@ The console displays real-time information about your running BotServer instance
 ║                                                            ║
 ║ Components:                                                ║
 ║   PostgreSQL: ✓ Connected                                  ║
-║   Valkey:     ✓ Connected                                  ║
+║   Cache:      ✓ Connected                                  ║
 ║   Storage:    ✓ Connected                                  ║
 ║   Vectors:    ✓ Connected                                  ║
 ║                                                            ║
@@ -96,7 +96,7 @@ In production, console mode can help with:
 Console mode provides basic monitoring only. For detailed analytics:
 - Check PostgreSQL directly for session data
 - Use system logs for detailed error information
-- Monitor Valkey for cache statistics
+- Monitor cache for statistics
 - Review application logs for debugging
 
 ## Terminal Requirements

docs/src/chapter-04-gbui/how-to/connect-whatsapp.md

@@ -725,16 +725,19 @@ Keep quality high by:
 | WhatsApp Cloud API Docs | https://developers.facebook.com/docs/whatsapp |
 | API Status | https://metastatus.com |
 
-### Environment Variables
+### Configuration via config.csv
 
-If configuring via environment variables:
+Configure WhatsApp in your bot's `config.csv`:
 
+```csv
+name,value
+whatsapp-phone-number-id,123456789012345
+whatsapp-access-token,EAAGm0PX4ZCp...
+whatsapp-verify-token,my_bot_verify_123
+whatsapp-business-account-id,987654321098765
 ```
-WHATSAPP_PHONE_NUMBER_ID=123456789012345
+
-WHATSAPP_ACCESS_TOKEN=EAAGm0PX4ZCp...
+**Note**: Do not use environment variables for WhatsApp configuration. All settings belong in `config.csv`.
-WHATSAPP_VERIFY_TOKEN=my_bot_verify_123
-WHATSAPP_BUSINESS_ACCOUNT_ID=987654321098765
-```
 
 ---
 

docs/src/chapter-04-gbui/how-to/monitor-sessions.md

@@ -70,7 +70,7 @@ Your General Bots deployment is a **living system** of interconnected components
 | **MinIO** | File storage, manages documents | Storage used, object count |
 | **BotModels** | LLM server, generates AI responses | Tokens/hour, model latency |
 | **Vault** | Secrets manager, stores API keys | Sealed status, policy count |
-| **Redis** | Cache layer, speeds up responses | Hit rate, memory usage |
+| **Cache** | Cache layer, speeds up responses | Hit rate, memory usage |
 | **InfluxDB** | Metrics database, stores analytics | Points/sec, retention |
 
 ---
@@ -213,7 +213,7 @@ The dashboard shows the health of all components:
 │  ● MinIO           Running    v2024.01    45.2 GB stored               │
 │  ● BotModels       Running    v2.1.0      gpt-4o active                │
 │  ● Vault           Sealed     v1.15.0     156 secrets                  │
-│  ● Redis           Running    v7.2.4      94.2% hit rate               │
+│  ● Cache           Running    v7.2.4      94.2% hit rate               │
 │  ● InfluxDB        Running    v2.7.3      2,450 pts/sec                │
 │                                                                         │
 │  Legend: ● Running  ● Warning  ● Stopped                               │

docs/src/chapter-04-gbui/monitoring.md

@@ -11,7 +11,7 @@ Your General Bots deployment is a living ecosystem of interconnected components.
 This animated diagram shows:
 - **BotServer** (center) - The core that orchestrates all interactions
 - **Data Layer** (left) - PostgreSQL, Qdrant, and MinIO for storage
-- **Services** (right) - BotModels, Vault, Redis for AI and security
+- **Services** (right) - BotModels, Vault, Cache for AI and security
 - **Analytics** (bottom) - InfluxDB for metrics collection
 - **Connection flows** - Animated data packets showing real-time communication
 
@@ -127,7 +127,7 @@ Each component in the system has specific health indicators:
 | **MinIO** | Storage usage, object count | > 80% storage used |
 | **BotModels** | Token usage, response latency | > 2s response time |
 | **Vault** | Seal status, policy count | Unsealed without auth |
-| **Redis** | Hit rate, memory usage | < 80% hit rate |
+| **Cache** | Hit rate, memory usage | < 80% hit rate |
 | **InfluxDB** | Write rate, retention | Write failures |
 
 ## Console Mode
@@ -149,10 +149,10 @@ Services: 4/5 running
 
 ## Alerts Configuration
 
-Configure alerts in `config.csv`:
+Configure alert thresholds in `config.csv`. Example values:
 
 ```csv
-key,value
+name,value
 alert-cpu-threshold,80
 alert-memory-threshold,85
 alert-disk-threshold,90
@@ -160,6 +160,8 @@ alert-response-time-ms,5000
 alert-email,admin@example.com
 ```
 
+**Note**: These are example configuration values. Adjust thresholds based on your infrastructure and requirements.
+
 ## Bot-Specific Metrics
 
 View metrics for individual bots:

docs/src/chapter-04-gbui/suite-manual.md

@@ -2,13 +2,13 @@
 
 > **The Complete Productivity Workspace**
 > 
-> *Like Microsoft Office, but with AI built-in*
+> *AI-native productivity suite for modern teams*
 
 ---
 
 ## Welcome to General Bots Suite
 
-General Bots Suite is your all-in-one workspace that combines communication, productivity, and AI assistance. Think of it like having Word, Outlook, Teams, and an AI assistant all in one place - but simpler and more powerful.
+General Bots Suite is your all-in-one workspace that combines communication, productivity, and AI assistance. All your productivity tools in one place - simple and powerful.
 
 **What makes it special:**
 - Everything works together seamlessly
@@ -1123,14 +1123,6 @@ You: Help me with tasks
 
 ---
 
-## Appendix C: Version History
-
-| Version | Date | Changes |
-|---------|------|---------|
-| 1.0 | March 2024 | Initial release with Chat, Drive, Tasks, Mail |
-| 1.1 | April 2024 | Added Meet, Paper, Research |
-| 1.2 | May 2024 | Added Designer, Sources, Compliance |
-
 ---
 
 *© General Bots - Built with ❤️ and AI*

docs/src/chapter-06-gbdialog/basic-vs-automation-tools.md

@@ -1,454 +1,238 @@
-# BASIC vs Everyone: The Complete Domination Guide
+# BASIC vs Automation Tools: A Practical Comparison
 
-> *"Embrace, Extend, Extinguish"* — We learned from the best. Now we're doing it **open source**.
+> **Understanding how General Bots BASIC compares to other automation platforms**
 
-## 🎯 The Mission: Kill Them All
+## Overview
 
-Just like Microsoft killed Lotus 1-2-3 by making Excel better AND cheaper, General Bots BASIC is here to **obliterate** every paid automation tool, AI assistant, and workflow platform.
+General Bots BASIC provides a conversational-first approach to automation. This chapter compares BASIC with popular automation tools to help you understand when each approach works best.
-
-**They charge you $30/user/month. We charge you $0. Forever.**
 
 ---
 
-## 💀 The Kill List
+## Comparison Matrix
 
-### Automation Platforms (DEAD)
+| Feature | Zapier | n8n | Make | Power Automate | **BASIC** |
-- ~~n8n~~ → BASIC does more
+|---------|--------|-----|------|----------------|-----------|
-- ~~Zapier~~ → BASIC is free
+| Webhooks | ✅ | ✅ | ✅ | ✅ | ✅ |
-- ~~Make.com~~ → BASIC has AI native
+| Scheduling | ✅ | ✅ | ✅ | ✅ | ✅ `SET SCHEDULE` |
-- ~~Power Automate~~ → BASIC is open source
+| HTTP/REST | ✅ | ✅ | ✅ | ✅ | ✅ |
-
+| GraphQL | ❌ | ✅ | ✅ | ❌ | ✅ |
-### AI Assistants (OBSOLETE)
+| SOAP | ❌ | ❌ | ✅ | ✅ | ✅ |
-- ~~Microsoft Copilot~~ → We support Claude Opus 4, GPT-4, AND local models
+| Database Native | ❌ | ✅ | ✅ | ✅ | ✅ |
-- ~~Google Gemini~~ → We're not locked to one vendor
+| **Conversations** | ❌ | ❌ | ❌ | ❌ | ✅ |
-- ~~ChatGPT Plus~~ → Our bots DO things, not just chat
+| **WhatsApp Native** | Plugin | Plugin | Plugin | Plugin | ✅ Built-in |
-
+| **Telegram Native** | Plugin | Plugin | Plugin | ❌ | ✅ Built-in |
-### Office Suites (DISRUPTED)
+| **Multi-Channel** | Limited | Limited | Limited | Limited | ✅ Native |
-- ~~Microsoft 365~~ → We have email, drive, calendar, meet
+| **LLM Integration** | Plugin | Plugin | Plugin | GPT-4 | ✅ Any model |
-- ~~Google Workspace~~ → Same features, zero cost
+| **Self-Hosted** | ❌ | ✅ | ❌ | ❌ | ✅ |
-- ~~Zoho~~ → More AI, less complexity
+| **Open Source** | ❌ | ✅ | ❌ | ❌ | ✅ AGPL |
 
 ---
 
-## 🏆 The Ultimate Comparison Matrix
+## Key Differences
 
-| Feature | Zapier | n8n | Make | Power Automate | **Copilot** | **Gemini** | **BASIC** |
+### Conversation-First Design
-|---------|--------|-----|------|----------------|-------------|------------|-----------|
-| **PRICE** | $50-800/mo | $24-500/mo | $10-350/mo | $15-40/user | **$30/user** | **$20/user** | **$0 FOREVER** |
-| Webhooks | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ |
-| Scheduling | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ `"every hour"` |
-| HTTP/REST | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ |
-| GraphQL | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ |
-| SOAP | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ |
-| Database Native | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ |
-| **Conversations** | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ |
-| **WhatsApp Native** | Plugin | Plugin | Plugin | Plugin | ❌ | ❌ | ✅ **NATIVE** |
-| **Telegram Native** | Plugin | Plugin | Plugin | ❌ | ❌ | ❌ | ✅ **NATIVE** |
-| **Image Generation** | ❌ | ❌ | ❌ | ❌ | ✅ DALL-E | ✅ Imagen | ✅ **ANY MODEL** |
-| **Video Generation** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
-| **Voice/TTS** | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ |
-| **Vision/OCR** | Plugin | Plugin | Plugin | ✅ | ✅ | ✅ | ✅ |
-| **Best AI Models** | ❌ | ❌ | ❌ | GPT-4 only | GPT-4 only | Gemini only | ✅ **ALL MODELS** |
-| Claude Opus 4 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
-| Local LLMs | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Llama/Mistral |
-| **Self-Hosted** | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ **100% YOURS** |
-| **Open Source** | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ **AGPL** |
-| **White Label** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
-| Version Control | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Git native |
-| Lead Scoring AI | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
-| CRM Native | Plugin | Plugin | Plugin | Plugin | ❌ | ❌ | ✅ |
-| Email Server | ❌ | ❌ | ❌ | Exchange | Gmail | Gmail | ✅ **BUILT-IN** |
-| Video Meetings | ❌ | ❌ | ❌ | Teams | Teams | Meet | ✅ **BUILT-IN** |
-| File Storage | ❌ | ❌ | ❌ | OneDrive | OneDrive | Drive | ✅ **BUILT-IN** |
 
----
+Traditional automation tools focus on backend workflows. BASIC adds interactive conversations:
 
-## 🔥 Why Copilot & Gemini Are ALREADY DEAD
+<div class="wa-chat">
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>I'll help you file an expense report. What's the amount?</p>
+      <div class="wa-time">10:30</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>$150</p>
+      <div class="wa-time">10:30</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>What category? (travel/meals/supplies)</p>
+      <div class="wa-time">10:30</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>meals</p>
+      <div class="wa-time">10:31</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>Upload the receipt photo</p>
+      <div class="wa-time">10:31</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>📎 receipt.jpg</p>
+      <div class="wa-time">10:31</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>✅ Expense submitted! Reference: EXP-2025-0142</p>
+      <div class="wa-time">10:31</div>
+    </div>
+  </div>
+</div>
 
-### Microsoft Copilot: $30/user/month for WHAT?
+### Multi-Channel Native
 
-**What Copilot gives you:**
+The same bot works across all channels without modification:
-- Chat with documents *(we do this)*
-- Email drafting *(we do this)*
-- Meeting summaries *(we do this)*
-- Excel formulas *(we do this)*
-- Locked to Microsoft ecosystem *(WE DON'T)*
-- GPT-4 only *(we support ALL models)*
 
-**What BASIC gives you for FREE:**
+<div class="wa-chat">
-```basic
+  <div class="wa-message bot">
-' Everything Copilot does, plus automation
+    <div class="wa-bubble">
-USE KB "company-docs"
+      <p>Your order has shipped! 📦</p>
-answer = LLM "Summarize Q4 results from the uploaded documents"
+      <p>Tracking: 7891234567890</p>
-SEND MAIL "team@company.com", "Q4 Summary", answer
+      <div class="wa-time">14:20</div>
+    </div>
+  </div>
+</div>
 
-' Schedule it!
+This message reaches users on WhatsApp, Telegram, Web, or any configured channel—same content, adapted formatting.
-SET SCHEDULE "every monday at 9am"
-```
 
-**Copilot can't:**
+### LLM Model Freedom
-- Send WhatsApp messages
-- Trigger webhooks
-- Run scheduled automations
-- Generate videos
-- Score leads
-- Build interactive bots
-- Work offline with local LLMs
 
-### Google Gemini: Locked in Google's Walled Garden
+BASIC supports any LLM provider:
 
-**Gemini limitations:**
+- OpenAI (GPT-4, GPT-3.5)
-- Only works with Google Workspace
+- Anthropic (Claude)
-- Only Gemini models (no Claude, no GPT)
+- Local models (Llama, Mistral via llama.cpp)
-- No automation capabilities
+- Groq, DeepSeek, and others
-- No webhook triggers
+- Any OpenAI-compatible API
-- No self-hosting
-- Your data trains their models
 
-**BASIC destroys this:**
+Configure in `config.csv`:
-```basic
-' Use ANY model - Claude Opus 4 is the BEST
-SET LLM MODEL "claude-opus-4"
-analysis = LLM "Deep analysis of customer feedback with nuanced understanding"
 
-' Or use local models for privacy
+```csv
-SET LLM MODEL "llama-3.1-70b"
+name,value
-private_analysis = LLM "Analyze confidential data locally"
+llm-url,http://localhost:8081
-
+llm-model,model.gguf
-' Or use Gemini if you want!
-SET LLM MODEL "gemini-pro"
 ```
 
 ---
 
-## 🏴‍☠️ The Microsoft Playbook (We're Using It)
+## When to Use Each Tool
 
-### 1. EMBRACE ✅
+### Choose BASIC When You Need
-We support everything they support:
-- All their file formats
-- All their APIs
-- All their integrations
-- All their workflows
 
-### 2. EXTEND ✅
+- **Interactive workflows** - Users participate in the process
-We add what they CAN'T:
+- **Multi-channel presence** - Same bot on WhatsApp, Telegram, Web
-- **Conversational AI** - Interactive bots, not just automation
+- **AI-powered conversations** - Natural language understanding
-- **Multi-model AI** - Claude Opus 4, GPT-4, Gemini, Llama, Mistral
+- **Self-hosted deployment** - Full data control
-- **Multimodal** - Image, video, audio generation
+- **Open source flexibility** - Modify and extend as needed
-- **Self-hosted** - Your data stays yours
-- **White label** - Your brand, not ours
-- **Zero cost** - Forever free
 
-### 3. EXTINGUISH 🎯
+### Choose Traditional Automation When You Need
-Why would anyone pay when they can get MORE for FREE?
 
-| Their Product | Their Cost | **BASIC Replacement** | **Your Cost** |
+- **Backend-only workflows** - No user interaction required
-|---------------|------------|----------------------|---------------|
+- **Visual workflow builders** - Prefer drag-and-drop interfaces
-| Zapier Business | $800/month | BASIC Scripts | **$0** |
+- **Existing integrations** - Specific pre-built connectors
-| n8n Cloud | $500/month | BASIC Scripts | **$0** |
+- **Team familiarity** - Team already knows the tool
-| Make Teams | $350/month | BASIC Scripts | **$0** |
-| Power Automate | $40/user/month | BASIC Scripts | **$0** |
-| Copilot Pro | $30/user/month | LLM Keyword + KB | **$0** |
-| Gemini Advanced | $20/user/month | LLM Keyword | **$0** |
-| ChatGPT Plus | $20/month | LLM Keyword | **$0** |
-| Microsoft 365 | $22/user/month | Full Office Suite | **$0** |
-| Google Workspace | $12/user/month | Full Office Suite | **$0** |
-| Intercom | $74/user/month | BASIC Bots | **$0** |
-| HubSpot | $800/month | CRM + Lead Scoring | **$0** |
-| Twilio | Pay per message | WhatsApp Native | **$0** |
-
-**100 users × $30/month Copilot = $3,000/month = $36,000/year**
-**100 users × BASIC = $0/month = $0/year**
 
 ---
 
-## 🚀 Claude Opus 4: The Best Model, Available HERE
+## Migration Examples
 
-While Copilot is stuck with GPT-4 and Gemini is stuck with... Gemini, **BASIC supports Claude Opus 4** — widely considered the most capable AI model for:
+### From Zapier
 
-- **Nuanced understanding** - Better at complex instructions
+Zapier workflow: Form submission → Slack notification → CRM entry → Welcome email
-- **Longer context** - 200K tokens vs GPT-4's 128K
-- **Better coding** - More accurate code generation
-- **Safer outputs** - Constitutional AI training
-- **Less hallucination** - More factual responses
 
-```basic
+BASIC equivalent:
-' Use the BEST model available
-SET LLM MODEL "claude-opus-4"
 
-' Complex multi-step reasoning
+<div class="wa-chat">
-analysis = LLM "
+  <div class="wa-message bot">
-    Analyze our Q4 sales data and:
+    <div class="wa-bubble">
-    1. Identify top 3 performing regions
+      <p>New lead captured! 🎉</p>
-    2. Find correlation with marketing spend
+      <p>👤 John Smith</p>
-    3. Predict Q1 trends
+      <p>📧 john@example.com</p>
-    4. Recommend budget allocation
+      <p>🏢 Acme Corp</p>
-    Be specific with numbers and confidence levels.
+      <p>Added to CRM and notified the team.</p>
-"
+      <div class="wa-time">09:15</div>
+    </div>
+  </div>
+</div>
 
-TALK analysis
+### From n8n
-```
 
-**Model Freedom:**
+n8n workflow: Monitor website → Alert on error → Create ticket
-```basic
+
-' Switch models based on task
+BASIC equivalent runs on schedule and notifies immediately:
-SET LLM MODEL "claude-opus-4"      ' Complex analysis
+
-SET LLM MODEL "gpt-4-turbo"        ' General tasks
+<div class="wa-chat">
-SET LLM MODEL "gemini-pro"         ' Google integration
+  <div class="wa-message bot">
-SET LLM MODEL "llama-3.1-70b"      ' Private/offline
+    <div class="wa-bubble">
-SET LLM MODEL "mistral-large"      ' European compliance
+      <p>🔴 Website Alert</p>
-SET LLM MODEL "deepseek-coder"     ' Code generation
+      <p>mysite.com returned status 503</p>
-```
+      <p>Ticket #IT-2025-0891 created</p>
+      <p>DevOps team notified</p>
+      <div class="wa-time">03:42</div>
+    </div>
+  </div>
+</div>
 
 ---
 
-## 💪 What We Do That NO ONE Else Can
+## Complete Office Suite
 
-### 1. Conversational Automation
+BASIC provides built-in capabilities for common office tasks:
-```basic
-' Interactive workflow - impossible in Zapier/n8n
-TALK "I'll help you file an expense report. What's the amount?"
-HEAR amount
 
-TALK "What category? (travel/meals/supplies)"
+| Capability | BASIC Keyword |
-HEAR category
+|------------|---------------|
+| Send email | `SEND MAIL` |
+| Create draft | `CREATE DRAFT` |
+| Schedule meetings | `BOOK` |
+| Manage files | `UPLOAD`, `DOWNLOAD`, `LIST` |
+| Create tasks | `CREATE TASK` |
+| Video meetings | `CREATE MEETING` |
 
-TALK "Upload the receipt photo"
+### Example: Daily Report Automation
-HEAR receipt AS FILE
 
-' AI extracts receipt data
+<div class="wa-chat">
-receipt_data = SEE receipt
+  <div class="wa-message bot">
-verified_amount = receipt_data.total
+    <div class="wa-bubble">
-
+      <p>📊 Daily Sales Report - May 15, 2025</p>
-IF verified_amount != amount THEN
+      <p>━━━━━━━━━━━━━━━━━━━━━</p>
-    TALK "Receipt shows $" + verified_amount + ", you entered $" + amount + ". Which is correct?"
+      <p>💰 Revenue: $15,340</p>
-    HEAR correct_amount
+      <p>📦 Orders: 47</p>
-    amount = correct_amount
+      <p>📈 +12% vs yesterday</p>
-END IF
+      <p>━━━━━━━━━━━━━━━━━━━━━</p>
-
+      <p>Report sent to executives@company.com</p>
-INSERT "expenses", amount, category, receipt, NOW()
+      <div class="wa-time">09:00</div>
-TALK "Expense submitted! Reference: " + LAST_INSERT_ID
+    </div>
-```
+  </div>
-
+</div>
-### 2. Multi-Channel Native
-```basic
-' Same bot works on ALL channels
-TALK "Your order has shipped!"  ' Works on WhatsApp, Telegram, Web, SMS
-
-' Channel-specific when needed
-IF channel = "whatsapp" THEN
-    SEND TEMPLATE "shipping_update", phone, tracking_number
-ELSE IF channel = "email" THEN
-    SEND MAIL email, "Shipping Update", tracking_email
-END IF
-```
-
-### 3. AI That DOES Things
-```basic
-' Not just chat - actual automation
-SET SCHEDULE "every day at 6am"
-
-' Analyze overnight support tickets
-tickets = FIND "support_tickets", "created_at > DATEADD('hour', -12, NOW())"
-
-FOR EACH ticket IN tickets
-    ' AI categorizes and prioritizes
-    analysis = LLM "Analyze this support ticket and return JSON with category, priority, suggested_response: " + ticket.content
-    
-    UPDATE "support_tickets", ticket.id, analysis.category, analysis.priority
-    
-    IF analysis.priority = "urgent" THEN
-        TALK TO on_call_agent, "🚨 Urgent ticket: " + ticket.subject
-    END IF
-NEXT
-
-' Generate daily summary
-summary = LLM "Create executive summary of " + LEN(tickets) + " overnight tickets"
-SEND MAIL "support-manager@company.com", "Overnight Ticket Summary", summary
-```
-
-### 4. Complete Office Suite Replacement
-```basic
-' Email
-SEND MAIL "team@company.com", "Subject", "Body"
-emails = GET "mail/inbox"
-
-' Calendar
-BOOK "Sales Meeting", "tomorrow at 2pm", "john@company.com, jane@company.com"
-
-' Files
-UPLOAD "report.pdf", "shared/reports/"
-file = DOWNLOAD "shared/templates/invoice.docx"
-GENERATE PDF "invoice_data", "invoice_template.docx"
-
-' Tasks
-CREATE TASK "Review proposal", "john", "friday"
-
-' Meetings (video)
-meeting_url = CREATE MEETING "Weekly Standup", "monday at 9am"
-```
 
 ---
 
-## 🎪 The Migration Massacre
+## Getting Started
 
-### From Zapier (RIP)
+### Quick Start
-```basic
-' Zapier: 5 zaps, $50/month
-' BASIC: 10 lines, $0/month
 
-ON FORM SUBMIT "contact-form"
+1. Download and run BotServer
-    ' Send to Slack
+2. Edit your bot's `.bas` files
-    POST "https://hooks.slack.com/...", { "text": "New lead: " + form.email }
+3. Configure settings in `config.csv`
-    
+4. Deploy to any channel
-    ' Add to CRM
-    INSERT "leads", form.name, form.email, form.company
-    
-    ' Send welcome email
-    SEND MAIL form.email, "Thanks for reaching out!", welcome_template
-    
-    ' Score the lead with AI
-    score = AI SCORE LEAD form.email, form.company, form.message
-    UPDATE "leads", LAST_INSERT_ID, "score", score
-END ON
-```
 
-### From n8n (Gone)
+### Resources
-```basic
-' n8n: Complex node setup, self-host headaches
-' BASIC: Just write what you want
-
-SET SCHEDULE "every 5 minutes"
-
-' Monitor website
-response = GET "https://mysite.com/health"
-IF response.status != 200 THEN
-    TALK TO ops_team, "🔴 Website down! Status: " + response.status
-    CREATE TASK "Investigate website outage", "devops", "urgent"
-END IF
-```
-
-### From Power Automate (Destroyed)
-```basic
-' Power Automate: $40/user/month, Microsoft lock-in
-' BASIC: Free, works everywhere
-
-' When email arrives with attachment
-ON EMAIL RECEIVED
-    IF email.has_attachments THEN
-        FOR EACH attachment IN email.attachments
-            ' Extract data with AI vision
-            data = SEE attachment
-            
-            ' Save to database
-            INSERT "documents", email.from, attachment.name, data
-            
-            ' Notify team
-            TALK TO document_team, "New document from " + email.from
-        NEXT
-    END IF
-END ON
-```
-
-### From Copilot (Obsolete)
-```basic
-' Copilot: $30/user, limited to Microsoft
-' BASIC: $0, unlimited potential
-
-' Everything Copilot does
-USE KB "company-knowledge"
-answer = LLM "Answer this question using company docs: " + question
-
-' Plus things Copilot CAN'T do
-SET SCHEDULE "every monday at 8am"
-report = LLM "Generate weekly report from sales data"
-SEND MAIL team, "Weekly Report", report
-POST "https://slack.com/api/...", { "text": report }
-```
-
----
-
-## 📊 TCO Calculator: The Massacre in Numbers
-
-### Small Business (10 users)
-| Solution | Monthly | Annual |
-|----------|---------|--------|
-| Zapier + Copilot | $300 + $300 = $600 | $7,200 |
-| n8n + ChatGPT | $50 + $200 = $250 | $3,000 |
-| **BASIC** | **$0** | **$0** |
-
-### Medium Business (100 users)
-| Solution | Monthly | Annual |
-|----------|---------|--------|
-| Zapier Pro + M365 + Copilot | $800 + $2,200 + $3,000 = $6,000 | $72,000 |
-| Make + Google + Gemini | $350 + $1,200 + $2,000 = $3,550 | $42,600 |
-| **BASIC** | **$0** | **$0** |
-
-### Enterprise (1,000 users)
-| Solution | Monthly | Annual |
-|----------|---------|--------|
-| Enterprise Stack | $50,000+ | $600,000+ |
-| **BASIC** | **$0** | **$0** |
-
-**SAVINGS: $600,000/year**
-
----
-
-## 🏁 The Endgame
-
-### Why They Can't Compete
-
-1. **We're open source** - They can't undercut free
-2. **We support ALL models** - They're locked to their own
-3. **We're self-hosted** - Your data is yours
-4. **We're conversation-first** - They're automation-only
-5. **We're multimodal native** - They bolt on features
-6. **We have no per-user pricing** - Deploy to millions, pay nothing
-
-### The Lotus 1-2-3 Moment
-
-Remember when Excel killed Lotus 1-2-3?
-- Excel was **cheaper** ✅
-- Excel was **more integrated** ✅
-- Excel **embraced their file format** ✅
-- Excel **extended with features** ✅
-
-**BASIC is doing the same thing to the entire automation/AI assistant industry.**
-
----
-
-## 🚀 Get Started (It's Free, Obviously)
-
-```bash
-# One command to rule them all
-curl -fsSL https://get.generalbots.com | sh
-
-# Or with Docker
-docker run -d generalbots/botserver
-```
-
-Then write your first automation:
-```basic
-SET SCHEDULE "every hour"
-TALK "Hello from the future of automation!"
-```
-
-**No credit card. No trial period. No user limits. No bullshit.**
-
----
-
-## Related
 
 - [Keywords Reference](./keywords.md) - Complete keyword documentation
-- [SET SCHEDULE](./keyword-set-schedule.md) - Natural language scheduling
+- [SET SCHEDULE](./keyword-set-schedule.md) - Automate with schedules
 - [WEBHOOK](./keyword-webhook.md) - Event-driven automation
-- [LLM](./keyword-llm.md) - AI integration with ANY model
+- [Templates](./templates.md) - Ready-to-use examples
-- [Templates](./templates.md) - Ready-to-use automation templates
 
 ---
 
-*"The best way to predict the future is to create it."*
+<style>
-— Alan Kay
+.wa-chat{background-color:#e5ddd5;border-radius:8px;padding:20px 15px;margin:20px 0;max-width:500px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;font-size:14px}
-
+.wa-message{margin-bottom:10px}
-*"The best way to compete with expensive software is to make it free."*
+.wa-message.user{text-align:right}
-— General Bots
+.wa-message.user .wa-bubble{background-color:#dcf8c6;display:inline-block;text-align:left}
+.wa-message.bot .wa-bubble{background-color:#fff;display:inline-block}
+.wa-bubble{padding:8px 12px;border-radius:8px;box-shadow:0 1px .5px rgba(0,0,0,.13);max-width:85%}
+.wa-bubble p{margin:0 0 4px 0;line-height:1.4;color:#303030}
+.wa-bubble p:last-child{margin-bottom:0}
+.wa-time{font-size:11px;color:#8696a0;text-align:right;margin-top:4px}
+</style>

docs/src/chapter-06-gbdialog/keyword-fill.md

@@ -1 +1,204 @@
 # FILL
+
+Populates a document template with data from variables or objects.
+
+## Syntax
+
+```basic
+result = FILL template, data
+FILL template, data TO output_path
+```
+
+## Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `template` | String | Path to template file (Word, Excel, PDF, or text) |
+| `data` | Object | Key-value pairs for placeholder replacement |
+| `output_path` | String | Optional destination path for filled document |
+
+## Description
+
+`FILL` replaces placeholders in document templates with actual data values. Placeholders use double curly braces like `{{name}}` or `{{company}}`. This is useful for generating personalized documents, contracts, invoices, and reports.
+
+## Examples
+
+### Basic Template Fill
+
+```basic
+data = #{
+    name: "John Smith",
+    company: "Acme Corp",
+    date: FORMAT(TODAY(), "MMMM d, yyyy")
+}
+
+result = FILL "templates/contract.docx", data
+TALK "Document generated: " + result.path
+```
+
+### Invoice Generation
+
+```basic
+invoice_data = #{
+    invoice_number: "INV-2025-001",
+    customer_name: customer.name,
+    customer_email: customer.email,
+    items: order_items,
+    subtotal: subtotal,
+    tax: tax_amount,
+    total: total_amount,
+    due_date: FORMAT(DATEADD("day", 30, TODAY()), "yyyy-MM-dd")
+}
+
+FILL "templates/invoice.docx", invoice_data TO "invoices/INV-2025-001.docx"
+TALK "Invoice generated and saved"
+```
+
+### Certificate Generation
+
+```basic
+certificate = #{
+    recipient: participant.name,
+    course: "AI Fundamentals",
+    completion_date: FORMAT(TODAY(), "MMMM d, yyyy"),
+    instructor: "Dr. Sarah Johnson",
+    certificate_id: GUID()
+}
+
+FILL "templates/certificate.docx", certificate TO "certificates/" + certificate.certificate_id + ".docx"
+```
+
+### Email Template
+
+```basic
+email_data = #{
+    first_name: user.first_name,
+    order_id: order.id,
+    tracking_number: shipment.tracking,
+    delivery_date: shipment.estimated_delivery
+}
+
+body = FILL "templates/shipping-notification.txt", email_data
+SEND MAIL user.email, "Your order has shipped!", body
+```
+
+## Supported Template Formats
+
+| Format | Extension | Placeholder Style |
+|--------|-----------|-------------------|
+| Word | `.docx` | `{{placeholder}}` |
+| Excel | `.xlsx` | `{{placeholder}}` |
+| Text | `.txt` | `{{placeholder}}` |
+| HTML | `.html` | `{{placeholder}}` |
+| Markdown | `.md` | `{{placeholder}}` |
+
+## Return Value
+
+Returns an object containing:
+
+| Property | Description |
+|----------|-------------|
+| `path` | Path to the generated document |
+| `content` | Document content (for text formats) |
+| `size` | File size in bytes |
+
+## Sample Conversation
+
+<div class="wa-chat">
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Generate a contract for Acme Corp</p>
+      <div class="wa-time">11:15</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>I'll create the contract. What's the contact person's name?</p>
+      <div class="wa-time">11:15</div>
+    </div>
+  </div>
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Jane Wilson</p>
+      <div class="wa-time">11:15</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>✅ Contract generated!</p>
+      <p></p>
+      <p>📄 <strong>Service Agreement</strong></p>
+      <p>• Company: Acme Corp</p>
+      <p>• Contact: Jane Wilson</p>
+      <p>• Date: May 15, 2025</p>
+      <p></p>
+      <p>The contract has been saved to your Drive.</p>
+      <p>Would you like me to email it to Jane?</p>
+      <div class="wa-time">11:15</div>
+    </div>
+  </div>
+</div>
+
+## Template Example
+
+A template file might look like:
+
+```
+SERVICE AGREEMENT
+
+This agreement is entered into on {{date}} between:
+
+Company: {{company_name}}
+Contact: {{contact_name}}
+Email: {{contact_email}}
+
+SERVICES:
+{{service_description}}
+
+TERMS:
+Duration: {{duration}} months
+Payment: ${{monthly_amount}} per month
+Start Date: {{start_date}}
+
+Signature: _____________________
+```
+
+## Advanced: Lists and Tables
+
+For repeating data, use array placeholders:
+
+```basic
+data = #{
+    customer: "Acme Corp",
+    items: [
+        #{name: "Widget", qty: 10, price: 29.99},
+        #{name: "Gadget", qty: 5, price: 49.99}
+    ],
+    total: 549.85
+}
+
+FILL "templates/order.docx", data TO "orders/order-123.docx"
+```
+
+In the template, use `{{#items}}...{{/items}}` for loops.
+
+## See Also
+
+- [GENERATE PDF](./keyword-generate-pdf.md) - Convert filled documents to PDF
+- [MERGE PDF](./keyword-merge-pdf.md) - Combine multiple documents
+- [UPLOAD](./keyword-upload.md) - Upload generated documents
+- [SEND MAIL](./keyword-send-mail.md) - Email generated documents
+
+---
+
+<style>
+.wa-chat{background-color:#e5ddd5;border-radius:8px;padding:20px 15px;margin:20px 0;max-width:500px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;font-size:14px}
+.wa-message{margin-bottom:10px}
+.wa-message.user{text-align:right}
+.wa-message.user .wa-bubble{background-color:#dcf8c6;display:inline-block;text-align:left}
+.wa-message.bot .wa-bubble{background-color:#fff;display:inline-block}
+.wa-bubble{padding:8px 12px;border-radius:8px;box-shadow:0 1px .5px rgba(0,0,0,.13);max-width:85%}
+.wa-bubble p{margin:0 0 4px 0;line-height:1.4;color:#303030}
+.wa-bubble p:last-child{margin-bottom:0}
+.wa-time{font-size:11px;color:#8696a0;text-align:right;margin-top:4px}
+</style>

docs/src/chapter-06-gbdialog/keyword-filter.md

@@ -1 +1,166 @@
 # FILTER
+
+Selects elements from an array that match a specified condition.
+
+## Syntax
+
+```basic
+result = FILTER(array, condition)
+```
+
+## Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `array` | Array | The source array to filter |
+| `condition` | String | Expression that evaluates to true/false for each element |
+
+## Description
+
+`FILTER` creates a new array containing only the elements from the input array that satisfy the given condition. The condition is evaluated for each element, and only elements where the condition is true are included in the result.
+
+## Examples
+
+### Filter by Field Value
+
+```basic
+orders = FIND "orders", "year=2025"
+large_orders = FILTER(orders, "item.total > 1000")
+
+TALK "Found " + LEN(large_orders) + " orders over $1000"
+```
+
+### Filter by String Match
+
+```basic
+contacts = FIND "contacts", "active=true"
+gmail_users = FILTER(contacts, "INSTR(item.email, 'gmail.com') > 0")
+
+FOR EACH contact IN gmail_users
+    TALK contact.name + " - " + contact.email
+NEXT
+```
+
+### Filter by Status
+
+```basic
+tasks = FIND "tasks", "assigned_to=me"
+pending = FILTER(tasks, "item.status = 'pending'")
+completed = FILTER(tasks, "item.status = 'completed'")
+
+TALK "Pending: " + LEN(pending) + ", Completed: " + LEN(completed)
+```
+
+### Filter Numbers
+
+```basic
+scores = [85, 92, 67, 78, 95, 88, 72]
+passing = FILTER(scores, "item >= 70")
+honors = FILTER(scores, "item >= 90")
+
+TALK "Passing: " + LEN(passing) + ", Honors: " + LEN(honors)
+```
+
+### Complex Conditions
+
+```basic
+products = FIND "products", "category=electronics"
+featured = FILTER(products, "item.in_stock = true AND item.rating >= 4.0")
+
+TALK "Featured products:"
+FOR EACH product IN featured
+    TALK "- " + product.name + " (★" + product.rating + ")"
+NEXT
+```
+
+## Return Value
+
+Returns a new array containing only elements where the condition evaluated to true.
+
+- Original array is not modified
+- Returns empty array if no elements match
+- Preserves order of matching elements
+
+## Sample Conversation
+
+<div class="wa-chat">
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Show me high-priority tasks</p>
+      <div class="wa-time">09:45</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>🔴 High Priority Tasks:</p>
+      <p></p>
+      <p>1. Review quarterly report (Due: Today)</p>
+      <p>2. Client presentation prep (Due: Tomorrow)</p>
+      <p>3. Budget approval meeting (Due: Friday)</p>
+      <p></p>
+      <p>You have 3 high-priority tasks.</p>
+      <div class="wa-time">09:45</div>
+    </div>
+  </div>
+</div>
+
+## Condition Operators
+
+| Operator | Description | Example |
+|----------|-------------|---------|
+| `=` | Equals | `"item.status = 'active'"` |
+| `!=` | Not equals | `"item.type != 'archived'"` |
+| `>` | Greater than | `"item.amount > 100"` |
+| `>=` | Greater or equal | `"item.score >= 70"` |
+| `<` | Less than | `"item.quantity < 10"` |
+| `<=` | Less or equal | `"item.age <= 30"` |
+| `AND` | Logical and | `"item.active = true AND item.verified = true"` |
+| `OR` | Logical or | `"item.priority = 'high' OR item.urgent = true"` |
+
+## Common Patterns
+
+### Filter then Count
+
+```basic
+users = FIND "users", "registered=true"
+premium = FILTER(users, "item.plan = 'premium'")
+TALK "Premium users: " + LEN(premium)
+```
+
+### Filter then Map
+
+```basic
+orders = FIND "orders", "status=shipped"
+recent = FILTER(orders, "item.ship_date > DATEADD('day', -7, NOW())")
+tracking = MAP(recent, "tracking_number")
+```
+
+### Chain Multiple Filters
+
+```basic
+products = FIND "products", "active=true"
+in_stock = FILTER(products, "item.quantity > 0")
+on_sale = FILTER(in_stock, "item.discount > 0")
+featured = FILTER(on_sale, "item.rating >= 4.5")
+```
+
+## See Also
+
+- [FIND](./keyword-find.md) - Retrieve data from database
+- [MAP](./keyword-map.md) - Transform filtered results
+- [FOR EACH](./keyword-for-each.md) - Iterate over filtered array
+- [AGGREGATE](./keyword-aggregate.md) - Calculate summary from filtered data
+
+---
+
+<style>
+.wa-chat{background-color:#e5ddd5;border-radius:8px;padding:20px 15px;margin:20px 0;max-width:500px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;font-size:14px}
+.wa-message{margin-bottom:10px}
+.wa-message.user{text-align:right}
+.wa-message.user .wa-bubble{background-color:#dcf8c6;display:inline-block;text-align:left}
+.wa-message.bot .wa-bubble{background-color:#fff;display:inline-block}
+.wa-bubble{padding:8px 12px;border-radius:8px;box-shadow:0 1px .5px rgba(0,0,0,.13);max-width:85%}
+.wa-bubble p{margin:0 0 4px 0;line-height:1.4;color:#303030}
+.wa-bubble p:last-child{margin-bottom:0}
+.wa-time{font-size:11px;color:#8696a0;text-align:right;margin-top:4px}
+</style>

docs/src/chapter-06-gbdialog/keyword-group-by.md

@@ -1 +1,130 @@
 # GROUP BY
+
+Groups data by specified columns and optionally applies aggregate functions.
+
+## Syntax
+
+```basic
+result = GROUP BY data, column
+result = GROUP BY data, column, aggregates
+```
+
+## Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `data` | Array | The data array to group |
+| `column` | String | Column name to group by |
+| `aggregates` | Object | Optional aggregate functions to apply |
+
+## Description
+
+`GROUP BY` organizes rows of data into groups based on matching values in a specified column. When combined with aggregate functions, it calculates summary values for each group.
+
+## Examples
+
+### Basic Grouping
+
+```basic
+orders = FIND "orders", "status=completed"
+grouped = GROUP BY orders, "category"
+
+FOR EACH group IN grouped
+    TALK "Category: " + group.key + " - Count: " + group.count
+NEXT
+```
+
+### With Aggregates
+
+```basic
+sales = FIND "sales", "year=2025"
+summary = GROUP BY sales, "region", #{
+    total: "SUM(amount)",
+    average: "AVG(amount)",
+    count: "COUNT(*)"
+}
+
+FOR EACH region IN summary
+    TALK region.key + ": $" + region.total + " (" + region.count + " sales)"
+NEXT
+```
+
+### Multiple Level Grouping
+
+```basic
+' First group by category, then by month
+products = FIND "orders", "year=2025"
+by_category = GROUP BY products, "category"
+
+FOR EACH cat IN by_category
+    TALK "Category: " + cat.key
+    by_month = GROUP BY cat.items, "month"
+    FOR EACH month IN by_month
+        TALK "  " + month.key + ": " + month.count + " orders"
+    NEXT
+NEXT
+```
+
+## Return Value
+
+Returns an array of group objects, each containing:
+
+| Property | Description |
+|----------|-------------|
+| `key` | The grouping value |
+| `items` | Array of items in this group |
+| `count` | Number of items in group |
+| Additional | Any requested aggregates |
+
+## Supported Aggregates
+
+| Function | Description |
+|----------|-------------|
+| `SUM(column)` | Sum of values |
+| `AVG(column)` | Average of values |
+| `MIN(column)` | Minimum value |
+| `MAX(column)` | Maximum value |
+| `COUNT(*)` | Number of rows |
+
+## Sample Conversation
+
+<div class="wa-chat">
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Show me sales by region</p>
+      <div class="wa-time">14:30</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>📊 Sales by Region (2025)</p>
+      <p></p>
+      <p><strong>North America:</strong> $245,000 (127 sales)</p>
+      <p><strong>Europe:</strong> $189,500 (98 sales)</p>
+      <p><strong>Asia Pacific:</strong> $156,200 (84 sales)</p>
+      <p><strong>Latin America:</strong> $67,300 (42 sales)</p>
+      <div class="wa-time">14:30</div>
+    </div>
+  </div>
+</div>
+
+## See Also
+
+- [AGGREGATE](./keyword-aggregate.md) - Single aggregate calculations
+- [PIVOT](./keyword-pivot.md) - Cross-tabulation of data
+- [FILTER](./keyword-filter.md) - Filter data before grouping
+- [FIND](./keyword-find.md) - Retrieve data to group
+
+---
+
+<style>
+.wa-chat{background-color:#e5ddd5;border-radius:8px;padding:20px 15px;margin:20px 0;max-width:500px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;font-size:14px}
+.wa-message{margin-bottom:10px}
+.wa-message.user{text-align:right}
+.wa-message.user .wa-bubble{background-color:#dcf8c6;display:inline-block;text-align:left}
+.wa-message.bot .wa-bubble{background-color:#fff;display:inline-block}
+.wa-bubble{padding:8px 12px;border-radius:8px;box-shadow:0 1px .5px rgba(0,0,0,.13);max-width:85%}
+.wa-bubble p{margin:0 0 4px 0;line-height:1.4;color:#303030}
+.wa-bubble p:last-child{margin-bottom:0}
+.wa-time{font-size:11px;color:#8696a0;text-align:right;margin-top:4px}
+</style>

docs/src/chapter-06-gbdialog/keyword-join.md

@@ -1 +1,132 @@
 # JOIN
+
+Combines elements of an array into a single string with a specified separator.
+
+## Syntax
+
+```basic
+result = JOIN(array, separator)
+```
+
+## Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `array` | Array | The array of elements to join |
+| `separator` | String | The delimiter to place between elements |
+
+## Description
+
+`JOIN` concatenates all elements of an array into a single string, inserting the specified separator between each element. This is commonly used for creating comma-separated lists, building display strings, or formatting data for output.
+
+## Examples
+
+### Basic Usage
+
+```basic
+names = ["Alice", "Bob", "Charlie"]
+result = JOIN(names, ", ")
+TALK result
+' Output: "Alice, Bob, Charlie"
+```
+
+### Creating Hashtag Lists
+
+```basic
+tags = ["GeneralBots", "AI", "Automation", "NoCode"]
+hashtags = JOIN(tags, " #")
+hashtags = "#" + hashtags
+TALK hashtags
+' Output: "#GeneralBots #AI #Automation #NoCode"
+```
+
+### Building File Paths
+
+```basic
+parts = ["documents", "reports", "2025", "sales.pdf"]
+path = JOIN(parts, "/")
+TALK "File: " + path
+' Output: "File: documents/reports/2025/sales.pdf"
+```
+
+### Email Recipients
+
+```basic
+recipients = ["john@example.com", "jane@example.com", "bob@example.com"]
+to_list = JOIN(recipients, "; ")
+SEND MAIL to_list, "Team Update", "Please review the attached report."
+```
+
+### Display Lists
+
+```basic
+items = FIND "products", "category=electronics"
+product_names = []
+
+FOR EACH item IN items
+    product_names = APPEND(product_names, item.name)
+NEXT
+
+TALK "Available products: " + JOIN(product_names, ", ")
+```
+
+## Return Value
+
+Returns a string containing all array elements concatenated with the separator.
+
+- If the array is empty, returns an empty string
+- If the array has one element, returns that element as a string
+- Null values in the array are converted to empty strings
+
+## Sample Conversation
+
+<div class="wa-chat">
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>What hashtags should I use for my post about AI automation?</p>
+      <div class="wa-time">15:20</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>Here are some great hashtags for your AI automation post:</p>
+      <p></p>
+      <p>#AI #Automation #MachineLearning #NoCode #Productivity #TechInnovation #DigitalTransformation</p>
+      <p></p>
+      <p>Copy and paste them directly into your post!</p>
+      <div class="wa-time">15:20</div>
+    </div>
+  </div>
+</div>
+
+## Common Separators
+
+| Separator | Use Case |
+|-----------|----------|
+| `", "` | Readable comma-separated lists |
+| `","` | CSV data |
+| `"\n"` | Multi-line output |
+| `" "` | Space-separated words |
+| `" \| "` | Table columns |
+| `"/"` | File paths |
+| `"; "` | Email recipients |
+
+## See Also
+
+- [SPLIT](./keyword-split.md) - Split a string into an array (opposite of JOIN)
+- [FOR EACH](./keyword-for-each.md) - Iterate over arrays
+- [FILTER](./keyword-filter.md) - Filter arrays before joining
+
+---
+
+<style>
+.wa-chat{background-color:#e5ddd5;border-radius:8px;padding:20px 15px;margin:20px 0;max-width:500px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;font-size:14px}
+.wa-message{margin-bottom:10px}
+.wa-message.user{text-align:right}
+.wa-message.user .wa-bubble{background-color:#dcf8c6;display:inline-block;text-align:left}
+.wa-message.bot .wa-bubble{background-color:#fff;display:inline-block}
+.wa-bubble{padding:8px 12px;border-radius:8px;box-shadow:0 1px .5px rgba(0,0,0,.13);max-width:85%}
+.wa-bubble p{margin:0 0 4px 0;line-height:1.4;color:#303030}
+.wa-bubble p:last-child{margin-bottom:0}
+.wa-time{font-size:11px;color:#8696a0;text-align:right;margin-top:4px}
+</style>

docs/src/chapter-06-gbdialog/keyword-map.md

@@ -1 +1,151 @@
 # MAP
+
+Transforms each element of an array by applying a function or expression.
+
+## Syntax
+
+```basic
+result = MAP(array, expression)
+result = MAP(array, field)
+```
+
+## Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `array` | Array | The source array to transform |
+| `expression` | String | Expression to apply to each element, or field name to extract |
+
+## Description
+
+`MAP` creates a new array by applying a transformation to each element of the input array. This is useful for extracting specific fields from objects, formatting data, or performing calculations on each item.
+
+## Examples
+
+### Extract Field from Objects
+
+```basic
+users = FIND "users", "status=active"
+names = MAP(users, "name")
+
+TALK "Active users: " + JOIN(names, ", ")
+```
+
+### Transform Values
+
+```basic
+prices = [100, 200, 300, 400]
+with_tax = MAP(prices, "item * 1.1")
+
+FOR EACH price IN with_tax
+    TALK "Price with tax: $" + price
+NEXT
+```
+
+### Format Data
+
+```basic
+orders = FIND "orders", "date=today"
+summaries = MAP(orders, "'Order #' + item.id + ': $' + item.total")
+
+FOR EACH summary IN summaries
+    TALK summary
+NEXT
+```
+
+### Extract Nested Properties
+
+```basic
+contacts = FIND "contacts", "company=Acme"
+emails = MAP(contacts, "email")
+
+email_list = JOIN(emails, "; ")
+TALK "Emails: " + email_list
+```
+
+### Uppercase Names
+
+```basic
+products = ["widget", "gadget", "gizmo"]
+upper_products = MAP(products, "UPPER(item)")
+
+TALK JOIN(upper_products, ", ")
+' Output: "WIDGET, GADGET, GIZMO"
+```
+
+## Return Value
+
+Returns a new array with the same length as the input, containing transformed values.
+
+- Original array is not modified
+- Null values in the source are preserved as null
+- If transformation fails for an element, that element becomes null
+
+## Sample Conversation
+
+<div class="wa-chat">
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>List all customer emails</p>
+      <div class="wa-time">11:30</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>📧 Customer Emails:</p>
+      <p></p>
+      <p>• john@example.com</p>
+      <p>• sarah@company.co</p>
+      <p>• mike@business.org</p>
+      <p>• lisa@startup.io</p>
+      <p></p>
+      <p>Found 4 customer emails.</p>
+      <div class="wa-time">11:30</div>
+    </div>
+  </div>
+</div>
+
+## Common Patterns
+
+### Extract IDs for API Calls
+
+```basic
+records = FIND "items", "sync=pending"
+ids = MAP(records, "id")
+' Use ids for batch API operations
+```
+
+### Create Display Labels
+
+```basic
+products = FIND "products", "in_stock=true"
+labels = MAP(products, "item.name + ' ($' + item.price + ')'")
+```
+
+### Calculate Derived Values
+
+```basic
+line_items = FIND "cart_items", "cart_id=123"
+totals = MAP(line_items, "item.quantity * item.unit_price")
+```
+
+## See Also
+
+- [FILTER](./keyword-filter.md) - Filter array elements
+- [FOR EACH](./keyword-for-each.md) - Iterate with more control
+- [JOIN](./keyword-join.md) - Combine mapped results into string
+- [AGGREGATE](./keyword-aggregate.md) - Calculate summary from mapped values
+
+---
+
+<style>
+.wa-chat{background-color:#e5ddd5;border-radius:8px;padding:20px 15px;margin:20px 0;max-width:500px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;font-size:14px}
+.wa-message{margin-bottom:10px}
+.wa-message.user{text-align:right}
+.wa-message.user .wa-bubble{background-color:#dcf8c6;display:inline-block;text-align:left}
+.wa-message.bot .wa-bubble{background-color:#fff;display:inline-block}
+.wa-bubble{padding:8px 12px;border-radius:8px;box-shadow:0 1px .5px rgba(0,0,0,.13);max-width:85%}
+.wa-bubble p{margin:0 0 4px 0;line-height:1.4;color:#303030}
+.wa-bubble p:last-child{margin-bottom:0}
+.wa-time{font-size:11px;color:#8696a0;text-align:right;margin-top:4px}
+</style>

docs/src/chapter-06-gbdialog/keyword-merge.md

@@ -1 +1,198 @@
 # MERGE
+
+Combines data from multiple sources or upserts records into a database table.
+
+## Syntax
+
+```basic
+MERGE table, data, key_column
+MERGE table, data, key_columns, update_columns
+```
+
+## Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `table` | String | Target database table name |
+| `data` | Array/Object | Data to merge (single record or array of records) |
+| `key_column` | String | Column(s) to match existing records |
+| `update_columns` | Array | Optional specific columns to update on match |
+
+## Description
+
+`MERGE` performs an "upsert" operation: it inserts new records or updates existing ones based on matching key columns. This is useful for synchronizing data from external sources, importing bulk data, or maintaining data consistency.
+
+## Examples
+
+### Basic Merge (Single Record)
+
+```basic
+contact = #{
+    email: "john@example.com",
+    name: "John Smith",
+    phone: "+1-555-0123"
+}
+
+MERGE "contacts", contact, "email"
+TALK "Contact merged successfully"
+```
+
+### Bulk Merge
+
+```basic
+new_products = GET "https://api.supplier.com/products"
+
+MERGE "products", new_products, "sku"
+TALK "Merged " + LEN(new_products) + " products"
+```
+
+### Merge with Specific Update Columns
+
+```basic
+price_updates = [
+    #{sku: "ABC123", price: 29.99},
+    #{sku: "DEF456", price: 49.99},
+    #{sku: "GHI789", price: 19.99}
+]
+
+MERGE "products", price_updates, "sku", ["price"]
+TALK "Prices updated"
+```
+
+### Composite Key Match
+
+```basic
+attendance = #{
+    employee_id: "EMP001",
+    date: TODAY(),
+    status: "present",
+    check_in: NOW()
+}
+
+MERGE "attendance", attendance, "employee_id,date"
+```
+
+### Sync from External API
+
+```basic
+SET SCHEDULE "every 6 hours"
+
+' Fetch latest data from CRM
+customers = GET "https://crm.example.com/api/customers"
+
+' Merge into local database
+MERGE "customers", customers, "crm_id"
+
+TALK "Synced " + LEN(customers) + " customer records"
+```
+
+## Return Value
+
+Returns an object with merge statistics:
+
+| Property | Description |
+|----------|-------------|
+| `inserted` | Number of new records created |
+| `updated` | Number of existing records updated |
+| `unchanged` | Number of records that matched but had no changes |
+| `total` | Total records processed |
+
+```basic
+result = MERGE "products", data, "sku"
+TALK "Inserted: " + result.inserted + ", Updated: " + result.updated
+```
+
+## Sample Conversation
+
+<div class="wa-chat">
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Sync customer data from the CRM</p>
+      <div class="wa-time">14:00</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>🔄 Syncing customer data...</p>
+      <div class="wa-time">14:00</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>✅ Customer sync complete!</p>
+      <p></p>
+      <p>📊 Results:</p>
+      <p>• New customers: 12</p>
+      <p>• Updated records: 45</p>
+      <p>• Unchanged: 238</p>
+      <p>• Total processed: 295</p>
+      <div class="wa-time">14:01</div>
+    </div>
+  </div>
+</div>
+
+## Behavior
+
+### On Match (Key Exists)
+- Updates all columns in the data (or only `update_columns` if specified)
+- Preserves columns not in the data
+- Updates `updated_at` timestamp if column exists
+
+### On No Match (New Record)
+- Inserts new row with all provided columns
+- Sets `created_at` timestamp if column exists
+
+## Common Patterns
+
+### Daily Data Import
+
+```basic
+SET SCHEDULE "every day at 2am"
+
+data = GET "https://data.provider.com/daily-export"
+result = MERGE "imported_data", data, "external_id"
+
+IF result.inserted > 0 THEN
+    SEND MAIL "admin@company.com", "Data Import", 
+        "Imported " + result.inserted + " new records"
+END IF
+```
+
+### Inventory Sync
+
+```basic
+inventory = GET "https://warehouse.api/stock-levels"
+MERGE "products", inventory, "sku", ["quantity", "last_restock"]
+```
+
+### User Profile Updates
+
+```basic
+profile = #{
+    user_id: current_user_id,
+    preferences: user_preferences,
+    last_active: NOW()
+}
+MERGE "user_profiles", profile, "user_id"
+```
+
+## See Also
+
+- [INSERT](./keyword-insert.md) - Insert new records only
+- [UPDATE](./keyword-update.md) - Update existing records only
+- [SAVE](./keyword-save.md) - Simple data persistence
+- [FIND](./keyword-find.md) - Query data before merging
+
+---
+
+<style>
+.wa-chat{background-color:#e5ddd5;border-radius:8px;padding:20px 15px;margin:20px 0;max-width:500px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;font-size:14px}
+.wa-message{margin-bottom:10px}
+.wa-message.user{text-align:right}
+.wa-message.user .wa-bubble{background-color:#dcf8c6;display:inline-block;text-align:left}
+.wa-message.bot .wa-bubble{background-color:#fff;display:inline-block}
+.wa-bubble{padding:8px 12px;border-radius:8px;box-shadow:0 1px .5px rgba(0,0,0,.13);max-width:85%}
+.wa-bubble p{margin:0 0 4px 0;line-height:1.4;color:#303030}
+.wa-bubble p:last-child{margin-bottom:0}
+.wa-time{font-size:11px;color:#8696a0;text-align:right;margin-top:4px}
+</style>

docs/src/chapter-06-gbdialog/keyword-pivot.md

@@ -1 +1,121 @@
 # PIVOT
+
+Transforms rows into columns, creating a cross-tabulation summary of data.
+
+## Syntax
+
+```basic
+result = PIVOT data, row_column, column_column, value_column
+result = PIVOT data, row_column, column_column, value_column, aggregate
+```
+
+## Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `data` | Array | The data array to pivot |
+| `row_column` | String | Column to use for row headers |
+| `column_column` | String | Column to use for column headers |
+| `value_column` | String | Column containing values to aggregate |
+| `aggregate` | String | Aggregate function: SUM, AVG, COUNT, MIN, MAX (default: SUM) |
+
+## Description
+
+`PIVOT` reorganizes data from a normalized format into a cross-tabulation format, making it easier to compare values across two dimensions. This is useful for creating summary reports and dashboards.
+
+## Examples
+
+### Basic Pivot
+
+```basic
+sales = FIND "sales", "year=2025"
+summary = PIVOT sales, "region", "quarter", "amount"
+
+' Result: regions as rows, quarters as columns
+' Shows total sales for each region/quarter combination
+```
+
+### With Aggregate Function
+
+```basic
+orders = FIND "orders", "status=completed"
+avg_order = PIVOT orders, "product", "month", "total", "AVG"
+
+FOR EACH row IN avg_order
+    TALK row.product + ": Q1=$" + row.Q1 + ", Q2=$" + row.Q2
+NEXT
+```
+
+### Sales by Region and Product
+
+```basic
+data = FIND "sales", "year=2025"
+pivot_table = PIVOT data, "region", "product", "revenue", "SUM"
+
+TALK "Revenue by Region and Product:"
+FOR EACH region IN pivot_table
+    TALK region.row_header + ":"
+    TALK "  Widgets: $" + region.Widgets
+    TALK "  Gadgets: $" + region.Gadgets
+NEXT
+```
+
+## Return Value
+
+Returns an array of objects where:
+- Each object represents a row
+- `row_header` contains the row label
+- Dynamic properties contain pivoted column values
+
+## Sample Conversation
+
+<div class="wa-chat">
+  <div class="wa-message user">
+    <div class="wa-bubble">
+      <p>Show quarterly sales by region</p>
+      <div class="wa-time">10:15</div>
+    </div>
+  </div>
+  <div class="wa-message bot">
+    <div class="wa-bubble">
+      <p>📊 Quarterly Sales Report 2025</p>
+      <p></p>
+      <p><strong>Region</strong> | Q1 | Q2 | Q3 | Q4</p>
+      <p>━━━━━━━━━━━━━━━━━━━━━━━━</p>
+      <p>North | $125K | $142K | $138K | $167K</p>
+      <p>South | $98K | $105K | $112K | $128K</p>
+      <p>East | $87K | $92K | $95K | $103K</p>
+      <p>West | $156K | $168K | $175K | $189K</p>
+      <div class="wa-time">10:15</div>
+    </div>
+  </div>
+</div>
+
+## Use Cases
+
+| Scenario | Row | Column | Value |
+|----------|-----|--------|-------|
+| Sales dashboard | Region | Quarter | Revenue |
+| Attendance report | Employee | Month | Days |
+| Product comparison | Product | Store | Units sold |
+| Time tracking | Project | Week | Hours |
+
+## See Also
+
+- [GROUP BY](./keyword-group-by.md) - Group data by columns
+- [AGGREGATE](./keyword-aggregate.md) - Calculate summary values
+- [TABLE](./keyword-table.md) - Display formatted tables
+
+---
+
+<style>
+.wa-chat{background-color:#e5ddd5;border-radius:8px;padding:20px 15px;margin:20px 0;max-width:500px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;font-size:14px}
+.wa-message{margin-bottom:10px}
+.wa-message.user{text-align:right}
+.wa-message.user .wa-bubble{background-color:#dcf8c6;display:inline-block;text-align:left}
+.wa-message.bot .wa-bubble{background-color:#fff;display:inline-block}
+.wa-bubble{padding:8px 12px;border-radius:8px;box-shadow:0 1px .5px rgba(0,0,0,.13);max-width:85%}
+.wa-bubble p{margin:0 0 4px 0;line-height:1.4;color:#303030}
+.wa-bubble p:last-child{margin-bottom:0}
+.wa-time{font-size:11px;color:#8696a0;text-align:right;margin-top:4px}
+</style>

docs/src/chapter-07-gbapp/observability.md

@@ -1,367 +1,95 @@
 # Observability
 
-General Bots uses a comprehensive observability stack for monitoring, logging, and metrics collection. This chapter explains how logging works and how Vector integrates without requiring code changes.
+General Bots provides built-in observability for monitoring, logging, and metrics collection.
 
-## Architecture Overview
+## Overview
 
-![Observability Flow](../assets/observability-flow.svg)
+The observability system works automatically without code changes:
 
-*Vector Agent collects logs from BotServer without requiring any code changes.*
+- **Log Collection** - Reads from `./botserver-stack/logs/`
+- **Log Parsing** - Extracts level, timestamp, message
+- **Routing** - Sends errors to alerts, metrics to storage
+- **Enrichment** - Adds hostname, service name, etc.
 
-## No Code Changes Required
+## Log Directory Structure
 
-**You do NOT need to replace `log::trace!()`, `log::info!()`, `log::error!()` calls.**
+| Directory | Contents |
+|-----------|----------|
+| `logs/system/` | BotServer application logs |
+| `logs/drive/` | Storage service logs |
+| `logs/tables/` | PostgreSQL logs |
+| `logs/cache/` | Cache component logs |
+| `logs/llm/` | LLM server logs |
+| `logs/email/` | Email service logs |
+| `logs/directory/` | Identity service logs |
+| `logs/vectordb/` | Vector database logs |
+| `logs/meet/` | Video meeting logs |
 
-Vector works by:
+## Installation
 
-1. **Tailing log files** - Reads from `./botserver-stack/logs/`
+The observability component is installed automatically during bootstrap, or manually:
-2. **Parsing log lines** - Extracts level, timestamp, message
-3. **Routing by level** - Sends errors to alerts, metrics to InfluxDB
-4. **Enriching data** - Adds hostname, service name, etc.
-
-Log directory structure:
-- `logs/system/` - BotServer application logs
-- `logs/drive/` - MinIO logs
-- `logs/tables/` - PostgreSQL logs
-- `logs/cache/` - Redis logs
-- `logs/llm/` - LLM server logs
-- `logs/email/` - Stalwart logs
-- `logs/directory/` - Zitadel logs
-- `logs/vectordb/` - Qdrant logs
-- `logs/meet/` - LiveKit logs
-- `logs/alm/` - Forgejo logs
-
-This approach:
-- Requires zero code changes
-- Works with existing logging
-- Can be added/removed without recompilation
-- Scales independently from the application
-
-## Vector Configuration
-
-### Installation
-
-Vector is installed as the **observability** component:
 
 ```bash
 ./botserver install observability
 ```
 
-### Configuration File
+Configuration is at `./botserver-stack/conf/monitoring/vector.toml`.
-
-Configuration is at `./botserver-stack/conf/monitoring/vector.toml`:
-
-```toml
-# Vector Configuration for General Bots
-# Collects logs without requiring code changes
-# Component: observability (Vector)
-# Config: ./botserver-stack/conf/monitoring/vector.toml
-
-#
-# SOURCES - Where logs come from
-#
-
-[sources.botserver_logs]
-type = "file"
-include = ["./botserver-stack/logs/system/*.log"]
-read_from = "beginning"
-
-[sources.drive_logs]
-type = "file"
-include = ["./botserver-stack/logs/drive/*.log"]
-read_from = "beginning"
-
-[sources.tables_logs]
-type = "file"
-include = ["./botserver-stack/logs/tables/*.log"]
-read_from = "beginning"
-
-[sources.cache_logs]
-type = "file"
-include = ["./botserver-stack/logs/cache/*.log"]
-read_from = "beginning"
-
-[sources.llm_logs]
-type = "file"
-include = ["./botserver-stack/logs/llm/*.log"]
-read_from = "beginning"
-
-[sources.service_logs]
-type = "file"
-include = [
-  "./botserver-stack/logs/email/*.log",
-  "./botserver-stack/logs/directory/*.log",
-  "./botserver-stack/logs/vectordb/*.log",
-  "./botserver-stack/logs/meet/*.log",
-  "./botserver-stack/logs/alm/*.log"
-]
-read_from = "beginning"
-
-#
-# TRANSFORMS - Parse and enrich logs
-#
-
-[transforms.parse_botserver]
-type = "remap"
-inputs = ["botserver_logs"]
-source = '''
-# Parse standard log format: [TIMESTAMP LEVEL target] message
-. = parse_regex!(.message, r'^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)\s+(?P<level>\w+)\s+(?P<target>\S+)\s+(?P<message>.*)$')
-
-# Convert timestamp
-.timestamp = parse_timestamp!(.timestamp, "%Y-%m-%dT%H:%M:%S%.fZ")
-
-# Normalize level
-.level = downcase!(.level)
-
-# Add service name
-.service = "botserver"
-
-# Extract session_id if present
-if contains(string!(.message), "session") {
-  session_match = parse_regex(.message, r'session[:\s]+(?P<session_id>[a-f0-9-]+)') ?? {}
-  if exists(session_match.session_id) {
-    .session_id = session_match.session_id
-  }
-}
-
-# Extract user_id if present
-if contains(string!(.message), "user") {
-  user_match = parse_regex(.message, r'user[:\s]+(?P<user_id>[a-f0-9-]+)') ?? {}
-  if exists(user_match.user_id) {
-    .user_id = user_match.user_id
-  }
-}
-'''
-
-[transforms.parse_service_logs]
-type = "remap"
-inputs = ["service_logs"]
-source = '''
-# Basic parsing for service logs
-.timestamp = now()
-.level = "info"
-
-# Detect errors
-if contains(string!(.message), "ERROR") || contains(string!(.message), "error") {
-  .level = "error"
-}
-if contains(string!(.message), "WARN") || contains(string!(.message), "warn") {
-  .level = "warn"
-}
-
-# Extract service name from file path
-.service = replace(string!(.file), r'.*/(\w+)\.log$', "$1")
-'''
-
-#
-# FILTERS - Route by log level
-#
-
-[transforms.filter_errors]
-type = "filter"
-inputs = ["parse_botserver", "parse_service_logs"]
-condition = '.level == "error"'
-
-[transforms.filter_warnings]
-type = "filter"
-inputs = ["parse_botserver", "parse_service_logs"]
-condition = '.level == "warn"'
-
-[transforms.filter_info]
-type = "filter"
-inputs = ["parse_botserver"]
-condition = '.level == "info" || .level == "debug"'
-
-#
-# METRICS - Convert logs to metrics
-#
-
-[transforms.log_to_metrics]
-type = "log_to_metric"
-inputs = ["parse_botserver"]
-
-[[transforms.log_to_metrics.metrics]]
-type = "counter"
-field = "level"
-name = "log_events_total"
-tags.level = "{{level}}"
-tags.service = "{{service}}"
-
-[[transforms.log_to_metrics.metrics]]
-type = "counter"
-field = "message"
-name = "errors_total"
-tags.service = "{{service}}"
-increment_by_value = false
-
-#
-# SINKS - Where logs go
-#
-
-# All logs to file (backup)
-[sinks.file_backup]
-type = "file"
-inputs = ["parse_botserver", "parse_service_logs"]
-path = "./botserver-stack/logs/vector/all-%Y-%m-%d.log"
-encoding.codec = "json"
-
-# Metrics to InfluxDB
-[sinks.influxdb]
-type = "influxdb_metrics"
-inputs = ["log_to_metrics"]
-endpoint = "http://localhost:8086"
-org = "pragmatismo"
-bucket = "metrics"
-token = "${INFLUXDB_TOKEN}"
-
-# Errors to alerting (webhook)
-[sinks.alert_webhook]
-type = "http"
-inputs = ["filter_errors"]
-uri = "http://localhost:8080/api/admin/alerts"
-method = "post"
-encoding.codec = "json"
-
-# Console output (for debugging)
-[sinks.console]
-type = "console"
-inputs = ["filter_errors"]
-encoding.codec = "text"
-```
 
 ## Log Format
 
-BotServer uses the standard Rust `log` crate format:
+BotServer uses the standard Rust log format:
 
 ```
 2024-01-15T10:30:45.123Z INFO botserver::core::bot Processing message for session: abc-123
-2024-01-15T10:30:45.456Z DEBUG botserver::llm::cache Cache hit for prompt hash: xyz789
 2024-01-15T10:30:45.789Z ERROR botserver::drive::upload Failed to upload file: permission denied
 ```
 
-Vector parses this automatically without code changes.
+Logs are parsed and routed automatically.
 
-## Metrics Collection
+## Metrics
 
-### Automatic Metrics
+### Available Metrics
-
-Vector converts log events to metrics:
 
 | Metric | Description |
 |--------|-------------|
 | `log_events_total` | Total log events by level |
 | `errors_total` | Error count by service |
-| `warnings_total` | Warning count by service |
+| `botserver_sessions_active` | Current active sessions |
+| `botserver_messages_total` | Total messages processed |
+| `botserver_llm_latency_seconds` | LLM response latency |
 
-### Application Metrics
+### Metrics Endpoint
 
-BotServer also exposes metrics via `/api/metrics` (Prometheus format):
+BotServer exposes Prometheus-compatible metrics at `/api/metrics`.
-
-```
-# HELP botserver_sessions_active Current active sessions
-# TYPE botserver_sessions_active gauge
-botserver_sessions_active 42
-
-# HELP botserver_messages_total Total messages processed
-# TYPE botserver_messages_total counter
-botserver_messages_total{channel="web"} 1234
-botserver_messages_total{channel="whatsapp"} 567
-
-# HELP botserver_llm_latency_seconds LLM response latency
-# TYPE botserver_llm_latency_seconds histogram
-botserver_llm_latency_seconds_bucket{le="0.5"} 100
-botserver_llm_latency_seconds_bucket{le="1.0"} 150
-botserver_llm_latency_seconds_bucket{le="2.0"} 180
-```
-
-Vector can scrape these directly:
-
-```toml
-[sources.prometheus_metrics]
-type = "prometheus_scrape"
-endpoints = ["http://localhost:8080/api/metrics"]
-scrape_interval_secs = 15
-```
 
 ## Alerting
 
-### Error Alerts
+Alerts are sent automatically when errors occur:
 
-Vector sends errors to a webhook for alerting:
+- **Webhook** - POST to `/api/admin/alerts`
+- **Slack** - Configure webhook URL in settings
+- **Email** - Configure SMTP in `config.csv`
 
-```toml
+Configure alert thresholds in `config.csv`:
-[sinks.alert_webhook]
-type = "http"
-inputs = ["filter_errors"]
-uri = "http://localhost:8080/api/admin/alerts"
-method = "post"
-encoding.codec = "json"
-```
-
-### Slack Integration
-
-```toml
-[sinks.slack_alerts]
-type = "http"
-inputs = ["filter_errors"]
-uri = "https://hooks.slack.com/services/YOUR/WEBHOOK/URL"
-method = "post"
-encoding.codec = "json"
-
-[sinks.slack_alerts.request]
-headers.content-type = "application/json"
-
-[sinks.slack_alerts.encoding]
-codec = "json"
-```
-
-### Email Alerts
-
-Use with an SMTP relay or webhook-to-email service:
-
-```toml
-[sinks.email_alerts]
-type = "http"
-inputs = ["filter_errors"]
-uri = "http://localhost:8025/api/send"
-method = "post"
-encoding.codec = "json"
-```
-
-## Grafana Dashboards
-
-### Pre-built Dashboard
-
-Import the General Bots dashboard from `templates/grafana-dashboard.json`:
-
-1. Open Grafana at `http://localhost:3000`
-2. Go to Dashboards → Import
-3. Upload `grafana-dashboard.json`
-4. Select InfluxDB data source
-
-### Key Panels
-
-| Panel | Query |
-|-------|-------|
-| Active Sessions | `from(bucket:"metrics") \|> filter(fn: (r) => r._measurement == "sessions_active")` |
-| Messages/Minute | `from(bucket:"metrics") \|> filter(fn: (r) => r._measurement == "messages_total") \|> derivative()` |
-| Error Rate | `from(bucket:"metrics") \|> filter(fn: (r) => r.level == "error") \|> count()` |
-| LLM Latency P95 | `from(bucket:"metrics") \|> filter(fn: (r) => r._measurement == "llm_latency") \|> quantile(q: 0.95)` |
-
-## Configuration Options
-
-### config.csv Settings
 
 ```csv
-# Observability settings
+name,value
-observability-enabled,true
+alert-cpu-threshold,80
-observability-log-level,info
+alert-memory-threshold,85
-observability-metrics-endpoint,/api/metrics
+alert-response-time-ms,5000
-observability-vector-enabled,true
 ```
 
-### Log Levels
+## Dashboards
+
+A pre-built Grafana dashboard is available at `templates/grafana-dashboard.json`.
+
+Key panels include:
+- Active Sessions
+- Messages per Minute
+- Error Rate
+- LLM Latency (P95)
+
+## Log Levels
 
 | Level | When to Use |
 |-------|-------------|
@@ -369,116 +97,37 @@ observability-vector-enabled,true
 | `warn` | Unexpected but handled, worth noting |
 | `info` | Normal operations, key events |
 | `debug` | Detailed flow, development |
-| `trace` | Very detailed, performance impact |
 
-Set in config.csv:
+Set in `config.csv`:
 
 ```csv
+name,value
 log-level,info
 ```
 
-Or environment:
-
-```bash
-RUST_LOG=info ./botserver
-```
-
 ## Troubleshooting
 
-### Vector Not Collecting Logs
+### Logs Not Collecting
 
-```bash
+1. Check observability service is running
-# Check Vector status
+2. Verify log directory permissions
-systemctl status gbo-observability
+3. Review service logs for errors
-
-# View Vector logs
-journalctl -u gbo-observability -f
-
-# Test configuration
-vector validate ./botserver-stack/conf/monitoring/vector.toml
-```
-
-### Missing Metrics in InfluxDB
-
-```bash
-# Check InfluxDB connection
-curl http://localhost:8086/health
-
-# Verify bucket exists
-influx bucket list
-
-# Check Vector sink status
-vector top
-```
 
 ### High Log Volume
 
-If logs are too verbose:
+1. Increase log level in `config.csv`
-
+2. Set retention policies in metrics storage
-1. Increase log level in config.csv
+3. Filter debug logs in production
-2. Add filters in Vector to drop debug logs
-3. Set retention policies in InfluxDB
-
-```toml
-# Drop debug logs before sending to InfluxDB
-[transforms.drop_debug]
-type = "filter"
-inputs = ["parse_botserver"]
-condition = '.level != "debug" && .level != "trace"'
-```
 
 ## Best Practices
 
-### 1. Don't Log Sensitive Data
+1. **Don't log sensitive data** - Never log passwords or tokens
+2. **Use appropriate levels** - Errors for failures, info for operations
+3. **Monitor trends** - Watch for gradual increases, not just spikes
+4. **Set up alerts early** - Don't wait for problems
 
-```rust
+## See Also
-// Bad
-log::info!("User password: {}", password);
 
-// Good
+- [Scaling and Load Balancing](./scaling.md) - Scale with your cluster
-log::info!("User {} authenticated successfully", user_id);
-```
-
-### 2. Use Structured Context
-
-```rust
-// Better for parsing
-log::info!("session={} user={} action=message_sent", session_id, user_id);
-```
-
-### 3. Set Appropriate Levels
-
-```rust
-// Errors: things that failed
-log::error!("Database connection failed: {}", err);
-
-// Warnings: unusual but handled
-log::warn!("Retrying LLM request after timeout");
-
-// Info: normal operations
-log::info!("Session {} started", session_id);
-
-// Debug: development details
-log::debug!("Cache lookup for key: {}", key);
-
-// Trace: very detailed
-log::trace!("Entering function process_message");
-```
-
-### 4. Keep Vector Config Simple
-
-Start with basic collection, add transforms as needed.
-
-## Summary
-
-- **No code changes needed** - Vector collects from log files
-- **Keep using log macros** - `log::info!()`, `log::error!()`, etc.
-- **Vector handles routing** - Errors to alerts, all to storage
-- **InfluxDB for metrics** - Time-series storage and queries
-- **Grafana for dashboards** - Visualize everything
-
-## Next Steps
-
-- [Scaling and Load Balancing](./scaling.md) - Scale observability with your cluster
 - [Infrastructure Design](./infrastructure.md) - Full architecture overview
 - [Monitoring Dashboard](../chapter-04-gbui/monitoring.md) - Built-in monitoring UI

docs/src/chapter-11-features/whats-new.md

@@ -1,10 +1,8 @@
-# What's New: Multi-Agent Features
+# What's New
 
-General Bots has been enhanced with powerful multi-agent orchestration capabilities. This document summarizes the new features, keywords, and configuration options.
+This page has been consolidated with the Multi-Agent Orchestration documentation.
 
-## Overview
+For the latest features including:
-
-The multi-agent update introduces:
 
 - **Agent-to-Agent (A2A) Protocol** - Bots communicate and delegate tasks
 - **Cross-Session User Memory** - User data persists across bots and sessions
@@ -14,288 +12,13 @@ The multi-agent update introduces:
 - **Agent Reflection** - Self-analysis for continuous improvement
 - **SSE Streaming** - Real-time response streaming
 
-## New BASIC Keywords
+Please see: **[Multi-Agent Orchestration](./multi-agent-orchestration.md)**
 
-### Multi-Agent Keywords
+---
 
-| Keyword | Description |
+## Quick Links
-|---------|-------------|
-| `ADD BOT` | Add a bot with triggers, tools, or schedules |
-| `DELEGATE TO BOT` | Send task to another bot and get response |
-| `BROADCAST TO BOTS` | Send message to all session bots |
-| `TRANSFER CONVERSATION` | Hand off conversation to another bot |
-| `BOT REFLECTION` | Enable agent self-analysis |
-| `BOT REFLECTION INSIGHTS` | Get reflection analysis results |
 
-### Memory Keywords
+- [Multi-Agent Orchestration](./multi-agent-orchestration.md) - Complete guide to multi-agent systems
-
+- [Memory Management](./memory-management.md) - User and bot memory systems
-| Keyword | Description |
+- [Hybrid RAG Search](./hybrid-search.md) - Search configuration and tuning
-|---------|-------------|
+- [Keywords Reference](../chapter-06-gbdialog/keywords.md) - All BASIC keywords
-| `SET USER MEMORY` | Store data at user level (cross-bot) |
-| `GET USER MEMORY` | Retrieve user-level data |
-| `SET USER FACT` | Store a fact about the user |
-| `USER FACTS` | Get all stored user facts |
-
-### Model Routing Keywords
-
-| Keyword | Description |
-|---------|-------------|
-| `USE MODEL` | Switch LLM model (fast/quality/code/auto) |
-
-### Code Execution Keywords
-
-| Keyword | Description |
-|---------|-------------|
-| `RUN PYTHON` | Execute Python in sandbox |
-| `RUN JAVASCRIPT` | Execute JavaScript in sandbox |
-| `RUN BASH` | Execute Bash script in sandbox |
-| `RUN ... WITH FILE` | Run script from file |
-
-## Quick Examples
-
-### Multi-Agent Routing
-
-```basic
-' Router bot directing queries to specialists
-HEAR userquery
-
-category = LLM "Classify into billing, technical, sales: " + userquery
-
-SWITCH category
-    CASE "billing"
-        result = DELEGATE userquery TO BOT "billing-bot"
-    CASE "technical"
-        result = DELEGATE userquery TO BOT "tech-bot"
-    CASE "sales"
-        result = DELEGATE userquery TO BOT "sales-bot"
-END SWITCH
-
-TALK result
-```
-
-### Cross-Bot User Memory
-
-```basic
-' Store user preference (accessible from any bot)
-SET USER MEMORY "language", "pt-BR"
-SET USER MEMORY "timezone", "America/Sao_Paulo"
-
-' In another bot - retrieve preference
-language = GET USER MEMORY("language")
-IF language = "pt-BR" THEN
-    TALK "Olá! Como posso ajudar?"
-END IF
-```
-
-### Dynamic Model Selection
-
-```basic
-' Use fast model for simple queries
-USE MODEL "fast"
-greeting = LLM "Say hello"
-
-' Switch to quality model for complex analysis
-USE MODEL "quality"
-analysis = LLM "Analyze market trends and provide recommendations"
-
-' Let system decide automatically
-USE MODEL "auto"
-```
-
-### Code Sandbox
-
-```basic
-' Execute Python for data processing
-code = "
-import json
-data = [1, 2, 3, 4, 5]
-print(json.dumps({'sum': sum(data), 'avg': sum(data)/len(data)}))
-"
-result = RUN PYTHON code
-TALK "Statistics: " + result
-```
-
-### Agent Reflection
-
-```basic
-' Enable self-analysis
-BOT REFLECTION true
-BOT REFLECTION ON "conversation_quality"
-
-' Later, check insights
-insights = BOT REFLECTION INSIGHTS()
-IF insights.qualityScore < 0.7 THEN
-    SEND MAIL admin, "Low Quality Alert", insights.summary
-END IF
-```
-
-## New Configuration Options
-
-Add these to your `config.csv`:
-
-### Multi-Agent (A2A)
-
-```csv
-name,value
-a2a-enabled,true
-a2a-timeout,30
-a2a-max-hops,5
-a2a-retry-count,3
-```
-
-### User Memory
-
-```csv
-name,value
-user-memory-enabled,true
-user-memory-max-keys,1000
-user-memory-default-ttl,0
-```
-
-### Model Routing
-
-```csv
-name,value
-model-routing-strategy,auto
-model-default,fast
-model-fast,DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf
-model-quality,gpt-4
-model-code,codellama-7b.gguf
-```
-
-### Hybrid RAG Search
-
-```csv
-name,value
-rag-hybrid-enabled,true
-rag-dense-weight,0.7
-rag-sparse-weight,0.3
-rag-reranker-enabled,true
-```
-
-### Code Sandbox
-
-```csv
-name,value
-sandbox-runtime,lxc
-sandbox-timeout,30
-sandbox-memory-mb,512
-sandbox-cpu-percent,50
-sandbox-network,false
-sandbox-python-packages,numpy,pandas,pillow
-```
-
-### Bot Reflection
-
-```csv
-name,value
-reflection-enabled,true
-reflection-interval,10
-reflection-min-messages,3
-reflection-model,quality
-```
-
-### SSE Streaming
-
-```csv
-name,value
-sse-enabled,true
-sse-heartbeat,30
-sse-max-connections,1000
-```
-
-## Database Migrations
-
-Run migrations to create the new tables:
-
-```bash
-cd botserver
-cargo run -- migrate
-```
-
-### New Tables
-
-| Table | Purpose |
-|-------|---------|
-| `user_memory` | Cross-session user preferences and facts |
-| `user_preferences` | Per-session user settings |
-| `a2a_messages` | Agent-to-Agent protocol messages |
-| `user_memory_extended` | Enhanced memory with TTL |
-| `kg_relationships` | Knowledge graph relationships |
-| `conversation_summaries` | Episodic memory summaries |
-| `conversation_costs` | LLM cost tracking |
-| `openapi_tools` | OpenAPI tool tracking |
-| `agent_reflections` | Agent self-analysis results |
-| `chat_history` | Conversation history |
-| `session_bots` | Multi-agent session tracking |
-
-## Architecture
-
-```
-┌─────────────────────────────────────────────────────────────┐
-│                    Multi-Agent System                        │
-├─────────────────────────────────────────────────────────────┤
-│                                                              │
-│  ┌──────────┐    A2A Protocol    ┌──────────┐               │
-│  │ Router   │◄──────────────────►│ Billing  │               │
-│  │ Bot      │                    │ Bot      │               │
-│  └────┬─────┘    ┌──────────┐    └──────────┘               │
-│       │          │ Support  │                                │
-│       └─────────►│ Bot      │◄──────────────────┐           │
-│                  └──────────┘                    │           │
-│                                                  │           │
-│  ┌──────────────────────────────────────────────┼──────┐    │
-│  │              Shared Resources                 │      │    │
-│  │  ┌────────────┐  ┌────────────┐  ┌──────────┴─┐    │    │
-│  │  │ User       │  │ Hybrid     │  │ Model      │    │    │
-│  │  │ Memory     │  │ RAG Search │  │ Router     │    │    │
-│  │  └────────────┘  └────────────┘  └────────────┘    │    │
-│  └───────────────────────────────────────────────────────┘  │
-│                                                              │
-└─────────────────────────────────────────────────────────────┘
-```
-
-## Design Principles
-
-These features follow General Bots' core principles:
-
-1. **BASIC-First** - All features accessible via simple BASIC keywords
-2. **KISS** - Simple syntax, predictable behavior
-3. **Pragmatismo** - Real-world utility over theoretical purity
-4. **No Lock-in** - Local deployment, own your data
-
-## Performance Considerations
-
-| Feature | Impact | Mitigation |
-|---------|--------|------------|
-| A2A Protocol | Adds network latency | Use timeouts, local bots |
-| User Memory | Database queries | Caching, indexing |
-| Hybrid Search | Dual search paths | Results cached |
-| Code Sandbox | Container startup | Warm containers |
-| Reflection | LLM calls | Run periodically, not per-message |
-| SSE Streaming | Connection overhead | Connection pooling |
-
-## Migration Guide
-
-### From Single-Bot to Multi-Agent
-
-1. **Identify Specializations** - What tasks need dedicated bots?
-2. **Create Specialist Bots** - Each with focused config
-3. **Build Router** - Central bot to direct traffic
-4. **Share Memory** - Move shared data to User Memory
-5. **Test Delegation** - Verify communication paths
-
-### Upgrading Existing Bots
-
-1. Run database migrations
-2. Add new config options as needed
-3. Existing keywords continue to work unchanged
-4. Gradually adopt new features
-
-## See Also
-
-- [Multi-Agent Orchestration](./multi-agent-orchestration.md) - Complete guide
-- [Memory Management](./memory-management.md) - Memory deep dive
-- [Hybrid RAG Search](./hybrid-search.md) - Search configuration
-- [Keywords Reference](../chapter-06-gbdialog/keywords.md) - All keywords
-- [Configuration Parameters](../chapter-08-config/parameters.md) - All config options

docs/src/glossary.md

@@ -2,246 +2,342 @@
 
 Quick lookup for General Bots terms. If you're lost, start here.
 
+---
+
 ## A
 
-**Auto-Bootstrap** - The magical first run that installs everything automatically. PostgreSQL, cache, storage - all configured without you lifting a finger.
+**A2A Protocol** - Agent-to-Agent Protocol. Enables bots to communicate and delegate tasks to each other in multi-agent systems. Messages include request, response, broadcast, and delegate types.
 
-**Argon2** - The Fort Knox of password hashing. Makes brute-force attacks computationally infeasible. We use it everywhere passwords are stored.
+**ADD BOT** - BASIC keyword to add a bot to the current session with triggers, tools, or schedules.
+
+**Argon2** - Password hashing algorithm used for secure credential storage. Makes brute-force attacks computationally infeasible.
+
+**Auto-Bootstrap** - The automatic first-run process that installs and configures all dependencies: PostgreSQL, cache, storage, and LLM servers.
+
+---
 
 ## B
 
-**BASIC** - Yes, that programming language from 1964. We brought it back because `TALK "Hello"` beats `await ctx.send()` any day. Powers all conversation scripts. Used since 2018 in production systems.
+**BASIC** - The scripting language for General Bots dialogs. Inspired by the 1964 language, simplified for conversational AI. Powers all `.bas` scripts with keywords like TALK, HEAR, and LLM.
 
-**Bot Package** - A folder ending in `.gbai` containing everything to run a bot. Scripts, documents, config. That's it. Copy folder = deploy bot.
+**BM25** - Best Match 25. Sparse retrieval algorithm for keyword-based search. Used in hybrid RAG alongside dense (semantic) search.
 
-**BotSession** - The conversation between user and bot. Remembers everything - who you are, what you said, where you left off. Persists to database, cached for speed.
+**BOOK** - BASIC keyword to schedule calendar appointments.
 
-**Bootstrap** - Initial setup process that automatically installs all dependencies. Runs on first launch, creates configuration, sets up database, configures services.
+**Bot Memory** - Persistent storage scoped to a single bot, shared across all users. Access with `SET BOT MEMORY` and `GET BOT MEMORY`.
+
+**Bot Package** - A folder ending in `.gbai` containing everything to run a bot: scripts, documents, and configuration.
+
+**BotSession** - The active conversation between user and bot. Tracks state, history, and context. Persists to database, cached for speed.
+
+**Bootstrap** - Initial setup process that installs all dependencies automatically on first launch.
+
+**BROADCAST TO BOTS** - BASIC keyword to send a message to all bots in the current session.
+
+---
 
 ## C
 
-**Collection** - A folder of documents in `.gbkb/` that becomes searchable knowledge. Drop PDFs in `policies/`, bot answers policy questions. Zero configuration.
+**Cache** - In-memory storage component for sessions, temporary data, and semantic caching. Provides sub-millisecond access times.
 
-**Context** - What the LLM knows right now. Includes conversation history, active knowledge bases, loaded tools. Limited by token window (usually 4-8k tokens).
+**Collection** - A folder of documents in `.gbkb/` that becomes searchable knowledge. Each subfolder is a separate collection.
 
-**config.csv** - The only config file you need. Simple key-value pairs in CSV format. Opens in Excel. Lives in `.gbot/` folder.
+**Chunking** - The process of splitting documents into smaller pieces for embedding and retrieval. Default chunk size is optimized for context windows.
+
+**config.csv** - The configuration file for each bot. Simple key-value pairs in CSV format. Lives in the `.gbot/` folder.
+
+**Context** - Information available to the LLM during a conversation. Includes history, knowledge base results, and user-provided context via `SET CONTEXT`.
+
+**Context Compaction** - Automatic summarization of older conversation history to fit within token limits while preserving important information.
+
+**CREATE DRAFT** - BASIC keyword to compose and save an email draft to the user's mailbox.
+
+**CREATE TASK** - BASIC keyword to create a task with assignee and due date.
+
+---
 
 ## D
 
-**Dialog** - A `.bas` script defining conversation flow. Not complex state machines - just simple BASIC code like `TALK "Hi"` and `answer = HEAR`.
+**DELEGATE TO BOT** - BASIC keyword to send a task to another bot and optionally wait for a response.
 
-**Drive** - Built-in S3-compatible storage (SeaweedFS). Stores documents, templates, uploads. Auto-installed during bootstrap. No AWS account needed.
+**Dense Search** - Semantic search using vector embeddings. Finds content by meaning rather than exact keywords.
+
+**Dialog** - A `.bas` script defining conversation flow. Contains BASIC code with keywords like TALK and HEAR.
+
+**Drive** - Built-in S3-compatible object storage. Stores documents, templates, and uploads. Auto-installed during bootstrap.
+
+---
 
 ## E
 
-**Embedding** - Text converted to numbers for similarity search. "dog" and "puppy" have similar embeddings. BGE model by default, replaceable with any GGUF model.
+**Embedding** - Text converted to numerical vectors for similarity search. Similar meanings produce similar vectors.
 
-**Event Handler** - BASIC code that runs when something happens. `ON "login"` runs at login. `ON "email"` when email arrives. `ON "0 8 * * *"` at 8 AM daily.
+**Embedding Model** - Neural network that generates embeddings. Default is BGE, replaceable with any GGUF-compatible model.
+
+**Episodic Memory** - Summaries of past conversations stored for long-term context. Automatically generated when conversations end.
+
+**Event Handler** - BASIC code triggered by events. Use `ON` keyword with triggers like `"login"`, `"email"`, or cron expressions.
+
+---
+
+## F
+
+**FIND** - BASIC keyword to search database tables with filter criteria. Returns matching records.
+
+**FOR EACH** - BASIC keyword for iterating over collections and query results.
+
+---
 
 ## G
 
-**.gbai** - "General Bot AI" folder. Contains entire bot. Example: `support.gbai/` becomes bot at `/support`. No manifest files, no build process.
+**.gbai** - "General Bot AI" package folder. Contains the entire bot. Example: `support.gbai/` becomes the bot at `/support`.
 
-**.gbdialog** - Subfolder with BASIC scripts. Must contain `start.bas` as entry point. Tools go in `tools/` subdirectory.
+**.gbdialog** - Subfolder containing BASIC scripts. Must include `start.bas` as the entry point. Tools go in `tools/` subdirectory.
 
-**.gbkb** - "Knowledge Base" subfolder. Each subdirectory becomes a searchable collection. PDFs, Word docs, text files - all automatically indexed.
+**.gbdrive** - File storage configuration subfolder. Maps to Drive buckets for document management.
 
-**.gbot** - Configuration subfolder. Contains single `config.csv` file with bot settings. Missing values use defaults.
+**.gbkb** - "Knowledge Base" subfolder. Each subdirectory becomes a searchable collection with automatic indexing.
 
-**.gbtheme** - Optional UI customization. CSS files, images, HTML templates. Most bots don't need this.
+**.gbot** - Configuration subfolder containing `config.csv` with bot settings.
 
-**.gbdrive** - File storage configuration. Maps to Drive (S3-compatible) buckets for document management.
+**.gbtheme** - Optional UI customization subfolder for CSS, images, and HTML templates.
 
-**General Bots** - The enterprise conversational AI platform. Combines LLMs with structured dialogs, knowledge bases, and multi-channel support.
+**General Bots** - Open-source enterprise conversational AI platform. Combines LLMs with structured dialogs, knowledge bases, and multi-channel support.
+
+**GET** - BASIC keyword to retrieve data from APIs, files, or session variables.
+
+**GET BOT MEMORY** - BASIC keyword to retrieve persistent bot-level data.
+
+**GET USER MEMORY** - BASIC keyword to retrieve cross-session user data accessible from any bot.
+
+**GraphQL** - Query language for APIs. Supported via the `GRAPHQL` keyword for complex data retrieval.
+
+---
 
 ## H
 
-**HEAR** - BASIC keyword to get user input. `answer = HEAR "What's your name?"` waits for response.
+**HEAR** - BASIC keyword to wait for and capture user input. `name = HEAR` stores the response in a variable.
 
-**Hot Reload** - Changes to BASIC scripts apply immediately. No restart needed. Edit, save, test - instant feedback loop.
+**Hot Reload** - Automatic reloading of BASIC scripts when files change. No restart needed.
+
+**Hybrid Search** - RAG approach combining dense (semantic) and sparse (keyword) retrieval using Reciprocal Rank Fusion.
+
+**HTMX** - Frontend library used for dynamic UI updates without full page reloads.
+
+---
 
 ## I
 
-**Installer** - Component that auto-configures services. Manages PostgreSQL, Cache, Drive, Directory Service, LLM servers. Everything runs locally.
+**INSERT** - BASIC keyword to add records to database tables.
 
-**Intent** - What the user wants to do. Detected from natural language. "I want to reset my password" → password_reset intent.
+**Intent** - What the user wants to accomplish. Detected from natural language via LLM classification.
+
+---
 
 ## K
 
-**Knowledge Base** - Documents that become searchable answers. PDFs, Word files, web pages. Automatically chunked, embedded, and indexed for semantic search.
+**Keyword** - A BASIC command like TALK, HEAR, or LLM. About 50+ available. Written in uppercase by convention.
+
+**Knowledge Base (KB)** - Documents searchable by the bot. Organized in folders under `.gbkb/`. Activate with `USE KB "foldername"`.
+
+---
 
 ## L
 
-**LLM** - Large Language Model. The AI brain. Default uses llama.cpp with GGUF models. Supports OpenAI, Anthropic, local models.
+**LiveKit** - WebRTC platform used for video meetings in General Bots.
 
-**Local-First** - Everything runs on your machine. No cloud dependencies. Database, storage, LLM - all local. Privacy by default.
+**LLM** - Large Language Model. The AI that powers natural conversation. Supports OpenAI, Anthropic, Groq, and local models via llama.cpp.
+
+**llama.cpp** - C++ library for running LLM inference locally. Used for self-hosted model deployment.
+
+**Local-First** - Architecture principle where everything runs locally by default. No cloud dependencies required.
+
+---
 
 ## M
 
-**Memory** - Bot and user memory storage. `SET BOT MEMORY "key", "value"` persists data. `GET BOT MEMORY "key"` retrieves it.
+**MCP** - Model Context Protocol. Standard format for defining tools that LLMs can call. Supported alongside OpenAI function format.
 
-**Multi-Channel** - Same bot works everywhere. WhatsApp, Teams, Web, API. Write once, deploy anywhere.
+**Memory** - Data persistence system with four scopes: User Memory (cross-bot), Bot Memory (per-bot), Session Memory (temporary), and Episodic Memory (conversation summaries).
+
+**Model Routing** - Dynamic selection of LLM models based on task requirements. Use `USE MODEL "fast"`, `"quality"`, `"code"`, or `"auto"`.
+
+**Multi-Agent** - Architecture where multiple specialized bots collaborate on complex tasks.
+
+**Multi-Channel** - Same bot works across WhatsApp, Telegram, Teams, Web, and other channels without modification.
+
+---
+
+## N
+
+**No Forms** - General Bots philosophy since 2017: people should converse, not fill forms. Conversations replace traditional UI forms.
+
+---
 
 ## O
 
-**OIDC** - OpenID Connect authentication. Handled by Directory Service (Zitadel). No passwords stored in General Bots.
+**ON** - BASIC keyword to define event handlers for triggers, schedules, or webhooks.
+
+**OIDC** - OpenID Connect. Authentication protocol handled by the Directory service (Zitadel).
+
+---
 
 ## P
 
-**Package Manager** - System that installs bot packages. Drop `.gbai` folder, it's automatically loaded. Remove folder, bot stops.
+**Package Manager** - Built-in system that installs bot packages. Drop a `.gbai` folder and it's automatically loaded.
 
-**PostgreSQL** - The database. Stores users, sessions, messages, memory. Auto-installed, auto-configured. Just works.
+**PARAM** - Declares tool parameters. `PARAM name, email` means the tool needs these inputs. LLM collects them automatically.
 
-**Pragmatismo** - Company behind General Bots. Brazilian software consultancy. Building bots since 2016.
+**PostgreSQL** - The database for General Bots. Stores users, sessions, messages, and bot configuration. Auto-installed and auto-configured.
+
+**POST** - BASIC keyword to make HTTP POST requests to external APIs.
+
+**Pragmatismo** - Brazilian software company that created and maintains General Bots.
+
+---
 
 ## Q
 
-**Qdrant** - Vector database for semantic search. Optional component for large-scale knowledge bases. Faster than PostgreSQL pgvector for millions of documents.
+**Qdrant** - Vector database for semantic search at scale. Optional component for large knowledge bases.
+
+---
 
 ## R
 
-**REPL** - Read-Eval-Print Loop. Interactive BASIC console for testing. Type commands, see results immediately.
+**RAG** - Retrieval-Augmented Generation. Pattern where relevant documents are retrieved and provided to the LLM as context.
+
+**Reranking** - Optional LLM-based scoring of search results for improved relevance. Adds latency but improves quality.
+
+**Rhai** - Rust scripting engine that powers the BASIC interpreter. Sandboxed and safe.
+
+**RRF** - Reciprocal Rank Fusion. Algorithm for combining rankings from multiple search methods in hybrid RAG.
+
+**RUN PYTHON / JAVASCRIPT / BASH** - BASIC keywords to execute code in sandboxed environments.
+
+---
 
 ## S
 
-**Semantic Search** - Finding by meaning, not keywords. "How do I change my password?" finds "reset credentials" documentation.
+**SAVE** - BASIC keyword to write data to CSV files or database tables.
 
-**Session** - Active conversation state. Tracks user, bot, context, memory. Expires after inactivity. Stored in PostgreSQL, cached in memory.
+**Script** - A `.bas` file with BASIC code. `start.bas` is the entry point; other scripts are tools or utilities.
 
-**SET CONTEXT** - BASIC command to add information to LLM context. `SET CONTEXT "User is premium customer"` influences all responses.
+**Semantic Cache** - Caching system that matches similar (not just identical) queries to reuse LLM responses.
+
+**Semantic Search** - Finding content by meaning rather than exact keywords. Powered by embeddings and vector similarity.
+
+**SEND MAIL** - BASIC keyword to send emails with optional HTML and attachments.
+
+**Session** - Active conversation state between user and bot. Expires after inactivity (default 30 minutes).
+
+**Session Memory** - Temporary storage for the current conversation. Access with `SET` and `GET`.
+
+**SET** - BASIC keyword to store values in session variables or update database records.
+
+**SET BOT MEMORY** - BASIC keyword to store persistent bot-level data.
+
+**SET CONTEXT** - BASIC keyword to add information to the LLM context. Influences all subsequent responses.
+
+**SET SCHEDULE** - BASIC keyword for cron-based task scheduling. Accepts natural language like `"every monday at 9am"`.
+
+**SET USER MEMORY** - BASIC keyword to store cross-session user data accessible from any bot.
+
+**Sparse Search** - Keyword-based search using algorithms like BM25. Excels at exact matches and rare terms.
+
+**SSE** - Server-Sent Events. Used for real-time streaming of LLM responses.
+
+**Stalwart** - Email server component providing IMAP/SMTP/JMAP support.
+
+**Suite** - The complete General Bots workspace application with Chat, Drive, Tasks, Mail, Calendar, and other apps.
+
+**SWITCH** - BASIC keyword for multi-way conditional branching.
+
+---
 
 ## T
 
-**TALK** - BASIC keyword for bot output. `TALK "Hello!"` sends message to user. Supports markdown, images, cards.
+**TALK** - BASIC keyword to send messages to the user. Supports text, markdown, and multimedia.
 
-**Token** - Unit of text for LLMs. Roughly 4 characters. Context windows measured in tokens. GPT-4: 8k tokens. Local models: 4k typically.
+**Template** - Pre-built bot configuration in the `templates/` folder. Copy and modify to create new bots.
 
-**Tool** - Function the bot can call. Defined in BASIC with parameters. `PARAM "city"` then `weather = GET "weather"` calls weather API.
+**Token** - Unit of text for LLMs. Roughly 4 characters. Context windows are measured in tokens.
+
+**Tool** - A `.bas` file the LLM can call automatically. Define with `PARAM` declarations and a `DESCRIPTION`. Place in the `tools/` folder.
+
+**TRANSFER CONVERSATION** - BASIC keyword to hand off the entire conversation to another bot.
+
+---
 
 ## U
 
-**USE KB** - BASIC command to activate knowledge base. `USE KB "policies"` makes policy documents searchable in conversation.
+**UPDATE** - BASIC keyword to modify existing database records.
 
-**USE TOOL** - Activate a tool for LLM to use. `USE TOOL "calculator"` lets bot do math.
+**USE KB** - BASIC keyword to activate a knowledge base for semantic search. `USE KB "policies"` makes the policies collection searchable.
+
+**USE MODEL** - BASIC keyword to switch LLM models. Options: `"fast"`, `"quality"`, `"code"`, or `"auto"`.
+
+**USE TOOL** - BASIC keyword to enable a tool for LLM use. The AI determines when to call it.
+
+**User Memory** - Persistent storage scoped to a user, accessible across all bots and sessions.
+
+---
 
 ## V
 
-**Valkey** - Redis-compatible cache. Stores sessions, temporary data. Faster than database for frequently accessed data.
+**Vault** - HashiCorp Vault. Secrets management service for storing credentials securely. Only `VAULT_*` environment variables are used.
 
-**Vector** - Mathematical representation of text meaning. Used for semantic search. Created by embedding models.
+**Vector** - Mathematical representation of meaning. Similar meanings produce similar vectors.
+
+**Vector Database** - Database optimized for storing and searching embeddings. Qdrant is the default option.
+
+---
 
 ## W
 
-**WebSocket** - Real-time connection for chat. Enables streaming responses, live updates. No polling needed.
+**WAIT** - BASIC keyword to pause execution for a specified duration.
 
-**Workflow** - Sequence of dialog steps. Login → Verify → Action → Confirm. Defined in BASIC, no complex orchestration.
+**WEBHOOK** - BASIC keyword to create HTTP endpoints that trigger bot actions.
+
+**WebSocket** - Real-time connection for chat. Enables instant messaging without polling. Path: `/ws`.
+
+---
 
 ## Z
 
-**Zitadel** - Current Directory Service implementation. Handles authentication, users, permissions. Can be replaced with Keycloak or other OIDC providers.
+**Zitadel** - Identity and access management service. Handles authentication, users, and permissions.
 
-## H
+---
 
-**HEAR** - BASIC keyword to get user input. `name = HEAR` waits for user to type, stores response in variable.
+## Package Extensions
 
-**Hot Reload** - Change scripts while bot runs. Edit file, bot uses new version immediately. No restart needed (unless changing config).
+| Extension | Purpose |
+|-----------|---------|
+| `.gbai` | Complete bot package |
+| `.gbdialog` | BASIC scripts |
+| `.gbkb` | Knowledge base documents |
+| `.gbot` | Bot configuration |
+| `.gbtheme` | UI customization |
+| `.gbdrive` | File storage mapping |
+| `.bas` | BASIC script file |
 
-## K
+---
-
-**Knowledge Base (KB)** - Documents the bot can search. Organized in folders under `.gbkb/`. Use with `USE KB "foldername"` in scripts.
-
-**Keyword** - BASIC command like TALK, HEAR, LLM. About 40 total. All caps by convention, not requirement.
-
-## L
-
-**LLM** - Large Language Model (ChatGPT, Claude, Llama). The AI brain that powers natural conversation understanding.
-
-**Local Mode** - Run everything on your machine. LLM runs locally (llama.cpp), no internet required. Slower but private.
-
-## M
-
-**MCP** - Model Context Protocol. Standard format for defining tools that LLMs can call. Alternative to OpenAI function format.
-
-**Memory** - Two types: Session (temporary, per conversation) and Bot (permanent, across all sessions). `SET` for session, `SET BOT MEMORY` for permanent.
-
-## P
-
-**Package Manager** - Built-in system that installs/manages components. Handles PostgreSQL, cache, storage, vector DB, LLM server. All automatic.
-
-**PARAM** - Defines tool parameters. `PARAM name, email, date` means tool needs these three inputs. LLM collects them automatically.
-
-**PostgreSQL** - The database. Stores users, sessions, messages, bot config. Auto-installed, auto-configured, auto-migrated.
-
-## Q
-
-**Qdrant** - Vector database for semantic search. Stores document embeddings. Finds similar content even with different words. Optional but recommended.
-
-## R
-
-**Rhai** - The scripting engine powering BASIC interpreter. Rust-based, sandboxed, safe. You never see it directly.
-
-## S
-
-**Script** - A `.bas` file with BASIC code. `start.bas` runs first. Can call others with `RUN "other.bas"`.
-
-**Semantic Search** - Finding by meaning, not keywords. "vacation policy" finds "time off guidelines". Powered by embeddings and vector similarity.
-
-**Session** - See BotSession. The container for a conversation.
-
-**Session Token** - Random string identifying a session. Stored in browser localStorage for web, passed as header for API. Unguessable for security.
-
-## T
-
-**TALK** - BASIC keyword to send message. `TALK "Hello world"` displays that text to user.
-
-**Template** - Example bot in `templates/` folder. Copy, modify, deploy. `default.gbai` is minimal starter, `announcements.gbai` shows advanced features.
-
-**Tool** - A `.bas` file the LLM can call. Has PARAM definitions and DESCRIPTION. Put in `tools/` folder. AI figures out when to use it.
-
-**Token** - Unit of text for LLMs. Roughly 4 characters. GPT-3.5 handles 4k tokens, GPT-4 handles 8k-128k. Includes prompt + response.
-
-## U
-
-**USE KB** - BASIC command to activate knowledge base. `USE KB "policies"` makes policies folder searchable for that session.
-
-**USE TOOL** - BASIC command to enable a tool. `USE TOOL "send-email"` lets LLM send emails when appropriate.
-
-## V
-
-**Valkey** - Redis-compatible cache (fork after Redis license change). Stores session data for fast access. Auto-installed during bootstrap.
-
-**Vector** - Mathematical representation of meaning. "Cat" might be [0.2, 0.8, -0.3, ...]. Similar meanings have similar vectors.
-
-**Vector Collection** - See Collection. Folder of documents converted to vectors for semantic search.
-
-## W
-
-**WebSocket** - Real-time connection for chat. Enables instant messaging without polling. Path: `/ws` on same port as HTTP.
-
-## Symbols
-
-**.bas** - File extension for BASIC scripts. Plain text files with BASIC code.
-
-**.csv** - Configuration format. Simple, editable in Excel, no JSON parsing needed.
-
-**.env** - Environment variables file. Auto-generated during bootstrap with credentials and settings.
 
 ## Common Confusions
 
-**"Do I need containers?"** - No. BotServer installs everything directly or in LXC containers.
+**"Do I need containers?"** - No. BotServer installs everything directly or in optional LXC containers.
 
-**"What database?"** - PostgreSQL, auto-installed, auto-configured.
+**"What database?"** - PostgreSQL, automatically installed and configured.
 
 **"What about scaling?"** - Single server handles 1000+ concurrent users. Scale by running multiple instances.
 
-**"Is BASIC really BASIC?"** - Inspired by BASIC, not strict implementation. Simpler, focused on conversations.
+**"Is BASIC really BASIC?"** - Inspired by BASIC, not strict implementation. Simplified and focused on conversations.
 
-**"Can I use TypeScript/Python/etc?"** - BASIC is used for conversation logic. However, you can integrate with any language through APIs. See [API documentation](./chapter-10-api/README.md) for REST endpoints and integration options.
+**"Can I use TypeScript/Python/etc?"** - BASIC handles conversation logic. Use `RUN PYTHON/JAVASCRIPT` for code execution, or integrate via REST API.
 
-**"Is it production-ready?"** - Yes. Used in production since 2016 (earlier versions), current Rust version since 2023.
+**"Is it production-ready?"** - Yes. Used in production since 2016, current Rust version since 2023.
 
 ---
 
 <div align="center">
-  <img src="https://pragmatismo.com.br/icons/general-bots-text.svg" alt="General Bots" width="200">
+  <img src="./assets/general-bots-logo.svg" alt="General Bots" width="200">
 </div>

docs/src/introduction.md

@@ -220,7 +220,7 @@ The default bot is ready. Ask it anything. Modify `templates/default.gbai/` to c
 
 - **Language**: Written in Rust for performance and safety
 - **Database**: PostgreSQL with Diesel ORM
-- **Cache**: Valkey (Redis-compatible) for sessions
+- **Cache**: Redis-compatible cache for sessions
 - **Storage**: S3-compatible object store (MinIO)
 - **Vectors**: Qdrant for semantic search
 - **Security**: Vault for secrets, Argon2 passwords, AES encryption

src/basic/keywords/add_bot.rs

@@ -173,8 +173,14 @@ fn add_bot_with_trigger_keyword(state: Arc<AppState>, user: UserSession, engine:
             std::thread::spawn(move || {
                 let rt = tokio::runtime::Runtime::new().expect("Failed to create runtime");
                 let result = rt.block_on(async {
-                    add_bot_to_session(&state_for_task, session_id, bot_id, &bot_name_clone, trigger)
+                    add_bot_to_session(
-                        .await
+                        &state_for_task,
+                        session_id,
+                        bot_id,
+                        &bot_name_clone,
+                        trigger,
+                    )
+                    .await
                 });
                 let _ = tx.send(result);
             });
@@ -238,8 +244,14 @@ fn add_bot_with_tools_keyword(state: Arc<AppState>, user: UserSession, engine: &
             std::thread::spawn(move || {
                 let rt = tokio::runtime::Runtime::new().expect("Failed to create runtime");
                 let result = rt.block_on(async {
-                    add_bot_to_session(&state_for_task, session_id, bot_id, &bot_name_clone, trigger)
+                    add_bot_to_session(
-                        .await
+                        &state_for_task,
+                        session_id,
+                        bot_id,
+                        &bot_name_clone,
+                        trigger,
+                    )
+                    .await
                 });
                 let _ = tx.send(result);
             });
@@ -297,8 +309,14 @@ fn add_bot_with_schedule_keyword(state: Arc<AppState>, user: UserSession, engine
             std::thread::spawn(move || {
                 let rt = tokio::runtime::Runtime::new().expect("Failed to create runtime");
                 let result = rt.block_on(async {
-                    add_bot_to_session(&state_for_task, session_id, bot_id, &bot_name_clone, trigger)
+                    add_bot_to_session(
-                        .await
+                        &state_for_task,
+                        session_id,
+                        bot_id,
+                        &bot_name_clone,
+                        trigger,
+                    )
+                    .await
                 });
                 let _ = tx.send(result);
             });
@@ -323,44 +341,44 @@ fn remove_bot_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engi
     let state_clone = Arc::clone(&state);
     let user_clone = user.clone();
 
-    engine.register_custom_syntax(&["REMOVE", "BOT", "$expr$"], false, move |context, inputs| {
+    engine.register_custom_syntax(
-        let bot_name = context
+        &["REMOVE", "BOT", "$expr$"],
-            .eval_expression_tree(&inputs[0])?
+        false,
-            .to_string()
+        move |context, inputs| {
-            .trim_matches('"')
+            let bot_name = context
-            .to_string();
+                .eval_expression_tree(&inputs[0])?
+                .to_string()
+                .trim_matches('"')
+                .to_string();
 
-        trace!(
+            trace!("REMOVE BOT '{}' from session: {}", bot_name, user_clone.id);
-            "REMOVE BOT '{}' from session: {}",
-            bot_name,
-            user_clone.id
-        );
 
-        let state_for_task = Arc::clone(&state_clone);
+            let state_for_task = Arc::clone(&state_clone);
-        let session_id = user_clone.id;
+            let session_id = user_clone.id;
 
-        let (tx, rx) = std::sync::mpsc::channel();
+            let (tx, rx) = std::sync::mpsc::channel();
 
-        std::thread::spawn(move || {
+            std::thread::spawn(move || {
-            let rt = tokio::runtime::Runtime::new().expect("Failed to create runtime");
+                let rt = tokio::runtime::Runtime::new().expect("Failed to create runtime");
-            let result = rt.block_on(async {
+                let result = rt.block_on(async {
-                remove_bot_from_session(&state_for_task, session_id, &bot_name).await
+                    remove_bot_from_session(&state_for_task, session_id, &bot_name).await
+                });
+                let _ = tx.send(result);
             });
-            let _ = tx.send(result);
-        });
 
-        match rx.recv_timeout(std::time::Duration::from_secs(30)) {
+            match rx.recv_timeout(std::time::Duration::from_secs(30)) {
-            Ok(Ok(msg)) => Ok(Dynamic::from(msg)),
+                Ok(Ok(msg)) => Ok(Dynamic::from(msg)),
-            Ok(Err(e)) => Err(Box::new(rhai::EvalAltResult::ErrorRuntime(
+                Ok(Err(e)) => Err(Box::new(rhai::EvalAltResult::ErrorRuntime(
-                e.into(),
+                    e.into(),
-                rhai::Position::NONE,
+                    rhai::Position::NONE,
-            ))),
+                ))),
-            Err(_) => Err(Box::new(rhai::EvalAltResult::ErrorRuntime(
+                Err(_) => Err(Box::new(rhai::EvalAltResult::ErrorRuntime(
-                "REMOVE BOT timed out".into(),
+                    "REMOVE BOT timed out".into(),
-                rhai::Position::NONE,
+                    rhai::Position::NONE,
-            ))),
+                ))),
-        }
+            }
-    });
+        },
+    );
 }
 
 /// LIST BOTS
@@ -378,8 +396,7 @@ fn list_bots_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engin
 
         std::thread::spawn(move || {
             let rt = tokio::runtime::Runtime::new().expect("Failed to create runtime");
-            let result =
+            let result = rt.block_on(async { get_session_bots(&state_for_task, session_id).await });
-                rt.block_on(async { get_session_bots(&state_for_task, session_id).await });
             let _ = tx.send(result);
         });
 
@@ -392,7 +409,10 @@ fn list_bots_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engin
                         let mut map = rhai::Map::new();
                         map.insert("name".into(), Dynamic::from(b.bot_name));
                         map.insert("priority".into(), Dynamic::from(b.priority));
-                        map.insert("trigger_type".into(), Dynamic::from(b.trigger.trigger_type.to_string()));
+                        map.insert(
+                            "trigger_type".into(),
+                            Dynamic::from(b.trigger.trigger_type.to_string()),
+                        );
                         map.insert("is_active".into(), Dynamic::from(b.is_active));
                         Dynamic::from(map)
                     })
@@ -480,11 +500,7 @@ fn delegate_to_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Eng
                 .trim_matches('"')
                 .to_string();
 
-            trace!(
+            trace!("DELEGATE TO '{}' for session: {}", bot_name, user_clone.id);
-                "DELEGATE TO '{}' for session: {}",
-                bot_name,
-                user_clone.id
-            );
 
             let state_for_task = Arc::clone(&state_clone);
             let session_id = user_clone.id;
@@ -538,7 +554,7 @@ async fn add_bot_to_session(
     .unwrap_or(false);
 
     // If bot doesn't exist, try to find it in templates or create a placeholder
-    let bot_id = if bot_exists {
+    let bot_id: String = if bot_exists {
         diesel::sql_query("SELECT id FROM bots WHERE name = $1 AND is_active = true")
             .bind::<diesel::sql_types::Text, _>(bot_name)
             .get_result::<UuidResult>(&mut *conn)
@@ -559,12 +575,12 @@ async fn add_bot_to_session(
         .execute(&mut *conn)
         .map_err(|e| format!("Failed to create bot: {}", e))?;
 
-        new_bot_id
+        new_bot_id.to_string()
     };
 
     // Serialize trigger to JSON
-    let trigger_json =
+    let trigger_json = serde_json::to_string(&trigger)
-        serde_json::to_string(&trigger).map_err(|e| format!("Failed to serialize trigger: {}", e))?;
+        .map_err(|e| format!("Failed to serialize trigger: {}", e))?;
 
     // Add bot to session
     let association_id = Uuid::new_v4();
@@ -690,13 +706,11 @@ async fn delegate_to_bot(
     }
 
     // Mark delegation in session
-    diesel::sql_query(
+    diesel::sql_query("UPDATE sessions SET delegated_to = $1, delegated_at = NOW() WHERE id = $2")
-        "UPDATE sessions SET delegated_to = $1, delegated_at = NOW() WHERE id = $2",
+        .bind::<diesel::sql_types::Text, _>(bot_name)
-    )
+        .bind::<diesel::sql_types::Text, _>(session_id.to_string())
-    .bind::<diesel::sql_types::Text, _>(bot_name)
+        .execute(&mut *conn)
-    .bind::<diesel::sql_types::Text, _>(session_id.to_string())
+        .map_err(|e| format!("Failed to delegate: {}", e))?;
-    .execute(&mut *conn)
-    .map_err(|e| format!("Failed to delegate: {}", e))?;
 
     Ok(format!("Conversation delegated to '{}'", bot_name))
 }

src/basic/keywords/api_tool_generator.rs

@@ -142,6 +142,9 @@ impl ApiToolGenerator {
         let api_configs = self.get_api_configs().await?;
         let mut result = SyncResult::default();
 
+        // Clone api_configs for use in cleanup after the loop
+        let api_configs_for_cleanup = api_configs.clone();
+
         for (api_name, spec_url) in api_configs {
             info!("Processing API: {} from {}", api_name, spec_url);
 
@@ -159,7 +162,7 @@ impl ApiToolGenerator {
         }
 
         // Clean up removed APIs
-        let removed = self.cleanup_removed_apis(&api_configs).await?;
+        let removed = self.cleanup_removed_apis(&api_configs_for_cleanup).await?;
         result.tools_removed = removed;
 
         Ok(result)

src/basic/keywords/hear_talk.rs

@@ -814,8 +814,10 @@ fn validate_language(input: &str) -> ValidationResult {
         ("tr", "turkish", "turco"),
     ];
 
-    for (code, variants @ ..) in &languages {
+    for entry in &languages {
-        if lower == *code || variants.iter().any(|v| lower == *v) {
+        let code = entry[0];
+        let variants = &entry[1..];
+        if lower == code || variants.iter().any(|v| lower == *v) {
             return ValidationResult::valid_with_metadata(
                 code.to_string(),
                 serde_json::json!({ "code": code, "input": input }),
@@ -1304,7 +1306,10 @@ pub async fn process_hear_input(
     match validation_type {
         InputType::Image | InputType::QrCode => {
             if let Some(atts) = &attachments {
-                if let Some(img) = atts.iter().find(|a| a.content_type.starts_with("image/")) {
+                if let Some(img) = atts
+                    .iter()
+                    .find(|a| a.mime_type.as_deref().unwrap_or("").starts_with("image/"))
+                {
                     if validation_type == InputType::QrCode {
                         // Call botmodels to read QR code
                         return process_qrcode(state, &img.url).await;
@@ -1319,7 +1324,10 @@ pub async fn process_hear_input(
         }
         InputType::Audio => {
             if let Some(atts) = &attachments {
-                if let Some(audio) = atts.iter().find(|a| a.content_type.starts_with("audio/")) {
+                if let Some(audio) = atts
+                    .iter()
+                    .find(|a| a.mime_type.as_deref().unwrap_or("").starts_with("audio/"))
+                {
                     // Call botmodels for speech-to-text
                     return process_audio_to_text(state, &audio.url).await;
                 }
@@ -1328,7 +1336,10 @@ pub async fn process_hear_input(
         }
         InputType::Video => {
             if let Some(atts) = &attachments {
-                if let Some(video) = atts.iter().find(|a| a.content_type.starts_with("video/")) {
+                if let Some(video) = atts
+                    .iter()
+                    .find(|a| a.mime_type.as_deref().unwrap_or("").starts_with("video/"))
+                {
                     // Call botmodels for video description
                     return process_video_description(state, &video.url).await;
                 }
@@ -1375,10 +1386,8 @@ async fn process_qrcode(
     image_url: &str,
 ) -> Result<(String, Option<serde_json::Value>), String> {
     // Call botmodels vision service
-    let botmodels_url = state
+    let botmodels_url =
-        .config
+        std::env::var("BOTMODELS_URL").unwrap_or_else(|_| "http://localhost:8001".to_string());
-        .get("botmodels-url")
-        .unwrap_or_else(|| "http://localhost:8001".to_string());
 
     let client = reqwest::Client::new();
 
@@ -1425,13 +1434,11 @@ async fn process_qrcode(
 
 /// Process audio to text using botmodels
 async fn process_audio_to_text(
-    state: &AppState,
+    _state: &AppState,
     audio_url: &str,
 ) -> Result<(String, Option<serde_json::Value>), String> {
-    let botmodels_url = state
+    let botmodels_url =
-        .config
+        std::env::var("BOTMODELS_URL").unwrap_or_else(|_| "http://localhost:8001".to_string());
-        .get("botmodels-url")
-        .unwrap_or_else(|| "http://localhost:8001".to_string());
 
     let client = reqwest::Client::new();
 
@@ -1479,13 +1486,11 @@ async fn process_audio_to_text(
 
 /// Process video description using botmodels
 async fn process_video_description(
-    state: &AppState,
+    _state: &AppState,
     video_url: &str,
 ) -> Result<(String, Option<serde_json::Value>), String> {
-    let botmodels_url = state
+    let botmodels_url =
-        .config
+        std::env::var("BOTMODELS_URL").unwrap_or_else(|_| "http://localhost:8001".to_string());
-        .get("botmodels-url")
-        .unwrap_or_else(|| "http://localhost:8001".to_string());
 
     let client = reqwest::Client::new();
 

src/basic/keywords/import_export.rs

@@ -235,7 +235,12 @@ fn resolve_file_path(
     }
 
     // Resolve relative to bot's gbdrive folder
-    let base_path = format!("{}/bots/{}/gbdrive", state.config.data_dir, user.bot_id);
+    let data_dir = state
+        .config
+        .as_ref()
+        .map(|c| c.data_dir.as_str())
+        .unwrap_or("./botserver-stack/data");
+    let base_path = format!("{}/bots/{}/gbdrive", data_dir, user.bot_id);
 
     let full_path = format!("{}/{}", base_path, file_path);
 
@@ -243,10 +248,7 @@ fn resolve_file_path(
         Ok(full_path)
     } else {
         // Try without gbdrive prefix
-        let alt_path = format!(
+        let alt_path = format!("{}/bots/{}/{}", data_dir, user.bot_id, file_path);
-            "{}/bots/{}/{}",
-            state.config.data_dir, user.bot_id, file_path
-        );
         if Path::new(&alt_path).exists() {
             Ok(alt_path)
         } else {
@@ -266,7 +268,12 @@ fn resolve_export_path(
     }
 
     // Resolve relative to bot's gbdrive folder
-    let base_path = format!("{}/bots/{}/gbdrive", state.config.data_dir, user.bot_id);
+    let data_dir = state
+        .config
+        .as_ref()
+        .map(|c| c.data_dir.as_str())
+        .unwrap_or("./botserver-stack/data");
+    let base_path = format!("{}/bots/{}/gbdrive", data_dir, user.bot_id);
 
     // Ensure directory exists
     std::fs::create_dir_all(&base_path)?;

src/basic/keywords/kb_statistics.rs

@@ -58,20 +58,18 @@ pub fn kb_statistics_keyword(state: Arc<AppState>, user: UserSession, engine: &m
             return Dynamic::UNIT;
         }
 
-        let result = rt.unwrap().block_on(async {
+        let result = rt
-            get_kb_statistics(&state, &user).await
+            .unwrap()
-        });
+            .block_on(async { get_kb_statistics(&state, &user).await });
 
         match result {
-            Ok(stats) => {
+            Ok(stats) => match serde_json::to_value(&stats) {
-                match serde_json::to_value(&stats) {
+                Ok(json) => Dynamic::from(json.to_string()),
-                    Ok(json) => Dynamic::from(json.to_string()),
+                Err(e) => {
-                    Err(e) => {
+                    error!("Failed to serialize KB statistics: {}", e);
-                        error!("Failed to serialize KB statistics: {}", e);
+                    Dynamic::UNIT
-                        Dynamic::UNIT
-                    }
                 }
-            }
+            },
             Err(e) => {
                 error!("Failed to get KB statistics: {}", e);
                 Dynamic::UNIT
@@ -83,44 +81,45 @@ pub fn kb_statistics_keyword(state: Arc<AppState>, user: UserSession, engine: &m
     let state_clone2 = Arc::clone(&state);
     let user_clone2 = user.clone();
 
-    engine.register_fn("KB COLLECTION STATS", move |collection_name: &str| -> Dynamic {
+    engine.register_fn(
-        let state = Arc::clone(&state_clone2);
+        "KB COLLECTION STATS",
-        let user = user_clone2.clone();
+        move |collection_name: &str| -> Dynamic {
+            let state = Arc::clone(&state_clone2);
+            let user = user_clone2.clone();
 
-        trace!(
+            trace!(
-            "KB COLLECTION STATS called for collection '{}' bot {} by user {}",
+                "KB COLLECTION STATS called for collection '{}' bot {} by user {}",
-            collection_name,
+                collection_name,
-            user.bot_id,
+                user.bot_id,
-            user.user_id
+                user.user_id
-        );
+            );
 
-        let rt = tokio::runtime::Handle::try_current();
+            let rt = tokio::runtime::Handle::try_current();
-        if rt.is_err() {
+            if rt.is_err() {
-            error!("KB COLLECTION STATS: No tokio runtime available");
+                error!("KB COLLECTION STATS: No tokio runtime available");
-            return Dynamic::UNIT;
+                return Dynamic::UNIT;
-        }
+            }
 
-        let collection = collection_name.to_string();
+            let collection = collection_name.to_string();
-        let result = rt.unwrap().block_on(async {
+            let result = rt
-            get_collection_statistics(&state, &collection).await
+                .unwrap()
-        });
+                .block_on(async { get_collection_statistics(&state, &collection).await });
 
-        match result {
+            match result {
-            Ok(stats) => {
+                Ok(stats) => match serde_json::to_value(&stats) {
-                match serde_json::to_value(&stats) {
                     Ok(json) => Dynamic::from(json.to_string()),
                     Err(e) => {
                         error!("Failed to serialize collection statistics: {}", e);
                         Dynamic::UNIT
                     }
+                },
+                Err(e) => {
+                    error!("Failed to get collection statistics: {}", e);
+                    Dynamic::UNIT
                 }
             }
-            Err(e) => {
+        },
-                error!("Failed to get collection statistics: {}", e);
+    );
-                Dynamic::UNIT
-            }
-        }
-    });
 
     // KB DOCUMENTS COUNT - Get total document count for bot
     let state_clone3 = Arc::clone(&state);
@@ -142,9 +141,9 @@ pub fn kb_statistics_keyword(state: Arc<AppState>, user: UserSession, engine: &m
             return 0;
         }
 
-        let result = rt.unwrap().block_on(async {
+        let result = rt
-            get_documents_count(&state, &user).await
+            .unwrap()
-        });
+            .block_on(async { get_documents_count(&state, &user).await });
 
         result.unwrap_or(0)
     });
@@ -170,9 +169,9 @@ pub fn kb_statistics_keyword(state: Arc<AppState>, user: UserSession, engine: &m
             return 0;
         }
 
-        let result = rt.unwrap().block_on(async {
+        let result = rt
-            get_documents_added_since(&state, &user, days).await
+            .unwrap()
-        });
+            .block_on(async { get_documents_added_since(&state, &user, days).await });
 
         result.unwrap_or(0)
     });
@@ -197,16 +196,13 @@ pub fn kb_statistics_keyword(state: Arc<AppState>, user: UserSession, engine: &m
             return Dynamic::UNIT;
         }
 
-        let result = rt.unwrap().block_on(async {
+        let result = rt
-            list_collections(&state, &user).await
+            .unwrap()
-        });
+            .block_on(async { list_collections(&state, &user).await });
 
         match result {
             Ok(collections) => {
-                let arr: Vec<Dynamic> = collections
+                let arr: Vec<Dynamic> = collections.into_iter().map(Dynamic::from).collect();
-                    .into_iter()
-                    .map(Dynamic::from)
-                    .collect();
                 Dynamic::from(arr)
             }
             Err(e) => {
@@ -236,9 +232,9 @@ pub fn kb_statistics_keyword(state: Arc<AppState>, user: UserSession, engine: &m
             return 0.0;
         }
 
-        let result = rt.unwrap().block_on(async {
+        let result = rt
-            get_storage_size(&state, &user).await
+            .unwrap()
-        });
+            .block_on(async { get_storage_size(&state, &user).await });
 
         result.unwrap_or(0.0)
     });
@@ -249,7 +245,8 @@ async fn get_kb_statistics(
     state: &AppState,
     user: &UserSession,
 ) -> Result<KBStatistics, Box<dyn std::error::Error + Send + Sync>> {
-    let qdrant_url = state.qdrant_url.clone().unwrap_or_else(|| "https://localhost:6334".to_string());
+    let qdrant_url =
+        std::env::var("QDRANT_URL").unwrap_or_else(|_| "https://localhost:6334".to_string());
     let client = reqwest::Client::builder()
         .danger_accept_invalid_certs(true)
         .build()?;
@@ -286,8 +283,11 @@ async fn get_kb_statistics(
     }
 
     // Get documents added in last week and month from database
-    let documents_added_last_week = get_documents_added_since(state, user, 7).await.unwrap_or(0) as u64;
+    let documents_added_last_week =
-    let documents_added_last_month = get_documents_added_since(state, user, 30).await.unwrap_or(0) as u64;
+        get_documents_added_since(state, user, 7).await.unwrap_or(0) as u64;
+    let documents_added_last_month = get_documents_added_since(state, user, 30)
+        .await
+        .unwrap_or(0) as u64;
 
     Ok(KBStatistics {
         total_collections: collection_names.len() as u64,
@@ -306,7 +306,8 @@ async fn get_collection_statistics(
     state: &AppState,
     collection_name: &str,
 ) -> Result<CollectionStats, Box<dyn std::error::Error + Send + Sync>> {
-    let qdrant_url = state.qdrant_url.clone().unwrap_or_else(|| "https://localhost:6334".to_string());
+    let qdrant_url =
+        std::env::var("QDRANT_URL").unwrap_or_else(|_| "https://localhost:6334".to_string());
     let client = reqwest::Client::builder()
         .danger_accept_invalid_certs(true)
         .build()?;
@@ -349,11 +350,10 @@ async fn get_documents_count(
     let mut conn = state.conn.get()?;
     let bot_id = user.bot_id.to_string();
 
-    let result: CountResult = sql_query(
+    let result: CountResult =
-        "SELECT COUNT(*) as count FROM kb_documents WHERE bot_id = $1"
+        sql_query("SELECT COUNT(*) as count FROM kb_documents WHERE bot_id = $1")
-    )
+            .bind::<diesel::sql_types::Text, _>(&bot_id)
-    .bind::<diesel::sql_types::Text, _>(&bot_id)
+            .get_result(&mut *conn)?;
-    .get_result(&mut *conn)?;
 
     Ok(result.count)
 }
@@ -366,7 +366,7 @@ async fn get_documents_added_since(
 ) -> Result<i64, Box<dyn std::error::Error + Send + Sync>> {
     use diesel::prelude::*;
     use diesel::sql_query;
-    use diesel::sql_types::{BigInt, Text, Integer};
+    use diesel::sql_types::{BigInt, Integer, Text};
 
     #[derive(QueryableByName)]
     struct CountResult {
@@ -380,7 +380,7 @@ async fn get_documents_added_since(
     let result: CountResult = sql_query(
         "SELECT COUNT(*) as count FROM kb_documents
          WHERE bot_id = $1
-         AND created_at >= NOW() - INTERVAL '1 day' * $2"
+         AND created_at >= NOW() - INTERVAL '1 day' * $2",
     )
     .bind::<Text, _>(&bot_id)
     .bind::<Integer, _>(days as i32)
@@ -394,7 +394,8 @@ async fn list_collections(
     state: &AppState,
     user: &UserSession,
 ) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
-    let qdrant_url = state.qdrant_url.clone().unwrap_or_else(|| "https://localhost:6334".to_string());
+    let qdrant_url =
+        std::env::var("QDRANT_URL").unwrap_or_else(|_| "https://localhost:6334".to_string());
     let client = reqwest::Client::builder()
         .danger_accept_invalid_certs(true)
         .build()?;

src/basic/keywords/play.rs

@@ -606,25 +606,26 @@ async fn send_play_to_client(
         serde_json::to_string(&message).map_err(|e| format!("Failed to serialize: {}", e))?;
 
     // Send via web adapter
-    let web_adapter = Arc::clone(&state.web_adapter);
+    let bot_response = crate::shared::models::BotResponse {
+        bot_id: String::new(),
+        user_id: String::new(),
+        session_id: session_id.to_string(),
+        channel: "web".to_string(),
+        content: message_str,
+        message_type: crate::shared::message_types::MessageType::BOT_RESPONSE,
+        stream_token: None,
+        is_complete: true,
+        suggestions: Vec::new(),
+        context_name: None,
+        context_length: 0,
+        context_max_length: 0,
+    };
 
-    if let Some(sender) = web_adapter
+    state
-        .sessions
+        .web_adapter
-        .lock()
+        .send_message_to_session(&session_id.to_string(), bot_response)
         .await
-        .get(&session_id.to_string())
+        .map_err(|e| format!("Failed to send to client: {}", e))?;
-    {
-        sender
-            .send(axum::extract::ws::Message::Text(message_str))
-            .await
-            .map_err(|e| format!("Failed to send to client: {}", e))?;
-    } else {
-        // Store for later delivery
-        trace!(
-            "No WebSocket connection for session {}, message queued",
-            session_id
-        );
-    }
 
     Ok(())
 }
@@ -643,19 +644,27 @@ async fn send_player_command(
     let message_str =
         serde_json::to_string(&message).map_err(|e| format!("Failed to serialize: {}", e))?;
 
-    let web_adapter = Arc::clone(&state.web_adapter);
+    // Use web adapter to send message
-
+    let _ = state
-    if let Some(sender) = web_adapter
+        .web_adapter
-        .sessions
+        .send_message_to_session(
-        .lock()
+            &session_id.to_string(),
-        .await
+            crate::shared::models::BotResponse {
-        .get(&session_id.to_string())
+                bot_id: String::new(),
-    {
+                user_id: String::new(),
-        sender
+                session_id: session_id.to_string(),
-            .send(axum::extract::ws::Message::Text(message_str))
+                channel: "web".to_string(),
-            .await
+                content: message_str,
-            .map_err(|e| format!("Failed to send command: {}", e))?;
+                message_type: crate::shared::message_types::MessageType::BOT_RESPONSE,
-    }
+                stream_token: None,
+                is_complete: true,
+                suggestions: Vec::new(),
+                context_name: None,
+                context_length: 0,
+                context_max_length: 0,
+            },
+        )
+        .await;
 
     Ok(())
 }

src/basic/keywords/qrcode.rs

@@ -244,20 +244,23 @@ fn execute_qr_code_generation(
     let image = code.render::<Luma<u8>>().min_dimensions(size, size).build();
 
     // Determine output path
+    let data_dir = state
+        .config
+        .as_ref()
+        .map(|c| c.data_dir.as_str())
+        .unwrap_or("./botserver-stack/data");
+
     let final_path = match output_path {
         Some(path) => {
             if Path::new(path).is_absolute() {
                 path.to_string()
             } else {
-                format!(
+                format!("{}/bots/{}/gbdrive/{}", data_dir, user.bot_id, path)
-                    "{}/bots/{}/gbdrive/{}",
-                    state.config.data_dir, user.bot_id, path
-                )
             }
         }
         None => {
             let filename = format!("qrcode_{}.png", Uuid::new_v4());
-            let base_path = format!("{}/bots/{}/gbdrive", state.config.data_dir, user.bot_id);
+            let base_path = format!("{}/bots/{}/gbdrive", data_dir, user.bot_id);
 
             // Ensure directory exists
             std::fs::create_dir_all(&base_path)?;

src/basic/keywords/sms.rs

@@ -273,7 +273,7 @@ async fn execute_send_sms(
         Some(p) => p.to_string(),
         None => config_manager
             .get_config(&bot_id, "sms-provider", None)
-            .unwrap_or_else(|| "twilio".to_string()),
+            .unwrap_or_else(|_| "twilio".to_string()),
     };
 
     let provider = SmsProvider::from(provider_name.as_str());
@@ -352,15 +352,15 @@ async fn send_via_twilio(
 
     let account_sid = config_manager
         .get_config(bot_id, "twilio-account-sid", None)
-        .ok_or("Twilio account SID not configured. Set twilio-account-sid in config.")?;
+        .map_err(|_| "Twilio account SID not configured. Set twilio-account-sid in config.")?;
 
     let auth_token = config_manager
         .get_config(bot_id, "twilio-auth-token", None)
-        .ok_or("Twilio auth token not configured. Set twilio-auth-token in config.")?;
+        .map_err(|_| "Twilio auth token not configured. Set twilio-auth-token in config.")?;
 
     let from_number = config_manager
         .get_config(bot_id, "twilio-from-number", None)
-        .ok_or("Twilio from number not configured. Set twilio-from-number in config.")?;
+        .map_err(|_| "Twilio from number not configured. Set twilio-from-number in config.")?;
 
     let client = reqwest::Client::new();
     let url = format!(
@@ -397,35 +397,56 @@ async fn send_via_aws_sns(
 
     let access_key = config_manager
         .get_config(bot_id, "aws-access-key", None)
-        .ok_or("AWS access key not configured. Set aws-access-key in config.")?;
+        .map_err(|_| "AWS access key not configured. Set aws-access-key in config.")?;
 
     let secret_key = config_manager
         .get_config(bot_id, "aws-secret-key", None)
-        .ok_or("AWS secret key not configured. Set aws-secret-key in config.")?;
+        .map_err(|_| "AWS secret key not configured. Set aws-secret-key in config.")?;
 
     let region = config_manager
-        .get_config(bot_id, "aws-region", None)
+        .get_config(bot_id, "aws-region", Some("us-east-1"))
-        .unwrap_or_else(|| "us-east-1".to_string());
+        .unwrap_or_else(|_| "us-east-1".to_string());
 
-    // Use AWS SDK for Rust
+    // Use HTTP API directly instead of AWS SDK
-    let config = aws_config::defaults(aws_config::BehaviorVersion::latest())
+    let client = reqwest::Client::new();
-        .region(aws_config::Region::new(region))
+    let url = format!("https://sns.{}.amazonaws.com/", region);
-        .credentials_provider(aws_credential_types::Credentials::new(
-            access_key, secret_key, None, None, "gb-sms",
-        ))
-        .load()
-        .await;
 
-    let client = aws_sdk_sns::Client::new(&config);
+    // Create timestamp for AWS Signature
+    let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
+    let date = &timestamp[..8];
 
-    let result = client
+    // Build the request parameters
-        .publish()
+    let params = [
-        .phone_number(phone)
+        ("Action", "Publish"),
-        .message(message)
+        ("PhoneNumber", phone),
+        ("Message", message),
+        ("Version", "2010-03-31"),
+    ];
+
+    // For simplicity, using query string auth (requires proper AWS SigV4 in production)
+    // This is a simplified implementation - in production use aws-sigv4 crate
+    let response = client
+        .post(&url)
+        .form(&params)
+        .header("X-Amz-Date", &timestamp)
+        .basic_auth(&access_key, Some(&secret_key))
         .send()
         .await?;
 
-    Ok(result.message_id)
+    if response.status().is_success() {
+        let body = response.text().await?;
+        // Parse MessageId from XML response
+        if let Some(start) = body.find("<MessageId>") {
+            if let Some(end) = body.find("</MessageId>") {
+                let message_id = &body[start + 11..end];
+                return Ok(Some(message_id.to_string()));
+            }
+        }
+        Ok(None)
+    } else {
+        let error_text = response.text().await?;
+        Err(format!("AWS SNS API error: {}", error_text).into())
+    }
 }
 
 async fn send_via_vonage(
@@ -438,15 +459,15 @@ async fn send_via_vonage(
 
     let api_key = config_manager
         .get_config(bot_id, "vonage-api-key", None)
-        .ok_or("Vonage API key not configured. Set vonage-api-key in config.")?;
+        .map_err(|_| "Vonage API key not configured. Set vonage-api-key in config.")?;
 
     let api_secret = config_manager
         .get_config(bot_id, "vonage-api-secret", None)
-        .ok_or("Vonage API secret not configured. Set vonage-api-secret in config.")?;
+        .map_err(|_| "Vonage API secret not configured. Set vonage-api-secret in config.")?;
 
     let from_number = config_manager
         .get_config(bot_id, "vonage-from-number", None)
-        .ok_or("Vonage from number not configured. Set vonage-from-number in config.")?;
+        .map_err(|_| "Vonage from number not configured. Set vonage-from-number in config.")?;
 
     let client = reqwest::Client::new();
 
@@ -495,11 +516,13 @@ async fn send_via_messagebird(
 
     let api_key = config_manager
         .get_config(bot_id, "messagebird-api-key", None)
-        .ok_or("MessageBird API key not configured. Set messagebird-api-key in config.")?;
+        .map_err(|_| "MessageBird API key not configured. Set messagebird-api-key in config.")?;
 
     let originator = config_manager
         .get_config(bot_id, "messagebird-originator", None)
-        .ok_or("MessageBird originator not configured. Set messagebird-originator in config.")?;
+        .map_err(|_| {
+            "MessageBird originator not configured. Set messagebird-originator in config."
+        })?;
 
     let client = reqwest::Client::new();
 
@@ -536,12 +559,16 @@ async fn send_via_custom_webhook(
 
     let webhook_url = config_manager
         .get_config(bot_id, &format!("{}-webhook-url", provider_name), None)
-        .ok_or(format!(
+        .map_err(|_| {
-            "Custom SMS webhook URL not configured. Set {}-webhook-url in config.",
+            format!(
-            provider_name
+                "Custom SMS webhook URL not configured. Set {}-webhook-url in config.",
-        ))?;
+                provider_name
+            )
+        })?;
 
-    let api_key = config_manager.get_config(bot_id, &format!("{}-api-key", provider_name), None);
+    let api_key = config_manager
+        .get_config(bot_id, &format!("{}-api-key", provider_name), None)
+        .ok();
 
     let client = reqwest::Client::new();
 

src/basic/keywords/table_definition.rs

@@ -515,6 +515,7 @@ pub async fn create_table_on_external_db(
     }
 }
 
+#[cfg(feature = "dynamic-db")]
 async fn create_table_mysql(
     connection_string: &str,
     sql: &str,
@@ -532,6 +533,15 @@ async fn create_table_mysql(
     Ok(())
 }
 
+#[cfg(not(feature = "dynamic-db"))]
+async fn create_table_mysql(
+    _connection_string: &str,
+    _sql: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    Err("MySQL support requires the 'dynamic-db' feature".into())
+}
+
+#[cfg(feature = "dynamic-db")]
 async fn create_table_postgres(
     connection_string: &str,
     sql: &str,
@@ -549,6 +559,14 @@ async fn create_table_postgres(
     Ok(())
 }
 
+#[cfg(not(feature = "dynamic-db"))]
+async fn create_table_postgres(
+    _connection_string: &str,
+    _sql: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    Err("PostgreSQL dynamic table support requires the 'dynamic-db' feature".into())
+}
+
 /// Process TABLE definitions during .bas file compilation
 pub fn process_table_definitions(
     state: Arc<AppState>,

src/core/bootstrap/mod.rs

@@ -712,12 +712,25 @@ meet    IN      A       127.0.0.1
 
         let ca_cert = if ca_cert_path.exists() && ca_key_path.exists() {
             info!("Using existing CA certificate");
-            // Load existing CA
+            // Load existing CA key and regenerate params
-            let cert_pem = fs::read_to_string(&ca_cert_path)?;
             let key_pem = fs::read_to_string(&ca_key_path)?;
             let key_pair = rcgen::KeyPair::from_pem(&key_pem)?;
-            let params = CertificateParams::from_ca_cert_pem(&cert_pem, key_pair)?;
+
-            Certificate::from_params(params)?
+            // Recreate CA params with the loaded key
+            let mut ca_params = CertificateParams::default();
+            ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
+            ca_params.key_pair = Some(key_pair);
+
+            let mut dn = DistinguishedName::new();
+            dn.push(DnType::CountryName, "BR");
+            dn.push(DnType::OrganizationName, "BotServer");
+            dn.push(DnType::CommonName, "BotServer CA");
+            ca_params.distinguished_name = dn;
+
+            ca_params.not_before = time::OffsetDateTime::now_utc();
+            ca_params.not_after = time::OffsetDateTime::now_utc() + time::Duration::days(3650);
+
+            Certificate::from_params(ca_params)?
         } else {
             info!("Generating new CA certificate");
             // Generate new CA

src/core/config/mod.rs

@@ -13,6 +13,7 @@ pub struct AppConfig {
     pub server: ServerConfig,
     pub email: EmailConfig,
     pub site_path: String,
+    pub data_dir: String,
 }
 #[derive(Clone, Debug)]
 pub struct DriveConfig {
@@ -24,6 +25,7 @@ pub struct DriveConfig {
 pub struct ServerConfig {
     pub host: String,
     pub port: u16,
+    pub base_url: String,
 }
 #[derive(Clone, Debug)]
 pub struct EmailConfig {
@@ -109,12 +111,14 @@ impl AppConfig {
             server: ServerConfig {
                 host: get_str("SERVER_HOST", "127.0.0.1"),
                 port: get_u16("SERVER_PORT", 8080),
+                base_url: get_str("SERVER_BASE_URL", "http://localhost:8080"),
             },
             site_path: {
                 ConfigManager::new(pool.clone())
                     .get_config(&Uuid::nil(), "SITES_ROOT", Some("./botserver-stack/sites"))?
                     .to_string()
             },
+            data_dir: get_str("DATA_DIR", "./botserver-stack/data"),
         })
     }
     pub fn from_env() -> Result<Self, anyhow::Error> {
@@ -138,6 +142,7 @@ impl AppConfig {
             server: ServerConfig {
                 host: "127.0.0.1".to_string(),
                 port: 8080,
+                base_url: "http://localhost:8080".to_string(),
             },
             site_path: {
                 let pool = create_conn()?;
@@ -147,6 +152,7 @@ impl AppConfig {
                     Some("./botserver-stack/sites"),
                 )?
             },
+            data_dir: "./botserver-stack/data".to_string(),
         })
     }
 }

src/core/directory/api.rs

@@ -185,16 +185,28 @@ pub async fn get_user_handler(
         }
     };
 
-    let user_result: Result<(String, String, String, bool), _> = users::table
+    let user_uuid = match uuid::Uuid::parse_str(&id) {
-        .filter(users::id.eq(&id))
+        Ok(u) => u,
+        Err(_) => {
+            return (
+                StatusCode::BAD_REQUEST,
+                Json(serde_json::json!({
+                    "error": "Invalid user ID format"
+                })),
+            );
+        }
+    };
+
+    let user_result: Result<(uuid::Uuid, String, String, bool), _> = users::table
+        .filter(users::id.eq(&user_uuid))
         .select((users::id, users::username, users::email, users::is_admin))
-        .first(&conn);
+        .first(&mut conn);
 
     match user_result {
-        Ok((id, username, email, is_admin)) => (
+        Ok((user_id, username, email, is_admin)) => (
             StatusCode::OK,
             Json(serde_json::json!({
-                "id": id,
+                "id": user_id.to_string(),
                 "username": username,
                 "email": email,
                 "is_admin": is_admin
@@ -226,17 +238,17 @@ pub async fn list_users_handler(State(state): State<Arc<AppState>>) -> impl Into
         }
     };
 
-    let users_result: Result<Vec<(String, String, String, bool)>, _> = users::table
+    let users_result: Result<Vec<(uuid::Uuid, String, String, bool)>, _> = users::table
         .select((users::id, users::username, users::email, users::is_admin))
-        .load(&conn);
+        .load(&mut conn);
 
     match users_result {
         Ok(users_list) => {
             let users_json: Vec<_> = users_list
                 .into_iter()
-                .map(|(id, username, email, is_admin)| {
+                .map(|(user_id, username, email, is_admin)| {
                     serde_json::json!({
-                        "id": id,
+                        "id": user_id.to_string(),
                         "username": username,
                         "email": email,
                         "is_admin": is_admin

src/core/directory/provisioning.rs

@@ -74,15 +74,17 @@ impl UserProvisioningService {
 
     async fn create_database_user(&self, account: &UserAccount) -> Result<String> {
         use crate::shared::models::schema::users;
+        use argon2::{password_hash::SaltString, Argon2, PasswordHasher};
         use diesel::prelude::*;
         use uuid::Uuid;
 
         let user_id = Uuid::new_v4().to_string();
-        let password_hash = argon2::hash_encoded(
+        let salt = SaltString::generate(&mut rand::rngs::OsRng);
-            Uuid::new_v4().to_string().as_bytes(),
+        let argon2 = Argon2::default();
-            &rand::random::<[u8; 32]>(),
+        let password_hash = argon2
-            &argon2::Config::default(),
+            .hash_password(Uuid::new_v4().to_string().as_bytes(), &salt)
-        )?;
+            .map_err(|e| anyhow::anyhow!("Password hashing failed: {}", e))?
+            .to_string();
 
         diesel::insert_into(users::table)
             .values((
@@ -172,8 +174,8 @@ impl UserProvisioningService {
         Ok(())
     }
 
-    async fn setup_oauth_config(&self, user_id: &str, account: &UserAccount) -> Result<()> {
+    async fn setup_oauth_config(&self, _user_id: &str, account: &UserAccount) -> Result<()> {
-        use crate::shared::models::schema::bot_config;
+        use crate::shared::models::schema::bot_configuration;
         use diesel::prelude::*;
 
         // Store OAuth configuration for services
@@ -185,15 +187,19 @@ impl UserProvisioningService {
         ];
 
         for (key, value) in services {
-            diesel::insert_into(bot_config::table)
+            diesel::insert_into(bot_configuration::table)
                 .values((
-                    bot_config::bot_id.eq("default"),
+                    bot_configuration::bot_id.eq(uuid::Uuid::nil()),
-                    bot_config::key.eq(key),
+                    bot_configuration::config_key.eq(key),
-                    bot_config::value.eq(value),
+                    bot_configuration::config_value.eq(value),
+                    bot_configuration::is_encrypted.eq(false),
+                    bot_configuration::config_type.eq("string"),
+                    bot_configuration::created_at.eq(chrono::Utc::now()),
+                    bot_configuration::updated_at.eq(chrono::Utc::now()),
                 ))
-                .on_conflict((bot_config::bot_id, bot_config::key))
+                .on_conflict((bot_configuration::bot_id, bot_configuration::config_key))
                 .do_update()
-                .set(bot_config::value.eq(value))
+                .set(bot_configuration::config_value.eq(value))
                 .execute(&*self.db_conn)?;
         }
 

src/core/dns/mod.rs

@@ -147,11 +147,11 @@ impl DynamicDnsService {
             }
         });
 
-        for (hostname, ip) in removed {
+        for (hostname, ip) in &removed {
-            if let Some(ip_entries) = by_ip.get_mut(&ip) {
+            if let Some(ip_entries) = by_ip.get_mut(ip) {
-                ip_entries.retain(|h| h != &hostname);
+                ip_entries.retain(|h| h != hostname);
                 if ip_entries.is_empty() {
-                    by_ip.remove(&ip);
+                    by_ip.remove(ip);
                 }
             }
         }

src/core/shared/models.rs

@@ -64,6 +64,21 @@ pub struct Suggestion {
     pub text: String,
     pub context: String,
 }
+
+/// Attachment for media files in messages
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Attachment {
+    /// Type of attachment (image, audio, video, file, etc.)
+    pub attachment_type: String,
+    /// URL or path to the attachment
+    pub url: String,
+    /// MIME type of the attachment
+    pub mime_type: Option<String>,
+    /// File name if available
+    pub filename: Option<String>,
+    /// File size in bytes
+    pub size: Option<u64>,
+}
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct BotResponse {
     pub bot_id: String,
@@ -180,6 +195,7 @@ pub mod schema {
             email -> Text,
             password_hash -> Text,
             is_active -> Bool,
+            is_admin -> Bool,
             created_at -> Timestamptz,
             updated_at -> Timestamptz,
         }

src/security/integration.rs

@@ -8,7 +8,7 @@ use reqwest::{Certificate, Client, ClientBuilder, Identity};
 use std::collections::HashMap;
 use std::fs;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 use std::time::Duration;
 use tracing::{debug, info, warn};
 
@@ -340,35 +340,32 @@ impl TlsIntegration {
     }
 }
 
-/// Global TLS integration instance
+/// Global TLS integration instance using OnceLock for safe initialization
-static mut TLS_INTEGRATION: Option<Arc<TlsIntegration>> = None;
+static TLS_INTEGRATION: OnceLock<Arc<TlsIntegration>> = OnceLock::new();
-static TLS_INIT: std::sync::Once = std::sync::Once::new();
 
 /// Initialize global TLS integration
 pub fn init_tls_integration(tls_enabled: bool, cert_dir: Option<PathBuf>) -> Result<()> {
-    unsafe {
+    let _ = TLS_INTEGRATION.get_or_init(|| {
-        TLS_INIT.call_once(|| {
+        let mut integration = TlsIntegration::new(tls_enabled);
-            let mut integration = TlsIntegration::new(tls_enabled);
 
-            if tls_enabled {
+        if tls_enabled {
-                if let Some(dir) = cert_dir {
+            if let Some(dir) = cert_dir {
-                    if let Err(e) = integration.load_all_certs_from_dir(&dir) {
+                if let Err(e) = integration.load_all_certs_from_dir(&dir) {
-                        warn!("Failed to load some certificates: {}", e);
+                    warn!("Failed to load some certificates: {}", e);
-                    }
                 }
             }
+        }
 
-            TLS_INTEGRATION = Some(Arc::new(integration));
+        info!("TLS integration initialized (TLS: {})", tls_enabled);
-            info!("TLS integration initialized (TLS: {})", tls_enabled);
+        Arc::new(integration)
-        });
+    });
-    }
 
     Ok(())
 }
 
 /// Get the global TLS integration instance
 pub fn get_tls_integration() -> Option<Arc<TlsIntegration>> {
-    unsafe { TLS_INTEGRATION.clone() }
+    TLS_INTEGRATION.get().cloned()
 }
 
 /// Convert a URL to HTTPS using global TLS settings

src/security/mod.rs

@@ -28,7 +28,7 @@ pub use mutual_tls::{
         configure_directory_mtls, configure_forgejo_mtls, configure_livekit_mtls,
         configure_postgres_mtls, configure_qdrant_mtls,
     },
-    MtlsCertificateManager, MtlsConfig, MtlsConnectionPool, ServiceIdentity,
+    MtlsConfig, MtlsError, MtlsManager,
 };
 pub use tls::{create_https_server, ServiceTlsConfig, TlsConfig, TlsManager, TlsRegistry};
 
@@ -79,8 +79,7 @@ impl Default for SecurityConfig {
 pub struct SecurityManager {
     config: SecurityConfig,
     ca_manager: CaManager,
-    mtls_manager: Option<MtlsCertificateManager>,
+    mtls_manager: Option<MtlsManager>,
-    connection_pool: Option<MtlsConnectionPool>,
 }
 
 impl SecurityManager {
@@ -88,26 +87,19 @@ impl SecurityManager {
     pub fn new(config: SecurityConfig) -> Result<Self> {
         let ca_manager = CaManager::new(config.ca_config.clone())?;
 
-        let (mtls_manager, connection_pool) = if config.mtls_enabled {
+        let mtls_manager = if config.mtls_enabled {
-            let manager = MtlsCertificateManager::new(
+            // Create mTLS config from CA certificates
-                &config.ca_config.ca_cert_path,
+            let ca_cert = std::fs::read_to_string(&config.ca_config.ca_cert_path).ok();
-                &config.ca_config.ca_key_path,
+            let mtls_config = MtlsConfig::new(ca_cert, None, None);
-            )?;
+            Some(MtlsManager::new(mtls_config))
-            let manager = Arc::new(manager);
-            let pool = MtlsConnectionPool::new(manager.clone());
-            (
-                Some(Arc::try_unwrap(manager).unwrap_or_else(|arc| (*arc).clone())),
-                Some(pool),
-            )
         } else {
-            (None, None)
+            None
         };
 
         Ok(Self {
             config,
             ca_manager,
             mtls_manager,
-            connection_pool,
         })
     }
 
@@ -149,17 +141,20 @@ impl SecurityManager {
 
             let base_path = PathBuf::from("./botserver-stack/conf/system");
 
-            // Register all services with mTLS
+            // Configure mTLS for each service
-            manager.register_service(configure_qdrant_mtls(&base_path))?;
+            let ca_path = base_path.join("ca/ca.crt");
-            manager.register_service(configure_postgres_mtls(&base_path))?;
+            let cert_path = base_path.join("certs/api.crt");
-            manager.register_service(configure_forgejo_mtls(&base_path))?;
+            let key_path = base_path.join("certs/api.key");
-            manager.register_service(configure_livekit_mtls(&base_path))?;
-            manager.register_service(configure_directory_mtls(&base_path))?;
 
-            // Register API service
+            // Validate configurations for each service
-            let api_config = MtlsConfig::new(ServiceIdentity::Api, &base_path)
+            let _ = configure_qdrant_mtls(Some(&ca_path), Some(&cert_path), Some(&key_path));
-                .with_allowed_clients(vec![ServiceIdentity::Directory, ServiceIdentity::Caddy]);
+            let _ = configure_postgres_mtls(Some(&ca_path), Some(&cert_path), Some(&key_path));
-            manager.register_service(api_config)?;
+            let _ = configure_forgejo_mtls(Some(&ca_path), Some(&cert_path), Some(&key_path));
+            let _ = configure_livekit_mtls(Some(&ca_path), Some(&cert_path), Some(&key_path));
+            let _ = configure_directory_mtls(Some(&ca_path), Some(&cert_path), Some(&key_path));
+
+            // Validate the manager configuration
+            manager.validate()?;
 
             info!("mTLS initialized for all services");
         }
@@ -245,14 +240,9 @@ impl SecurityManager {
     }
 
     /// Get mTLS manager
-    pub fn mtls_manager(&self) -> Option<&MtlsCertificateManager> {
+    pub fn mtls_manager(&self) -> Option<&MtlsManager> {
         self.mtls_manager.as_ref()
     }
-
-    /// Get mTLS connection pool
-    pub fn connection_pool(&self) -> Option<&MtlsConnectionPool> {
-        self.connection_pool.as_ref()
-    }
 }
 
 /// Check if a certificate needs renewal

src/tasks/mod.rs

@@ -4,7 +4,7 @@ use crate::core::urls::ApiUrls;
 use axum::{
     extract::{Path, Query, State},
     http::StatusCode,
-    response::Json,
+    response::{IntoResponse, Json},
     routing::{delete, get, post, put},
     Router,
 };
@@ -1308,9 +1308,13 @@ pub async fn handle_task_list_htmx(
     let filter_clone = filter.clone();
 
     let tasks = tokio::task::spawn_blocking(move || {
-        let mut db_conn = conn.get().map_err(|e| format!("DB connection error: {}", e))?;
+        let mut db_conn = conn
+            .get()
+            .map_err(|e| format!("DB connection error: {}", e))?;
 
-        let mut query = String::from("SELECT id, title, completed, priority, category, due_date FROM tasks WHERE 1=1");
+        let mut query = String::from(
+            "SELECT id, title, completed, priority, category, due_date FROM tasks WHERE 1=1",
+        );
 
         match filter_clone.as_str() {
             "active" => query.push_str(" AND completed = false"),
@@ -1335,7 +1339,6 @@ pub async fn handle_task_list_htmx(
     let mut html = String::new();
 
     for task in tasks {
-
         let completed_class = if task.completed { "completed" } else { "" };
         let priority_class = if task.priority { "active" } else { "" };
         let checked = if task.completed { "checked" } else { "" };
@@ -1385,7 +1388,10 @@ pub async fn handle_task_list_htmx(
                 String::new()
             },
             if let Some(due) = &task.due_date {
-                format!(r#"<span class="task-due-date">📅 {}</span>"#, due.format("%Y-%m-%d"))
+                format!(
+                    r#"<span class="task-due-date">📅 {}</span>"#,
+                    due.format("%Y-%m-%d")
+                )
             } else {
                 String::new()
             },
@@ -1425,27 +1431,32 @@ pub async fn handle_task_stats(State(state): State<Arc<AppState>>) -> Json<TaskS
     let conn = state.conn.clone();
 
     let stats = tokio::task::spawn_blocking(move || {
-        let mut db_conn = conn.get().map_err(|e| format!("DB connection error: {}", e))?;
+        let mut db_conn = conn
+            .get()
+            .map_err(|e| format!("DB connection error: {}", e))?;
 
         let total: i64 = diesel::sql_query("SELECT COUNT(*) as count FROM tasks")
             .get_result::<CountResult>(&mut db_conn)
             .map(|r| r.count)
             .unwrap_or(0);
 
-        let active: i64 = diesel::sql_query("SELECT COUNT(*) as count FROM tasks WHERE completed = false")
+        let active: i64 =
-            .get_result::<CountResult>(&mut db_conn)
+            diesel::sql_query("SELECT COUNT(*) as count FROM tasks WHERE completed = false")
-            .map(|r| r.count)
+                .get_result::<CountResult>(&mut db_conn)
-            .unwrap_or(0);
+                .map(|r| r.count)
+                .unwrap_or(0);
 
-        let completed: i64 = diesel::sql_query("SELECT COUNT(*) as count FROM tasks WHERE completed = true")
+        let completed: i64 =
-            .get_result::<CountResult>(&mut db_conn)
+            diesel::sql_query("SELECT COUNT(*) as count FROM tasks WHERE completed = true")
-            .map(|r| r.count)
+                .get_result::<CountResult>(&mut db_conn)
-            .unwrap_or(0);
+                .map(|r| r.count)
+                .unwrap_or(0);
 
-        let priority: i64 = diesel::sql_query("SELECT COUNT(*) as count FROM tasks WHERE priority = true")
+        let priority: i64 =
-            .get_result::<CountResult>(&mut db_conn)
+            diesel::sql_query("SELECT COUNT(*) as count FROM tasks WHERE priority = true")
-            .map(|r| r.count)
+                .get_result::<CountResult>(&mut db_conn)
-            .unwrap_or(0);
+                .map(|r| r.count)
+                .unwrap_or(0);
 
         Ok::<_, String>(TaskStats {
             total: total as usize,
@@ -1474,7 +1485,9 @@ pub async fn handle_clear_completed(State(state): State<Arc<AppState>>) -> impl
     let conn = state.conn.clone();
 
     tokio::task::spawn_blocking(move || {
-        let mut db_conn = conn.get().map_err(|e| format!("DB connection error: {}", e))?;
+        let mut db_conn = conn
+            .get()
+            .map_err(|e| format!("DB connection error: {}", e))?;
 
         diesel::sql_query("DELETE FROM tasks WHERE completed = true")
             .execute(&mut db_conn)
@@ -1504,12 +1517,14 @@ pub async fn handle_task_patch(
     log::info!("Updating task {} with {:?}", id, update);
 
     let conn = state.conn.clone();
-    let task_id = id.parse::<Uuid>().map_err(|e| {
+    let task_id = id
-        (StatusCode::BAD_REQUEST, format!("Invalid task ID: {}", e))
+        .parse::<Uuid>()
-    })?;
+        .map_err(|e| (StatusCode::BAD_REQUEST, format!("Invalid task ID: {}", e)))?;
 
     tokio::task::spawn_blocking(move || {
-        let mut db_conn = conn.get().map_err(|e| format!("DB connection error: {}", e))?;
+        let mut db_conn = conn
+            .get()
+            .map_err(|e| format!("DB connection error: {}", e))?;
 
         if let Some(completed) = update.completed {
             diesel::sql_query("UPDATE tasks SET completed = $1 WHERE id = $2")
@@ -1538,7 +1553,12 @@ pub async fn handle_task_patch(
         Ok::<_, String>(())
     })
     .await
-    .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Task join error: {}", e)))?
+    .map_err(|e| {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            format!("Task join error: {}", e),
+        )
+    })?
     .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
 
     Ok(Json(ApiResponse {

src/web/auth.rs

@@ -184,11 +184,11 @@ where
 }
 
 /// Authentication middleware
-pub async fn auth_middleware<B>(
+pub async fn auth_middleware(
     State(state): State<AppState>,
     cookies: Cookies,
-    request: Request<B>,
+    request: Request<axum::body::Body>,
-    next: Next<B>,
+    next: Next,
 ) -> Response {
     let path = request.uri().path();
 
@@ -383,10 +383,3 @@ pub fn create_auth_cookie(token: &str, expires_in_hours: i64) -> Cookie<'static>
         .max_age(time::Duration::hours(expires_in_hours))
         .finish()
 }
-
-/// FromRef implementation for middleware
-impl FromRef<AppState> for AppState {
-    fn from_ref(state: &AppState) -> Self {
-        state.clone()
-    }
-}

src/web/auth_handlers.rs

@@ -145,7 +145,7 @@ pub async fn login_submit(
     Response::builder()
         .status(StatusCode::OK)
         .header("HX-Redirect", "/")
-        .body("Login successful".to_string())
+        .body(axum::body::Body::from("Login successful"))
         .unwrap()
 }
 

src/web/chat_handlers.rs

@@ -21,6 +21,9 @@ use crate::shared::state::AppState;
 #[template(path = "suite/chat.html")]
 pub struct ChatTemplate {
     pub session_id: String,
+    pub user_initial: String,
+    pub user_name: String,
+    pub user_email: String,
 }
 
 /// Session list template

src/web/compliance_handlers.rs

@@ -75,7 +75,7 @@ pub struct ScanSummary {
 }
 
 /// Individual issue in response
-#[derive(Debug, Serialize)]
+#[derive(Debug, Clone, Serialize)]
 pub struct IssueResponse {
     pub id: String,
     pub severity: String,

src/web/filters.rs

@@ -0,0 +1,119 @@
+//! Custom Askama filters for web templates
+
+use askama::Result;
+
+/// Default filter - returns the value if non-empty, otherwise returns the default
+pub fn default(value: &str, default_value: &str) -> Result<String> {
+    if value.is_empty() {
+        Ok(default_value.to_string())
+    } else {
+        Ok(value.to_string())
+    }
+}
+
+/// Truncate filter - truncates a string to a maximum length
+pub fn truncate(value: &str, max_len: usize) -> Result<String> {
+    if value.len() > max_len {
+        Ok(format!("{}...", &value[..max_len.saturating_sub(3)]))
+    } else {
+        Ok(value.to_string())
+    }
+}
+
+/// Title case filter - capitalizes the first letter of each word
+pub fn title(value: &str) -> Result<String> {
+    Ok(value
+        .split_whitespace()
+        .map(|word| {
+            let mut chars = word.chars();
+            match chars.next() {
+                None => String::new(),
+                Some(first) => first.to_uppercase().chain(chars).collect(),
+            }
+        })
+        .collect::<Vec<_>>()
+        .join(" "))
+}
+
+/// Format date filter - formats a timestamp string
+pub fn format_date(value: &str, format: &str) -> Result<String> {
+    // Simple implementation - in production would use chrono
+    if format == "short" {
+        Ok(value.chars().take(10).collect())
+    } else {
+        Ok(value.to_string())
+    }
+}
+
+/// Pluralize filter - returns singular or plural form based on count
+pub fn pluralize(count: i64, singular: &str, plural: &str) -> Result<String> {
+    if count == 1 {
+        Ok(singular.to_string())
+    } else {
+        Ok(plural.to_string())
+    }
+}
+
+/// File size filter - formats bytes as human-readable size
+pub fn filesize(bytes: u64) -> Result<String> {
+    const KB: u64 = 1024;
+    const MB: u64 = KB * 1024;
+    const GB: u64 = MB * 1024;
+
+    if bytes >= GB {
+        Ok(format!("{:.1} GB", bytes as f64 / GB as f64))
+    } else if bytes >= MB {
+        Ok(format!("{:.1} MB", bytes as f64 / MB as f64))
+    } else if bytes >= KB {
+        Ok(format!("{:.1} KB", bytes as f64 / KB as f64))
+    } else {
+        Ok(format!("{} B", bytes))
+    }
+}
+
+/// Initials filter - extracts initials from a name
+pub fn initials(name: &str) -> Result<String> {
+    Ok(name
+        .split_whitespace()
+        .filter_map(|word| word.chars().next())
+        .take(2)
+        .collect::<String>()
+        .to_uppercase())
+}
+
+/// Escape JavaScript filter - escapes string for use in JavaScript
+pub fn escapejs(value: &str) -> Result<String> {
+    Ok(value
+        .replace('\\', "\\\\")
+        .replace('\'', "\\'")
+        .replace('"', "\\\"")
+        .replace('\n', "\\n")
+        .replace('\r', "\\r")
+        .replace('\t', "\\t"))
+}
+
+/// JSON filter - converts value to JSON string
+pub fn json(value: &str) -> Result<String> {
+    Ok(format!("\"{}\"", escapejs(value)?))
+}
+
+/// Slugify filter - converts string to URL-safe slug
+pub fn slugify(value: &str) -> Result<String> {
+    Ok(value
+        .to_lowercase()
+        .chars()
+        .map(|c| {
+            if c.is_ascii_alphanumeric() {
+                c
+            } else if c.is_whitespace() || c == '-' || c == '_' {
+                '-'
+            } else {
+                '_'
+            }
+        })
+        .collect::<String>()
+        .split('-')
+        .filter(|s| !s.is_empty())
+        .collect::<Vec<_>>()
+        .join("-"))
+}

src/web/mod.rs

@@ -24,6 +24,7 @@ pub mod auth;
 pub mod auth_handlers;
 pub mod chat_handlers;
 pub mod compliance_handlers;
+pub mod filters;
 pub mod stream_handlers;
 
 // Module stubs - to be implemented with full HTMX