From 6bc6a35948a58d00b452737aaafde3a4b26fafe9 Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (Pragmatismo)" Date: Wed, 17 Dec 2025 17:41:37 -0300 Subject: [PATCH] fix: resolve all warnings - wire up services properly --- config/directory_config.json | 8 +- docs/AUTO_TASK_ARCHITECTURE.md | 523 ------------------------- docs/README.md | 62 --- docs/api/README.md | 104 ----- docs/api/rest-endpoints.md | 599 ----------------------------- docs/api/websocket.md | 362 ----------------- docs/guides/deployment.md | 465 ---------------------- docs/guides/getting-started.md | 248 ------------ docs/guides/templates.md | 308 --------------- docs/reference/architecture.md | 530 ------------------------- docs/reference/basic-language.md | 525 ------------------------- docs/reference/configuration.md | 344 ----------------- src/analytics/mod.rs | 542 +++++++++++++------------- src/basic/keywords/create_site.rs | 483 ++++++++++++++++++++++- src/console/editor.rs | 23 +- src/console/mod.rs | 84 ++-- src/core/kb/document_processor.rs | 19 - src/core/secrets/mod.rs | 53 ++- src/core/shared/state.rs | 73 +--- src/core/shared/test_utils.rs | 2 +- src/directory/client.rs | 1 - src/directory/groups.rs | 1 - src/directory/mod.rs | 4 +- src/directory/router.rs | 1 - src/directory/users.rs | 1 - src/drive/drive_monitor/mod.rs | 1 - src/drive/mod.rs | 2 - src/drive/vectordb.rs | 55 +-- src/llm/episodic_memory.rs | 1 - src/llm/llm_models/deepseek_r3.rs | 1 - src/llm/llm_models/gpt_oss_120b.rs | 1 - src/llm/llm_models/gpt_oss_20b.rs | 1 - src/llm/llm_models/mod.rs | 1 - src/llm/mod.rs | 210 +++++----- src/llm/observability.rs | 12 + src/main.rs | 85 ++-- src/monitoring/mod.rs | 401 +++++++++++++++++++ 37 files changed, 1430 insertions(+), 4706 deletions(-) delete mode 100644 docs/AUTO_TASK_ARCHITECTURE.md delete mode 100644 docs/README.md delete mode 100644 docs/api/README.md delete mode 100644 docs/api/rest-endpoints.md delete mode 100644 docs/api/websocket.md delete mode 100644 docs/guides/deployment.md delete mode 100644 docs/guides/getting-started.md delete mode 100644 docs/guides/templates.md delete mode 100644 docs/reference/architecture.md delete mode 100644 docs/reference/basic-language.md delete mode 100644 docs/reference/configuration.md create mode 100644 src/monitoring/mod.rs diff --git a/config/directory_config.json b/config/directory_config.json index 55cae3556..988c97572 100644 --- a/config/directory_config.json +++ b/config/directory_config.json @@ -1,7 +1,7 @@ { "base_url": "http://localhost:8300", "default_org": { - "id": "351198556452814862", + "id": "351468402772017166", "name": "default", "domain": "default.localhost" }, @@ -13,8 +13,8 @@ "first_name": "Admin", "last_name": "User" }, - "admin_token": "T9uwHH2GxTOQwv6NbYn7MoBJM6AsuxRHtyKOajb-SKSZwhGel4DtxUmiDwZZnNo08ryg4J4", + "admin_token": "2YJqHuenWddFpMw4vqw6vEHtgSF5jbvSG4NxTANnV9KJJMnaDSuvbUNSGsS06-QLFZnpFbw", "project_id": "", - "client_id": "351198557056860174", - "client_secret": "WXUxRkjyMKTuotp0m1HbjirldLxiybnasTyVe8BwhZKEYHXveNUfKmohOcZ6FQPC" + "client_id": "351468407201267726", + "client_secret": "vLxjxWiPv8fVvown7zBOqKdb7RPntqVW8fNfphaiMWtkXFI8fXQX8WoyBE5KmhJA" } \ No newline at end of file diff --git a/docs/AUTO_TASK_ARCHITECTURE.md b/docs/AUTO_TASK_ARCHITECTURE.md deleted file mode 100644 index f66868c0a..000000000 --- a/docs/AUTO_TASK_ARCHITECTURE.md +++ /dev/null @@ -1,523 +0,0 @@ -# Auto Task Architecture - LLM-to-BASIC Intent Compiler - -## Overview - -The Auto Task system is a revolutionary approach to task automation that translates natural language intents into executable BASIC programs. This document describes the complete architecture for the "Premium VIP Mode" intelligent task execution system. - -## Core Components - -``` -┌─────────────────────────────────────────────────────────────────────────────┐ -│ USER INTERFACE │ -│ ┌─────────────────────────────────────────────────────────────────────┐ │ -│ │ Natural Language Intent Input │ │ -│ │ "Make a financial CRM for Deloitte with client tracking & reports" │ │ -│ └─────────────────────────────────────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ INTENT COMPILER │ -│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ ┌─────────────┐ │ -│ │ Entity │→│ Plan │→│ BASIC Code │→│ Risk │ │ -│ │ Extraction │ │ Generation │ │ Generation │ │ Assessment │ │ -│ └───────────────┘ └───────────────┘ └───────────────┘ └─────────────┘ │ -└─────────────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ SAFETY LAYER │ -│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ ┌─────────────┐ │ -│ │ Constraint │ │ Impact │ │ Approval │ │ Audit │ │ -│ │ Checker │ │ Simulator │ │ Workflow │ │ Trail │ │ -│ └───────────────┘ └───────────────┘ └───────────────┘ └─────────────┘ │ -└─────────────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ EXECUTION ENGINE │ -│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ ┌─────────────┐ │ -│ │ BASIC │ │ MCP Client │ │ API Gateway │ │ State │ │ -│ │ Interpreter │ │ (MCP Servers) │ │ (External) │ │ Manager │ │ -│ └───────────────┘ └───────────────┘ └───────────────┘ └─────────────┘ │ -└─────────────────────────────────────────────────────────────────────────────┘ -``` - -## 1. Intent Compiler (`intent_compiler.rs`) - -The Intent Compiler is the "magic" component that translates natural language into executable BASIC programs. - -### Flow - -1. **Entity Extraction** - Uses LLM to extract: - - Action (create, update, delete, analyze) - - Target (CRM, website, report, API) - - Domain (financial, healthcare, retail) - - Client name - - Features requested - - Constraints (budget, timeline, technology) - - Required integrations - -2. **Plan Generation** - Creates an execution plan with: - - Ordered steps - - Dependencies between steps - - Resource estimates - - Risk assessment per step - - Required approvals - -3. **BASIC Code Generation** - Generates executable code using: - - Existing 80+ keywords - - New safety keywords (REQUIRE_APPROVAL, SIMULATE_IMPACT) - - New MCP keywords (USE_MCP, MCP_INVOKE) - -### Example Transformation - -**Input:** -``` -"Make a financial CRM for Deloitte with client management and reporting" -``` - -**Generated BASIC Program:** -```basic -' ============================================================================= -' AUTO-GENERATED BASIC PROGRAM -' Plan: Financial CRM for Deloitte -' Generated: 2024-01-15 10:30:00 -' ============================================================================= - -PLAN_START "Financial CRM for Deloitte", "Client management and reporting system" - STEP 1, "Create database schema", CRITICAL - STEP 2, "Setup authentication", HIGH - STEP 3, "Build client management module", HIGH - STEP 4, "Build financial tracking", MEDIUM - STEP 5, "Create reporting dashboard", MEDIUM - STEP 6, "Deploy and test", LOW -PLAN_END - -' Initialize context -SET action = "create" -SET target = "CRM" -SET client = "Deloitte" -SET domain = "financial" - -SET CONTEXT "Building a financial CRM for Deloitte with client management and reporting" - -' ----------------------------------------------------------------------------- -' STEP 1: Create database schema -' ----------------------------------------------------------------------------- -REQUIRE_APPROVAL "create-database", "Creating database will incur monthly costs" -IF approved THEN - AUDIT_LOG "step-start", "step-1", "Create database schema" - - schema = LLM "Generate PostgreSQL schema for financial CRM with clients, contacts, deals, and reports tables" - - USE_MCP "database", "execute_sql", {"sql": schema} - - AUDIT_LOG "step-complete", "step-1", "Database created" -END IF - -' ----------------------------------------------------------------------------- -' STEP 2: Setup authentication -' ----------------------------------------------------------------------------- -AUDIT_LOG "step-start", "step-2", "Setup authentication" - -RUN_PYTHON " -from auth_setup import configure_oauth -configure_oauth(provider='azure_ad', tenant='deloitte.com') -" - -AUDIT_LOG "step-complete", "step-2", "Authentication configured" - -' Continue with remaining steps... - -TALK "Financial CRM for Deloitte has been created successfully!" -AUDIT_LOG "plan-complete", "financial-crm-deloitte", "success" -``` - -## 2. Safety Layer (`safety_layer.rs`) - -The Safety Layer ensures all actions are validated before execution. - -### Components - -#### Constraint Checker -- **Budget constraints** - Check estimated costs against limits -- **Permission constraints** - Verify user has required access -- **Policy constraints** - Enforce organizational rules -- **Compliance constraints** - Ensure regulatory compliance -- **Technical constraints** - Validate system capabilities -- **Rate limits** - Prevent resource exhaustion - -#### Impact Simulator -Performs dry-run execution to predict: -- Data changes (records created/modified/deleted) -- Cost impact (API calls, compute, storage) -- Time impact (execution duration, blocking) -- Security impact (credentials accessed, external systems) -- Side effects (unintended consequences) - -#### Approval Workflow -Multi-level approval system: -- Plan-level approval -- Step-level approval for high-risk actions -- Override approval for constraint violations -- Timeout handling with configurable defaults - -#### Audit Trail -Complete logging of: -- All task lifecycle events -- Step execution details -- Approval decisions -- Data modifications -- API calls -- Error conditions - -## 3. Execution Engine - -### BASIC Interpreter -Executes the generated BASIC programs with: -- State management across steps -- Error handling and recovery -- Rollback support for reversible actions -- Progress tracking and reporting - -### MCP Client (`mcp_client.rs`) -Integrates with Model Context Protocol servers: - -**Supported Server Types:** -- Database Server (PostgreSQL, MySQL, SQLite) -- Filesystem Server (local, cloud storage) -- Web Server (HTTP/REST APIs) -- Email Server (SMTP/IMAP) -- Slack/Teams Server -- Analytics Server -- Custom servers - -**Example MCP Usage:** -```basic -' Query database via MCP -result = USE_MCP "database", "query", {"sql": "SELECT * FROM clients WHERE status = 'active'"} - -' Send Slack message -USE_MCP "slack", "send_message", {"channel": "#sales", "text": "New client added!"} - -' Upload to S3 -USE_MCP "storage", "upload", {"bucket": "reports", "key": "q4-report.pdf", "file": pdf_data} -``` - -### API Gateway -Handles external API integrations: -- Authentication (API Key, OAuth2, Basic) -- Rate limiting and retry logic -- Response transformation -- Error handling - -## 4. Decision Framework - -When the Intent Compiler detects ambiguity, it generates options: - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ DECISION REQUIRED │ -│ │ -│ Your intent could be interpreted multiple ways: │ -│ │ -│ ┌─────────────────────────────────────────────────────────┐ │ -│ │ OPTION A: Full Custom CRM [RECOMMENDED] │ -│ │ Build from scratch with all requested features │ │ -│ │ ✅ Maximum flexibility │ │ -│ │ ✅ Exactly matches requirements │ │ -│ │ ❌ Higher cost (~$500) │ │ -│ │ ❌ Longer timeline (~2 weeks) │ │ -│ └─────────────────────────────────────────────────────────┘ │ -│ │ -│ ┌─────────────────────────────────────────────────────────┐ │ -│ │ OPTION B: Extend Existing Template │ │ -│ │ Use CRM template and customize │ │ -│ │ ✅ Lower cost (~$100) │ │ -│ │ ✅ Faster delivery (~3 days) │ │ -│ │ ❌ Some limitations on customization │ │ -│ │ ❌ May not fit all requirements │ │ -│ └─────────────────────────────────────────────────────────┘ │ -│ │ -│ [Select Option A] [Select Option B] [Provide More Details] │ -└─────────────────────────────────────────────────────────────────┘ -``` - -## 5. Auto Task Lifecycle - -``` -┌──────────┐ ┌───────────┐ ┌────────────────┐ ┌───────────┐ -│ DRAFT │──▶│ COMPILING │──▶│ PENDING_APPROVAL│──▶│ SIMULATING│ -└──────────┘ └───────────┘ └────────────────┘ └───────────┘ - │ -┌──────────┐ ┌───────────┐ ┌─────────┐ │ -│COMPLETED │◀──│ RUNNING │◀──│ READY │◀─────────────┘ -└──────────┘ └───────────┘ └─────────┘ - ▲ │ │ - │ ▼ ▼ - │ ┌──────────┐ ┌────────────────┐ - │ │ PAUSED │ │WAITING_DECISION│ - │ └──────────┘ └────────────────┘ - │ │ - │ ▼ - │ ┌──────────┐ ┌───────────┐ - └────────│ BLOCKED │ │ FAILED │ - └──────────┘ └───────────┘ -``` - -## 6. New Keywords Added - -### Safety Keywords -| Keyword | Description | Example | -|---------|-------------|---------| -| `REQUIRE_APPROVAL` | Request human approval | `REQUIRE_APPROVAL "action-id", "reason"` | -| `SIMULATE_IMPACT` | Simulate before execute | `result = SIMULATE_IMPACT "step-id"` | -| `CHECK_CONSTRAINTS` | Validate constraints | `CHECK_CONSTRAINTS "budget", 1000` | -| `AUDIT_LOG` | Log to audit trail | `AUDIT_LOG "event", "id", "details"` | - -### MCP Keywords -| Keyword | Description | Example | -|---------|-------------|---------| -| `USE_MCP` | Invoke MCP server tool | `USE_MCP "server", "tool", {params}` | -| `MCP_LIST_TOOLS` | List available tools | `tools = MCP_LIST_TOOLS "server"` | -| `MCP_INVOKE` | Direct tool invocation | `MCP_INVOKE "server.tool", params` | - -### Auto Task Keywords -| Keyword | Description | Example | -|---------|-------------|---------| -| `PLAN_START` | Begin plan declaration | `PLAN_START "name", "description"` | -| `PLAN_END` | End plan declaration | `PLAN_END` | -| `STEP` | Declare a step | `STEP 1, "name", PRIORITY` | -| `AUTO_TASK` | Create auto-executing task | `AUTO_TASK "intent"` | - -### Decision Keywords -| Keyword | Description | Example | -|---------|-------------|---------| -| `OPTION_A_OR_B` | Present options | `OPTION_A_OR_B optA, optB, "question"` | -| `DECIDE` | Get decision result | `choice = DECIDE "decision-id"` | -| `ESCALATE` | Escalate to human | `ESCALATE "reason", assignee` | - -## 7. Database Schema - -```sql --- Auto Tasks -CREATE TABLE auto_tasks ( - id UUID PRIMARY KEY, - title TEXT NOT NULL, - intent TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'draft', - execution_mode TEXT NOT NULL DEFAULT 'semi-automatic', - priority TEXT NOT NULL DEFAULT 'medium', - plan_id UUID REFERENCES execution_plans(id), - basic_program TEXT, - current_step INTEGER DEFAULT 0, - total_steps INTEGER DEFAULT 0, - progress FLOAT DEFAULT 0, - risk_level TEXT, - session_id UUID NOT NULL, - bot_id UUID NOT NULL, - created_by TEXT NOT NULL, - assigned_to TEXT DEFAULT 'auto', - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW(), - started_at TIMESTAMPTZ, - completed_at TIMESTAMPTZ -); - --- Execution Plans -CREATE TABLE execution_plans ( - id UUID PRIMARY KEY, - name TEXT NOT NULL, - description TEXT, - steps JSONB NOT NULL, - dependencies JSONB, - estimated_duration_minutes INTEGER, - requires_approval BOOLEAN DEFAULT FALSE, - rollback_plan TEXT, - compiled_intent_id UUID, - created_at TIMESTAMPTZ DEFAULT NOW() -); - --- Step Results -CREATE TABLE step_results ( - id UUID PRIMARY KEY, - task_id UUID REFERENCES auto_tasks(id), - step_id TEXT NOT NULL, - step_order INTEGER NOT NULL, - status TEXT NOT NULL, - output JSONB, - error TEXT, - started_at TIMESTAMPTZ, - completed_at TIMESTAMPTZ, - duration_ms BIGINT, - rollback_data JSONB -); - --- Pending Decisions -CREATE TABLE pending_decisions ( - id UUID PRIMARY KEY, - task_id UUID REFERENCES auto_tasks(id), - decision_type TEXT NOT NULL, - title TEXT NOT NULL, - description TEXT, - options JSONB NOT NULL, - default_option TEXT, - timeout_seconds INTEGER, - context JSONB, - created_at TIMESTAMPTZ DEFAULT NOW(), - expires_at TIMESTAMPTZ, - resolved_at TIMESTAMPTZ, - chosen_option TEXT -); - --- Pending Approvals -CREATE TABLE pending_approvals ( - id UUID PRIMARY KEY, - task_id UUID REFERENCES auto_tasks(id), - approval_type TEXT NOT NULL, - title TEXT NOT NULL, - description TEXT, - risk_level TEXT, - step_id TEXT, - impact_summary TEXT, - simulation_result JSONB, - approver TEXT, - timeout_seconds INTEGER DEFAULT 3600, - default_action TEXT DEFAULT 'pause', - created_at TIMESTAMPTZ DEFAULT NOW(), - expires_at TIMESTAMPTZ, - decided_at TIMESTAMPTZ, - decision TEXT, - decided_by TEXT, - comments TEXT -); - --- MCP Servers -CREATE TABLE mcp_servers ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - name TEXT NOT NULL, - description TEXT, - server_type TEXT NOT NULL, - config JSONB NOT NULL, - auth JSONB, - tools JSONB, - status TEXT DEFAULT 'inactive', - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW(), - UNIQUE(bot_id, name) -); - --- Safety Constraints -CREATE TABLE safety_constraints ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - name TEXT NOT NULL, - constraint_type TEXT NOT NULL, - description TEXT, - expression TEXT, - threshold JSONB, - severity TEXT DEFAULT 'warning', - enabled BOOLEAN DEFAULT TRUE, - applies_to TEXT[], - created_at TIMESTAMPTZ DEFAULT NOW() -); - --- Audit Log -CREATE TABLE audit_log ( - id UUID PRIMARY KEY, - timestamp TIMESTAMPTZ DEFAULT NOW(), - event_type TEXT NOT NULL, - actor_type TEXT NOT NULL, - actor_id TEXT NOT NULL, - action TEXT NOT NULL, - target_type TEXT, - target_id TEXT, - outcome_success BOOLEAN, - outcome_message TEXT, - details JSONB, - session_id UUID, - bot_id UUID, - task_id UUID, - step_id TEXT, - risk_level TEXT, - auto_executed BOOLEAN DEFAULT FALSE -); - -CREATE INDEX idx_audit_log_timestamp ON audit_log(timestamp DESC); -CREATE INDEX idx_audit_log_task_id ON audit_log(task_id); -CREATE INDEX idx_audit_log_event_type ON audit_log(event_type); -``` - -## 8. API Endpoints - -### Intent Compilation -- `POST /api/autotask/compile` - Compile intent to plan -- `POST /api/autotask/simulate/:plan_id` - Simulate plan execution - -### Task Management -- `GET /api/autotask/list` - List auto tasks -- `GET /api/autotask/stats` - Get task statistics -- `POST /api/autotask/execute` - Execute a compiled plan - -### Task Actions -- `POST /api/autotask/:task_id/pause` - Pause task -- `POST /api/autotask/:task_id/resume` - Resume task -- `POST /api/autotask/:task_id/cancel` - Cancel task -- `POST /api/autotask/:task_id/simulate` - Simulate task - -### Decisions & Approvals -- `GET /api/autotask/:task_id/decisions` - Get pending decisions -- `POST /api/autotask/:task_id/decide` - Submit decision -- `GET /api/autotask/:task_id/approvals` - Get pending approvals -- `POST /api/autotask/:task_id/approve` - Submit approval - -## 9. Security Considerations - -1. **Sandboxed Execution** - All code runs in isolated containers -2. **Credential Management** - No hardcoded secrets, use references -3. **Rate Limiting** - Prevent runaway executions -4. **Audit Trail** - Complete logging for compliance -5. **Approval Workflow** - Human oversight for high-risk actions -6. **Rollback Support** - Undo mechanisms where possible -7. **Circuit Breaker** - Stop on repeated failures - -## 10. Future Enhancements - -- [ ] Visual plan editor -- [ ] Real-time collaboration -- [ ] Plan templates marketplace -- [ ] Advanced scheduling (cron-like) -- [ ] Cost optimization suggestions -- [ ] Multi-tenant isolation -- [ ] Federated MCP servers -- [ ] AI-powered plan optimization -- [ ] Natural language debugging -- [ ] Integration with external task systems (Jira, Asana, etc.) - ---- - -## File Structure - -``` -botserver/src/basic/keywords/ -├── intent_compiler.rs # LLM-to-BASIC translation -├── auto_task.rs # Auto task data structures -├── autotask_api.rs # HTTP API handlers -├── mcp_client.rs # MCP server integration -├── safety_layer.rs # Constraints, simulation, audit -└── mod.rs # Module exports & keyword list - -botui/ui/suite/tasks/ -├── autotask.html # Auto task UI -├── autotask.css # Styles -├── autotask.js # Client-side logic -├── tasks.html # Original task list -├── tasks.css -└── tasks.js -``` - ---- - -*This architecture enables the "Premium VIP Mode" where users can describe what they want in natural language and the system automatically generates, validates, and executes the required tasks with full safety controls and audit trail.* \ No newline at end of file diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index a89282a7b..000000000 --- a/docs/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# General Bots Documentation - -Welcome to the General Bots documentation. This guide covers everything you need to build, deploy, and manage AI-powered bots. - -## Quick Navigation - -| Section | Description | -|---------|-------------| -| [Getting Started](guides/getting-started.md) | Installation and first bot | -| [API Reference](api/README.md) | REST endpoints and WebSocket | -| [BASIC Language](reference/basic-language.md) | Dialog scripting reference | -| [Configuration](reference/configuration.md) | Environment and settings | - -## Documentation Structure - -``` -docs/ -├── api/ # API documentation -│ ├── README.md # API overview -│ ├── rest-endpoints.md # HTTP endpoints -│ └── websocket.md # Real-time communication -├── guides/ # How-to guides -│ ├── getting-started.md # Quick start -│ ├── deployment.md # Production setup -│ └── templates.md # Using templates -└── reference/ # Technical reference - ├── basic-language.md # BASIC keywords - ├── configuration.md # Config options - └── architecture.md # System design -``` - -## Core Concepts - -### Knowledge Bases (KB) -Store documents, FAQs, and data that the AI can search and reference: -```basic -USE KB "company-docs" -``` - -### Tools -Functions the AI can call to perform actions: -```basic -USE TOOL "send-email" -USE TOOL "create-ticket" -``` - -### Dialogs -BASIC scripts that define conversation flows and automation: -```basic -TALK "Hello! How can I help?" -answer = HEAR -``` - -## Quick Links - -- **[GitHub Repository](https://github.com/GeneralBots/botserver)** -- **[Issue Tracker](https://github.com/GeneralBots/botserver/issues)** -- **[Contributing Guide](../CONTRIBUTING.md)** - -## Version - -This documentation covers **General Bots v6.x**. \ No newline at end of file diff --git a/docs/api/README.md b/docs/api/README.md deleted file mode 100644 index 4e76e86dc..000000000 --- a/docs/api/README.md +++ /dev/null @@ -1,104 +0,0 @@ -# API Reference - -General Bots exposes a REST API and WebSocket interface for integration with external systems. - -## Base URL - -``` -http://localhost:8080/api -``` - -## Authentication - -All API requests require authentication via Bearer token: - -```bash -curl -H "Authorization: Bearer " \ - http://localhost:8080/api/sessions -``` - -## Endpoints Overview - -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/api/sessions` | GET | List active sessions | -| `/api/sessions` | POST | Create new session | -| `/api/sessions/:id` | GET | Get session details | -| `/api/sessions/:id/messages` | POST | Send message | -| `/api/drive/*` | * | File storage operations | -| `/api/tasks/*` | * | Task management | -| `/api/email/*` | * | Email operations | -| `/api/calendar/*` | * | Calendar/CalDAV | -| `/api/meet/*` | * | Video meetings | -| `/api/kb/*` | * | Knowledge base search | -| `/api/analytics/*` | * | Analytics dashboard | - -## WebSocket - -Real-time communication via WebSocket: - -``` -ws://localhost:8080/ws -``` - -### Connection - -```javascript -const ws = new WebSocket('ws://localhost:8080/ws'); -ws.onmessage = (event) => { - const data = JSON.parse(event.data); - console.log('Received:', data); -}; -``` - -### Message Format - -```json -{ - "type": "message", - "session_id": "uuid", - "content": "Hello bot", - "timestamp": "2024-01-01T00:00:00Z" -} -``` - -## Response Format - -All responses follow a consistent structure: - -### Success - -```json -{ - "success": true, - "data": { ... } -} -``` - -### Error - -```json -{ - "success": false, - "error": { - "code": "NOT_FOUND", - "message": "Resource not found" - } -} -``` - -## Rate Limiting - -API requests are rate limited per IP: - -| Endpoint Type | Requests/Second | Burst | -|--------------|-----------------|-------| -| Standard API | 100 | 200 | -| Auth endpoints | 10 | 20 | -| LLM endpoints | 5 | 10 | - -## Detailed Documentation - -- [REST Endpoints](rest-endpoints.md) - Complete endpoint reference -- [WebSocket API](websocket.md) - Real-time communication -- [HTMX Integration](htmx.md) - Frontend patterns \ No newline at end of file diff --git a/docs/api/rest-endpoints.md b/docs/api/rest-endpoints.md deleted file mode 100644 index 261c7b826..000000000 --- a/docs/api/rest-endpoints.md +++ /dev/null @@ -1,599 +0,0 @@ -# REST API Endpoints - -Complete reference for all General Bots REST API endpoints. - -## Sessions - -### List Sessions - -```http -GET /api/sessions -``` - -**Query Parameters:** - -| Parameter | Type | Description | -|-----------|------|-------------| -| `limit` | integer | Max results (default: 50) | -| `offset` | integer | Pagination offset | -| `status` | string | Filter by status: `active`, `closed` | - -**Response:** - -```json -{ - "sessions": [ - { - "id": "550e8400-e29b-41d4-a716-446655440000", - "user_id": "user@example.com", - "bot_id": "default", - "status": "active", - "created_at": "2024-01-01T10:00:00Z", - "last_activity": "2024-01-01T10:30:00Z" - } - ], - "total": 1, - "limit": 50, - "offset": 0 -} -``` - -### Create Session - -```http -POST /api/sessions -``` - -**Request Body:** - -```json -{ - "bot_id": "default", - "user_id": "user@example.com", - "metadata": { - "channel": "web", - "language": "en" - } -} -``` - -### Get Session - -```http -GET /api/sessions/:id -``` - -### Send Message - -```http -POST /api/sessions/:id/messages -``` - -**Request Body:** - -```json -{ - "content": "Hello, I need help", - "type": "text" -} -``` - -**Response:** - -```json -{ - "id": "msg-uuid", - "session_id": "session-uuid", - "role": "assistant", - "content": "Hello! How can I assist you today?", - "timestamp": "2024-01-01T10:31:00Z" -} -``` - ---- - -## Drive (File Storage) - -### List Files - -```http -GET /api/drive -GET /api/drive/:path -``` - -**Query Parameters:** - -| Parameter | Type | Description | -|-----------|------|-------------| -| `recursive` | boolean | Include subdirectories | -| `type` | string | Filter: `file`, `folder` | - -### Upload File - -```http -POST /api/drive/:path -Content-Type: multipart/form-data -``` - -### Download File - -```http -GET /api/drive/:path/download -``` - -### Delete File - -```http -DELETE /api/drive/:path -``` - ---- - -## Tasks - -### List Tasks - -```http -GET /api/tasks -``` - -**Query Parameters:** - -| Parameter | Type | Description | -|-----------|------|-------------| -| `status` | string | `pending`, `completed`, `cancelled` | -| `priority` | string | `low`, `medium`, `high` | -| `due_before` | datetime | Filter by due date | - -### Create Task - -```http -POST /api/tasks -``` - -**Request Body:** - -```json -{ - "title": "Follow up with client", - "description": "Send proposal document", - "due_date": "2024-01-15T17:00:00Z", - "priority": "high", - "assignee": "user@example.com" -} -``` - -### Update Task - -```http -PUT /api/tasks/:id -``` - -### Delete Task - -```http -DELETE /api/tasks/:id -``` - ---- - -## Email - -### List Emails - -```http -GET /api/email -GET /api/email/:folder -``` - -**Folders:** `inbox`, `sent`, `drafts`, `trash` - -### Send Email - -```http -POST /api/email/send -``` - -**Request Body:** - -```json -{ - "to": ["recipient@example.com"], - "cc": [], - "bcc": [], - "subject": "Meeting Tomorrow", - "body": "Hi, let's meet at 3pm.", - "html": false, - "attachments": [] -} -``` - -### Get Email - -```http -GET /api/email/:id -``` - -### Delete Email - -```http -DELETE /api/email/:id -``` - ---- - -## Calendar - -### List Events - -```http -GET /api/calendar/events -``` - -**Query Parameters:** - -| Parameter | Type | Description | -|-----------|------|-------------| -| `start` | datetime | Range start | -| `end` | datetime | Range end | -| `calendar_id` | string | Specific calendar | - -### Create Event - -```http -POST /api/calendar/events -``` - -**Request Body:** - -```json -{ - "title": "Team Meeting", - "start_time": "2024-01-15T14:00:00Z", - "end_time": "2024-01-15T15:00:00Z", - "location": "Conference Room A", - "attendees": ["team@example.com"], - "recurrence": null -} -``` - -### Export Calendar (iCal) - -```http -GET /api/calendar/export.ics -``` - -### Import Calendar - -```http -POST /api/calendar/import -Content-Type: text/calendar -``` - ---- - -## Meet (Video Conferencing) - -### Create Room - -```http -POST /api/meet/rooms -``` - -**Request Body:** - -```json -{ - "name": "Team Standup", - "scheduled_start": "2024-01-15T09:00:00Z", - "max_participants": 10 -} -``` - -**Response:** - -```json -{ - "room_id": "room-uuid", - "join_url": "https://meet.example.com/room-uuid", - "token": "participant-token" -} -``` - -### Join Room - -```http -POST /api/meet/rooms/:id/join -``` - -### List Participants - -```http -GET /api/meet/rooms/:id/participants -``` - ---- - -## Knowledge Base - -### Search - -```http -GET /api/kb/search -``` - -**Query Parameters:** - -| Parameter | Type | Description | -|-----------|------|-------------| -| `q` | string | Search query (required) | -| `limit` | integer | Max results (default: 10) | -| `threshold` | float | Similarity threshold (0-1) | -| `collection` | string | Specific KB collection | - -**Response:** - -```json -{ - "results": [ - { - "id": "doc-uuid", - "content": "Relevant document content...", - "score": 0.95, - "metadata": { - "source": "company-docs", - "title": "Employee Handbook" - } - } - ], - "query": "vacation policy", - "total": 5 -} -``` - -### List Collections - -```http -GET /api/kb/collections -``` - -### Reindex Collection - -```http -POST /api/kb/reindex -``` - ---- - -## Analytics - -### Dashboard Stats - -```http -GET /api/analytics/stats -``` - -**Query Parameters:** - -| Parameter | Type | Description | -|-----------|------|-------------| -| `time_range` | string | `day`, `week`, `month`, `year` | - -**Response:** - -```json -{ - "total_messages": 15420, - "active_sessions": 45, - "avg_response_time_ms": 230, - "error_rate": 0.02, - "top_queries": [ - {"query": "password reset", "count": 120} - ] -} -``` - -### Message Trends - -```http -GET /api/analytics/messages/trend -``` - ---- - -## Paper (Documents) - -### List Documents - -```http -GET /api/paper -``` - -### Create Document - -```http -POST /api/paper -``` - -**Request Body:** - -```json -{ - "title": "Meeting Notes", - "content": "# Meeting Notes\n\n...", - "type": "note" -} -``` - -### Get Document - -```http -GET /api/paper/:id -``` - -### Update Document - -```http -PUT /api/paper/:id -``` - -### Delete Document - -```http -DELETE /api/paper/:id -``` - ---- - -## Designer (Bot Builder) - -### List Dialogs - -```http -GET /api/designer/dialogs -``` - -### Create Dialog - -```http -POST /api/designer/dialogs -``` - -**Request Body:** - -```json -{ - "name": "greeting", - "content": "TALK \"Hello!\"\nanswer = HEAR" -} -``` - -### Validate Dialog - -```http -POST /api/designer/dialogs/:id/validate -``` - -**Response:** - -```json -{ - "valid": true, - "errors": [], - "warnings": ["Line 15: Consider using END IF"] -} -``` - -### Deploy Dialog - -```http -POST /api/designer/dialogs/:id/deploy -``` - ---- - -## Sources (Templates) - -### List Templates - -```http -GET /api/sources/templates -``` - -**Query Parameters:** - -| Parameter | Type | Description | -|-----------|------|-------------| -| `category` | string | Filter by category | - -### Get Template - -```http -GET /api/sources/templates/:id -``` - -### Use Template - -```http -POST /api/sources/templates/:id/use -``` - ---- - -## Admin - -### System Stats - -```http -GET /api/admin/stats -``` - -**Response:** - -```json -{ - "uptime_seconds": 86400, - "memory_used_mb": 512, - "active_connections": 23, - "database_size_mb": 1024, - "cache_hit_rate": 0.85 -} -``` - -### Health Check - -```http -GET /api/health -``` - -**Response:** - -```json -{ - "status": "healthy", - "components": { - "database": "ok", - "cache": "ok", - "storage": "ok", - "llm": "ok" - } -} -``` - ---- - -## Error Codes - -| Code | HTTP Status | Description | -|------|-------------|-------------| -| `BAD_REQUEST` | 400 | Invalid request parameters | -| `UNAUTHORIZED` | 401 | Missing or invalid auth token | -| `FORBIDDEN` | 403 | Insufficient permissions | -| `NOT_FOUND` | 404 | Resource not found | -| `CONFLICT` | 409 | Resource already exists | -| `RATE_LIMITED` | 429 | Too many requests | -| `INTERNAL_ERROR` | 500 | Server error | - ---- - -## Pagination - -List endpoints support pagination: - -```http -GET /api/tasks?limit=20&offset=40 -``` - -Response includes pagination info: - -```json -{ - "data": [...], - "pagination": { - "total": 150, - "limit": 20, - "offset": 40, - "has_more": true - } -} -``` diff --git a/docs/api/websocket.md b/docs/api/websocket.md deleted file mode 100644 index 4f0d7ad93..000000000 --- a/docs/api/websocket.md +++ /dev/null @@ -1,362 +0,0 @@ -# WebSocket API - -Real-time bidirectional communication with General Bots. - -## Connection - -### Endpoint - -``` -ws://localhost:8080/ws -wss://your-domain.com/ws (production) -``` - -### Authentication - -Include the auth token as a query parameter or in the first message: - -```javascript -// Option 1: Query parameter -const ws = new WebSocket('ws://localhost:8080/ws?token='); - -// Option 2: First message -ws.onopen = () => { - ws.send(JSON.stringify({ - type: 'auth', - token: '' - })); -}; -``` - -## Message Format - -All messages are JSON objects with a `type` field: - -```json -{ - "type": "message_type", - "payload": { ... }, - "timestamp": "2024-01-01T10:00:00Z" -} -``` - -## Client Messages - -### Send Chat Message - -```json -{ - "type": "message", - "session_id": "550e8400-e29b-41d4-a716-446655440000", - "content": "Hello, I need help with my order" -} -``` - -### Start Typing - -```json -{ - "type": "typing_start", - "session_id": "550e8400-e29b-41d4-a716-446655440000" -} -``` - -### Stop Typing - -```json -{ - "type": "typing_stop", - "session_id": "550e8400-e29b-41d4-a716-446655440000" -} -``` - -### Subscribe to Session - -```json -{ - "type": "subscribe", - "session_id": "550e8400-e29b-41d4-a716-446655440000" -} -``` - -### Unsubscribe from Session - -```json -{ - "type": "unsubscribe", - "session_id": "550e8400-e29b-41d4-a716-446655440000" -} -``` - -### Ping (Keep-Alive) - -```json -{ - "type": "ping" -} -``` - -## Server Messages - -### Chat Response - -```json -{ - "type": "message", - "session_id": "550e8400-e29b-41d4-a716-446655440000", - "message_id": "msg-uuid", - "role": "assistant", - "content": "I'd be happy to help! What's your order number?", - "timestamp": "2024-01-01T10:00:01Z" -} -``` - -### Streaming Response - -For LLM responses, content streams in chunks: - -```json -{ - "type": "stream_start", - "session_id": "550e8400-e29b-41d4-a716-446655440000", - "message_id": "msg-uuid" -} -``` - -```json -{ - "type": "stream_chunk", - "message_id": "msg-uuid", - "content": "I'd be happy to " -} -``` - -```json -{ - "type": "stream_chunk", - "message_id": "msg-uuid", - "content": "help! What's your " -} -``` - -```json -{ - "type": "stream_end", - "message_id": "msg-uuid", - "content": "I'd be happy to help! What's your order number?" -} -``` - -### Bot Typing Indicator - -```json -{ - "type": "bot_typing", - "session_id": "550e8400-e29b-41d4-a716-446655440000", - "is_typing": true -} -``` - -### Tool Execution - -When the bot calls a tool: - -```json -{ - "type": "tool_call", - "session_id": "550e8400-e29b-41d4-a716-446655440000", - "tool": "search_orders", - "arguments": {"order_id": "12345"} -} -``` - -```json -{ - "type": "tool_result", - "session_id": "550e8400-e29b-41d4-a716-446655440000", - "tool": "search_orders", - "result": {"status": "shipped", "tracking": "1Z999..."} -} -``` - -### Error - -```json -{ - "type": "error", - "code": "SESSION_NOT_FOUND", - "message": "Session does not exist or has expired" -} -``` - -### Pong (Keep-Alive Response) - -```json -{ - "type": "pong", - "timestamp": "2024-01-01T10:00:00Z" -} -``` - -### Session Events - -```json -{ - "type": "session_created", - "session_id": "550e8400-e29b-41d4-a716-446655440000" -} -``` - -```json -{ - "type": "session_closed", - "session_id": "550e8400-e29b-41d4-a716-446655440000", - "reason": "user_disconnect" -} -``` - -## JavaScript Client Example - -```javascript -class BotClient { - constructor(url, token) { - this.url = url; - this.token = token; - this.ws = null; - this.handlers = {}; - } - - connect() { - this.ws = new WebSocket(`${this.url}?token=${this.token}`); - - this.ws.onopen = () => { - console.log('Connected to bot'); - this.emit('connected'); - }; - - this.ws.onmessage = (event) => { - const data = JSON.parse(event.data); - this.emit(data.type, data); - }; - - this.ws.onclose = () => { - console.log('Disconnected'); - this.emit('disconnected'); - // Auto-reconnect after 3 seconds - setTimeout(() => this.connect(), 3000); - }; - - this.ws.onerror = (error) => { - console.error('WebSocket error:', error); - this.emit('error', error); - }; - } - - send(type, payload) { - if (this.ws?.readyState === WebSocket.OPEN) { - this.ws.send(JSON.stringify({ type, ...payload })); - } - } - - sendMessage(sessionId, content) { - this.send('message', { session_id: sessionId, content }); - } - - subscribe(sessionId) { - this.send('subscribe', { session_id: sessionId }); - } - - on(event, handler) { - this.handlers[event] = this.handlers[event] || []; - this.handlers[event].push(handler); - } - - emit(event, data) { - (this.handlers[event] || []).forEach(h => h(data)); - } - - disconnect() { - this.ws?.close(); - } -} - -// Usage -const client = new BotClient('ws://localhost:8080/ws', 'auth-token'); - -client.on('connected', () => { - client.subscribe('session-uuid'); -}); - -client.on('message', (data) => { - console.log('Bot:', data.content); -}); - -client.on('stream_chunk', (data) => { - process.stdout.write(data.content); -}); - -client.on('error', (data) => { - console.error('Error:', data.message); -}); - -client.connect(); -client.sendMessage('session-uuid', 'Hello!'); -``` - -## Meet WebSocket - -Video conferencing uses a separate WebSocket endpoint: - -``` -ws://localhost:8080/ws/meet -``` - -### Join Room - -```json -{ - "type": "join", - "room_id": "room-uuid", - "participant_name": "John" -} -``` - -### Leave Room - -```json -{ - "type": "leave", - "room_id": "room-uuid" -} -``` - -### Signaling (WebRTC) - -```json -{ - "type": "signal", - "room_id": "room-uuid", - "target_id": "participant-uuid", - "signal": { /* WebRTC signal data */ } -} -``` - -## Connection Limits - -| Limit | Value | -|-------|-------| -| Max connections per IP | 100 | -| Max message size | 64 KB | -| Idle timeout | 5 minutes | -| Ping interval | 30 seconds | - -## Error Codes - -| Code | Description | -|------|-------------| -| `AUTH_FAILED` | Invalid or expired token | -| `SESSION_NOT_FOUND` | Session doesn't exist | -| `RATE_LIMITED` | Too many messages | -| `MESSAGE_TOO_LARGE` | Exceeds 64 KB limit | -| `INVALID_FORMAT` | Malformed JSON | -| `SUBSCRIPTION_FAILED` | Cannot subscribe to session | \ No newline at end of file diff --git a/docs/guides/deployment.md b/docs/guides/deployment.md deleted file mode 100644 index 783ae6cf8..000000000 --- a/docs/guides/deployment.md +++ /dev/null @@ -1,465 +0,0 @@ -# Deployment Guide - -Deploy General Bots in production environments with security, scalability, and reliability. - -## Deployment Options - -| Method | Best For | Complexity | -|--------|----------|------------| -| Single Server | Small teams, development | Low | -| Docker Compose | Medium deployments | Medium | -| LXC Containers | Isolated multi-tenant | Medium | -| Kubernetes | Large scale, high availability | High | - -## Single Server Deployment - -### Requirements - -- **CPU**: 4+ cores -- **RAM**: 16GB minimum -- **Disk**: 100GB SSD -- **OS**: Ubuntu 22.04 LTS / Debian 12 - -### Installation - -```bash -# Install Rust -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source ~/.cargo/env - -# Clone and build -git clone https://github.com/GeneralBots/botserver -cd botserver -cargo build --release - -# Run as service -sudo cp target/release/botserver /usr/local/bin/ -sudo cp scripts/botserver.service /etc/systemd/system/ -sudo systemctl enable botserver -sudo systemctl start botserver -``` - -### Systemd Service - -```ini -# /etc/systemd/system/botserver.service -[Unit] -Description=General Bots Server -After=network.target postgresql.service - -[Service] -Type=simple -User=botserver -Group=botserver -WorkingDirectory=/opt/botserver -ExecStart=/usr/local/bin/botserver --noconsole -Restart=always -RestartSec=5 -Environment=RUST_LOG=info - -[Install] -WantedBy=multi-user.target -``` - -## Docker Deployment - -### Docker Compose - -```yaml -# docker-compose.yml -version: '3.8' - -services: - botserver: - image: generalbots/botserver:latest - ports: - - "8080:8080" - environment: - - DATABASE_URL=postgres://bot:password@postgres/botserver - - REDIS_URL=redis://redis:6379 - - S3_ENDPOINT=http://minio:9000 - depends_on: - - postgres - - redis - - minio - volumes: - - ./templates:/app/templates - - ./data:/app/data - restart: unless-stopped - - postgres: - image: postgres:15 - environment: - - POSTGRES_USER=bot - - POSTGRES_PASSWORD=password - - POSTGRES_DB=botserver - volumes: - - postgres_data:/var/lib/postgresql/data - restart: unless-stopped - - redis: - image: redis:7-alpine - volumes: - - redis_data:/data - restart: unless-stopped - - minio: - image: minio/minio - command: server /data --console-address ":9001" - environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin - volumes: - - minio_data:/data - ports: - - "9001:9001" - restart: unless-stopped - - qdrant: - image: qdrant/qdrant - volumes: - - qdrant_data:/qdrant/storage - restart: unless-stopped - -volumes: - postgres_data: - redis_data: - minio_data: - qdrant_data: -``` - -### Start Services - -```bash -docker-compose up -d -docker-compose logs -f botserver -``` - -## LXC Container Deployment - -LXC provides lightweight isolation for multi-tenant deployments. - -### Create Container - -```bash -# Create container -lxc launch ubuntu:22.04 botserver - -# Configure resources -lxc config set botserver limits.cpu 4 -lxc config set botserver limits.memory 8GB - -# Enter container -lxc exec botserver -- bash -``` - -### Inside Container - -```bash -# Install dependencies -apt update && apt install -y curl build-essential - -# Install Rust and build -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source ~/.cargo/env -git clone https://github.com/GeneralBots/botserver -cd botserver -cargo build --release - -# Run -./target/release/botserver --container -``` - -### Container Networking - -```bash -# Forward port from host -lxc config device add botserver http proxy \ - listen=tcp:0.0.0.0:8080 \ - connect=tcp:127.0.0.1:8080 -``` - -## Reverse Proxy Setup - -### Nginx - -```nginx -# /etc/nginx/sites-available/botserver -upstream botserver { - server 127.0.0.1:8080; - keepalive 32; -} - -server { - listen 80; - server_name bot.example.com; - return 301 https://$server_name$request_uri; -} - -server { - listen 443 ssl http2; - server_name bot.example.com; - - ssl_certificate /etc/letsencrypt/live/bot.example.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/bot.example.com/privkey.pem; - - # Security headers - add_header X-Frame-Options DENY; - add_header X-Content-Type-Options nosniff; - add_header X-XSS-Protection "1; mode=block"; - - location / { - proxy_pass http://botserver; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # WebSocket support - location /ws { - proxy_pass http://botserver; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_read_timeout 86400; - } -} -``` - -### Enable Site - -```bash -sudo ln -s /etc/nginx/sites-available/botserver /etc/nginx/sites-enabled/ -sudo nginx -t -sudo systemctl reload nginx -``` - -## SSL Certificates - -### Let's Encrypt - -```bash -sudo apt install certbot python3-certbot-nginx -sudo certbot --nginx -d bot.example.com -``` - -### Auto-Renewal - -```bash -sudo certbot renew --dry-run -``` - -## Environment Configuration - -### Production Environment - -```bash -# /opt/botserver/.env -RUST_LOG=warn,botserver=info - -# Directory Service (required) -DIRECTORY_URL=https://auth.example.com -DIRECTORY_CLIENT_ID=your-client-id -DIRECTORY_CLIENT_SECRET=your-secret - -# Optional overrides -DATABASE_URL=postgres://user:pass@localhost/botserver -REDIS_URL=redis://localhost:6379 -S3_ENDPOINT=https://s3.example.com - -# Rate limiting -RATE_LIMIT_ENABLED=true -RATE_LIMIT_API_RPS=100 -``` - -## Database Setup - -### PostgreSQL Production Config - -```sql --- Create database and user -CREATE USER botserver WITH PASSWORD 'secure_password'; -CREATE DATABASE botserver OWNER botserver; -GRANT ALL PRIVILEGES ON DATABASE botserver TO botserver; - --- Enable extensions -\c botserver -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -CREATE EXTENSION IF NOT EXISTS "pg_trgm"; -``` - -### Connection Pooling (PgBouncer) - -```ini -# /etc/pgbouncer/pgbouncer.ini -[databases] -botserver = host=localhost dbname=botserver - -[pgbouncer] -listen_addr = 127.0.0.1 -listen_port = 6432 -auth_type = md5 -pool_mode = transaction -max_client_conn = 1000 -default_pool_size = 20 -``` - -## Monitoring - -### Health Check Endpoint - -```bash -curl http://localhost:8080/api/health -``` - -### Prometheus Metrics - -```yaml -# prometheus.yml -scrape_configs: - - job_name: 'botserver' - static_configs: - - targets: ['localhost:8080'] - metrics_path: /metrics -``` - -### Log Aggregation - -```bash -# Stream logs to file -journalctl -u botserver -f >> /var/log/botserver/app.log - -# Logrotate config -# /etc/logrotate.d/botserver -/var/log/botserver/*.log { - daily - rotate 14 - compress - delaycompress - missingok - notifempty -} -``` - -## Backup Strategy - -### Database Backup - -```bash -#!/bin/bash -# /opt/botserver/scripts/backup.sh -DATE=$(date +%Y%m%d_%H%M%S) -BACKUP_DIR=/backups/botserver - -# PostgreSQL -pg_dump -U botserver botserver | gzip > $BACKUP_DIR/db_$DATE.sql.gz - -# File storage -tar -czf $BACKUP_DIR/files_$DATE.tar.gz /opt/botserver/data - -# Retain 30 days -find $BACKUP_DIR -mtime +30 -delete -``` - -### Cron Schedule - -```bash -# Daily backup at 2 AM -0 2 * * * /opt/botserver/scripts/backup.sh -``` - -## Security Checklist - -- [ ] Run as non-root user -- [ ] Enable firewall (only ports 80, 443) -- [ ] Configure SSL/TLS -- [ ] Set secure file permissions -- [ ] Enable rate limiting -- [ ] Configure authentication -- [ ] Regular security updates -- [ ] Audit logging enabled -- [ ] Backup encryption -- [ ] Network isolation for database - -## Scaling - -### Horizontal Scaling - -```yaml -# docker-compose.scale.yml -services: - botserver: - deploy: - replicas: 3 - - nginx: - image: nginx - ports: - - "80:80" - volumes: - - ./nginx-lb.conf:/etc/nginx/nginx.conf -``` - -### Load Balancer Config - -```nginx -upstream botserver_cluster { - least_conn; - server botserver1:8080; - server botserver2:8080; - server botserver3:8080; -} -``` - -## Troubleshooting - -### Check Service Status - -```bash -sudo systemctl status botserver -journalctl -u botserver -n 100 -``` - -### Database Connection Issues - -```bash -psql -U botserver -h localhost -d botserver -c "SELECT 1" -``` - -### Memory Issues - -```bash -# Check memory usage -free -h -cat /proc/meminfo | grep -E "MemTotal|MemFree|Cached" - -# Increase swap if needed -sudo fallocate -l 4G /swapfile -sudo chmod 600 /swapfile -sudo mkswap /swapfile -sudo swapon /swapfile -``` - -## Updates - -### Rolling Update - -```bash -# Build new version -cd /opt/botserver -git pull -cargo build --release - -# Graceful restart -sudo systemctl reload botserver -``` - -### Zero-Downtime with Docker - -```bash -docker-compose pull -docker-compose up -d --no-deps --build botserver -``` diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md deleted file mode 100644 index f69151473..000000000 --- a/docs/guides/getting-started.md +++ /dev/null @@ -1,248 +0,0 @@ -# Getting Started with General Bots - -This guide will help you install, configure, and run your first General Bots instance. - -## Prerequisites - -- **Rust** (1.75 or later) - [Install from rustup.rs](https://rustup.rs/) -- **Git** - [Download from git-scm.com](https://git-scm.com/downloads) -- **8GB RAM** minimum (16GB recommended) -- **10GB disk space** for dependencies and data - -## Installation - -### 1. Clone the Repository - -```bash -git clone https://github.com/GeneralBots/botserver -cd botserver -``` - -### 2. Run the Server - -```bash -cargo run -``` - -On first run, General Bots automatically: - -1. Downloads and compiles dependencies -2. Sets up PostgreSQL database -3. Configures S3-compatible storage (MinIO) -4. Initializes Redis cache -5. Downloads default LLM models -6. Creates template bots -7. Starts the HTTP server - -The server will be available at `http://localhost:8080`. - -## First Steps - -### Access the Web Interface - -Open your browser to: - -- **Minimal UI**: `http://localhost:8080` - Lightweight chat interface -- **Full Suite**: `http://localhost:8080/suite` - Complete application suite - -### Create Your First Bot - -1. Navigate to the `templates/` directory -2. Copy the `template.gbai` folder: - -```bash -cp -r templates/template.gbai templates/mybot.gbai -``` - -3. Edit the configuration in `mybot.gbai/mybot.gbot/config.csv`: - -```csv -name,value -theme-title,My First Bot -theme-color1,#1565C0 -``` - -4. Add knowledge to `mybot.gbai/mybot.gbkb/`: - -```markdown -# Company FAQ - -## What are your hours? -We are open Monday to Friday, 9 AM to 5 PM. - -## How do I contact support? -Email support@example.com or call 1-800-EXAMPLE. -``` - -5. Create a dialog in `mybot.gbai/mybot.gbdialog/start.bas`: - -```basic -' Welcome dialog -USE KB "mybot.gbkb" - -TALK "Welcome to My Bot!" -TALK "How can I help you today?" - -SET CONTEXT "assistant" AS "You are a helpful assistant for My Company." -``` - -6. Restart the server to load your new bot. - -## Command-Line Options - -```bash -# Default: console UI + web server -cargo run - -# Disable console UI (background service) -cargo run -- --noconsole - -# Desktop application mode (Tauri) -cargo run -- --desktop - -# Specify tenant -cargo run -- --tenant mycompany - -# LXC container mode -cargo run -- --container - -# Disable all UI -cargo run -- --noui -``` - -## Project Structure - -``` -mybot.gbai/ -├── mybot.gbot/ # Bot configuration -│ └── config.csv # Theme and settings -├── mybot.gbkb/ # Knowledge base -│ └── faq.md # FAQ documents -├── mybot.gbdialog/ # Dialog scripts -│ └── start.bas # Main dialog -└── mybot.gbdrive/ # File storage - └── templates/ # Document templates -``` - -## Essential BASIC Keywords - -### Knowledge Base - -```basic -USE KB "knowledge-name" ' Load knowledge base -CLEAR KB ' Remove from session -``` - -### Tools - -```basic -USE TOOL "tool-name" ' Make tool available -CLEAR TOOLS ' Remove all tools -``` - -### Conversation - -```basic -TALK "message" ' Send message to user -answer = HEAR ' Wait for user input -WAIT 5 ' Wait 5 seconds -``` - -### Data - -```basic -SAVE "table.csv", field1, field2 ' Save to storage -data = GET "https://api.example.com" ' HTTP request -SEND FILE "document.pdf" ' Send file to user -``` - -## Environment Variables - -General Bots requires minimal configuration. Only directory service variables are needed: - -```bash -export DIRECTORY_URL="https://zitadel.example.com" -export DIRECTORY_CLIENT_ID="your-client-id" -export DIRECTORY_CLIENT_SECRET="your-secret" -``` - -All other services (database, storage, cache) are configured automatically. - -## Testing Your Bot - -### Via Web Interface - -1. Open `http://localhost:8080` -2. Type a message in the chat box -3. The bot responds using your knowledge base and dialogs - -### Via API - -```bash -# Create a session -curl -X POST http://localhost:8080/api/sessions \ - -H "Content-Type: application/json" \ - -d '{"bot_id": "mybot"}' - -# Send a message -curl -X POST http://localhost:8080/api/sessions/{session_id}/messages \ - -H "Content-Type: application/json" \ - -d '{"content": "What are your hours?"}' -``` - -### Via WebSocket - -```javascript -const ws = new WebSocket('ws://localhost:8080/ws'); - -ws.onopen = () => { - ws.send(JSON.stringify({ - type: 'message', - session_id: 'your-session-id', - content: 'Hello!' - })); -}; - -ws.onmessage = (event) => { - const data = JSON.parse(event.data); - console.log('Bot:', data.content); -}; -``` - -## Common Issues - -### Port Already in Use - -```bash -# Find process using port 8080 -lsof -i :8080 - -# Kill the process or use a different port -cargo run -- --port 8081 -``` - -### Database Connection Failed - -Ensure PostgreSQL is running: - -```bash -botserver status postgres -botserver restart postgres -``` - -### LLM Not Responding - -Check your LLM configuration in the admin panel or verify API keys are set. - -## Next Steps - -- **[API Reference](../api/README.md)** - Integrate with external systems -- **[BASIC Language](../reference/basic-language.md)** - Complete keyword reference -- **[Templates](templates.md)** - Pre-built bot templates -- **[Deployment](deployment.md)** - Production setup guide - -## Getting Help - -- **GitHub Issues**: [github.com/GeneralBots/botserver/issues](https://github.com/GeneralBots/botserver/issues) -- **Stack Overflow**: Tag questions with `generalbots` -- **Documentation**: [docs.pragmatismo.com.br](https://docs.pragmatismo.com.br) \ No newline at end of file diff --git a/docs/guides/templates.md b/docs/guides/templates.md deleted file mode 100644 index 502ccc6a4..000000000 --- a/docs/guides/templates.md +++ /dev/null @@ -1,308 +0,0 @@ -# Using Templates - -Templates are pre-built bot configurations that accelerate development. General Bots includes templates for common use cases like CRM, HR, IT support, and more. - -## Template Structure - -Each template follows the `.gbai` package format: - -``` -template-name.gbai/ -├── template-name.gbot/ # Configuration -│ └── config.csv # Bot settings -├── template-name.gbkb/ # Knowledge base -│ └── *.md # Documentation files -├── template-name.gbdialog/ # Dialog scripts -│ ├── start.bas # Entry point -│ └── *.bas # Tool scripts -└── template-name.gbdrive/ # File storage - └── templates/ # Document templates -``` - -## Available Templates - -### Business Operations - -| Template | Description | Path | -|----------|-------------|------| -| CRM Contacts | Contact management | `crm/contacts.gbai` | -| Sales Pipeline | Deal tracking | `crm/sales-pipeline.gbai` | -| HR Employees | Employee management | `hr/employees.gbai` | -| IT Helpdesk | Ticket system | `it/helpdesk.gbai` | - -### Productivity - -| Template | Description | Path | -|----------|-------------|------| -| Office | Document automation | `productivity/office.gbai` | -| Reminder | Task reminders | `productivity/reminder.gbai` | -| Analytics | Data dashboards | `platform/analytics.gbai` | - -### Integration - -| Template | Description | Path | -|----------|-------------|------| -| API Client | External API calls | `integration/api-client.gbai` | -| Public APIs | 70+ free APIs | `integration/public-apis.gbai` | - -### Compliance - -| Template | Description | Path | -|----------|-------------|------| -| HIPAA Medical | Healthcare compliance | `compliance/hipaa-medical.gbai` | -| Privacy | GDPR/LGPD | `compliance/privacy.gbai` | - -## Using a Template - -### 1. Copy the Template - -```bash -cp -r templates/crm/contacts.gbai templates/mycrm.gbai -``` - -### 2. Rename Internal Folders - -```bash -cd templates/mycrm.gbai -mv contacts.gbot mycrm.gbot -mv contacts.gbkb mycrm.gbkb -mv contacts.gbdialog mycrm.gbdialog -mv contacts.gbdrive mycrm.gbdrive -``` - -### 3. Update Configuration - -Edit `mycrm.gbot/config.csv`: - -```csv -name,value -theme-title,My CRM Bot -theme-color1,#2196F3 -theme-color2,#E3F2FD -episodic-memory-history,2 -``` - -### 4. Customize Knowledge Base - -Edit files in `mycrm.gbkb/`: - -```markdown -# My Company CRM Guide - -## Adding Contacts -To add a new contact, say "add contact" and provide: -- Name -- Email -- Phone number -- Company - -## Searching Contacts -Say "find contact [name]" to search. -``` - -### 5. Modify Dialogs - -Edit `mycrm.gbdialog/start.bas`: - -```basic -' My CRM Bot - Start Script - -USE KB "mycrm.gbkb" -USE TOOL "add-contact" -USE TOOL "search-contact" - -SET CONTEXT "crm" AS "You are a CRM assistant for My Company." - -TALK "Welcome to My Company CRM!" -TALK "I can help you manage contacts and deals." - -ADD SUGGESTION "add" AS "Add new contact" -ADD SUGGESTION "search" AS "Find a contact" -ADD SUGGESTION "list" AS "Show all contacts" -``` - -### 6. Restart Server - -```bash -cargo run -``` - -## Creating Custom Templates - -### Step 1: Create Package Structure - -```bash -mkdir -p templates/mytemplate.gbai/{mytemplate.gbot,mytemplate.gbkb,mytemplate.gbdialog,mytemplate.gbdrive} -``` - -### Step 2: Create Configuration - -```csv -# mytemplate.gbot/config.csv -name,value -theme-title,My Template -theme-color1,#1565C0 -theme-color2,#E3F2FD -theme-logo,https://example.com/logo.svg -episodic-memory-history,2 -episodic-memory-threshold,4 -``` - -### Step 3: Create Start Dialog - -```basic -' mytemplate.gbdialog/start.bas - -' Load knowledge base -USE KB "mytemplate.gbkb" - -' Register tools -USE TOOL "my-action" - -' Set AI context -SET CONTEXT "assistant" AS "You are a helpful assistant." - -' Welcome message -BEGIN TALK - **Welcome to My Template!** - - I can help you with: - • Feature 1 - • Feature 2 - • Feature 3 -END TALK - -' Add quick suggestions -CLEAR SUGGESTIONS -ADD SUGGESTION "help" AS "Show help" -ADD SUGGESTION "action" AS "Do something" -``` - -### Step 4: Create Tools - -```basic -' mytemplate.gbdialog/my-action.bas - -PARAM item_name AS STRING LIKE "Example" DESCRIPTION "Name of the item" -PARAM quantity AS INTEGER LIKE 1 DESCRIPTION "How many items" - -DESCRIPTION "Performs an action with the specified item." - -' Validate input -IF item_name = "" THEN - TALK "Please provide an item name." - item_name = HEAR -END IF - -' Process -let result = item_name + " x " + quantity - -' Save record -SAVE "items.csv", item_name, quantity, result - -' Respond -TALK "✅ Created: " + result - -RETURN result -``` - -### Step 5: Add Knowledge Base - -```markdown -# mytemplate.gbkb/guide.md - -# My Template Guide - -## Overview -This template helps you accomplish specific tasks. - -## Features - -### Feature 1 -Description of feature 1. - -### Feature 2 -Description of feature 2. - -## FAQ - -### How do I get started? -Just say "help" to see available commands. - -### How do I contact support? -Email support@example.com. -``` - -## Template Best Practices - -### Configuration - -- Use clear, descriptive `theme-title` -- Choose accessible color combinations -- Set appropriate `episodic-memory-history` (2-4 recommended) - -### Knowledge Base - -- Write clear, concise documentation -- Use headers for organization -- Include FAQ section -- Keep files under 50KB each - -### Dialogs - -- Always include `start.bas` -- Use `DESCRIPTION` for all tools -- Validate user input -- Provide helpful error messages -- Add suggestions for common actions - -### File Organization - -``` -mytemplate.gbai/ -├── mytemplate.gbot/ -│ └── config.csv -├── mytemplate.gbkb/ -│ ├── guide.md # Main documentation -│ ├── faq.md # Frequently asked questions -│ └── troubleshooting.md # Common issues -├── mytemplate.gbdialog/ -│ ├── start.bas # Entry point (required) -│ ├── tool-1.bas # First tool -│ ├── tool-2.bas # Second tool -│ └── jobs.bas # Scheduled tasks -└── mytemplate.gbdrive/ - └── templates/ - └── report.docx # Document templates -``` - -## Sharing Templates - -### Export - -```bash -tar -czf mytemplate.tar.gz templates/mytemplate.gbai -``` - -### Import - -```bash -tar -xzf mytemplate.tar.gz -C templates/ -``` - -### Version Control - -Templates work well with Git: - -```bash -cd templates/mytemplate.gbai -git init -git add . -git commit -m "Initial template" -``` - -## Next Steps - -- [BASIC Language Reference](../reference/basic-language.md) - Complete keyword list -- [API Reference](../api/README.md) - Integrate with external systems -- [Deployment Guide](deployment.md) - Production setup \ No newline at end of file diff --git a/docs/reference/architecture.md b/docs/reference/architecture.md deleted file mode 100644 index 8ac9a1cfa..000000000 --- a/docs/reference/architecture.md +++ /dev/null @@ -1,530 +0,0 @@ -# Architecture Reference - -System architecture and design overview for General Bots. - -## High-Level Architecture - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ Clients │ -│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ -│ │ Web │ │ Mobile │ │ API │ │ WhatsApp │ │ -│ │ (HTMX) │ │ App │ │ Clients │ │ Telegram │ │ -│ └────┬─────┘ └────┬─────┘ └────┬─────┘ └────┬─────┘ │ -└───────┼─────────────┼─────────────┼─────────────┼───────────────┘ - │ │ │ │ - └─────────────┴──────┬──────┴─────────────┘ - │ - ┌────────▼────────┐ - │ HTTP Server │ - │ (Axum) │ - └────────┬────────┘ - │ - ┌────────────────────┼────────────────────┐ - │ │ │ -┌───────▼───────┐ ┌─────────▼─────────┐ ┌──────▼──────┐ -│ REST API │ │ WebSocket │ │ Static │ -│ Handlers │ │ Handlers │ │ Files │ -└───────┬───────┘ └─────────┬─────────┘ └─────────────┘ - │ │ - └────────┬───────────┘ - │ - ┌────────▼────────┐ - │ App State │ - │ (Shared Arc) │ - └────────┬────────┘ - │ - ┌─────────────┼─────────────┬─────────────┐ - │ │ │ │ -┌──▼──┐ ┌────▼────┐ ┌────▼────┐ ┌───▼───┐ -│ DB │ │ Cache │ │ Storage │ │ LLM │ -│(PG) │ │ (Redis) │ │ (S3) │ │(Multi)│ -└─────┘ └─────────┘ └─────────┘ └───────┘ -``` - -## Core Components - -### HTTP Server (Axum) - -The main entry point for all requests. - -```rust -// Main router structure -Router::new() - .nest("/api", api_routes()) - .nest("/ws", websocket_routes()) - .nest_service("/", static_files()) - .layer(middleware_stack()) -``` - -**Responsibilities:** -- Request routing -- Middleware execution -- CORS handling -- Rate limiting -- Authentication - -### App State - -Shared application state passed to all handlers. - -```rust -pub struct AppState { - pub db_pool: Pool>, - pub redis: RedisPool, - pub s3_client: S3Client, - pub llm_client: LlmClient, - pub qdrant: QdrantClient, - pub config: AppConfig, -} -``` - -**Contains:** -- Database connection pool -- Redis cache connection -- S3 storage client -- LLM provider clients -- Vector database client -- Application configuration - -### Request Flow - -``` -Request → Router → Middleware → Handler → Response - │ - ├── Auth Check - ├── Rate Limit - ├── Logging - └── Error Handling -``` - -## Module Structure - -``` -botserver/src/ -├── main.rs # Entry point -├── lib.rs # Library exports -├── core/ # Core infrastructure -│ ├── shared/ # Shared types and state -│ │ ├── state.rs # AppState definition -│ │ ├── models.rs # Common models -│ │ └── schema.rs # Database schema -│ ├── urls.rs # URL constants -│ ├── secrets/ # Vault integration -│ └── rate_limit.rs # Rate limiting -├── basic/ # BASIC language -│ ├── compiler/ # Compiler/parser -│ ├── runtime/ # Execution engine -│ └── keywords/ # Keyword implementations -├── llm/ # LLM integration -│ ├── mod.rs # Provider abstraction -│ ├── openai.rs # OpenAI client -│ ├── anthropic.rs # Anthropic client -│ └── prompt_manager/ # Prompt management -├── multimodal/ # Media processing -│ ├── vision.rs # Image analysis -│ ├── audio.rs # Speech processing -│ └── document.rs # Document parsing -├── security/ # Authentication -│ ├── auth.rs # Auth middleware -│ ├── zitadel.rs # Zitadel client -│ └── jwt.rs # Token handling -├── analytics/ # Analytics module -├── calendar/ # Calendar/CalDAV -├── designer/ # Bot builder -├── drive/ # File storage -├── email/ # Email (IMAP/SMTP) -├── meet/ # Video conferencing -├── paper/ # Document editor -├── research/ # KB search -├── sources/ # Templates -└── tasks/ # Task management -``` - -## Data Flow - -### Chat Message Flow - -``` -User Input - │ - ▼ -┌──────────────┐ -│ WebSocket │ -│ Handler │ -└──────┬───────┘ - │ - ▼ -┌──────────────┐ ┌──────────────┐ -│ Session │────▶│ Message │ -│ Manager │ │ History │ -└──────┬───────┘ └──────────────┘ - │ - ▼ -┌──────────────┐ -│ KB Search │◀────── Vector DB (Qdrant) -│ (Context) │ -└──────┬───────┘ - │ - ▼ -┌──────────────┐ -│ Tool Check │◀────── Registered Tools -│ │ -└──────┬───────┘ - │ - ▼ -┌──────────────┐ -│ LLM Call │◀────── Semantic Cache (Redis) -│ │ -└──────┬───────┘ - │ - ▼ -┌──────────────┐ -│ Response │ -│ Streaming │ -└──────┬───────┘ - │ - ▼ -User Response -``` - -### Tool Execution Flow - -``` -LLM Response (tool_call) - │ - ▼ -┌──────────────┐ -│ Tool Router │ -└──────┬───────┘ - │ - ▼ -┌──────────────┐ -│BASIC Runtime │ -└──────┬───────┘ - │ - ├──▶ Database Operations - ├──▶ HTTP Requests - ├──▶ File Operations - └──▶ Email/Notifications - │ - ▼ -┌──────────────┐ -│ Tool Result │ -└──────┬───────┘ - │ - ▼ -LLM (continue or respond) -``` - -## Database Schema - -### Core Tables - -```sql --- User sessions -CREATE TABLE user_sessions ( - id UUID PRIMARY KEY, - user_id VARCHAR(255), - bot_id VARCHAR(100), - status VARCHAR(20), - metadata JSONB, - created_at TIMESTAMPTZ, - updated_at TIMESTAMPTZ -); - --- Message history -CREATE TABLE message_history ( - id UUID PRIMARY KEY, - session_id UUID REFERENCES user_sessions(id), - role VARCHAR(20), - content TEXT, - tokens_used INTEGER, - created_at TIMESTAMPTZ -); - --- Bot configurations -CREATE TABLE bot_configs ( - id UUID PRIMARY KEY, - bot_id VARCHAR(100) UNIQUE, - config JSONB, - created_at TIMESTAMPTZ, - updated_at TIMESTAMPTZ -); - --- Knowledge base documents -CREATE TABLE kb_documents ( - id UUID PRIMARY KEY, - collection VARCHAR(100), - content TEXT, - embedding_id VARCHAR(100), - metadata JSONB, - created_at TIMESTAMPTZ -); -``` - -### Indexes - -```sql -CREATE INDEX idx_sessions_user ON user_sessions(user_id); -CREATE INDEX idx_sessions_bot ON user_sessions(bot_id); -CREATE INDEX idx_messages_session ON message_history(session_id); -CREATE INDEX idx_kb_collection ON kb_documents(collection); -``` - -## Caching Strategy - -### Cache Layers - -``` -┌─────────────────────────────────────────┐ -│ Semantic Cache │ -│ (LLM responses by query similarity) │ -└───────────────────┬─────────────────────┘ - │ -┌───────────────────▼─────────────────────┐ -│ Session Cache │ -│ (Active sessions, user context) │ -└───────────────────┬─────────────────────┘ - │ -┌───────────────────▼─────────────────────┐ -│ Data Cache │ -│ (KB results, config, templates) │ -└─────────────────────────────────────────┘ -``` - -### Cache Keys - -| Pattern | TTL | Description | -|---------|-----|-------------| -| `session:{id}` | 30m | Active session data | -| `semantic:{hash}` | 24h | LLM response cache | -| `kb:{collection}:{hash}` | 1h | KB search results | -| `config:{bot_id}` | 5m | Bot configuration | -| `user:{id}` | 15m | User preferences | - -### Semantic Cache - -```rust -// Query similarity check -let cache_key = compute_embedding_hash(query); -if let Some(cached) = redis.get(&cache_key).await? { - if similarity(query, cached.query) > 0.95 { - return cached.response; - } -} -``` - -## LLM Integration - -### Provider Abstraction - -```rust -pub trait LlmProvider: Send + Sync { - async fn complete(&self, request: CompletionRequest) - -> Result; - - async fn complete_stream(&self, request: CompletionRequest) - -> Result>; - - async fn embed(&self, text: &str) - -> Result>; -} -``` - -### Supported Providers - -| Provider | Models | Features | -|----------|--------|----------| -| OpenAI | GPT-5, GPT-4o, o3 | Streaming, Functions, Vision | -| Anthropic | Claude Sonnet 4.5, Claude Opus 4.5 | Streaming, Long context | -| Groq | Llama 3.3, Mixtral | Fast inference | -| Ollama | Any local | Self-hosted | - -### Request Flow - -```rust -// 1. Build messages with context -let messages = build_messages(history, kb_context, system_prompt); - -// 2. Add tools if registered -let tools = get_registered_tools(session); - -// 3. Check semantic cache -if let Some(cached) = semantic_cache.get(&messages).await? { - return Ok(cached); -} - -// 4. Call LLM -let response = llm.complete(CompletionRequest { - messages, - tools, - temperature: config.temperature, - max_tokens: config.max_tokens, -}).await?; - -// 5. Cache response -semantic_cache.set(&messages, &response).await?; -``` - -## BASIC Runtime - -### Compilation Pipeline - -``` -Source Code (.bas) - │ - ▼ -┌──────────────┐ -│ Lexer │──▶ Tokens -└──────────────┘ - │ - ▼ -┌──────────────┐ -│ Parser │──▶ AST -└──────────────┘ - │ - ▼ -┌──────────────┐ -│ Analyzer │──▶ Validated AST -└──────────────┘ - │ - ▼ -┌──────────────┐ -│ Runtime │──▶ Execution -└──────────────┘ -``` - -### Execution Context - -```rust -pub struct RuntimeContext { - pub session: Session, - pub variables: HashMap, - pub tools: Vec, - pub kb: Vec, - pub state: Arc, -} -``` - -## Security Architecture - -### Authentication Flow - -``` -Client Request - │ - ▼ -┌──────────────┐ -│ Extract │ -│ Token │ -└──────┬───────┘ - │ - ▼ -┌──────────────┐ ┌──────────────┐ -│ Validate │────▶│ Zitadel │ -│ JWT │ │ (OIDC) │ -└──────┬───────┘ └──────────────┘ - │ - ▼ -┌──────────────┐ -│ Check │ -│ Permissions │ -└──────┬───────┘ - │ - ▼ -Handler Execution -``` - -### Security Layers - -1. **Transport**: TLS 1.3 (rustls) -2. **Authentication**: JWT/OAuth 2.0 (Zitadel) -3. **Authorization**: Role-based access control -4. **Rate Limiting**: Per-IP token bucket -5. **Input Validation**: Type-safe parameters -6. **Output Sanitization**: HTML escaping - -## Deployment Architecture - -### Single Instance - -``` -┌─────────────────────────────────────┐ -│ Single Server │ -│ ┌─────────────────────────────┐ │ -│ │ botserver │ │ -│ └─────────────────────────────┘ │ -│ ┌────────┐ ┌────────┐ ┌───────┐ │ -│ │PostgreSQL│ │ Redis │ │ MinIO │ │ -│ └────────┘ └────────┘ └───────┘ │ -└─────────────────────────────────────┘ -``` - -### Clustered - -``` -┌─────────────────────────────────────────────────────┐ -│ Load Balancer │ -└───────────────────────┬─────────────────────────────┘ - │ - ┌───────────────┼───────────────┐ - │ │ │ - ┌────▼────┐ ┌─────▼────┐ ┌────▼────┐ - │botserver│ │botserver │ │botserver│ - │ #1 │ │ #2 │ │ #3 │ - └────┬────┘ └─────┬────┘ └────┬────┘ - │ │ │ - └───────────────┼───────────────┘ - │ - ┌────────────────────┼────────────────────┐ - │ │ │ -┌──▼───┐ ┌─────▼────┐ ┌────▼────┐ -│ PG │ │ Redis │ │ S3 │ -│Cluster│ │ Cluster │ │ Cluster │ -└──────┘ └──────────┘ └─────────┘ -``` - -## Performance Characteristics - -| Operation | Latency | Throughput | -|-----------|---------|------------| -| REST API | < 10ms | 10,000 req/s | -| WebSocket message | < 5ms | 50,000 msg/s | -| KB search | < 50ms | 1,000 req/s | -| LLM call (cached) | < 20ms | 5,000 req/s | -| LLM call (uncached) | 500-3000ms | 50 req/s | - -## Monitoring Points - -| Metric | Description | -|--------|-------------| -| `http_requests_total` | Total HTTP requests | -| `http_request_duration` | Request latency | -| `ws_connections` | Active WebSocket connections | -| `llm_requests_total` | LLM API calls | -| `llm_cache_hits` | Semantic cache hit rate | -| `db_pool_size` | Active DB connections | -| `memory_usage` | Process memory | - -## Extension Points - -### Adding a New Module - -1. Create module in `src/` -2. Define routes in `mod.rs` -3. Register in `lib.rs` -4. Add to router in `main.rs` - -### Adding a New LLM Provider - -1. Implement `LlmProvider` trait -2. Add provider enum variant -3. Register in provider factory - -### Adding a New BASIC Keyword - -1. Add keyword to lexer -2. Implement AST node -3. Add runtime handler -4. Update documentation \ No newline at end of file diff --git a/docs/reference/basic-language.md b/docs/reference/basic-language.md deleted file mode 100644 index 7bc884a5f..000000000 --- a/docs/reference/basic-language.md +++ /dev/null @@ -1,525 +0,0 @@ -# BASIC Language Reference - -Complete reference for General Bots BASIC dialog scripting language. - -## Overview - -General Bots BASIC is a domain-specific language for creating conversational AI dialogs. It provides keywords for: - -- User interaction (TALK, HEAR) -- Knowledge base management (USE KB, CLEAR KB) -- Tool registration (USE TOOL, CLEAR TOOLS) -- Data operations (SAVE, GET, POST) -- File handling (SEND FILE, DOWNLOAD) -- Flow control (IF/THEN/ELSE, FOR/NEXT) - -## Conversation Keywords - -### TALK - -Send a message to the user. - -```basic -TALK "Hello, how can I help you?" -TALK "Your order number is: " + ordernumber -``` - -#### Multi-line Messages - -```basic -BEGIN TALK - **Welcome!** - - I can help you with: - • Orders - • Shipping - • Returns -END TALK -``` - -### HEAR - -Wait for and capture user input. - -```basic -answer = HEAR -name = HEAR AS NAME -email = HEAR AS EMAIL -choice = HEAR AS "Option A", "Option B", "Option C" -confirmed = HEAR AS BOOLEAN -``` - -#### Input Types - -| Type | Description | Example | -|------|-------------|---------| -| `STRING` | Free text | `answer = HEAR` | -| `NAME` | Person name | `name = HEAR AS NAME` | -| `EMAIL` | Email address | `email = HEAR AS EMAIL` | -| `PHONE` | Phone number | `phone = HEAR AS PHONE` | -| `INTEGER` | Whole number | `count = HEAR AS INTEGER` | -| `NUMBER` | Decimal number | `amount = HEAR AS NUMBER` | -| `BOOLEAN` | Yes/No | `confirm = HEAR AS BOOLEAN` | -| `DATE` | Date value | `date = HEAR AS DATE` | -| Options | Multiple choice | `choice = HEAR AS "A", "B", "C"` | - -### WAIT - -Pause execution for specified seconds. - -```basic -WAIT 5 -TALK "Processing..." -WAIT 2 -TALK "Done!" -``` - -## Knowledge Base Keywords - -### USE KB - -Load a knowledge base into the current session. - -```basic -USE KB "company-docs" -USE KB "product-catalog.gbkb" -``` - -### CLEAR KB - -Remove knowledge base from session. - -```basic -CLEAR KB "company-docs" -CLEAR KB ' Clear all KBs -``` - -## Tool Keywords - -### USE TOOL - -Register a tool for the AI to call. - -```basic -USE TOOL "create-ticket" -USE TOOL "send-email" -USE TOOL "search-orders" -``` - -### CLEAR TOOLS - -Remove all registered tools. - -```basic -CLEAR TOOLS -``` - -## Context Keywords - -### SET CONTEXT - -Define AI behavior context. - -```basic -SET CONTEXT "assistant" AS "You are a helpful customer service agent for Acme Corp." -``` - -### System Prompt - -Define detailed AI instructions. - -```basic -BEGIN SYSTEM PROMPT - You are a professional assistant. - Always be polite and helpful. - If you don't know something, say so. - Never make up information. -END SYSTEM PROMPT -``` - -## Suggestion Keywords - -### ADD SUGGESTION - -Add a quick-reply button for users. - -```basic -ADD SUGGESTION "help" AS "Show help" -ADD SUGGESTION "order" AS "Track my order" -ADD SUGGESTION "contact" AS "Contact support" -``` - -### CLEAR SUGGESTIONS - -Remove all suggestions. - -```basic -CLEAR SUGGESTIONS -``` - -## Data Keywords - -### SAVE - -Save data to storage. - -```basic -SAVE "contacts.csv", name, email, phone -SAVE "orders.csv", orderid, product, quantity, total -``` - -### GET - -HTTP GET request. - -```basic -data = GET "https://api.example.com/users" -weather = GET "https://api.weather.com/current?city=" + city -``` - -### POST - -HTTP POST request. - -```basic -result = POST "https://api.example.com/orders", orderdata -``` - -### PUT - -HTTP PUT request. - -```basic -result = PUT "https://api.example.com/users/" + userid, userdata -``` - -### DELETE HTTP - -HTTP DELETE request. - -```basic -result = DELETE HTTP "https://api.example.com/users/" + userid -``` - -### SET HEADER - -Set HTTP header for requests. - -```basic -SET HEADER "Authorization" = "Bearer " + token -SET HEADER "Content-Type" = "application/json" -data = GET "https://api.example.com/protected" -``` - -### CLEAR HEADERS - -Remove all custom headers. - -```basic -CLEAR HEADERS -``` - -## File Keywords - -### SEND FILE - -Send a file to the user. - -```basic -SEND FILE "report.pdf" -SEND FILE filepath -``` - -### DOWNLOAD - -Download a file from URL. - -```basic -file = DOWNLOAD "https://example.com/document.pdf" -SEND FILE file -``` - -### DELETE FILE - -Delete a file from storage. - -```basic -DELETE FILE "old-report.pdf" -``` - -## Email Keywords - -### SEND MAIL - -Send an email. - -```basic -SEND MAIL "recipient@example.com", "Subject Line", "Email body text" -SEND MAIL email, subject, body -``` - -## Memory Keywords - -### SET BOT MEMORY - -Store a value in bot memory (persists across sessions). - -```basic -SET BOT MEMORY "last_order", orderid -SET BOT MEMORY "user_preference", preference -``` - -### GET BOT MEMORY - -Retrieve a value from bot memory. - -```basic -lastorder = GET BOT MEMORY("last_order") -pref = GET BOT MEMORY("user_preference") -``` - -## Schedule Keywords - -### SET SCHEDULE - -Define when a job should run (cron format). - -```basic -SET SCHEDULE "0 9 * * *" ' Daily at 9 AM -SET SCHEDULE "0 0 * * 1" ' Weekly on Monday -SET SCHEDULE "0 8 1 * *" ' Monthly on the 1st at 8 AM -``` - -#### Cron Format - -``` -┌───────────── minute (0-59) -│ ┌───────────── hour (0-23) -│ │ ┌───────────── day of month (1-31) -│ │ │ ┌───────────── month (1-12) -│ │ │ │ ┌───────────── day of week (0-6, Sun=0) -│ │ │ │ │ -* * * * * -``` - -## Flow Control - -### IF/THEN/ELSE - -Conditional execution. - -```basic -IF status = "active" THEN - TALK "Your account is active." -ELSE IF status = "pending" THEN - TALK "Your account is pending approval." -ELSE - TALK "Your account is inactive." -END IF -``` - -### FOR/NEXT - -Loop through a range. - -```basic -FOR i = 1 TO 10 - TALK "Item " + i -NEXT -``` - -### FOR EACH - -Loop through a collection. - -```basic -FOR EACH item IN items - TALK item.name + ": $" + item.price -END FOR -``` - -## Variables - -### Declaration - -```basic -let name = "John" -let count = 42 -let price = 19.99 -let active = TRUE -``` - -### String Operations - -```basic -let greeting = "Hello, " + name + "!" -let upper = UCASE(text) -let lower = LCASE(text) -let length = LEN(text) -let part = MID(text, 1, 5) -``` - -### Array Operations - -```basic -let items = SPLIT(text, ",") -let first = items[0] -let count = LEN(items) -``` - -## Tool Definition - -Tools are BASIC files that the AI can call. - -### Structure - -```basic -' tool-name.bas - -PARAM parametername AS TYPE LIKE "example" DESCRIPTION "What this parameter is" -PARAM optionalparam AS STRING DESCRIPTION "Optional parameter" - -DESCRIPTION "What this tool does. Called when user wants to [action]." - -' Implementation -IF parametername = "" THEN - TALK "Please provide the parameter." - parametername = HEAR -END IF - -let result = "processed: " + parametername - -SAVE "records.csv", parametername, result - -TALK "✅ Done: " + result - -RETURN result -``` - -### Parameter Types - -| Type | Description | -|------|-------------| -| `STRING` | Text value | -| `INTEGER` | Whole number | -| `NUMBER` | Decimal number | -| `BOOLEAN` | True/False | -| `DATE` | Date value | -| `EMAIL` | Email address | -| `PHONE` | Phone number | - -## Comments - -```basic -' This is a single-line comment - -REM This is also a comment - -' Multi-line comments use multiple single-line comments -' Line 1 -' Line 2 -``` - -## Built-in Functions - -### String Functions - -| Function | Description | Example | -|----------|-------------|---------| -| `LEN(s)` | String length | `LEN("hello")` → `5` | -| `UCASE(s)` | Uppercase | `UCASE("hello")` → `"HELLO"` | -| `LCASE(s)` | Lowercase | `LCASE("HELLO")` → `"hello"` | -| `TRIM(s)` | Remove whitespace | `TRIM(" hi ")` → `"hi"` | -| `MID(s,start,len)` | Substring | `MID("hello",2,3)` → `"ell"` | -| `LEFT(s,n)` | Left characters | `LEFT("hello",2)` → `"he"` | -| `RIGHT(s,n)` | Right characters | `RIGHT("hello",2)` → `"lo"` | -| `SPLIT(s,delim)` | Split to array | `SPLIT("a,b,c",",")` → `["a","b","c"]` | -| `REPLACE(s,old,new)` | Replace text | `REPLACE("hello","l","x")` → `"hexxo"` | - -### Date Functions - -| Function | Description | Example | -|----------|-------------|---------| -| `NOW()` | Current datetime | `NOW()` | -| `TODAY()` | Current date | `TODAY()` | -| `YEAR(d)` | Extract year | `YEAR(date)` → `2024` | -| `MONTH(d)` | Extract month | `MONTH(date)` → `12` | -| `DAY(d)` | Extract day | `DAY(date)` → `15` | -| `DATEADD(d,n,unit)` | Add to date | `DATEADD(date,7,"days")` | -| `DATEDIFF(d1,d2,unit)` | Date difference | `DATEDIFF(date1,date2,"days")` | - -### Math Functions - -| Function | Description | Example | -|----------|-------------|---------| -| `ABS(n)` | Absolute value | `ABS(-5)` → `5` | -| `ROUND(n,d)` | Round number | `ROUND(3.456,2)` → `3.46` | -| `FLOOR(n)` | Round down | `FLOOR(3.7)` → `3` | -| `CEILING(n)` | Round up | `CEILING(3.2)` → `4` | -| `MIN(a,b)` | Minimum | `MIN(5,3)` → `3` | -| `MAX(a,b)` | Maximum | `MAX(5,3)` → `5` | -| `SUM(arr)` | Sum of array | `SUM(numbers)` | -| `AVG(arr)` | Average | `AVG(numbers)` | - -### Conversion Functions - -| Function | Description | Example | -|----------|-------------|---------| -| `STR(n)` | Number to string | `STR(42)` → `"42"` | -| `VAL(s)` | String to number | `VAL("42")` → `42` | -| `INT(n)` | To integer | `INT(3.7)` → `3` | - -## Complete Example - -```basic -' customer-support.bas - Main support dialog - -' Setup -USE KB "support-docs" -USE TOOL "create-ticket" -USE TOOL "check-order" -USE TOOL "request-refund" - -SET CONTEXT "support" AS "You are a helpful customer support agent for Acme Store." - -' Welcome -BEGIN TALK - **Welcome to Acme Support!** - - I can help you with: - • Order tracking - • Returns and refunds - • Product questions -END TALK - -' Quick actions -CLEAR SUGGESTIONS -ADD SUGGESTION "order" AS "Track my order" -ADD SUGGESTION "return" AS "Request a return" -ADD SUGGESTION "help" AS "Other questions" - -BEGIN SYSTEM PROMPT - Be friendly and professional. - Always verify order numbers before making changes. - For refunds over $100, escalate to human support. - If asked about competitors, politely redirect to our products. -END SYSTEM PROMPT -``` - -## Keyword Quick Reference - -| Category | Keywords | -|----------|----------| -| Conversation | `TALK`, `HEAR`, `WAIT` | -| Knowledge | `USE KB`, `CLEAR KB` | -| Tools | `USE TOOL`, `CLEAR TOOLS` | -| Context | `SET CONTEXT`, `SYSTEM PROMPT` | -| Suggestions | `ADD SUGGESTION`, `CLEAR SUGGESTIONS` | -| Data | `SAVE`, `GET`, `POST`, `PUT`, `DELETE HTTP` | -| HTTP | `SET HEADER`, `CLEAR HEADERS` | -| Files | `SEND FILE`, `DOWNLOAD`, `DELETE FILE` | -| Email | `SEND MAIL` | -| Memory | `SET BOT MEMORY`, `GET BOT MEMORY` | -| Schedule | `SET SCHEDULE` | -| Flow | `IF/THEN/ELSE/END IF`, `FOR/NEXT`, `FOR EACH` | -| Tools | `PARAM`, `DESCRIPTION`, `RETURN` | \ No newline at end of file diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md deleted file mode 100644 index 1e199667a..000000000 --- a/docs/reference/configuration.md +++ /dev/null @@ -1,344 +0,0 @@ -# Configuration Reference - -Complete reference for General Bots configuration options. - -## Configuration Files - -### Bot Configuration (`config.csv`) - -Located in each bot's `.gbot` folder: - -``` -mybot.gbai/ -└── mybot.gbot/ - └── config.csv -``` - -Format: CSV with `name,value` columns. - -```csv -name,value -theme-title,My Bot -theme-color1,#1565C0 -theme-color2,#E3F2FD -``` - -## Theme Settings - -| Setting | Description | Default | Example | -|---------|-------------|---------|---------| -| `theme-title` | Bot display name | Bot ID | `My Company Bot` | -| `theme-color1` | Primary color | `#1565C0` | `#2196F3` | -| `theme-color2` | Secondary/background | `#E3F2FD` | `#FFFFFF` | -| `theme-logo` | Logo URL | Default logo | `https://example.com/logo.svg` | -| `theme-favicon` | Favicon URL | Default | `https://example.com/favicon.ico` | - -### Color Examples - -```csv -name,value -theme-color1,#1565C0 -theme-color2,#E3F2FD -``` - -| Scheme | Primary | Secondary | -|--------|---------|-----------| -| Blue | `#1565C0` | `#E3F2FD` | -| Green | `#2E7D32` | `#E8F5E9` | -| Purple | `#7B1FA2` | `#F3E5F5` | -| Orange | `#EF6C00` | `#FFF3E0` | -| Red | `#C62828` | `#FFEBEE` | -| Dark | `#212121` | `#424242` | - -## Episodic Memory Settings - -| Setting | Description | Default | Range | -|---------|-------------|---------|-------| -| `episodic-memory-history` | Messages in context | `2` | 1-10 | -| `episodic-memory-threshold` | Compaction threshold | `4` | 2-20 | -| `episodic-memory-enabled` | Enable episodic memory | `true` | Boolean | -| `episodic-memory-model` | Model for summarization | `fast` | String | -| `episodic-memory-max-episodes` | Max episodes per user | `100` | 1-1000 | -| `episodic-memory-retention-days` | Days to retain episodes | `365` | 1-3650 | -| `episodic-memory-auto-summarize` | Auto-summarize conversations | `true` | Boolean | - -```csv -name,value -episodic-memory-history,2 -episodic-memory-threshold,4 -episodic-memory-enabled,true -episodic-memory-auto-summarize,true -``` - -### History Settings - -- `episodic-memory-history=1`: Minimal context, faster responses -- `episodic-memory-history=2`: Balanced (recommended) -- `episodic-memory-history=5`: More context, slower responses - -## LLM Settings - -| Setting | Description | Default | -|---------|-------------|---------| -| `llm-provider` | LLM provider | `openai` | -| `llm-model` | Model name | `gpt-5` | -| `llm-api-key` | API key (or use env) | - | -| `llm-endpoint` | Custom endpoint | Provider default | - -```csv -name,value -llm-provider,openai -llm-model,gpt-5 -``` - -### Supported Providers - -| Provider | Models | -|----------|--------| -| `openai` | `gpt-5`, `gpt-5-mini`, `o3` | -| `anthropic` | `claude-sonnet-4.5`, `claude-opus-4.5` | -| `groq` | `llama-3.3-70b`, `mixtral-8x7b` | -| `ollama` | Any local model | - -## Feature Flags - -| Setting | Description | Default | -|---------|-------------|---------| -| `feature-voice` | Enable voice input/output | `false` | -| `feature-file-upload` | Allow file uploads | `true` | -| `feature-suggestions` | Show quick replies | `true` | -| `feature-typing` | Show typing indicator | `true` | -| `feature-history` | Show chat history | `true` | - -```csv -name,value -feature-voice,true -feature-file-upload,true -feature-suggestions,true -``` - -## Security Settings - -| Setting | Description | Default | -|---------|-------------|---------| -| `auth-required` | Require authentication | `false` | -| `auth-provider` | Auth provider | `zitadel` | -| `allowed-domains` | Allowed email domains | `*` | -| `rate-limit` | Requests per minute | `60` | - -```csv -name,value -auth-required,true -auth-provider,zitadel -allowed-domains,example.com,company.org -rate-limit,30 -``` - -## Environment Variables - -### Required - -| Variable | Description | -|----------|-------------| -| `DIRECTORY_URL` | Zitadel/Auth instance URL | -| `DIRECTORY_CLIENT_ID` | OAuth client ID | -| `DIRECTORY_CLIENT_SECRET` | OAuth client secret | - -### Optional Overrides - -| Variable | Description | Default | -|----------|-------------|---------| -| `DATABASE_URL` | PostgreSQL connection | Auto-configured | -| `REDIS_URL` | Redis connection | Auto-configured | -| `S3_ENDPOINT` | S3/MinIO endpoint | Auto-configured | -| `S3_ACCESS_KEY` | S3 access key | Auto-configured | -| `S3_SECRET_KEY` | S3 secret key | Auto-configured | -| `QDRANT_URL` | Vector DB URL | Auto-configured | - -### LLM API Keys - -| Variable | Description | -|----------|-------------| -| `OPENAI_API_KEY` | OpenAI API key | -| `ANTHROPIC_API_KEY` | Anthropic API key | -| `GROQ_API_KEY` | Groq API key | - -### Server Settings - -| Variable | Description | Default | -|----------|-------------|---------| -| `PORT` | HTTP server port | `8080` | -| `HOST` | Bind address | `0.0.0.0` | -| `RUST_LOG` | Log level | `info` | -| `WORKERS` | Thread pool size | CPU cores | - -### Example `.env` File - -```bash -# Authentication (required) -DIRECTORY_URL=https://auth.example.com -DIRECTORY_CLIENT_ID=abc123 -DIRECTORY_CLIENT_SECRET=secret - -# LLM Provider -OPENAI_API_KEY=sk-... - -# Optional overrides -DATABASE_URL=postgres://user:pass@localhost/botserver -REDIS_URL=redis://localhost:6379 - -# Server -PORT=8080 -RUST_LOG=info,botserver=debug -``` - -## Rate Limiting - -### Environment Variables - -| Variable | Description | Default | -|----------|-------------|---------| -| `RATE_LIMIT_ENABLED` | Enable rate limiting | `true` | -| `RATE_LIMIT_API_RPS` | API requests/second | `100` | -| `RATE_LIMIT_API_BURST` | API burst limit | `200` | -| `RATE_LIMIT_AUTH_RPS` | Auth requests/second | `10` | -| `RATE_LIMIT_AUTH_BURST` | Auth burst limit | `20` | -| `RATE_LIMIT_LLM_RPS` | LLM requests/second | `5` | -| `RATE_LIMIT_LLM_BURST` | LLM burst limit | `10` | - -```bash -RATE_LIMIT_ENABLED=true -RATE_LIMIT_API_RPS=100 -RATE_LIMIT_LLM_RPS=5 -``` - -## Logging - -### Log Levels - -| Level | Description | -|-------|-------------| -| `error` | Errors only | -| `warn` | Warnings and errors | -| `info` | General information (default) | -| `debug` | Detailed debugging | -| `trace` | Very verbose | - -### Module-Specific Logging - -```bash -# General info, debug for botserver -RUST_LOG=info,botserver=debug - -# Quiet except errors, debug for specific module -RUST_LOG=error,botserver::llm=debug - -# Full trace for development -RUST_LOG=trace -``` - -## Database Configuration - -### Connection Pool - -| Setting | Description | Default | -|---------|-------------|---------| -| `DB_POOL_MIN` | Minimum connections | `2` | -| `DB_POOL_MAX` | Maximum connections | `10` | -| `DB_TIMEOUT` | Connection timeout (sec) | `30` | - -### PostgreSQL Tuning - -```sql --- Recommended settings for production -ALTER SYSTEM SET shared_buffers = '256MB'; -ALTER SYSTEM SET effective_cache_size = '1GB'; -ALTER SYSTEM SET max_connections = 200; -``` - -## Storage Configuration - -### S3/MinIO Settings - -| Variable | Description | Default | -|----------|-------------|---------| -| `S3_ENDPOINT` | Endpoint URL | Auto | -| `S3_REGION` | AWS region | `us-east-1` | -| `S3_BUCKET` | Default bucket | `botserver` | -| `S3_ACCESS_KEY` | Access key | Auto | -| `S3_SECRET_KEY` | Secret key | Auto | - -## Cache Configuration - -### Redis Settings - -| Variable | Description | Default | -|----------|-------------|---------| -| `REDIS_URL` | Connection URL | Auto | -| `CACHE_TTL` | Default TTL (seconds) | `3600` | -| `SEMANTIC_CACHE_ENABLED` | Enable LLM caching | `true` | -| `SEMANTIC_CACHE_THRESHOLD` | Similarity threshold | `0.95` | - -## Complete Example - -### config.csv - -```csv -name,value -theme-title,Acme Support Bot -theme-color1,#1565C0 -theme-color2,#E3F2FD -theme-logo,https://acme.com/logo.svg -episodic-memory-history,2 -episodic-memory-threshold,4 -llm-provider,openai -llm-model,gpt-5 -feature-voice,false -feature-file-upload,true -feature-suggestions,true -auth-required,true -rate-limit,30 -``` - -### .env - -```bash -# Auth -DIRECTORY_URL=https://auth.acme.com -DIRECTORY_CLIENT_ID=bot-client -DIRECTORY_CLIENT_SECRET=supersecret - -# LLM -OPENAI_API_KEY=sk-... - -# Server -PORT=8080 -RUST_LOG=info - -# Rate limiting -RATE_LIMIT_ENABLED=true -RATE_LIMIT_API_RPS=100 -``` - -## Configuration Precedence - -1. **Environment variables** (highest priority) -2. **Bot config.csv** -3. **Default values** (lowest priority) - -Environment variables always override config.csv settings. - -## Validation - -On startup, General Bots validates configuration and logs warnings for: - -- Missing required settings -- Invalid values -- Deprecated options -- Security concerns (e.g., weak rate limits) - -Check logs for configuration issues: - -```bash -RUST_LOG=info cargo run 2>&1 | grep -i config -``` diff --git a/src/analytics/mod.rs b/src/analytics/mod.rs index 86ae33149..ac08de10f 100644 --- a/src/analytics/mod.rs +++ b/src/analytics/mod.rs @@ -1,3 +1,4 @@ +use crate::llm::observability::{ObservabilityConfig, ObservabilityManager, QuickStats}; use crate::shared::state::AppState; use axum::{ extract::State, @@ -8,6 +9,7 @@ use axum::{ use diesel::prelude::*; use serde::{Deserialize, Serialize}; use std::sync::Arc; +use tokio::sync::RwLock; #[derive(Debug, Clone, Serialize, Deserialize, Queryable)] pub struct AnalyticsStats { @@ -47,9 +49,45 @@ pub struct AnalyticsQuery { pub time_range: Option, } +#[derive(Debug)] +pub struct AnalyticsService { + observability: Arc>, +} + +impl AnalyticsService { + pub fn new() -> Self { + let config = ObservabilityConfig::default(); + Self { + observability: Arc::new(RwLock::new(ObservabilityManager::new(config))), + } + } + + pub fn with_config(config: ObservabilityConfig) -> Self { + Self { + observability: Arc::new(RwLock::new(ObservabilityManager::new(config))), + } + } + + pub async fn get_quick_stats(&self) -> QuickStats { + let manager = self.observability.read().await; + manager.get_quick_stats() + } + + pub async fn get_observability_manager( + &self, + ) -> tokio::sync::RwLockReadGuard<'_, ObservabilityManager> { + self.observability.read().await + } +} + +impl Default for AnalyticsService { + fn default() -> Self { + Self::new() + } +} + pub fn configure_analytics_routes() -> Router> { Router::new() - // Metric cards - match frontend hx-get endpoints .route("/api/analytics/messages/count", get(handle_message_count)) .route( "/api/analytics/sessions/active", @@ -59,7 +97,6 @@ pub fn configure_analytics_routes() -> Router> { .route("/api/analytics/llm/tokens", get(handle_llm_tokens)) .route("/api/analytics/storage/usage", get(handle_storage_usage)) .route("/api/analytics/errors/count", get(handle_errors_count)) - // Timeseries charts .route( "/api/analytics/timeseries/messages", get(handle_timeseries_messages), @@ -68,7 +105,6 @@ pub fn configure_analytics_routes() -> Router> { "/api/analytics/timeseries/response_time", get(handle_timeseries_response), ) - // Distribution charts .route( "/api/analytics/channels/distribution", get(handle_channels_distribution), @@ -77,17 +113,16 @@ pub fn configure_analytics_routes() -> Router> { "/api/analytics/bots/performance", get(handle_bots_performance), ) - // Activity and queries .route( "/api/analytics/activity/recent", get(handle_recent_activity), ) .route("/api/analytics/queries/top", get(handle_top_queries)) - // Chat endpoint for analytics assistant .route("/api/analytics/chat", post(handle_analytics_chat)) + .route("/api/analytics/llm/stats", get(handle_llm_stats)) + .route("/api/analytics/budget/status", get(handle_budget_status)) } -/// GET /api/analytics/messages/count - Messages Today metric card pub async fn handle_message_count(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); @@ -110,9 +145,6 @@ pub async fn handle_message_count(State(state): State>) -> impl In .await .unwrap_or(0); - let trend = if count > 100 { "+12%" } else { "+5%" }; - let trend_class = "trend-up"; - let mut html = String::new(); html.push_str("
"); html.push_str(""); @@ -122,17 +154,11 @@ pub async fn handle_message_count(State(state): State>) -> impl In html.push_str(&format_number(count)); html.push_str(""); html.push_str("Messages Today"); - html.push_str(""); - html.push_str(trend); - html.push_str(""); html.push_str("
"); Html(html) } -/// GET /api/analytics/sessions/active - Active Sessions metric card pub async fn handle_active_sessions(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); @@ -146,7 +172,7 @@ pub async fn handle_active_sessions(State(state): State>) -> impl }; diesel::sql_query( - "SELECT COUNT(*) as count FROM user_sessions WHERE updated_at > NOW() - INTERVAL '1 hour'", + "SELECT COUNT(DISTINCT session_id) as count FROM message_history WHERE created_at > NOW() - INTERVAL '30 minutes'", ) .get_result::(&mut db_conn) .map(|r| r.count) @@ -163,13 +189,12 @@ pub async fn handle_active_sessions(State(state): State>) -> impl html.push_str(""); html.push_str(&count.to_string()); html.push_str(""); - html.push_str("Active Now"); + html.push_str("Active Sessions"); html.push_str(""); Html(html) } -/// GET /api/analytics/response/avg - Average Response Time metric card pub async fn handle_avg_response_time(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); @@ -183,7 +208,7 @@ pub async fn handle_avg_response_time(State(state): State>) -> imp }; diesel::sql_query( - "SELECT AVG(EXTRACT(EPOCH FROM (updated_at - created_at))) as avg FROM user_sessions WHERE created_at > NOW() - INTERVAL '24 hours'", + "SELECT AVG(EXTRACT(EPOCH FROM (updated_at - created_at))) as avg FROM message_history WHERE role = 1 AND created_at > NOW() - INTERVAL '24 hours'", ) .get_result::(&mut db_conn) .map(|r| r.avg.unwrap_or(0.0)) @@ -193,7 +218,7 @@ pub async fn handle_avg_response_time(State(state): State>) -> imp .unwrap_or(0.0); let display_time = if avg_time < 1.0 { - format!("{}ms", (avg_time * 1000.0) as i64) + format!("{}ms", (avg_time * 1000.0) as i32) } else { format!("{:.1}s", avg_time) }; @@ -212,7 +237,6 @@ pub async fn handle_avg_response_time(State(state): State>) -> imp Html(html) } -/// GET /api/analytics/llm/tokens - LLM Tokens Used metric card pub async fn handle_llm_tokens(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); @@ -225,7 +249,6 @@ pub async fn handle_llm_tokens(State(state): State>) -> impl IntoR } }; - // Try to get token count from analytics_events or estimate from messages diesel::sql_query( "SELECT COALESCE(SUM((metadata->>'tokens')::bigint), COUNT(*) * 150) as count FROM message_history WHERE created_at > NOW() - INTERVAL '24 hours'", ) @@ -250,9 +273,7 @@ pub async fn handle_llm_tokens(State(state): State>) -> impl IntoR Html(html) } -/// GET /api/analytics/storage/usage - Storage Usage metric card pub async fn handle_storage_usage(State(_state): State>) -> impl IntoResponse { - // In production, this would query S3/Drive storage usage let usage_gb = 2.4f64; let total_gb = 10.0f64; let percentage = (usage_gb / total_gb * 100.0) as i32; @@ -273,7 +294,6 @@ pub async fn handle_storage_usage(State(_state): State>) -> impl I Html(html) } -/// GET /api/analytics/errors/count - Errors Count metric card pub async fn handle_errors_count(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); @@ -286,7 +306,6 @@ pub async fn handle_errors_count(State(state): State>) -> impl Int } }; - // Count errors from analytics_events table diesel::sql_query( "SELECT COUNT(*) as count FROM analytics_events WHERE event_type = 'error' AND created_at > NOW() - INTERVAL '24 hours'", ) @@ -297,12 +316,12 @@ pub async fn handle_errors_count(State(state): State>) -> impl Int .await .unwrap_or(0); - let status_class = if count == 0 { - "status-good" - } else if count < 10 { - "status-warning" + let status_class = if count > 10 { + "error" + } else if count > 0 { + "warning" } else { - "status-error" + "success" }; let mut html = String::new(); @@ -321,11 +340,10 @@ pub async fn handle_errors_count(State(state): State>) -> impl Int Html(html) } -/// GET /api/analytics/timeseries/messages - Messages chart data pub async fn handle_timeseries_messages(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); - let data = tokio::task::spawn_blocking(move || { + let hourly_data = tokio::task::spawn_blocking(move || { let mut db_conn = match conn.get() { Ok(c) => c, Err(e) => { @@ -335,7 +353,11 @@ pub async fn handle_timeseries_messages(State(state): State>) -> i }; diesel::sql_query( - "SELECT EXTRACT(HOUR FROM created_at)::float8 as hour, COUNT(*) as count FROM message_history WHERE created_at > NOW() - INTERVAL '24 hours' GROUP BY EXTRACT(HOUR FROM created_at) ORDER BY hour", + "SELECT EXTRACT(HOUR FROM created_at) as hour, COUNT(*) as count + FROM message_history + WHERE created_at > NOW() - INTERVAL '24 hours' + GROUP BY EXTRACT(HOUR FROM created_at) + ORDER BY hour", ) .load::(&mut db_conn) .unwrap_or_default() @@ -343,35 +365,44 @@ pub async fn handle_timeseries_messages(State(state): State>) -> i .await .unwrap_or_default(); - let max_count = data.iter().map(|d| d.count).max().unwrap_or(1).max(1); + let hours: Vec = (0..24).collect(); + let mut counts: Vec = vec![0; 24]; + + for data in hourly_data { + let hour_idx = data.hour as usize; + if hour_idx < 24 { + counts[hour_idx] = data.count; + } + } + + let labels: Vec = hours.iter().map(|h| format!("{}:00", h)).collect(); + let max_count = counts.iter().max().copied().unwrap_or(1).max(1); let mut html = String::new(); + html.push_str("
"); html.push_str("
"); - for i in 0..24 { - let count = data - .iter() - .find(|d| d.hour as i32 == i) - .map(|d| d.count) - .unwrap_or(0); - let height = (count as f64 / max_count as f64 * 100.0) as i32; - - html.push_str("
"); + for (i, count) in counts.iter().enumerate() { + let height_pct = (*count as f64 / max_count as f64) * 100.0; + html.push_str(&format!( + "
", + height_pct, labels[i], count + )); } html.push_str("
"); html.push_str("
"); - html.push_str("0h6h12h18h24h"); + for (i, label) in labels.iter().enumerate() { + if i % 4 == 0 { + html.push_str(&format!("{}", label)); + } + } + html.push_str("
"); html.push_str("
"); Html(html) } -/// GET /api/analytics/timeseries/response_time - Response time chart data pub async fn handle_timeseries_response(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); @@ -384,7 +415,7 @@ pub async fn handle_timeseries_response(State(state): State>) -> i avg_time: Option, } - let data = tokio::task::spawn_blocking(move || { + let hourly_data = tokio::task::spawn_blocking(move || { let mut db_conn = match conn.get() { Ok(c) => c, Err(e) => { @@ -394,7 +425,12 @@ pub async fn handle_timeseries_response(State(state): State>) -> i }; diesel::sql_query( - "SELECT EXTRACT(HOUR FROM created_at)::float8 as hour, AVG(EXTRACT(EPOCH FROM (updated_at - created_at))) as avg_time FROM user_sessions WHERE created_at > NOW() - INTERVAL '24 hours' GROUP BY EXTRACT(HOUR FROM created_at) ORDER BY hour", + "SELECT EXTRACT(HOUR FROM created_at) as hour, + AVG(EXTRACT(EPOCH FROM (updated_at - created_at))) as avg_time + FROM message_history + WHERE role = 1 AND created_at > NOW() - INTERVAL '24 hours' + GROUP BY EXTRACT(HOUR FROM created_at) + ORDER BY hour", ) .load::(&mut db_conn) .unwrap_or_default() @@ -402,25 +438,42 @@ pub async fn handle_timeseries_response(State(state): State>) -> i .await .unwrap_or_default(); - let mut html = String::new(); - html.push_str("
"); - html.push_str(""); - html.push_str(" = vec![0.0; 24]; - for (_i, point) in data.iter().enumerate() { - let x = (point.hour as f64 / 24.0 * 288.0) as i32; - let y = 100 - (point.avg_time.unwrap_or(0.0).min(10.0) / 10.0 * 100.0) as i32; - html.push_str(&format!("L{},{} ", x, y)); + for data in hourly_data { + let hour_idx = data.hour as usize; + if hour_idx < 24 { + avgs[hour_idx] = data.avg_time.unwrap_or(0.0); + } } - html.push_str("\" fill=\"none\" stroke=\"var(--accent-color)\" stroke-width=\"2\"/>"); - html.push_str(""); + let labels: Vec = (0..24).map(|h| format!("{}:00", h)).collect(); + let max_avg = avgs.iter().cloned().fold(0.0f64, f64::max).max(0.1); + + let mut html = String::new(); + html.push_str("
"); + html.push_str(""); + html.push_str(""); + html.push_str("
"); + for (i, label) in labels.iter().enumerate() { + if i % 4 == 0 { + html.push_str(&format!("{}", label)); + } + } + html.push_str("
"); html.push_str("
"); Html(html) } -/// GET /api/analytics/channels/distribution - Channel distribution pie chart pub async fn handle_channels_distribution(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); @@ -433,67 +486,65 @@ pub async fn handle_channels_distribution(State(state): State>) -> count: i64, } - let data = tokio::task::spawn_blocking(move || { + let channel_data = tokio::task::spawn_blocking(move || { let mut db_conn = match conn.get() { Ok(c) => c, Err(e) => { log::error!("DB connection error: {}", e); - return vec![ - ("Web".to_string(), 45i64), - ("API".to_string(), 30i64), - ("WhatsApp".to_string(), 15i64), - ("Other".to_string(), 10i64), - ]; + return Vec::new(); } }; - // Try to get real channel distribution - let result: Result, _> = diesel::sql_query( - "SELECT COALESCE(context_data->>'channel', 'Web') as channel, COUNT(*) as count FROM user_sessions WHERE created_at > NOW() - INTERVAL '24 hours' GROUP BY context_data->>'channel' ORDER BY count DESC LIMIT 5", + diesel::sql_query( + "SELECT COALESCE(channel, 'web') as channel, COUNT(*) as count + FROM sessions + WHERE created_at > NOW() - INTERVAL '7 days' + GROUP BY channel + ORDER BY count DESC + LIMIT 5", ) - .load(&mut db_conn); - - match result { - Ok(channels) if !channels.is_empty() => { - channels.into_iter().map(|c| (c.channel, c.count)).collect() - } - _ => vec![ - ("Web".to_string(), 45i64), - ("API".to_string(), 30i64), - ("WhatsApp".to_string(), 15i64), - ("Other".to_string(), 10i64), - ], - } + .load::(&mut db_conn) + .unwrap_or_default() }) .await .unwrap_or_default(); - let total: i64 = data.iter().map(|(_, c)| c).sum(); + let total: i64 = channel_data.iter().map(|c| c.count).sum(); let colors = ["#4f46e5", "#10b981", "#f59e0b", "#ef4444", "#8b5cf6"]; let mut html = String::new(); html.push_str("
"); + html.push_str("
"); + + let mut offset = 0.0f64; + for (i, data) in channel_data.iter().enumerate() { + let pct = if total > 0 { + (data.count as f64 / total as f64) * 100.0 + } else { + 0.0 + }; + let color = colors[i % colors.len()]; + html.push_str(&format!( + "
", + offset, pct, color + )); + offset += pct; + } + + html.push_str("
"); html.push_str("
"); - for (i, (channel, count)) in data.iter().enumerate() { - let percentage = if total > 0 { - (*count as f64 / total as f64 * 100.0) as i32 + for (i, data) in channel_data.iter().enumerate() { + let pct = if total > 0 { + (data.count as f64 / total as f64) * 100.0 } else { - 0 + 0.0 }; - let color = colors.get(i).unwrap_or(&"#6b7280"); - - html.push_str("
"); - html.push_str(""); - html.push_str(""); - html.push_str(&html_escape(channel)); - html.push_str(""); - html.push_str(""); - html.push_str(&percentage.to_string()); - html.push_str("%"); - html.push_str("
"); + let color = colors[i % colors.len()]; + html.push_str(&format!( + "
{} ({:.0}%)
", + color, html_escape(&data.channel), pct + )); } html.push_str("
"); @@ -502,7 +553,6 @@ pub async fn handle_channels_distribution(State(state): State>) -> Html(html) } -/// GET /api/analytics/bots/performance - Bot performance chart pub async fn handle_bots_performance(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); @@ -515,58 +565,48 @@ pub async fn handle_bots_performance(State(state): State>) -> impl count: i64, } - let data = tokio::task::spawn_blocking(move || { + let bot_data = tokio::task::spawn_blocking(move || { let mut db_conn = match conn.get() { Ok(c) => c, Err(e) => { log::error!("DB connection error: {}", e); - return vec![ - ("Default Bot".to_string(), 150i64), - ("Support Bot".to_string(), 89i64), - ("Sales Bot".to_string(), 45i64), - ]; + return Vec::new(); } }; - let result: Result, _> = diesel::sql_query( - "SELECT b.name, COUNT(s.id) as count FROM bots b LEFT JOIN user_sessions s ON s.bot_id = b.id AND s.created_at > NOW() - INTERVAL '24 hours' GROUP BY b.id, b.name ORDER BY count DESC LIMIT 5", + diesel::sql_query( + "SELECT b.name, COUNT(mh.id) as count + FROM bots b + LEFT JOIN sessions s ON s.bot_id = b.id + LEFT JOIN message_history mh ON mh.session_id = s.id + WHERE mh.created_at > NOW() - INTERVAL '24 hours' OR mh.created_at IS NULL + GROUP BY b.id, b.name + ORDER BY count DESC + LIMIT 5", ) - .load(&mut db_conn); - - match result { - Ok(bots) if !bots.is_empty() => { - bots.into_iter().map(|b| (b.name, b.count)).collect() - } - _ => vec![ - ("Default Bot".to_string(), 150i64), - ("Support Bot".to_string(), 89i64), - ("Sales Bot".to_string(), 45i64), - ], - } + .load::(&mut db_conn) + .unwrap_or_default() }) .await .unwrap_or_default(); - let max_count = data.iter().map(|(_, c)| *c).max().unwrap_or(1).max(1); + let max_count = bot_data.iter().map(|b| b.count).max().unwrap_or(1).max(1); let mut html = String::new(); html.push_str("
"); - for (name, count) in &data { - let width = (*count as f64 / max_count as f64 * 100.0) as i32; - - html.push_str("
"); - html.push_str(""); - html.push_str(&html_escape(name)); - html.push_str(""); - html.push_str("
"); - html.push_str("
"); - html.push_str("
"); - html.push_str(""); - html.push_str(&count.to_string()); - html.push_str(""); + for data in bot_data.iter() { + let pct = (data.count as f64 / max_count as f64) * 100.0; + html.push_str("
"); + html.push_str(&format!( + "{}", + html_escape(&data.name) + )); + html.push_str(&format!( + "
", + pct + )); + html.push_str(&format!("{}", data.count)); html.push_str("
"); } @@ -575,12 +615,22 @@ pub async fn handle_bots_performance(State(state): State>) -> impl Html(html) } -/// GET /api/analytics/activity/recent - Recent activity feed +#[derive(Debug, QueryableByName, Clone)] +#[diesel(check_for_backend(diesel::pg::Pg))] +struct ActivityRow { + #[diesel(sql_type = diesel::sql_types::Text)] + activity_type: String, + #[diesel(sql_type = diesel::sql_types::Text)] + description: String, + #[diesel(sql_type = diesel::sql_types::Text)] + time_ago: String, +} + pub async fn handle_recent_activity(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); let activities = tokio::task::spawn_blocking(move || { - let mut db_conn = match conn.get() { + let db_conn = match conn.get() { Ok(c) => c, Err(e) => { log::error!("DB connection error: {}", e); @@ -588,103 +638,64 @@ pub async fn handle_recent_activity(State(state): State>) -> impl } }; - #[derive(Debug, QueryableByName)] - #[diesel(check_for_backend(diesel::pg::Pg))] - struct ActivityRow { - #[diesel(sql_type = diesel::sql_types::Text)] - activity_type: String, - #[diesel(sql_type = diesel::sql_types::Text)] - description: String, - #[diesel(sql_type = diesel::sql_types::Text)] - time_ago: String, - } - - let result: Result, _> = diesel::sql_query( - "SELECT 'session' as activity_type, 'New conversation started' as description, - CASE - WHEN created_at > NOW() - INTERVAL '1 minute' THEN 'just now' - WHEN created_at > NOW() - INTERVAL '1 hour' THEN EXTRACT(MINUTE FROM NOW() - created_at)::text || 'm ago' - ELSE EXTRACT(HOUR FROM NOW() - created_at)::text || 'h ago' - END as time_ago - FROM user_sessions - WHERE created_at > NOW() - INTERVAL '24 hours' - ORDER BY created_at DESC LIMIT 10", + diesel::sql_query( + "SELECT + CASE + WHEN role = 0 THEN 'message' + WHEN role = 1 THEN 'response' + ELSE 'system' + END as activity_type, + SUBSTRING(content FROM 1 FOR 50) as description, + CASE + WHEN created_at > NOW() - INTERVAL '1 minute' THEN 'just now' + WHEN created_at > NOW() - INTERVAL '1 hour' THEN CONCAT(EXTRACT(MINUTE FROM NOW() - created_at)::int, 'm ago') + ELSE CONCAT(EXTRACT(HOUR FROM NOW() - created_at)::int, 'h ago') + END as time_ago + FROM message_history + ORDER BY created_at DESC + LIMIT 10", ) - .load(&mut db_conn); - - match result { - Ok(items) if !items.is_empty() => items - .into_iter() - .map(|i| ActivityItemSimple { - activity_type: i.activity_type, - description: i.description, - time_ago: i.time_ago, - }) - .collect(), - _ => get_default_activities(), - } + .load::(&mut { db_conn }) + .unwrap_or_else(|_| get_default_activities()) }) .await .unwrap_or_else(|_| get_default_activities()); let mut html = String::new(); + html.push_str("
"); - for activity in &activities { + for activity in activities.iter() { let icon = match activity.activity_type.as_str() { - "session" => "", - "error" => "", - "bot" => "", - _ => "", + "message" => "💬", + "response" => "🤖", + "error" => "⚠️", + _ => "📋", }; html.push_str("
"); - html.push_str(""); - html.push_str(icon); - html.push_str(""); - html.push_str(""); - html.push_str(&html_escape(&activity.description)); - html.push_str(""); - html.push_str(""); - html.push_str(&html_escape(&activity.time_ago)); - html.push_str(""); + html.push_str(&format!("{}", icon)); + html.push_str("
"); + html.push_str(&format!( + "{}", + html_escape(&activity.description) + )); + html.push_str(&format!( + "{}", + html_escape(&activity.time_ago) + )); + html.push_str("
"); html.push_str("
"); } - if activities.is_empty() { - html.push_str("
No recent activity
"); - } + html.push_str("
"); Html(html) } -fn get_default_activities() -> Vec { - vec![ - ActivityItemSimple { - activity_type: "session".to_string(), - description: "New conversation started".to_string(), - time_ago: "2m ago".to_string(), - }, - ActivityItemSimple { - activity_type: "session".to_string(), - description: "User query processed".to_string(), - time_ago: "5m ago".to_string(), - }, - ActivityItemSimple { - activity_type: "bot".to_string(), - description: "Bot response generated".to_string(), - time_ago: "8m ago".to_string(), - }, - ] +fn get_default_activities() -> Vec { + vec![] } -#[derive(Debug)] -struct ActivityItemSimple { - activity_type: String, - description: String, - time_ago: String, -} - -/// GET /api/analytics/queries/top - Top queries list pub async fn handle_top_queries(State(state): State>) -> impl IntoResponse { let conn = state.conn.clone(); @@ -702,31 +713,20 @@ pub async fn handle_top_queries(State(state): State>) -> impl Into Ok(c) => c, Err(e) => { log::error!("DB connection error: {}", e); - return vec![ - ("How do I get started?".to_string(), 42i64), - ("What are the pricing plans?".to_string(), 38i64), - ("How to integrate API?".to_string(), 25i64), - ("Contact support".to_string(), 18i64), - ]; + return Vec::new(); } }; - let result: Result, _> = diesel::sql_query( - "SELECT query, COUNT(*) as count FROM research_search_history WHERE created_at > NOW() - INTERVAL '24 hours' GROUP BY query ORDER BY count DESC LIMIT 10", + diesel::sql_query( + "SELECT SUBSTRING(content FROM 1 FOR 100) as query, COUNT(*) as count + FROM message_history + WHERE role = 0 AND created_at > NOW() - INTERVAL '24 hours' + GROUP BY SUBSTRING(content FROM 1 FOR 100) + ORDER BY count DESC + LIMIT 10", ) - .load(&mut db_conn); - - match result { - Ok(items) if !items.is_empty() => { - items.into_iter().map(|q| (q.query, q.count)).collect() - } - _ => vec![ - ("How do I get started?".to_string(), 42i64), - ("What are the pricing plans?".to_string(), 38i64), - ("How to integrate API?".to_string(), 25i64), - ("Contact support".to_string(), 18i64), - ], - } + .load::(&mut db_conn) + .unwrap_or_default() }) .await .unwrap_or_default(); @@ -734,17 +734,14 @@ pub async fn handle_top_queries(State(state): State>) -> impl Into let mut html = String::new(); html.push_str("
"); - for (i, (query, count)) in queries.iter().enumerate() { + for (i, q) in queries.iter().enumerate() { html.push_str("
"); - html.push_str(""); - html.push_str(&(i + 1).to_string()); - html.push_str(""); - html.push_str(""); - html.push_str(&html_escape(query)); - html.push_str(""); - html.push_str(""); - html.push_str(&count.to_string()); - html.push_str(""); + html.push_str(&format!("{}", i + 1)); + html.push_str(&format!( + "{}", + html_escape(&q.query) + )); + html.push_str(&format!("{}", q.count)); html.push_str("
"); } @@ -753,27 +750,24 @@ pub async fn handle_top_queries(State(state): State>) -> impl Into Html(html) } -/// POST /api/analytics/chat - Analytics chat assistant pub async fn handle_analytics_chat( State(_state): State>, Json(payload): Json, ) -> impl IntoResponse { let query = payload.query.unwrap_or_default(); - // In production, this would use the LLM to analyze data let response = if query.to_lowercase().contains("message") { - "Based on the current data, message volume has increased by 12% compared to yesterday. Peak hours are between 10 AM and 2 PM." + "Based on current data, message volume trends are being analyzed." } else if query.to_lowercase().contains("error") { - "Error rate is currently at 0.5%, which is within normal parameters. No critical issues detected in the last 24 hours." + "Error rate analysis is available in the errors dashboard." } else if query.to_lowercase().contains("performance") { - "Average response time is 245ms, which is 15% faster than last week. All systems are performing optimally." + "Performance metrics show average response times within normal parameters." } else { - "I can help you analyze your analytics data. Try asking about messages, errors, performance, or user activity." + "I can help analyze your data. Ask about messages, errors, or performance." }; let mut html = String::new(); html.push_str("
"); - html.push_str("
"); html.push_str("
"); html.push_str(&html_escape(response)); html.push_str("
"); @@ -782,7 +776,37 @@ pub async fn handle_analytics_chat( Html(html) } -// Helper functions +pub async fn handle_llm_stats(State(_state): State>) -> impl IntoResponse { + let service = AnalyticsService::new(); + let stats = service.get_quick_stats().await; + + let mut html = String::new(); + html.push_str("
"); + html.push_str(&format!("
Total Requests{}
", stats.total_requests)); + html.push_str(&format!("
Total Tokens{}
", stats.total_tokens)); + html.push_str(&format!("
Cache Hits{}
", stats.cache_hits)); + html.push_str(&format!("
Cache Hit Rate{:.1}%
", stats.cache_hit_rate * 100.0)); + html.push_str(&format!("
Error Rate{:.1}%
", stats.error_rate * 100.0)); + html.push_str("
"); + + Html(html) +} + +pub async fn handle_budget_status(State(_state): State>) -> impl IntoResponse { + let service = AnalyticsService::new(); + let manager = service.get_observability_manager().await; + let status = manager.get_budget_status().await; + + let mut html = String::new(); + html.push_str("
"); + html.push_str(&format!("
Daily Spend${:.2} / ${:.2}
", status.daily_spend, status.daily_limit)); + html.push_str(&format!("
Monthly Spend${:.2} / ${:.2}
", status.monthly_spend, status.monthly_limit)); + html.push_str(&format!("
Daily Remaining${:.2} ({:.0}%)
", status.daily_remaining, status.daily_percentage * 100.0)); + html.push_str(&format!("
Monthly Remaining${:.2} ({:.0}%)
", status.monthly_remaining, status.monthly_percentage * 100.0)); + html.push_str("
"); + + Html(html) +} fn format_number(n: i64) -> String { if n >= 1_000_000 { diff --git a/src/basic/keywords/create_site.rs b/src/basic/keywords/create_site.rs index c135470fa..cf2003beb 100644 --- a/src/basic/keywords/create_site.rs +++ b/src/basic/keywords/create_site.rs @@ -1,14 +1,26 @@ +//! CREATE SITE keyword implementation +//! +//! Stores app source files in .gbdrive/apps/{app_name}/ (MinIO/S3) +//! then syncs to site_path for serving via HTTP. + +use crate::llm::LLMProvider; use crate::shared::models::UserSession; use crate::shared::state::AppState; +use log::{debug, info, warn}; use rhai::Dynamic; use rhai::Engine; +use serde_json::json; use std::error::Error; use std::fs; use std::io::Read; use std::path::PathBuf; +use std::sync::Arc; -pub fn create_site_keyword(state: &AppState, _user: UserSession, engine: &mut Engine) { +/// Register the CREATE SITE keyword +pub fn create_site_keyword(state: &AppState, user: UserSession, engine: &mut Engine) { let state_clone = state.clone(); + let user_clone = user.clone(); + engine .register_custom_syntax( &["CREATE", "SITE", "$expr$", ",", "$expr$", ",", "$expr$"], @@ -20,12 +32,23 @@ pub fn create_site_keyword(state: &AppState, _user: UserSession, engine: &mut En let alias = context.eval_expression_tree(&inputs[0])?; let template_dir = context.eval_expression_tree(&inputs[1])?; let prompt = context.eval_expression_tree(&inputs[2])?; + let config = state_clone .config .as_ref() .expect("Config must be initialized") .clone(); - let fut = create_site(&config, alias, template_dir, prompt); + + let s3 = state_clone.s3_client.clone().map(std::sync::Arc::new); + let bucket = state_clone.bucket_name.clone(); + let bot_id = user_clone.bot_id.to_string(); + + #[cfg(feature = "llm")] + let llm: Option> = Some(state_clone.llm_provider.clone()); + #[cfg(not(feature = "llm"))] + let llm: Option> = None; + + let fut = create_site(config, s3, bucket, bot_id, llm, alias, template_dir, prompt); let result = tokio::task::block_in_place(|| tokio::runtime::Handle::current().block_on(fut)) .map_err(|e| format!("Site creation failed: {}", e))?; @@ -35,41 +58,471 @@ pub fn create_site_keyword(state: &AppState, _user: UserSession, engine: &mut En .unwrap(); } +/// Create a new site/app +/// +/// 1. Load templates from template_dir +/// 2. Generate HTML via LLM +/// 3. Store in .gbdrive/apps/{alias}/ (S3/MinIO) +/// 4. Sync to site_path/{alias}/ for HTTP serving async fn create_site( - config: &crate::config::AppConfig, + config: crate::config::AppConfig, + s3: Option>, + bucket: String, + bot_id: String, + llm: Option>, alias: Dynamic, template_dir: Dynamic, prompt: Dynamic, ) -> Result> { + let alias_str = alias.to_string(); + let template_dir_str = template_dir.to_string(); + let prompt_str = prompt.to_string(); + + info!( + "CREATE SITE: {} from template {}", + alias_str, template_dir_str + ); + + // 1. Load templates let base_path = PathBuf::from(&config.site_path); - let template_path = base_path.join(template_dir.to_string()); - let alias_path = base_path.join(alias.to_string()); + let template_path = base_path.join(&template_dir_str); - fs::create_dir_all(&alias_path).map_err(|e| e.to_string())?; + let combined_content = load_templates(&template_path)?; + // 2. Generate HTML via LLM + let generated_html = generate_html_from_prompt(llm, &combined_content, &prompt_str).await?; + + // 3. Store in .gbdrive/apps/{alias}/ (S3/MinIO) + let drive_path = format!("apps/{}", alias_str); + store_to_drive(&s3, &bucket, &bot_id, &drive_path, &generated_html).await?; + + // 4. Sync to site_path for HTTP serving + let serve_path = base_path.join(&alias_str); + sync_to_serve_path(&serve_path, &generated_html, &template_path).await?; + + info!( + "CREATE SITE: {} completed, available at /apps/{}", + alias_str, alias_str + ); + + Ok(format!("/apps/{}", alias_str)) +} + +/// Load all HTML templates from a directory +fn load_templates(template_path: &PathBuf) -> Result> { let mut combined_content = String::new(); - for entry in fs::read_dir(&template_path).map_err(|e| e.to_string())? { + + if !template_path.exists() { + return Err(format!("Template directory not found: {:?}", template_path).into()); + } + + for entry in fs::read_dir(template_path).map_err(|e| e.to_string())? { let entry = entry.map_err(|e| e.to_string())?; let path = entry.path(); + if path.extension().map_or(false, |ext| ext == "html") { let mut file = fs::File::open(&path).map_err(|e| e.to_string())?; let mut contents = String::new(); file.read_to_string(&mut contents) .map_err(|e| e.to_string())?; + + combined_content.push_str(&format!("\n", path.display())); combined_content.push_str(&contents); combined_content.push_str("\n\n--- TEMPLATE SEPARATOR ---\n\n"); + + debug!("Loaded template: {:?}", path); } } - let _full_prompt = format!( - "TEMPLATE FILES:\n{}\n\nPROMPT: {}\n\nGenerate a new HTML file cloning all previous TEMPLATE (keeping only the local _assets libraries use, no external resources), but turning this into this prompt:", - combined_content, - prompt.to_string() + if combined_content.is_empty() { + return Err("No HTML templates found in template directory".into()); + } + + Ok(combined_content) +} + +/// Generate HTML from templates and prompt using LLM +async fn generate_html_from_prompt( + llm: Option>, + templates: &str, + prompt: &str, +) -> Result> { + let full_prompt = format!( + r#"You are an expert HTML/HTMX developer. Generate a complete HTML application. + +TEMPLATE FILES FOR REFERENCE: +{} + +USER REQUEST: +{} + +REQUIREMENTS: +1. Clone the template structure and styling +2. Use ONLY local _assets (htmx.min.js, app.js, styles.css) - NO external CDNs +3. Use HTMX for all data operations: + - hx-get="/api/db/TABLE" for lists + - hx-post="/api/db/TABLE" for create + - hx-put="/api/db/TABLE/ID" for update + - hx-delete="/api/db/TABLE/ID" for delete +4. Include search with hx-trigger="keyup changed delay:300ms" +5. Generate semantic, accessible HTML +6. App context is automatic - just use /api/db/* paths + +OUTPUT: Complete index.html file only, no explanations."#, + templates, prompt ); - let llm_result = "".to_string(); - let index_path = alias_path.join("index.html"); - fs::write(index_path, llm_result).map_err(|e| e.to_string())?; + let html = match llm { + Some(provider) => { + let messages = json!([{ + "role": "user", + "content": full_prompt + }]); - Ok(alias_path.to_string_lossy().into_owned()) + match provider + .generate(&full_prompt, &messages, "gpt-4o-mini", "") + .await + { + Ok(response) => { + let cleaned = extract_html_from_response(&response); + if cleaned.contains(" { + warn!("LLM generation failed: {}, using placeholder", e); + generate_placeholder_html(prompt) + } + } + } + None => { + debug!("No LLM provider configured, using placeholder HTML"); + generate_placeholder_html(prompt) + } + }; + + debug!("Generated HTML ({} bytes)", html.len()); + Ok(html) } + +/// Extract HTML content from LLM response (removes markdown code blocks if present) +fn extract_html_from_response(response: &str) -> String { + let trimmed = response.trim(); + + if trimmed.starts_with("```html") { + let without_prefix = trimmed.strip_prefix("```html").unwrap_or(trimmed); + let without_suffix = without_prefix + .trim() + .strip_suffix("```") + .unwrap_or(without_prefix); + return without_suffix.trim().to_string(); + } + + if trimmed.starts_with("```") { + let without_prefix = trimmed.strip_prefix("```").unwrap_or(trimmed); + let without_suffix = without_prefix + .trim() + .strip_suffix("```") + .unwrap_or(without_prefix); + return without_suffix.trim().to_string(); + } + + trimmed.to_string() +} + +/// Generate placeholder HTML (until LLM integration is complete) +fn generate_placeholder_html(prompt: &str) -> String { + format!( + r##" + + + + + App + + + + +
+

Generated App

+

Prompt: {}

+
+ +
+
+

Data

+
+ Loading... +
+ +
+ + +
+
+
+ + + +"##, + prompt + ) +} + +/// Store app files to .gbdrive (S3/MinIO) +async fn store_to_drive( + s3: &Option>, + bucket: &str, + bot_id: &str, + drive_path: &str, + html_content: &str, +) -> Result<(), Box> { + let Some(s3_client) = s3 else { + debug!("S3 not configured, skipping drive storage"); + return Ok(()); + }; + let key = format!("{}.gbdrive/{}/index.html", bot_id, drive_path); + + info!("Storing to drive: s3://{}/{}", bucket, key); + + s3_client + .put_object() + .bucket(bucket) + .key(&key) + .body(html_content.as_bytes().to_vec().into()) + .content_type("text/html") + .send() + .await + .map_err(|e| format!("Failed to store to drive: {}", e))?; + + // Also store schema.json for table definitions + let schema_key = format!("{}.gbdrive/{}/schema.json", bot_id, drive_path); + let schema = r#"{"tables": {}, "version": 1}"#; + + s3_client + .put_object() + .bucket(bucket) + .key(&schema_key) + .body(schema.as_bytes().to_vec().into()) + .content_type("application/json") + .send() + .await + .map_err(|e| format!("Failed to store schema: {}", e))?; + + Ok(()) +} + +/// Sync app files to serve path for HTTP serving +async fn sync_to_serve_path( + serve_path: &PathBuf, + html_content: &str, + template_path: &PathBuf, +) -> Result<(), Box> { + // Create app directory + fs::create_dir_all(serve_path).map_err(|e| format!("Failed to create serve path: {}", e))?; + + // Write index.html + let index_path = serve_path.join("index.html"); + fs::write(&index_path, html_content) + .map_err(|e| format!("Failed to write index.html: {}", e))?; + + info!("Written: {:?}", index_path); + + // Copy _assets from template + let template_assets = template_path.join("_assets"); + let serve_assets = serve_path.join("_assets"); + + if template_assets.exists() { + copy_dir_recursive(&template_assets, &serve_assets)?; + info!("Copied assets to: {:?}", serve_assets); + } else { + // Create default assets + fs::create_dir_all(&serve_assets) + .map_err(|e| format!("Failed to create assets dir: {}", e))?; + + // Write minimal htmx + let htmx_path = serve_assets.join("htmx.min.js"); + if !htmx_path.exists() { + // In production, this would copy from a known location + fs::write(&htmx_path, "/* HTMX - include from CDN or bundle */") + .map_err(|e| format!("Failed to write htmx: {}", e))?; + } + + // Write minimal styles + let styles_path = serve_assets.join("styles.css"); + if !styles_path.exists() { + fs::write(&styles_path, DEFAULT_STYLES) + .map_err(|e| format!("Failed to write styles: {}", e))?; + } + + // Write app.js + let app_js_path = serve_assets.join("app.js"); + if !app_js_path.exists() { + fs::write(&app_js_path, DEFAULT_APP_JS) + .map_err(|e| format!("Failed to write app.js: {}", e))?; + } + } + + // Write schema.json + let schema_path = serve_path.join("schema.json"); + fs::write(&schema_path, r#"{"tables": {}, "version": 1}"#) + .map_err(|e| format!("Failed to write schema.json: {}", e))?; + + Ok(()) +} + +/// Recursively copy a directory +fn copy_dir_recursive(src: &PathBuf, dst: &PathBuf) -> Result<(), Box> { + fs::create_dir_all(dst).map_err(|e| format!("Failed to create dir {:?}: {}", dst, e))?; + + for entry in fs::read_dir(src).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let src_path = entry.path(); + let dst_path = dst.join(entry.file_name()); + + if src_path.is_dir() { + copy_dir_recursive(&src_path, &dst_path)?; + } else { + fs::copy(&src_path, &dst_path) + .map_err(|e| format!("Failed to copy {:?}: {}", src_path, e))?; + } + } + + Ok(()) +} + +/// Default CSS styles for generated apps +const DEFAULT_STYLES: &str = r#" +:root { + --primary: #0ea5e9; + --success: #22c55e; + --warning: #f59e0b; + --danger: #ef4444; + --bg: #ffffff; + --text: #1e293b; + --border: #e2e8f0; + --radius: 8px; +} + +@media (prefers-color-scheme: dark) { + :root { + --bg: #0f172a; + --text: #f1f5f9; + --border: #334155; + } +} + +* { box-sizing: border-box; margin: 0; padding: 0; } + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: var(--bg); + color: var(--text); + line-height: 1.5; +} + +header { + padding: 1rem 2rem; + border-bottom: 1px solid var(--border); +} + +main { + padding: 2rem; + max-width: 1200px; + margin: 0 auto; +} + +h1, h2, h3 { margin-bottom: 1rem; } + +input, select, textarea { + padding: 0.5rem 1rem; + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--bg); + color: var(--text); + font-size: 1rem; +} + +input:focus, select:focus, textarea:focus { + outline: none; + border-color: var(--primary); +} + +button { + padding: 0.5rem 1rem; + background: var(--primary); + color: white; + border: none; + border-radius: var(--radius); + cursor: pointer; + font-size: 1rem; +} + +button:hover { opacity: 0.9; } + +form { + display: flex; + gap: 0.5rem; + margin: 1rem 0; +} + +table { + width: 100%; + border-collapse: collapse; +} + +th, td { + padding: 0.75rem; + text-align: left; + border-bottom: 1px solid var(--border); +} + +.htmx-indicator { + opacity: 0; + transition: opacity 0.2s; +} + +.htmx-request .htmx-indicator { + opacity: 1; +} +"#; + +/// Default JavaScript for generated apps +const DEFAULT_APP_JS: &str = r#" +// Toast notifications +function toast(message, type = 'info') { + const el = document.createElement('div'); + el.className = 'toast toast-' + type; + el.textContent = message; + el.style.cssText = 'position:fixed;bottom:20px;right:20px;padding:1rem;background:#333;color:#fff;border-radius:8px;z-index:9999;'; + document.body.appendChild(el); + setTimeout(() => el.remove(), 3000); +} + +// HTMX event handlers +document.body.addEventListener('htmx:afterSwap', function(e) { + console.log('Data updated:', e.detail.target.id); +}); + +document.body.addEventListener('htmx:responseError', function(e) { + toast('Error: ' + (e.detail.xhr.responseText || 'Request failed'), 'error'); +}); + +// Modal helpers +function openModal(id) { + document.getElementById(id)?.classList.add('active'); +} + +function closeModal(id) { + document.getElementById(id)?.classList.remove('active'); +} +"#; diff --git a/src/console/editor.rs b/src/console/editor.rs index 9919e5e50..05e79c5f6 100644 --- a/src/console/editor.rs +++ b/src/console/editor.rs @@ -66,11 +66,15 @@ impl Editor { pub fn file_path(&self) -> &str { &self.file_path } - #[allow(dead_code)] + pub fn set_visible_lines(&mut self, lines: usize) { self.visible_lines = lines.max(5); } + pub fn visible_lines(&self) -> usize { + self.visible_lines + } + fn get_cursor_line(&self) -> usize { self.content[..self.cursor_pos].lines().count() } @@ -90,10 +94,10 @@ impl Editor { } } - pub fn render(&self, cursor_blink: bool) -> String { + pub fn render(&mut self, cursor_blink: bool) -> String { let lines: Vec<&str> = self.content.lines().collect(); let total_lines = lines.len().max(1); - let visible_lines = self.visible_lines; + let visible_lines = self.visible_lines(); let cursor_line = self.get_cursor_line(); let cursor_col = self.content[..self.cursor_pos] .lines() @@ -189,19 +193,6 @@ impl Editor { } } - #[allow(dead_code)] - pub fn scroll_up(&mut self) { - self.scroll_offset = self.scroll_offset.saturating_sub(1); - } - - #[allow(dead_code)] - pub fn scroll_down(&mut self) { - let total_lines = self.content.lines().count().max(1); - let max_scroll = total_lines.saturating_sub(self.visible_lines.saturating_sub(3)); - if self.scroll_offset < max_scroll { - self.scroll_offset += 1; - } - } pub fn move_left(&mut self) { if self.cursor_pos > 0 { self.cursor_pos -= 1; diff --git a/src/console/mod.rs b/src/console/mod.rs index dfffc21fd..06a19238f 100644 --- a/src/console/mod.rs +++ b/src/console/mod.rs @@ -244,20 +244,35 @@ impl XtreeUI { title_bg, title_fg, ); - if let Some(editor) = &self.editor { - self.render_editor( - f, - content_chunks[1], - editor, - bg, - text, - border_active, - border_inactive, - highlight, - title_bg, - title_fg, - cursor_blink, - ); + if let Some(editor) = &mut self.editor { + let area = content_chunks[1]; + editor.set_visible_lines(area.height.saturating_sub(4) as usize); + let is_active = self.active_panel == ActivePanel::Editor; + let border_color = if is_active { + border_active + } else { + border_inactive + }; + let title_style = if is_active { + Style::default() + .fg(title_fg) + .bg(title_bg) + .add_modifier(Modifier::BOLD) + } else { + Style::default().fg(title_fg).bg(title_bg) + }; + let title_text = format!(" EDITOR: {} ", editor.file_path()); + let block = Block::default() + .title(Span::styled(title_text, title_style)) + .borders(Borders::ALL) + .border_style(Style::default().fg(border_color)) + .style(Style::default().bg(bg)); + let content = editor.render(cursor_blink); + let paragraph = Paragraph::new(content) + .block(block) + .style(Style::default().fg(text)) + .wrap(Wrap { trim: false }); + f.render_widget(paragraph, area); } self.render_chat( f, @@ -621,47 +636,6 @@ impl XtreeUI { .wrap(Wrap { trim: false }); f.render_widget(paragraph, area); } - fn render_editor( - &self, - f: &mut Frame, - area: Rect, - editor: &Editor, - bg: Color, - text: Color, - border_active: Color, - border_inactive: Color, - _highlight: Color, - title_bg: Color, - title_fg: Color, - cursor_blink: bool, - ) { - let is_active = self.active_panel == ActivePanel::Editor; - let border_color = if is_active { - border_active - } else { - border_inactive - }; - let title_style = if is_active { - Style::default() - .fg(title_fg) - .bg(title_bg) - .add_modifier(Modifier::BOLD) - } else { - Style::default().fg(title_fg).bg(title_bg) - }; - let title_text = format!(" EDITOR: {} ", editor.file_path()); - let block = Block::default() - .title(Span::styled(title_text, title_style)) - .borders(Borders::ALL) - .border_style(Style::default().fg(border_color)) - .style(Style::default().bg(bg)); - let content = editor.render(cursor_blink); - let paragraph = Paragraph::new(content) - .block(block) - .style(Style::default().fg(text)) - .wrap(Wrap { trim: false }); - f.render_widget(paragraph, area); - } fn render_chat( &self, f: &mut Frame, diff --git a/src/core/kb/document_processor.rs b/src/core/kb/document_processor.rs index c2f0d9cf5..5e3776b31 100644 --- a/src/core/kb/document_processor.rs +++ b/src/core/kb/document_processor.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - use anyhow::Result; use log::{error, info, warn}; use serde::{Deserialize, Serialize}; @@ -223,23 +221,6 @@ impl DocumentProcessor { } } - /// Extract PDF using poppler-utils - #[allow(dead_code)] - async fn extract_pdf_with_poppler(&self, file_path: &Path) -> Result { - let output = tokio::process::Command::new("pdftotext") - .arg(file_path) - .arg("-") - .output() - .await?; - - if output.status.success() { - Ok(String::from_utf8_lossy(&output.stdout).to_string()) - } else { - // Fallback to library extraction - self.extract_pdf_with_library(file_path).await - } - } - /// Extract PDF using rust library (fallback) async fn extract_pdf_with_library(&self, file_path: &Path) -> Result { use pdf_extract::extract_text; diff --git a/src/core/secrets/mod.rs b/src/core/secrets/mod.rs index e93c4b605..38e8baac6 100644 --- a/src/core/secrets/mod.rs +++ b/src/core/secrets/mod.rs @@ -378,23 +378,6 @@ impl SecretsManager { } } -#[allow(dead_code)] -fn parse_database_url(url: &str) -> Option> { - let url = url.strip_prefix("postgres://")?; - let (auth, rest) = url.split_once('@')?; - let (user, pass) = auth.split_once(':').unwrap_or((auth, "")); - let (host_port, database) = rest.split_once('/').unwrap_or((rest, "botserver")); - let (host, port) = host_port.split_once(':').unwrap_or((host_port, "5432")); - - Some(HashMap::from([ - ("username".into(), user.into()), - ("password".into(), pass.into()), - ("host".into(), host.into()), - ("port".into(), port.into()), - ("database".into(), database.into()), - ])) -} - pub fn init_secrets_manager() -> Result { SecretsManager::from_env() } @@ -422,6 +405,42 @@ impl BootstrapConfig { mod tests { use super::*; + /// Helper function to parse database URL into HashMap for tests + fn parse_database_url(url: &str) -> Result> { + let mut result = HashMap::new(); + if let Some(stripped) = url.strip_prefix("postgres://") { + let parts: Vec<&str> = stripped.split('@').collect(); + if parts.len() == 2 { + let user_pass: Vec<&str> = parts[0].split(':').collect(); + let host_db: Vec<&str> = parts[1].split('/').collect(); + + result.insert( + "username".to_string(), + user_pass.get(0).unwrap_or(&"").to_string(), + ); + result.insert( + "password".to_string(), + user_pass.get(1).unwrap_or(&"").to_string(), + ); + + let host_port: Vec<&str> = host_db[0].split(':').collect(); + result.insert( + "host".to_string(), + host_port.get(0).unwrap_or(&"").to_string(), + ); + result.insert( + "port".to_string(), + host_port.get(1).unwrap_or(&"5432").to_string(), + ); + + if host_db.len() >= 2 { + result.insert("database".to_string(), host_db[1].to_string()); + } + } + } + Ok(result) + } + #[test] fn test_parse_database_url() { let parsed = parse_database_url("postgres://user:pass@localhost:5432/mydb").unwrap(); diff --git a/src/core/shared/state.rs b/src/core/shared/state.rs index 4e4fc1c03..34e6c13e3 100644 --- a/src/core/shared/state.rs +++ b/src/core/shared/state.rs @@ -3,6 +3,10 @@ use crate::core::config::AppConfig; use crate::core::kb::KnowledgeBaseManager; use crate::core::session::SessionManager; use crate::core::shared::analytics::MetricsCollector; +#[cfg(all(test, feature = "directory"))] +use crate::core::shared::test_utils::create_mock_auth_service; +#[cfg(all(test, feature = "llm"))] +use crate::core::shared::test_utils::MockLLMProvider; #[cfg(feature = "directory")] use crate::directory::AuthService; #[cfg(feature = "llm")] @@ -198,71 +202,6 @@ impl std::fmt::Debug for AppState { } } -#[cfg(feature = "llm")] -#[derive(Debug)] -#[allow(dead_code)] -struct MockLLMProvider; - -#[cfg(feature = "llm")] -#[async_trait::async_trait] -impl LLMProvider for MockLLMProvider { - async fn generate( - &self, - _prompt: &str, - _config: &serde_json::Value, - _model: &str, - _key: &str, - ) -> Result> { - Ok("Mock response".to_string()) - } - - async fn generate_stream( - &self, - _prompt: &str, - _config: &serde_json::Value, - tx: mpsc::Sender, - _model: &str, - _key: &str, - ) -> Result<(), Box> { - let _ = tx.send("Mock response".to_string()).await; - Ok(()) - } - - async fn cancel_job( - &self, - _session_id: &str, - ) -> Result<(), Box> { - Ok(()) - } -} - -#[cfg(feature = "directory")] -#[allow(dead_code)] -fn create_mock_auth_service() -> AuthService { - use crate::directory::client::ZitadelConfig; - - let config = ZitadelConfig { - issuer_url: "http://localhost:8080".to_string(), - issuer: "http://localhost:8080".to_string(), - client_id: "mock_client_id".to_string(), - client_secret: "mock_client_secret".to_string(), - redirect_uri: "http://localhost:3000/callback".to_string(), - project_id: "mock_project_id".to_string(), - api_url: "http://localhost:8080".to_string(), - service_account_key: None, - }; - - let rt = tokio::runtime::Handle::try_current() - .map(|h| h.block_on(AuthService::new(config.clone()))) - .unwrap_or_else(|_| { - tokio::runtime::Runtime::new() - .expect("Failed to create runtime") - .block_on(AuthService::new(config)) - }); - - rt.expect("Failed to create mock AuthService") -} - /// Default implementation for AppState - ONLY FOR TESTS /// This will panic if Vault is not configured, so it must only be used in test contexts. #[cfg(test)] @@ -297,8 +236,8 @@ impl Default for AppState { session_manager: Arc::new(tokio::sync::Mutex::new(session_manager)), metrics_collector: MetricsCollector::new(), task_scheduler: None, - #[cfg(feature = "llm")] - llm_provider: Arc::new(MockLLMProvider), + #[cfg(all(test, feature = "llm"))] + llm_provider: Arc::new(MockLLMProvider::new()), #[cfg(feature = "directory")] auth_service: Arc::new(tokio::sync::Mutex::new(create_mock_auth_service())), channels: Arc::new(tokio::sync::Mutex::new(HashMap::new())), diff --git a/src/core/shared/test_utils.rs b/src/core/shared/test_utils.rs index 876e5622d..6458df5fa 100644 --- a/src/core/shared/test_utils.rs +++ b/src/core/shared/test_utils.rs @@ -221,7 +221,7 @@ impl Default for TestAppStateBuilder { } #[cfg(feature = "directory")] -fn create_mock_auth_service() -> AuthService { +pub fn create_mock_auth_service() -> AuthService { let config = ZitadelConfig { issuer_url: "http://localhost:8080".to_string(), issuer: "http://localhost:8080".to_string(), diff --git a/src/directory/client.rs b/src/directory/client.rs index 2c9143c61..a37a0d754 100644 --- a/src/directory/client.rs +++ b/src/directory/client.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use anyhow::{anyhow, Result}; use serde::{Deserialize, Serialize}; diff --git a/src/directory/groups.rs b/src/directory/groups.rs index 236a2c2f7..15bff08f8 100644 --- a/src/directory/groups.rs +++ b/src/directory/groups.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use axum::{ extract::{Path, Query, State}, diff --git a/src/directory/mod.rs b/src/directory/mod.rs index ab9bbebb6..5424d27ad 100644 --- a/src/directory/mod.rs +++ b/src/directory/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - use crate::shared::state::AppState; use axum::{ extract::{Query, State}, @@ -16,7 +14,7 @@ pub mod groups; pub mod router; pub mod users; -use self::client::{ZitadelClient, ZitadelConfig}; +pub use client::{ZitadelClient, ZitadelConfig}; pub struct AuthService { client: Arc, diff --git a/src/directory/router.rs b/src/directory/router.rs index 5341627c7..bf43bdf74 100644 --- a/src/directory/router.rs +++ b/src/directory/router.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use axum::{ routing::{delete, get, post, put}, diff --git a/src/directory/users.rs b/src/directory/users.rs index 6c1826f8d..b10d37157 100644 --- a/src/directory/users.rs +++ b/src/directory/users.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use axum::{ extract::{Path, Query, State}, diff --git a/src/drive/drive_monitor/mod.rs b/src/drive/drive_monitor/mod.rs index 7e236d206..2a802bdd8 100644 --- a/src/drive/drive_monitor/mod.rs +++ b/src/drive/drive_monitor/mod.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use crate::basic::compiler::BasicCompiler; use crate::config::ConfigManager; diff --git a/src/drive/mod.rs b/src/drive/mod.rs index 41d1d4f49..3b4e18820 100644 --- a/src/drive/mod.rs +++ b/src/drive/mod.rs @@ -348,8 +348,6 @@ pub async fn list_files( } #[cfg(feature = "console")] -/// Convert a FileTree to a list of FileItems for display in the console UI -#[allow(dead_code)] pub fn convert_tree_to_items(tree: &FileTree) -> Vec { let mut items = Vec::new(); diff --git a/src/drive/vectordb.rs b/src/drive/vectordb.rs index 266ed659e..163c27b5e 100644 --- a/src/drive/vectordb.rs +++ b/src/drive/vectordb.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - use anyhow::Result; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -60,9 +58,9 @@ pub struct FileSearchResult { /// Per-user drive vector DB manager #[derive(Debug)] pub struct UserDriveVectorDB { - _user_id: Uuid, - _bot_id: Uuid, - _collection_name: String, + user_id: Uuid, + bot_id: Uuid, + collection_name: String, db_path: PathBuf, #[cfg(feature = "vectordb")] client: Option>, @@ -74,15 +72,27 @@ impl UserDriveVectorDB { let collection_name = format!("drive_{}_{}", bot_id, user_id); Self { - _user_id: user_id, - _bot_id: bot_id, - _collection_name: collection_name, + user_id, + bot_id, + collection_name, db_path, #[cfg(feature = "vectordb")] client: None, } } + pub fn user_id(&self) -> Uuid { + self.user_id + } + + pub fn bot_id(&self) -> Uuid { + self.bot_id + } + + pub fn collection_name(&self) -> &str { + &self.collection_name + } + /// Initialize vector DB collection #[cfg(feature = "vectordb")] pub async fn initialize(&mut self, qdrant_url: &str) -> Result<()> { @@ -93,13 +103,13 @@ impl UserDriveVectorDB { let exists = collections .collections .iter() - .any(|c| c.name == self._collection_name); + .any(|c| c.name == self.collection_name); if !exists { // Create collection for file embeddings (1536 dimensions for OpenAI embeddings) client .create_collection(&CreateCollection { - collection_name: self._collection_name.clone(), + collection_name: self.collection_name.clone(), vectors_config: Some(VectorsConfig { config: Some(Config::Params(VectorParams { size: 1536, @@ -111,10 +121,7 @@ impl UserDriveVectorDB { }) .await?; - log::info!( - "Initialized vector DB collection: {}", - self._collection_name - ); + log::info!("Initialized vector DB collection: {}", self.collection_name); } self.client = Some(Arc::new(client)); @@ -143,7 +150,7 @@ impl UserDriveVectorDB { let point = PointStruct::new(file.id.clone(), embedding, payload); client - .upsert_points(self._collection_name.clone(), None, vec![point], None) + .upsert_points(self.collection_name.clone(), None, vec![point], None) .await?; log::debug!("Indexed file: {} - {}", file.id, file.file_name); @@ -181,7 +188,7 @@ impl UserDriveVectorDB { if !points.is_empty() { client - .upsert_points(self._collection_name.clone(), None, points, None) + .upsert_points(self.collection_name.clone(), None, points, None) .await?; } } @@ -241,7 +248,7 @@ impl UserDriveVectorDB { let search_result = client .search_points(&qdrant_client::qdrant::SearchPoints { - collection_name: self._collection_name.clone(), + collection_name: self.collection_name.clone(), vector: query_embedding, limit: query.limit as u64, filter, @@ -390,7 +397,7 @@ impl UserDriveVectorDB { client .delete_points( - self._collection_name.clone(), + self.collection_name.clone(), &vec![file_id.into()].into(), None, ) @@ -417,9 +424,7 @@ impl UserDriveVectorDB { .as_ref() .ok_or_else(|| anyhow::anyhow!("Vector DB not initialized"))?; - let info = client - .collection_info(self._collection_name.clone()) - .await?; + let info = client.collection_info(self.collection_name.clone()).await?; Ok(info.result.unwrap().points_count.unwrap_or(0)) } @@ -471,13 +476,13 @@ impl UserDriveVectorDB { .ok_or_else(|| anyhow::anyhow!("Vector DB not initialized"))?; client - .delete_collection(self._collection_name.clone()) + .delete_collection(self.collection_name.clone()) .await?; // Recreate empty collection client .create_collection(&CreateCollection { - collection_name: self._collection_name.clone(), + collection_name: self.collection_name.clone(), vectors_config: Some(VectorsConfig { config: Some(Config::Params(VectorParams { size: 1536, @@ -489,7 +494,7 @@ impl UserDriveVectorDB { }) .await?; - log::info!("Cleared drive vector collection: {}", self._collection_name); + log::info!("Cleared drive vector collection: {}", self.collection_name); Ok(()) } @@ -650,6 +655,6 @@ mod tests { let temp_dir = std::env::temp_dir().join("test_drive_vectordb"); let db = UserDriveVectorDB::new(Uuid::new_v4(), Uuid::new_v4(), temp_dir); - assert!(db._collection_name.starts_with("drive_")); + assert!(db.collection_name.starts_with("drive_")); } } diff --git a/src/llm/episodic_memory.rs b/src/llm/episodic_memory.rs index 1f0554a4b..a9a50b2c8 100644 --- a/src/llm/episodic_memory.rs +++ b/src/llm/episodic_memory.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use crate::core::config::ConfigManager; use crate::llm::llm_models; diff --git a/src/llm/llm_models/deepseek_r3.rs b/src/llm/llm_models/deepseek_r3.rs index 3d749aac9..e0e3bd801 100644 --- a/src/llm/llm_models/deepseek_r3.rs +++ b/src/llm/llm_models/deepseek_r3.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use super::ModelHandler; use regex; diff --git a/src/llm/llm_models/gpt_oss_120b.rs b/src/llm/llm_models/gpt_oss_120b.rs index e58b4e90d..50b68848a 100644 --- a/src/llm/llm_models/gpt_oss_120b.rs +++ b/src/llm/llm_models/gpt_oss_120b.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use super::ModelHandler; #[derive(Debug)] diff --git a/src/llm/llm_models/gpt_oss_20b.rs b/src/llm/llm_models/gpt_oss_20b.rs index 61115dda7..6cb958baf 100644 --- a/src/llm/llm_models/gpt_oss_20b.rs +++ b/src/llm/llm_models/gpt_oss_20b.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use super::ModelHandler; #[derive(Debug)] diff --git a/src/llm/llm_models/mod.rs b/src/llm/llm_models/mod.rs index be1347c2e..9eda5794a 100644 --- a/src/llm/llm_models/mod.rs +++ b/src/llm/llm_models/mod.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] pub mod deepseek_r3; pub mod gpt_oss_120b; diff --git a/src/llm/mod.rs b/src/llm/mod.rs index 60050d792..b8e79133d 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -1,15 +1,17 @@ -#![allow(dead_code)] - use async_trait::async_trait; use futures::StreamExt; use log::{info, trace}; use serde_json::Value; use tokio::sync::mpsc; + pub mod cache; pub mod episodic_memory; pub mod llm_models; pub mod local; pub mod observability; + +pub use llm_models::get_handler; + #[async_trait] pub trait LLMProvider: Send + Sync { async fn generate( @@ -19,6 +21,7 @@ pub trait LLMProvider: Send + Sync { model: &str, key: &str, ) -> Result>; + async fn generate_stream( &self, prompt: &str, @@ -27,114 +30,24 @@ pub trait LLMProvider: Send + Sync { model: &str, key: &str, ) -> Result<(), Box>; + async fn cancel_job( &self, session_id: &str, ) -> Result<(), Box>; } + #[derive(Debug)] pub struct OpenAIClient { client: reqwest::Client, base_url: String, } -#[async_trait] -impl LLMProvider for OpenAIClient { - async fn generate( - &self, - prompt: &str, - messages: &Value, - model: &str, - key: &str, - ) -> Result> { - let default_messages = serde_json::json!([{"role": "user", "content": prompt}]); - let response = self - .client - .post(&format!("{}/v1/chat/completions", self.base_url)) - .header("Authorization", format!("Bearer {}", key)) - .json(&serde_json::json!({ - "model": model, - "messages": if messages.is_array() && !messages.as_array().unwrap().is_empty() { - messages - } else { - &default_messages - } - })) - .send() - .await?; - let result: Value = response.json().await?; - let raw_content = result["choices"][0]["message"]["content"] - .as_str() - .unwrap_or(""); - let end_token = "final<|message|>"; - let content = if let Some(pos) = raw_content.find(end_token) { - raw_content[(pos + end_token.len())..].to_string() - } else { - raw_content.to_string() - }; - Ok(content) - } - async fn generate_stream( - &self, - prompt: &str, - messages: &Value, - tx: mpsc::Sender, - model: &str, - key: &str, - ) -> Result<(), Box> { - let default_messages = serde_json::json!([{"role": "user", "content": prompt}]); - let response = self - .client - .post(&format!("{}/v1/chat/completions", self.base_url)) - .header("Authorization", format!("Bearer {}", key)) - .json(&serde_json::json!({ - "model": model, - "messages": if messages.is_array() && !messages.as_array().unwrap().is_empty() { - info!("Using provided messages: {:?}", messages); - messages - } else { - &default_messages - }, - "stream": true - })) - .send() - .await?; - let status = response.status(); - if status != reqwest::StatusCode::OK { - let error_text = response.text().await.unwrap_or_default(); - trace!("LLM generate_stream error: {}", error_text); - return Err(format!("LLM request failed with status: {}", status).into()); - } - let mut stream = response.bytes_stream(); - let mut buffer = String::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - let chunk_str = String::from_utf8_lossy(&chunk); - for line in chunk_str.lines() { - if line.starts_with("data: ") && !line.contains("[DONE]") { - if let Ok(data) = serde_json::from_str::(&line[6..]) { - if let Some(content) = data["choices"][0]["delta"]["content"].as_str() { - buffer.push_str(content); - let _ = tx.send(content.to_string()).await; - } - } - } - } - } - Ok(()) - } - async fn cancel_job( - &self, - _session_id: &str, - ) -> Result<(), Box> { - Ok(()) - } -} impl OpenAIClient { pub fn new(_api_key: String, base_url: Option) -> Self { Self { client: reqwest::Client::new(), - base_url: base_url.unwrap(), + base_url: base_url.unwrap_or_else(|| "https://api.openai.com".to_string()), } } @@ -165,3 +78,110 @@ impl OpenAIClient { serde_json::Value::Array(messages) } } + +#[async_trait] +impl LLMProvider for OpenAIClient { + async fn generate( + &self, + prompt: &str, + messages: &Value, + model: &str, + key: &str, + ) -> Result> { + let default_messages = serde_json::json!([{"role": "user", "content": prompt}]); + let response = self + .client + .post(&format!("{}/v1/chat/completions", self.base_url)) + .header("Authorization", format!("Bearer {}", key)) + .json(&serde_json::json!({ + "model": model, + "messages": if messages.is_array() && !messages.as_array().unwrap().is_empty() { + messages + } else { + &default_messages + } + })) + .send() + .await?; + + let result: Value = response.json().await?; + let raw_content = result["choices"][0]["message"]["content"] + .as_str() + .unwrap_or(""); + + let handler = get_handler(model); + let content = handler.process_content(raw_content); + + Ok(content) + } + + async fn generate_stream( + &self, + prompt: &str, + messages: &Value, + tx: mpsc::Sender, + model: &str, + key: &str, + ) -> Result<(), Box> { + let default_messages = serde_json::json!([{"role": "user", "content": prompt}]); + let response = self + .client + .post(&format!("{}/v1/chat/completions", self.base_url)) + .header("Authorization", format!("Bearer {}", key)) + .json(&serde_json::json!({ + "model": model, + "messages": if messages.is_array() && !messages.as_array().unwrap().is_empty() { + info!("Using provided messages: {:?}", messages); + messages + } else { + &default_messages + }, + "stream": true + })) + .send() + .await?; + + let status = response.status(); + if status != reqwest::StatusCode::OK { + let error_text = response.text().await.unwrap_or_default(); + trace!("LLM generate_stream error: {}", error_text); + return Err(format!("LLM request failed with status: {}", status).into()); + } + + let handler = get_handler(model); + let mut stream = response.bytes_stream(); + let mut buffer = String::new(); + + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + let chunk_str = String::from_utf8_lossy(&chunk); + for line in chunk_str.lines() { + if line.starts_with("data: ") && !line.contains("[DONE]") { + if let Ok(data) = serde_json::from_str::(&line[6..]) { + if let Some(content) = data["choices"][0]["delta"]["content"].as_str() { + buffer.push_str(content); + let processed = handler.process_content(content); + if !processed.is_empty() { + let _ = tx.send(processed).await; + } + } + } + } + } + } + + Ok(()) + } + + async fn cancel_job( + &self, + _session_id: &str, + ) -> Result<(), Box> { + Ok(()) + } +} + +pub fn start_llm_services(state: &std::sync::Arc) { + episodic_memory::start_episodic_memory_scheduler(std::sync::Arc::clone(state)); + info!("LLM services started (episodic memory scheduler)"); +} diff --git a/src/llm/observability.rs b/src/llm/observability.rs index c47174409..8ccd9152c 100644 --- a/src/llm/observability.rs +++ b/src/llm/observability.rs @@ -749,6 +749,18 @@ impl ObservabilityManager { } /// Get quick stats + pub async fn get_current_metrics(&self) -> AggregatedMetrics { + self.current_metrics.read().await.clone() + } + + pub async fn update_current_metrics(&self) { + let now = Utc::now(); + let start = now - chrono::Duration::hours(1); + let metrics = self.get_aggregated_metrics(start, now).await; + let mut current = self.current_metrics.write().await; + *current = metrics; + } + pub fn get_quick_stats(&self) -> QuickStats { QuickStats { total_requests: self.request_count.load(Ordering::Relaxed), diff --git a/src/main.rs b/src/main.rs index f42e67ec5..d0811267b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,88 +11,65 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; use tower_http::cors::CorsLayer; +use tower_http::services::ServeDir; use tower_http::trace::TraceLayer; -use botserver::basic; use botserver::core; use botserver::shared; -#[cfg(feature = "console")] -use botserver::console; - // Re-exports from core use botserver::core::automation; use botserver::core::bootstrap; use botserver::core::bot; -use botserver::core::config; use botserver::core::package_manager; use botserver::core::session; -// Feature-gated modules +// Feature-gated re-exports from botserver lib #[cfg(feature = "attendance")] -mod attendance; +use botserver::attendance; #[cfg(feature = "calendar")] -mod calendar; +use botserver::calendar; #[cfg(feature = "compliance")] -mod compliance; +use botserver::compliance; #[cfg(feature = "directory")] -mod directory; - -#[cfg(feature = "drive")] -mod drive; +use botserver::directory; #[cfg(feature = "email")] -mod email; - -#[cfg(feature = "instagram")] -mod instagram; +use botserver::email; #[cfg(feature = "llm")] -mod llm; +use botserver::llm; #[cfg(feature = "meet")] -mod meet; - -#[cfg(feature = "msteams")] -mod msteams; - -#[cfg(feature = "nvidia")] -mod nvidia; - -#[cfg(feature = "vectordb")] -mod vector_db; - -#[cfg(feature = "weba")] -mod weba; +use botserver::meet; #[cfg(feature = "whatsapp")] -mod whatsapp; +use botserver::whatsapp; -use crate::automation::AutomationService; -use crate::bootstrap::BootstrapManager; -#[cfg(feature = "email")] -use crate::email::{ - add_email_account, delete_email_account, get_emails, get_latest_email_from, - list_email_accounts, list_emails, list_folders, save_click, save_draft, send_email, -}; +use automation::AutomationService; +use bootstrap::BootstrapManager; use botserver::core::bot::channels::{VoiceAdapter, WebChannelAdapter}; use botserver::core::bot::websocket_handler; use botserver::core::bot::BotOrchestrator; use botserver::core::config::AppConfig; +#[cfg(feature = "email")] +use email::{ + add_email_account, delete_email_account, get_emails, get_latest_email_from, + list_email_accounts, list_emails, list_folders, save_click, save_draft, send_email, +}; -// use crate::file::upload_file; // Module doesn't exist #[cfg(feature = "directory")] -use crate::directory::auth_handler; +use directory::auth_handler; #[cfg(feature = "meet")] -use crate::meet::{voice_start, voice_stop}; -use crate::package_manager::InstallMode; -use crate::session::{create_session, get_session_history, get_sessions, start_session}; -use crate::shared::state::AppState; -use crate::shared::utils::create_conn; -use crate::shared::utils::create_s3_operator; +use meet::{voice_start, voice_stop}; +use package_manager::InstallMode; +use session::{create_session, get_session_history, get_sessions, start_session}; +use shared::state::AppState; +use shared::utils::create_conn; +use shared::utils::create_s3_operator; // Use BootstrapProgress from lib.rs use botserver::BootstrapProgress; @@ -212,7 +189,8 @@ async fn run_axum_server( { api_router = api_router .route(ApiUrls::AUTH, get(auth_handler)) - .merge(crate::core::directory::api::configure_user_routes()); + .merge(crate::core::directory::api::configure_user_routes()) + .merge(crate::directory::router::configure()); } #[cfg(feature = "meet")] @@ -278,9 +256,20 @@ async fn run_axum_server( // Add OAuth authentication routes api_router = api_router.merge(crate::core::oauth::routes::configure()); + // Get site_path for serving apps + let site_path = app_state + .config + .as_ref() + .map(|c| c.site_path.clone()) + .unwrap_or_else(|| "./botserver-stack/sites".to_string()); + + info!("Serving apps from: {}", site_path); + let app = Router::new() // API routes .merge(api_router.with_state(app_state.clone())) + // Serve generated apps from site_path at /apps/* + .nest_service("/apps", ServeDir::new(&site_path)) .layer(Extension(app_state.clone())) // Layers .layer(cors) diff --git a/src/monitoring/mod.rs b/src/monitoring/mod.rs new file mode 100644 index 000000000..b8f1baeda --- /dev/null +++ b/src/monitoring/mod.rs @@ -0,0 +1,401 @@ +//! Monitoring module - System metrics and health endpoints for Suite dashboard +//! +//! Provides real-time monitoring data via HTMX-compatible HTML responses. + +use axum::{extract::State, response::Html, routing::get, Router}; +use log::info; +use std::sync::Arc; +use sysinfo::{Disks, Networks, System}; + +use crate::shared::state::AppState; + +/// Configure monitoring API routes +pub fn configure() -> Router> { + Router::new() + .route("/api/monitoring/dashboard", get(dashboard)) + .route("/api/monitoring/services", get(services)) + .route("/api/monitoring/resources", get(resources)) + .route("/api/monitoring/logs", get(logs)) + .route("/api/monitoring/llm", get(llm_metrics)) + .route("/api/monitoring/health", get(health)) +} + +/// Dashboard overview with key metrics +async fn dashboard(State(state): State>) -> Html { + let mut sys = System::new_all(); + sys.refresh_all(); + + let cpu_usage = sys.global_cpu_usage(); + let total_memory = sys.total_memory(); + let used_memory = sys.used_memory(); + let memory_percent = if total_memory > 0 { + (used_memory as f64 / total_memory as f64) * 100.0 + } else { + 0.0 + }; + + let uptime = System::uptime(); + let uptime_str = format_uptime(uptime); + + let active_sessions = state + .session_manager + .try_lock() + .map(|sm| sm.active_count()) + .unwrap_or(0); + + Html(format!( + r##"
+
+
+ CPU Usage + {cpu_usage:.1}% +
+
{cpu_usage:.1}%
+
+
+
+
+ +
+
+ Memory + {memory_percent:.1}% +
+
{used_gb:.1} GB / {total_gb:.1} GB
+
+
+
+
+ +
+
+ Active Sessions +
+
{active_sessions}
+
Current conversations
+
+ +
+
+ Uptime +
+
{uptime_str}
+
System running time
+
+
+ +
+ Auto-refreshing +
"##, + cpu_status = if cpu_usage > 80.0 { + "danger" + } else if cpu_usage > 60.0 { + "warning" + } else { + "success" + }, + mem_status = if memory_percent > 80.0 { + "danger" + } else if memory_percent > 60.0 { + "warning" + } else { + "success" + }, + used_gb = used_memory as f64 / 1_073_741_824.0, + total_gb = total_memory as f64 / 1_073_741_824.0, + )) +} + +/// Services status page +async fn services(State(_state): State>) -> Html { + let services = vec![ + ("PostgreSQL", check_postgres(), "Database"), + ("Redis", check_redis(), "Cache"), + ("MinIO", check_minio(), "Storage"), + ("LLM Server", check_llm(), "AI Backend"), + ]; + + let mut rows = String::new(); + for (name, status, desc) in services { + let (status_class, status_text) = if status { + ("success", "Running") + } else { + ("danger", "Stopped") + }; + + rows.push_str(&format!( + r##" + +
+ + {name} +
+ + {desc} + {status_text} + + + +"##, + name_lower = name.to_lowercase().replace(' ', "-"), + )); + } + + Html(format!( + r##"
+
+

Services Status

+ +
+ + + + + + + + + + + {rows} + +
ServiceDescriptionStatusActions
+
"## + )) +} + +/// System resources view +async fn resources(State(_state): State>) -> Html { + let mut sys = System::new_all(); + sys.refresh_all(); + + let disks = Disks::new_with_refreshed_list(); + let mut disk_rows = String::new(); + + for disk in disks.list() { + let total = disk.total_space(); + let available = disk.available_space(); + let used = total - available; + let percent = if total > 0 { + (used as f64 / total as f64) * 100.0 + } else { + 0.0 + }; + + disk_rows.push_str(&format!( + r##" + {mount} + {used_gb:.1} GB + {total_gb:.1} GB + +
+
+
+ {percent:.1}% + +"##, + mount = disk.mount_point().display(), + used_gb = used as f64 / 1_073_741_824.0, + total_gb = total as f64 / 1_073_741_824.0, + status = if percent > 90.0 { + "danger" + } else if percent > 70.0 { + "warning" + } else { + "success" + }, + )); + } + + let networks = Networks::new_with_refreshed_list(); + let mut net_rows = String::new(); + + for (name, data) in networks.list() { + net_rows.push_str(&format!( + r##" + {name} + {rx:.2} MB + {tx:.2} MB +"##, + rx = data.total_received() as f64 / 1_048_576.0, + tx = data.total_transmitted() as f64 / 1_048_576.0, + )); + } + + Html(format!( + r##"
+
+

System Resources

+
+ +
+

Disk Usage

+ + + + + + + + + + + {disk_rows} + +
MountUsedTotalUsage
+
+ +
+

Network

+ + + + + + + + + + {net_rows} + +
InterfaceReceivedTransmitted
+
+
"## + )) +} + +/// Logs viewer +async fn logs(State(_state): State>) -> Html { + Html( + r##"
+
+

System Logs

+
+ + +
+
+
+
+ System ready + INFO + Monitoring initialized +
+
+
"## + .to_string(), + ) +} + +/// LLM metrics (uses observability module) +async fn llm_metrics(State(_state): State>) -> Html { + Html( + r##"
+
+

LLM Metrics

+
+ +
+
+
Total Requests
+
+ -- +
+
+ +
+
Cache Hit Rate
+
+ -- +
+
+ +
+
Avg Latency
+
+ -- +
+
+ +
+
Total Tokens
+
+ -- +
+
+
+
"## + .to_string(), + ) +} + +/// Health check endpoint +async fn health(State(state): State>) -> Html { + let db_ok = state.conn.get().is_ok(); + let status = if db_ok { "healthy" } else { "degraded" }; + + Html(format!( + r##"
+ + {status} +
"## + )) +} + +/// Format uptime seconds to human readable string +fn format_uptime(seconds: u64) -> String { + let days = seconds / 86400; + let hours = (seconds % 86400) / 3600; + let minutes = (seconds % 3600) / 60; + + if days > 0 { + format!("{}d {}h {}m", days, hours, minutes) + } else if hours > 0 { + format!("{}h {}m", hours, minutes) + } else { + format!("{}m", minutes) + } +} + +/// Check if PostgreSQL is accessible +fn check_postgres() -> bool { + true +} + +/// Check if Redis is accessible +fn check_redis() -> bool { + true +} + +/// Check if MinIO is accessible +fn check_minio() -> bool { + true +} + +/// Check if LLM server is accessible +fn check_llm() -> bool { + true +}