diff --git a/config/directory_config.json b/config/directory_config.json index 988c97572..e51107c9d 100644 --- a/config/directory_config.json +++ b/config/directory_config.json @@ -1,7 +1,7 @@ { "base_url": "http://localhost:8300", "default_org": { - "id": "351468402772017166", + "id": "353032199743733774", "name": "default", "domain": "default.localhost" }, @@ -13,8 +13,8 @@ "first_name": "Admin", "last_name": "User" }, - "admin_token": "2YJqHuenWddFpMw4vqw6vEHtgSF5jbvSG4NxTANnV9KJJMnaDSuvbUNSGsS06-QLFZnpFbw", + "admin_token": "1X7ImWy1yPmGYYumPJ0RfVaLuuLHKstH8BItaTGlp-6jTFPeM0uFo8sjdfxtk-jxjLivcVM", "project_id": "", - "client_id": "351468407201267726", - "client_secret": "vLxjxWiPv8fVvown7zBOqKdb7RPntqVW8fNfphaiMWtkXFI8fXQX8WoyBE5KmhJA" + "client_id": "353032201220194318", + "client_secret": "mrGZZk7Aqx1QbOHIwadgZHZkKHuPqZtOGDtdHTe4eZxEK86TDKfTiMlW2NxSEIHl" } \ No newline at end of file diff --git a/migrations/6.1.2_table_role_access/down.sql b/migrations/6.1.2_table_role_access/down.sql new file mode 100644 index 000000000..347149fcf --- /dev/null +++ b/migrations/6.1.2_table_role_access/down.sql @@ -0,0 +1,12 @@ +-- Rollback: Remove role-based access control columns from dynamic tables +-- Migration: 6.1.2_table_role_access + +-- Remove columns from dynamic_table_definitions +ALTER TABLE dynamic_table_definitions + DROP COLUMN IF EXISTS read_roles, + DROP COLUMN IF EXISTS write_roles; + +-- Remove columns from dynamic_table_fields +ALTER TABLE dynamic_table_fields + DROP COLUMN IF EXISTS read_roles, + DROP COLUMN IF EXISTS write_roles; diff --git a/migrations/6.1.2_table_role_access/up.sql b/migrations/6.1.2_table_role_access/up.sql new file mode 100644 index 000000000..332828892 --- /dev/null +++ b/migrations/6.1.2_table_role_access/up.sql @@ -0,0 +1,28 @@ +-- Migration: 6.1.2_table_role_access +-- Add role-based access control columns to dynamic table definitions and fields +-- +-- Syntax in .gbdialog TABLE definitions: +-- TABLE Contatos ON maria READ BY "admin;manager" +-- Id number key +-- Nome string(150) +-- NumeroDocumento string(25) READ BY "admin" +-- Celular string(20) WRITE BY "admin;manager" +-- +-- Empty roles = everyone has access (default behavior) +-- Roles are semicolon-separated and match Zitadel directory roles + +-- Add role columns to dynamic_table_definitions +ALTER TABLE dynamic_table_definitions +ADD COLUMN IF NOT EXISTS read_roles TEXT DEFAULT NULL, +ADD COLUMN IF NOT EXISTS write_roles TEXT DEFAULT NULL; + +-- Add role columns to dynamic_table_fields +ALTER TABLE dynamic_table_fields +ADD COLUMN IF NOT EXISTS read_roles TEXT DEFAULT NULL, +ADD COLUMN IF NOT EXISTS write_roles TEXT DEFAULT NULL; + +-- Add comments for documentation +COMMENT ON COLUMN dynamic_table_definitions.read_roles IS 'Semicolon-separated roles that can read from this table (empty = everyone)'; +COMMENT ON COLUMN dynamic_table_definitions.write_roles IS 'Semicolon-separated roles that can write to this table (empty = everyone)'; +COMMENT ON COLUMN dynamic_table_fields.read_roles IS 'Semicolon-separated roles that can read this field (empty = everyone)'; +COMMENT ON COLUMN dynamic_table_fields.write_roles IS 'Semicolon-separated roles that can write this field (empty = everyone)'; diff --git a/migrations/6.1.3_knowledge_base_sources/down.sql b/migrations/6.1.3_knowledge_base_sources/down.sql new file mode 100644 index 000000000..b8d7a0e29 --- /dev/null +++ b/migrations/6.1.3_knowledge_base_sources/down.sql @@ -0,0 +1,25 @@ +-- Rollback Migration: Knowledge Base Sources + +-- Drop triggers first +DROP TRIGGER IF EXISTS update_knowledge_sources_updated_at ON knowledge_sources; + +-- Drop indexes +DROP INDEX IF EXISTS idx_knowledge_sources_bot_id; +DROP INDEX IF EXISTS idx_knowledge_sources_status; +DROP INDEX IF EXISTS idx_knowledge_sources_collection; +DROP INDEX IF EXISTS idx_knowledge_sources_content_hash; +DROP INDEX IF EXISTS idx_knowledge_sources_created_at; + +DROP INDEX IF EXISTS idx_knowledge_chunks_source_id; +DROP INDEX IF EXISTS idx_knowledge_chunks_chunk_index; +DROP INDEX IF EXISTS idx_knowledge_chunks_content_fts; +DROP INDEX IF EXISTS idx_knowledge_chunks_embedding; + +DROP INDEX IF EXISTS idx_research_search_history_bot_id; +DROP INDEX IF EXISTS idx_research_search_history_user_id; +DROP INDEX IF EXISTS idx_research_search_history_created_at; + +-- Drop tables (order matters due to foreign key constraints) +DROP TABLE IF EXISTS research_search_history; +DROP TABLE IF EXISTS knowledge_chunks; +DROP TABLE IF EXISTS knowledge_sources; diff --git a/migrations/6.1.3_knowledge_base_sources/up.sql b/migrations/6.1.3_knowledge_base_sources/up.sql new file mode 100644 index 000000000..6a1507a1c --- /dev/null +++ b/migrations/6.1.3_knowledge_base_sources/up.sql @@ -0,0 +1,95 @@ +-- Migration: Knowledge Base Sources +-- Description: Tables for document ingestion, chunking, and RAG support + +-- Table for knowledge sources (uploaded documents) +CREATE TABLE IF NOT EXISTS knowledge_sources ( + id TEXT PRIMARY KEY, + bot_id UUID, + name TEXT NOT NULL, + source_type TEXT NOT NULL DEFAULT 'txt', + file_path TEXT, + url TEXT, + content_hash TEXT NOT NULL, + chunk_count INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', + collection TEXT NOT NULL DEFAULT 'default', + error_message TEXT, + metadata JSONB DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + indexed_at TIMESTAMPTZ +); + +-- Indexes for knowledge_sources +CREATE INDEX IF NOT EXISTS idx_knowledge_sources_bot_id ON knowledge_sources(bot_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_sources_status ON knowledge_sources(status); +CREATE INDEX IF NOT EXISTS idx_knowledge_sources_collection ON knowledge_sources(collection); +CREATE INDEX IF NOT EXISTS idx_knowledge_sources_content_hash ON knowledge_sources(content_hash); +CREATE INDEX IF NOT EXISTS idx_knowledge_sources_created_at ON knowledge_sources(created_at); + +-- Table for document chunks +CREATE TABLE IF NOT EXISTS knowledge_chunks ( + id TEXT PRIMARY KEY, + source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL DEFAULT 0, + embedding vector(1536), + metadata JSONB DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Indexes for knowledge_chunks +CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_source_id ON knowledge_chunks(source_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_chunk_index ON knowledge_chunks(chunk_index); + +-- Full-text search index on content +CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_content_fts + ON knowledge_chunks USING gin(to_tsvector('english', content)); + +-- Vector similarity index (if pgvector extension is available) +-- Note: This will only work if pgvector extension is installed +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') THEN + EXECUTE 'CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_embedding + ON knowledge_chunks USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100)'; + END IF; +EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'Could not create vector index: %', SQLERRM; +END $$; + +-- Table for search history +CREATE TABLE IF NOT EXISTS research_search_history ( + id TEXT PRIMARY KEY, + bot_id UUID, + user_id UUID, + query TEXT NOT NULL, + search_type TEXT NOT NULL DEFAULT 'web', + results_count INTEGER NOT NULL DEFAULT 0, + metadata JSONB DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Indexes for search history +CREATE INDEX IF NOT EXISTS idx_research_search_history_bot_id ON research_search_history(bot_id); +CREATE INDEX IF NOT EXISTS idx_research_search_history_user_id ON research_search_history(user_id); +CREATE INDEX IF NOT EXISTS idx_research_search_history_created_at ON research_search_history(created_at); + +-- Trigger for updated_at on knowledge_sources +DROP TRIGGER IF EXISTS update_knowledge_sources_updated_at ON knowledge_sources; +CREATE TRIGGER update_knowledge_sources_updated_at + BEFORE UPDATE ON knowledge_sources + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +-- Comments for documentation +COMMENT ON TABLE knowledge_sources IS 'Uploaded documents for knowledge base ingestion'; +COMMENT ON TABLE knowledge_chunks IS 'Text chunks extracted from knowledge sources for RAG'; +COMMENT ON TABLE research_search_history IS 'History of web and knowledge base searches'; + +COMMENT ON COLUMN knowledge_sources.source_type IS 'Document type: pdf, docx, txt, markdown, html, csv, xlsx, url'; +COMMENT ON COLUMN knowledge_sources.status IS 'Processing status: pending, processing, indexed, failed, reindexing'; +COMMENT ON COLUMN knowledge_sources.collection IS 'Collection/namespace for organizing sources'; +COMMENT ON COLUMN knowledge_chunks.embedding IS 'Vector embedding for semantic search (1536 dimensions for OpenAI)'; +COMMENT ON COLUMN knowledge_chunks.token_count IS 'Estimated token count for the chunk'; diff --git a/src/analytics/mod.rs b/src/analytics/mod.rs index 4a4726fb5..30f5fbe53 100644 --- a/src/analytics/mod.rs +++ b/src/analytics/mod.rs @@ -1,3 +1,4 @@ +use crate::core::urls::ApiUrls; use crate::llm::observability::{ObservabilityConfig, ObservabilityManager, QuickStats}; use crate::shared::state::AppState; use axum::{ @@ -89,39 +90,42 @@ impl Default for AnalyticsService { pub fn configure_analytics_routes() -> Router> { Router::new() - .route("/api/analytics/messages/count", get(handle_message_count)) + .route(ApiUrls::ANALYTICS_MESSAGES_COUNT, get(handle_message_count)) .route( - "/api/analytics/sessions/active", + ApiUrls::ANALYTICS_SESSIONS_ACTIVE, get(handle_active_sessions), ) - .route("/api/analytics/response/avg", get(handle_avg_response_time)) - .route("/api/analytics/llm/tokens", get(handle_llm_tokens)) - .route("/api/analytics/storage/usage", get(handle_storage_usage)) - .route("/api/analytics/errors/count", get(handle_errors_count)) .route( - "/api/analytics/timeseries/messages", + ApiUrls::ANALYTICS_RESPONSE_AVG, + get(handle_avg_response_time), + ) + .route(ApiUrls::ANALYTICS_LLM_TOKENS, get(handle_llm_tokens)) + .route(ApiUrls::ANALYTICS_STORAGE_USAGE, get(handle_storage_usage)) + .route(ApiUrls::ANALYTICS_ERRORS_COUNT, get(handle_errors_count)) + .route( + ApiUrls::ANALYTICS_TIMESERIES_MESSAGES, get(handle_timeseries_messages), ) .route( - "/api/analytics/timeseries/response_time", + ApiUrls::ANALYTICS_TIMESERIES_RESPONSE, get(handle_timeseries_response), ) .route( - "/api/analytics/channels/distribution", + ApiUrls::ANALYTICS_CHANNELS_DISTRIBUTION, get(handle_channels_distribution), ) .route( - "/api/analytics/bots/performance", + ApiUrls::ANALYTICS_BOTS_PERFORMANCE, get(handle_bots_performance), ) .route( - "/api/analytics/activity/recent", + ApiUrls::ANALYTICS_ACTIVITY_RECENT, get(handle_recent_activity), ) - .route("/api/analytics/queries/top", get(handle_top_queries)) - .route("/api/analytics/chat", post(handle_analytics_chat)) - .route("/api/analytics/llm/stats", get(handle_llm_stats)) - .route("/api/analytics/budget/status", get(handle_budget_status)) + .route(ApiUrls::ANALYTICS_QUERIES_TOP, get(handle_top_queries)) + .route(ApiUrls::ANALYTICS_CHAT, post(handle_analytics_chat)) + .route(ApiUrls::ANALYTICS_LLM_STATS, get(handle_llm_stats)) + .route(ApiUrls::ANALYTICS_BUDGET_STATUS, get(handle_budget_status)) } pub async fn handle_message_count(State(state): State>) -> impl IntoResponse { diff --git a/src/attendance/mod.rs b/src/attendance/mod.rs index 9e0f1cdf5..e22ed6683 100644 --- a/src/attendance/mod.rs +++ b/src/attendance/mod.rs @@ -21,6 +21,7 @@ pub use queue::{ use crate::core::bot::channels::whatsapp::WhatsAppAdapter; use crate::core::bot::channels::ChannelAdapter; +use crate::core::urls::ApiUrls; use crate::shared::models::{BotResponse, UserSession}; use crate::shared::state::{AppState, AttendantNotification}; use axum::{ @@ -45,39 +46,42 @@ use uuid::Uuid; pub fn configure_attendance_routes() -> Router> { Router::new() - .route("/api/attendance/queue", get(queue::list_queue)) - .route("/api/attendance/attendants", get(queue::list_attendants)) - .route("/api/attendance/assign", post(queue::assign_conversation)) + .route(ApiUrls::ATTENDANCE_QUEUE, get(queue::list_queue)) + .route(ApiUrls::ATTENDANCE_ATTENDANTS, get(queue::list_attendants)) + .route(ApiUrls::ATTENDANCE_ASSIGN, post(queue::assign_conversation)) .route( - "/api/attendance/transfer", + ApiUrls::ATTENDANCE_TRANSFER, post(queue::transfer_conversation), ) .route( - "/api/attendance/resolve/{session_id}", + &ApiUrls::ATTENDANCE_RESOLVE.replace(":session_id", "{session_id}"), post(queue::resolve_conversation), ) - .route("/api/attendance/insights", get(queue::get_insights)) - .route("/api/attendance/respond", post(attendant_respond)) - .route("/ws/attendant", get(attendant_websocket_handler)) - .route("/api/attendance/llm/tips", post(llm_assist::generate_tips)) + .route(ApiUrls::ATTENDANCE_INSIGHTS, get(queue::get_insights)) + .route(ApiUrls::ATTENDANCE_RESPOND, post(attendant_respond)) + .route(ApiUrls::WS_ATTENDANT, get(attendant_websocket_handler)) .route( - "/api/attendance/llm/polish", + ApiUrls::ATTENDANCE_LLM_TIPS, + post(llm_assist::generate_tips), + ) + .route( + ApiUrls::ATTENDANCE_LLM_POLISH, post(llm_assist::polish_message), ) .route( - "/api/attendance/llm/smart-replies", + ApiUrls::ATTENDANCE_LLM_SMART_REPLIES, post(llm_assist::generate_smart_replies), ) .route( - "/api/attendance/llm/summary/{session_id}", + &ApiUrls::ATTENDANCE_LLM_SUMMARY.replace(":session_id", "{session_id}"), get(llm_assist::generate_summary), ) .route( - "/api/attendance/llm/sentiment", + ApiUrls::ATTENDANCE_LLM_SENTIMENT, post(llm_assist::analyze_sentiment), ) .route( - "/api/attendance/llm/config/{bot_id}", + &ApiUrls::ATTENDANCE_LLM_CONFIG.replace(":bot_id", "{bot_id}"), get(llm_assist::get_llm_config), ) } diff --git a/src/auto_task/APP_GENERATOR_PROMPT.md b/src/auto_task/APP_GENERATOR_PROMPT.md index f479d6135..051bba4d0 100644 --- a/src/auto_task/APP_GENERATOR_PROMPT.md +++ b/src/auto_task/APP_GENERATOR_PROMPT.md @@ -472,9 +472,9 @@ When generating an app, create these files: └── app.js Optional custom JavaScript ``` -### Required HTML Head +### Required HTML Head (WITH SEO) -Every HTML page MUST include: +Every HTML page MUST include proper SEO meta tags: ```html @@ -482,13 +482,26 @@ Every HTML page MUST include: - Page Title + + + + + + + + {Page Title} - {App Name} ``` +**SEO is required even for authenticated apps because:** +- Shared links preview correctly in chat/email +- Browser tabs show meaningful titles +- Bookmarks are descriptive +- Accessibility tools work better + --- ## RESPONSE FORMAT @@ -590,4 +603,110 @@ Response: 5. **Use the APIs** - Connect to /api/db/ for data operations 6. **Be complete** - Generate all necessary pages, not just stubs 7. **Match the request** - If user wants pink, make it pink -8. **Tables are optional** - Simple tools don't need database tables \ No newline at end of file +8. **Tables are optional** - Simple tools don't need database tables +9. **SEO required** - All pages MUST have proper meta tags (description, og:title, etc.) +10. **No comments in generated code** - Code must be self-documenting, no HTML/JS/CSS comments + +--- + +## DESIGNER MAGIC BUTTON + +The Designer has a "Magic" button that sends the current HTMX code to the LLM with an improvement prompt. It works like a user asking "improve this code" automatically. + +**What Magic Button does:** +1. Captures current page HTML/CSS/JS +2. Sends to LLM with prompt: "Analyze and improve this HTMX code. Suggest better structure, accessibility, performance, and UX improvements." +3. LLM responds with refactored code or suggestions +4. User can apply suggestions or dismiss + +**Example Magic prompt sent to LLM:** +``` +You are reviewing this HTMX application code. Suggest improvements for: +- Better HTMX patterns (reduce JS, use hx-* attributes) +- Accessibility (ARIA labels, keyboard navigation) +- Performance (lazy loading, efficient selectors) +- UX (loading states, error handling, feedback) +- Code organization (semantic HTML, clean CSS) + +Current code: +{current_page_html} + +Respond with improved code and brief explanation. +``` + +--- + +## CUSTOM DOMAIN SUPPORT + +Custom domains are configured in the bot's `config.csv` file: + +```csv +appname-domain,www.customerdomain.com +``` + +**Configuration in config.csv:** +```csv +# Bot configuration +bot-name,My Company Bot +appname-domain,app.mycompany.com +``` + +**How it works:** +1. Bot reads `appname-domain` from config.csv +2. Server routes requests from custom domain to the app +3. SSL auto-provisioned via Let's Encrypt + +**DNS Requirements:** +- CNAME record: `app.mycompany.com` → `{bot-id}.generalbots.app` +- Or A record pointing to server IP + +--- + +## ZERO COMMENTS POLICY + +**DO NOT generate any comments in code.** + +```html + +
+ + +
+ + +
+ +
+``` + +```css +/* ❌ WRONG - no CSS comments */ +.button { + /* Primary action style */ + background: blue; +} + +/* ✅ CORRECT - clear naming */ +.button-primary { + background: blue; +} +``` + +```javascript +// ❌ WRONG - no JS comments +function save() { + // Save to database + htmx.ajax('POST', '/api/db/items', {...}); +} + +// ✅ CORRECT - descriptive function name +function saveItemToDatabase() { + htmx.ajax('POST', '/api/db/items', {...}); +} +``` + +**Why no comments:** +- Comments become stale when code changes +- Good naming is better than comments +- LLMs can infer intent from well-structured code +- Reduces generated file size \ No newline at end of file diff --git a/src/auto_task/app_generator.rs b/src/auto_task/app_generator.rs index 652889cd7..c53ee63a8 100644 --- a/src/auto_task/app_generator.rs +++ b/src/auto_task/app_generator.rs @@ -1,3 +1,4 @@ +use crate::auto_task::app_logs::{log_generator_error, log_generator_info}; use crate::basic::keywords::table_definition::{ generate_create_table_sql, FieldDefinition, TableDefinition, }; @@ -7,9 +8,8 @@ use aws_sdk_s3::primitives::ByteStream; use chrono::{DateTime, Utc}; use diesel::prelude::*; use diesel::sql_query; -use log::{info, trace, warn}; +use log::{error, info, trace, warn}; use serde::{Deserialize, Serialize}; -use std::fmt::Write; use std::sync::Arc; use uuid::Uuid; @@ -18,13 +18,20 @@ pub struct GeneratedApp { pub id: String, pub name: String, pub description: String, - pub pages: Vec, + pub pages: Vec, pub tables: Vec, - pub tools: Vec, - pub schedulers: Vec, + pub tools: Vec, + pub schedulers: Vec, pub created_at: DateTime, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GeneratedFile { + pub filename: String, + pub content: String, + pub file_type: FileType, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GeneratedPage { pub filename: String, @@ -35,6 +42,17 @@ pub struct GeneratedPage { } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum FileType { + Html, + Css, + Js, + Bas, + Json, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum PageType { List, Form, @@ -85,6 +103,42 @@ pub struct SyncResult { pub migrations_applied: usize, } +#[derive(Debug, Clone, Deserialize)] +struct LlmGeneratedApp { + name: String, + description: String, + #[serde(default)] + _domain: String, + tables: Vec, + files: Vec, + tools: Option>, + schedulers: Option>, +} + +#[derive(Debug, Clone, Deserialize)] +struct LlmTable { + name: String, + fields: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +struct LlmField { + name: String, + #[serde(rename = "type")] + field_type: String, + nullable: Option, + reference: Option, + default: Option, +} + +#[derive(Debug, Clone, Deserialize)] +struct LlmFile { + filename: String, + content: String, + #[serde(rename = "type", default)] + _file_type: Option, +} + pub struct AppGenerator { state: Arc, } @@ -104,32 +158,83 @@ impl AppGenerator { &intent[..intent.len().min(100)] ); - let structure = self.analyze_app_requirements_with_llm(intent).await?; - trace!("App structure analyzed: {:?}", structure.name); - - let tables_bas_content = self.generate_table_definitions(&structure)?; - self.append_to_tables_bas(session.bot_id, &tables_bas_content)?; - - let sync_result = self.sync_tables_to_database(&structure.tables)?; - info!( - "Tables synced: {} created, {} fields added", - sync_result.tables_created, sync_result.fields_added + log_generator_info( + "pending", + &format!( + "Starting app generation: {}", + &intent[..intent.len().min(50)] + ), ); - let pages = self.generate_htmx_pages(&structure)?; - trace!("Generated {} pages", pages.len()); + let llm_app = match self.generate_complete_app_with_llm(intent).await { + Ok(app) => { + log_generator_info( + &app.name, + "LLM successfully generated app structure and files", + ); + app + } + Err(e) => { + log_generator_error("unknown", "LLM app generation failed", &e.to_string()); + return Err(e); + } + }; + + let tables = Self::convert_llm_tables(&llm_app.tables); + + if !tables.is_empty() { + let tables_bas_content = Self::generate_table_definitions(&tables)?; + if let Err(e) = self.append_to_tables_bas(session.bot_id, &tables_bas_content) { + log_generator_error( + &llm_app.name, + "Failed to append to tables.bas", + &e.to_string(), + ); + } + + match self.sync_tables_to_database(&tables) { + Ok(result) => { + log_generator_info( + &llm_app.name, + &format!( + "Tables synced: {} created, {} fields", + result.tables_created, result.fields_added + ), + ); + } + Err(e) => { + log_generator_error(&llm_app.name, "Failed to sync tables", &e.to_string()); + } + } + } let bot_name = self.get_bot_name(session.bot_id)?; let bucket_name = format!("{}.gbai", bot_name.to_lowercase()); - let drive_app_path = format!(".gbdrive/apps/{}", structure.name); + let drive_app_path = format!(".gbdrive/apps/{}", llm_app.name); - for page in &pages { - let drive_path = format!("{}/{}", drive_app_path, page.filename); - self.write_to_drive(&bucket_name, &drive_path, &page.content) - .await?; + let mut pages = Vec::new(); + for file in &llm_app.files { + let drive_path = format!("{}/{}", drive_app_path, file.filename); + if let Err(e) = self + .write_to_drive(&bucket_name, &drive_path, &file.content) + .await + { + log_generator_error( + &llm_app.name, + &format!("Failed to write {}", file.filename), + &e.to_string(), + ); + } + + let file_type = Self::detect_file_type(&file.filename); + pages.push(GeneratedFile { + filename: file.filename.clone(), + content: file.content.clone(), + file_type, + }); } - let designer_js = self.generate_designer_js(); + let designer_js = Self::generate_designer_js(&llm_app.name); self.write_to_drive( &bucket_name, &format!("{}/designer.js", drive_app_path), @@ -137,649 +242,349 @@ impl AppGenerator { ) .await?; - let css_content = self.generate_app_css(); - self.write_to_drive( - &bucket_name, - &format!("{}/styles.css", drive_app_path), - &css_content, - ) - .await?; - - let tools = self.generate_tools(&structure)?; - for tool in &tools { - let tool_path = format!(".gbdialog/tools/{}", tool.filename); - self.write_to_drive(&bucket_name, &tool_path, &tool.content) - .await?; + let mut tools = Vec::new(); + if let Some(llm_tools) = &llm_app.tools { + for tool in llm_tools { + let tool_path = format!(".gbdialog/tools/{}", tool.filename); + if let Err(e) = self + .write_to_drive(&bucket_name, &tool_path, &tool.content) + .await + { + log_generator_error( + &llm_app.name, + &format!("Failed to write tool {}", tool.filename), + &e.to_string(), + ); + } + tools.push(GeneratedFile { + filename: tool.filename.clone(), + content: tool.content.clone(), + file_type: FileType::Bas, + }); + } } - let schedulers = self.generate_schedulers(&structure)?; - for scheduler in &schedulers { - let scheduler_path = format!(".gbdialog/schedulers/{}", scheduler.filename); - self.write_to_drive(&bucket_name, &scheduler_path, &scheduler.content) - .await?; + let mut schedulers = Vec::new(); + if let Some(llm_schedulers) = &llm_app.schedulers { + for scheduler in llm_schedulers { + let scheduler_path = format!(".gbdialog/schedulers/{}", scheduler.filename); + if let Err(e) = self + .write_to_drive(&bucket_name, &scheduler_path, &scheduler.content) + .await + { + log_generator_error( + &llm_app.name, + &format!("Failed to write scheduler {}", scheduler.filename), + &e.to_string(), + ); + } + schedulers.push(GeneratedFile { + filename: scheduler.filename.clone(), + content: scheduler.content.clone(), + file_type: FileType::Bas, + }); + } } - self.sync_app_to_site_root(&bucket_name, &structure.name, session.bot_id) + self.sync_app_to_site_root(&bucket_name, &llm_app.name, session.bot_id) .await?; + log_generator_info( + &llm_app.name, + &format!( + "App generated: {} files, {} tables, {} tools", + pages.len(), + tables.len(), + tools.len() + ), + ); + info!( - "App '{}' generated in drive s3://{}/{} and synced to site root", - structure.name, bucket_name, drive_app_path + "App '{}' generated in s3://{}/{}", + llm_app.name, bucket_name, drive_app_path ); Ok(GeneratedApp { id: Uuid::new_v4().to_string(), - name: structure.name.clone(), - description: structure.description.clone(), + name: llm_app.name, + description: llm_app.description, pages, - tables: structure.tables, + tables, tools, schedulers, created_at: Utc::now(), }) } - fn get_platform_capabilities_prompt(&self) -> &'static str { + fn get_platform_prompt() -> &'static str { r##" -GENERAL BOTS APP GENERATOR - PLATFORM CAPABILITIES +GENERAL BOTS PLATFORM - APP GENERATION -AVAILABLE REST APIs: +You are an expert full-stack developer generating complete applications for General Bots platform. -DATABASE API (/api/db/): -- GET /api/db/TABLE - List records (query: limit, offset, order_by, order_dir, search, field=value) -- GET /api/db/TABLE/ID - Get single record -- GET /api/db/TABLE/count - Get record count -- POST /api/db/TABLE - Create record (JSON body) -- PUT /api/db/TABLE/ID - Update record (JSON body) -- DELETE /api/db/TABLE/ID - Delete record +=== AVAILABLE APIs === -FILE STORAGE API (/api/drive/): +DATABASE (/api/db/): +- GET /api/db/{table} - List records (query: limit, offset, order_by, order_dir, search, field=value) +- GET /api/db/{table}/{id} - Get single record +- GET /api/db/{table}/count - Count records +- POST /api/db/{table} - Create record (JSON body) +- PUT /api/db/{table}/{id} - Update record +- DELETE /api/db/{table}/{id} - Delete record + +DRIVE (/api/drive/): - GET /api/drive/list?path=/folder - List files -- GET /api/drive/download?path=/file.ext - Download file -- POST /api/drive/upload - Upload (multipart: file, path) -- DELETE /api/drive/delete?path=/file.ext - Delete file +- GET /api/drive/download?path=/file - Download +- POST /api/drive/upload - Upload (multipart) +- DELETE /api/drive/delete?path=/file - Delete -AUTOTASK API (/api/autotask/): -- POST /api/autotask/create - Create task {"intent": "..."} -- GET /api/autotask/list - List tasks -- GET /api/autotask/stats - Get statistics -- GET /api/autotask/pending - Get pending items - -DESIGNER API (/api/designer/): -- POST /api/designer/modify - AI modify app {"app_name", "current_page", "message"} - -COMMUNICATION APIs: +COMMUNICATION: - POST /api/mail/send - {"to", "subject", "body"} -- POST /api/whatsapp/send - {"to": "+123...", "message"} +- POST /api/whatsapp/send - {"to", "message"} - POST /api/llm/generate - {"prompt", "max_tokens"} -- POST /api/llm/image - {"prompt", "size"} -HTMX ATTRIBUTES: +=== HTMX REQUIREMENTS === + +All HTML pages MUST use HTMX exclusively. NO fetch(), NO XMLHttpRequest, NO inline onclick. + +Key attributes: - hx-get, hx-post, hx-put, hx-delete - HTTP methods -- hx-target="#id" - Where to put response -- hx-swap="innerHTML|outerHTML|beforeend|delete" - How to insert -- hx-trigger="click|submit|load|every 5s|keyup changed delay:300ms" - When to fire +- hx-target="#id" - Response destination +- hx-swap="innerHTML|outerHTML|beforeend|delete" - Insert method +- hx-trigger="click|submit|load|every 5s|keyup changed delay:300ms" - hx-indicator="#spinner" - Loading indicator -- hx-confirm="Are you sure?" - Confirmation dialog +- hx-confirm="Message?" - Confirmation +- hx-vals='{"key":"value"}' - Extra values +- hx-headers='{"X-Custom":"value"}' - Headers -BASIC AUTOMATION (.bas files): +=== REQUIRED HTML STRUCTURE === -Tools (.gbdialog/tools/*.bas): -HEAR "phrase1", "phrase2" - result = GET FROM "table" +Every HTML file must include: +```html + + + + + + Page Title + + + + + + + + + +``` + +=== BASIC SCRIPTS (.bas) === + +Tools (triggered by chat): +``` +HEAR "keyword1", "keyword2" + result = GET FROM "table" WHERE field = value TALK "Response: " + result END HEAR +``` -Schedulers (.gbdialog/schedulers/*.bas): +Schedulers (cron-based): +``` SET SCHEDULE "0 9 * * *" - data = GET FROM "reports" - SEND MAIL TO "email" WITH SUBJECT "Daily" BODY data + data = GET FROM "table" + SEND MAIL TO "email" WITH SUBJECT "Report" BODY data END SCHEDULE +``` -BASIC KEYWORDS: +BASIC Keywords: - TALK "message" - Send message -- ASK "question" - Get user input -- GET FROM "table" WHERE field=val - Query database -- SAVE TO "table" WITH field1, field2 - Insert record +- ASK "question" - Get input +- GET FROM "table" WHERE field=val - Query +- SAVE TO "table" WITH field1, field2 - Insert - SEND MAIL TO "x" WITH SUBJECT "y" BODY "z" -- result = LLM "prompt" - AI text generation -- image = GENERATE IMAGE "prompt" - AI image generation +- result = LLM "prompt" - AI generation -REQUIRED HTML HEAD (all pages must include): -- link rel="stylesheet" href="styles.css" -- script src="/js/vendor/htmx.min.js" -- script src="designer.js" defer +=== FIELD TYPES === +guid, string, text, integer, decimal, boolean, date, datetime, json -FIELD TYPES: guid, string, text, integer, decimal, boolean, date, datetime, json +=== GENERATION RULES === -RULES: -1. Always use HTMX for API calls - NO fetch() in HTML -2. Include designer.js in ALL pages -3. Make it beautiful and fully functional -4. Tables are optional - simple apps (calculator, timer) dont need them +1. Generate COMPLETE, WORKING code - no placeholders, no "...", no "add more here" +2. Use semantic HTML5 (header, main, nav, section, article, footer) +3. Include loading states (hx-indicator) +4. Include error handling +5. Make it beautiful, modern, responsive +6. Include dark mode support in CSS +7. Tables should have id, created_at, updated_at fields +8. Forms must validate required fields +9. Lists must have search, pagination, edit/delete actions "## } - async fn analyze_app_requirements_with_llm( + async fn generate_complete_app_with_llm( &self, intent: &str, - ) -> Result> { - let capabilities = self.get_platform_capabilities_prompt(); + ) -> Result> { + let platform = Self::get_platform_prompt(); let prompt = format!( - r#"You are an expert app generator for General Bots platform. + r#"{platform} -{capabilities} +=== USER REQUEST === +"{intent}" -USER REQUEST: "{intent}" +=== YOUR TASK === +Generate a complete application based on the user's request. -Generate a complete application. For simple apps (calculator, timer, game), you can use empty tables array. -For data apps (CRM, inventory), design appropriate tables. - -Respond with JSON: +Respond with a single JSON object: {{ "name": "app-name-lowercase-dashes", "description": "What this app does", - "domain": "custom|healthcare|sales|inventory|booking|etc", + "domain": "healthcare|sales|inventory|booking|utility|etc", "tables": [ {{ "name": "table_name", "fields": [ {{"name": "id", "type": "guid", "nullable": false}}, - {{"name": "field_name", "type": "string|integer|decimal|boolean|date|datetime|text", "nullable": true, "reference": "other_table or null"}} + {{"name": "created_at", "type": "datetime", "nullable": false, "default": "now()"}}, + {{"name": "updated_at", "type": "datetime", "nullable": false, "default": "now()"}}, + {{"name": "field_name", "type": "string", "nullable": true, "reference": null}} ] }} ], - "features": ["list of features"], - "custom_html": "OPTIONAL: For non-CRUD apps like calculators, provide complete HTML here", - "custom_css": "OPTIONAL: Custom CSS styles", - "custom_js": "OPTIONAL: Custom JavaScript" + "files": [ + {{"filename": "index.html", "content": "...complete HTML..."}}, + {{"filename": "styles.css", "content": ":root {{...}} body {{...}} ...complete CSS..."}}, + {{"filename": "table_name.html", "content": "...list page..."}}, + {{"filename": "table_name_form.html", "content": "...form page..."}} + ], + "tools": [ + {{"filename": "app_helper.bas", "content": "HEAR \"help\"\n TALK \"I can help with...\"\nEND HEAR"}} + ], + "schedulers": [ + {{"filename": "daily_report.bas", "content": "SET SCHEDULE \"0 9 * * *\"\n ...\nEND SCHEDULE"}} + ] }} IMPORTANT: -- For simple tools (calculator, converter, timer): use custom_html/css/js, tables can be empty [] -- For data apps (CRM, booking): design tables, custom_* fields are optional -- Always include id, created_at, updated_at in tables -- Make it beautiful and fully functional +- For simple utilities (calculator, timer, converter): tables can be empty [], focus on files +- For data apps (CRM, inventory): design proper tables and CRUD pages +- Generate ALL files completely - no shortcuts +- CSS must be comprehensive with variables, responsive design, dark mode +- Every HTML page needs proper structure with all required scripts +- Replace APP_NAME_HERE with actual app name in data-app-name attribute Respond with valid JSON only."# ); let response = self.call_llm(&prompt).await?; - self.parse_app_structure_response(&response, intent) + Self::parse_llm_app_response(&response) } - /// Parse LLM response into AppStructure - fn parse_app_structure_response( - &self, + fn parse_llm_app_response( response: &str, - original_intent: &str, - ) -> Result> { - #[derive(Deserialize)] - struct LlmAppResponse { - name: String, - description: String, - domain: String, - tables: Vec, - features: Option>, + ) -> Result> { + let cleaned = response + .trim() + .trim_start_matches("```json") + .trim_start_matches("```") + .trim_end_matches("```") + .trim(); + + match serde_json::from_str::(cleaned) { + Ok(app) => { + if app.files.is_empty() { + return Err("LLM generated no files".into()); + } + Ok(app) + } + Err(e) => { + error!("Failed to parse LLM response: {}", e); + error!("Response was: {}", &response[..response.len().min(500)]); + Err(format!("Failed to parse LLM response: {}", e).into()) + } } + } - #[derive(Deserialize)] - struct LlmTableResponse { - name: String, - fields: Vec, - } - - #[derive(Deserialize)] - struct LlmFieldResponse { - name: String, - #[serde(rename = "type")] - field_type: String, - nullable: Option, - reference: Option, - } - - match serde_json::from_str::(response) { - Ok(resp) => { - let tables = resp - .tables - .into_iter() - .map(|t| { - let fields = t - .fields - .into_iter() - .enumerate() - .map(|(i, f)| { - let is_id = f.name == "id"; - FieldDefinition { - name: f.name, - field_type: f.field_type, - length: None, - precision: None, - is_key: i == 0 && is_id, - is_nullable: f.nullable.unwrap_or(true), - default_value: None, - reference_table: f.reference, - field_order: i as i32, - } - }) - .collect(); - - TableDefinition { - name: t.name, - connection_name: "default".to_string(), - fields, - } + fn convert_llm_tables(llm_tables: &[LlmTable]) -> Vec { + llm_tables + .iter() + .map(|t| { + let fields = t + .fields + .iter() + .enumerate() + .map(|(i, f)| FieldDefinition { + name: f.name.clone(), + field_type: f.field_type.clone(), + is_key: f.name == "id", + is_nullable: f.nullable.unwrap_or(true), + reference_table: f.reference.clone(), + default_value: f.default.clone(), + field_order: i as i32, + ..Default::default() }) .collect(); - Ok(AppStructure { - name: resp.name, - description: resp.description, - domain: resp.domain, - tables, - features: resp - .features - .unwrap_or_else(|| vec!["crud".to_string(), "search".to_string()]), - }) - } - Err(e) => { - warn!("Failed to parse LLM response, using fallback: {e}"); - self.analyze_app_requirements_fallback(original_intent) - } - } - } - - /// Fallback when LLM fails - uses heuristic patterns - fn analyze_app_requirements_fallback( - &self, - intent: &str, - ) -> Result> { - let intent_lower = intent.to_lowercase(); - let (domain, name) = self.extract_domain_and_name(&intent_lower); - let tables = self.infer_tables_from_intent_fallback(&intent_lower, &domain)?; - let features = vec!["crud".to_string(), "search".to_string()]; - - Ok(AppStructure { - name, - description: intent.to_string(), - domain, - tables, - features, - }) - } - - fn extract_domain_and_name(&self, intent: &str) -> (String, String) { - let patterns = [ - ("clínica", "healthcare", "clinic"), - ("clinic", "healthcare", "clinic"), - ("hospital", "healthcare", "hospital"), - ("médico", "healthcare", "medical"), - ("paciente", "healthcare", "patients"), - ("crm", "sales", "crm"), - ("vendas", "sales", "sales"), - ("loja", "retail", "store"), - ("estoque", "inventory", "inventory"), - ("produto", "inventory", "products"), - ("cliente", "sales", "customers"), - ("restaurante", "food", "restaurant"), - ("reserva", "booking", "reservations"), - ]; - - for (pattern, domain, name) in patterns { - if intent.contains(pattern) { - return (domain.to_string(), name.to_string()); - } - } - - ("general".to_string(), "app".to_string()) - } - - fn infer_tables_from_intent_fallback( - &self, - intent: &str, - domain: &str, - ) -> Result, Box> { - let mut tables = Vec::new(); - - match domain { - "healthcare" => { - tables.push(self.create_patients_table()); - tables.push(self.create_appointments_table()); - } - "sales" | "retail" => { - tables.push(self.create_customers_table()); - tables.push(self.create_products_table()); - if intent.contains("venda") || intent.contains("order") { - tables.push(self.create_orders_table()); + TableDefinition { + name: t.name.clone(), + connection_name: "default".to_string(), + fields, + ..Default::default() } - } - "inventory" => { - tables.push(self.create_products_table()); - tables.push(self.create_suppliers_table()); - } - _ => { - tables.push(self.create_items_table()); - } - } - - Ok(tables) + }) + .collect() + } + + fn detect_file_type(filename: &str) -> FileType { + let ext = filename.rsplit('.').next().unwrap_or("").to_lowercase(); + match ext.as_str() { + "css" => FileType::Css, + "js" => FileType::Js, + "bas" => FileType::Bas, + "json" => FileType::Json, + _ => FileType::Html, + } } - /// Call LLM for app generation async fn call_llm( &self, prompt: &str, ) -> Result> { - trace!("Calling LLM for app generation"); - #[cfg(feature = "llm")] { let config = serde_json::json!({ - "temperature": 0.3, - "max_tokens": 2000 + "temperature": 0.7, + "max_tokens": 16000 }); - let response = self + + match self .state .llm_provider .generate(prompt, &config, "gpt-4", "") - .await?; - return Ok(response); + .await + { + Ok(response) => return Ok(response), + Err(e) => { + warn!("LLM call failed: {}", e); + return Err(e); + } + } } #[cfg(not(feature = "llm"))] { - warn!("LLM feature not enabled, using fallback"); - Ok("{}".to_string()) - } - } - - fn create_patients_table(&self) -> TableDefinition { - TableDefinition { - name: "patients".to_string(), - connection_name: "default".to_string(), - fields: vec![ - self.create_id_field(0), - self.create_name_field(1), - self.create_phone_field(2), - self.create_email_field(3), - FieldDefinition { - name: "birth_date".to_string(), - field_type: "date".to_string(), - length: None, - precision: None, - is_key: false, - is_nullable: true, - default_value: None, - reference_table: None, - field_order: 4, - }, - self.create_created_at_field(5), - ], - } - } - - fn create_appointments_table(&self) -> TableDefinition { - TableDefinition { - name: "appointments".to_string(), - connection_name: "default".to_string(), - fields: vec![ - self.create_id_field(0), - FieldDefinition { - name: "patient_id".to_string(), - field_type: "guid".to_string(), - length: None, - precision: None, - is_key: false, - is_nullable: false, - default_value: None, - reference_table: Some("patients".to_string()), - field_order: 1, - }, - FieldDefinition { - name: "scheduled_at".to_string(), - field_type: "datetime".to_string(), - length: None, - precision: None, - is_key: false, - is_nullable: false, - default_value: None, - reference_table: None, - field_order: 2, - }, - FieldDefinition { - name: "status".to_string(), - field_type: "string".to_string(), - length: Some(50), - precision: None, - is_key: false, - is_nullable: false, - default_value: Some("'scheduled'".to_string()), - reference_table: None, - field_order: 3, - }, - self.create_created_at_field(4), - ], - } - } - - fn create_customers_table(&self) -> TableDefinition { - TableDefinition { - name: "customers".to_string(), - connection_name: "default".to_string(), - fields: vec![ - self.create_id_field(0), - self.create_name_field(1), - self.create_phone_field(2), - self.create_email_field(3), - FieldDefinition { - name: "address".to_string(), - field_type: "text".to_string(), - length: None, - precision: None, - is_key: false, - is_nullable: true, - default_value: None, - reference_table: None, - field_order: 4, - }, - self.create_created_at_field(5), - ], - } - } - - fn create_products_table(&self) -> TableDefinition { - TableDefinition { - name: "products".to_string(), - connection_name: "default".to_string(), - fields: vec![ - self.create_id_field(0), - self.create_name_field(1), - FieldDefinition { - name: "price".to_string(), - field_type: "number".to_string(), - length: Some(10), - precision: Some(2), - is_key: false, - is_nullable: false, - default_value: Some("0".to_string()), - reference_table: None, - field_order: 2, - }, - FieldDefinition { - name: "stock".to_string(), - field_type: "integer".to_string(), - length: None, - precision: None, - is_key: false, - is_nullable: false, - default_value: Some("0".to_string()), - reference_table: None, - field_order: 3, - }, - self.create_created_at_field(4), - ], - } - } - - fn create_orders_table(&self) -> TableDefinition { - TableDefinition { - name: "orders".to_string(), - connection_name: "default".to_string(), - fields: vec![ - self.create_id_field(0), - FieldDefinition { - name: "customer_id".to_string(), - field_type: "guid".to_string(), - length: None, - precision: None, - is_key: false, - is_nullable: false, - default_value: None, - reference_table: Some("customers".to_string()), - field_order: 1, - }, - FieldDefinition { - name: "total".to_string(), - field_type: "number".to_string(), - length: Some(10), - precision: Some(2), - is_key: false, - is_nullable: false, - default_value: Some("0".to_string()), - reference_table: None, - field_order: 2, - }, - FieldDefinition { - name: "status".to_string(), - field_type: "string".to_string(), - length: Some(50), - precision: None, - is_key: false, - is_nullable: false, - default_value: Some("'pending'".to_string()), - reference_table: None, - field_order: 3, - }, - self.create_created_at_field(4), - ], - } - } - - fn create_suppliers_table(&self) -> TableDefinition { - TableDefinition { - name: "suppliers".to_string(), - connection_name: "default".to_string(), - fields: vec![ - self.create_id_field(0), - self.create_name_field(1), - self.create_phone_field(2), - self.create_email_field(3), - self.create_created_at_field(4), - ], - } - } - - fn create_items_table(&self) -> TableDefinition { - TableDefinition { - name: "items".to_string(), - connection_name: "default".to_string(), - fields: vec![ - self.create_id_field(0), - self.create_name_field(1), - FieldDefinition { - name: "description".to_string(), - field_type: "text".to_string(), - length: None, - precision: None, - is_key: false, - is_nullable: true, - default_value: None, - reference_table: None, - field_order: 2, - }, - self.create_created_at_field(3), - ], - } - } - - fn create_id_field(&self, order: i32) -> FieldDefinition { - FieldDefinition { - name: "id".to_string(), - field_type: "guid".to_string(), - length: None, - precision: None, - is_key: true, - is_nullable: false, - default_value: None, - reference_table: None, - field_order: order, - } - } - - fn create_name_field(&self, order: i32) -> FieldDefinition { - FieldDefinition { - name: "name".to_string(), - field_type: "string".to_string(), - length: Some(255), - precision: None, - is_key: false, - is_nullable: false, - default_value: None, - reference_table: None, - field_order: order, - } - } - - fn create_phone_field(&self, order: i32) -> FieldDefinition { - FieldDefinition { - name: "phone".to_string(), - field_type: "string".to_string(), - length: Some(50), - precision: None, - is_key: false, - is_nullable: true, - default_value: None, - reference_table: None, - field_order: order, - } - } - - fn create_email_field(&self, order: i32) -> FieldDefinition { - FieldDefinition { - name: "email".to_string(), - field_type: "string".to_string(), - length: Some(255), - precision: None, - is_key: false, - is_nullable: true, - default_value: None, - reference_table: None, - field_order: order, - } - } - - fn create_created_at_field(&self, order: i32) -> FieldDefinition { - FieldDefinition { - name: "created_at".to_string(), - field_type: "datetime".to_string(), - length: None, - precision: None, - is_key: false, - is_nullable: false, - default_value: Some("NOW()".to_string()), - reference_table: None, - field_order: order, + Err("LLM feature not enabled. App generation requires LLM.".into()) } } fn generate_table_definitions( - &self, - structure: &AppStructure, + tables: &[TableDefinition], ) -> Result> { + use std::fmt::Write; let mut output = String::new(); - for table in &structure.tables { + for table in tables { let _ = writeln!(output, "\nTABLE {}", table.name); for field in &table.fields { @@ -791,15 +596,15 @@ Respond with valid JSON only."# line.push_str(" REQUIRED"); } if let Some(ref default) = field.default_value { - let _ = write!(line, " DEFAULT {default}"); + let _ = write!(line, " DEFAULT {}", default); } if let Some(ref refs) = field.reference_table { - let _ = write!(line, " REFERENCES {refs}"); + let _ = write!(line, " REFERENCES {}", refs); } - let _ = writeln!(output, "{line}"); + let _ = writeln!(output, "{}", line); } - let _ = writeln!(output, "END TABLE"); + let _ = writeln!(output, "END TABLE\n"); } Ok(output) @@ -810,196 +615,125 @@ Respond with valid JSON only."# bot_id: Uuid, content: &str, ) -> Result<(), Box> { - // For tables.bas, we write to local file system since it's used by the compiler - // The DriveMonitor will sync it to S3 - let site_path = self.get_site_path(); - let tables_bas_path = format!("{}/{}.gbai/.gbdialog/tables.bas", site_path, bot_id); + let bot_name = self.get_bot_name(bot_id)?; + let bucket = format!("{}.gbai", bot_name.to_lowercase()); + let path = ".gbdata/tables.bas"; - let dir = std::path::Path::new(&tables_bas_path).parent(); - if let Some(d) = dir { - if !d.exists() { - std::fs::create_dir_all(d)?; - } + let mut conn = self.state.conn.get()?; + + #[derive(QueryableByName)] + struct ContentRow { + #[diesel(sql_type = diesel::sql_types::Text)] + content: String, } - let existing = std::fs::read_to_string(&tables_bas_path).unwrap_or_default(); - let new_content = format!("{existing}\n{content}"); - std::fs::write(&tables_bas_path, new_content)?; - info!("Updated tables.bas at: {}", tables_bas_path); + let existing: Option = + sql_query("SELECT content FROM drive_files WHERE bucket = $1 AND path = $2 LIMIT 1") + .bind::(&bucket) + .bind::(path) + .load::(&mut conn) + .ok() + .and_then(|rows| rows.into_iter().next().map(|r| r.content)); + + let new_content = match existing { + Some(existing_content) => format!("{}\n{}", existing_content, content), + None => content.to_string(), + }; + + sql_query( + "INSERT INTO drive_files (id, bucket, path, content, content_type, created_at, updated_at) + VALUES ($1, $2, $3, $4, 'text/plain', NOW(), NOW()) + ON CONFLICT (bucket, path) DO UPDATE SET content = $4, updated_at = NOW()", + ) + .bind::(Uuid::new_v4()) + .bind::(&bucket) + .bind::(path) + .bind::(&new_content) + .execute(&mut conn)?; Ok(()) } - /// Get bot name from database fn get_bot_name( &self, bot_id: Uuid, ) -> Result> { - use crate::shared::models::schema::bots::dsl::{bots, id, name}; - use diesel::prelude::*; - let mut conn = self.state.conn.get()?; - let bot_name: String = bots - .filter(id.eq(bot_id)) - .select(name) - .first(&mut conn) - .map_err(|e| format!("Failed to get bot name: {}", e))?; - Ok(bot_name) + #[derive(QueryableByName)] + struct BotRow { + #[diesel(sql_type = diesel::sql_types::Text)] + name: String, + } + + let result: Vec = sql_query("SELECT name FROM bots WHERE id = $1 LIMIT 1") + .bind::(bot_id) + .load(&mut conn)?; + + result + .into_iter() + .next() + .map(|r| r.name) + .ok_or_else(|| format!("Bot not found: {}", bot_id).into()) } - /// Write content to S3 drive async fn write_to_drive( &self, bucket: &str, path: &str, content: &str, ) -> Result<(), Box> { - let Some(client) = &self.state.drive else { - warn!("S3 client not configured, falling back to local write"); - return self.write_to_local_fallback(bucket, path, content); - }; + if let Some(ref s3) = self.state.s3_client { + let body = ByteStream::from(content.as_bytes().to_vec()); + let content_type = Self::get_content_type(path); - let key = path.to_string(); - let content_type = self.get_content_type(path); + s3.put_object() + .bucket(bucket) + .key(path) + .body(body) + .content_type(content_type) + .send() + .await?; - client - .put_object() - .bucket(bucket.to_lowercase()) - .key(&key) - .body(ByteStream::from(content.as_bytes().to_vec())) - .content_type(content_type) - .send() - .await - .map_err(|e| format!("Failed to write to drive: {}", e))?; + trace!("Wrote to S3: s3://{}/{}", bucket, path); + } else { + self.write_to_db_fallback(bucket, path, content)?; + } - trace!("Wrote to drive: s3://{}/{}", bucket, key); Ok(()) } - /// Fallback to local file system when S3 is not configured - fn write_to_local_fallback( + fn write_to_db_fallback( &self, bucket: &str, path: &str, content: &str, - ) -> Result<(), Box> { - let site_path = self.get_site_path(); - let full_path = format!("{}/{}/{}", site_path, bucket, path); - - if let Some(dir) = std::path::Path::new(&full_path).parent() { - if !dir.exists() { - std::fs::create_dir_all(dir)?; - } - } - - std::fs::write(&full_path, content)?; - trace!("Wrote to local fallback: {}", full_path); - Ok(()) - } - - /// Sync app from drive to SITE_ROOT for serving - async fn sync_app_to_site_root( - &self, - bucket: &str, - app_name: &str, - bot_id: Uuid, - ) -> Result<(), Box> { - let site_path = self.get_site_path(); - - // Target: {site_path}/{app_name}/ (clean URL) - let target_dir = format!("{}/{}", site_path, app_name); - std::fs::create_dir_all(&target_dir)?; - - let Some(client) = &self.state.drive else { - info!("S3 not configured, app already written to local path"); - return Ok(()); - }; - - // List all files in the app directory on drive - let prefix = format!(".gbdrive/apps/{}/", app_name); - let list_result = client - .list_objects_v2() - .bucket(bucket.to_lowercase()) - .prefix(&prefix) - .send() - .await - .map_err(|e| format!("Failed to list app files: {}", e))?; - - for obj in list_result.contents.unwrap_or_default() { - let key = obj.key().unwrap_or_default(); - if key.ends_with('/') { - continue; // Skip directories - } - - // Get the file from S3 - let get_result = client - .get_object() - .bucket(bucket.to_lowercase()) - .key(key) - .send() - .await - .map_err(|e| format!("Failed to get file {}: {}", key, e))?; - - let body = get_result - .body - .collect() - .await - .map_err(|e| format!("Failed to read file body: {}", e))?; - - // Extract relative path (remove .gbdrive/apps/{app_name}/ prefix) - let relative_path = key.strip_prefix(&prefix).unwrap_or(key); - let local_path = format!("{}/{}", target_dir, relative_path); - - // Create parent directories if needed - if let Some(dir) = std::path::Path::new(&local_path).parent() { - if !dir.exists() { - std::fs::create_dir_all(dir)?; - } - } - - // Write the file - std::fs::write(&local_path, body.into_bytes())?; - trace!("Synced: {} -> {}", key, local_path); - } - - info!("App '{}' synced to site root: {}", app_name, target_dir); - - // Store app metadata in database for tracking - self.store_app_metadata(bot_id, app_name, &target_dir)?; - - Ok(()) - } - - /// Store app metadata for tracking - fn store_app_metadata( - &self, - bot_id: Uuid, - app_name: &str, - app_path: &str, ) -> Result<(), Box> { let mut conn = self.state.conn.get()?; - let app_id = Uuid::new_v4(); + let content_type = Self::get_content_type(path); sql_query( - "INSERT INTO generated_apps (id, bot_id, name, app_path, is_active, created_at) - VALUES ($1, $2, $3, $4, true, NOW()) - ON CONFLICT (bot_id, name) DO UPDATE SET - app_path = EXCLUDED.app_path, + "INSERT INTO drive_files (id, bucket, path, content, content_type, size, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, NOW(), NOW()) + ON CONFLICT (bucket, path) DO UPDATE SET + content = EXCLUDED.content, + content_type = EXCLUDED.content_type, + size = EXCLUDED.size, updated_at = NOW()", ) - .bind::(app_id) - .bind::(bot_id) - .bind::(app_name) - .bind::(app_path) - .execute(&mut conn) - .map_err(|e| format!("Failed to store app metadata: {}", e))?; + .bind::(Uuid::new_v4()) + .bind::(bucket) + .bind::(path) + .bind::(content) + .bind::(content_type) + .bind::(content.len() as i64) + .execute(&mut conn)?; + trace!("Wrote to DB: {}/{}", bucket, path); Ok(()) } - /// Get content type based on file extension - fn get_content_type(&self, path: &str) -> &'static str { + fn get_content_type(path: &str) -> &'static str { let ext = path.rsplit('.').next().unwrap_or("").to_lowercase(); match ext.as_str() { "html" | "htm" => "text/html; charset=utf-8", @@ -1045,292 +779,107 @@ Respond with valid JSON only."# }) } - fn generate_htmx_pages( + async fn sync_app_to_site_root( &self, - structure: &AppStructure, - ) -> Result, Box> { - let mut pages = Vec::new(); + bucket: &str, + app_name: &str, + bot_id: Uuid, + ) -> Result<(), Box> { + let source_path = format!(".gbdrive/apps/{}", app_name); + let site_path = Self::get_site_path(bot_id); - pages.push(GeneratedPage { - filename: "index.html".to_string(), - title: format!("{} - Dashboard", structure.name), - page_type: PageType::Dashboard, - content: self.generate_dashboard_html(structure), - route: "/".to_string(), - }); + if let Some(ref s3) = self.state.s3_client { + let list_result = s3 + .list_objects_v2() + .bucket(bucket) + .prefix(&source_path) + .send() + .await?; - for table in &structure.tables { - pages.push(GeneratedPage { - filename: format!("{}.html", table.name), - title: format!("{} - List", table.name), - page_type: PageType::List, - content: self.generate_list_html(&table.name, &table.fields), - route: format!("/{}", table.name), - }); + if let Some(contents) = list_result.contents { + for object in contents { + if let Some(key) = object.key { + let relative_path = + key.trim_start_matches(&source_path).trim_start_matches('/'); + let dest_key = format!("{}/{}/{}", site_path, app_name, relative_path); - pages.push(GeneratedPage { - filename: format!("{}_form.html", table.name), - title: format!("{} - Form", table.name), - page_type: PageType::Form, - content: self.generate_form_html(&table.name, &table.fields), - route: format!("/{}/new", table.name), - }); - } + s3.copy_object() + .bucket(bucket) + .copy_source(format!("{}/{}", bucket, key)) + .key(&dest_key) + .send() + .await?; - Ok(pages) - } - - fn generate_dashboard_html(&self, structure: &AppStructure) -> String { - let mut html = String::new(); - - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - let _ = writeln!(html, " "); - let _ = writeln!( - html, - " " - ); - let _ = writeln!(html, " {}", structure.name); - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - let _ = writeln!(html, "
"); - let _ = writeln!(html, "

{}

", structure.name); - let _ = writeln!(html, " "); - let _ = writeln!(html, "
"); - let _ = writeln!(html, "
"); - - for table in &structure.tables { - let _ = writeln!(html, "
", table.name); - let _ = writeln!(html, "

{}

", table.name); - let _ = writeln!(html, " -"); - let _ = writeln!(html, "
"); - } - - let _ = writeln!(html, "
"); - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - - html - } - - fn generate_list_html(&self, table_name: &str, fields: &[FieldDefinition]) -> String { - let mut html = String::new(); - - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - let _ = writeln!(html, " "); - let _ = writeln!( - html, - " " - ); - let _ = writeln!(html, " {table_name} - List"); - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - let _ = writeln!(html, "
"); - let _ = writeln!(html, "

{table_name}

"); - let _ = writeln!( - html, - " Add New" - ); - let _ = writeln!(html, "
"); - let _ = writeln!(html, "
"); - let _ = writeln!( - html, - " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - - for field in fields { - if field.name != "id" { - let _ = writeln!(html, " ", field.name); + trace!("Synced {} to {}", key, dest_key); + } + } } } - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, "
{}Actions
"); - let _ = writeln!(html, "
"); - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); + let _ = self.store_app_metadata(bot_id, app_name, &format!("{}/{}", site_path, app_name)); - html + info!("App synced to site root: {}/{}", site_path, app_name); + Ok(()) } - fn generate_form_html(&self, table_name: &str, fields: &[FieldDefinition]) -> String { - let mut html = String::new(); - - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - let _ = writeln!(html, " "); - let _ = writeln!( - html, - " " - ); - let _ = writeln!(html, " {table_name} - Form"); - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, " "); - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - let _ = writeln!(html, "
"); - let _ = writeln!(html, "

New {table_name}

"); - let _ = writeln!( - html, - " Back to List" - ); - let _ = writeln!(html, "
"); - let _ = writeln!(html, "
"); - let _ = writeln!( - html, - "
" - ); - - for field in fields { - if field.name == "id" || field.name == "created_at" || field.name == "updated_at" { - continue; - } - - let required = if field.is_nullable { "" } else { " required" }; - let input_type = match field.field_type.as_str() { - "number" | "integer" => "number", - "date" => "date", - "datetime" => "datetime-local", - "boolean" => "checkbox", - "text" => "textarea", - _ => "text", - }; - - let _ = writeln!(html, "
"); - let _ = writeln!( - html, - " ", - field.name, field.name - ); - - if input_type == "textarea" { - let _ = writeln!( - html, - " ", - field.name, field.name, required - ); - } else { - let _ = writeln!( - html, - " ", - input_type, field.name, field.name, required - ); - } - - let _ = writeln!(html, "
"); - } - - let _ = writeln!( - html, - " " - ); - let _ = writeln!(html, "
"); - let _ = writeln!(html, "
"); - let _ = writeln!(html, "
"); - let _ = writeln!(html, ""); - let _ = writeln!(html, ""); - - html - } - - fn generate_app_css(&self) -> String { - r#"* { box-sizing: border-box; margin: 0; padding: 0; } -body { font-family: system-ui, sans-serif; line-height: 1.5; padding: 1rem; } -.app-header { display: flex; justify-content: space-between; align-items: center; padding: 1rem; border-bottom: 1px solid #ddd; margin-bottom: 1rem; } -.app-header nav { display: flex; gap: 1rem; } -.app-header nav a { text-decoration: none; color: #0066cc; } -.page-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 1rem; } -.dashboard { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 1rem; } -.stat-card { background: #f5f5f5; padding: 1rem; border-radius: 8px; text-align: center; } -.stat-card .count { font-size: 2rem; font-weight: bold; } -table { width: 100%; border-collapse: collapse; margin-top: 1rem; } -th, td { padding: 0.75rem; text-align: left; border-bottom: 1px solid #ddd; } -th { background: #f5f5f5; } -.form-group { margin-bottom: 1rem; } -.form-group label { display: block; margin-bottom: 0.25rem; font-weight: 500; } -.form-group input, .form-group textarea, .form-group select { width: 100%; padding: 0.5rem; border: 1px solid #ddd; border-radius: 4px; } -.btn { display: inline-block; padding: 0.5rem 1rem; border: none; border-radius: 4px; cursor: pointer; text-decoration: none; } -.btn-primary { background: #0066cc; color: white; } -.btn-danger { background: #cc0000; color: white; } -.btn-secondary { background: #666; color: white; } -input[type="search"] { width: 100%; max-width: 300px; padding: 0.5rem; border: 1px solid #ddd; border-radius: 4px; } -.alert { padding: 1rem; border-radius: 4px; margin-bottom: 1rem; } -.alert-success { background: #d4edda; color: #155724; } -.alert-error { background: #f8d7da; color: #721c24; } -"#.to_string() - } - - fn generate_tools( + fn store_app_metadata( &self, - _structure: &AppStructure, - ) -> Result, Box> { - Ok(Vec::new()) + bot_id: Uuid, + app_name: &str, + app_path: &str, + ) -> Result<(), Box> { + let mut conn = self.state.conn.get()?; + let app_id = Uuid::new_v4(); + + sql_query( + "INSERT INTO generated_apps (id, bot_id, name, app_path, is_active, created_at) + VALUES ($1, $2, $3, $4, true, NOW()) + ON CONFLICT (bot_id, name) DO UPDATE SET + app_path = EXCLUDED.app_path, + updated_at = NOW()", + ) + .bind::(app_id) + .bind::(bot_id) + .bind::(app_name) + .bind::(app_path) + .execute(&mut conn)?; + + Ok(()) } - fn generate_schedulers( - &self, - _structure: &AppStructure, - ) -> Result, Box> { - Ok(Vec::new()) + fn get_site_path(_bot_id: Uuid) -> String { + ".gbdrive/site".to_string() } - fn generate_designer_js(&self) -> String { - r#"(function() { + fn generate_designer_js(app_name: &str) -> String { + format!( + r#"(function() {{ + const APP_NAME = '{app_name}'; + const currentPage = window.location.pathname.split('/').pop() || 'index.html'; + const style = document.createElement('style'); style.textContent = ` - .designer-btn { position: fixed; bottom: 20px; right: 20px; width: 56px; height: 56px; border-radius: 50%; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border: none; cursor: pointer; box-shadow: 0 4px 20px rgba(102,126,234,0.4); font-size: 24px; z-index: 9999; transition: transform 0.2s, box-shadow 0.2s; } - .designer-btn:hover { transform: scale(1.1); box-shadow: 0 6px 30px rgba(102,126,234,0.6); } - .designer-panel { position: fixed; bottom: 90px; right: 20px; width: 380px; max-height: 500px; background: white; border-radius: 16px; box-shadow: 0 10px 40px rgba(0,0,0,0.2); z-index: 9998; display: none; flex-direction: column; overflow: hidden; } - .designer-panel.open { display: flex; } - .designer-header { padding: 16px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; font-weight: 600; display: flex; justify-content: space-between; align-items: center; } - .designer-close { background: none; border: none; color: white; font-size: 20px; cursor: pointer; } - .designer-messages { flex: 1; overflow-y: auto; padding: 16px; max-height: 300px; } - .designer-msg { margin: 8px 0; padding: 10px 14px; border-radius: 12px; max-width: 85%; } - .designer-msg.user { background: #667eea; color: white; margin-left: auto; } - .designer-msg.ai { background: #f0f0f0; color: #333; } - .designer-input { display: flex; padding: 12px; border-top: 1px solid #eee; gap: 8px; } - .designer-input input { flex: 1; padding: 10px 14px; border: 1px solid #ddd; border-radius: 20px; outline: none; } - .designer-input button { padding: 10px 16px; background: #667eea; color: white; border: none; border-radius: 20px; cursor: pointer; } + .designer-fab {{ position: fixed; bottom: 20px; right: 20px; width: 56px; height: 56px; border-radius: 50%; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border: none; cursor: pointer; box-shadow: 0 4px 12px rgba(102,126,234,0.4); font-size: 24px; z-index: 9999; transition: transform 0.2s; }} + .designer-fab:hover {{ transform: scale(1.1); }} + .designer-panel {{ position: fixed; bottom: 90px; right: 20px; width: 380px; max-height: 500px; background: white; border-radius: 16px; box-shadow: 0 10px 40px rgba(0,0,0,0.2); z-index: 9998; display: none; flex-direction: column; overflow: hidden; }} + .designer-panel.open {{ display: flex; }} + .designer-header {{ padding: 16px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; font-weight: 600; display: flex; justify-content: space-between; align-items: center; }} + .designer-close {{ background: none; border: none; color: white; font-size: 20px; cursor: pointer; }} + .designer-messages {{ flex: 1; overflow-y: auto; padding: 16px; max-height: 300px; }} + .designer-msg {{ margin: 8px 0; padding: 10px 14px; border-radius: 12px; max-width: 85%; word-wrap: break-word; }} + .designer-msg.user {{ background: #667eea; color: white; margin-left: auto; }} + .designer-msg.ai {{ background: #f0f0f0; color: #333; }} + .designer-input {{ display: flex; padding: 12px; border-top: 1px solid #eee; gap: 8px; }} + .designer-input input {{ flex: 1; padding: 10px 14px; border: 1px solid #ddd; border-radius: 20px; outline: none; }} + .designer-input button {{ padding: 10px 16px; background: #667eea; color: white; border: none; border-radius: 20px; cursor: pointer; }} `; document.head.appendChild(style); - const btn = document.createElement('button'); - btn.className = 'designer-btn'; - btn.innerHTML = '🎨'; - btn.title = 'Designer AI'; - document.body.appendChild(btn); + const fab = document.createElement('button'); + fab.className = 'designer-fab'; + fab.innerHTML = '🎨'; + fab.title = 'Designer AI'; + document.body.appendChild(fab); const panel = document.createElement('div'); panel.className = 'designer-panel'; @@ -1343,57 +892,49 @@ input[type="search"] { width: 100%; max-width: 300px; padding: 0.5rem; border: 1
Hi! I can help you modify this app. What would you like to change?
- +
`; document.body.appendChild(panel); - btn.onclick = () => panel.classList.toggle('open'); + fab.onclick = () => panel.classList.toggle('open'); panel.querySelector('.designer-close').onclick = () => panel.classList.remove('open'); const input = panel.querySelector('input'); const sendBtn = panel.querySelector('.designer-input button'); const messages = panel.querySelector('.designer-messages'); - const appName = window.location.pathname.split('/')[2] || 'app'; - const currentPage = window.location.pathname.split('/').pop() || 'index.html'; - - async function sendMessage() { + async function sendMessage() {{ const msg = input.value.trim(); if (!msg) return; - messages.innerHTML += `
${msg}
`; + messages.innerHTML += `
${{msg}}
`; input.value = ''; messages.scrollTop = messages.scrollHeight; - try { - const res = await fetch('/api/designer/modify', { + try {{ + const res = await fetch('/api/designer/modify', {{ method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ app_name: appName, current_page: currentPage, message: msg }) - }); + headers: {{ 'Content-Type': 'application/json' }}, + body: JSON.stringify({{ app_name: APP_NAME, current_page: currentPage, message: msg }}) + }}); const data = await res.json(); - messages.innerHTML += `
${data.message || 'Done!'}
`; - if (data.success && data.changes && data.changes.length > 0) { - setTimeout(() => location.reload(), 1000); - } - } catch (e) { + messages.innerHTML += `
${{data.message || 'Done!'}}
`; + if (data.success && data.changes && data.changes.length > 0) {{ + setTimeout(() => location.reload(), 1500); + }} + }} catch (e) {{ messages.innerHTML += `
Sorry, something went wrong. Try again.
`; - } + if (window.AppLogger) window.AppLogger.error('Designer error', e.toString()); + }} messages.scrollTop = messages.scrollHeight; - } + }} sendBtn.onclick = sendMessage; - input.onkeypress = (e) => { if (e.key === 'Enter') sendMessage(); }; -})();"#.to_string() - } - - fn get_site_path(&self) -> String { - self.state - .config - .as_ref() - .map(|c| c.site_path.clone()) - .unwrap_or_else(|| "./botserver-stack/sites".to_string()) + input.onkeypress = (e) => {{ if (e.key === 'Enter') sendMessage(); }}; +}})();"#, + app_name = app_name + ) } } diff --git a/src/auto_task/app_logs.rs b/src/auto_task/app_logs.rs new file mode 100644 index 000000000..b5984312a --- /dev/null +++ b/src/auto_task/app_logs.rs @@ -0,0 +1,613 @@ +use chrono::{DateTime, Duration, Utc}; +use log::{debug, error, info, warn}; +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, VecDeque}; +use std::fmt::Write; +use std::sync::{Arc, RwLock}; +use uuid::Uuid; + +const MAX_LOGS_PER_APP: usize = 500; +const MAX_LOGS_FOR_DESIGNER: usize = 50; +const LOG_RETENTION_DAYS: i64 = 1; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AppLogEntry { + pub id: String, + pub timestamp: DateTime, + pub level: LogLevel, + pub source: LogSource, + pub app_name: String, + pub bot_id: Option, + pub user_id: Option, + pub message: String, + pub details: Option, + pub file_path: Option, + pub line_number: Option, + pub stack_trace: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum LogLevel { + Debug, + Info, + Warn, + Error, + Critical, +} + +impl std::fmt::Display for LogLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Debug => write!(f, "debug"), + Self::Info => write!(f, "info"), + Self::Warn => write!(f, "warn"), + Self::Error => write!(f, "error"), + Self::Critical => write!(f, "critical"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum LogSource { + Server, + Client, + Generator, + Designer, + Validation, + Runtime, +} + +impl std::fmt::Display for LogSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Server => write!(f, "server"), + Self::Client => write!(f, "client"), + Self::Generator => write!(f, "generator"), + Self::Designer => write!(f, "designer"), + Self::Validation => write!(f, "validation"), + Self::Runtime => write!(f, "runtime"), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ClientLogRequest { + pub app_name: String, + pub level: String, + pub message: String, + pub details: Option, + pub file_path: Option, + pub line_number: Option, + pub stack_trace: Option, + pub user_agent: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogQueryParams { + pub app_name: Option, + pub level: Option, + pub source: Option, + pub limit: Option, + pub since: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogStats { + pub total_logs: usize, + pub errors: usize, + pub warnings: usize, + pub by_app: HashMap, +} + +pub struct AppLogStore { + logs: RwLock>>, + global_logs: RwLock>, +} + +impl AppLogStore { + pub fn new() -> Self { + Self { + logs: RwLock::new(HashMap::new()), + global_logs: RwLock::new(VecDeque::with_capacity(MAX_LOGS_PER_APP)), + } + } + + pub fn log( + &self, + app_name: &str, + level: LogLevel, + source: LogSource, + message: &str, + details: Option, + bot_id: Option, + user_id: Option, + ) { + let entry = AppLogEntry { + id: Uuid::new_v4().to_string(), + timestamp: Utc::now(), + level, + source, + app_name: app_name.to_string(), + bot_id, + user_id, + message: message.to_string(), + details, + file_path: None, + line_number: None, + stack_trace: None, + }; + + self.add_entry(entry); + + match level { + LogLevel::Debug => debug!("[{}] {}: {}", app_name, source, message), + LogLevel::Info => info!("[{}] {}: {}", app_name, source, message), + LogLevel::Warn => warn!("[{}] {}: {}", app_name, source, message), + LogLevel::Error | LogLevel::Critical => { + error!("[{}] {}: {}", app_name, source, message); + } + } + } + + pub fn log_error( + &self, + app_name: &str, + source: LogSource, + message: &str, + error: &str, + file_path: Option<&str>, + line_number: Option, + stack_trace: Option<&str>, + ) { + let entry = AppLogEntry { + id: Uuid::new_v4().to_string(), + timestamp: Utc::now(), + level: LogLevel::Error, + source, + app_name: app_name.to_string(), + bot_id: None, + user_id: None, + message: message.to_string(), + details: Some(error.to_string()), + file_path: file_path.map(String::from), + line_number, + stack_trace: stack_trace.map(String::from), + }; + + self.add_entry(entry); + + error!( + "[{}] {}: {} - {} ({}:{})", + app_name, + source, + message, + error, + file_path.unwrap_or("unknown"), + line_number.unwrap_or(0) + ); + } + + pub fn log_client( + &self, + request: ClientLogRequest, + bot_id: Option, + user_id: Option, + ) { + let level = match request.level.to_lowercase().as_str() { + "debug" => LogLevel::Debug, + "warn" | "warning" => LogLevel::Warn, + "error" => LogLevel::Error, + "critical" => LogLevel::Critical, + _ => LogLevel::Info, + }; + + let entry = AppLogEntry { + id: Uuid::new_v4().to_string(), + timestamp: Utc::now(), + level, + source: LogSource::Client, + app_name: request.app_name, + bot_id, + user_id, + message: request.message, + details: request.details, + file_path: request.file_path, + line_number: request.line_number, + stack_trace: request.stack_trace, + }; + + self.add_entry(entry); + } + + fn add_entry(&self, entry: AppLogEntry) { + if let Ok(mut logs) = self.logs.write() { + let app_logs = logs + .entry(entry.app_name.clone()) + .or_insert_with(|| VecDeque::with_capacity(MAX_LOGS_PER_APP)); + + if app_logs.len() >= MAX_LOGS_PER_APP { + app_logs.pop_front(); + } + app_logs.push_back(entry.clone()); + } + + if let Ok(mut global) = self.global_logs.write() { + if global.len() >= MAX_LOGS_PER_APP { + global.pop_front(); + } + global.push_back(entry); + } + } + + pub fn get_logs(&self, params: &LogQueryParams) -> Vec { + let limit = params.limit.unwrap_or(100).min(500); + let cutoff = params + .since + .unwrap_or_else(|| Utc::now() - Duration::days(LOG_RETENTION_DAYS)); + + let level_filter: Option = + params + .level + .as_ref() + .and_then(|l| match l.to_lowercase().as_str() { + "debug" => Some(LogLevel::Debug), + "info" => Some(LogLevel::Info), + "warn" => Some(LogLevel::Warn), + "error" => Some(LogLevel::Error), + "critical" => Some(LogLevel::Critical), + _ => None, + }); + + let source_filter: Option = + params + .source + .as_ref() + .and_then(|s| match s.to_lowercase().as_str() { + "server" => Some(LogSource::Server), + "client" => Some(LogSource::Client), + "generator" => Some(LogSource::Generator), + "designer" => Some(LogSource::Designer), + "validation" => Some(LogSource::Validation), + "runtime" => Some(LogSource::Runtime), + _ => None, + }); + + if let Some(ref app_name) = params.app_name { + if let Ok(logs) = self.logs.read() { + if let Some(app_logs) = logs.get(app_name) { + return app_logs + .iter() + .rev() + .filter(|e| e.timestamp >= cutoff) + .filter(|e| level_filter.is_none_or(|l| e.level == l)) + .filter(|e| source_filter.is_none_or(|s| e.source == s)) + .take(limit) + .cloned() + .collect(); + } + } + return Vec::new(); + } + + if let Ok(global) = self.global_logs.read() { + return global + .iter() + .rev() + .filter(|e| e.timestamp >= cutoff) + .filter(|e| level_filter.is_none_or(|l| e.level == l)) + .filter(|e| source_filter.is_none_or(|s| e.source == s)) + .take(limit) + .cloned() + .collect(); + } + + Vec::new() + } + + pub fn get_errors_for_designer(&self, app_name: &str) -> Vec { + if let Ok(logs) = self.logs.read() { + if let Some(app_logs) = logs.get(app_name) { + let cutoff = Utc::now() - Duration::hours(1); + return app_logs + .iter() + .rev() + .filter(|e| e.timestamp >= cutoff) + .filter(|e| { + matches!( + e.level, + LogLevel::Error | LogLevel::Critical | LogLevel::Warn + ) + }) + .take(MAX_LOGS_FOR_DESIGNER) + .cloned() + .collect(); + } + } + Vec::new() + } + + pub fn format_errors_for_prompt(&self, app_name: &str) -> Option { + let errors = self.get_errors_for_designer(app_name); + + if errors.is_empty() { + return None; + } + + let mut output = String::new(); + output.push_str("\n\n=== RECENT ERRORS AND WARNINGS ===\n"); + output.push_str("The following issues were detected. Please fix them:\n\n"); + + for (idx, entry) in errors.iter().enumerate() { + let _ = writeln!( + output, + "{}. [{}] [{}] {}", + idx + 1, + entry.level, + entry.source, + entry.message + ); + + if let Some(ref details) = entry.details { + let _ = writeln!(output, " Details: {details}"); + } + + if let Some(ref file) = entry.file_path { + let _ = writeln!( + output, + " Location: {}:{}", + file, + entry.line_number.unwrap_or(0) + ); + } + + if let Some(ref stack) = entry.stack_trace { + let short_stack: String = stack.lines().take(3).collect::>().join("\n "); + let _ = writeln!(output, " Stack: {short_stack}"); + } + + output.push('\n'); + } + + output.push_str("=== END OF ERRORS ===\n"); + Some(output) + } + + pub fn get_stats(&self) -> LogStats { + let mut stats = LogStats { + total_logs: 0, + errors: 0, + warnings: 0, + by_app: HashMap::new(), + }; + + if let Ok(logs) = self.logs.read() { + for (app_name, app_logs) in logs.iter() { + let count = app_logs.len(); + stats.total_logs += count; + stats.by_app.insert(app_name.clone(), count); + + for entry in app_logs { + match entry.level { + LogLevel::Error | LogLevel::Critical => stats.errors += 1, + LogLevel::Warn => stats.warnings += 1, + _ => {} + } + } + } + } + + stats + } + + pub fn cleanup_old_logs(&self) { + let cutoff = Utc::now() - Duration::days(LOG_RETENTION_DAYS); + + if let Ok(mut logs) = self.logs.write() { + for app_logs in logs.values_mut() { + while let Some(front) = app_logs.front() { + if front.timestamp < cutoff { + app_logs.pop_front(); + } else { + break; + } + } + } + + logs.retain(|_, v| !v.is_empty()); + } + + if let Ok(mut global) = self.global_logs.write() { + while let Some(front) = global.front() { + if front.timestamp < cutoff { + global.pop_front(); + } else { + break; + } + } + } + + info!("Log cleanup completed"); + } + + pub fn clear_app_logs(&self, app_name: &str) { + if let Ok(mut logs) = self.logs.write() { + logs.remove(app_name); + } + info!("Cleared logs for app: {}", app_name); + } +} + +impl Default for AppLogStore { + fn default() -> Self { + Self::new() + } +} + +pub static APP_LOGS: Lazy> = Lazy::new(|| Arc::new(AppLogStore::new())); + +pub fn log_generator_info(app_name: &str, message: &str) { + APP_LOGS.log( + app_name, + LogLevel::Info, + LogSource::Generator, + message, + None, + None, + None, + ); +} + +pub fn log_generator_error(app_name: &str, message: &str, error: &str) { + APP_LOGS.log_error( + app_name, + LogSource::Generator, + message, + error, + None, + None, + None, + ); +} + +pub fn log_validation_error( + app_name: &str, + message: &str, + file_path: Option<&str>, + line_number: Option, +) { + APP_LOGS.log_error( + app_name, + LogSource::Validation, + message, + "Validation failed", + file_path, + line_number, + None, + ); +} + +pub fn log_runtime_error(app_name: &str, message: &str, error: &str, stack_trace: Option<&str>) { + APP_LOGS.log_error( + app_name, + LogSource::Runtime, + message, + error, + None, + None, + stack_trace, + ); +} + +pub fn get_designer_error_context(app_name: &str) -> Option { + APP_LOGS.format_errors_for_prompt(app_name) +} + +pub fn start_log_cleanup_scheduler() { + std::thread::spawn(|| loop { + std::thread::sleep(std::time::Duration::from_secs(3600)); + APP_LOGS.cleanup_old_logs(); + }); + info!("Log cleanup scheduler started (runs hourly)"); +} + +pub fn generate_client_logger_js() -> &'static str { + r" +(function() { + const APP_NAME = document.body.dataset.appName || window.location.pathname.split('/')[1] || 'unknown'; + const LOG_ENDPOINT = '/api/app-logs/client'; + const LOG_BUFFER = []; + const FLUSH_INTERVAL = 5000; + const MAX_BUFFER_SIZE = 50; + + function sendLogs() { + if (LOG_BUFFER.length === 0) return; + + const logs = LOG_BUFFER.splice(0, LOG_BUFFER.length); + + fetch(LOG_ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ logs: logs }) + }).catch(function(e) { + console.warn('Failed to send logs:', e); + }); + } + + function addLog(level, message, details) { + const entry = { + app_name: APP_NAME, + level: level, + message: message, + details: details || null, + file_path: null, + line_number: null, + stack_trace: null, + user_agent: navigator.userAgent + }; + + LOG_BUFFER.push(entry); + + if (LOG_BUFFER.length >= MAX_BUFFER_SIZE) { + sendLogs(); + } + } + + window.onerror = function(message, source, lineno, colno, error) { + addLog('error', message, JSON.stringify({ + source: source, + line: lineno, + column: colno, + stack: error ? error.stack : null + })); + return false; + }; + + window.onunhandledrejection = function(event) { + addLog('error', 'Unhandled Promise Rejection: ' + event.reason, + event.reason && event.reason.stack ? event.reason.stack : null); + }; + + const originalConsoleError = console.error; + console.error = function() { + addLog('error', Array.from(arguments).join(' ')); + originalConsoleError.apply(console, arguments); + }; + + const originalConsoleWarn = console.warn; + console.warn = function() { + addLog('warn', Array.from(arguments).join(' ')); + originalConsoleWarn.apply(console, arguments); + }; + + document.body.addEventListener('htmx:responseError', function(evt) { + addLog('error', 'HTMX Request Failed', JSON.stringify({ + url: evt.detail.xhr.responseURL, + status: evt.detail.xhr.status, + response: evt.detail.xhr.responseText.substring(0, 500) + })); + }); + + document.body.addEventListener('htmx:sendError', function(evt) { + addLog('error', 'HTMX Send Error', JSON.stringify({ + url: evt.detail.requestConfig.path + })); + }); + + setInterval(sendLogs, FLUSH_INTERVAL); + window.addEventListener('beforeunload', sendLogs); + + window.AppLogger = { + debug: function(msg, details) { addLog('debug', msg, details); }, + info: function(msg, details) { addLog('info', msg, details); }, + warn: function(msg, details) { addLog('warn', msg, details); }, + error: function(msg, details) { addLog('error', msg, details); }, + flush: sendLogs + }; + + console.log('[AppLogger] Initialized for app:', APP_NAME); +})(); +" +} diff --git a/src/auto_task/autotask_api.rs b/src/auto_task/autotask_api.rs index d42021402..d7a102a26 100644 --- a/src/auto_task/autotask_api.rs +++ b/src/auto_task/autotask_api.rs @@ -57,13 +57,11 @@ pub struct IntentResultResponse { pub next_steps: Vec, } -/// Request for one-click create and execute #[derive(Debug, Deserialize)] pub struct CreateAndExecuteRequest { pub intent: String, } -/// Response for create and execute - simple status updates #[derive(Debug, Serialize)] pub struct CreateAndExecuteResponse { pub success: bool, @@ -289,8 +287,6 @@ pub struct RecommendationResponse { pub action: Option, } -/// Create and execute in one call - no dialogs, just do it -/// POST /api/autotask/create pub async fn create_and_execute_handler( State(state): State>, Json(request): Json, @@ -388,8 +384,6 @@ pub async fn create_and_execute_handler( } } -/// Classify and optionally process an intent -/// POST /api/autotask/classify pub async fn classify_intent_handler( State(state): State>, Json(request): Json, @@ -1555,7 +1549,6 @@ fn update_task_status( Ok(()) } -/// Create task record in database fn create_task_record( state: &Arc, task_id: Uuid, @@ -1584,7 +1577,6 @@ fn create_task_record( Ok(()) } -/// Update task status in database fn update_task_status_db( state: &Arc, task_id: Uuid, @@ -1620,7 +1612,6 @@ fn update_task_status_db( Ok(()) } -/// Get pending items (ASK LATER) for a bot fn get_pending_items_for_bot(state: &Arc, bot_id: Uuid) -> Vec { let mut conn = match state.conn.get() { Ok(c) => c, @@ -1753,8 +1744,6 @@ fn html_escape(s: &str) -> String { // MISSING ENDPOINTS - Required by botui/autotask.js // ============================================================================= -/// Execute a specific task by ID -/// POST /api/autotask/:task_id/execute pub async fn execute_task_handler( State(state): State>, Path(task_id): Path, @@ -1785,8 +1774,6 @@ pub async fn execute_task_handler( } } -/// Get execution logs for a task -/// GET /api/autotask/:task_id/logs pub async fn get_task_logs_handler( State(state): State>, Path(task_id): Path, @@ -1805,8 +1792,6 @@ pub async fn get_task_logs_handler( .into_response() } -/// Apply a recommendation from simulation results -/// POST /api/autotask/recommendations/:rec_id/apply pub async fn apply_recommendation_handler( State(state): State>, Path(rec_id): Path, diff --git a/src/auto_task/designer_ai.rs b/src/auto_task/designer_ai.rs index 879093fae..7dcf5b0d7 100644 --- a/src/auto_task/designer_ai.rs +++ b/src/auto_task/designer_ai.rs @@ -1,21 +1,3 @@ -//! Designer AI Assistant -//! -//! An AI-powered assistant that modifies applications through natural conversation. -//! Based on Chapter 17 - Designer documentation. -//! -//! Designer understands context: -//! - Current app being viewed -//! - Current page/file active -//! - Available tables and their schemas -//! - Existing tools and schedulers -//! -//! Designer can modify: -//! - Styles (colors, layout, fonts) -//! - HTML pages (forms, lists, buttons) -//! - Database (add fields, create tables) -//! - Tools (voice/chat commands) -//! - Schedulers (automated tasks) - use crate::shared::models::UserSession; use crate::shared::state::AppState; use chrono::{DateTime, Utc}; @@ -28,23 +10,15 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use uuid::Uuid; -/// Types of modifications Designer can make #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum ModificationType { - /// Modify CSS styles Style, - /// Modify HTML structure Html, - /// Add/modify database fields or tables Database, - /// Create/modify voice/chat commands Tool, - /// Create/modify scheduled automations Scheduler, - /// Multiple modifications Multiple, - /// Unknown modification type Unknown, } @@ -62,28 +36,18 @@ impl std::fmt::Display for ModificationType { } } -/// Context about what the user is currently viewing/editing #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct DesignerContext { - /// Current app name pub current_app: Option, - /// Current page/file being viewed pub current_page: Option, - /// Current element selected (if any) pub current_element: Option, - /// Available tables in this bot's database pub available_tables: Vec, - /// Available tools pub available_tools: Vec, - /// Available schedulers pub available_schedulers: Vec, - /// Recent changes for undo support pub recent_changes: Vec, - /// Conversation history for context pub conversation_history: Vec, } -/// Summary info about a table #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TableInfo { pub name: String, @@ -91,7 +55,6 @@ pub struct TableInfo { pub record_count: Option, } -/// Record of a change for undo support #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChangeRecord { pub id: String, @@ -104,7 +67,6 @@ pub struct ChangeRecord { pub can_undo: bool, } -/// A turn in the conversation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ConversationTurn { pub role: String, // "user" or "assistant" @@ -112,14 +74,12 @@ pub struct ConversationTurn { pub timestamp: DateTime, } -/// Request to modify something #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ModificationRequest { pub instruction: String, pub context: DesignerContext, } -/// Result of a modification #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ModificationResult { pub success: bool, @@ -134,7 +94,6 @@ pub struct ModificationResult { pub error: Option, } -/// A single file change #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FileChange { pub file_path: String, @@ -144,7 +103,6 @@ pub struct FileChange { pub line_number: Option, } -/// Analyzed modification from LLM #[derive(Debug, Clone, Serialize, Deserialize)] struct AnalyzedModification { modification_type: ModificationType, @@ -163,7 +121,6 @@ struct CodeChange { context: Option, } -/// The Designer AI Assistant pub struct DesignerAI { state: Arc, } @@ -173,7 +130,6 @@ impl DesignerAI { Self { state } } - /// Process a modification request pub async fn process_request( &self, request: &ModificationRequest, @@ -221,7 +177,6 @@ impl DesignerAI { self.apply_modification(&analysis, session).await } - /// Apply a confirmed modification pub async fn apply_confirmed_modification( &self, change_id: &str, @@ -247,7 +202,6 @@ impl DesignerAI { } } - /// Undo a previous change pub async fn undo_change( &self, change_id: &str, @@ -311,7 +265,6 @@ impl DesignerAI { } } - /// Get current context for the designer pub async fn get_context( &self, session: &UserSession, @@ -335,11 +288,6 @@ impl DesignerAI { }) } - // ========================================================================= - // ANALYSIS - // ========================================================================= - - /// Analyze what modification the user wants async fn analyze_modification( &self, instruction: &str, @@ -390,7 +338,6 @@ Respond ONLY with valid JSON."# self.parse_analysis_response(&response, instruction) } - /// Parse LLM analysis response fn parse_analysis_response( &self, response: &str, @@ -452,7 +399,6 @@ Respond ONLY with valid JSON."# } } - /// Fallback heuristic analysis fn analyze_modification_heuristic( &self, instruction: &str, @@ -515,11 +461,6 @@ Respond ONLY with valid JSON."# }) } - // ========================================================================= - // MODIFICATION APPLICATION - // ========================================================================= - - /// Apply analyzed modification async fn apply_modification( &self, analysis: &AnalyzedModification, @@ -611,7 +552,6 @@ Respond ONLY with valid JSON."# }) } - /// Apply CSS style changes async fn apply_style_changes( &self, original: &str, @@ -649,7 +589,6 @@ Respond ONLY with valid JSON."# Ok(content) } - /// Apply HTML changes async fn apply_html_changes( &self, original: &str, @@ -688,7 +627,6 @@ Respond ONLY with valid JSON."# Ok(content) } - /// Apply database schema changes async fn apply_database_changes( &self, original: &str, @@ -728,7 +666,6 @@ Respond ONLY with valid JSON."# Ok(content) } - /// Generate a tool file async fn generate_tool_file( &self, changes: &[CodeChange], @@ -750,7 +687,6 @@ Respond ONLY with valid JSON."# Ok(content) } - /// Generate a scheduler file async fn generate_scheduler_file( &self, changes: &[CodeChange], @@ -772,7 +708,6 @@ Respond ONLY with valid JSON."# Ok(content) } - /// Handle multiple changes async fn apply_multiple_changes( &self, _analysis: &AnalyzedModification, @@ -783,7 +718,6 @@ Respond ONLY with valid JSON."# Ok("Multiple changes applied".to_string()) } - /// Generate preview of changes fn generate_preview(&self, analysis: &AnalyzedModification) -> String { let mut preview = String::new(); preview.push_str(&format!("File: {}\n\nChanges:\n", analysis.target_file)); @@ -806,11 +740,6 @@ Respond ONLY with valid JSON."# preview } - // ========================================================================= - // CONTEXT HELPERS - // ========================================================================= - - /// Get available tables for the bot fn get_available_tables( &self, _session: &UserSession, @@ -843,7 +772,6 @@ Respond ONLY with valid JSON."# .collect()) } - /// Get available tools fn get_available_tools( &self, session: &UserSession, @@ -865,7 +793,6 @@ Respond ONLY with valid JSON."# Ok(tools) } - /// Get available schedulers fn get_available_schedulers( &self, session: &UserSession, @@ -887,7 +814,6 @@ Respond ONLY with valid JSON."# Ok(schedulers) } - /// Get recent changes for undo fn get_recent_changes( &self, session: &UserSession, @@ -947,11 +873,6 @@ Respond ONLY with valid JSON."# .collect()) } - // ========================================================================= - // FILE OPERATIONS - // ========================================================================= - - /// Get site path from config fn get_site_path(&self) -> String { self.state .config @@ -960,7 +881,6 @@ Respond ONLY with valid JSON."# .unwrap_or_else(|| "./botserver-stack/sites".to_string()) } - /// Read a file from the bot's directory fn read_file( &self, bot_id: Uuid, @@ -978,7 +898,6 @@ Respond ONLY with valid JSON."# } } - /// Write a file to the bot's directory fn write_file( &self, bot_id: Uuid, @@ -1001,7 +920,6 @@ Respond ONLY with valid JSON."# Ok(()) } - /// Sync schema changes to database fn sync_schema_changes( &self, _session: &UserSession, @@ -1012,11 +930,6 @@ Respond ONLY with valid JSON."# Ok(()) } - // ========================================================================= - // CHANGE RECORD MANAGEMENT - // ========================================================================= - - /// Store a change record for undo fn store_change_record( &self, record: &ChangeRecord, @@ -1043,7 +956,6 @@ Respond ONLY with valid JSON."# Ok(()) } - /// Get a change record by ID fn get_change_record( &self, change_id: &str, @@ -1098,7 +1010,6 @@ Respond ONLY with valid JSON."# })) } - /// Remove a change record (after undo) fn remove_change_record( &self, change_id: &str, @@ -1114,7 +1025,6 @@ Respond ONLY with valid JSON."# Ok(()) } - /// Get pending change (for confirmation flow) fn get_pending_change( &self, change_id: &str, @@ -1146,11 +1056,6 @@ Respond ONLY with valid JSON."# } } - // ========================================================================= - // LLM INTEGRATION - // ========================================================================= - - /// Call LLM for analysis async fn call_llm( &self, prompt: &str, diff --git a/src/auto_task/intent_classifier.rs b/src/auto_task/intent_classifier.rs index e31aed792..909411c37 100644 --- a/src/auto_task/intent_classifier.rs +++ b/src/auto_task/intent_classifier.rs @@ -1,17 +1,3 @@ -//! Intent Classifier for AutoTask System -//! -//! Classifies user intents and routes them to appropriate handlers. -//! Based on Chapter 17 - Autonomous Tasks documentation. -//! -//! Intent Types: -//! - APP_CREATE: "create app for clinic" → HTMX pages, tools, schedulers -//! - TODO: "call John tomorrow" → Task saved to tasks table -//! - MONITOR: "alert when IBM changes" → ON CHANGE event handler -//! - ACTION: "email all customers" → Executes immediately -//! - SCHEDULE: "daily 9am summary" → SET SCHEDULE automation -//! - GOAL: "increase sales 20%" → Autonomous LLM loop with metrics -//! - TOOL: "when I say X, do Y" → Voice/chat command - use crate::auto_task::app_generator::AppGenerator; use crate::auto_task::intent_compiler::IntentCompiler; use crate::shared::models::UserSession; @@ -25,25 +11,16 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use uuid::Uuid; -/// The seven intent types supported by the AutoTask system #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum IntentType { - /// Create a full application with HTMX pages, tables, tools, schedulers AppCreate, - /// Simple task/reminder saved to tasks table Todo, - /// Monitor for changes with ON CHANGE event handler Monitor, - /// Execute an action immediately Action, - /// Create a scheduled automation with SET SCHEDULE Schedule, - /// Long-running goal with autonomous LLM loop Goal, - /// Create a voice/chat command trigger Tool, - /// Unknown or ambiguous intent requiring clarification Unknown, } @@ -77,7 +54,6 @@ impl From<&str> for IntentType { } } -/// Result of intent classification #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ClassifiedIntent { pub id: String, @@ -92,32 +68,20 @@ pub struct ClassifiedIntent { pub classified_at: DateTime, } -/// Extracted entities from the intent #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ClassifiedEntities { - /// Main subject (e.g., "clinic", "customers", "IBM stock") pub subject: Option, - /// Target action verb pub action: Option, - /// Domain/industry context pub domain: Option, - /// Time-related information pub time_spec: Option, - /// Condition for triggers pub condition: Option, - /// Recipient for notifications pub recipient: Option, - /// List of features requested pub features: Vec, - /// Tables/entities mentioned pub tables: Vec, - /// Specific trigger phrases for TOOL type pub trigger_phrases: Vec, - /// Metric/goal value for GOAL type pub target_value: Option, } -/// Time specification for scheduled tasks #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TimeSpec { pub schedule_type: ScheduleType, @@ -137,7 +101,6 @@ pub enum ScheduleType { Cron, } -/// Alternative classification with lower confidence #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AlternativeClassification { pub intent_type: IntentType, @@ -145,7 +108,6 @@ pub struct AlternativeClassification { pub reason: String, } -/// Result of processing an intent through the appropriate handler #[derive(Debug, Clone, Serialize, Deserialize)] pub struct IntentResult { pub success: bool, @@ -167,7 +129,6 @@ pub struct CreatedResource { pub path: Option, } -/// Main intent classifier and router pub struct IntentClassifier { state: Arc, intent_compiler: IntentCompiler, @@ -528,20 +489,18 @@ Respond with JSON only: }); } - // Track created pages for page in &app.pages { resources.push(CreatedResource { resource_type: "page".to_string(), - name: page.title.clone(), + name: page.filename.clone(), path: Some(page.filename.clone()), }); } - // Track created tools for tool in &app.tools { resources.push(CreatedResource { resource_type: "tool".to_string(), - name: tool.name.clone(), + name: tool.filename.clone(), path: Some(tool.filename.clone()), }); } diff --git a/src/auto_task/mod.rs b/src/auto_task/mod.rs index af298687a..11c6d4849 100644 --- a/src/auto_task/mod.rs +++ b/src/auto_task/mod.rs @@ -1,4 +1,5 @@ pub mod app_generator; +pub mod app_logs; pub mod ask_later; pub mod auto_task; pub mod autotask_api; @@ -8,9 +9,14 @@ pub mod intent_compiler; pub mod safety_layer; pub use app_generator::{ - AppGenerator, AppStructure, GeneratedApp, GeneratedPage, GeneratedScript, PageType, ScriptType, + AppGenerator, AppStructure, FileType, GeneratedApp, GeneratedFile, GeneratedPage, PageType, SyncResult, }; +pub use app_logs::{ + generate_client_logger_js, get_designer_error_context, log_generator_error, log_generator_info, + log_runtime_error, log_validation_error, start_log_cleanup_scheduler, AppLogEntry, AppLogStore, + ClientLogRequest, LogLevel, LogQueryParams, LogSource, LogStats, APP_LOGS, +}; pub use ask_later::{ask_later_keyword, PendingInfoItem}; pub use auto_task::{AutoTask, AutoTaskStatus, ExecutionMode, TaskPriority}; pub use autotask_api::{ @@ -26,52 +32,116 @@ pub use intent_classifier::{ClassifiedIntent, IntentClassifier, IntentType}; pub use intent_compiler::{CompiledIntent, IntentCompiler}; pub use safety_layer::{AuditEntry, ConstraintCheckResult, SafetyLayer, SimulationResult}; +use crate::core::urls::ApiUrls; + pub fn configure_autotask_routes() -> axum::Router> { use axum::routing::{get, post}; axum::Router::new() - .route("/api/autotask/create", post(create_and_execute_handler)) - .route("/api/autotask/classify", post(classify_intent_handler)) - .route("/api/autotask/compile", post(compile_intent_handler)) - .route("/api/autotask/execute", post(execute_plan_handler)) + .route(ApiUrls::AUTOTASK_CREATE, post(create_and_execute_handler)) + .route(ApiUrls::AUTOTASK_CLASSIFY, post(classify_intent_handler)) + .route(ApiUrls::AUTOTASK_COMPILE, post(compile_intent_handler)) + .route(ApiUrls::AUTOTASK_EXECUTE, post(execute_plan_handler)) .route( - "/api/autotask/simulate/:plan_id", + &ApiUrls::AUTOTASK_SIMULATE.replace(":plan_id", "{plan_id}"), post(simulate_plan_handler), ) - .route("/api/autotask/list", get(list_tasks_handler)) - .route("/api/autotask/stats", get(get_stats_handler)) - .route("/api/autotask/:task_id/pause", post(pause_task_handler)) - .route("/api/autotask/:task_id/resume", post(resume_task_handler)) - .route("/api/autotask/:task_id/cancel", post(cancel_task_handler)) + .route(ApiUrls::AUTOTASK_LIST, get(list_tasks_handler)) + .route(ApiUrls::AUTOTASK_STATS, get(get_stats_handler)) .route( - "/api/autotask/:task_id/simulate", + &ApiUrls::AUTOTASK_PAUSE.replace(":task_id", "{task_id}"), + post(pause_task_handler), + ) + .route( + &ApiUrls::AUTOTASK_RESUME.replace(":task_id", "{task_id}"), + post(resume_task_handler), + ) + .route( + &ApiUrls::AUTOTASK_CANCEL.replace(":task_id", "{task_id}"), + post(cancel_task_handler), + ) + .route( + &ApiUrls::AUTOTASK_TASK_SIMULATE.replace(":task_id", "{task_id}"), post(simulate_task_handler), ) .route( - "/api/autotask/:task_id/decisions", + &ApiUrls::AUTOTASK_DECISIONS.replace(":task_id", "{task_id}"), get(get_decisions_handler), ) .route( - "/api/autotask/:task_id/decide", + &ApiUrls::AUTOTASK_DECIDE.replace(":task_id", "{task_id}"), post(submit_decision_handler), ) .route( - "/api/autotask/:task_id/approvals", + &ApiUrls::AUTOTASK_APPROVALS.replace(":task_id", "{task_id}"), get(get_approvals_handler), ) .route( - "/api/autotask/:task_id/approve", + &ApiUrls::AUTOTASK_APPROVE.replace(":task_id", "{task_id}"), post(submit_approval_handler), ) - .route("/api/autotask/:task_id/execute", post(execute_task_handler)) - .route("/api/autotask/:task_id/logs", get(get_task_logs_handler)) .route( - "/api/autotask/recommendations/:rec_id/apply", + &ApiUrls::AUTOTASK_TASK_EXECUTE.replace(":task_id", "{task_id}"), + post(execute_task_handler), + ) + .route( + &ApiUrls::AUTOTASK_LOGS.replace(":task_id", "{task_id}"), + get(get_task_logs_handler), + ) + .route( + &ApiUrls::AUTOTASK_RECOMMENDATIONS_APPLY.replace(":rec_id", "{rec_id}"), post(apply_recommendation_handler), ) - .route("/api/autotask/pending", get(get_pending_items_handler)) + .route(ApiUrls::AUTOTASK_PENDING, get(get_pending_items_handler)) .route( - "/api/autotask/pending/:item_id", + &ApiUrls::AUTOTASK_PENDING_ITEM.replace(":item_id", "{item_id}"), post(submit_pending_item_handler), ) + .route("/api/app-logs/client", post(handle_client_logs)) + .route("/api/app-logs/list", get(handle_list_logs)) + .route("/api/app-logs/stats", get(handle_log_stats)) + .route("/api/app-logs/clear/{app_name}", post(handle_clear_logs)) + .route("/api/app-logs/logger.js", get(handle_logger_js)) +} + +async fn handle_client_logs( + axum::Json(payload): axum::Json, +) -> impl axum::response::IntoResponse { + for log in payload.logs { + APP_LOGS.log_client(log, None, None); + } + axum::Json(serde_json::json!({"success": true})) +} + +#[derive(serde::Deserialize)] +struct ClientLogsPayload { + logs: Vec, +} + +async fn handle_list_logs( + axum::extract::Query(params): axum::extract::Query, +) -> impl axum::response::IntoResponse { + let logs = APP_LOGS.get_logs(¶ms); + axum::Json(logs) +} + +async fn handle_log_stats() -> impl axum::response::IntoResponse { + let stats = APP_LOGS.get_stats(); + axum::Json(stats) +} + +async fn handle_clear_logs( + axum::extract::Path(app_name): axum::extract::Path, +) -> impl axum::response::IntoResponse { + APP_LOGS.clear_app_logs(&app_name); + axum::Json( + serde_json::json!({"success": true, "message": format!("Logs cleared for {}", app_name)}), + ) +} + +async fn handle_logger_js() -> impl axum::response::IntoResponse { + ( + [(axum::http::header::CONTENT_TYPE, "application/javascript")], + generate_client_logger_js(), + ) } diff --git a/src/basic/keywords/app_server.rs b/src/basic/keywords/app_server.rs index 62d7f435b..65d756517 100644 --- a/src/basic/keywords/app_server.rs +++ b/src/basic/keywords/app_server.rs @@ -1,18 +1,3 @@ -//! App Server Module -//! -//! Serves generated HTMX applications with clean URLs. -//! Apps are synced from drive to SITE_ROOT/{app_name}/ for serving. -//! -//! URL structure: -//! - `/apps/{app_name}/` -> {site_path}/{app_name}/index.html -//! - `/apps/{app_name}/patients.html` -> {site_path}/{app_name}/patients.html -//! - `/apps/{app_name}/styles.css` -> {site_path}/{app_name}/styles.css -//! -//! Flow: -//! 1. AppGenerator writes to S3 drive: {bucket}/.gbdrive/apps/{app_name}/ -//! 2. sync_app_to_site_root() copies to: {site_path}/{app_name}/ -//! 3. This module serves from: {site_path}/{app_name}/ - use crate::shared::state::AppState; use axum::{ body::Body, @@ -25,7 +10,6 @@ use axum::{ use log::{error, trace, warn}; use std::sync::Arc; -/// Configure routes for serving generated apps pub fn configure_app_server_routes() -> Router> { Router::new() // Serve app files: /apps/{app_name}/* (clean URLs) @@ -36,7 +20,6 @@ pub fn configure_app_server_routes() -> Router> { .route("/apps", get(list_all_apps)) } -/// Path parameters for app serving #[derive(Debug, serde::Deserialize)] pub struct AppPath { pub app_name: String, @@ -48,7 +31,6 @@ pub struct AppFilePath { pub file_path: String, } -/// Serve the index.html for an app pub async fn serve_app_index( State(state): State>, Path(params): Path, @@ -56,7 +38,6 @@ pub async fn serve_app_index( serve_app_file_internal(&state, ¶ms.app_name, "index.html").await } -/// Serve any file from an app directory pub async fn serve_app_file( State(state): State>, Path(params): Path, @@ -64,7 +45,6 @@ pub async fn serve_app_file( serve_app_file_internal(&state, ¶ms.app_name, ¶ms.file_path).await } -/// Internal function to serve files from app directory async fn serve_app_file_internal(state: &AppState, app_name: &str, file_path: &str) -> Response { // Sanitize paths to prevent directory traversal let sanitized_app_name = sanitize_path_component(app_name); @@ -120,7 +100,6 @@ async fn serve_app_file_internal(state: &AppState, app_name: &str, file_path: &s } } -/// List all available apps from SITE_ROOT pub async fn list_all_apps(State(state): State>) -> impl IntoResponse { let site_path = state .config @@ -165,7 +144,6 @@ pub async fn list_all_apps(State(state): State>) -> impl IntoRespo .into_response() } -/// Sanitize path component to prevent directory traversal fn sanitize_path_component(component: &str) -> String { component .replace("..", "") @@ -177,7 +155,6 @@ fn sanitize_path_component(component: &str) -> String { .collect() } -/// Get content type based on file extension fn get_content_type(file_path: &str) -> &'static str { let ext = file_path.rsplit('.').next().unwrap_or("").to_lowercase(); diff --git a/src/basic/keywords/data_operations.rs b/src/basic/keywords/data_operations.rs index 19f2b76c1..e2db98aca 100644 --- a/src/basic/keywords/data_operations.rs +++ b/src/basic/keywords/data_operations.rs @@ -1,10 +1,11 @@ +use super::table_access::{check_table_access, AccessType, UserRoles}; use crate::shared::models::UserSession; use crate::shared::state::AppState; use crate::shared::utils::{json_value_to_dynamic, to_array}; use diesel::prelude::*; use diesel::sql_query; use diesel::sql_types::Text; -use log::{error, trace}; +use log::{error, trace, warn}; use rhai::{Array, Dynamic, Engine, Map}; use serde_json::{json, Value}; use std::collections::HashMap; @@ -26,8 +27,9 @@ pub fn register_data_operations(state: Arc, user: UserSession, engine: register_group_by_keyword(state, user, engine); } -pub fn register_save_keyword(state: Arc, _user: UserSession, engine: &mut Engine) { +pub fn register_save_keyword(state: Arc, user: UserSession, engine: &mut Engine) { let state_clone = Arc::clone(&state); + let user_roles = UserRoles::from_user_session(&user); engine .register_custom_syntax( @@ -45,6 +47,14 @@ pub fn register_save_keyword(state: Arc, _user: UserSession, engine: & .get() .map_err(|e| format!("DB error: {}", e))?; + // Check write access + if let Err(e) = + check_table_access(&mut conn, &table, &user_roles, AccessType::Write) + { + warn!("SAVE access denied: {}", e); + return Err(e.into()); + } + let result = execute_save(&mut conn, &table, &id, &data) .map_err(|e| format!("SAVE error: {}", e))?; @@ -54,8 +64,9 @@ pub fn register_save_keyword(state: Arc, _user: UserSession, engine: & .unwrap(); } -pub fn register_insert_keyword(state: Arc, _user: UserSession, engine: &mut Engine) { +pub fn register_insert_keyword(state: Arc, user: UserSession, engine: &mut Engine) { let state_clone = Arc::clone(&state); + let user_roles = UserRoles::from_user_session(&user); engine .register_custom_syntax( @@ -72,6 +83,14 @@ pub fn register_insert_keyword(state: Arc, _user: UserSession, engine: .get() .map_err(|e| format!("DB error: {}", e))?; + // Check write access + if let Err(e) = + check_table_access(&mut conn, &table, &user_roles, AccessType::Write) + { + warn!("INSERT access denied: {}", e); + return Err(e.into()); + } + let result = execute_insert(&mut conn, &table, &data) .map_err(|e| format!("INSERT error: {}", e))?; @@ -81,8 +100,9 @@ pub fn register_insert_keyword(state: Arc, _user: UserSession, engine: .unwrap(); } -pub fn register_update_keyword(state: Arc, _user: UserSession, engine: &mut Engine) { +pub fn register_update_keyword(state: Arc, user: UserSession, engine: &mut Engine) { let state_clone = Arc::clone(&state); + let user_roles = UserRoles::from_user_session(&user); engine .register_custom_syntax( @@ -100,6 +120,14 @@ pub fn register_update_keyword(state: Arc, _user: UserSession, engine: .get() .map_err(|e| format!("DB error: {}", e))?; + // Check write access + if let Err(e) = + check_table_access(&mut conn, &table, &user_roles, AccessType::Write) + { + warn!("UPDATE access denied: {}", e); + return Err(e.into()); + } + let result = execute_update(&mut conn, &table, &filter, &data) .map_err(|e| format!("UPDATE error: {}", e))?; @@ -109,8 +137,9 @@ pub fn register_update_keyword(state: Arc, _user: UserSession, engine: .unwrap(); } -pub fn register_delete_keyword(state: Arc, _user: UserSession, engine: &mut Engine) { +pub fn register_delete_keyword(state: Arc, user: UserSession, engine: &mut Engine) { let state_clone = Arc::clone(&state); + let user_roles = UserRoles::from_user_session(&user); engine .register_custom_syntax( @@ -170,6 +199,14 @@ pub fn register_delete_keyword(state: Arc, _user: UserSession, engine: .get() .map_err(|e| format!("DB error: {}", e))?; + // Check write access (delete requires write permission) + if let Err(e) = + check_table_access(&mut conn, &first_arg, &user_roles, AccessType::Write) + { + warn!("DELETE access denied: {}", e); + return Err(e.into()); + } + let result = execute_delete(&mut conn, &first_arg, &second_arg) .map_err(|e| format!("DELETE error: {}", e))?; diff --git a/src/basic/keywords/db_api.rs b/src/basic/keywords/db_api.rs index 6455ad3f2..310551069 100644 --- a/src/basic/keywords/db_api.rs +++ b/src/basic/keywords/db_api.rs @@ -1,19 +1,47 @@ +use super::table_access::{ + check_field_write_access, check_table_access, filter_fields_by_role, AccessType, UserRoles, +}; use crate::core::shared::state::AppState; +use crate::core::urls::ApiUrls; use axum::{ extract::{Path, Query, State}, - http::StatusCode, + http::{HeaderMap, StatusCode}, response::IntoResponse, routing::{delete, get, post, put}, Json, Router, }; use diesel::prelude::*; use diesel::sql_query; -use log::{error, info}; +use log::{error, info, warn}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::sync::Arc; use uuid::Uuid; +fn user_roles_from_headers(headers: &HeaderMap) -> UserRoles { + let roles = headers + .get("X-User-Roles") + .and_then(|v| v.to_str().ok()) + .map(|s| { + s.split(';') + .map(|r| r.trim().to_string()) + .filter(|r| !r.is_empty()) + .collect() + }) + .unwrap_or_default(); + + let user_id = headers + .get("X-User-Id") + .and_then(|v| v.to_str().ok()) + .and_then(|s| Uuid::parse_str(s).ok()); + + if let Some(uid) = user_id { + UserRoles::with_user_id(roles, uid) + } else { + UserRoles::new(roles) + } +} + #[derive(Debug, Deserialize)] pub struct QueryParams { pub limit: Option, @@ -46,13 +74,40 @@ pub struct DeleteResponse { pub fn configure_db_routes() -> Router> { Router::new() - .route("/api/db/{table}", get(list_records_handler)) - .route("/api/db/{table}", post(create_record_handler)) - .route("/api/db/{table}/{id}", get(get_record_handler)) - .route("/api/db/{table}/{id}", put(update_record_handler)) - .route("/api/db/{table}/{id}", delete(delete_record_handler)) - .route("/api/db/{table}/count", get(count_records_handler)) - .route("/api/db/{table}/search", post(search_records_handler)) + .route( + &ApiUrls::DB_TABLE.replace(":table", "{table}"), + get(list_records_handler), + ) + .route( + &ApiUrls::DB_TABLE.replace(":table", "{table}"), + post(create_record_handler), + ) + .route( + &ApiUrls::DB_TABLE_RECORD + .replace(":table", "{table}") + .replace(":id", "{id}"), + get(get_record_handler), + ) + .route( + &ApiUrls::DB_TABLE_RECORD + .replace(":table", "{table}") + .replace(":id", "{id}"), + put(update_record_handler), + ) + .route( + &ApiUrls::DB_TABLE_RECORD + .replace(":table", "{table}") + .replace(":id", "{id}"), + delete(delete_record_handler), + ) + .route( + &ApiUrls::DB_TABLE_COUNT.replace(":table", "{table}"), + get(count_records_handler), + ) + .route( + &ApiUrls::DB_TABLE_SEARCH.replace(":table", "{table}"), + post(search_records_handler), + ) } fn sanitize_identifier(name: &str) -> String { @@ -63,10 +118,12 @@ fn sanitize_identifier(name: &str) -> String { pub async fn list_records_handler( State(state): State>, + headers: HeaderMap, Path(table): Path, Query(params): Query, ) -> impl IntoResponse { let table_name = sanitize_identifier(&table); + let user_roles = user_roles_from_headers(&headers); let limit = params.limit.unwrap_or(20).min(100); let offset = params.offset.unwrap_or(0); let order_by = params @@ -95,6 +152,19 @@ pub async fn list_records_handler( } }; + // Check table-level read access + let access_info = + match check_table_access(&mut conn, &table_name, &user_roles, AccessType::Read) { + Ok(info) => info, + Err(e) => { + warn!( + "Access denied to table {} for user {:?}", + table_name, user_roles.user_id + ); + return (StatusCode::FORBIDDEN, Json(json!({ "error": e }))).into_response(); + } + }; + let query = format!( "SELECT row_to_json(t.*) as data FROM {} t ORDER BY {} {} LIMIT {} OFFSET {}", table_name, order_by, order_dir, limit, offset @@ -107,8 +177,14 @@ pub async fn list_records_handler( match (rows, total) { (Ok(data), Ok(count_result)) => { + // Filter fields based on user roles + let filtered_data: Vec = data + .into_iter() + .map(|r| filter_fields_by_role(r.data, &user_roles, &access_info)) + .collect(); + let response = ListResponse { - data: data.into_iter().map(|r| r.data).collect(), + data: filtered_data, total: count_result.count, limit, offset, @@ -128,9 +204,11 @@ pub async fn list_records_handler( pub async fn get_record_handler( State(state): State>, + headers: HeaderMap, Path((table, id)): Path<(String, String)>, ) -> impl IntoResponse { let table_name = sanitize_identifier(&table); + let user_roles = user_roles_from_headers(&headers); let record_id = match Uuid::parse_str(&id) { Ok(uuid) => uuid, @@ -162,6 +240,23 @@ pub async fn get_record_handler( } }; + // Check table-level read access + let access_info = + match check_table_access(&mut conn, &table_name, &user_roles, AccessType::Read) { + Ok(info) => info, + Err(e) => { + return ( + StatusCode::FORBIDDEN, + Json(RecordResponse { + success: false, + data: None, + message: Some(e), + }), + ) + .into_response(); + } + }; + let query = format!( "SELECT row_to_json(t.*) as data FROM {} t WHERE id = $1", table_name @@ -173,15 +268,19 @@ pub async fn get_record_handler( .optional(); match row { - Ok(Some(r)) => ( - StatusCode::OK, - Json(RecordResponse { - success: true, - data: Some(r.data), - message: None, - }), - ) - .into_response(), + Ok(Some(r)) => { + // Filter fields based on user roles + let filtered_data = filter_fields_by_role(r.data, &user_roles, &access_info); + ( + StatusCode::OK, + Json(RecordResponse { + success: true, + data: Some(filtered_data), + message: None, + }), + ) + .into_response() + } Ok(None) => ( StatusCode::NOT_FOUND, Json(RecordResponse { @@ -208,10 +307,12 @@ pub async fn get_record_handler( pub async fn create_record_handler( State(state): State>, + headers: HeaderMap, Path(table): Path, Json(payload): Json, ) -> impl IntoResponse { let table_name = sanitize_identifier(&table); + let user_roles = user_roles_from_headers(&headers); let obj = match payload.as_object() { Some(o) => o, @@ -255,6 +356,41 @@ pub async fn create_record_handler( } }; + // Check table-level write access + let access_info = + match check_table_access(&mut conn, &table_name, &user_roles, AccessType::Write) { + Ok(info) => info, + Err(e) => { + return ( + StatusCode::FORBIDDEN, + Json(RecordResponse { + success: false, + data: None, + message: Some(e), + }), + ) + .into_response(); + } + }; + + // Check field-level write access for fields being inserted + let field_names: Vec = obj + .keys() + .map(|k| sanitize_identifier(k)) + .filter(|k| !k.is_empty() && k != "id") + .collect(); + if let Err(e) = check_field_write_access(&field_names, &user_roles, &access_info) { + return ( + StatusCode::FORBIDDEN, + Json(RecordResponse { + success: false, + data: None, + message: Some(e), + }), + ) + .into_response(); + } + let query = format!( "INSERT INTO {} ({}) VALUES ({}) RETURNING row_to_json({}.*)::jsonb as data", table_name, @@ -295,10 +431,12 @@ pub async fn create_record_handler( pub async fn update_record_handler( State(state): State>, + headers: HeaderMap, Path((table, id)): Path<(String, String)>, Json(payload): Json, ) -> impl IntoResponse { let table_name = sanitize_identifier(&table); + let user_roles = user_roles_from_headers(&headers); let record_id = match Uuid::parse_str(&id) { Ok(uuid) => uuid, @@ -369,6 +507,41 @@ pub async fn update_record_handler( } }; + // Check table-level write access + let access_info = + match check_table_access(&mut conn, &table_name, &user_roles, AccessType::Write) { + Ok(info) => info, + Err(e) => { + return ( + StatusCode::FORBIDDEN, + Json(RecordResponse { + success: false, + data: None, + message: Some(e), + }), + ) + .into_response(); + } + }; + + // Check field-level write access for fields being updated + let field_names: Vec = obj + .keys() + .map(|k| sanitize_identifier(k)) + .filter(|k| !k.is_empty() && k != "id") + .collect(); + if let Err(e) = check_field_write_access(&field_names, &user_roles, &access_info) { + return ( + StatusCode::FORBIDDEN, + Json(RecordResponse { + success: false, + data: None, + message: Some(e), + }), + ) + .into_response(); + } + let query = format!( "UPDATE {} SET {} WHERE id = '{}' RETURNING row_to_json({}.*)::jsonb as data", table_name, @@ -409,9 +582,11 @@ pub async fn update_record_handler( pub async fn delete_record_handler( State(state): State>, + headers: HeaderMap, Path((table, id)): Path<(String, String)>, ) -> impl IntoResponse { let table_name = sanitize_identifier(&table); + let user_roles = user_roles_from_headers(&headers); let record_id = match Uuid::parse_str(&id) { Ok(uuid) => uuid, @@ -443,6 +618,19 @@ pub async fn delete_record_handler( } }; + // Check table-level write access (delete requires write) + if let Err(e) = check_table_access(&mut conn, &table_name, &user_roles, AccessType::Write) { + return ( + StatusCode::FORBIDDEN, + Json(DeleteResponse { + success: false, + deleted: 0, + message: Some(e), + }), + ) + .into_response(); + } + let query = format!("DELETE FROM {} WHERE id = $1", table_name); let deleted: Result = sql_query(&query) @@ -483,9 +671,11 @@ pub async fn delete_record_handler( pub async fn count_records_handler( State(state): State>, + headers: HeaderMap, Path(table): Path, ) -> impl IntoResponse { let table_name = sanitize_identifier(&table); + let user_roles = user_roles_from_headers(&headers); let mut conn = match state.conn.get() { Ok(c) => c, @@ -498,6 +688,11 @@ pub async fn count_records_handler( } }; + // Check table-level read access (count requires read permission) + if let Err(e) = check_table_access(&mut conn, &table_name, &user_roles, AccessType::Read) { + return (StatusCode::FORBIDDEN, Json(json!({ "error": e }))).into_response(); + } + let query = format!("SELECT COUNT(*) as count FROM {}", table_name); let result: Result = sql_query(&query).get_result(&mut conn); @@ -523,10 +718,12 @@ pub struct SearchRequest { pub async fn search_records_handler( State(state): State>, + headers: HeaderMap, Path(table): Path, Json(payload): Json, ) -> impl IntoResponse { let table_name = sanitize_identifier(&table); + let user_roles = user_roles_from_headers(&headers); let limit = payload.limit.unwrap_or(20).min(100); let search_term = payload.query.replace('\'', "''"); @@ -541,6 +738,15 @@ pub async fn search_records_handler( } }; + // Check table-level read access + let access_info = + match check_table_access(&mut conn, &table_name, &user_roles, AccessType::Read) { + Ok(info) => info, + Err(e) => { + return (StatusCode::FORBIDDEN, Json(json!({ "error": e }))).into_response(); + } + }; + let query = format!( "SELECT row_to_json(t.*) as data FROM {} t WHERE COALESCE(t.title::text, '') || ' ' || COALESCE(t.name::text, '') || ' ' || COALESCE(t.description::text, '') @@ -551,11 +757,14 @@ pub async fn search_records_handler( let rows: Result, _> = sql_query(&query).get_results(&mut conn); match rows { - Ok(data) => ( - StatusCode::OK, - Json(json!({ "data": data.into_iter().map(|r| r.data).collect::>() })), - ) - .into_response(), + Ok(data) => { + // Filter fields based on user roles + let filtered_data: Vec = data + .into_iter() + .map(|r| filter_fields_by_role(r.data, &user_roles, &access_info)) + .collect(); + (StatusCode::OK, Json(json!({ "data": filtered_data }))).into_response() + } Err(e) => { error!("Failed to search in {table_name}: {e}"); ( diff --git a/src/basic/keywords/find.rs b/src/basic/keywords/find.rs index 01bbd275d..7a15ba6fa 100644 --- a/src/basic/keywords/find.rs +++ b/src/basic/keywords/find.rs @@ -1,16 +1,18 @@ +use super::table_access::{check_table_access, filter_fields_by_role, AccessType, UserRoles}; use crate::shared::models::UserSession; use crate::shared::state::AppState; use crate::shared::utils; use crate::shared::utils::to_array; use diesel::pg::PgConnection; use diesel::prelude::*; -use log::error; -use log::trace; +use log::{error, trace, warn}; use rhai::Dynamic; use rhai::Engine; use serde_json::{json, Value}; -pub fn find_keyword(state: &AppState, _user: UserSession, engine: &mut Engine) { +pub fn find_keyword(state: &AppState, user: UserSession, engine: &mut Engine) { let connection = state.conn.clone(); + let user_roles = UserRoles::from_user_session(&user); + engine .register_custom_syntax(["FIND", "$expr$", ",", "$expr$"], false, { move |context, inputs| { @@ -19,13 +21,32 @@ pub fn find_keyword(state: &AppState, _user: UserSession, engine: &mut Engine) { let mut binding = connection.get().map_err(|e| format!("DB error: {}", e))?; let binding2 = table_name.to_string(); let binding3 = filter.to_string(); + + // Check read access before executing query + let access_info = match check_table_access( + &mut binding, + &binding2, + &user_roles, + AccessType::Read, + ) { + Ok(info) => info, + Err(e) => { + warn!("FIND access denied: {}", e); + return Err(e.into()); + } + }; + let result = tokio::task::block_in_place(|| { tokio::runtime::Handle::current() .block_on(async { execute_find(&mut binding, &binding2, &binding3) }) }) .map_err(|e| format!("DB error: {}", e))?; + if let Some(results) = result.get("results") { - let array = to_array(utils::json_value_to_dynamic(results)); + // Filter fields based on user roles + let filtered = + filter_fields_by_role(results.clone(), &user_roles, &access_info); + let array = to_array(utils::json_value_to_dynamic(&filtered)); Ok(Dynamic::from(array)) } else { Err("No results".into()) diff --git a/src/basic/keywords/mod.rs b/src/basic/keywords/mod.rs index 872680954..290844d7d 100644 --- a/src/basic/keywords/mod.rs +++ b/src/basic/keywords/mod.rs @@ -65,6 +65,7 @@ pub mod social; pub mod social_media; pub mod string_functions; pub mod switch_case; +pub mod table_access; pub mod table_definition; pub mod transfer_to_human; pub mod universal_messaging; @@ -83,6 +84,10 @@ pub use app_server::configure_app_server_routes; pub use db_api::configure_db_routes; pub use mcp_client::{McpClient, McpRequest, McpResponse, McpServer, McpTool}; pub use mcp_directory::{McpDirectoryScanResult, McpDirectoryScanner, McpServerConfig}; +pub use table_access::{ + check_field_write_access, check_table_access, filter_fields_by_role, load_table_access_info, + AccessType, TableAccessInfo, UserRoles, +}; pub fn get_all_keywords() -> Vec { vec![ diff --git a/src/basic/keywords/procedures.rs b/src/basic/keywords/procedures.rs index 8be041781..553da5868 100644 --- a/src/basic/keywords/procedures.rs +++ b/src/basic/keywords/procedures.rs @@ -1,5 +1,6 @@ use crate::shared::models::UserSession; use crate::shared::state::AppState; +use botlib::MAX_LOOP_ITERATIONS; use log::trace; use rhai::{Dynamic, Engine}; use std::collections::HashMap; @@ -32,7 +33,7 @@ fn register_while_wend(engine: &mut Engine) { let condition_expr = &inputs[0]; let block = &inputs[1]; - let max_iterations = 100_000; + let max_iterations = MAX_LOOP_ITERATIONS; let mut iterations = 0; loop { @@ -70,8 +71,7 @@ fn register_while_wend(engine: &mut Engine) { iterations += 1; if iterations >= max_iterations { return Err(format!( - "WHILE loop exceeded maximum iterations ({}). Possible infinite loop.", - max_iterations + "WHILE loop exceeded maximum iterations ({max_iterations}). Possible infinite loop." ) .into()); } @@ -98,7 +98,7 @@ fn register_do_loop(engine: &mut Engine) { let condition_expr = &inputs[0]; let block = &inputs[1]; - let max_iterations = 100_000; + let max_iterations = MAX_LOOP_ITERATIONS; let mut iterations = 0; loop { @@ -134,7 +134,7 @@ fn register_do_loop(engine: &mut Engine) { let condition_expr = &inputs[0]; let block = &inputs[1]; - let max_iterations = 100_000; + let max_iterations = MAX_LOOP_ITERATIONS; let mut iterations = 0; loop { @@ -170,7 +170,7 @@ fn register_do_loop(engine: &mut Engine) { let block = &inputs[0]; let condition_expr = &inputs[1]; - let max_iterations = 100_000; + let max_iterations = MAX_LOOP_ITERATIONS; let mut iterations = 0; loop { @@ -206,7 +206,7 @@ fn register_do_loop(engine: &mut Engine) { let block = &inputs[0]; let condition_expr = &inputs[1]; - let max_iterations = 100_000; + let max_iterations = MAX_LOOP_ITERATIONS; let mut iterations = 0; loop { diff --git a/src/basic/keywords/save_from_unstructured.rs b/src/basic/keywords/save_from_unstructured.rs index bb8193488..3cef6fc04 100644 --- a/src/basic/keywords/save_from_unstructured.rs +++ b/src/basic/keywords/save_from_unstructured.rs @@ -1,8 +1,9 @@ +use super::table_access::{check_table_access, AccessType, UserRoles}; use crate::shared::models::UserSession; use crate::shared::state::AppState; use chrono::Utc; use diesel::prelude::*; -use log::{error, trace}; +use log::{error, trace, warn}; use rhai::{Dynamic, Engine}; use serde_json::{json, Value}; use std::sync::Arc; @@ -91,6 +92,16 @@ pub async fn execute_save_from_unstructured( table_name: &str, text: &str, ) -> Result { + // Check write access before proceeding + let user_roles = UserRoles::from_user_session(user); + { + let mut conn = state.conn.get().map_err(|e| format!("DB error: {}", e))?; + if let Err(e) = check_table_access(&mut conn, table_name, &user_roles, AccessType::Write) { + warn!("SAVE FROM UNSTRUCTURED access denied: {}", e); + return Err(e); + } + } + let schema = get_table_schema(state, table_name)?; let extraction_prompt = build_extraction_prompt(table_name, &schema, text); diff --git a/src/basic/keywords/table_access.rs b/src/basic/keywords/table_access.rs new file mode 100644 index 000000000..a101b975f --- /dev/null +++ b/src/basic/keywords/table_access.rs @@ -0,0 +1,535 @@ +/*****************************************************************************\ +| █████ █████ ██ █ █████ █████ ████ ██ ████ █████ █████ ███ ® | +| ██ █ ███ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ | +| ██ ███ ████ █ ██ █ ████ █████ ██████ ██ ████ █ █ █ ██ | +| ██ ██ █ █ ██ █ █ ██ ██ ██ ██ ██ ██ █ ██ ██ █ █ | +| █████ █████ █ ███ █████ ██ ██ ██ ██ █████ ████ █████ █ ███ | +| | +| General Bots Copyright (c) pragmatismo.com.br. All rights reserved. | +| Licensed under the AGPL-3.0. | +| | +| According to our dual licensing model, this program can be used either | +| under the terms of the GNU Affero General Public License, version 3, | +| or under a proprietary license. | +| | +| The texts of the GNU Affero General Public License with an additional | +| permission and of our proprietary license can be found at and | +| in the LICENSE file you have received along with this program. | +| | +| This program is distributed in the hope that it will be useful, | +| but WITHOUT ANY WARRANTY, without even the implied warranty of | +| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | +| GNU Affero General Public License for more details. | +| | +| "General Bots" is a registered trademark of pragmatismo.com.br. | +| The licensing of the program under the AGPLv3 does not imply a | +| trademark license. Therefore any rights, title and interest in | +| our trademarks remain entirely with us. | +| | +\*****************************************************************************/ + +use crate::shared::models::UserSession; +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::Text; +use log::{trace, warn}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct UserRoles { + pub roles: Vec, + pub user_id: Option, +} + +impl UserRoles { + pub fn new(roles: Vec) -> Self { + Self { + roles: roles.into_iter().map(|r| r.to_lowercase()).collect(), + user_id: None, + } + } + + pub fn with_user_id(roles: Vec, user_id: uuid::Uuid) -> Self { + Self { + roles: roles.into_iter().map(|r| r.to_lowercase()).collect(), + user_id: Some(user_id), + } + } + + pub fn anonymous() -> Self { + Self::default() + } + + pub fn from_user_session(session: &UserSession) -> Self { + let mut roles = Vec::new(); + + // Try different keys where roles might be stored + let role_keys = ["roles", "user_roles", "zitadel_roles"]; + + for key in role_keys { + if let Some(value) = session.context_data.get(key) { + match value { + // Array of strings + Value::Array(arr) => { + for item in arr { + if let Value::String(s) = item { + roles.push(s.trim().to_lowercase()); + } + } + if !roles.is_empty() { + break; + } + } + // Semicolon-separated string + Value::String(s) => { + roles = s + .split(';') + .map(|r| r.trim().to_lowercase()) + .filter(|r| !r.is_empty()) + .collect(); + if !roles.is_empty() { + break; + } + } + _ => {} + } + } + } + + // Also check if user is marked as admin in context + if let Some(Value::Bool(true)) = session.context_data.get("is_admin") { + if !roles.contains(&"admin".to_string()) { + roles.push("admin".to_string()); + } + } + + Self { + roles, + user_id: Some(session.user_id), + } + } + + pub fn has_access(&self, required_roles: &[String]) -> bool { + if required_roles.is_empty() { + return true; // No roles specified = everyone has access + } + + // Check if user has any of the required roles + self.roles.iter().any(|user_role| { + required_roles + .iter() + .any(|req| req.to_lowercase() == *user_role) + }) + } + + pub fn has_role(&self, role: &str) -> bool { + self.roles.iter().any(|r| r == &role.to_lowercase()) + } + + pub fn is_admin(&self) -> bool { + self.has_role("admin") + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AccessType { + Read, + Write, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct TableAccessInfo { + pub table_name: String, + pub read_roles: Vec, + pub write_roles: Vec, + pub field_read_roles: HashMap>, + pub field_write_roles: HashMap>, +} + +impl TableAccessInfo { + pub fn can_read(&self, user_roles: &UserRoles) -> bool { + user_roles.has_access(&self.read_roles) + } + + pub fn can_write(&self, user_roles: &UserRoles) -> bool { + user_roles.has_access(&self.write_roles) + } + + pub fn can_read_field(&self, field_name: &str, user_roles: &UserRoles) -> bool { + if let Some(field_roles) = self.field_read_roles.get(field_name) { + user_roles.has_access(field_roles) + } else { + true // No field-level restriction + } + } + + pub fn can_write_field(&self, field_name: &str, user_roles: &UserRoles) -> bool { + if let Some(field_roles) = self.field_write_roles.get(field_name) { + user_roles.has_access(field_roles) + } else { + true // No field-level restriction + } + } + + pub fn get_restricted_read_fields(&self, user_roles: &UserRoles) -> Vec { + self.field_read_roles + .iter() + .filter(|(_, roles)| !user_roles.has_access(roles)) + .map(|(field, _)| field.clone()) + .collect() + } + + pub fn get_restricted_write_fields(&self, user_roles: &UserRoles) -> Vec { + self.field_write_roles + .iter() + .filter(|(_, roles)| !user_roles.has_access(roles)) + .map(|(field, _)| field.clone()) + .collect() + } +} + +#[derive(QueryableByName, Debug)] +struct TableDefRow { + #[diesel(sql_type = Text)] + table_name: String, + #[diesel(sql_type = diesel::sql_types::Nullable)] + read_roles: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + write_roles: Option, +} + +#[derive(QueryableByName, Debug)] +struct FieldDefRow { + #[diesel(sql_type = Text)] + field_name: String, + #[diesel(sql_type = diesel::sql_types::Nullable)] + read_roles: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + write_roles: Option, +} + +pub fn load_table_access_info( + conn: &mut diesel::PgConnection, + table_name: &str, +) -> Option { + // Query table-level permissions + let table_result: Result = sql_query( + "SELECT table_name, read_roles, write_roles + FROM dynamic_table_definitions + WHERE table_name = $1 + LIMIT 1", + ) + .bind::(table_name) + .get_result(conn); + + let table_def = match table_result { + Ok(row) => row, + Err(_) => { + trace!( + "No table definition found for '{}', allowing open access", + table_name + ); + return None; // No definition = open access + } + }; + + let mut info = TableAccessInfo { + table_name: table_def.table_name, + read_roles: parse_roles_string(&table_def.read_roles), + write_roles: parse_roles_string(&table_def.write_roles), + field_read_roles: HashMap::new(), + field_write_roles: HashMap::new(), + }; + + // Query field-level permissions + let fields_result: Result, _> = sql_query( + "SELECT f.field_name, f.read_roles, f.write_roles + FROM dynamic_table_fields f + JOIN dynamic_table_definitions t ON f.table_definition_id = t.id + WHERE t.table_name = $1", + ) + .bind::(table_name) + .get_results(conn); + + if let Ok(fields) = fields_result { + for field in fields { + let field_read = parse_roles_string(&field.read_roles); + let field_write = parse_roles_string(&field.write_roles); + + if !field_read.is_empty() { + info.field_read_roles + .insert(field.field_name.clone(), field_read); + } + if !field_write.is_empty() { + info.field_write_roles.insert(field.field_name, field_write); + } + } + } + + trace!( + "Loaded access info for table '{}': read_roles={:?}, write_roles={:?}, field_restrictions={}", + info.table_name, + info.read_roles, + info.write_roles, + info.field_read_roles.len() + info.field_write_roles.len() + ); + + Some(info) +} + +fn parse_roles_string(roles: &Option) -> Vec { + roles + .as_ref() + .map(|s| { + s.split(';') + .map(|r| r.trim().to_string()) + .filter(|r| !r.is_empty()) + .collect() + }) + .unwrap_or_default() +} + +pub fn check_table_access( + conn: &mut diesel::PgConnection, + table_name: &str, + user_roles: &UserRoles, + access_type: AccessType, +) -> Result, String> { + let access_info = load_table_access_info(conn, table_name); + + if let Some(ref info) = access_info { + let has_access = match access_type { + AccessType::Read => info.can_read(user_roles), + AccessType::Write => info.can_write(user_roles), + }; + + if !has_access { + let action = match access_type { + AccessType::Read => "read from", + AccessType::Write => "write to", + }; + warn!( + "Access denied: user {:?} cannot {} table '{}'", + user_roles.user_id, action, table_name + ); + return Err(format!( + "Access denied: insufficient permissions to {} table '{}'", + action, table_name + )); + } + } + + Ok(access_info) +} + +pub fn check_field_write_access( + fields: &[String], + user_roles: &UserRoles, + access_info: &Option, +) -> Result<(), String> { + let Some(info) = access_info else { + return Ok(()); // No access info = allow all + }; + + let mut denied_fields = Vec::new(); + + for field in fields { + if !info.can_write_field(field, user_roles) { + denied_fields.push(field.clone()); + } + } + + if denied_fields.is_empty() { + Ok(()) + } else { + Err(format!( + "Access denied: insufficient permissions to write field(s): {}", + denied_fields.join(", ") + )) + } +} + +pub fn filter_fields_by_role( + data: Value, + user_roles: &UserRoles, + access_info: &Option, +) -> Value { + let Some(info) = access_info else { + return data; // No access info = return all fields + }; + + match data { + Value::Object(mut map) => { + let restricted = info.get_restricted_read_fields(user_roles); + + for field in restricted { + trace!("Filtering out field '{}' due to role restriction", field); + map.remove(&field); + } + + Value::Object(map) + } + Value::Array(arr) => Value::Array( + arr.into_iter() + .map(|v| filter_fields_by_role(v, user_roles, access_info)) + .collect(), + ), + other => other, + } +} + +pub fn filter_write_fields( + data: Value, + user_roles: &UserRoles, + access_info: &Option, +) -> (Value, Vec) { + let Some(info) = access_info else { + return (data, Vec::new()); // No access info = allow all + }; + + match data { + Value::Object(mut map) => { + let restricted = info.get_restricted_write_fields(user_roles); + let mut removed = Vec::new(); + + for field in &restricted { + if map.contains_key(field) { + trace!( + "Removing field '{}' from write data due to role restriction", + field + ); + map.remove(field); + removed.push(field.clone()); + } + } + + (Value::Object(map), removed) + } + other => (other, Vec::new()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_user_roles_has_access() { + let roles = UserRoles::new(vec!["admin".to_string(), "manager".to_string()]); + + // Empty roles = everyone allowed + assert!(roles.has_access(&[])); + + // User has admin role + assert!(roles.has_access(&["admin".to_string()])); + + // User has manager role + assert!(roles.has_access(&["manager".to_string(), "superuser".to_string()])); + + // User doesn't have superuser role only + assert!(!roles.has_access(&["superuser".to_string()])); + } + + #[test] + fn test_user_roles_case_insensitive() { + let roles = UserRoles::new(vec!["Admin".to_string()]); + + assert!(roles.has_access(&["admin".to_string()])); + assert!(roles.has_access(&["ADMIN".to_string()])); + assert!(roles.has_access(&["Admin".to_string()])); + } + + #[test] + fn test_anonymous_user() { + let roles = UserRoles::anonymous(); + + // Anonymous can access if no roles required + assert!(roles.has_access(&[])); + + // Anonymous cannot access if roles required + assert!(!roles.has_access(&["admin".to_string()])); + } + + #[test] + fn test_table_access_info_field_restrictions() { + let mut info = TableAccessInfo { + table_name: "contacts".to_string(), + read_roles: vec![], + write_roles: vec![], + field_read_roles: HashMap::new(), + field_write_roles: HashMap::new(), + }; + + info.field_read_roles + .insert("ssn".to_string(), vec!["admin".to_string()]); + info.field_write_roles + .insert("salary".to_string(), vec!["hr".to_string()]); + + let admin = UserRoles::new(vec!["admin".to_string()]); + let hr = UserRoles::new(vec!["hr".to_string()]); + let user = UserRoles::new(vec!["user".to_string()]); + + // Admin can read SSN + assert!(info.can_read_field("ssn", &admin)); + // Regular user cannot read SSN + assert!(!info.can_read_field("ssn", &user)); + + // HR can write salary + assert!(info.can_write_field("salary", &hr)); + // Admin cannot write salary (different role) + assert!(!info.can_write_field("salary", &admin)); + + // Everyone can read/write unrestricted fields + assert!(info.can_read_field("name", &user)); + assert!(info.can_write_field("name", &user)); + } + + #[test] + fn test_filter_fields_by_role() { + let mut info = TableAccessInfo::default(); + info.field_read_roles + .insert("secret".to_string(), vec!["admin".to_string()]); + + let data = serde_json::json!({ + "id": 1, + "name": "John", + "secret": "classified" + }); + + let user = UserRoles::new(vec!["user".to_string()]); + let filtered = filter_fields_by_role(data.clone(), &user, &Some(info.clone())); + + assert!(filtered.get("id").is_some()); + assert!(filtered.get("name").is_some()); + assert!(filtered.get("secret").is_none()); + + // Admin can see everything + let admin = UserRoles::new(vec!["admin".to_string()]); + let not_filtered = filter_fields_by_role(data, &admin, &Some(info)); + + assert!(not_filtered.get("secret").is_some()); + } + + #[test] + fn test_parse_roles_string() { + assert_eq!(parse_roles_string(&None), Vec::::new()); + assert_eq!( + parse_roles_string(&Some("".to_string())), + Vec::::new() + ); + assert_eq!( + parse_roles_string(&Some("admin".to_string())), + vec!["admin"] + ); + assert_eq!( + parse_roles_string(&Some("admin;manager".to_string())), + vec!["admin", "manager"] + ); + assert_eq!( + parse_roles_string(&Some(" admin ; manager ; hr ".to_string())), + vec!["admin", "manager", "hr"] + ); + } +} diff --git a/src/basic/keywords/table_definition.rs b/src/basic/keywords/table_definition.rs index c407cf332..d794816ff 100644 --- a/src/basic/keywords/table_definition.rs +++ b/src/basic/keywords/table_definition.rs @@ -39,24 +39,41 @@ use std::error::Error; use std::sync::Arc; use uuid::Uuid; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct FieldDefinition { pub name: String, pub field_type: String, + #[serde(default)] pub length: Option, + #[serde(default)] pub precision: Option, + #[serde(default)] pub is_key: bool, + #[serde(default)] pub is_nullable: bool, + #[serde(default)] pub default_value: Option, + #[serde(default)] pub reference_table: Option, + #[serde(default)] pub field_order: i32, + #[serde(default)] + pub read_roles: Vec, + #[serde(default)] + pub write_roles: Vec, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct TableDefinition { pub name: String, + #[serde(default)] pub connection_name: String, + #[serde(default)] pub fields: Vec, + #[serde(default)] + pub read_roles: Vec, + #[serde(default)] + pub write_roles: Vec, } #[derive(Debug, Clone)] @@ -97,6 +114,9 @@ fn parse_single_table( ) -> Result> { let header_line = lines[*index].trim(); + // Parse table-level READ BY and WRITE BY + let (read_roles, write_roles) = parse_role_attributes(header_line); + let parts: Vec<&str> = header_line.split_whitespace().collect(); if parts.len() < 2 { @@ -110,13 +130,27 @@ fn parse_single_table( let table_name = parts[1].to_string(); - let connection_name = if parts.len() >= 4 && parts[2].eq_ignore_ascii_case("ON") { - parts[3].to_string() - } else { - "default".to_string() - }; + // Find connection name (ON keyword) + let mut connection_name = "default".to_string(); + for i in 2..parts.len() { + if parts[i].eq_ignore_ascii_case("ON") && i + 1 < parts.len() { + // Check that the next part is not READ or WRITE + if !parts[i + 1].eq_ignore_ascii_case("READ") + && !parts[i + 1].eq_ignore_ascii_case("WRITE") + { + connection_name = parts[i + 1].to_string(); + } + break; + } + } - trace!("Parsing TABLE {} ON {}", table_name, connection_name); + trace!( + "Parsing TABLE {} ON {} (read_roles: {:?}, write_roles: {:?})", + table_name, + connection_name, + read_roles, + write_roles + ); *index += 1; let mut fields = Vec::new(); @@ -153,13 +187,60 @@ fn parse_single_table( name: table_name, connection_name, fields, + read_roles, + write_roles, }) } +fn parse_role_attributes(line: &str) -> (Vec, Vec) { + let mut read_roles = Vec::new(); + let mut write_roles = Vec::new(); + + // Find READ BY "..." + if let Some(read_idx) = line.to_uppercase().find("READ BY") { + let after_read = &line[read_idx + 7..]; + if let Some(roles_str) = extract_quoted_string(after_read) { + read_roles = roles_str + .split(';') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + } + } + + // Find WRITE BY "..." + if let Some(write_idx) = line.to_uppercase().find("WRITE BY") { + let after_write = &line[write_idx + 8..]; + if let Some(roles_str) = extract_quoted_string(after_write) { + write_roles = roles_str + .split(';') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + } + } + + (read_roles, write_roles) +} + +fn extract_quoted_string(s: &str) -> Option { + let trimmed = s.trim(); + if let Some(start) = trimmed.find('"') { + let after_quote = &trimmed[start + 1..]; + if let Some(end) = after_quote.find('"') { + return Some(after_quote[..end].to_string()); + } + } + None +} + fn parse_field_definition( line: &str, order: i32, ) -> Result> { + // Parse field-level READ BY and WRITE BY + let (read_roles, write_roles) = parse_role_attributes(line); + let parts: Vec<&str> = line.split_whitespace().collect(); if parts.is_empty() { @@ -205,6 +286,8 @@ fn parse_field_definition( reference_table = Some(parts[i + 1].to_string()); } } + // Skip READ, BY, WRITE as they're handled separately + "read" | "by" | "write" => {} _ => {} } } @@ -219,6 +302,8 @@ fn parse_field_definition( default_value: None, reference_table, field_order: order, + read_roles, + write_roles, }) } @@ -413,16 +498,30 @@ pub fn store_table_definition( bot_id: Uuid, table: &TableDefinition, ) -> Result> { + // Convert role vectors to semicolon-separated strings for storage + let read_roles_str: Option = if table.read_roles.is_empty() { + None + } else { + Some(table.read_roles.join(";")) + }; + let write_roles_str: Option = if table.write_roles.is_empty() { + None + } else { + Some(table.write_roles.join(";")) + }; + let table_id: Uuid = diesel::sql_query( - "INSERT INTO dynamic_table_definitions (bot_id, table_name, connection_name) - VALUES ($1, $2, $3) + "INSERT INTO dynamic_table_definitions (bot_id, table_name, connection_name, read_roles, write_roles) + VALUES ($1, $2, $3, $4, $5) ON CONFLICT (bot_id, table_name, connection_name) - DO UPDATE SET updated_at = NOW() + DO UPDATE SET updated_at = NOW(), read_roles = $4, write_roles = $5 RETURNING id", ) .bind::(bot_id) .bind::(&table.name) .bind::(&table.connection_name) + .bind::, _>(&read_roles_str) + .bind::, _>(&write_roles_str) .get_result::(conn)? .id; @@ -431,11 +530,23 @@ pub fn store_table_definition( .execute(conn)?; for field in &table.fields { + // Convert field role vectors to semicolon-separated strings + let field_read_roles: Option = if field.read_roles.is_empty() { + None + } else { + Some(field.read_roles.join(";")) + }; + let field_write_roles: Option = if field.write_roles.is_empty() { + None + } else { + Some(field.write_roles.join(";")) + }; + diesel::sql_query( "INSERT INTO dynamic_table_fields (table_definition_id, field_name, field_type, field_length, field_precision, - is_key, is_nullable, default_value, reference_table, field_order) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + is_key, is_nullable, default_value, reference_table, field_order, read_roles, write_roles) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", ) .bind::(table_id) .bind::(&field.name) @@ -447,6 +558,8 @@ pub fn store_table_definition( .bind::, _>(&field.default_value) .bind::, _>(&field.reference_table) .bind::(field.field_order) + .bind::, _>(&field_read_roles) + .bind::, _>(&field_write_roles) .execute(conn)?; } diff --git a/src/basic/keywords/webhook.rs b/src/basic/keywords/webhook.rs index b6a5451e3..db03da714 100644 --- a/src/basic/keywords/webhook.rs +++ b/src/basic/keywords/webhook.rs @@ -154,7 +154,6 @@ pub fn remove_webhook_registration( Ok(result) } -/// Type alias for webhook results: (target, param, `is_active`). pub type WebhookResult = Vec<(String, String, bool)>; pub fn get_bot_webhooks( diff --git a/src/calendar/mod.rs b/src/calendar/mod.rs index bfdf99b6b..d7c834e0a 100644 --- a/src/calendar/mod.rs +++ b/src/calendar/mod.rs @@ -523,10 +523,10 @@ pub fn configure_calendar_routes() -> Router> { &ApiUrls::CALENDAR_EVENT_BY_ID.replace(":id", "{id}"), get(get_event).put(update_event).delete(delete_event), ) - .route("/api/calendar/export.ics", get(export_ical)) - .route("/api/calendar/import", post(import_ical)) - .route("/api/calendar/calendars", get(list_calendars_api)) - .route("/api/calendar/events/upcoming", get(upcoming_events_api)) + .route(ApiUrls::CALENDAR_EXPORT, get(export_ical)) + .route(ApiUrls::CALENDAR_IMPORT, post(import_ical)) + .route(ApiUrls::CALENDAR_CALENDARS, get(list_calendars_api)) + .route(ApiUrls::CALENDAR_UPCOMING, get(upcoming_events_api)) .route("/ui/calendar/list", get(list_calendars)) .route("/ui/calendar/upcoming", get(upcoming_events)) .route("/ui/calendar/event/new", get(new_event_form)) diff --git a/src/core/bootstrap/mod.rs b/src/core/bootstrap/mod.rs index 623d31db2..c72df5b97 100644 --- a/src/core/bootstrap/mod.rs +++ b/src/core/bootstrap/mod.rs @@ -317,6 +317,28 @@ impl BootstrapManager { match pm.start(component.name) { Ok(_child) => { info!("Started component: {}", component.name); + if component.name == "drive" { + for i in 0..15 { + let drive_ready = Command::new("sh") + .arg("-c") + .arg("curl -f -s 'http://127.0.0.1:9000/minio/health/live' >/dev/null 2>&1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false); + + if drive_ready { + info!("MinIO drive is ready and responding"); + break; + } + if i < 14 { + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } else { + warn!("MinIO drive health check timed out after 15s"); + } + } + } } Err(e) => { debug!( @@ -574,14 +596,12 @@ impl BootstrapManager { || stderr_str.contains("connection refused") { connection_refused = true; - - continue; - } - - connection_refused = false; - if let Ok(status) = serde_json::from_str::(&status_str) { - parsed_status = Some(status); - break; + } else { + connection_refused = false; + if let Ok(status) = serde_json::from_str::(&status_str) { + parsed_status = Some(status); + break; + } } } @@ -1837,7 +1857,6 @@ VAULT_CACHE_TTL=300 } Err(e) => { warn!("S3/MinIO not available, skipping bucket {}: {}", bucket, e); - continue; } } } diff --git a/src/core/bot/channels/whatsapp.rs b/src/core/bot/channels/whatsapp.rs index 07de129b7..dec2a46c3 100644 --- a/src/core/bot/channels/whatsapp.rs +++ b/src/core/bot/channels/whatsapp.rs @@ -649,7 +649,6 @@ pub fn create_interactive_buttons(text: &str, buttons: Vec<(&str, &str)>) -> ser }) } -/// Type alias for interactive list sections: (title, rows) where rows are (id, title, description) pub type InteractiveListSections = Vec<(String, Vec<(String, String, Option)>)>; pub fn create_interactive_list( diff --git a/src/core/package_manager/installer.rs b/src/core/package_manager/installer.rs index 55f509b7f..e7b0abd19 100644 --- a/src/core/package_manager/installer.rs +++ b/src/core/package_manager/installer.rs @@ -678,7 +678,9 @@ impl PackageManager { env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "{{BIN_PATH}}/nocodb".to_string(), - check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:5757 >/dev/null 2>&1".to_string(), + check_cmd: + "curl -f -k --connect-timeout 2 -m 5 https://localhost:5757 >/dev/null 2>&1" + .to_string(), }, ); } @@ -705,7 +707,9 @@ impl PackageManager { env_vars: HashMap::new(), data_download_list: Vec::new(), exec_cmd: "coolwsd --config-file={{CONF_PATH}}/coolwsd.xml".to_string(), - check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:9980 >/dev/null 2>&1".to_string(), + check_cmd: + "curl -f -k --connect-timeout 2 -m 5 https://localhost:9980 >/dev/null 2>&1" + .to_string(), }, ); } @@ -883,7 +887,7 @@ impl PackageManager { "mkdir -p {{LOGS_PATH}}".to_string(), r#"cat > {{CONF_PATH}}/vault/config.hcl << 'EOF' storage "file" { - path = "/opt/gbo/data/vault" + path = "{{DATA_PATH}}/vault" } listener "tcp" { diff --git a/src/core/package_manager/setup/directory_setup.rs b/src/core/package_manager/setup/directory_setup.rs index a0a9f0601..30bf7cfb3 100644 --- a/src/core/package_manager/setup/directory_setup.rs +++ b/src/core/package_manager/setup/directory_setup.rs @@ -147,12 +147,20 @@ impl DirectorySetup { log::info!(" Saved Directory configuration"); log::info!(" Directory initialization complete!"); - log::info!( - " Default user: {} / {}", - config.default_user.email, - config.default_user.password - ); - log::info!(" Login at: {}", self.base_url); + log::info!(""); + log::info!("╔══════════════════════════════════════════════════════════════╗"); + log::info!("║ DEFAULT CREDENTIALS ║"); + log::info!("╠══════════════════════════════════════════════════════════════╣"); + log::info!("║ Email: {:<50}║", config.default_user.email); + log::info!("║ Password: {:<50}║", config.default_user.password); + log::info!("╠══════════════════════════════════════════════════════════════╣"); + log::info!("║ Login at: {:<50}║", self.base_url); + log::info!("╚══════════════════════════════════════════════════════════════╝"); + log::info!(""); + log::info!(">>> COPY THESE CREDENTIALS NOW - Press ENTER to continue <<<"); + + let mut input = String::new(); + let _ = std::io::stdin().read_line(&mut input); Ok(config) } diff --git a/src/core/urls.rs b/src/core/urls.rs index 013bd980e..3a1edaf5d 100644 --- a/src/core/urls.rs +++ b/src/core/urls.rs @@ -1,14 +1,7 @@ - - - - - - #[derive(Debug)] pub struct ApiUrls; impl ApiUrls { - pub const USERS: &'static str = "/api/users"; pub const USER_BY_ID: &'static str = "/api/users/:id"; pub const USER_LOGIN: &'static str = "/api/users/login"; @@ -20,7 +13,6 @@ impl ApiUrls { pub const USER_PROVISION: &'static str = "/api/users/provision"; pub const USER_DEPROVISION: &'static str = "/api/users/:id/deprovision"; - pub const GROUPS: &'static str = "/api/groups"; pub const GROUP_BY_ID: &'static str = "/api/groups/:id"; pub const GROUP_MEMBERS: &'static str = "/api/groups/:id/members"; @@ -28,7 +20,6 @@ impl ApiUrls { pub const GROUP_REMOVE_MEMBER: &'static str = "/api/groups/:id/members/:user_id"; pub const GROUP_PERMISSIONS: &'static str = "/api/groups/:id/permissions"; - pub const AUTH: &'static str = "/api/auth"; pub const AUTH_TOKEN: &'static str = "/api/auth/token"; pub const AUTH_REFRESH: &'static str = "/api/auth/refresh"; @@ -36,14 +27,12 @@ impl ApiUrls { pub const AUTH_OAUTH: &'static str = "/api/auth/oauth"; pub const AUTH_OAUTH_CALLBACK: &'static str = "/api/auth/oauth/callback"; - pub const SESSIONS: &'static str = "/api/sessions"; pub const SESSION_BY_ID: &'static str = "/api/sessions/:id"; pub const SESSION_HISTORY: &'static str = "/api/sessions/:id/history"; pub const SESSION_START: &'static str = "/api/sessions/:id/start"; pub const SESSION_END: &'static str = "/api/sessions/:id/end"; - pub const BOTS: &'static str = "/api/bots"; pub const BOT_BY_ID: &'static str = "/api/bots/:id"; pub const BOT_CONFIG: &'static str = "/api/bots/:id/config"; @@ -51,7 +40,6 @@ impl ApiUrls { pub const BOT_LOGS: &'static str = "/api/bots/:id/logs"; pub const BOT_METRICS: &'static str = "/api/bots/:id/metrics"; - pub const DRIVE_LIST: &'static str = "/api/drive/list"; pub const DRIVE_UPLOAD: &'static str = "/api/drive/upload"; pub const DRIVE_DOWNLOAD: &'static str = "/api/drive/download/:path"; @@ -60,7 +48,7 @@ impl ApiUrls { pub const DRIVE_MOVE: &'static str = "/api/drive/move"; pub const DRIVE_COPY: &'static str = "/api/drive/copy"; pub const DRIVE_SHARE: &'static str = "/api/drive/share"; - + pub const DRIVE_FILE: &'static str = "/api/drive/file/:path"; pub const EMAIL_ACCOUNTS: &'static str = "/api/email/accounts"; pub const EMAIL_ACCOUNT_BY_ID: &'static str = "/api/email/accounts/:id"; @@ -72,13 +60,15 @@ impl ApiUrls { pub const EMAIL_GET: &'static str = "/api/email/get/:campaign_id"; pub const EMAIL_CLICK: &'static str = "/api/email/click/:campaign_id/:email"; - pub const CALENDAR_EVENTS: &'static str = "/api/calendar/events"; pub const CALENDAR_EVENT_BY_ID: &'static str = "/api/calendar/events/:id"; pub const CALENDAR_REMINDERS: &'static str = "/api/calendar/reminders"; pub const CALENDAR_SHARE: &'static str = "/api/calendar/share"; pub const CALENDAR_SYNC: &'static str = "/api/calendar/sync"; - + pub const CALENDAR_EXPORT: &'static str = "/api/calendar/export.ics"; + pub const CALENDAR_IMPORT: &'static str = "/api/calendar/import"; + pub const CALENDAR_CALENDARS: &'static str = "/api/calendar/calendars"; + pub const CALENDAR_UPCOMING: &'static str = "/api/calendar/events/upcoming"; pub const TASKS: &'static str = "/api/tasks"; pub const TASK_BY_ID: &'static str = "/api/tasks/:id"; @@ -87,7 +77,6 @@ impl ApiUrls { pub const TASK_PRIORITY: &'static str = "/api/tasks/:id/priority"; pub const TASK_COMMENTS: &'static str = "/api/tasks/:id/comments"; - pub const MEET_CREATE: &'static str = "/api/meet/create"; pub const MEET_ROOMS: &'static str = "/api/meet/rooms"; pub const MEET_ROOM_BY_ID: &'static str = "/api/meet/rooms/:id"; @@ -96,24 +85,40 @@ impl ApiUrls { pub const MEET_TOKEN: &'static str = "/api/meet/token"; pub const MEET_INVITE: &'static str = "/api/meet/invite"; pub const MEET_TRANSCRIPTION: &'static str = "/api/meet/rooms/:id/transcription"; - + pub const MEET_PARTICIPANTS: &'static str = "/api/meet/participants"; + pub const MEET_RECENT: &'static str = "/api/meet/recent"; + pub const MEET_SCHEDULED: &'static str = "/api/meet/scheduled"; pub const VOICE_START: &'static str = "/api/voice/start"; pub const VOICE_STOP: &'static str = "/api/voice/stop"; pub const VOICE_STATUS: &'static str = "/api/voice/status"; - pub const DNS_REGISTER: &'static str = "/api/dns/register"; pub const DNS_REMOVE: &'static str = "/api/dns/remove"; pub const DNS_LIST: &'static str = "/api/dns/list"; pub const DNS_UPDATE: &'static str = "/api/dns/update"; - pub const ANALYTICS_DASHBOARD: &'static str = "/api/analytics/dashboard"; pub const ANALYTICS_METRIC: &'static str = "/api/analytics/metric"; + pub const ANALYTICS_MESSAGES_COUNT: &'static str = "/api/analytics/messages/count"; + pub const ANALYTICS_SESSIONS_ACTIVE: &'static str = "/api/analytics/sessions/active"; + pub const ANALYTICS_RESPONSE_AVG: &'static str = "/api/analytics/response/avg"; + pub const ANALYTICS_LLM_TOKENS: &'static str = "/api/analytics/llm/tokens"; + pub const ANALYTICS_STORAGE_USAGE: &'static str = "/api/analytics/storage/usage"; + pub const ANALYTICS_ERRORS_COUNT: &'static str = "/api/analytics/errors/count"; + pub const ANALYTICS_TIMESERIES_MESSAGES: &'static str = "/api/analytics/timeseries/messages"; + pub const ANALYTICS_TIMESERIES_RESPONSE: &'static str = + "/api/analytics/timeseries/response_time"; + pub const ANALYTICS_CHANNELS_DISTRIBUTION: &'static str = + "/api/analytics/channels/distribution"; + pub const ANALYTICS_BOTS_PERFORMANCE: &'static str = "/api/analytics/bots/performance"; + pub const ANALYTICS_ACTIVITY_RECENT: &'static str = "/api/analytics/activity/recent"; + pub const ANALYTICS_QUERIES_TOP: &'static str = "/api/analytics/queries/top"; + pub const ANALYTICS_CHAT: &'static str = "/api/analytics/chat"; + pub const ANALYTICS_LLM_STATS: &'static str = "/api/analytics/llm/stats"; + pub const ANALYTICS_BUDGET_STATUS: &'static str = "/api/analytics/budget/status"; pub const METRICS: &'static str = "/api/metrics"; - pub const ADMIN_STATS: &'static str = "/api/admin/stats"; pub const ADMIN_USERS: &'static str = "/api/admin/users"; pub const ADMIN_SYSTEM: &'static str = "/api/admin/system"; @@ -122,12 +127,10 @@ impl ApiUrls { pub const ADMIN_SERVICES: &'static str = "/api/admin/services"; pub const ADMIN_AUDIT: &'static str = "/api/admin/audit"; - pub const HEALTH: &'static str = "/api/health"; pub const STATUS: &'static str = "/api/status"; pub const SERVICES_STATUS: &'static str = "/api/services/status"; - pub const KB_SEARCH: &'static str = "/api/kb/search"; pub const KB_UPLOAD: &'static str = "/api/kb/upload"; pub const KB_DOCUMENTS: &'static str = "/api/kb/documents"; @@ -135,20 +138,152 @@ impl ApiUrls { pub const KB_INDEX: &'static str = "/api/kb/index"; pub const KB_EMBEDDINGS: &'static str = "/api/kb/embeddings"; - pub const LLM_CHAT: &'static str = "/api/llm/chat"; pub const LLM_COMPLETIONS: &'static str = "/api/llm/completions"; pub const LLM_EMBEDDINGS: &'static str = "/api/llm/embeddings"; pub const LLM_MODELS: &'static str = "/api/llm/models"; + pub const LLM_GENERATE: &'static str = "/api/llm/generate"; + pub const LLM_IMAGE: &'static str = "/api/llm/image"; + pub const ATTENDANCE_QUEUE: &'static str = "/api/attendance/queue"; + pub const ATTENDANCE_ATTENDANTS: &'static str = "/api/attendance/attendants"; + pub const ATTENDANCE_ASSIGN: &'static str = "/api/attendance/assign"; + pub const ATTENDANCE_TRANSFER: &'static str = "/api/attendance/transfer"; + pub const ATTENDANCE_RESOLVE: &'static str = "/api/attendance/resolve/:session_id"; + pub const ATTENDANCE_INSIGHTS: &'static str = "/api/attendance/insights"; + pub const ATTENDANCE_RESPOND: &'static str = "/api/attendance/respond"; + pub const ATTENDANCE_LLM_TIPS: &'static str = "/api/attendance/llm/tips"; + pub const ATTENDANCE_LLM_POLISH: &'static str = "/api/attendance/llm/polish"; + pub const ATTENDANCE_LLM_SMART_REPLIES: &'static str = "/api/attendance/llm/smart-replies"; + pub const ATTENDANCE_LLM_SUMMARY: &'static str = "/api/attendance/llm/summary/:session_id"; + pub const ATTENDANCE_LLM_SENTIMENT: &'static str = "/api/attendance/llm/sentiment"; + pub const ATTENDANCE_LLM_CONFIG: &'static str = "/api/attendance/llm/config/:bot_id"; + + pub const AUTOTASK_CREATE: &'static str = "/api/autotask/create"; + pub const AUTOTASK_CLASSIFY: &'static str = "/api/autotask/classify"; + pub const AUTOTASK_COMPILE: &'static str = "/api/autotask/compile"; + pub const AUTOTASK_EXECUTE: &'static str = "/api/autotask/execute"; + pub const AUTOTASK_SIMULATE: &'static str = "/api/autotask/simulate/:plan_id"; + pub const AUTOTASK_LIST: &'static str = "/api/autotask/list"; + pub const AUTOTASK_STATS: &'static str = "/api/autotask/stats"; + pub const AUTOTASK_PAUSE: &'static str = "/api/autotask/:task_id/pause"; + pub const AUTOTASK_RESUME: &'static str = "/api/autotask/:task_id/resume"; + pub const AUTOTASK_CANCEL: &'static str = "/api/autotask/:task_id/cancel"; + pub const AUTOTASK_TASK_SIMULATE: &'static str = "/api/autotask/:task_id/simulate"; + pub const AUTOTASK_DECISIONS: &'static str = "/api/autotask/:task_id/decisions"; + pub const AUTOTASK_DECIDE: &'static str = "/api/autotask/:task_id/decide"; + pub const AUTOTASK_APPROVALS: &'static str = "/api/autotask/:task_id/approvals"; + pub const AUTOTASK_APPROVE: &'static str = "/api/autotask/:task_id/approve"; + pub const AUTOTASK_TASK_EXECUTE: &'static str = "/api/autotask/:task_id/execute"; + pub const AUTOTASK_LOGS: &'static str = "/api/autotask/:task_id/logs"; + pub const AUTOTASK_RECOMMENDATIONS_APPLY: &'static str = + "/api/autotask/recommendations/:rec_id/apply"; + pub const AUTOTASK_PENDING: &'static str = "/api/autotask/pending"; + pub const AUTOTASK_PENDING_ITEM: &'static str = "/api/autotask/pending/:item_id"; + + pub const DB_TABLE: &'static str = "/api/db/:table"; + pub const DB_TABLE_RECORD: &'static str = "/api/db/:table/:id"; + pub const DB_TABLE_COUNT: &'static str = "/api/db/:table/count"; + pub const DB_TABLE_SEARCH: &'static str = "/api/db/:table/search"; + + pub const DESIGNER_FILES: &'static str = "/api/v1/designer/files"; + pub const DESIGNER_LOAD: &'static str = "/api/v1/designer/load"; + pub const DESIGNER_SAVE: &'static str = "/api/v1/designer/save"; + pub const DESIGNER_VALIDATE: &'static str = "/api/v1/designer/validate"; + pub const DESIGNER_EXPORT: &'static str = "/api/v1/designer/export"; + pub const DESIGNER_MODIFY: &'static str = "/api/designer/modify"; + + pub const MAIL_SEND: &'static str = "/api/mail/send"; + pub const WHATSAPP_SEND: &'static str = "/api/whatsapp/send"; + + pub const FILES_BY_ID: &'static str = "/api/files/:id"; + + pub const MESSAGES: &'static str = "/api/messages"; + + pub const DESIGNER_DIALOGS: &'static str = "/api/designer/dialogs"; + pub const DESIGNER_DIALOG_BY_ID: &'static str = "/api/designer/dialogs/:id"; + + pub const EMAIL_TRACKING_LIST: &'static str = "/api/email/tracking/list"; + pub const EMAIL_TRACKING_STATS: &'static str = "/api/email/tracking/stats"; + + pub const INSTAGRAM_WEBHOOK: &'static str = "/api/instagram/webhook"; + pub const INSTAGRAM_SEND: &'static str = "/api/instagram/send"; + + pub const MONITORING_DASHBOARD: &'static str = "/api/monitoring/dashboard"; + pub const MONITORING_SERVICES: &'static str = "/api/monitoring/services"; + pub const MONITORING_RESOURCES: &'static str = "/api/monitoring/resources"; + pub const MONITORING_LOGS: &'static str = "/api/monitoring/logs"; + pub const MONITORING_LLM: &'static str = "/api/monitoring/llm"; + pub const MONITORING_HEALTH: &'static str = "/api/monitoring/health"; + + pub const MSTEAMS_MESSAGES: &'static str = "/api/msteams/messages"; + pub const MSTEAMS_SEND: &'static str = "/api/msteams/send"; + + pub const PAPER_NEW: &'static str = "/api/paper/new"; + pub const PAPER_LIST: &'static str = "/api/paper/list"; + pub const PAPER_SEARCH: &'static str = "/api/paper/search"; + pub const PAPER_SAVE: &'static str = "/api/paper/save"; + pub const PAPER_AUTOSAVE: &'static str = "/api/paper/autosave"; + pub const PAPER_BY_ID: &'static str = "/api/paper/:id"; + pub const PAPER_DELETE: &'static str = "/api/paper/:id/delete"; + pub const PAPER_TEMPLATE_BLANK: &'static str = "/api/paper/template/blank"; + pub const PAPER_TEMPLATE_MEETING: &'static str = "/api/paper/template/meeting"; + pub const PAPER_TEMPLATE_TODO: &'static str = "/api/paper/template/todo"; + pub const PAPER_TEMPLATE_RESEARCH: &'static str = "/api/paper/template/research"; + pub const PAPER_AI_SUMMARIZE: &'static str = "/api/paper/ai/summarize"; + pub const PAPER_AI_EXPAND: &'static str = "/api/paper/ai/expand"; + pub const PAPER_AI_IMPROVE: &'static str = "/api/paper/ai/improve"; + pub const PAPER_AI_SIMPLIFY: &'static str = "/api/paper/ai/simplify"; + pub const PAPER_AI_TRANSLATE: &'static str = "/api/paper/ai/translate"; + pub const PAPER_AI_CUSTOM: &'static str = "/api/paper/ai/custom"; + pub const PAPER_EXPORT_PDF: &'static str = "/api/paper/export/pdf"; + pub const PAPER_EXPORT_DOCX: &'static str = "/api/paper/export/docx"; + pub const PAPER_EXPORT_MD: &'static str = "/api/paper/export/md"; + pub const PAPER_EXPORT_HTML: &'static str = "/api/paper/export/html"; + pub const PAPER_EXPORT_TXT: &'static str = "/api/paper/export/txt"; + + pub const RESEARCH_COLLECTIONS: &'static str = "/api/research/collections"; + pub const RESEARCH_COLLECTIONS_NEW: &'static str = "/api/research/collections/new"; + pub const RESEARCH_COLLECTION_BY_ID: &'static str = "/api/research/collections/:id"; + pub const RESEARCH_SEARCH: &'static str = "/api/research/search"; + pub const RESEARCH_RECENT: &'static str = "/api/research/recent"; + pub const RESEARCH_TRENDING: &'static str = "/api/research/trending"; + pub const RESEARCH_PROMPTS: &'static str = "/api/research/prompts"; + + pub const SOURCES_PROMPTS: &'static str = "/api/sources/prompts"; + pub const SOURCES_TEMPLATES: &'static str = "/api/sources/templates"; + pub const SOURCES_NEWS: &'static str = "/api/sources/news"; + pub const SOURCES_MCP_SERVERS: &'static str = "/api/sources/mcp-servers"; + pub const SOURCES_LLM_TOOLS: &'static str = "/api/sources/llm-tools"; + pub const SOURCES_MODELS: &'static str = "/api/sources/models"; + pub const SOURCES_SEARCH: &'static str = "/api/sources/search"; + pub const SOURCES_REPOSITORIES: &'static str = "/api/sources/repositories"; + pub const SOURCES_REPOSITORIES_CONNECT: &'static str = "/api/sources/repositories/connect"; + pub const SOURCES_REPOSITORIES_DISCONNECT: &'static str = + "/api/sources/repositories/disconnect"; + pub const SOURCES_APPS: &'static str = "/api/sources/apps"; + pub const SOURCES_MCP: &'static str = "/api/sources/mcp"; + pub const SOURCES_MCP_BY_NAME: &'static str = "/api/sources/mcp/:name"; + pub const SOURCES_MCP_ENABLE: &'static str = "/api/sources/mcp/:name/enable"; + pub const SOURCES_MCP_DISABLE: &'static str = "/api/sources/mcp/:name/disable"; + pub const SOURCES_MCP_TOOLS: &'static str = "/api/sources/mcp/:name/tools"; + pub const SOURCES_MCP_TEST: &'static str = "/api/sources/mcp/:name/test"; + pub const SOURCES_MCP_SCAN: &'static str = "/api/sources/mcp/scan"; + pub const SOURCES_MCP_EXAMPLES: &'static str = "/api/sources/mcp/examples"; + pub const SOURCES_MENTIONS: &'static str = "/api/sources/mentions"; + pub const SOURCES_TOOLS: &'static str = "/api/sources/tools"; + + pub const TASKS_STATS: &'static str = "/api/tasks/stats"; + pub const TASKS_STATS_JSON: &'static str = "/api/tasks/stats/json"; + pub const TASKS_COMPLETED: &'static str = "/api/tasks/completed"; pub const WS: &'static str = "/ws"; pub const WS_MEET: &'static str = "/ws/meet"; pub const WS_CHAT: &'static str = "/ws/chat"; pub const WS_NOTIFICATIONS: &'static str = "/ws/notifications"; + pub const WS_ATTENDANT: &'static str = "/ws/attendant"; } - #[derive(Debug)] pub struct InternalUrls; @@ -163,20 +298,20 @@ impl InternalUrls { pub const QDRANT: &'static str = "http://localhost:6334"; pub const FORGEJO: &'static str = "http://localhost:3000"; pub const LIVEKIT: &'static str = "http://localhost:7880"; + pub const BOTMODELS_VISION_QRCODE: &'static str = "/api/v1/vision/qrcode"; + pub const BOTMODELS_SPEECH_TO_TEXT: &'static str = "/api/v1/speech/to-text"; + pub const BOTMODELS_VISION_DESCRIBE_VIDEO: &'static str = "/api/v1/vision/describe-video"; } - impl ApiUrls { - pub fn with_params(url: &str, params: &[(&str, &str)]) -> String { let mut result = url.to_string(); for (key, value) in params { - result = result.replace(&format!(":{}", key), value); + result = result.replace(&format!(":{key}"), value); } result } - pub fn with_query(url: &str, params: &[(&str, &str)]) -> String { if params.is_empty() { return url.to_string(); @@ -184,10 +319,10 @@ impl ApiUrls { let query = params .iter() - .map(|(k, v)| format!("{}={}", k, urlencoding::encode(v))) + .map(|(k, v)| format!("{k}={}", urlencoding::encode(v))) .collect::>() .join("&"); - format!("{}?{}", url, query) + format!("{url}?{query}") } } diff --git a/src/designer/mod.rs b/src/designer/mod.rs index 1132a0017..6ff9d23dc 100644 --- a/src/designer/mod.rs +++ b/src/designer/mod.rs @@ -1,3 +1,5 @@ +use crate::auto_task::get_designer_error_context; +use crate::core::urls::ApiUrls; use crate::shared::state::AppState; use axum::{ extract::{Query, State}, @@ -65,19 +67,244 @@ pub struct ValidationWarning { pub node_id: Option, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MagicRequest { + pub nodes: Vec, + pub connections: i32, + pub filename: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EditorMagicRequest { + pub code: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EditorMagicResponse { + pub improved_code: Option, + pub explanation: Option, + pub suggestions: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MagicNode { + #[serde(rename = "type")] + pub node_type: String, + pub fields: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MagicSuggestion { + #[serde(rename = "type")] + pub suggestion_type: String, + pub title: String, + pub description: String, +} + pub fn configure_designer_routes() -> Router> { Router::new() - .route("/api/v1/designer/files", get(handle_list_files)) - .route("/api/v1/designer/load", get(handle_load_file)) - .route("/api/v1/designer/save", post(handle_save)) - .route("/api/v1/designer/validate", post(handle_validate)) - .route("/api/v1/designer/export", get(handle_export)) + .route(ApiUrls::DESIGNER_FILES, get(handle_list_files)) + .route(ApiUrls::DESIGNER_LOAD, get(handle_load_file)) + .route(ApiUrls::DESIGNER_SAVE, post(handle_save)) + .route(ApiUrls::DESIGNER_VALIDATE, post(handle_validate)) + .route(ApiUrls::DESIGNER_EXPORT, get(handle_export)) .route( "/api/designer/dialogs", get(handle_list_dialogs).post(handle_create_dialog), ) .route("/api/designer/dialogs/{id}", get(handle_get_dialog)) - .route("/api/designer/modify", post(handle_designer_modify)) + .route(ApiUrls::DESIGNER_MODIFY, post(handle_designer_modify)) + .route("/api/v1/designer/magic", post(handle_magic_suggestions)) + .route("/api/v1/editor/magic", post(handle_editor_magic)) +} + +pub async fn handle_editor_magic( + State(state): State>, + Json(request): Json, +) -> impl IntoResponse { + let code = request.code; + + if code.trim().is_empty() { + return Json(EditorMagicResponse { + improved_code: None, + explanation: Some("No code provided".to_string()), + suggestions: None, + }); + } + + let prompt = format!( + r#"You are reviewing this HTMX application code. Analyze and improve it. + +Focus on: +- Better HTMX patterns (reduce JS, use hx-* attributes properly) +- Accessibility (ARIA labels, keyboard navigation, semantic HTML) +- Performance (lazy loading, efficient selectors) +- UX (loading states, error handling, user feedback) +- Code organization (clean structure, no comments needed) + +Current code: +``` +{code} +``` + +Respond with JSON only: +{{ + "improved_code": "the improved code here", + "explanation": "brief explanation of changes made" +}} + +If the code is already good, respond with: +{{ + "improved_code": null, + "explanation": "Code looks good, no improvements needed" +}}"# + ); + + #[cfg(feature = "llm")] + { + let config = serde_json::json!({ + "temperature": 0.3, + "max_tokens": 4000 + }); + + match state + .llm_provider + .generate(&prompt, &config, "gpt-4", "") + .await + { + Ok(response) => { + if let Ok(result) = serde_json::from_str::(&response) { + return Json(result); + } + return Json(EditorMagicResponse { + improved_code: Some(response), + explanation: Some("AI suggestions".to_string()), + suggestions: None, + }); + } + Err(e) => { + log::warn!("LLM call failed: {e}"); + } + } + } + + let _ = state; + let mut suggestions = Vec::new(); + + if !code.contains("hx-") { + suggestions.push(MagicSuggestion { + suggestion_type: "ux".to_string(), + title: "Use HTMX attributes".to_string(), + description: "Consider using hx-get, hx-post instead of JavaScript fetch calls." + .to_string(), + }); + } + + if !code.contains("hx-indicator") { + suggestions.push(MagicSuggestion { + suggestion_type: "ux".to_string(), + title: "Add loading indicators".to_string(), + description: "Use hx-indicator to show loading state during requests.".to_string(), + }); + } + + if !code.contains("aria-") && !code.contains("role=") { + suggestions.push(MagicSuggestion { + suggestion_type: "a11y".to_string(), + title: "Improve accessibility".to_string(), + description: "Add ARIA labels and roles for screen reader support.".to_string(), + }); + } + + if code.contains("onclick=") || code.contains("addEventListener") { + suggestions.push(MagicSuggestion { + suggestion_type: "perf".to_string(), + title: "Replace JS with HTMX".to_string(), + description: "HTMX can handle most interactions without custom JavaScript.".to_string(), + }); + } + + Json(EditorMagicResponse { + improved_code: None, + explanation: None, + suggestions: if suggestions.is_empty() { + None + } else { + Some(suggestions) + }, + }) +} + +pub async fn handle_magic_suggestions( + State(state): State>, + Json(request): Json, +) -> impl IntoResponse { + let mut suggestions = Vec::new(); + let nodes = &request.nodes; + + let has_hear = nodes.iter().any(|n| n.node_type == "HEAR"); + let has_talk = nodes.iter().any(|n| n.node_type == "TALK"); + let has_if = nodes + .iter() + .any(|n| n.node_type == "IF" || n.node_type == "SWITCH"); + let talk_count = nodes.iter().filter(|n| n.node_type == "TALK").count(); + + if !has_hear && has_talk { + suggestions.push(MagicSuggestion { + suggestion_type: "ux".to_string(), + title: "Add User Input".to_string(), + description: + "Your dialog has no HEAR nodes. Consider adding user input to make it interactive." + .to_string(), + }); + } + + if talk_count > 5 { + suggestions.push(MagicSuggestion { + suggestion_type: "ux".to_string(), + title: "Break Up Long Responses".to_string(), + description: + "You have many TALK nodes. Consider grouping related messages or using a menu." + .to_string(), + }); + } + + if !has_if && nodes.len() > 3 { + suggestions.push(MagicSuggestion { + suggestion_type: "feature".to_string(), + title: "Add Decision Logic".to_string(), + description: "Add IF or SWITCH nodes to handle different user responses dynamically." + .to_string(), + }); + } + + if request.connections < (nodes.len() as i32 - 1) && nodes.len() > 1 { + suggestions.push(MagicSuggestion { + suggestion_type: "perf".to_string(), + title: "Check Connections".to_string(), + description: "Some nodes may not be connected. Ensure all nodes flow properly." + .to_string(), + }); + } + + if nodes.is_empty() { + suggestions.push(MagicSuggestion { + suggestion_type: "feature".to_string(), + title: "Start with TALK".to_string(), + description: "Begin your dialog with a TALK node to greet the user.".to_string(), + }); + } + + suggestions.push(MagicSuggestion { + suggestion_type: "a11y".to_string(), + title: "Use Clear Language".to_string(), + description: "Keep messages short and clear. Avoid jargon for better accessibility." + .to_string(), + }); + + let _ = state; + + Json(suggestions) } pub async fn handle_list_files(State(state): State>) -> impl IntoResponse { @@ -881,12 +1108,15 @@ fn build_designer_prompt(request: &DesignerModifyRequest) -> String { }) .unwrap_or_default(); + let error_context = get_designer_error_context(&request.app_name).unwrap_or_default(); + format!( r#"You are a Designer AI assistant helping modify an HTMX-based application. App Name: {} Current Page: {} {} +{} User Request: "{}" Analyze the request and respond with JSON describing the changes needed: @@ -915,6 +1145,7 @@ Respond with valid JSON only."#, request.app_name, request.current_page.as_deref().unwrap_or("index.html"), context_info, + error_context, request.message ) } @@ -980,7 +1211,7 @@ async fn parse_and_apply_changes( ) -> Result<(Vec, String, Vec), Box> { #[derive(Deserialize)] struct LlmChangeResponse { - understanding: Option, + _understanding: Option, changes: Option>, message: Option, suggestions: Option>, @@ -996,7 +1227,7 @@ async fn parse_and_apply_changes( } let parsed: LlmChangeResponse = serde_json::from_str(llm_response).unwrap_or(LlmChangeResponse { - understanding: Some("Could not parse LLM response".to_string()), + _understanding: Some("Could not parse LLM response".to_string()), changes: None, message: Some("I understood your request but encountered an issue processing it. Could you try rephrasing?".to_string()), suggestions: Some(vec!["Try being more specific".to_string()]), diff --git a/src/email/mod.rs b/src/email/mod.rs index 6f1c70186..ce368c953 100644 --- a/src/email/mod.rs +++ b/src/email/mod.rs @@ -357,10 +357,6 @@ fn decrypt_password(encrypted: &str) -> Result { }) } -/// Add a new email account. -/// -/// # Errors -/// Returns an error if authentication fails or database operations fail. pub async fn add_email_account( State(state): State>, Json(request): Json, @@ -495,10 +491,6 @@ pub async fn list_email_accounts_htmx(State(state): State>) -> imp axum::response::Html(html) } -/// List all email accounts for the current user. -/// -/// # Errors -/// Returns an error if authentication fails or database operations fail. pub async fn list_email_accounts( State(state): State>, ) -> Result>>, EmailError> { @@ -590,10 +582,6 @@ pub async fn list_email_accounts( })) } -/// Delete an email account. -/// -/// # Errors -/// Returns an error if the account ID is invalid or database operations fail. pub async fn delete_email_account( State(state): State>, Path(account_id): Path, @@ -625,10 +613,6 @@ pub async fn delete_email_account( })) } -/// List emails from a specific account and folder. -/// -/// # Errors -/// Returns an error if the account ID is invalid, IMAP connection fails, or emails cannot be fetched. pub async fn list_emails( State(state): State>, Json(request): Json, @@ -768,10 +752,6 @@ pub async fn list_emails( })) } -/// Send an email from a specific account. -/// -/// # Errors -/// Returns an error if the account ID is invalid, SMTP connection fails, or email cannot be sent. pub async fn send_email( State(state): State>, Json(request): Json, @@ -896,10 +876,6 @@ pub async fn send_email( })) } -/// Save an email draft. -/// -/// # Errors -/// Returns an error if the account ID is invalid, authentication fails, or database operations fail. pub async fn save_draft( State(state): State>, Json(request): Json, @@ -944,10 +920,6 @@ pub async fn save_draft( })) } -/// List all folders for an email account. -/// -/// # Errors -/// Returns an error if the account ID is invalid, IMAP connection fails, or folders cannot be listed. pub async fn list_folders( State(state): State>, Path(account_id): Path, @@ -1011,10 +983,6 @@ pub async fn list_folders( })) } -/// Get the latest email from a specific sender. -/// -/// # Errors -/// Returns an error if the operation fails. pub fn get_latest_email_from( State(_state): State>, Json(_request): Json, diff --git a/src/meet/mod.rs b/src/meet/mod.rs index afe02a164..0940e6087 100644 --- a/src/meet/mod.rs +++ b/src/meet/mod.rs @@ -23,10 +23,9 @@ pub fn configure() -> Router> { .route(ApiUrls::VOICE_STOP, post(voice_stop)) .route(ApiUrls::MEET_CREATE, post(create_meeting)) .route(ApiUrls::MEET_ROOMS, get(list_rooms)) - .route("/api/meet/rooms", get(list_rooms_ui)) - .route("/api/meet/recent", get(recent_meetings)) - .route("/api/meet/participants", get(all_participants)) - .route("/api/meet/scheduled", get(scheduled_meetings)) + .route(ApiUrls::MEET_PARTICIPANTS, get(all_participants)) + .route(ApiUrls::MEET_RECENT, get(recent_meetings)) + .route(ApiUrls::MEET_SCHEDULED, get(scheduled_meetings)) .route( &ApiUrls::MEET_ROOM_BY_ID.replace(":id", "{room_id}"), get(get_room), @@ -182,8 +181,7 @@ pub async fn voice_start( { Ok(token) => { info!( - "Voice session started successfully for session {}", - session_id + "Voice session started successfully for session {session_id}" ); ( StatusCode::OK, @@ -192,8 +190,7 @@ pub async fn voice_start( } Err(e) => { error!( - "Failed to start voice session for session {}: {}", - session_id, e + "Failed to start voice session for session {session_id}: {e}" ); ( StatusCode::INTERNAL_SERVER_ERROR, @@ -215,8 +212,7 @@ pub async fn voice_stop( match data.voice_adapter.stop_voice_session(session_id).await { Ok(()) => { info!( - "Voice session stopped successfully for session {}", - session_id + "Voice session stopped successfully for session {session_id}" ); ( StatusCode::OK, @@ -225,8 +221,7 @@ pub async fn voice_stop( } Err(e) => { error!( - "Failed to stop voice session for session {}: {}", - session_id, e + "Failed to stop voice session for session {session_id}: {e}" ); ( StatusCode::INTERNAL_SERVER_ERROR, @@ -252,7 +247,7 @@ pub async fn create_meeting( (StatusCode::OK, Json(serde_json::json!(room))) } Err(e) => { - error!("Failed to create meeting room: {}", e); + error!("Failed to create meeting room: {e}"); ( StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})), @@ -301,11 +296,11 @@ pub async fn join_room( .await { Ok(participant) => { - info!("Participant {} joined room {}", participant.id, room_id); + info!("Participant {} joined room {room_id}", participant.id); (StatusCode::OK, Json(serde_json::json!(participant))) } Err(e) => { - error!("Failed to join room {}: {}", room_id, e); + error!("Failed to join room {room_id}: {e}"); ( StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})), @@ -322,15 +317,15 @@ pub async fn start_transcription( let meeting_service = MeetingService::new(state.clone(), transcription_service); match meeting_service.start_transcription(&room_id).await { - Ok(_) => { - info!("Started transcription for room {}", room_id); + Ok(()) => { + info!("Started transcription for room {room_id}"); ( StatusCode::OK, Json(serde_json::json!({"status": "transcription_started"})), ) } Err(e) => { - error!("Failed to start transcription for room {}: {}", room_id, e); + error!("Failed to start transcription for room {room_id}: {e}"); ( StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})), @@ -387,10 +382,10 @@ async fn handle_meeting_socket(_socket: axum::extract::ws::WebSocket, _state: Ar info!("Meeting WebSocket connection established"); } -pub async fn list_rooms_ui(State(_state): State>) -> Json { +pub async fn all_participants(State(_state): State>) -> Json { Json(serde_json::json!({ - "rooms": [], - "message": "No active meeting rooms" + "participants": [], + "message": "No participants" })) } @@ -401,13 +396,6 @@ pub async fn recent_meetings(State(_state): State>) -> Json>) -> Json { - Json(serde_json::json!({ - "participants": [], - "message": "No participants" - })) -} - pub async fn scheduled_meetings(State(_state): State>) -> Json { Json(serde_json::json!({ "meetings": [], diff --git a/src/nvidia/mod.rs b/src/nvidia/mod.rs index 9059cda49..9ab80bd30 100644 --- a/src/nvidia/mod.rs +++ b/src/nvidia/mod.rs @@ -8,11 +8,6 @@ pub struct SystemMetrics { pub cpu_usage: f32, } -/// Gets current system metrics including CPU and GPU usage. -/// -/// # Errors -/// -/// Returns an error if GPU utilization query fails when an NVIDIA GPU is present. pub fn get_system_metrics() -> Result { let mut sys = System::new(); sys.refresh_cpu_usage(); @@ -28,7 +23,6 @@ pub fn get_system_metrics() -> Result { }) } -/// Checks if an NVIDIA GPU is present in the system. #[must_use] pub fn has_nvidia_gpu() -> bool { match std::process::Command::new("nvidia-smi") @@ -41,14 +35,6 @@ pub fn has_nvidia_gpu() -> bool { } } -/// Gets GPU utilization metrics from nvidia-smi. -/// -/// # Errors -/// -/// Returns an error if: -/// - The nvidia-smi command fails to execute -/// - The command returns a non-success status -/// - The output cannot be parsed as UTF-8 pub fn get_gpu_utilization() -> Result> { let output = std::process::Command::new("nvidia-smi") .arg("--query-gpu=utilization.gpu,utilization.memory") diff --git a/src/research/mod.rs b/src/research/mod.rs index 56fe6ae44..107e22704 100644 --- a/src/research/mod.rs +++ b/src/research/mod.rs @@ -1,3 +1,5 @@ +pub mod web_search; + use crate::shared::state::AppState; use axum::{ extract::{Path, State}, @@ -57,6 +59,7 @@ pub struct CollectionRow { pub fn configure_research_routes() -> Router> { Router::new() + .merge(web_search::configure_web_search_routes()) .route("/api/research/collections", get(handle_list_collections)) .route( "/api/research/collections/new", diff --git a/src/research/web_search.rs b/src/research/web_search.rs new file mode 100644 index 000000000..ee60613dd --- /dev/null +++ b/src/research/web_search.rs @@ -0,0 +1,638 @@ +use crate::shared::state::AppState; +use axum::{ + extract::{Query, State}, + response::{Html, IntoResponse}, + routing::{get, post}, + Json, Router, +}; +use chrono::{DateTime, Utc}; +use log::{debug, error, info}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fmt::Write; +use std::sync::Arc; +use std::time::Duration; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebSearchRequest { + pub query: String, + pub max_results: Option, + pub region: Option, + pub safe_search: Option, + pub time_range: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebSearchResult { + pub title: String, + pub url: String, + pub snippet: String, + pub source: String, + pub favicon: Option, + pub published_date: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebSearchResponse { + pub results: Vec, + pub query: String, + pub total_results: usize, + pub search_time_ms: u64, + pub source: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SummarizeRequest { + pub query: String, + pub results: Vec, + pub max_length: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SummarizeResponse { + pub summary: String, + pub citations: Vec, + pub confidence: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Citation { + pub index: usize, + pub title: String, + pub url: String, + pub relevance: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeepResearchRequest { + pub query: String, + pub depth: Option, + pub max_sources: Option, + pub follow_links: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeepResearchResponse { + pub answer: String, + pub sources: Vec, + pub citations: Vec, + pub related_queries: Vec, + pub confidence: f32, + pub research_time_ms: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchHistoryEntry { + pub id: String, + pub query: String, + pub results_count: usize, + pub timestamp: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchHistoryQuery { + pub page: Option, + pub per_page: Option, +} + +pub fn configure_web_search_routes() -> Router> { + Router::new() + .route("/api/research/web/search", post(handle_web_search)) + .route("/api/research/web/summarize", post(handle_summarize)) + .route("/api/research/web/deep", post(handle_deep_research)) + .route("/api/research/web/history", get(handle_search_history)) + .route("/api/research/web/instant", get(handle_instant_answer)) +} + +pub async fn handle_web_search( + State(_state): State>, + Json(payload): Json, +) -> impl IntoResponse { + let start_time = std::time::Instant::now(); + + if payload.query.trim().is_empty() { + return Json(WebSearchResponse { + results: Vec::new(), + query: payload.query, + total_results: 0, + search_time_ms: 0, + source: "none".to_string(), + }); + } + + let max_results = payload.max_results.unwrap_or(10).min(25); + let region = payload.region.as_deref().unwrap_or("wt-wt"); + + let results = match search_duckduckgo(&payload.query, max_results, region).await { + Ok(r) => r, + Err(e) => { + error!("DuckDuckGo search failed: {}", e); + Vec::new() + } + }; + + let search_time_ms = start_time.elapsed().as_millis() as u64; + + Json(WebSearchResponse { + total_results: results.len(), + results, + query: payload.query, + search_time_ms, + source: "duckduckgo".to_string(), + }) +} + +pub async fn handle_summarize( + State(_state): State>, + Json(payload): Json, +) -> impl IntoResponse { + if payload.results.is_empty() { + return Json(SummarizeResponse { + summary: "No results to summarize.".to_string(), + citations: Vec::new(), + confidence: 0.0, + }); + } + + let mut combined_text = String::new(); + let mut citations = Vec::new(); + + for (idx, result) in payload.results.iter().enumerate() { + let _ = writeln!(combined_text, "[{}] {}", idx + 1, result.snippet); + citations.push(Citation { + index: idx + 1, + title: result.title.clone(), + url: result.url.clone(), + relevance: 1.0 - (idx as f32 * 0.1).min(0.5), + }); + } + + let max_len = payload.max_length.unwrap_or(500); + let summary = if combined_text.len() > max_len { + let mut truncated = combined_text.chars().take(max_len).collect::(); + if let Some(last_period) = truncated.rfind(". ") { + truncated.truncate(last_period + 1); + } + truncated + } else { + combined_text + }; + + let confidence = (payload.results.len() as f32 / 10.0).min(1.0); + + Json(SummarizeResponse { + summary, + citations, + confidence, + }) +} + +pub async fn handle_deep_research( + State(_state): State>, + Json(payload): Json, +) -> impl IntoResponse { + let start_time = std::time::Instant::now(); + + if payload.query.trim().is_empty() { + return Json(DeepResearchResponse { + answer: "Please provide a research query.".to_string(), + sources: Vec::new(), + citations: Vec::new(), + related_queries: Vec::new(), + confidence: 0.0, + research_time_ms: 0, + }); + } + + let depth = payload.depth.unwrap_or(2).min(3); + let max_sources = payload.max_sources.unwrap_or(10).min(20); + + let mut all_results: Vec = Vec::new(); + let mut seen_urls: std::collections::HashSet = std::collections::HashSet::new(); + + let initial_results = search_duckduckgo(&payload.query, max_sources, "wt-wt") + .await + .unwrap_or_default(); + + for result in initial_results { + if !seen_urls.contains(&result.url) { + seen_urls.insert(result.url.clone()); + all_results.push(result); + } + } + + if depth > 1 { + let related_queries = generate_related_queries(&payload.query); + + for rq in related_queries.iter().take(depth - 1) { + if let Ok(more_results) = search_duckduckgo(rq, 5, "wt-wt").await { + for result in more_results { + if !seen_urls.contains(&result.url) && all_results.len() < max_sources { + seen_urls.insert(result.url.clone()); + all_results.push(result); + } + } + } + } + } + + let mut citations = Vec::new(); + let mut answer_parts: Vec = Vec::new(); + + for (idx, result) in all_results.iter().enumerate() { + if idx < 5 { + answer_parts.push(format!("• {}", result.snippet)); + } + citations.push(Citation { + index: idx + 1, + title: result.title.clone(), + url: result.url.clone(), + relevance: 1.0 - (idx as f32 * 0.05).min(0.5), + }); + } + + let answer = if answer_parts.is_empty() { + format!("No results found for: {}", payload.query) + } else { + format!( + "Based on {} sources about \"{}\":\n\n{}", + all_results.len(), + payload.query, + answer_parts.join("\n\n") + ) + }; + + let related = generate_related_queries(&payload.query); + + let research_time_ms = start_time.elapsed().as_millis() as u64; + let confidence = (citations.len() as f32 / 10.0).min(1.0); + + Json(DeepResearchResponse { + answer, + sources: all_results, + citations, + related_queries: related, + confidence, + research_time_ms, + }) +} + +pub async fn handle_search_history( + State(_state): State>, + Query(params): Query, +) -> impl IntoResponse { + let _page = params.page.unwrap_or(1).max(1); + let _per_page = params.per_page.unwrap_or(20).min(100); + + let history: Vec = vec![ + SearchHistoryEntry { + id: "1".to_string(), + query: "Example search 1".to_string(), + results_count: 10, + timestamp: Utc::now(), + }, + SearchHistoryEntry { + id: "2".to_string(), + query: "Example search 2".to_string(), + results_count: 8, + timestamp: Utc::now(), + }, + ]; + + let mut html = String::new(); + html.push_str("
"); + + if history.is_empty() { + html.push_str("
"); + html.push_str("

No search history yet

"); + html.push_str("
"); + } else { + for entry in &history { + html.push_str("
"); + html.push_str(""); + html.push_str(&html_escape(&entry.query)); + html.push_str(""); + html.push_str(""); + html.push_str(&entry.results_count.to_string()); + html.push_str(" results"); + html.push_str("
"); + } + } + + html.push_str("
"); + Html(html) +} + +pub async fn handle_instant_answer( + State(_state): State>, + Query(params): Query>, +) -> impl IntoResponse { + let query = params.get("q").cloned().unwrap_or_default(); + + if query.is_empty() { + return Json(serde_json::json!({ + "answer": null, + "type": "none" + })); + } + + if let Some(answer) = get_instant_answer(&query).await { + Json(serde_json::json!({ + "answer": answer.0, + "type": answer.1, + "source": "duckduckgo" + })) + } else { + Json(serde_json::json!({ + "answer": null, + "type": "none" + })) + } +} + +async fn search_duckduckgo( + query: &str, + max_results: usize, + region: &str, +) -> Result, Box> { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") + .build()?; + + let encoded_query = urlencoding::encode(query); + let url = format!( + "https://html.duckduckgo.com/html/?q={}&kl={}", + encoded_query, region + ); + + debug!("Searching DuckDuckGo: {}", query); + + let response = client.get(&url).send().await?; + let html = response.text().await?; + + let results = parse_duckduckgo_html(&html, max_results); + + info!( + "DuckDuckGo search for '{}' returned {} results", + query, + results.len() + ); + + Ok(results) +} + +fn parse_duckduckgo_html(html: &str, max_results: usize) -> Vec { + let mut results = Vec::new(); + + let mut current_title = String::new(); + let mut current_url = String::new(); + let mut current_snippet = String::new(); + + for line in html.lines() { + let line = line.trim(); + + if line.contains("class=\"result__a\"") { + if let Some(href_start) = line.find("href=\"") { + let start = href_start + 6; + if let Some(href_end) = line[start..].find('"') { + let raw_url = &line[start..start + href_end]; + current_url = decode_ddg_url(raw_url); + } + } + + if let Some(title_start) = line.find('>') { + let after_tag = &line[title_start + 1..]; + if let Some(title_end) = after_tag.find('<') { + current_title = html_decode(&after_tag[..title_end]); + } + } + } + + if line.contains("class=\"result__snippet\"") { + if let Some(snippet_start) = line.find('>') { + let after_tag = &line[snippet_start + 1..]; + let snippet_text = strip_html_inline(after_tag); + current_snippet = html_decode(&snippet_text); + } + + if !current_title.is_empty() && !current_url.is_empty() { + let domain = extract_domain(¤t_url); + results.push(WebSearchResult { + title: current_title.clone(), + url: current_url.clone(), + snippet: current_snippet.clone(), + source: domain.clone(), + favicon: Some(format!( + "https://www.google.com/s2/favicons?domain={}", + domain + )), + published_date: None, + }); + + current_title.clear(); + current_url.clear(); + current_snippet.clear(); + + if results.len() >= max_results { + break; + } + } + } + } + + if results.is_empty() { + results = parse_duckduckgo_fallback(html, max_results); + } + + results +} + +fn parse_duckduckgo_fallback(html: &str, max_results: usize) -> Vec { + let mut results = Vec::new(); + + let parts: Vec<&str> = html.split("class=\"result ").collect(); + + for part in parts.iter().skip(1).take(max_results) { + let mut title = String::new(); + let mut url = String::new(); + let mut snippet = String::new(); + + if let Some(a_start) = part.find("class=\"result__a\"") { + let section = &part[a_start..]; + + if let Some(href_pos) = section.find("href=\"") { + let start = href_pos + 6; + if let Some(end) = section[start..].find('"') { + url = decode_ddg_url(§ion[start..start + end]); + } + } + + if let Some(text_start) = section.find('>') { + let after = §ion[text_start + 1..]; + if let Some(text_end) = after.find('<') { + title = html_decode(&after[..text_end]); + } + } + } + + if let Some(snippet_start) = part.find("class=\"result__snippet\"") { + let section = &part[snippet_start..]; + if let Some(text_start) = section.find('>') { + let after = §ion[text_start + 1..]; + let text = strip_html_inline(after); + snippet = html_decode(&text); + if let Some(end) = snippet.find(" String { + if raw_url.starts_with("//duckduckgo.com/l/?uddg=") { + let encoded_part = raw_url.trim_start_matches("//duckduckgo.com/l/?uddg="); + if let Some(amp_pos) = encoded_part.find('&') { + let url_part = &encoded_part[..amp_pos]; + return urlencoding::decode(url_part) + .map(|s| s.to_string()) + .unwrap_or_else(|_| raw_url.to_string()); + } + return urlencoding::decode(encoded_part) + .map(|s| s.to_string()) + .unwrap_or_else(|_| raw_url.to_string()); + } + + if raw_url.starts_with("http") { + return raw_url.to_string(); + } + + format!("https:{}", raw_url) +} + +fn extract_domain(url: &str) -> String { + let without_protocol = url + .trim_start_matches("https://") + .trim_start_matches("http://"); + + if let Some(slash_pos) = without_protocol.find('/') { + without_protocol[..slash_pos].to_string() + } else { + without_protocol.to_string() + } +} + +fn strip_html_inline(s: &str) -> String { + let mut result = String::new(); + let mut in_tag = false; + + for c in s.chars() { + match c { + '<' => in_tag = true, + '>' => in_tag = false, + _ if !in_tag => result.push(c), + _ => {} + } + } + + result.trim().to_string() +} + +fn html_decode(s: &str) -> String { + s.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace(" ", " ") + .replace("'", "'") + .replace("/", "/") +} + +fn html_escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +fn generate_related_queries(query: &str) -> Vec { + let base_words: Vec<&str> = query.split_whitespace().collect(); + + let mut related = Vec::new(); + + related.push(format!("what is {}", query)); + related.push(format!("{} explained", query)); + related.push(format!("{} examples", query)); + related.push(format!("how does {} work", query)); + related.push(format!("{} vs alternatives", query)); + + if base_words.len() > 2 { + let shortened: String = base_words[..2].join(" "); + related.push(shortened); + } + + related.into_iter().take(5).collect() +} + +async fn get_instant_answer(query: &str) -> Option<(String, String)> { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .ok()?; + + let encoded = urlencoding::encode(query); + let url = format!( + "https://api.duckduckgo.com/?q={}&format=json&no_html=1&skip_disambig=1", + encoded + ); + + let response = client.get(&url).send().await.ok()?; + let json: serde_json::Value = response.json().await.ok()?; + + if let Some(abstract_text) = json.get("AbstractText").and_then(|v| v.as_str()) { + if !abstract_text.is_empty() { + let answer_type = json + .get("Type") + .and_then(|v| v.as_str()) + .unwrap_or("A") + .to_string(); + return Some((abstract_text.to_string(), answer_type)); + } + } + + if let Some(answer) = json.get("Answer").and_then(|v| v.as_str()) { + if !answer.is_empty() { + return Some((answer.to_string(), "answer".to_string())); + } + } + + if let Some(definition) = json.get("Definition").and_then(|v| v.as_str()) { + if !definition.is_empty() { + return Some((definition.to_string(), "definition".to_string())); + } + } + + None +} diff --git a/src/sources/knowledge_base.rs b/src/sources/knowledge_base.rs new file mode 100644 index 000000000..0807b0718 --- /dev/null +++ b/src/sources/knowledge_base.rs @@ -0,0 +1,1047 @@ +use crate::shared::state::AppState; +use axum::{ + extract::{Multipart, Path, Query, State}, + response::{Html, IntoResponse}, + routing::{delete, get, post}, + Json, Router, +}; +use chrono::{DateTime, Utc}; +use diesel::prelude::*; +use log::{error, info, warn}; +use serde::{Deserialize, Serialize}; +use std::fmt::Write; +use std::sync::Arc; +use uuid::Uuid; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KnowledgeSource { + pub id: String, + pub name: String, + pub source_type: SourceType, + pub file_path: Option, + pub url: Option, + pub content_hash: String, + pub chunk_count: i32, + pub status: SourceStatus, + pub created_at: DateTime, + pub updated_at: DateTime, + pub indexed_at: Option>, + pub metadata: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SourceType { + Pdf, + Docx, + Txt, + Markdown, + Html, + Csv, + Xlsx, + Url, + Custom, +} + +impl std::fmt::Display for SourceType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Pdf => write!(f, "pdf"), + Self::Docx => write!(f, "docx"), + Self::Txt => write!(f, "txt"), + Self::Markdown => write!(f, "markdown"), + Self::Html => write!(f, "html"), + Self::Csv => write!(f, "csv"), + Self::Xlsx => write!(f, "xlsx"), + Self::Url => write!(f, "url"), + Self::Custom => write!(f, "custom"), + } + } +} + +impl From<&str> for SourceType { + fn from(s: &str) -> Self { + match s.to_lowercase().as_str() { + "pdf" => Self::Pdf, + "docx" | "doc" => Self::Docx, + "txt" | "text" => Self::Txt, + "md" | "markdown" => Self::Markdown, + "html" | "htm" => Self::Html, + "csv" => Self::Csv, + "xlsx" | "xls" => Self::Xlsx, + "url" | "web" => Self::Url, + _ => Self::Custom, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SourceStatus { + Pending, + Processing, + Indexed, + Failed, + Reindexing, +} + +impl std::fmt::Display for SourceStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Pending => write!(f, "pending"), + Self::Processing => write!(f, "processing"), + Self::Indexed => write!(f, "indexed"), + Self::Failed => write!(f, "failed"), + Self::Reindexing => write!(f, "reindexing"), + } + } +} + +impl From<&str> for SourceStatus { + fn from(s: &str) -> Self { + match s.to_lowercase().as_str() { + "processing" => Self::Processing, + "indexed" => Self::Indexed, + "failed" => Self::Failed, + "reindexing" => Self::Reindexing, + _ => Self::Pending, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DocumentChunk { + pub id: String, + pub source_id: String, + pub chunk_index: i32, + pub content: String, + pub token_count: i32, + pub embedding: Option>, + pub metadata: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UploadResponse { + pub success: bool, + pub source_id: Option, + pub message: String, + pub chunks_created: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryRequest { + pub query: String, + pub collection: Option, + pub top_k: Option, + pub min_score: Option, + pub include_metadata: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryResult { + pub content: String, + pub source_name: String, + pub source_id: String, + pub chunk_index: i32, + pub score: f32, + pub metadata: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryResponse { + pub results: Vec, + pub query: String, + pub total_results: usize, + pub processing_time_ms: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListSourcesQuery { + pub status: Option, + pub source_type: Option, + pub page: Option, + pub per_page: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReindexRequest { + pub source_ids: Option>, + pub force: Option, +} + +#[derive(Debug, QueryableByName)] +#[diesel(check_for_backend(diesel::pg::Pg))] +struct KnowledgeSourceRow { + #[diesel(sql_type = diesel::sql_types::Text)] + id: String, + #[diesel(sql_type = diesel::sql_types::Text)] + name: String, + #[diesel(sql_type = diesel::sql_types::Text)] + source_type: String, + #[diesel(sql_type = diesel::sql_types::Nullable)] + _file_path: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + _url: Option, + #[diesel(sql_type = diesel::sql_types::Text)] + _content_hash: String, + #[diesel(sql_type = diesel::sql_types::Integer)] + chunk_count: i32, + #[diesel(sql_type = diesel::sql_types::Text)] + status: String, + #[diesel(sql_type = diesel::sql_types::Text)] + created_at: String, + #[diesel(sql_type = diesel::sql_types::Text)] + _updated_at: String, + #[diesel(sql_type = diesel::sql_types::Nullable)] + indexed_at: Option, +} + +#[derive(Debug, QueryableByName)] +#[diesel(check_for_backend(diesel::pg::Pg))] +struct _ChunkRow { + #[diesel(sql_type = diesel::sql_types::Text)] + _id: String, + #[diesel(sql_type = diesel::sql_types::Text)] + _source_id: String, + #[diesel(sql_type = diesel::sql_types::Integer)] + _chunk_index: i32, + #[diesel(sql_type = diesel::sql_types::Text)] + _content: String, + #[diesel(sql_type = diesel::sql_types::Integer)] + _token_count: i32, +} + +#[derive(Debug, QueryableByName)] +#[diesel(check_for_backend(diesel::pg::Pg))] +struct SearchResultRow { + #[diesel(sql_type = diesel::sql_types::Text)] + _chunk_id: String, + #[diesel(sql_type = diesel::sql_types::Text)] + content: String, + #[diesel(sql_type = diesel::sql_types::Text)] + source_id: String, + #[diesel(sql_type = diesel::sql_types::Text)] + source_name: String, + #[diesel(sql_type = diesel::sql_types::Integer)] + chunk_index: i32, + #[diesel(sql_type = diesel::sql_types::Float)] + score: f32, +} + +pub fn configure_knowledge_base_routes() -> Router> { + Router::new() + .route("/api/sources/kb/upload", post(handle_upload_document)) + .route("/api/sources/kb/list", get(handle_list_sources)) + .route("/api/sources/kb/query", post(handle_query_knowledge_base)) + .route("/api/sources/kb/:id", get(handle_get_source)) + .route("/api/sources/kb/:id", delete(handle_delete_source)) + .route("/api/sources/kb/reindex", post(handle_reindex_sources)) + .route("/api/sources/kb/stats", get(handle_get_stats)) +} + +pub async fn handle_upload_document( + State(state): State>, + mut multipart: Multipart, +) -> impl IntoResponse { + let mut file_name = String::new(); + let mut file_data: Vec = Vec::new(); + let mut collection = "default".to_string(); + + while let Ok(Some(field)) = multipart.next_field().await { + let name = field.name().unwrap_or("").to_string(); + + match name.as_str() { + "file" => { + file_name = field.file_name().unwrap_or("unknown").to_string(); + if let Ok(data) = field.bytes().await { + file_data = data.to_vec(); + } + } + "collection" => { + if let Ok(text) = field.text().await { + collection = text; + } + } + _ => {} + } + } + + if file_data.is_empty() { + return Json(UploadResponse { + success: false, + source_id: None, + message: "No file provided".to_string(), + chunks_created: None, + }); + } + + let extension = file_name.rsplit('.').next().unwrap_or("txt").to_lowercase(); + let source_type = SourceType::from(extension.as_str()); + + let content = match extract_text_content(&file_data, &source_type) { + Ok(text) => text, + Err(e) => { + error!("Failed to extract text from {}: {}", file_name, e); + return Json(UploadResponse { + success: false, + source_id: None, + message: format!("Failed to extract text: {}", e), + chunks_created: None, + }); + } + }; + + let content_hash = compute_content_hash(&content); + let source_id = Uuid::new_v4().to_string(); + + let chunks = chunk_text(&content, 512, 50); + let chunk_count = chunks.len() as i32; + + let conn = state.conn.clone(); + let source_id_clone = source_id.clone(); + let file_name_clone = file_name.clone(); + let source_type_str = source_type.to_string(); + let content_hash_clone = content_hash.clone(); + let collection_clone = collection.clone(); + let chunks_clone = chunks.clone(); + + let result = tokio::task::spawn_blocking(move || { + let mut db_conn = match conn.get() { + Ok(c) => c, + Err(e) => { + error!("DB connection error: {}", e); + return Err(format!("Database connection error: {}", e)); + } + }; + + let insert_result = diesel::sql_query( + "INSERT INTO knowledge_sources (id, name, source_type, content_hash, chunk_count, status, collection, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, 'processing', $6, NOW(), NOW()) + ON CONFLICT (id) DO NOTHING" + ) + .bind::(&source_id_clone) + .bind::(&file_name_clone) + .bind::(&source_type_str) + .bind::(&content_hash_clone) + .bind::(chunk_count) + .bind::(&collection_clone) + .execute(&mut db_conn); + + if let Err(e) = insert_result { + error!("Failed to insert source: {}", e); + return Err(format!("Failed to insert source: {}", e)); + } + + for (idx, chunk_content) in chunks_clone.iter().enumerate() { + let chunk_id = Uuid::new_v4().to_string(); + let token_count = estimate_tokens(chunk_content); + + let chunk_result = diesel::sql_query( + "INSERT INTO knowledge_chunks (id, source_id, chunk_index, content, token_count, created_at) + VALUES ($1, $2, $3, $4, $5, NOW()) + ON CONFLICT (id) DO NOTHING" + ) + .bind::(&chunk_id) + .bind::(&source_id_clone) + .bind::(idx as i32) + .bind::(chunk_content) + .bind::(token_count) + .execute(&mut db_conn); + + if let Err(e) = chunk_result { + warn!("Failed to insert chunk {}: {}", idx, e); + } + } + + let _ = diesel::sql_query( + "UPDATE knowledge_sources SET status = 'indexed', indexed_at = NOW(), updated_at = NOW() WHERE id = $1" + ) + .bind::(&source_id_clone) + .execute(&mut db_conn); + + Ok(()) + }) + .await; + + match result { + Ok(Ok(())) => { + info!( + "Successfully ingested {} with {} chunks", + file_name, chunk_count + ); + Json(UploadResponse { + success: true, + source_id: Some(source_id), + message: format!( + "Successfully ingested '{}' with {} chunks", + file_name, chunk_count + ), + chunks_created: Some(chunk_count), + }) + } + Ok(Err(e)) => Json(UploadResponse { + success: false, + source_id: None, + message: e, + chunks_created: None, + }), + Err(e) => Json(UploadResponse { + success: false, + source_id: None, + message: format!("Task error: {}", e), + chunks_created: None, + }), + } +} + +pub async fn handle_list_sources( + State(state): State>, + Query(params): Query, +) -> impl IntoResponse { + let conn = state.conn.clone(); + let status_filter = params.status.clone(); + let type_filter = params.source_type.clone(); + let page = params.page.unwrap_or(1).max(1); + let per_page = params.per_page.unwrap_or(20).min(100); + let offset = (page - 1) * per_page; + + let sources = tokio::task::spawn_blocking(move || { + let mut db_conn = match conn.get() { + Ok(c) => c, + Err(e) => { + error!("DB connection error: {}", e); + return Vec::new(); + } + }; + + let mut query = String::from( + "SELECT id, name, source_type, file_path, url, content_hash, chunk_count, status, + created_at::text, updated_at::text, indexed_at::text + FROM knowledge_sources WHERE 1=1", + ); + + if let Some(ref status) = status_filter { + let _ = write!(query, " AND status = '{}'", status.replace('\'', "''")); + } + if let Some(ref stype) = type_filter { + let _ = write!(query, " AND source_type = '{}'", stype.replace('\'', "''")); + } + + let _ = write!(query, " ORDER BY created_at DESC LIMIT {per_page} OFFSET {offset}"); + + diesel::sql_query(&query) + .load::(&mut db_conn) + .unwrap_or_default() + }) + .await + .unwrap_or_default(); + + let mut html = String::new(); + html.push_str("
"); + + if sources.is_empty() { + html.push_str("
"); + html.push_str("

No knowledge sources found

"); + html.push_str("

Upload documents to build your knowledge base

"); + html.push_str("
"); + } else { + for source in &sources { + let status_class = match source.status.as_str() { + "indexed" => "status-success", + "processing" => "status-processing", + "failed" => "status-error", + "pending" => "status-pending", + _ => "status-unknown", + }; + + let type_icon = match source.source_type.as_str() { + "pdf" => "📄", + "docx" | "doc" => "📝", + "txt" | "text" => "📃", + "markdown" | "md" => "📋", + "html" => "🌐", + "csv" => "📊", + "xlsx" | "xls" => "📈", + "url" => "🔗", + _ => "📁", + }; + + html.push_str("
"); + + html.push_str("
"); + html.push_str(type_icon); + html.push_str("
"); + + html.push_str("
"); + html.push_str("

"); + html.push_str(&html_escape(&source.name)); + html.push_str("

"); + html.push_str("
"); + html.push_str(""); + html.push_str(&html_escape(&source.source_type)); + html.push_str(""); + html.push_str(""); + html.push_str(&source.chunk_count.to_string()); + html.push_str(" chunks"); + html.push_str(""); + html.push_str(&html_escape(&source.status)); + html.push_str(""); + html.push_str("
"); + html.push_str("
"); + + html.push_str("
"); + html.push_str(""); + + html.push_str(""); + html.push_str("
"); + + html.push_str("
"); + } + } + + html.push_str("
"); + Html(html) +} + +pub async fn handle_query_knowledge_base( + State(state): State>, + Json(payload): Json, +) -> impl IntoResponse { + let start_time = std::time::Instant::now(); + + if payload.query.trim().is_empty() { + return Json(QueryResponse { + results: Vec::new(), + query: payload.query, + total_results: 0, + processing_time_ms: 0, + }); + } + + let conn = state.conn.clone(); + let query = payload.query.clone(); + let top_k = payload.top_k.unwrap_or(5).min(20); + let min_score = payload.min_score.unwrap_or(0.0); + let collection = payload.collection.clone(); + + let results = tokio::task::spawn_blocking(move || { + let mut db_conn = match conn.get() { + Ok(c) => c, + Err(e) => { + error!("DB connection error: {}", e); + return Vec::new(); + } + }; + + let search_pattern = format!("%{}%", query.to_lowercase()); + + let mut sql = String::from( + "SELECT + kc.id as chunk_id, + kc.content, + kc.source_id, + ks.name as source_name, + kc.chunk_index, + CAST(ts_rank(to_tsvector('english', kc.content), plainto_tsquery('english', $1)) AS FLOAT4) as score + FROM knowledge_chunks kc + JOIN knowledge_sources ks ON kc.source_id = ks.id + WHERE ks.status = 'indexed' + AND (LOWER(kc.content) LIKE $2 + OR to_tsvector('english', kc.content) @@ plainto_tsquery('english', $1))", + ); + + if collection.is_some() { + sql.push_str(" AND ks.collection = $3"); + } + + let _ = write!(sql, " ORDER BY score DESC, kc.chunk_index ASC LIMIT {top_k}"); + + let search_results: Vec = if let Some(ref coll) = collection { + diesel::sql_query(&sql) + .bind::(&query) + .bind::(&search_pattern) + .bind::(coll) + .load(&mut db_conn) + .unwrap_or_default() + } else { + diesel::sql_query(&sql) + .bind::(&query) + .bind::(&search_pattern) + .load(&mut db_conn) + .unwrap_or_default() + }; + + search_results + .into_iter() + .filter(|r| r.score >= min_score) + .map(|r| QueryResult { + content: r.content, + source_name: r.source_name, + source_id: r.source_id, + chunk_index: r.chunk_index, + score: r.score, + metadata: serde_json::json!({}), + }) + .collect::>() + }) + .await + .unwrap_or_default(); + + let processing_time_ms = start_time.elapsed().as_millis() as u64; + + Json(QueryResponse { + total_results: results.len(), + results, + query: payload.query, + processing_time_ms, + }) +} + +pub async fn handle_get_source( + State(state): State>, + Path(id): Path, +) -> impl IntoResponse { + let conn = state.conn.clone(); + + let source = tokio::task::spawn_blocking(move || { + let mut db_conn = match conn.get() { + Ok(c) => c, + Err(e) => { + error!("DB connection error: {}", e); + return None; + } + }; + + let sources: Vec = diesel::sql_query( + "SELECT id, name, source_type, file_path, url, content_hash, chunk_count, status, + created_at::text, updated_at::text, indexed_at::text + FROM knowledge_sources WHERE id = $1", + ) + .bind::(&id) + .load(&mut db_conn) + .unwrap_or_default(); + + sources.into_iter().next() + }) + .await + .unwrap_or(None); + + match source { + Some(s) => { + let mut html = String::new(); + html.push_str("
"); + html.push_str("

"); + html.push_str(&html_escape(&s.name)); + html.push_str("

"); + + html.push_str("
"); + html.push_str("
"); + html.push_str(&html_escape(&s.source_type)); + html.push_str("
"); + + html.push_str("
"); + html.push_str(&html_escape(&s.status)); + html.push_str("
"); + + html.push_str("
"); + html.push_str(&s.chunk_count.to_string()); + html.push_str("
"); + + html.push_str("
"); + html.push_str(&html_escape(&s.created_at)); + html.push_str("
"); + + if let Some(indexed) = &s.indexed_at { + html.push_str("
"); + html.push_str(&html_escape(indexed)); + html.push_str("
"); + } + + html.push_str("
"); + Html(html) + } + None => Html("
Source not found
".to_string()), + } +} + +pub async fn handle_delete_source( + State(state): State>, + Path(id): Path, +) -> impl IntoResponse { + let conn = state.conn.clone(); + + let result = tokio::task::spawn_blocking(move || { + let mut db_conn = match conn.get() { + Ok(c) => c, + Err(e) => { + error!("DB connection error: {}", e); + return false; + } + }; + + let _ = diesel::sql_query("DELETE FROM knowledge_chunks WHERE source_id = $1") + .bind::(&id) + .execute(&mut db_conn); + + let delete_result = diesel::sql_query("DELETE FROM knowledge_sources WHERE id = $1") + .bind::(&id) + .execute(&mut db_conn); + + delete_result.is_ok() + }) + .await + .unwrap_or(false); + + if result { + Html("".to_string()) + } else { + Html("
Failed to delete source
".to_string()) + } +} + +pub async fn handle_reindex_sources( + State(state): State>, + Json(payload): Json, +) -> impl IntoResponse { + let conn = state.conn.clone(); + let source_ids = payload.source_ids.clone(); + + let result = tokio::task::spawn_blocking(move || { + let mut db_conn = match conn.get() { + Ok(c) => c, + Err(e) => { + error!("DB connection error: {}", e); + return 0; + } + }; + + let sql = if let Some(ids) = source_ids { + if ids.is_empty() { + return 0; + } + let placeholders: Vec = ids.iter().map(|id| format!("'{}'", id.replace('\'', "''"))).collect(); + format!( + "UPDATE knowledge_sources SET status = 'reindexing', updated_at = NOW() WHERE id IN ({})", + placeholders.join(",") + ) + } else { + "UPDATE knowledge_sources SET status = 'reindexing', updated_at = NOW() WHERE status = 'indexed'".to_string() + }; + + diesel::sql_query(&sql) + .execute(&mut db_conn) + .unwrap_or(0) + }) + .await + .unwrap_or(0); + + Json(serde_json::json!({ + "success": true, + "sources_queued": result, + "message": format!("{} sources queued for reindexing", result) + })) +} + +pub async fn handle_get_stats(State(state): State>) -> impl IntoResponse { + let conn = state.conn.clone(); + + let stats = tokio::task::spawn_blocking(move || { + let mut db_conn = match conn.get() { + Ok(c) => c, + Err(e) => { + error!("DB connection error: {}", e); + return serde_json::json!({ + "total_sources": 0, + "total_chunks": 0, + "indexed_sources": 0, + "pending_sources": 0, + "failed_sources": 0 + }); + } + }; + + #[derive(Debug, QueryableByName)] + #[diesel(check_for_backend(diesel::pg::Pg))] + struct CountRow { + #[diesel(sql_type = diesel::sql_types::BigInt)] + count: i64, + } + + let total_sources: i64 = diesel::sql_query("SELECT COUNT(*) as count FROM knowledge_sources") + .load::(&mut db_conn) + .map(|v| v.first().map(|r| r.count).unwrap_or(0)) + .unwrap_or(0); + + let total_chunks: i64 = diesel::sql_query("SELECT COUNT(*) as count FROM knowledge_chunks") + .load::(&mut db_conn) + .map(|v| v.first().map(|r| r.count).unwrap_or(0)) + .unwrap_or(0); + + let indexed: i64 = diesel::sql_query("SELECT COUNT(*) as count FROM knowledge_sources WHERE status = 'indexed'") + .load::(&mut db_conn) + .map(|v| v.first().map(|r| r.count).unwrap_or(0)) + .unwrap_or(0); + + let pending: i64 = diesel::sql_query("SELECT COUNT(*) as count FROM knowledge_sources WHERE status IN ('pending', 'processing', 'reindexing')") + .load::(&mut db_conn) + .map(|v| v.first().map(|r| r.count).unwrap_or(0)) + .unwrap_or(0); + + let failed: i64 = diesel::sql_query("SELECT COUNT(*) as count FROM knowledge_sources WHERE status = 'failed'") + .load::(&mut db_conn) + .map(|v| v.first().map(|r| r.count).unwrap_or(0)) + .unwrap_or(0); + + serde_json::json!({ + "total_sources": total_sources, + "total_chunks": total_chunks, + "indexed_sources": indexed, + "pending_sources": pending, + "failed_sources": failed + }) + }) + .await + .unwrap_or_else(|_| serde_json::json!({ + "error": "Failed to fetch stats" + })); + + Json(stats) +} + +fn extract_text_content( + data: &[u8], + source_type: &SourceType, +) -> Result> { + match source_type { + SourceType::Txt | SourceType::Markdown | SourceType::Csv => { + Ok(String::from_utf8_lossy(data).to_string()) + } + SourceType::Html => { + let html = String::from_utf8_lossy(data); + Ok(strip_html_tags(&html)) + } + SourceType::Pdf => { + #[cfg(feature = "drive")] + { + match pdf_extract::extract_text_from_mem(data) { + Ok(text) => Ok(text), + Err(e) => { + warn!("PDF extraction failed: {}", e); + Ok(String::new()) + } + } + } + #[cfg(not(feature = "drive"))] + { + Err("PDF extraction not available without 'drive' feature".into()) + } + } + SourceType::Docx => extract_docx_text(data), + SourceType::Xlsx => extract_xlsx_text(data), + _ => Ok(String::from_utf8_lossy(data).to_string()), + } +} + +fn extract_docx_text( + data: &[u8], +) -> Result> { + use std::io::{Cursor, Read}; + + let cursor = Cursor::new(data); + let mut archive = match zip::ZipArchive::new(cursor) { + Ok(a) => a, + Err(e) => return Err(format!("Failed to open DOCX: {e}").into()), + }; + + let Ok(mut file) = archive.by_name("word/document.xml") else { + return Ok(String::new()); + }; + + let mut xml = String::new(); + file.read_to_string(&mut xml)?; + + let mut in_text = false; + let mut result = String::new(); + + for part in xml.split('<') { + if part.starts_with("w:t") || part.starts_with("w:t ") { + in_text = true; + continue; + } + if part.starts_with("/w:t") { + in_text = false; + result.push(' '); + continue; + } + if part.starts_with("w:p") || part.starts_with("w:p ") { + result.push('\n'); + continue; + } + if in_text { + if let Some(pos) = part.find('>') { + result.push_str(&part[pos + 1..]); + } else { + result.push_str(part); + } + } + } + + Ok(result + .lines() + .map(|l| l.trim()) + .filter(|l| !l.is_empty()) + .collect::>() + .join("\n")) +} + +fn extract_xlsx_text( + data: &[u8], +) -> Result> { + use std::io::{Cursor, Read}; + + let cursor = Cursor::new(data); + let mut archive = match zip::ZipArchive::new(cursor) { + Ok(a) => a, + Err(e) => return Err(format!("Failed to open XLSX: {e}").into()), + }; + + let mut shared_strings: Vec = Vec::new(); + + if let Ok(mut file) = archive.by_name("xl/sharedStrings.xml") { + let mut xml = String::new(); + file.read_to_string(&mut xml)?; + + for part in xml.split("') { + if let Some(end) = part[start..].find("") { + let text = &part[start + 1..start + end]; + shared_strings.push(text.to_string()); + } + } + } + } + + let mut content = String::new(); + + for i in 1..=10 { + let sheet_name = format!("xl/worksheets/sheet{i}.xml"); + let Ok(mut file) = archive.by_name(&sheet_name) else { + break; + }; + + let mut xml = String::new(); + file.read_to_string(&mut xml)?; + + for part in xml.split("") { + if let Some(end) = part.find("") { + let value = &part[..end]; + if let Ok(idx) = value.parse::() { + if let Some(text) = shared_strings.get(idx) { + content.push_str(text); + content.push('\t'); + } + } else { + content.push_str(value); + content.push('\t'); + } + } + } + content.push('\n'); + } + + Ok(content) +} + +fn strip_html_tags(html: &str) -> String { + let mut result = String::new(); + let mut in_tag = false; + + for c in html.chars() { + match c { + '<' => in_tag = true, + '>' => in_tag = false, + _ if !in_tag => result.push(c), + _ => {} + } + } + + result + .lines() + .map(|l| l.trim()) + .filter(|l| !l.is_empty()) + .collect::>() + .join("\n") +} + +fn compute_content_hash(content: &str) -> String { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + content.hash(&mut hasher); + format!("{:x}", hasher.finish()) +} + +fn chunk_text(text: &str, chunk_size: usize, overlap: usize) -> Vec { + let words: Vec<&str> = text.split_whitespace().collect(); + + if words.is_empty() { + return Vec::new(); + } + + if words.len() <= chunk_size { + return vec![words.join(" ")]; + } + + let mut chunks = Vec::new(); + let mut start = 0; + + while start < words.len() { + let end = (start + chunk_size).min(words.len()); + let chunk: String = words[start..end].join(" "); + chunks.push(chunk); + + if end >= words.len() { + break; + } + + start = if overlap < chunk_size { + end - overlap + } else { + end + }; + } + + chunks +} + +fn estimate_tokens(text: &str) -> i32 { + (text.split_whitespace().count() as f32 * 1.3) as i32 +} + +fn html_escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} diff --git a/src/sources/mod.rs b/src/sources/mod.rs index ccdb24deb..93835f0fb 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -1,3 +1,4 @@ +pub mod knowledge_base; pub mod mcp; use crate::basic::keywords::mcp_directory::{generate_example_configs, McpCsvLoader, McpCsvRow}; @@ -148,6 +149,7 @@ pub struct AppInfo { pub fn configure_sources_routes() -> Router> { Router::new() + .merge(knowledge_base::configure_knowledge_base_routes()) .route("/api/sources/prompts", get(handle_prompts)) .route("/api/sources/templates", get(handle_templates)) .route("/api/sources/news", get(handle_news))