diff --git a/config/llm_releases.json b/3rdparty/llm_releases.json similarity index 100% rename from config/llm_releases.json rename to 3rdparty/llm_releases.json diff --git a/3rdparty/mcp_servers.json b/3rdparty/mcp_servers.json new file mode 100644 index 000000000..160f548fc --- /dev/null +++ b/3rdparty/mcp_servers.json @@ -0,0 +1,466 @@ +{ + "mcp_servers": [ + { + "id": "azure-cosmos-db", + "name": "Azure Cosmos DB", + "description": "Enables Agents to interact with and retrieve data from Azure Cosmos DB accounts.", + "icon": "azure-cosmos-db", + "type": "Local", + "category": "Database", + "provider": "Microsoft" + }, + { + "id": "azure-database-postgresql", + "name": "Azure Database for PostgreSQL", + "description": "Enables Agents to interact with and retrieve data from Azure Database for PostgreSQL resources using natural language prompts.", + "icon": "azure-database-postgresql", + "type": "Local", + "category": "Database", + "provider": "Microsoft" + }, + { + "id": "azure-databricks-genie", + "name": "Azure Databricks Genie", + "description": "Azure Databricks Genie MCP server lets AI agents connect to Genie spaces so users can ask natural language questions and get specialized answers from their data easily.", + "icon": "azure-databricks-genie", + "type": "Remote", + "category": "Analytics", + "provider": "Microsoft" + }, + { + "id": "azure-managed-redis", + "name": "Azure Managed Redis", + "description": "Azure Managed Redis MCP Server provides a natural language interface for agentic apps to interact with Azure Managed Redisโ€”a high-speed, in-memory datastore that is ideal for low-latency use cases like agent memory, vector data store and semantic caching.", + "icon": "azure-managed-redis", + "type": "Local", + "category": "Database", + "provider": "Microsoft" + }, + { + "id": "azure-sql", + "name": "Azure SQL MCP Server", + "description": "A secure, self-hosted MCP for interacting with SQL data (Azure SQL, SQL MI, SQL DW, SQL Server).", + "icon": "azure-sql", + "type": "Local", + "category": "Database", + "provider": "Microsoft" + }, + { + "id": "elasticsearch", + "name": "Elasticsearch", + "description": "Search, retrieve, and analyze Elasticsearch data in developer and agentic workflows.", + "icon": "elasticsearch", + "type": "Remote", + "category": "Search", + "provider": "Elastic" + }, + { + "id": "mongodb", + "name": "MongoDB MCP Server", + "description": "MongoDB MCP Server allows any MCP-aware LLM to connect to MongoDB Atlas for admin tasks and to MongoDB databases for data operations, all through natural language.", + "icon": "mongodb", + "type": "Local", + "category": "Database", + "provider": "MongoDB" + }, + { + "id": "pinecone-assistant", + "name": "Pinecone Assistant MCP Server", + "description": "Pinecone Assistant MCP server helps prototype and deploy assistants that retrieve context-aware answers grounded in proprietary data.", + "icon": "pinecone", + "type": "Remote", + "category": "Vector Database", + "provider": "Pinecone" + }, + { + "id": "vercel", + "name": "Vercel", + "description": "With Vercel MCP, you can explore projects, inspect failed deployments, fetch logs, and more right from your AI client.", + "icon": "vercel", + "type": "Remote", + "category": "Deployment", + "provider": "Vercel" + }, + { + "id": "amplitude", + "name": "Amplitude MCP Server", + "description": "Search, access, and get insights on your Amplitude product analytics data.", + "icon": "amplitude", + "type": "Remote", + "category": "Analytics", + "provider": "Amplitude" + }, + { + "id": "atlan", + "name": "Atlan", + "description": "The Atlan MCP server provides a set of tools that enable AI agents to work directly with Atlan metadata. These tools supply real-time context to AI environments, making it easier to search, explore, and update metadata without leaving your workflow.", + "icon": "atlan", + "type": "Remote", + "category": "Data Catalog", + "provider": "Atlan" + }, + { + "id": "atlassian", + "name": "Atlassian", + "description": "Connect to Jira and Confluence for issue tracking and documentation.", + "icon": "atlassian", + "type": "Remote", + "category": "Productivity", + "provider": "Atlassian" + }, + { + "id": "azure-language-foundry", + "name": "Azure Language in Foundry Tools", + "description": "The MCP server enables AI agents to access Azure Language in Foundry Tools for accurate, explainable and compliant NLP capabilities.", + "icon": "azure-language", + "type": "Remote", + "category": "AI/ML", + "provider": "Microsoft" + }, + { + "id": "azure-speech", + "name": "Azure Speech MCP Server", + "description": "A hosted MCP server that exposes Azure Speech capabilities (speech-to-text, text-to-speech and streaming speech I/O) to agents and LLM workflows.", + "icon": "azure-speech", + "type": "Remote", + "category": "AI/ML", + "provider": "Microsoft" + }, + { + "id": "box", + "name": "Box MCP Server", + "description": "Access and manage your Box content with AI-powered tools for file operations, collaboration, and metadata extraction.", + "icon": "box", + "type": "Remote", + "category": "Storage", + "provider": "Box" + }, + { + "id": "cast-imaging", + "name": "CAST Imaging MCP Server", + "description": "Deterministic mapping of application architecture and code objects to support discovery, impact analysis, and technical debt remediation.", + "icon": "cast-imaging", + "type": "Remote", + "category": "DevOps", + "provider": "CAST" + }, + { + "id": "celonis", + "name": "Celonis PI Graph MCP Server", + "description": "Agent toolkit that provides process intelligence context, action triggering, and write-back capabilities into Celonis.", + "icon": "celonis", + "type": "Remote", + "category": "Process Mining", + "provider": "Celonis" + }, + { + "id": "exa", + "name": "Exa Web Search", + "description": "Exa MCP is a powerful web search and web crawling MCP. It lets you do real-time web searches, extract content from any URL, and even run deep research for detailed reports.", + "icon": "exa", + "type": "Remote", + "category": "Search", + "provider": "Exa" + }, + { + "id": "factory-rca", + "name": "Factory RCA MCP", + "description": "Toolset for manufacturing root-cause analysis, anomaly detection, and telemetry-driven recommendations.", + "icon": "factory", + "type": "Remote", + "category": "Manufacturing", + "provider": "Factory" + }, + { + "id": "github", + "name": "GitHub", + "description": "Access GitHub repositories, issues, and pull requests through secure API integration. If you need the GitHub MCP server to access your private repo, make sure you have installed the GitHub app.", + "icon": "github", + "type": "Remote", + "category": "Development", + "provider": "GitHub" + }, + { + "id": "huggingface", + "name": "Hugging Face MCP Server", + "description": "Search through millions of Hugging Face models, datasets, applications and research papers, and use the Spaces applications you've selected.", + "icon": "huggingface", + "type": "Remote", + "category": "AI/ML", + "provider": "Hugging Face" + }, + { + "id": "infobip-rcs", + "name": "Infobip RCS MCP server", + "description": "Infobip RCS MCP server enables seamless integration with our communication platform that allows you to reach your customers globally through RCS.", + "icon": "infobip", + "type": "Remote", + "category": "Communication", + "provider": "Infobip" + }, + { + "id": "infobip-sms", + "name": "Infobip SMS MCP server", + "description": "The Infobip SMS MCP server enables agentic and developer workflows to send and manage SMS messages through Infobip's platform.", + "icon": "infobip", + "type": "Remote", + "category": "Communication", + "provider": "Infobip" + }, + { + "id": "infobip-whatsapp", + "name": "Infobip WhatsApp MCP server", + "description": "Infobip WhatsApp MCP server enables seamless integration with our communication platform that allows you to reach your customers globally through WhatsApp.", + "icon": "infobip", + "type": "Remote", + "category": "Communication", + "provider": "Infobip" + }, + { + "id": "intercom", + "name": "Intercom MCP Server", + "description": "Secure, read-only access to Intercom conversations and contacts for MCP-compatible AI tools.", + "icon": "intercom", + "type": "Remote", + "category": "Customer Support", + "provider": "Intercom" + }, + { + "id": "marketnode", + "name": "Marketnode MCP Server", + "description": "AI-powered document data extraction, workflow automation, transaction management and tokenization for financial institutions and enterprises.", + "icon": "marketnode", + "type": "Remote", + "category": "Finance", + "provider": "Marketnode" + }, + { + "id": "foundry", + "name": "Foundry MCP Server (preview)", + "description": "Foundry MCP Server (preview) offers instant access to model exploration, deployment of models and agents, and performance evaluation. This fully cloud-native MCP server is integrated with Visual Studio Code and Foundry agents, and secured by Microsoft Entra ID, RBAC, and tenant-level conditional access with Azure Policy for enterprise control.", + "icon": "foundry", + "type": "Remote", + "category": "AI/ML", + "provider": "Microsoft" + }, + { + "id": "microsoft-enterprise", + "name": "Microsoft MCP Server for Enterprise", + "description": "Official Microsoft MCP Server to query Microsoft Entra data using natural language.", + "icon": "microsoft", + "type": "Remote", + "category": "Enterprise", + "provider": "Microsoft" + }, + { + "id": "mihcm", + "name": "MiHCM MCP Server", + "description": "Provides secure access to employee and leave management data from the MiHCM HR platform through standardized MCP server.", + "icon": "mihcm", + "type": "Remote", + "category": "HR", + "provider": "MiHCM" + }, + { + "id": "morningstar", + "name": "Morningstar MCP Server", + "description": "Access Morningstar data, research, and capabilities through specialized MCP tools for global securities.", + "icon": "morningstar", + "type": "Remote", + "category": "Finance", + "provider": "Morningstar" + }, + { + "id": "microsoft-sentinel", + "name": "Microsoft Sentinel Data Exploration", + "description": "The data exploration tool collection in the Microsoft Sentinel MCP server lets you search for relevant tables and retrieve data from Microsoft Sentinel's data lake using natural language.", + "icon": "microsoft-sentinel", + "type": "Remote", + "category": "Security", + "provider": "Microsoft" + }, + { + "id": "microsoft-learn", + "name": "Microsoft Learn", + "description": "AI assistant with real-time access to official Microsoft documentation.", + "icon": "microsoft-learn", + "type": "Remote", + "category": "Documentation", + "provider": "Microsoft" + }, + { + "id": "neon", + "name": "Neon", + "description": "Manage and query Neon Postgres databases with natural language.", + "icon": "neon", + "type": "Remote", + "category": "Database", + "provider": "Neon" + }, + { + "id": "netlify", + "name": "Netlify", + "description": "Deploy, secure, and manage websites with Netlify.", + "icon": "netlify", + "type": "Remote", + "category": "Deployment", + "provider": "Netlify" + }, + { + "id": "pipedream", + "name": "Pipedream", + "description": "Securely connect to 10,000+ tools from 3,000+ APIs with Pipedream MCP.", + "icon": "pipedream", + "type": "Remote", + "category": "Integration", + "provider": "Pipedream" + }, + { + "id": "postman", + "name": "Postman", + "description": "Postman's remote MCP server connects AI agents, assistants, and chatbots directly to your APIs on Postman.", + "icon": "postman", + "type": "Remote", + "category": "API", + "provider": "Postman" + }, + { + "id": "sophos-intelix", + "name": "Sophos Intelix MCP Server", + "description": "Sophos Intelix delivers threat intelligence into analyst workflows, enabling agents to access file, URL, and IP reputation and threat analysis.", + "icon": "sophos", + "type": "Remote", + "category": "Security", + "provider": "Sophos" + }, + { + "id": "stripe", + "name": "Stripe", + "description": "Payment processing and financial infrastructure tools.", + "icon": "stripe", + "type": "Remote", + "category": "Payments", + "provider": "Stripe" + }, + { + "id": "supabase", + "name": "Supabase", + "description": "Connect your Supabase projects to AI agents: design tables and migrations; create database branches; build custom APIs with Edge Functions; retrieve logs and more.", + "icon": "supabase", + "type": "Remote", + "category": "Database", + "provider": "Supabase" + }, + { + "id": "tavily", + "name": "Tavily MCP", + "description": "Real-time web search, extraction, crawling and mapping tools for agentic workflows with source citations.", + "icon": "tavily", + "type": "Remote", + "category": "Search", + "provider": "Tavily" + }, + { + "id": "tomtom", + "name": "TomTom Maps", + "description": "Give your application real-time geospatial context from TomTom โ€” including maps, routing, search, geocoding and traffic.", + "icon": "tomtom", + "type": "Remote", + "category": "Maps", + "provider": "TomTom" + }, + { + "id": "wix", + "name": "Wix MCP", + "description": "Unified access to Wix's development ecosystem for documentation, implementation, and site management.", + "icon": "wix", + "type": "Remote", + "category": "Web Development", + "provider": "Wix" + }, + { + "id": "10to8", + "name": "10to8 Appointment Scheduling", + "description": "10to8 is a powerful appointment management, communications & online booking system.", + "icon": "10to8", + "type": "Custom", + "category": "Scheduling", + "provider": "10to8" + }, + { + "id": "1docstop", + "name": "1DocStop", + "description": "The best document management system for your web & mobile apps. Store, Manage, and Access all your documents whenever and wherever you are.", + "icon": "1docstop", + "type": "Custom", + "category": "Document Management", + "provider": "1DocStop" + }, + { + "id": "1me-corporate", + "name": "1Me Corporate", + "description": "1Me is the easiest and fastest way to share your contact information. With 1Me, you can have an unlimited number of contact cards.", + "icon": "1me", + "type": "Custom", + "category": "Contact Management", + "provider": "1Me" + }, + { + "id": "1pt", + "name": "1pt (Independent Publisher)", + "description": "1pt is a URL shortening service and hosts over 15,000+ redirects with 200,000+ visits.", + "icon": "1pt", + "type": "Custom", + "category": "URL Shortener", + "provider": "1pt" + } + ], + "categories": [ + "Database", + "Analytics", + "Search", + "Vector Database", + "Deployment", + "Data Catalog", + "Productivity", + "AI/ML", + "Storage", + "DevOps", + "Process Mining", + "Development", + "Communication", + "Customer Support", + "Finance", + "Enterprise", + "HR", + "Security", + "Documentation", + "Integration", + "API", + "Payments", + "Maps", + "Web Development", + "Scheduling", + "Document Management", + "Contact Management", + "URL Shortener", + "Manufacturing" + ], + "types": [ + { + "id": "Local", + "name": "MCP: Local", + "description": "Runs locally on your machine" + }, + { + "id": "Remote", + "name": "MCP: Remote", + "description": "Hosted remote MCP server" + }, + { + "id": "Custom", + "name": "Custom", + "description": "Custom integration" + } + ] +} diff --git a/Cargo.toml b/Cargo.toml index 442d5204d..8b0dedcf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ features = ["database"] [features] # ===== DEFAULT FEATURE SET ===== -default = ["console", "chat", "automation", "tasks", "drive", "llm", "cache", "progress-bars", "directory"] +default = ["console", "chat", "automation", "tasks", "drive", "llm", "cache", "progress-bars", "directory", "calendar", "meet", "email"] # ===== UI FEATURES ===== console = ["dep:crossterm", "dep:ratatui", "monitoring"] diff --git a/config/directory_config.json b/config/directory_config.json deleted file mode 100644 index 85dbd5b30..000000000 --- a/config/directory_config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "base_url": "http://localhost:8300", - "default_org": { - "id": "353379211173429262", - "name": "default", - "domain": "default.localhost" - }, - "default_user": { - "id": "admin", - "username": "admin", - "email": "admin@localhost", - "password": "", - "first_name": "Admin", - "last_name": "User" - }, - "admin_token": "pY9ruIghlJlAVn-a-vpbtM0L9yQ3WtweXkXJKEk2aVBL4HEeIppxCA8MPx60ZjQJRghq9zU", - "project_id": "", - "client_id": "353379211962023950", - "client_secret": "vD8uaGZubV4pOMqxfFkGc0YOfmzYII8W7L25V7cGWieQlw0UHuvDQkSuJbQ3Rhgp" -} \ No newline at end of file diff --git a/src/auto_task/app_generator.rs b/src/auto_task/app_generator.rs index d43acc5b7..27b27a2aa 100644 --- a/src/auto_task/app_generator.rs +++ b/src/auto_task/app_generator.rs @@ -958,8 +958,8 @@ impl AppGenerator { self.update_item_status(SectionType::DatabaseModels, &table.name, crate::auto_task::ItemStatus::Running); self.broadcast_manifest_update(); - // Sync the individual table - match self.sync_single_table_to_database(table) { + // Sync the individual table to the bot's specific database + match self.sync_single_table_to_database(table, session.bot_id) { Ok(field_count) => { tables_created += 1; fields_added += field_count; @@ -2738,18 +2738,32 @@ NO QUESTIONS. JUST BUILD."# Ok(()) } - /// Sync a single table to database - used for real-time progress updates + /// Sync a single table to the bot's specific database - used for real-time progress updates fn sync_single_table_to_database( &self, table: &TableDefinition, + bot_id: Uuid, ) -> Result> { - let mut conn = self.state.conn.get()?; let create_sql = generate_create_table_sql(table, "postgres"); - sql_query(&create_sql).execute(&mut conn)?; - info!("Created table: {}", table.name); - - Ok(table.fields.len()) + // Try to use bot's specific database first + match self.state.bot_database_manager.create_table_in_bot_database(bot_id, &create_sql) { + Ok(()) => { + info!("Created table '{}' in bot database (bot_id: {})", table.name, bot_id); + Ok(table.fields.len()) + } + Err(e) => { + // Log warning and fall back to main database + warn!( + "Failed to create table '{}' in bot database: {}, falling back to main database", + table.name, e + ); + let mut conn = self.state.conn.get()?; + sql_query(&create_sql).execute(&mut conn)?; + info!("Created table '{}' in main database (fallback)", table.name); + Ok(table.fields.len()) + } + } } fn update_task_app_url( diff --git a/src/calendar/mod.rs b/src/calendar/mod.rs index d7c834e0a..6c6e34ef6 100644 --- a/src/calendar/mod.rs +++ b/src/calendar/mod.rs @@ -515,6 +515,7 @@ pub async fn start_reminder_job(engine: Arc) { pub fn configure_calendar_routes() -> Router> { Router::new() + // JSON APIs .route( ApiUrls::CALENDAR_EVENTS, get(list_events).post(create_event), @@ -525,12 +526,13 @@ pub fn configure_calendar_routes() -> Router> { ) .route(ApiUrls::CALENDAR_EXPORT, get(export_ical)) .route(ApiUrls::CALENDAR_IMPORT, post(import_ical)) - .route(ApiUrls::CALENDAR_CALENDARS, get(list_calendars_api)) - .route(ApiUrls::CALENDAR_UPCOMING, get(upcoming_events_api)) - .route("/ui/calendar/list", get(list_calendars)) - .route("/ui/calendar/upcoming", get(upcoming_events)) - .route("/ui/calendar/event/new", get(new_event_form)) - .route("/ui/calendar/new", get(new_calendar_form)) + .route(ApiUrls::CALENDAR_CALENDARS_JSON, get(list_calendars_api)) + .route(ApiUrls::CALENDAR_UPCOMING_JSON, get(upcoming_events_api)) + // HTMX/HTML APIs + .route(ApiUrls::CALENDAR_CALENDARS, get(list_calendars)) + .route(ApiUrls::CALENDAR_UPCOMING, get(upcoming_events)) + .route(ApiUrls::CALENDAR_NEW_EVENT_FORM, get(new_event_form)) + .route(ApiUrls::CALENDAR_NEW_CALENDAR_FORM, get(new_calendar_form)) } #[cfg(test)] diff --git a/src/core/bot_database.rs b/src/core/bot_database.rs new file mode 100644 index 000000000..ffcbb543a --- /dev/null +++ b/src/core/bot_database.rs @@ -0,0 +1,360 @@ +//! Bot Database Management Module +//! +//! This module handles per-bot database management, including: +//! - Getting bot database names from the bots table +//! - Creating connection pools to bot-specific databases +//! - Ensuring bot databases exist and are properly initialized +//! - Syncing bot databases on server startup + +use diesel::prelude::*; +use diesel::r2d2::{ConnectionManager, Pool}; +use diesel::sql_query; +use diesel::PgConnection; +use log::{error, info}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use uuid::Uuid; + +use crate::shared::utils::DbPool; + +/// Cache for bot database connection pools +pub struct BotDatabaseManager { + /// Main database pool (for accessing bots table) + main_pool: DbPool, + /// Cached connection pools for bot databases + bot_pools: Arc>>, + /// Base connection URL (without database name) + base_url: String, +} + +#[derive(QueryableByName, Debug)] +pub struct BotDatabaseInfo { + #[diesel(sql_type = diesel::sql_types::Uuid)] + pub id: Uuid, + #[diesel(sql_type = diesel::sql_types::Varchar)] + pub name: String, + #[diesel(sql_type = diesel::sql_types::Nullable)] + pub database_name: Option, +} + +#[derive(QueryableByName)] +struct DbExists { + #[diesel(sql_type = diesel::sql_types::Bool)] + exists: bool, +} + +impl BotDatabaseManager { + /// Create a new BotDatabaseManager + pub fn new(main_pool: DbPool, database_url: &str) -> Self { + let base_url = Self::extract_base_url(database_url); + Self { + main_pool, + bot_pools: Arc::new(RwLock::new(HashMap::new())), + base_url, + } + } + + /// Extract base URL without database name + /// Converts "postgres://user:pass@host:port/dbname" to "postgres://user:pass@host:port" + fn extract_base_url(database_url: &str) -> String { + if let Some(last_slash_pos) = database_url.rfind('/') { + // Check if there's a query string + let db_part = &database_url[last_slash_pos..]; + if let Some(query_pos) = db_part.find('?') { + // Keep query string, just remove db name + format!( + "{}{}", + &database_url[..last_slash_pos], + &db_part[query_pos..] + ) + } else { + database_url[..last_slash_pos].to_string() + } + } else { + database_url.to_string() + } + } + + /// Get the database name for a specific bot + pub fn get_bot_database_name( + &self, + bot_id: Uuid, + ) -> Result, Box> { + let mut conn = self.main_pool.get()?; + + let result: Option = sql_query( + "SELECT id, name, database_name FROM bots WHERE id = $1 AND is_active = true", + ) + .bind::(bot_id) + .get_result(&mut conn) + .optional()?; + + Ok(result.and_then(|info| info.database_name)) + } + + /// Get or create a connection pool to a bot's specific database + pub fn get_bot_pool( + &self, + bot_id: Uuid, + ) -> Result> { + // Check cache first + { + let pools = self.bot_pools.read().map_err(|e| format!("Lock error: {}", e))?; + if let Some(pool) = pools.get(&bot_id) { + return Ok(pool.clone()); + } + } + + // Get database name for this bot + let db_name = self.get_bot_database_name(bot_id)? + .ok_or_else(|| format!("No database configured for bot {}", bot_id))?; + + // Create new pool + let pool = self.create_pool_for_database(&db_name)?; + + // Cache it + { + let mut pools = self.bot_pools.write().map_err(|e| format!("Lock error: {}", e))?; + pools.insert(bot_id, pool.clone()); + } + + Ok(pool) + } + + /// Create a connection pool for a specific database + fn create_pool_for_database( + &self, + database_name: &str, + ) -> Result> { + let database_url = format!("{}/{}", self.base_url, database_name); + let manager = ConnectionManager::::new(&database_url); + + Pool::builder() + .max_size(5) // Smaller pool for per-bot databases + .min_idle(Some(0)) + .connection_timeout(std::time::Duration::from_secs(5)) + .idle_timeout(Some(std::time::Duration::from_secs(300))) + .max_lifetime(Some(std::time::Duration::from_secs(1800))) + .build(manager) + .map_err(|e| format!("Failed to create pool for database {}: {}", database_name, e).into()) + } + + /// Create a database if it doesn't exist + pub fn ensure_database_exists( + &self, + database_name: &str, + ) -> Result> { + let safe_db_name: String = database_name + .chars() + .filter(|c| c.is_alphanumeric() || *c == '_') + .collect(); + + if safe_db_name.is_empty() || safe_db_name.len() > 63 { + return Err("Invalid database name".into()); + } + + let mut conn = self.main_pool.get()?; + + // Check if database exists + let check_query = format!( + "SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = '{}') as exists", + safe_db_name + ); + + let exists = sql_query(&check_query) + .get_result::(&mut conn) + .map(|r| r.exists) + .unwrap_or(false); + + if exists { + info!("Database {} already exists", safe_db_name); + return Ok(false); // Already existed + } + + // Create database + let create_query = format!("CREATE DATABASE {}", safe_db_name); + if let Err(e) = sql_query(&create_query).execute(&mut conn) { + let err_str = e.to_string(); + if err_str.contains("already exists") { + info!("Database {} already exists (concurrent creation)", safe_db_name); + return Ok(false); + } + return Err(format!("Failed to create database: {}", e).into()); + } + + info!("Created database: {}", safe_db_name); + Ok(true) // Newly created + } + + /// Generate a database name for a bot + pub fn generate_database_name(bot_name: &str) -> String { + format!( + "bot_{}", + bot_name + .replace('-', "_") + .replace(' ', "_") + .to_lowercase() + .chars() + .filter(|c| c.is_alphanumeric() || *c == '_') + .collect::() + ) + } + + /// Ensure a bot has a database and update the bots table if needed + pub fn ensure_bot_has_database( + &self, + bot_id: Uuid, + bot_name: &str, + ) -> Result> { + // Check if bot already has a database_name + let existing_db_name = self.get_bot_database_name(bot_id)?; + + let db_name = if let Some(name) = existing_db_name { + name + } else { + // Generate and set database name + let new_db_name = Self::generate_database_name(bot_name); + let mut conn = self.main_pool.get()?; + + sql_query("UPDATE bots SET database_name = $1 WHERE id = $2") + .bind::(&new_db_name) + .bind::(bot_id) + .execute(&mut conn)?; + + info!("Set database_name for bot {} to {}", bot_id, new_db_name); + new_db_name + }; + + // Ensure the database exists + self.ensure_database_exists(&db_name)?; + + Ok(db_name) + } + + /// Get all active bots and their database info + pub fn get_all_bots(&self) -> Result, Box> { + let mut conn = self.main_pool.get()?; + + let bots: Vec = sql_query( + "SELECT id, name, database_name FROM bots WHERE is_active = true", + ) + .get_results(&mut conn)?; + + Ok(bots) + } + + /// Sync all bot databases - ensures each bot has a database + /// Call this during server startup + pub fn sync_all_bot_databases(&self) -> Result> { + let bots = self.get_all_bots()?; + let mut result = SyncResult::default(); + + for bot in bots { + match self.ensure_bot_has_database(bot.id, &bot.name) { + Ok(db_name) => { + if bot.database_name.is_none() { + result.databases_created += 1; + info!("Created database for bot {}: {}", bot.name, db_name); + } else { + result.databases_verified += 1; + } + } + Err(e) => { + error!("Failed to ensure database for bot {}: {}", bot.name, e); + result.errors.push(format!("Bot {}: {}", bot.name, e)); + } + } + } + + info!( + "Bot database sync complete: {} created, {} verified, {} errors", + result.databases_created, + result.databases_verified, + result.errors.len() + ); + + Ok(result) + } + + /// Execute a table creation SQL in a bot's database + pub fn create_table_in_bot_database( + &self, + bot_id: Uuid, + create_sql: &str, + ) -> Result<(), Box> { + let pool = self.get_bot_pool(bot_id)?; + let mut conn = pool.get()?; + + sql_query(create_sql).execute(&mut conn)?; + + Ok(()) + } + + /// Clear cached pool for a bot (useful when database is recreated) + pub fn clear_bot_pool_cache(&self, bot_id: Uuid) { + if let Ok(mut pools) = self.bot_pools.write() { + pools.remove(&bot_id); + } + } + + /// Clear all cached pools + pub fn clear_all_pool_caches(&self) { + if let Ok(mut pools) = self.bot_pools.write() { + pools.clear(); + } + } +} + +/// Result of syncing bot databases +#[derive(Default, Debug)] +pub struct SyncResult { + pub databases_created: usize, + pub databases_verified: usize, + pub errors: Vec, +} + +/// Helper function to create a bot database manager from AppState +pub fn create_bot_database_manager(pool: DbPool, database_url: &str) -> BotDatabaseManager { + BotDatabaseManager::new(pool, database_url) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_base_url() { + assert_eq!( + BotDatabaseManager::extract_base_url("postgres://user:pass@localhost:5432/mydb"), + "postgres://user:pass@localhost:5432" + ); + + assert_eq!( + BotDatabaseManager::extract_base_url("postgres://user:pass@localhost:5432/mydb?sslmode=require"), + "postgres://user:pass@localhost:5432?sslmode=require" + ); + + assert_eq!( + BotDatabaseManager::extract_base_url("postgres://user:pass@localhost/mydb"), + "postgres://user:pass@localhost" + ); + } + + #[test] + fn test_generate_database_name() { + assert_eq!( + BotDatabaseManager::generate_database_name("my-bot"), + "bot_my_bot" + ); + + assert_eq!( + BotDatabaseManager::generate_database_name("My Bot 2"), + "bot_my_bot_2" + ); + + assert_eq!( + BotDatabaseManager::generate_database_name("test@bot!"), + "bot_testbot" + ); + } +} diff --git a/src/core/kb/kb_indexer.rs b/src/core/kb/kb_indexer.rs index 24d1d3cbb..a5585d108 100644 --- a/src/core/kb/kb_indexer.rs +++ b/src/core/kb/kb_indexer.rs @@ -183,11 +183,29 @@ impl KbIndexer { MemoryStats::format_bytes(before_embed.rss_bytes) ); + // Re-validate embedding server is still available before generating embeddings + // This prevents memory from being held if server went down during document processing + if !is_embedding_server_ready() { + warn!("[KB_INDEXER] Embedding server became unavailable during indexing, aborting"); + return Err(anyhow::anyhow!( + "Embedding server became unavailable during KB indexing. Processed {} documents before failure.", + indexed_documents + )); + } + trace!("[KB_INDEXER] Calling generate_embeddings for {} chunks...", chunks.len()); - let embeddings = self + let embeddings = match self .embedding_generator .generate_embeddings(&chunks) - .await?; + .await + { + Ok(emb) => emb, + Err(e) => { + warn!("[KB_INDEXER] Embedding generation failed for {}: {}", doc_path, e); + // Continue with next document instead of failing entire batch + continue; + } + }; let after_embed = MemoryStats::current(); trace!("[KB_INDEXER] After generate_embeddings: {} embeddings, RSS={} (delta={})", diff --git a/src/core/mod.rs b/src/core/mod.rs index a8dbda94c..b497ef5bf 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -1,6 +1,7 @@ pub mod automation; pub mod bootstrap; pub mod bot; +pub mod bot_database; pub mod config; pub mod directory; pub mod dns; diff --git a/src/core/shared/state.rs b/src/core/shared/state.rs index 6c1a2d757..eafa3c1ac 100644 --- a/src/core/shared/state.rs +++ b/src/core/shared/state.rs @@ -1,5 +1,6 @@ use crate::auto_task::TaskManifest; use crate::core::bot::channels::{ChannelAdapter, VoiceAdapter, WebChannelAdapter}; +use crate::core::bot_database::BotDatabaseManager; use crate::core::config::AppConfig; use crate::core::kb::KnowledgeBaseManager; use crate::core::session::SessionManager; @@ -328,6 +329,7 @@ pub struct AppState { pub config: Option, pub conn: DbPool, pub database_url: String, + pub bot_database_manager: Arc, pub session_manager: Arc>, pub metrics_collector: MetricsCollector, pub task_scheduler: Option>, @@ -357,6 +359,7 @@ impl Clone for AppState { config: self.config.clone(), conn: self.conn.clone(), database_url: self.database_url.clone(), + bot_database_manager: Arc::clone(&self.bot_database_manager), #[cfg(feature = "cache")] cache: self.cache.clone(), session_manager: Arc::clone(&self.session_manager), @@ -397,6 +400,7 @@ impl std::fmt::Debug for AppState { .field("config", &self.config.is_some()) .field("conn", &"DbPool") .field("database_url", &"[REDACTED]") + .field("bot_database_manager", &"Arc") .field("session_manager", &"Arc>") .field("metrics_collector", &"MetricsCollector") .field("task_scheduler", &self.task_scheduler.is_some()); @@ -533,6 +537,8 @@ impl Default for AppState { let (attendant_tx, _) = broadcast::channel(100); let (task_progress_tx, _) = broadcast::channel(100); + let bot_database_manager = Arc::new(BotDatabaseManager::new(pool.clone(), &database_url)); + Self { #[cfg(feature = "drive")] drive: None, @@ -543,6 +549,7 @@ impl Default for AppState { config: None, conn: pool.clone(), database_url, + bot_database_manager, session_manager: Arc::new(tokio::sync::Mutex::new(session_manager)), metrics_collector: MetricsCollector::new(), task_scheduler: None, diff --git a/src/core/shared/test_utils.rs b/src/core/shared/test_utils.rs index cdee6bb51..a2a92386f 100644 --- a/src/core/shared/test_utils.rs +++ b/src/core/shared/test_utils.rs @@ -1,4 +1,5 @@ use crate::core::bot::channels::{ChannelAdapter, VoiceAdapter, WebChannelAdapter}; +use crate::core::bot_database::BotDatabaseManager; use crate::core::config::AppConfig; use crate::core::session::SessionManager; use crate::core::shared::analytics::MetricsCollector; @@ -188,6 +189,8 @@ impl TestAppStateBuilder { let (task_progress_tx, _) = broadcast::channel(100); + let bot_database_manager = Arc::new(BotDatabaseManager::new(pool.clone(), &database_url)); + Ok(AppState { #[cfg(feature = "drive")] drive: None, @@ -198,6 +201,7 @@ impl TestAppStateBuilder { config: self.config, conn: pool.clone(), database_url, + bot_database_manager, session_manager: Arc::new(tokio::sync::Mutex::new(session_manager)), metrics_collector: MetricsCollector::new(), task_scheduler: None, diff --git a/src/core/urls.rs b/src/core/urls.rs index 45632a7c7..9acd60141 100644 --- a/src/core/urls.rs +++ b/src/core/urls.rs @@ -2,6 +2,7 @@ pub struct ApiUrls; impl ApiUrls { + // User management - JSON APIs pub const USERS: &'static str = "/api/users"; pub const USER_BY_ID: &'static str = "/api/users/:id"; pub const USER_LOGIN: &'static str = "/api/users/login"; @@ -13,6 +14,7 @@ impl ApiUrls { pub const USER_PROVISION: &'static str = "/api/users/provision"; pub const USER_DEPROVISION: &'static str = "/api/users/:id/deprovision"; + // Groups - JSON APIs pub const GROUPS: &'static str = "/api/groups"; pub const GROUP_BY_ID: &'static str = "/api/groups/:id"; pub const GROUP_MEMBERS: &'static str = "/api/groups/:id/members"; @@ -20,6 +22,7 @@ impl ApiUrls { pub const GROUP_REMOVE_MEMBER: &'static str = "/api/groups/:id/members/:user_id"; pub const GROUP_PERMISSIONS: &'static str = "/api/groups/:id/permissions"; + // Auth - JSON APIs pub const AUTH: &'static str = "/api/auth"; pub const AUTH_TOKEN: &'static str = "/api/auth/token"; pub const AUTH_REFRESH: &'static str = "/api/auth/refresh"; @@ -27,12 +30,14 @@ impl ApiUrls { pub const AUTH_OAUTH: &'static str = "/api/auth/oauth"; pub const AUTH_OAUTH_CALLBACK: &'static str = "/api/auth/oauth/callback"; + // Sessions - JSON APIs pub const SESSIONS: &'static str = "/api/sessions"; pub const SESSION_BY_ID: &'static str = "/api/sessions/:id"; pub const SESSION_HISTORY: &'static str = "/api/sessions/:id/history"; pub const SESSION_START: &'static str = "/api/sessions/:id/start"; pub const SESSION_END: &'static str = "/api/sessions/:id/end"; + // Bots - JSON APIs pub const BOTS: &'static str = "/api/bots"; pub const BOT_BY_ID: &'static str = "/api/bots/:id"; pub const BOT_CONFIG: &'static str = "/api/bots/:id/config"; @@ -40,6 +45,7 @@ impl ApiUrls { pub const BOT_LOGS: &'static str = "/api/bots/:id/logs"; pub const BOT_METRICS: &'static str = "/api/bots/:id/metrics"; + // Drive - JSON APIs pub const DRIVE_LIST: &'static str = "/api/drive/list"; pub const DRIVE_UPLOAD: &'static str = "/api/drive/upload"; pub const DRIVE_DOWNLOAD: &'static str = "/api/drive/download/:path"; @@ -50,6 +56,7 @@ impl ApiUrls { pub const DRIVE_SHARE: &'static str = "/api/drive/share"; pub const DRIVE_FILE: &'static str = "/api/drive/file/:path"; + // Email - JSON APIs pub const EMAIL_ACCOUNTS: &'static str = "/api/email/accounts"; pub const EMAIL_ACCOUNT_BY_ID: &'static str = "/api/email/accounts/:id"; pub const EMAIL_LIST: &'static str = "/api/email/list"; @@ -60,6 +67,20 @@ impl ApiUrls { pub const EMAIL_GET: &'static str = "/api/email/get/:campaign_id"; pub const EMAIL_CLICK: &'static str = "/api/email/click/:campaign_id/:email"; + // Email - HTMX/HTML APIs + pub const EMAIL_ACCOUNTS_HTMX: &'static str = "/api/ui/email/accounts"; + pub const EMAIL_LIST_HTMX: &'static str = "/api/ui/email/list"; + pub const EMAIL_FOLDERS_HTMX: &'static str = "/api/ui/email/folders/:account_id"; + pub const EMAIL_COMPOSE_HTMX: &'static str = "/api/ui/email/compose"; + pub const EMAIL_CONTENT_HTMX: &'static str = "/api/ui/email/content/:id"; + pub const EMAIL_LABELS_HTMX: &'static str = "/api/ui/email/labels"; + pub const EMAIL_TEMPLATES_HTMX: &'static str = "/api/ui/email/templates"; + pub const EMAIL_SIGNATURES_HTMX: &'static str = "/api/ui/email/signatures"; + pub const EMAIL_RULES_HTMX: &'static str = "/api/ui/email/rules"; + pub const EMAIL_SEARCH_HTMX: &'static str = "/api/ui/email/search"; + pub const EMAIL_AUTO_RESPONDER_HTMX: &'static str = "/api/ui/email/auto-responder"; + + // Calendar - JSON APIs pub const CALENDAR_EVENTS: &'static str = "/api/calendar/events"; pub const CALENDAR_EVENT_BY_ID: &'static str = "/api/calendar/events/:id"; pub const CALENDAR_REMINDERS: &'static str = "/api/calendar/reminders"; @@ -67,16 +88,32 @@ impl ApiUrls { pub const CALENDAR_SYNC: &'static str = "/api/calendar/sync"; pub const CALENDAR_EXPORT: &'static str = "/api/calendar/export.ics"; pub const CALENDAR_IMPORT: &'static str = "/api/calendar/import"; - pub const CALENDAR_CALENDARS: &'static str = "/api/calendar/calendars"; - pub const CALENDAR_UPCOMING: &'static str = "/api/calendar/events/upcoming"; + pub const CALENDAR_CALENDARS_JSON: &'static str = "/api/calendar/calendars"; + pub const CALENDAR_UPCOMING_JSON: &'static str = "/api/calendar/events/upcoming"; + // Calendar - HTMX/HTML APIs + pub const CALENDAR_CALENDARS: &'static str = "/api/ui/calendar/calendars"; + pub const CALENDAR_UPCOMING: &'static str = "/api/ui/calendar/events/upcoming"; + pub const CALENDAR_NEW_EVENT_FORM: &'static str = "/api/ui/calendar/events/new"; + pub const CALENDAR_NEW_CALENDAR_FORM: &'static str = "/api/ui/calendar/calendars/new"; + + // Tasks - JSON APIs pub const TASKS: &'static str = "/api/tasks"; pub const TASK_BY_ID: &'static str = "/api/tasks/:id"; pub const TASK_ASSIGN: &'static str = "/api/tasks/:id/assign"; pub const TASK_STATUS: &'static str = "/api/tasks/:id/status"; pub const TASK_PRIORITY: &'static str = "/api/tasks/:id/priority"; pub const TASK_COMMENTS: &'static str = "/api/tasks/:id/comments"; + pub const TASKS_STATS_JSON: &'static str = "/api/tasks/stats/json"; + // Tasks - HTMX/HTML APIs + pub const TASKS_LIST_HTMX: &'static str = "/api/ui/tasks"; + pub const TASKS_GET_HTMX: &'static str = "/api/ui/tasks/:id"; + pub const TASKS_STATS: &'static str = "/api/ui/tasks/stats"; + pub const TASKS_COMPLETED: &'static str = "/api/ui/tasks/completed"; + pub const TASKS_TIME_SAVED: &'static str = "/api/ui/tasks/time-saved"; + + // Meet - JSON APIs pub const MEET_CREATE: &'static str = "/api/meet/create"; pub const MEET_ROOMS: &'static str = "/api/meet/rooms"; pub const MEET_ROOM_BY_ID: &'static str = "/api/meet/rooms/:id"; @@ -89,36 +126,42 @@ impl ApiUrls { pub const MEET_RECENT: &'static str = "/api/meet/recent"; pub const MEET_SCHEDULED: &'static str = "/api/meet/scheduled"; + // Voice - JSON APIs pub const VOICE_START: &'static str = "/api/voice/start"; pub const VOICE_STOP: &'static str = "/api/voice/stop"; pub const VOICE_STATUS: &'static str = "/api/voice/status"; + // DNS - JSON APIs pub const DNS_REGISTER: &'static str = "/api/dns/register"; pub const DNS_REMOVE: &'static str = "/api/dns/remove"; pub const DNS_LIST: &'static str = "/api/dns/list"; pub const DNS_UPDATE: &'static str = "/api/dns/update"; + // Analytics - JSON APIs pub const ANALYTICS_DASHBOARD: &'static str = "/api/analytics/dashboard"; pub const ANALYTICS_METRIC: &'static str = "/api/analytics/metric"; - pub const ANALYTICS_MESSAGES_COUNT: &'static str = "/api/analytics/messages/count"; - pub const ANALYTICS_SESSIONS_ACTIVE: &'static str = "/api/analytics/sessions/active"; - pub const ANALYTICS_RESPONSE_AVG: &'static str = "/api/analytics/response/avg"; - pub const ANALYTICS_LLM_TOKENS: &'static str = "/api/analytics/llm/tokens"; - pub const ANALYTICS_STORAGE_USAGE: &'static str = "/api/analytics/storage/usage"; - pub const ANALYTICS_ERRORS_COUNT: &'static str = "/api/analytics/errors/count"; - pub const ANALYTICS_TIMESERIES_MESSAGES: &'static str = "/api/analytics/timeseries/messages"; - pub const ANALYTICS_TIMESERIES_RESPONSE: &'static str = - "/api/analytics/timeseries/response_time"; - pub const ANALYTICS_CHANNELS_DISTRIBUTION: &'static str = - "/api/analytics/channels/distribution"; - pub const ANALYTICS_BOTS_PERFORMANCE: &'static str = "/api/analytics/bots/performance"; - pub const ANALYTICS_ACTIVITY_RECENT: &'static str = "/api/analytics/activity/recent"; - pub const ANALYTICS_QUERIES_TOP: &'static str = "/api/analytics/queries/top"; - pub const ANALYTICS_CHAT: &'static str = "/api/analytics/chat"; - pub const ANALYTICS_LLM_STATS: &'static str = "/api/analytics/llm/stats"; - pub const ANALYTICS_BUDGET_STATUS: &'static str = "/api/analytics/budget/status"; pub const METRICS: &'static str = "/api/metrics"; + // Analytics - HTMX/HTML APIs + pub const ANALYTICS_MESSAGES_COUNT: &'static str = "/api/ui/analytics/messages/count"; + pub const ANALYTICS_SESSIONS_ACTIVE: &'static str = "/api/ui/analytics/sessions/active"; + pub const ANALYTICS_RESPONSE_AVG: &'static str = "/api/ui/analytics/response/avg"; + pub const ANALYTICS_LLM_TOKENS: &'static str = "/api/ui/analytics/llm/tokens"; + pub const ANALYTICS_STORAGE_USAGE: &'static str = "/api/ui/analytics/storage/usage"; + pub const ANALYTICS_ERRORS_COUNT: &'static str = "/api/ui/analytics/errors/count"; + pub const ANALYTICS_TIMESERIES_MESSAGES: &'static str = "/api/ui/analytics/timeseries/messages"; + pub const ANALYTICS_TIMESERIES_RESPONSE: &'static str = + "/api/ui/analytics/timeseries/response_time"; + pub const ANALYTICS_CHANNELS_DISTRIBUTION: &'static str = + "/api/ui/analytics/channels/distribution"; + pub const ANALYTICS_BOTS_PERFORMANCE: &'static str = "/api/ui/analytics/bots/performance"; + pub const ANALYTICS_ACTIVITY_RECENT: &'static str = "/api/ui/analytics/activity/recent"; + pub const ANALYTICS_QUERIES_TOP: &'static str = "/api/ui/analytics/queries/top"; + pub const ANALYTICS_CHAT: &'static str = "/api/ui/analytics/chat"; + pub const ANALYTICS_LLM_STATS: &'static str = "/api/ui/analytics/llm/stats"; + pub const ANALYTICS_BUDGET_STATUS: &'static str = "/api/ui/analytics/budget/status"; + + // Admin - JSON APIs pub const ADMIN_STATS: &'static str = "/api/admin/stats"; pub const ADMIN_USERS: &'static str = "/api/admin/users"; pub const ADMIN_SYSTEM: &'static str = "/api/admin/system"; @@ -127,10 +170,12 @@ impl ApiUrls { pub const ADMIN_SERVICES: &'static str = "/api/admin/services"; pub const ADMIN_AUDIT: &'static str = "/api/admin/audit"; + // Health/Status - JSON APIs pub const HEALTH: &'static str = "/api/health"; pub const STATUS: &'static str = "/api/status"; pub const SERVICES_STATUS: &'static str = "/api/services/status"; + // Knowledge Base - JSON APIs pub const KB_SEARCH: &'static str = "/api/kb/search"; pub const KB_UPLOAD: &'static str = "/api/kb/upload"; pub const KB_DOCUMENTS: &'static str = "/api/kb/documents"; @@ -138,6 +183,7 @@ impl ApiUrls { pub const KB_INDEX: &'static str = "/api/kb/index"; pub const KB_EMBEDDINGS: &'static str = "/api/kb/embeddings"; + // LLM - JSON APIs pub const LLM_CHAT: &'static str = "/api/llm/chat"; pub const LLM_COMPLETIONS: &'static str = "/api/llm/completions"; pub const LLM_EMBEDDINGS: &'static str = "/api/llm/embeddings"; @@ -145,6 +191,7 @@ impl ApiUrls { pub const LLM_GENERATE: &'static str = "/api/llm/generate"; pub const LLM_IMAGE: &'static str = "/api/llm/image"; + // Attendance - JSON APIs pub const ATTENDANCE_QUEUE: &'static str = "/api/attendance/queue"; pub const ATTENDANCE_ATTENDANTS: &'static str = "/api/attendance/attendants"; pub const ATTENDANCE_ASSIGN: &'static str = "/api/attendance/assign"; @@ -159,12 +206,12 @@ impl ApiUrls { pub const ATTENDANCE_LLM_SENTIMENT: &'static str = "/api/attendance/llm/sentiment"; pub const ATTENDANCE_LLM_CONFIG: &'static str = "/api/attendance/llm/config/:bot_id"; + // AutoTask - JSON APIs pub const AUTOTASK_CREATE: &'static str = "/api/autotask/create"; pub const AUTOTASK_CLASSIFY: &'static str = "/api/autotask/classify"; pub const AUTOTASK_COMPILE: &'static str = "/api/autotask/compile"; pub const AUTOTASK_EXECUTE: &'static str = "/api/autotask/execute"; pub const AUTOTASK_SIMULATE: &'static str = "/api/autotask/simulate/:plan_id"; - pub const AUTOTASK_LIST: &'static str = "/api/autotask/list"; pub const AUTOTASK_GET: &'static str = "/api/autotask/tasks/:task_id"; pub const AUTOTASK_STATS: &'static str = "/api/autotask/stats"; pub const AUTOTASK_PAUSE: &'static str = "/api/autotask/:task_id/pause"; @@ -182,102 +229,127 @@ impl ApiUrls { pub const AUTOTASK_PENDING: &'static str = "/api/autotask/pending"; pub const AUTOTASK_PENDING_ITEM: &'static str = "/api/autotask/pending/:item_id"; + // AutoTask - HTMX/HTML APIs + pub const AUTOTASK_LIST: &'static str = "/api/ui/autotask/list"; + + // DB - JSON APIs pub const DB_TABLE: &'static str = "/api/db/:table"; pub const DB_TABLE_RECORD: &'static str = "/api/db/:table/:id"; pub const DB_TABLE_COUNT: &'static str = "/api/db/:table/count"; pub const DB_TABLE_SEARCH: &'static str = "/api/db/:table/search"; - pub const DESIGNER_FILES: &'static str = "/api/v1/designer/files"; - pub const DESIGNER_LOAD: &'static str = "/api/v1/designer/load"; - pub const DESIGNER_SAVE: &'static str = "/api/v1/designer/save"; - pub const DESIGNER_VALIDATE: &'static str = "/api/v1/designer/validate"; - pub const DESIGNER_EXPORT: &'static str = "/api/v1/designer/export"; - pub const DESIGNER_MODIFY: &'static str = "/api/designer/modify"; + // Designer - HTMX/HTML APIs + pub const DESIGNER_FILES: &'static str = "/api/ui/designer/files"; + pub const DESIGNER_LOAD: &'static str = "/api/ui/designer/load"; + pub const DESIGNER_SAVE: &'static str = "/api/ui/designer/save"; + pub const DESIGNER_VALIDATE: &'static str = "/api/ui/designer/validate"; + pub const DESIGNER_EXPORT: &'static str = "/api/ui/designer/export"; + pub const DESIGNER_MODIFY: &'static str = "/api/ui/designer/modify"; + pub const DESIGNER_DIALOGS: &'static str = "/api/ui/designer/dialogs"; + pub const DESIGNER_DIALOG_BY_ID: &'static str = "/api/ui/designer/dialogs/:id"; + // Mail/WhatsApp - JSON APIs pub const MAIL_SEND: &'static str = "/api/mail/send"; pub const WHATSAPP_SEND: &'static str = "/api/whatsapp/send"; + // Files - JSON APIs pub const FILES_BY_ID: &'static str = "/api/files/:id"; + // Messages - JSON APIs pub const MESSAGES: &'static str = "/api/messages"; - pub const DESIGNER_DIALOGS: &'static str = "/api/designer/dialogs"; - pub const DESIGNER_DIALOG_BY_ID: &'static str = "/api/designer/dialogs/:id"; - + // Email Tracking - JSON APIs pub const EMAIL_TRACKING_LIST: &'static str = "/api/email/tracking/list"; pub const EMAIL_TRACKING_STATS: &'static str = "/api/email/tracking/stats"; + // Instagram - JSON APIs pub const INSTAGRAM_WEBHOOK: &'static str = "/api/instagram/webhook"; pub const INSTAGRAM_SEND: &'static str = "/api/instagram/send"; - pub const MONITORING_DASHBOARD: &'static str = "/api/monitoring/dashboard"; - pub const MONITORING_SERVICES: &'static str = "/api/monitoring/services"; - pub const MONITORING_RESOURCES: &'static str = "/api/monitoring/resources"; - pub const MONITORING_LOGS: &'static str = "/api/monitoring/logs"; - pub const MONITORING_LLM: &'static str = "/api/monitoring/llm"; - pub const MONITORING_HEALTH: &'static str = "/api/monitoring/health"; + // Monitoring - HTMX/HTML APIs + pub const MONITORING_DASHBOARD: &'static str = "/api/ui/monitoring/dashboard"; + pub const MONITORING_SERVICES: &'static str = "/api/ui/monitoring/services"; + pub const MONITORING_RESOURCES: &'static str = "/api/ui/monitoring/resources"; + pub const MONITORING_LOGS: &'static str = "/api/ui/monitoring/logs"; + pub const MONITORING_LLM: &'static str = "/api/ui/monitoring/llm"; + pub const MONITORING_HEALTH: &'static str = "/api/ui/monitoring/health"; + // MS Teams - JSON APIs pub const MSTEAMS_MESSAGES: &'static str = "/api/msteams/messages"; pub const MSTEAMS_SEND: &'static str = "/api/msteams/send"; - pub const PAPER_NEW: &'static str = "/api/paper/new"; - pub const PAPER_LIST: &'static str = "/api/paper/list"; - pub const PAPER_SEARCH: &'static str = "/api/paper/search"; - pub const PAPER_SAVE: &'static str = "/api/paper/save"; - pub const PAPER_AUTOSAVE: &'static str = "/api/paper/autosave"; - pub const PAPER_BY_ID: &'static str = "/api/paper/:id"; - pub const PAPER_DELETE: &'static str = "/api/paper/:id/delete"; - pub const PAPER_TEMPLATE_BLANK: &'static str = "/api/paper/template/blank"; - pub const PAPER_TEMPLATE_MEETING: &'static str = "/api/paper/template/meeting"; - pub const PAPER_TEMPLATE_TODO: &'static str = "/api/paper/template/todo"; - pub const PAPER_TEMPLATE_RESEARCH: &'static str = "/api/paper/template/research"; - pub const PAPER_AI_SUMMARIZE: &'static str = "/api/paper/ai/summarize"; - pub const PAPER_AI_EXPAND: &'static str = "/api/paper/ai/expand"; - pub const PAPER_AI_IMPROVE: &'static str = "/api/paper/ai/improve"; - pub const PAPER_AI_SIMPLIFY: &'static str = "/api/paper/ai/simplify"; - pub const PAPER_AI_TRANSLATE: &'static str = "/api/paper/ai/translate"; - pub const PAPER_AI_CUSTOM: &'static str = "/api/paper/ai/custom"; - pub const PAPER_EXPORT_PDF: &'static str = "/api/paper/export/pdf"; - pub const PAPER_EXPORT_DOCX: &'static str = "/api/paper/export/docx"; - pub const PAPER_EXPORT_MD: &'static str = "/api/paper/export/md"; - pub const PAPER_EXPORT_HTML: &'static str = "/api/paper/export/html"; - pub const PAPER_EXPORT_TXT: &'static str = "/api/paper/export/txt"; + // Paper - HTMX/HTML APIs + pub const PAPER_NEW: &'static str = "/api/ui/paper/new"; + pub const PAPER_LIST: &'static str = "/api/ui/paper/list"; + pub const PAPER_SEARCH: &'static str = "/api/ui/paper/search"; + pub const PAPER_SAVE: &'static str = "/api/ui/paper/save"; + pub const PAPER_AUTOSAVE: &'static str = "/api/ui/paper/autosave"; + pub const PAPER_BY_ID: &'static str = "/api/ui/paper/:id"; + pub const PAPER_DELETE: &'static str = "/api/ui/paper/:id/delete"; + pub const PAPER_TEMPLATE_BLANK: &'static str = "/api/ui/paper/template/blank"; + pub const PAPER_TEMPLATE_MEETING: &'static str = "/api/ui/paper/template/meeting"; + pub const PAPER_TEMPLATE_TODO: &'static str = "/api/ui/paper/template/todo"; + pub const PAPER_TEMPLATE_RESEARCH: &'static str = "/api/ui/paper/template/research"; + pub const PAPER_AI_SUMMARIZE: &'static str = "/api/ui/paper/ai/summarize"; + pub const PAPER_AI_EXPAND: &'static str = "/api/ui/paper/ai/expand"; + pub const PAPER_AI_IMPROVE: &'static str = "/api/ui/paper/ai/improve"; + pub const PAPER_AI_SIMPLIFY: &'static str = "/api/ui/paper/ai/simplify"; + pub const PAPER_AI_TRANSLATE: &'static str = "/api/ui/paper/ai/translate"; + pub const PAPER_AI_CUSTOM: &'static str = "/api/ui/paper/ai/custom"; + pub const PAPER_EXPORT_PDF: &'static str = "/api/ui/paper/export/pdf"; + pub const PAPER_EXPORT_DOCX: &'static str = "/api/ui/paper/export/docx"; + pub const PAPER_EXPORT_MD: &'static str = "/api/ui/paper/export/md"; + pub const PAPER_EXPORT_HTML: &'static str = "/api/ui/paper/export/html"; + pub const PAPER_EXPORT_TXT: &'static str = "/api/ui/paper/export/txt"; - pub const RESEARCH_COLLECTIONS: &'static str = "/api/research/collections"; - pub const RESEARCH_COLLECTIONS_NEW: &'static str = "/api/research/collections/new"; - pub const RESEARCH_COLLECTION_BY_ID: &'static str = "/api/research/collections/:id"; - pub const RESEARCH_SEARCH: &'static str = "/api/research/search"; - pub const RESEARCH_RECENT: &'static str = "/api/research/recent"; - pub const RESEARCH_TRENDING: &'static str = "/api/research/trending"; - pub const RESEARCH_PROMPTS: &'static str = "/api/research/prompts"; + // Research - HTMX/HTML APIs + pub const RESEARCH_COLLECTIONS: &'static str = "/api/ui/research/collections"; + pub const RESEARCH_COLLECTIONS_NEW: &'static str = "/api/ui/research/collections/new"; + pub const RESEARCH_COLLECTION_BY_ID: &'static str = "/api/ui/research/collections/:id"; + pub const RESEARCH_SEARCH: &'static str = "/api/ui/research/search"; + pub const RESEARCH_RECENT: &'static str = "/api/ui/research/recent"; + pub const RESEARCH_TRENDING: &'static str = "/api/ui/research/trending"; + pub const RESEARCH_PROMPTS: &'static str = "/api/ui/research/prompts"; + pub const RESEARCH_WEB_SEARCH: &'static str = "/api/ui/research/web/search"; + pub const RESEARCH_WEB_SUMMARIZE: &'static str = "/api/ui/research/web/summarize"; + pub const RESEARCH_WEB_DEEP: &'static str = "/api/ui/research/web/deep"; + pub const RESEARCH_WEB_HISTORY: &'static str = "/api/ui/research/web/history"; + pub const RESEARCH_WEB_INSTANT: &'static str = "/api/ui/research/web/instant"; + pub const RESEARCH_EXPORT_CITATIONS: &'static str = "/api/ui/research/export/citations"; - pub const SOURCES_PROMPTS: &'static str = "/api/sources/prompts"; - pub const SOURCES_TEMPLATES: &'static str = "/api/sources/templates"; - pub const SOURCES_NEWS: &'static str = "/api/sources/news"; - pub const SOURCES_MCP_SERVERS: &'static str = "/api/sources/mcp-servers"; - pub const SOURCES_LLM_TOOLS: &'static str = "/api/sources/llm-tools"; - pub const SOURCES_MODELS: &'static str = "/api/sources/models"; - pub const SOURCES_SEARCH: &'static str = "/api/sources/search"; - pub const SOURCES_REPOSITORIES: &'static str = "/api/sources/repositories"; - pub const SOURCES_REPOSITORIES_CONNECT: &'static str = "/api/sources/repositories/connect"; + // Sources - HTMX/HTML APIs + pub const SOURCES_PROMPTS: &'static str = "/api/ui/sources/prompts"; + pub const SOURCES_TEMPLATES: &'static str = "/api/ui/sources/templates"; + pub const SOURCES_NEWS: &'static str = "/api/ui/sources/news"; + pub const SOURCES_MCP_SERVERS: &'static str = "/api/ui/sources/mcp-servers"; + pub const SOURCES_LLM_TOOLS: &'static str = "/api/ui/sources/llm-tools"; + pub const SOURCES_MODELS: &'static str = "/api/ui/sources/models"; + pub const SOURCES_SEARCH: &'static str = "/api/ui/sources/search"; + pub const SOURCES_REPOSITORIES: &'static str = "/api/ui/sources/repositories"; + pub const SOURCES_REPOSITORIES_CONNECT: &'static str = "/api/ui/sources/repositories/connect"; pub const SOURCES_REPOSITORIES_DISCONNECT: &'static str = - "/api/sources/repositories/disconnect"; - pub const SOURCES_APPS: &'static str = "/api/sources/apps"; - pub const SOURCES_MCP: &'static str = "/api/sources/mcp"; - pub const SOURCES_MCP_BY_NAME: &'static str = "/api/sources/mcp/:name"; - pub const SOURCES_MCP_ENABLE: &'static str = "/api/sources/mcp/:name/enable"; - pub const SOURCES_MCP_DISABLE: &'static str = "/api/sources/mcp/:name/disable"; - pub const SOURCES_MCP_TOOLS: &'static str = "/api/sources/mcp/:name/tools"; - pub const SOURCES_MCP_TEST: &'static str = "/api/sources/mcp/:name/test"; - pub const SOURCES_MCP_SCAN: &'static str = "/api/sources/mcp/scan"; - pub const SOURCES_MCP_EXAMPLES: &'static str = "/api/sources/mcp/examples"; - pub const SOURCES_MENTIONS: &'static str = "/api/sources/mentions"; - pub const SOURCES_TOOLS: &'static str = "/api/sources/tools"; + "/api/ui/sources/repositories/disconnect"; + pub const SOURCES_APPS: &'static str = "/api/ui/sources/apps"; + pub const SOURCES_MCP: &'static str = "/api/ui/sources/mcp"; + pub const SOURCES_MCP_BY_NAME: &'static str = "/api/ui/sources/mcp/:name"; + pub const SOURCES_MCP_ENABLE: &'static str = "/api/ui/sources/mcp/:name/enable"; + pub const SOURCES_MCP_DISABLE: &'static str = "/api/ui/sources/mcp/:name/disable"; + pub const SOURCES_MCP_TOOLS: &'static str = "/api/ui/sources/mcp/:name/tools"; + pub const SOURCES_MCP_TEST: &'static str = "/api/ui/sources/mcp/:name/test"; + pub const SOURCES_MCP_SCAN: &'static str = "/api/ui/sources/mcp/scan"; + pub const SOURCES_MCP_EXAMPLES: &'static str = "/api/ui/sources/mcp/examples"; + pub const SOURCES_MENTIONS: &'static str = "/api/ui/sources/mentions"; + pub const SOURCES_TOOLS: &'static str = "/api/ui/sources/tools"; - pub const TASKS_STATS: &'static str = "/api/tasks/stats"; - pub const TASKS_STATS_JSON: &'static str = "/api/tasks/stats/json"; - pub const TASKS_COMPLETED: &'static str = "/api/tasks/completed"; + // Sources Knowledge Base - HTMX/HTML APIs + pub const SOURCES_KB_UPLOAD: &'static str = "/api/ui/sources/kb/upload"; + pub const SOURCES_KB_LIST: &'static str = "/api/ui/sources/kb/list"; + pub const SOURCES_KB_QUERY: &'static str = "/api/ui/sources/kb/query"; + pub const SOURCES_KB_BY_ID: &'static str = "/api/ui/sources/kb/:id"; + pub const SOURCES_KB_REINDEX: &'static str = "/api/ui/sources/kb/reindex"; + pub const SOURCES_KB_STATS: &'static str = "/api/ui/sources/kb/stats"; + // WebSocket endpoints pub const WS: &'static str = "/ws"; pub const WS_MEET: &'static str = "/ws/meet"; pub const WS_CHAT: &'static str = "/ws/chat"; diff --git a/src/designer/mod.rs b/src/designer/mod.rs index f914683d8..1e24f9849 100644 --- a/src/designer/mod.rs +++ b/src/designer/mod.rs @@ -111,13 +111,13 @@ pub fn configure_designer_routes() -> Router> { .route(ApiUrls::DESIGNER_VALIDATE, post(handle_validate)) .route(ApiUrls::DESIGNER_EXPORT, get(handle_export)) .route( - "/api/designer/dialogs", + ApiUrls::DESIGNER_DIALOGS, get(handle_list_dialogs).post(handle_create_dialog), ) - .route("/api/designer/dialogs/{id}", get(handle_get_dialog)) + .route(&ApiUrls::DESIGNER_DIALOG_BY_ID.replace(":id", "{id}"), get(handle_get_dialog)) .route(ApiUrls::DESIGNER_MODIFY, post(handle_designer_modify)) - .route("/api/v1/designer/magic", post(handle_magic_suggestions)) - .route("/api/v1/editor/magic", post(handle_editor_magic)) + .route("/api/ui/designer/magic", post(handle_magic_suggestions)) + .route("/api/ui/editor/magic", post(handle_editor_magic)) } pub async fn handle_editor_magic( diff --git a/src/drive/drive_monitor/mod.rs b/src/drive/drive_monitor/mod.rs index 365b08d64..7c841c20b 100644 --- a/src/drive/drive_monitor/mod.rs +++ b/src/drive/drive_monitor/mod.rs @@ -7,11 +7,12 @@ use crate::shared::message_types::MessageType; use crate::shared::state::AppState; use aws_sdk_s3::Client; use log::{debug, error, info, trace, warn}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::error::Error; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::Arc; +use tokio::sync::RwLock as TokioRwLock; use tokio::time::Duration; const KB_INDEXING_TIMEOUT_SECS: u64 = 60; @@ -31,6 +32,8 @@ pub struct DriveMonitor { work_root: PathBuf, is_processing: Arc, consecutive_failures: Arc, + /// Track KB folders currently being indexed to prevent duplicate tasks + kb_indexing_in_progress: Arc>>, } impl DriveMonitor { pub fn new(state: Arc, bucket_name: String, bot_id: uuid::Uuid) -> Self { @@ -46,6 +49,7 @@ impl DriveMonitor { work_root, is_processing: Arc::new(AtomicBool::new(false)), consecutive_failures: Arc::new(AtomicU32::new(0)), + kb_indexing_in_progress: Arc::new(TokioRwLock::new(HashSet::new())), } } @@ -735,10 +739,30 @@ impl DriveMonitor { continue; } + // Create a unique key for this KB folder to track indexing state + let kb_key = format!("{}_{}", bot_name, kb_name); + + // Check if this KB folder is already being indexed + { + let indexing_set = self.kb_indexing_in_progress.read().await; + if indexing_set.contains(&kb_key) { + debug!("[DRIVE_MONITOR] KB folder {} already being indexed, skipping duplicate task", kb_key); + continue; + } + } + + // Mark this KB folder as being indexed + { + let mut indexing_set = self.kb_indexing_in_progress.write().await; + indexing_set.insert(kb_key.clone()); + } + let kb_manager = Arc::clone(&self.kb_manager); let bot_name_owned = bot_name.to_string(); let kb_name_owned = kb_name.to_string(); let kb_folder_owned = kb_folder_path.clone(); + let indexing_tracker = Arc::clone(&self.kb_indexing_in_progress); + let kb_key_owned = kb_key.clone(); tokio::spawn(async move { info!( @@ -746,10 +770,18 @@ impl DriveMonitor { kb_folder_owned.display() ); - match tokio::time::timeout( + let result = tokio::time::timeout( Duration::from_secs(KB_INDEXING_TIMEOUT_SECS), kb_manager.handle_gbkb_change(&bot_name_owned, &kb_folder_owned) - ).await { + ).await; + + // Always remove from tracking set when done, regardless of outcome + { + let mut indexing_set = indexing_tracker.write().await; + indexing_set.remove(&kb_key_owned); + } + + match result { Ok(Ok(_)) => { debug!( "Successfully processed KB change for {}/{}", diff --git a/src/email/mod.rs b/src/email/mod.rs index 21e874537..07a89670d 100644 --- a/src/email/mod.rs +++ b/src/email/mod.rs @@ -124,18 +124,19 @@ pub fn configure() -> Router> { ) .route("/api/email/tracking/list", get(list_sent_emails_tracking)) .route("/api/email/tracking/stats", get(get_tracking_stats)) - .route("/ui/email/accounts", get(list_email_accounts_htmx)) - .route("/ui/email/list", get(list_emails_htmx)) - .route("/ui/email/folders", get(list_folders_htmx)) - .route("/ui/email/compose", get(compose_email_htmx)) - .route("/ui/email/:id", get(get_email_content_htmx)) - .route("/ui/email/:id/delete", delete(delete_email_htmx)) - .route("/ui/email/labels", get(list_labels_htmx)) - .route("/ui/email/templates", get(list_templates_htmx)) - .route("/ui/email/signatures", get(list_signatures_htmx)) - .route("/ui/email/rules", get(list_rules_htmx)) - .route("/ui/email/search", get(search_emails_htmx)) - .route("/ui/email/auto-responder", post(save_auto_responder)) + // HTMX/HTML APIs + .route(ApiUrls::EMAIL_ACCOUNTS_HTMX, get(list_email_accounts_htmx)) + .route(ApiUrls::EMAIL_LIST_HTMX, get(list_emails_htmx)) + .route(ApiUrls::EMAIL_FOLDERS_HTMX, get(list_folders_htmx)) + .route(ApiUrls::EMAIL_COMPOSE_HTMX, get(compose_email_htmx)) + .route(&ApiUrls::EMAIL_CONTENT_HTMX.replace(":id", "{id}"), get(get_email_content_htmx)) + .route("/api/ui/email/{id}/delete", delete(delete_email_htmx)) + .route(ApiUrls::EMAIL_LABELS_HTMX, get(list_labels_htmx)) + .route(ApiUrls::EMAIL_TEMPLATES_HTMX, get(list_templates_htmx)) + .route(ApiUrls::EMAIL_SIGNATURES_HTMX, get(list_signatures_htmx)) + .route(ApiUrls::EMAIL_RULES_HTMX, get(list_rules_htmx)) + .route(ApiUrls::EMAIL_SEARCH_HTMX, get(search_emails_htmx)) + .route(ApiUrls::EMAIL_AUTO_RESPONDER_HTMX, post(save_auto_responder)) } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1717,39 +1718,73 @@ struct EmailContent { pub async fn list_emails_htmx( State(state): State>, Query(params): Query>, -) -> Result { +) -> impl IntoResponse { let folder = params .get("folder") .cloned() .unwrap_or_else(|| "inbox".to_string()); - let user_id = extract_user_from_session(&state) - .map_err(|_| EmailError("Authentication required".to_string()))?; + let user_id = match extract_user_from_session(&state) { + Ok(id) => id, + Err(_) => { + return axum::response::Html( + r#"
+

Authentication required

+

Please sign in to view your emails

+
"# + .to_string(), + ); + } + }; let conn = state.conn.clone(); - let account = tokio::task::spawn_blocking(move || { - let mut db_conn = conn - .get() - .map_err(|e| format!("DB connection error: {}", e))?; + let account_result = tokio::task::spawn_blocking(move || { + let db_conn_result = conn.get(); + let mut db_conn = match db_conn_result { + Ok(c) => c, + Err(e) => return Err(format!("DB connection error: {}", e)), + }; - diesel::sql_query("SELECT * FROM email_accounts WHERE user_id = $1 LIMIT 1") + diesel::sql_query("SELECT * FROM user_email_accounts WHERE user_id = $1 LIMIT 1") .bind::(user_id) .get_result::(&mut db_conn) .optional() .map_err(|e| format!("Failed to get email account: {}", e)) }) - .await - .map_err(|e| EmailError(format!("Task join error: {e}")))? - .map_err(EmailError)?; + .await; - let Some(account) = account else { - return Ok(axum::response::Html( - r#"
-

No email account configured

-

Please add an email account first

-
"# - .to_string(), - )); + let account = match account_result { + Ok(Ok(Some(acc))) => acc, + Ok(Ok(None)) => { + return axum::response::Html( + r##"
+

No email account configured

+

Please add an email account in settings to get started

+ Add Email Account +
"## + .to_string(), + ); + } + Ok(Err(e)) => { + log::error!("Email account query error: {}", e); + return axum::response::Html( + r#"
+

Unable to load emails

+

There was an error connecting to the database. Please try again later.

+
"# + .to_string(), + ); + } + Err(e) => { + log::error!("Task join error: {}", e); + return axum::response::Html( + r#"
+

Unable to load emails

+

An internal error occurred. Please try again later.

+
"# + .to_string(), + ); + } }; let config = EmailConfig { @@ -1795,20 +1830,28 @@ pub async fn list_emails_htmx( ); } - Ok(axum::response::Html(html)) + axum::response::Html(html) } pub async fn list_folders_htmx( State(state): State>, -) -> Result { - let user_id = extract_user_from_session(&state) - .map_err(|_| EmailError("Authentication required".to_string()))?; +) -> impl IntoResponse { + let user_id = match extract_user_from_session(&state) { + Ok(id) => id, + Err(_) => { + return axum::response::Html( + r#"
Please sign in
"#.to_string(), + ); + } + }; let conn = state.conn.clone(); - let account = tokio::task::spawn_blocking(move || { - let mut db_conn = conn - .get() - .map_err(|e| format!("DB connection error: {}", e))?; + let account_result = tokio::task::spawn_blocking(move || { + let db_conn_result = conn.get(); + let mut db_conn = match db_conn_result { + Ok(c) => c, + Err(e) => return Err(format!("DB connection error: {}", e)), + }; diesel::sql_query("SELECT * FROM email_accounts WHERE user_id = $1 LIMIT 1") .bind::(user_id) @@ -1816,20 +1859,27 @@ pub async fn list_folders_htmx( .optional() .map_err(|e| format!("Failed to get email account: {}", e)) }) - .await - .map_err(|e| EmailError(format!("Task join error: {e}")))? - .map_err(EmailError)?; + .await; - if account.is_none() { - return Ok(axum::response::Html( - r#"
No account configured
"#.to_string(), - )); - } - - let Some(account) = account else { - return Ok(Html( - r#"
Account not found
"#.to_string(), - )); + let account = match account_result { + Ok(Ok(Some(acc))) => acc, + Ok(Ok(None)) => { + return axum::response::Html( + r#"
No account configured
"#.to_string(), + ); + } + Ok(Err(e)) => { + log::error!("Email folder query error: {}", e); + return axum::response::Html( + r#"
Error loading folders
"#.to_string(), + ); + } + Err(e) => { + log::error!("Task join error: {}", e); + return axum::response::Html( + r#"
Error loading folders
"#.to_string(), + ); + } }; let config = EmailConfig { @@ -1889,7 +1939,7 @@ pub async fn list_folders_htmx( ); } - Ok(axum::response::Html(html)) + axum::response::Html(html) } pub async fn compose_email_htmx( @@ -2016,15 +2066,27 @@ pub async fn get_email_content_htmx( pub async fn delete_email_htmx( State(state): State>, Path(id): Path, -) -> Result { - let user_id = extract_user_from_session(&state) - .map_err(|_| EmailError("Authentication required".to_string()))?; +) -> impl IntoResponse { + let user_id = match extract_user_from_session(&state) { + Ok(id) => id, + Err(_) => { + return axum::response::Html( + r#"
+

Authentication required

+

Please sign in to delete emails

+
"# + .to_string(), + ); + } + }; let conn = state.conn.clone(); - let account = tokio::task::spawn_blocking(move || { - let mut db_conn = conn - .get() - .map_err(|e| format!("DB connection error: {}", e))?; + let account_result = tokio::task::spawn_blocking(move || { + let db_conn_result = conn.get(); + let mut db_conn = match db_conn_result { + Ok(c) => c, + Err(e) => return Err(format!("DB connection error: {}", e)), + }; diesel::sql_query("SELECT * FROM email_accounts WHERE user_id = $1 LIMIT 1") .bind::(user_id) @@ -2032,28 +2094,75 @@ pub async fn delete_email_htmx( .optional() .map_err(|e| format!("Failed to get email account: {}", e)) }) - .await - .map_err(|e| EmailError(format!("Task join error: {e}")))? - .map_err(EmailError)?; + .await; - if let Some(account) = account { - let config = EmailConfig { - username: account.username.clone(), - password: account.password.clone(), - server: account.imap_server.clone(), - port: account.imap_port as u16, - from: account.email.clone(), - smtp_server: account.smtp_server.clone(), - smtp_port: account.smtp_port as u16, - }; + let account = match account_result { + Ok(Ok(Some(acc))) => acc, + Ok(Ok(None)) => { + return axum::response::Html( + r#"
+

No email account configured

+

Please add an email account first

+
"# + .to_string(), + ); + } + Ok(Err(e)) => { + log::error!("Email account query error: {}", e); + return axum::response::Html( + r#"
+

Error deleting email

+

Database error occurred

+
"# + .to_string(), + ); + } + Err(e) => { + log::error!("Task join error: {}", e); + return axum::response::Html( + r#"
+

Error deleting email

+

An internal error occurred

+
"# + .to_string(), + ); + } + }; - move_email_to_trash(&config, &id) - .map_err(|e| EmailError(format!("Failed to delete email: {}", e)))?; + let config = EmailConfig { + username: account.username.clone(), + password: account.password.clone(), + server: account.imap_server.clone(), + port: account.imap_port as u16, + from: account.email.clone(), + smtp_server: account.smtp_server.clone(), + smtp_port: account.smtp_port as u16, + }; + + if let Err(e) = move_email_to_trash(&config, &id) { + log::error!("Failed to delete email: {}", e); + return axum::response::Html( + r#"
+

Error deleting email

+

Failed to move email to trash

+
"# + .to_string(), + ); } info!("Email {} moved to trash", id); - list_emails_htmx(State(state), Query(std::collections::HashMap::new())).await + axum::response::Html( + r#"
+

Email moved to trash

+
+ "# + .to_string(), + ) } pub async fn get_latest_email( diff --git a/src/lib.rs b/src/lib.rs index fb68f7159..c139c28ed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -69,6 +69,10 @@ pub use llm::DynamicLLMProvider; #[cfg(feature = "meet")] pub mod meet; +pub mod monitoring; + +pub mod settings; + #[cfg(feature = "msteams")] pub mod msteams; diff --git a/src/main.rs b/src/main.rs index f0c960518..bd4d5161f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,7 +26,7 @@ use tower_http::trace::TraceLayer; async fn ensure_vendor_files_in_minio(drive: &aws_sdk_s3::Client) { use aws_sdk_s3::primitives::ByteStream; - let htmx_content = include_bytes!("../../botserver-stack/static/js/vendor/htmx.min.js"); + let htmx_content = include_bytes!("../botserver-stack/static/js/vendor/htmx.min.js"); let bucket = "default.gbai"; let key = "default.gblib/vendor/htmx.min.js"; @@ -91,6 +91,7 @@ use bootstrap::BootstrapManager; use botserver::core::bot::channels::{VoiceAdapter, WebChannelAdapter}; use botserver::core::bot::websocket_handler; use botserver::core::bot::BotOrchestrator; +use botserver::core::bot_database::BotDatabaseManager; use botserver::core::config::AppConfig; #[cfg(feature = "directory")] @@ -283,6 +284,8 @@ async fn run_axum_server( api_router = api_router.merge(botserver::research::configure_research_routes()); api_router = api_router.merge(botserver::sources::configure_sources_routes()); api_router = api_router.merge(botserver::designer::configure_designer_routes()); + api_router = api_router.merge(botserver::monitoring::configure()); + api_router = api_router.merge(botserver::settings::configure_settings_routes()); api_router = api_router.merge(botserver::basic::keywords::configure_db_routes()); api_router = api_router.merge(botserver::basic::keywords::configure_app_server_routes()); api_router = api_router.merge(botserver::auto_task::configure_autotask_routes()); @@ -858,12 +861,36 @@ async fn main() -> std::io::Result<()> { botserver::core::shared::state::TaskProgressEvent, >(1000); + // Initialize BotDatabaseManager for per-bot database support + let database_url = crate::shared::utils::get_database_url_sync().unwrap_or_default(); + let bot_database_manager = Arc::new(BotDatabaseManager::new(pool.clone(), &database_url)); + + // Sync all bot databases on startup - ensures each bot has its own database + info!("Syncing bot databases on startup..."); + match bot_database_manager.sync_all_bot_databases() { + Ok(sync_result) => { + info!( + "Bot database sync complete: {} created, {} verified, {} errors", + sync_result.databases_created, + sync_result.databases_verified, + sync_result.errors.len() + ); + for err in &sync_result.errors { + warn!("Bot database sync error: {}", err); + } + } + Err(e) => { + error!("Failed to sync bot databases: {}", e); + } + } + let app_state = Arc::new(AppState { drive: Some(drive.clone()), s3_client: Some(drive), config: Some(cfg.clone()), conn: pool.clone(), - database_url: crate::shared::utils::get_database_url_sync().unwrap_or_default(), + database_url: database_url.clone(), + bot_database_manager: bot_database_manager.clone(), bucket_name: "default.gbai".to_string(), cache: redis_client.clone(), session_manager: session_manager.clone(), diff --git a/src/meet/mod.rs b/src/meet/mod.rs index 0940e6087..2b3f19953 100644 --- a/src/meet/mod.rs +++ b/src/meet/mod.rs @@ -1,7 +1,7 @@ use axum::{ extract::{Path, State}, http::StatusCode, - response::{IntoResponse, Json}, + response::{Html, IntoResponse, Json}, routing::{get, post}, Router, }; @@ -256,14 +256,39 @@ pub async fn create_meeting( } } -pub async fn list_rooms(State(state): State>) -> impl IntoResponse { +pub async fn list_rooms(State(state): State>) -> Html { let transcription_service = Arc::new(DefaultTranscriptionService); let meeting_service = MeetingService::new(state.clone(), transcription_service); let rooms = meeting_service.rooms.read().await; - let room_list: Vec<_> = rooms.values().cloned().collect(); - (StatusCode::OK, Json(serde_json::json!(room_list))) + if rooms.is_empty() { + return Html(r##"
+
๐Ÿ“น
+

No active rooms

+

Create a new meeting to get started

+
"##.to_string()); + } + + let mut html = String::new(); + for room in rooms.values() { + let participant_count = room.participants.len(); + html.push_str(&format!( + r##"
+
๐Ÿ“น
+
+

{name}

+ {count} participant(s) +
+ +
"##, + id = room.id, + name = room.name, + count = participant_count, + )); + } + + Html(html) } pub async fn get_room( @@ -382,23 +407,22 @@ async fn handle_meeting_socket(_socket: axum::extract::ws::WebSocket, _state: Ar info!("Meeting WebSocket connection established"); } -pub async fn all_participants(State(_state): State>) -> Json { - Json(serde_json::json!({ - "participants": [], - "message": "No participants" - })) +pub async fn all_participants(State(_state): State>) -> Html { + Html(r##"
+

No participants

+
"##.to_string()) } -pub async fn recent_meetings(State(_state): State>) -> Json { - Json(serde_json::json!({ - "meetings": [], - "message": "No recent meetings" - })) +pub async fn recent_meetings(State(_state): State>) -> Html { + Html(r##"
+
๐Ÿ“‹
+

No recent meetings

+
"##.to_string()) } -pub async fn scheduled_meetings(State(_state): State>) -> Json { - Json(serde_json::json!({ - "meetings": [], - "message": "No scheduled meetings" - })) +pub async fn scheduled_meetings(State(_state): State>) -> Html { + Html(r##"
+
๐Ÿ“…
+

No scheduled meetings

+
"##.to_string()) } diff --git a/src/monitoring/mod.rs b/src/monitoring/mod.rs index 19620e265..ad232ac81 100644 --- a/src/monitoring/mod.rs +++ b/src/monitoring/mod.rs @@ -1,23 +1,35 @@ - - - use axum::{extract::State, response::Html, routing::get, Router}; -use log::info; +use chrono::Local; use std::sync::Arc; use sysinfo::{Disks, Networks, System}; +use crate::core::urls::ApiUrls; use crate::shared::state::AppState; pub fn configure() -> Router> { Router::new() - .route("/api/monitoring/dashboard", get(dashboard)) - .route("/api/monitoring/services", get(services)) - .route("/api/monitoring/resources", get(resources)) - .route("/api/monitoring/logs", get(logs)) - .route("/api/monitoring/llm", get(llm_metrics)) - .route("/api/monitoring/health", get(health)) + .route(ApiUrls::MONITORING_DASHBOARD, get(dashboard)) + .route(ApiUrls::MONITORING_SERVICES, get(services)) + .route(ApiUrls::MONITORING_RESOURCES, get(resources)) + .route(ApiUrls::MONITORING_LOGS, get(logs)) + .route(ApiUrls::MONITORING_LLM, get(llm_metrics)) + .route(ApiUrls::MONITORING_HEALTH, get(health)) + // Additional endpoints expected by the frontend + .route("/api/ui/monitoring/timestamp", get(timestamp)) + .route("/api/ui/monitoring/bots", get(bots)) + .route("/api/ui/monitoring/services/status", get(services_status)) + .route("/api/ui/monitoring/resources/bars", get(resources_bars)) + .route("/api/ui/monitoring/activity/latest", get(activity_latest)) + .route("/api/ui/monitoring/metric/sessions", get(metric_sessions)) + .route("/api/ui/monitoring/metric/messages", get(metric_messages)) + .route("/api/ui/monitoring/metric/response_time", get(metric_response_time)) + .route("/api/ui/monitoring/trend/sessions", get(trend_sessions)) + .route("/api/ui/monitoring/rate/messages", get(rate_messages)) + // Aliases for frontend compatibility + .route("/api/ui/monitoring/sessions", get(sessions_panel)) + .route("/api/ui/monitoring/messages", get(messages_panel)) } @@ -399,3 +411,160 @@ fn check_minio() -> bool { fn check_llm() -> bool { true } + + +async fn timestamp(State(_state): State>) -> Html { + let now = Local::now(); + Html(format!("Last updated: {}", now.format("%H:%M:%S"))) +} + + +async fn bots(State(state): State>) -> Html { + let active_sessions = state + .session_manager + .try_lock() + .map(|sm| sm.active_count()) + .unwrap_or(0); + + Html(format!( + r##"
+
+ Active Sessions + {active_sessions} +
+
"## + )) +} + + +async fn services_status(State(_state): State>) -> Html { + let services = vec![ + ("postgresql", check_postgres()), + ("redis", check_redis()), + ("minio", check_minio()), + ("llm", check_llm()), + ]; + + let mut status_updates = String::new(); + for (name, running) in services { + let status = if running { "running" } else { "stopped" }; + status_updates.push_str(&format!( + r##""## + )); + } + + Html(status_updates) +} + + +async fn resources_bars(State(_state): State>) -> Html { + let mut sys = System::new_all(); + sys.refresh_all(); + + let cpu_usage = sys.global_cpu_usage(); + let total_memory = sys.total_memory(); + let used_memory = sys.used_memory(); + let memory_percent = if total_memory > 0 { + (used_memory as f64 / total_memory as f64) * 100.0 + } else { + 0.0 + }; + + Html(format!( + r##" + CPU + + + {cpu_usage:.0}% + + + MEM + + + {memory_percent:.0}% +"##, + cpu_width = cpu_usage.min(100.0), + mem_width = memory_percent.min(100.0), + )) +} + + +async fn activity_latest(State(_state): State>) -> Html { + Html("System monitoring active...".to_string()) +} + + +async fn metric_sessions(State(state): State>) -> Html { + let active_sessions = state + .session_manager + .try_lock() + .map(|sm| sm.active_count()) + .unwrap_or(0); + + Html(format!("{}", active_sessions)) +} + + +async fn metric_messages(State(_state): State>) -> Html { + Html("--".to_string()) +} + + +async fn metric_response_time(State(_state): State>) -> Html { + Html("--".to_string()) +} + + +async fn trend_sessions(State(_state): State>) -> Html { + Html("โ†‘ 0%".to_string()) +} + + +async fn rate_messages(State(_state): State>) -> Html { + Html("0/hr".to_string()) +} + + +async fn sessions_panel(State(state): State>) -> Html { + let active_sessions = state + .session_manager + .try_lock() + .map(|sm| sm.active_count()) + .unwrap_or(0); + + Html(format!( + r##"
+
+

Active Sessions

+ {active_sessions} +
+
+
+

No active sessions

+
+
+
"## + )) +} + + +async fn messages_panel(State(_state): State>) -> Html { + Html( + r##"
+
+

Recent Messages

+
+
+
+

No recent messages

+
+
+
"## + .to_string(), + ) +} diff --git a/src/paper/mod.rs b/src/paper/mod.rs index edd424072..28914d21a 100644 --- a/src/paper/mod.rs +++ b/src/paper/mod.rs @@ -1,5 +1,6 @@ #[cfg(feature = "llm")] use crate::llm::OpenAIClient; +use crate::core::urls::ApiUrls; use crate::shared::state::AppState; use aws_sdk_s3::primitives::ByteStream; use axum::{ @@ -78,32 +79,34 @@ pub struct UserRow { } pub fn configure_paper_routes() -> Router> { + use crate::core::urls::ApiUrls; + Router::new() - .route("/api/paper/new", post(handle_new_document)) - .route("/api/paper/list", get(handle_list_documents)) - .route("/api/paper/search", get(handle_search_documents)) - .route("/api/paper/save", post(handle_save_document)) - .route("/api/paper/autosave", post(handle_autosave)) - .route("/api/paper/{id}", get(handle_get_document)) - .route("/api/paper/{id}/delete", post(handle_delete_document)) - .route("/api/paper/template/blank", post(handle_template_blank)) - .route("/api/paper/template/meeting", post(handle_template_meeting)) - .route("/api/paper/template/todo", post(handle_template_todo)) + .route(ApiUrls::PAPER_NEW, post(handle_new_document)) + .route(ApiUrls::PAPER_LIST, get(handle_list_documents)) + .route(ApiUrls::PAPER_SEARCH, get(handle_search_documents)) + .route(ApiUrls::PAPER_SAVE, post(handle_save_document)) + .route(ApiUrls::PAPER_AUTOSAVE, post(handle_autosave)) + .route(&ApiUrls::PAPER_BY_ID.replace(":id", "{id}"), get(handle_get_document)) + .route(&ApiUrls::PAPER_DELETE.replace(":id", "{id}"), post(handle_delete_document)) + .route(ApiUrls::PAPER_TEMPLATE_BLANK, post(handle_template_blank)) + .route(ApiUrls::PAPER_TEMPLATE_MEETING, post(handle_template_meeting)) + .route(ApiUrls::PAPER_TEMPLATE_TODO, post(handle_template_todo)) .route( - "/api/paper/template/research", + ApiUrls::PAPER_TEMPLATE_RESEARCH, post(handle_template_research), ) - .route("/api/paper/ai/summarize", post(handle_ai_summarize)) - .route("/api/paper/ai/expand", post(handle_ai_expand)) - .route("/api/paper/ai/improve", post(handle_ai_improve)) - .route("/api/paper/ai/simplify", post(handle_ai_simplify)) - .route("/api/paper/ai/translate", post(handle_ai_translate)) - .route("/api/paper/ai/custom", post(handle_ai_custom)) - .route("/api/paper/export/pdf", get(handle_export_pdf)) - .route("/api/paper/export/docx", get(handle_export_docx)) - .route("/api/paper/export/md", get(handle_export_md)) - .route("/api/paper/export/html", get(handle_export_html)) - .route("/api/paper/export/txt", get(handle_export_txt)) + .route(ApiUrls::PAPER_AI_SUMMARIZE, post(handle_ai_summarize)) + .route(ApiUrls::PAPER_AI_EXPAND, post(handle_ai_expand)) + .route(ApiUrls::PAPER_AI_IMPROVE, post(handle_ai_improve)) + .route(ApiUrls::PAPER_AI_SIMPLIFY, post(handle_ai_simplify)) + .route(ApiUrls::PAPER_AI_TRANSLATE, post(handle_ai_translate)) + .route(ApiUrls::PAPER_AI_CUSTOM, post(handle_ai_custom)) + .route(ApiUrls::PAPER_EXPORT_PDF, get(handle_export_pdf)) + .route(ApiUrls::PAPER_EXPORT_DOCX, get(handle_export_docx)) + .route(ApiUrls::PAPER_EXPORT_MD, get(handle_export_md)) + .route(ApiUrls::PAPER_EXPORT_HTML, get(handle_export_html)) + .route(ApiUrls::PAPER_EXPORT_TXT, get(handle_export_txt)) } async fn get_current_user( @@ -552,9 +555,8 @@ pub async fn handle_new_document( html.push_str(""); html.push_str(""); @@ -588,7 +590,7 @@ pub async fn handle_list_documents( if documents.is_empty() { html.push_str("
"); html.push_str("

No documents yet

"); - html.push_str(""); + html.push_str(&format!("", ApiUrls::PAPER_NEW)); html.push_str("
"); } else { for doc in documents { @@ -768,7 +770,7 @@ pub async fn handle_delete_document( match delete_document_from_drive(&state, &user_identifier, &id).await { Ok(()) => { log::info!("Document deleted: {}", id); - Html("
".to_string()) + Html(format!("
", ApiUrls::PAPER_LIST)) } Err(e) => { log::error!("Failed to delete document {}: {}", id, e); @@ -1218,8 +1220,8 @@ fn format_document_list_item(id: &str, title: &str, time: &str, is_new: bool) -> html.push_str(new_class); html.push_str("\" data-id=\""); html.push_str(&html_escape(id)); - html.push_str("\" hx-get=\"/api/paper/"); - html.push_str(&html_escape(id)); + html.push_str("\" hx-get=\""); + html.push_str(&ApiUrls::PAPER_BY_ID.replace(":id", &html_escape(id))); html.push_str("\" hx-target=\"#editor-content\" hx-swap=\"innerHTML\">"); html.push_str("
๐Ÿ“„
"); html.push_str("
"); diff --git a/src/research/mod.rs b/src/research/mod.rs index 107e22704..fa4d4094c 100644 --- a/src/research/mod.rs +++ b/src/research/mod.rs @@ -5,7 +5,7 @@ use axum::{ extract::{Path, State}, response::{Html, IntoResponse}, routing::{get, post}, - Json, Router, + Form, Json, Router, }; use diesel::prelude::*; use serde::{Deserialize, Serialize}; @@ -58,20 +58,22 @@ pub struct CollectionRow { } pub fn configure_research_routes() -> Router> { + use crate::core::urls::ApiUrls; + Router::new() .merge(web_search::configure_web_search_routes()) - .route("/api/research/collections", get(handle_list_collections)) + .route(ApiUrls::RESEARCH_COLLECTIONS, get(handle_list_collections)) .route( - "/api/research/collections/new", + ApiUrls::RESEARCH_COLLECTIONS_NEW, post(handle_create_collection), ) - .route("/api/research/collections/{id}", get(handle_get_collection)) - .route("/api/research/search", post(handle_search)) - .route("/api/research/recent", get(handle_recent_searches)) - .route("/api/research/trending", get(handle_trending_tags)) - .route("/api/research/prompts", get(handle_prompts)) + .route(&ApiUrls::RESEARCH_COLLECTION_BY_ID.replace(":id", "{id}"), get(handle_get_collection)) + .route(ApiUrls::RESEARCH_SEARCH, post(handle_search)) + .route(ApiUrls::RESEARCH_RECENT, get(handle_recent_searches)) + .route(ApiUrls::RESEARCH_TRENDING, get(handle_trending_tags)) + .route(ApiUrls::RESEARCH_PROMPTS, get(handle_prompts)) .route( - "/api/research/export-citations", + ApiUrls::RESEARCH_EXPORT_CITATIONS, get(handle_export_citations), ) } @@ -264,7 +266,7 @@ pub async fn handle_get_collection( pub async fn handle_search( State(state): State>, - Json(payload): Json, + Form(payload): Form, ) -> impl IntoResponse { let query = payload.query.unwrap_or_default(); diff --git a/src/research/web_search.rs b/src/research/web_search.rs index ee60613dd..999fcb2fc 100644 --- a/src/research/web_search.rs +++ b/src/research/web_search.rs @@ -96,12 +96,14 @@ pub struct SearchHistoryQuery { } pub fn configure_web_search_routes() -> Router> { + use crate::core::urls::ApiUrls; + Router::new() - .route("/api/research/web/search", post(handle_web_search)) - .route("/api/research/web/summarize", post(handle_summarize)) - .route("/api/research/web/deep", post(handle_deep_research)) - .route("/api/research/web/history", get(handle_search_history)) - .route("/api/research/web/instant", get(handle_instant_answer)) + .route(ApiUrls::RESEARCH_WEB_SEARCH, post(handle_web_search)) + .route(ApiUrls::RESEARCH_WEB_SUMMARIZE, post(handle_summarize)) + .route(ApiUrls::RESEARCH_WEB_DEEP, post(handle_deep_research)) + .route(ApiUrls::RESEARCH_WEB_HISTORY, get(handle_search_history)) + .route(ApiUrls::RESEARCH_WEB_INSTANT, get(handle_instant_answer)) } pub async fn handle_web_search( diff --git a/src/settings/mod.rs b/src/settings/mod.rs new file mode 100644 index 000000000..515e42f60 --- /dev/null +++ b/src/settings/mod.rs @@ -0,0 +1,149 @@ +use axum::{ + extract::State, + response::Html, + routing::{get, post}, + Router, +}; +use std::sync::Arc; + +use crate::shared::state::AppState; + +pub fn configure_settings_routes() -> Router> { + Router::new() + .route("/api/user/storage", get(get_storage_info)) + .route("/api/user/storage/connections", get(get_storage_connections)) + .route("/api/user/security/2fa/status", get(get_2fa_status)) + .route("/api/user/security/2fa/enable", post(enable_2fa)) + .route("/api/user/security/2fa/disable", post(disable_2fa)) + .route("/api/user/security/sessions", get(get_active_sessions)) + .route( + "/api/user/security/sessions/revoke-all", + post(revoke_all_sessions), + ) + .route("/api/user/security/devices", get(get_trusted_devices)) +} + +async fn get_storage_info(State(_state): State>) -> Html { + Html( + r##"
+
+
+
+
+ 2.5 GB used + of 10 GB +
+
+
+ ๐Ÿ“„ + Documents + 1.2 GB +
+
+ ๐Ÿ–ผ๏ธ + Images + 800 MB +
+
+ ๐Ÿ“ง + Emails + 500 MB +
+
+
"## + .to_string(), + ) +} + +async fn get_storage_connections(State(_state): State>) -> Html { + Html( + r##"
+

No external storage connections configured

+ +
"## + .to_string(), + ) +} + +async fn get_2fa_status(State(_state): State>) -> Html { + Html( + r##"
+ + Two-factor authentication is not enabled +
"## + .to_string(), + ) +} + +async fn enable_2fa(State(_state): State>) -> Html { + Html( + r##"
+ + Two-factor authentication enabled +
"## + .to_string(), + ) +} + +async fn disable_2fa(State(_state): State>) -> Html { + Html( + r##"
+ + Two-factor authentication disabled +
"## + .to_string(), + ) +} + +async fn get_active_sessions(State(_state): State>) -> Html { + Html( + r##"
+
+
+ ๐Ÿ’ป + Current Session + This device +
+
+ Current browser session + Active now +
+
+
+
+

No other active sessions

+
"## + .to_string(), + ) +} + +async fn revoke_all_sessions(State(_state): State>) -> Html { + Html( + r##"
+ โœ“ + All other sessions have been revoked +
"## + .to_string(), + ) +} + +async fn get_trusted_devices(State(_state): State>) -> Html { + Html( + r##"
+
+ ๐Ÿ’ป +
+ Current Device + Last active: Just now +
+
+ Trusted +
+
+

No other trusted devices

+
"## + .to_string(), + ) +} diff --git a/src/sources/knowledge_base.rs b/src/sources/knowledge_base.rs index 0807b0718..fdba40cda 100644 --- a/src/sources/knowledge_base.rs +++ b/src/sources/knowledge_base.rs @@ -229,14 +229,16 @@ struct SearchResultRow { } pub fn configure_knowledge_base_routes() -> Router> { + use crate::core::urls::ApiUrls; + Router::new() - .route("/api/sources/kb/upload", post(handle_upload_document)) - .route("/api/sources/kb/list", get(handle_list_sources)) - .route("/api/sources/kb/query", post(handle_query_knowledge_base)) - .route("/api/sources/kb/:id", get(handle_get_source)) - .route("/api/sources/kb/:id", delete(handle_delete_source)) - .route("/api/sources/kb/reindex", post(handle_reindex_sources)) - .route("/api/sources/kb/stats", get(handle_get_stats)) + .route(ApiUrls::SOURCES_KB_UPLOAD, post(handle_upload_document)) + .route(ApiUrls::SOURCES_KB_LIST, get(handle_list_sources)) + .route(ApiUrls::SOURCES_KB_QUERY, post(handle_query_knowledge_base)) + .route(&ApiUrls::SOURCES_KB_BY_ID.replace(":id", "{id}"), get(handle_get_source)) + .route(&ApiUrls::SOURCES_KB_BY_ID.replace(":id", "{id}"), delete(handle_delete_source)) + .route(ApiUrls::SOURCES_KB_REINDEX, post(handle_reindex_sources)) + .route(ApiUrls::SOURCES_KB_STATS, get(handle_get_stats)) } pub async fn handle_upload_document( diff --git a/src/sources/mod.rs b/src/sources/mod.rs index 93835f0fb..40b9fe77e 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -148,47 +148,49 @@ pub struct AppInfo { } pub fn configure_sources_routes() -> Router> { + use crate::core::urls::ApiUrls; + Router::new() .merge(knowledge_base::configure_knowledge_base_routes()) - .route("/api/sources/prompts", get(handle_prompts)) - .route("/api/sources/templates", get(handle_templates)) - .route("/api/sources/news", get(handle_news)) - .route("/api/sources/mcp-servers", get(handle_mcp_servers)) - .route("/api/sources/llm-tools", get(handle_llm_tools)) - .route("/api/sources/models", get(handle_models)) - .route("/api/sources/search", get(handle_search)) - .route("/api/sources/repositories", get(handle_list_repositories)) + .route(ApiUrls::SOURCES_PROMPTS, get(handle_prompts)) + .route(ApiUrls::SOURCES_TEMPLATES, get(handle_templates)) + .route(ApiUrls::SOURCES_NEWS, get(handle_news)) + .route(ApiUrls::SOURCES_MCP_SERVERS, get(handle_mcp_servers)) + .route(ApiUrls::SOURCES_LLM_TOOLS, get(handle_llm_tools)) + .route(ApiUrls::SOURCES_MODELS, get(handle_models)) + .route(ApiUrls::SOURCES_SEARCH, get(handle_search)) + .route(ApiUrls::SOURCES_REPOSITORIES, get(handle_list_repositories)) .route( - "/api/sources/repositories/:id/connect", + ApiUrls::SOURCES_REPOSITORIES_CONNECT, post(handle_connect_repository), ) .route( - "/api/sources/repositories/:id/disconnect", + ApiUrls::SOURCES_REPOSITORIES_DISCONNECT, post(handle_disconnect_repository), ) - .route("/api/sources/apps", get(handle_list_apps)) - .route("/api/sources/mcp", get(handle_list_mcp_servers_json)) - .route("/api/sources/mcp", post(handle_add_mcp_server)) - .route("/api/sources/mcp/:name", get(handle_get_mcp_server)) - .route("/api/sources/mcp/:name", put(handle_update_mcp_server)) - .route("/api/sources/mcp/:name", delete(handle_delete_mcp_server)) + .route(ApiUrls::SOURCES_APPS, get(handle_list_apps)) + .route(ApiUrls::SOURCES_MCP, get(handle_list_mcp_servers_json)) + .route(ApiUrls::SOURCES_MCP, post(handle_add_mcp_server)) + .route(&ApiUrls::SOURCES_MCP_BY_NAME.replace(":name", "{name}"), get(handle_get_mcp_server)) + .route(&ApiUrls::SOURCES_MCP_BY_NAME.replace(":name", "{name}"), put(handle_update_mcp_server)) + .route(&ApiUrls::SOURCES_MCP_BY_NAME.replace(":name", "{name}"), delete(handle_delete_mcp_server)) .route( - "/api/sources/mcp/:name/enable", + &ApiUrls::SOURCES_MCP_ENABLE.replace(":name", "{name}"), post(handle_enable_mcp_server), ) .route( - "/api/sources/mcp/:name/disable", + &ApiUrls::SOURCES_MCP_DISABLE.replace(":name", "{name}"), post(handle_disable_mcp_server), ) .route( - "/api/sources/mcp/:name/tools", + &ApiUrls::SOURCES_MCP_TOOLS.replace(":name", "{name}"), get(handle_list_mcp_server_tools), ) - .route("/api/sources/mcp/:name/test", post(handle_test_mcp_server)) - .route("/api/sources/mcp/scan", post(handle_scan_mcp_directory)) - .route("/api/sources/mcp/examples", get(handle_get_mcp_examples)) - .route("/api/sources/mentions", get(handle_mentions_autocomplete)) - .route("/api/sources/tools", get(handle_list_all_tools)) + .route(&ApiUrls::SOURCES_MCP_TEST.replace(":name", "{name}"), post(handle_test_mcp_server)) + .route(ApiUrls::SOURCES_MCP_SCAN, post(handle_scan_mcp_directory)) + .route(ApiUrls::SOURCES_MCP_EXAMPLES, get(handle_get_mcp_examples)) + .route(ApiUrls::SOURCES_MENTIONS, get(handle_mentions_autocomplete)) + .route(ApiUrls::SOURCES_TOOLS, get(handle_list_all_tools)) } pub async fn handle_list_mcp_servers_json( @@ -676,7 +678,71 @@ pub async fn handle_list_repositories(State(_state): State>) -> im last_sync: Some("2024-01-15T10:30:00Z".to_string()), }]; - Json(ApiResponse::success(repos)) + let mut html = String::new(); + html.push_str("
"); + + for repo in &repos { + let status_class = if repo.status == "connected" { "connected" } else { "disconnected" }; + let status_text = if repo.status == "connected" { "Connected" } else { "Disconnected" }; + let language = repo.language.as_deref().unwrap_or("Unknown"); + let last_sync = repo.last_sync.as_deref().unwrap_or("Never"); + + let _ = write!( + html, + r#"
+
+
+ + + +
+
+

{}

+ {} +
+ {} +
+

{}

+
+ + + + + {} + + โญ {} + ๐Ÿด {} + Last sync: {} +
+
+ +
+
"#, + html_escape(&repo.name), + html_escape(&repo.owner), + status_class, + status_text, + html_escape(&repo.description), + language, + repo.stars, + repo.forks, + last_sync, + html_escape(&repo.url) + ); + } + + if repos.is_empty() { + html.push_str(r#"
+ + + +

No Repositories

+

Connect your GitHub repositories to get started

+
"#); + } + + html.push_str("
"); + Html(html) } pub async fn handle_connect_repository( @@ -707,7 +773,56 @@ pub async fn handle_list_apps(State(_state): State>) -> impl IntoR status: "active".to_string(), }]; - Json(ApiResponse::success(apps)) + let mut html = String::new(); + html.push_str("
"); + + for app in &apps { + let app_icon = match app.app_type.as_str() { + "htmx" => "๐Ÿ“ฑ", + "react" => "โš›๏ธ", + "vue" => "๐Ÿ’š", + _ => "๐Ÿ”ท", + }; + + let _ = write!( + html, + r#"
+
+
{}
+
+

{}

+ {} +
+
+

{}

+
+ + +
+
"#, + app_icon, + html_escape(&app.name), + html_escape(&app.app_type), + html_escape(&app.description), + html_escape(&app.url) + ); + } + + if apps.is_empty() { + html.push_str(r#"
+ + + + + + +

No Apps

+

Create your first app to get started

+
"#); + } + + html.push_str("
"); + Html(html) } pub async fn handle_prompts( @@ -826,6 +941,98 @@ pub async fn handle_news(State(_state): State>) -> impl IntoRespon Html(html) } +/// MCP Server from JSON catalog +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpServerCatalogEntry { + pub id: String, + pub name: String, + pub description: String, + pub icon: String, + #[serde(rename = "type")] + pub server_type: String, + pub category: String, + pub provider: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpServersCatalog { + pub mcp_servers: Vec, + pub categories: Vec, + pub types: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpServerType { + pub id: String, + pub name: String, + pub description: String, +} + +fn load_mcp_servers_catalog() -> Option { + let catalog_path = std::path::Path::new("./3rdparty/mcp_servers.json"); + if catalog_path.exists() { + match std::fs::read_to_string(catalog_path) { + Ok(content) => match serde_json::from_str(&content) { + Ok(catalog) => Some(catalog), + Err(e) => { + error!("Failed to parse mcp_servers.json: {}", e); + None + } + }, + Err(e) => { + error!("Failed to read mcp_servers.json: {}", e); + None + } + } + } else { + None + } +} + +fn get_type_badge_class(server_type: &str) -> &'static str { + match server_type { + "Local" => "badge-local", + "Remote" => "badge-remote", + "Custom" => "badge-custom", + _ => "badge-default", + } +} + +fn get_category_icon(category: &str) -> &'static str { + match category { + "Database" => "๐Ÿ—„๏ธ", + "Analytics" => "๐Ÿ“Š", + "Search" => "๐Ÿ”", + "Vector Database" => "๐Ÿงฎ", + "Deployment" => "๐Ÿš€", + "Data Catalog" => "๐Ÿ“š", + "Productivity" => "โœ…", + "AI/ML" => "๐Ÿค–", + "Storage" => "๐Ÿ’พ", + "DevOps" => "โš™๏ธ", + "Process Mining" => "โ›๏ธ", + "Development" => "๐Ÿ’ป", + "Communication" => "๐Ÿ’ฌ", + "Customer Support" => "๐ŸŽง", + "Finance" => "๐Ÿ’ฐ", + "Enterprise" => "๐Ÿข", + "HR" => "๐Ÿ‘ฅ", + "Security" => "๐Ÿ”’", + "Documentation" => "๐Ÿ“–", + "Integration" => "๐Ÿ”—", + "API" => "๐Ÿ”Œ", + "Payments" => "๐Ÿ’ณ", + "Maps" => "๐Ÿ—บ๏ธ", + "Web Development" => "๐ŸŒ", + "Scheduling" => "๐Ÿ“…", + "Document Management" => "๐Ÿ“", + "Contact Management" => "๐Ÿ“‡", + "URL Shortener" => "๐Ÿ”—", + "Manufacturing" => "๐Ÿญ", + _ => "๐Ÿ“ฆ", + } +} + pub async fn handle_mcp_servers( State(_state): State>, Query(params): Query, @@ -836,49 +1043,66 @@ pub async fn handle_mcp_servers( let loader = McpCsvLoader::new(&work_path, &bot_id); let scan_result = loader.load(); + // Load MCP servers catalog from JSON + let catalog = load_mcp_servers_catalog(); + let mut html = String::new(); - html.push_str("
"); - html.push_str("
"); - html.push_str("

MCP Servers

"); - html.push_str("

Model Context Protocol servers extend your bot's capabilities. Configure servers in mcp.csv.

"); - html.push_str("
"); - html.push_str(""); - html.push_str( - "", - ); + html.push_str("
"); + + // Header section + html.push_str("
"); + html.push_str("

MCP Servers

"); + html.push_str("

Model Context Protocol servers extend your bot's capabilities

"); + html.push_str("
"); + html.push_str(""); + html.push_str(""); html.push_str("
"); + // Configured Servers Section (from CSV) + html.push_str("
"); + html.push_str("

๐Ÿ”ง Configured Servers

"); let _ = write!( html, - "
MCP Config:{}{}
", + "
Config: {}{}
", scan_result.file_path.to_string_lossy(), - if loader.csv_exists() { "" } else { "Not Found" } + if loader.csv_exists() { "" } else { " Not Found" } ); - html.push_str("
"); + html.push_str("
"); if scan_result.servers.is_empty() { - html.push_str("
๐Ÿ”Œ

No MCP Servers Found

Add MCP server configuration files to your .gbmcp directory.

"); + html.push_str("
๐Ÿ”ŒNo servers configured. Add from catalog below or create mcp.csv.
"); } else { for server in &scan_result.servers { let is_active = matches!( server.status, crate::basic::keywords::mcp_client::McpServerStatus::Active ); - let status_class = if is_active { - "status-active" - } else { - "status-inactive" - }; + let status_class = if is_active { "status-active" } else { "status-inactive" }; let status_text = if is_active { "Active" } else { "Inactive" }; + let status_bg = if is_active { "#e8f5e9" } else { "#ffebee" }; + let status_color = if is_active { "#2e7d32" } else { "#c62828" }; + let _ = write!( html, - "
{}

{}

{}
{}

{}

{} tools
", + "
+
+
{}
+

{}

{}
+ {} +
+

{}

+
+ {} tools + +
+
", mcp::get_server_type_icon(&server.server_type.to_string()), html_escape(&server.name), server.server_type, - status_class, + status_bg, + status_color, status_text, if server.description.is_empty() { "No description".to_string() } else { html_escape(&server.description) }, server.tools.len(), @@ -886,8 +1110,84 @@ pub async fn handle_mcp_servers( ); } } - html.push_str("
"); + + // MCP Server Catalog Section (from JSON) + if let Some(ref catalog) = catalog { + html.push_str("
"); + html.push_str("

๐Ÿ“ฆ Available MCP Servers

"); + html.push_str("

Browse and add MCP servers from the catalog

"); + + // Category filter with inline onclick handlers + html.push_str("
"); + html.push_str(""); + for category in &catalog.categories { + let _ = write!( + html, + "", + html_escape(category), + html_escape(category) + ); + } + html.push_str("
"); + + html.push_str("
"); + for server in &catalog.mcp_servers { + let badge_bg = match server.server_type.as_str() { + "Local" => "#e3f2fd", + "Remote" => "#e8f5e9", + "Custom" => "#fff3e0", + _ => "#f5f5f5", + }; + let badge_color = match server.server_type.as_str() { + "Local" => "#1565c0", + "Remote" => "#2e7d32", + "Custom" => "#ef6c00", + _ => "#333", + }; + let category_icon = get_category_icon(&server.category); + + let _ = write!( + html, + "
+
+
{}
+
+

{}

+ {} +
+ MCP: {} +
+

{}

+
+ {} {} + +
+
", + html_escape(&server.category), + html_escape(&server.id), + category_icon, + html_escape(&server.name), + html_escape(&server.provider), + badge_bg, + badge_color, + html_escape(&server.server_type), + html_escape(&server.description), + category_icon, + html_escape(&server.category), + html_escape(&server.id), + html_escape(&server.name) + ); + } + html.push_str("
"); + } else { + html.push_str("
"); + html.push_str("
๐Ÿ“ฆ

MCP Catalog Not Found

Create 3rdparty/mcp_servers.json to browse available servers.

"); + html.push_str("
"); + } + + html.push_str("
"); + Html(html) } diff --git a/src/tasks/mod.rs b/src/tasks/mod.rs index 4cce8a8c2..b98c7a76f 100644 --- a/src/tasks/mod.rs +++ b/src/tasks/mod.rs @@ -2067,32 +2067,36 @@ pub async fn handle_task_set_dependencies( } pub fn configure_task_routes() -> Router> { + use crate::core::urls::ApiUrls; + log::info!("[ROUTES] Registering task routes with /api/tasks/:id pattern"); Router::new() - // Task list and create + // JSON API - Task create + .route(ApiUrls::TASKS, post(handle_task_create)) + // HTMX/HTML APIs + .route(ApiUrls::TASKS_LIST_HTMX, get(handle_task_list_htmx)) + .route(ApiUrls::TASKS_STATS, get(handle_task_stats_htmx)) + .route(ApiUrls::TASKS_TIME_SAVED, get(handle_time_saved)) + .route(ApiUrls::TASKS_COMPLETED, delete(handle_clear_completed)) .route( - "/api/tasks", - post(handle_task_create).get(handle_task_list_htmx), + &ApiUrls::TASKS_GET_HTMX.replace(":id", "{id}"), + get(handle_task_get), ) - // Specific routes MUST come before parameterized route - .route("/api/tasks/stats", get(handle_task_stats_htmx)) - .route("/api/tasks/stats/json", get(handle_task_stats)) - .route("/api/tasks/time-saved", get(handle_time_saved)) - .route("/api/tasks/completed", delete(handle_clear_completed)) - // Parameterized task routes - use :id for axum path params + // JSON API - Stats + .route(ApiUrls::TASKS_STATS_JSON, get(handle_task_stats)) + // JSON API - Parameterized task routes .route( - "/api/tasks/:id", - get(handle_task_get) - .put(handle_task_update) + &ApiUrls::TASK_BY_ID.replace(":id", "{id}"), + put(handle_task_update) .delete(handle_task_delete) .patch(handle_task_patch), ) - .route("/api/tasks/:id/assign", post(handle_task_assign)) - .route("/api/tasks/:id/status", put(handle_task_status_update)) - .route("/api/tasks/:id/priority", put(handle_task_priority_set)) - .route("/api/tasks/:id/dependencies", put(handle_task_set_dependencies)) - .route("/api/tasks/:id/cancel", post(handle_task_cancel)) + .route(&ApiUrls::TASK_ASSIGN.replace(":id", "{id}"), post(handle_task_assign)) + .route(&ApiUrls::TASK_STATUS.replace(":id", "{id}"), put(handle_task_status_update)) + .route(&ApiUrls::TASK_PRIORITY.replace(":id", "{id}"), put(handle_task_priority_set)) + .route("/api/tasks/{id}/dependencies", put(handle_task_set_dependencies)) + .route("/api/tasks/{id}/cancel", post(handle_task_cancel)) } pub async fn handle_task_cancel(