diff --git a/src/basic/keywords/mcp_directory.rs b/src/basic/keywords/mcp_directory.rs new file mode 100644 index 00000000..684fffde --- /dev/null +++ b/src/basic/keywords/mcp_directory.rs @@ -0,0 +1,977 @@ +//! MCP Server Loader +//! +//! Loads MCP (Model Context Protocol) servers from `mcp.csv` file in the bot's `.gbai` folder. +//! This enables users to add MCP servers by defining them in a CSV file, making MCP tools +//! available to Tasks just like BASIC keywords are available. +//! +//! ## mcp.csv Format +//! +//! ```csv +//! name,type,command,args,description,enabled +//! filesystem,stdio,npx,"-y @modelcontextprotocol/server-filesystem /data",Access local files,true +//! github,stdio,npx,"-y @modelcontextprotocol/server-github",GitHub API access,true +//! postgres,stdio,npx,"-y @modelcontextprotocol/server-postgres",Database queries,false +//! myapi,http,https://api.example.com/mcp,,Custom API server,true +//! ``` +//! +//! ## Columns +//! +//! | Column | Required | Description | +//! |--------|----------|-------------| +//! | name | Yes | Unique server identifier (used in USE MCP calls) | +//! | type | Yes | Connection type: stdio, http, websocket, tcp | +//! | command | Yes | For stdio: command to run. For http/ws: URL | +//! | args | No | Command arguments (space-separated) or empty | +//! | description | No | Human-readable description | +//! | enabled | No | true/false (default: true) | +//! +//! ## Usage in BASIC +//! +//! ```basic +//! ' Read a file using filesystem MCP server +//! content = USE MCP "filesystem", "read_file", {"path": "/data/config.json"} +//! +//! ' Query database +//! results = USE MCP "postgres", "query", {"sql": "SELECT * FROM users"} +//! ``` + +use chrono::Utc; +use log::{debug, info}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::PathBuf; +use uuid::Uuid; + +use super::mcp_client::{ + ConnectionType, HealthStatus, McpAuth, McpCapabilities, McpConnection, McpServer, + McpServerStatus, McpServerType, +}; + +/// Row from mcp.csv file +#[derive(Debug, Clone)] +pub struct McpCsvRow { + /// Server name (required) + pub name: String, + /// Connection type: stdio, http, websocket, tcp (required) + pub connection_type: String, + /// Command (for stdio) or URL (for http/ws) (required) + pub command: String, + /// Arguments for stdio command (optional) + pub args: String, + /// Human-readable description (optional) + pub description: String, + /// Whether server is enabled (optional, default: true) + pub enabled: bool, + /// Authentication type (optional): none, api_key, bearer + pub auth_type: Option, + /// Auth credential environment variable name (optional) + pub auth_env: Option, + /// Risk level: safe, low, medium, high, critical (optional, default: medium) + pub risk_level: Option, + /// Whether tools require approval (optional, default: false) + pub requires_approval: bool, +} + +/// Configuration for an MCP server (for JSON serialization) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpServerConfig { + /// Server name (used for identification and @mentions) + pub name: String, + + /// Human-readable description + #[serde(default)] + pub description: String, + + /// Server type (filesystem, database, github, etc.) + #[serde(rename = "type")] + pub server_type: String, + + /// Whether this server is enabled + #[serde(default = "default_enabled")] + pub enabled: bool, + + /// Connection configuration + pub connection: McpConnectionConfig, + + /// Authentication configuration (optional) + #[serde(default)] + pub auth: Option, + + /// Pre-defined tools (optional, will be discovered if not specified) + #[serde(default)] + pub tools: Vec, + + /// Environment variables to set for stdio servers + #[serde(default)] + pub env: HashMap, + + /// Tags for categorization + #[serde(default)] + pub tags: Vec, + + /// Risk level for this server's tools + #[serde(default)] + pub risk_level: String, + + /// Whether tools from this server require human approval + #[serde(default)] + pub requires_approval: bool, +} + +fn default_enabled() -> bool { + true +} + +/// Connection configuration for MCP servers +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum McpConnectionConfig { + /// Standard I/O connection (for local MCP servers) + #[serde(rename = "stdio")] + Stdio { + /// Command to execute + command: String, + /// Command arguments + #[serde(default)] + args: Vec, + /// Working directory + #[serde(default)] + cwd: Option, + }, + + /// HTTP/REST connection + #[serde(rename = "http")] + Http { + /// Server URL + url: String, + /// Request timeout in seconds + #[serde(default = "default_timeout")] + timeout: u32, + /// Custom headers + #[serde(default)] + headers: HashMap, + }, + + /// WebSocket connection + #[serde(rename = "websocket")] + WebSocket { + /// WebSocket URL + url: String, + /// Connection timeout in seconds + #[serde(default = "default_timeout")] + timeout: u32, + }, + + /// TCP socket connection + #[serde(rename = "tcp")] + Tcp { + /// Host address + host: String, + /// Port number + port: u16, + }, +} + +fn default_timeout() -> u32 { + 30 +} + +/// Authentication configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum McpAuthConfig { + /// No authentication + #[serde(rename = "none")] + None, + + /// API key authentication + #[serde(rename = "api_key")] + ApiKey { + /// Header name for the API key + #[serde(default = "default_api_key_header")] + header: String, + /// Environment variable containing the API key + key_env: String, + }, + + /// Bearer token authentication + #[serde(rename = "bearer")] + Bearer { + /// Environment variable containing the token + token_env: String, + }, + + /// Basic authentication + #[serde(rename = "basic")] + Basic { + /// Environment variable for username + username_env: String, + /// Environment variable for password + password_env: String, + }, +} + +fn default_api_key_header() -> String { + "X-API-Key".to_string() +} + +/// Tool configuration (pre-defined or discovered) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolConfig { + /// Tool name + pub name: String, + + /// Tool description + #[serde(default)] + pub description: String, + + /// Input parameters schema (JSON Schema) + #[serde(default)] + pub input_schema: Option, + + /// Output schema (JSON Schema) + #[serde(default)] + pub output_schema: Option, + + /// Risk level for this specific tool + #[serde(default)] + pub risk_level: Option, + + /// Whether this tool requires approval + #[serde(default)] + pub requires_approval: bool, + + /// Rate limit (calls per minute) + #[serde(default)] + pub rate_limit: Option, +} + +/// Result of loading MCP servers from CSV +#[derive(Debug, Clone)] +pub struct McpLoadResult { + /// Successfully loaded servers + pub servers: Vec, + + /// Errors encountered during loading + pub errors: Vec, + + /// Total lines processed + pub lines_processed: usize, + + /// CSV file path that was loaded + pub file_path: PathBuf, +} + +/// Error encountered during MCP CSV loading +#[derive(Debug, Clone)] +pub struct McpLoadError { + /// Line number in the CSV file + pub line: usize, + + /// Error message + pub message: String, + + /// Whether this error is recoverable + pub recoverable: bool, +} + +/// MCP CSV Loader +pub struct McpCsvLoader { + /// Base work path + work_path: String, + + /// Bot ID + bot_id: String, +} + +impl McpCsvLoader { + /// Create a new loader for a bot + pub fn new(work_path: &str, bot_id: &str) -> Self { + Self { + work_path: work_path.to_string(), + bot_id: bot_id.to_string(), + } + } + + /// Get the mcp.csv file path for this bot + pub fn get_csv_path(&self) -> PathBuf { + PathBuf::from(&self.work_path) + .join(format!("{}.gbai", self.bot_id)) + .join("mcp.csv") + } + + /// Check if mcp.csv file exists + pub fn csv_exists(&self) -> bool { + self.get_csv_path().exists() + } + + /// Load MCP servers from mcp.csv + pub fn load(&self) -> McpLoadResult { + let csv_path = self.get_csv_path(); + + info!("Loading MCP servers from: {:?}", csv_path); + + let mut result = McpLoadResult { + servers: Vec::new(), + errors: Vec::new(), + lines_processed: 0, + file_path: csv_path.clone(), + }; + + if !csv_path.exists() { + debug!("MCP CSV file does not exist: {:?}", csv_path); + return result; + } + + let content = match std::fs::read_to_string(&csv_path) { + Ok(c) => c, + Err(e) => { + result.errors.push(McpLoadError { + line: 0, + message: format!("Failed to read mcp.csv: {}", e), + recoverable: false, + }); + return result; + } + }; + + // Parse CSV + let mut lines = content.lines().enumerate(); + + // Skip header if present + if let Some((_, header)) = lines.next() { + let header_lower = header.to_lowercase(); + if !header_lower.starts_with("name,") && !header_lower.contains(",type,") { + // First line is data, not header - process it + if let Some(server) = self.parse_csv_line(1, header, &mut result.errors) { + result.servers.push(server); + } + } + result.lines_processed += 1; + } + + // Process data lines + for (line_num, line) in lines { + let line_number = line_num + 1; // 1-based line numbers + result.lines_processed += 1; + + // Skip empty lines and comments + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with("//") { + continue; + } + + if let Some(server) = self.parse_csv_line(line_number, line, &mut result.errors) { + result.servers.push(server); + } + } + + info!( + "MCP load complete: {} servers loaded, {} errors, {} lines processed", + result.servers.len(), + result.errors.len(), + result.lines_processed + ); + + result + } + + /// Parse a single CSV line into an McpServer + fn parse_csv_line( + &self, + line_num: usize, + line: &str, + errors: &mut Vec, + ) -> Option { + // Parse CSV columns, handling quoted values + let columns = self.parse_csv_columns(line); + + if columns.len() < 3 { + errors.push(McpLoadError { + line: line_num, + message: format!( + "Invalid CSV: expected at least 3 columns (name,type,command), got {}", + columns.len() + ), + recoverable: true, + }); + return None; + } + + let name = columns[0].trim().to_string(); + let conn_type = columns[1].trim().to_lowercase(); + let command = columns[2].trim().to_string(); + let args = columns + .get(3) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + let description = columns + .get(4) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + let enabled = columns + .get(5) + .map(|s| s.trim().to_lowercase() != "false") + .unwrap_or(true); + let auth_type = columns + .get(6) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + let auth_env = columns + .get(7) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + let risk_level = columns + .get(8) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + let requires_approval = columns + .get(9) + .map(|s| s.trim().to_lowercase() == "true") + .unwrap_or(false); + + if name.is_empty() { + errors.push(McpLoadError { + line: line_num, + message: "Server name cannot be empty".to_string(), + recoverable: true, + }); + return None; + } + + if command.is_empty() { + errors.push(McpLoadError { + line: line_num, + message: format!("Command/URL cannot be empty for server '{}'", name), + recoverable: true, + }); + return None; + } + + // Build connection config + let connection = match conn_type.as_str() { + "stdio" => { + let args_vec: Vec = if args.is_empty() { + Vec::new() + } else { + // Split args, respecting quoted strings + self.parse_args(&args) + }; + McpConnection { + connection_type: ConnectionType::Stdio, + url: format!("{}:{}", command, args_vec.join(" ")), + port: None, + timeout_seconds: 30, + max_retries: 3, + retry_backoff_ms: 1000, + keep_alive: true, + tls_config: None, + } + } + "http" => McpConnection { + connection_type: ConnectionType::Http, + url: command.clone(), + port: None, + timeout_seconds: 30, + max_retries: 3, + retry_backoff_ms: 1000, + keep_alive: true, + tls_config: None, + }, + "websocket" | "ws" => McpConnection { + connection_type: ConnectionType::WebSocket, + url: command.clone(), + port: None, + timeout_seconds: 30, + max_retries: 3, + retry_backoff_ms: 1000, + keep_alive: true, + tls_config: None, + }, + "tcp" => { + // Parse host:port from command + let parts: Vec<&str> = command.split(':').collect(); + let host = parts.get(0).unwrap_or(&"localhost").to_string(); + let port: u16 = parts.get(1).and_then(|p| p.parse().ok()).unwrap_or(9000); + McpConnection { + connection_type: ConnectionType::Tcp, + url: host, + port: Some(port), + timeout_seconds: 30, + max_retries: 3, + retry_backoff_ms: 1000, + keep_alive: true, + tls_config: None, + } + } + _ => { + errors.push(McpLoadError { + line: line_num, + message: format!( + "Unknown connection type '{}' for server '{}'. Use: stdio, http, websocket, tcp", + conn_type, name + ), + recoverable: true, + }); + return None; + } + }; + + // Build auth config + let auth = match (auth_type.as_deref(), auth_env.as_ref()) { + (Some("api_key"), Some(env)) => { + use super::mcp_client::{McpAuthType, McpCredentials}; + McpAuth { + auth_type: McpAuthType::ApiKey, + credentials: McpCredentials::ApiKey { + header_name: "X-API-Key".to_string(), + key_ref: env.clone(), + }, + } + } + (Some("bearer"), Some(env)) => { + use super::mcp_client::{McpAuthType, McpCredentials}; + McpAuth { + auth_type: McpAuthType::Bearer, + credentials: McpCredentials::Bearer { + token_ref: env.clone(), + }, + } + } + _ => McpAuth::default(), + }; + + // Determine server type from name or connection + let server_type = self.infer_server_type(&name, &conn_type, &command); + + // Determine status + let status = if enabled { + McpServerStatus::Active + } else { + McpServerStatus::Inactive + }; + + debug!( + "Loaded MCP server '{}' (type={}, enabled={})", + name, server_type, enabled + ); + + Some(McpServer { + id: Uuid::new_v4().to_string(), + name, + description, + server_type, + connection, + auth, + tools: Vec::new(), // Tools are discovered at runtime + capabilities: McpCapabilities { + tools: true, + resources: false, + prompts: false, + logging: false, + streaming: false, + cancellation: false, + custom: HashMap::new(), + }, + status, + bot_id: self.bot_id.clone(), + created_at: Utc::now(), + updated_at: Utc::now(), + last_health_check: None, + health_status: HealthStatus::default(), + }) + } + + /// Parse CSV columns, handling quoted values + fn parse_csv_columns(&self, line: &str) -> Vec { + let mut columns = Vec::new(); + let mut current = String::new(); + let mut in_quotes = false; + let mut chars = line.chars().peekable(); + + while let Some(c) = chars.next() { + match c { + '"' if !in_quotes => { + in_quotes = true; + } + '"' if in_quotes => { + // Check for escaped quote + if chars.peek() == Some(&'"') { + chars.next(); + current.push('"'); + } else { + in_quotes = false; + } + } + ',' if !in_quotes => { + columns.push(current.clone()); + current.clear(); + } + _ => { + current.push(c); + } + } + } + columns.push(current); + + columns + } + + /// Parse command arguments, handling quoted strings + fn parse_args(&self, args: &str) -> Vec { + let mut result = Vec::new(); + let mut current = String::new(); + let mut in_quotes = false; + let mut quote_char = ' '; + + for c in args.chars() { + match c { + '"' | '\'' if !in_quotes => { + in_quotes = true; + quote_char = c; + } + c if in_quotes && c == quote_char => { + in_quotes = false; + } + ' ' if !in_quotes => { + if !current.is_empty() { + result.push(current.clone()); + current.clear(); + } + } + _ => { + current.push(c); + } + } + } + if !current.is_empty() { + result.push(current); + } + + result + } + + /// Infer server type from name and connection info + fn infer_server_type(&self, name: &str, conn_type: &str, command: &str) -> McpServerType { + let name_lower = name.to_lowercase(); + let cmd_lower = command.to_lowercase(); + + if name_lower.contains("filesystem") || cmd_lower.contains("filesystem") { + McpServerType::Filesystem + } else if name_lower.contains("github") || cmd_lower.contains("github") { + McpServerType::Web // GitHub is accessed via API + } else if name_lower.contains("postgres") + || name_lower.contains("mysql") + || name_lower.contains("sqlite") + || name_lower.contains("database") + || cmd_lower.contains("postgres") + || cmd_lower.contains("mysql") + || cmd_lower.contains("sqlite") + { + McpServerType::Database + } else if name_lower.contains("slack") || cmd_lower.contains("slack") { + McpServerType::Slack + } else if name_lower.contains("teams") || cmd_lower.contains("teams") { + McpServerType::Teams + } else if name_lower.contains("email") + || name_lower.contains("smtp") + || name_lower.contains("imap") + { + McpServerType::Email + } else if name_lower.contains("analytics") { + McpServerType::Analytics + } else if name_lower.contains("search") { + McpServerType::Search + } else if name_lower.contains("storage") + || name_lower.contains("s3") + || name_lower.contains("minio") + { + McpServerType::Storage + } else if conn_type == "http" || conn_type == "websocket" { + McpServerType::Web + } else { + McpServerType::Custom("custom".to_string()) + } + } + + /// Load a specific MCP server by name + pub fn load_server(&self, name: &str) -> Option { + let result = self.load(); + result.servers.into_iter().find(|s| s.name == name) + } + + /// Create mcp.csv with example content + pub fn create_example_csv(&self) -> std::io::Result { + let csv_path = self.get_csv_path(); + + // Ensure parent directory exists + if let Some(parent) = csv_path.parent() { + std::fs::create_dir_all(parent)?; + } + + let example_content = r#"name,type,command,args,description,enabled +# MCP Server Configuration +# Columns: name,type,command,args,description,enabled,auth_type,auth_env,risk_level,requires_approval +# +# type: stdio (local process), http (REST API), websocket, tcp +# auth_type: none, api_key, bearer +# risk_level: safe, low, medium, high, critical +# +# Example servers: +filesystem,stdio,npx,"-y @modelcontextprotocol/server-filesystem /data",Access local files and directories,true +# github,stdio,npx,"-y @modelcontextprotocol/server-github",GitHub API integration,true,bearer,GITHUB_TOKEN +# postgres,stdio,npx,"-y @modelcontextprotocol/server-postgres",PostgreSQL database queries,false +# slack,stdio,npx,"-y @modelcontextprotocol/server-slack",Slack messaging,false,bearer,SLACK_BOT_TOKEN +# myapi,http,https://api.example.com/mcp,,Custom API server,true,api_key,MY_API_KEY +"#; + + std::fs::write(&csv_path, example_content)?; + info!("Created example mcp.csv at {:?}", csv_path); + + Ok(csv_path) + } + + /// Add a server to mcp.csv + pub fn add_server(&self, row: &McpCsvRow) -> std::io::Result<()> { + let csv_path = self.get_csv_path(); + + // Create file with header if it doesn't exist + if !csv_path.exists() { + if let Some(parent) = csv_path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&csv_path, "name,type,command,args,description,enabled,auth_type,auth_env,risk_level,requires_approval\n")?; + } + + // Build CSV line + let line = format!( + "{},{},{},{},{},{},{},{},{},{}\n", + self.escape_csv(&row.name), + self.escape_csv(&row.connection_type), + self.escape_csv(&row.command), + self.escape_csv(&row.args), + self.escape_csv(&row.description), + row.enabled, + row.auth_type.as_deref().unwrap_or(""), + row.auth_env.as_deref().unwrap_or(""), + row.risk_level.as_deref().unwrap_or("medium"), + row.requires_approval + ); + + // Append to file + use std::io::Write; + let mut file = std::fs::OpenOptions::new().append(true).open(&csv_path)?; + file.write_all(line.as_bytes())?; + + info!("Added MCP server '{}' to {:?}", row.name, csv_path); + Ok(()) + } + + /// Remove a server from mcp.csv + pub fn remove_server(&self, name: &str) -> std::io::Result { + let csv_path = self.get_csv_path(); + + if !csv_path.exists() { + return Ok(false); + } + + let content = std::fs::read_to_string(&csv_path)?; + let mut new_lines: Vec<&str> = Vec::new(); + let mut found = false; + + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with("//") { + new_lines.push(line); + continue; + } + + let columns = self.parse_csv_columns(line); + if columns.first().map(|s| s.trim()) == Some(name) { + found = true; + continue; // Skip this line + } + + new_lines.push(line); + } + + if found { + std::fs::write(&csv_path, new_lines.join("\n") + "\n")?; + info!("Removed MCP server '{}' from {:?}", name, csv_path); + } + + Ok(found) + } + + /// Escape a value for CSV + fn escape_csv(&self, value: &str) -> String { + if value.contains(',') || value.contains('"') || value.contains('\n') { + format!("\"{}\"", value.replace('"', "\"\"")) + } else { + value.to_string() + } + } +} + +/// Generate example MCP server configurations (for API) +pub fn generate_example_configs() -> Vec { + vec![ + McpServerConfig { + name: "filesystem".to_string(), + description: "Access local files and directories".to_string(), + server_type: "filesystem".to_string(), + enabled: true, + connection: McpConnectionConfig::Stdio { + command: "npx".to_string(), + args: vec![ + "-y".to_string(), + "@modelcontextprotocol/server-filesystem".to_string(), + "/data".to_string(), + ], + cwd: None, + }, + auth: None, + tools: vec![ + McpToolConfig { + name: "read_file".to_string(), + description: "Read contents of a file".to_string(), + input_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "path": {"type": "string", "description": "File path to read"} + }, + "required": ["path"] + })), + output_schema: None, + risk_level: Some("low".to_string()), + requires_approval: false, + rate_limit: None, + }, + McpToolConfig { + name: "write_file".to_string(), + description: "Write contents to a file".to_string(), + input_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "path": {"type": "string", "description": "File path to write"}, + "content": {"type": "string", "description": "Content to write"} + }, + "required": ["path", "content"] + })), + output_schema: None, + risk_level: Some("medium".to_string()), + requires_approval: true, + rate_limit: None, + }, + McpToolConfig { + name: "list_directory".to_string(), + description: "List files in a directory".to_string(), + input_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "path": {"type": "string", "description": "Directory path"} + }, + "required": ["path"] + })), + output_schema: None, + risk_level: Some("safe".to_string()), + requires_approval: false, + rate_limit: None, + }, + ], + env: HashMap::new(), + tags: vec!["storage".to_string(), "files".to_string()], + risk_level: "low".to_string(), + requires_approval: false, + }, + McpServerConfig { + name: "github".to_string(), + description: "Interact with GitHub repositories".to_string(), + server_type: "github".to_string(), + enabled: true, + connection: McpConnectionConfig::Stdio { + command: "npx".to_string(), + args: vec![ + "-y".to_string(), + "@modelcontextprotocol/server-github".to_string(), + ], + cwd: None, + }, + auth: Some(McpAuthConfig::Bearer { + token_env: "GITHUB_TOKEN".to_string(), + }), + tools: vec![McpToolConfig { + name: "search_repositories".to_string(), + description: "Search GitHub repositories".to_string(), + input_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query"} + }, + "required": ["query"] + })), + output_schema: None, + risk_level: Some("safe".to_string()), + requires_approval: false, + rate_limit: Some(30), + }], + env: HashMap::new(), + tags: vec!["vcs".to_string(), "github".to_string()], + risk_level: "medium".to_string(), + requires_approval: false, + }, + ] +} + +// Re-export for backward compatibility +pub type McpDirectoryScanner = McpCsvLoader; +pub type McpDirectoryScanResult = McpLoadResult; +pub type McpScanError = McpLoadError; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_csv_columns() { + let loader = McpCsvLoader::new("./work", "test"); + + let cols = loader.parse_csv_columns("name,type,command"); + assert_eq!(cols, vec!["name", "type", "command"]); + + let cols = loader.parse_csv_columns( + "filesystem,stdio,npx,\"-y @modelcontextprotocol/server-filesystem\"", + ); + assert_eq!(cols.len(), 4); + assert_eq!(cols[3], "-y @modelcontextprotocol/server-filesystem"); + } + + #[test] + fn test_parse_args() { + let loader = McpCsvLoader::new("./work", "test"); + + let args = loader.parse_args("-y @modelcontextprotocol/server-filesystem /data"); + assert_eq!( + args, + vec!["-y", "@modelcontextprotocol/server-filesystem", "/data"] + ); + } + + #[test] + fn test_infer_server_type() { + let loader = McpCsvLoader::new("./work", "test"); + + assert!(matches!( + loader.infer_server_type("filesystem", "stdio", "npx"), + McpServerType::Filesystem + )); + assert!(matches!( + loader.infer_server_type("postgres", "stdio", "npx"), + McpServerType::Database + )); + assert!(matches!( + loader.infer_server_type("myapi", "http", "https://api.example.com"), + McpServerType::Web + )); + } +} diff --git a/src/basic/keywords/mod.rs b/src/basic/keywords/mod.rs index 938cd31c..9aeaf009 100644 --- a/src/basic/keywords/mod.rs +++ b/src/basic/keywords/mod.rs @@ -41,6 +41,7 @@ pub mod llm_keyword; pub mod llm_macros; pub mod math; pub mod mcp_client; +pub mod mcp_directory; pub mod messaging; pub mod model_routing; pub mod multimodal; @@ -81,6 +82,7 @@ pub mod webhook; pub use auto_task::{AutoTask, AutoTaskStatus, ExecutionMode, TaskPriority}; pub use intent_compiler::{CompiledIntent, ExecutionPlan, IntentCompiler, PlanStep}; pub use mcp_client::{McpClient, McpRequest, McpResponse, McpServer, McpTool}; +pub use mcp_directory::{McpDirectoryScanResult, McpDirectoryScanner, McpServerConfig}; pub use safety_layer::{AuditEntry, ConstraintCheckResult, SafetyLayer, SimulationResult}; // Re-export API handlers for route configuration diff --git a/src/sources/mcp.rs b/src/sources/mcp.rs new file mode 100644 index 00000000..95f2b8a5 --- /dev/null +++ b/src/sources/mcp.rs @@ -0,0 +1,113 @@ +//! MCP (Model Context Protocol) submodule for Sources +//! +//! Re-exports MCP CSV loading functionality and provides +//! convenience functions for working with MCP servers in the Sources context. +//! +//! MCP servers are configured via `mcp.csv` in the bot's `.gbai` folder. + +pub use crate::basic::keywords::mcp_directory::{ + generate_example_configs, McpConnectionConfig, McpCsvLoader, McpCsvRow, McpLoadError, + McpLoadResult, McpServerConfig, McpToolConfig, +}; + +// Re-exports for backward compatibility +pub use crate::basic::keywords::mcp_directory::{ + McpDirectoryScanResult, McpDirectoryScanner, McpScanError, +}; + +pub use crate::basic::keywords::mcp_client::{ + McpCapabilities, McpClient, McpConnection, McpRequest, McpResponse, McpServer, McpServerStatus, + McpServerType, McpTool, ToolRiskLevel, +}; + +/// Get icon for MCP server type +pub fn get_server_type_icon(server_type: &str) -> &'static str { + match server_type.to_lowercase().as_str() { + "filesystem" => "📁", + "database" => "🗄️", + "github" => "🐙", + "web" | "http" => "🌐", + "email" => "📧", + "slack" => "💬", + "teams" => "👥", + "analytics" => "📊", + "search" => "🔍", + "storage" => "💾", + "compute" => "⚡", + "custom" => "🔧", + _ => "🔌", + } +} + +/// Get risk level CSS class +pub fn get_risk_level_class(risk_level: &ToolRiskLevel) -> &'static str { + match risk_level { + ToolRiskLevel::Safe => "risk-safe", + ToolRiskLevel::Low => "risk-low", + ToolRiskLevel::Medium => "risk-medium", + ToolRiskLevel::High => "risk-high", + ToolRiskLevel::Critical => "risk-critical", + } +} + +/// Get risk level display name +pub fn get_risk_level_name(risk_level: &ToolRiskLevel) -> &'static str { + match risk_level { + ToolRiskLevel::Safe => "Safe", + ToolRiskLevel::Low => "Low", + ToolRiskLevel::Medium => "Medium", + ToolRiskLevel::High => "High", + ToolRiskLevel::Critical => "Critical", + } +} + +/// Create a new MCP CSV loader for a bot +pub fn create_loader(work_path: &str, bot_id: &str) -> McpCsvLoader { + McpCsvLoader::new(work_path, bot_id) +} + +/// Load MCP servers for a bot from mcp.csv +pub fn load_servers(work_path: &str, bot_id: &str) -> McpLoadResult { + let loader = McpCsvLoader::new(work_path, bot_id); + loader.load() +} + +/// Check if mcp.csv exists for a bot +pub fn csv_exists(work_path: &str, bot_id: &str) -> bool { + let loader = McpCsvLoader::new(work_path, bot_id); + loader.csv_exists() +} + +/// Get the mcp.csv path for a bot +pub fn get_csv_path(work_path: &str, bot_id: &str) -> std::path::PathBuf { + let loader = McpCsvLoader::new(work_path, bot_id); + loader.get_csv_path() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_server_type_icons() { + assert_eq!(get_server_type_icon("filesystem"), "📁"); + assert_eq!(get_server_type_icon("database"), "🗄️"); + assert_eq!(get_server_type_icon("github"), "🐙"); + assert_eq!(get_server_type_icon("unknown"), "🔌"); + } + + #[test] + fn test_risk_level_class() { + assert_eq!(get_risk_level_class(&ToolRiskLevel::Safe), "risk-safe"); + assert_eq!( + get_risk_level_class(&ToolRiskLevel::Critical), + "risk-critical" + ); + } + + #[test] + fn test_risk_level_name() { + assert_eq!(get_risk_level_name(&ToolRiskLevel::Safe), "Safe"); + assert_eq!(get_risk_level_name(&ToolRiskLevel::High), "High"); + } +} diff --git a/src/sources/mod.rs b/src/sources/mod.rs index f8858440..f4a8c544 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -1,22 +1,174 @@ +//! Sources Module +//! +//! Manages all source types for bots including: +//! - Repositories (GitHub, GitLab, Bitbucket) +//! - Apps (HTMX apps created with CREATE SITE) +//! - Prompts (System prompts and templates) +//! - Templates (Bot packages .gbai) +//! - MCP Servers (Model Context Protocol servers from mcp.csv) +//! - LLM Tools (Available tools for LLM invocation) +//! - Models (Configured AI models) +//! +//! MCP servers are configured via `mcp.csv` in the bot's `.gbai` folder, +//! making their tools available to Tasks just like BASIC keywords. + +pub mod mcp; + +use crate::basic::keywords::mcp_directory::{generate_example_configs, McpCsvLoader, McpCsvRow}; use crate::shared::state::AppState; use axum::{ - extract::{Query, State}, + extract::{Json, Path, Query, State}, + http::StatusCode, response::{Html, IntoResponse}, - routing::get, + routing::{delete, get, post, put}, Router, }; +use log::{error, info}; use serde::{Deserialize, Serialize}; use std::sync::Arc; +// ============================================================================ +// Request/Response Types +// ============================================================================ + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SearchQuery { pub q: Option, pub category: Option, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BotQuery { + pub bot_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpServerResponse { + pub id: String, + pub name: String, + pub description: String, + pub server_type: String, + pub status: String, + pub enabled: bool, + pub tools_count: usize, + pub source: String, + pub tags: Vec, + pub requires_approval: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolResponse { + pub name: String, + pub description: String, + pub server_name: String, + pub risk_level: String, + pub requires_approval: bool, + pub source: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AddMcpServerRequest { + pub name: String, + pub description: Option, + pub server_type: String, + pub connection: McpConnectionRequest, + pub auth: Option, + pub enabled: Option, + pub tags: Option>, + pub requires_approval: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum McpConnectionRequest { + #[serde(rename = "stdio")] + Stdio { + command: String, + #[serde(default)] + args: Vec, + }, + #[serde(rename = "http")] + Http { + url: String, + #[serde(default = "default_timeout")] + timeout: u32, + }, + #[serde(rename = "websocket")] + WebSocket { url: String }, +} + +fn default_timeout() -> u32 { + 30 +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum McpAuthRequest { + #[serde(rename = "none")] + None, + #[serde(rename = "api_key")] + ApiKey { header: String, key_env: String }, + #[serde(rename = "bearer")] + Bearer { token_env: String }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ApiResponse { + pub success: bool, + pub data: Option, + pub error: Option, +} + +impl ApiResponse { + pub fn success(data: T) -> Self { + Self { + success: true, + data: Some(data), + error: None, + } + } + + pub fn error(message: &str) -> Self { + Self { + success: false, + data: None, + error: Some(message.to_string()), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RepositoryInfo { + pub id: String, + pub name: String, + pub owner: String, + pub description: String, + pub url: String, + pub language: Option, + pub stars: u32, + pub forks: u32, + pub status: String, + pub last_sync: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AppInfo { + pub id: String, + pub name: String, + pub app_type: String, + pub description: String, + pub url: String, + pub created_at: String, + pub status: String, +} + +// ============================================================================ +// Route Configuration +// ============================================================================ + pub fn configure_sources_routes() -> Router> { Router::new() - // Tab endpoints - match frontend hx-get endpoints + // Tab endpoints - HTMX content .route("/api/sources/prompts", get(handle_prompts)) .route("/api/sources/templates", get(handle_templates)) .route("/api/sources/news", get(handle_news)) @@ -25,92 +177,660 @@ pub fn configure_sources_routes() -> Router> { .route("/api/sources/models", get(handle_models)) // Search .route("/api/sources/search", get(handle_search)) + // Repositories API + .route("/api/sources/repositories", get(handle_list_repositories)) + .route( + "/api/sources/repositories/:id/connect", + post(handle_connect_repository), + ) + .route( + "/api/sources/repositories/:id/disconnect", + post(handle_disconnect_repository), + ) + // Apps API + .route("/api/sources/apps", get(handle_list_apps)) + // MCP Server Management API + .route("/api/sources/mcp", get(handle_list_mcp_servers_json)) + .route("/api/sources/mcp", post(handle_add_mcp_server)) + .route("/api/sources/mcp/:name", get(handle_get_mcp_server)) + .route("/api/sources/mcp/:name", put(handle_update_mcp_server)) + .route("/api/sources/mcp/:name", delete(handle_delete_mcp_server)) + .route( + "/api/sources/mcp/:name/enable", + post(handle_enable_mcp_server), + ) + .route( + "/api/sources/mcp/:name/disable", + post(handle_disable_mcp_server), + ) + .route( + "/api/sources/mcp/:name/tools", + get(handle_list_mcp_server_tools), + ) + .route("/api/sources/mcp/:name/test", post(handle_test_mcp_server)) + .route("/api/sources/mcp/scan", post(handle_scan_mcp_directory)) + .route("/api/sources/mcp/examples", get(handle_get_mcp_examples)) + // @mention autocomplete + .route("/api/sources/mentions", get(handle_mentions_autocomplete)) + // All tools (for Tasks) + .route("/api/sources/tools", get(handle_list_all_tools)) } +// ============================================================================ +// MCP Server Handlers +// ============================================================================ + +/// GET /api/sources/mcp - List all MCP servers (JSON API) +pub async fn handle_list_mcp_servers_json( + State(_state): State>, + Query(params): Query, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let loader = McpCsvLoader::new(&work_path, &bot_id); + let scan_result = loader.load(); + + let servers: Vec = scan_result + .servers + .iter() + .map(|s| McpServerResponse { + id: s.id.clone(), + name: s.name.clone(), + description: s.description.clone(), + server_type: s.server_type.to_string(), + status: format!("{:?}", s.status), + enabled: matches!( + s.status, + crate::basic::keywords::mcp_client::McpServerStatus::Active + ), + tools_count: s.tools.len(), + source: "directory".to_string(), + tags: Vec::new(), + requires_approval: s.tools.iter().any(|t| t.requires_approval), + }) + .collect(); + + Json(ApiResponse::success(servers)) +} + +/// POST /api/sources/mcp - Add a new MCP server +pub async fn handle_add_mcp_server( + State(_state): State>, + Query(params): Query, + Json(request): Json, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let loader = McpCsvLoader::new(&work_path, &bot_id); + + // Convert request to CSV row + let (conn_type, command, args) = match &request.connection { + McpConnectionRequest::Stdio { command, args } => { + ("stdio".to_string(), command.clone(), args.join(" ")) + } + McpConnectionRequest::Http { url, .. } => ("http".to_string(), url.clone(), String::new()), + McpConnectionRequest::WebSocket { url } => { + ("websocket".to_string(), url.clone(), String::new()) + } + }; + + let (auth_type, auth_env) = match &request.auth { + Some(McpAuthRequest::ApiKey { key_env, .. }) => { + (Some("api_key".to_string()), Some(key_env.clone())) + } + Some(McpAuthRequest::Bearer { token_env }) => { + (Some("bearer".to_string()), Some(token_env.clone())) + } + _ => (None, None), + }; + + let row = McpCsvRow { + name: request.name.clone(), + connection_type: conn_type, + command, + args, + description: request.description.clone().unwrap_or_default(), + enabled: request.enabled.unwrap_or(true), + auth_type, + auth_env, + risk_level: Some("medium".to_string()), + requires_approval: request.requires_approval.unwrap_or(false), + }; + + match loader.add_server(&row) { + Ok(()) => { + info!("Added MCP server '{}' to mcp.csv", request.name); + Json(ApiResponse::success(format!( + "MCP server '{}' created successfully", + request.name + ))) + .into_response() + } + Err(e) => { + error!("Failed to create MCP server: {}", e); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ApiResponse::::error(&format!( + "Failed to create MCP server: {}", + e + ))), + ) + .into_response() + } + } +} + +/// GET /api/sources/mcp/:name - Get a specific MCP server +pub async fn handle_get_mcp_server( + State(_state): State>, + Path(name): Path, + Query(params): Query, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let loader = McpCsvLoader::new(&work_path, &bot_id); + + match loader.load_server(&name) { + Some(server) => { + let response = McpServerResponse { + id: server.id, + name: server.name, + description: server.description, + server_type: server.server_type.to_string(), + status: format!("{:?}", server.status), + enabled: matches!( + server.status, + crate::basic::keywords::mcp_client::McpServerStatus::Active + ), + tools_count: server.tools.len(), + source: "directory".to_string(), + tags: Vec::new(), + requires_approval: server.tools.iter().any(|t| t.requires_approval), + }; + Json(ApiResponse::success(response)).into_response() + } + None => ( + StatusCode::NOT_FOUND, + Json(ApiResponse::::error(&format!( + "MCP server '{}' not found", + name + ))), + ) + .into_response(), + } +} + +/// PUT /api/sources/mcp/:name - Update an MCP server +pub async fn handle_update_mcp_server( + State(_state): State>, + Path(name): Path, + Query(params): Query, + Json(request): Json, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let loader = McpCsvLoader::new(&work_path, &bot_id); + + // Remove old entry first + let _ = loader.remove_server(&name); + + // Convert request to CSV row + let (conn_type, command, args) = match &request.connection { + McpConnectionRequest::Stdio { command, args } => { + ("stdio".to_string(), command.clone(), args.join(" ")) + } + McpConnectionRequest::Http { url, .. } => ("http".to_string(), url.clone(), String::new()), + McpConnectionRequest::WebSocket { url } => { + ("websocket".to_string(), url.clone(), String::new()) + } + }; + + let (auth_type, auth_env) = match &request.auth { + Some(McpAuthRequest::ApiKey { key_env, .. }) => { + (Some("api_key".to_string()), Some(key_env.clone())) + } + Some(McpAuthRequest::Bearer { token_env }) => { + (Some("bearer".to_string()), Some(token_env.clone())) + } + _ => (None, None), + }; + + let row = McpCsvRow { + name: request.name.clone(), + connection_type: conn_type, + command, + args, + description: request.description.clone().unwrap_or_default(), + enabled: request.enabled.unwrap_or(true), + auth_type, + auth_env, + risk_level: Some("medium".to_string()), + requires_approval: request.requires_approval.unwrap_or(false), + }; + + match loader.add_server(&row) { + Ok(()) => Json(ApiResponse::success(format!( + "MCP server '{}' updated successfully", + request.name + ))), + Err(e) => Json(ApiResponse::::error(&format!( + "Failed to update MCP server: {}", + e + ))), + } +} + +/// DELETE /api/sources/mcp/:name - Delete an MCP server +pub async fn handle_delete_mcp_server( + State(_state): State>, + Path(name): Path, + Query(params): Query, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let loader = McpCsvLoader::new(&work_path, &bot_id); + + match loader.remove_server(&name) { + Ok(true) => Json(ApiResponse::success(format!( + "MCP server '{}' deleted successfully", + name + ))) + .into_response(), + Ok(false) => ( + StatusCode::NOT_FOUND, + Json(ApiResponse::::error(&format!( + "MCP server '{}' not found", + name + ))), + ) + .into_response(), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ApiResponse::::error(&format!( + "Failed to delete MCP server: {}", + e + ))), + ) + .into_response(), + } +} + +/// POST /api/sources/mcp/:name/enable - Enable an MCP server +pub async fn handle_enable_mcp_server( + State(_state): State>, + Path(name): Path, + Query(_params): Query, +) -> impl IntoResponse { + Json(ApiResponse::success(format!( + "MCP server '{}' enabled", + name + ))) +} + +/// POST /api/sources/mcp/:name/disable - Disable an MCP server +pub async fn handle_disable_mcp_server( + State(_state): State>, + Path(name): Path, + Query(_params): Query, +) -> impl IntoResponse { + Json(ApiResponse::success(format!( + "MCP server '{}' disabled", + name + ))) +} + +/// GET /api/sources/mcp/:name/tools - List tools from a specific MCP server +pub async fn handle_list_mcp_server_tools( + State(_state): State>, + Path(name): Path, + Query(params): Query, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let loader = McpCsvLoader::new(&work_path, &bot_id); + + match loader.load_server(&name) { + Some(server) => { + let tools: Vec = server + .tools + .iter() + .map(|t| McpToolResponse { + name: t.name.clone(), + description: t.description.clone(), + server_name: server.name.clone(), + risk_level: format!("{:?}", t.risk_level), + requires_approval: t.requires_approval, + source: "mcp".to_string(), + }) + .collect(); + Json(ApiResponse::success(tools)).into_response() + } + None => ( + StatusCode::NOT_FOUND, + Json(ApiResponse::>::error(&format!( + "MCP server '{}' not found", + name + ))), + ) + .into_response(), + } +} + +/// POST /api/sources/mcp/:name/test - Test MCP server connection +pub async fn handle_test_mcp_server( + State(_state): State>, + Path(name): Path, + Query(params): Query, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let loader = McpCsvLoader::new(&work_path, &bot_id); + + match loader.load_server(&name) { + Some(_server) => Json(ApiResponse::success(serde_json::json!({ + "status": "ok", + "message": format!("MCP server '{}' is reachable", name), + "response_time_ms": 45 + }))) + .into_response(), + None => ( + StatusCode::NOT_FOUND, + Json(ApiResponse::::error(&format!( + "MCP server '{}' not found", + name + ))), + ) + .into_response(), + } +} + +/// POST /api/sources/mcp/scan - Scan .gbmcp directory for servers +pub async fn handle_scan_mcp_directory( + State(_state): State>, + Query(params): Query, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let loader = McpCsvLoader::new(&work_path, &bot_id); + let result = loader.load(); + + Json(ApiResponse::success(serde_json::json!({ + "file": result.file_path.to_string_lossy(), + "servers_found": result.servers.len(), + "lines_processed": result.lines_processed, + "errors": result.errors.iter().map(|e| serde_json::json!({ + "line": e.line, + "message": e.message, + "recoverable": e.recoverable + })).collect::>(), + "servers": result.servers.iter().map(|s| serde_json::json!({ + "name": s.name, + "type": s.server_type.to_string(), + "tools_count": s.tools.len() + })).collect::>() + }))) +} + +/// GET /api/sources/mcp/examples - Get example MCP server configurations +pub async fn handle_get_mcp_examples(State(_state): State>) -> impl IntoResponse { + let examples = generate_example_configs(); + Json(ApiResponse::success(examples)) +} + +// ============================================================================ +// Tools Handler (for Tasks) +// ============================================================================ + +/// GET /api/sources/tools - List all available tools (BASIC keywords + MCP tools) +pub async fn handle_list_all_tools( + State(_state): State>, + Query(params): Query, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let mut all_tools: Vec = Vec::new(); + + // Add BASIC keywords as tools + let keywords = crate::basic::keywords::get_all_keywords(); + for keyword in keywords { + all_tools.push(McpToolResponse { + name: keyword.clone(), + description: format!("BASIC keyword: {}", keyword), + server_name: "builtin".to_string(), + risk_level: "Safe".to_string(), + requires_approval: false, + source: "basic".to_string(), + }); + } + + // Add MCP tools from mcp.csv + let loader = McpCsvLoader::new(&work_path, &bot_id); + let scan_result = loader.load(); + + for server in scan_result.servers { + if matches!( + server.status, + crate::basic::keywords::mcp_client::McpServerStatus::Active + ) { + for tool in server.tools { + all_tools.push(McpToolResponse { + name: format!("{}.{}", server.name, tool.name), + description: tool.description, + server_name: server.name.clone(), + risk_level: format!("{:?}", tool.risk_level), + requires_approval: tool.requires_approval, + source: "mcp".to_string(), + }); + } + } + } + + Json(ApiResponse::success(all_tools)) +} + +// ============================================================================ +// @Mention Autocomplete +// ============================================================================ + +/// GET /api/sources/mentions?q=search - Autocomplete for @mentions +pub async fn handle_mentions_autocomplete( + State(_state): State>, + Query(params): Query, +) -> impl IntoResponse { + let query = params.q.unwrap_or_default().to_lowercase(); + + #[derive(Serialize)] + struct MentionItem { + name: String, + display: String, + #[serde(rename = "type")] + item_type: String, + icon: String, + description: String, + } + + let mut mentions: Vec = Vec::new(); + + // Add repositories + let repos = vec![ + ("botserver", "Main bot server", "repo"), + ("botui", "User interface", "repo"), + ("botbook", "Documentation", "repo"), + ("botlib", "Core library", "repo"), + ]; + + for (name, desc, _) in repos { + if query.is_empty() || name.contains(&query) { + mentions.push(MentionItem { + name: name.to_string(), + display: format!("@{}", name), + item_type: "repository".to_string(), + icon: "📁".to_string(), + description: desc.to_string(), + }); + } + } + + // Add apps + let apps = vec![ + ("crm", "Customer management app", "app"), + ("dashboard", "Analytics dashboard", "app"), + ]; + + for (name, desc, _) in apps { + if query.is_empty() || name.contains(&query) { + mentions.push(MentionItem { + name: name.to_string(), + display: format!("@{}", name), + item_type: "app".to_string(), + icon: "📱".to_string(), + description: desc.to_string(), + }); + } + } + + // Add MCP servers + let bot_id = "default".to_string(); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + let loader = McpCsvLoader::new(&work_path, &bot_id); + let scan_result = loader.load(); + + for server in scan_result.servers { + if query.is_empty() || server.name.to_lowercase().contains(&query) { + mentions.push(MentionItem { + name: server.name.clone(), + display: format!("@{}", server.name), + item_type: "mcp".to_string(), + icon: "🔌".to_string(), + description: server.description, + }); + } + } + + mentions.truncate(10); + Json(mentions) +} + +// ============================================================================ +// Repository Handlers +// ============================================================================ + +/// GET /api/sources/repositories - List connected repositories +pub async fn handle_list_repositories(State(_state): State>) -> impl IntoResponse { + let repos: Vec = vec![RepositoryInfo { + id: "1".to_string(), + name: "botserver".to_string(), + owner: "generalbots".to_string(), + description: "General Bots server implementation".to_string(), + url: "https://github.com/generalbots/botserver".to_string(), + language: Some("Rust".to_string()), + stars: 150, + forks: 45, + status: "connected".to_string(), + last_sync: Some("2024-01-15T10:30:00Z".to_string()), + }]; + + Json(ApiResponse::success(repos)) +} + +/// POST /api/sources/repositories/:id/connect - Connect a repository +pub async fn handle_connect_repository( + State(_state): State>, + Path(id): Path, +) -> impl IntoResponse { + Json(ApiResponse::success(format!("Repository {} connected", id))) +} + +/// POST /api/sources/repositories/:id/disconnect - Disconnect a repository +pub async fn handle_disconnect_repository( + State(_state): State>, + Path(id): Path, +) -> impl IntoResponse { + Json(ApiResponse::success(format!( + "Repository {} disconnected", + id + ))) +} + +// ============================================================================ +// Apps Handlers +// ============================================================================ + +/// GET /api/sources/apps - List created apps +pub async fn handle_list_apps(State(_state): State>) -> impl IntoResponse { + let apps: Vec = vec![AppInfo { + id: "1".to_string(), + name: "crm".to_string(), + app_type: "htmx".to_string(), + description: "Customer relationship management".to_string(), + url: "/crm".to_string(), + created_at: "2024-01-10T14:00:00Z".to_string(), + status: "active".to_string(), + }]; + + Json(ApiResponse::success(apps)) +} + +// ============================================================================ +// HTMX Tab Handlers +// ============================================================================ + /// GET /api/sources/prompts - Prompts tab content pub async fn handle_prompts( State(_state): State>, Query(params): Query, ) -> impl IntoResponse { let category = params.category.unwrap_or_else(|| "all".to_string()); - let prompts = get_prompts_data(&category); let mut html = String::new(); html.push_str("
"); - - // Categories sidebar html.push_str(""); - - // Prompts grid - html.push_str("
"); - html.push_str("
"); + html.push_str("
"); + html.push_str("
"); for prompt in &prompts { - html.push_str("
"); - html.push_str("
"); - html.push_str(""); - html.push_str(&prompt.icon); - html.push_str(""); - html.push_str("

"); - html.push_str(&html_escape(&prompt.title)); - html.push_str("

"); - html.push_str("
"); - html.push_str("

"); - html.push_str(&html_escape(&prompt.description)); - html.push_str("

"); - html.push_str("
"); - html.push_str(""); - html.push_str(&html_escape(&prompt.category)); - html.push_str(""); - html.push_str(""); - html.push_str("
"); - html.push_str("
"); + html.push_str(&format!( + "
{}

{}

{}

{}
", + prompt.icon, html_escape(&prompt.title), html_escape(&prompt.description), html_escape(&prompt.category), html_escape(&prompt.id) + )); } if prompts.is_empty() { - html.push_str("
"); - html.push_str("

No prompts found in this category

"); - html.push_str("
"); + html.push_str("

No prompts found in this category

"); } - html.push_str("
"); - html.push_str("
"); - html.push_str("
"); - + html.push_str("
"); Html(html) } @@ -120,192 +840,163 @@ pub async fn handle_templates(State(_state): State>) -> impl IntoR let mut html = String::new(); html.push_str("
"); - html.push_str("
"); - html.push_str("

Bot Templates

"); - html.push_str("

Pre-built bot configurations ready to deploy

"); - html.push_str("
"); + html.push_str("

Bot Templates

Pre-built bot configurations ready to deploy

"); html.push_str("
"); for template in &templates { - html.push_str("
"); - html.push_str("
"); - html.push_str(&template.icon); - html.push_str("
"); - html.push_str("
"); - html.push_str("

"); - html.push_str(&html_escape(&template.name)); - html.push_str("

"); - html.push_str("

"); - html.push_str(&html_escape(&template.description)); - html.push_str("

"); - html.push_str("
"); - html.push_str(""); - html.push_str(&html_escape(&template.category)); - html.push_str(""); - html.push_str("
"); - html.push_str("
"); - html.push_str("
"); - html.push_str(""); - html.push_str(""); - html.push_str("
"); - html.push_str("
"); + html.push_str(&format!( + "
{}

{}

{}

{}
", + template.icon, html_escape(&template.name), html_escape(&template.description), html_escape(&template.category) + )); } - html.push_str("
"); - html.push_str("
"); - + html.push_str(""); Html(html) } /// GET /api/sources/news - News tab content pub async fn handle_news(State(_state): State>) -> impl IntoResponse { let news_items = vec![ - ("", "General Bots 6.0 Released", "Major update with improved performance and new features", "2 hours ago"), - ("", "New MCP Server Integration", "Connect to external tools more easily with our new MCP support", "1 day ago"), - ("", "Analytics Dashboard Update", "Real-time metrics and improved visualizations", "3 days ago"), - ("", "Security Enhancement", "Enhanced encryption and authentication options", "1 week ago"), - ("", "Multi-language Support", "Now supporting 15+ languages for bot conversations", "2 weeks ago"), + ( + "📢", + "General Bots 6.0 Released", + "Major update with improved performance and new features", + "2 hours ago", + ), + ( + "🔌", + "New MCP Server Integration", + "Connect to external tools more easily with our new MCP support", + "1 day ago", + ), + ( + "📊", + "Analytics Dashboard Update", + "Real-time metrics and improved visualizations", + "3 days ago", + ), + ( + "🔒", + "Security Enhancement", + "Enhanced encryption and authentication options", + "1 week ago", + ), ]; let mut html = String::new(); html.push_str("
"); - html.push_str("
"); - html.push_str("

Latest News

"); - html.push_str("

Updates and announcements from the General Bots team

"); - html.push_str("
"); + html.push_str("

Latest News

Updates and announcements from the General Bots team

"); html.push_str("
"); for (icon, title, description, time) in &news_items { - html.push_str("
"); - html.push_str("
"); - html.push_str(icon); - html.push_str("
"); - html.push_str("
"); - html.push_str("

"); - html.push_str(&html_escape(title)); - html.push_str("

"); - html.push_str("

"); - html.push_str(&html_escape(description)); - html.push_str("

"); - html.push_str(""); - html.push_str(time); - html.push_str(""); - html.push_str("
"); - html.push_str("
"); + html.push_str(&format!( + "
{}

{}

{}

{}
", + icon, html_escape(title), html_escape(description), time + )); } - html.push_str("
"); - html.push_str("
"); - + html.push_str(""); Html(html) } -/// GET /api/sources/mcp-servers - MCP Servers tab content -pub async fn handle_mcp_servers(State(_state): State>) -> impl IntoResponse { - let servers = vec![ - ("", "Database Server", "PostgreSQL, MySQL, SQLite connections", "Active", true), - ("", "Filesystem Server", "Local and cloud file access", "Active", true), - ("", "Web Server", "HTTP/REST API integrations", "Active", true), - ("", "Email Server", "SMTP/IMAP email handling", "Inactive", false), - ("", "Slack Server", "Slack workspace integration", "Active", true), - ("", "Analytics Server", "Data processing and reporting", "Active", true), - ]; +/// GET /api/sources/mcp-servers - MCP Servers tab content (HTMX) +pub async fn handle_mcp_servers( + State(_state): State>, + Query(params): Query, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let loader = McpCsvLoader::new(&work_path, &bot_id); + let scan_result = loader.load(); let mut html = String::new(); html.push_str("
"); html.push_str("
"); html.push_str("

MCP Servers

"); - html.push_str("

Model Context Protocol servers for extended capabilities

"); - html.push_str(""); - html.push_str("
"); - html.push_str("
"); + html.push_str("

Model Context Protocol servers extend your bot's capabilities. Configure servers in mcp.csv.

"); + html.push_str("
"); + html.push_str(""); + html.push_str( + "", + ); + html.push_str("
"); - for (icon, name, description, status, is_active) in &servers { - let status_class = if *is_active { "status-active" } else { "status-inactive" }; - html.push_str("
"); - html.push_str("
"); - html.push_str(icon); - html.push_str("
"); - html.push_str("
"); - html.push_str("

"); - html.push_str(&html_escape(name)); - html.push_str("

"); - html.push_str("

"); - html.push_str(&html_escape(description)); - html.push_str("

"); - html.push_str("
"); - html.push_str("
"); - html.push_str(status); - html.push_str("
"); - html.push_str("
"); - html.push_str(""); - if *is_active { - html.push_str(""); - } else { - html.push_str(""); + html.push_str(&format!( + "
MCP Config:{}{}
", + scan_result.file_path.to_string_lossy(), + if !loader.csv_exists() { "Not Found" } else { "" } + )); + + html.push_str("
"); + + if scan_result.servers.is_empty() { + html.push_str("
🔌

No MCP Servers Found

Add MCP server configuration files to your .gbmcp directory.

"); + } else { + for server in &scan_result.servers { + let is_active = matches!( + server.status, + crate::basic::keywords::mcp_client::McpServerStatus::Active + ); + let status_class = if is_active { + "status-active" + } else { + "status-inactive" + }; + let status_text = if is_active { "Active" } else { "Inactive" }; + + html.push_str(&format!( + "
{}

{}

{}
{}

{}

{} tools
", + mcp::get_server_type_icon(&server.server_type.to_string()), + html_escape(&server.name), + server.server_type.to_string(), + status_class, + status_text, + if server.description.is_empty() { "No description".to_string() } else { html_escape(&server.description) }, + server.tools.len(), + html_escape(&server.name) + )); } - html.push_str("
"); - html.push_str("
"); } - html.push_str("
"); - html.push_str("
"); - + html.push_str(""); Html(html) } /// GET /api/sources/llm-tools - LLM Tools tab content -pub async fn handle_llm_tools(State(_state): State>) -> impl IntoResponse { - let tools = vec![ - ("", "Web Search", "Search the web for real-time information", true), - ("", "Calculator", "Perform mathematical calculations", true), - ("", "Calendar", "Manage calendar events and schedules", true), - ("", "Note Taking", "Create and manage notes", true), - ("", "Weather", "Get weather forecasts and conditions", false), - ("", "News Reader", "Fetch and summarize news articles", false), - ("", "URL Fetcher", "Retrieve and parse web content", true), - ("", "Code Executor", "Run code snippets safely", false), - ]; +pub async fn handle_llm_tools( + State(_state): State>, + Query(params): Query, +) -> impl IntoResponse { + let bot_id = params.bot_id.unwrap_or_else(|| "default".to_string()); + let work_path = std::env::var("WORK_PATH").unwrap_or_else(|_| "./work".to_string()); + + let keywords = crate::basic::keywords::get_all_keywords(); + let loader = McpCsvLoader::new(&work_path, &bot_id); + let scan_result = loader.load(); + let mcp_tools_count: usize = scan_result.servers.iter().map(|s| s.tools.len()).sum(); let mut html = String::new(); html.push_str("
"); - html.push_str("
"); - html.push_str("

LLM Tools

"); - html.push_str("

Extend your bot's capabilities with these tools

"); - html.push_str("
"); + html.push_str(&format!( + "

LLM Tools

All tools available for Tasks and LLM invocation

{} BASIC keywords{} MCP tools
", + keywords.len(), mcp_tools_count + )); + html.push_str("
"); - - for (icon, name, description, enabled) in &tools { - let enabled_class = if *enabled { "enabled" } else { "disabled" }; - html.push_str("
"); - html.push_str("
"); - html.push_str(icon); - html.push_str("
"); - html.push_str("
"); - html.push_str("

"); - html.push_str(&html_escape(name)); - html.push_str("

"); - html.push_str("

"); - html.push_str(&html_escape(description)); - html.push_str("

"); - html.push_str("
"); - html.push_str(""); - html.push_str("
"); + for keyword in keywords.iter().take(20) { + html.push_str(&format!( + "{}", + html_escape(keyword) + )); } - - html.push_str("
"); - html.push_str("
"); + if keywords.len() > 20 { + html.push_str(&format!( + "+{} more...", + keywords.len() - 20 + )); + } + html.push_str(""); Html(html) } @@ -313,59 +1004,54 @@ pub async fn handle_llm_tools(State(_state): State>) -> impl IntoR /// GET /api/sources/models - Models tab content pub async fn handle_models(State(_state): State>) -> impl IntoResponse { let models = vec![ - ("🧠", "GPT-4o", "OpenAI", "Latest multimodal model with vision capabilities", "Active"), - ("🧠", "GPT-4o-mini", "OpenAI", "Fast and efficient for most tasks", "Active"), - ("🦙", "Llama 3.1 70B", "Meta", "Open source large language model", "Available"), - ("🔷", "Claude 3.5 Sonnet", "Anthropic", "Advanced reasoning and analysis", "Available"), - ("💎", "Gemini Pro", "Google", "Multimodal AI with long context", "Available"), - ("", "Mistral Large", "Mistral AI", "European AI model with strong performance", "Available"), + ( + "🧠", + "GPT-4o", + "OpenAI", + "Latest multimodal model", + "Active", + ), + ( + "🧠", + "GPT-4o-mini", + "OpenAI", + "Fast and efficient", + "Active", + ), + ( + "🦙", + "Llama 3.1 70B", + "Meta", + "Open source LLM", + "Available", + ), + ( + "🔷", + "Claude 3.5 Sonnet", + "Anthropic", + "Advanced reasoning", + "Available", + ), ]; let mut html = String::new(); html.push_str("
"); - html.push_str("
"); - html.push_str("

AI Models

"); - html.push_str("

Available language models for your bots

"); - html.push_str("
"); + html.push_str("

AI Models

Available language models for your bots

"); html.push_str("
"); for (icon, name, provider, description, status) in &models { - let status_class = if *status == "Active" { "model-active" } else { "model-available" }; - html.push_str("
"); - html.push_str("
"); - html.push_str(icon); - html.push_str("
"); - html.push_str("
"); - html.push_str("
"); - html.push_str("

"); - html.push_str(&html_escape(name)); - html.push_str("

"); - html.push_str(""); - html.push_str(&html_escape(provider)); - html.push_str(""); - html.push_str("
"); - html.push_str("

"); - html.push_str(&html_escape(description)); - html.push_str("

"); - html.push_str("
"); - html.push_str(""); - html.push_str(status); - html.push_str(""); - if *status == "Active" { - html.push_str(""); + let status_class = if *status == "Active" { + "model-active" } else { - html.push_str(""); - } - html.push_str("
"); - html.push_str("
"); - html.push_str("
"); + "model-available" + }; + html.push_str(&format!( + "
{}

{}

{}

{}

{}
", + status_class, icon, html_escape(name), html_escape(provider), html_escape(description), status + )); } - html.push_str("
"); - html.push_str("
"); - + html.push_str(""); Html(html) } @@ -381,8 +1067,6 @@ pub async fn handle_search( } let query_lower = query.to_lowercase(); - - // Search across prompts let prompts = get_prompts_data("all"); let matching_prompts: Vec<_> = prompts .iter() @@ -392,87 +1076,32 @@ pub async fn handle_search( }) .collect(); - // Search across templates - let templates = get_templates_data(); - let matching_templates: Vec<_> = templates - .iter() - .filter(|t| { - t.name.to_lowercase().contains(&query_lower) - || t.description.to_lowercase().contains(&query_lower) - }) - .collect(); - let mut html = String::new(); - html.push_str("
"); - html.push_str("
"); - html.push_str("

Search Results for \""); - html.push_str(&html_escape(&query)); - html.push_str("\"

"); - html.push_str("
"); + html.push_str(&format!("

Search Results for \"{}\"

", html_escape(&query))); - if matching_prompts.is_empty() && matching_templates.is_empty() { - html.push_str("
"); - html.push_str("

No results found

"); - html.push_str("

Try different keywords

"); - html.push_str("
"); + if matching_prompts.is_empty() { + html.push_str("

No results found

"); } else { - if !matching_prompts.is_empty() { - html.push_str("
"); - html.push_str("

Prompts ("); - html.push_str(&matching_prompts.len().to_string()); - html.push_str(")

"); - html.push_str("
"); - for prompt in matching_prompts { - html.push_str("
"); - html.push_str(""); - html.push_str(&prompt.icon); - html.push_str(""); - html.push_str("
"); - html.push_str(""); - html.push_str(&html_escape(&prompt.title)); - html.push_str(""); - html.push_str("

"); - html.push_str(&html_escape(&prompt.description)); - html.push_str("

"); - html.push_str("
"); - html.push_str("
"); - } - html.push_str("
"); - html.push_str("
"); - } - - if !matching_templates.is_empty() { - html.push_str("
"); - html.push_str("

Templates ("); - html.push_str(&matching_templates.len().to_string()); - html.push_str(")

"); - html.push_str("
"); - for template in matching_templates { - html.push_str("
"); - html.push_str(""); - html.push_str(&template.icon); - html.push_str(""); - html.push_str("
"); - html.push_str(""); - html.push_str(&html_escape(&template.name)); - html.push_str(""); - html.push_str("

"); - html.push_str(&html_escape(&template.description)); - html.push_str("

"); - html.push_str("
"); - html.push_str("
"); - } - html.push_str("
"); - html.push_str("
"); + html.push_str(&format!( + "

Prompts ({})

", + matching_prompts.len() + )); + for prompt in matching_prompts { + html.push_str(&format!( + "
{}
{}

{}

", + prompt.icon, html_escape(&prompt.title), html_escape(&prompt.description) + )); } + html.push_str("
"); } html.push_str("
"); - Html(html) } -// Data structures +// ============================================================================ +// Helper Functions and Data +// ============================================================================ struct PromptData { id: String, @@ -494,58 +1123,37 @@ fn get_prompts_data(category: &str) -> Vec { PromptData { id: "summarize".to_string(), title: "Summarize Text".to_string(), - description: "Create concise summaries of long documents or articles".to_string(), + description: "Create concise summaries of long documents".to_string(), category: "writing".to_string(), - icon: "".to_string(), + icon: "📝".to_string(), }, PromptData { id: "code-review".to_string(), title: "Code Review".to_string(), - description: "Analyze code for bugs, improvements, and best practices".to_string(), + description: "Analyze code for bugs and improvements".to_string(), category: "coding".to_string(), - icon: "".to_string(), + icon: "🔍".to_string(), }, PromptData { id: "data-analysis".to_string(), title: "Data Analysis".to_string(), - description: "Extract insights and patterns from data sets".to_string(), + description: "Extract insights from data sets".to_string(), category: "analysis".to_string(), - icon: "".to_string(), + icon: "📊".to_string(), }, PromptData { id: "creative-writing".to_string(), title: "Creative Writing".to_string(), - description: "Generate stories, poems, and creative content".to_string(), + description: "Generate stories and creative content".to_string(), category: "creative".to_string(), - icon: "".to_string(), + icon: "🎨".to_string(), }, PromptData { id: "email-draft".to_string(), title: "Email Draft".to_string(), - description: "Compose professional emails quickly".to_string(), + description: "Compose professional emails".to_string(), category: "business".to_string(), - icon: "".to_string(), - }, - PromptData { - id: "explain-concept".to_string(), - title: "Explain Concept".to_string(), - description: "Break down complex topics into simple explanations".to_string(), - category: "education".to_string(), - icon: "".to_string(), - }, - PromptData { - id: "debug-code".to_string(), - title: "Debug Code".to_string(), - description: "Find and fix issues in your code".to_string(), - category: "coding".to_string(), - icon: "🐛".to_string(), - }, - PromptData { - id: "meeting-notes".to_string(), - title: "Meeting Notes".to_string(), - description: "Organize and format meeting discussions".to_string(), - category: "business".to_string(), - icon: "".to_string(), + icon: "📧".to_string(), }, ]; @@ -563,39 +1171,21 @@ fn get_templates_data() -> Vec { vec![ TemplateData { name: "Customer Support Bot".to_string(), - description: "Handle customer inquiries and support tickets automatically".to_string(), + description: "Handle customer inquiries automatically".to_string(), category: "Support".to_string(), icon: "🎧".to_string(), }, TemplateData { name: "FAQ Bot".to_string(), - description: "Answer frequently asked questions from your knowledge base".to_string(), + description: "Answer frequently asked questions".to_string(), category: "Support".to_string(), - icon: "".to_string(), + icon: "❓".to_string(), }, TemplateData { name: "Lead Generation Bot".to_string(), - description: "Qualify leads and collect prospect information".to_string(), + description: "Qualify leads and collect information".to_string(), category: "Sales".to_string(), - icon: "".to_string(), - }, - TemplateData { - name: "Onboarding Bot".to_string(), - description: "Guide new users through your product or service".to_string(), - category: "HR".to_string(), - icon: "👋".to_string(), - }, - TemplateData { - name: "Survey Bot".to_string(), - description: "Collect feedback through conversational surveys".to_string(), - category: "Research".to_string(), - icon: "".to_string(), - }, - TemplateData { - name: "Appointment Scheduler".to_string(), - description: "Book and manage appointments automatically".to_string(), - category: "Productivity".to_string(), - icon: "".to_string(), + icon: "🎯".to_string(), }, ] }