From b0baf36b116594396980d93e044f044654b909f6 Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (Pragmatismo)" Date: Mon, 29 Dec 2025 18:21:03 -0300 Subject: [PATCH] Fix TLS configuration for MinIO, Qdrant, and template structure - Fix MinIO health check to use HTTPS instead of HTTP - Add Vault connectivity check before fetching credentials - Add CA cert configuration for S3 client - Add Qdrant vector_db setup with TLS configuration - Fix Qdrant default URL to use HTTPS - Always sync templates to S3 buckets (not just on create) - Skip .gbkb root files, only index files in subfolders --- config/directory_config.json | 8 +- src/basic/keywords/kb_statistics.rs | 36 +- src/core/bootstrap/mod.rs | 44 +- src/core/directory/api.rs | 10 +- src/core/kb/kb_indexer.rs | 28 +- src/core/package_manager/facade.rs | 24 +- src/core/package_manager/installer.rs | 23 +- src/core/package_manager/setup/mod.rs | 2 + .../package_manager/setup/vector_db_setup.rs | 93 ++ src/core/shared/enums.rs | 33 +- src/core/shared/schema.rs | 1107 ++++++----------- src/core/shared/utils.rs | 75 +- src/core/urls.rs | 2 +- src/drive/drive_monitor/mod.rs | 5 +- src/main.rs | 9 +- src/multimodal/mod.rs | 7 +- src/security/integration.rs | 4 +- src/security/zitadel_auth.rs | 13 +- src/timeseries/mod.rs | 7 +- src/whatsapp/mod.rs | 1 - 20 files changed, 737 insertions(+), 794 deletions(-) create mode 100644 src/core/package_manager/setup/vector_db_setup.rs diff --git a/config/directory_config.json b/config/directory_config.json index e51107c9d..d12ac678e 100644 --- a/config/directory_config.json +++ b/config/directory_config.json @@ -1,7 +1,7 @@ { "base_url": "http://localhost:8300", "default_org": { - "id": "353032199743733774", + "id": "353229789043097614", "name": "default", "domain": "default.localhost" }, @@ -13,8 +13,8 @@ "first_name": "Admin", "last_name": "User" }, - "admin_token": "1X7ImWy1yPmGYYumPJ0RfVaLuuLHKstH8BItaTGlp-6jTFPeM0uFo8sjdfxtk-jxjLivcVM", + "admin_token": "7QNrwws4y1X5iIuTUCtXpQj9RoQf4fYi144yEY87tNAbLMZOOD57t3YDAqCtIyIkBS1EZ5k", "project_id": "", - "client_id": "353032201220194318", - "client_secret": "mrGZZk7Aqx1QbOHIwadgZHZkKHuPqZtOGDtdHTe4eZxEK86TDKfTiMlW2NxSEIHl" + "client_id": "353229789848469518", + "client_secret": "COd3gKdMO43jkUztTckCNtHrjxa5RtcIlBn7Cbp4GFoXs6mw6iZalB3m4Vv3FK5Y" } \ No newline at end of file diff --git a/src/basic/keywords/kb_statistics.rs b/src/basic/keywords/kb_statistics.rs index 1038c53c0..09710052a 100644 --- a/src/basic/keywords/kb_statistics.rs +++ b/src/basic/keywords/kb_statistics.rs @@ -1,5 +1,7 @@ +use crate::config::ConfigManager; use crate::shared::models::UserSession; use crate::shared::state::AppState; +use crate::shared::utils::create_tls_client; use log::{error, trace}; use rhai::{Dynamic, Engine}; use serde::{Deserialize, Serialize}; @@ -225,11 +227,11 @@ async fn get_kb_statistics( state: &AppState, user: &UserSession, ) -> Result> { - let qdrant_url = - std::env::var("QDRANT_URL").unwrap_or_else(|_| "https://localhost:6334".to_string()); - let client = reqwest::Client::builder() - .danger_accept_invalid_certs(true) - .build()?; + let config_manager = ConfigManager::new(state.conn.clone()); + let qdrant_url = config_manager + .get_config(&user.bot_id, "vectordb-url", Some("https://localhost:6333")) + .unwrap_or_else(|_| "https://localhost:6333".to_string()); + let client = create_tls_client(Some(30)); let collections_response = client .get(format!("{}/collections", qdrant_url)) @@ -277,14 +279,14 @@ async fn get_kb_statistics( } async fn get_collection_statistics( - _state: &AppState, + state: &AppState, collection_name: &str, ) -> Result> { - let qdrant_url = - std::env::var("QDRANT_URL").unwrap_or_else(|_| "https://localhost:6334".to_string()); - let client = reqwest::Client::builder() - .danger_accept_invalid_certs(true) - .build()?; + let config_manager = ConfigManager::new(state.conn.clone()); + let qdrant_url = config_manager + .get_config(&uuid::Uuid::nil(), "vectordb-url", Some("https://localhost:6333")) + .unwrap_or_else(|_| "https://localhost:6333".to_string()); + let client = create_tls_client(Some(30)); let response = client .get(format!("{}/collections/{}", qdrant_url, collection_name)) @@ -362,14 +364,14 @@ fn get_documents_added_since( } async fn list_collections( - _state: &AppState, + state: &AppState, user: &UserSession, ) -> Result, Box> { - let qdrant_url = - std::env::var("QDRANT_URL").unwrap_or_else(|_| "https://localhost:6334".to_string()); - let client = reqwest::Client::builder() - .danger_accept_invalid_certs(true) - .build()?; + let config_manager = ConfigManager::new(state.conn.clone()); + let qdrant_url = config_manager + .get_config(&user.bot_id, "vectordb-url", Some("https://localhost:6333")) + .unwrap_or_else(|_| "https://localhost:6333".to_string()); + let client = create_tls_client(Some(30)); let response = client .get(format!("{}/collections", qdrant_url)) diff --git a/src/core/bootstrap/mod.rs b/src/core/bootstrap/mod.rs index 1af44f4f6..7bd6bf9e3 100644 --- a/src/core/bootstrap/mod.rs +++ b/src/core/bootstrap/mod.rs @@ -1,5 +1,5 @@ use crate::config::AppConfig; -use crate::package_manager::setup::{DirectorySetup, EmailSetup}; +use crate::package_manager::setup::{DirectorySetup, EmailSetup, VectorDbSetup}; use crate::package_manager::{InstallMode, PackageManager}; use crate::shared::utils::{establish_pg_connection, init_secrets_manager}; use anyhow::Result; @@ -321,7 +321,7 @@ impl BootstrapManager { for i in 0..15 { let drive_ready = Command::new("sh") .arg("-c") - .arg("curl -f -s 'http://127.0.0.1:9000/minio/health/live' >/dev/null 2>&1") + .arg("curl -sfk 'https://127.0.0.1:9000/minio/health/live' >/dev/null 2>&1") .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .status() @@ -975,6 +975,15 @@ impl BootstrapManager { error!("Failed to setup CoreDNS: {}", e); } } + + if component == "vector_db" { + info!("Configuring Qdrant vector database with TLS..."); + let conf_path = self.stack_dir("conf"); + let data_path = self.stack_dir("data"); + if let Err(e) = VectorDbSetup::setup(conf_path, data_path).await { + error!("Failed to setup vector_db: {}", e); + } + } } } info!("=== BOOTSTRAP COMPLETED SUCCESSFULLY ==="); @@ -1797,6 +1806,13 @@ VAULT_CACHE_TTL=300 ) }; + // Set CA cert for self-signed TLS (dev stack) + let ca_cert_path = "./botserver-stack/conf/system/certificates/ca/ca.crt"; + if std::path::Path::new(ca_cert_path).exists() { + std::env::set_var("AWS_CA_BUNDLE", ca_cert_path); + std::env::set_var("SSL_CERT_FILE", ca_cert_path); + } + let base_config = aws_config::defaults(BehaviorVersion::latest()) .endpoint_url(endpoint) .region("auto") @@ -1850,16 +1866,17 @@ VAULT_CACHE_TTL=300 { let bot_name = path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); let bucket = bot_name.trim_start_matches('/').to_string(); + // Create bucket if it doesn't exist if client.head_bucket().bucket(&bucket).send().await.is_err() { - match client.create_bucket().bucket(&bucket).send().await { - Ok(_) => { - Self::upload_directory_recursive(&client, &path, &bucket, "/").await?; - } - Err(e) => { - warn!("S3/MinIO not available, skipping bucket {}: {}", bucket, e); - } + if let Err(e) = client.create_bucket().bucket(&bucket).send().await { + warn!("S3/MinIO not available, skipping bucket {}: {}", bucket, e); + continue; } } + // Always sync templates to bucket + if let Err(e) = Self::upload_directory_recursive(&client, &path, &bucket, "/").await { + warn!("Failed to upload templates to bucket {}: {}", bucket, e); + } } } Ok(()) @@ -2290,6 +2307,15 @@ log_level = "info" fs::copy(&ca_cert_path, service_dir.join("ca.crt"))?; } + let minio_certs_dir = PathBuf::from("./botserver-stack/conf/drive/certs"); + fs::create_dir_all(&minio_certs_dir)?; + let drive_cert_dir = cert_dir.join("drive"); + fs::copy(drive_cert_dir.join("server.crt"), minio_certs_dir.join("public.crt"))?; + fs::copy(drive_cert_dir.join("server.key"), minio_certs_dir.join("private.key"))?; + let minio_ca_dir = minio_certs_dir.join("CAs"); + fs::create_dir_all(&minio_ca_dir)?; + fs::copy(&ca_cert_path, minio_ca_dir.join("ca.crt"))?; + info!("TLS certificates generated successfully"); Ok(()) } diff --git a/src/core/directory/api.rs b/src/core/directory/api.rs index 67f06a211..2952b5d3a 100644 --- a/src/core/directory/api.rs +++ b/src/core/directory/api.rs @@ -1,6 +1,7 @@ use crate::core::directory::{BotAccess, UserAccount, UserProvisioningService, UserRole}; use crate::core::urls::ApiUrls; use crate::shared::state::AppState; +use crate::shared::utils::create_tls_client; use anyhow::Result; use axum::{ extract::{Json, Path, State}, @@ -254,14 +255,7 @@ pub async fn check_services_status(State(state): State>) -> impl I } } - let client = reqwest::Client::builder() - .danger_accept_invalid_certs(true) - .timeout(std::time::Duration::from_secs(2)) - .build() - .unwrap_or_else(|e| { - log::warn!("Failed to create HTTP client: {}, using default", e); - reqwest::Client::new() - }); + let client = create_tls_client(Some(2)); if let Ok(response) = client.get("https://localhost:8300/healthz").send().await { status.directory = response.status().is_success(); diff --git a/src/core/kb/kb_indexer.rs b/src/core/kb/kb_indexer.rs index 5ccfaafed..36450d37b 100644 --- a/src/core/kb/kb_indexer.rs +++ b/src/core/kb/kb_indexer.rs @@ -5,6 +5,9 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use uuid::Uuid; +use crate::config::ConfigManager; +use crate::shared::utils::{create_tls_client, DbPool}; + use super::document_processor::{DocumentProcessor, TextChunk}; use super::embedding_generator::{Embedding, EmbeddingConfig, KbEmbeddingGenerator}; @@ -18,7 +21,21 @@ pub struct QdrantConfig { impl Default for QdrantConfig { fn default() -> Self { Self { - url: "http://localhost:6333".to_string(), + url: "https://localhost:6333".to_string(), + api_key: None, + timeout_secs: 30, + } + } +} + +impl QdrantConfig { + pub fn from_config(pool: DbPool, bot_id: &Uuid) -> Self { + let config_manager = ConfigManager::new(pool); + let url = config_manager + .get_config(bot_id, "vectordb-url", Some("https://localhost:6333")) + .unwrap_or_else(|_| "https://localhost:6333".to_string()); + Self { + url, api_key: None, timeout_secs: 30, } @@ -77,13 +94,8 @@ impl KbIndexer { let document_processor = DocumentProcessor::default(); let embedding_generator = KbEmbeddingGenerator::new(embedding_config); - let http_client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(qdrant_config.timeout_secs)) - .build() - .unwrap_or_else(|e| { - log::warn!("Failed to create HTTP client with timeout: {}, using default", e); - reqwest::Client::new() - }); + // Use shared TLS client with local CA certificate + let http_client = create_tls_client(Some(qdrant_config.timeout_secs)); Self { document_processor, diff --git a/src/core/package_manager/facade.rs b/src/core/package_manager/facade.rs index 38328643e..7ecd66c01 100644 --- a/src/core/package_manager/facade.rs +++ b/src/core/package_manager/facade.rs @@ -859,13 +859,27 @@ Store credentials in Vault: temp_file: &std::path::Path, bin_path: &std::path::Path, ) -> Result<()> { + // Check if tarball has a top-level directory or files at root + let list_output = Command::new("tar") + .args(["-tzf", temp_file.to_str().unwrap_or_default()]) + .output()?; + + let has_subdir = if list_output.status.success() { + let contents = String::from_utf8_lossy(&list_output.stdout); + // If first entry contains '/', there's a subdirectory structure + contents.lines().next().map(|l| l.contains('/')).unwrap_or(false) + } else { + false + }; + + let mut args = vec!["-xzf", temp_file.to_str().unwrap_or_default()]; + if has_subdir { + args.push("--strip-components=1"); + } + let output = Command::new("tar") .current_dir(bin_path) - .args([ - "-xzf", - temp_file.to_str().unwrap_or_default(), - "--strip-components=1", - ]) + .args(&args) .output()?; if !output.status.success() { return Err(anyhow::anyhow!( diff --git a/src/core/package_manager/installer.rs b/src/core/package_manager/installer.rs index 9426dd768..07276b467 100644 --- a/src/core/package_manager/installer.rs +++ b/src/core/package_manager/installer.rs @@ -251,8 +251,8 @@ impl PackageManager { ("MINIO_ROOT_PASSWORD".to_string(), "$DRIVE_SECRET".to_string()), ]), data_download_list: Vec::new(), - exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(), - check_cmd: "curl -sf http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1".to_string(), + exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 --certs-dir {{CONF_PATH}}/drive/certs > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(), + check_cmd: "curl -sfk https://127.0.0.1:9000/minio/health/live >/dev/null 2>&1".to_string(), }, ); } @@ -801,7 +801,7 @@ impl PackageManager { ComponentConfig { name: "vector_db".to_string(), - ports: vec![6333], + ports: vec![6334], dependencies: vec![], linux_packages: vec![], macos_packages: vec![], @@ -818,8 +818,8 @@ impl PackageManager { post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), - exec_cmd: "{{BIN_PATH}}/qdrant --storage-path {{DATA_PATH}} --enable-tls --cert {{CONF_PATH}}/system/certificates/qdrant/server.crt --key {{CONF_PATH}}/system/certificates/qdrant/server.key".to_string(), - check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:6334/metrics >/dev/null 2>&1".to_string(), + exec_cmd: "nohup {{BIN_PATH}}/qdrant --config-path {{CONF_PATH}}/vector_db/config.yaml > {{LOGS_PATH}}/qdrant.log 2>&1 &".to_string(), + check_cmd: "curl -sfk https://localhost:6333/collections >/dev/null 2>&1".to_string(), }, ); } @@ -1164,6 +1164,19 @@ EOF"#.to_string(), return credentials; } + // Check if Vault is reachable before trying to fetch credentials + let vault_check = std::process::Command::new("sh") + .arg("-c") + .arg(format!("curl -sf {}/v1/sys/health >/dev/null 2>&1", vault_addr)) + .status() + .map(|s| s.success()) + .unwrap_or(false); + + if !vault_check { + trace!("Vault not reachable at {}, skipping credential fetch", vault_addr); + return credentials; + } + let base_path = std::env::var("BOTSERVER_STACK_PATH") .map(std::path::PathBuf::from) .unwrap_or_else(|_| { diff --git a/src/core/package_manager/setup/mod.rs b/src/core/package_manager/setup/mod.rs index c5d736ab3..983219dbc 100644 --- a/src/core/package_manager/setup/mod.rs +++ b/src/core/package_manager/setup/mod.rs @@ -1,5 +1,7 @@ pub mod directory_setup; pub mod email_setup; +pub mod vector_db_setup; pub use directory_setup::{DirectorySetup, DefaultUser}; pub use email_setup::EmailSetup; +pub use vector_db_setup::VectorDbSetup; diff --git a/src/core/package_manager/setup/vector_db_setup.rs b/src/core/package_manager/setup/vector_db_setup.rs new file mode 100644 index 000000000..0b66c18fc --- /dev/null +++ b/src/core/package_manager/setup/vector_db_setup.rs @@ -0,0 +1,93 @@ +use anyhow::Result; +use std::path::PathBuf; +use std::fs; +use tracing::info; + +pub struct VectorDbSetup; + +impl VectorDbSetup { + pub async fn setup(conf_path: PathBuf, data_path: PathBuf) -> Result<()> { + let config_dir = conf_path.join("vector_db"); + fs::create_dir_all(&config_dir)?; + + let data_dir = data_path.join("vector_db"); + fs::create_dir_all(&data_dir)?; + + let cert_dir = conf_path.join("system/certificates/vectordb"); + + // Convert to absolute paths for Qdrant config + let data_dir_abs = fs::canonicalize(&data_dir).unwrap_or(data_dir); + let cert_dir_abs = fs::canonicalize(&cert_dir).unwrap_or(cert_dir); + + let config_content = generate_qdrant_config(&data_dir_abs, &cert_dir_abs); + + let config_path = config_dir.join("config.yaml"); + fs::write(&config_path, config_content)?; + + info!("Qdrant vector_db configuration written to {:?}", config_path); + + Ok(()) + } +} + +pub fn generate_qdrant_config(data_dir: &PathBuf, cert_dir: &PathBuf) -> String { + let data_path = data_dir.to_string_lossy(); + let cert_path = cert_dir.join("server.crt").to_string_lossy().to_string(); + let key_path = cert_dir.join("server.key").to_string_lossy().to_string(); + let ca_path = cert_dir.join("ca.crt").to_string_lossy().to_string(); + + format!( + r#"# Qdrant configuration with TLS enabled +# Generated by BotServer bootstrap + +log_level: INFO + +storage: + storage_path: {data_path} + snapshots_path: {data_path}/snapshots + on_disk_payload: true + +service: + host: 0.0.0.0 + http_port: 6333 + grpc_port: 6334 + enable_tls: true + +tls: + cert: {cert_path} + key: {key_path} + ca_cert: {ca_path} + verify_https_client_certificate: false + +cluster: + enabled: false + +telemetry_disabled: true +"# + ) +} + +pub async fn generate_vector_db_config(config_path: PathBuf, data_path: PathBuf) -> Result<()> { + VectorDbSetup::setup(config_path, data_path).await +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn test_generate_qdrant_config() { + let data_dir = PathBuf::from("/tmp/qdrant/data"); + let cert_dir = PathBuf::from("/tmp/qdrant/certs"); + + let config = generate_qdrant_config(&data_dir, &cert_dir); + + assert!(config.contains("enable_tls: true")); + assert!(config.contains("http_port: 6333")); + assert!(config.contains("grpc_port: 6334")); + assert!(config.contains("/tmp/qdrant/data")); + assert!(config.contains("/tmp/qdrant/certs/server.crt")); + assert!(config.contains("/tmp/qdrant/certs/server.key")); + } +} diff --git a/src/core/shared/enums.rs b/src/core/shared/enums.rs index c4ccaa98b..50348f653 100644 --- a/src/core/shared/enums.rs +++ b/src/core/shared/enums.rs @@ -55,7 +55,7 @@ impl ToSql for ChannelType { impl FromSql for ChannelType { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::Web), 1 => Ok(Self::WhatsApp), @@ -142,7 +142,7 @@ impl ToSql for MessageRole { impl FromSql for MessageRole { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 1 => Ok(Self::User), 2 => Ok(Self::Assistant), @@ -220,7 +220,7 @@ impl ToSql for MessageType { impl FromSql for MessageType { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::Text), 1 => Ok(Self::Image), @@ -290,7 +290,7 @@ impl ToSql for LlmProvider { impl FromSql for LlmProvider { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::OpenAi), 1 => Ok(Self::Anthropic), @@ -359,7 +359,7 @@ impl ToSql for ContextProvider { impl FromSql for ContextProvider { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::None), 1 => Ok(Self::Qdrant), @@ -409,7 +409,7 @@ impl ToSql for TaskStatus { impl FromSql for TaskStatus { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::Pending), 1 => Ok(Self::Ready), @@ -489,7 +489,7 @@ impl ToSql for TaskPriority { impl FromSql for TaskPriority { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::Low), 1 => Ok(Self::Normal), @@ -558,7 +558,7 @@ impl ToSql for ExecutionMode { impl FromSql for ExecutionMode { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::Manual), 1 => Ok(Self::Supervised), @@ -611,7 +611,7 @@ impl ToSql for RiskLevel { impl FromSql for RiskLevel { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::None), 1 => Ok(Self::Low), @@ -668,7 +668,7 @@ impl ToSql for ApprovalStatus { impl FromSql for ApprovalStatus { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::Pending), 1 => Ok(Self::Approved), @@ -717,7 +717,7 @@ impl ToSql for ApprovalDecision { impl FromSql for ApprovalDecision { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::Approve), 1 => Ok(Self::Reject), @@ -774,7 +774,7 @@ impl ToSql for IntentType { impl FromSql for IntentType { fn from_sql(bytes: PgValue<'_>) -> deserialize::Result { - let value = i16::from_sql(bytes)?; + let value = >::from_sql(bytes)?; match value { 0 => Ok(Self::Unknown), 1 => Ok(Self::AppCreate), @@ -814,3 +814,12 @@ impl std::str::FromStr for IntentType { "APP_CREATE" | "APPCREATE" | "APP" | "APPLICATION" | "CREATE_APP" => Ok(Self::AppCreate), "TODO" | "TASK" | "REMINDER" => Ok(Self::Todo), "MONITOR" | "WATCH" | "ALERT" | "ON_CHANGE" => Ok(Self::Monitor), + "ACTION" | "DO" | "EXECUTE" | "RUN" => Ok(Self::Action), + "SCHEDULE" | "SCHEDULED" | "CRON" | "TIMER" => Ok(Self::Schedule), + "GOAL" | "OBJECTIVE" | "TARGET" => Ok(Self::Goal), + "TOOL" | "FUNCTION" | "UTILITY" => Ok(Self::Tool), + "QUERY" | "SEARCH" | "FIND" | "LOOKUP" => Ok(Self::Query), + _ => Ok(Self::Unknown), + } + } +} diff --git a/src/core/shared/schema.rs b/src/core/shared/schema.rs index f88244a58..f419c1e69 100644 --- a/src/core/shared/schema.rs +++ b/src/core/shared/schema.rs @@ -1,449 +1,25 @@ -// @generated automatically by Diesel CLI. -// This schema matches the consolidated migration 20250101000000_consolidated_schema - diesel::table! { - shard_config (shard_id) { - shard_id -> Int2, - region_code -> Bpchar, - datacenter -> Varchar, - connection_string -> Text, - is_primary -> Nullable, - is_active -> Nullable, - min_tenant_id -> Int8, - max_tenant_id -> Int8, - created_at -> Nullable, - } -} - -diesel::table! { - tenant_shard_map (tenant_id) { - tenant_id -> Int8, - shard_id -> Int2, - region_code -> Bpchar, - created_at -> Nullable, - } -} - -diesel::table! { - tenants (id) { - id -> Int8, - shard_id -> Int2, - external_id -> Nullable, - name -> Varchar, - slug -> Varchar, - region_code -> Bpchar, - plan_tier -> Int2, - settings -> Nullable, - limits -> Nullable, - is_active -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - users (id) { - id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - username -> Varchar, - email -> Varchar, - password_hash -> Nullable, - phone_number -> Nullable, - display_name -> Nullable, - avatar_url -> Nullable, - locale -> Nullable, - timezone -> Nullable, - is_active -> Nullable, - last_login_at -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, + organizations (org_id) { + org_id -> Uuid, + name -> Text, + slug -> Text, + created_at -> Timestamptz, } } diesel::table! { bots (id) { id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, name -> Varchar, description -> Nullable, - llm_provider -> Int2, - llm_config -> Nullable, - context_provider -> Int2, - context_config -> Nullable, - system_prompt -> Nullable, - personality -> Nullable, - capabilities -> Nullable, + llm_provider -> Varchar, + llm_config -> Jsonb, + context_provider -> Varchar, + context_config -> Jsonb, + created_at -> Timestamptz, + updated_at -> Timestamptz, is_active -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - bot_configuration (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - config_key -> Varchar, - config_value -> Text, - value_type -> Int2, - is_secret -> Nullable, - vault_path -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - bot_channels (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - channel_type -> Int2, - channel_identifier -> Nullable, - config -> Nullable, - credentials_vault_path -> Nullable, - is_active -> Nullable, - last_activity_at -> Nullable, - created_at -> Nullable, - } -} - -diesel::table! { - user_sessions (id) { - id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - user_id -> Uuid, - bot_id -> Uuid, - channel_type -> Int2, - title -> Nullable, - context_data -> Nullable, - current_tool -> Nullable, - answer_mode -> Nullable, - message_count -> Nullable, - total_tokens -> Nullable, - last_activity_at -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - message_history (id) { - id -> Uuid, - session_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - user_id -> Uuid, - role -> Int2, - message_type -> Int2, - content_encrypted -> Text, - content_hash -> Nullable, - media_url -> Nullable, - metadata -> Nullable, - token_count -> Nullable, - processing_time_ms -> Nullable, - llm_model -> Nullable, - message_index -> Int4, - created_at -> Nullable, - } -} - -diesel::table! { - bot_memories (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - user_id -> Nullable, - session_id -> Nullable, - memory_type -> Int2, - content -> Text, - embedding_id -> Nullable, - importance_score -> Nullable, - access_count -> Nullable, - last_accessed_at -> Nullable, - expires_at -> Nullable, - created_at -> Nullable, - } -} - -diesel::table! { - auto_tasks (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - session_id -> Nullable, - title -> Varchar, - intent -> Text, - status -> Int2, - execution_mode -> Int2, - priority -> Int2, - plan_id -> Nullable, - basic_program -> Nullable, - current_step -> Nullable, - total_steps -> Nullable, - progress -> Nullable, - step_results -> Nullable, - error_message -> Nullable, - started_at -> Nullable, - completed_at -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - execution_plans (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - task_id -> Nullable, - intent -> Text, - intent_type -> Nullable, - confidence -> Nullable, - status -> Int2, - steps -> Jsonb, - context -> Nullable, - basic_program -> Nullable, - simulation_result -> Nullable, - risk_level -> Nullable, - approved_by -> Nullable, - approved_at -> Nullable, - executed_at -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - task_approvals (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - task_id -> Uuid, - plan_id -> Nullable, - step_index -> Nullable, - action_type -> Varchar, - action_description -> Text, - risk_level -> Nullable, - status -> Int2, - decision -> Nullable, - decision_reason -> Nullable, - decided_by -> Nullable, - decided_at -> Nullable, - expires_at -> Nullable, - created_at -> Nullable, - } -} - -diesel::table! { - task_decisions (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - task_id -> Uuid, - question -> Text, - options -> Jsonb, - context -> Nullable, - status -> Int2, - selected_option -> Nullable, - decision_reason -> Nullable, - decided_by -> Nullable, - decided_at -> Nullable, - timeout_seconds -> Nullable, - created_at -> Nullable, - } -} - -diesel::table! { - safety_audit_log (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - task_id -> Nullable, - plan_id -> Nullable, - action_type -> Varchar, - action_details -> Jsonb, - constraint_checks -> Nullable, - simulation_result -> Nullable, - risk_assessment -> Nullable, - outcome -> Int2, - error_message -> Nullable, - created_at -> Nullable, - } -} - -diesel::table! { - intent_classifications (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - session_id -> Nullable, - original_text -> Text, - intent_type -> Int2, - confidence -> Float4, - entities -> Nullable, - suggested_name -> Nullable, - was_correct -> Nullable, - corrected_type -> Nullable, - feedback -> Nullable, - created_at -> Nullable, - } -} - -diesel::table! { - generated_apps (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - name -> Varchar, - description -> Nullable, - domain -> Nullable, - intent_source -> Nullable, - pages -> Nullable, - tables_created -> Nullable, - tools -> Nullable, - schedulers -> Nullable, - app_path -> Nullable, - is_active -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - designer_changes (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - session_id -> Nullable, - change_type -> Int2, - description -> Text, - file_path -> Varchar, - original_content -> Text, - new_content -> Text, - created_at -> Nullable, - } -} - -diesel::table! { - designer_pending_changes (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - session_id -> Nullable, - analysis_json -> Text, - instruction -> Text, - expires_at -> Timestamptz, - created_at -> Nullable, - } -} - -diesel::table! { - kb_collections (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - name -> Varchar, - description -> Nullable, - folder_path -> Nullable, - qdrant_collection -> Nullable, - document_count -> Nullable, - chunk_count -> Nullable, - total_tokens -> Nullable, - last_indexed_at -> Nullable, - is_active -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - kb_documents (id) { - id -> Uuid, - collection_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - file_path -> Varchar, - file_name -> Varchar, - file_type -> Nullable, - file_size -> Nullable, - content_hash -> Nullable, - chunk_count -> Nullable, - is_indexed -> Nullable, - indexed_at -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - session_kb_associations (id) { - id -> Uuid, - session_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - bot_id -> Uuid, - kb_name -> Varchar, - kb_folder_path -> Nullable, - qdrant_collection -> Nullable, - added_by_tool -> Nullable, - is_active -> Nullable, - added_at -> Nullable, - } -} - -diesel::table! { - kb_sources (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - name -> Varchar, - source_type -> Varchar, - connection_config -> Jsonb, - sync_schedule -> Nullable, - last_sync_at -> Nullable, - sync_status -> Nullable, - document_count -> Nullable, - is_active -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - tools (id) { - id -> Uuid, - bot_id -> Nullable, - tenant_id -> Int8, - shard_id -> Int2, - name -> Varchar, - description -> Text, - parameters -> Nullable, - script -> Text, - tool_type -> Nullable, - is_system -> Nullable, - is_active -> Nullable, - usage_count -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, + tenant_id -> Nullable, } } @@ -451,308 +27,455 @@ diesel::table! { system_automations (id) { id -> Uuid, bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - name -> Nullable, - kind -> Int2, - target -> Nullable, - schedule -> Nullable, - param -> Nullable, - is_active -> Nullable, + kind -> Int4, + target -> Nullable, + schedule -> Nullable, + param -> Text, + is_active -> Bool, last_triggered -> Nullable, - next_trigger -> Nullable, - run_count -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, } } diesel::table! { - pending_info (id) { + user_sessions (id) { id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - field_name -> Varchar, - field_label -> Varchar, - field_type -> Varchar, - reason -> Nullable, - config_key -> Varchar, - is_filled -> Nullable, - filled_at -> Nullable, - filled_value -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - usage_analytics (id) { - id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, user_id -> Uuid, bot_id -> Uuid, - session_id -> Nullable, - date -> Date, - session_count -> Nullable, - message_count -> Nullable, - total_tokens -> Nullable, - total_processing_time_ms -> Nullable, - avg_response_time_ms -> Nullable, - created_at -> Nullable, + title -> Text, + context_data -> Jsonb, + current_tool -> Nullable, + created_at -> Timestamptz, + updated_at -> Timestamptz, } } diesel::table! { - analytics_events (id) { + message_history (id) { id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - user_id -> Nullable, - session_id -> Nullable, - bot_id -> Nullable, - event_type -> Varchar, - event_data -> Nullable, - created_at -> Nullable, + session_id -> Uuid, + user_id -> Uuid, + role -> Int4, + content_encrypted -> Text, + message_type -> Int4, + message_index -> Int8, + created_at -> Timestamptz, + } +} + +diesel::table! { + users (id) { + id -> Uuid, + username -> Text, + email -> Text, + password_hash -> Text, + is_active -> Bool, + is_admin -> Bool, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + clicks (id) { + id -> Uuid, + campaign_id -> Text, + email -> Text, + updated_at -> Timestamptz, + } +} + +diesel::table! { + bot_memories (id) { + id -> Uuid, + bot_id -> Uuid, + key -> Text, + value -> Text, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + kb_documents (id) { + id -> Text, + bot_id -> Text, + user_id -> Text, + collection_name -> Text, + file_path -> Text, + file_size -> Integer, + file_hash -> Text, + first_published_at -> Text, + last_modified_at -> Text, + indexed_at -> Nullable, + metadata -> Text, + created_at -> Text, + updated_at -> Text, + } +} + +diesel::table! { + basic_tools (id) { + id -> Text, + bot_id -> Text, + tool_name -> Text, + file_path -> Text, + ast_path -> Text, + file_hash -> Text, + mcp_json -> Nullable, + tool_json -> Nullable, + compiled_at -> Text, + is_active -> Integer, + created_at -> Text, + updated_at -> Text, + } +} + +diesel::table! { + kb_collections (id) { + id -> Text, + bot_id -> Text, + user_id -> Text, + name -> Text, + folder_path -> Text, + qdrant_collection -> Text, + document_count -> Integer, + is_active -> Integer, + created_at -> Text, + updated_at -> Text, + } +} + +diesel::table! { + user_kb_associations (id) { + id -> Text, + user_id -> Text, + bot_id -> Text, + kb_name -> Text, + is_website -> Integer, + website_url -> Nullable, + created_at -> Text, + updated_at -> Text, + } +} + +diesel::table! { + session_tool_associations (id) { + id -> Text, + session_id -> Text, + tool_name -> Text, + added_at -> Text, + } +} + +diesel::table! { + bot_configuration (id) { + id -> Uuid, + bot_id -> Uuid, + config_key -> Text, + config_value -> Text, + is_encrypted -> Bool, + config_type -> Text, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + user_email_accounts (id) { + id -> Uuid, + user_id -> Uuid, + email -> Varchar, + display_name -> Nullable, + imap_server -> Varchar, + imap_port -> Int4, + smtp_server -> Varchar, + smtp_port -> Int4, + username -> Varchar, + password_encrypted -> Text, + is_primary -> Bool, + is_active -> Bool, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + email_drafts (id) { + id -> Uuid, + user_id -> Uuid, + account_id -> Uuid, + to_address -> Text, + cc_address -> Nullable, + bcc_address -> Nullable, + subject -> Nullable, + body -> Nullable, + attachments -> Jsonb, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + email_folders (id) { + id -> Uuid, + account_id -> Uuid, + folder_name -> Varchar, + folder_path -> Varchar, + unread_count -> Int4, + total_count -> Int4, + last_synced -> Nullable, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + user_preferences (id) { + id -> Uuid, + user_id -> Uuid, + preference_key -> Varchar, + preference_value -> Jsonb, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + user_login_tokens (id) { + id -> Uuid, + user_id -> Uuid, + token_hash -> Varchar, + expires_at -> Timestamptz, + created_at -> Timestamptz, + last_used -> Timestamptz, + user_agent -> Nullable, + ip_address -> Nullable, + is_active -> Bool, } } diesel::table! { tasks (id) { id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - bot_id -> Nullable, - title -> Varchar, + title -> Text, description -> Nullable, + status -> Text, + priority -> Text, assignee_id -> Nullable, reporter_id -> Nullable, project_id -> Nullable, - parent_task_id -> Nullable, - status -> Int2, - priority -> Int2, due_date -> Nullable, - estimated_hours -> Nullable, - actual_hours -> Nullable, - progress -> Nullable, - tags -> Nullable>>, - dependencies -> Nullable>>, + tags -> Array, + dependencies -> Array, + estimated_hours -> Nullable, + actual_hours -> Nullable, + progress -> Int4, + created_at -> Timestamptz, + updated_at -> Timestamptz, completed_at -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, } } -diesel::table! { - task_comments (id) { - id -> Uuid, - task_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - author_id -> Uuid, - content -> Text, - created_at -> Nullable, - updated_at -> Nullable, - } -} + diesel::table! { - connected_accounts (id) { - id -> Uuid, - user_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - provider -> Varchar, - provider_user_id -> Nullable, - email -> Nullable, - display_name -> Nullable, - access_token_vault -> Nullable, - refresh_token_vault -> Nullable, - token_expires_at -> Nullable, - scopes -> Nullable>>, - sync_status -> Nullable, - last_sync_at -> Nullable, - is_active -> Nullable, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - session_account_associations (id) { - id -> Uuid, - session_id -> Uuid, - bot_id -> Uuid, - account_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - email -> Nullable, - provider -> Nullable, - qdrant_collection -> Nullable, - is_active -> Nullable, - added_at -> Nullable, - } -} - -diesel::table! { - whatsapp_numbers (id) { + global_email_signatures (id) { id -> Uuid, bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - phone_number -> Varchar, - is_active -> Nullable, - created_at -> Nullable, - } -} - -diesel::table! { - clicks (campaign_id, email) { - campaign_id -> Varchar, - email -> Varchar, - tenant_id -> Int8, - click_count -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - table_role_access (id) { - id -> Uuid, - bot_id -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - table_name -> Varchar, - role_name -> Varchar, - can_read -> Nullable, - can_write -> Nullable, - can_delete -> Nullable, - row_filter -> Nullable, - column_filter -> Nullable>>, - created_at -> Nullable, - updated_at -> Nullable, - } -} - -diesel::table! { - context_injections (id) { - id -> Uuid, - session_id -> Uuid, - injected_by -> Uuid, - tenant_id -> Int8, - shard_id -> Int2, - context_data -> Jsonb, - reason -> Nullable, - created_at -> Nullable, - } -} - -diesel::table! { - organizations (org_id) { - org_id -> Uuid, - tenant_id -> Int8, name -> Varchar, - slug -> Varchar, - created_at -> Nullable, - updated_at -> Nullable, + content_html -> Text, + content_plain -> Text, + position -> Varchar, + is_active -> Bool, + created_at -> Timestamptz, + updated_at -> Timestamptz, } } diesel::table! { - user_organizations (id) { + email_signatures (id) { id -> Uuid, user_id -> Uuid, - org_id -> Uuid, - role -> Nullable, - created_at -> Nullable, + bot_id -> Nullable, + name -> Varchar, + content_html -> Text, + content_plain -> Text, + is_default -> Bool, + is_active -> Bool, + created_at -> Timestamptz, + updated_at -> Timestamptz, } } -// Foreign key relationships -diesel::joinable!(tenant_shard_map -> shard_config (shard_id)); -diesel::joinable!(users -> tenants (tenant_id)); -diesel::joinable!(bots -> tenants (tenant_id)); -diesel::joinable!(bot_configuration -> bots (bot_id)); -diesel::joinable!(bot_channels -> bots (bot_id)); -diesel::joinable!(user_sessions -> users (user_id)); -diesel::joinable!(user_sessions -> bots (bot_id)); -diesel::joinable!(user_sessions -> tenants (tenant_id)); -diesel::joinable!(message_history -> user_sessions (session_id)); -diesel::joinable!(message_history -> users (user_id)); -diesel::joinable!(bot_memories -> bots (bot_id)); -diesel::joinable!(auto_tasks -> bots (bot_id)); -diesel::joinable!(auto_tasks -> user_sessions (session_id)); -diesel::joinable!(execution_plans -> bots (bot_id)); -diesel::joinable!(execution_plans -> auto_tasks (task_id)); -diesel::joinable!(task_approvals -> bots (bot_id)); -diesel::joinable!(task_approvals -> auto_tasks (task_id)); -diesel::joinable!(task_decisions -> bots (bot_id)); -diesel::joinable!(task_decisions -> auto_tasks (task_id)); -diesel::joinable!(safety_audit_log -> bots (bot_id)); -diesel::joinable!(intent_classifications -> bots (bot_id)); -diesel::joinable!(generated_apps -> bots (bot_id)); -diesel::joinable!(designer_changes -> bots (bot_id)); -diesel::joinable!(designer_pending_changes -> bots (bot_id)); -diesel::joinable!(kb_collections -> bots (bot_id)); -diesel::joinable!(kb_documents -> kb_collections (collection_id)); -diesel::joinable!(session_kb_associations -> user_sessions (session_id)); -diesel::joinable!(session_kb_associations -> bots (bot_id)); -diesel::joinable!(kb_sources -> bots (bot_id)); -diesel::joinable!(system_automations -> bots (bot_id)); -diesel::joinable!(pending_info -> bots (bot_id)); -diesel::joinable!(usage_analytics -> users (user_id)); -diesel::joinable!(usage_analytics -> bots (bot_id)); -diesel::joinable!(task_comments -> tasks (task_id)); -diesel::joinable!(task_comments -> users (author_id)); -diesel::joinable!(connected_accounts -> users (user_id)); -diesel::joinable!(session_account_associations -> user_sessions (session_id)); -diesel::joinable!(session_account_associations -> bots (bot_id)); -diesel::joinable!(session_account_associations -> connected_accounts (account_id)); -diesel::joinable!(whatsapp_numbers -> bots (bot_id)); -diesel::joinable!(table_role_access -> bots (bot_id)); -diesel::joinable!(context_injections -> user_sessions (session_id)); -diesel::joinable!(organizations -> tenants (tenant_id)); -diesel::joinable!(user_organizations -> users (user_id)); -diesel::joinable!(user_organizations -> organizations (org_id)); +diesel::table! { + scheduled_emails (id) { + id -> Uuid, + user_id -> Uuid, + bot_id -> Uuid, + to_addresses -> Text, + cc_addresses -> Nullable, + bcc_addresses -> Nullable, + subject -> Text, + body_html -> Text, + body_plain -> Nullable, + attachments_json -> Text, + scheduled_at -> Timestamptz, + sent_at -> Nullable, + status -> Varchar, + retry_count -> Int4, + error_message -> Nullable, + created_at -> Timestamptz, + } +} + +diesel::table! { + email_templates (id) { + id -> Uuid, + bot_id -> Uuid, + user_id -> Nullable, + name -> Varchar, + description -> Nullable, + subject_template -> Text, + body_html_template -> Text, + body_plain_template -> Nullable, + variables_json -> Text, + category -> Nullable, + is_shared -> Bool, + usage_count -> Int4, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + email_auto_responders (id) { + id -> Uuid, + user_id -> Uuid, + bot_id -> Uuid, + responder_type -> Varchar, + subject -> Text, + body_html -> Text, + body_plain -> Nullable, + start_date -> Nullable, + end_date -> Nullable, + send_to_internal_only -> Bool, + exclude_addresses -> Nullable, + is_active -> Bool, + stalwart_sieve_id -> Nullable, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + email_rules (id) { + id -> Uuid, + user_id -> Uuid, + bot_id -> Uuid, + name -> Varchar, + priority -> Int4, + conditions_json -> Text, + actions_json -> Text, + stop_processing -> Bool, + is_active -> Bool, + stalwart_sieve_id -> Nullable, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + email_labels (id) { + id -> Uuid, + user_id -> Uuid, + bot_id -> Uuid, + name -> Varchar, + color -> Varchar, + parent_id -> Nullable, + is_system -> Bool, + created_at -> Timestamptz, + } +} + +diesel::table! { + email_label_assignments (id) { + id -> Uuid, + email_message_id -> Varchar, + label_id -> Uuid, + assigned_at -> Timestamptz, + } +} + +diesel::table! { + distribution_lists (id) { + id -> Uuid, + bot_id -> Uuid, + owner_id -> Uuid, + name -> Varchar, + email_alias -> Nullable, + description -> Nullable, + members_json -> Text, + is_public -> Bool, + stalwart_principal_id -> Nullable, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +diesel::table! { + shared_mailboxes (id) { + id -> Uuid, + bot_id -> Uuid, + email_address -> Varchar, + display_name -> Varchar, + description -> Nullable, + settings_json -> Text, + stalwart_account_id -> Nullable, + is_active -> Bool, + created_at -> Timestamptz, + } +} + +diesel::table! { + shared_mailbox_members (id) { + id -> Uuid, + mailbox_id -> Uuid, + user_id -> Uuid, + permission_level -> Varchar, + added_at -> Timestamptz, + } +} diesel::allow_tables_to_appear_in_same_query!( - shard_config, - tenant_shard_map, - tenants, - users, + organizations, bots, - bot_configuration, - bot_channels, + system_automations, user_sessions, message_history, - bot_memories, - auto_tasks, - execution_plans, - task_approvals, - task_decisions, - safety_audit_log, - intent_classifications, - generated_apps, - designer_changes, - designer_pending_changes, - kb_collections, - kb_documents, - session_kb_associations, - kb_sources, - tools, - system_automations, - pending_info, - usage_analytics, - analytics_events, - tasks, - task_comments, - connected_accounts, - session_account_associations, - whatsapp_numbers, + users, clicks, - table_role_access, - context_injections, - organizations, - user_organizations, + bot_memories, + kb_documents, + basic_tools, + kb_collections, + user_kb_associations, + session_tool_associations, + bot_configuration, + user_email_accounts, + email_drafts, + email_folders, + user_preferences, + user_login_tokens, + tasks, + global_email_signatures, + email_signatures, + scheduled_emails, + email_templates, + email_auto_responders, + email_rules, + email_labels, + email_label_assignments, + distribution_lists, + shared_mailboxes, + shared_mailbox_members, ); diff --git a/src/core/shared/utils.rs b/src/core/shared/utils.rs index 479a8f476..bcc22efd8 100644 --- a/src/core/shared/utils.rs +++ b/src/core/shared/utils.rs @@ -11,12 +11,14 @@ use diesel::{ use futures_util::StreamExt; #[cfg(feature = "progress-bars")] use indicatif::{ProgressBar, ProgressStyle}; -use reqwest::Client; +use log::{debug, warn}; +use reqwest::{Certificate, Client}; use rhai::{Array, Dynamic}; use serde_json::Value; use smartstring::SmartString; use std::error::Error; use std::sync::Arc; +use std::time::Duration; use tokio::fs::File as TokioFile; use tokio::io::AsyncWriteExt; use tokio::sync::RwLock; @@ -99,6 +101,12 @@ pub async fn create_s3_operator( (config.access_key.clone(), config.secret_key.clone()) }; + if std::path::Path::new(CA_CERT_PATH).exists() { + std::env::set_var("AWS_CA_BUNDLE", CA_CERT_PATH); + std::env::set_var("SSL_CERT_FILE", CA_CERT_PATH); + debug!("Set AWS_CA_BUNDLE and SSL_CERT_FILE to {} for S3 client", CA_CERT_PATH); + } + let base_config = aws_config::defaults(BehaviorVersion::latest()) .endpoint_url(endpoint) .region("auto") @@ -362,3 +370,68 @@ pub fn get_content_type(path: &str) -> &'static str { pub fn sanitize_sql_value(value: &str) -> String { value.replace('\'', "''") } + +/// Default path to the local CA certificate used for internal service TLS (dev stack) +pub const CA_CERT_PATH: &str = "./botserver-stack/conf/system/certificates/ca/ca.crt"; + +/// Creates an HTTP client with proper TLS verification. +/// +/// **Behavior:** +/// - If local CA cert exists (dev stack): uses it for verification +/// - If local CA cert doesn't exist (production): uses system CA store +/// +/// # Arguments +/// * `timeout_secs` - Request timeout in seconds (default: 30) +/// +/// # Returns +/// A reqwest::Client configured for TLS verification +pub fn create_tls_client(timeout_secs: Option) -> Client { + create_tls_client_with_ca(CA_CERT_PATH, timeout_secs) +} + +/// Creates an HTTP client with a custom CA certificate path. +/// +/// **Behavior:** +/// - If CA cert file exists: adds it as trusted root (for self-signed/internal CA) +/// - If CA cert file doesn't exist: uses system CA store (for public CAs like Let's Encrypt) +/// +/// This allows seamless transition from dev (local CA) to production (public CA). +/// +/// # Arguments +/// * `ca_cert_path` - Path to the CA certificate file (ignored if file doesn't exist) +/// * `timeout_secs` - Request timeout in seconds (default: 30) +/// +/// # Returns +/// A reqwest::Client configured for TLS verification +pub fn create_tls_client_with_ca(ca_cert_path: &str, timeout_secs: Option) -> Client { + let timeout = Duration::from_secs(timeout_secs.unwrap_or(30)); + let mut builder = Client::builder().timeout(timeout); + + // Try to load local CA cert (dev stack with self-signed certs) + // If it doesn't exist, we use system CA store (production with public certs) + if std::path::Path::new(ca_cert_path).exists() { + match std::fs::read(ca_cert_path) { + Ok(ca_cert_pem) => { + match Certificate::from_pem(&ca_cert_pem) { + Ok(ca_cert) => { + builder = builder.add_root_certificate(ca_cert); + debug!("Using local CA certificate from {} (dev stack mode)", ca_cert_path); + } + Err(e) => { + warn!("Failed to parse CA certificate from {}: {}", ca_cert_path, e); + } + } + } + Err(e) => { + warn!("Failed to read CA certificate from {}: {}", ca_cert_path, e); + } + } + } else { + debug!("Local CA cert not found at {}, using system CA store (production mode)", ca_cert_path); + } + + builder.build().unwrap_or_else(|e| { + warn!("Failed to create TLS client: {}, using default client", e); + Client::new() + }) +} diff --git a/src/core/urls.rs b/src/core/urls.rs index 3a1edaf5d..b7a2c4cc4 100644 --- a/src/core/urls.rs +++ b/src/core/urls.rs @@ -291,7 +291,7 @@ impl InternalUrls { pub const DIRECTORY_BASE: &'static str = "http://localhost:8080"; pub const DATABASE: &'static str = "postgres://localhost:5432"; pub const CACHE: &'static str = "redis://localhost:6379"; - pub const DRIVE: &'static str = "http://localhost:9000"; + pub const DRIVE: &'static str = "https://localhost:9000"; pub const EMAIL: &'static str = "http://localhost:8025"; pub const LLM: &'static str = "http://localhost:8081"; pub const EMBEDDING: &'static str = "http://localhost:8082"; diff --git a/src/drive/drive_monitor/mod.rs b/src/drive/drive_monitor/mod.rs index 313e2d84f..5551c1fb8 100644 --- a/src/drive/drive_monitor/mod.rs +++ b/src/drive/drive_monitor/mod.rs @@ -561,7 +561,10 @@ impl DriveMonitor { } let path_parts: Vec<&str> = path.split('/').collect(); - if path_parts.len() >= 2 { + // path_parts: [0] = "bot.gbkb", [1] = folder or file, [2+] = nested files + // Skip files directly in .gbkb root (path_parts.len() == 2 means root file) + // Only process files inside subfolders (path_parts.len() >= 3) + if path_parts.len() >= 3 { let kb_name = path_parts[1]; let kb_folder_path = self .work_root diff --git a/src/main.rs b/src/main.rs index 5f0eadac4..aa36e201b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -62,8 +62,7 @@ use botserver::core::config::AppConfig; #[cfg(feature = "directory")] use directory::auth_handler; -#[cfg(feature = "meet")] -use meet::{voice_start, voice_stop}; + use package_manager::InstallMode; use session::{create_session, get_session_history, get_sessions, start_session}; use shared::state::AppState; @@ -216,11 +215,7 @@ async fn run_axum_server( #[cfg(feature = "meet")] { - api_router = api_router - .route(ApiUrls::VOICE_START, post(voice_start)) - .route(ApiUrls::VOICE_STOP, post(voice_stop)) - .route(ApiUrls::WS_MEET, get(crate::meet::meeting_websocket)) - .merge(crate::meet::configure()); + api_router = api_router.merge(crate::meet::configure()); } #[cfg(feature = "email")] diff --git a/src/multimodal/mod.rs b/src/multimodal/mod.rs index cc6ff84b5..1a287b1ce 100644 --- a/src/multimodal/mod.rs +++ b/src/multimodal/mod.rs @@ -1,4 +1,5 @@ use crate::config::ConfigManager; +use crate::shared::utils::create_tls_client; use crate::shared::state::AppState; use log::{error, info, trace}; use reqwest::Client; @@ -232,11 +233,7 @@ impl BotModelsClient { image_config: ImageGeneratorConfig, video_config: VideoGeneratorConfig, ) -> Self { - let client = Client::builder() - .danger_accept_invalid_certs(true) - .timeout(std::time::Duration::from_secs(300)) - .build() - .unwrap_or_else(|_| Client::new()); + let client = create_tls_client(Some(300)); Self { client, diff --git a/src/security/integration.rs b/src/security/integration.rs index 17db44c3e..260daaa52 100644 --- a/src/security/integration.rs +++ b/src/security/integration.rs @@ -209,9 +209,7 @@ impl TlsIntegration { builder = builder.identity(identity.clone()); } - if cfg!(debug_assertions) { - builder = builder.danger_accept_invalid_certs(true); - } + if self.https_only { builder = builder.https_only(true); diff --git a/src/security/zitadel_auth.rs b/src/security/zitadel_auth.rs index 06c07bc35..148d5df24 100644 --- a/src/security/zitadel_auth.rs +++ b/src/security/zitadel_auth.rs @@ -1,6 +1,7 @@ use crate::core::secrets::SecretsManager; use crate::security::auth::{AuthConfig, AuthError, AuthenticatedUser, BotAccess, Permission, Role}; -use anyhow::{anyhow, Result}; +use crate::shared::utils::create_tls_client; +use anyhow::Result; use axum::{ body::Body, http::{header, Request}, @@ -203,15 +204,7 @@ struct ServiceToken { impl ZitadelAuthProvider { pub fn new(config: ZitadelAuthConfig) -> Result { - let http_client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .danger_accept_invalid_certs( - std::env::var("ZITADEL_SKIP_TLS_VERIFY") - .map(|v| v == "true" || v == "1") - .unwrap_or(false), - ) - .build() - .map_err(|e| anyhow!("Failed to create HTTP client: {}", e))?; + let http_client = create_tls_client(Some(30)); Ok(Self { config, diff --git a/src/timeseries/mod.rs b/src/timeseries/mod.rs index e176bc3df..2f0afb94f 100644 --- a/src/timeseries/mod.rs +++ b/src/timeseries/mod.rs @@ -17,6 +17,7 @@ +use crate::shared::utils::create_tls_client; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -224,11 +225,7 @@ pub struct TimeSeriesClient { impl TimeSeriesClient { pub async fn new(config: TimeSeriesConfig) -> Result { - let http_client = reqwest::Client::builder() - .danger_accept_invalid_certs(!config.verify_tls) - .timeout(std::time::Duration::from_secs(30)) - .build() - .map_err(|e| TimeSeriesError::ConnectionError(e.to_string()))?; + let http_client = create_tls_client(Some(30)); let write_buffer = Arc::new(RwLock::new(Vec::with_capacity(config.batch_size))); let (write_sender, write_receiver) = mpsc::channel::(10000); diff --git a/src/whatsapp/mod.rs b/src/whatsapp/mod.rs index ef51b6057..4a02df68f 100644 --- a/src/whatsapp/mod.rs +++ b/src/whatsapp/mod.rs @@ -175,7 +175,6 @@ pub fn configure() -> Router> { .route("/webhook/whatsapp", get(verify_webhook)) .route("/webhook/whatsapp", post(handle_webhook)) .route("/api/whatsapp/send", post(send_message)) - .route("/api/attendance/respond", post(attendant_respond)) } pub async fn verify_webhook(