GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests Projects Releases 323 Packages Wiki Activity Actions

Fix TLS configuration for MinIO, Qdrant, and template structure

Browse source 
- Fix MinIO health check to use HTTPS instead of HTTP
- Add Vault connectivity check before fetching credentials
- Add CA cert configuration for S3 client
- Add Qdrant vector_db setup with TLS configuration
- Fix Qdrant default URL to use HTTPS
- Always sync templates to S3 buckets (not just on create)
- Skip .gbkb root files, only index files in subfolders
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-12-29 18:21:03 -03:00
parent 23868e4c7d
commit b0baf36b11
20 changed files with 737 additions and 794 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
config/directory_config.json
View file

@ -1,7 +1,7 @@
{
"base_url": "http://localhost:8300",
"default_org": {
"id": "353032199743733774",
"id": "353229789043097614",
"name": "default",
"domain": "default.localhost"
},
@ -13,8 +13,8 @@
"first_name": "Admin",
"last_name": "User"
},
"admin_token": "1X7ImWy1yPmGYYumPJ0RfVaLuuLHKstH8BItaTGlp-6jTFPeM0uFo8sjdfxtk-jxjLivcVM",
"admin_token": "7QNrwws4y1X5iIuTUCtXpQj9RoQf4fYi144yEY87tNAbLMZOOD57t3YDAqCtIyIkBS1EZ5k",
"project_id": "",
"client_id": "353032201220194318",
"client_secret": "mrGZZk7Aqx1QbOHIwadgZHZkKHuPqZtOGDtdHTe4eZxEK86TDKfTiMlW2NxSEIHl"
"client_id": "353229789848469518",
"client_secret": "COd3gKdMO43jkUztTckCNtHrjxa5RtcIlBn7Cbp4GFoXs6mw6iZalB3m4Vv3FK5Y"
}

36
src/basic/keywords/kb_statistics.rs
View file

@ -1,5 +1,7 @@
use crate::config::ConfigManager;
use crate::shared::models::UserSession;
use crate::shared::state::AppState;
use crate::shared::utils::create_tls_client;
use log::{error, trace};
use rhai::{Dynamic, Engine};
use serde::{Deserialize, Serialize};
@ -225,11 +227,11 @@ async fn get_kb_statistics(
state: &AppState,
user: &UserSession,
) -> Result<KBStatistics, Box<dyn std::error::Error + Send + Sync>> {
let qdrant_url =
std::env::var("QDRANT_URL").unwrap_or_else(|_| "https://localhost:6334".to_string());
let client = reqwest::Client::builder()
.danger_accept_invalid_certs(true)
.build()?;
let config_manager = ConfigManager::new(state.conn.clone());
let qdrant_url = config_manager
.get_config(&user.bot_id, "vectordb-url", Some("https://localhost:6333"))
.unwrap_or_else(|_| "https://localhost:6333".to_string());
let client = create_tls_client(Some(30));
let collections_response = client
.get(format!("{}/collections", qdrant_url))
@ -277,14 +279,14 @@ async fn get_kb_statistics(
}
async fn get_collection_statistics(
_state: &AppState,
state: &AppState,
collection_name: &str,
) -> Result<CollectionStats, Box<dyn std::error::Error + Send + Sync>> {
let qdrant_url =
std::env::var("QDRANT_URL").unwrap_or_else(|_| "https://localhost:6334".to_string());
let client = reqwest::Client::builder()
.danger_accept_invalid_certs(true)
.build()?;
let config_manager = ConfigManager::new(state.conn.clone());
let qdrant_url = config_manager
.get_config(&uuid::Uuid::nil(), "vectordb-url", Some("https://localhost:6333"))
.unwrap_or_else(|_| "https://localhost:6333".to_string());
let client = create_tls_client(Some(30));
let response = client
.get(format!("{}/collections/{}", qdrant_url, collection_name))
@ -362,14 +364,14 @@ fn get_documents_added_since(
}
async fn list_collections(
_state: &AppState,
state: &AppState,
user: &UserSession,
) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
let qdrant_url =
std::env::var("QDRANT_URL").unwrap_or_else(|_| "https://localhost:6334".to_string());
let client = reqwest::Client::builder()
.danger_accept_invalid_certs(true)
.build()?;
let config_manager = ConfigManager::new(state.conn.clone());
let qdrant_url = config_manager
.get_config(&user.bot_id, "vectordb-url", Some("https://localhost:6333"))
.unwrap_or_else(|_| "https://localhost:6333".to_string());
let client = create_tls_client(Some(30));
let response = client
.get(format!("{}/collections", qdrant_url))

44
src/core/bootstrap/mod.rs
View file

@ -1,5 +1,5 @@
use crate::config::AppConfig;
use crate::package_manager::setup::{DirectorySetup, EmailSetup};
use crate::package_manager::setup::{DirectorySetup, EmailSetup, VectorDbSetup};
use crate::package_manager::{InstallMode, PackageManager};
use crate::shared::utils::{establish_pg_connection, init_secrets_manager};
use anyhow::Result;
@ -321,7 +321,7 @@ impl BootstrapManager {
for i in 0..15 {
let drive_ready = Command::new("sh")
.arg("-c")
.arg("curl -f -s 'http://127.0.0.1:9000/minio/health/live' >/dev/null 2>&1")
.arg("curl -sfk 'https://127.0.0.1:9000/minio/health/live' >/dev/null 2>&1")
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
@ -975,6 +975,15 @@ impl BootstrapManager {
error!("Failed to setup CoreDNS: {}", e);
}
}
if component == "vector_db" {
info!("Configuring Qdrant vector database with TLS...");
let conf_path = self.stack_dir("conf");
let data_path = self.stack_dir("data");
if let Err(e) = VectorDbSetup::setup(conf_path, data_path).await {
error!("Failed to setup vector_db: {}", e);
}
}
}
}
info!("=== BOOTSTRAP COMPLETED SUCCESSFULLY ===");
@ -1797,6 +1806,13 @@ VAULT_CACHE_TTL=300
)
};
// Set CA cert for self-signed TLS (dev stack)
let ca_cert_path = "./botserver-stack/conf/system/certificates/ca/ca.crt";
if std::path::Path::new(ca_cert_path).exists() {
std::env::set_var("AWS_CA_BUNDLE", ca_cert_path);
std::env::set_var("SSL_CERT_FILE", ca_cert_path);
}
let base_config = aws_config::defaults(BehaviorVersion::latest())
.endpoint_url(endpoint)
.region("auto")
@ -1850,16 +1866,17 @@ VAULT_CACHE_TTL=300
{
let bot_name = path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let bucket = bot_name.trim_start_matches('/').to_string();
// Create bucket if it doesn't exist
if client.head_bucket().bucket(&bucket).send().await.is_err() {
match client.create_bucket().bucket(&bucket).send().await {
Ok(_) => {
Self::upload_directory_recursive(&client, &path, &bucket, "/").await?;
}
Err(e) => {
warn!("S3/MinIO not available, skipping bucket {}: {}", bucket, e);
}
if let Err(e) = client.create_bucket().bucket(&bucket).send().await {
warn!("S3/MinIO not available, skipping bucket {}: {}", bucket, e);
continue;
}
}
// Always sync templates to bucket
if let Err(e) = Self::upload_directory_recursive(&client, &path, &bucket, "/").await {
warn!("Failed to upload templates to bucket {}: {}", bucket, e);
}
}
}
Ok(())
@ -2290,6 +2307,15 @@ log_level = "info"
fs::copy(&ca_cert_path, service_dir.join("ca.crt"))?;
}
let minio_certs_dir = PathBuf::from("./botserver-stack/conf/drive/certs");
fs::create_dir_all(&minio_certs_dir)?;
let drive_cert_dir = cert_dir.join("drive");
fs::copy(drive_cert_dir.join("server.crt"), minio_certs_dir.join("public.crt"))?;
fs::copy(drive_cert_dir.join("server.key"), minio_certs_dir.join("private.key"))?;
let minio_ca_dir = minio_certs_dir.join("CAs");
fs::create_dir_all(&minio_ca_dir)?;
fs::copy(&ca_cert_path, minio_ca_dir.join("ca.crt"))?;
info!("TLS certificates generated successfully");
Ok(())
}

10
src/core/directory/api.rs
View file

@ -1,6 +1,7 @@
use crate::core::directory::{BotAccess, UserAccount, UserProvisioningService, UserRole};
use crate::core::urls::ApiUrls;
use crate::shared::state::AppState;
use crate::shared::utils::create_tls_client;
use anyhow::Result;
use axum::{
extract::{Json, Path, State},
@ -254,14 +255,7 @@ pub async fn check_services_status(State(state): State<Arc<AppState>>) -> impl I
}
}
let client = reqwest::Client::builder()
.danger_accept_invalid_certs(true)
.timeout(std::time::Duration::from_secs(2))
.build()
.unwrap_or_else(|e| {
log::warn!("Failed to create HTTP client: {}, using default", e);
reqwest::Client::new()
});
let client = create_tls_client(Some(2));
if let Ok(response) = client.get("https://localhost:8300/healthz").send().await {
status.directory = response.status().is_success();

28
src/core/kb/kb_indexer.rs
View file

@ -5,6 +5,9 @@ use std::collections::HashMap;
use std::path::{Path, PathBuf};
use uuid::Uuid;
use crate::config::ConfigManager;
use crate::shared::utils::{create_tls_client, DbPool};
use super::document_processor::{DocumentProcessor, TextChunk};
use super::embedding_generator::{Embedding, EmbeddingConfig, KbEmbeddingGenerator};
@ -18,7 +21,21 @@ pub struct QdrantConfig {
impl Default for QdrantConfig {
fn default() -> Self {
Self {
url: "http://localhost:6333".to_string(),
url: "https://localhost:6333".to_string(),
api_key: None,
timeout_secs: 30,
}
}
}
impl QdrantConfig {
pub fn from_config(pool: DbPool, bot_id: &Uuid) -> Self {
let config_manager = ConfigManager::new(pool);
let url = config_manager
.get_config(bot_id, "vectordb-url", Some("https://localhost:6333"))
.unwrap_or_else(|_| "https://localhost:6333".to_string());
Self {
url,
api_key: None,
timeout_secs: 30,
}
@ -77,13 +94,8 @@ impl KbIndexer {
let document_processor = DocumentProcessor::default();
let embedding_generator = KbEmbeddingGenerator::new(embedding_config);
let http_client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(qdrant_config.timeout_secs))
.build()
.unwrap_or_else(|e| {
log::warn!("Failed to create HTTP client with timeout: {}, using default", e);
reqwest::Client::new()
});
// Use shared TLS client with local CA certificate
let http_client = create_tls_client(Some(qdrant_config.timeout_secs));
Self {
document_processor,

24
src/core/package_manager/facade.rs
View file

@ -859,13 +859,27 @@ Store credentials in Vault:
temp_file: &std::path::Path,
bin_path: &std::path::Path,
) -> Result<()> {
// Check if tarball has a top-level directory or files at root
let list_output = Command::new("tar")
.args(["-tzf", temp_file.to_str().unwrap_or_default()])
.output()?;
let has_subdir = if list_output.status.success() {
let contents = String::from_utf8_lossy(&list_output.stdout);
// If first entry contains '/', there's a subdirectory structure
contents.lines().next().map(|l| l.contains('/')).unwrap_or(false)
} else {
false
};
let mut args = vec!["-xzf", temp_file.to_str().unwrap_or_default()];
if has_subdir {
args.push("--strip-components=1");
}
let output = Command::new("tar")
.current_dir(bin_path)
.args([
"-xzf",
temp_file.to_str().unwrap_or_default(),
"--strip-components=1",
])
.args(&args)
.output()?;
if !output.status.success() {
return Err(anyhow::anyhow!(

23
src/core/package_manager/installer.rs
View file

@ -251,8 +251,8 @@ impl PackageManager {
("MINIO_ROOT_PASSWORD".to_string(), "$DRIVE_SECRET".to_string()),
]),
data_download_list: Vec::new(),
exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(),
check_cmd: "curl -sf http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1".to_string(),
exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 --certs-dir {{CONF_PATH}}/drive/certs > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(),
check_cmd: "curl -sfk https://127.0.0.1:9000/minio/health/live >/dev/null 2>&1".to_string(),
},
);
}
@ -801,7 +801,7 @@ impl PackageManager {
ComponentConfig {
name: "vector_db".to_string(),
ports: vec![6333],
ports: vec![6334],
dependencies: vec![],
linux_packages: vec![],
macos_packages: vec![],
@ -818,8 +818,8 @@ impl PackageManager {
post_install_cmds_windows: vec![],
env_vars: HashMap::new(),
data_download_list: Vec::new(),
exec_cmd: "{{BIN_PATH}}/qdrant --storage-path {{DATA_PATH}} --enable-tls --cert {{CONF_PATH}}/system/certificates/qdrant/server.crt --key {{CONF_PATH}}/system/certificates/qdrant/server.key".to_string(),
check_cmd: "curl -f -k --connect-timeout 2 -m 5 https://localhost:6334/metrics >/dev/null 2>&1".to_string(),
exec_cmd: "nohup {{BIN_PATH}}/qdrant --config-path {{CONF_PATH}}/vector_db/config.yaml > {{LOGS_PATH}}/qdrant.log 2>&1 &".to_string(),
check_cmd: "curl -sfk https://localhost:6333/collections >/dev/null 2>&1".to_string(),
},
);
}
@ -1164,6 +1164,19 @@ EOF"#.to_string(),
return credentials;
}
// Check if Vault is reachable before trying to fetch credentials
let vault_check = std::process::Command::new("sh")
.arg("-c")
.arg(format!("curl -sf {}/v1/sys/health >/dev/null 2>&1", vault_addr))
.status()
.map(|s| s.success())
.unwrap_or(false);
if !vault_check {
trace!("Vault not reachable at {}, skipping credential fetch", vault_addr);
return credentials;
}
let base_path = std::env::var("BOTSERVER_STACK_PATH")
.map(std::path::PathBuf::from)
.unwrap_or_else(|_| {

2
src/core/package_manager/setup/mod.rs
View file

@ -1,5 +1,7 @@
pub mod directory_setup;
pub mod email_setup;
pub mod vector_db_setup;
pub use directory_setup::{DirectorySetup, DefaultUser};
pub use email_setup::EmailSetup;
pub use vector_db_setup::VectorDbSetup;

93
src/core/package_manager/setup/vector_db_setup.rs Normal file
View file

@ -0,0 +1,93 @@
use anyhow::Result;
use std::path::PathBuf;
use std::fs;
use tracing::info;
pub struct VectorDbSetup;
impl VectorDbSetup {
pub async fn setup(conf_path: PathBuf, data_path: PathBuf) -> Result<()> {
let config_dir = conf_path.join("vector_db");
fs::create_dir_all(&config_dir)?;
let data_dir = data_path.join("vector_db");
fs::create_dir_all(&data_dir)?;
let cert_dir = conf_path.join("system/certificates/vectordb");
// Convert to absolute paths for Qdrant config
let data_dir_abs = fs::canonicalize(&data_dir).unwrap_or(data_dir);
let cert_dir_abs = fs::canonicalize(&cert_dir).unwrap_or(cert_dir);
let config_content = generate_qdrant_config(&data_dir_abs, &cert_dir_abs);
let config_path = config_dir.join("config.yaml");
fs::write(&config_path, config_content)?;
info!("Qdrant vector_db configuration written to {:?}", config_path);
Ok(())
}
}
pub fn generate_qdrant_config(data_dir: &PathBuf, cert_dir: &PathBuf) -> String {
let data_path = data_dir.to_string_lossy();
let cert_path = cert_dir.join("server.crt").to_string_lossy().to_string();
let key_path = cert_dir.join("server.key").to_string_lossy().to_string();
let ca_path = cert_dir.join("ca.crt").to_string_lossy().to_string();
format!(
r#"# Qdrant configuration with TLS enabled
# Generated by BotServer bootstrap
log_level: INFO
storage:
storage_path: {data_path}
snapshots_path: {data_path}/snapshots
on_disk_payload: true
service:
host: 0.0.0.0
http_port: 6333
grpc_port: 6334
enable_tls: true
tls:
cert: {cert_path}
key: {key_path}
ca_cert: {ca_path}
verify_https_client_certificate: false
cluster:
enabled: false
telemetry_disabled: true
"#
)
}
pub async fn generate_vector_db_config(config_path: PathBuf, data_path: PathBuf) -> Result<()> {
VectorDbSetup::setup(config_path, data_path).await
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
#[test]
fn test_generate_qdrant_config() {
let data_dir = PathBuf::from("/tmp/qdrant/data");
let cert_dir = PathBuf::from("/tmp/qdrant/certs");
let config = generate_qdrant_config(&data_dir, &cert_dir);
assert!(config.contains("enable_tls: true"));
assert!(config.contains("http_port: 6333"));
assert!(config.contains("grpc_port: 6334"));
assert!(config.contains("/tmp/qdrant/data"));
assert!(config.contains("/tmp/qdrant/certs/server.crt"));
assert!(config.contains("/tmp/qdrant/certs/server.key"));
}
}

33
src/core/shared/enums.rs
View file

@ -55,7 +55,7 @@ impl ToSql<SmallInt, Pg> for ChannelType {
impl FromSql<SmallInt, Pg> for ChannelType {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::Web),
1 => Ok(Self::WhatsApp),
@ -142,7 +142,7 @@ impl ToSql<SmallInt, Pg> for MessageRole {
impl FromSql<SmallInt, Pg> for MessageRole {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
1 => Ok(Self::User),
2 => Ok(Self::Assistant),
@ -220,7 +220,7 @@ impl ToSql<SmallInt, Pg> for MessageType {
impl FromSql<SmallInt, Pg> for MessageType {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::Text),
1 => Ok(Self::Image),
@ -290,7 +290,7 @@ impl ToSql<SmallInt, Pg> for LlmProvider {
impl FromSql<SmallInt, Pg> for LlmProvider {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::OpenAi),
1 => Ok(Self::Anthropic),
@ -359,7 +359,7 @@ impl ToSql<SmallInt, Pg> for ContextProvider {
impl FromSql<SmallInt, Pg> for ContextProvider {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::None),
1 => Ok(Self::Qdrant),
@ -409,7 +409,7 @@ impl ToSql<SmallInt, Pg> for TaskStatus {
impl FromSql<SmallInt, Pg> for TaskStatus {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::Pending),
1 => Ok(Self::Ready),
@ -489,7 +489,7 @@ impl ToSql<SmallInt, Pg> for TaskPriority {
impl FromSql<SmallInt, Pg> for TaskPriority {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::Low),
1 => Ok(Self::Normal),
@ -558,7 +558,7 @@ impl ToSql<SmallInt, Pg> for ExecutionMode {
impl FromSql<SmallInt, Pg> for ExecutionMode {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::Manual),
1 => Ok(Self::Supervised),
@ -611,7 +611,7 @@ impl ToSql<SmallInt, Pg> for RiskLevel {
impl FromSql<SmallInt, Pg> for RiskLevel {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::None),
1 => Ok(Self::Low),
@ -668,7 +668,7 @@ impl ToSql<SmallInt, Pg> for ApprovalStatus {
impl FromSql<SmallInt, Pg> for ApprovalStatus {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::Pending),
1 => Ok(Self::Approved),
@ -717,7 +717,7 @@ impl ToSql<SmallInt, Pg> for ApprovalDecision {
impl FromSql<SmallInt, Pg> for ApprovalDecision {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::Approve),
1 => Ok(Self::Reject),
@ -774,7 +774,7 @@ impl ToSql<SmallInt, Pg> for IntentType {
impl FromSql<SmallInt, Pg> for IntentType {
fn from_sql(bytes: PgValue<'_>) -> deserialize::Result<Self> {
let value = i16::from_sql(bytes)?;
let value = <i16 as FromSql<SmallInt, Pg>>::from_sql(bytes)?;
match value {
0 => Ok(Self::Unknown),
1 => Ok(Self::AppCreate),
@ -814,3 +814,12 @@ impl std::str::FromStr for IntentType {
"APP_CREATE" | "APPCREATE" | "APP" | "APPLICATION" | "CREATE_APP" => Ok(Self::AppCreate),
"TODO" | "TASK" | "REMINDER" => Ok(Self::Todo),
"MONITOR" | "WATCH" | "ALERT" | "ON_CHANGE" => Ok(Self::Monitor),
"ACTION" | "DO" | "EXECUTE" | "RUN" => Ok(Self::Action),
"SCHEDULE" | "SCHEDULED" | "CRON" | "TIMER" => Ok(Self::Schedule),
"GOAL" | "OBJECTIVE" | "TARGET" => Ok(Self::Goal),
"TOOL" | "FUNCTION" | "UTILITY" => Ok(Self::Tool),
"QUERY" | "SEARCH" | "FIND" | "LOOKUP" => Ok(Self::Query),
_ => Ok(Self::Unknown),
}
}
}

1107
src/core/shared/schema.rs
View file

File diff suppressed because it is too large Load diff

75
src/core/shared/utils.rs
View file

@ -11,12 +11,14 @@ use diesel::{
use futures_util::StreamExt;
#[cfg(feature = "progress-bars")]
use indicatif::{ProgressBar, ProgressStyle};
use reqwest::Client;
use log::{debug, warn};
use reqwest::{Certificate, Client};
use rhai::{Array, Dynamic};
use serde_json::Value;
use smartstring::SmartString;
use std::error::Error;
use std::sync::Arc;
use std::time::Duration;
use tokio::fs::File as TokioFile;
use tokio::io::AsyncWriteExt;
use tokio::sync::RwLock;
@ -99,6 +101,12 @@ pub async fn create_s3_operator(
(config.access_key.clone(), config.secret_key.clone())
};
if std::path::Path::new(CA_CERT_PATH).exists() {
std::env::set_var("AWS_CA_BUNDLE", CA_CERT_PATH);
std::env::set_var("SSL_CERT_FILE", CA_CERT_PATH);
debug!("Set AWS_CA_BUNDLE and SSL_CERT_FILE to {} for S3 client", CA_CERT_PATH);
}
let base_config = aws_config::defaults(BehaviorVersion::latest())
.endpoint_url(endpoint)
.region("auto")
@ -362,3 +370,68 @@ pub fn get_content_type(path: &str) -> &'static str {
pub fn sanitize_sql_value(value: &str) -> String {
value.replace('\'', "''")
}
/// Default path to the local CA certificate used for internal service TLS (dev stack)
pub const CA_CERT_PATH: &str = "./botserver-stack/conf/system/certificates/ca/ca.crt";
/// Creates an HTTP client with proper TLS verification.
///
/// **Behavior:**
/// - If local CA cert exists (dev stack): uses it for verification
/// - If local CA cert doesn't exist (production): uses system CA store
///
/// # Arguments
/// * `timeout_secs` - Request timeout in seconds (default: 30)
///
/// # Returns
/// A reqwest::Client configured for TLS verification
pub fn create_tls_client(timeout_secs: Option<u64>) -> Client {
create_tls_client_with_ca(CA_CERT_PATH, timeout_secs)
}
/// Creates an HTTP client with a custom CA certificate path.
///
/// **Behavior:**
/// - If CA cert file exists: adds it as trusted root (for self-signed/internal CA)
/// - If CA cert file doesn't exist: uses system CA store (for public CAs like Let's Encrypt)
///
/// This allows seamless transition from dev (local CA) to production (public CA).
///
/// # Arguments
/// * `ca_cert_path` - Path to the CA certificate file (ignored if file doesn't exist)
/// * `timeout_secs` - Request timeout in seconds (default: 30)
///
/// # Returns
/// A reqwest::Client configured for TLS verification
pub fn create_tls_client_with_ca(ca_cert_path: &str, timeout_secs: Option<u64>) -> Client {
let timeout = Duration::from_secs(timeout_secs.unwrap_or(30));
let mut builder = Client::builder().timeout(timeout);
// Try to load local CA cert (dev stack with self-signed certs)
// If it doesn't exist, we use system CA store (production with public certs)
if std::path::Path::new(ca_cert_path).exists() {
match std::fs::read(ca_cert_path) {
Ok(ca_cert_pem) => {
match Certificate::from_pem(&ca_cert_pem) {
Ok(ca_cert) => {
builder = builder.add_root_certificate(ca_cert);
debug!("Using local CA certificate from {} (dev stack mode)", ca_cert_path);
}
Err(e) => {
warn!("Failed to parse CA certificate from {}: {}", ca_cert_path, e);
}
}
}
Err(e) => {
warn!("Failed to read CA certificate from {}: {}", ca_cert_path, e);
}
}
} else {
debug!("Local CA cert not found at {}, using system CA store (production mode)", ca_cert_path);
}
builder.build().unwrap_or_else(|e| {
warn!("Failed to create TLS client: {}, using default client", e);
Client::new()
})
}

2
src/core/urls.rs
View file

@ -291,7 +291,7 @@ impl InternalUrls {
pub const DIRECTORY_BASE: &'static str = "http://localhost:8080";
pub const DATABASE: &'static str = "postgres://localhost:5432";
pub const CACHE: &'static str = "redis://localhost:6379";
pub const DRIVE: &'static str = "http://localhost:9000";
pub const DRIVE: &'static str = "https://localhost:9000";
pub const EMAIL: &'static str = "http://localhost:8025";
pub const LLM: &'static str = "http://localhost:8081";
pub const EMBEDDING: &'static str = "http://localhost:8082";

5
src/drive/drive_monitor/mod.rs
View file

@ -561,7 +561,10 @@ impl DriveMonitor {
}
let path_parts: Vec<&str> = path.split('/').collect();
if path_parts.len() >= 2 {
// path_parts: [0] = "bot.gbkb", [1] = folder or file, [2+] = nested files
// Skip files directly in .gbkb root (path_parts.len() == 2 means root file)
// Only process files inside subfolders (path_parts.len() >= 3)
if path_parts.len() >= 3 {
let kb_name = path_parts[1];
let kb_folder_path = self
.work_root

9
src/main.rs
View file

@ -62,8 +62,7 @@ use botserver::core::config::AppConfig;
#[cfg(feature = "directory")]
use directory::auth_handler;
#[cfg(feature = "meet")]
use meet::{voice_start, voice_stop};
use package_manager::InstallMode;
use session::{create_session, get_session_history, get_sessions, start_session};
use shared::state::AppState;
@ -216,11 +215,7 @@ async fn run_axum_server(
#[cfg(feature = "meet")]
{
api_router = api_router
.route(ApiUrls::VOICE_START, post(voice_start))
.route(ApiUrls::VOICE_STOP, post(voice_stop))
.route(ApiUrls::WS_MEET, get(crate::meet::meeting_websocket))
.merge(crate::meet::configure());
api_router = api_router.merge(crate::meet::configure());
}
#[cfg(feature = "email")]

7
src/multimodal/mod.rs
View file

@ -1,4 +1,5 @@
use crate::config::ConfigManager;
use crate::shared::utils::create_tls_client;
use crate::shared::state::AppState;
use log::{error, info, trace};
use reqwest::Client;
@ -232,11 +233,7 @@ impl BotModelsClient {
image_config: ImageGeneratorConfig,
video_config: VideoGeneratorConfig,
) -> Self {
let client = Client::builder()
.danger_accept_invalid_certs(true)
.timeout(std::time::Duration::from_secs(300))
.build()
.unwrap_or_else(|_| Client::new());
let client = create_tls_client(Some(300));
Self {
client,

4
src/security/integration.rs
View file

@ -209,9 +209,7 @@ impl TlsIntegration {
builder = builder.identity(identity.clone());
}
if cfg!(debug_assertions) {
builder = builder.danger_accept_invalid_certs(true);
}
if self.https_only {
builder = builder.https_only(true);

13
src/security/zitadel_auth.rs
View file

@ -1,6 +1,7 @@
use crate::core::secrets::SecretsManager;
use crate::security::auth::{AuthConfig, AuthError, AuthenticatedUser, BotAccess, Permission, Role};
use anyhow::{anyhow, Result};
use crate::shared::utils::create_tls_client;
use anyhow::Result;
use axum::{
body::Body,
http::{header, Request},
@ -203,15 +204,7 @@ struct ServiceToken {
impl ZitadelAuthProvider {
pub fn new(config: ZitadelAuthConfig) -> Result<Self> {
let http_client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.danger_accept_invalid_certs(
std::env::var("ZITADEL_SKIP_TLS_VERIFY")
.map(|v| v == "true" || v == "1")
.unwrap_or(false),
)
.build()
.map_err(|e| anyhow!("Failed to create HTTP client: {}", e))?;
let http_client = create_tls_client(Some(30));
Ok(Self {
config,

7
src/timeseries/mod.rs
View file

@ -17,6 +17,7 @@
use crate::shared::utils::create_tls_client;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@ -224,11 +225,7 @@ pub struct TimeSeriesClient {
impl TimeSeriesClient {
pub async fn new(config: TimeSeriesConfig) -> Result<Self, TimeSeriesError> {
let http_client = reqwest::Client::builder()
.danger_accept_invalid_certs(!config.verify_tls)
.timeout(std::time::Duration::from_secs(30))
.build()
.map_err(|e| TimeSeriesError::ConnectionError(e.to_string()))?;
let http_client = create_tls_client(Some(30));
let write_buffer = Arc::new(RwLock::new(Vec::with_capacity(config.batch_size)));
let (write_sender, write_receiver) = mpsc::channel::<MetricPoint>(10000);

1
src/whatsapp/mod.rs
View file

@ -175,7 +175,6 @@ pub fn configure() -> Router<Arc<AppState>> {
.route("/webhook/whatsapp", get(verify_webhook))
.route("/webhook/whatsapp", post(handle_webhook))
.route("/api/whatsapp/send", post(send_message))
.route("/api/attendance/respond", post(attendant_respond))
}
pub async fn verify_webhook(