Merge branch 'main' into main

This commit is contained in:
Rodrigo Rodriguez 2025-10-31 16:05:14 -03:00 committed by GitHub
commit ea765811a2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 2003 additions and 802 deletions

6
.vscode/launch.json vendored
View file

@ -16,6 +16,9 @@
} }
}, },
"args": [], "args": [],
"env": {
"RUST_LOG": "debug"
},
"cwd": "${workspaceFolder}" "cwd": "${workspaceFolder}"
}, },
{ {
@ -30,6 +33,9 @@
} }
}, },
"args": [], "args": [],
"env": {
"RUST_LOG": "trace"
},
"cwd": "${workspaceFolder}" "cwd": "${workspaceFolder}"
} }
] ]

1073
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -53,6 +53,8 @@ anyhow = "1.0"
argon2 = "0.5" argon2 = "0.5"
async-stream = "0.3" async-stream = "0.3"
async-trait = "0.1" async-trait = "0.1"
aws-config = "1.8.8"
aws-sdk-s3 = { version = "1.109.0", features = ["behavior-version-latest"] }
base64 = "0.22" base64 = "0.22"
bytes = "1.8" bytes = "1.8"
chrono = { version = "0.4", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] }
@ -64,6 +66,7 @@ env_logger = "0.11"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
headless_chrome = { version = "1.0.18", optional = true } headless_chrome = { version = "1.0.18", optional = true }
hmac = "0.12.1"
imap = { version = "3.0.0-alpha.15", optional = true } imap = { version = "3.0.0-alpha.15", optional = true }
include_dir = "0.7" include_dir = "0.7"
indicatif = "0.18.0" indicatif = "0.18.0"
@ -71,9 +74,9 @@ lettre = { version = "0.11", features = ["smtp-transport", "builder", "tokio1",
livekit = "0.7" livekit = "0.7"
log = "0.4" log = "0.4"
mailparse = "0.15" mailparse = "0.15"
mockito = "1.7.0"
native-tls = "0.2" native-tls = "0.2"
num-format = "0.4" num-format = "0.4"
opendal = { version = "0.54.1", features = ["services-s3"] }
pdf-extract = "0.10.0" pdf-extract = "0.10.0"
qdrant-client = { version = "1.12", optional = true } qdrant-client = { version = "1.12", optional = true }
rand = "0.9.2" rand = "0.9.2"

View file

@ -22,9 +22,9 @@ dirs=(
# "auth" # "auth"
# "automation" # "automation"
# "basic" # "basic"
# "bot" "bot"
"bootstrap" "bootstrap"
"package_manager" #"package_manager"
# "channels" # "channels"
# "config" # "config"
# "context" # "context"

View file

@ -183,31 +183,6 @@ BEGIN
END IF; END IF;
END $$; END $$;
-- ============================================================================
-- DEFAULT SERVER CONFIGURATION
-- Insert default values that replace .env
-- ============================================================================
INSERT INTO server_configuration (id, config_key, config_value, config_type, description) VALUES
(gen_random_uuid()::text, 'SERVER_HOST', '127.0.0.1', 'string', 'Server bind address'),
(gen_random_uuid()::text, 'SERVER_PORT', '8080', 'integer', 'Server port'),
(gen_random_uuid()::text, 'TABLES_SERVER', 'localhost', 'string', 'PostgreSQL server address'),
(gen_random_uuid()::text, 'TABLES_PORT', '5432', 'integer', 'PostgreSQL port'),
(gen_random_uuid()::text, 'TABLES_DATABASE', 'botserver', 'string', 'PostgreSQL database name'),
(gen_random_uuid()::text, 'TABLES_USERNAME', 'botserver', 'string', 'PostgreSQL username'),
(gen_random_uuid()::text, 'DRIVE_SERVER', 'localhost:9000', 'string', 'MinIO server address'),
(gen_random_uuid()::text, 'DRIVE_USE_SSL', 'false', 'boolean', 'Use SSL for drive'),
(gen_random_uuid()::text, 'DRIVE_ORG_PREFIX', 'botserver', 'string', 'Drive organization prefix'),
(gen_random_uuid()::text, 'DRIVE_BUCKET', 'default', 'string', 'Default S3 bucket'),
(gen_random_uuid()::text, 'VECTORDB_URL', 'http://localhost:6333', 'string', 'Qdrant vector database URL'),
(gen_random_uuid()::text, 'CACHE_URL', 'redis://localhost:6379', 'string', 'Redis cache URL'),
(gen_random_uuid()::text, 'STACK_PATH', './botserver-stack', 'string', 'Base path for all components'),
(gen_random_uuid()::text, 'SITES_ROOT', './botserver-stack/sites', 'string', 'Root path for sites')
ON CONFLICT (config_key) DO NOTHING;
-- ============================================================================
-- DEFAULT TENANT
-- Create default tenant for single-tenant installations
-- ============================================================================
INSERT INTO tenants (id, name, slug, is_active) VALUES INSERT INTO tenants (id, name, slug, is_active) VALUES
(gen_random_uuid(), 'Default Tenant', 'default', true) (gen_random_uuid(), 'Default Tenant', 'default', true)
ON CONFLICT (slug) DO NOTHING; ON CONFLICT (slug) DO NOTHING;

View file

@ -1,11 +0,0 @@
-- Migration 6.0.6: Add LLM configuration defaults
-- Description: Configure local LLM server settings with model paths
-- Insert LLM configuration with defaults
INSERT INTO server_configuration (id, config_key, config_value, config_type, description) VALUES
(gen_random_uuid()::text, 'LLM_LOCAL', 'true', 'boolean', 'Enable local LLM server'),
(gen_random_uuid()::text, 'LLM_MODEL_PATH', 'botserver-stack/data/llm/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf', 'string', 'Path to LLM model file'),
(gen_random_uuid()::text, 'LLM_URL', 'http://localhost:8081', 'string', 'Local LLM server URL'),
(gen_random_uuid()::text, 'EMBEDDING_MODEL_PATH', 'botserver-stack/data/llm/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf', 'string', 'Path to embedding model file'),
(gen_random_uuid()::text, 'EMBEDDING_URL', 'http://localhost:8082', 'string', 'Embedding server URL')
ON CONFLICT (config_key) DO NOTHING;

View file

@ -1,11 +1,11 @@
use actix_web::{web, HttpResponse, Result}; use actix_web::{HttpRequest, HttpResponse, Result, web};
use argon2::{ use argon2::{
password_hash::{rand_core::OsRng, PasswordHash, PasswordHasher, PasswordVerifier, SaltString}, password_hash::{rand_core::OsRng, PasswordHash, PasswordHasher, PasswordVerifier, SaltString},
Argon2, Argon2,
}; };
use diesel::pg::PgConnection; use diesel::pg::PgConnection;
use diesel::prelude::*; use diesel::prelude::*;
use log::{error, warn}; use log::{error};
use redis::Client; use redis::Client;
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
@ -148,6 +148,7 @@ impl AuthService {
#[actix_web::get("/api/auth")] #[actix_web::get("/api/auth")]
async fn auth_handler( async fn auth_handler(
req: HttpRequest,
data: web::Data<AppState>, data: web::Data<AppState>,
web::Query(params): web::Query<HashMap<String, String>>, web::Query(params): web::Query<HashMap<String, String>>,
) -> Result<HttpResponse> { ) -> Result<HttpResponse> {
@ -166,45 +167,10 @@ async fn auth_handler(
} }
}; };
let bot_id = if let Ok(bot_guid) = std::env::var("BOT_GUID") { let mut db_conn = data.conn.lock().unwrap();
match Uuid::parse_str(&bot_guid) { let (bot_id, bot_name) = match crate::bot::bot_from_url(&mut *db_conn, req.path()) {
Ok(uuid) => uuid, Ok((id, name)) => (id, name),
Err(e) => { Err(res) => return Ok(res),
warn!("Invalid BOT_GUID from env: {}", e);
return Ok(HttpResponse::BadRequest()
.json(serde_json::json!({"error": "Invalid BOT_GUID"})));
}
}
} else {
// BOT_GUID not set, get first available bot from database
use crate::shared::models::schema::bots::dsl::*;
use diesel::prelude::*;
let mut db_conn = data.conn.lock().unwrap();
match bots
.filter(is_active.eq(true))
.select(id)
.first::<Uuid>(&mut *db_conn)
.optional()
{
Ok(Some(first_bot_id)) => {
log::info!(
"BOT_GUID not set, using first available bot: {}",
first_bot_id
);
first_bot_id
}
Ok(None) => {
error!("No active bots found in database");
return Ok(HttpResponse::ServiceUnavailable()
.json(serde_json::json!({"error": "No bots available"})));
}
Err(e) => {
error!("Failed to query bots: {}", e);
return Ok(HttpResponse::InternalServerError()
.json(serde_json::json!({"error": "Failed to query bots"})));
}
}
}; };
let session = { let session = {
@ -224,35 +190,40 @@ async fn auth_handler(
} }
}; };
let session_id_clone = session.id.clone(); let auth_script_path = format!("./work/{}.gbai/{}.gbdialog/auth.ast", bot_name, bot_name);
let auth_script_path = "./templates/annoucements.gbai/annoucements.gbdialog/auth.bas"; if std::path::Path::new(&auth_script_path).exists() {
let auth_script = match std::fs::read_to_string(auth_script_path) { let auth_script = match std::fs::read_to_string(&auth_script_path) {
Ok(content) => content, Ok(content) => content,
Err(_) => r#"SET_USER "00000000-0000-0000-0000-000000000001""#.to_string(), Err(e) => {
}; error!("Failed to read auth script: {}", e);
return Ok(HttpResponse::InternalServerError()
let script_service = crate::basic::ScriptService::new(Arc::clone(&data), session.clone()); .json(serde_json::json!({"error": "Failed to read auth script"})));
match script_service }
.compile(&auth_script) };
.and_then(|ast| script_service.run(&ast))
{ let script_service = crate::basic::ScriptService::new(Arc::clone(&data), session.clone());
Ok(result) => { match script_service
if result.to_string() == "false" { .compile(&auth_script)
error!("Auth script returned false, authentication failed"); .and_then(|ast| script_service.run(&ast))
return Ok(HttpResponse::Unauthorized() {
.json(serde_json::json!({"error": "Authentication failed"}))); Ok(result) => {
if result.to_string() == "false" {
error!("Auth script returned false, authentication failed");
return Ok(HttpResponse::Unauthorized()
.json(serde_json::json!({"error": "Authentication failed"})));
}
}
Err(e) => {
error!("Failed to run auth script: {}", e);
return Ok(HttpResponse::InternalServerError()
.json(serde_json::json!({"error": "Auth failed"})));
} }
}
Err(e) => {
error!("Failed to run auth script: {}", e);
return Ok(HttpResponse::InternalServerError()
.json(serde_json::json!({"error": "Auth failed"})));
} }
} }
let session = { let session = {
let mut sm = data.session_manager.lock().await; let mut sm = data.session_manager.lock().await;
match sm.get_session_by_id(session_id_clone) { match sm.get_session_by_id(session.id) {
Ok(Some(s)) => s, Ok(Some(s)) => s,
Ok(None) => { Ok(None) => {
error!("Failed to retrieve session"); error!("Failed to retrieve session");

View file

@ -331,7 +331,7 @@ impl AutomationService {
e e
); );
if let Some(s3_operator) = &self.state.s3_operator { if let Some(client) = &self.state.s3_client {
let bucket_name = format!( let bucket_name = format!(
"{}{}.gbai", "{}{}.gbai",
env::var("MINIO_ORG_PREFIX").unwrap_or_else(|_| "org1_".to_string()), env::var("MINIO_ORG_PREFIX").unwrap_or_else(|_| "org1_".to_string()),
@ -341,10 +341,9 @@ impl AutomationService {
trace!("Downloading from bucket={} key={}", bucket_name, s3_key); trace!("Downloading from bucket={} key={}", bucket_name, s3_key);
match s3_operator.read(&format!("{}/{}", bucket_name, s3_key)).await { match crate::kb::minio_handler::get_file_content(client, &bucket_name, &s3_key).await {
Ok(data) => { Ok(data) => {
let bytes: Vec<u8> = data.to_vec(); match String::from_utf8(data) {
match String::from_utf8(bytes) {
Ok(content) => { Ok(content) => {
info!("Downloaded script '{}' from MinIO", param); info!("Downloaded script '{}' from MinIO", param);

View file

@ -2,6 +2,7 @@ use crate::shared::models::UserSession;
use crate::shared::state::AppState; use crate::shared::state::AppState;
use log::{debug, error, info}; use log::{debug, error, info};
use reqwest::{self, Client}; use reqwest::{self, Client};
use crate::kb::minio_handler;
use rhai::{Dynamic, Engine}; use rhai::{Dynamic, Engine};
use std::error::Error; use std::error::Error;
use std::path::Path; use std::path::Path;
@ -158,13 +159,7 @@ pub async fn get_from_bucket(
return Err("Invalid file path".into()); return Err("Invalid file path".into());
} }
let s3_operator = match &state.s3_operator { let client = state.s3_client.as_ref().ok_or("S3 client not configured")?;
Some(operator) => operator,
None => {
error!("S3 operator not configured");
return Err("S3 operator not configured".into());
}
};
let bucket_name = { let bucket_name = {
let cfg = state let cfg = state
@ -187,11 +182,11 @@ pub async fn get_from_bucket(
bucket bucket
}; };
let response = match tokio::time::timeout( let bytes = match tokio::time::timeout(
Duration::from_secs(30), Duration::from_secs(30),
s3_operator.read(&format!("{}/{}", bucket_name, file_path)) minio_handler::get_file_content(client, &bucket_name, file_path)
).await { ).await {
Ok(Ok(response)) => response, Ok(Ok(data)) => data,
Ok(Err(e)) => { Ok(Err(e)) => {
error!("S3 read failed: {}", e); error!("S3 read failed: {}", e);
return Err(format!("S3 operation failed: {}", e).into()); return Err(format!("S3 operation failed: {}", e).into());
@ -202,15 +197,7 @@ pub async fn get_from_bucket(
} }
}; };
let bytes = response.to_vec();
debug!(
"Retrieved {} bytes from S3 for key: {}",
bytes.len(),
file_path
);
let content = if file_path.to_ascii_lowercase().ends_with(".pdf") { let content = if file_path.to_ascii_lowercase().ends_with(".pdf") {
debug!("Processing as PDF file: {}", file_path);
match pdf_extract::extract_text_from_mem(&bytes) { match pdf_extract::extract_text_from_mem(&bytes) {
Ok(text) => text, Ok(text) => text,
Err(e) => { Err(e) => {

View file

@ -1,20 +1,25 @@
use crate::config::AppConfig; use crate::config::AppConfig;
use crate::package_manager::{InstallMode, PackageManager}; use crate::package_manager::{InstallMode, PackageManager};
use anyhow::Result; use anyhow::Result;
use diesel::connection::SimpleConnection; use diesel::{connection::SimpleConnection, RunQueryDsl, Connection, QueryableByName};
use diesel::RunQueryDsl;
use diesel::{Connection, QueryableByName};
use dotenvy::dotenv; use dotenvy::dotenv;
use log::{debug, error, info, trace}; use log::{debug, error, info, trace};
use opendal::Operator; use aws_sdk_s3::Client;
use aws_config::BehaviorVersion;
use rand::distr::Alphanumeric; use rand::distr::Alphanumeric;
use rand::Rng; use rand::Rng;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use std::io::{self, Write}; use std::io::{self, Write};
use std::path::Path; use std::path::Path;
use std::process::Command; use std::process::Command;
use std::sync::{Arc, Mutex};
use diesel::Queryable;
#[derive(QueryableByName)] #[derive(QueryableByName)]
#[diesel(check_for_backend(diesel::pg::Pg))]
#[derive(Queryable)]
#[diesel(table_name = crate::shared::models::schema::bots)]
struct BotIdRow { struct BotIdRow {
#[diesel(sql_type = diesel::sql_types::Uuid)] #[diesel(sql_type = diesel::sql_types::Uuid)]
id: uuid::Uuid, id: uuid::Uuid,
@ -28,21 +33,21 @@ pub struct ComponentInfo {
pub struct BootstrapManager { pub struct BootstrapManager {
pub install_mode: InstallMode, pub install_mode: InstallMode,
pub tenant: Option<String>, pub tenant: Option<String>,
pub s3_operator: Operator, pub s3_client: Client,
} }
impl BootstrapManager { impl BootstrapManager {
pub fn new(install_mode: InstallMode, tenant: Option<String>) -> Self { pub async fn new(install_mode: InstallMode, tenant: Option<String>) -> Self {
info!( info!(
"Initializing BootstrapManager with mode {:?} and tenant {:?}", "Initializing BootstrapManager with mode {:?} and tenant {:?}",
install_mode, tenant install_mode, tenant
); );
let config = AppConfig::from_env(); let config = AppConfig::from_env();
let s3_operator = Self::create_s3_operator(&config); let s3_client = futures::executor::block_on(Self::create_s3_operator(&config));
Self { Self {
install_mode, install_mode,
tenant, tenant,
s3_operator, s3_client,
} }
} }
@ -140,8 +145,8 @@ impl BootstrapManager {
let mut conn = diesel::pg::PgConnection::establish(&database_url) let mut conn = diesel::pg::PgConnection::establish(&database_url)
.map_err(|e| anyhow::anyhow!("Failed to connect to database: {}", e))?; .map_err(|e| anyhow::anyhow!("Failed to connect to database: {}", e))?;
let default_bot_id: uuid::Uuid = diesel::sql_query("SELECT id FROM bots LIMIT 1") let default_bot_id: uuid::Uuid = diesel::sql_query("SELECT id FROM bots LIMIT 1")
.get_result::<BotIdRow>(&mut conn) .load::<BotIdRow>(&mut conn)
.map(|row| row.id) .map(|rows| rows.first().map(|r| r.id).unwrap_or_else(|| uuid::Uuid::new_v4()))
.unwrap_or_else(|_| uuid::Uuid::new_v4()); .unwrap_or_else(|_| uuid::Uuid::new_v4());
if let Err(e) = self.update_bot_config(&default_bot_id, component.name) { if let Err(e) = self.update_bot_config(&default_bot_id, component.name) {
@ -156,7 +161,8 @@ impl BootstrapManager {
Ok(()) Ok(())
} }
pub fn bootstrap(&mut self) -> Result<AppConfig> { pub async fn bootstrap(&mut self) -> Result<AppConfig> {
// First check for legacy mode
if let Ok(tables_server) = std::env::var("TABLES_SERVER") { if let Ok(tables_server) = std::env::var("TABLES_SERVER") {
if !tables_server.is_empty() { if !tables_server.is_empty() {
info!( info!(
@ -164,7 +170,7 @@ impl BootstrapManager {
); );
let database_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| { let database_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
let username = let username =
std::env::var("TABLES_USERNAME").unwrap_or_else(|_| "postgres".to_string()); std::env::var("TABLES_USERNAME").unwrap_or_else(|_| "gbuser".to_string());
let password = let password =
std::env::var("TABLES_PASSWORD").unwrap_or_else(|_| "postgres".to_string()); std::env::var("TABLES_PASSWORD").unwrap_or_else(|_| "postgres".to_string());
let server = let server =
@ -178,6 +184,11 @@ impl BootstrapManager {
) )
}); });
// In legacy mode, still try to load config.csv if available
if let Ok(config) = self.load_config_from_csv().await {
return Ok(config);
}
match diesel::PgConnection::establish(&database_url) { match diesel::PgConnection::establish(&database_url) {
Ok(mut conn) => { Ok(mut conn) => {
if let Err(e) = self.apply_migrations(&mut conn) { if let Err(e) = self.apply_migrations(&mut conn) {
@ -292,47 +303,50 @@ impl BootstrapManager {
} }
} }
self.s3_operator = Self::create_s3_operator(&config); self.s3_client = futures::executor::block_on(Self::create_s3_operator(&config));
let default_bucket_path = Path::new("templates/default.gbai/default.gbot/config.csv");
if default_bucket_path.exists() { // Load config from CSV if available
info!("Found initial config.csv, uploading to default.gbai/default.gbot"); if let Ok(csv_config) = self.load_config_from_csv().await {
let operator = &self.s3_operator; Ok(csv_config)
futures::executor::block_on(async { } else {
let content = std::fs::read(default_bucket_path).expect("Failed to read config.csv"); Ok(config)
operator.write("default.gbai/default.gbot/config.csv", content).await
})?;
debug!("Initial config.csv uploaded successfully");
} }
Ok(config)
} }
fn create_s3_operator(config: &AppConfig) -> Operator {
use opendal::Scheme;
use std::collections::HashMap;
let mut endpoint = config.drive.server.clone();
if !endpoint.ends_with('/') {
endpoint.push('/');
}
let mut map = HashMap::new();
map.insert("endpoint".to_string(), endpoint);
map.insert("access_key_id".to_string(), config.drive.access_key.clone());
map.insert(
"secret_access_key".to_string(),
config.drive.secret_key.clone(),
);
map.insert(
"bucket".to_string(),
format!("default.gbai"),
);
map.insert("region".to_string(), "auto".to_string());
map.insert("force_path_style".to_string(), "true".to_string());
trace!("Creating S3 operator with endpoint {}", config.drive.server); async fn create_s3_operator(config: &AppConfig) -> Client {
let endpoint = if !config.drive.server.ends_with('/') {
format!("{}/", config.drive.server)
} else {
config.drive.server.clone()
};
Operator::via_iter(Scheme::S3, map).expect("Failed to initialize S3 operator") let base_config = aws_config::defaults(BehaviorVersion::latest())
.endpoint_url(endpoint)
.region("auto")
.credentials_provider(
aws_sdk_s3::config::Credentials::new(
config.drive.access_key.clone(),
config.drive.secret_key.clone(),
None,
None,
"static",
)
)
.load()
.await;
let s3_config = aws_sdk_s3::config::Builder::from(&base_config)
.force_path_style(true)
.build();
aws_sdk_s3::Client::from_conf(s3_config)
} }
fn generate_secure_password(&self, length: usize) -> String { fn generate_secure_password(&self, length: usize) -> String {
let mut rng = rand::rng(); let mut rng = rand::rng();
std::iter::repeat_with(|| rng.sample(Alphanumeric) as char) std::iter::repeat_with(|| rng.sample(Alphanumeric) as char)
@ -381,7 +395,7 @@ impl BootstrapManager {
if !templates_dir.exists() { if !templates_dir.exists() {
return Ok(()); return Ok(());
} }
let operator = &self.s3_operator; let client = &self.s3_client;
for entry in std::fs::read_dir(templates_dir)? { for entry in std::fs::read_dir(templates_dir)? {
let entry = entry?; let entry = entry?;
let path = entry.path(); let path = entry.path();
@ -395,27 +409,30 @@ impl BootstrapManager {
let bot_name = path.file_name().unwrap().to_string_lossy().to_string(); let bot_name = path.file_name().unwrap().to_string_lossy().to_string();
let bucket = bot_name.trim_start_matches('/').to_string(); let bucket = bot_name.trim_start_matches('/').to_string();
info!("Uploading template {} to Drive bucket {}", bot_name, bucket); info!("Uploading template {} to Drive bucket {}", bot_name, bucket);
if operator.stat(&bucket).await.is_err() {
info!("Bucket {} not found, creating it", bucket); // Check if bucket exists
let bucket_path = if bucket.ends_with('/') { bucket.clone() } else { format!("{}/", bucket) }; if client.head_bucket().bucket(&bucket).send().await.is_err() {
match operator.create_dir(&bucket_path).await { info!("Bucket {} not found, creating it", bucket);
Ok(_) => { match client.create_bucket()
debug!("Bucket {} created successfully", bucket); .bucket(&bucket)
} .send()
Err(e) => { .await {
let err_msg = format!("{}", e); Ok(_) => {
if err_msg.contains("BucketAlreadyOwnedByYou") { debug!("Bucket {} created successfully", bucket);
log::warn!("Bucket {} already exists, reusing default.gbai", bucket); }
self.upload_directory_recursive(&operator, &Path::new("templates/default.gbai"), "default.gbai").await?; Err(e) => {
continue; error!("Failed to create bucket {}: {:?}", bucket, e);
} else { return Err(anyhow::anyhow!(
return Err(e.into()); "Failed to create bucket {}: {}. Check S3 credentials and endpoint configuration",
} bucket, e
} ));
} }
} }
self.upload_directory_recursive(&operator, &path, &bucket).await?; }
info!("Uploaded template {} to Drive bucket {}", bot_name, bucket);
self.upload_directory_recursive(client, &path, &bucket, "/")
.await?;
info!("Uploaded template {} to Drive bucket {}", bot_name, bucket);
} }
} }
Ok(()) Ok(())
@ -436,22 +453,9 @@ info!("Uploaded template {} to Drive bucket {}", bot_name, bucket);
if path.is_dir() && path.extension().map(|e| e == "gbai").unwrap_or(false) { if path.is_dir() && path.extension().map(|e| e == "gbai").unwrap_or(false) {
let bot_folder = path.file_name().unwrap().to_string_lossy().to_string(); let bot_folder = path.file_name().unwrap().to_string_lossy().to_string();
let bot_name = bot_folder.trim_end_matches(".gbai"); let bot_name = bot_folder.trim_end_matches(".gbai");
let formatted_name = bot_name
.split('_')
.map(|word| {
let mut chars = word.chars();
match chars.next() {
None => String::new(),
Some(first) => {
first.to_uppercase().collect::<String>() + chars.as_str()
}
}
})
.collect::<Vec<_>>()
.join(" ");
let existing: Option<String> = bots::table let existing: Option<String> = bots::table
.filter(bots::name.eq(&formatted_name)) .filter(bots::name.eq(&bot_name))
.select(bots::name) .select(bots::name)
.first(conn) .first(conn)
.optional()?; .optional()?;
@ -461,11 +465,11 @@ info!("Uploaded template {} to Drive bucket {}", bot_name, bucket);
"INSERT INTO bots (id, name, description, llm_provider, llm_config, context_provider, context_config, is_active) \ "INSERT INTO bots (id, name, description, llm_provider, llm_config, context_provider, context_config, is_active) \
VALUES (gen_random_uuid(), $1, $2, 'openai', '{\"model\": \"gpt-4\", \"temperature\": 0.7}', 'database', '{}', true)" VALUES (gen_random_uuid(), $1, $2, 'openai', '{\"model\": \"gpt-4\", \"temperature\": 0.7}', 'database', '{}', true)"
) )
.bind::<diesel::sql_types::Text, _>(&formatted_name) .bind::<diesel::sql_types::Text, _>(&bot_name)
.bind::<diesel::sql_types::Text, _>(format!("Bot for {} template", bot_name)) .bind::<diesel::sql_types::Text, _>(format!("Bot for {} template", bot_name))
.execute(conn)?; .execute(conn)?;
} else { } else {
log::trace!("Bot {} already exists", formatted_name); log::trace!("Bot {} already exists", bot_name);
} }
} }
} }
@ -475,8 +479,9 @@ info!("Uploaded template {} to Drive bucket {}", bot_name, bucket);
fn upload_directory_recursive<'a>( fn upload_directory_recursive<'a>(
&'a self, &'a self,
client: &'a Operator, client: &'a Client,
local_path: &'a Path, local_path: &'a Path,
bucket: &'a str,
prefix: &'a str, prefix: &'a str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> { ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {
Box::pin(async move { Box::pin(async move {
@ -490,26 +495,79 @@ info!("Uploaded template {} to Drive bucket {}", bot_name, bucket);
let entry = entry?; let entry = entry?;
let path = entry.path(); let path = entry.path();
let file_name = path.file_name().unwrap().to_string_lossy().to_string(); let file_name = path.file_name().unwrap().to_string_lossy().to_string();
let key = if prefix.is_empty() {
file_name.clone() // Construct key path, ensuring no duplicate slashes
} else { let mut key = prefix.trim_matches('/').to_string();
format!("{}/{}", prefix.trim_end_matches('/'), file_name) if !key.is_empty() {
}; key.push('/');
}
key.push_str(&file_name);
if path.is_file() { if path.is_file() {
info!("Uploading file: {} with key: {}", path.display(), key); info!("Uploading file: {} to bucket {} with key: {}",
path.display(), bucket, key);
let content = std::fs::read(&path)?; let content = std::fs::read(&path)?;
trace!("Writing file {} with key {}", path.display(), key); client.put_object()
client.write(&key, content).await?; .bucket(bucket)
trace!("Successfully wrote file {}", path.display()); .key(&key)
.body(content.into())
.send()
.await?;
} else if path.is_dir() { } else if path.is_dir() {
self.upload_directory_recursive(client, &path, &key).await?; self.upload_directory_recursive(client, &path, bucket, &key).await?;
} }
} }
Ok(()) Ok(())
}) })
} }
async fn load_config_from_csv(&self) -> Result<AppConfig> {
use crate::config::ConfigManager;
use uuid::Uuid;
let client = &self.s3_client;
let bucket = "default.gbai";
let config_key = "default.gbot/config.csv";
match client.get_object()
.bucket(bucket)
.key(config_key)
.send()
.await
{
Ok(response) => {
let bytes = response.body.collect().await?.into_bytes();
let csv_content = String::from_utf8(bytes.to_vec())?;
let database_url = std::env::var("DATABASE_URL")
.unwrap_or_else(|_| "postgres://gbuser:@localhost:5432/botserver".to_string());
// Create new connection for config loading
let config_conn = diesel::PgConnection::establish(&database_url)?;
let config_manager = ConfigManager::new(Arc::new(Mutex::new(config_conn)));
// Use default bot ID or create one if needed
let default_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?;
// Write CSV to temp file for ConfigManager
let temp_path = std::env::temp_dir().join("config.csv");
std::fs::write(&temp_path, csv_content)?;
config_manager.sync_gbot_config(&default_bot_id, temp_path.to_str().unwrap())
.map_err(|e| anyhow::anyhow!("Failed to sync gbot config: {}", e))?;
// Load config from database which now has the CSV values
let mut config_conn = diesel::PgConnection::establish(&database_url)?;
let config = AppConfig::from_database(&mut config_conn);
info!("Successfully loaded config from CSV");
Ok(config)
}
Err(e) => {
debug!("No config.csv found: {}", e);
Err(e.into())
}
}
}
fn apply_migrations(&self, conn: &mut diesel::PgConnection) -> Result<()> { fn apply_migrations(&self, conn: &mut diesel::PgConnection) -> Result<()> {
let migrations_dir = std::path::Path::new("migrations"); let migrations_dir = std::path::Path::new("migrations");
if !migrations_dir.exists() { if !migrations_dir.exists() {

View file

@ -3,6 +3,7 @@ use crate::shared::models::{BotResponse, UserMessage, UserSession};
use crate::shared::state::AppState; use crate::shared::state::AppState;
use actix_web::{web, HttpRequest, HttpResponse, Result}; use actix_web::{web, HttpRequest, HttpResponse, Result};
use actix_ws::Message as WsMessage; use actix_ws::Message as WsMessage;
use diesel::PgConnection;
use log::{debug, error, info, warn}; use log::{debug, error, info, warn};
use chrono::Utc; use chrono::Utc;
use serde_json; use serde_json;
@ -11,18 +12,140 @@ use std::sync::Arc;
use tokio::sync::mpsc; use tokio::sync::mpsc;
use crate::kb::embeddings::generate_embeddings; use crate::kb::embeddings::generate_embeddings;
use uuid::Uuid; use uuid::Uuid;
use crate::kb::qdrant_client::{ensure_collection_exists, get_qdrant_client, QdrantPoint}; use crate::kb::qdrant_client::{ensure_collection_exists, get_qdrant_client, QdrantPoint};
use crate::context::langcache::{get_langcache_client}; use crate::context::langcache::get_langcache_client;
use crate::drive_monitor::DriveMonitor;
use tokio::sync::Mutex as AsyncMutex;
pub struct BotOrchestrator { pub struct BotOrchestrator {
pub state: Arc<AppState>, pub state: Arc<AppState>,
pub mounted_bots: Arc<AsyncMutex<HashMap<String, Arc<DriveMonitor>>>>,
} }
impl BotOrchestrator { impl BotOrchestrator {
pub fn new(state: Arc<AppState>) -> Self { pub fn new(state: Arc<AppState>) -> Self {
Self { state } Self {
state,
mounted_bots: Arc::new(AsyncMutex::new(HashMap::new())),
}
}
pub async fn mount_all_bots(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
use crate::shared::models::schema::bots::dsl::*;
use diesel::prelude::*;
let mut db_conn = self.state.conn.lock().unwrap();
let active_bots = bots
.filter(is_active.eq(true))
.select(id)
.load::<uuid::Uuid>(&mut *db_conn)
.map_err(|e| {
error!("Failed to query active bots: {}", e);
e
})?;
for bot_guid in active_bots {
let state_clone = self.state.clone();
let mounted_bots_clone = self.mounted_bots.clone();
let bot_guid_str = bot_guid.to_string();
tokio::spawn(async move {
if let Err(e) = Self::mount_bot_task(state_clone, mounted_bots_clone, bot_guid_str.clone()).await {
error!("Failed to mount bot {}: {}", bot_guid_str, e);
}
});
}
Ok(())
}
async fn mount_bot_task(
state: Arc<AppState>,
mounted_bots: Arc<AsyncMutex<HashMap<String, Arc<DriveMonitor>>>>,
bot_guid: String,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
use diesel::prelude::*;
use crate::shared::models::schema::bots::dsl::*;
let bot_name: String = {
let mut db_conn = state.conn.lock().unwrap();
bots
.filter(id.eq(Uuid::parse_str(&bot_guid)?))
.select(name)
.first(&mut *db_conn)
.map_err(|e| {
error!("Failed to query bot name for {}: {}", bot_guid, e);
e
})?
};
let bucket_name = format!("{}.gbai", bot_name);
{
let mounted = mounted_bots.lock().await;
if mounted.contains_key(&bot_guid) {
warn!("Bot {} is already mounted", bot_guid);
return Ok(());
}
}
let drive_monitor = Arc::new(DriveMonitor::new(state.clone(), bucket_name));
let _handle = drive_monitor.clone().spawn().await;
{
let mut mounted = mounted_bots.lock().await;
mounted.insert(bot_guid.clone(), drive_monitor);
}
info!("Bot {} mounted successfully", bot_guid);
Ok(())
}
pub async fn create_bot(&self, bot_guid: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let bucket_name = format!("{}{}.gbai", self.state.config.as_ref().unwrap().drive.org_prefix, bot_guid);
crate::create_bucket::create_bucket(&bucket_name)?;
Ok(())
}
pub async fn mount_bot(&self, bot_guid: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let bot_guid = bot_guid.strip_suffix(".gbai").unwrap_or(bot_guid).to_string();
use diesel::prelude::*;
use crate::shared::models::schema::bots::dsl::*;
let bot_name: String = {
let mut db_conn = self.state.conn.lock().unwrap();
bots
.filter(id.eq(Uuid::parse_str(&bot_guid)?))
.select(name)
.first(&mut *db_conn)
.map_err(|e| {
error!("Failed to query bot name for {}: {}", bot_guid, e);
e
})?
};
let bucket_name = format!("{}.gbai", bot_name);
{
let mounted_bots = self.mounted_bots.lock().await;
if mounted_bots.contains_key(&bot_guid) {
warn!("Bot {} is already mounted", bot_guid);
return Ok(());
}
}
let drive_monitor = Arc::new(DriveMonitor::new(self.state.clone(), bucket_name));
let _handle = drive_monitor.clone().spawn().await;
{
let mut mounted_bots = self.mounted_bots.lock().await;
mounted_bots.insert(bot_guid.clone(), drive_monitor);
}
Ok(())
} }
pub async fn handle_user_input( pub async fn handle_user_input(
@ -30,10 +153,7 @@ impl BotOrchestrator {
session_id: Uuid, session_id: Uuid,
user_input: &str, user_input: &str,
) -> Result<Option<String>, Box<dyn std::error::Error + Send + Sync>> { ) -> Result<Option<String>, Box<dyn std::error::Error + Send + Sync>> {
info!( info!("Handling user input for session {}: '{}'", session_id, user_input);
"Handling user input for session {}: '{}'",
session_id, user_input
);
let mut session_manager = self.state.session_manager.lock().await; let mut session_manager = self.state.session_manager.lock().await;
session_manager.provide_input(session_id, user_input.to_string())?; session_manager.provide_input(session_id, user_input.to_string())?;
Ok(None) Ok(None)
@ -74,10 +194,7 @@ impl BotOrchestrator {
bot_id: &str, bot_id: &str,
mode: i32, mode: i32,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!( info!("Setting answer mode for user {} with bot {} to mode {}", user_id, bot_id, mode);
"Setting answer mode for user {} with bot {} to mode {}",
user_id, bot_id, mode
);
let mut session_manager = self.state.session_manager.lock().await; let mut session_manager = self.state.session_manager.lock().await;
session_manager.update_answer_mode(user_id, bot_id, mode)?; session_manager.update_answer_mode(user_id, bot_id, mode)?;
Ok(()) Ok(())
@ -92,10 +209,7 @@ impl BotOrchestrator {
event_type: &str, event_type: &str,
data: serde_json::Value, data: serde_json::Value,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!( info!("Sending event '{}' to session {} on channel {}", event_type, session_id, channel);
"Sending event '{}' to session {} on channel {}",
event_type, session_id, channel
);
let event_response = BotResponse { let event_response = BotResponse {
bot_id: bot_id.to_string(), bot_id: bot_id.to_string(),
user_id: user_id.to_string(), user_id: user_id.to_string(),
@ -113,8 +227,9 @@ impl BotOrchestrator {
if let Some(adapter) = self.state.channels.lock().unwrap().get(channel) { if let Some(adapter) = self.state.channels.lock().unwrap().get(channel) {
adapter.send_message(event_response).await?; adapter.send_message(event_response).await?;
} else { } else {
warn!("No channel adapter found for channel 1: {}", channel); warn!("No channel adapter found for channel: {}", channel);
} }
Ok(()) Ok(())
} }
@ -124,10 +239,7 @@ impl BotOrchestrator {
channel: &str, channel: &str,
content: &str, content: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!( info!("Sending direct message to session {}: '{}'", session_id, content);
"Sending direct message to session {}: '{}'",
session_id, content
);
let bot_response = BotResponse { let bot_response = BotResponse {
bot_id: "default_bot".to_string(), bot_id: "default_bot".to_string(),
user_id: "default_user".to_string(), user_id: "default_user".to_string(),
@ -142,8 +254,9 @@ impl BotOrchestrator {
if let Some(adapter) = self.state.channels.lock().unwrap().get(channel) { if let Some(adapter) = self.state.channels.lock().unwrap().get(channel) {
adapter.send_message(bot_response).await?; adapter.send_message(bot_response).await?;
} else { } else {
warn!("No channel adapter found for channel 2: {}", channel); warn!("No channel adapter found for direct message on channel: {}", channel);
} }
Ok(()) Ok(())
} }
@ -151,60 +264,35 @@ impl BotOrchestrator {
&self, &self,
message: UserMessage, message: UserMessage,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!( info!("Processing message from channel: {}, user: {}, session: {}", message.channel, message.user_id, message.session_id);
"Processing message from channel: {}, user: {}, session: {}", debug!("Message content: '{}', type: {}", message.content, message.message_type);
message.channel, message.user_id, message.session_id
);
debug!(
"Message content: '{}', type: {}",
message.content, message.message_type
);
let user_id = Uuid::parse_str(&message.user_id).map_err(|e| { let user_id = Uuid::parse_str(&message.user_id).map_err(|e| {
error!("Invalid user ID provided: {}", e); error!("Invalid user ID provided: {}", e);
e e
})?; })?;
let bot_id = if let Ok(bot_guid) = std::env::var("BOT_GUID") { let bot_id = Uuid::nil();
Uuid::parse_str(&bot_guid).map_err(|e| {
warn!("Invalid BOT_GUID from env: {}", e);
e
})?
} else {
warn!("BOT_GUID not set in environment, using nil UUID");
Uuid::nil()
};
let session = { let session = {
let mut sm = self.state.session_manager.lock().await; let mut sm = self.state.session_manager.lock().await;
let session_id = Uuid::parse_str(&message.session_id).map_err(|e| { let session_id = Uuid::parse_str(&message.session_id).map_err(|e| {
error!("Invalid session ID: {}", e); error!("Invalid session ID: {}", e);
e e
})?; })?;
match sm.get_session_by_id(session_id)? { match sm.get_session_by_id(session_id)? {
Some(session) => session, Some(session) => session,
None => { None => {
error!( error!("Failed to create session for user {} with bot {}", user_id, bot_id);
"Failed to create session for user {} with bot {}",
user_id, bot_id
);
return Err("Failed to create session".into()); return Err("Failed to create session".into());
} }
} }
}; };
if self.is_waiting_for_input(session.id).await { if self.is_waiting_for_input(session.id).await {
debug!( debug!("Session {} is waiting for input, processing as variable input", session.id);
"Session {} is waiting for input, processing as variable input", if let Some(variable_name) = self.handle_user_input(session.id, &message.content).await? {
session.id info!("Stored user input in variable '{}' for session {}", variable_name, session.id);
);
if let Some(variable_name) =
self.handle_user_input(session.id, &message.content).await?
{
debug!(
"Stored user input in variable '{}' for session {}",
variable_name, session.id
);
if let Some(adapter) = self.state.channels.lock().unwrap().get(&message.channel) { if let Some(adapter) = self.state.channels.lock().unwrap().get(&message.channel) {
let ack_response = BotResponse { let ack_response = BotResponse {
bot_id: message.bot_id.clone(), bot_id: message.bot_id.clone(),
@ -263,10 +351,7 @@ impl BotOrchestrator {
if let Some(adapter) = self.state.channels.lock().unwrap().get(&message.channel) { if let Some(adapter) = self.state.channels.lock().unwrap().get(&message.channel) {
adapter.send_message(bot_response).await?; adapter.send_message(bot_response).await?;
} else { } else {
warn!( warn!("No channel adapter found for message channel: {}", message.channel);
"No channel adapter found for channel 3: {}",
message.channel
);
} }
Ok(()) Ok(())
@ -298,7 +383,6 @@ impl BotOrchestrator {
session_manager.get_conversation_history(session.id, session.user_id)? session_manager.get_conversation_history(session.id, session.user_id)?
}; };
// Prompt compactor: keep only last 10 entries
let recent_history = if history.len() > 10 { let recent_history = if history.len() > 10 {
&history[history.len() - 10..] &history[history.len() - 10..]
} else { } else {
@ -308,31 +392,23 @@ impl BotOrchestrator {
for (role, content) in recent_history { for (role, content) in recent_history {
prompt.push_str(&format!("{}: {}\n", role, content)); prompt.push_str(&format!("{}: {}\n", role, content));
} }
prompt.push_str(&format!("User: {}\nAssistant:", message.content)); prompt.push_str(&format!("User: {}\nAssistant:", message.content));
// Determine which cache backend to use
let use_langcache = std::env::var("LLM_CACHE") let use_langcache = std::env::var("LLM_CACHE")
.unwrap_or_else(|_| "false".to_string()) .unwrap_or_else(|_| "false".to_string())
.eq_ignore_ascii_case("true"); .eq_ignore_ascii_case("true");
if use_langcache { if use_langcache {
// Ensure LangCache collection exists
ensure_collection_exists(&self.state, "semantic_cache").await?; ensure_collection_exists(&self.state, "semantic_cache").await?;
// Get LangCache client
let langcache_client = get_langcache_client()?; let langcache_client = get_langcache_client()?;
// Isolate the user question (ignore conversation history)
let isolated_question = message.content.trim().to_string(); let isolated_question = message.content.trim().to_string();
// Generate embedding for the isolated question
let question_embeddings = generate_embeddings(vec![isolated_question.clone()]).await?; let question_embeddings = generate_embeddings(vec![isolated_question.clone()]).await?;
let question_embedding = question_embeddings let question_embedding = question_embeddings
.get(0) .get(0)
.ok_or_else(|| "Failed to generate embedding for question")? .ok_or_else(|| "Failed to generate embedding for question")?
.clone(); .clone();
// Search for similar question in LangCache
let search_results = langcache_client let search_results = langcache_client
.search("semantic_cache", question_embedding.clone(), 1) .search("semantic_cache", question_embedding.clone(), 1)
.await?; .await?;
@ -344,13 +420,11 @@ impl BotOrchestrator {
} }
} }
// Generate response via LLM provider using full prompt (including history)
let response = self.state let response = self.state
.llm_provider .llm_provider
.generate(&prompt, &serde_json::Value::Null) .generate(&prompt, &serde_json::Value::Null)
.await?; .await?;
// Store isolated question and response in LangCache
let point = QdrantPoint { let point = QdrantPoint {
id: uuid::Uuid::new_v4().to_string(), id: uuid::Uuid::new_v4().to_string(),
vector: question_embedding, vector: question_embedding,
@ -360,26 +434,21 @@ impl BotOrchestrator {
"response": response "response": response
}), }),
}; };
langcache_client langcache_client
.upsert_points("semantic_cache", vec![point]) .upsert_points("semantic_cache", vec![point])
.await?; .await?;
Ok(response) Ok(response)
} else { } else {
// Ensure semantic cache collection exists
ensure_collection_exists(&self.state, "semantic_cache").await?; ensure_collection_exists(&self.state, "semantic_cache").await?;
// Get Qdrant client
let qdrant_client = get_qdrant_client(&self.state)?; let qdrant_client = get_qdrant_client(&self.state)?;
// Generate embedding for the prompt
let embeddings = generate_embeddings(vec![prompt.clone()]).await?; let embeddings = generate_embeddings(vec![prompt.clone()]).await?;
let embedding = embeddings let embedding = embeddings
.get(0) .get(0)
.ok_or_else(|| "Failed to generate embedding")? .ok_or_else(|| "Failed to generate embedding")?
.clone(); .clone();
// Search for similar prompt in Qdrant
let search_results = qdrant_client let search_results = qdrant_client
.search("semantic_cache", embedding.clone(), 1) .search("semantic_cache", embedding.clone(), 1)
.await?; .await?;
@ -392,13 +461,11 @@ impl BotOrchestrator {
} }
} }
// Generate response via LLM provider
let response = self.state let response = self.state
.llm_provider .llm_provider
.generate(&prompt, &serde_json::Value::Null) .generate(&prompt, &serde_json::Value::Null)
.await?; .await?;
// Store prompt and response in Qdrant
let point = QdrantPoint { let point = QdrantPoint {
id: uuid::Uuid::new_v4().to_string(), id: uuid::Uuid::new_v4().to_string(),
vector: embedding, vector: embedding,
@ -407,14 +474,13 @@ impl BotOrchestrator {
"response": response "response": response
}), }),
}; };
qdrant_client qdrant_client
.upsert_points("semantic_cache", vec![point]) .upsert_points("semantic_cache", vec![point])
.await?; .await?;
Ok(response) Ok(response)
} }
} }
pub async fn stream_response( pub async fn stream_response(
@ -422,10 +488,7 @@ impl BotOrchestrator {
message: UserMessage, message: UserMessage,
response_tx: mpsc::Sender<BotResponse>, response_tx: mpsc::Sender<BotResponse>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!( info!("Streaming response for user: {}, session: {}", message.user_id, message.session_id);
"Streaming response for user: {}, session: {}",
message.user_id, message.session_id
);
let user_id = Uuid::parse_str(&message.user_id).map_err(|e| { let user_id = Uuid::parse_str(&message.user_id).map_err(|e| {
error!("Invalid user ID: {}", e); error!("Invalid user ID: {}", e);
@ -448,6 +511,7 @@ impl BotOrchestrator {
error!("Invalid session ID: {}", e); error!("Invalid session ID: {}", e);
e e
})?; })?;
match sm.get_session_by_id(session_id)? { match sm.get_session_by_id(session_id)? {
Some(sess) => sess, Some(sess) => sess,
None => { None => {
@ -500,12 +564,9 @@ impl BotOrchestrator {
for (role, content) in &history { for (role, content) in &history {
p.push_str(&format!("{}: {}\n", role, content)); p.push_str(&format!("{}: {}\n", role, content));
} }
p.push_str(&format!("User: {}\nAssistant:", message.content));
debug!( p.push_str(&format!("User: {}\nAssistant:", message.content));
"Stream prompt constructed with {} history entries", info!("Stream prompt constructed with {} history entries", history.len());
history.len()
);
p p
}; };
@ -556,22 +617,20 @@ impl BotOrchestrator {
if !first_word_received && !chunk.trim().is_empty() { if !first_word_received && !chunk.trim().is_empty() {
first_word_received = true; first_word_received = true;
debug!("First word received in stream: '{}'", chunk);
} }
analysis_buffer.push_str(&chunk); analysis_buffer.push_str(&chunk);
if analysis_buffer.contains("**") && !in_analysis { if analysis_buffer.contains("**") && !in_analysis {
in_analysis = true; in_analysis = true;
} }
if in_analysis { if in_analysis {
if analysis_buffer.ends_with("final") { if analysis_buffer.ends_with("final") {
debug!( info!("Analysis section completed, buffer length: {}", analysis_buffer.len());
"Analysis section completed, buffer length: {}",
analysis_buffer.len()
);
in_analysis = false; in_analysis = false;
analysis_buffer.clear(); analysis_buffer.clear();
if message.channel == "web" { if message.channel == "web" {
let orchestrator = BotOrchestrator::new(Arc::clone(&self.state)); let orchestrator = BotOrchestrator::new(Arc::clone(&self.state));
orchestrator orchestrator
@ -595,6 +654,7 @@ impl BotOrchestrator {
} }
full_response.push_str(&chunk); full_response.push_str(&chunk);
let partial = BotResponse { let partial = BotResponse {
bot_id: message.bot_id.clone(), bot_id: message.bot_id.clone(),
user_id: message.user_id.clone(), user_id: message.user_id.clone(),
@ -612,10 +672,7 @@ impl BotOrchestrator {
} }
} }
debug!( info!("Stream processing completed, {} chunks processed", chunk_count);
"Stream processing completed, {} chunks processed",
chunk_count
);
{ {
let mut sm = self.state.session_manager.lock().await; let mut sm = self.state.session_manager.lock().await;
@ -632,8 +689,8 @@ impl BotOrchestrator {
stream_token: None, stream_token: None,
is_complete: true, is_complete: true,
}; };
response_tx.send(final_msg).await?;
response_tx.send(final_msg).await?;
Ok(()) Ok(())
} }
@ -651,10 +708,7 @@ impl BotOrchestrator {
session_id: Uuid, session_id: Uuid,
user_id: Uuid, user_id: Uuid,
) -> Result<Vec<(String, String)>, Box<dyn std::error::Error + Send + Sync>> { ) -> Result<Vec<(String, String)>, Box<dyn std::error::Error + Send + Sync>> {
info!( info!("Getting conversation history for session {} user {}", session_id, user_id);
"Getting conversation history for session {} user {}",
session_id, user_id
);
let mut session_manager = self.state.session_manager.lock().await; let mut session_manager = self.state.session_manager.lock().await;
let history = session_manager.get_conversation_history(session_id, user_id)?; let history = session_manager.get_conversation_history(session_id, user_id)?;
Ok(history) Ok(history)
@ -665,12 +719,11 @@ impl BotOrchestrator {
state: Arc<AppState>, state: Arc<AppState>,
token: Option<String>, token: Option<String>,
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> { ) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
info!( info!("Running start script for session: {} with token: {:?}", session.id, token);
"Running start script for session: {} with token: {:?}",
session.id, token let bot_guid = std::env::var("BOT_GUID").unwrap_or_else(|_| String::from("default_bot"));
);
let bot_guid = std::env::var("BOT_GUID").unwrap_or_else(|_| "default_bot".to_string());
let start_script_path = format!("./{}.gbai/.gbdialog/start.bas", bot_guid); let start_script_path = format!("./{}.gbai/.gbdialog/start.bas", bot_guid);
let start_script = match std::fs::read_to_string(&start_script_path) { let start_script = match std::fs::read_to_string(&start_script_path) {
Ok(content) => content, Ok(content) => content,
Err(_) => { Err(_) => {
@ -678,10 +731,8 @@ impl BotOrchestrator {
return Ok(true); return Ok(true);
} }
}; };
debug!(
"Start script content for session {}: {}", info!("Start script content for session {}: {}", session.id, start_script);
session.id, start_script
);
let session_clone = session.clone(); let session_clone = session.clone();
let state_clone = state.clone(); let state_clone = state.clone();
@ -694,17 +745,11 @@ impl BotOrchestrator {
.and_then(|ast| script_service.run(&ast)) .and_then(|ast| script_service.run(&ast))
{ {
Ok(result) => { Ok(result) => {
info!( info!("Start script executed successfully for session {}, result: {}", session_clone.id, result);
"Start script executed successfully for session {}, result: {}",
session_clone.id, result
);
Ok(true) Ok(true)
} }
Err(e) => { Err(e) => {
error!( error!("Failed to run start script for session {}: {}", session_clone.id, e);
"Failed to run start script for session {}: {}",
session_clone.id, e
);
Ok(false) Ok(false)
} }
} }
@ -716,10 +761,8 @@ impl BotOrchestrator {
channel: &str, channel: &str,
message: &str, message: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
warn!( warn!("Sending warning to session {} on channel {}: {}", session_id, channel, message);
"Sending warning to session {} on channel {}: {}",
session_id, channel, message
);
if channel == "web" { if channel == "web" {
self.send_event( self.send_event(
"system", "system",
@ -747,10 +790,7 @@ impl BotOrchestrator {
}; };
adapter.send_message(warn_response).await adapter.send_message(warn_response).await
} else { } else {
warn!( warn!("No channel adapter found for warning on channel: {}", channel);
"No channel adapter found for warning on channel: {}",
channel
);
Ok(()) Ok(())
} }
} }
@ -763,10 +803,8 @@ impl BotOrchestrator {
_bot_id: &str, _bot_id: &str,
token: Option<String>, token: Option<String>,
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> { ) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
info!( info!("Triggering auto welcome for user: {}, session: {}, token: {:?}", user_id, session_id, token);
"Triggering auto welcome for user: {}, session: {}, token: {:?}",
user_id, session_id, token
);
let session_uuid = Uuid::parse_str(session_id).map_err(|e| { let session_uuid = Uuid::parse_str(session_id).map_err(|e| {
error!("Invalid session ID: {}", e); error!("Invalid session ID: {}", e);
e e
@ -784,22 +822,53 @@ impl BotOrchestrator {
}; };
let result = Self::run_start_script(&session, Arc::clone(&self.state), token).await?; let result = Self::run_start_script(&session, Arc::clone(&self.state), token).await?;
info!( info!("Auto welcome completed for session: {} with result: {}", session_id, result);
"Auto welcome completed for session: {} with result: {}",
session_id, result
);
Ok(result) Ok(result)
} }
}
async fn get_web_response_channel( pub fn bot_from_url(
&self, db_conn: &mut PgConnection,
session_id: &str, path: &str
) -> Result<mpsc::Sender<BotResponse>, Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(Uuid, String), HttpResponse> {
let response_channels = self.state.response_channels.lock().await; use crate::shared::models::schema::bots::dsl::*;
if let Some(tx) = response_channels.get(session_id) { use diesel::prelude::*;
Ok(tx.clone())
} else { // Extract bot name from first path segment
Err("No response channel found for session".into()) if let Some(bot_name) = path.split('/').nth(1).filter(|s| !s.is_empty()) {
match bots
.filter(name.eq(bot_name))
.filter(is_active.eq(true))
.select((id, name))
.first::<(Uuid, String)>(db_conn)
.optional()
{
Ok(Some((bot_id, bot_name))) => return Ok((bot_id, bot_name)),
Ok(None) => warn!("No active bot found with name: {}", bot_name),
Err(e) => error!("Failed to query bot by name: {}", e),
}
}
// Fall back to first available bot
match bots
.filter(is_active.eq(true))
.select((id, name))
.first::<(Uuid, String)>(db_conn)
.optional()
{
Ok(Some((first_bot_id, first_bot_name))) => {
log::info!("Using first available bot: {} ({})", first_bot_id, first_bot_name);
Ok((first_bot_id, first_bot_name))
}
Ok(None) => {
error!("No active bots found in database");
Err(HttpResponse::ServiceUnavailable()
.json(serde_json::json!({"error": "No bots available"})))
}
Err(e) => {
error!("Failed to query bots: {}", e);
Err(HttpResponse::InternalServerError()
.json(serde_json::json!({"error": "Failed to query bots"})))
} }
} }
} }
@ -808,6 +877,7 @@ impl Default for BotOrchestrator {
fn default() -> Self { fn default() -> Self {
Self { Self {
state: Arc::new(AppState::default()), state: Arc::new(AppState::default()),
mounted_bots: Arc::new(AsyncMutex::new(HashMap::new())),
} }
} }
} }
@ -826,7 +896,6 @@ async fn websocket_handler(
.unwrap_or_else(|| Uuid::new_v4().to_string()) .unwrap_or_else(|| Uuid::new_v4().to_string())
.replace("undefined", &Uuid::new_v4().to_string()); .replace("undefined", &Uuid::new_v4().to_string());
// Ensure user exists in database before proceeding
let user_id = { let user_id = {
let user_uuid = Uuid::parse_str(&user_id_string).unwrap_or_else(|_| Uuid::new_v4()); let user_uuid = Uuid::parse_str(&user_id_string).unwrap_or_else(|_| Uuid::new_v4());
let mut sm = data.session_manager.lock().await; let mut sm = data.session_manager.lock().await;
@ -841,8 +910,8 @@ async fn websocket_handler(
let (res, mut session, mut msg_stream) = actix_ws::handle(&req, stream)?; let (res, mut session, mut msg_stream) = actix_ws::handle(&req, stream)?;
let (tx, mut rx) = mpsc::channel::<BotResponse>(100); let (tx, mut rx) = mpsc::channel::<BotResponse>(100);
let orchestrator = BotOrchestrator::new(Arc::clone(&data));
let orchestrator = BotOrchestrator::new(Arc::clone(&data));
orchestrator orchestrator
.register_response_channel(session_id.clone(), tx.clone()) .register_response_channel(session_id.clone(), tx.clone())
.await; .await;
@ -855,7 +924,6 @@ async fn websocket_handler(
.add_connection(session_id.clone(), tx.clone()) .add_connection(session_id.clone(), tx.clone())
.await; .await;
// Get first available bot from database
let bot_id = { let bot_id = {
use crate::shared::models::schema::bots::dsl::*; use crate::shared::models::schema::bots::dsl::*;
use diesel::prelude::*; use diesel::prelude::*;
@ -897,16 +965,13 @@ async fn websocket_handler(
.await .await
.ok(); .ok();
info!( info!("WebSocket connection established for session: {}, user: {}", session_id, user_id);
"WebSocket connection established for session: {}, user: {}",
session_id, user_id
);
// Trigger auto welcome (start.bas)
let orchestrator_clone = BotOrchestrator::new(Arc::clone(&data)); let orchestrator_clone = BotOrchestrator::new(Arc::clone(&data));
let user_id_welcome = user_id.clone(); let user_id_welcome = user_id.clone();
let session_id_welcome = session_id.clone(); let session_id_welcome = session_id.clone();
let bot_id_welcome = bot_id.clone(); let bot_id_welcome = bot_id.clone();
actix_web::rt::spawn(async move { actix_web::rt::spawn(async move {
if let Err(e) = orchestrator_clone if let Err(e) = orchestrator_clone
.trigger_auto_welcome(&session_id_welcome, &user_id_welcome, &bot_id_welcome, None) .trigger_auto_welcome(&session_id_welcome, &user_id_welcome, &bot_id_welcome, None)
@ -922,10 +987,7 @@ async fn websocket_handler(
let user_id_clone = user_id.clone(); let user_id_clone = user_id.clone();
actix_web::rt::spawn(async move { actix_web::rt::spawn(async move {
info!( info!("Starting WebSocket sender for session {}", session_id_clone1);
"Starting WebSocket sender for session {}",
session_id_clone1
);
let mut message_count = 0; let mut message_count = 0;
while let Some(msg) = rx.recv().await { while let Some(msg) = rx.recv().await {
message_count += 1; message_count += 1;
@ -936,23 +998,17 @@ async fn websocket_handler(
} }
} }
} }
info!( info!("WebSocket sender terminated for session {}, sent {} messages", session_id_clone1, message_count);
"WebSocket sender terminated for session {}, sent {} messages",
session_id_clone1, message_count
);
}); });
actix_web::rt::spawn(async move { actix_web::rt::spawn(async move {
info!( info!("Starting WebSocket receiver for session {}", session_id_clone2);
"Starting WebSocket receiver for session {}",
session_id_clone2
);
let mut message_count = 0; let mut message_count = 0;
while let Some(Ok(msg)) = msg_stream.recv().await { while let Some(Ok(msg)) = msg_stream.recv().await {
match msg { match msg {
WsMessage::Text(text) => { WsMessage::Text(text) => {
message_count += 1; message_count += 1;
// Get first available bot from database
let bot_id = { let bot_id = {
use crate::shared::models::schema::bots::dsl::*; use crate::shared::models::schema::bots::dsl::*;
use diesel::prelude::*; use diesel::prelude::*;
@ -976,42 +1032,37 @@ async fn websocket_handler(
} }
}; };
// Parse the text as JSON to extract the content field
let json_value: serde_json::Value = match serde_json::from_str(&text) { let json_value: serde_json::Value = match serde_json::from_str(&text) {
Ok(value) => value, Ok(value) => value,
Err(e) => { Err(e) => {
error!("Error parsing JSON message {}: {}", message_count, e); error!("Error parsing JSON message {}: {}", message_count, e);
continue; // Skip processing this message continue;
} }
}; };
// Extract content from JSON, fallback to original text if content field doesn't exist
let content = json_value["content"] let content = json_value["content"]
.as_str() .as_str()
.map(|s| s.to_string()) .map(|s| s.to_string())
.unwrap(); .unwrap();
let user_message = UserMessage { let user_message = UserMessage {
bot_id: bot_id, bot_id,
user_id: user_id_clone.clone(), user_id: user_id_clone.clone(),
session_id: session_id_clone2.clone(), session_id: session_id_clone2.clone(),
channel: "web".to_string(), channel: "web".to_string(),
content: content, content,
message_type: 1, message_type: 1,
media_url: None, media_url: None,
timestamp: Utc::now(), timestamp: Utc::now(),
}; };
if let Err(e) = orchestrator.stream_response(user_message, tx.clone()).await { if let Err(e) = orchestrator.stream_response(user_message, tx.clone()).await {
error!( error!("Error processing WebSocket message {}: {}", message_count, e);
"Error processing WebSocket message {}: {}",
message_count, e
);
} }
} }
WsMessage::Close(reason) => {
debug!("WebSocket closing for session {} - reason: {:?}", session_id_clone2, reason);
WsMessage::Close(_) => {
// Get first available bot from database
let bot_id = { let bot_id = {
use crate::shared::models::schema::bots::dsl::*; use crate::shared::models::schema::bots::dsl::*;
use diesel::prelude::*; use diesel::prelude::*;
@ -1034,7 +1085,9 @@ async fn websocket_handler(
} }
} }
}; };
orchestrator
debug!("Sending session_end event for {}", session_id_clone2);
if let Err(e) = orchestrator
.send_event( .send_event(
&user_id_clone, &user_id_clone,
&bot_id, &bot_id,
@ -1044,26 +1097,28 @@ async fn websocket_handler(
serde_json::json!({}), serde_json::json!({}),
) )
.await .await
.ok(); {
error!("Failed to send session_end event: {}", e);
}
debug!("Removing WebSocket connection for {}", session_id_clone2);
web_adapter.remove_connection(&session_id_clone2).await; web_adapter.remove_connection(&session_id_clone2).await;
debug!("Unregistering response channel for {}", session_id_clone2);
orchestrator orchestrator
.unregister_response_channel(&session_id_clone2) .unregister_response_channel(&session_id_clone2)
.await; .await;
info!("WebSocket fully closed for session {}", session_id_clone2);
break; break;
} }
_ => {} _ => {}
} }
} }
info!( info!("WebSocket receiver terminated for session {}, processed {} messages", session_id_clone2, message_count);
"WebSocket receiver terminated for session {}, processed {} messages",
session_id_clone2, message_count
);
}); });
info!( info!("WebSocket handler setup completed for session {}", session_id);
"WebSocket handler setup completed for session {}",
session_id
);
Ok(res) Ok(res)
} }
@ -1076,6 +1131,7 @@ async fn start_session(
.get("session_id") .get("session_id")
.and_then(|s| s.as_str()) .and_then(|s| s.as_str())
.unwrap_or(""); .unwrap_or("");
let token = info let token = info
.get("token") .get("token")
.and_then(|t| t.as_str()) .and_then(|t| t.as_str())
@ -1109,12 +1165,10 @@ async fn start_session(
}; };
let result = BotOrchestrator::run_start_script(&session, Arc::clone(&data), token).await; let result = BotOrchestrator::run_start_script(&session, Arc::clone(&data), token).await;
match result { match result {
Ok(true) => { Ok(true) => {
info!( info!("Start script completed successfully for session: {}", session_id);
"Start script completed successfully for session: {}",
session_id
);
Ok(HttpResponse::Ok().json(serde_json::json!({ Ok(HttpResponse::Ok().json(serde_json::json!({
"status": "started", "status": "started",
"session_id": session.id, "session_id": session.id,
@ -1130,10 +1184,7 @@ async fn start_session(
}))) })))
} }
Err(e) => { Err(e) => {
error!( error!("Error running start script for session {}: {}", session_id, e);
"Error running start script for session {}: {}",
session_id, e
);
Ok(HttpResponse::InternalServerError() Ok(HttpResponse::InternalServerError()
.json(serde_json::json!({"error": e.to_string()}))) .json(serde_json::json!({"error": e.to_string()})))
} }
@ -1148,14 +1199,12 @@ async fn send_warning_handler(
let default_session = "default".to_string(); let default_session = "default".to_string();
let default_channel = "web".to_string(); let default_channel = "web".to_string();
let default_message = "Warning!".to_string(); let default_message = "Warning!".to_string();
let session_id = info.get("session_id").unwrap_or(&default_session); let session_id = info.get("session_id").unwrap_or(&default_session);
let channel = info.get("channel").unwrap_or(&default_channel); let channel = info.get("channel").unwrap_or(&default_channel);
let message = info.get("message").unwrap_or(&default_message); let message = info.get("message").unwrap_or(&default_message);
info!( info!("Sending warning via API - session: {}, channel: {}", session_id, channel);
"Sending warning via API - session: {}, channel: {}",
session_id, channel
);
let orchestrator = BotOrchestrator::new(Arc::clone(&data)); let orchestrator = BotOrchestrator::new(Arc::clone(&data));
if let Err(e) = orchestrator if let Err(e) = orchestrator

View file

@ -3,6 +3,8 @@ use diesel::sql_types::Text;
use log::{info, warn}; use log::{info, warn};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
use std::fs::OpenOptions;
use std::io::Write;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
@ -190,11 +192,18 @@ impl AppConfig {
.and_then(|p| p.parse().ok()) .and_then(|p| p.parse().ok())
.unwrap_or_else(|| get_u32("CUSTOM_PORT", 5432)), .unwrap_or_else(|| get_u32("CUSTOM_PORT", 5432)),
database: std::env::var("CUSTOM_DATABASE") database: std::env::var("CUSTOM_DATABASE")
.unwrap_or_else(|_| get_str("CUSTOM_DATABASE", "botserver")), .unwrap_or_else(|_| get_str("CUSTOM_DATABASE", "gbuser")),
}; };
let minio = DriveConfig { let minio = DriveConfig {
server: get_str("DRIVE_SERVER", "http://localhost:9000"), server: {
let server = get_str("DRIVE_SERVER", "http://localhost:9000");
if !server.starts_with("http://") && !server.starts_with("https://") {
format!("http://{}", server)
} else {
server
}
},
access_key: get_str("DRIVE_ACCESSKEY", "minioadmin"), access_key: get_str("DRIVE_ACCESSKEY", "minioadmin"),
secret_key: get_str("DRIVE_SECRET", "minioadmin"), secret_key: get_str("DRIVE_SECRET", "minioadmin"),
use_ssl: get_bool("DRIVE_USE_SSL", false), use_ssl: get_bool("DRIVE_USE_SSL", false),
@ -216,6 +225,11 @@ impl AppConfig {
endpoint: get_str("AI_ENDPOINT", "https://api.openai.com"), endpoint: get_str("AI_ENDPOINT", "https://api.openai.com"),
}; };
// Write drive config to .env file
if let Err(e) = write_drive_config_to_env(&minio) {
warn!("Failed to write drive config to .env: {}", e);
}
AppConfig { AppConfig {
drive: minio, drive: minio,
server: ServerConfig { server: ServerConfig {
@ -367,6 +381,22 @@ impl AppConfig {
} }
} }
fn write_drive_config_to_env(drive: &DriveConfig) -> std::io::Result<()> {
let mut file = OpenOptions::new()
.append(true)
.create(true)
.open(".env")?;
writeln!(file,"")?;
writeln!(file, "DRIVE_SERVER={}", drive.server)?;
writeln!(file, "DRIVE_ACCESSKEY={}", drive.access_key)?;
writeln!(file, "DRIVE_SECRET={}", drive.secret_key)?;
writeln!(file, "DRIVE_USE_SSL={}", drive.use_ssl)?;
writeln!(file, "DRIVE_ORG_PREFIX={}", drive.org_prefix)?;
Ok(())
}
fn parse_database_url(url: &str) -> (String, String, String, u32, String) { fn parse_database_url(url: &str) -> (String, String, String, u32, String) {
if let Some(stripped) = url.strip_prefix("postgres://") { if let Some(stripped) = url.strip_prefix("postgres://") {
let parts: Vec<&str> = stripped.split('@').collect(); let parts: Vec<&str> = stripped.split('@').collect();

8
src/create_bucket.rs Normal file
View file

@ -0,0 +1,8 @@
use std::fs;
use std::path::Path;
pub fn create_bucket(bucket_name: &str) -> std::io::Result<()> {
let bucket_path = Path::new("buckets").join(bucket_name);
fs::create_dir_all(&bucket_path)?;
Ok(())
}

View file

@ -2,8 +2,8 @@ use crate::basic::compiler::BasicCompiler;
use crate::kb::embeddings; use crate::kb::embeddings;
use crate::kb::qdrant_client; use crate::kb::qdrant_client;
use crate::shared::state::AppState; use crate::shared::state::AppState;
use aws_sdk_s3::Client;
use log::{debug, error, info, warn}; use log::{debug, error, info, warn};
use opendal::Operator;
use std::collections::HashMap; use std::collections::HashMap;
use std::error::Error; use std::error::Error;
use std::sync::Arc; use std::sync::Arc;
@ -34,7 +34,10 @@ impl DriveMonitor {
pub fn spawn(self: Arc<Self>) -> tokio::task::JoinHandle<()> { pub fn spawn(self: Arc<Self>) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move { tokio::spawn(async move {
info!("Drive Monitor service started for bucket: {}", self.bucket_name); info!(
"Drive Monitor service started for bucket: {}",
self.bucket_name
);
let mut tick = interval(Duration::from_secs(30)); let mut tick = interval(Duration::from_secs(30));
loop { loop {
tick.tick().await; tick.tick().await;
@ -46,17 +49,17 @@ impl DriveMonitor {
} }
async fn check_for_changes(&self) -> Result<(), Box<dyn Error + Send + Sync>> { async fn check_for_changes(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
let op = match &self.state.s3_operator { let client = match &self.state.s3_client {
Some(op) => op, Some(client) => client,
None => { None => {
return Ok(()); return Ok(());
} }
}; };
self.check_gbdialog_changes(op).await?; self.check_gbdialog_changes(client).await?;
self.check_gbkb_changes(op).await?; self.check_gbkb_changes(client).await?;
if let Err(e) = self.check_default_gbot(op).await { if let Err(e) = self.check_gbot(client).await {
error!("Error checking default bot config: {}", e); error!("Error checking default bot config: {}", e);
} }
@ -65,40 +68,57 @@ impl DriveMonitor {
async fn check_gbdialog_changes( async fn check_gbdialog_changes(
&self, &self,
op: &Operator, client: &Client,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let prefix = ".gbdialog/"; let prefix = ".gbdialog/";
let mut current_files = HashMap::new(); let mut current_files = HashMap::new();
let mut lister = op.lister_with(prefix).recursive(true).await?; let mut continuation_token = None;
while let Some(entry) = futures::TryStreamExt::try_next(&mut lister).await? { loop {
let path = entry.path().to_string(); let list_objects = client
.list_objects_v2()
.bucket(&self.bucket_name.to_lowercase())
.set_continuation_token(continuation_token)
.send()
.await?;
debug!("List objects result: {:?}", list_objects);
if path.ends_with('/') || !path.ends_with(".bas") { for obj in list_objects.contents.unwrap_or_default() {
continue; let path = obj.key().unwrap_or_default().to_string();
let path_parts: Vec<&str> = path.split('/').collect();
if path_parts.len() < 2 || !path_parts[0].ends_with(".gbdialog") {
continue;
}
if path.ends_with('/') || !path.ends_with(".bas") {
continue;
}
let file_state = FileState {
path: path.clone(),
size: obj.size().unwrap_or(0),
etag: obj.e_tag().unwrap_or_default().to_string(),
last_modified: obj.last_modified().map(|dt| dt.to_string()),
};
current_files.insert(path, file_state);
} }
let meta = op.stat(&path).await?; if !list_objects.is_truncated.unwrap_or(false) {
let file_state = FileState { break;
path: path.clone(), }
size: meta.content_length() as i64, continuation_token = list_objects.next_continuation_token;
etag: meta.etag().unwrap_or_default().to_string(),
last_modified: meta.last_modified().map(|dt| dt.to_rfc3339()),
};
current_files.insert(path, file_state);
} }
let mut file_states = self.file_states.write().await; let mut file_states = self.file_states.write().await;
for (path, current_state) in current_files.iter() { for (path, current_state) in current_files.iter() {
if let Some(previous_state) = file_states.get(path) { if let Some(previous_state) = file_states.get(path) {
if current_state.etag != previous_state.etag { if current_state.etag != previous_state.etag {
if let Err(e) = self.compile_tool(op, path).await { if let Err(e) = self.compile_tool(client, path).await {
error!("Failed to compile tool {}: {}", path, e); error!("Failed to compile tool {}: {}", path, e);
} }
} }
} else { } else {
if let Err(e) = self.compile_tool(op, path).await { if let Err(e) = self.compile_tool(client, path).await {
error!("Failed to compile tool {}: {}", path, e); error!("Failed to compile tool {}: {}", path, e);
} }
} }
@ -125,45 +145,67 @@ impl DriveMonitor {
async fn check_gbkb_changes( async fn check_gbkb_changes(
&self, &self,
op: &Operator, client: &Client,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let prefix = ".gbkb/"; let prefix = ".gbkb/";
let mut current_files = HashMap::new(); let mut current_files = HashMap::new();
let mut lister = op.lister_with(prefix).recursive(true).await?; let mut continuation_token = None;
while let Some(entry) = futures::TryStreamExt::try_next(&mut lister).await? { loop {
let path = entry.path().to_string(); let list_objects = client
.list_objects_v2()
.bucket(&self.bucket_name.to_lowercase())
.prefix(prefix)
.set_continuation_token(continuation_token)
.send()
.await?;
debug!("List objects result: {:?}", list_objects);
if path.ends_with('/') { for obj in list_objects.contents.unwrap_or_default() {
continue; let path = obj.key().unwrap_or_default().to_string();
let path_parts: Vec<&str> = path.split('/').collect();
if path_parts.len() < 2 || !path_parts[0].ends_with(".gbkb") {
continue;
}
if path.ends_with('/') {
continue;
}
let ext = path.rsplit('.').next().unwrap_or("").to_lowercase();
if !["pdf", "txt", "md", "docx"].contains(&ext.as_str()) {
continue;
}
let file_state = FileState {
path: path.clone(),
size: obj.size().unwrap_or(0),
etag: obj.e_tag().unwrap_or_default().to_string(),
last_modified: obj.last_modified().map(|dt| dt.to_string()),
};
current_files.insert(path, file_state);
} }
let ext = path.rsplit('.').next().unwrap_or("").to_lowercase(); if !list_objects.is_truncated.unwrap_or(false) {
if !["pdf", "txt", "md", "docx"].contains(&ext.as_str()) { break;
continue;
} }
continuation_token = list_objects.next_continuation_token;
let meta = op.stat(&path).await?;
let file_state = FileState {
path: path.clone(),
size: meta.content_length() as i64,
etag: meta.etag().unwrap_or_default().to_string(),
last_modified: meta.last_modified().map(|dt| dt.to_rfc3339()),
};
current_files.insert(path, file_state);
} }
let mut file_states = self.file_states.write().await; let mut file_states = self.file_states.write().await;
for (path, current_state) in current_files.iter() { for (path, current_state) in current_files.iter() {
if let Some(previous_state) = file_states.get(path) { if let Some(previous_state) = file_states.get(path) {
if current_state.etag != previous_state.etag { if current_state.etag != previous_state.etag {
if let Err(e) = self.index_document(op, path).await { if let Err(e) = self.index_document(client, path).await {
error!("Failed to index document {}: {}", path, e); error!("Failed to index document {}: {}", path, e);
} }
} }
} else { } else {
if let Err(e) = self.index_document(op, path).await { if let Err(e) = self.index_document(client, path).await {
error!("Failed to index document {}: {}", path, e); error!("Failed to index document {}: {}", path, e);
} }
} }
@ -188,38 +230,103 @@ impl DriveMonitor {
Ok(()) Ok(())
} }
async fn check_default_gbot( async fn check_gbot(
&self, &self,
op: &Operator, client: &Client,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let prefix = format!("{}default.gbot/", self.bucket_name); let prefix = ".gbot/";
let config_key = format!("{}config.csv", prefix); let mut continuation_token = None;
match op.stat(&config_key).await { loop {
Ok(_) => { let list_objects = client
let content = op.read(&config_key).await?; .list_objects_v2()
let csv_content = String::from_utf8(content.to_vec()) .bucket(&self.bucket_name.to_lowercase())
.map_err(|e| format!("UTF-8 error in config.csv: {}", e))?; .prefix(prefix)
debug!("Found config.csv: {} bytes", csv_content.len()); .set_continuation_token(continuation_token)
Ok(()) .send()
.await?;
for obj in list_objects.contents.unwrap_or_default() {
let path = obj.key().unwrap_or_default().to_string();
let path_parts: Vec<&str> = path.split('/').collect();
if path_parts.len() < 2 || !path_parts[0].ends_with(".gbot") {
continue;
}
if !path.ends_with("config.csv") {
continue;
}
debug!("Checking config file at path: {}", path);
match client
.head_object()
.bucket(&self.bucket_name)
.key(&path)
.send()
.await
{
Ok(head_res) => {
debug!("HeadObject successful for {}, metadata: {:?}", path, head_res);
let response = client
.get_object()
.bucket(&self.bucket_name)
.key(&path)
.send()
.await?;
debug!("GetObject successful for {}, content length: {}", path, response.content_length().unwrap_or(0));
let bytes = response.body.collect().await?.into_bytes();
debug!("Collected {} bytes for {}", bytes.len(), path);
let csv_content = String::from_utf8(bytes.to_vec())
.map_err(|e| format!("UTF-8 error in {}: {}", path, e))?;
debug!("Found {}: {} bytes", path, csv_content.len());
}
Err(e) => {
debug!("Config file {} not found or inaccessible: {}", path, e);
}
}
} }
Err(e) => {
debug!("Config file not found or inaccessible: {}", e); if !list_objects.is_truncated.unwrap_or(false) {
Ok(()) break;
} }
continuation_token = list_objects.next_continuation_token;
} }
Ok(())
} }
async fn compile_tool( async fn compile_tool(
&self, &self,
op: &Operator, client: &Client,
file_path: &str, file_path: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let content = op.read(file_path).await?; debug!("Fetching object from S3: bucket={}, key={}", &self.bucket_name, file_path);
let source_content = String::from_utf8(content.to_vec())?; let response = match client
.get_object()
.bucket(&self.bucket_name)
.key(file_path)
.send()
.await {
Ok(res) => {
debug!("Successfully fetched object from S3: bucket={}, key={}, size={}",
&self.bucket_name, file_path, res.content_length().unwrap_or(0));
res
}
Err(e) => {
error!("Failed to fetch object from S3: bucket={}, key={}, error={:?}",
&self.bucket_name, file_path, e);
return Err(e.into());
}
};
let bytes = response.body.collect().await?.into_bytes();
let source_content = String::from_utf8(bytes.to_vec())?;
let tool_name = file_path let tool_name = file_path
.strip_prefix(".gbdialog/") .split('/')
.last()
.unwrap_or(file_path) .unwrap_or(file_path)
.strip_suffix(".bas") .strip_suffix(".bas")
.unwrap_or(file_path) .unwrap_or(file_path)
@ -229,7 +336,7 @@ impl DriveMonitor {
.bucket_name .bucket_name
.strip_suffix(".gbai") .strip_suffix(".gbai")
.unwrap_or(&self.bucket_name); .unwrap_or(&self.bucket_name);
let work_dir = format!("./work/{}.gbai/.gbdialog", bot_name); let work_dir = format!("./work/{}.gbai/{}.gbdialog", bot_name, bot_name);
std::fs::create_dir_all(&work_dir)?; std::fs::create_dir_all(&work_dir)?;
let local_source_path = format!("{}/{}.bas", work_dir, tool_name); let local_source_path = format!("{}/{}.bas", work_dir, tool_name);
@ -254,7 +361,7 @@ impl DriveMonitor {
async fn index_document( async fn index_document(
&self, &self,
op: &Operator, client: &Client,
file_path: &str, file_path: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let parts: Vec<&str> = file_path.split('/').collect(); let parts: Vec<&str> = file_path.split('/').collect();
@ -264,8 +371,13 @@ impl DriveMonitor {
} }
let collection_name = parts[1]; let collection_name = parts[1];
let content = op.read(file_path).await?; let response = client
let bytes = content.to_vec(); .get_object()
.bucket(&self.bucket_name)
.key(file_path)
.send()
.await?;
let bytes = response.body.collect().await?.into_bytes();
let text_content = self.extract_text(file_path, &bytes)?; let text_content = self.extract_text(file_path, &bytes)?;
if text_content.trim().is_empty() { if text_content.trim().is_empty() {

View file

@ -3,10 +3,12 @@ use crate::shared::state::AppState;
use actix_multipart::Multipart; use actix_multipart::Multipart;
use actix_web::web; use actix_web::web;
use actix_web::{post, HttpResponse}; use actix_web::{post, HttpResponse};
use opendal::Operator; use aws_sdk_s3::{Client as S3Client, config::Builder as S3ConfigBuilder};
use aws_config::BehaviorVersion;
use std::io::Write; use std::io::Write;
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
use tokio_stream::StreamExt as TokioStreamExt; use tokio_stream::StreamExt as TokioStreamExt;
// Removed unused import
#[post("/files/upload/{folder_path}")] #[post("/files/upload/{folder_path}")]
pub async fn upload_file( pub async fn upload_file(
@ -40,13 +42,13 @@ pub async fn upload_file(
let file_name = file_name.unwrap_or_else(|| "unnamed_file".to_string()); let file_name = file_name.unwrap_or_else(|| "unnamed_file".to_string());
let temp_file_path = temp_file.into_temp_path(); let temp_file_path = temp_file.into_temp_path();
let op = state.get_ref().s3_operator.as_ref().ok_or_else(|| { let client = state.get_ref().s3_client.as_ref().ok_or_else(|| {
actix_web::error::ErrorInternalServerError("S3 operator is not initialized") actix_web::error::ErrorInternalServerError("S3 client is not initialized")
})?; })?;
let s3_key = format!("{}/{}", folder_path, file_name); let s3_key = format!("{}/{}", folder_path, file_name);
match upload_to_s3(op, &s3_key, &temp_file_path).await { match upload_to_s3(client, &state.get_ref().bucket_name, &s3_key, &temp_file_path).await {
Ok(_) => { Ok(_) => {
let _ = std::fs::remove_file(&temp_file_path); let _ = std::fs::remove_file(&temp_file_path);
Ok(HttpResponse::Ok().body(format!( Ok(HttpResponse::Ok().body(format!(
@ -64,27 +66,149 @@ pub async fn upload_file(
} }
} }
pub async fn init_drive(config: &DriveConfig) -> Result<Operator, Box<dyn std::error::Error>> { pub async fn aws_s3_bucket_delete(
use opendal::services::S3; bucket: &str,
use opendal::Operator; endpoint: &str,
let client = Operator::new( access_key: &str,
S3::default() secret_key: &str,
.root("/") ) -> Result<(), Box<dyn std::error::Error>> {
.endpoint(&config.server) let config = aws_config::defaults(BehaviorVersion::latest())
.access_key_id(&config.access_key) .endpoint_url(endpoint)
.secret_access_key(&config.secret_key), .region("auto")
)? .credentials_provider(
.finish(); aws_sdk_s3::config::Credentials::new(
access_key.to_string(),
secret_key.to_string(),
None,
None,
"static",
)
)
.load()
.await;
Ok(client) let client = S3Client::new(&config);
client.delete_bucket()
.bucket(bucket)
.send()
.await?;
Ok(())
}
pub async fn aws_s3_bucket_create(
bucket: &str,
endpoint: &str,
access_key: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let config = aws_config::defaults(BehaviorVersion::latest())
.endpoint_url(endpoint)
.region("auto")
.credentials_provider(
aws_sdk_s3::config::Credentials::new(
access_key.to_string(),
secret_key.to_string(),
None,
None,
"static",
)
)
.load()
.await;
let client = S3Client::new(&config);
client.create_bucket()
.bucket(bucket)
.send()
.await?;
Ok(())
}
pub async fn init_drive(config: &DriveConfig) -> Result<S3Client, Box<dyn std::error::Error>> {
let endpoint = if !config.server.ends_with('/') {
format!("{}/", config.server)
} else {
config.server.clone()
};
let base_config = aws_config::defaults(BehaviorVersion::latest())
.endpoint_url(endpoint)
.region("auto")
.credentials_provider(
aws_sdk_s3::config::Credentials::new(
config.access_key.clone(),
config.secret_key.clone(),
None,
None,
"static",
)
)
.load()
.await;
let s3_config = S3ConfigBuilder::from(&base_config)
.force_path_style(true)
.build();
Ok(S3Client::from_conf(s3_config))
} }
async fn upload_to_s3( async fn upload_to_s3(
op: &Operator, client: &S3Client,
bucket: &str,
key: &str, key: &str,
file_path: &std::path::Path, file_path: &std::path::Path,
) -> Result<(), Box<dyn std::error::Error>> { ) -> Result<(), Box<dyn std::error::Error>> {
let data = std::fs::read(file_path)?; let data = std::fs::read(file_path)?;
op.write(key, data).await?; client.put_object()
.bucket(bucket)
.key(key)
.body(data.into())
.send()
.await?;
Ok(()) Ok(())
} }
async fn create_s3_client(
) -> Result<S3Client, Box<dyn std::error::Error>> {
let config = DriveConfig {
server: std::env::var("DRIVE_SERVER").expect("DRIVE_SERVER not set"),
access_key: std::env::var("DRIVE_ACCESS_KEY").expect("DRIVE_ACCESS_KEY not set"),
secret_key: std::env::var("DRIVE_SECRET_KEY").expect("DRIVE_SECRET_KEY not set"),
org_prefix: "".to_string(),
use_ssl: false,
};
Ok(init_drive(&config).await?)
}
pub async fn bucket_exists(client: &S3Client, bucket: &str) -> Result<bool, Box<dyn std::error::Error>> {
match client.head_bucket().bucket(bucket).send().await {
Ok(_) => Ok(true),
Err(e) => {
if e.to_string().contains("NoSuchBucket") {
Ok(false)
} else {
Err(Box::new(e))
}
}
}
}
pub async fn create_bucket(client: &S3Client, bucket: &str) -> Result<(), Box<dyn std::error::Error>> {
client.create_bucket()
.bucket(bucket)
.send()
.await?;
Ok(())
}
#[cfg(test)]
mod bucket_tests {
include!("tests/bucket_tests.rs");
}
#[cfg(test)]
mod tests {
include!("tests/tests.rs");
}

View file

@ -0,0 +1,70 @@
use super::*;
use aws_sdk_s3::Client as S3Client;
use std::env;
#[tokio::test]
async fn test_aws_s3_bucket_create() {
if env::var("CI").is_ok() {
return; // Skip in CI environment
}
let bucket = "test-bucket-aws";
let endpoint = "http://localhost:4566"; // LocalStack default endpoint
let access_key = "test";
let secret_key = "test";
match aws_s3_bucket_create(bucket, endpoint, access_key, secret_key).await {
Ok(_) => {
// Verify bucket exists
let config = aws_config::defaults(BehaviorVersion::latest())
.endpoint_url(endpoint)
.region("auto")
.load()
.await;
let client = S3Client::new(&config);
let exists = bucket_exists(&client, bucket).await.unwrap_or(false);
assert!(exists, "Bucket should exist after creation");
},
Err(e) => {
println!("Bucket creation failed: {:?}", e);
}
}
}
#[tokio::test]
async fn test_aws_s3_bucket_delete() {
if env::var("CI").is_ok() {
return; // Skip in CI environment
}
let bucket = "test-delete-bucket-aws";
let endpoint = "http://localhost:4566"; // LocalStack default endpoint
let access_key = "test";
let secret_key = "test";
// First create the bucket
if let Err(e) = aws_s3_bucket_create(bucket, endpoint, access_key, secret_key).await {
println!("Failed to create test bucket: {:?}", e);
return;
}
// Then test deletion
match aws_s3_bucket_delete(bucket, endpoint, access_key, secret_key).await {
Ok(_) => {
// Verify bucket no longer exists
let config = aws_config::defaults(BehaviorVersion::latest())
.endpoint_url(endpoint)
.region("auto")
.load()
.await;
let client = S3Client::new(&config);
let exists = bucket_exists(&client, bucket).await.unwrap_or(false);
assert!(!exists, "Bucket should not exist after deletion");
},
Err(e) => {
println!("Bucket deletion failed: {:?}", e);
}
}
}

80
src/file/tests/tests.rs Normal file
View file

@ -0,0 +1,80 @@
use super::*;
#[tokio::test]
async fn test_create_s3_client() {
if std::env::var("CI").is_ok() {
return; // Skip in CI environment
}
// Setup test environment variables
std::env::set_var("DRIVE_SERVER", "http://localhost:9000");
std::env::set_var("DRIVE_ACCESS_KEY", "minioadmin");
std::env::set_var("DRIVE_SECRET_KEY", "minioadmin");
match create_s3_client().await {
Ok(client) => {
// Verify client creation
assert!(client.config().region().is_some());
// Test bucket operations
if let Err(e) = create_bucket(&client, "test.gbai").await {
println!("Bucket creation failed: {:?}", e);
}
},
Err(e) => {
// Skip if no S3 server available
println!("S3 client creation failed: {:?}", e);
}
}
// Cleanup
std::env::remove_var("DRIVE_SERVER");
std::env::remove_var("DRIVE_ACCESS_KEY");
std::env::remove_var("DRIVE_SECRET_KEY");
}
#[tokio::test]
async fn test_bucket_exists() {
if std::env::var("CI").is_ok() {
return; // Skip in CI environment
}
// Setup test environment variables
std::env::set_var("DRIVE_SERVER", "http://localhost:9000");
std::env::set_var("DRIVE_ACCESS_KEY", "minioadmin");
std::env::set_var("DRIVE_SECRET_KEY", "minioadmin");
match create_s3_client().await {
Ok(client) => {
// Verify client creation
assert!(client.config().region().is_some());
},
Err(e) => {
// Skip if no S3 server available
println!("S3 client creation failed: {:?}", e);
}
}
}
#[tokio::test]
async fn test_create_bucket() {
if std::env::var("CI").is_ok() {
return; // Skip in CI environment
}
// Setup test environment variables
std::env::set_var("DRIVE_SERVER", "http://localhost:9000");
std::env::set_var("DRIVE_ACCESS_KEY", "minioadmin");
std::env::set_var("DRIVE_SECRET_KEY", "minioadmin");
match create_s3_client().await {
Ok(client) => {
// Verify client creation
assert!(client.config().region().is_some());
},
Err(e) => {
// Skip if no S3 server available
println!("S3 client creation failed: {:?}", e);
}
}
}

View file

@ -1,7 +1,6 @@
use crate::shared::state::AppState; use crate::shared::state::AppState;
use log::error; use log::error;
use opendal::Operator; use aws_sdk_s3::Client;
use tokio_stream::StreamExt;
use std::collections::HashMap; use std::collections::HashMap;
use std::error::Error; use std::error::Error;
use std::sync::Arc; use std::sync::Arc;
@ -17,14 +16,32 @@ pub struct FileState {
pub struct MinIOHandler { pub struct MinIOHandler {
state: Arc<AppState>, state: Arc<AppState>,
s3: Arc<Client>,
watched_prefixes: Arc<tokio::sync::RwLock<Vec<String>>>, watched_prefixes: Arc<tokio::sync::RwLock<Vec<String>>>,
file_states: Arc<tokio::sync::RwLock<HashMap<String, FileState>>>, file_states: Arc<tokio::sync::RwLock<HashMap<String, FileState>>>,
} }
pub async fn get_file_content(
client: &aws_sdk_s3::Client,
bucket: &str,
key: &str
) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
let response = client.get_object()
.bucket(bucket)
.key(key)
.send()
.await?;
let bytes = response.body.collect().await?.into_bytes().to_vec();
Ok(bytes)
}
impl MinIOHandler { impl MinIOHandler {
pub fn new(state: Arc<AppState>) -> Self { pub fn new(state: Arc<AppState>) -> Self {
let client = state.s3_client.as_ref().expect("S3 client must be initialized").clone();
Self { Self {
state, state: Arc::clone(&state),
s3: Arc::new(client),
watched_prefixes: Arc::new(tokio::sync::RwLock::new(Vec::new())), watched_prefixes: Arc::new(tokio::sync::RwLock::new(Vec::new())),
file_states: Arc::new(tokio::sync::RwLock::new(HashMap::new())), file_states: Arc::new(tokio::sync::RwLock::new(HashMap::new())),
} }
@ -61,16 +78,9 @@ impl MinIOHandler {
&self, &self,
callback: &Arc<dyn Fn(FileChangeEvent) + Send + Sync>, callback: &Arc<dyn Fn(FileChangeEvent) + Send + Sync>,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let op = match &self.state.s3_operator {
Some(op) => op,
None => {
return Ok(());
}
};
let prefixes = self.watched_prefixes.read().await; let prefixes = self.watched_prefixes.read().await;
for prefix in prefixes.iter() { for prefix in prefixes.iter() {
if let Err(e) = self.check_prefix_changes(op, prefix, callback).await { if let Err(e) = self.check_prefix_changes(&self.s3, prefix, callback).await {
error!("Error checking prefix {}: {}", prefix, e); error!("Error checking prefix {}: {}", prefix, e);
} }
} }
@ -79,28 +89,41 @@ impl MinIOHandler {
async fn check_prefix_changes( async fn check_prefix_changes(
&self, &self,
op: &Operator, client: &Client,
prefix: &str, prefix: &str,
callback: &Arc<dyn Fn(FileChangeEvent) + Send + Sync>, callback: &Arc<dyn Fn(FileChangeEvent) + Send + Sync>,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let mut current_files = HashMap::new(); let mut current_files = HashMap::new();
let mut lister = op.lister_with(prefix).recursive(true).await?; let mut continuation_token = None;
while let Some(entry) = lister.try_next().await? { loop {
let path = entry.path().to_string(); let list_objects = client.list_objects_v2()
.bucket(&self.state.bucket_name)
.prefix(prefix)
.set_continuation_token(continuation_token)
.send()
.await?;
if path.ends_with('/') { for obj in list_objects.contents.unwrap_or_default() {
continue; let path = obj.key().unwrap_or_default().to_string();
if path.ends_with('/') {
continue;
}
let file_state = FileState {
path: path.clone(),
size: obj.size().unwrap_or(0),
etag: obj.e_tag().unwrap_or_default().to_string(),
last_modified: obj.last_modified().map(|dt| dt.to_string()),
};
current_files.insert(path, file_state);
} }
let meta = op.stat(&path).await?; if !list_objects.is_truncated.unwrap_or(false) {
let file_state = FileState { break;
path: path.clone(), }
size: meta.content_length() as i64, continuation_token = list_objects.next_continuation_token;
etag: meta.etag().unwrap_or_default().to_string(),
last_modified: meta.last_modified().map(|dt| dt.to_rfc3339()),
};
current_files.insert(path, file_state);
} }
let mut file_states = self.file_states.write().await; let mut file_states = self.file_states.write().await;
@ -146,7 +169,7 @@ impl MinIOHandler {
pub async fn get_file_state(&self, path: &str) -> Option<FileState> { pub async fn get_file_state(&self, path: &str) -> Option<FileState> {
let states = self.file_states.read().await; let states = self.file_states.read().await;
states.get(path).cloned() states.get(&path.to_string()).cloned()
} }
pub async fn clear_state(&self) { pub async fn clear_state(&self) {

View file

@ -1,7 +1,8 @@
use crate::shared::models::KBCollection; use crate::shared::models::KBCollection;
use crate::shared::state::AppState; use crate::shared::state::AppState;
use log::{ error, info, warn}; use log::{ error, info, warn};
use tokio_stream::StreamExt; // Removed unused import
// Removed duplicate import since we're using the module directly
use std::collections::HashMap; use std::collections::HashMap;
use std::error::Error; use std::error::Error;
use std::sync::Arc; use std::sync::Arc;
@ -95,35 +96,16 @@ impl KBManager {
&self, &self,
collection: &KBCollection, collection: &KBCollection,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let op = match &self.state.s3_operator { let _client = match &self.state.s3_client {
Some(op) => op, Some(client) => client,
None => { None => {
warn!("S3 operator not configured"); warn!("S3 client not configured");
return Ok(()); return Ok(());
} }
}; };
let mut lister = op.lister_with(&collection.folder_path).recursive(true).await?; let minio_handler = minio_handler::MinIOHandler::new(self.state.clone());
while let Some(entry) = lister.try_next().await? { minio_handler.watch_prefix(collection.folder_path.clone()).await;
let path = entry.path().to_string();
if path.ends_with('/') {
continue;
}
let meta = op.stat(&path).await?;
if let Err(e) = self
.process_file(
&collection,
&path,
meta.content_length() as i64,
meta.last_modified().map(|dt| dt.to_rfc3339()),
)
.await
{
error!("Error processing file {}: {}", path, e);
}
}
Ok(()) Ok(())
} }
@ -135,7 +117,8 @@ impl KBManager {
file_size: i64, file_size: i64,
_last_modified: Option<String>, _last_modified: Option<String>,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let content = self.get_file_content(file_path).await?; let client = self.state.s3_client.as_ref().ok_or("S3 client not configured")?;
let content = minio_handler::get_file_content(client, &self.state.bucket_name, file_path).await?;
let file_hash = if content.len() > 100 { let file_hash = if content.len() > 100 {
format!( format!(
"{:x}_{:x}_{}", "{:x}_{:x}_{}",
@ -183,20 +166,6 @@ impl KBManager {
Ok(()) Ok(())
} }
async fn get_file_content(
&self,
file_path: &str,
) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
let op = self
.state
.s3_operator
.as_ref()
.ok_or("S3 operator not configured")?;
let content = op.read(file_path).await?;
Ok(content.to_vec())
}
async fn extract_text( async fn extract_text(
&self, &self,
file_path: &str, file_path: &str,

View file

@ -36,6 +36,7 @@ mod tools;
mod web_automation; mod web_automation;
mod web_server; mod web_server;
mod whatsapp; mod whatsapp;
mod create_bucket;
use crate::auth::auth_handler; use crate::auth::auth_handler;
use crate::automation::AutomationService; use crate::automation::AutomationService;
@ -43,7 +44,6 @@ use crate::bootstrap::BootstrapManager;
use crate::bot::{start_session, websocket_handler}; use crate::bot::{start_session, websocket_handler};
use crate::channels::{VoiceAdapter, WebChannelAdapter}; use crate::channels::{VoiceAdapter, WebChannelAdapter};
use crate::config::AppConfig; use crate::config::AppConfig;
use crate::drive_monitor::DriveMonitor;
#[cfg(feature = "email")] #[cfg(feature = "email")]
use crate::email::{ use crate::email::{
get_emails, get_latest_email_from, list_emails, save_click, save_draft, send_email, get_emails, get_latest_email_from, list_emails, save_click, save_draft, send_email,
@ -59,10 +59,17 @@ use crate::shared::state::AppState;
use crate::web_server::{bot_index, index, static_files}; use crate::web_server::{bot_index, index, static_files};
use crate::whatsapp::whatsapp_webhook_verify; use crate::whatsapp::whatsapp_webhook_verify;
use crate::whatsapp::WhatsAppAdapter; use crate::whatsapp::WhatsAppAdapter;
use crate::bot::BotOrchestrator;
#[cfg(not(feature = "desktop"))] #[cfg(not(feature = "desktop"))]
#[tokio::main] #[tokio::main]
async fn main() -> std::io::Result<()> { async fn main() -> std::io::Result<()> {
// Test bucket creation
match create_bucket::create_bucket("test-bucket") {
Ok(_) => println!("Bucket created successfully"),
Err(e) => eprintln!("Failed to create bucket: {}", e),
}
let args: Vec<String> = std::env::args().collect(); let args: Vec<String> = std::env::args().collect();
if args.len() > 1 { if args.len() > 1 {
let command = &args[1]; let command = &args[1];
@ -89,6 +96,7 @@ async fn main() -> std::io::Result<()> {
} }
} }
// Rest of the original main function remains unchanged...
dotenv().ok(); dotenv().ok();
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
.write_style(env_logger::WriteStyle::Always) .write_style(env_logger::WriteStyle::Always)
@ -106,7 +114,7 @@ async fn main() -> std::io::Result<()> {
None None
}; };
let mut bootstrap = BootstrapManager::new(install_mode.clone(), tenant.clone()); let mut bootstrap = BootstrapManager::new(install_mode.clone(), tenant.clone()).await;
// Prevent double bootstrap: skip if environment already initialized // Prevent double bootstrap: skip if environment already initialized
let env_path = std::env::current_dir()?.join("botserver-stack").join(".env"); let env_path = std::env::current_dir()?.join("botserver-stack").join(".env");
@ -120,7 +128,7 @@ async fn main() -> std::io::Result<()> {
Err(_) => AppConfig::from_env(), Err(_) => AppConfig::from_env(),
} }
} else { } else {
match bootstrap.bootstrap() { match bootstrap.bootstrap().await {
Ok(config) => { Ok(config) => {
info!("Bootstrap completed successfully"); info!("Bootstrap completed successfully");
config config
@ -138,9 +146,13 @@ async fn main() -> std::io::Result<()> {
} }
}; };
// Start all services (synchronous)
if let Err(e) = bootstrap.start_all() {
log::warn!("Failed to start all services: {}", e);
}
let _ = bootstrap.start_all(); // Upload templates (asynchronous)
if let Err(e) = bootstrap.upload_templates_to_drive(&cfg).await { if let Err(e) = futures::executor::block_on(bootstrap.upload_templates_to_drive(&cfg)) {
log::warn!("Failed to upload templates to MinIO: {}", e); log::warn!("Failed to upload templates to MinIO: {}", e);
} }
@ -193,7 +205,6 @@ async fn main() -> std::io::Result<()> {
)); ));
let tool_api = Arc::new(tools::ToolApi::new()); let tool_api = Arc::new(tools::ToolApi::new());
let drive = init_drive(&config.drive) let drive = init_drive(&config.drive)
.await .await
.expect("Failed to initialize Drive"); .expect("Failed to initialize Drive");
@ -209,9 +220,10 @@ async fn main() -> std::io::Result<()> {
))); )));
let app_state = Arc::new(AppState { let app_state = Arc::new(AppState {
s3_operator: Some(drive.clone()), s3_client: Some(drive),
config: Some(cfg.clone()), config: Some(cfg.clone()),
conn: db_pool.clone(), conn: db_pool.clone(),
bucket_name: "default.gbai".to_string(), // Default bucket name
custom_conn: db_custom_pool.clone(), custom_conn: db_custom_pool.clone(),
redis_client: redis_client.clone(), redis_client: redis_client.clone(),
session_manager: session_manager.clone(), session_manager: session_manager.clone(),
@ -246,18 +258,20 @@ async fn main() -> std::io::Result<()> {
.expect("Failed to create runtime for automation"); .expect("Failed to create runtime for automation");
let local = tokio::task::LocalSet::new(); let local = tokio::task::LocalSet::new();
local.block_on(&rt, async move { local.block_on(&rt, async move {
let bot_guid = std::env::var("BOT_GUID").unwrap_or_else(|_| "default_bot".to_string()); let scripts_dir = "work/default.gbai/.gbdialog".to_string();
let scripts_dir = format!("work/{}.gbai/.gbdialog", bot_guid);
let automation = AutomationService::new(automation_state, &scripts_dir); let automation = AutomationService::new(automation_state, &scripts_dir);
automation.spawn().await.ok(); automation.spawn().await.ok();
}); });
}); });
let drive_state = app_state.clone(); // Initialize bot orchestrator and mount all bots
let bot_guid = std::env::var("BOT_GUID").unwrap_or_else(|_| "default_bot".to_string()); let bot_orchestrator = BotOrchestrator::new(app_state.clone());
let bucket_name = format!("{}{}.gbai", cfg.drive.org_prefix, bot_guid);
let drive_monitor = Arc::new(DriveMonitor::new(drive_state, bucket_name)); // Mount all active bots from database
let _drive_handle = drive_monitor.spawn(); if let Err(e) = bot_orchestrator.mount_all_bots().await {
log::error!("Failed to mount bots: {}", e);
}
HttpServer::new(move || { HttpServer::new(move || {
let cors = Cors::default() let cors = Cors::default()
@ -271,24 +285,20 @@ async fn main() -> std::io::Result<()> {
.wrap(cors) .wrap(cors)
.wrap(Logger::default()) .wrap(Logger::default())
.wrap(Logger::new("HTTP REQUEST: %a %{User-Agent}i")) .wrap(Logger::new("HTTP REQUEST: %a %{User-Agent}i"))
.app_data(web::Data::from(app_state_clone)); .app_data(web::Data::from(app_state_clone))
app = app
.service(upload_file)
.service(index)
.service(static_files)
.service(websocket_handler)
.service(auth_handler) .service(auth_handler)
.service(whatsapp_webhook_verify) .service(chat_completions_local)
.service(create_session)
.service(embeddings_local)
.service(get_session_history)
.service(get_sessions)
.service(index)
.service(start_session)
.service(upload_file)
.service(voice_start) .service(voice_start)
.service(voice_stop) .service(voice_stop)
.service(create_session) .service(whatsapp_webhook_verify)
.service(get_sessions) .service(websocket_handler);
.service(start_session)
.service(get_session_history)
.service(chat_completions_local)
.service(embeddings_local)
.service(bot_index); // Must be last - catches all remaining paths
#[cfg(feature = "email")] #[cfg(feature = "email")]
{ {
@ -299,9 +309,12 @@ async fn main() -> std::io::Result<()> {
.service(send_email) .service(send_email)
.service(save_draft) .service(save_draft)
.service(save_click); .service(save_click);
}
}
app = app.service(static_files);
app = app.service(bot_index);
app app
}) })
.workers(worker_count) .workers(worker_count)
.bind((config.server.host.clone(), config.server.port))? .bind((config.server.host.clone(), config.server.port))?

View file

@ -89,17 +89,9 @@ impl PackageManager {
), ),
binary_name: Some("minio".to_string()), binary_name: Some("minio".to_string()),
pre_install_cmds_linux: vec![], pre_install_cmds_linux: vec![],
post_install_cmds_linux: vec![ post_install_cmds_linux: vec![],
"wget https://dl.min.io/client/mc/release/linux-amd64/mc -O {{BIN_PATH}}/mc"
.to_string(),
"chmod +x {{BIN_PATH}}/mc".to_string(),
],
pre_install_cmds_macos: vec![], pre_install_cmds_macos: vec![],
post_install_cmds_macos: vec![ post_install_cmds_macos: vec![],
"wget https://dl.min.io/client/mc/release/darwin-amd64/mc -O {{BIN_PATH}}/mc"
.to_string(),
"chmod +x {{BIN_PATH}}/mc".to_string(),
],
pre_install_cmds_windows: vec![], pre_install_cmds_windows: vec![],
post_install_cmds_windows: vec![], post_install_cmds_windows: vec![],
env_vars: HashMap::from([ env_vars: HashMap::from([
@ -107,7 +99,7 @@ impl PackageManager {
("DRIVE_ROOT_PASSWORD".to_string(), drive_password.clone()), ("DRIVE_ROOT_PASSWORD".to_string(), drive_password.clone()),
]), ]),
data_download_list: Vec::new(), data_download_list: Vec::new(),
exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 > {{LOGS_PATH}}/minio.log 2>&1 & sleep 5 && {{BIN_PATH}}/mc alias set drive http://localhost:9000 minioadmin minioadmin && {{BIN_PATH}}/mc admin user add drive $DRIVE_ROOT_USER $DRIVE_ROOT_PASSWORD && {{BIN_PATH}}/mc admin policy attach drive readwrite --user $DRIVE_ROOT_USER && {{BIN_PATH}}/mc mb drive/default.gbai || true".to_string(), exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(),
}, },
); );

View file

@ -395,7 +395,7 @@ async fn create_session(data: web::Data<AppState>) -> Result<HttpResponse> {
#[actix_web::get("/api/sessions")] #[actix_web::get("/api/sessions")]
async fn get_sessions(data: web::Data<AppState>) -> Result<HttpResponse> { async fn get_sessions(data: web::Data<AppState>) -> Result<HttpResponse> {
let user_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap(); let user_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
let orchestrator = BotOrchestrator::new(Arc::clone(&data)); let orchestrator = BotOrchestrator::new(Arc::new(data.get_ref().clone()));
match orchestrator.get_user_sessions(user_id).await { match orchestrator.get_user_sessions(user_id).await {
Ok(sessions) => Ok(HttpResponse::Ok().json(sessions)), Ok(sessions) => Ok(HttpResponse::Ok().json(sessions)),
Err(e) => { Err(e) => {
@ -416,7 +416,7 @@ async fn get_session_history(
match Uuid::parse_str(&session_id) { match Uuid::parse_str(&session_id) {
Ok(session_uuid) => { Ok(session_uuid) => {
let orchestrator = BotOrchestrator::new(Arc::clone(&data)); let orchestrator = BotOrchestrator::new(Arc::new(data.get_ref().clone()));
match orchestrator match orchestrator
.get_conversation_history(session_uuid, user_id) .get_conversation_history(session_uuid, user_id)
.await .await

View file

@ -6,8 +6,8 @@ use crate::session::SessionManager;
use crate::tools::{ToolApi, ToolManager}; use crate::tools::{ToolApi, ToolManager};
use crate::whatsapp::WhatsAppAdapter; use crate::whatsapp::WhatsAppAdapter;
use diesel::{Connection, PgConnection}; use diesel::{Connection, PgConnection};
use opendal::Operator; use aws_sdk_s3::Client as S3Client;
use redis::Client; use redis::Client as RedisClient;
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use std::sync::Mutex; use std::sync::Mutex;
@ -15,11 +15,12 @@ use tokio::sync::mpsc;
use crate::shared::models::BotResponse; use crate::shared::models::BotResponse;
pub struct AppState { pub struct AppState {
pub s3_operator: Option<Operator>, pub s3_client: Option<S3Client>,
pub bucket_name: String,
pub config: Option<AppConfig>, pub config: Option<AppConfig>,
pub conn: Arc<Mutex<PgConnection>>, pub conn: Arc<Mutex<PgConnection>>,
pub custom_conn: Arc<Mutex<PgConnection>>, pub custom_conn: Arc<Mutex<PgConnection>>,
pub redis_client: Option<Arc<Client>>, pub redis_client: Option<Arc<RedisClient>>,
pub session_manager: Arc<tokio::sync::Mutex<SessionManager>>, pub session_manager: Arc<tokio::sync::Mutex<SessionManager>>,
pub tool_manager: Arc<ToolManager>, pub tool_manager: Arc<ToolManager>,
pub llm_provider: Arc<dyn LLMProvider>, pub llm_provider: Arc<dyn LLMProvider>,
@ -35,7 +36,8 @@ pub struct AppState {
impl Clone for AppState { impl Clone for AppState {
fn clone(&self) -> Self { fn clone(&self) -> Self {
Self { Self {
s3_operator: self.s3_operator.clone(), s3_client: self.s3_client.clone(),
bucket_name: self.bucket_name.clone(),
config: self.config.clone(), config: self.config.clone(),
conn: Arc::clone(&self.conn), conn: Arc::clone(&self.conn),
custom_conn: Arc::clone(&self.custom_conn), custom_conn: Arc::clone(&self.custom_conn),
@ -57,7 +59,8 @@ impl Clone for AppState {
impl Default for AppState { impl Default for AppState {
fn default() -> Self { fn default() -> Self {
Self { Self {
s3_operator: None, s3_client: None,
bucket_name: "default.gbai".to_string(),
config: None, config: None,
conn: Arc::new(Mutex::new( conn: Arc::new(Mutex::new(
diesel::PgConnection::establish("postgres://localhost/test").unwrap(), diesel::PgConnection::establish("postgres://localhost/test").unwrap(),

View file

@ -13,6 +13,7 @@ use std::io::Write;
use std::str::FromStr; use std::str::FromStr;
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
#[tokio::test] #[tokio::test]
async fn test_successful_file_upload() -> Result<()> { async fn test_successful_file_upload() -> Result<()> {
// Setup test environment and MinIO client // Setup test environment and MinIO client

View file

@ -26,7 +26,7 @@ async fn bot_index(req: HttpRequest) -> Result<HttpResponse> {
} }
} }
#[actix_web::get("/{filename:.*}")] #[actix_web::get("/static/{filename:.*}")]
async fn static_files(req: HttpRequest) -> Result<HttpResponse> { async fn static_files(req: HttpRequest) -> Result<HttpResponse> {
let filename = req.match_info().query("filename"); let filename = req.match_info().query("filename");
let path = format!("web/html/{}", filename); let path = format!("web/html/{}", filename);