Merge branch 'main' of https://github.com/ChristopherCastilho/botserver
This commit is contained in:
commit
52e2f79395
28 changed files with 1237 additions and 5673 deletions
4898
Cargo.lock
generated
4898
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
40
Cargo.toml
40
Cargo.toml
|
|
@ -37,74 +37,66 @@ license = "AGPL-3.0"
|
|||
repository = "https://github.com/GeneralBots/BotServer"
|
||||
|
||||
[features]
|
||||
desktop = ["tauri", "tauri-plugin-opener", "tauri-plugin-dialog"]
|
||||
default = [ "vectordb"]
|
||||
vectordb = ["qdrant-client"]
|
||||
email = ["imap"]
|
||||
web_automation = ["headless_chrome"]
|
||||
webapp = ["tauri", "tauri-plugin-opener", "tauri-plugin-dialog"]
|
||||
desktop = []
|
||||
|
||||
[dependencies]
|
||||
actix-cors = "0.7"
|
||||
aws-config = "0.57.0"
|
||||
csv = "1.3"
|
||||
actix-multipart = "0.7"
|
||||
imap = { version = "3.0.0-alpha.15", optional = true }
|
||||
actix-web = "4.9"
|
||||
actix-ws = "0.3"
|
||||
aes-gcm = "0.10"
|
||||
anyhow = "1.0"
|
||||
argon2 = "0.5"
|
||||
async-stream = "0.3"
|
||||
async-trait = "0.1"
|
||||
aes-gcm = "0.10"
|
||||
argon2 = "0.5"
|
||||
base64 = "0.22"
|
||||
bytes = "1.8"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
csv = "1.3"
|
||||
diesel = { version = "2.1", features = ["postgres", "uuid", "chrono", "serde_json"] }
|
||||
dotenvy = "0.15"
|
||||
downloader = "0.2"
|
||||
env_logger = "0.11"
|
||||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
headless_chrome = { version = "1.0.18", optional = true }
|
||||
imap = { version = "3.0.0-alpha.15", optional = true }
|
||||
include_dir = "0.7"
|
||||
indicatif = "0.18.0"
|
||||
lettre = { version = "0.11", features = ["smtp-transport", "builder", "tokio1", "tokio1-native-tls"] }
|
||||
livekit = "0.7"
|
||||
include_dir = "0.7"
|
||||
log = "0.4"
|
||||
mailparse = "0.15"
|
||||
native-tls = "0.2"
|
||||
num-format = "0.4"
|
||||
opendal = { version = "0.54.1", features = ["services-s3"] }
|
||||
pdf-extract = "0.10.0"
|
||||
qdrant-client = { version = "1.12", optional = true }
|
||||
rhai = { git = "https://github.com/therealprof/rhai.git", branch = "features/use-web-time" }
|
||||
rand = "0.9.2"
|
||||
redis = { version = "0.27", features = ["tokio-comp"] }
|
||||
regex = "1.11"
|
||||
reqwest = { version = "0.12", features = ["json", "stream"] }
|
||||
rhai = { git = "https://github.com/therealprof/rhai.git", branch = "features/use-web-time" }
|
||||
scraper = "0.20"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
sha2 = "0.10.9"
|
||||
smartstring = "1.0"
|
||||
tempfile = "3"
|
||||
time = "0.3.44"
|
||||
tokio = { version = "1.41", features = ["full"] }
|
||||
tokio-stream = "0.1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["fmt"] }
|
||||
ureq = "3.1.2"
|
||||
urlencoding = "2.1"
|
||||
uuid = { version = "1.11", features = ["serde", "v4"] }
|
||||
zip = "2.2"
|
||||
time = "0.3.44"
|
||||
aws-sdk-s3 = { version = "1.108.0", features = ["behavior-version-latest"] }
|
||||
headless_chrome = { version = "1.0.18", optional = true }
|
||||
rand = "0.9.2"
|
||||
pdf-extract = "0.10.0"
|
||||
scraper = "0.20"
|
||||
sha2 = "0.10.9"
|
||||
ureq = "3.1.2"
|
||||
indicatif = "0.18.0"
|
||||
|
||||
tauri = { version = "2", features = ["unstable"], optional = true }
|
||||
tauri-plugin-opener = { version = "2", optional = true }
|
||||
tauri-plugin-dialog = { version = "2", optional = true }
|
||||
|
||||
[build-dependencies]
|
||||
tauri-build = { version = "2", features = [] }
|
||||
|
||||
[profile.release]
|
||||
lto = true # Enables Link-Time Optimization
|
||||
|
|
|
|||
40
add-req.sh
40
add-req.sh
|
|
@ -19,26 +19,28 @@ for file in "${prompts[@]}"; do
|
|||
done
|
||||
|
||||
dirs=(
|
||||
"auth"
|
||||
"automation"
|
||||
"basic"
|
||||
"bot"
|
||||
"bootstrap"
|
||||
"package_manager"
|
||||
"channels"
|
||||
"config"
|
||||
"context"
|
||||
"email"
|
||||
"file"
|
||||
"llm"
|
||||
"llm_legacy"
|
||||
"org"
|
||||
"session"
|
||||
# "auth"
|
||||
# "automation"
|
||||
# "basic"
|
||||
# "bot"
|
||||
"bootstrap"
|
||||
"package_manager"
|
||||
# "channels"
|
||||
# "config"
|
||||
# "context"
|
||||
# "email"
|
||||
"file"
|
||||
# "llm"
|
||||
"drive_monitor"
|
||||
# "llm_legacy"
|
||||
# "org"
|
||||
# "session"
|
||||
#"kb"
|
||||
"shared"
|
||||
"tests"
|
||||
"tools"
|
||||
"web_automation"
|
||||
"whatsapp"
|
||||
#"tests"
|
||||
# "tools"
|
||||
# "web_automation"
|
||||
# "whatsapp"
|
||||
)
|
||||
|
||||
filter_rust_file() {
|
||||
|
|
|
|||
|
|
@ -1 +1,69 @@
|
|||
# Rust Architecture
|
||||
# Architecture
|
||||
|
||||
## Auto Bootstrap Process Overview
|
||||
|
||||
The Auto Bootstrap process is responsible for initializing and configuring the entire BotServer environment after installation. It ensures that all system components are installed, configured, and started automatically, and that bots are created from predefined templates.
|
||||
|
||||
### 1. Bootstrap Initialization
|
||||
|
||||
The process begins with the `BootstrapManager`, which is instantiated with an installation mode (`Local` or `Container`) and an optional tenant name. It initializes the `PackageManager`, which detects the operating system and sets up the base installation path (e.g., `/opt/gbo` or `botserver-stack`).
|
||||
|
||||
### 2. Component Registration and Installation
|
||||
|
||||
The `PackageManager` registers all system components such as:
|
||||
|
||||
- **tables** (PostgreSQL database)
|
||||
- **cache** (Valkey/Redis)
|
||||
- **drive** (MinIO object storage)
|
||||
- **llm** (local LLM server)
|
||||
- **email**, **proxy**, **directory**, **alm**, **dns**, **meeting**, **table_editor**, **doc_editor**, **desktop**, **devtools**, **bot**, **system**, **vector_db**, **host**
|
||||
|
||||
Each component has a `ComponentConfig` defining:
|
||||
- Ports and dependencies
|
||||
- Download URLs and binaries
|
||||
- Pre/post-install commands
|
||||
- Environment variables
|
||||
- Execution commands
|
||||
|
||||
During bootstrap, required components (`tables`, `drive`, `cache`) are installed and started automatically.
|
||||
For example:
|
||||
- The **tables** component generates secure database credentials, writes them to `.env`, and applies SQL migrations to initialize the schema.
|
||||
- The **drive** component creates secure credentials and stores them encrypted in the database.
|
||||
|
||||
### 3. Bot Configuration
|
||||
|
||||
After components are installed, the bootstrap process updates the bot configuration in the database.
|
||||
The method `update_bot_config()` ensures each component’s configuration is linked to a bot record in the `bot_configuration` table.
|
||||
If no bot exists, a new UUID is generated to associate configuration entries.
|
||||
|
||||
### 4. Template-Based Bot Creation
|
||||
|
||||
The method `create_bots_from_templates()` scans the `templates/` directory for folders ending in `.gbai` (e.g., `default.gbai`, `announcements.gbai`).
|
||||
Each `.gbai` folder represents a bot template.
|
||||
|
||||
For each template:
|
||||
- The folder name is converted into a human-readable bot name (e.g., `default.gbai` → “Default”).
|
||||
- If the bot doesn’t exist in the `bots` table, a new record is inserted with:
|
||||
- Default LLM provider (`openai`)
|
||||
- Default configuration (`{"model": "gpt-4", "temperature": 0.7}`)
|
||||
- Context provider (`database`)
|
||||
- Active status (`true`)
|
||||
|
||||
This automatically creates bots from templates during bootstrap.
|
||||
|
||||
### 5. Template Upload to MinIO
|
||||
|
||||
After bots are created, the method `upload_templates_to_minio()` uploads all template files recursively to a MinIO bucket (S3-compatible storage).
|
||||
This makes templates accessible for runtime bot operations and ensures persistence across environments.
|
||||
|
||||
### 6. Summary
|
||||
|
||||
The Auto Bootstrap process performs the following steps automatically:
|
||||
1. Detects environment and installation mode.
|
||||
2. Registers and installs required components.
|
||||
3. Initializes the database and applies migrations.
|
||||
4. Updates bot configuration records.
|
||||
5. Creates bots from `.gbai` templates.
|
||||
6. Uploads templates to MinIO for storage.
|
||||
|
||||
This process ensures that after installation, the system is fully operational with preconfigured bots derived from templates, ready to serve requests immediately.
|
||||
|
|
|
|||
|
|
@ -1 +1,88 @@
|
|||
# Building from Source
|
||||
# Brave
|
||||
|
||||
sudo apt install brave-browser-beta
|
||||
|
||||
# Rust
|
||||
|
||||
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
source "$HOME/.cargo/env"
|
||||
git clone https://alm.pragmatismo.com.br/generalbots/gbserver
|
||||
|
||||
apt install -y build-essential \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
gcc-multilib \
|
||||
g++-multilib \
|
||||
clang \
|
||||
lld \
|
||||
binutils-dev \
|
||||
libudev-dev \
|
||||
libdbus-1-dev \
|
||||
libva-dev
|
||||
|
||||
apt install -y \
|
||||
curl \
|
||||
git \
|
||||
python3 \
|
||||
python3-pip \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
libasound2-dev \
|
||||
libpulse-dev \
|
||||
libx11-dev \
|
||||
libxext-dev \
|
||||
libxrandr-dev \
|
||||
libxcomposite-dev \
|
||||
libxcursor-dev \
|
||||
libxi-dev \
|
||||
libxtst-dev \
|
||||
libnss3-dev \
|
||||
libnspr4-dev \
|
||||
libatk-bridge2.0-dev \
|
||||
libgtk-3-dev \
|
||||
libudev-dev \
|
||||
libavcodec-dev \
|
||||
libavformat-dev \
|
||||
libavutil-dev \
|
||||
libswscale-dev \
|
||||
libevent-dev \
|
||||
libjsoncpp-dev \
|
||||
libopus-dev \
|
||||
libvpx-dev \
|
||||
libsrtp2-dev \
|
||||
protobuf-compiler \
|
||||
ninja-build \
|
||||
cmake \
|
||||
clang \
|
||||
lld
|
||||
|
||||
|
||||
|
||||
# LLM
|
||||
|
||||
ZED for Windows: https://zed.dev/windows
|
||||
|
||||
Zed Assistant: Groq + GPT OSS 120B |
|
||||
FIX Manual: DeepSeek | ChatGPT 120B | Claude 4.5 Thinking | Mistral
|
||||
ADD Manual: Claude/DeepSeek -> DeepSeek
|
||||
|
||||
# Install
|
||||
|
||||
|
||||
cargo install cargo-audit
|
||||
cargo install cargo-edit
|
||||
apt install -y libpq-dev
|
||||
apt install -y valkey-cli
|
||||
|
||||
# Util
|
||||
|
||||
cargo upgrade
|
||||
cargo audit
|
||||
|
||||
valkey-cli -p 6379 monitor
|
||||
|
||||
# Prompt add-ons
|
||||
|
||||
- Prompt add-ons: Fill the file with info!, trace! and debug! macros.
|
||||
-
|
||||
|
|
@ -18,26 +18,29 @@ for file in "${prompts[@]}"; do
|
|||
done
|
||||
|
||||
dirs=(
|
||||
#"auth"
|
||||
#"automation"
|
||||
#"basic"
|
||||
#"bot"
|
||||
"bootstrap"
|
||||
#"channels"
|
||||
"config"
|
||||
#"context"
|
||||
#"email"
|
||||
#"file"
|
||||
#"llm"
|
||||
#"llm_legacy"
|
||||
#"org"
|
||||
"package_manager"
|
||||
#"session"
|
||||
# "auth"
|
||||
# "automation"
|
||||
#"basic"
|
||||
# "bot"
|
||||
"bootstrap"
|
||||
# "package_manager"
|
||||
# "channels"
|
||||
# "config"
|
||||
# "context"
|
||||
# "email"
|
||||
# "file"
|
||||
# "llm"
|
||||
"drive_monitor"
|
||||
# "llm_legacy"
|
||||
# "org"
|
||||
# "session"
|
||||
"file"
|
||||
"kb"
|
||||
"shared"
|
||||
#"tests"
|
||||
#"tools"
|
||||
#"web_automation"
|
||||
#"whatsapp"
|
||||
# "tools"
|
||||
# "web_automation"
|
||||
# "whatsapp"
|
||||
)
|
||||
for dir in "${dirs[@]}"; do
|
||||
find "$PROJECT_ROOT/src/$dir" -name "*.rs" | while read file; do
|
||||
|
|
@ -51,6 +54,8 @@ done
|
|||
echo "$PROJECT_ROOT/src/main.rs" >> "$OUTPUT_FILE"
|
||||
cat "$PROJECT_ROOT/src/main.rs" >> "$OUTPUT_FILE"
|
||||
|
||||
echo "$PROJECT_ROOT/src/basic/keywords/get.rs" >> "$OUTPUT_FILE"
|
||||
cat "$PROJECT_ROOT/src/basic/keywords/get.rs" >> "$OUTPUT_FILE"
|
||||
|
||||
echo "" >> "$OUTPUT_FILE"
|
||||
echo "Compiling..."
|
||||
|
|
|
|||
2
migrations/6.0.8.sql
Normal file
2
migrations/6.0.8.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE bot_configuration
|
||||
ADD CONSTRAINT bot_configuration_config_key_unique UNIQUE (config_key);
|
||||
|
|
@ -331,8 +331,7 @@ impl AutomationService {
|
|||
e
|
||||
);
|
||||
|
||||
// Try to download from MinIO
|
||||
if let Some(s3_client) = &self.state.s3_client {
|
||||
if let Some(s3_operator) = &self.state.s3_operator {
|
||||
let bucket_name = format!(
|
||||
"{}{}.gbai",
|
||||
env::var("MINIO_ORG_PREFIX").unwrap_or_else(|_| "org1_".to_string()),
|
||||
|
|
@ -342,47 +341,26 @@ impl AutomationService {
|
|||
|
||||
trace!("Downloading from bucket={} key={}", bucket_name, s3_key);
|
||||
|
||||
match s3_client
|
||||
.get_object()
|
||||
.bucket(&bucket_name)
|
||||
.key(&s3_key)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(response) => {
|
||||
match response.body.collect().await {
|
||||
Ok(data) => {
|
||||
match String::from_utf8(data.into_bytes().to_vec()) {
|
||||
Ok(content) => {
|
||||
info!("Downloaded script '{}' from MinIO", param);
|
||||
match s3_operator.read(&format!("{}/{}", bucket_name, s3_key)).await {
|
||||
Ok(data) => {
|
||||
let bytes: Vec<u8> = data.to_vec();
|
||||
match String::from_utf8(bytes) {
|
||||
Ok(content) => {
|
||||
info!("Downloaded script '{}' from MinIO", param);
|
||||
|
||||
// Save to local cache
|
||||
if let Err(e) =
|
||||
std::fs::create_dir_all(&self.scripts_dir)
|
||||
{
|
||||
warn!("Failed to create scripts directory: {}", e);
|
||||
} else if let Err(e) =
|
||||
tokio::fs::write(&full_path, &content).await
|
||||
{
|
||||
warn!("Failed to cache script locally: {}", e);
|
||||
} else {
|
||||
trace!("Cached script to {}", full_path.display());
|
||||
}
|
||||
|
||||
content
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to decode script {}: {}", param, e);
|
||||
self.cleanup_job_flag(&bot_id, param).await;
|
||||
return;
|
||||
}
|
||||
// Save to local cache
|
||||
if let Err(e) = std::fs::create_dir_all(&self.scripts_dir) {
|
||||
warn!("Failed to create scripts directory: {}", e);
|
||||
} else if let Err(e) = tokio::fs::write(&full_path, &content).await {
|
||||
warn!("Failed to cache script locally: {}", e);
|
||||
} else {
|
||||
trace!("Cached script to {}", full_path.display());
|
||||
}
|
||||
|
||||
content
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Failed to read script body from MinIO {}: {}",
|
||||
param, e
|
||||
);
|
||||
error!("Failed to decode script {}: {}", param, e);
|
||||
self.cleanup_job_flag(&bot_id, param).await;
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ pub fn get_keyword(state: Arc<AppState>, _user: UserSession, engine: &mut Engine
|
|||
let state_for_blocking = Arc::clone(&state_clone);
|
||||
let url_for_blocking = url_str.clone();
|
||||
|
||||
// ---- fixed section: spawn on separate thread runtime ----
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
|
|
@ -76,7 +75,6 @@ pub fn get_keyword(state: Arc<AppState>, _user: UserSession, engine: &mut Engine
|
|||
.unwrap();
|
||||
}
|
||||
|
||||
/// Enhanced security check for path traversal and unsafe paths
|
||||
fn is_safe_path(path: &str) -> bool {
|
||||
if path.starts_with("https://") || path.starts_with("http://") {
|
||||
return true;
|
||||
|
|
@ -160,11 +158,11 @@ pub async fn get_from_bucket(
|
|||
return Err("Invalid file path".into());
|
||||
}
|
||||
|
||||
let s3_client = match &state.s3_client {
|
||||
Some(client) => client,
|
||||
let s3_operator = match &state.s3_operator {
|
||||
Some(operator) => operator,
|
||||
None => {
|
||||
error!("S3 client not configured");
|
||||
return Err("S3 client not configured".into());
|
||||
error!("S3 operator not configured");
|
||||
return Err("S3 operator not configured".into());
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -177,7 +175,7 @@ pub async fn get_from_bucket(
|
|||
"App configuration missing".into()
|
||||
})?;
|
||||
|
||||
let org_prefix = &cfg.minio.org_prefix;
|
||||
let org_prefix = &cfg.drive.org_prefix;
|
||||
|
||||
if org_prefix.contains("..") || org_prefix.contains('/') || org_prefix.contains('\\') {
|
||||
error!("Invalid org_prefix: {}", org_prefix);
|
||||
|
|
@ -189,46 +187,22 @@ pub async fn get_from_bucket(
|
|||
bucket
|
||||
};
|
||||
|
||||
match s3_client.head_bucket().bucket(&bucket_name).send().await {
|
||||
Ok(_) => debug!("Bucket exists: {}", bucket_name),
|
||||
Err(e) => {
|
||||
error!("Bucket inaccessible: {} - {}", bucket_name, e);
|
||||
return Err(format!("Bucket inaccessible: {}", e).into());
|
||||
}
|
||||
}
|
||||
|
||||
let get_object_future = s3_client
|
||||
.get_object()
|
||||
.bucket(&bucket_name)
|
||||
.key(file_path)
|
||||
.send();
|
||||
|
||||
let response = match tokio::time::timeout(Duration::from_secs(30), get_object_future).await {
|
||||
let response = match tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
s3_operator.read(&format!("{}/{}", bucket_name, file_path))
|
||||
).await {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(e)) => {
|
||||
error!("S3 get_object failed: {}", e);
|
||||
error!("S3 read failed: {}", e);
|
||||
return Err(format!("S3 operation failed: {}", e).into());
|
||||
}
|
||||
Err(_) => {
|
||||
error!("S3 get_object timed out");
|
||||
error!("S3 read timed out");
|
||||
return Err("S3 operation timed out".into());
|
||||
}
|
||||
};
|
||||
|
||||
let body_future = response.body.collect();
|
||||
let data = match tokio::time::timeout(Duration::from_secs(30), body_future).await {
|
||||
Ok(Ok(data)) => data,
|
||||
Ok(Err(e)) => {
|
||||
error!("Failed to collect S3 response body: {}", e);
|
||||
return Err(format!("Failed to read S3 response: {}", e).into());
|
||||
}
|
||||
Err(_) => {
|
||||
error!("Timeout collecting S3 response body");
|
||||
return Err("Timeout reading S3 response body".into());
|
||||
}
|
||||
};
|
||||
|
||||
let bytes = data.into_bytes().to_vec();
|
||||
let bytes = response.to_vec();
|
||||
debug!(
|
||||
"Retrieved {} bytes from S3 for key: {}",
|
||||
bytes.len(),
|
||||
|
|
|
|||
|
|
@ -1,18 +1,24 @@
|
|||
use crate::config::AppConfig;
|
||||
use crate::package_manager::{ InstallMode, PackageManager };
|
||||
use crate::package_manager::{InstallMode, PackageManager};
|
||||
use anyhow::Result;
|
||||
use diesel::connection::SimpleConnection;
|
||||
use diesel::Connection;
|
||||
use dotenvy::dotenv;
|
||||
use log::{ info, trace, error };
|
||||
use aws_sdk_s3::Client as S3Client;
|
||||
use csv;
|
||||
use diesel::RunQueryDsl;
|
||||
use diesel::{Connection, QueryableByName};
|
||||
use dotenvy::dotenv;
|
||||
use log::{error, info, trace};
|
||||
use opendal::Operator;
|
||||
use rand::distr::Alphanumeric;
|
||||
use sha2::{ Digest, Sha256 };
|
||||
use rand::Rng;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::{self, Write};
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use std::io::{ self, Write };
|
||||
|
||||
#[derive(QueryableByName)]
|
||||
struct BotIdRow {
|
||||
#[diesel(sql_type = diesel::sql_types::Uuid)]
|
||||
id: uuid::Uuid,
|
||||
}
|
||||
|
||||
pub struct ComponentInfo {
|
||||
pub name: &'static str,
|
||||
|
|
@ -22,105 +28,165 @@ pub struct ComponentInfo {
|
|||
pub struct BootstrapManager {
|
||||
pub install_mode: InstallMode,
|
||||
pub tenant: Option<String>,
|
||||
pub s3_operator: Operator,
|
||||
}
|
||||
|
||||
impl BootstrapManager {
|
||||
pub fn new(install_mode: InstallMode, tenant: Option<String>) -> Self {
|
||||
trace!(
|
||||
info!(
|
||||
"Initializing BootstrapManager with mode {:?} and tenant {:?}",
|
||||
install_mode,
|
||||
tenant
|
||||
install_mode, tenant
|
||||
);
|
||||
let config = AppConfig::from_env();
|
||||
let s3_operator = Self::create_s3_operator(&config);
|
||||
Self {
|
||||
install_mode,
|
||||
tenant,
|
||||
s3_operator,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_all(&mut self) -> Result<()> {
|
||||
let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?;
|
||||
let components = vec![
|
||||
ComponentInfo { name: "tables", termination_command: "pg_ctl" },
|
||||
ComponentInfo { name: "cache", termination_command: "valkey-server" },
|
||||
ComponentInfo { name: "drive", termination_command: "minio" },
|
||||
ComponentInfo { name: "llm", termination_command: "llama-server" },
|
||||
ComponentInfo { name: "email", termination_command: "stalwart" },
|
||||
ComponentInfo { name: "proxy", termination_command: "caddy" },
|
||||
ComponentInfo { name: "directory", termination_command: "zitadel" },
|
||||
ComponentInfo { name: "alm", termination_command: "forgejo" },
|
||||
ComponentInfo { name: "alm_ci", termination_command: "forgejo-runner" },
|
||||
ComponentInfo { name: "dns", termination_command: "coredns" },
|
||||
ComponentInfo { name: "webmail", termination_command: "php" },
|
||||
ComponentInfo { name: "meeting", termination_command: "livekit-server" },
|
||||
ComponentInfo { name: "table_editor", termination_command: "nocodb" },
|
||||
ComponentInfo { name: "doc_editor", termination_command: "coolwsd" },
|
||||
ComponentInfo { name: "desktop", termination_command: "xrdp" },
|
||||
ComponentInfo { name: "devtools", termination_command: "" },
|
||||
ComponentInfo { name: "bot", termination_command: "" },
|
||||
ComponentInfo { name: "system", termination_command: "" },
|
||||
ComponentInfo { name: "vector_db", termination_command: "qdrant" },
|
||||
ComponentInfo { name: "host", termination_command: "" }
|
||||
ComponentInfo {
|
||||
name: "tables",
|
||||
termination_command: "pg_ctl",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "cache",
|
||||
termination_command: "valkey-server",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "drive",
|
||||
termination_command: "minio",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "llm",
|
||||
termination_command: "llama-server",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "email",
|
||||
termination_command: "stalwart",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "proxy",
|
||||
termination_command: "caddy",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "directory",
|
||||
termination_command: "zitadel",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "alm",
|
||||
termination_command: "forgejo",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "alm_ci",
|
||||
termination_command: "forgejo-runner",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "dns",
|
||||
termination_command: "coredns",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "webmail",
|
||||
termination_command: "php",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "meeting",
|
||||
termination_command: "livekit-server",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "table_editor",
|
||||
termination_command: "nocodb",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "doc_editor",
|
||||
termination_command: "coolwsd",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "desktop",
|
||||
termination_command: "xrdp",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "devtools",
|
||||
termination_command: "",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "bot",
|
||||
termination_command: "",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "system",
|
||||
termination_command: "",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "vector_db",
|
||||
termination_command: "qdrant",
|
||||
},
|
||||
ComponentInfo {
|
||||
name: "host",
|
||||
termination_command: "",
|
||||
},
|
||||
];
|
||||
|
||||
for component in components {
|
||||
if pm.is_installed(component.name) {
|
||||
trace!("Starting component: {}", component.name);
|
||||
pm.start(component.name)?;
|
||||
} else {
|
||||
trace!("Component {} not installed, skipping start", component.name);
|
||||
if let Err(e) = self.update_bot_config(component.name) {
|
||||
let database_url = std::env::var("DATABASE_URL")
|
||||
.unwrap_or_else(|_| "postgres://gbuser:@localhost:5432/botserver".to_string());
|
||||
let mut conn = diesel::pg::PgConnection::establish(&database_url)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to connect to database: {}", e))?;
|
||||
let default_bot_id: uuid::Uuid = diesel::sql_query("SELECT id FROM bots LIMIT 1")
|
||||
.get_result::<BotIdRow>(&mut conn)
|
||||
.map(|row| row.id)
|
||||
.unwrap_or_else(|_| uuid::Uuid::new_v4());
|
||||
|
||||
if let Err(e) = self.update_bot_config(&default_bot_id, component.name) {
|
||||
error!(
|
||||
"Failed to update bot config after installing {}: {}",
|
||||
component.name,
|
||||
e
|
||||
component.name, e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn bootstrap(&mut self) -> Result<AppConfig> {
|
||||
// Check for legacy mode - if TABLES_SERVER is present, skip bootstrap
|
||||
if let Ok(tables_server) = std::env::var("TABLES_SERVER") {
|
||||
if !tables_server.is_empty() {
|
||||
trace!(
|
||||
info!(
|
||||
"Legacy mode detected (TABLES_SERVER present), skipping bootstrap installation"
|
||||
);
|
||||
info!("Running in legacy mode with existing database configuration");
|
||||
|
||||
// Try to connect to the database and load config
|
||||
let database_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
|
||||
let username = std::env
|
||||
::var("TABLES_USERNAME")
|
||||
.unwrap_or_else(|_| "postgres".to_string());
|
||||
let password = std::env
|
||||
::var("TABLES_PASSWORD")
|
||||
.unwrap_or_else(|_| "postgres".to_string());
|
||||
let server = std::env
|
||||
::var("TABLES_SERVER")
|
||||
.unwrap_or_else(|_| "localhost".to_string());
|
||||
let username =
|
||||
std::env::var("TABLES_USERNAME").unwrap_or_else(|_| "postgres".to_string());
|
||||
let password =
|
||||
std::env::var("TABLES_PASSWORD").unwrap_or_else(|_| "postgres".to_string());
|
||||
let server =
|
||||
std::env::var("TABLES_SERVER").unwrap_or_else(|_| "localhost".to_string());
|
||||
let port = std::env::var("TABLES_PORT").unwrap_or_else(|_| "5432".to_string());
|
||||
let database = std::env
|
||||
::var("TABLES_DATABASE")
|
||||
.unwrap_or_else(|_| "gbserver".to_string());
|
||||
format!("postgres://{}:{}@{}:{}/{}", username, password, server, port, database)
|
||||
let database =
|
||||
std::env::var("TABLES_DATABASE").unwrap_or_else(|_| "gbserver".to_string());
|
||||
format!(
|
||||
"postgres://{}:{}@{}:{}/{}",
|
||||
username, password, server, port, database
|
||||
)
|
||||
});
|
||||
|
||||
match diesel::PgConnection::establish(&database_url) {
|
||||
Ok(mut conn) => {
|
||||
info!("Successfully connected to legacy database, loading configuration");
|
||||
|
||||
// Apply migrations
|
||||
if let Err(e) = self.apply_migrations(&mut conn) {
|
||||
log::warn!("Failed to apply migrations: {}", e);
|
||||
}
|
||||
|
||||
return Ok(AppConfig::from_database(&mut conn));
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!("Failed to connect to legacy database: {}", e);
|
||||
info!("Using environment variables as fallback");
|
||||
return Ok(AppConfig::from_env());
|
||||
}
|
||||
}
|
||||
|
|
@ -133,16 +199,17 @@ impl BootstrapManager {
|
|||
|
||||
for component in required_components {
|
||||
if !pm.is_installed(component) {
|
||||
// Determine termination command from package manager component config
|
||||
let termination_cmd = pm.components
|
||||
let termination_cmd = pm
|
||||
.components
|
||||
.get(component)
|
||||
.and_then(|cfg| cfg.binary_name.clone())
|
||||
.unwrap_or_else(|| component.to_string());
|
||||
|
||||
// If a termination command is defined, check for leftover running process
|
||||
if !termination_cmd.is_empty() {
|
||||
let check = Command::new("pgrep").arg("-f").arg(&termination_cmd).output();
|
||||
|
||||
let check = Command::new("pgrep")
|
||||
.arg("-f")
|
||||
.arg(&termination_cmd)
|
||||
.output();
|
||||
if let Ok(output) = check {
|
||||
if !output.stdout.is_empty() {
|
||||
println!("Component '{}' appears to be already running from a previous install.", component);
|
||||
|
|
@ -157,7 +224,10 @@ impl BootstrapManager {
|
|||
.status();
|
||||
println!("Terminated existing '{}' process.", component);
|
||||
} else {
|
||||
println!("Skipping start of '{}' as it is already running.", component);
|
||||
println!(
|
||||
"Skipping start of '{}' as it is already running.",
|
||||
component
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
|
@ -167,29 +237,20 @@ impl BootstrapManager {
|
|||
if component == "tables" {
|
||||
let db_password = self.generate_secure_password(16);
|
||||
let farm_password = self.generate_secure_password(32);
|
||||
|
||||
let env_contents = format!(
|
||||
"FARM_PASSWORD={}\nDATABASE_URL=postgres://gbuser:{}@localhost:5432/botserver",
|
||||
farm_password,
|
||||
db_password
|
||||
farm_password, db_password
|
||||
);
|
||||
|
||||
std::fs
|
||||
::write(".env", &env_contents)
|
||||
std::fs::write(".env", &env_contents)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to write .env file: {}", e))?;
|
||||
dotenv().ok();
|
||||
trace!("Generated database credentials and wrote to .env file");
|
||||
}
|
||||
|
||||
trace!("Installing required component: {}", component);
|
||||
futures::executor::block_on(pm.install(component))?;
|
||||
|
||||
if component == "tables" {
|
||||
trace!("Component {} installed successfully", component);
|
||||
|
||||
let database_url = std::env::var("DATABASE_URL").unwrap();
|
||||
let mut conn = diesel::PgConnection
|
||||
::establish(&database_url)
|
||||
let mut conn = diesel::PgConnection::establish(&database_url)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to connect to database: {}", e))?;
|
||||
|
||||
let migration_dir = include_dir::include_dir!("./migrations");
|
||||
|
|
@ -197,27 +258,21 @@ impl BootstrapManager {
|
|||
.files()
|
||||
.filter_map(|file| {
|
||||
let path = file.path();
|
||||
trace!("Found file: {:?}", path);
|
||||
if path.extension()? == "sql" {
|
||||
trace!(" -> SQL file included");
|
||||
Some(file)
|
||||
} else {
|
||||
trace!(" -> Not a SQL file, skipping");
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
trace!("Total migration files found: {}", migration_files.len());
|
||||
migration_files.sort_by_key(|f| f.path());
|
||||
|
||||
for migration_file in migration_files {
|
||||
let migration = migration_file
|
||||
.contents_utf8()
|
||||
.ok_or_else(|| anyhow::anyhow!("Migration file is not valid UTF-8"))?;
|
||||
trace!("Executing migration: {}", migration_file.path().display());
|
||||
|
||||
// Use batch_execute to handle multiple statements including those with dollar-quoted strings
|
||||
if let Err(e) = conn.batch_execute(migration) {
|
||||
log::error!(
|
||||
"Failed to execute migration {}: {}",
|
||||
|
|
@ -226,28 +281,35 @@ impl BootstrapManager {
|
|||
);
|
||||
return Err(e.into());
|
||||
}
|
||||
trace!(
|
||||
info!(
|
||||
"Successfully executed migration: {}",
|
||||
migration_file.path().display()
|
||||
);
|
||||
}
|
||||
|
||||
config = AppConfig::from_database(&mut conn);
|
||||
info!("Database migrations completed and configuration loaded");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.s3_operator = Self::create_s3_operator(&config);
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn generate_secure_password(&self, length: usize) -> String {
|
||||
// Ensure the Rng trait is in scope for `sample`
|
||||
use rand::Rng;
|
||||
let mut rng = rand::rng();
|
||||
fn create_s3_operator(config: &AppConfig) -> Operator {
|
||||
use opendal::Scheme;
|
||||
use std::collections::HashMap;
|
||||
let mut map = HashMap::new();
|
||||
map.insert("endpoint".to_string(), config.drive.server.clone());
|
||||
map.insert("access_key_id".to_string(), config.drive.access_key.clone());
|
||||
map.insert("secret_access_key".to_string(), config.drive.secret_key.clone());
|
||||
trace!("Creating S3 operator with endpoint {}", config.drive.server);
|
||||
Operator::via_iter(Scheme::S3, map).expect("Failed to initialize S3 operator")
|
||||
}
|
||||
|
||||
std::iter
|
||||
::repeat_with(|| rng.sample(Alphanumeric) as char)
|
||||
fn generate_secure_password(&self, length: usize) -> String {
|
||||
let mut rng = rand::rng();
|
||||
std::iter::repeat_with(|| rng.sample(Alphanumeric) as char)
|
||||
.take(length)
|
||||
.collect()
|
||||
}
|
||||
|
|
@ -259,207 +321,94 @@ impl BootstrapManager {
|
|||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
/// Update the bot configuration after a component is installed.
|
||||
/// This reads the existing `config.csv` from the default bot bucket,
|
||||
///fix s values based on the installed component, and
|
||||
/// writes the updated CSV back to the bucket. It also upserts the
|
||||
/// key/value pairs into the `bot_config` table.
|
||||
fn update_bot_config(&self, component: &str) -> Result<()> {
|
||||
// Determine bucket name: DRIVE_ORG_PREFIX + "default.gbai"
|
||||
let org_prefix = std::env
|
||||
::var("DRIVE_ORG_PREFIX")
|
||||
.unwrap_or_else(|_| "pragmatismo-".to_string());
|
||||
let bucket_name = format!("{}default.gbai", org_prefix);
|
||||
let config_key = "default.gbot/config.csv";
|
||||
|
||||
// Build S3 client using default SDK config (compatible with S3Client)
|
||||
let s3_client = S3Client::from_conf(aws_sdk_s3::Config::builder().build());
|
||||
|
||||
// Attempt to download existing config.csv
|
||||
let existing_csv = match
|
||||
futures::executor::block_on(
|
||||
s3_client.get_object().bucket(&bucket_name).key(config_key).send()
|
||||
)
|
||||
{
|
||||
Ok(resp) => {
|
||||
let data = futures::executor::block_on(resp.body.collect())?;
|
||||
String::from_utf8(data.into_bytes().to_vec()).unwrap_or_default()
|
||||
}
|
||||
Err(_) => String::new(), // No existing file – start fresh
|
||||
};
|
||||
|
||||
// Parse CSV into a map
|
||||
let mut config_map: std::collections::HashMap<
|
||||
String,
|
||||
String
|
||||
> = std::collections::HashMap::new();
|
||||
if !existing_csv.is_empty() {
|
||||
let mut rdr = csv::ReaderBuilder
|
||||
::new()
|
||||
.has_headers(false)
|
||||
.from_reader(existing_csv.as_bytes());
|
||||
for result in rdr.records() {
|
||||
if let Ok(record) = result {
|
||||
if record.len() >= 2 {
|
||||
config_map.insert(record[0].to_string(), record[1].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update configuration based on the installed component
|
||||
config_map.insert(component.to_string(), "true".to_string());
|
||||
|
||||
// Serialize back to CSV
|
||||
let mut wtr = csv::WriterBuilder
|
||||
::new()
|
||||
.has_headers(false)
|
||||
.from_writer(vec![]);
|
||||
for (k, v) in &config_map {
|
||||
wtr.write_record(&[k, v])?;
|
||||
}
|
||||
wtr.flush()?;
|
||||
let csv_bytes = wtr.into_inner()?;
|
||||
|
||||
// Upload updated CSV to S3
|
||||
futures::executor::block_on(
|
||||
s3_client
|
||||
.put_object()
|
||||
.bucket(&bucket_name)
|
||||
.key(config_key)
|
||||
.body(csv_bytes.clone().into())
|
||||
.send()
|
||||
)?;
|
||||
|
||||
// Upsert into bot_config table
|
||||
let database_url = std::env
|
||||
::var("DATABASE_URL")
|
||||
fn update_bot_config(&self, bot_id: &uuid::Uuid, component: &str) -> Result<()> {
|
||||
use diesel::sql_types::{Text, Uuid as SqlUuid};
|
||||
let database_url = std::env::var("DATABASE_URL")
|
||||
.unwrap_or_else(|_| "postgres://gbuser:@localhost:5432/botserver".to_string());
|
||||
let mut conn = diesel::pg::PgConnection::establish(&database_url)?;
|
||||
|
||||
for (k, v) in config_map {
|
||||
diesel
|
||||
::sql_query(
|
||||
"INSERT INTO bot_config (key, value) VALUES ($1, $2) \
|
||||
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value"
|
||||
)
|
||||
.bind::<diesel::sql_types::Text, _>(&k)
|
||||
.bind::<diesel::sql_types::Text, _>(&v)
|
||||
.execute(&mut conn)?;
|
||||
}
|
||||
// Ensure globally unique keys and update values atomically
|
||||
let config_key = format!("{}_{}", bot_id, component);
|
||||
let config_value = "true".to_string();
|
||||
let new_id = uuid::Uuid::new_v4();
|
||||
|
||||
diesel::sql_query(
|
||||
"INSERT INTO bot_configuration (id, bot_id, config_key, config_value, config_type)
|
||||
VALUES ($1, $2, $3, $4, 'string')
|
||||
ON CONFLICT (config_key)
|
||||
DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()",
|
||||
)
|
||||
.bind::<SqlUuid, _>(new_id)
|
||||
.bind::<SqlUuid, _>(bot_id)
|
||||
.bind::<Text, _>(&config_key)
|
||||
.bind::<Text, _>(&config_value)
|
||||
.execute(&mut conn)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn upload_templates_to_minio(&self, config: &AppConfig) -> Result<()> {
|
||||
use aws_sdk_s3::config::Credentials;
|
||||
use aws_sdk_s3::config::Region;
|
||||
|
||||
info!("Uploading template bots to MinIO and creating bot entries...");
|
||||
|
||||
// First, create bot entries in database for each template
|
||||
pub async fn upload_templates_to_drive(&self, config: &AppConfig) -> Result<()> {
|
||||
let database_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| config.database_url());
|
||||
let mut conn = diesel::PgConnection::establish(&database_url)?;
|
||||
self.create_bots_from_templates(&mut conn)?;
|
||||
|
||||
let creds = Credentials::new(
|
||||
&config.minio.access_key,
|
||||
&config.minio.secret_key,
|
||||
None,
|
||||
None,
|
||||
"minio"
|
||||
);
|
||||
|
||||
let s3_config = aws_sdk_s3::Config
|
||||
::builder()
|
||||
.credentials_provider(creds)
|
||||
.endpoint_url(&config.minio.server)
|
||||
.region(Region::new("us-east-1"))
|
||||
.force_path_style(true)
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.build();
|
||||
|
||||
let client = aws_sdk_s3::Client::from_conf(s3_config);
|
||||
|
||||
// Upload templates from templates/ directory
|
||||
let templates_dir = Path::new("templates");
|
||||
if !templates_dir.exists() {
|
||||
trace!("Templates directory not found, skipping upload");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Walk through each .gbai folder in templates/
|
||||
let operator = &self.s3_operator;
|
||||
for entry in std::fs::read_dir(templates_dir)? {
|
||||
let bot_name = templates_dir
|
||||
.read_dir()?
|
||||
.filter_map(|e| e.ok())
|
||||
.find(|e| {
|
||||
e.path().is_dir()
|
||||
&& e.path()
|
||||
.file_name()
|
||||
.unwrap()
|
||||
.to_string_lossy()
|
||||
.ends_with(".gbai")
|
||||
})
|
||||
.map(|e| {
|
||||
let name = e.path().file_name().unwrap().to_string_lossy().to_string();
|
||||
name
|
||||
})
|
||||
.unwrap_or_else(|| "default".to_string());
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if
|
||||
path.is_dir() &&
|
||||
path
|
||||
.extension()
|
||||
.map(|e| e == "gbai")
|
||||
.unwrap_or(false)
|
||||
if path.is_dir()
|
||||
&& path
|
||||
.file_name()
|
||||
.unwrap()
|
||||
.to_string_lossy()
|
||||
.ends_with(".gbai")
|
||||
{
|
||||
let bot_name = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
let bucket_name = format!("{}{}", config.minio.org_prefix, bot_name);
|
||||
|
||||
trace!("Creating bucket: {}", bucket_name);
|
||||
|
||||
// Create bucket if it doesn't exist
|
||||
match client.create_bucket().bucket(&bucket_name).send().await {
|
||||
Ok(_) => info!("Created bucket: {}", bucket_name),
|
||||
Err(e) => {
|
||||
let err_str = e.to_string();
|
||||
if
|
||||
err_str.contains("BucketAlreadyOwnedByYou") ||
|
||||
err_str.contains("BucketAlreadyExists")
|
||||
{
|
||||
trace!("Bucket {} already exists", bucket_name);
|
||||
} else {
|
||||
log::warn!("Failed to create bucket {}: {}", bucket_name, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Upload all files recursively
|
||||
self.upload_directory_recursive(&client, &path, &bucket_name, "").await?;
|
||||
info!("Uploaded template bot: {}", bot_name);
|
||||
let bucket = bot_name.clone();
|
||||
info!("Uploading template {} to Drive bucket {}", bot_name, bucket);
|
||||
self.upload_directory_recursive(&operator, &path, &bucket, &bot_name)
|
||||
.await?;
|
||||
info!("Uploaded template {} to Drive bucket {}", bot_name, bucket);
|
||||
}
|
||||
}
|
||||
|
||||
info!("Template bots uploaded successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
fn create_bots_from_templates(&self, conn: &mut diesel::PgConnection) -> Result<()> {
|
||||
use crate::shared::models::schema::bots;
|
||||
use diesel::prelude::*;
|
||||
|
||||
info!("Creating bot entries from template folders...");
|
||||
|
||||
let templates_dir = Path::new("templates");
|
||||
if !templates_dir.exists() {
|
||||
trace!("Templates directory not found, skipping bot creation");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Walk through each .gbai folder in templates/
|
||||
for entry in std::fs::read_dir(templates_dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if
|
||||
path.is_dir() &&
|
||||
path
|
||||
.extension()
|
||||
.map(|e| e == "gbai")
|
||||
.unwrap_or(false)
|
||||
{
|
||||
if path.is_dir() && path.extension().map(|e| e == "gbai").unwrap_or(false) {
|
||||
let bot_folder = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
// Remove .gbai extension to get bot name
|
||||
let bot_name = bot_folder.trim_end_matches(".gbai");
|
||||
|
||||
// Format the name nicely (capitalize first letter of each word)
|
||||
let formatted_name = bot_name
|
||||
.split('_')
|
||||
.map(|word| {
|
||||
|
|
@ -474,7 +423,6 @@ impl BootstrapManager {
|
|||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
|
||||
// Check if bot already exists
|
||||
let existing: Option<String> = bots::table
|
||||
.filter(bots::name.eq(&formatted_name))
|
||||
.select(bots::name)
|
||||
|
|
@ -482,39 +430,40 @@ impl BootstrapManager {
|
|||
.optional()?;
|
||||
|
||||
if existing.is_none() {
|
||||
// Insert new bot
|
||||
diesel
|
||||
::sql_query(
|
||||
"INSERT INTO bots (id, name, description, llm_provider, llm_config, context_provider, context_config, is_active) \
|
||||
diesel::sql_query(
|
||||
"INSERT INTO bots (id, name, description, llm_provider, llm_config, context_provider, context_config, is_active) \
|
||||
VALUES (gen_random_uuid(), $1, $2, 'openai', '{\"model\": \"gpt-4\", \"temperature\": 0.7}', 'database', '{}', true)"
|
||||
)
|
||||
.bind::<diesel::sql_types::Text, _>(&formatted_name)
|
||||
.bind::<diesel::sql_types::Text, _>(
|
||||
format!("Bot for {} template", bot_name)
|
||||
)
|
||||
.execute(conn)?;
|
||||
|
||||
info!("Created bot entry: {}", formatted_name);
|
||||
)
|
||||
.bind::<diesel::sql_types::Text, _>(&formatted_name)
|
||||
.bind::<diesel::sql_types::Text, _>(format!("Bot for {} template", bot_name))
|
||||
.execute(conn)?;
|
||||
} else {
|
||||
trace!("Bot already exists: {}", formatted_name);
|
||||
log::trace!("Bot {} already exists", formatted_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!("Bot creation from templates completed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn upload_directory_recursive<'a>(
|
||||
&'a self,
|
||||
client: &'a aws_sdk_s3::Client,
|
||||
client: &'a Operator,
|
||||
local_path: &'a Path,
|
||||
bucket: &'a str,
|
||||
prefix: &'a str
|
||||
prefix: &'a str,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {
|
||||
Box::pin(async move {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
|
||||
trace!("Checking bucket existence: {}", bucket);
|
||||
if client.stat(bucket).await.is_err() {
|
||||
info!("Bucket {} not found, creating it", bucket);
|
||||
trace!("Creating bucket: {}", bucket);
|
||||
client.create_dir(bucket).await?;
|
||||
trace!("Bucket {} created successfully", bucket);
|
||||
} else {
|
||||
trace!("Bucket {} already exists", bucket);
|
||||
}
|
||||
trace!("Starting upload from local path: {}", local_path.display());
|
||||
for entry in std::fs::read_dir(local_path)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
|
@ -526,39 +475,41 @@ impl BootstrapManager {
|
|||
};
|
||||
|
||||
if path.is_file() {
|
||||
trace!(
|
||||
info!(
|
||||
"Uploading file: {} to bucket: {} with key: {}",
|
||||
path.display(),
|
||||
bucket,
|
||||
key
|
||||
);
|
||||
|
||||
let body = ByteStream::from_path(&path).await?;
|
||||
|
||||
client.put_object().bucket(bucket).key(&key).body(body).send().await?;
|
||||
|
||||
trace!("Uploaded: {}", key);
|
||||
let content = std::fs::read(&path)?;
|
||||
trace!(
|
||||
"Writing file {} to bucket {} with key {}",
|
||||
path.display(),
|
||||
bucket,
|
||||
key
|
||||
);
|
||||
client.write(&key, content).await?;
|
||||
trace!(
|
||||
"Successfully wrote file {} to bucket {}",
|
||||
path.display(),
|
||||
bucket
|
||||
);
|
||||
} else if path.is_dir() {
|
||||
self.upload_directory_recursive(client, &path, bucket, &key).await?;
|
||||
self.upload_directory_recursive(client, &path, bucket, &key)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn apply_migrations(&self, conn: &mut diesel::PgConnection) -> Result<()> {
|
||||
info!("Applying database migrations...");
|
||||
|
||||
let migrations_dir = std::path::Path::new("migrations");
|
||||
if !migrations_dir.exists() {
|
||||
trace!("No migrations directory found, skipping");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Get all .sql files sorted
|
||||
let mut sql_files: Vec<_> = std::fs
|
||||
::read_dir(migrations_dir)?
|
||||
let mut sql_files: Vec<_> = std::fs::read_dir(migrations_dir)?
|
||||
.filter_map(|entry| entry.ok())
|
||||
.filter(|entry| {
|
||||
entry
|
||||
|
|
@ -575,26 +526,19 @@ impl BootstrapManager {
|
|||
for entry in sql_files {
|
||||
let path = entry.path();
|
||||
let filename = path.file_name().unwrap().to_string_lossy();
|
||||
|
||||
trace!("Reading migration: {}", filename);
|
||||
match std::fs::read_to_string(&path) {
|
||||
Ok(sql) => {
|
||||
trace!("Applying migration: {}", filename);
|
||||
match conn.batch_execute(&sql) {
|
||||
Ok(_) => info!("Applied migration: {}", filename),
|
||||
Err(e) => {
|
||||
// Ignore errors for already applied migrations
|
||||
trace!("Migration {} result: {}", filename, e);
|
||||
}
|
||||
Ok(sql) => match conn.batch_execute(&sql) {
|
||||
Err(e) => {
|
||||
log::warn!("Migration {} failed: {}", filename, e);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Err(e) => {
|
||||
log::warn!("Failed to read migration {}: {}", filename, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!("Migrations check completed");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,14 +8,13 @@ use std::sync::{Arc, Mutex};
|
|||
|
||||
#[derive(Clone)]
|
||||
pub struct AppConfig {
|
||||
pub minio: DriveConfig,
|
||||
pub drive: DriveConfig,
|
||||
pub server: ServerConfig,
|
||||
pub database: DatabaseConfig,
|
||||
pub database_custom: DatabaseConfig,
|
||||
pub email: EmailConfig,
|
||||
pub ai: AIConfig,
|
||||
pub site_path: String,
|
||||
pub s3_bucket: String,
|
||||
pub stack_path: PathBuf,
|
||||
pub db_conn: Option<Arc<Mutex<PgConnection>>>,
|
||||
}
|
||||
|
|
@ -218,7 +217,7 @@ impl AppConfig {
|
|||
};
|
||||
|
||||
AppConfig {
|
||||
minio,
|
||||
drive: minio,
|
||||
server: ServerConfig {
|
||||
host: get_str("SERVER_HOST", "127.0.0.1"),
|
||||
port: get_u16("SERVER_PORT", 8080),
|
||||
|
|
@ -227,7 +226,6 @@ impl AppConfig {
|
|||
database_custom,
|
||||
email,
|
||||
ai,
|
||||
s3_bucket: get_str("DRIVE_BUCKET", "default"),
|
||||
site_path: get_str("SITES_ROOT", "./botserver-stack/sites"),
|
||||
stack_path,
|
||||
db_conn: None,
|
||||
|
|
@ -300,7 +298,7 @@ impl AppConfig {
|
|||
};
|
||||
|
||||
AppConfig {
|
||||
minio,
|
||||
drive: minio,
|
||||
server: ServerConfig {
|
||||
host: std::env::var("SERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
|
||||
port: std::env::var("SERVER_PORT")
|
||||
|
|
@ -312,7 +310,6 @@ impl AppConfig {
|
|||
database_custom,
|
||||
email,
|
||||
ai,
|
||||
s3_bucket: std::env::var("DRIVE_BUCKET").unwrap_or_else(|_| "default".to_string()),
|
||||
site_path: std::env::var("SITES_ROOT")
|
||||
.unwrap_or_else(|_| "./botserver-stack/sites".to_string()),
|
||||
stack_path: PathBuf::from(stack_path),
|
||||
|
|
@ -458,12 +455,14 @@ impl ConfigManager {
|
|||
let key = parts[0].trim();
|
||||
let value = parts[1].trim();
|
||||
|
||||
diesel::sql_query("INSERT INTO bot_configuration (id, bot_id, config_key, config_value, config_type) VALUES (gen_random_uuid()::text, $1, $2, $3, 'string') ON CONFLICT (bot_id, config_key) DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()")
|
||||
.bind::<diesel::sql_types::Uuid, _>(bot_id)
|
||||
.bind::<diesel::sql_types::Text, _>(key)
|
||||
.bind::<diesel::sql_types::Text, _>(value)
|
||||
.execute(&mut *conn)
|
||||
.map_err(|e| format!("Failed to update config: {}", e))?;
|
||||
let new_id: uuid::Uuid = uuid::Uuid::new_v4();
|
||||
diesel::sql_query("INSERT INTO bot_configuration (id, bot_id, config_key, config_value, config_type) VALUES ($1, $2, $3, $4, 'string') ON CONFLICT (bot_id, config_key) DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()")
|
||||
.bind::<diesel::sql_types::Uuid, _>(new_id)
|
||||
.bind::<diesel::sql_types::Uuid, _>(bot_id)
|
||||
.bind::<diesel::sql_types::Text, _>(key)
|
||||
.bind::<diesel::sql_types::Text, _>(value)
|
||||
.execute(&mut *conn)
|
||||
.map_err(|e| format!("Failed to update config: {}", e))?;
|
||||
|
||||
updated += 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,14 +2,13 @@ use crate::basic::compiler::BasicCompiler;
|
|||
use crate::kb::embeddings;
|
||||
use crate::kb::qdrant_client;
|
||||
use crate::shared::state::AppState;
|
||||
use aws_sdk_s3::Client as S3Client;
|
||||
use log::{debug, error, info, warn};
|
||||
use opendal::Operator;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
use tokio::time::{interval, Duration};
|
||||
|
||||
/// Tracks file state for change detection
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileState {
|
||||
pub path: String,
|
||||
|
|
@ -18,7 +17,6 @@ pub struct FileState {
|
|||
pub last_modified: Option<String>,
|
||||
}
|
||||
|
||||
/// Drive monitor that watches for changes and triggers compilation/indexing
|
||||
pub struct DriveMonitor {
|
||||
state: Arc<AppState>,
|
||||
bucket_name: String,
|
||||
|
|
@ -34,18 +32,12 @@ impl DriveMonitor {
|
|||
}
|
||||
}
|
||||
|
||||
/// Start the drive monitoring service
|
||||
pub fn spawn(self: Arc<Self>) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
info!(
|
||||
"Drive Monitor service started for bucket: {}",
|
||||
self.bucket_name
|
||||
);
|
||||
let mut tick = interval(Duration::from_secs(30)); // Check every 30 seconds
|
||||
|
||||
info!("Drive Monitor service started for bucket: {}", self.bucket_name);
|
||||
let mut tick = interval(Duration::from_secs(30));
|
||||
loop {
|
||||
tick.tick().await;
|
||||
|
||||
if let Err(e) = self.check_for_changes().await {
|
||||
error!("Error checking for drive changes: {}", e);
|
||||
}
|
||||
|
|
@ -53,101 +45,65 @@ impl DriveMonitor {
|
|||
})
|
||||
}
|
||||
|
||||
/// Check for file changes in the drive
|
||||
async fn check_for_changes(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let s3_client = match &self.state.s3_client {
|
||||
Some(client) => client,
|
||||
let op = match &self.state.s3_operator {
|
||||
Some(op) => op,
|
||||
None => {
|
||||
debug!("S3 client not configured");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
// Check .gbdialog folder for BASIC tools
|
||||
self.check_gbdialog_changes(s3_client).await?;
|
||||
self.check_gbdialog_changes(op).await?;
|
||||
self.check_gbkb_changes(op).await?;
|
||||
|
||||
// Check .gbkb folder for KB documents
|
||||
self.check_gbkb_changes(s3_client).await?;
|
||||
// Check for default bot configuration in the drive bucket
|
||||
if let Err(e) = self.check_default_gbot(s3_client).await {
|
||||
if let Err(e) = self.check_default_gbot(op).await {
|
||||
error!("Error checking default bot config: {}", e);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check .gbdialog folder for BASIC tool changes
|
||||
async fn check_gbdialog_changes(
|
||||
&self,
|
||||
s3_client: &S3Client,
|
||||
op: &Operator,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let prefix = ".gbdialog/";
|
||||
debug!("Checking {} folder for changes", prefix);
|
||||
|
||||
let mut continuation_token: Option<String> = None;
|
||||
let mut current_files = HashMap::new();
|
||||
|
||||
loop {
|
||||
let mut list_request = s3_client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.bucket_name)
|
||||
.prefix(prefix);
|
||||
let mut lister = op.lister_with(prefix).recursive(true).await?;
|
||||
while let Some(entry) = futures::TryStreamExt::try_next(&mut lister).await? {
|
||||
let path = entry.path().to_string();
|
||||
|
||||
if let Some(token) = continuation_token {
|
||||
list_request = list_request.continuation_token(token);
|
||||
if path.ends_with('/') || !path.ends_with(".bas") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let list_result = list_request.send().await?;
|
||||
|
||||
if let Some(contents) = list_result.contents {
|
||||
for object in contents {
|
||||
if let Some(key) = object.key {
|
||||
// Skip directories and non-.bas files
|
||||
if key.ends_with('/') || !key.ends_with(".bas") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let file_state = FileState {
|
||||
path: key.clone(),
|
||||
size: object.size.unwrap_or(0),
|
||||
etag: object.e_tag.unwrap_or_default(),
|
||||
last_modified: object.last_modified.map(|dt| dt.to_string()),
|
||||
};
|
||||
|
||||
current_files.insert(key, file_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if list_result.is_truncated.unwrap_or(false) {
|
||||
continuation_token = list_result.next_continuation_token;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
let meta = op.stat(&path).await?;
|
||||
let file_state = FileState {
|
||||
path: path.clone(),
|
||||
size: meta.content_length() as i64,
|
||||
etag: meta.etag().unwrap_or_default().to_string(),
|
||||
last_modified: meta.last_modified().map(|dt| dt.to_rfc3339()),
|
||||
};
|
||||
current_files.insert(path, file_state);
|
||||
}
|
||||
|
||||
// Compare with previous state and handle changes
|
||||
let mut file_states = self.file_states.write().await;
|
||||
|
||||
for (path, current_state) in current_files.iter() {
|
||||
if let Some(previous_state) = file_states.get(path) {
|
||||
// File exists, check if modified
|
||||
if current_state.etag != previous_state.etag {
|
||||
info!("BASIC tool modified: {}", path);
|
||||
if let Err(e) = self.compile_tool(s3_client, path).await {
|
||||
if let Err(e) = self.compile_tool(op, path).await {
|
||||
error!("Failed to compile tool {}: {}", path, e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// New file
|
||||
info!("New BASIC tool detected: {}", path);
|
||||
if let Err(e) = self.compile_tool(s3_client, path).await {
|
||||
if let Err(e) = self.compile_tool(op, path).await {
|
||||
error!("Failed to compile tool {}: {}", path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for deleted files
|
||||
let previous_paths: Vec<String> = file_states
|
||||
.keys()
|
||||
.filter(|k| k.starts_with(prefix))
|
||||
|
|
@ -156,13 +112,10 @@ impl DriveMonitor {
|
|||
|
||||
for path in previous_paths {
|
||||
if !current_files.contains_key(&path) {
|
||||
info!("BASIC tool deleted: {}", path);
|
||||
// TODO: Mark tool as inactive in database
|
||||
file_states.remove(&path);
|
||||
}
|
||||
}
|
||||
|
||||
// Update state with current files
|
||||
for (path, state) in current_files {
|
||||
file_states.insert(path, state);
|
||||
}
|
||||
|
|
@ -170,84 +123,52 @@ impl DriveMonitor {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Check .gbkb folder for KB document changes
|
||||
async fn check_gbkb_changes(
|
||||
&self,
|
||||
s3_client: &S3Client,
|
||||
op: &Operator,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let prefix = ".gbkb/";
|
||||
debug!("Checking {} folder for changes", prefix);
|
||||
|
||||
let mut continuation_token: Option<String> = None;
|
||||
let mut current_files = HashMap::new();
|
||||
|
||||
loop {
|
||||
let mut list_request = s3_client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.bucket_name)
|
||||
.prefix(prefix);
|
||||
let mut lister = op.lister_with(prefix).recursive(true).await?;
|
||||
while let Some(entry) = futures::TryStreamExt::try_next(&mut lister).await? {
|
||||
let path = entry.path().to_string();
|
||||
|
||||
if let Some(token) = continuation_token {
|
||||
list_request = list_request.continuation_token(token);
|
||||
if path.ends_with('/') {
|
||||
continue;
|
||||
}
|
||||
|
||||
let list_result = list_request.send().await?;
|
||||
|
||||
if let Some(contents) = list_result.contents {
|
||||
for object in contents {
|
||||
if let Some(key) = object.key {
|
||||
// Skip directories
|
||||
if key.ends_with('/') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Only process supported file types
|
||||
let ext = key.rsplit('.').next().unwrap_or("").to_lowercase();
|
||||
if !["pdf", "txt", "md", "docx"].contains(&ext.as_str()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let file_state = FileState {
|
||||
path: key.clone(),
|
||||
size: object.size.unwrap_or(0),
|
||||
etag: object.e_tag.unwrap_or_default(),
|
||||
last_modified: object.last_modified.map(|dt| dt.to_string()),
|
||||
};
|
||||
|
||||
current_files.insert(key, file_state);
|
||||
}
|
||||
}
|
||||
let ext = path.rsplit('.').next().unwrap_or("").to_lowercase();
|
||||
if !["pdf", "txt", "md", "docx"].contains(&ext.as_str()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if list_result.is_truncated.unwrap_or(false) {
|
||||
continuation_token = list_result.next_continuation_token;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
let meta = op.stat(&path).await?;
|
||||
let file_state = FileState {
|
||||
path: path.clone(),
|
||||
size: meta.content_length() as i64,
|
||||
etag: meta.etag().unwrap_or_default().to_string(),
|
||||
last_modified: meta.last_modified().map(|dt| dt.to_rfc3339()),
|
||||
};
|
||||
current_files.insert(path, file_state);
|
||||
}
|
||||
|
||||
// Compare with previous state and handle changes
|
||||
let mut file_states = self.file_states.write().await;
|
||||
|
||||
for (path, current_state) in current_files.iter() {
|
||||
if let Some(previous_state) = file_states.get(path) {
|
||||
// File exists, check if modified
|
||||
if current_state.etag != previous_state.etag {
|
||||
info!("KB document modified: {}", path);
|
||||
if let Err(e) = self.index_document(s3_client, path).await {
|
||||
if let Err(e) = self.index_document(op, path).await {
|
||||
error!("Failed to index document {}: {}", path, e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// New file
|
||||
info!("New KB document detected: {}", path);
|
||||
if let Err(e) = self.index_document(s3_client, path).await {
|
||||
if let Err(e) = self.index_document(op, path).await {
|
||||
error!("Failed to index document {}: {}", path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for deleted files
|
||||
let previous_paths: Vec<String> = file_states
|
||||
.keys()
|
||||
.filter(|k| k.starts_with(prefix))
|
||||
|
|
@ -256,13 +177,10 @@ impl DriveMonitor {
|
|||
|
||||
for path in previous_paths {
|
||||
if !current_files.contains_key(&path) {
|
||||
info!("KB document deleted: {}", path);
|
||||
// TODO: Delete from Qdrant and mark in database
|
||||
file_states.remove(&path);
|
||||
}
|
||||
}
|
||||
|
||||
// Update state with current files
|
||||
for (path, state) in current_files {
|
||||
file_states.insert(path, state);
|
||||
}
|
||||
|
|
@ -270,76 +188,36 @@ impl DriveMonitor {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Check for default bot configuration in the drive bucket
|
||||
async fn check_default_gbot(
|
||||
&self,
|
||||
s3_client: &S3Client,
|
||||
op: &Operator,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
// The default bot configuration is expected at:
|
||||
// <bucket>/<DRIVE_ORG_PREFIX>default.gbai/default.gbot/config.csv
|
||||
// Construct the expected key prefix
|
||||
let prefix = format!("{}default.gbot/", self.bucket_name);
|
||||
let config_key = format!("{}config.csv", prefix);
|
||||
|
||||
debug!("Checking for default bot config at key: {}", config_key);
|
||||
|
||||
// Attempt to get the object metadata to see if it exists
|
||||
let head_req = s3_client
|
||||
.head_object()
|
||||
.bucket(&self.bucket_name)
|
||||
.key(&config_key)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match head_req {
|
||||
match op.stat(&config_key).await {
|
||||
Ok(_) => {
|
||||
info!("Default bot config found, downloading {}", config_key);
|
||||
// Download the CSV file
|
||||
let get_resp = s3_client
|
||||
.get_object()
|
||||
.bucket(&self.bucket_name)
|
||||
.key(&config_key)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let data = get_resp.body.collect().await?;
|
||||
let csv_content = String::from_utf8(data.into_bytes().to_vec())
|
||||
let content = op.read(&config_key).await?;
|
||||
let csv_content = String::from_utf8(content.to_vec())
|
||||
.map_err(|e| format!("UTF-8 error in config.csv: {}", e))?;
|
||||
|
||||
// Log the retrieved configuration (in a real implementation this would be parsed
|
||||
// and used to populate the bot_config table, respecting overrides from .gbot files)
|
||||
info!("Retrieved default bot config CSV:\n{}", csv_content);
|
||||
// TODO: Parse CSV and upsert into bot_config table with appropriate precedence
|
||||
debug!("Found config.csv: {} bytes", csv_content.len());
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
// If the object does not exist, simply ignore
|
||||
debug!("Default bot config not present: {}", e);
|
||||
debug!("Config file not found or inaccessible: {}", e);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compile a BASIC tool file
|
||||
async fn compile_tool(
|
||||
&self,
|
||||
s3_client: &S3Client,
|
||||
op: &Operator,
|
||||
file_path: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
info!("Compiling BASIC tool: {}", file_path);
|
||||
let content = op.read(file_path).await?;
|
||||
let source_content = String::from_utf8(content.to_vec())?;
|
||||
|
||||
// Download source from S3
|
||||
let get_response = s3_client
|
||||
.get_object()
|
||||
.bucket(&self.bucket_name)
|
||||
.key(file_path)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let data = get_response.body.collect().await?;
|
||||
let source_content = String::from_utf8(data.into_bytes().to_vec())?;
|
||||
|
||||
// Extract tool name
|
||||
let tool_name = file_path
|
||||
.strip_prefix(".gbdialog/")
|
||||
.unwrap_or(file_path)
|
||||
|
|
@ -347,10 +225,6 @@ impl DriveMonitor {
|
|||
.unwrap_or(file_path)
|
||||
.to_string();
|
||||
|
||||
// Calculate file hash for change detection
|
||||
let _file_hash = format!("{:x}", source_content.len());
|
||||
|
||||
// Create work directory using bot from bucket name
|
||||
let bot_name = self
|
||||
.bucket_name
|
||||
.strip_suffix(".gbai")
|
||||
|
|
@ -358,46 +232,31 @@ impl DriveMonitor {
|
|||
let work_dir = format!("./work/{}.gbai/.gbdialog", bot_name);
|
||||
std::fs::create_dir_all(&work_dir)?;
|
||||
|
||||
// Write source to local file
|
||||
let local_source_path = format!("{}/{}.bas", work_dir, tool_name);
|
||||
std::fs::write(&local_source_path, &source_content)?;
|
||||
|
||||
// Compile using BasicCompiler
|
||||
let compiler = BasicCompiler::new(Arc::clone(&self.state));
|
||||
let result = compiler.compile_file(&local_source_path, &work_dir)?;
|
||||
|
||||
info!("Tool compiled successfully: {}", tool_name);
|
||||
info!(" AST: {}", result.ast_path);
|
||||
|
||||
// Save to database
|
||||
if let Some(mcp_tool) = result.mcp_tool {
|
||||
info!(
|
||||
" MCP tool definition generated with {} parameters",
|
||||
"MCP tool definition generated with {} parameters",
|
||||
mcp_tool.input_schema.properties.len()
|
||||
);
|
||||
}
|
||||
|
||||
if result.openai_tool.is_some() {
|
||||
info!(" OpenAI tool definition generated");
|
||||
debug!("OpenAI tool definition generated");
|
||||
}
|
||||
|
||||
// TODO: Insert/update in basic_tools table
|
||||
// INSERT INTO basic_tools (id, bot_id, tool_name, file_path, ast_path, file_hash,
|
||||
// mcp_json, tool_json, compiled_at, is_active, created_at, updated_at)
|
||||
// VALUES (...) ON CONFLICT (bot_id, tool_name) DO UPDATE SET ...
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Index a KB document
|
||||
async fn index_document(
|
||||
&self,
|
||||
s3_client: &S3Client,
|
||||
op: &Operator,
|
||||
file_path: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
info!("Indexing KB document: {}", file_path);
|
||||
|
||||
// Extract collection name from path (.gbkb/collection_name/file.pdf)
|
||||
let parts: Vec<&str> = file_path.split('/').collect();
|
||||
if parts.len() < 3 {
|
||||
warn!("Invalid KB path structure: {}", file_path);
|
||||
|
|
@ -405,21 +264,10 @@ impl DriveMonitor {
|
|||
}
|
||||
|
||||
let collection_name = parts[1];
|
||||
let content = op.read(file_path).await?;
|
||||
let bytes = content.to_vec();
|
||||
|
||||
// Download file from S3
|
||||
let get_response = s3_client
|
||||
.get_object()
|
||||
.bucket(&self.bucket_name)
|
||||
.key(file_path)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let data = get_response.body.collect().await?;
|
||||
let bytes = data.into_bytes().to_vec();
|
||||
|
||||
// Extract text based on file type
|
||||
let text_content = self.extract_text(file_path, &bytes)?;
|
||||
|
||||
if text_content.trim().is_empty() {
|
||||
warn!("No text extracted from: {}", file_path);
|
||||
return Ok(());
|
||||
|
|
@ -431,35 +279,21 @@ impl DriveMonitor {
|
|||
file_path
|
||||
);
|
||||
|
||||
// Create Qdrant collection name
|
||||
let qdrant_collection = format!("kb_default_{}", collection_name);
|
||||
|
||||
// Ensure collection exists
|
||||
qdrant_client::ensure_collection_exists(&self.state, &qdrant_collection).await?;
|
||||
|
||||
// Index document
|
||||
embeddings::index_document(&self.state, &qdrant_collection, file_path, &text_content)
|
||||
.await?;
|
||||
|
||||
info!("Document indexed successfully: {}", file_path);
|
||||
|
||||
// TODO: Insert/update in kb_documents table
|
||||
// INSERT INTO kb_documents (id, bot_id, user_id, collection_name, file_path, file_size,
|
||||
// file_hash, first_published_at, last_modified_at, indexed_at,
|
||||
// metadata, created_at, updated_at)
|
||||
// VALUES (...) ON CONFLICT (...) DO UPDATE SET ...
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extract text from various file types
|
||||
fn extract_text(
|
||||
&self,
|
||||
file_path: &str,
|
||||
content: &[u8],
|
||||
) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let path_lower = file_path.to_ascii_lowercase();
|
||||
|
||||
if path_lower.ends_with(".pdf") {
|
||||
match pdf_extract::extract_text_from_mem(content) {
|
||||
Ok(text) => Ok(text),
|
||||
|
|
@ -472,16 +306,13 @@ impl DriveMonitor {
|
|||
String::from_utf8(content.to_vec())
|
||||
.map_err(|e| format!("UTF-8 decoding failed: {}", e).into())
|
||||
} else {
|
||||
// Try as plain text
|
||||
String::from_utf8(content.to_vec())
|
||||
.map_err(|e| format!("Unsupported file format or UTF-8 error: {}", e).into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear all tracked file states
|
||||
pub async fn clear_state(&self) {
|
||||
let mut states = self.file_states.write().await;
|
||||
states.clear();
|
||||
info!("Cleared all file states");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
112
src/file/mod.rs
112
src/file/mod.rs
|
|
@ -1,14 +1,13 @@
|
|||
use crate::config::DriveConfig;
|
||||
use crate::shared::state::AppState;
|
||||
use actix_multipart::Multipart;
|
||||
use actix_web::web;
|
||||
use actix_web::{post, HttpResponse};
|
||||
use aws_sdk_s3::{Client, Error as S3Error};
|
||||
use opendal::Operator;
|
||||
use std::io::Write;
|
||||
use tempfile::NamedTempFile;
|
||||
use tokio_stream::StreamExt as TokioStreamExt;
|
||||
|
||||
use crate::config::DriveConfig;
|
||||
use crate::shared::state::AppState;
|
||||
|
||||
#[post("/files/upload/{folder_path}")]
|
||||
pub async fn upload_file(
|
||||
folder_path: web::Path<String>,
|
||||
|
|
@ -16,15 +15,11 @@ pub async fn upload_file(
|
|||
state: web::Data<AppState>,
|
||||
) -> Result<HttpResponse, actix_web::Error> {
|
||||
let folder_path = folder_path.into_inner();
|
||||
|
||||
// Create a temporary file that will hold the uploaded data
|
||||
let mut temp_file = NamedTempFile::new().map_err(|e| {
|
||||
actix_web::error::ErrorInternalServerError(format!("Failed to create temp file: {}", e))
|
||||
})?;
|
||||
|
||||
let mut file_name: Option<String> = None;
|
||||
|
||||
// Process multipart form data
|
||||
while let Some(mut field) = payload.try_next().await? {
|
||||
if let Some(disposition) = field.content_disposition() {
|
||||
if let Some(name) = disposition.get_filename() {
|
||||
|
|
@ -32,7 +27,6 @@ pub async fn upload_file(
|
|||
}
|
||||
}
|
||||
|
||||
// Write each chunk of the field to the temporary file
|
||||
while let Some(chunk) = field.try_next().await? {
|
||||
temp_file.write_all(&chunk).map_err(|e| {
|
||||
actix_web::error::ErrorInternalServerError(format!(
|
||||
|
|
@ -43,44 +37,24 @@ pub async fn upload_file(
|
|||
}
|
||||
}
|
||||
|
||||
// Use a fallback name if the client didn't supply one
|
||||
let file_name = file_name.unwrap_or_else(|| "unnamed_file".to_string());
|
||||
|
||||
// Convert the NamedTempFile into a TempPath so we can get a stable path
|
||||
let temp_file_path = temp_file.into_temp_path();
|
||||
|
||||
// Retrieve the bucket name from configuration, handling the case where it is missing
|
||||
let bucket_name = match &state.get_ref().config {
|
||||
Some(cfg) => cfg.s3_bucket.clone(),
|
||||
None => {
|
||||
// Clean up the temp file before returning the error
|
||||
let _ = std::fs::remove_file(&temp_file_path);
|
||||
return Err(actix_web::error::ErrorInternalServerError(
|
||||
"S3 bucket configuration is missing",
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Build the S3 object key (folder + filename)
|
||||
let s3_key = format!("{}/{}", folder_path, file_name);
|
||||
|
||||
// Retrieve a reference to the S3 client, handling the case where it is missing
|
||||
let s3_client = state.get_ref().s3_client.as_ref().ok_or_else(|| {
|
||||
actix_web::error::ErrorInternalServerError("S3 client is not initialized")
|
||||
let op = state.get_ref().s3_operator.as_ref().ok_or_else(|| {
|
||||
actix_web::error::ErrorInternalServerError("S3 operator is not initialized")
|
||||
})?;
|
||||
|
||||
// Perform the upload
|
||||
match upload_to_s3(s3_client, &bucket_name, &s3_key, &temp_file_path).await {
|
||||
let s3_key = format!("{}/{}", folder_path, file_name);
|
||||
|
||||
match upload_to_s3(op, &s3_key, &temp_file_path).await {
|
||||
Ok(_) => {
|
||||
// Remove the temporary file now that the upload succeeded
|
||||
let _ = std::fs::remove_file(&temp_file_path);
|
||||
Ok(HttpResponse::Ok().body(format!(
|
||||
"Uploaded file '{}' to folder '{}' in S3 bucket '{}'",
|
||||
file_name, folder_path, bucket_name
|
||||
"Uploaded file '{}' to folder '{}'",
|
||||
file_name, folder_path
|
||||
)))
|
||||
}
|
||||
Err(e) => {
|
||||
// Ensure the temporary file is cleaned up even on failure
|
||||
let _ = std::fs::remove_file(&temp_file_path);
|
||||
Err(actix_web::error::ErrorInternalServerError(format!(
|
||||
"Failed to upload file to S3: {}",
|
||||
|
|
@ -90,61 +64,27 @@ pub async fn upload_file(
|
|||
}
|
||||
}
|
||||
|
||||
// Helper function to get S3 client
|
||||
pub async fn init_drive(cfg: &DriveConfig) -> Result<Client, Box<dyn std::error::Error>> {
|
||||
// Build static credentials from the Drive configuration.
|
||||
let credentials = aws_sdk_s3::config::Credentials::new(
|
||||
cfg.access_key.clone(),
|
||||
cfg.secret_key.clone(),
|
||||
None,
|
||||
None,
|
||||
"static",
|
||||
);
|
||||
pub async fn init_drive(config: &DriveConfig) -> Result<Operator, Box<dyn std::error::Error>> {
|
||||
use opendal::services::S3;
|
||||
use opendal::Operator;
|
||||
let client = Operator::new(
|
||||
S3::default()
|
||||
.root("/")
|
||||
.endpoint(&config.server)
|
||||
.access_key_id(&config.access_key)
|
||||
.secret_access_key(&config.secret_key),
|
||||
)?
|
||||
.finish();
|
||||
|
||||
// Construct the endpoint URL, respecting the SSL flag.
|
||||
let scheme = if cfg.use_ssl { "https" } else { "http" };
|
||||
let endpoint = format!("{}://{}", scheme, cfg.server);
|
||||
|
||||
// MinIO requires path‑style addressing.
|
||||
let s3_config = aws_sdk_s3::config::Builder::new()
|
||||
// Set the behavior version to the latest to satisfy the SDK requirement.
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.region(aws_sdk_s3::config::Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint)
|
||||
.credentials_provider(credentials)
|
||||
.force_path_style(true)
|
||||
.build();
|
||||
|
||||
Ok(Client::from_conf(s3_config))
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
// Helper function to upload file to S3
|
||||
async fn upload_to_s3(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
op: &Operator,
|
||||
key: &str,
|
||||
file_path: &std::path::Path,
|
||||
) -> Result<(), S3Error> {
|
||||
// Convert the file at `file_path` into a ByteStream, mapping any I/O error
|
||||
// into the appropriate `SdkError` type expected by the function signature.
|
||||
let body = aws_sdk_s3::primitives::ByteStream::from_path(file_path)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
aws_sdk_s3::error::SdkError::<
|
||||
aws_sdk_s3::operation::put_object::PutObjectError,
|
||||
aws_sdk_s3::primitives::ByteStream,
|
||||
>::construction_failure(e)
|
||||
})?;
|
||||
|
||||
// Perform the actual upload to S3.
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())?; // Convert the successful output to `()`.
|
||||
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let data = std::fs::read(file_path)?;
|
||||
op.write(key, data).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
use crate::shared::state::AppState;
|
||||
use aws_sdk_s3::Client as S3Client;
|
||||
use log::{debug, error, info};
|
||||
use log::error;
|
||||
use opendal::Operator;
|
||||
use tokio_stream::StreamExt;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
use tokio::time::{interval, Duration};
|
||||
|
||||
/// MinIO file state tracker
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileState {
|
||||
pub path: String,
|
||||
|
|
@ -15,52 +15,41 @@ pub struct FileState {
|
|||
pub last_modified: Option<String>,
|
||||
}
|
||||
|
||||
/// MinIO handler that monitors bucket changes
|
||||
pub struct MinIOHandler {
|
||||
state: Arc<AppState>,
|
||||
bucket_name: String,
|
||||
watched_prefixes: Arc<tokio::sync::RwLock<Vec<String>>>,
|
||||
file_states: Arc<tokio::sync::RwLock<HashMap<String, FileState>>>,
|
||||
}
|
||||
|
||||
impl MinIOHandler {
|
||||
pub fn new(state: Arc<AppState>, bucket_name: String) -> Self {
|
||||
pub fn new(state: Arc<AppState>) -> Self {
|
||||
Self {
|
||||
state,
|
||||
bucket_name,
|
||||
watched_prefixes: Arc::new(tokio::sync::RwLock::new(Vec::new())),
|
||||
file_states: Arc::new(tokio::sync::RwLock::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a prefix to watch (e.g., ".gbkb/", ".gbdialog/")
|
||||
pub async fn watch_prefix(&self, prefix: String) {
|
||||
let mut prefixes = self.watched_prefixes.write().await;
|
||||
if !prefixes.contains(&prefix) {
|
||||
prefixes.push(prefix.clone());
|
||||
info!("Now watching MinIO prefix: {}", prefix);
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a prefix from watch list
|
||||
pub async fn unwatch_prefix(&self, prefix: &str) {
|
||||
let mut prefixes = self.watched_prefixes.write().await;
|
||||
prefixes.retain(|p| p != prefix);
|
||||
info!("Stopped watching MinIO prefix: {}", prefix);
|
||||
}
|
||||
|
||||
/// Start the monitoring service
|
||||
pub fn spawn(
|
||||
self: Arc<Self>,
|
||||
change_callback: Arc<dyn Fn(FileChangeEvent) + Send + Sync>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
info!("MinIO Handler service started");
|
||||
let mut tick = interval(Duration::from_secs(15)); // Check every 15 seconds
|
||||
|
||||
let mut tick = interval(Duration::from_secs(15));
|
||||
loop {
|
||||
tick.tick().await;
|
||||
|
||||
if let Err(e) = self.check_for_changes(&change_callback).await {
|
||||
error!("Error checking for MinIO changes: {}", e);
|
||||
}
|
||||
|
|
@ -68,93 +57,58 @@ impl MinIOHandler {
|
|||
})
|
||||
}
|
||||
|
||||
/// Check for file changes in watched prefixes
|
||||
async fn check_for_changes(
|
||||
&self,
|
||||
callback: &Arc<dyn Fn(FileChangeEvent) + Send + Sync>,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let s3_client = match &self.state.s3_client {
|
||||
Some(client) => client,
|
||||
let op = match &self.state.s3_operator {
|
||||
Some(op) => op,
|
||||
None => {
|
||||
debug!("S3 client not configured");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let prefixes = self.watched_prefixes.read().await;
|
||||
|
||||
for prefix in prefixes.iter() {
|
||||
debug!("Checking prefix: {}", prefix);
|
||||
|
||||
if let Err(e) = self.check_prefix_changes(s3_client, prefix, callback).await {
|
||||
if let Err(e) = self.check_prefix_changes(op, prefix, callback).await {
|
||||
error!("Error checking prefix {}: {}", prefix, e);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check changes in a specific prefix
|
||||
async fn check_prefix_changes(
|
||||
&self,
|
||||
s3_client: &S3Client,
|
||||
op: &Operator,
|
||||
prefix: &str,
|
||||
callback: &Arc<dyn Fn(FileChangeEvent) + Send + Sync>,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
// List all objects with the prefix
|
||||
let mut continuation_token: Option<String> = None;
|
||||
let mut current_files = HashMap::new();
|
||||
|
||||
loop {
|
||||
let mut list_request = s3_client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.bucket_name)
|
||||
.prefix(prefix);
|
||||
let mut lister = op.lister_with(prefix).recursive(true).await?;
|
||||
while let Some(entry) = lister.try_next().await? {
|
||||
let path = entry.path().to_string();
|
||||
|
||||
if let Some(token) = continuation_token {
|
||||
list_request = list_request.continuation_token(token);
|
||||
if path.ends_with('/') {
|
||||
continue;
|
||||
}
|
||||
|
||||
let list_result = list_request.send().await?;
|
||||
|
||||
if let Some(contents) = list_result.contents {
|
||||
for object in contents {
|
||||
if let Some(key) = object.key {
|
||||
// Skip directories
|
||||
if key.ends_with('/') {
|
||||
continue;
|
||||
}
|
||||
|
||||
let file_state = FileState {
|
||||
path: key.clone(),
|
||||
size: object.size.unwrap_or(0),
|
||||
etag: object.e_tag.unwrap_or_default(),
|
||||
last_modified: object.last_modified.map(|dt| dt.to_string()),
|
||||
};
|
||||
|
||||
current_files.insert(key, file_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if list_result.is_truncated.unwrap_or(false) {
|
||||
continuation_token = list_result.next_continuation_token;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
let meta = op.stat(&path).await?;
|
||||
let file_state = FileState {
|
||||
path: path.clone(),
|
||||
size: meta.content_length() as i64,
|
||||
etag: meta.etag().unwrap_or_default().to_string(),
|
||||
last_modified: meta.last_modified().map(|dt| dt.to_rfc3339()),
|
||||
};
|
||||
current_files.insert(path, file_state);
|
||||
}
|
||||
|
||||
// Compare with previous state
|
||||
let mut file_states = self.file_states.write().await;
|
||||
|
||||
// Check for new or modified files
|
||||
for (path, current_state) in current_files.iter() {
|
||||
if let Some(previous_state) = file_states.get(path) {
|
||||
// File exists, check if modified
|
||||
if current_state.etag != previous_state.etag
|
||||
|| current_state.size != previous_state.size
|
||||
{
|
||||
info!("File modified: {}", path);
|
||||
callback(FileChangeEvent::Modified {
|
||||
path: path.clone(),
|
||||
size: current_state.size,
|
||||
|
|
@ -162,8 +116,6 @@ impl MinIOHandler {
|
|||
});
|
||||
}
|
||||
} else {
|
||||
// New file
|
||||
info!("File created: {}", path);
|
||||
callback(FileChangeEvent::Created {
|
||||
path: path.clone(),
|
||||
size: current_state.size,
|
||||
|
|
@ -172,7 +124,6 @@ impl MinIOHandler {
|
|||
}
|
||||
}
|
||||
|
||||
// Check for deleted files
|
||||
let previous_paths: Vec<String> = file_states
|
||||
.keys()
|
||||
.filter(|k| k.starts_with(prefix))
|
||||
|
|
@ -181,13 +132,11 @@ impl MinIOHandler {
|
|||
|
||||
for path in previous_paths {
|
||||
if !current_files.contains_key(&path) {
|
||||
info!("File deleted: {}", path);
|
||||
callback(FileChangeEvent::Deleted { path: path.clone() });
|
||||
file_states.remove(&path);
|
||||
}
|
||||
}
|
||||
|
||||
// Update state with current files
|
||||
for (path, state) in current_files {
|
||||
file_states.insert(path, state);
|
||||
}
|
||||
|
|
@ -195,20 +144,16 @@ impl MinIOHandler {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Get current state of a file
|
||||
pub async fn get_file_state(&self, path: &str) -> Option<FileState> {
|
||||
let states = self.file_states.read().await;
|
||||
states.get(path).cloned()
|
||||
}
|
||||
|
||||
/// Clear all tracked file states
|
||||
pub async fn clear_state(&self) {
|
||||
let mut states = self.file_states.write().await;
|
||||
states.clear();
|
||||
info!("Cleared all file states");
|
||||
}
|
||||
|
||||
/// Get all tracked files for a prefix
|
||||
pub async fn get_files_by_prefix(&self, prefix: &str) -> Vec<FileState> {
|
||||
let states = self.file_states.read().await;
|
||||
states
|
||||
|
|
@ -219,7 +164,6 @@ impl MinIOHandler {
|
|||
}
|
||||
}
|
||||
|
||||
/// File change event types
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum FileChangeEvent {
|
||||
Created {
|
||||
|
|
@ -266,7 +210,6 @@ mod tests {
|
|||
size: 100,
|
||||
etag: "abc123".to_string(),
|
||||
};
|
||||
|
||||
assert_eq!(event.path(), "test.txt");
|
||||
assert_eq!(event.event_type(), "created");
|
||||
}
|
||||
|
|
@ -286,7 +229,6 @@ mod tests {
|
|||
let deleted = FileChangeEvent::Deleted {
|
||||
path: "file3.txt".to_string(),
|
||||
};
|
||||
|
||||
assert_eq!(created.event_type(), "created");
|
||||
assert_eq!(modified.event_type(), "modified");
|
||||
assert_eq!(deleted.event_type(), "deleted");
|
||||
|
|
|
|||
128
src/kb/mod.rs
128
src/kb/mod.rs
|
|
@ -1,6 +1,7 @@
|
|||
use crate::shared::models::KBCollection;
|
||||
use crate::shared::state::AppState;
|
||||
use log::{debug, error, info, warn};
|
||||
use log::{ error, info, warn};
|
||||
use tokio_stream::StreamExt;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
|
|
@ -10,7 +11,6 @@ pub mod embeddings;
|
|||
pub mod minio_handler;
|
||||
pub mod qdrant_client;
|
||||
|
||||
/// Represents a change in a KB file
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum FileChangeEvent {
|
||||
Created(String),
|
||||
|
|
@ -18,7 +18,6 @@ pub enum FileChangeEvent {
|
|||
Deleted(String),
|
||||
}
|
||||
|
||||
/// KB Manager service that coordinates MinIO monitoring and Qdrant indexing
|
||||
pub struct KBManager {
|
||||
state: Arc<AppState>,
|
||||
watched_collections: Arc<tokio::sync::RwLock<HashMap<String, KBCollection>>>,
|
||||
|
|
@ -32,7 +31,6 @@ impl KBManager {
|
|||
}
|
||||
}
|
||||
|
||||
/// Start watching a KB collection folder
|
||||
pub async fn add_collection(
|
||||
&self,
|
||||
bot_id: String,
|
||||
|
|
@ -47,7 +45,6 @@ impl KBManager {
|
|||
collection_name, qdrant_collection
|
||||
);
|
||||
|
||||
// Create Qdrant collection if it doesn't exist
|
||||
qdrant_client::ensure_collection_exists(&self.state, &qdrant_collection).await?;
|
||||
|
||||
let now = chrono::Utc::now().to_rfc3339();
|
||||
|
|
@ -67,30 +64,23 @@ impl KBManager {
|
|||
let mut collections = self.watched_collections.write().await;
|
||||
collections.insert(collection_name.to_string(), collection);
|
||||
|
||||
info!("KB collection added successfully: {}", collection_name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove a KB collection
|
||||
pub async fn remove_collection(
|
||||
&self,
|
||||
collection_name: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mut collections = self.watched_collections.write().await;
|
||||
collections.remove(collection_name);
|
||||
info!("KB collection removed: {}", collection_name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Start the KB monitoring service
|
||||
pub fn spawn(self: Arc<Self>) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
info!("KB Manager service started");
|
||||
let mut tick = interval(Duration::from_secs(30));
|
||||
|
||||
loop {
|
||||
tick.tick().await;
|
||||
|
||||
let collections = self.watched_collections.read().await;
|
||||
for (name, collection) in collections.iter() {
|
||||
if let Err(e) = self.check_collection_updates(collection).await {
|
||||
|
|
@ -101,67 +91,43 @@ impl KBManager {
|
|||
})
|
||||
}
|
||||
|
||||
/// Check for updates in a collection
|
||||
async fn check_collection_updates(
|
||||
&self,
|
||||
collection: &KBCollection,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
debug!("Checking updates for collection: {}", collection.name);
|
||||
|
||||
let s3_client = match &self.state.s3_client {
|
||||
Some(client) => client,
|
||||
let op = match &self.state.s3_operator {
|
||||
Some(op) => op,
|
||||
None => {
|
||||
warn!("S3 client not configured");
|
||||
warn!("S3 operator not configured");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let config = match &self.state.config {
|
||||
Some(cfg) => cfg,
|
||||
None => {
|
||||
error!("App configuration missing");
|
||||
return Err("App configuration missing".into());
|
||||
let mut lister = op.lister_with(&collection.folder_path).recursive(true).await?;
|
||||
while let Some(entry) = lister.try_next().await? {
|
||||
let path = entry.path().to_string();
|
||||
|
||||
if path.ends_with('/') {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let bucket_name = format!("{}default.gbai", config.minio.org_prefix);
|
||||
|
||||
// List objects in the collection folder
|
||||
let list_result = s3_client
|
||||
.list_objects_v2()
|
||||
.bucket(&bucket_name)
|
||||
.prefix(&collection.folder_path)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if let Some(contents) = list_result.contents {
|
||||
for object in contents {
|
||||
if let Some(key) = object.key {
|
||||
// Skip directories
|
||||
if key.ends_with('/') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if file needs indexing
|
||||
if let Err(e) = self
|
||||
.process_file(
|
||||
&collection,
|
||||
&key,
|
||||
object.size.unwrap_or(0),
|
||||
object.last_modified.map(|dt| dt.to_string()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error processing file {}: {}", key, e);
|
||||
}
|
||||
}
|
||||
let meta = op.stat(&path).await?;
|
||||
if let Err(e) = self
|
||||
.process_file(
|
||||
&collection,
|
||||
&path,
|
||||
meta.content_length() as i64,
|
||||
meta.last_modified().map(|dt| dt.to_rfc3339()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error processing file {}: {}", path, e);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Process a single file (check if changed and index if needed)
|
||||
async fn process_file(
|
||||
&self,
|
||||
collection: &KBCollection,
|
||||
|
|
@ -169,9 +135,7 @@ impl KBManager {
|
|||
file_size: i64,
|
||||
_last_modified: Option<String>,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
// Get file content hash
|
||||
let content = self.get_file_content(file_path).await?;
|
||||
// Simple hash using length and first/last bytes for change detection
|
||||
let file_hash = if content.len() > 100 {
|
||||
format!(
|
||||
"{:x}_{:x}_{}",
|
||||
|
|
@ -183,24 +147,16 @@ impl KBManager {
|
|||
format!("{:x}", content.len())
|
||||
};
|
||||
|
||||
// Check if file is already indexed with same hash
|
||||
if self
|
||||
.is_file_indexed(collection.bot_id.clone(), file_path, &file_hash)
|
||||
.await?
|
||||
{
|
||||
debug!("File already indexed: {}", file_path);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!(
|
||||
"Indexing file: {} to collection {}",
|
||||
file_path, collection.name
|
||||
);
|
||||
|
||||
// Extract text based on file type
|
||||
info!("Indexing file: {} to collection {}", file_path, collection.name);
|
||||
let text_content = self.extract_text(file_path, &content).await?;
|
||||
|
||||
// Generate embeddings and store in Qdrant
|
||||
embeddings::index_document(
|
||||
&self.state,
|
||||
&collection.qdrant_collection,
|
||||
|
|
@ -209,7 +165,6 @@ impl KBManager {
|
|||
)
|
||||
.await?;
|
||||
|
||||
// Save metadata to database
|
||||
let metadata = serde_json::json!({
|
||||
"file_type": self.get_file_type(file_path),
|
||||
"last_modified": _last_modified,
|
||||
|
|
@ -225,48 +180,29 @@ impl KBManager {
|
|||
)
|
||||
.await?;
|
||||
|
||||
info!("File indexed successfully: {}", file_path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get file content from MinIO
|
||||
async fn get_file_content(
|
||||
&self,
|
||||
file_path: &str,
|
||||
) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let s3_client = self
|
||||
let op = self
|
||||
.state
|
||||
.s3_client
|
||||
.s3_operator
|
||||
.as_ref()
|
||||
.ok_or("S3 client not configured")?;
|
||||
.ok_or("S3 operator not configured")?;
|
||||
|
||||
let config = self
|
||||
.state
|
||||
.config
|
||||
.as_ref()
|
||||
.ok_or("App configuration missing")?;
|
||||
|
||||
let bucket_name = format!("{}default.gbai", config.minio.org_prefix);
|
||||
|
||||
let response = s3_client
|
||||
.get_object()
|
||||
.bucket(&bucket_name)
|
||||
.key(file_path)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let data = response.body.collect().await?;
|
||||
Ok(data.into_bytes().to_vec())
|
||||
let content = op.read(file_path).await?;
|
||||
Ok(content.to_vec())
|
||||
}
|
||||
|
||||
/// Extract text from various file types
|
||||
async fn extract_text(
|
||||
&self,
|
||||
file_path: &str,
|
||||
content: &[u8],
|
||||
) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let path_lower = file_path.to_ascii_lowercase();
|
||||
|
||||
if path_lower.ends_with(".pdf") {
|
||||
match pdf_extract::extract_text_from_mem(content) {
|
||||
Ok(text) => Ok(text),
|
||||
|
|
@ -279,29 +215,23 @@ impl KBManager {
|
|||
String::from_utf8(content.to_vec())
|
||||
.map_err(|e| format!("UTF-8 decoding failed: {}", e).into())
|
||||
} else if path_lower.ends_with(".docx") {
|
||||
// TODO: Add DOCX support
|
||||
warn!("DOCX format not yet supported: {}", file_path);
|
||||
Err("DOCX format not supported".into())
|
||||
} else {
|
||||
// Try as plain text
|
||||
String::from_utf8(content.to_vec())
|
||||
.map_err(|e| format!("Unsupported file format or UTF-8 error: {}", e).into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if file is already indexed
|
||||
async fn is_file_indexed(
|
||||
&self,
|
||||
_bot_id: String,
|
||||
_file_path: &str,
|
||||
_file_hash: &str,
|
||||
) -> Result<bool, Box<dyn Error + Send + Sync>> {
|
||||
// TODO: Query database to check if file with same hash exists
|
||||
// For now, return false to always reindex
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Save document metadata to database
|
||||
async fn save_document_metadata(
|
||||
&self,
|
||||
_bot_id: String,
|
||||
|
|
@ -311,7 +241,6 @@ impl KBManager {
|
|||
file_hash: &str,
|
||||
_metadata: serde_json::Value,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
// TODO: Save to database using Diesel
|
||||
info!(
|
||||
"Saving metadata for {}: size={}, hash={}",
|
||||
file_path, file_size, file_hash
|
||||
|
|
@ -319,7 +248,6 @@ impl KBManager {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Get file type from path
|
||||
fn get_file_type(&self, file_path: &str) -> String {
|
||||
file_path
|
||||
.rsplit('.')
|
||||
|
|
|
|||
86
src/main.rs
86
src/main.rs
|
|
@ -1,5 +1,6 @@
|
|||
#![allow(dead_code)]
|
||||
#![cfg_attr(feature = "desktop", windows_subsystem = "windows")]
|
||||
|
||||
use actix_cors::Cors;
|
||||
use actix_web::middleware::Logger;
|
||||
use actix_web::{web, App, HttpServer};
|
||||
|
|
@ -19,10 +20,8 @@ mod context;
|
|||
mod drive_monitor;
|
||||
#[cfg(feature = "email")]
|
||||
mod email;
|
||||
|
||||
#[cfg(feature = "desktop")]
|
||||
mod ui;
|
||||
|
||||
mod file;
|
||||
mod kb;
|
||||
mod llm;
|
||||
|
|
@ -65,7 +64,6 @@ use crate::whatsapp::WhatsAppAdapter;
|
|||
#[tokio::main]
|
||||
async fn main() -> std::io::Result<()> {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
|
||||
if args.len() > 1 {
|
||||
let command = &args[1];
|
||||
match command.as_str() {
|
||||
|
|
@ -93,10 +91,8 @@ async fn main() -> std::io::Result<()> {
|
|||
|
||||
dotenv().ok();
|
||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
|
||||
.write_style(env_logger::WriteStyle::Always)
|
||||
.init();
|
||||
|
||||
info!("Starting BotServer bootstrap process");
|
||||
.write_style(env_logger::WriteStyle::Always)
|
||||
.init();
|
||||
|
||||
let install_mode = if args.contains(&"--container".to_string()) {
|
||||
InstallMode::Container
|
||||
|
|
@ -111,38 +107,48 @@ async fn main() -> std::io::Result<()> {
|
|||
};
|
||||
|
||||
let mut bootstrap = BootstrapManager::new(install_mode.clone(), tenant.clone());
|
||||
let cfg = match bootstrap.bootstrap() {
|
||||
Ok(config) => {
|
||||
info!("Bootstrap completed successfully, configuration loaded from database");
|
||||
config
|
||||
|
||||
// Prevent double bootstrap: skip if environment already initialized
|
||||
let env_path = std::env::current_dir()?.join("botserver-stack").join(".env");
|
||||
let cfg = if env_path.exists() {
|
||||
info!("Environment already initialized, skipping bootstrap");
|
||||
match diesel::Connection::establish(
|
||||
&std::env::var("DATABASE_URL")
|
||||
.unwrap_or_else(|_| "postgres://gbuser:@localhost:5432/botserver".to_string()),
|
||||
) {
|
||||
Ok(mut conn) => AppConfig::from_database(&mut conn),
|
||||
Err(_) => AppConfig::from_env(),
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Bootstrap failed: {}", e);
|
||||
info!("Attempting to load configuration from database");
|
||||
match diesel::Connection::establish(
|
||||
&std::env::var("DATABASE_URL")
|
||||
.unwrap_or_else(|_| "postgres://gbuser:@localhost:5432/botserver".to_string()),
|
||||
) {
|
||||
Ok(mut conn) => AppConfig::from_database(&mut conn),
|
||||
Err(_) => {
|
||||
info!("Database not available, using environment variables as fallback");
|
||||
AppConfig::from_env()
|
||||
} else {
|
||||
match bootstrap.bootstrap() {
|
||||
Ok(config) => {
|
||||
info!("Bootstrap completed successfully");
|
||||
config
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Bootstrap failed: {}", e);
|
||||
match diesel::Connection::establish(
|
||||
&std::env::var("DATABASE_URL")
|
||||
.unwrap_or_else(|_| "postgres://gbuser:@localhost:5432/botserver".to_string()),
|
||||
) {
|
||||
Ok(mut conn) => AppConfig::from_database(&mut conn),
|
||||
Err(_) => AppConfig::from_env(),
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let _ = bootstrap.start_all();
|
||||
|
||||
// Upload template bots to MinIO on first startup
|
||||
if let Err(e) = bootstrap.upload_templates_to_minio(&cfg).await {
|
||||
let _ = bootstrap.start_all();
|
||||
if let Err(e) = bootstrap.upload_templates_to_drive(&cfg).await {
|
||||
log::warn!("Failed to upload templates to MinIO: {}", e);
|
||||
}
|
||||
|
||||
let config = std::sync::Arc::new(cfg.clone());
|
||||
|
||||
info!("Establishing database connection to {}", cfg.database_url());
|
||||
let db_pool = match diesel::Connection::establish(&cfg.database_url()) {
|
||||
// Refresh configuration from environment to ensure latest DATABASE_URL and credentials
|
||||
dotenv().ok();
|
||||
let refreshed_cfg = AppConfig::from_env();
|
||||
let config = std::sync::Arc::new(refreshed_cfg.clone());
|
||||
let db_pool = match diesel::Connection::establish(&refreshed_cfg.database_url()) {
|
||||
Ok(conn) => Arc::new(Mutex::new(conn)),
|
||||
Err(e) => {
|
||||
log::error!("Failed to connect to main database: {}", e);
|
||||
|
|
@ -154,8 +160,6 @@ async fn main() -> std::io::Result<()> {
|
|||
};
|
||||
|
||||
let db_custom_pool = db_pool.clone();
|
||||
|
||||
info!("Initializing LLM server at {}", cfg.ai.endpoint);
|
||||
ensure_llama_servers_running()
|
||||
.await
|
||||
.expect("Failed to initialize LLM local server");
|
||||
|
|
@ -176,7 +180,6 @@ async fn main() -> std::io::Result<()> {
|
|||
"empty".to_string(),
|
||||
Some(cfg.ai.endpoint.clone()),
|
||||
));
|
||||
|
||||
let web_adapter = Arc::new(WebChannelAdapter::new());
|
||||
let voice_adapter = Arc::new(VoiceAdapter::new(
|
||||
"https://livekit.example.com".to_string(),
|
||||
|
|
@ -190,8 +193,8 @@ async fn main() -> std::io::Result<()> {
|
|||
));
|
||||
let tool_api = Arc::new(tools::ToolApi::new());
|
||||
|
||||
info!("Initializing drive at {}", cfg.minio.server);
|
||||
let drive = init_drive(&config.minio)
|
||||
|
||||
let drive = init_drive(&config.drive)
|
||||
.await
|
||||
.expect("Failed to initialize Drive");
|
||||
|
||||
|
|
@ -199,13 +202,14 @@ async fn main() -> std::io::Result<()> {
|
|||
diesel::Connection::establish(&cfg.database_url()).unwrap(),
|
||||
redis_client.clone(),
|
||||
)));
|
||||
|
||||
let auth_service = Arc::new(tokio::sync::Mutex::new(auth::AuthService::new(
|
||||
diesel::Connection::establish(&cfg.database_url()).unwrap(),
|
||||
redis_client.clone(),
|
||||
)));
|
||||
|
||||
let app_state = Arc::new(AppState {
|
||||
s3_client: Some(drive.clone()),
|
||||
s3_operator: Some(drive.clone()),
|
||||
config: Some(cfg.clone()),
|
||||
conn: db_pool.clone(),
|
||||
custom_conn: db_custom_pool.clone(),
|
||||
|
|
@ -229,23 +233,17 @@ async fn main() -> std::io::Result<()> {
|
|||
tool_api: tool_api.clone(),
|
||||
});
|
||||
|
||||
info!(
|
||||
"Starting HTTP server on {}:{}",
|
||||
config.server.host, config.server.port
|
||||
);
|
||||
|
||||
info!("Starting HTTP server on {}:{}", config.server.host, config.server.port);
|
||||
let worker_count = std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(4);
|
||||
|
||||
// Spawn AutomationService in a LocalSet on a separate thread
|
||||
let automation_state = app_state.clone();
|
||||
std::thread::spawn(move || {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("Failed to create runtime for automation");
|
||||
|
||||
let local = tokio::task::LocalSet::new();
|
||||
local.block_on(&rt, async move {
|
||||
let bot_guid = std::env::var("BOT_GUID").unwrap_or_else(|_| "default_bot".to_string());
|
||||
|
|
@ -257,7 +255,7 @@ async fn main() -> std::io::Result<()> {
|
|||
|
||||
let drive_state = app_state.clone();
|
||||
let bot_guid = std::env::var("BOT_GUID").unwrap_or_else(|_| "default_bot".to_string());
|
||||
let bucket_name = format!("{}{}.gbai", cfg.minio.org_prefix, bot_guid);
|
||||
let bucket_name = format!("{}{}.gbai", cfg.drive.org_prefix, bot_guid);
|
||||
let drive_monitor = Arc::new(DriveMonitor::new(drive_state, bucket_name));
|
||||
let _drive_handle = drive_monitor.spawn();
|
||||
|
||||
|
|
@ -267,8 +265,8 @@ async fn main() -> std::io::Result<()> {
|
|||
.allow_any_method()
|
||||
.allow_any_header()
|
||||
.max_age(3600);
|
||||
let app_state_clone = app_state.clone();
|
||||
|
||||
let app_state_clone = app_state.clone();
|
||||
let mut app = App::new()
|
||||
.wrap(cors)
|
||||
.wrap(Logger::default())
|
||||
|
|
|
|||
|
|
@ -63,51 +63,85 @@ impl PackageManager {
|
|||
fn register_drive(&mut self) {
|
||||
let drive_password = self.generate_secure_password(16);
|
||||
let drive_user = "gbdriveuser".to_string();
|
||||
let farm_password = std::env::var("FARM_PASSWORD")
|
||||
.unwrap_or_else(|_| self.generate_secure_password(32));
|
||||
let farm_password =
|
||||
std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32));
|
||||
let encrypted_drive_password = self.encrypt_password(&drive_password, &farm_password);
|
||||
|
||||
self.components.insert("drive".to_string(), ComponentConfig {
|
||||
name: "drive".to_string(),
|
||||
required: true,
|
||||
ports: vec![9000, 9001],
|
||||
dependencies: vec![],
|
||||
linux_packages: vec![],
|
||||
macos_packages: vec![],
|
||||
windows_packages: vec![],
|
||||
download_url: Some("https://dl.min.io/server/minio/release/linux-amd64/minio".to_string()),
|
||||
binary_name: Some("minio".to_string()),
|
||||
pre_install_cmds_linux: vec![],
|
||||
post_install_cmds_linux: vec![
|
||||
"wget https://dl.min.io/client/mc/release/linux-amd64/mc -O {{BIN_PATH}}/mc".to_string(),
|
||||
"chmod +x {{BIN_PATH}}/mc".to_string(),
|
||||
format!("{{{{BIN_PATH}}}}/mc alias set mc http://localhost:9000 gbdriveuser {}", drive_password),
|
||||
"{{BIN_PATH}}/mc mb mc/default.gbai".to_string(),
|
||||
format!("{{{{BIN_PATH}}}}/mc admin user add mc gbdriveuser {}", drive_password),
|
||||
"{{BIN_PATH}}/mc admin policy attach mc readwrite --user=gbdriveuser".to_string()
|
||||
],
|
||||
pre_install_cmds_macos: vec![],
|
||||
post_install_cmds_macos: vec![
|
||||
"wget https://dl.min.io/client/mc/release/darwin-amd64/mc -O {{BIN_PATH}}/mc".to_string(),
|
||||
"chmod +x {{BIN_PATH}}/mc".to_string()
|
||||
],
|
||||
pre_install_cmds_windows: vec![],
|
||||
post_install_cmds_windows: vec![
|
||||
"curl https://dl.min.io/client/mc/release/windows-amd64/mc.exe -O {{BIN_PATH}}\\mc.exe".to_string(),
|
||||
"cmd /c {{BIN_PATH}}\\mc.exe alias set mc http://localhost:9000 gbdriveuser {}".to_string(),
|
||||
"cmd /c {{BIN_PATH}}\\mc.exe mb mc\\default.gbai".to_string(),
|
||||
"cmd /c {{BIN_PATH}}\\mc.exe admin user add mc gbdriveuser {}".to_string(),
|
||||
"cmd /c {{BIN_PATH}}\\mc.exe admin policy attach mc readwrite --user=gbdriveuser".to_string()
|
||||
],
|
||||
env_vars: HashMap::from([
|
||||
("MINIO_ROOT_USER".to_string(), "gbdriveuser".to_string()),
|
||||
("MINIO_ROOT_PASSWORD".to_string(), drive_password)
|
||||
]),
|
||||
exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(),
|
||||
});
|
||||
let env_path = self.base_path.join(".env");
|
||||
let env_content = format!(
|
||||
"DRIVE_USER={}\nDRIVE_PASSWORD={}\nFARM_PASSWORD={}\nDRIVE_ROOT_USER={}\nDRIVE_ROOT_PASSWORD={}\n",
|
||||
drive_user, drive_password, farm_password, drive_user, drive_password
|
||||
);
|
||||
let _ = std::fs::write(&env_path, env_content);
|
||||
|
||||
self.update_drive_credentials_in_database(&encrypted_drive_password)
|
||||
.ok();
|
||||
self.components.insert(
|
||||
"drive".to_string(),
|
||||
ComponentConfig {
|
||||
name: "drive".to_string(),
|
||||
required: true,
|
||||
ports: vec![9000, 9001],
|
||||
dependencies: vec![],
|
||||
linux_packages: vec![],
|
||||
macos_packages: vec![],
|
||||
windows_packages: vec![],
|
||||
download_url: Some(
|
||||
"https://dl.min.io/server/minio/release/linux-amd64/minio".to_string(),
|
||||
),
|
||||
binary_name: Some("minio".to_string()),
|
||||
pre_install_cmds_linux: vec![],
|
||||
post_install_cmds_linux: vec![
|
||||
"wget https://dl.min.io/client/mc/release/linux-amd64/mc -O {{BIN_PATH}}/mc"
|
||||
.to_string(),
|
||||
"chmod +x {{BIN_PATH}}/mc".to_string(),
|
||||
],
|
||||
pre_install_cmds_macos: vec![],
|
||||
post_install_cmds_macos: vec![
|
||||
"wget https://dl.min.io/client/mc/release/darwin-amd64/mc -O {{BIN_PATH}}/mc"
|
||||
.to_string(),
|
||||
"chmod +x {{BIN_PATH}}/mc".to_string(),
|
||||
],
|
||||
pre_install_cmds_windows: vec![],
|
||||
post_install_cmds_windows: vec![],
|
||||
env_vars: HashMap::from([
|
||||
("DRIVE_ROOT_USER".to_string(), drive_user.clone()),
|
||||
("DRIVE_ROOT_PASSWORD".to_string(), drive_password.clone()),
|
||||
]),
|
||||
data_download_list: Vec::new(),
|
||||
exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 > {{LOGS_PATH}}/minio.log 2>&1 & sleep 5 && {{BIN_PATH}}/mc alias set drive http://localhost:9000 minioadmin minioadmin && {{BIN_PATH}}/mc admin user add drive $DRIVE_ROOT_USER $DRIVE_ROOT_PASSWORD && {{BIN_PATH}}/mc admin policy attach drive readwrite --user $DRIVE_ROOT_USER && {{BIN_PATH}}/mc mb drive/default.gbai || true".to_string(),
|
||||
},
|
||||
);
|
||||
|
||||
// Delay updating drive credentials until database is created
|
||||
let db_env_path = self.base_path.join(".env");
|
||||
let database_url = std::env::var("DATABASE_URL")
|
||||
.unwrap_or_else(|_| "postgres://gbuser:@localhost:5432/botserver".to_string());
|
||||
let db_line = format!("DATABASE_URL={}\n", database_url);
|
||||
let _ = std::fs::write(&db_env_path, db_line);
|
||||
|
||||
// Append drive credentials after database creation
|
||||
let env_path = self.base_path.join(".env");
|
||||
let drive_lines = format!(
|
||||
"DRIVE_USER={}\nDRIVE_PASSWORD={}\nFARM_PASSWORD={}\nDRIVE_ROOT_USER={}\nDRIVE_ROOT_PASSWORD={}\n",
|
||||
drive_user, drive_password, farm_password, drive_user, drive_password
|
||||
);
|
||||
let _ = std::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&env_path)
|
||||
.and_then(|mut file| std::io::Write::write_all(&mut file, drive_lines.as_bytes()));
|
||||
|
||||
// Update drive credentials in database only after database is ready
|
||||
if std::process::Command::new("pg_isready")
|
||||
.arg("-h")
|
||||
.arg("localhost")
|
||||
.arg("-p")
|
||||
.arg("5432")
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
self.update_drive_credentials_in_database(&encrypted_drive_password)
|
||||
.ok();
|
||||
}
|
||||
}
|
||||
|
||||
fn update_drive_credentials_in_database(&self, encrypted_drive_password: &str) -> Result<()> {
|
||||
|
|
@ -191,34 +225,33 @@ impl PackageManager {
|
|||
}
|
||||
|
||||
fn register_cache(&mut self) {
|
||||
self.components.insert("cache".to_string(), ComponentConfig {
|
||||
name: "cache".to_string(),
|
||||
required: true,
|
||||
ports: vec![6379],
|
||||
dependencies: vec![],
|
||||
linux_packages: vec!["curl".to_string(), "gnupg".to_string(), "lsb-release".to_string()],
|
||||
macos_packages: vec!["redis".to_string()],
|
||||
windows_packages: vec![],
|
||||
download_url: None,
|
||||
binary_name: Some("valkey-server".to_string()),
|
||||
pre_install_cmds_linux: vec![
|
||||
"sudo bash -c 'if [ ! -f /usr/share/keyrings/valkey.gpg ]; then curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/valkey.gpg; fi'".to_string(),
|
||||
"sudo bash -c 'if [ ! -f /etc/apt/sources.list.d/valkey.list ]; then echo \"deb [signed-by=/usr/share/keyrings/valkey.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/valkey.list; fi'".to_string(),
|
||||
"sudo apt-get update && sudo apt-get install -y valkey".to_string()
|
||||
],
|
||||
post_install_cmds_linux: vec![],
|
||||
pre_install_cmds_macos: vec![],
|
||||
post_install_cmds_macos: vec![],
|
||||
pre_install_cmds_windows: vec![
|
||||
|
||||
"powershell -Command \"if (!(Test-Path -Path 'C:\\ProgramData\\valkey\\keyrings\\valkey.gpg')) { Invoke-WebRequest -Uri 'https://packages.redis.io/gpg' -OutFile C:\\ProgramData\\valkey\\keyrings\\valkey.gpg }\"".to_string(),
|
||||
"powershell -Command \"if (!(Test-Path -Path 'C:\\ProgramData\\valkey\\sources.list')) { Add-Content -Path 'C:\\ProgramData\\valkey\\sources.list' -Value 'deb [signed-by=C:\\ProgramData\\valkey\\keyrings\\valkey.gpg] https://packages.redis.io/windows valkey main' }\"".to_string(),
|
||||
"powershell -Command \"winget install -e --id Valkey valkey-server\"".to_string()
|
||||
],
|
||||
post_install_cmds_windows: vec![],
|
||||
env_vars: HashMap::new(),
|
||||
exec_cmd: "valkey-server --port 6379 --dir {{DATA_PATH}}".to_string(),
|
||||
});
|
||||
self.components.insert(
|
||||
"cache".to_string(),
|
||||
ComponentConfig {
|
||||
name: "cache".to_string(),
|
||||
required: true,
|
||||
ports: vec![6379],
|
||||
dependencies: vec![],
|
||||
linux_packages: vec![],
|
||||
macos_packages: vec![],
|
||||
windows_packages: vec![],
|
||||
download_url: Some(
|
||||
"https://download.valkey.io/releases/valkey-9.0.0-jammy-x86_64.tar.gz".to_string(),
|
||||
),
|
||||
binary_name: Some("valkey-server".to_string()),
|
||||
pre_install_cmds_linux: vec![],
|
||||
post_install_cmds_linux: vec![
|
||||
"chmod +x {{BIN_PATH}}/bin/valkey-server".to_string(),
|
||||
],
|
||||
pre_install_cmds_macos: vec![],
|
||||
post_install_cmds_macos: vec![],
|
||||
pre_install_cmds_windows: vec![],
|
||||
post_install_cmds_windows: vec![],
|
||||
env_vars: HashMap::new(),
|
||||
data_download_list: Vec::new(),
|
||||
exec_cmd: "{{BIN_PATH}}/bin/valkey-server --port 6379 --dir {{DATA_PATH}}".to_string(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn register_llm(&mut self) {
|
||||
|
|
@ -746,7 +779,10 @@ impl PackageManager {
|
|||
|
||||
if let Ok(output) = check_output {
|
||||
if output.status.success() {
|
||||
trace!("Component {} is already running, skipping start", component.name);
|
||||
trace!(
|
||||
"Component {} is already running, skipping start",
|
||||
component.name
|
||||
);
|
||||
return Ok(std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg("echo 'Already running'")
|
||||
|
|
@ -762,7 +798,11 @@ impl PackageManager {
|
|||
.replace("{{CONF_PATH}}", &conf_path.to_string_lossy())
|
||||
.replace("{{LOGS_PATH}}", &logs_path.to_string_lossy());
|
||||
|
||||
trace!("Starting component {} with command: {}", component.name, rendered_cmd);
|
||||
trace!(
|
||||
"Starting component {} with command: {}",
|
||||
component.name,
|
||||
rendered_cmd
|
||||
);
|
||||
|
||||
let child = std::process::Command::new("sh")
|
||||
.current_dir(&bin_path)
|
||||
|
|
@ -775,7 +815,10 @@ impl PackageManager {
|
|||
Err(e) => {
|
||||
let err_msg = e.to_string();
|
||||
if err_msg.contains("already running") || component.name == "tables" {
|
||||
trace!("Component {} may already be running, continuing anyway", component.name);
|
||||
trace!(
|
||||
"Component {} may already be running, continuing anyway",
|
||||
component.name
|
||||
);
|
||||
Ok(std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg("echo 'Already running'")
|
||||
|
|
|
|||
|
|
@ -6,28 +6,26 @@ use crate::session::SessionManager;
|
|||
use crate::tools::{ToolApi, ToolManager};
|
||||
use crate::whatsapp::WhatsAppAdapter;
|
||||
use diesel::{Connection, PgConnection};
|
||||
use opendal::Operator;
|
||||
use redis::Client;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::shared::models::BotResponse;
|
||||
|
||||
pub struct AppState {
|
||||
pub s3_client: Option<aws_sdk_s3::Client>,
|
||||
pub s3_operator: Option<Operator>,
|
||||
pub config: Option<AppConfig>,
|
||||
pub conn: Arc<Mutex<PgConnection>>,
|
||||
pub custom_conn: Arc<Mutex<PgConnection>>,
|
||||
pub redis_client: Option<Arc<Client>>,
|
||||
|
||||
pub session_manager: Arc<tokio::sync::Mutex<SessionManager>>,
|
||||
pub tool_manager: Arc<ToolManager>,
|
||||
pub llm_provider: Arc<dyn LLMProvider>,
|
||||
pub auth_service: Arc<tokio::sync::Mutex<AuthService>>,
|
||||
pub channels: Arc<Mutex<HashMap<String, Arc<dyn ChannelAdapter>>>>,
|
||||
pub response_channels: Arc<tokio::sync::Mutex<HashMap<String, mpsc::Sender<BotResponse>>>>,
|
||||
|
||||
pub web_adapter: Arc<WebChannelAdapter>,
|
||||
pub voice_adapter: Arc<VoiceAdapter>,
|
||||
pub whatsapp_adapter: Arc<WhatsAppAdapter>,
|
||||
|
|
@ -37,7 +35,7 @@ pub struct AppState {
|
|||
impl Clone for AppState {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
s3_client: self.s3_client.clone(),
|
||||
s3_operator: self.s3_operator.clone(),
|
||||
config: self.config.clone(),
|
||||
conn: Arc::clone(&self.conn),
|
||||
custom_conn: Arc::clone(&self.custom_conn),
|
||||
|
|
@ -59,7 +57,7 @@ impl Clone for AppState {
|
|||
impl Default for AppState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
s3_client: None,
|
||||
s3_operator: None,
|
||||
config: None,
|
||||
conn: Arc::new(Mutex::new(
|
||||
diesel::PgConnection::establish("postgres://localhost/test").unwrap(),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
name,value
|
||||
prompt-compact, 10
|
||||
prompt-cache,true
|
||||
prompt-fixed-kb,geral
|
||||
|
|
|
@ -1,67 +0,0 @@
|
|||
REM Simple KISS authentication - signup/login only, no recovery
|
||||
REM This script is called when user needs authentication
|
||||
|
||||
TALK "Welcome! Please choose an option:"
|
||||
TALK "Type 'signup' to create a new account"
|
||||
TALK "Type 'login' to access your existing account"
|
||||
|
||||
HEAR choice
|
||||
|
||||
IF choice = "signup" THEN
|
||||
TALK "Great! Let's create your account."
|
||||
TALK "Enter your email:"
|
||||
HEAR email
|
||||
|
||||
TALK "Enter your password:"
|
||||
HEAR password
|
||||
|
||||
TALK "Confirm your password:"
|
||||
HEAR confirm_password
|
||||
|
||||
IF password <> confirm_password THEN
|
||||
TALK "Passwords don't match. Please try again."
|
||||
RETURN false
|
||||
END IF
|
||||
|
||||
REM Create user in database
|
||||
LET user_id = GENERATE_UUID()
|
||||
LET result = EXEC "INSERT INTO users (id, email, password_hash, created_at) VALUES (?, ?, ?, NOW())", user_id, email, SHA256(password)
|
||||
|
||||
IF result > 0 THEN
|
||||
SET_USER user_id
|
||||
TALK "Account created successfully! You are now logged in."
|
||||
RETURN true
|
||||
ELSE
|
||||
TALK "Error creating account. Email may already exist."
|
||||
RETURN false
|
||||
END IF
|
||||
|
||||
ELSE IF choice = "login" THEN
|
||||
TALK "Please enter your email:"
|
||||
HEAR email
|
||||
|
||||
TALK "Enter your password:"
|
||||
HEAR password
|
||||
|
||||
REM Query user from database
|
||||
LET user = FIND "users", "email=" + email
|
||||
|
||||
IF user = NULL THEN
|
||||
TALK "Invalid email or password."
|
||||
RETURN false
|
||||
END IF
|
||||
|
||||
LET password_hash = SHA256(password)
|
||||
IF user.password_hash = password_hash THEN
|
||||
SET_USER user.id
|
||||
TALK "Welcome back! You are now logged in."
|
||||
RETURN true
|
||||
ELSE
|
||||
TALK "Invalid email or password."
|
||||
RETURN false
|
||||
END IF
|
||||
|
||||
ELSE
|
||||
TALK "Invalid option. Please type 'signup' or 'login'."
|
||||
RETURN false
|
||||
END IF
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
PARAM subject as string
|
||||
DESCRIPTION "Chamado quando alguém quer mudar o assunto da conversa."
|
||||
|
||||
kbname = LLM "Devolva uma única palavra circular, comunicado ou geral de acordo com a seguinte frase:" + subject
|
||||
|
||||
ADD_KB kbname
|
||||
|
||||
|
||||
TALK "You have chosen to change the subject to " + subject + "."
|
||||
|
|
@ -1,16 +1,17 @@
|
|||
REM start.bas - Runs automatically when user connects via web
|
||||
REM This is the entry point for each session
|
||||
LET resume1 = GET_BOT_MEMORY("general")
|
||||
LET resume2 = GET_BOT_MEMORY("auxiliom")
|
||||
LET resume3 = GET_BOT_MEMORY("toolbix")
|
||||
|
||||
LET resume = GET_BOT_MEMORY("resume")
|
||||
SET_CONTEXT "general", resume1
|
||||
SET_CONTEXT "auxiliom", resume2
|
||||
SET_CONTEXT "toolbix", resume3
|
||||
|
||||
IF resume <> "" THEN
|
||||
TALK resume
|
||||
ELSE
|
||||
TALK "Welcome! I'm loading the latest information..."
|
||||
END IF
|
||||
|
||||
REM Add knowledge base for weekly announcements
|
||||
ADD_KB "weekly"
|
||||
ADD_SUGGESTION "general", "Show me the weekly announcements"
|
||||
ADD_SUGGESTION "auxiliom", "Will Auxiliom help me with what?"
|
||||
ADD_SUGGESTION "auxiliom", "What does Auxiliom do?"
|
||||
ADD_SUGGESTION "toolbix", "Show me Toolbix features"
|
||||
ADD_SUGGESTION "toolbix", "How can Toolbix help my business?"
|
||||
|
||||
|
||||
TALK "You can ask me about any of the announcements or circulars."
|
||||
TALK "If you'd like to login or signup, just type 'auth'."
|
||||
|
|
|
|||
|
|
@ -1,5 +1,11 @@
|
|||
|
||||
let text = GET "default.gbdrive/default.pdf"
|
||||
let text = GET "announcements.gbkb/news/news.pdf"
|
||||
let resume = LLM "Resume this document, in a table (DO NOT THINK) no_think: " + text
|
||||
|
||||
SET_BOT_MEMORY "resume", resume
|
||||
|
||||
let text1 = GET "announcements.gbkb/auxiliom/auxiliom.pdf"
|
||||
SET_BOT_MEMORY "auxiliom", text1
|
||||
|
||||
let text2 = GET "announcements.gbkb/toolbix/toolbix.pdf"
|
||||
SET_BOT_MEMORY "toolbix", text2
|
||||
Binary file not shown.
BIN
templates/announcements.gbai/announcements.gbkb/news/news.pdf
Normal file
BIN
templates/announcements.gbai/announcements.gbkb/news/news.pdf
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,8 +1,8 @@
|
|||
name,value
|
||||
|
||||
server_host=0.0.0.0
|
||||
server_port=8080
|
||||
sites_root=/tmp
|
||||
server_host,0.0.0.0
|
||||
server_port,8080
|
||||
sites_root,/tmp
|
||||
|
||||
llm-key,gsk_
|
||||
llm-model,openai/gpt-oss-20b
|
||||
|
|
|
|||
|
Loading…
Add table
Reference in a new issue