fix: critical bugs - LLM context truncation, bot creation, S3 endpoint, vectordb seed
All checks were successful
BotServer CI/CD / build (push) Successful in 3m28s
All checks were successful
BotServer CI/CD / build (push) Successful in 3m28s
1. Fix model.starts_with('') always true - was limiting ALL models to 768 tokens
(local llama limit), truncating system prompts and KB context. Now only
applies when model=='local' or empty string, default is 32k tokens.
2. Fix create_bot_from_drive missing NOT NULL columns (llm_provider,
context_provider) - bots auto-created from S3 buckets failed to persist.
3. Fix S3 endpoint URL construction missing port 9100.
4. Fix Vault seed: vectordb.url was empty string, now defaults to
http://localhost:6333.
5. Fix Vault credential regeneration on recovery - added vault_seeds_exist().
6. Fix CA cert path for Vault TLS (botserver-stack vs botserver-stack).
7. Add bot verification after insert in create_bot_from_drive.
This commit is contained in:
parent
a131120638
commit
a4a3837c4c
6 changed files with 120 additions and 24 deletions
2
migrations/6.2.7-add-bots-slug/down.sql
Normal file
2
migrations/6.2.7-add-bots-slug/down.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
-- Remove slug column from bots table
|
||||
ALTER TABLE public.bots DROP COLUMN IF EXISTS slug;
|
||||
8
migrations/6.2.7-add-bots-slug/up.sql
Normal file
8
migrations/6.2.7-add-bots-slug/up.sql
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
-- Add slug column to bots table for drive-based bot creation
|
||||
ALTER TABLE public.bots ADD COLUMN IF NOT EXISTS slug VARCHAR(255);
|
||||
|
||||
-- Create unique constraint on slug (without WHERE clause for ON CONFLICT to work)
|
||||
ALTER TABLE public.bots ADD CONSTRAINT bots_slug_key UNIQUE (slug);
|
||||
|
||||
-- Backfill slug from name for existing bots
|
||||
UPDATE public.bots SET slug = name WHERE slug IS NULL;
|
||||
|
|
@ -1049,9 +1049,7 @@ EOF"#.to_string(),
|
|||
|
||||
exec_cmd: "{{BIN_PATH}}/vector --config {{CONF_PATH}}/monitoring/vector.toml"
|
||||
.to_string(),
|
||||
check_cmd:
|
||||
"curl -f --connect-timeout 2 -m 5 /health >/dev/null 2>&1"
|
||||
.to_string(),
|
||||
check_cmd: "curl -f --connect-timeout 2 -m 5 /health >/dev/null 2>&1".to_string(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -1549,6 +1547,51 @@ VAULT_CACERT={}
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if Vault already has seeded credentials (to avoid overwriting on recovery)
|
||||
fn vault_seeds_exist(
|
||||
&self,
|
||||
vault_addr: &str,
|
||||
root_token: &str,
|
||||
ca_cert: &std::path::Path,
|
||||
vault_bin: &std::path::Path,
|
||||
) -> Result<bool> {
|
||||
let args = vec![
|
||||
"kv".to_string(),
|
||||
"get".to_string(),
|
||||
"-tls-skip-verify".to_string(),
|
||||
format!("-address={}", vault_addr),
|
||||
"-field=accesskey".to_string(),
|
||||
"secret/gbo/drive".to_string(),
|
||||
];
|
||||
|
||||
let result = SafeCommand::new(vault_bin.to_str().unwrap_or("vault"))
|
||||
.and_then(|c| {
|
||||
let mut cmd = c;
|
||||
for arg in &args {
|
||||
cmd = cmd.trusted_arg(arg)?;
|
||||
}
|
||||
Ok(cmd)
|
||||
})
|
||||
.and_then(|c| {
|
||||
c.env("VAULT_ADDR", vault_addr)
|
||||
.and_then(|c| c.env("VAULT_TOKEN", root_token))
|
||||
.and_then(|c| c.env("VAULT_CACERT", ca_cert.to_str().unwrap_or("")))
|
||||
})
|
||||
.and_then(|c| c.execute());
|
||||
|
||||
match result {
|
||||
Ok(output) => {
|
||||
if output.status.success() {
|
||||
let value = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||
Ok(!value.is_empty())
|
||||
} else {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
Err(_) => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed default credentials into Vault KV2 after initialization
|
||||
fn seed_vault_defaults(
|
||||
&self,
|
||||
|
|
@ -1633,10 +1676,7 @@ VAULT_CACERT={}
|
|||
("model".to_string(), "gpt-4".to_string()),
|
||||
("openai_key".to_string(), "none".to_string()),
|
||||
("anthropic_key".to_string(), "none".to_string()),
|
||||
(
|
||||
"ollama_url".to_string(),
|
||||
"".to_string(),
|
||||
),
|
||||
("ollama_url".to_string(), "".to_string()),
|
||||
],
|
||||
),
|
||||
(
|
||||
|
|
@ -1656,7 +1696,7 @@ VAULT_CACERT={}
|
|||
(
|
||||
"secret/gbo/vectordb",
|
||||
vec![
|
||||
("url".to_string(), "".to_string()),
|
||||
("url".to_string(), "http://localhost:6333".to_string()),
|
||||
("host".to_string(), "localhost".to_string()),
|
||||
("port".to_string(), "6333".to_string()),
|
||||
("grpc_port".to_string(), "6334".to_string()),
|
||||
|
|
@ -1813,9 +1853,13 @@ VAULT_CACERT={}
|
|||
warn!("No root token found - Vault may need manual recovery");
|
||||
}
|
||||
|
||||
// Seed defaults if we have a token (ensure credentials exist even during recovery)
|
||||
// Seed defaults ONLY if not already present (skip during recovery to preserve credentials)
|
||||
if let Some(ref token) = root_token {
|
||||
let _ = self.seed_vault_defaults(&vault_addr, token, &ca_cert, &vault_bin);
|
||||
if self.vault_seeds_exist(&vault_addr, token, &ca_cert, &vault_bin)? {
|
||||
info!("Vault credentials already exist, skipping seed on recovery");
|
||||
} else {
|
||||
let _ = self.seed_vault_defaults(&vault_addr, token, &ca_cert, &vault_bin);
|
||||
}
|
||||
}
|
||||
|
||||
info!("Vault recovery complete");
|
||||
|
|
|
|||
|
|
@ -140,19 +140,30 @@ pub fn get_stack_path() -> String {
|
|||
pub async fn create_s3_operator(
|
||||
config: &DriveConfig,
|
||||
) -> Result<S3Client, Box<dyn std::error::Error>> {
|
||||
log::info!("Creating S3 operator with server: {}, access_key: {}", config.server, config.access_key);
|
||||
let endpoint = {
|
||||
let server = if config.server.starts_with("http://") || config.server.starts_with("https://") {
|
||||
let base = if config.server.starts_with("http://") || config.server.starts_with("https://") {
|
||||
config.server.clone()
|
||||
} else {
|
||||
format!("http://{}", config.server)
|
||||
};
|
||||
if server.ends_with('/') {
|
||||
server
|
||||
let with_port = if base.contains("://") {
|
||||
let without_scheme = base.split("://").nth(1).unwrap_or("");
|
||||
let has_port = without_scheme.contains(':');
|
||||
if has_port || without_scheme.is_empty() {
|
||||
base
|
||||
} else {
|
||||
format!("{}:9100", base.trim_end_matches('/'))
|
||||
}
|
||||
} else {
|
||||
format!("{}/", server)
|
||||
format!("http://{}:9100", base)
|
||||
};
|
||||
if with_port.ends_with('/') {
|
||||
with_port
|
||||
} else {
|
||||
format!("{}/", with_port)
|
||||
}
|
||||
};
|
||||
log::info!("Creating S3 operator with endpoint: {}, access_key: {}", endpoint, config.access_key);
|
||||
|
||||
let (access_key, secret_key) = if config.access_key.is_empty() || config.secret_key.is_empty() {
|
||||
let (manager, is_vault_enabled) = {
|
||||
|
|
|
|||
|
|
@ -289,7 +289,7 @@ impl LLMProvider for OpenAIClient {
|
|||
128000 // Cerebras gpt-oss models and GPT-4 variants
|
||||
} else if model.contains("gpt-3.5") {
|
||||
16385
|
||||
} else if model.starts_with("") || model == "local" {
|
||||
} else if model == "local" || model.is_empty() {
|
||||
768 // Local llama.cpp server context limit
|
||||
} else {
|
||||
32768 // Default conservative limit for modern models
|
||||
|
|
@ -378,7 +378,7 @@ impl LLMProvider for OpenAIClient {
|
|||
128000 // Cerebras gpt-oss models and GPT-4 variants
|
||||
} else if model.contains("gpt-3.5") {
|
||||
16385
|
||||
} else if model.starts_with("") || model == "local" {
|
||||
} else if model == "local" || model.is_empty() {
|
||||
768 // Local llama.cpp server context limit
|
||||
} else {
|
||||
32768 // Default conservative limit for modern models
|
||||
|
|
|
|||
|
|
@ -966,13 +966,22 @@ async fn start_drive_monitors(
|
|||
let create_state = state_for_scan.clone();
|
||||
let bn = bot_name.to_string();
|
||||
let pool_create = pool_clone.clone();
|
||||
if let Err(e) = tokio::task::spawn_blocking(move || {
|
||||
match tokio::task::spawn_blocking(move || {
|
||||
create_bot_from_drive(&create_state, &pool_create, &bn)
|
||||
})
|
||||
.await
|
||||
{
|
||||
error!("Failed to create bot '{}': {}", bot_name, e);
|
||||
continue;
|
||||
Ok(Err(e)) => {
|
||||
error!("Failed to create bot '{}': {}", bot_name, e);
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Task failed to create bot '{}': {}", bot_name, e);
|
||||
continue;
|
||||
}
|
||||
Ok(Ok(())) => {
|
||||
info!("Bot '{}' created successfully", bot_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1080,12 +1089,22 @@ fn create_bot_from_drive(
|
|||
let bot_id_str = bot_id.to_string();
|
||||
let org_id_str = org_result.org_id.to_string();
|
||||
|
||||
sql_query(format!(
|
||||
"INSERT INTO bots (id, name, slug, org_id, is_active, created_at) VALUES ('{}', '{}', '{}', '{}', true, NOW())",
|
||||
// Try to insert, if conflict on name, update instead
|
||||
let result = sql_query(format!(
|
||||
"INSERT INTO bots (id, name, slug, org_id, is_active, created_at, llm_provider, llm_config, context_provider, context_config) VALUES ('{}', '{}', '{}', '{}', true, NOW(), 'openai', '{{}}', 'openai', '{{}}') ON CONFLICT (id) DO UPDATE SET is_active = true",
|
||||
bot_id_str, bot_name, bot_name, org_id_str
|
||||
))
|
||||
.execute(&mut conn)
|
||||
.map_err(|e| format!("Failed to create bot '{}': {}", bot_name, e))?;
|
||||
.execute(&mut conn);
|
||||
|
||||
if result.is_err() {
|
||||
// Bot might already exist with different id, try to update by name
|
||||
sql_query(format!(
|
||||
"UPDATE bots SET is_active = true, slug = '{}', llm_provider = 'openai', llm_config = '{{}}', context_provider = 'openai', context_config = '{{}}' WHERE name = '{}'",
|
||||
bot_name, bot_name
|
||||
))
|
||||
.execute(&mut conn)
|
||||
.map_err(|e| format!("Failed to update bot '{}': {}", bot_name, e))?;
|
||||
}
|
||||
|
||||
let db_name = format!("bot_{}", bot_name.replace('-', "_"));
|
||||
let _ = sql_query(format!(
|
||||
|
|
@ -1094,6 +1113,18 @@ fn create_bot_from_drive(
|
|||
))
|
||||
.execute(&mut conn);
|
||||
|
||||
// Verify the bot was actually inserted
|
||||
let exists: bool = sql_query(format!(
|
||||
"SELECT 1 FROM bots WHERE name = '{}' LIMIT 1",
|
||||
bot_name
|
||||
))
|
||||
.execute(&mut conn)
|
||||
.is_ok();
|
||||
|
||||
if !exists {
|
||||
return Err(format!("Bot '{}' was not found in database after insert", bot_name));
|
||||
}
|
||||
|
||||
info!("Bot '{}' created successfully with id {}", bot_name, bot_id);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue