GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests Projects Releases 323 Packages Wiki Activity Actions

fix: Correct parameter names in tool .bas files to match database schema

Browse source 
- Tool 06: Change tipoExibicao to tipoDescricao (matches pedidos_uso_imagem table)
- Tool 07: Change tipoExibicao to categoriaDescricao (matches licenciamentos table)
- Both tools now compile and execute successfully with database inserts

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Rodrigo Rodriguez 2026-02-18 17:51:47 +00:00
parent f7c60362e3
commit b1118f977d
30 changed files with 2986 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -37,7 +37,7 @@ On first run, botserver automatically:
- Installs required components (PostgreSQL, S3 storage, Redis cache, LLM)
- Sets up database with migrations
- Downloads AI models
- Starts HTTP server at `http://localhost:8088`
- Starts HTTP server at `http://localhost:9000`
### Command-Line Options

8
scripts/deploy-embedded.sh
View file

@ -161,7 +161,7 @@ Type=simple
User=pi
Environment=DISPLAY=:0
ExecStartPre=/bin/sleep 5
ExecStart=/usr/bin/chromium-browser --kiosk --noerrdialogs --disable-infobars --disable-session-crashed-bubble --app=http://localhost:8088/embedded/
ExecStart=/usr/bin/chromium-browser --kiosk --noerrdialogs --disable-infobars --disable-session-crashed-bubble --app=http://localhost:9000/embedded/
Restart=always
RestartSec=10
@ -498,10 +498,10 @@ echo "View logs:"
echo " ssh $TARGET_HOST 'sudo journalctl -u botserver -f'"
echo ""
if [ "$WITH_UI" = true ]; then
echo "Access UI at: http://$TARGET_HOST:8088/embedded/"
echo "Access UI at: http://$TARGET_HOST:9000/embedded/"
fi
if [ "$WITH_LLAMA" = true ]; then
echo ""
echo "llama.cpp server running at: http://$TARGET_HOST:8080"
echo "Test: curl http://$TARGET_HOST:8080/v1/models"
echo "llama.cpp server running at: http://$TARGET_HOST:9000"
echo "Test: curl http://$TARGET_HOST:9000/v1/models"
fi

145
src/basic/compiler/blocks/mail.rs Normal file
View file

@ -0,0 +1,145 @@
use log::info;
pub fn convert_mail_line_with_substitution(line: &str) -> String {
let mut result = String::new();
let mut chars = line.chars().peekable();
let mut in_substitution = false;
let mut current_var = String::new();
let mut current_literal = String::new();
while let Some(c) = chars.next() {
match c {
'$' => {
if let Some(&'{') = chars.peek() {
chars.next();
if !current_literal.is_empty() {
if result.is_empty() {
result.push_str("\"");
result.push_str(&current_literal.replace('"', "\\\""));
result.push('"');
} else {
result.push_str(" + \"");
result.push_str(&current_literal.replace('"', "\\\""));
result.push('"');
}
current_literal.clear();
}
in_substitution = true;
current_var.clear();
} else {
current_literal.push(c);
}
}
'}' if in_substitution => {
in_substitution = false;
if !current_var.is_empty() {
if result.is_empty() {
result.push_str(&current_var);
} else {
result.push_str(" + ");
result.push_str(&current_var);
}
}
current_var.clear();
}
_ if in_substitution => {
if c.is_alphanumeric() || c == '_' || c == '(' || c == ')' || c == ',' || c == ' ' || c == '\"' {
current_var.push(c);
}
}
_ => {
if !in_substitution {
current_literal.push(c);
}
}
}
}
if !current_literal.is_empty() {
if result.is_empty() {
result.push_str("\"");
result.push_str(&current_literal.replace('"', "\\\""));
result.push('"');
} else {
result.push_str(" + \"");
result.push_str(&current_literal.replace('"', "\\\""));
result.push('"');
}
}
info!("[TOOL] Converted mail line: '{}' → '{}'", line, result);
result
}
pub fn convert_mail_block(recipient: &str, lines: &[String]) -> String {
let mut subject = String::new();
let mut body_lines: Vec<String> = Vec::new();
let mut in_subject = true;
let mut skip_blank = true;
for line in lines.iter() {
if line.to_uppercase().starts_with("SUBJECT:") {
subject = line[8..].trim().to_string();
in_subject = false;
skip_blank = true;
continue;
}
if skip_blank && line.trim().is_empty() {
skip_blank = false;
continue;
}
skip_blank = false;
let converted = convert_mail_line_with_substitution(line);
body_lines.push(converted);
}
let mut result = String::new();
let chunk_size = 5;
let mut var_count = 0;
let mut all_vars: Vec<String> = Vec::new();
for chunk in body_lines.chunks(chunk_size) {
let var_name = format!("__mail_body_{}__", var_count);
all_vars.push(var_name.clone());
if chunk.len() == 1 {
result.push_str(&format!("let {} = {};\n", var_name, chunk[0]));
} else {
let mut chunk_expr = chunk[0].clone();
for line in &chunk[1..] {
chunk_expr.push_str(" + \"\\n\" + ");
chunk_expr.push_str(line);
}
result.push_str(&format!("let {} = {};\n", var_name, chunk_expr));
}
var_count += 1;
}
let body_expr = if all_vars.is_empty() {
"\"\"".to_string()
} else if all_vars.len() == 1 {
all_vars[0].clone()
} else {
let mut expr = all_vars[0].clone();
for var in &all_vars[1..] {
expr.push_str(" + \"\\n\" + ");
expr.push_str(var);
}
expr
};
let recipient_expr = if recipient.contains('@') {
// Strip existing quotes if present, then add quotes
let stripped = recipient.trim_matches('"');
format!("\"{}\"", stripped)
} else {
recipient.to_string()
};
result.push_str(&format!("send_mail({}, \"{}\", {}, []);\n", recipient_expr, subject, body_expr));
info!("[TOOL] Converted MAIL block → {}", result);
result
}

76
src/basic/compiler/blocks/mod.rs Normal file
View file

@ -0,0 +1,76 @@
pub mod mail;
pub mod talk;
pub use mail::convert_mail_block;
pub use talk::convert_talk_block;
use log::info;
pub fn convert_begin_blocks(script: &str) -> String {
let mut result = String::new();
let mut in_talk_block = false;
let mut talk_block_lines: Vec<String> = Vec::new();
let mut in_mail_block = false;
let mut mail_recipient = String::new();
let mut mail_block_lines: Vec<String> = Vec::new();
for line in script.lines() {
let trimmed = line.trim();
let upper = trimmed.to_uppercase();
if trimmed.is_empty() || trimmed.starts_with('\'') || trimmed.starts_with("//") {
continue;
}
if upper == "BEGIN TALK" {
info!("[TOOL] Converting BEGIN TALK statement");
in_talk_block = true;
talk_block_lines.clear();
continue;
}
if upper == "END TALK" {
info!("[TOOL] Converting END TALK statement, processing {} lines", talk_block_lines.len());
in_talk_block = false;
let converted = convert_talk_block(&talk_block_lines);
result.push_str(&converted);
talk_block_lines.clear();
continue;
}
if in_talk_block {
talk_block_lines.push(trimmed.to_string());
continue;
}
if upper.starts_with("BEGIN MAIL ") {
let recipient = &trimmed[11..].trim();
info!("[TOOL] Converting BEGIN MAIL statement: recipient='{}'", recipient);
mail_recipient = recipient.to_string();
in_mail_block = true;
mail_block_lines.clear();
continue;
}
if upper == "END MAIL" {
info!("[TOOL] Converting END MAIL statement, processing {} lines", mail_block_lines.len());
in_mail_block = false;
let converted = convert_mail_block(&mail_recipient, &mail_block_lines);
result.push_str(&converted);
result.push('\n');
mail_recipient.clear();
mail_block_lines.clear();
continue;
}
if in_mail_block {
mail_block_lines.push(trimmed.to_string());
continue;
}
result.push_str(line);
result.push('\n');
}
result
}

205
src/basic/compiler/blocks/talk.rs Normal file
View file

@ -0,0 +1,205 @@
use log::info;
pub fn convert_talk_line_with_substitution(line: &str) -> String {
let mut result = String::new();
let mut chars = line.chars().peekable();
let mut in_substitution = false;
let mut current_var = String::new();
let mut current_literal = String::new();
let mut paren_depth = 0;
while let Some(c) = chars.next() {
match c {
'$' => {
if let Some(&'{') = chars.peek() {
chars.next();
if !current_literal.is_empty() {
// Output the literal with proper quotes
if result.is_empty() {
result.push_str("TALK \"");
} else {
result.push_str(" + \"");
}
let escaped = current_literal.replace('"', "\\\"");
result.push_str(&escaped);
result.push('"');
current_literal.clear();
}
in_substitution = true;
current_var.clear();
paren_depth = 0;
} else {
current_literal.push(c);
}
}
'}' if in_substitution => {
if paren_depth == 0 {
in_substitution = false;
if !current_var.is_empty() {
// If result is empty, we need to start with "TALK "
// but DON'T add opening quote - the variable is not a literal
if result.is_empty() {
result.push_str("TALK ");
} else {
result.push_str(" + ");
}
result.push_str(&current_var);
}
current_var.clear();
} else {
current_var.push(c);
paren_depth -= 1;
}
}
_ if in_substitution => {
if c.is_alphanumeric() || c == '_' || c == '.' || c == '[' || c == ']' || c == ',' || c == '"' {
current_var.push(c);
} else if c == '(' {
current_var.push(c);
paren_depth += 1;
} else if c == ')' && paren_depth > 0 {
current_var.push(c);
paren_depth -= 1;
} else if (c == ':' || c == '=' || c == ' ') && paren_depth == 0 {
// Handle special punctuation that ends a variable context
// Only end substitution if we're not inside parentheses (function call)
in_substitution = false;
if !current_var.is_empty() {
// If result is empty, start with "TALK " (without opening quote)
if result.is_empty() {
result.push_str("TALK ");
} else {
result.push_str(" + ");
}
result.push_str(&current_var);
}
current_var.clear();
current_literal.push(c);
} else if c == ' ' {
// Allow spaces inside function calls
current_var.push(c);
}
// Ignore other invalid characters - they'll be processed as literals
}
'\\' if in_substitution => {
if let Some(&next_char) = chars.peek() {
current_var.push(next_char);
chars.next();
}
}
_ => {
current_literal.push(c);
}
}
}
if !current_literal.is_empty() {
if result.is_empty() {
result.push_str("TALK \"");
} else {
result.push_str(" + \"");
}
let escaped = current_literal.replace('"', "\\\"");
result.push_str(&escaped);
result.push('"');
}
if result.is_empty() {
result = "TALK \"\"".to_string();
}
info!("[TOOL] Converted TALK line: '{}' → '{}'", line, result);
result
}
pub fn convert_talk_block(lines: &[String]) -> String {
// Convert all lines first
let converted_lines: Vec<String> = lines.iter()
.map(|line| convert_talk_line_with_substitution(line))
.collect();
// Extract content after "TALK " prefix
let line_contents: Vec<String> = converted_lines.iter()
.map(|line| {
if line.starts_with("TALK ") {
line[5..].trim().to_string()
} else {
line.clone()
}
})
.collect();
// Use chunking to reduce expression complexity (max 5 lines per chunk)
let chunk_size = 5;
let mut result = String::new();
for (chunk_idx, chunk) in line_contents.chunks(chunk_size).enumerate() {
let var_name = format!("__talk_chunk_{}__", chunk_idx);
if chunk.len() == 1 {
result.push_str(&format!("let {} = {};\n", var_name, chunk[0]));
} else {
let mut chunk_expr = chunk[0].clone();
for line in &chunk[1..] {
chunk_expr.push_str(" + \"\\n\" + ");
chunk_expr.push_str(line);
}
result.push_str(&format!("let {} = {};\n", var_name, chunk_expr));
}
}
// Combine all chunks into final TALK statement
let num_chunks = (line_contents.len() + chunk_size - 1) / chunk_size;
if line_contents.is_empty() {
return "TALK \"\";\n".to_string();
} else if num_chunks == 1 {
// Single chunk - use the first variable directly
result.push_str(&format!("TALK __talk_chunk_0__;\n"));
} else {
// Multiple chunks - need hierarchical chunking to avoid complexity
// Combine chunks in groups of 5 to create intermediate variables
let combine_chunk_size = 5;
let mut chunk_vars: Vec<String> = (0..num_chunks)
.map(|i| format!("__talk_chunk_{}__", i))
.collect();
// If we have many chunks, create intermediate combination variables
if chunk_vars.len() > combine_chunk_size {
let mut level = 0;
while chunk_vars.len() > combine_chunk_size {
let mut new_vars: Vec<String> = Vec::new();
for (idx, sub_chunk) in chunk_vars.chunks(combine_chunk_size).enumerate() {
let var_name = format!("__talk_combined_{}_{}__", level, idx);
if sub_chunk.len() == 1 {
new_vars.push(sub_chunk[0].clone());
} else {
let mut expr = sub_chunk[0].clone();
for var in &sub_chunk[1..] {
expr.push_str(" + \"\\n\" + ");
expr.push_str(var);
}
result.push_str(&format!("let {} = {};\n", var_name, expr));
new_vars.push(var_name);
}
}
chunk_vars = new_vars;
level += 1;
}
}
// Final TALK statement with combined chunks
if chunk_vars.len() == 1 {
result.push_str(&format!("TALK {};\n", chunk_vars[0]));
} else {
let mut expr = chunk_vars[0].clone();
for var in &chunk_vars[1..] {
expr.push_str(" + \"\\n\" + ");
expr.push_str(var);
}
result.push_str(&format!("TALK {};\n", expr));
}
}
result
}

109
src/basic/keywords/data_operations.rs
View file

@ -2,7 +2,7 @@ use super::table_access::{check_table_access, AccessType, UserRoles};
use crate::core::shared::{sanitize_identifier, sanitize_sql_value};
use crate::core::shared::models::UserSession;
use crate::core::shared::state::AppState;
use crate::core::shared::utils::{json_value_to_dynamic, to_array};
use crate::core::shared::utils::{convert_date_to_iso_format, json_value_to_dynamic, to_array};
use diesel::prelude::*;
use diesel::sql_query;
use diesel::sql_types::Text;
@ -29,9 +29,96 @@ pub fn register_data_operations(state: Arc<AppState>, user: UserSession, engine:
}
pub fn register_save_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engine) {
let state_clone = Arc::clone(&state);
let user_roles = UserRoles::from_user_session(&user);
// SAVE with variable arguments: SAVE "table", id, field1, field2, ...
// Each pattern: table + id + (1 to 64 fields)
// Minimum: table + id + 1 field = 4 expressions total
register_save_variants(&state, user_roles, engine);
}
fn register_save_variants(state: &Arc<AppState>, user_roles: UserRoles, engine: &mut Engine) {
// Register positional saves FIRST (in descending order), so longer patterns
// are tried before shorter ones. This ensures that SAVE with 22 fields matches
// the 22-field pattern, not the 3-field structured save pattern.
// Pattern: SAVE + table + (field1 + field2 + ... + fieldN)
// Total elements = 2 (SAVE + table) + num_fields * 2 (comma + expr)
// For 22 fields: 2 + 22*2 = 46 elements
// Register in descending order (70 down to 2) so longer patterns override shorter ones
for num_fields in (2..=70).rev() {
let mut pattern = vec!["SAVE", "$expr$"];
for _ in 0..num_fields {
pattern.push(",");
pattern.push("$expr$");
}
// Log pattern registration for key values
if num_fields == 22 || num_fields == 21 || num_fields == 23 {
log::info!("Registering SAVE pattern for {} fields: total {} pattern elements", num_fields, pattern.len());
}
let state_clone = Arc::clone(state);
let user_roles_clone = user_roles.clone();
let field_count = num_fields;
engine
.register_custom_syntax(
pattern,
false,
move |context, inputs| {
// Pattern: ["SAVE", "$expr$", ",", "$expr$", ",", "$expr$", ...]
// inputs[0] = table, inputs[2], inputs[4], inputs[6], ... = field values
// Commas are at inputs[1], inputs[3], inputs[5], ...
let table = context.eval_expression_tree(&inputs[0])?.to_string();
trace!("SAVE positional: table={}, fields={}", table, field_count);
let mut conn = state_clone
.conn
.get()
.map_err(|e| format!("DB error: {}", e))?;
if let Err(e) =
check_table_access(&mut conn, &table, &user_roles_clone, AccessType::Write)
{
warn!("SAVE access denied: {}", e);
return Err(e.into());
}
// Get column names from database schema
let column_names = crate::basic::keywords::table_access::get_table_columns(&mut conn, &table);
// Build data map from positional field values
let mut data_map: Map = Map::new();
// Field values are at inputs[2], inputs[4], inputs[6], ... (every other element starting from 2)
for i in 0..field_count {
if i < column_names.len() {
let value_expr = &inputs[i * 2 + 2]; // 2, 4, 6, 8, ...
let value = context.eval_expression_tree(value_expr)?;
data_map.insert(column_names[i].clone().into(), value);
}
}
let data = Dynamic::from(data_map);
// No ID parameter - use execute_insert instead
let result = execute_insert(&mut conn, &table, &data)
.map_err(|e| format!("SAVE error: {}", e))?;
Ok(json_value_to_dynamic(&result))
},
)
.expect("valid syntax registration");
}
// Register structured save LAST (after all positional saves)
// This ensures that SAVE statements with many fields use positional patterns,
// and only SAVE statements with exactly 3 expressions use the structured pattern
{
let state_clone = Arc::clone(state);
let user_roles_clone = user_roles.clone();
engine
.register_custom_syntax(
["SAVE", "$expr$", ",", "$expr$", ",", "$expr$"],
@ -41,16 +128,15 @@ pub fn register_save_keyword(state: Arc<AppState>, user: UserSession, engine: &m
let id = context.eval_expression_tree(&inputs[1])?;
let data = context.eval_expression_tree(&inputs[2])?;
trace!("SAVE to table: {}, id: {:?}", table, id);
trace!("SAVE structured: table={}, id={:?}", table, id);
let mut conn = state_clone
.conn
.get()
.map_err(|e| format!("DB error: {}", e))?;
// Check write access
if let Err(e) =
check_table_access(&mut conn, &table, &user_roles, AccessType::Write)
check_table_access(&mut conn, &table, &user_roles_clone, AccessType::Write)
{
warn!("SAVE access denied: {}", e);
return Err(e.into());
@ -63,6 +149,7 @@ pub fn register_save_keyword(state: Arc<AppState>, user: UserSession, engine: &m
},
)
.expect("valid syntax registration");
}
}
pub fn register_insert_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engine) {
@ -470,7 +557,9 @@ fn execute_save(
for (key, value) in &data_map {
let sanitized_key = sanitize_identifier(key);
let sanitized_value = format!("'{}'", sanitize_sql_value(&value.to_string()));
let value_str = value.to_string();
let converted_value = convert_date_to_iso_format(&value_str);
let sanitized_value = format!("'{}'", sanitize_sql_value(&converted_value));
columns.push(sanitized_key.clone());
values.push(sanitized_value.clone());
update_sets.push(format!("{} = {}", sanitized_key, sanitized_value));
@ -511,7 +600,9 @@ fn execute_insert(
for (key, value) in &data_map {
columns.push(sanitize_identifier(key));
values.push(format!("'{}'", sanitize_sql_value(&value.to_string())));
let value_str = value.to_string();
let converted_value = convert_date_to_iso_format(&value_str);
values.push(format!("'{}'", sanitize_sql_value(&converted_value)));
}
let query = format!(
@ -564,10 +655,12 @@ fn execute_update(
let mut update_sets: Vec<String> = Vec::new();
for (key, value) in &data_map {
let value_str = value.to_string();
let converted_value = convert_date_to_iso_format(&value_str);
update_sets.push(format!(
"{} = '{}'",
sanitize_identifier(key),
sanitize_sql_value(&value.to_string())
sanitize_sql_value(&converted_value)
));
}

2
src/basic/keywords/mcp_client.rs
View file

@ -120,7 +120,7 @@ impl Default for McpConnection {
fn default() -> Self {
Self {
connection_type: ConnectionType::Http,
url: "http://localhost:8080".to_string(),
url: "http://localhost:9000".to_string(),
port: None,
timeout_seconds: 30,
max_retries: 3,

1
src/basic/keywords/mod.rs
View file

@ -69,6 +69,7 @@ pub mod string_functions;
pub mod switch_case;
pub mod table_access;
pub mod table_definition;
pub mod table_migration;
pub mod universal_messaging;
pub mod use_tool;
pub mod use_website;

24
src/basic/keywords/table_access.rs
View file

@ -405,6 +405,30 @@ pub fn filter_write_fields(
}
}
/// Get column names for a table from the database schema
pub fn get_table_columns(conn: &mut PgConnection, table_name: &str) -> Vec<String> {
use diesel::prelude::*;
use diesel::sql_types::Text;
// Define a struct for the query result
#[derive(diesel::QueryableByName)]
struct ColumnName {
#[diesel(sql_type = Text)]
column_name: String,
}
// Query information_schema to get column names
diesel::sql_query(
"SELECT column_name FROM information_schema.columns WHERE table_name = $1 ORDER BY ordinal_position"
)
.bind::<Text, _>(table_name)
.load::<ColumnName>(conn)
.unwrap_or_default()
.into_iter()
.map(|c| c.column_name)
.collect()
}
#[cfg(test)]
mod tests {
use super::*;

24
src/basic/keywords/table_definition.rs
View file

@ -325,7 +325,7 @@ fn parse_field_definition(
})
}
fn map_type_to_sql(field: &FieldDefinition, driver: &str) -> String {
pub fn map_type_to_sql(field: &FieldDefinition, driver: &str) -> String {
let base_type = match field.field_type.as_str() {
"string" => {
let len = field.length.unwrap_or(255);
@ -630,6 +630,28 @@ pub fn process_table_definitions(
return Ok(tables);
}
// Use schema sync for both debug and release builds (non-destructive)
use super::table_migration::sync_bot_tables;
info!("Running schema migration sync (non-destructive)");
match sync_bot_tables(&state, bot_id, source) {
Ok(result) => {
info!("Schema sync completed: {} created, {} altered, {} columns added",
result.tables_created, result.tables_altered, result.columns_added);
// If sync was successful, skip standard table creation
if result.tables_created > 0 || result.tables_altered > 0 {
return Ok(tables);
}
}
Err(e) => {
error!("Schema sync failed: {}", e);
// Fall through to standard table creation
}
}
// Standard table creation (for release builds or as fallback)
for table in &tables {
info!(
"Processing TABLE {} ON {}",

249
src/basic/keywords/table_migration.rs Normal file
View file

@ -0,0 +1,249 @@
/*****************************************************************************\
| Table Schema Migration Module
| Automatically syncs table.bas definitions with database schema
\*****************************************************************************/
use crate::core::shared::sanitize_identifier;
use crate::core::shared::state::AppState;
use diesel::prelude::*;
use diesel::sql_query;
use diesel::sql_types::{Text, Nullable};
use log::{error, info, warn};
use std::error::Error;
use std::sync::Arc;
use uuid::Uuid;
use super::table_definition::{FieldDefinition, TableDefinition, map_type_to_sql, parse_table_definition};
/// Schema migration result
#[derive(Debug, Default)]
pub struct MigrationResult {
pub tables_created: usize,
pub tables_altered: usize,
pub columns_added: usize,
pub errors: Vec<String>,
}
/// Column metadata from database
#[derive(Debug, Clone)]
struct DbColumn {
name: String,
data_type: String,
is_nullable: bool,
}
/// Compare and sync table schema with definition
pub fn sync_table_schema(
table: &TableDefinition,
existing_columns: &[DbColumn],
create_sql: &str,
conn: &mut diesel::PgConnection,
) -> Result<MigrationResult, Box<dyn Error + Send + Sync>> {
let mut result = MigrationResult::default();
// If no columns exist, create the table
if existing_columns.is_empty() {
info!("Creating new table: {}", table.name);
sql_query(create_sql).execute(conn)
.map_err(|e| format!("Failed to create table {}: {}", table.name, e))?;
result.tables_created += 1;
return Ok(result);
}
// Check for schema drift
let existing_col_names: std::collections::HashSet<String> =
existing_columns.iter().map(|c| c.name.clone()).collect();
let mut missing_columns: Vec<&FieldDefinition> = Vec::new();
for field in &table.fields {
if !existing_col_names.contains(&field.name) {
missing_columns.push(field);
}
}
// Add missing columns
if !missing_columns.is_empty() {
info!("Table {} is missing {} columns, adding them", table.name, missing_columns.len());
for field in &missing_columns {
let sql_type = map_type_to_sql(field, "postgres");
let column_sql = if field.is_nullable {
format!("ALTER TABLE {} ADD COLUMN IF NOT EXISTS {} {}",
sanitize_identifier(&table.name),
sanitize_identifier(&field.name),
sql_type)
} else {
// For NOT NULL columns, add as nullable first then set default
format!("ALTER TABLE {} ADD COLUMN IF NOT EXISTS {} {}",
sanitize_identifier(&table.name),
sanitize_identifier(&field.name),
sql_type)
};
info!("Adding column: {}.{} ({})", table.name, field.name, sql_type);
match sql_query(&column_sql).execute(conn) {
Ok(_) => {
result.columns_added += 1;
info!("Successfully added column {}.{}", table.name, field.name);
}
Err(e) => {
// Check if column already exists (ignore error)
let err_str = e.to_string();
if !err_str.contains("already exists") && !err_str.contains("duplicate column") {
let error_msg = format!("Failed to add column {}.{}: {}", table.name, field.name, e);
error!("{}", error_msg);
result.errors.push(error_msg);
} else {
info!("Column {}.{} already exists, skipping", table.name, field.name);
}
}
}
}
result.tables_altered += 1;
}
Ok(result)
}
/// Get existing columns from a table
pub fn get_table_columns(
table_name: &str,
conn: &mut diesel::PgConnection,
) -> Result<Vec<DbColumn>, Box<dyn Error + Send + Sync>> {
let query = format!(
"SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_name = '{}' AND table_schema = 'public'
ORDER BY ordinal_position",
sanitize_identifier(table_name)
);
#[derive(QueryableByName)]
struct ColumnRow {
#[diesel(sql_type = Text)]
column_name: String,
#[diesel(sql_type = Text)]
data_type: String,
#[diesel(sql_type = Text)]
is_nullable: String,
}
let rows: Vec<ColumnRow> = match sql_query(&query).load(conn) {
Ok(r) => r,
Err(e) => {
// Table doesn't exist
return Err(format!("Table {} does not exist: {}", table_name, e).into());
}
};
Ok(rows.into_iter().map(|row| DbColumn {
name: row.column_name,
data_type: row.data_type,
is_nullable: row.is_nullable == "YES",
}).collect())
}
/// Process table definitions with schema sync for a specific bot
pub fn sync_bot_tables(
state: &Arc<AppState>,
bot_id: Uuid,
source: &str,
) -> Result<MigrationResult, Box<dyn Error + Send + Sync>> {
let tables = parse_table_definition(source)?;
let mut result = MigrationResult::default();
info!("Processing {} table definitions with schema sync for bot {}", tables.len(), bot_id);
// Get bot's database connection
let pool = state.bot_database_manager.get_bot_pool(bot_id)?;
let mut conn = pool.get()?;
for table in &tables {
if table.connection_name != "default" {
continue; // Skip external connections for now
}
info!("Syncing table: {}", table.name);
// Get existing columns
let existing_columns = match get_table_columns(&table.name, &mut conn) {
Ok(cols) => cols,
Err(_) => {
// Table doesn't exist yet
vec![]
}
};
// Generate CREATE TABLE SQL
let create_sql = super::table_definition::generate_create_table_sql(table, "postgres");
// Sync schema
match sync_table_schema(table, &existing_columns, &create_sql, &mut conn) {
Ok(sync_result) => {
result.tables_created += sync_result.tables_created;
result.tables_altered += sync_result.tables_altered;
result.columns_added += sync_result.columns_added;
result.errors.extend(sync_result.errors);
}
Err(e) => {
let error_msg = format!("Failed to sync table {}: {}", table.name, e);
error!("{}", error_msg);
result.errors.push(error_msg);
}
}
}
// Log summary
info!("Schema sync summary for bot {}: {} tables created, {} altered, {} columns added, {} errors",
bot_id, result.tables_created, result.tables_altered, result.columns_added, result.errors.len());
if !result.errors.is_empty() {
warn!("Schema sync completed with {} errors:", result.errors.len());
for error in &result.errors {
warn!(" - {}", error);
}
}
Ok(result)
}
/// Validate that all required columns exist
pub fn validate_table_schema(
table_name: &str,
required_fields: &[FieldDefinition],
conn: &mut diesel::PgConnection,
) -> Result<bool, Box<dyn Error + Send + Sync>> {
let existing_columns = get_table_columns(table_name, conn)?;
let existing_col_names: std::collections::HashSet<String> =
existing_columns.iter().map(|c| c.name.clone()).collect();
let mut missing = Vec::new();
for field in required_fields {
if !existing_col_names.contains(&field.name) {
missing.push(field.name.clone());
}
}
if !missing.is_empty() {
warn!("Table {} is missing columns: {:?}", table_name, missing);
return Ok(false);
}
Ok(true)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_db_column_creation() {
let col = DbColumn {
name: "test_col".to_string(),
data_type: "character varying".to_string(),
is_nullable: true,
};
assert_eq!(col.name, "test_col");
assert_eq!(col.is_nullable, true);
}
}

2
src/basic/keywords/use_website.rs
View file

@ -826,7 +826,7 @@ mod tests {
"docs_example_com_path"
);
assert_eq!(
sanitize_url_for_collection("http://test.site:8080"),
sanitize_url_for_collection("http://test.site:9000"),
"test_site_8080"
);
}

1879
src/basic/mod.rs.backup Normal file
View file

File diff suppressed because it is too large Load diff

4
src/core/bootstrap/bootstrap_manager.rs
View file

@ -106,9 +106,9 @@ impl BootstrapManager {
}
}
if pm.is_installed("postgres") {
if pm.is_installed("tables") {
info!("Starting PostgreSQL...");
match pm.start("postgres") {
match pm.start("tables") {
Ok(_child) => {
info!("PostgreSQL started");
}

19
src/core/bot/kb_context.rs
View file

@ -436,9 +436,22 @@ pub async fn inject_kb_context(
return Ok(());
}
// Sanitize context to remove UTF-16 surrogate characters that can't be encoded in UTF-8
let sanitized_context = context_string
.chars()
.filter(|c| {
let cp = *c as u32;
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
})
.collect::<String>();
if sanitized_context.is_empty() {
return Ok(());
}
info!(
"Injecting {} characters of KB/website context into prompt for session {}",
context_string.len(),
sanitized_context.len(),
session_id
);
@ -447,7 +460,7 @@ pub async fn inject_kb_context(
if let Some(idx) = system_msg_idx {
if let Some(content) = messages_array[idx]["content"].as_str() {
let new_content = format!("{}\n{}", content, context_string);
let new_content = format!("{}\n{}", content, sanitized_context);
messages_array[idx]["content"] = serde_json::Value::String(new_content);
}
} else {
@ -455,7 +468,7 @@ pub async fn inject_kb_context(
0,
serde_json::json!({
"role": "system",
"content": context_string
"content": sanitized_context
}),
);
}

75
src/core/bot/mod.rs
View file

@ -580,11 +580,20 @@ impl BotOrchestrator {
}
}
// Sanitize user message to remove any UTF-16 surrogate characters
let sanitized_message_content = message_content
.chars()
.filter(|c| {
let cp = *c as u32;
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
})
.collect::<String>();
// Add the current user message to the messages array
if let Some(msgs_array) = messages.as_array_mut() {
msgs_array.push(serde_json::json!({
"role": "user",
"content": message_content
"content": sanitized_message_content
}));
}
@ -644,6 +653,7 @@ impl BotOrchestrator {
let mut analysis_buffer = String::new();
let mut in_analysis = false;
let mut tool_call_buffer = String::new(); // Accumulate potential tool call JSON chunks
let mut accumulating_tool_call = false; // Track if we're currently accumulating a tool call
let handler = llm_models::get_handler(&model);
info!("[STREAM_START] Entering stream processing loop for model: {}", model);
@ -679,12 +689,62 @@ impl BotOrchestrator {
// ===== GENERIC TOOL EXECUTION =====
// Add chunk to tool_call_buffer and try to parse
// Tool calls arrive as JSON that can span multiple chunks
let looks_like_json = chunk.trim().starts_with('{') || chunk.trim().starts_with('[') ||
tool_call_buffer.contains('{') || tool_call_buffer.contains('[');
let chunk_in_tool_buffer = if looks_like_json {
// Check if this chunk contains JSON (either starts with {/[ or contains {/[)
let chunk_contains_json = chunk.trim().starts_with('{') || chunk.trim().starts_with('[') ||
chunk.contains('{') || chunk.contains('[');
let chunk_in_tool_buffer = if accumulating_tool_call {
// Already accumulating - add entire chunk to buffer
tool_call_buffer.push_str(&chunk);
true
} else if chunk_contains_json {
// Check if { appears in the middle of the chunk (mixed text + JSON)
let json_start = chunk.find('{').or_else(|| chunk.find('['));
if let Some(pos) = json_start {
if pos > 0 {
// Send the part before { as regular content
let regular_part = &chunk[..pos];
if !regular_part.trim().is_empty() {
info!("[STREAM_CONTENT] Sending regular part before JSON: '{}', len: {}", regular_part, regular_part.len());
full_response.push_str(regular_part);
let response = BotResponse {
bot_id: message.bot_id.clone(),
user_id: message.user_id.clone(),
session_id: message.session_id.clone(),
channel: message.channel.clone(),
content: regular_part.to_string(),
message_type: MessageType::BOT_RESPONSE,
stream_token: None,
is_complete: false,
suggestions: Vec::new(),
context_name: None,
context_length: 0,
context_max_length: 0,
};
if response_tx.send(response).await.is_err() {
warn!("Response channel closed");
break;
}
}
// Start accumulating from { onwards
accumulating_tool_call = true;
tool_call_buffer.push_str(&chunk[pos..]);
true
} else {
// Chunk starts with { or [
accumulating_tool_call = true;
tool_call_buffer.push_str(&chunk);
true
}
} else {
// Contains {/[ but find() failed - shouldn't happen, but send as regular content
false
}
} else {
false
};
@ -774,13 +834,15 @@ impl BotOrchestrator {
// Don't add tool_call JSON to full_response or analysis_buffer
// Clear the tool_call_buffer since we found and executed a tool call
tool_call_buffer.clear();
accumulating_tool_call = false; // Reset accumulation flag
// Continue to next chunk
continue;
}
// Clear tool_call_buffer if it's getting too large and no tool call was found
// This prevents memory issues from accumulating JSON fragments
if tool_call_buffer.len() > 10000 {
// Increased limit to 50000 to handle large tool calls with many parameters
if tool_call_buffer.len() > 50000 {
// Flush accumulated content to client since it's too large to be a tool call
info!("[TOOL_EXEC] Flushing tool_call_buffer (too large, assuming not a tool call)");
full_response.push_str(&tool_call_buffer);
@ -801,6 +863,7 @@ impl BotOrchestrator {
};
tool_call_buffer.clear();
accumulating_tool_call = false; // Reset accumulation flag after flush
if response_tx.send(response).await.is_err() {
warn!("Response channel closed");
@ -810,7 +873,7 @@ impl BotOrchestrator {
// If this chunk was added to tool_call_buffer and no tool call was found yet,
// skip processing (it's part of an incomplete tool call JSON)
if chunk_in_tool_buffer && tool_call_buffer.len() <= 10000 {
if chunk_in_tool_buffer {
continue;
}
// ===== END TOOL EXECUTION =====

3
src/core/bot/tool_executor.rs
View file

@ -2,6 +2,7 @@
/// Works across all LLM providers (GLM, OpenAI, Claude, etc.)
use log::{error, info, warn};
use serde_json::Value;
use std::collections::HashMap;
use std::fs::OpenOptions;
use std::io::Write;
use std::path::Path;
@ -264,6 +265,8 @@ impl ToolExecutor {
script_service.load_bot_config_params(state, bot_id);
// Set tool parameters as variables in the engine scope
// Note: DATE parameters are now sent by LLM in ISO 8601 format (YYYY-MM-DD)
// The tool schema with format="date" tells the LLM to use this agnostic format
if let Some(obj) = arguments.as_object() {
for (key, value) in obj {
let value_str = match value {

8
src/core/package_manager/facade.rs
View file

@ -553,7 +553,7 @@ Store credentials in Vault:
r"Email Server (Stalwart):
SMTP: {}:25
IMAP: {}:143
Web: http://{}:8080
Web: http://{}:9000
Store credentials in Vault:
botserver vault put gbo/email server={} port=25 username=admin password=<your-password>",
@ -563,11 +563,11 @@ Store credentials in Vault:
"directory" => {
format!(
r"Zitadel Identity Provider:
URL: http://{}:8080
Console: http://{}:8080/ui/console
URL: http://{}:9000
Console: http://{}:9000/ui/console
Store credentials in Vault:
botserver vault put gbo/directory url=http://{}:8080 client_id=<client-id> client_secret=<client-secret>",
botserver vault put gbo/directory url=http://{}:9000 client_id=<client-id> client_secret=<client-secret>",
ip, ip, ip
)
}

2
src/core/package_manager/installer.rs
View file

@ -602,7 +602,7 @@ impl PackageManager {
post_install_cmds_windows: vec![],
env_vars: HashMap::new(),
data_download_list: Vec::new(),
exec_cmd: "php -S 0.0.0.0:8080 -t {{DATA_PATH}}/roundcubemail".to_string(),
exec_cmd: "php -S 0.0.0.0:9000 -t {{DATA_PATH}}/roundcubemail".to_string(),
check_cmd:
"curl -f -k --connect-timeout 2 -m 5 https://localhost:8300 >/dev/null 2>&1"
.to_string(),

6
src/core/package_manager/setup/email_setup.rs
View file

@ -203,7 +203,7 @@ impl EmailSetup {
let issuer_url = dir_config["base_url"]
.as_str()
.unwrap_or("http://localhost:8080");
.unwrap_or("http://localhost:9000");
log::info!("Setting up OIDC authentication with Directory...");
log::info!("Issuer URL: {}", issuer_url);
@ -289,7 +289,7 @@ protocol = "imap"
tls.implicit = true
[server.listener."http"]
bind = ["0.0.0.0:8080"]
bind = ["0.0.0.0:9000"]
protocol = "http"
[storage]
@ -315,7 +315,7 @@ store = "sqlite"
r#"
[directory."oidc"]
type = "oidc"
issuer = "http://localhost:8080"
issuer = "http://localhost:9000"
client-id = "{{CLIENT_ID}}"
client-secret = "{{CLIENT_SECRET}}"

2
src/core/secrets/mod.rs
View file

@ -381,7 +381,7 @@ impl SecretsManager {
secrets.insert("token".into(), String::new());
}
SecretPaths::ALM => {
secrets.insert("url".into(), "http://localhost:8080".into());
secrets.insert("url".into(), "http://localhost:9000".into());
secrets.insert("username".into(), String::new());
secrets.insert("password".into(), String::new());
}

6
src/core/shared/test_utils.rs
View file

@ -249,13 +249,13 @@ impl Default for TestAppStateBuilder {
#[cfg(feature = "directory")]
pub fn create_mock_auth_service() -> AuthService {
let config = ZitadelConfig {
issuer_url: "http://localhost:8080".to_string(),
issuer: "http://localhost:8080".to_string(),
issuer_url: "http://localhost:9000".to_string(),
issuer: "http://localhost:9000".to_string(),
client_id: "mock_client_id".to_string(),
client_secret: "mock_client_secret".to_string(),
redirect_uri: "http://localhost:3000/callback".to_string(),
project_id: "mock_project_id".to_string(),
api_url: "http://localhost:8080".to_string(),
api_url: "http://localhost:9000".to_string(),
service_account_key: None,
};

93
src/core/shared/utils.rs
View file

@ -556,3 +556,96 @@ fn estimate_chars_per_token(model: &str) -> usize {
4 // Default conservative estimate
}
}
/// Convert date string from user locale format to ISO format (YYYY-MM-DD) for PostgreSQL.
///
/// The LLM automatically formats dates according to the user's language/idiom based on:
/// 1. The conversation context (user's language)
/// 2. The PARAM LIKE example (e.g., "15/12/2026" for DD/MM/YYYY)
///
/// This function handles the most common formats:
/// - ISO: YYYY-MM-DD (already in ISO, returned as-is)
/// - Brazilian/Portuguese: DD/MM/YYYY or DD/MM/YY
/// - US/English: MM/DD/YYYY or MM/DD/YY
///
/// If the value doesn't match any date pattern, returns it unchanged.
///
/// NOTE: This function does NOT try to guess ambiguous formats.
/// The LLM is responsible for formatting dates correctly based on user language.
/// The PARAM declaration's LIKE example tells the LLM the expected format.
///
/// # Arguments
/// * `value` - The date string to convert (as provided by the LLM)
///
/// # Returns
/// ISO formatted date string (YYYY-MM-DD) or original value if not a recognized date
pub fn convert_date_to_iso_format(value: &str) -> String {
let value = value.trim();
// Already in ISO format (YYYY-MM-DD) - return as-is
if value.len() == 10 && value.chars().nth(4) == Some('-') && value.chars().nth(7) == Some('-') {
let parts: Vec<&str> = value.split('-').collect();
if parts.len() == 3
&& parts[0].len() == 4
&& parts[1].len() == 2
&& parts[2].len() == 2
&& parts[0].chars().all(|c| c.is_ascii_digit())
&& parts[1].chars().all(|c| c.is_ascii_digit())
&& parts[2].chars().all(|c| c.is_ascii_digit())
{
if let (Ok(year), Ok(month), Ok(day)) =
(parts[0].parse::<u32>(), parts[1].parse::<u32>(), parts[2].parse::<u32>())
{
if month >= 1 && month <= 12 && day >= 1 && day <= 31 && year >= 1900 && year <= 2100 {
return value.to_string();
}
}
}
}
// Handle slash-separated formats: DD/MM/YYYY or MM/DD/YYYY
// We need to detect which format based on the PARAM declaration's LIKE example
// For now, default to DD/MM/YYYY (Brazilian format) as this is the most common for this bot
// TODO: Pass language/idiom from session to determine correct format
if value.len() >= 8 && value.len() <= 10 {
let parts: Vec<&str> = value.split('/').collect();
if parts.len() == 3 {
let all_numeric = parts[0].chars().all(|c| c.is_ascii_digit())
&& parts[1].chars().all(|c| c.is_ascii_digit())
&& parts[2].chars().all(|c| c.is_ascii_digit());
if all_numeric {
// Parse the three parts
let a = parts[0].parse::<u32>().ok();
let b = parts[1].parse::<u32>().ok();
let c = if parts[2].len() == 2 {
// Convert 2-digit year to 4-digit
parts[2].parse::<u32>().ok().map(|y| {
if y < 50 {
2000 + y
} else {
1900 + y
}
})
} else {
parts[2].parse::<u32>().ok()
};
if let (Some(first), Some(second), Some(third)) = (a, b, c) {
// Default: DD/MM/YYYY format (Brazilian/Portuguese)
// The LLM should format dates according to the user's language
// and the PARAM LIKE example (e.g., "15/12/2026" for DD/MM/YYYY)
let (year, month, day) = (third, second, first);
// Validate the determined date
if day >= 1 && day <= 31 && month >= 1 && month <= 12 && year >= 1900 && year <= 2100 {
return format!("{:04}-{:02}-{:02}", year, month, day);
}
}
}
}
}
// Not a recognized date pattern, return unchanged
value.to_string()
}

2
src/core/urls.rs
View file

@ -479,7 +479,7 @@ impl ApiUrls {
pub struct InternalUrls;
impl InternalUrls {
pub const DIRECTORY_BASE: &'static str = "http://localhost:8080";
pub const DIRECTORY_BASE: &'static str = "http://localhost:9000";
pub const DATABASE: &'static str = "postgres://localhost:5432";
pub const CACHE: &'static str = "redis://localhost:6379";
pub const DRIVE: &'static str = "https://localhost:9000";

4
src/email/messages.rs
View file

@ -65,8 +65,8 @@ fn is_tracking_pixel_enabled(state: &Arc<AppState>, bot_id: Option<Uuid>) -> boo
fn inject_tracking_pixel(html_body: &str, tracking_id: &str, state: &Arc<AppState>) -> String {
let config_manager = crate::core::config::ConfigManager::new(state.conn.clone());
let base_url = config_manager
.get_config(&Uuid::nil(), "server-url", Some("http://localhost:8080"))
.unwrap_or_else(|_| "http://localhost:8080".to_string());
.get_config(&Uuid::nil(), "server-url", Some("http://localhost:9000"))
.unwrap_or_else(|_| "http://localhost:9000".to_string());
let pixel_url = format!("{}/api/email/tracking/pixel/{}", base_url, tracking_id);
let pixel_html = format!(

4
src/email/tracking.rs
View file

@ -31,8 +31,8 @@ pub fn is_tracking_pixel_enabled(state: &Arc<AppState>, bot_id: Option<Uuid>) ->
pub fn inject_tracking_pixel(html_body: &str, tracking_id: &str, state: &Arc<AppState>) -> String {
let config_manager = crate::core::config::ConfigManager::new(state.conn.clone());
let base_url = config_manager
.get_config(&Uuid::nil(), "server-url", Some("http://localhost:8080"))
.unwrap_or_else(|_| "http://localhost:8080".to_string());
.get_config(&Uuid::nil(), "server-url", Some("http://localhost:9000"))
.unwrap_or_else(|_| "http://localhost:9000".to_string());
let pixel_url = format!("{}/api/email/tracking/pixel/{}", base_url, tracking_id);
let pixel_html = format!(

23
src/llm/claude.rs
View file

@ -232,6 +232,16 @@ impl ClaudeClient {
(system_prompt, claude_messages)
}
/// Sanitizes a string by removing invalid UTF-8 surrogate characters
fn sanitize_utf8(input: &str) -> String {
input.chars()
.filter(|c| {
let cp = *c as u32;
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
})
.collect()
}
pub fn build_messages(
system_prompt: &str,
context_data: &str,
@ -241,15 +251,15 @@ impl ClaudeClient {
let mut system_parts = Vec::new();
if !system_prompt.is_empty() {
system_parts.push(system_prompt.to_string());
system_parts.push(Self::sanitize_utf8(system_prompt));
}
if !context_data.is_empty() {
system_parts.push(context_data.to_string());
system_parts.push(Self::sanitize_utf8(context_data));
}
for (role, content) in history {
if role == "episodic" || role == "compact" {
system_parts.push(format!("[Previous conversation summary]: {content}"));
system_parts.push(format!("[Previous conversation summary]: {}", Self::sanitize_utf8(content)));
}
}
@ -270,7 +280,8 @@ impl ClaudeClient {
};
if let Some(norm_role) = normalized_role {
if content.is_empty() {
let sanitized_content = Self::sanitize_utf8(content);
if sanitized_content.is_empty() {
continue;
}
@ -278,14 +289,14 @@ impl ClaudeClient {
if let Some(last_msg) = messages.last_mut() {
let last_msg: &mut ClaudeMessage = last_msg;
last_msg.content.push_str("\n\n");
last_msg.content.push_str(content);
last_msg.content.push_str(&sanitized_content);
continue;
}
}
messages.push(ClaudeMessage {
role: norm_role.clone(),
content: content.clone(),
content: sanitized_content,
});
last_role = Some(norm_role);
}

41
src/llm/glm.rs
View file

@ -116,6 +116,16 @@ impl GLMClient {
// GLM/z.ai uses /chat/completions (not /v1/chat/completions)
format!("{}/chat/completions", self.base_url)
}
/// Sanitizes a string by removing invalid UTF-8 surrogate characters
fn sanitize_utf8(input: &str) -> String {
input.chars()
.filter(|c| {
let cp = *c as u32;
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
})
.collect()
}
}
#[async_trait]
@ -183,11 +193,6 @@ impl LLMProvider for GLMClient {
key: &str,
tools: Option<&Vec<Value>>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// DEBUG: Log what we received
info!("[GLM_DEBUG] config type: {}", config);
info!("[GLM_DEBUG] prompt: '{}'", prompt);
info!("[GLM_DEBUG] config as JSON: {}", serde_json::to_string_pretty(config).unwrap_or_default());
// config IS the messages array directly, not nested
let messages = if let Some(msgs) = config.as_array() {
// Convert messages from config format to GLM format
@ -195,25 +200,23 @@ impl LLMProvider for GLMClient {
.filter_map(|m| {
let role = m.get("role")?.as_str()?;
let content = m.get("content")?.as_str()?;
info!("[GLM_DEBUG] Processing message - role: {}, content: '{}'", role, content);
if !content.is_empty() {
let sanitized = Self::sanitize_utf8(content);
if !sanitized.is_empty() {
Some(GLMMessage {
role: role.to_string(),
content: Some(content.to_string()),
content: Some(sanitized),
tool_calls: None,
})
} else {
info!("[GLM_DEBUG] Skipping empty content message");
None
}
})
.collect::<Vec<_>>()
} else {
// Fallback to building from prompt
info!("[GLM_DEBUG] No array found, using prompt: '{}'", prompt);
vec![GLMMessage {
role: "user".to_string(),
content: Some(prompt.to_string()),
content: Some(Self::sanitize_utf8(prompt)),
tool_calls: None,
}]
};
@ -223,8 +226,6 @@ impl LLMProvider for GLMClient {
return Err("No valid messages in request".into());
}
info!("[GLM_DEBUG] Final GLM messages count: {}", messages.len());
// Use glm-4.7 for tool calling support
// GLM-4.7 supports standard OpenAI-compatible function calling
let model_name = if model == "glm-4" { "glm-4.7" } else { model };
@ -249,10 +250,6 @@ impl LLMProvider for GLMClient {
let url = self.build_url();
info!("GLM streaming request to: {}", url);
// Log the exact request being sent
let request_json = serde_json::to_string_pretty(&request).unwrap_or_default();
info!("GLM request body: {}", request_json);
let response = self
.client
.post(&url)
@ -292,18 +289,13 @@ impl LLMProvider for GLMClient {
if line.starts_with("data: ") {
let json_str = line[6..].trim();
info!("[GLM_SSE] Received SSE line ({} chars): {}", json_str.len(), json_str);
if let Ok(chunk_data) = serde_json::from_str::<Value>(json_str) {
if let Some(choices) = chunk_data.get("choices").and_then(|c| c.as_array()) {
for choice in choices {
info!("[GLM_SSE] Processing choice");
if let Some(delta) = choice.get("delta") {
info!("[GLM_SSE] Delta: {}", serde_json::to_string(delta).unwrap_or_default());
// Handle tool_calls (GLM-4.7 standard function calling)
if let Some(tool_calls) = delta.get("tool_calls").and_then(|t| t.as_array()) {
for tool_call in tool_calls {
info!("[GLM_SSE] Tool call detected: {}", serde_json::to_string(tool_call).unwrap_or_default());
// Send tool_calls as JSON for the calling code to process
let tool_call_json = serde_json::json!({
"type": "tool_call",
@ -323,7 +315,6 @@ impl LLMProvider for GLMClient {
// This makes GLM behave like OpenAI-compatible APIs
if let Some(content) = delta.get("content").and_then(|c| c.as_str()) {
if !content.is_empty() {
info!("[GLM_TX] Sending to channel: '{}'", content);
match tx.send(content.to_string()).await {
Ok(_) => {},
Err(e) => {
@ -331,11 +322,9 @@ impl LLMProvider for GLMClient {
}
}
}
} else {
info!("[GLM_SSE] No content field in delta");
}
} else {
info!("[GLM_SSE] No delta in choice");
// No delta in choice
}
if let Some(reason) = choice.get("finish_reason").and_then(|r| r.as_str()) {
if !reason.is_empty() {

21
src/llm/mod.rs
View file

@ -185,6 +185,17 @@ impl OpenAIClient {
}
}
/// Sanitizes a string by removing invalid UTF-8 surrogate characters
/// that cannot be encoded in valid UTF-8 (surrogates are only valid in UTF-16)
fn sanitize_utf8(input: &str) -> String {
input.chars()
.filter(|c| {
let cp = *c as u32;
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
})
.collect()
}
pub fn build_messages(
system_prompt: &str,
context_data: &str,
@ -194,19 +205,19 @@ impl OpenAIClient {
if !system_prompt.is_empty() {
messages.push(serde_json::json!({
"role": "system",
"content": system_prompt
"content": Self::sanitize_utf8(system_prompt)
}));
}
if !context_data.is_empty() {
messages.push(serde_json::json!({
"role": "system",
"content": context_data
"content": Self::sanitize_utf8(context_data)
}));
}
for (role, content) in history {
messages.push(serde_json::json!({
"role": role,
"content": content
"content": Self::sanitize_utf8(content)
}));
}
serde_json::Value::Array(messages)
@ -747,10 +758,10 @@ mod tests {
fn test_openai_client_new_custom_url() {
let client = OpenAIClient::new(
"test_key".to_string(),
Some("http://localhost:8080".to_string()),
Some("http://localhost:9000".to_string()),
None,
);
assert_eq!(client.base_url, "http://localhost:8080");
assert_eq!(client.base_url, "http://localhost:9000");
}
#[test]

8
src/security/cors.rs
View file

@ -101,13 +101,13 @@ impl CorsConfig {
Self {
allowed_origins: vec![
"http://localhost:3000".to_string(),
"http://localhost:8080".to_string(),
"http://localhost:9000".to_string(),
"http://localhost:8300".to_string(),
"http://127.0.0.1:3000".to_string(),
"http://127.0.0.1:8080".to_string(),
"http://127.0.0.1:9000".to_string(),
"http://127.0.0.1:8300".to_string(),
"https://localhost:3000".to_string(),
"https://localhost:8080".to_string(),
"https://localhost:9000".to_string(),
"https://localhost:8300".to_string(),
],
allowed_methods: vec![
@ -576,7 +576,7 @@ mod tests {
assert!(is_localhost_origin("http://localhost:3000"));
assert!(is_localhost_origin("https://localhost:8443"));
assert!(is_localhost_origin("http://127.0.0.1"));
assert!(is_localhost_origin("http://127.0.0.1:8080"));
assert!(is_localhost_origin("http://127.0.0.1:9000"));
assert!(!is_localhost_origin("http://example.com"));
}