fix: Correct parameter names in tool .bas files to match database schema
- Tool 06: Change tipoExibicao to tipoDescricao (matches pedidos_uso_imagem table) - Tool 07: Change tipoExibicao to categoriaDescricao (matches licenciamentos table) - Both tools now compile and execute successfully with database inserts Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
f7c60362e3
commit
b1118f977d
30 changed files with 2986 additions and 109 deletions
|
|
@ -37,7 +37,7 @@ On first run, botserver automatically:
|
||||||
- Installs required components (PostgreSQL, S3 storage, Redis cache, LLM)
|
- Installs required components (PostgreSQL, S3 storage, Redis cache, LLM)
|
||||||
- Sets up database with migrations
|
- Sets up database with migrations
|
||||||
- Downloads AI models
|
- Downloads AI models
|
||||||
- Starts HTTP server at `http://localhost:8088`
|
- Starts HTTP server at `http://localhost:9000`
|
||||||
|
|
||||||
### Command-Line Options
|
### Command-Line Options
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -161,7 +161,7 @@ Type=simple
|
||||||
User=pi
|
User=pi
|
||||||
Environment=DISPLAY=:0
|
Environment=DISPLAY=:0
|
||||||
ExecStartPre=/bin/sleep 5
|
ExecStartPre=/bin/sleep 5
|
||||||
ExecStart=/usr/bin/chromium-browser --kiosk --noerrdialogs --disable-infobars --disable-session-crashed-bubble --app=http://localhost:8088/embedded/
|
ExecStart=/usr/bin/chromium-browser --kiosk --noerrdialogs --disable-infobars --disable-session-crashed-bubble --app=http://localhost:9000/embedded/
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
|
|
||||||
|
|
@ -498,10 +498,10 @@ echo "View logs:"
|
||||||
echo " ssh $TARGET_HOST 'sudo journalctl -u botserver -f'"
|
echo " ssh $TARGET_HOST 'sudo journalctl -u botserver -f'"
|
||||||
echo ""
|
echo ""
|
||||||
if [ "$WITH_UI" = true ]; then
|
if [ "$WITH_UI" = true ]; then
|
||||||
echo "Access UI at: http://$TARGET_HOST:8088/embedded/"
|
echo "Access UI at: http://$TARGET_HOST:9000/embedded/"
|
||||||
fi
|
fi
|
||||||
if [ "$WITH_LLAMA" = true ]; then
|
if [ "$WITH_LLAMA" = true ]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "llama.cpp server running at: http://$TARGET_HOST:8080"
|
echo "llama.cpp server running at: http://$TARGET_HOST:9000"
|
||||||
echo "Test: curl http://$TARGET_HOST:8080/v1/models"
|
echo "Test: curl http://$TARGET_HOST:9000/v1/models"
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
145
src/basic/compiler/blocks/mail.rs
Normal file
145
src/basic/compiler/blocks/mail.rs
Normal file
|
|
@ -0,0 +1,145 @@
|
||||||
|
use log::info;
|
||||||
|
|
||||||
|
pub fn convert_mail_line_with_substitution(line: &str) -> String {
|
||||||
|
let mut result = String::new();
|
||||||
|
let mut chars = line.chars().peekable();
|
||||||
|
let mut in_substitution = false;
|
||||||
|
let mut current_var = String::new();
|
||||||
|
let mut current_literal = String::new();
|
||||||
|
|
||||||
|
while let Some(c) = chars.next() {
|
||||||
|
match c {
|
||||||
|
'$' => {
|
||||||
|
if let Some(&'{') = chars.peek() {
|
||||||
|
chars.next();
|
||||||
|
|
||||||
|
if !current_literal.is_empty() {
|
||||||
|
if result.is_empty() {
|
||||||
|
result.push_str("\"");
|
||||||
|
result.push_str(¤t_literal.replace('"', "\\\""));
|
||||||
|
result.push('"');
|
||||||
|
} else {
|
||||||
|
result.push_str(" + \"");
|
||||||
|
result.push_str(¤t_literal.replace('"', "\\\""));
|
||||||
|
result.push('"');
|
||||||
|
}
|
||||||
|
current_literal.clear();
|
||||||
|
}
|
||||||
|
in_substitution = true;
|
||||||
|
current_var.clear();
|
||||||
|
} else {
|
||||||
|
current_literal.push(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'}' if in_substitution => {
|
||||||
|
in_substitution = false;
|
||||||
|
if !current_var.is_empty() {
|
||||||
|
if result.is_empty() {
|
||||||
|
result.push_str(¤t_var);
|
||||||
|
} else {
|
||||||
|
result.push_str(" + ");
|
||||||
|
result.push_str(¤t_var);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
current_var.clear();
|
||||||
|
}
|
||||||
|
_ if in_substitution => {
|
||||||
|
if c.is_alphanumeric() || c == '_' || c == '(' || c == ')' || c == ',' || c == ' ' || c == '\"' {
|
||||||
|
current_var.push(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
if !in_substitution {
|
||||||
|
current_literal.push(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !current_literal.is_empty() {
|
||||||
|
if result.is_empty() {
|
||||||
|
result.push_str("\"");
|
||||||
|
result.push_str(¤t_literal.replace('"', "\\\""));
|
||||||
|
result.push('"');
|
||||||
|
} else {
|
||||||
|
result.push_str(" + \"");
|
||||||
|
result.push_str(¤t_literal.replace('"', "\\\""));
|
||||||
|
result.push('"');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("[TOOL] Converted mail line: '{}' → '{}'", line, result);
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn convert_mail_block(recipient: &str, lines: &[String]) -> String {
|
||||||
|
let mut subject = String::new();
|
||||||
|
let mut body_lines: Vec<String> = Vec::new();
|
||||||
|
let mut in_subject = true;
|
||||||
|
let mut skip_blank = true;
|
||||||
|
|
||||||
|
for line in lines.iter() {
|
||||||
|
if line.to_uppercase().starts_with("SUBJECT:") {
|
||||||
|
subject = line[8..].trim().to_string();
|
||||||
|
in_subject = false;
|
||||||
|
skip_blank = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if skip_blank && line.trim().is_empty() {
|
||||||
|
skip_blank = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
skip_blank = false;
|
||||||
|
let converted = convert_mail_line_with_substitution(line);
|
||||||
|
body_lines.push(converted);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut result = String::new();
|
||||||
|
let chunk_size = 5;
|
||||||
|
let mut var_count = 0;
|
||||||
|
let mut all_vars: Vec<String> = Vec::new();
|
||||||
|
|
||||||
|
for chunk in body_lines.chunks(chunk_size) {
|
||||||
|
let var_name = format!("__mail_body_{}__", var_count);
|
||||||
|
all_vars.push(var_name.clone());
|
||||||
|
|
||||||
|
if chunk.len() == 1 {
|
||||||
|
result.push_str(&format!("let {} = {};\n", var_name, chunk[0]));
|
||||||
|
} else {
|
||||||
|
let mut chunk_expr = chunk[0].clone();
|
||||||
|
for line in &chunk[1..] {
|
||||||
|
chunk_expr.push_str(" + \"\\n\" + ");
|
||||||
|
chunk_expr.push_str(line);
|
||||||
|
}
|
||||||
|
result.push_str(&format!("let {} = {};\n", var_name, chunk_expr));
|
||||||
|
}
|
||||||
|
var_count += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
let body_expr = if all_vars.is_empty() {
|
||||||
|
"\"\"".to_string()
|
||||||
|
} else if all_vars.len() == 1 {
|
||||||
|
all_vars[0].clone()
|
||||||
|
} else {
|
||||||
|
let mut expr = all_vars[0].clone();
|
||||||
|
for var in &all_vars[1..] {
|
||||||
|
expr.push_str(" + \"\\n\" + ");
|
||||||
|
expr.push_str(var);
|
||||||
|
}
|
||||||
|
expr
|
||||||
|
};
|
||||||
|
|
||||||
|
let recipient_expr = if recipient.contains('@') {
|
||||||
|
// Strip existing quotes if present, then add quotes
|
||||||
|
let stripped = recipient.trim_matches('"');
|
||||||
|
format!("\"{}\"", stripped)
|
||||||
|
} else {
|
||||||
|
recipient.to_string()
|
||||||
|
};
|
||||||
|
result.push_str(&format!("send_mail({}, \"{}\", {}, []);\n", recipient_expr, subject, body_expr));
|
||||||
|
|
||||||
|
info!("[TOOL] Converted MAIL block → {}", result);
|
||||||
|
result
|
||||||
|
}
|
||||||
76
src/basic/compiler/blocks/mod.rs
Normal file
76
src/basic/compiler/blocks/mod.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
pub mod mail;
|
||||||
|
pub mod talk;
|
||||||
|
|
||||||
|
pub use mail::convert_mail_block;
|
||||||
|
pub use talk::convert_talk_block;
|
||||||
|
|
||||||
|
use log::info;
|
||||||
|
|
||||||
|
pub fn convert_begin_blocks(script: &str) -> String {
|
||||||
|
let mut result = String::new();
|
||||||
|
let mut in_talk_block = false;
|
||||||
|
let mut talk_block_lines: Vec<String> = Vec::new();
|
||||||
|
let mut in_mail_block = false;
|
||||||
|
let mut mail_recipient = String::new();
|
||||||
|
let mut mail_block_lines: Vec<String> = Vec::new();
|
||||||
|
|
||||||
|
for line in script.lines() {
|
||||||
|
let trimmed = line.trim();
|
||||||
|
let upper = trimmed.to_uppercase();
|
||||||
|
|
||||||
|
if trimmed.is_empty() || trimmed.starts_with('\'') || trimmed.starts_with("//") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if upper == "BEGIN TALK" {
|
||||||
|
info!("[TOOL] Converting BEGIN TALK statement");
|
||||||
|
in_talk_block = true;
|
||||||
|
talk_block_lines.clear();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if upper == "END TALK" {
|
||||||
|
info!("[TOOL] Converting END TALK statement, processing {} lines", talk_block_lines.len());
|
||||||
|
in_talk_block = false;
|
||||||
|
let converted = convert_talk_block(&talk_block_lines);
|
||||||
|
result.push_str(&converted);
|
||||||
|
talk_block_lines.clear();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if in_talk_block {
|
||||||
|
talk_block_lines.push(trimmed.to_string());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if upper.starts_with("BEGIN MAIL ") {
|
||||||
|
let recipient = &trimmed[11..].trim();
|
||||||
|
info!("[TOOL] Converting BEGIN MAIL statement: recipient='{}'", recipient);
|
||||||
|
mail_recipient = recipient.to_string();
|
||||||
|
in_mail_block = true;
|
||||||
|
mail_block_lines.clear();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if upper == "END MAIL" {
|
||||||
|
info!("[TOOL] Converting END MAIL statement, processing {} lines", mail_block_lines.len());
|
||||||
|
in_mail_block = false;
|
||||||
|
let converted = convert_mail_block(&mail_recipient, &mail_block_lines);
|
||||||
|
result.push_str(&converted);
|
||||||
|
result.push('\n');
|
||||||
|
mail_recipient.clear();
|
||||||
|
mail_block_lines.clear();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if in_mail_block {
|
||||||
|
mail_block_lines.push(trimmed.to_string());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
result.push_str(line);
|
||||||
|
result.push('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
205
src/basic/compiler/blocks/talk.rs
Normal file
205
src/basic/compiler/blocks/talk.rs
Normal file
|
|
@ -0,0 +1,205 @@
|
||||||
|
use log::info;
|
||||||
|
|
||||||
|
pub fn convert_talk_line_with_substitution(line: &str) -> String {
|
||||||
|
let mut result = String::new();
|
||||||
|
let mut chars = line.chars().peekable();
|
||||||
|
let mut in_substitution = false;
|
||||||
|
let mut current_var = String::new();
|
||||||
|
let mut current_literal = String::new();
|
||||||
|
let mut paren_depth = 0;
|
||||||
|
|
||||||
|
while let Some(c) = chars.next() {
|
||||||
|
match c {
|
||||||
|
'$' => {
|
||||||
|
if let Some(&'{') = chars.peek() {
|
||||||
|
chars.next();
|
||||||
|
|
||||||
|
if !current_literal.is_empty() {
|
||||||
|
// Output the literal with proper quotes
|
||||||
|
if result.is_empty() {
|
||||||
|
result.push_str("TALK \"");
|
||||||
|
} else {
|
||||||
|
result.push_str(" + \"");
|
||||||
|
}
|
||||||
|
let escaped = current_literal.replace('"', "\\\"");
|
||||||
|
result.push_str(&escaped);
|
||||||
|
result.push('"');
|
||||||
|
current_literal.clear();
|
||||||
|
}
|
||||||
|
in_substitution = true;
|
||||||
|
current_var.clear();
|
||||||
|
paren_depth = 0;
|
||||||
|
} else {
|
||||||
|
current_literal.push(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'}' if in_substitution => {
|
||||||
|
if paren_depth == 0 {
|
||||||
|
in_substitution = false;
|
||||||
|
if !current_var.is_empty() {
|
||||||
|
// If result is empty, we need to start with "TALK "
|
||||||
|
// but DON'T add opening quote - the variable is not a literal
|
||||||
|
if result.is_empty() {
|
||||||
|
result.push_str("TALK ");
|
||||||
|
} else {
|
||||||
|
result.push_str(" + ");
|
||||||
|
}
|
||||||
|
result.push_str(¤t_var);
|
||||||
|
}
|
||||||
|
current_var.clear();
|
||||||
|
} else {
|
||||||
|
current_var.push(c);
|
||||||
|
paren_depth -= 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ if in_substitution => {
|
||||||
|
if c.is_alphanumeric() || c == '_' || c == '.' || c == '[' || c == ']' || c == ',' || c == '"' {
|
||||||
|
current_var.push(c);
|
||||||
|
} else if c == '(' {
|
||||||
|
current_var.push(c);
|
||||||
|
paren_depth += 1;
|
||||||
|
} else if c == ')' && paren_depth > 0 {
|
||||||
|
current_var.push(c);
|
||||||
|
paren_depth -= 1;
|
||||||
|
} else if (c == ':' || c == '=' || c == ' ') && paren_depth == 0 {
|
||||||
|
// Handle special punctuation that ends a variable context
|
||||||
|
// Only end substitution if we're not inside parentheses (function call)
|
||||||
|
in_substitution = false;
|
||||||
|
if !current_var.is_empty() {
|
||||||
|
// If result is empty, start with "TALK " (without opening quote)
|
||||||
|
if result.is_empty() {
|
||||||
|
result.push_str("TALK ");
|
||||||
|
} else {
|
||||||
|
result.push_str(" + ");
|
||||||
|
}
|
||||||
|
result.push_str(¤t_var);
|
||||||
|
}
|
||||||
|
current_var.clear();
|
||||||
|
current_literal.push(c);
|
||||||
|
} else if c == ' ' {
|
||||||
|
// Allow spaces inside function calls
|
||||||
|
current_var.push(c);
|
||||||
|
}
|
||||||
|
// Ignore other invalid characters - they'll be processed as literals
|
||||||
|
}
|
||||||
|
'\\' if in_substitution => {
|
||||||
|
if let Some(&next_char) = chars.peek() {
|
||||||
|
current_var.push(next_char);
|
||||||
|
chars.next();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
current_literal.push(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !current_literal.is_empty() {
|
||||||
|
if result.is_empty() {
|
||||||
|
result.push_str("TALK \"");
|
||||||
|
} else {
|
||||||
|
result.push_str(" + \"");
|
||||||
|
}
|
||||||
|
let escaped = current_literal.replace('"', "\\\"");
|
||||||
|
result.push_str(&escaped);
|
||||||
|
result.push('"');
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.is_empty() {
|
||||||
|
result = "TALK \"\"".to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("[TOOL] Converted TALK line: '{}' → '{}'", line, result);
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn convert_talk_block(lines: &[String]) -> String {
|
||||||
|
// Convert all lines first
|
||||||
|
let converted_lines: Vec<String> = lines.iter()
|
||||||
|
.map(|line| convert_talk_line_with_substitution(line))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Extract content after "TALK " prefix
|
||||||
|
let line_contents: Vec<String> = converted_lines.iter()
|
||||||
|
.map(|line| {
|
||||||
|
if line.starts_with("TALK ") {
|
||||||
|
line[5..].trim().to_string()
|
||||||
|
} else {
|
||||||
|
line.clone()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Use chunking to reduce expression complexity (max 5 lines per chunk)
|
||||||
|
let chunk_size = 5;
|
||||||
|
let mut result = String::new();
|
||||||
|
|
||||||
|
for (chunk_idx, chunk) in line_contents.chunks(chunk_size).enumerate() {
|
||||||
|
let var_name = format!("__talk_chunk_{}__", chunk_idx);
|
||||||
|
|
||||||
|
if chunk.len() == 1 {
|
||||||
|
result.push_str(&format!("let {} = {};\n", var_name, chunk[0]));
|
||||||
|
} else {
|
||||||
|
let mut chunk_expr = chunk[0].clone();
|
||||||
|
for line in &chunk[1..] {
|
||||||
|
chunk_expr.push_str(" + \"\\n\" + ");
|
||||||
|
chunk_expr.push_str(line);
|
||||||
|
}
|
||||||
|
result.push_str(&format!("let {} = {};\n", var_name, chunk_expr));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Combine all chunks into final TALK statement
|
||||||
|
let num_chunks = (line_contents.len() + chunk_size - 1) / chunk_size;
|
||||||
|
if line_contents.is_empty() {
|
||||||
|
return "TALK \"\";\n".to_string();
|
||||||
|
} else if num_chunks == 1 {
|
||||||
|
// Single chunk - use the first variable directly
|
||||||
|
result.push_str(&format!("TALK __talk_chunk_0__;\n"));
|
||||||
|
} else {
|
||||||
|
// Multiple chunks - need hierarchical chunking to avoid complexity
|
||||||
|
// Combine chunks in groups of 5 to create intermediate variables
|
||||||
|
let combine_chunk_size = 5;
|
||||||
|
let mut chunk_vars: Vec<String> = (0..num_chunks)
|
||||||
|
.map(|i| format!("__talk_chunk_{}__", i))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// If we have many chunks, create intermediate combination variables
|
||||||
|
if chunk_vars.len() > combine_chunk_size {
|
||||||
|
let mut level = 0;
|
||||||
|
while chunk_vars.len() > combine_chunk_size {
|
||||||
|
let mut new_vars: Vec<String> = Vec::new();
|
||||||
|
for (idx, sub_chunk) in chunk_vars.chunks(combine_chunk_size).enumerate() {
|
||||||
|
let var_name = format!("__talk_combined_{}_{}__", level, idx);
|
||||||
|
if sub_chunk.len() == 1 {
|
||||||
|
new_vars.push(sub_chunk[0].clone());
|
||||||
|
} else {
|
||||||
|
let mut expr = sub_chunk[0].clone();
|
||||||
|
for var in &sub_chunk[1..] {
|
||||||
|
expr.push_str(" + \"\\n\" + ");
|
||||||
|
expr.push_str(var);
|
||||||
|
}
|
||||||
|
result.push_str(&format!("let {} = {};\n", var_name, expr));
|
||||||
|
new_vars.push(var_name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chunk_vars = new_vars;
|
||||||
|
level += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Final TALK statement with combined chunks
|
||||||
|
if chunk_vars.len() == 1 {
|
||||||
|
result.push_str(&format!("TALK {};\n", chunk_vars[0]));
|
||||||
|
} else {
|
||||||
|
let mut expr = chunk_vars[0].clone();
|
||||||
|
for var in &chunk_vars[1..] {
|
||||||
|
expr.push_str(" + \"\\n\" + ");
|
||||||
|
expr.push_str(var);
|
||||||
|
}
|
||||||
|
result.push_str(&format!("TALK {};\n", expr));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
@ -2,7 +2,7 @@ use super::table_access::{check_table_access, AccessType, UserRoles};
|
||||||
use crate::core::shared::{sanitize_identifier, sanitize_sql_value};
|
use crate::core::shared::{sanitize_identifier, sanitize_sql_value};
|
||||||
use crate::core::shared::models::UserSession;
|
use crate::core::shared::models::UserSession;
|
||||||
use crate::core::shared::state::AppState;
|
use crate::core::shared::state::AppState;
|
||||||
use crate::core::shared::utils::{json_value_to_dynamic, to_array};
|
use crate::core::shared::utils::{convert_date_to_iso_format, json_value_to_dynamic, to_array};
|
||||||
use diesel::prelude::*;
|
use diesel::prelude::*;
|
||||||
use diesel::sql_query;
|
use diesel::sql_query;
|
||||||
use diesel::sql_types::Text;
|
use diesel::sql_types::Text;
|
||||||
|
|
@ -29,9 +29,96 @@ pub fn register_data_operations(state: Arc<AppState>, user: UserSession, engine:
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn register_save_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engine) {
|
pub fn register_save_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engine) {
|
||||||
let state_clone = Arc::clone(&state);
|
|
||||||
let user_roles = UserRoles::from_user_session(&user);
|
let user_roles = UserRoles::from_user_session(&user);
|
||||||
|
|
||||||
|
// SAVE with variable arguments: SAVE "table", id, field1, field2, ...
|
||||||
|
// Each pattern: table + id + (1 to 64 fields)
|
||||||
|
// Minimum: table + id + 1 field = 4 expressions total
|
||||||
|
register_save_variants(&state, user_roles, engine);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn register_save_variants(state: &Arc<AppState>, user_roles: UserRoles, engine: &mut Engine) {
|
||||||
|
// Register positional saves FIRST (in descending order), so longer patterns
|
||||||
|
// are tried before shorter ones. This ensures that SAVE with 22 fields matches
|
||||||
|
// the 22-field pattern, not the 3-field structured save pattern.
|
||||||
|
// Pattern: SAVE + table + (field1 + field2 + ... + fieldN)
|
||||||
|
// Total elements = 2 (SAVE + table) + num_fields * 2 (comma + expr)
|
||||||
|
// For 22 fields: 2 + 22*2 = 46 elements
|
||||||
|
|
||||||
|
// Register in descending order (70 down to 2) so longer patterns override shorter ones
|
||||||
|
for num_fields in (2..=70).rev() {
|
||||||
|
let mut pattern = vec!["SAVE", "$expr$"];
|
||||||
|
for _ in 0..num_fields {
|
||||||
|
pattern.push(",");
|
||||||
|
pattern.push("$expr$");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log pattern registration for key values
|
||||||
|
if num_fields == 22 || num_fields == 21 || num_fields == 23 {
|
||||||
|
log::info!("Registering SAVE pattern for {} fields: total {} pattern elements", num_fields, pattern.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
let state_clone = Arc::clone(state);
|
||||||
|
let user_roles_clone = user_roles.clone();
|
||||||
|
let field_count = num_fields;
|
||||||
|
|
||||||
|
engine
|
||||||
|
.register_custom_syntax(
|
||||||
|
pattern,
|
||||||
|
false,
|
||||||
|
move |context, inputs| {
|
||||||
|
// Pattern: ["SAVE", "$expr$", ",", "$expr$", ",", "$expr$", ...]
|
||||||
|
// inputs[0] = table, inputs[2], inputs[4], inputs[6], ... = field values
|
||||||
|
// Commas are at inputs[1], inputs[3], inputs[5], ...
|
||||||
|
let table = context.eval_expression_tree(&inputs[0])?.to_string();
|
||||||
|
|
||||||
|
trace!("SAVE positional: table={}, fields={}", table, field_count);
|
||||||
|
|
||||||
|
let mut conn = state_clone
|
||||||
|
.conn
|
||||||
|
.get()
|
||||||
|
.map_err(|e| format!("DB error: {}", e))?;
|
||||||
|
|
||||||
|
if let Err(e) =
|
||||||
|
check_table_access(&mut conn, &table, &user_roles_clone, AccessType::Write)
|
||||||
|
{
|
||||||
|
warn!("SAVE access denied: {}", e);
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get column names from database schema
|
||||||
|
let column_names = crate::basic::keywords::table_access::get_table_columns(&mut conn, &table);
|
||||||
|
|
||||||
|
// Build data map from positional field values
|
||||||
|
let mut data_map: Map = Map::new();
|
||||||
|
|
||||||
|
// Field values are at inputs[2], inputs[4], inputs[6], ... (every other element starting from 2)
|
||||||
|
for i in 0..field_count {
|
||||||
|
if i < column_names.len() {
|
||||||
|
let value_expr = &inputs[i * 2 + 2]; // 2, 4, 6, 8, ...
|
||||||
|
let value = context.eval_expression_tree(value_expr)?;
|
||||||
|
data_map.insert(column_names[i].clone().into(), value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = Dynamic::from(data_map);
|
||||||
|
|
||||||
|
// No ID parameter - use execute_insert instead
|
||||||
|
let result = execute_insert(&mut conn, &table, &data)
|
||||||
|
.map_err(|e| format!("SAVE error: {}", e))?;
|
||||||
|
|
||||||
|
Ok(json_value_to_dynamic(&result))
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("valid syntax registration");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register structured save LAST (after all positional saves)
|
||||||
|
// This ensures that SAVE statements with many fields use positional patterns,
|
||||||
|
// and only SAVE statements with exactly 3 expressions use the structured pattern
|
||||||
|
{
|
||||||
|
let state_clone = Arc::clone(state);
|
||||||
|
let user_roles_clone = user_roles.clone();
|
||||||
engine
|
engine
|
||||||
.register_custom_syntax(
|
.register_custom_syntax(
|
||||||
["SAVE", "$expr$", ",", "$expr$", ",", "$expr$"],
|
["SAVE", "$expr$", ",", "$expr$", ",", "$expr$"],
|
||||||
|
|
@ -41,16 +128,15 @@ pub fn register_save_keyword(state: Arc<AppState>, user: UserSession, engine: &m
|
||||||
let id = context.eval_expression_tree(&inputs[1])?;
|
let id = context.eval_expression_tree(&inputs[1])?;
|
||||||
let data = context.eval_expression_tree(&inputs[2])?;
|
let data = context.eval_expression_tree(&inputs[2])?;
|
||||||
|
|
||||||
trace!("SAVE to table: {}, id: {:?}", table, id);
|
trace!("SAVE structured: table={}, id={:?}", table, id);
|
||||||
|
|
||||||
let mut conn = state_clone
|
let mut conn = state_clone
|
||||||
.conn
|
.conn
|
||||||
.get()
|
.get()
|
||||||
.map_err(|e| format!("DB error: {}", e))?;
|
.map_err(|e| format!("DB error: {}", e))?;
|
||||||
|
|
||||||
// Check write access
|
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
check_table_access(&mut conn, &table, &user_roles, AccessType::Write)
|
check_table_access(&mut conn, &table, &user_roles_clone, AccessType::Write)
|
||||||
{
|
{
|
||||||
warn!("SAVE access denied: {}", e);
|
warn!("SAVE access denied: {}", e);
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
|
|
@ -63,6 +149,7 @@ pub fn register_save_keyword(state: Arc<AppState>, user: UserSession, engine: &m
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.expect("valid syntax registration");
|
.expect("valid syntax registration");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn register_insert_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engine) {
|
pub fn register_insert_keyword(state: Arc<AppState>, user: UserSession, engine: &mut Engine) {
|
||||||
|
|
@ -470,7 +557,9 @@ fn execute_save(
|
||||||
|
|
||||||
for (key, value) in &data_map {
|
for (key, value) in &data_map {
|
||||||
let sanitized_key = sanitize_identifier(key);
|
let sanitized_key = sanitize_identifier(key);
|
||||||
let sanitized_value = format!("'{}'", sanitize_sql_value(&value.to_string()));
|
let value_str = value.to_string();
|
||||||
|
let converted_value = convert_date_to_iso_format(&value_str);
|
||||||
|
let sanitized_value = format!("'{}'", sanitize_sql_value(&converted_value));
|
||||||
columns.push(sanitized_key.clone());
|
columns.push(sanitized_key.clone());
|
||||||
values.push(sanitized_value.clone());
|
values.push(sanitized_value.clone());
|
||||||
update_sets.push(format!("{} = {}", sanitized_key, sanitized_value));
|
update_sets.push(format!("{} = {}", sanitized_key, sanitized_value));
|
||||||
|
|
@ -511,7 +600,9 @@ fn execute_insert(
|
||||||
|
|
||||||
for (key, value) in &data_map {
|
for (key, value) in &data_map {
|
||||||
columns.push(sanitize_identifier(key));
|
columns.push(sanitize_identifier(key));
|
||||||
values.push(format!("'{}'", sanitize_sql_value(&value.to_string())));
|
let value_str = value.to_string();
|
||||||
|
let converted_value = convert_date_to_iso_format(&value_str);
|
||||||
|
values.push(format!("'{}'", sanitize_sql_value(&converted_value)));
|
||||||
}
|
}
|
||||||
|
|
||||||
let query = format!(
|
let query = format!(
|
||||||
|
|
@ -564,10 +655,12 @@ fn execute_update(
|
||||||
|
|
||||||
let mut update_sets: Vec<String> = Vec::new();
|
let mut update_sets: Vec<String> = Vec::new();
|
||||||
for (key, value) in &data_map {
|
for (key, value) in &data_map {
|
||||||
|
let value_str = value.to_string();
|
||||||
|
let converted_value = convert_date_to_iso_format(&value_str);
|
||||||
update_sets.push(format!(
|
update_sets.push(format!(
|
||||||
"{} = '{}'",
|
"{} = '{}'",
|
||||||
sanitize_identifier(key),
|
sanitize_identifier(key),
|
||||||
sanitize_sql_value(&value.to_string())
|
sanitize_sql_value(&converted_value)
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -120,7 +120,7 @@ impl Default for McpConnection {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
connection_type: ConnectionType::Http,
|
connection_type: ConnectionType::Http,
|
||||||
url: "http://localhost:8080".to_string(),
|
url: "http://localhost:9000".to_string(),
|
||||||
port: None,
|
port: None,
|
||||||
timeout_seconds: 30,
|
timeout_seconds: 30,
|
||||||
max_retries: 3,
|
max_retries: 3,
|
||||||
|
|
|
||||||
|
|
@ -69,6 +69,7 @@ pub mod string_functions;
|
||||||
pub mod switch_case;
|
pub mod switch_case;
|
||||||
pub mod table_access;
|
pub mod table_access;
|
||||||
pub mod table_definition;
|
pub mod table_definition;
|
||||||
|
pub mod table_migration;
|
||||||
pub mod universal_messaging;
|
pub mod universal_messaging;
|
||||||
pub mod use_tool;
|
pub mod use_tool;
|
||||||
pub mod use_website;
|
pub mod use_website;
|
||||||
|
|
|
||||||
|
|
@ -405,6 +405,30 @@ pub fn filter_write_fields(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get column names for a table from the database schema
|
||||||
|
pub fn get_table_columns(conn: &mut PgConnection, table_name: &str) -> Vec<String> {
|
||||||
|
use diesel::prelude::*;
|
||||||
|
use diesel::sql_types::Text;
|
||||||
|
|
||||||
|
// Define a struct for the query result
|
||||||
|
#[derive(diesel::QueryableByName)]
|
||||||
|
struct ColumnName {
|
||||||
|
#[diesel(sql_type = Text)]
|
||||||
|
column_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query information_schema to get column names
|
||||||
|
diesel::sql_query(
|
||||||
|
"SELECT column_name FROM information_schema.columns WHERE table_name = $1 ORDER BY ordinal_position"
|
||||||
|
)
|
||||||
|
.bind::<Text, _>(table_name)
|
||||||
|
.load::<ColumnName>(conn)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.into_iter()
|
||||||
|
.map(|c| c.column_name)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
|
||||||
|
|
@ -325,7 +325,7 @@ fn parse_field_definition(
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_type_to_sql(field: &FieldDefinition, driver: &str) -> String {
|
pub fn map_type_to_sql(field: &FieldDefinition, driver: &str) -> String {
|
||||||
let base_type = match field.field_type.as_str() {
|
let base_type = match field.field_type.as_str() {
|
||||||
"string" => {
|
"string" => {
|
||||||
let len = field.length.unwrap_or(255);
|
let len = field.length.unwrap_or(255);
|
||||||
|
|
@ -630,6 +630,28 @@ pub fn process_table_definitions(
|
||||||
return Ok(tables);
|
return Ok(tables);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Use schema sync for both debug and release builds (non-destructive)
|
||||||
|
use super::table_migration::sync_bot_tables;
|
||||||
|
|
||||||
|
info!("Running schema migration sync (non-destructive)");
|
||||||
|
|
||||||
|
match sync_bot_tables(&state, bot_id, source) {
|
||||||
|
Ok(result) => {
|
||||||
|
info!("Schema sync completed: {} created, {} altered, {} columns added",
|
||||||
|
result.tables_created, result.tables_altered, result.columns_added);
|
||||||
|
|
||||||
|
// If sync was successful, skip standard table creation
|
||||||
|
if result.tables_created > 0 || result.tables_altered > 0 {
|
||||||
|
return Ok(tables);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Schema sync failed: {}", e);
|
||||||
|
// Fall through to standard table creation
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Standard table creation (for release builds or as fallback)
|
||||||
for table in &tables {
|
for table in &tables {
|
||||||
info!(
|
info!(
|
||||||
"Processing TABLE {} ON {}",
|
"Processing TABLE {} ON {}",
|
||||||
|
|
|
||||||
249
src/basic/keywords/table_migration.rs
Normal file
249
src/basic/keywords/table_migration.rs
Normal file
|
|
@ -0,0 +1,249 @@
|
||||||
|
/*****************************************************************************\
|
||||||
|
| Table Schema Migration Module
|
||||||
|
| Automatically syncs table.bas definitions with database schema
|
||||||
|
\*****************************************************************************/
|
||||||
|
|
||||||
|
use crate::core::shared::sanitize_identifier;
|
||||||
|
use crate::core::shared::state::AppState;
|
||||||
|
use diesel::prelude::*;
|
||||||
|
use diesel::sql_query;
|
||||||
|
use diesel::sql_types::{Text, Nullable};
|
||||||
|
use log::{error, info, warn};
|
||||||
|
use std::error::Error;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use super::table_definition::{FieldDefinition, TableDefinition, map_type_to_sql, parse_table_definition};
|
||||||
|
|
||||||
|
/// Schema migration result
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct MigrationResult {
|
||||||
|
pub tables_created: usize,
|
||||||
|
pub tables_altered: usize,
|
||||||
|
pub columns_added: usize,
|
||||||
|
pub errors: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Column metadata from database
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct DbColumn {
|
||||||
|
name: String,
|
||||||
|
data_type: String,
|
||||||
|
is_nullable: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compare and sync table schema with definition
|
||||||
|
pub fn sync_table_schema(
|
||||||
|
table: &TableDefinition,
|
||||||
|
existing_columns: &[DbColumn],
|
||||||
|
create_sql: &str,
|
||||||
|
conn: &mut diesel::PgConnection,
|
||||||
|
) -> Result<MigrationResult, Box<dyn Error + Send + Sync>> {
|
||||||
|
let mut result = MigrationResult::default();
|
||||||
|
|
||||||
|
// If no columns exist, create the table
|
||||||
|
if existing_columns.is_empty() {
|
||||||
|
info!("Creating new table: {}", table.name);
|
||||||
|
sql_query(create_sql).execute(conn)
|
||||||
|
.map_err(|e| format!("Failed to create table {}: {}", table.name, e))?;
|
||||||
|
result.tables_created += 1;
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for schema drift
|
||||||
|
let existing_col_names: std::collections::HashSet<String> =
|
||||||
|
existing_columns.iter().map(|c| c.name.clone()).collect();
|
||||||
|
|
||||||
|
let mut missing_columns: Vec<&FieldDefinition> = Vec::new();
|
||||||
|
for field in &table.fields {
|
||||||
|
if !existing_col_names.contains(&field.name) {
|
||||||
|
missing_columns.push(field);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add missing columns
|
||||||
|
if !missing_columns.is_empty() {
|
||||||
|
info!("Table {} is missing {} columns, adding them", table.name, missing_columns.len());
|
||||||
|
|
||||||
|
for field in &missing_columns {
|
||||||
|
let sql_type = map_type_to_sql(field, "postgres");
|
||||||
|
let column_sql = if field.is_nullable {
|
||||||
|
format!("ALTER TABLE {} ADD COLUMN IF NOT EXISTS {} {}",
|
||||||
|
sanitize_identifier(&table.name),
|
||||||
|
sanitize_identifier(&field.name),
|
||||||
|
sql_type)
|
||||||
|
} else {
|
||||||
|
// For NOT NULL columns, add as nullable first then set default
|
||||||
|
format!("ALTER TABLE {} ADD COLUMN IF NOT EXISTS {} {}",
|
||||||
|
sanitize_identifier(&table.name),
|
||||||
|
sanitize_identifier(&field.name),
|
||||||
|
sql_type)
|
||||||
|
};
|
||||||
|
|
||||||
|
info!("Adding column: {}.{} ({})", table.name, field.name, sql_type);
|
||||||
|
match sql_query(&column_sql).execute(conn) {
|
||||||
|
Ok(_) => {
|
||||||
|
result.columns_added += 1;
|
||||||
|
info!("Successfully added column {}.{}", table.name, field.name);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
// Check if column already exists (ignore error)
|
||||||
|
let err_str = e.to_string();
|
||||||
|
if !err_str.contains("already exists") && !err_str.contains("duplicate column") {
|
||||||
|
let error_msg = format!("Failed to add column {}.{}: {}", table.name, field.name, e);
|
||||||
|
error!("{}", error_msg);
|
||||||
|
result.errors.push(error_msg);
|
||||||
|
} else {
|
||||||
|
info!("Column {}.{} already exists, skipping", table.name, field.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.tables_altered += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get existing columns from a table
|
||||||
|
pub fn get_table_columns(
|
||||||
|
table_name: &str,
|
||||||
|
conn: &mut diesel::PgConnection,
|
||||||
|
) -> Result<Vec<DbColumn>, Box<dyn Error + Send + Sync>> {
|
||||||
|
let query = format!(
|
||||||
|
"SELECT column_name, data_type, is_nullable
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_name = '{}' AND table_schema = 'public'
|
||||||
|
ORDER BY ordinal_position",
|
||||||
|
sanitize_identifier(table_name)
|
||||||
|
);
|
||||||
|
|
||||||
|
#[derive(QueryableByName)]
|
||||||
|
struct ColumnRow {
|
||||||
|
#[diesel(sql_type = Text)]
|
||||||
|
column_name: String,
|
||||||
|
#[diesel(sql_type = Text)]
|
||||||
|
data_type: String,
|
||||||
|
#[diesel(sql_type = Text)]
|
||||||
|
is_nullable: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows: Vec<ColumnRow> = match sql_query(&query).load(conn) {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(e) => {
|
||||||
|
// Table doesn't exist
|
||||||
|
return Err(format!("Table {} does not exist: {}", table_name, e).into());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(rows.into_iter().map(|row| DbColumn {
|
||||||
|
name: row.column_name,
|
||||||
|
data_type: row.data_type,
|
||||||
|
is_nullable: row.is_nullable == "YES",
|
||||||
|
}).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Process table definitions with schema sync for a specific bot
|
||||||
|
pub fn sync_bot_tables(
|
||||||
|
state: &Arc<AppState>,
|
||||||
|
bot_id: Uuid,
|
||||||
|
source: &str,
|
||||||
|
) -> Result<MigrationResult, Box<dyn Error + Send + Sync>> {
|
||||||
|
let tables = parse_table_definition(source)?;
|
||||||
|
let mut result = MigrationResult::default();
|
||||||
|
|
||||||
|
info!("Processing {} table definitions with schema sync for bot {}", tables.len(), bot_id);
|
||||||
|
|
||||||
|
// Get bot's database connection
|
||||||
|
let pool = state.bot_database_manager.get_bot_pool(bot_id)?;
|
||||||
|
let mut conn = pool.get()?;
|
||||||
|
|
||||||
|
for table in &tables {
|
||||||
|
if table.connection_name != "default" {
|
||||||
|
continue; // Skip external connections for now
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Syncing table: {}", table.name);
|
||||||
|
|
||||||
|
// Get existing columns
|
||||||
|
let existing_columns = match get_table_columns(&table.name, &mut conn) {
|
||||||
|
Ok(cols) => cols,
|
||||||
|
Err(_) => {
|
||||||
|
// Table doesn't exist yet
|
||||||
|
vec![]
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Generate CREATE TABLE SQL
|
||||||
|
let create_sql = super::table_definition::generate_create_table_sql(table, "postgres");
|
||||||
|
|
||||||
|
// Sync schema
|
||||||
|
match sync_table_schema(table, &existing_columns, &create_sql, &mut conn) {
|
||||||
|
Ok(sync_result) => {
|
||||||
|
result.tables_created += sync_result.tables_created;
|
||||||
|
result.tables_altered += sync_result.tables_altered;
|
||||||
|
result.columns_added += sync_result.columns_added;
|
||||||
|
result.errors.extend(sync_result.errors);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
let error_msg = format!("Failed to sync table {}: {}", table.name, e);
|
||||||
|
error!("{}", error_msg);
|
||||||
|
result.errors.push(error_msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log summary
|
||||||
|
info!("Schema sync summary for bot {}: {} tables created, {} altered, {} columns added, {} errors",
|
||||||
|
bot_id, result.tables_created, result.tables_altered, result.columns_added, result.errors.len());
|
||||||
|
|
||||||
|
if !result.errors.is_empty() {
|
||||||
|
warn!("Schema sync completed with {} errors:", result.errors.len());
|
||||||
|
for error in &result.errors {
|
||||||
|
warn!(" - {}", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validate that all required columns exist
|
||||||
|
pub fn validate_table_schema(
|
||||||
|
table_name: &str,
|
||||||
|
required_fields: &[FieldDefinition],
|
||||||
|
conn: &mut diesel::PgConnection,
|
||||||
|
) -> Result<bool, Box<dyn Error + Send + Sync>> {
|
||||||
|
let existing_columns = get_table_columns(table_name, conn)?;
|
||||||
|
let existing_col_names: std::collections::HashSet<String> =
|
||||||
|
existing_columns.iter().map(|c| c.name.clone()).collect();
|
||||||
|
|
||||||
|
let mut missing = Vec::new();
|
||||||
|
for field in required_fields {
|
||||||
|
if !existing_col_names.contains(&field.name) {
|
||||||
|
missing.push(field.name.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !missing.is_empty() {
|
||||||
|
warn!("Table {} is missing columns: {:?}", table_name, missing);
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_db_column_creation() {
|
||||||
|
let col = DbColumn {
|
||||||
|
name: "test_col".to_string(),
|
||||||
|
data_type: "character varying".to_string(),
|
||||||
|
is_nullable: true,
|
||||||
|
};
|
||||||
|
assert_eq!(col.name, "test_col");
|
||||||
|
assert_eq!(col.is_nullable, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -826,7 +826,7 @@ mod tests {
|
||||||
"docs_example_com_path"
|
"docs_example_com_path"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
sanitize_url_for_collection("http://test.site:8080"),
|
sanitize_url_for_collection("http://test.site:9000"),
|
||||||
"test_site_8080"
|
"test_site_8080"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
1879
src/basic/mod.rs.backup
Normal file
1879
src/basic/mod.rs.backup
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -106,9 +106,9 @@ impl BootstrapManager {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if pm.is_installed("postgres") {
|
if pm.is_installed("tables") {
|
||||||
info!("Starting PostgreSQL...");
|
info!("Starting PostgreSQL...");
|
||||||
match pm.start("postgres") {
|
match pm.start("tables") {
|
||||||
Ok(_child) => {
|
Ok(_child) => {
|
||||||
info!("PostgreSQL started");
|
info!("PostgreSQL started");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -436,9 +436,22 @@ pub async fn inject_kb_context(
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sanitize context to remove UTF-16 surrogate characters that can't be encoded in UTF-8
|
||||||
|
let sanitized_context = context_string
|
||||||
|
.chars()
|
||||||
|
.filter(|c| {
|
||||||
|
let cp = *c as u32;
|
||||||
|
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
|
||||||
|
})
|
||||||
|
.collect::<String>();
|
||||||
|
|
||||||
|
if sanitized_context.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
"Injecting {} characters of KB/website context into prompt for session {}",
|
"Injecting {} characters of KB/website context into prompt for session {}",
|
||||||
context_string.len(),
|
sanitized_context.len(),
|
||||||
session_id
|
session_id
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -447,7 +460,7 @@ pub async fn inject_kb_context(
|
||||||
|
|
||||||
if let Some(idx) = system_msg_idx {
|
if let Some(idx) = system_msg_idx {
|
||||||
if let Some(content) = messages_array[idx]["content"].as_str() {
|
if let Some(content) = messages_array[idx]["content"].as_str() {
|
||||||
let new_content = format!("{}\n{}", content, context_string);
|
let new_content = format!("{}\n{}", content, sanitized_context);
|
||||||
messages_array[idx]["content"] = serde_json::Value::String(new_content);
|
messages_array[idx]["content"] = serde_json::Value::String(new_content);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -455,7 +468,7 @@ pub async fn inject_kb_context(
|
||||||
0,
|
0,
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"role": "system",
|
"role": "system",
|
||||||
"content": context_string
|
"content": sanitized_context
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -580,11 +580,20 @@ impl BotOrchestrator {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sanitize user message to remove any UTF-16 surrogate characters
|
||||||
|
let sanitized_message_content = message_content
|
||||||
|
.chars()
|
||||||
|
.filter(|c| {
|
||||||
|
let cp = *c as u32;
|
||||||
|
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
|
||||||
|
})
|
||||||
|
.collect::<String>();
|
||||||
|
|
||||||
// Add the current user message to the messages array
|
// Add the current user message to the messages array
|
||||||
if let Some(msgs_array) = messages.as_array_mut() {
|
if let Some(msgs_array) = messages.as_array_mut() {
|
||||||
msgs_array.push(serde_json::json!({
|
msgs_array.push(serde_json::json!({
|
||||||
"role": "user",
|
"role": "user",
|
||||||
"content": message_content
|
"content": sanitized_message_content
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -644,6 +653,7 @@ impl BotOrchestrator {
|
||||||
let mut analysis_buffer = String::new();
|
let mut analysis_buffer = String::new();
|
||||||
let mut in_analysis = false;
|
let mut in_analysis = false;
|
||||||
let mut tool_call_buffer = String::new(); // Accumulate potential tool call JSON chunks
|
let mut tool_call_buffer = String::new(); // Accumulate potential tool call JSON chunks
|
||||||
|
let mut accumulating_tool_call = false; // Track if we're currently accumulating a tool call
|
||||||
let handler = llm_models::get_handler(&model);
|
let handler = llm_models::get_handler(&model);
|
||||||
|
|
||||||
info!("[STREAM_START] Entering stream processing loop for model: {}", model);
|
info!("[STREAM_START] Entering stream processing loop for model: {}", model);
|
||||||
|
|
@ -679,12 +689,62 @@ impl BotOrchestrator {
|
||||||
// ===== GENERIC TOOL EXECUTION =====
|
// ===== GENERIC TOOL EXECUTION =====
|
||||||
// Add chunk to tool_call_buffer and try to parse
|
// Add chunk to tool_call_buffer and try to parse
|
||||||
// Tool calls arrive as JSON that can span multiple chunks
|
// Tool calls arrive as JSON that can span multiple chunks
|
||||||
let looks_like_json = chunk.trim().starts_with('{') || chunk.trim().starts_with('[') ||
|
|
||||||
tool_call_buffer.contains('{') || tool_call_buffer.contains('[');
|
|
||||||
|
|
||||||
let chunk_in_tool_buffer = if looks_like_json {
|
// Check if this chunk contains JSON (either starts with {/[ or contains {/[)
|
||||||
|
let chunk_contains_json = chunk.trim().starts_with('{') || chunk.trim().starts_with('[') ||
|
||||||
|
chunk.contains('{') || chunk.contains('[');
|
||||||
|
|
||||||
|
let chunk_in_tool_buffer = if accumulating_tool_call {
|
||||||
|
// Already accumulating - add entire chunk to buffer
|
||||||
tool_call_buffer.push_str(&chunk);
|
tool_call_buffer.push_str(&chunk);
|
||||||
true
|
true
|
||||||
|
} else if chunk_contains_json {
|
||||||
|
// Check if { appears in the middle of the chunk (mixed text + JSON)
|
||||||
|
let json_start = chunk.find('{').or_else(|| chunk.find('['));
|
||||||
|
|
||||||
|
if let Some(pos) = json_start {
|
||||||
|
if pos > 0 {
|
||||||
|
// Send the part before { as regular content
|
||||||
|
let regular_part = &chunk[..pos];
|
||||||
|
if !regular_part.trim().is_empty() {
|
||||||
|
info!("[STREAM_CONTENT] Sending regular part before JSON: '{}', len: {}", regular_part, regular_part.len());
|
||||||
|
full_response.push_str(regular_part);
|
||||||
|
|
||||||
|
let response = BotResponse {
|
||||||
|
bot_id: message.bot_id.clone(),
|
||||||
|
user_id: message.user_id.clone(),
|
||||||
|
session_id: message.session_id.clone(),
|
||||||
|
channel: message.channel.clone(),
|
||||||
|
content: regular_part.to_string(),
|
||||||
|
message_type: MessageType::BOT_RESPONSE,
|
||||||
|
stream_token: None,
|
||||||
|
is_complete: false,
|
||||||
|
suggestions: Vec::new(),
|
||||||
|
context_name: None,
|
||||||
|
context_length: 0,
|
||||||
|
context_max_length: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
if response_tx.send(response).await.is_err() {
|
||||||
|
warn!("Response channel closed");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start accumulating from { onwards
|
||||||
|
accumulating_tool_call = true;
|
||||||
|
tool_call_buffer.push_str(&chunk[pos..]);
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
// Chunk starts with { or [
|
||||||
|
accumulating_tool_call = true;
|
||||||
|
tool_call_buffer.push_str(&chunk);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Contains {/[ but find() failed - shouldn't happen, but send as regular content
|
||||||
|
false
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
false
|
false
|
||||||
};
|
};
|
||||||
|
|
@ -774,13 +834,15 @@ impl BotOrchestrator {
|
||||||
// Don't add tool_call JSON to full_response or analysis_buffer
|
// Don't add tool_call JSON to full_response or analysis_buffer
|
||||||
// Clear the tool_call_buffer since we found and executed a tool call
|
// Clear the tool_call_buffer since we found and executed a tool call
|
||||||
tool_call_buffer.clear();
|
tool_call_buffer.clear();
|
||||||
|
accumulating_tool_call = false; // Reset accumulation flag
|
||||||
// Continue to next chunk
|
// Continue to next chunk
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clear tool_call_buffer if it's getting too large and no tool call was found
|
// Clear tool_call_buffer if it's getting too large and no tool call was found
|
||||||
// This prevents memory issues from accumulating JSON fragments
|
// This prevents memory issues from accumulating JSON fragments
|
||||||
if tool_call_buffer.len() > 10000 {
|
// Increased limit to 50000 to handle large tool calls with many parameters
|
||||||
|
if tool_call_buffer.len() > 50000 {
|
||||||
// Flush accumulated content to client since it's too large to be a tool call
|
// Flush accumulated content to client since it's too large to be a tool call
|
||||||
info!("[TOOL_EXEC] Flushing tool_call_buffer (too large, assuming not a tool call)");
|
info!("[TOOL_EXEC] Flushing tool_call_buffer (too large, assuming not a tool call)");
|
||||||
full_response.push_str(&tool_call_buffer);
|
full_response.push_str(&tool_call_buffer);
|
||||||
|
|
@ -801,6 +863,7 @@ impl BotOrchestrator {
|
||||||
};
|
};
|
||||||
|
|
||||||
tool_call_buffer.clear();
|
tool_call_buffer.clear();
|
||||||
|
accumulating_tool_call = false; // Reset accumulation flag after flush
|
||||||
|
|
||||||
if response_tx.send(response).await.is_err() {
|
if response_tx.send(response).await.is_err() {
|
||||||
warn!("Response channel closed");
|
warn!("Response channel closed");
|
||||||
|
|
@ -810,7 +873,7 @@ impl BotOrchestrator {
|
||||||
|
|
||||||
// If this chunk was added to tool_call_buffer and no tool call was found yet,
|
// If this chunk was added to tool_call_buffer and no tool call was found yet,
|
||||||
// skip processing (it's part of an incomplete tool call JSON)
|
// skip processing (it's part of an incomplete tool call JSON)
|
||||||
if chunk_in_tool_buffer && tool_call_buffer.len() <= 10000 {
|
if chunk_in_tool_buffer {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// ===== END TOOL EXECUTION =====
|
// ===== END TOOL EXECUTION =====
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
/// Works across all LLM providers (GLM, OpenAI, Claude, etc.)
|
/// Works across all LLM providers (GLM, OpenAI, Claude, etc.)
|
||||||
use log::{error, info, warn};
|
use log::{error, info, warn};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::fs::OpenOptions;
|
use std::fs::OpenOptions;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
@ -264,6 +265,8 @@ impl ToolExecutor {
|
||||||
script_service.load_bot_config_params(state, bot_id);
|
script_service.load_bot_config_params(state, bot_id);
|
||||||
|
|
||||||
// Set tool parameters as variables in the engine scope
|
// Set tool parameters as variables in the engine scope
|
||||||
|
// Note: DATE parameters are now sent by LLM in ISO 8601 format (YYYY-MM-DD)
|
||||||
|
// The tool schema with format="date" tells the LLM to use this agnostic format
|
||||||
if let Some(obj) = arguments.as_object() {
|
if let Some(obj) = arguments.as_object() {
|
||||||
for (key, value) in obj {
|
for (key, value) in obj {
|
||||||
let value_str = match value {
|
let value_str = match value {
|
||||||
|
|
|
||||||
|
|
@ -553,7 +553,7 @@ Store credentials in Vault:
|
||||||
r"Email Server (Stalwart):
|
r"Email Server (Stalwart):
|
||||||
SMTP: {}:25
|
SMTP: {}:25
|
||||||
IMAP: {}:143
|
IMAP: {}:143
|
||||||
Web: http://{}:8080
|
Web: http://{}:9000
|
||||||
|
|
||||||
Store credentials in Vault:
|
Store credentials in Vault:
|
||||||
botserver vault put gbo/email server={} port=25 username=admin password=<your-password>",
|
botserver vault put gbo/email server={} port=25 username=admin password=<your-password>",
|
||||||
|
|
@ -563,11 +563,11 @@ Store credentials in Vault:
|
||||||
"directory" => {
|
"directory" => {
|
||||||
format!(
|
format!(
|
||||||
r"Zitadel Identity Provider:
|
r"Zitadel Identity Provider:
|
||||||
URL: http://{}:8080
|
URL: http://{}:9000
|
||||||
Console: http://{}:8080/ui/console
|
Console: http://{}:9000/ui/console
|
||||||
|
|
||||||
Store credentials in Vault:
|
Store credentials in Vault:
|
||||||
botserver vault put gbo/directory url=http://{}:8080 client_id=<client-id> client_secret=<client-secret>",
|
botserver vault put gbo/directory url=http://{}:9000 client_id=<client-id> client_secret=<client-secret>",
|
||||||
ip, ip, ip
|
ip, ip, ip
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -602,7 +602,7 @@ impl PackageManager {
|
||||||
post_install_cmds_windows: vec![],
|
post_install_cmds_windows: vec![],
|
||||||
env_vars: HashMap::new(),
|
env_vars: HashMap::new(),
|
||||||
data_download_list: Vec::new(),
|
data_download_list: Vec::new(),
|
||||||
exec_cmd: "php -S 0.0.0.0:8080 -t {{DATA_PATH}}/roundcubemail".to_string(),
|
exec_cmd: "php -S 0.0.0.0:9000 -t {{DATA_PATH}}/roundcubemail".to_string(),
|
||||||
check_cmd:
|
check_cmd:
|
||||||
"curl -f -k --connect-timeout 2 -m 5 https://localhost:8300 >/dev/null 2>&1"
|
"curl -f -k --connect-timeout 2 -m 5 https://localhost:8300 >/dev/null 2>&1"
|
||||||
.to_string(),
|
.to_string(),
|
||||||
|
|
|
||||||
|
|
@ -203,7 +203,7 @@ impl EmailSetup {
|
||||||
|
|
||||||
let issuer_url = dir_config["base_url"]
|
let issuer_url = dir_config["base_url"]
|
||||||
.as_str()
|
.as_str()
|
||||||
.unwrap_or("http://localhost:8080");
|
.unwrap_or("http://localhost:9000");
|
||||||
|
|
||||||
log::info!("Setting up OIDC authentication with Directory...");
|
log::info!("Setting up OIDC authentication with Directory...");
|
||||||
log::info!("Issuer URL: {}", issuer_url);
|
log::info!("Issuer URL: {}", issuer_url);
|
||||||
|
|
@ -289,7 +289,7 @@ protocol = "imap"
|
||||||
tls.implicit = true
|
tls.implicit = true
|
||||||
|
|
||||||
[server.listener."http"]
|
[server.listener."http"]
|
||||||
bind = ["0.0.0.0:8080"]
|
bind = ["0.0.0.0:9000"]
|
||||||
protocol = "http"
|
protocol = "http"
|
||||||
|
|
||||||
[storage]
|
[storage]
|
||||||
|
|
@ -315,7 +315,7 @@ store = "sqlite"
|
||||||
r#"
|
r#"
|
||||||
[directory."oidc"]
|
[directory."oidc"]
|
||||||
type = "oidc"
|
type = "oidc"
|
||||||
issuer = "http://localhost:8080"
|
issuer = "http://localhost:9000"
|
||||||
client-id = "{{CLIENT_ID}}"
|
client-id = "{{CLIENT_ID}}"
|
||||||
client-secret = "{{CLIENT_SECRET}}"
|
client-secret = "{{CLIENT_SECRET}}"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -381,7 +381,7 @@ impl SecretsManager {
|
||||||
secrets.insert("token".into(), String::new());
|
secrets.insert("token".into(), String::new());
|
||||||
}
|
}
|
||||||
SecretPaths::ALM => {
|
SecretPaths::ALM => {
|
||||||
secrets.insert("url".into(), "http://localhost:8080".into());
|
secrets.insert("url".into(), "http://localhost:9000".into());
|
||||||
secrets.insert("username".into(), String::new());
|
secrets.insert("username".into(), String::new());
|
||||||
secrets.insert("password".into(), String::new());
|
secrets.insert("password".into(), String::new());
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -249,13 +249,13 @@ impl Default for TestAppStateBuilder {
|
||||||
#[cfg(feature = "directory")]
|
#[cfg(feature = "directory")]
|
||||||
pub fn create_mock_auth_service() -> AuthService {
|
pub fn create_mock_auth_service() -> AuthService {
|
||||||
let config = ZitadelConfig {
|
let config = ZitadelConfig {
|
||||||
issuer_url: "http://localhost:8080".to_string(),
|
issuer_url: "http://localhost:9000".to_string(),
|
||||||
issuer: "http://localhost:8080".to_string(),
|
issuer: "http://localhost:9000".to_string(),
|
||||||
client_id: "mock_client_id".to_string(),
|
client_id: "mock_client_id".to_string(),
|
||||||
client_secret: "mock_client_secret".to_string(),
|
client_secret: "mock_client_secret".to_string(),
|
||||||
redirect_uri: "http://localhost:3000/callback".to_string(),
|
redirect_uri: "http://localhost:3000/callback".to_string(),
|
||||||
project_id: "mock_project_id".to_string(),
|
project_id: "mock_project_id".to_string(),
|
||||||
api_url: "http://localhost:8080".to_string(),
|
api_url: "http://localhost:9000".to_string(),
|
||||||
service_account_key: None,
|
service_account_key: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -556,3 +556,96 @@ fn estimate_chars_per_token(model: &str) -> usize {
|
||||||
4 // Default conservative estimate
|
4 // Default conservative estimate
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Convert date string from user locale format to ISO format (YYYY-MM-DD) for PostgreSQL.
|
||||||
|
///
|
||||||
|
/// The LLM automatically formats dates according to the user's language/idiom based on:
|
||||||
|
/// 1. The conversation context (user's language)
|
||||||
|
/// 2. The PARAM LIKE example (e.g., "15/12/2026" for DD/MM/YYYY)
|
||||||
|
///
|
||||||
|
/// This function handles the most common formats:
|
||||||
|
/// - ISO: YYYY-MM-DD (already in ISO, returned as-is)
|
||||||
|
/// - Brazilian/Portuguese: DD/MM/YYYY or DD/MM/YY
|
||||||
|
/// - US/English: MM/DD/YYYY or MM/DD/YY
|
||||||
|
///
|
||||||
|
/// If the value doesn't match any date pattern, returns it unchanged.
|
||||||
|
///
|
||||||
|
/// NOTE: This function does NOT try to guess ambiguous formats.
|
||||||
|
/// The LLM is responsible for formatting dates correctly based on user language.
|
||||||
|
/// The PARAM declaration's LIKE example tells the LLM the expected format.
|
||||||
|
///
|
||||||
|
/// # Arguments
|
||||||
|
/// * `value` - The date string to convert (as provided by the LLM)
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
/// ISO formatted date string (YYYY-MM-DD) or original value if not a recognized date
|
||||||
|
pub fn convert_date_to_iso_format(value: &str) -> String {
|
||||||
|
let value = value.trim();
|
||||||
|
|
||||||
|
// Already in ISO format (YYYY-MM-DD) - return as-is
|
||||||
|
if value.len() == 10 && value.chars().nth(4) == Some('-') && value.chars().nth(7) == Some('-') {
|
||||||
|
let parts: Vec<&str> = value.split('-').collect();
|
||||||
|
if parts.len() == 3
|
||||||
|
&& parts[0].len() == 4
|
||||||
|
&& parts[1].len() == 2
|
||||||
|
&& parts[2].len() == 2
|
||||||
|
&& parts[0].chars().all(|c| c.is_ascii_digit())
|
||||||
|
&& parts[1].chars().all(|c| c.is_ascii_digit())
|
||||||
|
&& parts[2].chars().all(|c| c.is_ascii_digit())
|
||||||
|
{
|
||||||
|
if let (Ok(year), Ok(month), Ok(day)) =
|
||||||
|
(parts[0].parse::<u32>(), parts[1].parse::<u32>(), parts[2].parse::<u32>())
|
||||||
|
{
|
||||||
|
if month >= 1 && month <= 12 && day >= 1 && day <= 31 && year >= 1900 && year <= 2100 {
|
||||||
|
return value.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle slash-separated formats: DD/MM/YYYY or MM/DD/YYYY
|
||||||
|
// We need to detect which format based on the PARAM declaration's LIKE example
|
||||||
|
// For now, default to DD/MM/YYYY (Brazilian format) as this is the most common for this bot
|
||||||
|
// TODO: Pass language/idiom from session to determine correct format
|
||||||
|
if value.len() >= 8 && value.len() <= 10 {
|
||||||
|
let parts: Vec<&str> = value.split('/').collect();
|
||||||
|
if parts.len() == 3 {
|
||||||
|
let all_numeric = parts[0].chars().all(|c| c.is_ascii_digit())
|
||||||
|
&& parts[1].chars().all(|c| c.is_ascii_digit())
|
||||||
|
&& parts[2].chars().all(|c| c.is_ascii_digit());
|
||||||
|
|
||||||
|
if all_numeric {
|
||||||
|
// Parse the three parts
|
||||||
|
let a = parts[0].parse::<u32>().ok();
|
||||||
|
let b = parts[1].parse::<u32>().ok();
|
||||||
|
let c = if parts[2].len() == 2 {
|
||||||
|
// Convert 2-digit year to 4-digit
|
||||||
|
parts[2].parse::<u32>().ok().map(|y| {
|
||||||
|
if y < 50 {
|
||||||
|
2000 + y
|
||||||
|
} else {
|
||||||
|
1900 + y
|
||||||
|
}
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
parts[2].parse::<u32>().ok()
|
||||||
|
};
|
||||||
|
|
||||||
|
if let (Some(first), Some(second), Some(third)) = (a, b, c) {
|
||||||
|
// Default: DD/MM/YYYY format (Brazilian/Portuguese)
|
||||||
|
// The LLM should format dates according to the user's language
|
||||||
|
// and the PARAM LIKE example (e.g., "15/12/2026" for DD/MM/YYYY)
|
||||||
|
let (year, month, day) = (third, second, first);
|
||||||
|
|
||||||
|
// Validate the determined date
|
||||||
|
if day >= 1 && day <= 31 && month >= 1 && month <= 12 && year >= 1900 && year <= 2100 {
|
||||||
|
return format!("{:04}-{:02}-{:02}", year, month, day);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not a recognized date pattern, return unchanged
|
||||||
|
value.to_string()
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -479,7 +479,7 @@ impl ApiUrls {
|
||||||
pub struct InternalUrls;
|
pub struct InternalUrls;
|
||||||
|
|
||||||
impl InternalUrls {
|
impl InternalUrls {
|
||||||
pub const DIRECTORY_BASE: &'static str = "http://localhost:8080";
|
pub const DIRECTORY_BASE: &'static str = "http://localhost:9000";
|
||||||
pub const DATABASE: &'static str = "postgres://localhost:5432";
|
pub const DATABASE: &'static str = "postgres://localhost:5432";
|
||||||
pub const CACHE: &'static str = "redis://localhost:6379";
|
pub const CACHE: &'static str = "redis://localhost:6379";
|
||||||
pub const DRIVE: &'static str = "https://localhost:9000";
|
pub const DRIVE: &'static str = "https://localhost:9000";
|
||||||
|
|
|
||||||
|
|
@ -65,8 +65,8 @@ fn is_tracking_pixel_enabled(state: &Arc<AppState>, bot_id: Option<Uuid>) -> boo
|
||||||
fn inject_tracking_pixel(html_body: &str, tracking_id: &str, state: &Arc<AppState>) -> String {
|
fn inject_tracking_pixel(html_body: &str, tracking_id: &str, state: &Arc<AppState>) -> String {
|
||||||
let config_manager = crate::core::config::ConfigManager::new(state.conn.clone());
|
let config_manager = crate::core::config::ConfigManager::new(state.conn.clone());
|
||||||
let base_url = config_manager
|
let base_url = config_manager
|
||||||
.get_config(&Uuid::nil(), "server-url", Some("http://localhost:8080"))
|
.get_config(&Uuid::nil(), "server-url", Some("http://localhost:9000"))
|
||||||
.unwrap_or_else(|_| "http://localhost:8080".to_string());
|
.unwrap_or_else(|_| "http://localhost:9000".to_string());
|
||||||
|
|
||||||
let pixel_url = format!("{}/api/email/tracking/pixel/{}", base_url, tracking_id);
|
let pixel_url = format!("{}/api/email/tracking/pixel/{}", base_url, tracking_id);
|
||||||
let pixel_html = format!(
|
let pixel_html = format!(
|
||||||
|
|
|
||||||
|
|
@ -31,8 +31,8 @@ pub fn is_tracking_pixel_enabled(state: &Arc<AppState>, bot_id: Option<Uuid>) ->
|
||||||
pub fn inject_tracking_pixel(html_body: &str, tracking_id: &str, state: &Arc<AppState>) -> String {
|
pub fn inject_tracking_pixel(html_body: &str, tracking_id: &str, state: &Arc<AppState>) -> String {
|
||||||
let config_manager = crate::core::config::ConfigManager::new(state.conn.clone());
|
let config_manager = crate::core::config::ConfigManager::new(state.conn.clone());
|
||||||
let base_url = config_manager
|
let base_url = config_manager
|
||||||
.get_config(&Uuid::nil(), "server-url", Some("http://localhost:8080"))
|
.get_config(&Uuid::nil(), "server-url", Some("http://localhost:9000"))
|
||||||
.unwrap_or_else(|_| "http://localhost:8080".to_string());
|
.unwrap_or_else(|_| "http://localhost:9000".to_string());
|
||||||
|
|
||||||
let pixel_url = format!("{}/api/email/tracking/pixel/{}", base_url, tracking_id);
|
let pixel_url = format!("{}/api/email/tracking/pixel/{}", base_url, tracking_id);
|
||||||
let pixel_html = format!(
|
let pixel_html = format!(
|
||||||
|
|
|
||||||
|
|
@ -232,6 +232,16 @@ impl ClaudeClient {
|
||||||
(system_prompt, claude_messages)
|
(system_prompt, claude_messages)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sanitizes a string by removing invalid UTF-8 surrogate characters
|
||||||
|
fn sanitize_utf8(input: &str) -> String {
|
||||||
|
input.chars()
|
||||||
|
.filter(|c| {
|
||||||
|
let cp = *c as u32;
|
||||||
|
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn build_messages(
|
pub fn build_messages(
|
||||||
system_prompt: &str,
|
system_prompt: &str,
|
||||||
context_data: &str,
|
context_data: &str,
|
||||||
|
|
@ -241,15 +251,15 @@ impl ClaudeClient {
|
||||||
let mut system_parts = Vec::new();
|
let mut system_parts = Vec::new();
|
||||||
|
|
||||||
if !system_prompt.is_empty() {
|
if !system_prompt.is_empty() {
|
||||||
system_parts.push(system_prompt.to_string());
|
system_parts.push(Self::sanitize_utf8(system_prompt));
|
||||||
}
|
}
|
||||||
if !context_data.is_empty() {
|
if !context_data.is_empty() {
|
||||||
system_parts.push(context_data.to_string());
|
system_parts.push(Self::sanitize_utf8(context_data));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (role, content) in history {
|
for (role, content) in history {
|
||||||
if role == "episodic" || role == "compact" {
|
if role == "episodic" || role == "compact" {
|
||||||
system_parts.push(format!("[Previous conversation summary]: {content}"));
|
system_parts.push(format!("[Previous conversation summary]: {}", Self::sanitize_utf8(content)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -270,7 +280,8 @@ impl ClaudeClient {
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(norm_role) = normalized_role {
|
if let Some(norm_role) = normalized_role {
|
||||||
if content.is_empty() {
|
let sanitized_content = Self::sanitize_utf8(content);
|
||||||
|
if sanitized_content.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -278,14 +289,14 @@ impl ClaudeClient {
|
||||||
if let Some(last_msg) = messages.last_mut() {
|
if let Some(last_msg) = messages.last_mut() {
|
||||||
let last_msg: &mut ClaudeMessage = last_msg;
|
let last_msg: &mut ClaudeMessage = last_msg;
|
||||||
last_msg.content.push_str("\n\n");
|
last_msg.content.push_str("\n\n");
|
||||||
last_msg.content.push_str(content);
|
last_msg.content.push_str(&sanitized_content);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
messages.push(ClaudeMessage {
|
messages.push(ClaudeMessage {
|
||||||
role: norm_role.clone(),
|
role: norm_role.clone(),
|
||||||
content: content.clone(),
|
content: sanitized_content,
|
||||||
});
|
});
|
||||||
last_role = Some(norm_role);
|
last_role = Some(norm_role);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -116,6 +116,16 @@ impl GLMClient {
|
||||||
// GLM/z.ai uses /chat/completions (not /v1/chat/completions)
|
// GLM/z.ai uses /chat/completions (not /v1/chat/completions)
|
||||||
format!("{}/chat/completions", self.base_url)
|
format!("{}/chat/completions", self.base_url)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sanitizes a string by removing invalid UTF-8 surrogate characters
|
||||||
|
fn sanitize_utf8(input: &str) -> String {
|
||||||
|
input.chars()
|
||||||
|
.filter(|c| {
|
||||||
|
let cp = *c as u32;
|
||||||
|
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
|
|
@ -183,11 +193,6 @@ impl LLMProvider for GLMClient {
|
||||||
key: &str,
|
key: &str,
|
||||||
tools: Option<&Vec<Value>>,
|
tools: Option<&Vec<Value>>,
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
// DEBUG: Log what we received
|
|
||||||
info!("[GLM_DEBUG] config type: {}", config);
|
|
||||||
info!("[GLM_DEBUG] prompt: '{}'", prompt);
|
|
||||||
info!("[GLM_DEBUG] config as JSON: {}", serde_json::to_string_pretty(config).unwrap_or_default());
|
|
||||||
|
|
||||||
// config IS the messages array directly, not nested
|
// config IS the messages array directly, not nested
|
||||||
let messages = if let Some(msgs) = config.as_array() {
|
let messages = if let Some(msgs) = config.as_array() {
|
||||||
// Convert messages from config format to GLM format
|
// Convert messages from config format to GLM format
|
||||||
|
|
@ -195,25 +200,23 @@ impl LLMProvider for GLMClient {
|
||||||
.filter_map(|m| {
|
.filter_map(|m| {
|
||||||
let role = m.get("role")?.as_str()?;
|
let role = m.get("role")?.as_str()?;
|
||||||
let content = m.get("content")?.as_str()?;
|
let content = m.get("content")?.as_str()?;
|
||||||
info!("[GLM_DEBUG] Processing message - role: {}, content: '{}'", role, content);
|
let sanitized = Self::sanitize_utf8(content);
|
||||||
if !content.is_empty() {
|
if !sanitized.is_empty() {
|
||||||
Some(GLMMessage {
|
Some(GLMMessage {
|
||||||
role: role.to_string(),
|
role: role.to_string(),
|
||||||
content: Some(content.to_string()),
|
content: Some(sanitized),
|
||||||
tool_calls: None,
|
tool_calls: None,
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
info!("[GLM_DEBUG] Skipping empty content message");
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
} else {
|
} else {
|
||||||
// Fallback to building from prompt
|
// Fallback to building from prompt
|
||||||
info!("[GLM_DEBUG] No array found, using prompt: '{}'", prompt);
|
|
||||||
vec![GLMMessage {
|
vec![GLMMessage {
|
||||||
role: "user".to_string(),
|
role: "user".to_string(),
|
||||||
content: Some(prompt.to_string()),
|
content: Some(Self::sanitize_utf8(prompt)),
|
||||||
tool_calls: None,
|
tool_calls: None,
|
||||||
}]
|
}]
|
||||||
};
|
};
|
||||||
|
|
@ -223,8 +226,6 @@ impl LLMProvider for GLMClient {
|
||||||
return Err("No valid messages in request".into());
|
return Err("No valid messages in request".into());
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("[GLM_DEBUG] Final GLM messages count: {}", messages.len());
|
|
||||||
|
|
||||||
// Use glm-4.7 for tool calling support
|
// Use glm-4.7 for tool calling support
|
||||||
// GLM-4.7 supports standard OpenAI-compatible function calling
|
// GLM-4.7 supports standard OpenAI-compatible function calling
|
||||||
let model_name = if model == "glm-4" { "glm-4.7" } else { model };
|
let model_name = if model == "glm-4" { "glm-4.7" } else { model };
|
||||||
|
|
@ -249,10 +250,6 @@ impl LLMProvider for GLMClient {
|
||||||
let url = self.build_url();
|
let url = self.build_url();
|
||||||
info!("GLM streaming request to: {}", url);
|
info!("GLM streaming request to: {}", url);
|
||||||
|
|
||||||
// Log the exact request being sent
|
|
||||||
let request_json = serde_json::to_string_pretty(&request).unwrap_or_default();
|
|
||||||
info!("GLM request body: {}", request_json);
|
|
||||||
|
|
||||||
let response = self
|
let response = self
|
||||||
.client
|
.client
|
||||||
.post(&url)
|
.post(&url)
|
||||||
|
|
@ -292,18 +289,13 @@ impl LLMProvider for GLMClient {
|
||||||
|
|
||||||
if line.starts_with("data: ") {
|
if line.starts_with("data: ") {
|
||||||
let json_str = line[6..].trim();
|
let json_str = line[6..].trim();
|
||||||
info!("[GLM_SSE] Received SSE line ({} chars): {}", json_str.len(), json_str);
|
|
||||||
if let Ok(chunk_data) = serde_json::from_str::<Value>(json_str) {
|
if let Ok(chunk_data) = serde_json::from_str::<Value>(json_str) {
|
||||||
if let Some(choices) = chunk_data.get("choices").and_then(|c| c.as_array()) {
|
if let Some(choices) = chunk_data.get("choices").and_then(|c| c.as_array()) {
|
||||||
for choice in choices {
|
for choice in choices {
|
||||||
info!("[GLM_SSE] Processing choice");
|
|
||||||
if let Some(delta) = choice.get("delta") {
|
if let Some(delta) = choice.get("delta") {
|
||||||
info!("[GLM_SSE] Delta: {}", serde_json::to_string(delta).unwrap_or_default());
|
|
||||||
|
|
||||||
// Handle tool_calls (GLM-4.7 standard function calling)
|
// Handle tool_calls (GLM-4.7 standard function calling)
|
||||||
if let Some(tool_calls) = delta.get("tool_calls").and_then(|t| t.as_array()) {
|
if let Some(tool_calls) = delta.get("tool_calls").and_then(|t| t.as_array()) {
|
||||||
for tool_call in tool_calls {
|
for tool_call in tool_calls {
|
||||||
info!("[GLM_SSE] Tool call detected: {}", serde_json::to_string(tool_call).unwrap_or_default());
|
|
||||||
// Send tool_calls as JSON for the calling code to process
|
// Send tool_calls as JSON for the calling code to process
|
||||||
let tool_call_json = serde_json::json!({
|
let tool_call_json = serde_json::json!({
|
||||||
"type": "tool_call",
|
"type": "tool_call",
|
||||||
|
|
@ -323,7 +315,6 @@ impl LLMProvider for GLMClient {
|
||||||
// This makes GLM behave like OpenAI-compatible APIs
|
// This makes GLM behave like OpenAI-compatible APIs
|
||||||
if let Some(content) = delta.get("content").and_then(|c| c.as_str()) {
|
if let Some(content) = delta.get("content").and_then(|c| c.as_str()) {
|
||||||
if !content.is_empty() {
|
if !content.is_empty() {
|
||||||
info!("[GLM_TX] Sending to channel: '{}'", content);
|
|
||||||
match tx.send(content.to_string()).await {
|
match tx.send(content.to_string()).await {
|
||||||
Ok(_) => {},
|
Ok(_) => {},
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -331,11 +322,9 @@ impl LLMProvider for GLMClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
info!("[GLM_SSE] No content field in delta");
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
info!("[GLM_SSE] No delta in choice");
|
// No delta in choice
|
||||||
}
|
}
|
||||||
if let Some(reason) = choice.get("finish_reason").and_then(|r| r.as_str()) {
|
if let Some(reason) = choice.get("finish_reason").and_then(|r| r.as_str()) {
|
||||||
if !reason.is_empty() {
|
if !reason.is_empty() {
|
||||||
|
|
|
||||||
|
|
@ -185,6 +185,17 @@ impl OpenAIClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sanitizes a string by removing invalid UTF-8 surrogate characters
|
||||||
|
/// that cannot be encoded in valid UTF-8 (surrogates are only valid in UTF-16)
|
||||||
|
fn sanitize_utf8(input: &str) -> String {
|
||||||
|
input.chars()
|
||||||
|
.filter(|c| {
|
||||||
|
let cp = *c as u32;
|
||||||
|
!(0xD800..=0xDBFF).contains(&cp) && !(0xDC00..=0xDFFF).contains(&cp)
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn build_messages(
|
pub fn build_messages(
|
||||||
system_prompt: &str,
|
system_prompt: &str,
|
||||||
context_data: &str,
|
context_data: &str,
|
||||||
|
|
@ -194,19 +205,19 @@ impl OpenAIClient {
|
||||||
if !system_prompt.is_empty() {
|
if !system_prompt.is_empty() {
|
||||||
messages.push(serde_json::json!({
|
messages.push(serde_json::json!({
|
||||||
"role": "system",
|
"role": "system",
|
||||||
"content": system_prompt
|
"content": Self::sanitize_utf8(system_prompt)
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
if !context_data.is_empty() {
|
if !context_data.is_empty() {
|
||||||
messages.push(serde_json::json!({
|
messages.push(serde_json::json!({
|
||||||
"role": "system",
|
"role": "system",
|
||||||
"content": context_data
|
"content": Self::sanitize_utf8(context_data)
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
for (role, content) in history {
|
for (role, content) in history {
|
||||||
messages.push(serde_json::json!({
|
messages.push(serde_json::json!({
|
||||||
"role": role,
|
"role": role,
|
||||||
"content": content
|
"content": Self::sanitize_utf8(content)
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
serde_json::Value::Array(messages)
|
serde_json::Value::Array(messages)
|
||||||
|
|
@ -747,10 +758,10 @@ mod tests {
|
||||||
fn test_openai_client_new_custom_url() {
|
fn test_openai_client_new_custom_url() {
|
||||||
let client = OpenAIClient::new(
|
let client = OpenAIClient::new(
|
||||||
"test_key".to_string(),
|
"test_key".to_string(),
|
||||||
Some("http://localhost:8080".to_string()),
|
Some("http://localhost:9000".to_string()),
|
||||||
None,
|
None,
|
||||||
);
|
);
|
||||||
assert_eq!(client.base_url, "http://localhost:8080");
|
assert_eq!(client.base_url, "http://localhost:9000");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
|
|
@ -101,13 +101,13 @@ impl CorsConfig {
|
||||||
Self {
|
Self {
|
||||||
allowed_origins: vec![
|
allowed_origins: vec![
|
||||||
"http://localhost:3000".to_string(),
|
"http://localhost:3000".to_string(),
|
||||||
"http://localhost:8080".to_string(),
|
"http://localhost:9000".to_string(),
|
||||||
"http://localhost:8300".to_string(),
|
"http://localhost:8300".to_string(),
|
||||||
"http://127.0.0.1:3000".to_string(),
|
"http://127.0.0.1:3000".to_string(),
|
||||||
"http://127.0.0.1:8080".to_string(),
|
"http://127.0.0.1:9000".to_string(),
|
||||||
"http://127.0.0.1:8300".to_string(),
|
"http://127.0.0.1:8300".to_string(),
|
||||||
"https://localhost:3000".to_string(),
|
"https://localhost:3000".to_string(),
|
||||||
"https://localhost:8080".to_string(),
|
"https://localhost:9000".to_string(),
|
||||||
"https://localhost:8300".to_string(),
|
"https://localhost:8300".to_string(),
|
||||||
],
|
],
|
||||||
allowed_methods: vec![
|
allowed_methods: vec![
|
||||||
|
|
@ -576,7 +576,7 @@ mod tests {
|
||||||
assert!(is_localhost_origin("http://localhost:3000"));
|
assert!(is_localhost_origin("http://localhost:3000"));
|
||||||
assert!(is_localhost_origin("https://localhost:8443"));
|
assert!(is_localhost_origin("https://localhost:8443"));
|
||||||
assert!(is_localhost_origin("http://127.0.0.1"));
|
assert!(is_localhost_origin("http://127.0.0.1"));
|
||||||
assert!(is_localhost_origin("http://127.0.0.1:8080"));
|
assert!(is_localhost_origin("http://127.0.0.1:9000"));
|
||||||
assert!(!is_localhost_origin("http://example.com"));
|
assert!(!is_localhost_origin("http://example.com"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue