diff --git a/src/auto_task/intent_classifier.rs b/src/auto_task/intent_classifier.rs index 2fd86012..8ea7ab1a 100644 --- a/src/auto_task/intent_classifier.rs +++ b/src/auto_task/intent_classifier.rs @@ -721,8 +721,8 @@ END ON context_vars.insert("original_intent".to_string(), compiled.original_intent.clone()); script_service.inject_config_variables(context_vars); - // Compile and execute the BASIC program - let ast = match script_service.compile(basic_program) { + // Compile and execute dynamically generated BASIC program + let ast = match script_service.engine.compile(basic_program) { Ok(ast) => ast, Err(e) => { let error_msg = format!("Failed to compile BASIC program: {}", e); @@ -746,7 +746,7 @@ END ON } }; - let execution_result = script_service.run(&ast); + let execution_result: Result> = script_service.engine.eval_ast_with_scope(&mut script_service.scope, &ast); match execution_result { Ok(result) => { diff --git a/src/basic/compiler/mod.rs b/src/basic/compiler/mod.rs index 5eb15734..ceca63ec 100644 --- a/src/basic/compiler/mod.rs +++ b/src/basic/compiler/mod.rs @@ -448,6 +448,9 @@ impl BasicCompiler { source.to_string() }; let source = source.as_str(); + + // Preprocess LLM keyword to add WITH OPTIMIZE FOR "speed" syntax + let source = crate::basic::ScriptService::preprocess_llm_keyword(source); let mut has_schedule = false; let script_name = Path::new(source_path) .file_stem() @@ -615,6 +618,12 @@ impl BasicCompiler { }; // Convert BEGIN TALK and BEGIN MAIL blocks to Rhai code let result = crate::basic::compiler::blocks::convert_begin_blocks(&result); + // Convert ALL multi-word keywords to underscore versions (e.g., "USE KB" → "USE_KB") + let result = crate::basic::ScriptService::convert_multiword_keywords(&result); + // Convert WHILE...WEND to Rhai while { } blocks BEFORE if/then conversion + let result = crate::basic::ScriptService::convert_while_wend_syntax(&result); + // Pre-declare all variables at outer scope so assignments inside blocks work correctly + let result = crate::basic::ScriptService::predeclare_variables(&result); // Convert IF ... THEN / END IF to if ... { } let result = crate::basic::ScriptService::convert_if_then_syntax(&result); // Convert SELECT ... CASE / END SELECT to match expressions diff --git a/src/basic/mod.rs b/src/basic/mod.rs index 2bfde63a..38297dbc 100644 --- a/src/basic/mod.rs +++ b/src/basic/mod.rs @@ -6,7 +6,6 @@ use crate::basic::keywords::switch_case::switch_keyword; use crate::core::shared::models::UserSession; use crate::core::shared::state::AppState; use diesel::prelude::*; -use log::trace; use rhai::{Dynamic, Engine, EvalAltResult, Scope}; use std::collections::HashMap; use std::sync::Arc; @@ -47,7 +46,6 @@ use self::keywords::http_operations::register_http_operations; use self::keywords::last::last_keyword; #[cfg(feature = "automation")] use self::keywords::on_form_submit::on_form_submit_keyword; -use self::keywords::switch_case::preprocess_switch; use self::keywords::use_tool::use_tool_keyword; use self::keywords::use_website::{clear_websites_keyword, register_use_website_function}; use self::keywords::web_data::register_web_data_keywords; @@ -321,334 +319,20 @@ impl ScriptService { } } } - fn preprocess_basic_script(&self, script: &str) -> Result { - let _ = self; // silence unused self warning - kept for API consistency - let script = preprocess_switch(script); - - // Preprocess LLM keyword to add WITH OPTIMIZE FOR "speed" syntax - // This is needed because Rhai's custom syntax requires the full syntax - let script = Self::preprocess_llm_keyword(&script); - - // Convert ALL multi-word keywords to underscore versions (e.g., "USE WEBSITE" → "USE_WEBSITE") - // This avoids Rhai custom syntax conflicts and makes the system more secure - let script = Self::convert_multiword_keywords(&script); - - let script = Self::normalize_variables_to_lowercase(&script); - - let mut result = String::new(); - let mut for_stack: Vec = Vec::new(); - let mut current_indent = 0; - for line in script.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with("//") || trimmed.starts_with('\'') { - continue; - } - if trimmed.starts_with("FOR EACH") { - for_stack.push(current_indent); - result.push_str(&" ".repeat(current_indent)); - result.push_str(trimmed); - result.push_str("{\n"); - current_indent += 4; - result.push_str(&" ".repeat(current_indent)); - result.push('\n'); - continue; - } - if trimmed.starts_with("NEXT") { - if let Some(expected_indent) = for_stack.pop() { - if (current_indent - 4) != expected_indent { - return Err("NEXT without matching FOR EACH (indentation mismatch)".to_string()); - } - current_indent -= 4; - result.push_str(&" ".repeat(current_indent)); - result.push_str("}\n"); - result.push_str(&" ".repeat(current_indent)); - result.push_str(trimmed); - result.push(';'); - result.push('\n'); - continue; - } - log::error!("NEXT without matching FOR EACH"); - return Err("NEXT without matching FOR EACH".to_string()); - } - if trimmed == "EXIT FOR" { - result.push_str(&" ".repeat(current_indent)); - result.push_str(trimmed); - result.push('\n'); - continue; - } - result.push_str(&" ".repeat(current_indent)); - let basic_commands = [ - "SET", - "CREATE", - "PRINT", - "FOR", - "FIND", - "GET", - "EXIT", - "IF", - "THEN", - "ELSE", - "END IF", - "WHILE", - "WEND", - "DO", - "LOOP", - "HEAR", - "TALK", - "SET CONTEXT", - "SET USER", - "GET BOT MEMORY", - "SET BOT MEMORY", - "IMAGE", - "VIDEO", - "AUDIO", - "SEE", - "SEND FILE", - "SWITCH", - "CASE", - "DEFAULT", - "END SWITCH", - "USE KB", - "CLEAR KB", - "USE TOOL", - "CLEAR TOOLS", - "ADD SUGGESTION", - "CLEAR SUGGESTIONS", - "INSTR", - "IS_NUMERIC", - "IS NUMERIC", - "POST", - "PUT", - "PATCH", - "DELETE", - "SET HEADER", - "CLEAR HEADERS", - "GRAPHQL", - "SOAP", - "SAVE", - "INSERT", - "UPDATE", - "DELETE", - "MERGE", - "FILL", - "MAP", - "FILTER", - "AGGREGATE", - "JOIN", - "PIVOT", - "GROUP BY", - "READ", - "WRITE", - "COPY", - "MOVE", - "LIST", - "COMPRESS", - "EXTRACT", - "UPLOAD", - "DOWNLOAD", - "GENERATE PDF", - "MERGE PDF", - "WEBHOOK", - "POST TO", - "POST TO INSTAGRAM", - "POST TO FACEBOOK", - "POST TO LINKEDIN", - "POST TO TWITTER", - "GET INSTAGRAM METRICS", - "GET FACEBOOK METRICS", - "GET LINKEDIN METRICS", - "GET TWITTER METRICS", - "DELETE POST", - "SEND MAIL", - "SEND TEMPLATE", - "CREATE TEMPLATE", - "GET TEMPLATE", - "ON ERROR RESUME NEXT", - "ON ERROR GOTO", - "CLEAR ERROR", - "ERROR MESSAGE", - "ON FORM SUBMIT", - "SCORE LEAD", - "GET LEAD SCORE", - "QUALIFY LEAD", - "UPDATE LEAD SCORE", - "AI SCORE LEAD", - "ABS", - "ROUND", - "INT", - "FIX", - "FLOOR", - "CEIL", - "MAX", - "MIN", - "MOD", - "RANDOM", - "RND", - "SGN", - "SQR", - "SQRT", - "LOG", - "EXP", - "POW", - "SIN", - "COS", - "TAN", - "SUM", - "AVG", - "NOW", - "TODAY", - "DATE", - "TIME", - "YEAR", - "MONTH", - "DAY", - "HOUR", - "MINUTE", - "SECOND", - "WEEKDAY", - "DATEADD", - "DATEDIFF", - "FORMAT_DATE", - "ISDATE", - "VAL", - "STR", - "CINT", - "CDBL", - "CSTR", - "ISNULL", - "ISEMPTY", - "TYPEOF", - "ISARRAY", - "ISOBJECT", - "ISSTRING", - "ISNUMBER", - "NVL", - "IIF", - "ARRAY", - "UBOUND", - "LBOUND", - "COUNT", - "SORT", - "UNIQUE", - "CONTAINS", - "INDEX_OF", - "PUSH", - "POP", - "SHIFT", - "REVERSE", - "SLICE", - "SPLIT", - "CONCAT", - "FLATTEN", - "RANGE", - "THROW", - "ERROR", - "IS_ERROR", - "ASSERT", - "LOG_ERROR", - "LOG_WARN", - "LOG_INFO", - ]; - let is_basic_command = basic_commands.iter().any(|&cmd| trimmed.starts_with(cmd)); - let is_control_flow = trimmed.starts_with("IF") - || trimmed.starts_with("ELSE") - || trimmed.starts_with("END IF"); - result.push_str(trimmed); - let needs_semicolon = is_basic_command - || !for_stack.is_empty() - || is_control_flow - || (!trimmed.ends_with(';') && !trimmed.ends_with('{') && !trimmed.ends_with('}')); - if needs_semicolon { - result.push(';'); - } - result.push('\n'); - } - if !for_stack.is_empty() { - return Err("Unclosed FOR EACH loop".to_string()); - } - Ok(result) - } - pub fn compile(&self, script: &str) -> Result> { - let processed_script = match self.preprocess_basic_script(script) { - Ok(s) => s, - Err(e) => return Err(Box::new(EvalAltResult::ErrorRuntime(Dynamic::from(e), rhai::Position::NONE))), + /// Run a pre-compiled .ast script (loaded from Drive). + /// Compilation happens only in BasicCompiler (Drive Monitor). + /// Runtime only compiles the already-preprocessed Rhai source and executes it. + pub fn run(&mut self, ast_content: &str) -> Result> { + let ast = match self.engine.compile(ast_content) { + Ok(ast) => ast, + Err(e) => return Err(Box::new(e.into())), }; - trace!("Processed Script:\n{}", processed_script); - match self.engine.compile(&processed_script) { - Ok(ast) => Ok(ast), - Err(parse_error) => Err(Box::new(parse_error.into())), - } - } - - /// Compile preprocessed script content (from .ast file) - skips preprocessing - pub fn compile_preprocessed(&self, script: &str) -> Result> { - trace!("Compiling preprocessed script directly"); - match self.engine.compile(script) { - Ok(ast) => Ok(ast), - Err(parse_error) => Err(Box::new(parse_error.into())), - } - } - - /// Compile a tool script (.bas file with PARAM/DESCRIPTION metadata lines) - /// Filters out tool metadata before compiling - pub fn compile_tool_script(&self, script: &str) -> Result> { - // Filter out PARAM, DESCRIPTION, comment, and empty lines (tool metadata) - let executable_script: String = script - .lines() - .filter(|line| { - let trimmed = line.trim(); - // Keep lines that are NOT PARAM, DESCRIPTION, comments, or empty - !(trimmed.starts_with("PARAM ") || - trimmed.starts_with("PARAM\t") || - trimmed.starts_with("DESCRIPTION ") || - trimmed.starts_with("DESCRIPTION\t") || - trimmed.starts_with("REM ") || - trimmed.starts_with("REM\t") || - trimmed == "REM" || // bare REM line - trimmed.starts_with('\'') || // BASIC comment lines - trimmed.starts_with('#') || // Hash comment lines - trimmed.is_empty()) - }) - .collect::>() - .join("\n"); - - trace!("Filtered tool metadata: {} -> {} chars", script.len(), executable_script.len()); - - // Apply minimal preprocessing for tools (skip variable normalization to avoid breaking multi-line strings) - let script = preprocess_switch(&executable_script); - let script = Self::convert_multiword_keywords(&script); - // Skip normalize_variables_to_lowercase for tools - it breaks multi-line strings - - trace!("Preprocessed tool script for Rhai compilation"); - // Convert SAVE statements with field lists to map-based SAVE (simplified version for tools) - let script = Self::convert_save_for_tools(&script); - // Convert BEGIN TALK and BEGIN MAIL blocks to single calls - let script = crate::basic::compiler::blocks::convert_begin_blocks(&script); - // Convert WHILE...WEND to Rhai while { } blocks BEFORE if/then conversion - let script = Self::convert_while_wend_syntax(&script); - // Pre-declare all variables at outer scope so assignments inside blocks work correctly. - // In Rhai, a plain `x = val` inside a block updates the outer variable - - // but only if `x` was declared outside with `let`. - let script = Self::predeclare_variables(&script); - // Convert IF ... THEN / END IF to if ... { } - let script = Self::convert_if_then_syntax(&script); - // Convert SELECT ... CASE / END SELECT to match expressions - let script = Self::convert_select_case_syntax(&script); - // Convert BASIC keywords to lowercase (but preserve variable casing) - let script = Self::convert_keywords_to_lowercase(&script); - // Save to file for debugging - if let Err(e) = std::fs::write("/tmp/tool_preprocessed.bas", &script) { - log::warn!("Failed to write preprocessed script: {}", e); - } - match self.engine.compile(&script) { - Ok(ast) => Ok(ast), - Err(parse_error) => Err(Box::new(parse_error.into())), - } + self.engine.eval_ast_with_scope(&mut self.scope, &ast) } /// Pre-declare all BASIC variables at the top of the script with `let var = ();`. /// This allows assignments inside loops/if-blocks to update outer-scope variables in Rhai. - fn predeclare_variables(script: &str) -> String { + pub(crate) fn predeclare_variables(script: &str) -> String { use std::collections::BTreeSet; let reserved: std::collections::HashSet<&str> = [ "if", "else", "while", "for", "loop", "return", "break", "continue", @@ -701,114 +385,23 @@ impl ScriptService { declarations.push_str(script); declarations } - pub fn run(&mut self, ast: &rhai::AST) -> Result> { - self.engine.eval_ast_with_scope(&mut self.scope, ast) - } - /// Execute a BASIC script asynchronously + /// Execute a pre-compiled .ast script asynchronously pub async fn execute_script( state: Arc, user: UserSession, - script: &str, + ast_content: &str, ) -> Result { let mut script_service = Self::new(state.clone(), user.clone()); script_service.load_bot_config_params(&state, user.bot_id); - - match script_service.compile(script) { - Ok(ast) => { - match script_service.run(&ast) { - Ok(result) => Ok(result.to_string()), - Err(e) => Err(format!("Execution error: {}", e)), - } - } - Err(e) => Err(format!("Compilation error: {}", e)), + + match script_service.run(ast_content) { + Ok(result) => Ok(result.to_string()), + Err(e) => Err(format!("Script error: {}", e)), } } - /// Convert SAVE statements for tool compilation (simplified, no DB lookup) - /// SAVE "table", var1, var2, ... -> let __data__ = #{var1: var1, var2: var2, ...}; SAVE "table", __data__ - fn convert_save_for_tools(script: &str) -> String { - let mut result = String::new(); - let mut save_counter = 0; - - for line in script.lines() { - let trimmed = line.trim(); - - // Check if this is a SAVE statement - if trimmed.to_uppercase().starts_with("SAVE ") { - // Parse SAVE statement - // Format: SAVE "table", value1, value2, ... - let content = &trimmed[4..].trim(); - - // Simple parse by splitting on commas (outside quotes) - let parts = Self::parse_save_parts(content); - - // If more than 2 parts, convert to map-based SAVE - if parts.len() > 2 { - let table_name = parts[0].trim_matches('"'); - let values: Vec<&str> = parts.iter().skip(1).map(|s| s.trim()).collect(); - - // Build map with variable names as keys - let map_pairs: Vec = values.iter().map(|v| format!("{}: {}", v, v)).collect(); - let map_expr = format!("#{{{}}}", map_pairs.join(", ")); - let data_var = format!("__save_data_{}__", save_counter); - save_counter += 1; - - let converted = format!("let {} = {};\nINSERT \"{}\", {};", data_var, map_expr, table_name, data_var); - result.push_str(&converted); - result.push('\n'); - continue; - } - } - - result.push_str(line); - result.push('\n'); - } - - result - } - - /// Parse SAVE statement parts (handles quoted strings) - fn parse_save_parts(s: &str) -> Vec { - let mut parts = Vec::new(); - let mut current = String::new(); - let mut in_quotes = false; - let mut chars = s.chars().peekable(); - - while let Some(c) = chars.next() { - match c { - '"' if !in_quotes => { - in_quotes = true; - current.push(c); - } - '"' if in_quotes => { - in_quotes = false; - current.push(c); - } - ',' if !in_quotes => { - parts.push(current.trim().to_string()); - current = String::new(); - // Skip whitespace after comma - while let Some(&next_c) = chars.peek() { - if next_c.is_whitespace() { - chars.next(); - } else { - break; - } - } - } - _ => current.push(c), - } - } - - if !current.is_empty() { - parts.push(current.trim().to_string()); - } - - parts - } - - /// Set a variable in the script scope (for tool parameters) + /// Pre-declare all BASIC variables at the top of the script with `let var = ();`. pub fn set_variable(&mut self, name: &str, value: &str) -> Result<(), Box> { use rhai::Dynamic; self.scope.set_or_push(name, Dynamic::from(value.to_string())); @@ -1682,312 +1275,6 @@ impl ScriptService { result } - fn normalize_variables_to_lowercase(script: &str) -> String { - use regex::Regex; - - let mut result = String::new(); - - let keywords = [ - "SET", - "CREATE", - "PRINT", - "FOR", - "FIND", - "GET", - "EXIT", - "IF", - "THEN", - "ELSE", - "END", - "WHILE", - "WEND", - "DO", - "LOOP", - "HEAR", - "TALK", - "NEXT", - "FUNCTION", - "SUB", - "CALL", - "RETURN", - "DIM", - "AS", - "NEW", - "ARRAY", - "OBJECT", - "LET", - "REM", - "AND", - "OR", - "NOT", - "TRUE", - "FALSE", - "NULL", - "SWITCH", - "CASE", - "DEFAULT", - "USE", - "KB", - "TOOL", - "CLEAR", - "ADD", - "SUGGESTION", - "SUGGESTIONS", - "TOOLS", - "CONTEXT", - "USER", - "BOT", - "MEMORY", - "IMAGE", - "VIDEO", - "AUDIO", - "SEE", - "SEND", - "FILE", - "POST", - "PUT", - "PATCH", - "DELETE", - "SAVE", - "INSERT", - "UPDATE", - "MERGE", - "FILL", - "MAP", - "FILTER", - "AGGREGATE", - "JOIN", - "PIVOT", - "GROUP", - "BY", - "READ", - "WRITE", - "COPY", - "MOVE", - "LIST", - "COMPRESS", - "EXTRACT", - "UPLOAD", - "DOWNLOAD", - "GENERATE", - "PDF", - "WEBHOOK", - "TEMPLATE", - "FORM", - "SUBMIT", - "SCORE", - "LEAD", - "QUALIFY", - "AI", - "ABS", - "ROUND", - "INT", - "FIX", - "FLOOR", - "CEIL", - "MAX", - "MIN", - "MOD", - "RANDOM", - "RND", - "SGN", - "SQR", - "SQRT", - "LOG", - "EXP", - "POW", - "SIN", - "COS", - "TAN", - "SUM", - "AVG", - "NOW", - "TODAY", - "DATE", - "TIME", - "YEAR", - "MONTH", - "DAY", - "HOUR", - "MINUTE", - "SECOND", - "WEEKDAY", - "DATEADD", - "DATEDIFF", - "FORMAT", - "ISDATE", - "VAL", - "STR", - "CINT", - "CDBL", - "CSTR", - "ISNULL", - "ISEMPTY", - "TYPEOF", - "ISARRAY", - "ISOBJECT", - "ISSTRING", - "ISNUMBER", - "NVL", - "IIF", - "UBOUND", - "LBOUND", - "COUNT", - "SORT", - "UNIQUE", - "CONTAINS", - "INDEX", - "OF", - "PUSH", - "POP", - "SHIFT", - "REVERSE", - "SLICE", - "SPLIT", - "CONCAT", - "FLATTEN", - "RANGE", - "THROW", - "ERROR", - "IS", - "ASSERT", - "WARN", - "INFO", - "EACH", - "WITH", - "TO", - "STEP", - "BEGIN", - "SYSTEM", - "PROMPT", - "SCHEDULE", - "REFRESH", - "ALLOW", - "ROLE", - "ANSWER", - "MODE", - "SYNCHRONIZE", - "TABLE", - "ON", - "EMAIL", - "REPORT", - "RESET", - "WAIT", - "FIRST", - "LAST", - "LLM", - "INSTR", - "NUMERIC", - "LEN", - "LEFT", - "RIGHT", - "MID", - "LOWER", - "UPPER", - "TRIM", - "LTRIM", - "RTRIM", - "REPLACE", - "LIKE", - "DELEGATE", - "PRIORITY", - "BOTS", - "REMOVE", - "MEMBER", - "BOOK", - "REMEMBER", - "TASK", - "SITE", - "DRAFT", - "INSTAGRAM", - "FACEBOOK", - "LINKEDIN", - "TWITTER", - "METRICS", - "HEADER", - "HEADERS", - "GRAPHQL", - "SOAP", - "HTTP", - "DESCRIPTION", - "PARAM", - "REQUIRED", - "WEBSITE", - "MODEL", - "DETECT", - "LLM", - "TALK", - ]; - - let _identifier_re = Regex::new(r"([a-zA-Z_][a-zA-Z0-9_]*)").expect("valid regex"); - - for line in script.lines() { - let trimmed = line.trim(); - - if trimmed.starts_with("REM") || trimmed.starts_with('\'') || trimmed.starts_with("//") - { - continue; - } - - // Skip lines with custom syntax that should not be lowercased - // These are registered directly with Rhai in uppercase - let trimmed_upper = trimmed.to_uppercase(); - if trimmed_upper.contains("ADD_SUGGESTION_TOOL") || - trimmed_upper.contains("ADD_SUGGESTION_TEXT") || - trimmed_upper.starts_with("ADD_SUGGESTION_") || - trimmed_upper.starts_with("ADD_MEMBER") { - // Keep original line as-is - result.push_str(line); - result.push('\n'); - continue; - } - - let mut processed_line = String::new(); - let mut chars = line.chars().peekable(); - let mut in_string = false; - let mut string_char = '"'; - let mut current_word = String::new(); - - while let Some(c) = chars.next() { - if in_string { - processed_line.push(c); - if c == string_char { - in_string = false; - } else if c == '\\' { - if let Some(&next) = chars.peek() { - processed_line.push(next); - chars.next(); - } - } - } else if c == '"' || c == '\'' { - if !current_word.is_empty() { - processed_line.push_str(&Self::normalize_word(¤t_word, &keywords)); - current_word.clear(); - } - in_string = true; - string_char = c; - processed_line.push(c); - } else if c.is_alphanumeric() || c == '_' { - current_word.push(c); - } else { - if !current_word.is_empty() { - processed_line.push_str(&Self::normalize_word(¤t_word, &keywords)); - current_word.clear(); - } - processed_line.push(c); - } - } - - if !current_word.is_empty() { - processed_line.push_str(&Self::normalize_word(¤t_word, &keywords)); - } - - result.push_str(&processed_line); - result.push('\n'); - } - - result - } /// Convert ALL multi-word keywords to underscore versions (function calls) /// This avoids Rhai custom syntax conflicts and makes the system more secure @@ -2163,24 +1450,7 @@ impl ScriptService { params } - fn normalize_word(word: &str, keywords: &[&str]) -> String { - let upper = word.to_uppercase(); - - if keywords.contains(&upper.as_str()) { - upper - } else if word - .chars() - .next() - .map(|c| c.is_ascii_digit()) - .unwrap_or(false) - { - word.to_string() - } else { - word.to_lowercase() - } - } - - fn preprocess_llm_keyword(script: &str) -> String { + pub(crate) fn preprocess_llm_keyword(script: &str) -> String { // Transform LLM "prompt" to LLM "prompt" WITH OPTIMIZE FOR "speed" // Handle cases like: // LLM "text" diff --git a/src/core/automation/mod.rs b/src/core/automation/mod.rs index 79b38b94..1ed5b2d5 100644 --- a/src/core/automation/mod.rs +++ b/src/core/automation/mod.rs @@ -145,14 +145,10 @@ impl AutomationService { script_service.load_bot_config_params(&self.state, automation.bot_id); - match script_service.compile(&script_content) { - Ok(ast) => { - if let Err(e) = script_service.run(&ast) { - error!("Script execution failed: {}", e); - } - } + match script_service.run(&script_content) { + Ok(_) => {} Err(e) => { - error!("Script compilation failed: {}", e); + error!("Script execution failed: {}", e); } } Ok(()) diff --git a/src/core/bot/mod.rs b/src/core/bot/mod.rs index 8b5674b2..09f1a009 100644 --- a/src/core/bot/mod.rs +++ b/src/core/bot/mod.rs @@ -634,78 +634,73 @@ impl BotOrchestrator { trace!("Executing start.bas for session {} at: {}", actual_session_id, start_script_path); - // Use pre-compiled .ast if available (avoids preprocessing) + // Load pre-compiled .ast only (compilation happens in Drive Monitor) let ast_path = start_script_path.replace(".bas", ".ast"); - let (script_content, is_preprocessed) = if std::path::Path::new(&ast_path).exists() { - (tokio::fs::read_to_string(&ast_path).await.unwrap_or_default(), true) - } else { - (tokio::fs::read_to_string(&start_script_path).await.unwrap_or_default(), false) + let ast_content = match tokio::fs::read_to_string(&ast_path).await { + Ok(content) if !content.is_empty() => content, + _ => { + let content = tokio::fs::read_to_string(&start_script_path).await.unwrap_or_default(); + if content.is_empty() { + trace!("No start.bas/start.ast found for bot {}", bot_name_for_context); + return Ok(()); + } + content + } }; - if !script_content.is_empty() { - let state_clone = self.state.clone(); - let actual_session_id_for_task = session.id; - let bot_id_clone = session.bot_id; + let state_clone = self.state.clone(); + let actual_session_id_for_task = session.id; + let bot_id_clone = session.bot_id; - // Execute start.bas synchronously (blocking) - let result = tokio::task::spawn_blocking(move || { - let session_result = { - let mut sm = state_clone.session_manager.blocking_lock(); - sm.get_session_by_id(actual_session_id_for_task) - }; + // Execute start.bas synchronously (blocking) + let result = tokio::task::spawn_blocking(move || { + let session_result = { + let mut sm = state_clone.session_manager.blocking_lock(); + sm.get_session_by_id(actual_session_id_for_task) + }; - let sess = match session_result { - Ok(Some(s)) => s, - Ok(None) => { - return Err(format!("Session {} not found during start.bas execution", actual_session_id_for_task)); - } - Err(e) => return Err(format!("Failed to get session: {}", e)), - }; - - let mut script_service = crate::basic::ScriptService::new( - state_clone.clone(), - sess - ); - script_service.load_bot_config_params(&state_clone, bot_id_clone); - - let compile_result = if is_preprocessed { - script_service.compile_preprocessed(&script_content) - } else { - script_service.compile(&script_content) - }; - - match compile_result { - Ok(ast) => match script_service.run(&ast) { - Ok(_) => Ok(()), - Err(e) => Err(format!("Script execution error: {}", e)), - }, - Err(e) => Err(format!("Script compilation error: {}", e)), + let sess = match session_result { + Ok(Some(s)) => s, + Ok(None) => { + return Err(format!("Session {} not found during start.bas execution", actual_session_id_for_task)); } - }).await; + Err(e) => return Err(format!("Failed to get session: {}", e)), + }; - match result { - Ok(Ok(())) => { - trace!("start.bas completed successfully for session {}", actual_session_id); + let mut script_service = crate::basic::ScriptService::new( + state_clone.clone(), + sess + ); + script_service.load_bot_config_params(&state_clone, bot_id_clone); - // Mark start.bas as executed for this session to prevent re-running - if let Some(cache) = &self.state.cache { - if let Ok(mut conn) = cache.get_multiplexed_async_connection().await { - let _: Result<(), redis::RedisError> = redis::cmd("SET") - .arg(&start_bas_key) - .arg("1") - .arg("EX") - .arg("86400") // Expire after 24 hours - .query_async(&mut conn) - .await; - } + match script_service.run(&ast_content) { + Ok(_) => Ok(()), + Err(e) => Err(format!("Script execution error: {}", e)), + } + }).await; + + match result { + Ok(Ok(())) => { + trace!("start.bas completed successfully for session {}", actual_session_id); + + // Mark start.bas as executed for this session to prevent re-running + if let Some(cache) = &self.state.cache { + if let Ok(mut conn) = cache.get_multiplexed_async_connection().await { + let _: Result<(), redis::RedisError> = redis::cmd("SET") + .arg(&start_bas_key) + .arg("1") + .arg("EX") + .arg("86400") // Expire after 24 hours + .query_async(&mut conn) + .await; } } - Ok(Err(e)) => { - error!("start.bas error for session {}: {}", actual_session_id, e); - } - Err(e) => { - error!("start.bas task error for session {}: {}", actual_session_id, e); - } + } + Ok(Err(e)) => { + error!("start.bas error for session {}: {}", actual_session_id, e); + } + Err(e) => { + error!("start.bas task error for session {}: {}", actual_session_id, e); } } } // End of if should_execute_start_bas @@ -1147,13 +1142,17 @@ impl BotOrchestrator { // DEBUG: Log LLM output for troubleshooting HTML rendering issues let has_html = full_response.contains(" 500 { - format!("{}... ({} chars total)", &full_response[..500], full_response.len()) + let has_div = full_response.contains(""); + let has_style = full_response.contains("") && has_div; + let preview = if full_response.len() > 800 { + format!("{}... ({} chars total)", &full_response[..800], full_response.len()) } else { full_response.clone() }; - info!("[LLM_OUTPUT] session={} has_html={} preview=\"{}\"", - session_id, has_html, preview.replace('\n', "\\n")); + info!("[LLM_OUTPUT] session={} has_html={} has_div={} has_style={} is_truncated={} len={} preview=\"{}\"", + session_id, has_html, has_div, has_style, is_truncated, full_response.len(), + preview.replace('\n', "\\n")); trace!("LLM stream complete. Full response: {}", full_response); @@ -1433,27 +1432,22 @@ async fn handle_websocket( info!("Looking for start.bas at: {}", start_script_path); - // Check for pre-compiled .ast file first (avoids preprocessing overhead) + // Load pre-compiled .ast only (compilation happens in Drive Monitor) let ast_path = start_script_path.replace(".bas", ".ast"); - let (script_content, is_preprocessed) = if tokio::fs::metadata(&ast_path).await.is_ok() { - if let Ok(content) = tokio::fs::read_to_string(&ast_path).await { - info!("Using pre-compiled start.ast for {}", bot_name); - (content, true) - } else { - (String::new(), false) + let ast_content = match tokio::fs::read_to_string(&ast_path).await { + Ok(content) if !content.is_empty() => content, + _ => { + let content = tokio::fs::read_to_string(&start_script_path).await.unwrap_or_default(); + if content.is_empty() { + info!("No start.bas/start.ast found for bot {}", bot_name); + String::new() + } else { + content + } } - } else if tokio::fs::metadata(&start_script_path).await.is_ok() { - if let Ok(content) = tokio::fs::read_to_string(&start_script_path).await { - info!("Compiling start.bas for {}", bot_name); - (content, false) - } else { - (String::new(), false) - } - } else { - (String::new(), false) }; - if !script_content.is_empty() { + if !ast_content.is_empty() { info!( "Executing start.bas for bot {} on session {}", bot_name, session_id @@ -1464,8 +1458,6 @@ async fn handle_websocket( let bot_id_str = bot_id.to_string(); let session_id_str = session_id.to_string(); let mut send_ready_rx = send_ready_rx; - let script_content_owned = script_content.clone(); - let is_preprocessed_owned = is_preprocessed; tokio::spawn(async move { let _ = send_ready_rx.recv().await; @@ -1502,18 +1494,9 @@ async fn handle_websocket( ); script_service.load_bot_config_params(&state_for_start, bot_id); - let compile_result = if is_preprocessed_owned { - script_service.compile_preprocessed(&script_content_owned) - } else { - script_service.compile(&script_content_owned) - }; - - match compile_result { - Ok(ast) => match script_service.run(&ast) { - Ok(_) => Ok(()), - Err(e) => Err(format!("Script execution error: {}", e)), - }, - Err(e) => Err(format!("Script compilation error: {}", e)), + match script_service.run(&ast_content) { + Ok(_) => Ok(()), + Err(e) => Err(format!("Script execution error: {}", e)), } }).await; diff --git a/src/core/bot/tool_executor.rs b/src/core/bot/tool_executor.rs index f9e8da01..8c54f9b3 100644 --- a/src/core/bot/tool_executor.rs +++ b/src/core/bot/tool_executor.rs @@ -157,46 +157,25 @@ impl ToolExecutor { } }; - // Load the .bas tool file (prefer pre-compiled .ast if available) - let bas_path = Self::get_tool_bas_path(bot_name, &tool_call.tool_name); - let ast_path = bas_path.with_extension("ast"); + // Load the pre-compiled .ast file (compilation happens only in Drive Monitor) + let ast_path = Self::get_tool_ast_path(bot_name, &tool_call.tool_name); - // Check for .ast first (pre-compiled), fallback to .bas - let (script_content, is_preprocessed) = if ast_path.exists() { - match tokio::fs::read_to_string(&ast_path).await { - Ok(script) => { - trace!("Using pre-compiled .ast for tool: {}", tool_call.tool_name); - (script, true) - } - Err(_) => (String::new(), false) + let ast_content = match tokio::fs::read_to_string(&ast_path).await { + Ok(content) => content, + Err(e) => { + let error_msg = format!("Failed to read tool .ast file {}: {}", ast_path.display(), e); + Self::log_tool_error(bot_name, &tool_call.tool_name, &error_msg); + return ToolExecutionResult { + tool_call_id: tool_call.id.clone(), + success: false, + result: String::new(), + error: Some(Self::format_user_friendly_error(&tool_call.tool_name, &error_msg)), + }; } - } else if bas_path.exists() { - match tokio::fs::read_to_string(&bas_path).await { - Ok(script) => (script, false), - Err(e) => { - let error_msg = format!("Failed to read tool file: {}", e); - Self::log_tool_error(bot_name, &tool_call.tool_name, &error_msg); - return ToolExecutionResult { - tool_call_id: tool_call.id.clone(), - success: false, - result: String::new(), - error: Some(Self::format_user_friendly_error(&tool_call.tool_name, &error_msg)), - }; - } - } - } else { - let error_msg = format!("Tool file not found: {:?}", bas_path); - Self::log_tool_error(bot_name, &tool_call.tool_name, &error_msg); - return ToolExecutionResult { - tool_call_id: tool_call.id.clone(), - success: false, - result: String::new(), - error: Some(Self::format_user_friendly_error(&tool_call.tool_name, &error_msg)), - }; }; - if script_content.is_empty() { - let error_msg = "Tool script is empty".to_string(); + if ast_content.is_empty() { + let error_msg = "Tool .ast file is empty".to_string(); Self::log_tool_error(bot_name, &tool_call.tool_name, &error_msg); return ToolExecutionResult { tool_call_id: tool_call.id.clone(), @@ -245,10 +224,9 @@ impl ToolExecutor { &bot_name_clone, bot_id_clone, &session, - &script_content, + &ast_content, &tool_name_clone, &arguments_clone, - is_preprocessed, ) }) .await; @@ -274,10 +252,9 @@ impl ToolExecutor { bot_name: &str, bot_id: Uuid, session: &crate::core::shared::models::UserSession, - script_content: &str, + ast_content: &str, tool_name: &str, arguments: &Value, - is_preprocessed: bool, ) -> ToolExecutionResult { let tool_call_id = format!("tool_{}", uuid::Uuid::new_v4()); @@ -304,30 +281,8 @@ impl ToolExecutor { } } - // Compile: use compile_preprocessed for .ast files, compile_tool_script for .bas - let ast = if is_preprocessed { - script_service.compile_preprocessed(script_content) - } else { - script_service.compile_tool_script(script_content) - }; - - let ast = match ast { - Ok(ast) => ast, - Err(e) => { - let error_msg = format!("Compilation error: {}", e); - Self::log_tool_error(bot_name, tool_name, &error_msg); - let user_message = Self::format_user_friendly_error(tool_name, &error_msg); - return ToolExecutionResult { - tool_call_id, - success: false, - result: String::new(), - error: Some(user_message), - }; - } - }; - - // Run the script - match script_service.run(&ast) { + // Run the pre-compiled .ast content (compilation happens only in Drive Monitor) + match script_service.run(ast_content) { Ok(result) => { trace!("Tool '{}' executed successfully", tool_name); @@ -365,13 +320,12 @@ impl ToolExecutor { .ok() } - /// Get the path to a tool's .bas file - fn get_tool_bas_path(bot_name: &str, tool_name: &str) -> std::path::PathBuf { - // Use work directory for compiled .bas files + /// Get the path to a tool's pre-compiled .ast file + fn get_tool_ast_path(bot_name: &str, tool_name: &str) -> std::path::PathBuf { let work_path = std::path::PathBuf::from(crate::core::shared::utils::get_work_path()) .join(format!("{}.gbai", bot_name)) .join(format!("{}.gbdialog", bot_name)) - .join(format!("{}.bas", tool_name)); + .join(format!("{}.ast", tool_name)); work_path } diff --git a/src/directory/mod.rs b/src/directory/mod.rs index 56942c3e..2401181e 100644 --- a/src/directory/mod.rs +++ b/src/directory/mod.rs @@ -213,13 +213,11 @@ pub async fn auth_handler( let mut script_service = crate::basic::ScriptService::new(state_clone.clone(), session_clone); - script_service.load_bot_config_params(&state_clone, bot_id); - match script_service.compile(&auth_script) { - Ok(ast) => match script_service.run(&ast) { - Ok(_) => Ok(()), - Err(e) => Err(format!("Script execution error: {}", e)), - }, - Err(e) => Err(format!("Script compilation error: {}", e)), +script_service.load_bot_config_params(&state_clone, bot_id); + + match script_service.run(&auth_script) { + Ok(_) => Ok(()), + Err(e) => Err(format!("Script execution error: {}", e)), } }) .await diff --git a/src/llm/mod.rs b/src/llm/mod.rs index 9b9d514c..72631637 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use futures::StreamExt; -use log::{error, info}; +use log::{error, info, trace}; use serde_json::Value; use std::sync::Arc; use tokio::sync::{mpsc, RwLock}; @@ -413,7 +413,8 @@ impl LLMProvider for OpenAIClient { let mut request_body = serde_json::json!({ "model": model, "messages": messages, - "stream": true + "stream": true, + "max_tokens": 16384 }); // Add tools to the request if provided @@ -477,7 +478,17 @@ impl LLMProvider for OpenAIClient { for line in chunk_str.lines() { if line.starts_with("data: ") && !line.contains("[DONE]") { if let Ok(data) = serde_json::from_str::(&line[6..]) { + // Kimi K2.5 and other reasoning models send thinking in "reasoning" field + // Only process "content" (actual response), ignore "reasoning" (thinking) let content = data["choices"][0]["delta"]["content"].as_str(); + let reasoning = data["choices"][0]["delta"]["reasoning"].as_str(); + + // Log first chunk to help debug reasoning models + if reasoning.is_some() && content.is_none() { + trace!("[LLM] Kimi reasoning chunk (no content yet): {} chars", + reasoning.unwrap_or("").len()); + } + if let Some(content) = content { let processed = handler.process_content(content); if !processed.is_empty() {