botserver/src/basic/mod.rs

#[cfg(feature = "chat")]
use crate::basic::keywords::add_suggestion::clear_suggestions_keyword;
use crate::basic::keywords::set_user::set_user_keyword;
use crate::basic::keywords::string_functions::register_string_functions;
use crate::basic::keywords::switch_case::switch_keyword;
use crate::core::shared::models::UserSession;
use crate::core::shared::state::AppState;
use diesel::prelude::*;
use log::info;
use rhai::{Dynamic, Engine, EvalAltResult, Scope};
use std::collections::HashMap;
use std::sync::Arc;
pub mod compiler;
pub mod keywords;

#[derive(QueryableByName)]
struct ParamConfigRow {
    #[diesel(sql_type = diesel::sql_types::Text)]
    config_key: String,
    #[diesel(sql_type = diesel::sql_types::Text)]
    config_value: String,
}

// ===== CORE KEYWORD IMPORTS (always available) =====
#[cfg(feature = "chat")]
use self::keywords::add_bot::register_bot_keywords;
#[cfg(feature = "chat")]
use self::keywords::add_member::add_member_keyword;
#[cfg(feature = "chat")]
use self::keywords::add_suggestion::add_suggestion_keyword;
#[cfg(feature = "llm")]
use self::keywords::ai_tools::register_ai_tools_keywords;
use self::keywords::bot_memory::{get_bot_memory_keyword, set_bot_memory_keyword};
use self::keywords::clear_tools::clear_tools_keyword;
use self::keywords::core_functions::register_core_functions;
use self::keywords::data_operations::register_data_operations;
use self::keywords::find::find_keyword;
use self::keywords::search::search_keyword;
#[cfg(feature = "billing")]
use self::keywords::products::products_keyword;
use self::keywords::first::first_keyword;
use self::keywords::for_next::for_keyword;
use self::keywords::format::format_keyword;
use self::keywords::get::get_keyword;
use self::keywords::hear_talk::{hear_keyword, talk_keyword};
use self::keywords::http_operations::register_http_operations;
use self::keywords::last::last_keyword;
#[cfg(feature = "automation")]
use self::keywords::on_form_submit::on_form_submit_keyword;
use self::keywords::switch_case::preprocess_switch;
use self::keywords::use_tool::use_tool_keyword;
use self::keywords::use_website::{clear_websites_keyword, register_use_website_function};
use self::keywords::web_data::register_web_data_keywords;
#[cfg(feature = "automation")]
use self::keywords::webhook::webhook_keyword;
#[cfg(feature = "llm")]
use self::keywords::llm_keyword::llm_keyword;
use self::keywords::on::on_keyword;
use self::keywords::print::print_keyword;
use self::keywords::set::set_keyword;
use self::keywords::set_context::set_context_keyword;
use self::keywords::wait::wait_keyword;

// ===== CALENDAR FEATURE IMPORTS =====
#[cfg(feature = "calendar")]
use self::keywords::book::book_keyword;

// ===== MAIL FEATURE IMPORTS =====
#[cfg(feature = "mail")]
use self::keywords::create_draft::create_draft_keyword;
#[cfg(feature = "mail")]
use self::keywords::on_email::on_email_keyword;
#[cfg(feature = "mail")]
use self::keywords::send_mail::send_mail_keyword;
#[cfg(feature = "mail")]
use self::keywords::send_template::register_send_template_keywords;

// ===== TASKS FEATURE IMPORTS =====
#[cfg(feature = "tasks")]
use self::keywords::create_task::create_task_keyword;

// ===== SOCIAL FEATURE IMPORTS =====
#[cfg(feature = "social")]
use self::keywords::social_media::register_social_media_keywords;

// ===== LLM FEATURE IMPORTS =====
#[cfg(feature = "llm")]
use self::keywords::model_routing::register_model_routing_keywords;
#[cfg(feature = "llm")]
use self::keywords::multimodal::register_multimodal_keywords;
#[cfg(feature = "llm")]
use self::keywords::remember::remember_keyword;
#[cfg(feature = "llm")]
use self::keywords::save_from_unstructured::save_from_unstructured_keyword;

// ===== VECTORDB FEATURE IMPORTS =====
#[cfg(feature = "vectordb")]
use self::keywords::clear_kb::register_clear_kb_keyword;
#[cfg(feature = "vectordb")]
use self::keywords::use_kb::register_use_kb_keyword;

// ===== DRIVE FEATURE IMPORTS =====
#[cfg(feature = "drive")]
use self::keywords::file_operations::register_file_operations;
#[cfg(feature = "drive")]
use self::keywords::create_site::create_site_keyword;

// ===== PEOPLE FEATURE IMPORTS =====
#[cfg(feature = "people")]
use self::keywords::lead_scoring::register_lead_scoring_keywords;

// ===== COMMUNICATIONS FEATURE IMPORTS =====
#[cfg(any(feature = "whatsapp", feature = "telegram", feature = "mail"))]
use self::keywords::sms::register_sms_keywords;

// ===== CHAT FEATURE IMPORTS =====
#[cfg(feature = "chat")]
use self::keywords::transfer_to_human::register_transfer_to_human_keyword;

#[derive(Debug)]
pub struct ScriptService {
    pub engine: Engine,
    pub scope: Scope<'static>,
}

impl ScriptService {
    #[must_use]
    pub fn new(state: Arc<AppState>, user: UserSession) -> Self {
        let mut engine = Engine::new();
        let scope = Scope::new();
        engine.set_allow_anonymous_fn(true);
        engine.set_allow_looping(true);

        // ===== CORE KEYWORDS (always available) =====
        set_bot_memory_keyword(state.clone(), user.clone(), &mut engine);
        get_bot_memory_keyword(state.clone(), user.clone(), &mut engine);
        find_keyword(&state, user.clone(), &mut engine);
        search_keyword(&state, user.clone(), &mut engine);
        #[cfg(feature = "billing")]
        products_keyword(&state, user.clone(), &mut engine);
        for_keyword(&state, user.clone(), &mut engine);
        first_keyword(&mut engine);
        last_keyword(&mut engine);
        format_keyword(&mut engine);
        #[cfg(feature = "llm")]
        llm_keyword(state.clone(), user.clone(), &mut engine);
        get_keyword(state.clone(), user.clone(), &mut engine);
        set_keyword(&state, user.clone(), &mut engine);
        wait_keyword(&state, user.clone(), &mut engine);
        print_keyword(&state, user.clone(), &mut engine);
        on_keyword(&state, user.clone(), &mut engine);
        hear_keyword(state.clone(), user.clone(), &mut engine);
        talk_keyword(state.clone(), user.clone(), &mut engine);
        set_context_keyword(state.clone(), user.clone(), &mut engine);
        set_user_keyword(state.clone(), user.clone(), &mut engine);
        #[cfg(feature = "chat")]
        clear_suggestions_keyword(state.clone(), user.clone(), &mut engine);
        use_tool_keyword(state.clone(), user.clone(), &mut engine);
        clear_tools_keyword(state.clone(), user.clone(), &mut engine);
        clear_websites_keyword(state.clone(), user.clone(), &mut engine);
        #[cfg(feature = "chat")]
        add_suggestion_keyword(state.clone(), user.clone(), &mut engine);
        #[cfg(feature = "chat")]
        add_member_keyword(state.clone(), user.clone(), &mut engine);
        #[cfg(feature = "chat")]
        register_bot_keywords(&state, &user, &mut engine);

        // ===== PROCEDURE KEYWORDS (RETURN, etc.) =====
        keywords::procedures::register_procedure_keywords(state.clone(), user.clone(), &mut engine);

        // ===== WORKFLOW ORCHESTRATION KEYWORDS =====
        keywords::orchestration::register_orchestrate_workflow(state.clone(), user.clone(), &mut engine);
        keywords::orchestration::register_step_keyword(state.clone(), user.clone(), &mut engine);
        keywords::events::register_on_event(state.clone(), user.clone(), &mut engine);
        keywords::events::register_publish_event(state.clone(), user.clone(), &mut engine);
        keywords::events::register_wait_for_event(state.clone(), user.clone(), &mut engine);
        keywords::enhanced_memory::register_bot_share_memory(state.clone(), user.clone(), &mut engine);
        keywords::enhanced_memory::register_bot_sync_memory(state.clone(), user.clone(), &mut engine);
        keywords::enhanced_llm::register_enhanced_llm_keyword(state.clone(), user.clone(), &mut engine);

        keywords::universal_messaging::register_universal_messaging(
            state.clone(),
            user.clone(),
            &mut engine,
        );
        register_string_functions(state.clone(), user.clone(), &mut engine);
        switch_keyword(&state, user.clone(), &mut engine);
        register_http_operations(state.clone(), user.clone(), &mut engine);
        register_data_operations(state.clone(), user.clone(), &mut engine);
        #[cfg(feature = "automation")]
        webhook_keyword(&state, user.clone(), &mut engine);
        #[cfg(feature = "automation")]
        on_form_submit_keyword(state.clone(), user.clone(), &mut engine);
        #[cfg(feature = "llm")]
        register_ai_tools_keywords(state.clone(), user.clone(), &mut engine);
        register_web_data_keywords(state.clone(), user.clone(), &mut engine);
        register_core_functions(state.clone(), user.clone(), &mut engine);

        // ===== MAIL FEATURE KEYWORDS =====
        #[cfg(feature = "mail")]
        {
            create_draft_keyword(&state, user.clone(), &mut engine);
            on_email_keyword(&state, user.clone(), &mut engine);
            send_mail_keyword(state.clone(), user.clone(), &mut engine);
            register_send_template_keywords(state.clone(), user.clone(), &mut engine);
        }

        // ===== CALENDAR FEATURE KEYWORDS =====
        #[cfg(feature = "calendar")]
        {
            book_keyword(state.clone(), user.clone(), &mut engine);
        }

        // ===== TASKS FEATURE KEYWORDS =====
        #[cfg(feature = "tasks")]
        {
            create_task_keyword(state.clone(), user.clone(), &mut engine);
        }

        // ===== LLM FEATURE KEYWORDS =====
        #[cfg(feature = "llm")]
        {
            register_model_routing_keywords(state.clone(), user.clone(), &mut engine);
            register_multimodal_keywords(state.clone(), user.clone(), &mut engine);
            remember_keyword(state.clone(), user.clone(), &mut engine);
            save_from_unstructured_keyword(state.clone(), user.clone(), &mut engine);
        }

        // Register USE WEBSITE after all other USE keywords to avoid conflicts
        // USE WEBSITE is now preprocessed to USE_WEBSITE function call
        // Register it as a regular function instead of custom syntax
        register_use_website_function(state.clone(), user.clone(), &mut engine);

        // ===== VECTORDB FEATURE KEYWORDS =====
        #[cfg(feature = "vectordb")]
        {
            let _ = register_use_kb_keyword(&mut engine, state.clone(), Arc::new(user.clone()));
            let _ = register_clear_kb_keyword(&mut engine, state.clone(), Arc::new(user.clone()));
        }

        // ===== DRIVE FEATURE KEYWORDS =====
        #[cfg(feature = "drive")]
        {
            create_site_keyword(&state, user.clone(), &mut engine);
            register_file_operations(state.clone(), user.clone(), &mut engine);
        }

        // ===== SOCIAL FEATURE KEYWORDS =====
        #[cfg(feature = "social")]
        {
            register_social_media_keywords(state.clone(), user.clone(), &mut engine);
        }

        // ===== PEOPLE FEATURE KEYWORDS =====
        #[cfg(feature = "people")]
        {
            register_lead_scoring_keywords(state.clone(), user.clone(), &mut engine);
        }

        // ===== CHAT FEATURE KEYWORDS =====
        #[cfg(feature = "chat")]
        {
            register_transfer_to_human_keyword(state.clone(), user.clone(), &mut engine);
        }

        // ===== COMMUNICATIONS FEATURE KEYWORDS =====
        #[cfg(any(feature = "whatsapp", feature = "telegram", feature = "mail"))]
        {
            register_sms_keywords(state.clone(), user.clone(), &mut engine);
        }

        // Silence unused variable warning when features are disabled
        let _ = user;

        Self { engine, scope }
    }

    pub fn inject_config_variables(&mut self, config_vars: HashMap<String, String>) {
        for (key, value) in config_vars {
            let var_name = if key.starts_with("param-") {
                key.strip_prefix("param-").unwrap_or(&key).to_lowercase()
            } else {
                key.to_lowercase()
            };

            if let Ok(int_val) = value.parse::<i64>() {
                self.scope.push(&var_name, int_val);
            } else if let Ok(float_val) = value.parse::<f64>() {
                self.scope.push(&var_name, float_val);
            } else if value.eq_ignore_ascii_case("true") {
                self.scope.push(&var_name, true);
            } else if value.eq_ignore_ascii_case("false") {
                self.scope.push(&var_name, false);
            } else {
                self.scope.push(&var_name, value);
            }
        }
    }

    pub fn load_bot_config_params(&mut self, state: &AppState, bot_id: uuid::Uuid) {
        if let Ok(mut conn) = state.conn.get() {
            let result = diesel::sql_query(
                "SELECT config_key, config_value FROM bot_configuration WHERE bot_id = $1 AND config_key LIKE 'param-%'"
            )
            .bind::<diesel::sql_types::Uuid, _>(bot_id)
            .load::<ParamConfigRow>(&mut conn);

            if let Ok(params) = result {
                let config_vars: HashMap<String, String> = params
                    .into_iter()
                    .map(|row| (row.config_key, row.config_value))
                    .collect();
                self.inject_config_variables(config_vars);
            }
        }
    }
    fn preprocess_basic_script(&self, script: &str) -> String {
        let _ = self; // silence unused self warning - kept for API consistency
        let script = preprocess_switch(script);

        // Convert ALL multi-word keywords to underscore versions (e.g., "USE WEBSITE" → "USE_WEBSITE")
        // This avoids Rhai custom syntax conflicts and makes the system more secure
        let script = Self::convert_multiword_keywords(&script);

        let script = Self::normalize_variables_to_lowercase(&script);

        let mut result = String::new();
        let mut for_stack: Vec<usize> = Vec::new();
        let mut current_indent = 0;
        for line in script.lines() {
            let trimmed = line.trim();
            if trimmed.is_empty() || trimmed.starts_with("//") || trimmed.starts_with('\'') {
                continue;
            }
            if trimmed.starts_with("FOR EACH") {
                for_stack.push(current_indent);
                result.push_str(&" ".repeat(current_indent));
                result.push_str(trimmed);
                result.push_str("{\n");
                current_indent += 4;
                result.push_str(&" ".repeat(current_indent));
                result.push('\n');
                continue;
            }
            if trimmed.starts_with("NEXT") {
                if let Some(expected_indent) = for_stack.pop() {
                    assert!(
                        (current_indent - 4) == expected_indent,
                        "NEXT without matching FOR EACH"
                    );
                    current_indent -= 4;
                    result.push_str(&" ".repeat(current_indent));
                    result.push_str("}\n");
                    result.push_str(&" ".repeat(current_indent));
                    result.push_str(trimmed);
                    result.push(';');
                    result.push('\n');
                    continue;
                }
                log::error!("NEXT without matching FOR EACH");
                return result;
            }
            if trimmed == "EXIT FOR" {
                result.push_str(&" ".repeat(current_indent));
                result.push_str(trimmed);
                result.push('\n');
                continue;
            }
            result.push_str(&" ".repeat(current_indent));
            let basic_commands = [
                "SET",
                "CREATE",
                "PRINT",
                "FOR",
                "FIND",
                "GET",
                "EXIT",
                "IF",
                "THEN",
                "ELSE",
                "END IF",
                "WHILE",
                "WEND",
                "DO",
                "LOOP",
                "HEAR",
                "TALK",
                "SET CONTEXT",
                "SET USER",
                "GET BOT MEMORY",
                "SET BOT MEMORY",
                "IMAGE",
                "VIDEO",
                "AUDIO",
                "SEE",
                "SEND FILE",
                "SWITCH",
                "CASE",
                "DEFAULT",
                "END SWITCH",
                "USE KB",
                "CLEAR KB",
                "USE TOOL",
                "CLEAR TOOLS",
                "ADD SUGGESTION",
                "CLEAR SUGGESTIONS",
                "INSTR",
                "IS_NUMERIC",
                "IS NUMERIC",
                "POST",
                "PUT",
                "PATCH",
                "DELETE",
                "SET HEADER",
                "CLEAR HEADERS",
                "GRAPHQL",
                "SOAP",
                "SAVE",
                "INSERT",
                "UPDATE",
                "DELETE",
                "MERGE",
                "FILL",
                "MAP",
                "FILTER",
                "AGGREGATE",
                "JOIN",
                "PIVOT",
                "GROUP BY",
                "READ",
                "WRITE",
                "COPY",
                "MOVE",
                "LIST",
                "COMPRESS",
                "EXTRACT",
                "UPLOAD",
                "DOWNLOAD",
                "GENERATE PDF",
                "MERGE PDF",
                "WEBHOOK",
                "POST TO",
                "POST TO INSTAGRAM",
                "POST TO FACEBOOK",
                "POST TO LINKEDIN",
                "POST TO TWITTER",
                "GET INSTAGRAM METRICS",
                "GET FACEBOOK METRICS",
                "GET LINKEDIN METRICS",
                "GET TWITTER METRICS",
                "DELETE POST",
                "SEND MAIL",
                "SEND TEMPLATE",
                "CREATE TEMPLATE",
                "GET TEMPLATE",
                "ON ERROR RESUME NEXT",
                "ON ERROR GOTO",
                "CLEAR ERROR",
                "ERROR MESSAGE",
                "ON FORM SUBMIT",
                "SCORE LEAD",
                "GET LEAD SCORE",
                "QUALIFY LEAD",
                "UPDATE LEAD SCORE",
                "AI SCORE LEAD",
                "ABS",
                "ROUND",
                "INT",
                "FIX",
                "FLOOR",
                "CEIL",
                "MAX",
                "MIN",
                "MOD",
                "RANDOM",
                "RND",
                "SGN",
                "SQR",
                "SQRT",
                "LOG",
                "EXP",
                "POW",
                "SIN",
                "COS",
                "TAN",
                "SUM",
                "AVG",
                "NOW",
                "TODAY",
                "DATE",
                "TIME",
                "YEAR",
                "MONTH",
                "DAY",
                "HOUR",
                "MINUTE",
                "SECOND",
                "WEEKDAY",
                "DATEADD",
                "DATEDIFF",
                "FORMAT_DATE",
                "ISDATE",
                "VAL",
                "STR",
                "CINT",
                "CDBL",
                "CSTR",
                "ISNULL",
                "ISEMPTY",
                "TYPEOF",
                "ISARRAY",
                "ISOBJECT",
                "ISSTRING",
                "ISNUMBER",
                "NVL",
                "IIF",
                "ARRAY",
                "UBOUND",
                "LBOUND",
                "COUNT",
                "SORT",
                "UNIQUE",
                "CONTAINS",
                "INDEX_OF",
                "PUSH",
                "POP",
                "SHIFT",
                "REVERSE",
                "SLICE",
                "SPLIT",
                "CONCAT",
                "FLATTEN",
                "RANGE",
                "THROW",
                "ERROR",
                "IS_ERROR",
                "ASSERT",
                "LOG_ERROR",
                "LOG_WARN",
                "LOG_INFO",
            ];
            let is_basic_command = basic_commands.iter().any(|&cmd| trimmed.starts_with(cmd));
            let is_control_flow = trimmed.starts_with("IF")
                || trimmed.starts_with("ELSE")
                || trimmed.starts_with("END IF");
            result.push_str(trimmed);
            let needs_semicolon = is_basic_command
                || !for_stack.is_empty()
                || is_control_flow
                || (!trimmed.ends_with(';') && !trimmed.ends_with('{') && !trimmed.ends_with('}'));
            if needs_semicolon {
                result.push(';');
            }
            result.push('\n');
        }
        assert!(for_stack.is_empty(), "Unclosed FOR EACH loop");
        result
    }
    pub fn compile(&self, script: &str) -> Result<rhai::AST, Box<EvalAltResult>> {
        let processed_script = self.preprocess_basic_script(script);
        info!("Processed Script:\n{}", processed_script);
        match self.engine.compile(&processed_script) {
            Ok(ast) => Ok(ast),
            Err(parse_error) => Err(Box::new(parse_error.into())),
        }
    }

    /// Compile a tool script (.bas file with PARAM/DESCRIPTION metadata lines)
    /// Filters out tool metadata before compiling
    pub fn compile_tool_script(&self, script: &str) -> Result<rhai::AST, Box<EvalAltResult>> {
        // Filter out PARAM, DESCRIPTION, comment, and empty lines (tool metadata)
        let executable_script: String = script
            .lines()
            .filter(|line| {
                let trimmed = line.trim();
                // Keep lines that are NOT PARAM, DESCRIPTION, comments, or empty
                !(trimmed.starts_with("PARAM ") ||
                  trimmed.starts_with("PARAM\t") ||
                  trimmed.starts_with("DESCRIPTION ") ||
                  trimmed.starts_with("DESCRIPTION\t") ||
                  trimmed.starts_with('\'') || // BASIC comment lines
                  trimmed.is_empty())
            })
            .collect::<Vec<&str>>()
            .join("\n");

        info!("[TOOL] Filtered tool metadata: {} -> {} chars", script.len(), executable_script.len());

        // Apply minimal preprocessing for tools (skip variable normalization to avoid breaking multi-line strings)
        let script = preprocess_switch(&executable_script);
        let script = Self::convert_multiword_keywords(&script);
        // Skip normalize_variables_to_lowercase for tools - it breaks multi-line strings
        // Note: FORMAT is registered as a regular function, so FORMAT(expr, pattern) works directly

        info!("[TOOL] Preprocessed tool script for Rhai compilation");
        // Convert IF ... THEN / END IF to if ... { }
        let script = Self::convert_if_then_syntax(&script);
        // Convert SELECT ... CASE / END SELECT to match expressions
        let script = Self::convert_select_case_syntax(&script);
        // Convert BASIC keywords to lowercase (but preserve variable casing)
        let script = Self::convert_keywords_to_lowercase(&script);
        // Save to file for debugging
        if let Err(e) = std::fs::write("/tmp/tool_preprocessed.bas", &script) {
            log::warn!("Failed to write preprocessed script: {}", e);
        }
        match self.engine.compile(&script) {
            Ok(ast) => Ok(ast),
            Err(parse_error) => Err(Box::new(parse_error.into())),
        }
    }
    pub fn run(&mut self, ast: &rhai::AST) -> Result<Dynamic, Box<EvalAltResult>> {
        self.engine.eval_ast_with_scope(&mut self.scope, ast)
    }

    /// Set a variable in the script scope (for tool parameters)
    pub fn set_variable(&mut self, name: &str, value: &str) -> Result<(), Box<dyn std::error::Error>> {
        use rhai::Dynamic;
        self.scope.set_or_push(name, Dynamic::from(value.to_string()));
        Ok(())
    }

    #[allow(dead_code)]
    /// Convert FORMAT(expr, pattern) to FORMAT expr pattern (custom syntax format)
    /// Also handles RANDOM and other functions that need space-separated arguments
    fn convert_format_syntax(script: &str) -> String {
        use regex::Regex;
        let mut result = script.to_string();

        // First, process RANDOM to ensure commas are preserved
        // RANDOM(min, max) stays as RANDOM(min, max) - no conversion needed

        // Convert FORMAT(expr, pattern) → FORMAT expr pattern
        // Need to handle nested functions carefully
        // Match FORMAT( ... ) but don't include inner function parentheses
        // This regex matches FORMAT followed by parentheses containing two comma-separated expressions
        if let Ok(re) = Regex::new(r"(?i)FORMAT\s*\(([^()]+(?:\([^()]*\)[^()]*)*),([^)]+)\)") {
            result = re.replace_all(&result, "FORMAT $1$2").to_string();
        }

        result
    }

    /// Convert BASIC IF ... THEN / END IF syntax to Rhai's if ... { } syntax
    fn convert_if_then_syntax(script: &str) -> String {
        let mut result = String::new();
        let mut if_stack: Vec<bool> = Vec::new();
        let mut in_with_block = false;

        log::info!("[TOOL] Converting IF/THEN syntax, input has {} lines", script.lines().count());

        for line in script.lines() {
            let trimmed = line.trim();
            let upper = trimmed.to_uppercase();

            // Skip empty lines and comments
            if trimmed.is_empty() || trimmed.starts_with('\'') || trimmed.starts_with("//") {
                continue;
            }

            // Handle IF ... THEN
            if upper.starts_with("IF ") && upper.contains(" THEN") {
                let then_pos = match upper.find(" THEN") {
                    Some(pos) => pos,
                    None => continue, // Skip invalid IF statement
                };
                let condition = &trimmed[3..then_pos].trim();
                // Convert BASIC "NOT IN" to Rhai "!in"
                let condition = condition.replace(" NOT IN ", " !in ").replace(" not in ", " !in ");
                log::info!("[TOOL] Converting IF statement: condition='{}'", condition);
                result.push_str("if ");
                result.push_str(&condition);
                result.push_str(" {\n");
                if_stack.push(true);
                continue;
            }

            // Handle ELSE
            if upper == "ELSE" {
                log::info!("[TOOL] Converting ELSE statement");
                result.push_str("} else {\n");
                continue;
            }

            // Handle END IF
            if upper == "END IF" {
                log::info!("[TOOL] Converting END IF statement");
                if let Some(_) = if_stack.pop() {
                    result.push_str("}\n");
                }
                continue;
            }

            // Handle WITH ... END WITH (BASIC object creation)
            if upper.starts_with("WITH ") {
                let object_name = &trimmed[5..].trim();
                log::info!("[TOOL] Converting WITH statement: object='{}'", object_name);
                // Convert WITH obj → let obj = #{  (start object literal)
                result.push_str("let ");
                result.push_str(object_name);
                result.push_str(" = #{\n");
                in_with_block = true;
                continue;
            }

            if upper == "END WITH" {
                log::info!("[TOOL] Converting END WITH statement");
                result.push_str("};\n");
                in_with_block = false;
                continue;
            }

            // Inside a WITH block - convert property assignments (key = value → key: value)
            if in_with_block {
                // Check if this is a property assignment (identifier = value)
                if trimmed.contains('=') && !trimmed.contains("==") && !trimmed.contains("!=") && !trimmed.contains("+=") && !trimmed.contains("-=") {
                    // Convert assignment to object property syntax
                    let parts: Vec<&str> = trimmed.splitn(2, '=').collect();
                    if parts.len() == 2 {
                        let property_name = parts[0].trim();
                        let property_value = parts[1].trim();
                        // Remove trailing semicolon if present
                        let property_value = property_value.trim_end_matches(';');
                        result.push_str(&format!("    {}: {},\n", property_name, property_value));
                        continue;
                    }
                }
                // Regular line in WITH block - add indentation
                result.push_str("    ");
            }

            // Handle SAVE table, object → INSERT table, object
            // BASIC SAVE uses 2 parameters but Rhai SAVE needs 3
            // INSERT uses 2 parameters which matches the BASIC syntax
            if upper.starts_with("SAVE") && upper.contains(',') {
                log::info!("[TOOL] Processing SAVE line: '{}'", trimmed);
                // Extract table and object name
                let after_save = &trimmed[4..].trim(); // Skip "SAVE"
                let parts: Vec<&str> = after_save.split(',').collect();
                log::info!("[TOOL] SAVE parts: {:?}", parts);
                if parts.len() == 2 {
                    let table = parts[0].trim().trim_matches('"');
                    let object_name = parts[1].trim().trim_end_matches(';');
                    // Convert to INSERT table, object
                    let converted = format!("INSERT \"{}\", {};\n", table, object_name);
                    log::info!("[TOOL] Converted SAVE to INSERT: '{}'", converted);
                    result.push_str(&converted);
                    continue;
                }
            }

            // Handle SEND EMAIL → send_mail (function call style)
            // Syntax: SEND EMAIL to, subject, body → send_mail(to, subject, body, [])
            if upper.starts_with("SEND EMAIL") {
                log::info!("[TOOL] Processing SEND EMAIL line: '{}'", trimmed);
                let after_send = &trimmed[11..].trim(); // Skip "SEND EMAIL " (10 chars + space = 11)
                let parts: Vec<&str> = after_send.split(',').collect();
                log::info!("[TOOL] SEND EMAIL parts: {:?}", parts);
                if parts.len() == 3 {
                    let to = parts[0].trim();
                    let subject = parts[1].trim();
                    let body = parts[2].trim().trim_end_matches(';');
                    // Convert to send_mail(to, subject, body, []) function call
                    let converted = format!("send_mail({}, {}, {}, []);\n", to, subject, body);
                    log::info!("[TOOL] Converted SEND EMAIL to: '{}'", converted);
                    result.push_str(&converted);
                    continue;
                }
            }

            // Regular line - add indentation if inside IF block
            if !if_stack.is_empty() {
                result.push_str("    ");
            }

            // Check if line is a simple statement (not containing THEN or other control flow)
            if !upper.starts_with("IF ") && !upper.starts_with("ELSE") && !upper.starts_with("END IF") {
                // Check if this is a variable assignment (identifier = expression)
                // Pattern: starts with letter/underscore, contains = but not ==, !=, <=, >=, +=, -=
                let is_var_assignment = trimmed.chars().next().map_or(false, |c| c.is_alphabetic() || c == '_')
                    && trimmed.contains('=')
                    && !trimmed.contains("==")
                    && !trimmed.contains("!=")
                    && !trimmed.contains("<=")
                    && !trimmed.contains(">=")
                    && !trimmed.contains("+=")
                    && !trimmed.contains("-=")
                    && !trimmed.contains("*=")
                    && !trimmed.contains("/=");

                if is_var_assignment {
                    // Add 'let' for variable declarations
                    result.push_str("let ");
                }
                result.push_str(trimmed);
                // Add semicolon if line doesn't have one and doesn't end with { or }
                if !trimmed.ends_with(';') && !trimmed.ends_with('{') && !trimmed.ends_with('}') {
                    result.push(';');
                }
                result.push('\n');
            } else {
                result.push_str(trimmed);
                result.push('\n');
            }
        }

        log::info!("[TOOL] IF/THEN conversion complete, output has {} lines", result.lines().count());
        result
    }

    /// Convert BASIC SELECT ... CASE / END SELECT to Rhai match expressions
    /// Transforms: SELECT var ... CASE "value" ... END SELECT
    /// Into: match var { "value" => { ... } ... }
    fn convert_select_case_syntax(script: &str) -> String {
        let mut result = String::new();
        let mut lines: Vec<&str> = script.lines().collect();
        let mut i = 0;

        log::info!("[TOOL] Converting SELECT/CASE syntax");

        while i < lines.len() {
            let trimmed = lines[i].trim();
            let upper = trimmed.to_uppercase();

            // Detect SELECT statement (e.g., "SELECT tipoMissa")
            if upper.starts_with("SELECT ") && !upper.contains(" THEN") {
                // Extract the variable being selected
                let select_var = trimmed[7..].trim(); // Skip "SELECT "
                log::info!("[TOOL] Converting SELECT statement for variable: '{}'", select_var);

                // Start match expression
                result.push_str(&format!("match {} {{\n", select_var));

                // Skip the SELECT line
                i += 1;

                // Process CASE statements until END SELECT
                let mut current_case_body: Vec<String> = Vec::new();
                let mut in_case = false;

                while i < lines.len() {
                    let case_trimmed = lines[i].trim();
                    let case_upper = case_trimmed.to_uppercase();

                    if case_upper == "END SELECT" {
                        // Close any open case
                        if in_case {
                            for body_line in &current_case_body {
                                result.push_str("    ");
                                result.push_str(body_line);
                                result.push('\n');
                            }
                            current_case_body.clear();
                            in_case = false;
                        }
                        // Close the match expression
                        result.push_str("}\n");
                        i += 1;
                        break;
                    } else if case_upper.starts_with("CASE ") {
                        // Close previous case if any
                        if in_case {
                            for body_line in &current_case_body {
                                result.push_str("    ");
                                result.push_str(body_line);
                                result.push('\n');
                            }
                            current_case_body.clear();
                        }

                        // Extract the case value (handle both CASE "value" and CASE value)
                        let case_value = if case_trimmed[5..].trim().starts_with('"') {
                            // CASE "value" format
                            case_trimmed[5..].trim().to_string()
                        } else {
                            // CASE value format (variable/enum)
                            format!("\"{}\"", case_trimmed[5..].trim())
                        };

                        result.push_str(&format!("    {} => {{\n", case_value));
                        in_case = true;
                        i += 1;
                    } else {
                        // Collect body lines for the current case
                        if in_case {
                            current_case_body.push(lines[i].to_string());
                        }
                        i += 1;
                    }
                }

                continue;
            }

            // Not a SELECT statement - just copy the line
            result.push_str(lines[i]);
            result.push('\n');
            i += 1;
        }

        result
    }

    /// Convert BASIC keywords to lowercase without touching variables
    /// This is a simplified version of normalize_variables_to_lowercase for tools
    fn convert_keywords_to_lowercase(script: &str) -> String {
        let keywords = [
            "IF", "THEN", "ELSE", "END IF", "FOR", "NEXT", "WHILE", "WEND",
            "DO", "LOOP", "RETURN", "EXIT",
            "WITH", "END WITH", "AND", "OR", "NOT", "MOD",
            "DIM", "AS", "NEW", "FUNCTION", "SUB", "CALL",
        ];

        let mut result = String::new();
        for line in script.lines() {
            let mut processed_line = line.to_string();
            for keyword in &keywords {
                // Use word boundaries to avoid replacing parts of variable names
                let pattern = format!(r"\b{}\b", regex::escape(keyword));
                if let Ok(re) = regex::Regex::new(&pattern) {
                    processed_line = re.replace_all(&processed_line, keyword.to_lowercase()).to_string();
                }
            }
            result.push_str(&processed_line);
            result.push('\n');
        }
        result
    }

    fn normalize_variables_to_lowercase(script: &str) -> String {
        use regex::Regex;

        let mut result = String::new();

        let keywords = [
            "SET",
            "CREATE",
            "PRINT",
            "FOR",
            "FIND",
            "GET",
            "EXIT",
            "IF",
            "THEN",
            "ELSE",
            "END",
            "WHILE",
            "WEND",
            "DO",
            "LOOP",
            "HEAR",
            "TALK",
            "NEXT",
            "FUNCTION",
            "SUB",
            "CALL",
            "RETURN",
            "DIM",
            "AS",
            "NEW",
            "ARRAY",
            "OBJECT",
            "LET",
            "REM",
            "AND",
            "OR",
            "NOT",
            "TRUE",
            "FALSE",
            "NULL",
            "SWITCH",
            "CASE",
            "DEFAULT",
            "USE",
            "KB",
            "TOOL",
            "CLEAR",
            "ADD",
            "SUGGESTION",
            "SUGGESTIONS",
            "TOOLS",
            "CONTEXT",
            "USER",
            "BOT",
            "MEMORY",
            "IMAGE",
            "VIDEO",
            "AUDIO",
            "SEE",
            "SEND",
            "FILE",
            "POST",
            "PUT",
            "PATCH",
            "DELETE",
            "SAVE",
            "INSERT",
            "UPDATE",
            "MERGE",
            "FILL",
            "MAP",
            "FILTER",
            "AGGREGATE",
            "JOIN",
            "PIVOT",
            "GROUP",
            "BY",
            "READ",
            "WRITE",
            "COPY",
            "MOVE",
            "LIST",
            "COMPRESS",
            "EXTRACT",
            "UPLOAD",
            "DOWNLOAD",
            "GENERATE",
            "PDF",
            "WEBHOOK",
            "TEMPLATE",
            "FORM",
            "SUBMIT",
            "SCORE",
            "LEAD",
            "QUALIFY",
            "AI",
            "ABS",
            "ROUND",
            "INT",
            "FIX",
            "FLOOR",
            "CEIL",
            "MAX",
            "MIN",
            "MOD",
            "RANDOM",
            "RND",
            "SGN",
            "SQR",
            "SQRT",
            "LOG",
            "EXP",
            "POW",
            "SIN",
            "COS",
            "TAN",
            "SUM",
            "AVG",
            "NOW",
            "TODAY",
            "DATE",
            "TIME",
            "YEAR",
            "MONTH",
            "DAY",
            "HOUR",
            "MINUTE",
            "SECOND",
            "WEEKDAY",
            "DATEADD",
            "DATEDIFF",
            "FORMAT",
            "ISDATE",
            "VAL",
            "STR",
            "CINT",
            "CDBL",
            "CSTR",
            "ISNULL",
            "ISEMPTY",
            "TYPEOF",
            "ISARRAY",
            "ISOBJECT",
            "ISSTRING",
            "ISNUMBER",
            "NVL",
            "IIF",
            "UBOUND",
            "LBOUND",
            "COUNT",
            "SORT",
            "UNIQUE",
            "CONTAINS",
            "INDEX",
            "OF",
            "PUSH",
            "POP",
            "SHIFT",
            "REVERSE",
            "SLICE",
            "SPLIT",
            "CONCAT",
            "FLATTEN",
            "RANGE",
            "THROW",
            "ERROR",
            "IS",
            "ASSERT",
            "WARN",
            "INFO",
            "EACH",
            "WITH",
            "TO",
            "STEP",
            "BEGIN",
            "SYSTEM",
            "PROMPT",
            "SCHEDULE",
            "REFRESH",
            "ALLOW",
            "ROLE",
            "ANSWER",
            "MODE",
            "SYNCHRONIZE",
            "TABLE",
            "ON",
            "EMAIL",
            "REPORT",
            "RESET",
            "WAIT",
            "FIRST",
            "LAST",
            "LLM",
            "INSTR",
            "NUMERIC",
            "LEN",
            "LEFT",
            "RIGHT",
            "MID",
            "LOWER",
            "UPPER",
            "TRIM",
            "LTRIM",
            "RTRIM",
            "REPLACE",
            "LIKE",
            "DELEGATE",
            "PRIORITY",
            "BOTS",
            "REMOVE",
            "MEMBER",
            "BOOK",
            "REMEMBER",
            "TASK",
            "SITE",
            "DRAFT",
            "INSTAGRAM",
            "FACEBOOK",
            "LINKEDIN",
            "TWITTER",
            "METRICS",
            "HEADER",
            "HEADERS",
            "GRAPHQL",
            "SOAP",
            "HTTP",
            "DESCRIPTION",
            "PARAM",
            "REQUIRED",
            "WEBSITE",
            "MODEL",
        ];

        let _identifier_re = Regex::new(r"([a-zA-Z_][a-zA-Z0-9_]*)").expect("valid regex");

        for line in script.lines() {
            let trimmed = line.trim();

            if trimmed.starts_with("REM") || trimmed.starts_with('\'') || trimmed.starts_with("//")
            {
                continue;
            }

            // Skip lines with custom syntax that should not be lowercased
            // These are registered directly with Rhai in uppercase
            let trimmed_upper = trimmed.to_uppercase();
            if trimmed_upper.contains("ADD_SUGGESTION_TOOL") ||
               trimmed_upper.contains("ADD_SUGGESTION_TEXT") ||
               trimmed_upper.starts_with("ADD_SUGGESTION_") ||
               trimmed_upper.starts_with("ADD_MEMBER") {
                // Keep original line as-is
                result.push_str(line);
                result.push('\n');
                continue;
            }

            let mut processed_line = String::new();
            let mut chars = line.chars().peekable();
            let mut in_string = false;
            let mut string_char = '"';
            let mut current_word = String::new();

            while let Some(c) = chars.next() {
                if in_string {
                    processed_line.push(c);
                    if c == string_char {
                        in_string = false;
                    } else if c == '\\' {
                        if let Some(&next) = chars.peek() {
                            processed_line.push(next);
                            chars.next();
                        }
                    }
                } else if c == '"' || c == '\'' {
                    if !current_word.is_empty() {
                        processed_line.push_str(&Self::normalize_word(&current_word, &keywords));
                        current_word.clear();
                    }
                    in_string = true;
                    string_char = c;
                    processed_line.push(c);
                } else if c.is_alphanumeric() || c == '_' {
                    current_word.push(c);
                } else {
                    if !current_word.is_empty() {
                        processed_line.push_str(&Self::normalize_word(&current_word, &keywords));
                        current_word.clear();
                    }
                    processed_line.push(c);
                }
            }

            if !current_word.is_empty() {
                processed_line.push_str(&Self::normalize_word(&current_word, &keywords));
            }

            result.push_str(&processed_line);
            result.push('\n');
        }

        result
    }

    /// Convert ALL multi-word keywords to underscore versions (function calls)
    /// This avoids Rhai custom syntax conflicts and makes the system more secure
    ///
    /// Examples:
    /// - "USE WEBSITE "url"" → "USE_WEBSITE("url")"
    /// - "USE WEBSITE "url" REFRESH "interval"" → "USE_WEBSITE("url", "interval")"
    /// - "SET BOT MEMORY key AS value" → "SET_BOT_MEMORY(key, value)"
    /// - "CLEAR SUGGESTIONS" → "CLEAR_SUGGESTIONS()"
    fn convert_multiword_keywords(script: &str) -> String {
        use regex::Regex;

        // Known multi-word keywords with their conversion patterns
        // Format: (keyword_pattern, min_params, max_params, param_names)
        let multiword_patterns = vec![
            // USE family
            (r#"USE\s+WEBSITE"#, 1, 2, vec!["url", "refresh"]),
            (r#"USE\s+MODEL"#, 1, 1, vec!["model"]),
            (r#"USE\s+KB"#, 1, 1, vec!["kb_name"]),
            (r#"USE\s+TOOL"#, 1, 1, vec!["tool_path"]),

            // SET family
            (r#"SET\s+BOT\s+MEMORY"#, 2, 2, vec!["key", "value"]),
            (r#"SET\s+CONTEXT"#, 2, 2, vec!["key", "value"]),
            (r#"SET\s+USER"#, 1, 1, vec!["user_id"]),

            // GET family
            (r#"GET\s+BOT\s+MEMORY"#, 1, 1, vec!["key"]),

            // CLEAR family
            (r#"CLEAR\s+SUGGESTIONS"#, 0, 0, vec![]),
            (r#"CLEAR\s+TOOLS"#, 0, 0, vec![]),
            (r#"CLEAR\s+WEBSITES"#, 0, 0, vec![]),

            // ADD family - ADD_SUGGESTION_TOOL must come before ADD\s+SUGGESTION
            (r#"ADD_SUGGESTION_TOOL"#, 2, 2, vec!["tool", "text"]),
            (r#"ADD\s+SUGGESTION\s+TEXT"#, 2, 2, vec!["value", "text"]),
            (r#"ADD\s+SUGGESTION(?!\s*TEXT|\s*TOOL|_TOOL)"#, 2, 2, vec!["context", "text"]),
            (r#"ADD\s+MEMBER"#, 2, 2, vec!["name", "role"]),

            // CREATE family
            (r#"CREATE\s+TASK"#, 1, 1, vec!["task"]),
            (r#"CREATE\s+DRAFT"#, 4, 4, vec!["to", "subject", "body", "attachments"]),
            (r#"CREATE\s+SITE"#, 1, 1, vec!["site"]),

            // ON family
            (r#"ON\s+FORM\s+SUBMIT"#, 1, 1, vec!["form"]),
            (r#"ON\s+EMAIL"#, 1, 1, vec!["filter"]),
            (r#"ON\s+EVENT"#, 1, 1, vec!["event"]),

            // SEND family
            (r#"SEND\s+MAIL"#, 4, 4, vec!["to", "subject", "body", "attachments"]),

            // BOOK (calendar)
            (r#"BOOK"#, 1, 1, vec!["event"]),
        ];

        let mut result = String::new();

        for line in script.lines() {
            let trimmed = line.trim();
            let mut converted = false;

            // Skip lines that already use underscore-style custom syntax
            // These are registered directly with Rhai and should not be converted
            let trimmed_upper = trimmed.to_uppercase();
            if trimmed_upper.contains("ADD_SUGGESTION_TOOL") ||
               trimmed_upper.contains("ADD_SUGGESTION_TEXT") ||
               trimmed_upper.starts_with("ADD_SUGGESTION_") ||
               trimmed_upper.starts_with("ADD_MEMBER") ||
               (trimmed_upper.starts_with("USE_") && trimmed.contains('(')) {
                // Keep original line and add semicolon if needed
                result.push_str(line);
                if !trimmed.ends_with(';') && !trimmed.ends_with('{') && !trimmed.ends_with('}') {
                    result.push(';');
                }
                result.push('\n');
                continue;
            }

            // Try each pattern
            for (pattern, min_params, max_params, _param_names) in &multiword_patterns {
                // Build regex pattern: KEYWORD params...
                // Handle quoted strings and unquoted identifiers
                let regex_str = format!(
                    r#"(?i)^\s*{}\s+(.*?)(?:\s*)$"#,
                    pattern
                );

                if let Ok(re) = Regex::new(&regex_str) {
                    if let Some(caps) = re.captures(trimmed) {
                        if let Some(params_str) = caps.get(1) {
                            let params = Self::parse_parameters(params_str.as_str());
                            let param_count = params.len();

                            // Validate parameter count
                            if param_count >= *min_params && param_count <= *max_params {
                                // Convert keyword to underscores
                                let keyword = pattern.replace(r"\s+", "_");

                                // Build function call
                                let params_str = if params.is_empty() {
                                    String::new()
                                } else {
                                    params.join(", ")
                                };

                                result.push_str(&format!("{}({});", keyword, params_str));
                                result.push('\n');
                                converted = true;
                                break;
                            }
                        }
                    }
                }
            }

            // If not converted, keep original line
            if !converted {
                result.push_str(line);
                result.push('\n');
            }
        }

        result
    }

    /// Parse parameters from a keyword line
    /// Handles quoted strings, AS keyword, and comma-separated values
    fn parse_parameters(params_str: &str) -> Vec<String> {
        let mut params = Vec::new();
        let mut current = String::new();
        let mut in_quotes = false;
        let mut quote_char = '"';
        let mut chars = params_str.chars().peekable();

        while let Some(c) = chars.next() {
            match c {
                '"' | '\'' if !in_quotes => {
                    in_quotes = true;
                    quote_char = c;
                    current.push(c);
                }
                '"' | '\'' if in_quotes && c == quote_char => {
                    in_quotes = false;
                    current.push(c);
                }
                ' ' | '\t' if !in_quotes => {
                    // End of parameter if we have content
                    if !current.is_empty() {
                        params.push(current.trim().to_string());
                        current = String::new();
                    }
                }
                ',' if !in_quotes => {
                    // Comma separator
                    if !current.is_empty() {
                        params.push(current.trim().to_string());
                        current = String::new();
                    }
                }
                _ => {
                    current.push(c);
                }
            }
        }

        // Don't forget the last parameter
        if !current.is_empty() {
            params.push(current.trim().to_string());
        }

        params
    }

    fn normalize_word(word: &str, keywords: &[&str]) -> String {
        let upper = word.to_uppercase();

        if keywords.contains(&upper.as_str()) {
            upper
        } else if word
            .chars()
            .next()
            .map(|c| c.is_ascii_digit())
            .unwrap_or(false)
        {
            word.to_string()
        } else {
            word.to_lowercase()
        }
    }
}



#[cfg(test)]
pub mod tests;