diff --git a/docs/keywords/format.md b/docs/keywords/format.md new file mode 100644 index 0000000..a5c4f15 --- /dev/null +++ b/docs/keywords/format.md @@ -0,0 +1,55 @@ +**FORMAT FUNCTION - PATTERN REFERENCE** + +**SYNTAX:** `FORMAT(value, pattern)` + +**PATTERN TABLE:** + +| CATEGORY | PATTERN | OUTPUT EXAMPLE | DESCRIPTION | +|----------|---------|----------------|-------------| +| **DATE** | `yyyy` | 2024 | 4-digit year | +| | `yy` | 24 | 2-digit year | +| | `MM` | 01 | 2-digit month | +| | `M` | 1 | 1-2 digit month | +| | `dd` | 05 | 2-digit day | +| | `d` | 5 | 1-2 digit day | +| **TIME** | `HH` | 14 | 24-hour, 2-digit | +| | `H` | 14 | 24-hour, 1-2 digit | +| | `hh` | 02 | 12-hour, 2-digit | +| | `h` | 2 | 12-hour, 1-2 digit | +| | `mm` | 08 | 2-digit minutes | +| | `m` | 8 | 1-2 digit minutes | +| | `ss` | 09 | 2-digit seconds | +| | `s` | 9 | 1-2 digit seconds | +| | `tt` | PM | AM/PM designator | +| | `t` | P | A/P designator | +| | `fff` | 123 | Milliseconds | +| **CURRENCY** | `C` | $ | Currency symbol | +| | `c` | 123.45 | Currency amount | +| | `N` | 1,234.56 | Number with commas | +| | `n` | 1234.56 | Number without commas | +| | `F` | 123.00 | Fixed decimal | +| | `f` | 123.45 | Float decimal | +| | `0` | 0.00 | Zero placeholder | +| | `#` | #.## | Digit placeholder | +| **NUMERIC** | `0` | 0 | Required digit | +| | `#` | # | Optional digit | +| | `.` | . | Decimal point | +| | `,` | , | Thousands separator | +| | `%` | % | Percentage | +| **TEXT** | `@` | TEXT | Character placeholder | +| | `&` | text | Lowercase text | +| | `>` | TEXT | Uppercase text | +| | `<` | text | Force lowercase | +| | `!` | T | Force uppercase | + +**COMMON COMBINATIONS:** +- `yyyy-MM-dd` → 2024-01-15 +- `MM/dd/yy` → 01/15/24 +- `HH:mm:ss` → 14:30:45 +- `C0.00` → $123.45 +- `N2` → 1,234.56 + +**USAGE:** +`FORMAT(123.456, "C2")` → "$123.46" +`FORMAT(NOW(), "yyyy-MM-dd HH:mm")` → "2024-01-15 14:30" +`FORMAT(0.15, "0%")` → "15%" \ No newline at end of file diff --git a/src/scripts/containers/alm-ci.sh b/src/scripts/containers/alm-ci.sh index 8f9a856..5f34c98 100644 --- a/src/scripts/containers/alm-ci.sh +++ b/src/scripts/containers/alm-ci.sh @@ -6,7 +6,7 @@ ALM_CI_LABELS="gbo" FORGEJO_RUNNER_VERSION="v6.3.1" FORGEJO_RUNNER_BINARY="forgejo-runner-6.3.1-linux-amd64" CONTAINER_IMAGE="images:debian/12" - + # Paths HOST_BASE="/opt/gbo/tenants/$PARAM_TENANT/alm-ci" HOST_DATA="$HOST_BASE/data" @@ -149,6 +149,7 @@ User=$CONTAINER_NAME Group=$CONTAINER_NAME ExecStart=$BIN_PATH/forgejo-runner daemon Restart=always +RestartSec=5 StandardOutput=append:/opt/gbo/logs/output.log StandardError=append:/opt/gbo/logs/error.log @@ -168,5 +169,5 @@ LXC_PROXY="/opt/gbo/tenants/$PARAM_TENANT/proxy/data/websites" LXC_SYSTEM="/opt/gbo/tenants/$PARAM_TENANT/system/bin" lxc config device add "$CONTAINER_NAME" almbot disk source="$LXC_BOT" path=/opt/gbo/bin/bot -lxc config device add "$CONTAINER_NAME" almproxy disk source="$LXC_PROXY" path=/opt/gbo/bin/proxy +lxc config device add "$CONTAINER_NAME" almproxy disk source="$LXC_PROXY" path=/opt/gbo/bin/proxy lxc config device add "$CONTAINER_NAME" almsystem disk source="$LXC_SYSTEM" path=/opt/gbo/bin/syst em || exit 1 diff --git a/src/scripts/utils/set-size-5GB.sh b/src/scripts/utils/set-size-5GB.sh index e2d5de2..6de4216 100644 --- a/src/scripts/utils/set-size-5GB.sh +++ b/src/scripts/utils/set-size-5GB.sh @@ -1,10 +1,7 @@ -export container="pragmatismo-alm-ci" -lxc stop "$container" +lxc config device override $CONTAINER_NAME root +lxc config device set $CONTAINER_NAME root size 6GB -lxc config device override "$container" root size=15GB -lxc config device set "$container" root size=15GB -lxc start "$container" -ROOT_DEV=$(lxc exec "$container" -- df / --output=source | tail -1) - -lxc exec "$container" -- growpart "$(dirname "$ROOT_DEV")" "$(basename "$ROOT_DEV")" -lxc exec "$container" -- resize2fs "$ROOT_DEV" +zpool set autoexpand=on default +zpool online -e default /var/snap/lxd/common/lxd/disks/default.img +zpool list +zfs list diff --git a/src/services/keywords/first.rs b/src/services/keywords/first.rs index 5d4c7fd..344eeb2 100644 --- a/src/services/keywords/first.rs +++ b/src/services/keywords/first.rs @@ -7,15 +7,179 @@ pub fn first_keyword(engine: &mut Engine) { move |context, inputs| { let input_string = context.eval_expression_tree(&inputs[0])?; let input_str = input_string.to_string(); - + // Extract first word by splitting on whitespace - let first_word = input_str.split_whitespace() + let first_word = input_str + .split_whitespace() .next() .unwrap_or("") .to_string(); - + Ok(Dynamic::from(first_word)) } }) .unwrap(); -} \ No newline at end of file +} + +#[cfg(test)] +mod tests { + use super::*; + use rhai::{Dynamic, Engine}; + + fn setup_engine() -> Engine { + let mut engine = Engine::new(); + first_keyword(&mut engine); + engine + } + + #[test] + fn test_first_keyword_basic() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + FIRST "hello world" + "#, + ) + .unwrap(); + + assert_eq!(result, "hello"); + } + + #[test] + fn test_first_keyword_single_word() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + FIRST "single" + "#, + ) + .unwrap(); + + assert_eq!(result, "single"); + } + + #[test] + fn test_first_keyword_multiple_spaces() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + FIRST " leading spaces" + "#, + ) + .unwrap(); + + assert_eq!(result, "leading"); + } + + #[test] + fn test_first_keyword_empty_string() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + FIRST "" + "#, + ) + .unwrap(); + + assert_eq!(result, ""); + } + + #[test] + fn test_first_keyword_whitespace_only() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + FIRST " " + "#, + ) + .unwrap(); + + assert_eq!(result, ""); + } + + #[test] + fn test_first_keyword_with_tabs() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + FIRST " tab separated words" + "#, + ) + .unwrap(); + + assert_eq!(result, "tab"); + } + + #[test] + fn test_first_keyword_with_variable() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + let text = "variable test"; + FIRST text + "#, + ) + .unwrap(); + + assert_eq!(result, "variable"); + } + + #[test] + fn test_first_keyword_with_expression() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + FIRST "one two " + "three four" + "#, + ) + .unwrap(); + + assert_eq!(result, "one"); + } + + #[test] + fn test_first_keyword_mixed_whitespace() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + FIRST " multiple spaces between words " + "#, + ) + .unwrap(); + + assert_eq!(result, "multiple"); + } + + #[test] + fn test_first_keyword_special_characters() { + let engine = setup_engine(); + + let result = engine + .eval::( + r#" + FIRST "hello-world example" + "#, + ) + .unwrap(); + + assert_eq!(result, "hello-world"); + } +} diff --git a/src/services/llm_generic.rs b/src/services/llm_generic.rs index f20384d..6f802bf 100644 --- a/src/services/llm_generic.rs +++ b/src/services/llm_generic.rs @@ -1,4 +1,4 @@ -use log::info; +use log::{error, info}; use actix_web::{post, web, HttpRequest, HttpResponse, Result}; use dotenv::dotenv; @@ -41,7 +41,6 @@ fn clean_request_body(body: &str) -> String { let re = Regex::new(r#","?\s*"(max_completion_tokens|parallel_tool_calls|top_p|frequency_penalty|presence_penalty)"\s*:\s*[^,}]*"#).unwrap(); re.replace_all(body, "").to_string() } - #[post("/v1/chat/completions")] pub async fn generic_chat_completions(body: web::Bytes, _req: HttpRequest) -> Result { // Log raw POST data @@ -58,9 +57,19 @@ pub async fn generic_chat_completions(body: web::Bytes, _req: HttpRequest) -> Re let endpoint = env::var("AI_ENDPOINT") .map_err(|_| actix_web::error::ErrorInternalServerError("AI_ENDPOINT not set."))?; - // Clean the request body (remove unsupported parameters) - let cleaned_body_str = clean_request_body(body_str); - info!("Cleaned POST Data: {}", cleaned_body_str); + // Parse and modify the request body + let mut json_value: serde_json::Value = serde_json::from_str(body_str) + .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to parse JSON"))?; + + // Add model parameter + if let Some(obj) = json_value.as_object_mut() { + obj.insert("model".to_string(), serde_json::Value::String(model)); + } + + let modified_body_str = serde_json::to_string(&json_value) + .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to serialize JSON"))?; + + info!("Modified POST Data: {}", modified_body_str); // Set up headers let mut headers = reqwest::header::HeaderMap::new(); @@ -74,21 +83,7 @@ pub async fn generic_chat_completions(body: web::Bytes, _req: HttpRequest) -> Re reqwest::header::HeaderValue::from_static("application/json"), ); - // After cleaning the request body, add the unused parameter - let mut json_value: serde_json::Value = serde_json::from_str(&cleaned_body_str) - .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to parse JSON"))?; - - // Add the unused parameter - json_value - .as_object_mut() - .unwrap() - .insert("model".to_string(), serde_json::Value::String(model)); - - // Serialize the modified JSON - let modified_body_str = serde_json::to_string(&json_value) - .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to serialize JSON"))?; - - // Send request to the OpenAI-compatible provider + // Send request to the AI provider let client = Client::new(); let response = client .post(&endpoint) @@ -108,13 +103,104 @@ pub async fn generic_chat_completions(body: web::Bytes, _req: HttpRequest) -> Re info!("Provider response status: {}", status); info!("Provider response body: {}", raw_response); - // Return the response with appropriate status code + // Convert response to OpenAI format if successful if status.is_success() { - Ok(HttpResponse::Ok().body(raw_response)) + match convert_to_openai_format(&raw_response) { + Ok(openai_response) => Ok(HttpResponse::Ok() + .content_type("application/json") + .body(openai_response)), + Err(e) => { + error!("Failed to convert response format: {}", e); + // Return the original response if conversion fails + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(raw_response)) + } + } } else { + // Return error as-is let actix_status = actix_web::http::StatusCode::from_u16(status.as_u16()) .unwrap_or(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR); - Ok(HttpResponse::build(actix_status).body(raw_response)) + Ok(HttpResponse::build(actix_status) + .content_type("application/json") + .body(raw_response)) } } + +/// Converts provider response to OpenAI-compatible format +fn convert_to_openai_format(provider_response: &str) -> Result> { + #[derive(serde::Deserialize)] + struct ProviderResponse { + text: String, + #[serde(default)] + generated_tokens: Option, + #[serde(default)] + input_tokens: Option, + } + + #[derive(serde::Serialize)] + struct OpenAIResponse { + id: String, + object: String, + created: u64, + model: String, + choices: Vec, + usage: OpenAIUsage, + } + + #[derive(serde::Serialize)] + struct OpenAIChoice { + index: u32, + message: OpenAIMessage, + finish_reason: String, + } + + #[derive(serde::Serialize)] + struct OpenAIMessage { + role: String, + content: String, + } + + #[derive(serde::Serialize)] + struct OpenAIUsage { + prompt_tokens: u32, + completion_tokens: u32, + total_tokens: u32, + } + + // Parse the provider response + let provider: ProviderResponse = serde_json::from_str(provider_response)?; + + let completion_tokens = provider + .generated_tokens + .unwrap_or_else(|| provider.text.split_whitespace().count() as u32); + + let prompt_tokens = provider.input_tokens.unwrap_or(0); + let total_tokens = prompt_tokens + completion_tokens; + + let openai_response = OpenAIResponse { + id: format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()), + object: "chat.completion".to_string(), + created: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), + model: "llama".to_string(), + choices: vec![OpenAIChoice { + index: 0, + message: OpenAIMessage { + role: "assistant".to_string(), + content: provider.text, + }, + finish_reason: "stop".to_string(), + }], + usage: OpenAIUsage { + prompt_tokens, + completion_tokens, + total_tokens, + }, + }; + + serde_json::to_string(&openai_response).map_err(|e| e.into()) +} diff --git a/src/services/llm_local.rs b/src/services/llm_local.rs index 5cebd8d..023a5f6 100644 --- a/src/services/llm_local.rs +++ b/src/services/llm_local.rs @@ -456,7 +456,7 @@ struct LlamaCppEmbeddingRequest { struct LlamaCppEmbeddingResponseItem { #[serde(rename = "index")] pub _index: usize, - pub embedding: Vec>, // This is the fucked up part - embedding is an array of arrays + pub embedding: Vec>, // This is the up part - embedding is an array of arrays } // Proxy endpoint for embeddings