- Open AI compatible endpoint in GB.

api.json

@@ -1,537 +0,0 @@
-openapi: 3.0.0
-info:
-  title: General Bots API
-  description: API for managing files, documents, groups, conversations, and more.
-  version: 1.0.0
-servers:
-  - url: https://api.generalbots.com/v1
-    description: Production server
-paths:
-  /files/upload:
-    post:
-      summary: Upload a file
-      operationId: uploadFile
-      requestBody:
-        required: true
-        content:
-          multipart/form-data:
-            schema:
-              type: object
-              properties:
-                file:
-                  type: string
-                  format: binary
-      responses:
-        '200':
-          description: File uploaded successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  fileId:
-                    type: string
-                  url:
-                    type: string
-
-  /files/download:
-    post:
-      summary: Download a file
-      operationId: downloadFile
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                fileId:
-                  type: string
-      responses:
-        '200':
-          description: File downloaded successfully
-          content:
-            application/octet-stream:
-              schema:
-                type: string
-                format: binary
-
-  /files/copy:
-    post:
-      summary: Copy a file
-      operationId: copyFile
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                sourcePath:
-                  type: string
-                destinationPath:
-                  type: string
-      responses:
-        '200':
-          description: File copied successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  message:
-                    type: string
-
-  /files/move:
-    post:
-      summary: Move a file
-      operationId: moveFile
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                sourcePath:
-                  type: string
-                destinationPath:
-                  type: string
-      responses:
-        '200':
-          description: File moved successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  message:
-                    type: string
-
-  /files/delete:
-    post:
-      summary: Delete a file
-      operationId: deleteFile
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                fileId:
-                  type: string
-      responses:
-        '200':
-          description: File deleted successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  message:
-                    type: string
-
-  /files/getContents:
-    post:
-      summary: Get file contents
-      operationId: getFileContents
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                fileId:
-                  type: string
-      responses:
-        '200':
-          description: File contents retrieved successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  contents:
-                    type: string
-
-  /files/save:
-    post:
-      summary: Save a file
-      operationId: saveFile
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                fileId:
-                  type: string
-                contents:
-                  type: string
-      responses:
-        '200':
-          description: File saved successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  message:
-                    type: string
-
-  /files/createFolder:
-    post:
-      summary: Create a folder
-      operationId: createFolder
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                folderName:
-                  type: string
-                parentFolderId:
-                  type: string
-      responses:
-        '200':
-          description: Folder created successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  folderId:
-                    type: string
-
-  /files/shareFolder:
-    post:
-      summary: Share a folder
-      operationId: shareFolder
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                folderId:
-                  type: string
-                userIds:
-                  type: array
-                  items:
-                    type: string
-      responses:
-        '200':
-          description: Folder shared successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  message:
-                    type: string
-
-  /files/dirFolder:
-    post:
-      summary: List folder contents
-      operationId: dirFolder
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                folderId:
-                  type: string
-      responses:
-        '200':
-          description: Folder contents retrieved successfully
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: object
-                  properties:
-                    name:
-                      type: string
-                    type:
-                      type: string
-                    size:
-                      type: integer
-
-  /files/list:
-    post:
-      summary: List files
-      operationId: getFiles
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                folderId:
-                  type: string
-      responses:
-        '200':
-          description: Files listed successfully
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: object
-                  properties:
-                    name:
-                      type: string
-                    type:
-                      type: string
-                    size:
-                      type: integer
-
-  /files/search:
-    post:
-      summary: Search files
-      operationId: searchFiles
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                query:
-                  type: string
-      responses:
-        '200':
-          description: Files searched successfully
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: object
-                  properties:
-                    name:
-                      type: string
-                    type:
-                      type: string
-                    size:
-                      type: integer
-
-  /files/recent:
-    post:
-      summary: Get recent files
-      operationId: getRecentFiles
-      responses:
-        '200':
-          description: Recent files retrieved successfully
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: object
-                  properties:
-                    name:
-                      type: string
-                    type:
-                      type: string
-                    size:
-                      type: integer
-
-  /files/favorite:
-    post:
-      summary: Toggle favorite status of a file
-      operationId: toggleFavorite
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                fileId:
-                  type: string
-      responses:
-        '200':
-          description: Favorite status toggled successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  isFavorite:
-                    type: boolean
-
-  /files/versions:
-    post:
-      summary: Get file versions
-      operationId: getFileVersions
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                fileId:
-                  type: string
-      responses:
-        '200':
-          description: File versions retrieved successfully
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: object
-                  properties:
-                    versionId:
-                      type: string
-                    timestamp:
-                      type: string
-                    size:
-                      type: integer
-
-  /files/restore:
-    post:
-      summary: Restore a file version
-      operationId: restoreFileVersion
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                fileId:
-                  type: string
-                versionId:
-                  type: string
-      responses:
-        '200':
-          description: File version restored successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  message:
-                    type: string
-
-  /files/permissions:
-    post:
-      summary: Set file permissions
-      operationId: setFilePermissions
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                fileId:
-                  type: string
-                permissions:
-                  type: object
-      responses:
-        '200':
-          description: File permissions updated successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  message:
-                    type: string
-
-  /files/quota:
-    get:
-      summary: Get storage quota
-      operationId: getStorageQuota
-      responses:
-        '200':
-          description: Storage quota retrieved successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  used:
-                    type: integer
-                  total:
-                    type: integer
-
-  /files/shared:
-    get:
-      summary: Get shared files
-      operationId: getSharedFiles
-      responses:
-        '200':
-          description: Shared files retrieved successfully
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: object
-                  properties:
-                    name:
-                      type: string
-                    type:
-                      type: string
-                    size:
-                      type: integer
-
-  /files/sync/status:
-    get:
-      summary: Get sync status
-      operationId: getSyncStatus
-      responses:
-        '200':
-          description: Sync status retrieved successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  status:
-                    type: string
-
-  /files/sync/start:
-    post:
-      summary: Start sync
-      operationId: startSync
-      responses:
-        '200':
-          description: Sync started successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  message:
-                    type: string
-
-  /files/sync/stop:
-    post:
-      summary: Stop sync
-      operationId: stopSync
-      responses:
-        '200':
-          description: Sync stopped successfully
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  message:
-                    type: string

src/main.rs

@@ -1,7 +1,6 @@
+use actix_web::middleware::Logger;
 use std::sync::Arc;
 
-use actix_cors::Cors;
-use actix_web::http::header;
 use actix_web::{web, App, HttpServer};
 use dotenv::dotenv;
 use services::state::*;
@@ -11,6 +10,7 @@ use sqlx::PgPool;
 use crate::services::automation::AutomationService;
 use crate::services::email::{get_emails, list_emails, save_click, send_email};
 use crate::services::llm::{chat, chat_stream};
+use crate::services::llm_local::chat_completions_local;
 use crate::services::llm_provider::chat_completions;
 use crate::services::web_automation::{initialize_browser_pool, BrowserPool};
 
@@ -38,7 +38,7 @@ async fn main() -> std::io::Result<()> {
         "/usr/bin/brave-browser-beta".to_string(),
     ));
 
-    #[cfg(feature = "local_llm")]
+    #[cfg(feature = "default")]
     {
         use crate::services::llm_local::ensure_llama_server_running;
 
@@ -67,15 +67,17 @@ async fn main() -> std::io::Result<()> {
 
     // Start HTTP server
     HttpServer::new(move || {
-        let cors = Cors::default()
-            .send_wildcard()
-            .allowed_methods(vec!["GET", "POST", "PUT", "DELETE"])
-            .allowed_headers(vec![header::AUTHORIZATION, header::ACCEPT])
-            .allowed_header(header::CONTENT_TYPE)
-            .max_age(3600);
+        // let cors = Cors::default()
+        //     .send_wildcard()
+        //     .allowed_methods(vec!["GET", "POST", "PUT", "DELETE"])
+        //     .allowed_headers(vec![header::AUTHORIZATION, header::ACCEPT])
+        //     .allowed_header(header::CONTENT_TYPE)
+        //     .max_age(3600);
+        //.wrap(cors)
 
         App::new()
-            .wrap(cors)
+            .wrap(Logger::default())
+            .wrap(Logger::new("%a %{User-Agent}i"))
             .app_data(app_state.clone())
             .service(upload_file)
             .service(list_file)
@@ -85,6 +87,7 @@ async fn main() -> std::io::Result<()> {
             .service(send_email)
             .service(chat_stream)
             .service(chat_completions)
+            .service(chat_completions_local)
             .service(chat)
     })
     .bind((config.server.host.clone(), config.server.port))?

src/scripts/utils/set-limits.sh

@@ -27,16 +27,16 @@ CPU_PRIORITY=5
 
 for pattern in "${!container_limits[@]}"; do
     echo "Configuring $container..."
-    
+
     memory=$DEFAULT_MEMORY
     cpu_allowance=$DEFAULT_CPU_ALLOWANCE
-    
+
     # Configure all containers
     for container in $(lxc list -c n --format csv); do
     # Check if container matches any pattern
         if [[ $container == $pattern ]]; then
             IFS=':' read -r memory cpu_allowance <<< "${container_limits[$pattern]}"
-            
+
             # Apply configuration
             lxc config set "$container" limits.memory "$memory"
             lxc config set "$container" limits.cpu.allowance "$cpu_allowance"
@@ -50,4 +50,4 @@ for pattern in "${!container_limits[@]}"; do
             break
         fi
     done
-done
+done

src/services/llm_local.rs

@@ -93,7 +93,7 @@ async fn start_llama_server() -> Result<(), Box<dyn std::error::Error + Send + S
     // Get environment variables for llama.cpp configuration
     let llama_path = env::var("LLM_CPP_PATH").unwrap_or_else(|_| "llama-server".to_string());
     let model_path = env::var("LLM_MODEL_PATH")
-        .unwrap_or_else(|_| "./models/tinyllama-1.1b-q4_01.gguf".to_string());
+        .unwrap_or_else(|_| "./tinyllama-1.1b-chat-v1.0.Q4_0.gguf".to_string());
     let cpu_limit = env::var("CPU_LIMIT").unwrap_or_else(|_| "50".to_string());
     let port = env::var("LLM_PORT").unwrap_or_else(|_| "8080".to_string());
 
@@ -105,15 +105,15 @@ async fn start_llama_server() -> Result<(), Box<dyn std::error::Error + Send + S
 
     // Kill any existing llama processes
     println!("🧹 Cleaning up existing processes...");
-    let _ = Command::new("pkill").arg("-f").arg("llama-server").output();
+    let _ = Command::new("pkill").arg("-f").arg("llama-cli").output();
 
     // Wait a bit for cleanup
     sleep(Duration::from_secs(2)).await;
 
     // Build the command
     let full_command = format!(
-        "cpulimit -l {} -- {} -m '{}' --n-gpu-layers 18 --temp 0.7 --ctx-size 1024 --batch-size 256 --no-mmap --mlock --port {} --host 127.0.0.1 --tensor-split 1.0 --main-gpu 0",
-        cpu_limit, llama_path, model_path, port
+        "{}/llama-server -m {} --mlock --port {} --host 127.0.0.1",
+        llama_path, model_path, port
     );
 
     println!("📝 Executing command: {}", full_command);
@@ -238,12 +238,12 @@ fn messages_to_prompt(messages: &[ChatMessage]) -> String {
 }
 
 // Proxy endpoint
-#[post("/v1/chat/completions")]
-pub async fn chat_completions(
+#[post("/v1/chat/completions1")]
+pub async fn chat_completions_local(
     req_body: web::Json<ChatCompletionRequest>,
     _req: HttpRequest,
 ) -> Result<HttpResponse> {
-    dotenv().ok();
+    dotenv().ok().unwrap();
 
     // Ensure llama.cpp server is running
     if let Err(e) = ensure_llama_server_running().await {

src/services/llm_provider.rs

@@ -1,5 +1,6 @@
 use actix_web::{post, web, HttpRequest, HttpResponse, Result};
 use dotenv::dotenv;
+use regex::Regex;
 use reqwest::Client;
 use serde::{Deserialize, Serialize};
 use std::env;
@@ -33,12 +34,15 @@ struct Choice {
     finish_reason: String,
 }
 
-// Proxy endpoint
 #[post("/v1/chat/completions")]
-async fn chat_completions(
-    req_body: web::Json<ChatCompletionRequest>,
-    _req: HttpRequest,
-) -> Result<HttpResponse> {
+async fn chat_completions(body: web::Bytes, _req: HttpRequest) -> Result<HttpResponse> {
+    // Always log raw POST data
+    if let Ok(body_str) = std::str::from_utf8(&body) {
+        println!("POST Data: {}", body_str);
+    } else {
+        println!("POST Data (binary): {:?}", body);
+    }
+
     dotenv().ok();
 
     // Environment variables
@@ -67,12 +71,23 @@ async fn chat_completions(
         reqwest::header::HeaderValue::from_static("application/json"),
     );
 
+    let body_str = std::str::from_utf8(&body).unwrap_or("");
+    println!("Original POST Data: {}", body_str);
+
+    // Remove the problematic params
+    let re =
+        Regex::new(r#","?\s*"(max_completion_tokens|parallel_tool_calls)"\s*:\s*[^,}]*"#).unwrap();
+    let cleaned = re.replace_all(body_str, "");
+    let cleaned_body = web::Bytes::from(cleaned.to_string());
+
+    println!("Cleaned POST Data: {}", cleaned);
+
     // Send request to Azure
     let client = Client::new();
     let response = client
         .post(&url)
         .headers(headers)
-        .json(&req_body.into_inner())
+        .body(cleaned_body)
         .send()
         .await
         .map_err(actix_web::error::ErrorInternalServerError)?;
@@ -88,11 +103,7 @@ async fn chat_completions(
     println!("Raw Azure response: {}", raw_response);
 
     if status.is_success() {
-        // Parse the raw response as JSON
-        let azure_response: serde_json::Value = serde_json::from_str(&raw_response)
-            .map_err(actix_web::error::ErrorInternalServerError)?;
-
-        Ok(HttpResponse::Ok().json(azure_response))
+        Ok(HttpResponse::Ok().body(raw_response))
     } else {
         // Handle error responses properly
         let actix_status = actix_web::http::StatusCode::from_u16(status.as_u16())