Fix task UI and MinIO app generation

- Fix MinIO bucket name sanitization (replace spaces with hyphens)
- Write apps to MinIO path: botname.gbapp/appname/files
- Serve apps directly from MinIO via /apps/:app_name route
- Add WebSocket reconnection on HTMX page load
- Remove sync_app_to_site_root (drive monitor handles sync)
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-12-31 12:38:35 -03:00
parent 061c14b4a2
commit 50d58ff59f
10 changed files with 1738 additions and 639 deletions

View file

@ -120,6 +120,7 @@ dotenvy = "0.15"
env_logger = "0.11" env_logger = "0.11"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio-util = { version = "0.7", features = ["io", "compat"] }
hex = "0.4" hex = "0.4"
hmac = "0.12.1" hmac = "0.12.1"
hyper = { version = "1.4", features = ["full"] } hyper = { version = "1.4", features = ["full"] }
@ -241,6 +242,8 @@ rss = "2.0"
# HTML parsing/web scraping # HTML parsing/web scraping
scraper = "0.25" scraper = "0.25"
walkdir = "2.5.0" walkdir = "2.5.0"
hyper-util = { version = "0.1.19", features = ["client-legacy", "tokio"] }
http-body-util = "0.1.3"
[dev-dependencies] [dev-dependencies]
mockito = "1.7.0" mockito = "1.7.0"

View file

@ -13,6 +13,7 @@ use diesel::sql_query;
use log::{error, info, trace, warn}; use log::{error, info, trace, warn};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::mpsc;
use uuid::Uuid; use uuid::Uuid;
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@ -105,42 +106,50 @@ pub struct SyncResult {
pub migrations_applied: usize, pub migrations_applied: usize,
} }
#[derive(Debug, Clone, Deserialize)] /// Streaming format parsed app structure
#[derive(Debug, Clone, Default)]
struct LlmGeneratedApp { struct LlmGeneratedApp {
name: String, name: String,
description: String, description: String,
#[serde(default)] domain: String,
_domain: String,
tables: Vec<LlmTable>, tables: Vec<LlmTable>,
files: Vec<LlmFile>, files: Vec<LlmFile>,
tools: Option<Vec<LlmFile>>, tools: Vec<LlmFile>,
schedulers: Option<Vec<LlmFile>>, schedulers: Vec<LlmFile>,
} }
#[derive(Debug, Clone, Deserialize)] #[derive(Debug, Clone, Default)]
struct LlmTable { struct LlmTable {
name: String, name: String,
fields: Vec<LlmField>, fields: Vec<LlmField>,
} }
#[derive(Debug, Clone, Deserialize)] #[derive(Debug, Clone, Default)]
struct LlmField { struct LlmField {
name: String, name: String,
#[serde(rename = "type")]
field_type: String, field_type: String,
nullable: Option<bool>, nullable: bool,
reference: Option<String>, reference: Option<String>,
default: Option<String>, default: Option<String>,
} }
#[derive(Debug, Clone, Deserialize)] #[derive(Debug, Clone, Default)]
struct LlmFile { struct LlmFile {
filename: String, filename: String,
content: String, content: String,
#[serde(rename = "type", default)]
_file_type: Option<String>,
} }
/// Streaming delimiter constants
const DELIM_APP_START: &str = "<<<APP_START>>>";
const DELIM_APP_END: &str = "<<<APP_END>>>";
const DELIM_TABLES_START: &str = "<<<TABLES_START>>>";
const DELIM_TABLES_END: &str = "<<<TABLES_END>>>";
const DELIM_TABLE_PREFIX: &str = "<<<TABLE:";
const DELIM_FILE_PREFIX: &str = "<<<FILE:";
const DELIM_TOOL_PREFIX: &str = "<<<TOOL:";
const DELIM_SCHEDULER_PREFIX: &str = "<<<SCHEDULER:";
const DELIM_END: &str = ">>>";
pub struct AppGenerator { pub struct AppGenerator {
state: Arc<AppState>, state: Arc<AppState>,
task_id: Option<String>, task_id: Option<String>,
@ -253,13 +262,13 @@ impl AppGenerator {
activity activity
); );
info!("[APP_GENERATOR] Calling generate_complete_app_with_llm for intent: {}", &intent[..intent.len().min(50)]); trace!("APP_GENERATOR Calling LLM for intent: {}", &intent[..intent.len().min(50)]);
let llm_start = std::time::Instant::now(); let llm_start = std::time::Instant::now();
let llm_app = match self.generate_complete_app_with_llm(intent, session.bot_id).await { let llm_app = match self.generate_complete_app_with_llm(intent, session.bot_id).await {
Ok(app) => { Ok(app) => {
let llm_elapsed = llm_start.elapsed(); let llm_elapsed = llm_start.elapsed();
info!("[APP_GENERATOR] LLM generation completed in {:?}: app={}, files={}, tables={}", info!("APP_GENERATOR LLM completed in {:?}: app={}, files={}, tables={}",
llm_elapsed, app.name, app.files.len(), app.tables.len()); llm_elapsed, app.name, app.files.len(), app.tables.len());
log_generator_info( log_generator_info(
&app.name, &app.name,
@ -286,7 +295,7 @@ impl AppGenerator {
} }
Err(e) => { Err(e) => {
let llm_elapsed = llm_start.elapsed(); let llm_elapsed = llm_start.elapsed();
error!("[APP_GENERATOR] LLM generation failed after {:?}: {}", llm_elapsed, e); error!("APP_GENERATOR LLM failed after {:?}: {}", llm_elapsed, e);
log_generator_error("unknown", "LLM app generation failed", &e.to_string()); log_generator_error("unknown", "LLM app generation failed", &e.to_string());
if let Some(ref task_id) = self.task_id { if let Some(ref task_id) = self.task_id {
self.state.emit_task_error(task_id, "llm_request", &e.to_string()); self.state.emit_task_error(task_id, "llm_request", &e.to_string());
@ -356,8 +365,12 @@ impl AppGenerator {
} }
let bot_name = self.get_bot_name(session.bot_id)?; let bot_name = self.get_bot_name(session.bot_id)?;
let bucket_name = format!("{}.gbai", bot_name.to_lowercase()); // Sanitize bucket name - replace spaces and invalid characters
let drive_app_path = format!(".gbdrive/apps/{}", llm_app.name); let sanitized_name = bot_name.to_lowercase().replace(' ', "-").replace('_', "-");
let bucket_name = format!("{}.gbai", sanitized_name);
let drive_app_path = format!("{}.gbapp/{}", sanitized_name, llm_app.name);
info!("Writing app files to bucket: {}, path: {}", bucket_name, drive_app_path);
let total_files = llm_app.files.len(); let total_files = llm_app.files.len();
let activity = self.build_activity("writing", 0, Some(total_files as u32), Some("Preparing files")); let activity = self.build_activity("writing", 0, Some(total_files as u32), Some("Preparing files"));
@ -390,6 +403,7 @@ impl AppGenerator {
activity activity
); );
// Write to MinIO - drive monitor will sync to SITES_ROOT
if let Err(e) = self if let Err(e) = self
.write_to_drive(&bucket_name, &drive_path, &file.content) .write_to_drive(&bucket_name, &drive_path, &file.content)
.await .await
@ -415,6 +429,8 @@ impl AppGenerator {
let designer_js = Self::generate_designer_js(&llm_app.name); let designer_js = Self::generate_designer_js(&llm_app.name);
self.bytes_generated += designer_js.len() as u64; self.bytes_generated += designer_js.len() as u64;
// Write designer.js to MinIO
self.write_to_drive( self.write_to_drive(
&bucket_name, &bucket_name,
&format!("{}/designer.js", drive_app_path), &format!("{}/designer.js", drive_app_path),
@ -423,8 +439,8 @@ impl AppGenerator {
.await?; .await?;
let mut tools = Vec::new(); let mut tools = Vec::new();
if let Some(llm_tools) = &llm_app.tools { if !llm_app.tools.is_empty() {
let tools_count = llm_tools.len(); let tools_count = llm_app.tools.len();
let activity = self.build_activity("tools", 0, Some(tools_count as u32), Some("Creating BASIC tools")); let activity = self.build_activity("tools", 0, Some(tools_count as u32), Some("Creating BASIC tools"));
self.emit_activity( self.emit_activity(
"write_tools", "write_tools",
@ -434,7 +450,7 @@ impl AppGenerator {
activity activity
); );
for (idx, tool) in llm_tools.iter().enumerate() { for (idx, tool) in llm_app.tools.iter().enumerate() {
let tool_path = format!(".gbdialog/tools/{}", tool.filename); let tool_path = format!(".gbdialog/tools/{}", tool.filename);
self.files_written.push(format!("tools/{}", tool.filename)); self.files_written.push(format!("tools/{}", tool.filename));
self.bytes_generated += tool.content.len() as u64; self.bytes_generated += tool.content.len() as u64;
@ -461,8 +477,8 @@ impl AppGenerator {
} }
let mut schedulers = Vec::new(); let mut schedulers = Vec::new();
if let Some(llm_schedulers) = &llm_app.schedulers { if !llm_app.schedulers.is_empty() {
let sched_count = llm_schedulers.len(); let sched_count = llm_app.schedulers.len();
let activity = self.build_activity("schedulers", 0, Some(sched_count as u32), Some("Creating schedulers")); let activity = self.build_activity("schedulers", 0, Some(sched_count as u32), Some("Creating schedulers"));
self.emit_activity( self.emit_activity(
"write_schedulers", "write_schedulers",
@ -472,7 +488,7 @@ impl AppGenerator {
activity activity
); );
for (idx, scheduler) in llm_schedulers.iter().enumerate() { for (idx, scheduler) in llm_app.schedulers.iter().enumerate() {
let scheduler_path = format!(".gbdialog/schedulers/{}", scheduler.filename); let scheduler_path = format!(".gbdialog/schedulers/{}", scheduler.filename);
self.files_written.push(format!("schedulers/{}", scheduler.filename)); self.files_written.push(format!("schedulers/{}", scheduler.filename));
self.bytes_generated += scheduler.content.len() as u64; self.bytes_generated += scheduler.content.len() as u64;
@ -498,11 +514,8 @@ impl AppGenerator {
} }
} }
let activity = self.build_activity("syncing", TOTAL_STEPS as u32 - 1, Some(TOTAL_STEPS as u32), Some("Deploying to site")); let activity = self.build_activity("complete", TOTAL_STEPS as u32, Some(TOTAL_STEPS as u32), Some("App ready"));
self.emit_activity("sync_site", "Syncing app to site...", 8, TOTAL_STEPS, activity); self.emit_activity("complete", "App written to drive, ready to serve from MinIO", 8, TOTAL_STEPS, activity);
self.sync_app_to_site_root(&bucket_name, &llm_app.name, session.bot_id)
.await?;
let elapsed = self.generation_start.map(|s| s.elapsed().as_secs()).unwrap_or(0); let elapsed = self.generation_start.map(|s| s.elapsed().as_secs()).unwrap_or(0);
@ -679,55 +692,279 @@ If user says "inventory" → build stock tracking with products, categories, mov
If user says "booking" build appointment scheduler with calendar, slots, confirmations If user says "booking" build appointment scheduler with calendar, slots, confirmations
If user says ANYTHING interpret creatively and BUILD SOMETHING AWESOME If user says ANYTHING interpret creatively and BUILD SOMETHING AWESOME
Respond with a single JSON object: === OUTPUT FORMAT (STREAMING DELIMITERS) ===
{{
"name": "app-name-lowercase-dashes",
"description": "What this app does",
"domain": "healthcare|sales|inventory|booking|utility|etc",
"tables": [
{{
"name": "table_name",
"fields": [
{{"name": "id", "type": "guid", "nullable": false}},
{{"name": "created_at", "type": "datetime", "nullable": false, "default": "now()"}},
{{"name": "updated_at", "type": "datetime", "nullable": false, "default": "now()"}},
{{"name": "field_name", "type": "string", "nullable": true, "reference": null}}
]
}}
],
"files": [
{{"filename": "index.html", "content": "<!DOCTYPE html>...complete HTML..."}},
{{"filename": "styles.css", "content": ":root {{...}} body {{...}} ...complete CSS..."}},
{{"filename": "table_name.html", "content": "<!DOCTYPE html>...list page..."}},
{{"filename": "table_name_form.html", "content": "<!DOCTYPE html>...form page..."}}
],
"tools": [
{{"filename": "app_helper.bas", "content": "HEAR \"help\"\n TALK \"I can help with...\"\nEND HEAR"}}
],
"schedulers": [
{{"filename": "daily_report.bas", "content": "SET SCHEDULE \"0 9 * * *\"\n ...\nEND SCHEDULE"}}
]
}}
CRITICAL RULES: Use this EXACT format with delimiters (NOT JSON) so content can stream safely:
- For utilities (calculator, timer, converter, BMI, mortgage): tables = [], focus on interactive HTML/JS
<<<APP_START>>>
name: app-name-lowercase-dashes
description: What this app does
domain: healthcare|sales|inventory|booking|utility|etc
<<<TABLES_START>>>
<<<TABLE:table_name>>>
id:guid:false
created_at:datetime:false:now()
updated_at:datetime:false:now()
field_name:string:true
foreign_key:guid:false:ref:other_table
<<<TABLE:another_table>>>
id:guid:false
name:string:true
<<<TABLES_END>>>
<<<FILE:index.html>>>
<!DOCTYPE html>
<html lang="en">
... complete HTML content here ...
</html>
<<<FILE:styles.css>>>
:root {{ --primary: #3b82f6; }}
body {{ margin: 0; font-family: system-ui; }}
... complete CSS content here ...
<<<FILE:table_name.html>>>
<!DOCTYPE html>
... complete list page ...
<<<FILE:table_name_form.html>>>
<!DOCTYPE html>
... complete form page ...
<<<TOOL:app_helper.bas>>>
HEAR "help"
TALK "I can help with..."
END HEAR
<<<SCHEDULER:daily_report.bas>>>
SET SCHEDULE "0 9 * * *"
data = GET FROM "table"
SEND MAIL TO "admin@example.com" WITH SUBJECT "Daily Report" BODY data
END SCHEDULE
<<<APP_END>>>
=== TABLE FIELD FORMAT ===
Each field on its own line: name:type:nullable[:default][:ref:table]
- Types: guid, string, text, integer, decimal, boolean, date, datetime, json
- nullable: true or false
- default: optional, e.g., now(), 0, ''
- ref:table: optional foreign key reference
=== CRITICAL RULES ===
- For utilities (calculator, timer, converter): TABLES_START/END with nothing between, focus on HTML/JS
- For data apps (CRM, inventory): design proper tables and CRUD pages - For data apps (CRM, inventory): design proper tables and CRUD pages
- Generate ALL files completely - no placeholders, no "...", no shortcuts - Generate ALL files completely - no placeholders, no "...", no shortcuts
- CSS must be comprehensive with variables, responsive design, dark mode - CSS must be comprehensive with variables, responsive design, dark mode
- Every HTML page needs proper structure with all required scripts - Every HTML page needs proper structure with all required scripts
- Replace APP_NAME_HERE with actual app name in data-app-name attribute - Replace APP_NAME_HERE with actual app name in data-app-name attribute
- BE CREATIVE - add extra features the user didn't ask for but would love - BE CREATIVE - add extra features the user didn't ask for but would love
- Use the EXACT delimiter format above - this allows streaming progress!
Respond with valid JSON only. NO QUESTIONS. JUST BUILD."# NO QUESTIONS. JUST BUILD."#
); );
let response = self.call_llm(&prompt, bot_id).await?; let response = self.call_llm(&prompt, bot_id).await?;
Self::parse_llm_app_response(&response) Self::parse_streaming_response(&response)
} }
fn parse_llm_app_response( /// Parse streaming delimiter format response
fn parse_streaming_response(
response: &str, response: &str,
) -> Result<LlmGeneratedApp, Box<dyn std::error::Error + Send + Sync>> { ) -> Result<LlmGeneratedApp, Box<dyn std::error::Error + Send + Sync>> {
let mut app = LlmGeneratedApp::default();
// Find APP_START and APP_END
let start_idx = response.find(DELIM_APP_START);
let end_idx = response.find(DELIM_APP_END);
let content = match (start_idx, end_idx) {
(Some(s), Some(e)) => &response[s + DELIM_APP_START.len()..e],
(Some(s), None) => {
warn!("No APP_END found, using rest of response");
&response[s + DELIM_APP_START.len()..]
}
_ => {
// Fallback: try to parse as JSON for backwards compatibility
return Self::parse_json_fallback(response);
}
};
let lines: Vec<&str> = content.lines().collect();
let mut current_section = "header";
let mut current_table: Option<LlmTable> = None;
let mut current_file: Option<(String, String, String)> = None; // (type, filename, content)
for raw_line in lines.iter() {
let line = raw_line.trim();
// Parse header fields
if current_section == "header" {
if line.starts_with("name:") {
app.name = line[5..].trim().to_string();
continue;
}
if line.starts_with("description:") {
app.description = line[12..].trim().to_string();
continue;
}
if line.starts_with("domain:") {
app.domain = line[7..].trim().to_string();
continue;
}
}
// Section transitions
if line == DELIM_TABLES_START {
current_section = "tables";
continue;
}
if line == DELIM_TABLES_END {
// Save any pending table
if let Some(table) = current_table.take() {
if !table.name.is_empty() {
app.tables.push(table);
}
}
current_section = "files";
continue;
}
// Table definitions
if line.starts_with(DELIM_TABLE_PREFIX) && line.ends_with(DELIM_END) {
// Save previous table
if let Some(table) = current_table.take() {
if !table.name.is_empty() {
app.tables.push(table);
}
}
let table_name = &line[DELIM_TABLE_PREFIX.len()..line.len() - DELIM_END.len()];
current_table = Some(LlmTable {
name: table_name.to_string(),
fields: Vec::new(),
});
continue;
}
// Table field (when in tables section with active table)
if current_section == "tables" && current_table.is_some() && !line.is_empty() && !line.starts_with("<<<") {
if let Some(ref mut table) = current_table {
if let Some(field) = Self::parse_field_line(line) {
table.fields.push(field);
}
}
continue;
}
// File definitions
if line.starts_with(DELIM_FILE_PREFIX) && line.ends_with(DELIM_END) {
// Save previous file
if let Some((file_type, filename, content)) = current_file.take() {
Self::save_parsed_file(&mut app, &file_type, filename, content);
}
let filename = &line[DELIM_FILE_PREFIX.len()..line.len() - DELIM_END.len()];
current_file = Some(("file".to_string(), filename.to_string(), String::new()));
continue;
}
// Tool definitions
if line.starts_with(DELIM_TOOL_PREFIX) && line.ends_with(DELIM_END) {
if let Some((file_type, filename, content)) = current_file.take() {
Self::save_parsed_file(&mut app, &file_type, filename, content);
}
let filename = &line[DELIM_TOOL_PREFIX.len()..line.len() - DELIM_END.len()];
current_file = Some(("tool".to_string(), filename.to_string(), String::new()));
continue;
}
// Scheduler definitions
if line.starts_with(DELIM_SCHEDULER_PREFIX) && line.ends_with(DELIM_END) {
if let Some((file_type, filename, content)) = current_file.take() {
Self::save_parsed_file(&mut app, &file_type, filename, content);
}
let filename = &line[DELIM_SCHEDULER_PREFIX.len()..line.len() - DELIM_END.len()];
current_file = Some(("scheduler".to_string(), filename.to_string(), String::new()));
continue;
}
// Accumulate file content (use original line to preserve indentation)
if let Some((_, _, ref mut file_content)) = current_file {
if !file_content.is_empty() {
file_content.push('\n');
}
file_content.push_str(raw_line);
}
}
// Save any remaining file
if let Some((file_type, filename, content)) = current_file.take() {
Self::save_parsed_file(&mut app, &file_type, filename, content);
}
// Validate
if app.name.is_empty() {
return Err("No app name found in response".into());
}
if app.files.is_empty() {
return Err("No files generated".into());
}
info!(
"Parsed streaming response: name={}, tables={}, files={}, tools={}, schedulers={}",
app.name,
app.tables.len(),
app.files.len(),
app.tools.len(),
app.schedulers.len()
);
Ok(app)
}
/// Parse a table field line in format: name:type:nullable[:default][:ref:table]
fn parse_field_line(line: &str) -> Option<LlmField> {
let parts: Vec<&str> = line.split(':').collect();
if parts.len() < 3 {
return None;
}
let mut field = LlmField {
name: parts[0].trim().to_string(),
field_type: parts[1].trim().to_string(),
nullable: parts[2].trim() == "true",
reference: None,
default: None,
};
// Parse optional parts
let mut i = 3;
while i < parts.len() {
if parts[i].trim() == "ref" && i + 1 < parts.len() {
field.reference = Some(parts[i + 1].trim().to_string());
i += 2;
} else {
// It's a default value
field.default = Some(parts[i].trim().to_string());
i += 1;
}
}
Some(field)
}
/// Save a parsed file to the appropriate collection
fn save_parsed_file(app: &mut LlmGeneratedApp, file_type: &str, filename: String, content: String) {
let file = LlmFile {
filename,
content: content.trim().to_string(),
};
match file_type {
"tool" => app.tools.push(file),
"scheduler" => app.schedulers.push(file),
_ => app.files.push(file),
}
}
/// Fallback to JSON parsing for backwards compatibility
fn parse_json_fallback(
response: &str,
) -> Result<LlmGeneratedApp, Box<dyn std::error::Error + Send + Sync>> {
warn!("Falling back to JSON parsing");
let cleaned = response let cleaned = response
.trim() .trim()
.trim_start_matches("```json") .trim_start_matches("```json")
@ -735,8 +972,93 @@ Respond with valid JSON only. NO QUESTIONS. JUST BUILD."#
.trim_end_matches("```") .trim_end_matches("```")
.trim(); .trim();
match serde_json::from_str::<LlmGeneratedApp>(cleaned) { #[derive(Debug, Deserialize)]
Ok(app) => { struct JsonApp {
name: String,
description: String,
#[serde(default)]
domain: String,
#[serde(default)]
tables: Vec<JsonTable>,
#[serde(default)]
files: Vec<JsonFile>,
#[serde(default)]
tools: Option<Vec<JsonFile>>,
#[serde(default)]
schedulers: Option<Vec<JsonFile>>,
}
#[derive(Debug, Deserialize)]
struct JsonTable {
name: String,
fields: Vec<JsonField>,
}
#[derive(Debug, Deserialize)]
struct JsonField {
name: String,
#[serde(rename = "type")]
field_type: String,
#[serde(default)]
nullable: Option<bool>,
#[serde(default)]
reference: Option<String>,
#[serde(default, deserialize_with = "deserialize_default_value")]
default: Option<String>,
}
#[derive(Debug, Deserialize)]
struct JsonFile {
filename: String,
content: String,
}
/// Deserialize default value that can be string, bool, number, or null
fn deserialize_default_value<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: serde::Deserializer<'de>,
{
let value: Option<serde_json::Value> = Option::deserialize(deserializer)?;
match value {
None => Ok(None),
Some(serde_json::Value::Null) => Ok(None),
Some(serde_json::Value::String(s)) => Ok(Some(s)),
Some(serde_json::Value::Bool(b)) => Ok(Some(b.to_string())),
Some(serde_json::Value::Number(n)) => Ok(Some(n.to_string())),
Some(v) => Ok(Some(v.to_string())),
}
}
match serde_json::from_str::<JsonApp>(cleaned) {
Ok(json_app) => {
let app = LlmGeneratedApp {
name: json_app.name,
description: json_app.description,
domain: json_app.domain,
tables: json_app.tables.into_iter().map(|t| LlmTable {
name: t.name,
fields: t.fields.into_iter().map(|f| LlmField {
name: f.name,
field_type: f.field_type,
nullable: f.nullable.unwrap_or(true),
reference: f.reference,
default: f.default,
}).collect(),
}).collect(),
files: json_app.files.into_iter().map(|f| LlmFile {
filename: f.filename,
content: f.content,
}).collect(),
tools: json_app.tools.unwrap_or_default().into_iter().map(|f| LlmFile {
filename: f.filename,
content: f.content,
}).collect(),
schedulers: json_app.schedulers.unwrap_or_default().into_iter().map(|f| LlmFile {
filename: f.filename,
content: f.content,
}).collect(),
};
if app.files.is_empty() { if app.files.is_empty() {
return Err("LLM generated no files".into()); return Err("LLM generated no files".into());
} }
@ -762,7 +1084,7 @@ Respond with valid JSON only. NO QUESTIONS. JUST BUILD."#
name: f.name.clone(), name: f.name.clone(),
field_type: f.field_type.clone(), field_type: f.field_type.clone(),
is_key: f.name == "id", is_key: f.name == "id",
is_nullable: f.nullable.unwrap_or(true), is_nullable: f.nullable,
reference_table: f.reference.clone(), reference_table: f.reference.clone(),
default_value: f.default.clone(), default_value: f.default.clone(),
field_order: i as i32, field_order: i as i32,
@ -820,23 +1142,101 @@ Respond with valid JSON only. NO QUESTIONS. JUST BUILD."#
}); });
let prompt_len = prompt.len(); let prompt_len = prompt.len();
info!("[APP_GENERATOR] Starting LLM call: model={}, prompt_len={} chars", model, prompt_len); trace!("APP_GENERATOR Starting LLM streaming: model={}, prompt_len={}", model, prompt_len);
let start = std::time::Instant::now(); let start = std::time::Instant::now();
// Use streaming to provide real-time feedback
let (tx, mut rx) = mpsc::channel::<String>(100);
let state = self.state.clone();
let task_id = self.task_id.clone();
// Spawn a task to receive stream chunks and broadcast them
let stream_task = tokio::spawn(async move {
let mut full_response = String::new();
let mut chunk_buffer = String::new();
let mut last_emit = std::time::Instant::now();
let mut chunk_count = 0u32;
let stream_start = std::time::Instant::now();
trace!("APP_GENERATOR Stream receiver started");
while let Some(chunk) = rx.recv().await {
chunk_count += 1;
full_response.push_str(&chunk);
chunk_buffer.push_str(&chunk);
// Log progress periodically
if chunk_count == 1 || chunk_count % 500 == 0 {
trace!("APP_GENERATOR Stream progress: {} chunks, {} chars, {:?}",
chunk_count, full_response.len(), stream_start.elapsed());
}
// Emit chunks every 100ms or when buffer has enough content
if last_emit.elapsed().as_millis() > 100 || chunk_buffer.len() > 50 {
if let Some(ref tid) = task_id {
state.emit_llm_stream(tid, &chunk_buffer);
}
chunk_buffer.clear();
last_emit = std::time::Instant::now();
}
}
trace!("APP_GENERATOR Stream finished: {} chunks, {} chars in {:?}",
chunk_count, full_response.len(), stream_start.elapsed());
// Emit any remaining buffer
if !chunk_buffer.is_empty() {
trace!("APP_GENERATOR Emitting final buffer: {} chars", chunk_buffer.len());
if let Some(ref tid) = task_id {
state.emit_llm_stream(tid, &chunk_buffer);
}
}
// Log response preview
if full_response.len() > 0 {
let preview = if full_response.len() > 200 {
format!("{}...", &full_response[..200])
} else {
full_response.clone()
};
trace!("APP_GENERATOR Response preview: {}", preview.replace('\n', "\\n"));
}
full_response
});
// Start the streaming LLM call
trace!("APP_GENERATOR Starting generate_stream...");
match self match self
.state .state
.llm_provider .llm_provider
.generate(prompt, &llm_config, &model, &key) .generate_stream(prompt, &llm_config, tx, &model, &key)
.await .await
{ {
Ok(response) => { Ok(()) => {
let elapsed = start.elapsed(); trace!("APP_GENERATOR generate_stream completed, waiting for stream_task");
info!("[APP_GENERATOR] LLM call succeeded: response_len={} chars, elapsed={:?}", response.len(), elapsed); // Wait for the stream task to complete and get the full response
return Ok(response); match stream_task.await {
Ok(response) => {
let elapsed = start.elapsed();
trace!("APP_GENERATOR LLM streaming succeeded: {} chars in {:?}", response.len(), elapsed);
if response.is_empty() {
error!("APP_GENERATOR Empty response from LLM");
}
return Ok(response);
}
Err(e) => {
let elapsed = start.elapsed();
error!("APP_GENERATOR LLM stream task failed after {:?}: {}", elapsed, e);
return Err(format!("Stream task failed: {}", e).into());
}
}
} }
Err(e) => { Err(e) => {
let elapsed = start.elapsed(); let elapsed = start.elapsed();
error!("[APP_GENERATOR] LLM call failed after {:?}: {}", elapsed, e); error!("APP_GENERATOR LLM streaming failed after {:?}: {}", elapsed, e);
// Abort the stream task
stream_task.abort();
return Err(e); return Err(e);
} }
} }
@ -947,26 +1347,100 @@ Respond with valid JSON only. NO QUESTIONS. JUST BUILD."#
.ok_or_else(|| format!("Bot not found: {}", bot_id).into()) .ok_or_else(|| format!("Bot not found: {}", bot_id).into())
} }
/// Ensure the bucket exists, creating it if necessary
async fn ensure_bucket_exists(
&self,
bucket: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if let Some(ref s3) = self.state.drive {
// Check if bucket exists
match s3.head_bucket().bucket(bucket).send().await {
Ok(_) => {
trace!("Bucket {} already exists", bucket);
return Ok(());
}
Err(_) => {
// Bucket doesn't exist, try to create it
info!("Bucket {} does not exist, creating...", bucket);
match s3.create_bucket().bucket(bucket).send().await {
Ok(_) => {
info!("Created bucket: {}", bucket);
return Ok(());
}
Err(e) => {
// Check if error is "bucket already exists" (race condition)
let err_str = format!("{:?}", e);
if err_str.contains("BucketAlreadyExists") || err_str.contains("BucketAlreadyOwnedByYou") {
trace!("Bucket {} already exists (race condition)", bucket);
return Ok(());
}
error!("Failed to create bucket {}: {}", bucket, e);
return Err(Box::new(e));
}
}
}
}
} else {
// No S3 client, we'll use DB fallback - no bucket needed
trace!("No S3 client, using DB fallback for storage");
Ok(())
}
}
async fn write_to_drive( async fn write_to_drive(
&self, &self,
bucket: &str, bucket: &str,
path: &str, path: &str,
content: &str, content: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if let Some(ref s3) = self.state.s3_client { info!("write_to_drive: bucket={}, path={}, content_len={}", bucket, path, content.len());
if let Some(ref s3) = self.state.drive {
let body = ByteStream::from(content.as_bytes().to_vec()); let body = ByteStream::from(content.as_bytes().to_vec());
let content_type = get_content_type(path); let content_type = get_content_type(path);
s3.put_object() info!("S3 client available, attempting put_object to s3://{}/{}", bucket, path);
match s3.put_object()
.bucket(bucket) .bucket(bucket)
.key(path) .key(path)
.body(body) .body(body)
.content_type(content_type) .content_type(content_type)
.send() .send()
.await?; .await
{
Ok(_) => {
info!("Successfully wrote to S3: s3://{}/{}", bucket, path);
}
Err(e) => {
// Log detailed error info
error!("S3 put_object failed: bucket={}, path={}, error={:?}", bucket, path, e);
error!("S3 error details: {}", e);
trace!("Wrote to S3: s3://{}/{}", bucket, path); // If bucket doesn't exist, try to create it and retry
let err_str = format!("{:?}", e);
if err_str.contains("NoSuchBucket") || err_str.contains("NotFound") {
warn!("Bucket {} not found, attempting to create...", bucket);
self.ensure_bucket_exists(bucket).await?;
// Retry the write
let body = ByteStream::from(content.as_bytes().to_vec());
s3.put_object()
.bucket(bucket)
.key(path)
.body(body)
.content_type(get_content_type(path))
.send()
.await?;
info!("Wrote to S3 after creating bucket: s3://{}/{}", bucket, path);
} else {
error!("S3 write failed (not a bucket issue): {}", err_str);
return Err(Box::new(e));
}
}
}
} else { } else {
warn!("No S3/drive client available, using DB fallback for {}/{}", bucket, path);
self.write_to_db_fallback(bucket, path, content)?; self.write_to_db_fallback(bucket, path, content)?;
} }
@ -1034,49 +1508,6 @@ Respond with valid JSON only. NO QUESTIONS. JUST BUILD."#
}) })
} }
async fn sync_app_to_site_root(
&self,
bucket: &str,
app_name: &str,
bot_id: Uuid,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let source_path = format!(".gbdrive/apps/{}", app_name);
let site_path = Self::get_site_path(bot_id);
if let Some(ref s3) = self.state.s3_client {
let list_result = s3
.list_objects_v2()
.bucket(bucket)
.prefix(&source_path)
.send()
.await?;
if let Some(contents) = list_result.contents {
for object in contents {
if let Some(key) = object.key {
let relative_path =
key.trim_start_matches(&source_path).trim_start_matches('/');
let dest_key = format!("{}/{}/{}", site_path, app_name, relative_path);
s3.copy_object()
.bucket(bucket)
.copy_source(format!("{}/{}", bucket, key))
.key(&dest_key)
.send()
.await?;
trace!("Synced {} to {}", key, dest_key);
}
}
}
}
let _ = self.store_app_metadata(bot_id, app_name, &format!("{}/{}", site_path, app_name));
info!("App synced to site root: {}/{}", site_path, app_name);
Ok(())
}
fn store_app_metadata( fn store_app_metadata(
&self, &self,
bot_id: Uuid, bot_id: Uuid,
@ -1102,9 +1533,7 @@ Respond with valid JSON only. NO QUESTIONS. JUST BUILD."#
Ok(()) Ok(())
} }
fn get_site_path(_bot_id: Uuid) -> String {
".gbdrive/site".to_string()
}
fn generate_designer_js(app_name: &str) -> String { fn generate_designer_js(app_name: &str) -> String {
format!( format!(

View file

@ -319,69 +319,78 @@ pub async fn create_and_execute_handler(
let task_id = Uuid::new_v4(); let task_id = Uuid::new_v4();
if let Err(e) = create_task_record(&state, task_id, &session, &request.intent) { if let Err(e) = create_task_record(&state, task_id, &session, &request.intent) {
error!("Failed to create task record: {}", e); error!("Failed to create task record: {}", e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(CreateAndExecuteResponse {
success: false,
task_id: String::new(),
status: "error".to_string(),
message: format!("Failed to create task: {}", e),
app_url: None,
created_resources: Vec::new(),
pending_items: Vec::new(),
error: Some(e.to_string()),
}),
);
} }
// Update status to running // Update status to running
let _ = update_task_status_db(&state, task_id, "running", None); let _ = update_task_status_db(&state, task_id, "running", None);
// Use IntentClassifier to classify and process with task tracking // Clone what we need for the background task
let classifier = IntentClassifier::new(Arc::clone(&state)); let state_clone = Arc::clone(&state);
let intent = request.intent.clone();
let session_clone = session.clone();
let task_id_str = task_id.to_string();
match classifier // Spawn background task to do the actual work
.classify_and_process_with_task_id(&request.intent, &session, Some(task_id.to_string())) tokio::spawn(async move {
.await info!("[AUTOTASK] Background task started for task_id={}", task_id_str);
{
Ok(result) => {
let status = if result.success {
"completed"
} else {
"failed"
};
let _ = update_task_status_db(&state, task_id, status, result.error.as_deref());
// Get any pending items (ASK LATER) // Use IntentClassifier to classify and process with task tracking
let pending_items = get_pending_items_for_bot(&state, session.bot_id); let classifier = IntentClassifier::new(state_clone.clone());
( match classifier
StatusCode::OK, .classify_and_process_with_task_id(&intent, &session_clone, Some(task_id_str.clone()))
Json(CreateAndExecuteResponse { .await
success: result.success, {
task_id: task_id.to_string(), Ok(result) => {
status: status.to_string(), let status = if result.success {
message: result.message, "completed"
app_url: result.app_url, } else {
created_resources: result "failed"
.created_resources };
.into_iter() let _ = update_task_status_db(&state_clone, task_id, status, result.error.as_deref());
.map(|r| CreatedResourceResponse { info!(
resource_type: r.resource_type, "[AUTOTASK] Background task completed: task_id={}, status={}, message={}",
name: r.name, task_id_str, status, result.message
path: r.path, );
}) }
.collect(), Err(e) => {
pending_items, let _ = update_task_status_db(&state_clone, task_id, "failed", Some(&e.to_string()));
error: result.error, error!(
}), "[AUTOTASK] Background task failed: task_id={}, error={}",
) task_id_str, e
);
}
} }
Err(e) => { });
let _ = update_task_status_db(&state, task_id, "failed", Some(&e.to_string()));
error!("Failed to process intent: {}", e); // Return immediately with task_id - client will poll for status
( info!("[AUTOTASK] Returning immediately with task_id={}", task_id);
StatusCode::INTERNAL_SERVER_ERROR, (
Json(CreateAndExecuteResponse { StatusCode::ACCEPTED,
success: false, Json(CreateAndExecuteResponse {
task_id: task_id.to_string(), success: true,
status: "failed".to_string(), task_id: task_id.to_string(),
message: "Failed to process request".to_string(), status: "running".to_string(),
app_url: None, message: "Task started, poll for status".to_string(),
created_resources: Vec::new(), app_url: None,
pending_items: Vec::new(), created_resources: Vec::new(),
error: Some(e.to_string()), pending_items: Vec::new(),
}), error: None,
) }),
} )
}
} }
pub async fn classify_intent_handler( pub async fn classify_intent_handler(
@ -754,6 +763,42 @@ pub async fn list_tasks_handler(
} }
} }
/// Get a single task by ID - used for polling task status
pub async fn get_task_handler(
State(state): State<Arc<AppState>>,
Path(task_id): Path<String>,
) -> impl IntoResponse {
info!("Getting task: {}", task_id);
match get_auto_task_by_id(&state, &task_id) {
Ok(Some(task)) => {
let error_str = task.error.as_ref().map(|e| e.message.clone());
(StatusCode::OK, Json(serde_json::json!({
"id": task.id,
"name": task.title,
"description": task.intent,
"status": format!("{:?}", task.status).to_lowercase(),
"progress": task.progress,
"current_step": task.current_step,
"total_steps": task.total_steps,
"error": error_str,
"created_at": task.created_at,
"updated_at": task.updated_at,
"completed_at": task.completed_at,
})))
},
Ok(None) => (StatusCode::NOT_FOUND, Json(serde_json::json!({
"error": "Task not found"
}))),
Err(e) => {
error!("Failed to get task {}: {}", task_id, e);
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({
"error": e.to_string()
})))
}
}
}
pub async fn get_stats_handler(State(state): State<Arc<AppState>>) -> impl IntoResponse { pub async fn get_stats_handler(State(state): State<Arc<AppState>>) -> impl IntoResponse {
match get_auto_task_stats(&state) { match get_auto_task_stats(&state) {
Ok(stats) => (StatusCode::OK, Json(stats)), Ok(stats) => (StatusCode::OK, Json(stats)),
@ -1366,6 +1411,102 @@ fn start_task_execution(
Ok(()) Ok(())
} }
/// Get a single auto task by ID
fn get_auto_task_by_id(
state: &Arc<AppState>,
task_id: &str,
) -> Result<Option<AutoTask>, Box<dyn std::error::Error + Send + Sync>> {
let mut conn = state.conn.get()?;
#[derive(QueryableByName)]
struct TaskRow {
#[diesel(sql_type = Text)]
id: String,
#[diesel(sql_type = Text)]
title: String,
#[diesel(sql_type = Text)]
intent: String,
#[diesel(sql_type = Text)]
status: String,
#[diesel(sql_type = diesel::sql_types::Float8)]
progress: f64,
#[diesel(sql_type = diesel::sql_types::Nullable<Text>)]
current_step: Option<String>,
#[diesel(sql_type = diesel::sql_types::Nullable<Text>)]
error: Option<String>,
#[diesel(sql_type = diesel::sql_types::Timestamptz)]
created_at: chrono::DateTime<Utc>,
#[diesel(sql_type = diesel::sql_types::Timestamptz)]
updated_at: chrono::DateTime<Utc>,
#[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Timestamptz>)]
completed_at: Option<chrono::DateTime<Utc>>,
}
let query = format!(
"SELECT id::text, title, intent, status, progress, current_step, error, created_at, updated_at, completed_at \
FROM auto_tasks WHERE id = '{}'",
task_id
);
let rows: Vec<TaskRow> = sql_query(&query).get_results(&mut conn).unwrap_or_default();
if let Some(r) = rows.into_iter().next() {
Ok(Some(AutoTask {
id: r.id,
title: r.title.clone(),
intent: r.intent,
status: match r.status.as_str() {
"running" => AutoTaskStatus::Running,
"completed" => AutoTaskStatus::Completed,
"failed" => AutoTaskStatus::Failed,
"paused" => AutoTaskStatus::Paused,
"cancelled" => AutoTaskStatus::Cancelled,
_ => AutoTaskStatus::Draft,
},
mode: ExecutionMode::FullyAutomatic,
priority: TaskPriority::Medium,
plan_id: None,
basic_program: None,
current_step: r.current_step.as_ref().and_then(|s| s.parse().ok()).unwrap_or(0),
total_steps: 0,
progress: r.progress,
step_results: Vec::new(),
pending_decisions: Vec::new(),
pending_approvals: Vec::new(),
risk_summary: None,
resource_usage: Default::default(),
error: r.error.map(|msg| crate::auto_task::task_types::TaskError {
code: "TASK_ERROR".to_string(),
message: msg,
details: None,
recoverable: false,
step_id: None,
occurred_at: Utc::now(),
}),
rollback_state: None,
session_id: String::new(),
bot_id: String::new(),
created_by: String::new(),
assigned_to: String::new(),
schedule: None,
tags: Vec::new(),
parent_task_id: None,
subtask_ids: Vec::new(),
depends_on: Vec::new(),
dependents: Vec::new(),
mcp_servers: Vec::new(),
external_apis: Vec::new(),
created_at: r.created_at,
updated_at: r.updated_at,
started_at: None,
completed_at: r.completed_at,
estimated_completion: None,
}))
} else {
Ok(None)
}
}
fn list_auto_tasks( fn list_auto_tasks(
state: &Arc<AppState>, state: &Arc<AppState>,
filter: &str, filter: &str,

View file

@ -609,13 +609,12 @@ Respond with JSON only:
let mut conn = self.state.conn.get()?; let mut conn = self.state.conn.get()?;
// Insert into tasks table // Insert into tasks table (no bot_id column in tasks table)
sql_query( sql_query(
"INSERT INTO tasks (id, bot_id, title, description, status, priority, created_at) "INSERT INTO tasks (id, title, description, status, priority, created_at)
VALUES ($1, $2, $3, $4, 'pending', 'normal', NOW())", VALUES ($1, $2, $3, 'pending', 'normal', NOW())",
) )
.bind::<DieselUuid, _>(task_id) .bind::<DieselUuid, _>(task_id)
.bind::<DieselUuid, _>(session.bot_id)
.bind::<Text, _>(&title) .bind::<Text, _>(&title)
.bind::<Text, _>(&classification.original_text) .bind::<Text, _>(&classification.original_text)
.execute(&mut conn)?; .execute(&mut conn)?;

View file

@ -22,7 +22,7 @@ pub use autotask_api::{
apply_recommendation_handler, cancel_task_handler, classify_intent_handler, apply_recommendation_handler, cancel_task_handler, classify_intent_handler,
compile_intent_handler, create_and_execute_handler, execute_plan_handler, execute_task_handler, compile_intent_handler, create_and_execute_handler, execute_plan_handler, execute_task_handler,
get_approvals_handler, get_decisions_handler, get_pending_items_handler, get_stats_handler, get_approvals_handler, get_decisions_handler, get_pending_items_handler, get_stats_handler,
get_task_logs_handler, list_tasks_handler, pause_task_handler, resume_task_handler, get_task_handler, get_task_logs_handler, list_tasks_handler, pause_task_handler, resume_task_handler,
simulate_plan_handler, simulate_task_handler, submit_approval_handler, submit_decision_handler, simulate_plan_handler, simulate_task_handler, submit_approval_handler, submit_decision_handler,
submit_pending_item_handler, submit_pending_item_handler,
}; };
@ -59,6 +59,10 @@ pub fn configure_autotask_routes() -> axum::Router<std::sync::Arc<crate::shared:
post(simulate_plan_handler), post(simulate_plan_handler),
) )
.route(ApiUrls::AUTOTASK_LIST, get(list_tasks_handler)) .route(ApiUrls::AUTOTASK_LIST, get(list_tasks_handler))
.route(
&ApiUrls::AUTOTASK_GET.replace(":task_id", "{task_id}"),
get(get_task_handler),
)
.route(ApiUrls::AUTOTASK_STATS, get(get_stats_handler)) .route(ApiUrls::AUTOTASK_STATS, get(get_stats_handler))
.route( .route(
&ApiUrls::AUTOTASK_PAUSE.replace(":task_id", "{task_id}"), &ApiUrls::AUTOTASK_PAUSE.replace(":task_id", "{task_id}"),

View file

@ -8,7 +8,7 @@ use axum::{
routing::get, routing::get,
Router, Router,
}; };
use log::{error, trace, warn}; use log::{error, info, trace, warn};
use std::sync::Arc; use std::sync::Arc;
pub fn configure_app_server_routes() -> Router<Arc<AppState>> { pub fn configure_app_server_routes() -> Router<Arc<AppState>> {
@ -36,17 +36,17 @@ pub async fn serve_app_index(
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
Path(params): Path<AppPath>, Path(params): Path<AppPath>,
) -> impl IntoResponse { ) -> impl IntoResponse {
serve_app_file_internal(&state, &params.app_name, "index.html") serve_app_file_internal(&state, &params.app_name, "index.html").await
} }
pub async fn serve_app_file( pub async fn serve_app_file(
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
Path(params): Path<AppFilePath>, Path(params): Path<AppFilePath>,
) -> impl IntoResponse { ) -> impl IntoResponse {
serve_app_file_internal(&state, &params.app_name, &params.file_path) serve_app_file_internal(&state, &params.app_name, &params.file_path).await
} }
fn serve_app_file_internal(state: &AppState, app_name: &str, file_path: &str) -> Response { async fn serve_app_file_internal(state: &AppState, app_name: &str, file_path: &str) -> Response {
let sanitized_app_name = sanitize_path_component(app_name); let sanitized_app_name = sanitize_path_component(app_name);
let sanitized_file_path = sanitize_path_component(file_path); let sanitized_file_path = sanitize_path_component(file_path);
@ -54,6 +54,55 @@ fn serve_app_file_internal(state: &AppState, app_name: &str, file_path: &str) ->
return (StatusCode::BAD_REQUEST, "Invalid path").into_response(); return (StatusCode::BAD_REQUEST, "Invalid path").into_response();
} }
// Get bot name from bucket_name config (default to "default")
let bot_name = state.bucket_name
.trim_end_matches(".gbai")
.to_string();
let sanitized_bot_name = bot_name.to_lowercase().replace(' ', "-").replace('_', "-");
// MinIO bucket and path: botname.gbai / botname.gbapp/appname/file
let bucket = format!("{}.gbai", sanitized_bot_name);
let key = format!("{}.gbapp/{}/{}", sanitized_bot_name, sanitized_app_name, sanitized_file_path);
info!("Serving app file from MinIO: bucket={}, key={}", bucket, key);
// Try to serve from MinIO
if let Some(ref drive) = state.drive {
match drive
.get_object()
.bucket(&bucket)
.key(&key)
.send()
.await
{
Ok(response) => {
match response.body.collect().await {
Ok(body) => {
let content = body.into_bytes();
let content_type = get_content_type(&sanitized_file_path);
return Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, content_type)
.header(header::CACHE_CONTROL, "public, max-age=3600")
.body(Body::from(content.to_vec()))
.unwrap_or_else(|_| {
(StatusCode::INTERNAL_SERVER_ERROR, "Failed to build response")
.into_response()
});
}
Err(e) => {
error!("Failed to read MinIO response body: {}", e);
}
}
}
Err(e) => {
warn!("MinIO get_object failed for {}/{}: {}", bucket, key, e);
}
}
}
// Fallback to filesystem if MinIO fails
let site_path = state let site_path = state
.config .config
.as_ref() .as_ref()
@ -61,11 +110,11 @@ fn serve_app_file_internal(state: &AppState, app_name: &str, file_path: &str) ->
.unwrap_or_else(|| "./botserver-stack/sites".to_string()); .unwrap_or_else(|| "./botserver-stack/sites".to_string());
let full_path = format!( let full_path = format!(
"{}/{}/{}", "{}/{}.gbai/{}.gbapp/{}/{}",
site_path, sanitized_app_name, sanitized_file_path site_path, sanitized_bot_name, sanitized_bot_name, sanitized_app_name, sanitized_file_path
); );
trace!("Serving app file: {full_path}"); trace!("Fallback: serving app file from filesystem: {full_path}");
let path = std::path::Path::new(&full_path); let path = std::path::Path::new(&full_path);
if !path.exists() { if !path.exists() {

View file

@ -158,6 +158,8 @@ pub struct TaskProgressEvent {
pub error: Option<String>, pub error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub activity: Option<AgentActivity>, pub activity: Option<AgentActivity>,
#[serde(skip_serializing_if = "Option::is_none")]
pub text: Option<String>,
} }
impl TaskProgressEvent { impl TaskProgressEvent {
@ -174,6 +176,24 @@ impl TaskProgressEvent {
details: None, details: None,
error: None, error: None,
activity: None, activity: None,
text: None,
}
}
pub fn llm_stream(task_id: impl Into<String>, text: impl Into<String>) -> Self {
Self {
event_type: "llm_stream".to_string(),
task_id: task_id.into(),
step: "llm_stream".to_string(),
message: String::new(),
progress: 0,
total_steps: 0,
current_step: 0,
timestamp: chrono::Utc::now().to_rfc3339(),
details: None,
error: None,
activity: None,
text: Some(text.into()),
} }
} }
@ -181,7 +201,7 @@ impl TaskProgressEvent {
pub fn with_progress(mut self, current: u8, total: u8) -> Self { pub fn with_progress(mut self, current: u8, total: u8) -> Self {
self.current_step = current; self.current_step = current;
self.total_steps = total; self.total_steps = total;
self.progress = if total > 0 { (current * 100) / total } else { 0 }; self.progress = if total > 0 { ((current as u16 * 100) / total as u16) as u8 } else { 0 };
self self
} }
@ -224,6 +244,7 @@ impl TaskProgressEvent {
details: None, details: None,
error: None, error: None,
activity: None, activity: None,
text: None,
} }
} }
} }
@ -473,6 +494,14 @@ impl AppState {
.with_error(error); .with_error(error);
self.broadcast_task_progress(event); self.broadcast_task_progress(event);
} }
pub fn emit_llm_stream(&self, task_id: &str, text: &str) {
let event = TaskProgressEvent::llm_stream(task_id, text);
if let Some(tx) = &self.task_progress_broadcast {
// Don't log every stream chunk - too noisy
let _ = tx.send(event);
}
}
} }
#[cfg(test)] #[cfg(test)]

View file

@ -165,6 +165,7 @@ impl ApiUrls {
pub const AUTOTASK_EXECUTE: &'static str = "/api/autotask/execute"; pub const AUTOTASK_EXECUTE: &'static str = "/api/autotask/execute";
pub const AUTOTASK_SIMULATE: &'static str = "/api/autotask/simulate/:plan_id"; pub const AUTOTASK_SIMULATE: &'static str = "/api/autotask/simulate/:plan_id";
pub const AUTOTASK_LIST: &'static str = "/api/autotask/list"; pub const AUTOTASK_LIST: &'static str = "/api/autotask/list";
pub const AUTOTASK_GET: &'static str = "/api/autotask/tasks/:task_id";
pub const AUTOTASK_STATS: &'static str = "/api/autotask/stats"; pub const AUTOTASK_STATS: &'static str = "/api/autotask/stats";
pub const AUTOTASK_PAUSE: &'static str = "/api/autotask/:task_id/pause"; pub const AUTOTASK_PAUSE: &'static str = "/api/autotask/:task_id/pause";
pub const AUTOTASK_RESUME: &'static str = "/api/autotask/:task_id/resume"; pub const AUTOTASK_RESUME: &'static str = "/api/autotask/:task_id/resume";

File diff suppressed because it is too large Load diff

View file

@ -353,13 +353,18 @@ pub async fn handle_task_get(
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
Path(id): Path<String>, Path(id): Path<String>,
) -> impl IntoResponse { ) -> impl IntoResponse {
log::info!("[TASK_GET] *** Handler called for task: {} ***", id);
let conn = state.conn.clone(); let conn = state.conn.clone();
let task_id = id.clone(); let task_id = id.clone();
let result = tokio::task::spawn_blocking(move || { let result = tokio::task::spawn_blocking(move || {
let mut db_conn = conn let mut db_conn = conn
.get() .get()
.map_err(|e| format!("DB connection error: {}", e))?; .map_err(|e| {
log::error!("[TASK_GET] DB connection error: {}", e);
format!("DB connection error: {}", e)
})?;
#[derive(Debug, QueryableByName, serde::Serialize)] #[derive(Debug, QueryableByName, serde::Serialize)]
struct AutoTaskRow { struct AutoTaskRow {
@ -375,27 +380,46 @@ pub async fn handle_task_get(
pub intent: Option<String>, pub intent: Option<String>,
#[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Text>)] #[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Text>)]
pub error: Option<String>, pub error: Option<String>,
#[diesel(sql_type = diesel::sql_types::Float)] #[diesel(sql_type = diesel::sql_types::Double)]
pub progress: f32, pub progress: f64,
#[diesel(sql_type = diesel::sql_types::Integer)]
pub current_step: i32,
#[diesel(sql_type = diesel::sql_types::Integer)]
pub total_steps: i32,
#[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Jsonb>)]
pub step_results: Option<serde_json::Value>,
#[diesel(sql_type = diesel::sql_types::Timestamptz)] #[diesel(sql_type = diesel::sql_types::Timestamptz)]
pub created_at: chrono::DateTime<chrono::Utc>, pub created_at: chrono::DateTime<chrono::Utc>,
#[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Timestamptz>)] #[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Timestamptz>)]
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
#[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Timestamptz>)]
pub completed_at: Option<chrono::DateTime<chrono::Utc>>, pub completed_at: Option<chrono::DateTime<chrono::Utc>>,
} }
let parsed_uuid = match Uuid::parse_str(&task_id) { let parsed_uuid = match Uuid::parse_str(&task_id) {
Ok(u) => u, Ok(u) => {
Err(_) => return Err(format!("Invalid task ID: {}", task_id)), log::info!("[TASK_GET] Parsed UUID: {}", u);
u
}
Err(e) => {
log::error!("[TASK_GET] Invalid task ID '{}': {}", task_id, e);
return Err(format!("Invalid task ID: {}", task_id));
}
}; };
let task: Option<AutoTaskRow> = diesel::sql_query( let task: Option<AutoTaskRow> = diesel::sql_query(
"SELECT id, title, status, priority, intent, error, progress, created_at, completed_at "SELECT id, title, status, priority, intent, error, progress, current_step, total_steps, step_results, created_at, started_at, completed_at
FROM auto_tasks WHERE id = $1 LIMIT 1" FROM auto_tasks WHERE id = $1 LIMIT 1"
) )
.bind::<diesel::sql_types::Uuid, _>(parsed_uuid) .bind::<diesel::sql_types::Uuid, _>(parsed_uuid)
.get_result(&mut db_conn) .get_result(&mut db_conn)
.map_err(|e| {
log::error!("[TASK_GET] Query error for {}: {}", parsed_uuid, e);
e
})
.ok(); .ok();
log::info!("[TASK_GET] Query result for {}: found={}", parsed_uuid, task.is_some());
Ok::<_, String>(task) Ok::<_, String>(task)
}) })
.await .await
@ -406,6 +430,7 @@ pub async fn handle_task_get(
match result { match result {
Ok(Some(task)) => { Ok(Some(task)) => {
log::info!("[TASK_GET] Returning task: {} - {}", task.id, task.title);
let status_class = match task.status.as_str() { let status_class = match task.status.as_str() {
"completed" | "done" => "completed", "completed" | "done" => "completed",
"running" | "pending" => "running", "running" | "pending" => "running",
@ -414,49 +439,333 @@ pub async fn handle_task_get(
}; };
let progress_percent = (task.progress * 100.0) as u8; let progress_percent = (task.progress * 100.0) as u8;
let created = task.created_at.format("%Y-%m-%d %H:%M").to_string(); let created = task.created_at.format("%Y-%m-%d %H:%M").to_string();
let completed = task.completed_at.map(|d| d.format("%Y-%m-%d %H:%M").to_string()).unwrap_or_default();
// Calculate runtime
let runtime = if let Some(started) = task.started_at {
let end_time = task.completed_at.unwrap_or_else(chrono::Utc::now);
let duration = end_time.signed_duration_since(started);
let mins = duration.num_minutes();
let secs = duration.num_seconds() % 60;
if mins > 0 {
format!("{}m {}s", mins, secs)
} else {
format!("{}s", secs)
}
} else {
"Not started".to_string()
};
let task_id = task.id.to_string();
let intent_text = task.intent.clone().unwrap_or_else(|| task.title.clone());
let error_html = task.error.clone().map(|e| format!(
r#"<div class="error-alert">
<span class="error-icon"></span>
<span class="error-text">{}</span>
</div>"#, e
)).unwrap_or_default();
let current_step = task.current_step;
let total_steps = if task.total_steps > 0 { task.total_steps } else { 1 };
let status_label = match task.status.as_str() {
"completed" | "done" => "Completed",
"running" => "Running",
"pending" => "Pending",
"failed" | "error" => "Failed",
"paused" => "Paused",
"waiting_approval" => "Awaiting Approval",
_ => &task.status
};
// Build progress log HTML from step_results
let progress_log_html = build_progress_log_html(&task.step_results, current_step, total_steps);
// Build terminal output from recent activity
let terminal_html = build_terminal_html(&task.step_results, &task.status);
let html = format!(r#" let html = format!(r#"
<div class="task-detail-header"> <div class="task-detail-rich" data-task-id="{task_id}">
<h2 class="task-detail-title">{}</h2> <!-- Header with title and status badge -->
<span class="task-status task-status-{}">{}</span> <div class="detail-header-rich">
</div> <h2 class="detail-title-rich">{title}</h2>
<div class="task-detail-meta"> <span class="status-badge-rich status-{status_class}">{status_label}</span>
<div class="meta-item"><span class="meta-label">Priority:</span> <span class="meta-value">{}</span></div> </div>
<div class="meta-item"><span class="meta-label">Created:</span> <span class="meta-value">{}</span></div>
{} <!-- Status Section -->
</div> <div class="detail-section-box status-section">
<div class="task-detail-progress"> <div class="section-label">STATUS</div>
<div class="progress-label">Progress: {}%</div> <div class="status-content">
<div class="progress-bar-container"> <div class="status-main">
<div class="progress-bar-fill" style="width: {}%"></div> <span class="status-dot status-{status_class}"></span>
<span class="status-text">{title}</span>
</div>
<div class="status-meta">
<span class="meta-runtime">Runtime: {runtime}</span>
<span class="meta-estimated">Step {current_step}/{total_steps}</span>
</div>
</div>
{error_html}
<div class="status-details">
<div class="status-row">
<span class="status-indicator {status_indicator}"></span>
<span class="status-step-name">{status_label} (Step {current_step}/{total_steps})</span>
<span class="status-step-note">{priority} priority</span>
</div>
</div>
</div>
<!-- Progress Bar -->
<div class="detail-progress-rich">
<div class="progress-bar-rich">
<div class="progress-fill-rich" style="width: {progress_percent}%"></div>
</div>
<div class="progress-info-rich">
<span class="progress-label-rich">Progress: {progress_percent}%</span>
</div>
</div>
<!-- Progress Log Section -->
<div class="detail-section-box progress-log-section">
<div class="section-label">PROGRESS LOG</div>
<div class="progress-log-content" id="progress-log-{task_id}">
{progress_log_html}
</div>
</div>
<!-- Terminal Section -->
<div class="detail-section-box terminal-section-rich">
<div class="section-header-rich">
<div class="section-label">
<span class="terminal-dot-rich {terminal_active}"></span>
TERMINAL (LIVE AGENT ACTIVITY)
</div>
<div class="terminal-stats-rich">
<span>Step: <strong>{current_step}</strong> of <strong>{total_steps}</strong></span>
</div>
</div>
<div class="terminal-output-rich" id="terminal-output-{task_id}">
{terminal_html}
</div>
<div class="terminal-footer-rich">
<span class="terminal-eta">Started: <strong>{created}</strong></span>
</div>
</div>
<!-- Intent Section -->
<div class="detail-section-box intent-section">
<div class="section-label">INTENT</div>
<p class="intent-text-rich">{intent_text}</p>
</div>
<!-- Actions -->
<div class="detail-actions-rich">
<button class="btn-action-rich btn-pause" onclick="pauseTask('{task_id}')">
<span class="btn-icon"></span> Pause
</button>
<button class="btn-action-rich btn-cancel" onclick="cancelTask('{task_id}')">
<span class="btn-icon"></span> Cancel
</button>
<button class="btn-action-rich btn-detailed" onclick="showDetailedView('{task_id}')">
Detailed View
</button>
</div> </div>
</div> </div>
{}
{}
"#, "#,
task.title, task_id = task_id,
status_class, title = task.title,
task.status, status_class = status_class,
task.priority, status_label = status_label,
created, runtime = runtime,
if !completed.is_empty() { format!(r#"<div class="meta-item"><span class="meta-label">Completed:</span> <span class="meta-value">{}</span></div>"#, completed) } else { String::new() }, current_step = current_step,
progress_percent, total_steps = total_steps,
progress_percent, error_html = error_html,
task.intent.map(|i| format!(r#"<div class="task-detail-section"><h3>Intent</h3><p class="intent-text">{}</p></div>"#, i)).unwrap_or_default(), status_indicator = if task.status == "running" { "active" } else { "" },
task.error.map(|e| format!(r#"<div class="task-detail-section error-section"><h3>Error</h3><p class="error-text">{}</p></div>"#, e)).unwrap_or_default() priority = task.priority,
progress_percent = progress_percent,
progress_log_html = progress_log_html,
terminal_active = if task.status == "running" { "active" } else { "" },
terminal_html = terminal_html,
created = created,
intent_text = intent_text,
); );
(StatusCode::OK, axum::response::Html(html)).into_response() (StatusCode::OK, axum::response::Html(html)).into_response()
} }
Ok(None) => { Ok(None) => {
log::warn!("[TASK_GET] Task not found: {}", id);
(StatusCode::NOT_FOUND, axum::response::Html("<div class='error'>Task not found</div>".to_string())).into_response() (StatusCode::NOT_FOUND, axum::response::Html("<div class='error'>Task not found</div>".to_string())).into_response()
} }
Err(e) => { Err(e) => {
log::error!("[TASK_GET] Error fetching task {}: {}", id, e);
(StatusCode::INTERNAL_SERVER_ERROR, axum::response::Html(format!("<div class='error'>{}</div>", e))).into_response() (StatusCode::INTERNAL_SERVER_ERROR, axum::response::Html(format!("<div class='error'>{}</div>", e))).into_response()
} }
} }
} }
/// Build HTML for the progress log section from step_results JSON
fn build_progress_log_html(step_results: &Option<serde_json::Value>, current_step: i32, total_steps: i32) -> String {
let mut html = String::new();
if let Some(serde_json::Value::Array(steps)) = step_results {
if steps.is_empty() {
// No steps yet - show current status
html.push_str(&format!(r#"
<div class="log-group">
<div class="log-group-header">
<span class="log-group-name">Task Execution</span>
<span class="log-step-badge">Step {}/{}</span>
<span class="log-status-badge running">In Progress</span>
</div>
<div class="log-group-items">
<div class="log-item">
<span class="log-dot running"></span>
<span class="log-item-name">Waiting for execution steps...</span>
</div>
</div>
</div>
"#, current_step, total_steps));
} else {
// Group steps and show real data
for (idx, step) in steps.iter().enumerate() {
let step_name = step.get("step_name")
.and_then(|v| v.as_str())
.unwrap_or("Step");
let step_status = step.get("status")
.and_then(|v| v.as_str())
.unwrap_or("pending");
let step_order = step.get("step_order")
.and_then(|v| v.as_i64())
.unwrap_or((idx + 1) as i64);
let duration_ms = step.get("duration_ms")
.and_then(|v| v.as_i64());
let status_class = match step_status {
"completed" | "Completed" => "completed",
"running" | "Running" => "running",
"failed" | "Failed" => "failed",
_ => "pending"
};
let duration_str = duration_ms.map(|ms| {
if ms > 60000 {
format!("{}m {}s", ms / 60000, (ms % 60000) / 1000)
} else if ms > 1000 {
format!("{}s", ms / 1000)
} else {
format!("{}ms", ms)
}
}).unwrap_or_else(|| "--".to_string());
html.push_str(&format!(r#"
<div class="log-item">
<span class="log-dot {status_class}"></span>
<span class="log-item-name">{step_name}</span>
<span class="log-item-badge">Step {step_order}/{total_steps}</span>
<span class="log-item-status">{step_status}</span>
<span class="log-duration">Duration: {duration_str}</span>
</div>
"#,
status_class = status_class,
step_name = step_name,
step_order = step_order,
total_steps = total_steps,
step_status = step_status,
duration_str = duration_str,
));
// Show logs if present
if let Some(serde_json::Value::Array(logs)) = step.get("logs") {
for log_entry in logs.iter().take(3) {
let msg = log_entry.get("message")
.and_then(|v| v.as_str())
.unwrap_or("");
if !msg.is_empty() {
html.push_str(&format!(r#"
<div class="log-subitem">
<span class="log-subdot {status_class}"></span>
<span class="log-subitem-name">{msg}</span>
</div>
"#, status_class = status_class, msg = msg));
}
}
}
}
}
} else {
// No step results - show placeholder based on current progress
html.push_str(&format!(r#"
<div class="log-group">
<div class="log-group-header">
<span class="log-group-name">Task Progress</span>
<span class="log-step-badge">Step {}/{}</span>
<span class="log-status-badge pending">Pending</span>
</div>
<div class="log-group-items">
<div class="log-item">
<span class="log-dot pending"></span>
<span class="log-item-name">No execution steps recorded yet</span>
</div>
</div>
</div>
"#, current_step, total_steps));
}
html
}
/// Build HTML for terminal output from step results
fn build_terminal_html(step_results: &Option<serde_json::Value>, status: &str) -> String {
let mut html = String::new();
let mut lines: Vec<String> = Vec::new();
if let Some(serde_json::Value::Array(steps)) = step_results {
for step in steps.iter() {
// Add step name as a line
if let Some(step_name) = step.get("step_name").and_then(|v| v.as_str()) {
let step_status = step.get("status").and_then(|v| v.as_str()).unwrap_or("");
let prefix = match step_status {
"completed" | "Completed" => "",
"running" | "Running" => "",
"failed" | "Failed" => "",
_ => ""
};
lines.push(format!("{} {}", prefix, step_name));
}
// Add log messages
if let Some(serde_json::Value::Array(logs)) = step.get("logs") {
for log_entry in logs.iter() {
if let Some(msg) = log_entry.get("message").and_then(|v| v.as_str()) {
lines.push(format!(" {}", msg));
}
}
}
}
}
if lines.is_empty() {
// Show default message based on status
let default_msg = match status {
"running" => "Task is running...",
"pending" => "Waiting to start...",
"completed" | "done" => "Task completed successfully",
"failed" | "error" => "Task failed - check error details",
"paused" => "Task is paused",
_ => "Initializing..."
};
html.push_str(&format!(r#"<div class="terminal-line current">{}</div>"#, default_msg));
} else {
// Show last 10 lines, with the last one marked as current
let start = if lines.len() > 10 { lines.len() - 10 } else { 0 };
for (idx, line) in lines[start..].iter().enumerate() {
let is_last = idx == lines[start..].len() - 1;
let class = if is_last && status == "running" { "terminal-line current" } else { "terminal-line" };
html.push_str(&format!(r#"<div class="{}">{}</div>"#, class, line));
}
}
html
}
impl TaskEngine { impl TaskEngine {
pub async fn create_task_with_db( pub async fn create_task_with_db(
&self, &self,
@ -1295,39 +1604,31 @@ pub async fn handle_task_set_dependencies(
} }
pub fn configure_task_routes() -> Router<Arc<AppState>> { pub fn configure_task_routes() -> Router<Arc<AppState>> {
log::info!("[ROUTES] Registering task routes with /api/tasks/:id pattern");
Router::new() Router::new()
// Task list and create
.route( .route(
ApiUrls::TASKS, "/api/tasks",
post(handle_task_create).get(handle_task_list_htmx), post(handle_task_create).get(handle_task_list_htmx),
) )
// Specific routes MUST come before parameterized route
.route("/api/tasks/stats", get(handle_task_stats_htmx)) .route("/api/tasks/stats", get(handle_task_stats_htmx))
.route("/api/tasks/stats/json", get(handle_task_stats)) .route("/api/tasks/stats/json", get(handle_task_stats))
.route("/api/tasks/time-saved", get(handle_time_saved)) .route("/api/tasks/time-saved", get(handle_time_saved))
.route("/api/tasks/completed", delete(handle_clear_completed)) .route("/api/tasks/completed", delete(handle_clear_completed))
// Parameterized task routes - use :id for axum path params
.route( .route(
&ApiUrls::TASK_BY_ID.replace(":id", "{id}"), "/api/tasks/:id",
get(handle_task_get).put(handle_task_update), get(handle_task_get)
) .put(handle_task_update)
.route( .delete(handle_task_delete)
&ApiUrls::TASK_BY_ID.replace(":id", "{id}"), .patch(handle_task_patch),
delete(handle_task_delete).patch(handle_task_patch),
)
.route(
&ApiUrls::TASK_ASSIGN.replace(":id", "{id}"),
post(handle_task_assign),
)
.route(
&ApiUrls::TASK_STATUS.replace(":id", "{id}"),
put(handle_task_status_update),
)
.route(
&ApiUrls::TASK_PRIORITY.replace(":id", "{id}"),
put(handle_task_priority_set),
)
.route(
"/api/tasks/{id}/dependencies",
put(handle_task_set_dependencies),
) )
.route("/api/tasks/:id/assign", post(handle_task_assign))
.route("/api/tasks/:id/status", put(handle_task_status_update))
.route("/api/tasks/:id/priority", put(handle_task_priority_set))
.route("/api/tasks/:id/dependencies", put(handle_task_set_dependencies))
} }
pub fn configure(router: Router<Arc<TaskEngine>>) -> Router<Arc<TaskEngine>> { pub fn configure(router: Router<Arc<TaskEngine>>) -> Router<Arc<TaskEngine>> {