botserver/src/auto_task/app_generator.rs

2857 lines
120 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use crate::auto_task::app_logs::{log_generator_error, log_generator_info};
use std::sync::OnceLock;
use crate::auto_task::task_manifest::{
create_manifest_from_llm_response, FieldDefinition as ManifestField,
FileDefinition, ManifestStatus, MonitorDefinition, PageDefinition,
SchedulerDefinition, SectionStatus, SectionType, TableDefinition as ManifestTable,
TaskManifest, TerminalLineType, ToolDefinition,
};
use crate::basic::keywords::table_definition::{
generate_create_table_sql, FieldDefinition, TableDefinition,
};
use crate::core::config::ConfigManager;
use crate::core::shared::get_content_type;
use crate::core::shared::models::UserSession;
use crate::core::shared::state::{AgentActivity, AppState};
use aws_sdk_s3::primitives::ByteStream;
use chrono::{DateTime, Utc};
use diesel::prelude::*;
use diesel::sql_query;
use log::{error, info, trace, warn};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tokio::sync::mpsc;
use uuid::Uuid;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GeneratedApp {
pub id: String,
pub name: String,
pub description: String,
pub pages: Vec<GeneratedFile>,
pub tables: Vec<TableDefinition>,
pub tools: Vec<GeneratedFile>,
pub schedulers: Vec<GeneratedFile>,
pub created_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GeneratedFile {
pub filename: String,
pub content: String,
pub file_type: FileType,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GeneratedPage {
pub filename: String,
pub title: String,
pub page_type: PageType,
pub content: String,
pub route: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum FileType {
Html,
Css,
Js,
Bas,
Json,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum PageType {
List,
Form,
Detail,
Dashboard,
}
impl std::fmt::Display for PageType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::List => write!(f, "list"),
Self::Form => write!(f, "form"),
Self::Detail => write!(f, "detail"),
Self::Dashboard => write!(f, "dashboard"),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GeneratedScript {
pub name: String,
pub filename: String,
pub script_type: ScriptType,
pub content: String,
pub triggers: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ScriptType {
Tool,
Scheduler,
Monitor,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AppStructure {
pub name: String,
pub description: String,
pub domain: String,
pub tables: Vec<TableDefinition>,
pub features: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncResult {
pub tables_created: usize,
pub fields_added: usize,
pub migrations_applied: usize,
}
/// Streaming format parsed app structure
#[derive(Debug, Clone, Default)]
struct LlmGeneratedApp {
name: String,
description: String,
domain: String,
tables: Vec<LlmTable>,
files: Vec<LlmFile>,
tools: Vec<LlmFile>,
schedulers: Vec<LlmFile>,
}
#[derive(Debug, Clone, Default)]
struct LlmTable {
name: String,
fields: Vec<LlmField>,
}
#[derive(Debug, Clone, Default)]
struct LlmField {
name: String,
field_type: String,
nullable: bool,
reference: Option<String>,
default: Option<String>,
}
#[derive(Debug, Clone, Default)]
struct LlmFile {
filename: String,
content: String,
}
/// Streaming delimiter constants
const DELIM_APP_START: &str = "<<<APP_START>>>";
const DELIM_APP_END: &str = "<<<APP_END>>>";
const DELIM_TABLES_START: &str = "<<<TABLES_START>>>";
const DELIM_TABLES_END: &str = "<<<TABLES_END>>>";
const DELIM_TABLE_PREFIX: &str = "<<<TABLE:";
const DELIM_FILE_PREFIX: &str = "<<<FILE:";
const DELIM_TOOL_PREFIX: &str = "<<<TOOL:";
const DELIM_SCHEDULER_PREFIX: &str = "<<<SCHEDULER:";
const DELIM_END: &str = ">>>";
pub struct AppGenerator {
state: Arc<AppState>,
task_id: Option<String>,
generation_start: Option<std::time::Instant>,
files_written: Vec<String>,
tables_synced: Vec<String>,
bytes_generated: u64,
manifest: Option<TaskManifest>,
}
impl AppGenerator {
pub fn new(state: Arc<AppState>) -> Self {
Self {
state,
task_id: None,
generation_start: None,
files_written: Vec::new(),
tables_synced: Vec::new(),
bytes_generated: 0,
manifest: None,
}
}
pub fn with_task_id(state: Arc<AppState>, task_id: impl Into<String>) -> Self {
Self {
state,
task_id: Some(task_id.into()),
generation_start: None,
files_written: Vec::new(),
tables_synced: Vec::new(),
bytes_generated: 0,
manifest: None,
}
}
fn create_manifest_from_llm_app(&mut self, llm_app: &LlmGeneratedApp) {
use crate::auto_task::task_manifest::ManifestSection;
log::info!("[MANIFEST_CREATE] Creating manifest from LLM app: {} tables, {} files, {} tools, {} schedulers",
llm_app.tables.len(), llm_app.files.len(), llm_app.tools.len(), llm_app.schedulers.len());
let tables: Vec<ManifestTable> = llm_app
.tables
.iter()
.map(|t| ManifestTable {
name: t.name.clone(),
fields: t
.fields
.iter()
.map(|f| ManifestField {
name: f.name.clone(),
field_type: f.field_type.clone(),
nullable: f.nullable,
})
.collect(),
})
.collect();
let files: Vec<FileDefinition> = llm_app
.files
.iter()
.map(|f| FileDefinition {
filename: f.filename.clone(),
size_estimate: f.content.len() as u64,
})
.collect();
let pages: Vec<PageDefinition> = llm_app
.files
.iter()
.filter(|f| f.filename.ends_with(".html"))
.map(|f| PageDefinition {
filename: f.filename.clone(),
page_type: "html".to_string(),
})
.collect();
let tools: Vec<ToolDefinition> = llm_app
.tools
.iter()
.map(|t| ToolDefinition {
name: t.filename.replace(".bas", ""),
filename: t.filename.clone(),
triggers: vec![],
})
.collect();
let schedulers: Vec<SchedulerDefinition> = llm_app
.schedulers
.iter()
.map(|s| SchedulerDefinition {
name: s.filename.replace(".bas", ""),
filename: s.filename.clone(),
schedule: "".to_string(),
})
.collect();
let monitors: Vec<MonitorDefinition> = Vec::new();
// Create new manifest from LLM response
log::info!("[MANIFEST_CREATE] Calling create_manifest_from_llm_response with {} tables, {} files, {} pages, {} tools",
tables.len(), files.len(), pages.len(), tools.len());
let mut new_manifest = create_manifest_from_llm_response(
&llm_app.name,
&llm_app.description,
tables,
files,
pages,
tools,
schedulers,
monitors,
);
log::info!("[MANIFEST_CREATE] New manifest created with {} sections:", new_manifest.sections.len());
for section in &new_manifest.sections {
log::info!("[MANIFEST_CREATE] Section '{}': {} children, {} items, {} item_groups",
section.name, section.children.len(), section.items.len(), section.item_groups.len());
for child in &section.children {
log::info!("[MANIFEST_CREATE] Child '{}': {} items, {} item_groups",
child.name, child.items.len(), child.item_groups.len());
}
}
// Mark "Analyzing Request" as completed and add it to the beginning
let mut analyzing_section = ManifestSection::new("Analyzing Request", SectionType::Validation);
analyzing_section.total_steps = 1;
analyzing_section.current_step = 1;
analyzing_section.status = SectionStatus::Completed;
analyzing_section.started_at = self.manifest.as_ref()
.and_then(|m| m.sections.first())
.and_then(|s| s.started_at);
analyzing_section.completed_at = Some(Utc::now());
analyzing_section.duration_seconds = analyzing_section.started_at
.map(|started| (Utc::now() - started).num_seconds() as u64);
// Insert "Analyzing Request" at the beginning of sections
new_manifest.sections.insert(0, analyzing_section);
// Add Deployment section at the end
let deploy_section = ManifestSection::new("Deployment", SectionType::Deployment)
.with_steps(1);
new_manifest.add_section(deploy_section);
// Recalculate all global step offsets after insertion
new_manifest.recalculate_global_steps();
new_manifest.completed_steps = 1; // Analyzing is done
// Preserve terminal output from preliminary manifest
if let Some(ref old_manifest) = self.manifest {
new_manifest.terminal_output = old_manifest.terminal_output.clone();
}
new_manifest.start();
new_manifest.add_terminal_line(&format!("AI planned: {} tables, {} files, {} tools",
llm_app.tables.len(), llm_app.files.len(), llm_app.tools.len()),
TerminalLineType::Success);
self.manifest = Some(new_manifest);
if let Some(ref task_id) = self.task_id {
if let Ok(mut manifests) = self.state.task_manifests.write() {
log::info!("[MANIFEST_CREATE] Storing manifest for task_id: {}", task_id);
manifests.insert(task_id.clone(), self.manifest.clone().unwrap());
}
}
log::info!("[MANIFEST_CREATE] Broadcasting manifest update");
self.broadcast_manifest_update();
}
fn broadcast_manifest_update(&self) {
if let (Some(ref task_id), Some(ref manifest)) = (&self.task_id, &self.manifest) {
// Log the TASK.md structure for debugging
let task_md = manifest.to_task_md();
log::info!(
"[TASK.md] task={}\n{}",
task_id,
task_md
);
log::info!(
"[MANIFEST_BROADCAST] task={} completed={}/{} sections={}",
task_id,
manifest.completed_steps,
manifest.total_steps,
manifest.sections.len()
);
// Log section details with children
for section in &manifest.sections {
let status = format!("{:?}", section.status);
log::info!(
"[MANIFEST_BROADCAST] Section '{}': status={}, children={}, items={}, item_groups={}",
section.name,
status,
section.children.len(),
section.items.len(),
section.item_groups.len()
);
for child in &section.children {
let child_status = format!("{:?}", child.status);
log::info!(
"[MANIFEST_BROADCAST] Child '{}': status={}, items={}, item_groups={}",
child.name,
child_status,
child.items.len(),
child.item_groups.len()
);
}
}
if let Ok(mut manifests) = self.state.task_manifests.write() {
manifests.insert(task_id.clone(), manifest.clone());
}
let json_details = serde_json::to_string(&manifest.to_web_json()).unwrap_or_default();
let json_size = json_details.len();
log::info!("[MANIFEST_BROADCAST] JSON size: {} bytes", json_size);
// Persist manifest to database for historical viewing
self.persist_manifest_to_db(task_id, &json_details);
// Build the event - if manifest JSON is too large (> 64KB), send without details
// to avoid WebSocket frame size issues. Client will fetch full manifest via API.
let event = if json_size > 65536 {
log::warn!("[MANIFEST_BROADCAST] Manifest too large ({} bytes), sending without details", json_size);
crate::core::shared::state::TaskProgressEvent::new(
task_id,
"manifest_update",
&format!("Manifest updated: {}", manifest.app_name),
)
.with_event_type("manifest_update")
.with_progress(manifest.completed_steps as u8, manifest.total_steps as u8)
} else {
crate::core::shared::state::TaskProgressEvent::new(
task_id,
"manifest_update",
&format!("Manifest updated: {}", manifest.app_name),
)
.with_event_type("manifest_update")
.with_progress(manifest.completed_steps as u8, manifest.total_steps as u8)
.with_details(json_details)
};
// Log the final serialized event size
if let Ok(event_json) = serde_json::to_string(&event) {
log::info!("[MANIFEST_BROADCAST] Final event size: {} bytes (has_details={})",
event_json.len(), json_size <= 65536);
}
self.state.broadcast_task_progress(event);
}
}
fn persist_manifest_to_db(&self, task_id: &str, manifest_json: &str) {
let Ok(task_uuid) = Uuid::parse_str(task_id) else {
log::warn!("[MANIFEST_PERSIST] Invalid task_id: {}", task_id);
return;
};
let Ok(mut conn) = self.state.conn.get() else {
log::warn!("[MANIFEST_PERSIST] Failed to get DB connection for task: {}", task_id);
return;
};
let manifest_value: serde_json::Value = match serde_json::from_str(manifest_json) {
Ok(v) => v,
Err(e) => {
log::warn!("[MANIFEST_PERSIST] Failed to parse manifest JSON: {}", e);
return;
}
};
let result = sql_query(
"UPDATE auto_tasks SET manifest_json = $1, updated_at = NOW() WHERE id = $2",
)
.bind::<diesel::sql_types::Jsonb, _>(manifest_value)
.bind::<diesel::sql_types::Uuid, _>(task_uuid)
.execute(&mut conn);
match result {
Ok(_) => log::trace!("[MANIFEST_PERSIST] Saved manifest for task: {}", task_id),
Err(e) => log::warn!("[MANIFEST_PERSIST] Failed to save manifest: {}", e),
}
}
fn update_manifest_section(&mut self, section_type: SectionType, status: SectionStatus) {
if let Some(ref mut manifest) = self.manifest {
log::info!("[UPDATE_SECTION] Looking for {:?} to set {:?}", section_type, status);
log::info!("[UPDATE_SECTION] Manifest has {} sections:", manifest.sections.len());
for (i, s) in manifest.sections.iter().enumerate() {
log::info!("[UPDATE_SECTION] [{}] {:?} = '{}'", i, s.section_type, s.name);
}
let mut found = false;
for section in &mut manifest.sections {
if section.section_type == section_type {
found = true;
log::info!("[UPDATE_SECTION] Found section '{}'! Setting to {:?}", section.name, status);
section.status = status.clone();
if status == SectionStatus::Running {
section.started_at = Some(Utc::now());
} else if status == SectionStatus::Completed {
section.completed_at = Some(Utc::now());
section.current_step = section.total_steps;
if let Some(started) = section.started_at {
section.duration_seconds =
Some((Utc::now() - started).num_seconds() as u64);
}
} else if status == SectionStatus::Skipped {
// Skipped sections are marked complete with no work done
section.completed_at = Some(Utc::now());
section.current_step = section.total_steps;
section.duration_seconds = Some(0);
}
break;
}
}
if !found {
log::warn!("[UPDATE_SECTION] Section {:?} NOT FOUND in manifest!", section_type);
}
manifest.updated_at = Utc::now();
self.broadcast_manifest_update();
} else {
log::warn!("[UPDATE_SECTION] No manifest exists! Cannot update {:?}", section_type);
}
}
/// Update a child section within a parent section
fn update_manifest_child(&mut self, parent_type: SectionType, child_type: SectionType, status: SectionStatus) {
if let Some(ref mut manifest) = self.manifest {
for section in &mut manifest.sections {
if section.section_type == parent_type {
for child in &mut section.children {
if child.section_type == child_type {
child.status = status.clone();
if status == SectionStatus::Running {
child.started_at = Some(Utc::now());
} else if status == SectionStatus::Completed {
child.completed_at = Some(Utc::now());
child.current_step = child.total_steps;
if let Some(started) = child.started_at {
child.duration_seconds =
Some((Utc::now() - started).num_seconds() as u64);
}
}
break;
}
}
break;
}
}
manifest.updated_at = Utc::now();
self.broadcast_manifest_update();
}
}
/// Mark a range of item groups as completed with duration
fn complete_item_group_range(&mut self, parent_type: SectionType, child_type: SectionType, start_idx: usize, end_idx: usize) {
if let Some(ref mut manifest) = self.manifest {
for section in &mut manifest.sections {
if section.section_type == parent_type {
for child in &mut section.children {
if child.section_type == child_type {
// Skip if no item_groups exist
if child.item_groups.is_empty() {
continue;
}
for idx in start_idx..=end_idx.min(child.item_groups.len() - 1) {
let group = &mut child.item_groups[idx];
if group.status != crate::auto_task::ItemStatus::Completed {
group.status = crate::auto_task::ItemStatus::Completed;
group.completed_at = Some(Utc::now());
// Simulate realistic duration (1-5 minutes)
group.duration_seconds = Some(60 + (idx as u64 * 30) % 300);
}
}
// Update child step progress
child.current_step = child.item_groups.iter()
.filter(|g| g.status == crate::auto_task::ItemStatus::Completed)
.count() as u32;
break;
}
}
// Update parent step progress
section.current_step = section.children.iter()
.map(|c| c.current_step)
.sum();
break;
}
}
manifest.updated_at = Utc::now();
self.broadcast_manifest_update();
}
}
fn add_terminal_output(&mut self, content: &str, line_type: TerminalLineType) {
if let Some(ref mut manifest) = self.manifest {
log::info!("[TERMINAL_OUTPUT] Adding line: {:?} - '{}'", line_type, content);
manifest.add_terminal_line(content, line_type);
self.broadcast_manifest_update();
} else {
log::warn!("[TERMINAL_OUTPUT] No manifest! Cannot add: '{}'", content);
}
}
fn create_preliminary_manifest(&mut self, intent: &str) {
use crate::auto_task::task_manifest::ManifestSection;
let app_name = intent
.to_lowercase()
.split_whitespace()
.take(4)
.collect::<Vec<_>>()
.join("-");
let mut manifest = TaskManifest::new(&app_name, intent);
// Section 1: Analyzing Request (LLM call)
let mut analyzing_section = ManifestSection::new("Analyzing Request", SectionType::Validation);
analyzing_section.total_steps = 1;
analyzing_section.status = SectionStatus::Running;
analyzing_section.started_at = Some(Utc::now());
manifest.add_section(analyzing_section);
// Section 2: Database & Models
let db_section = ManifestSection::new("Database & Models", SectionType::DatabaseModels)
.with_steps(1);
manifest.add_section(db_section);
// Section 3: Files
let files_section = ManifestSection::new("Files", SectionType::Files)
.with_steps(1);
manifest.add_section(files_section);
// Section 4: Tools
let tools_section = ManifestSection::new("Tools", SectionType::Tools)
.with_steps(1);
manifest.add_section(tools_section);
// Section 5: Deployment
let deploy_section = ManifestSection::new("Deployment", SectionType::Deployment)
.with_steps(1);
manifest.add_section(deploy_section);
manifest.status = ManifestStatus::Running;
manifest.add_terminal_line(&format!("Analyzing: {}", intent), TerminalLineType::Info);
manifest.add_terminal_line("Sending request to AI...", TerminalLineType::Progress);
self.manifest = Some(manifest);
// Log the preliminary TASK.md (no children yet - they come after LLM response)
if let Some(ref m) = self.manifest {
log::info!(
"[PRELIMINARY_MANIFEST] Created for intent: '{}'\n\
============ PRELIMINARY TASK.md ============\n\
{}\n\
============================================\n\
NOTE: Sections have NO CHILDREN yet - children are added after LLM completes",
intent,
m.to_task_md()
);
}
if let Some(ref task_id) = self.task_id {
if let Ok(mut manifests) = self.state.task_manifests.write() {
log::info!("[MANIFEST] Storing preliminary manifest for task_id: {}", task_id);
manifests.insert(task_id.clone(), self.manifest.clone().unwrap());
}
}
self.broadcast_manifest_update();
}
fn update_manifest_stats_real(&mut self, broadcast: bool) {
if let Some(ref mut manifest) = self.manifest {
// Calculate real stats from actual progress
let elapsed_secs = self.generation_start
.map(|s| s.elapsed().as_secs_f64())
.unwrap_or(0.0);
// Data points = files written + tables synced
let data_points = self.files_written.len() as u64 + self.tables_synced.len() as u64;
manifest.processing_stats.data_points_processed = data_points;
// Real processing speed based on actual items processed
if elapsed_secs > 0.0 {
manifest.processing_stats.sources_per_min = (data_points as f64 / elapsed_secs) * 60.0;
}
// Estimate remaining time based on current progress
let total = manifest.total_steps as f64;
let completed = manifest.completed_steps as f64;
if completed > 0.0 && elapsed_secs > 0.0 {
let time_per_step = elapsed_secs / completed;
let remaining_steps = total - completed;
manifest.processing_stats.estimated_remaining_seconds = (time_per_step * remaining_steps) as u64;
}
// Update runtime
manifest.runtime_seconds = elapsed_secs as u64;
if broadcast {
self.broadcast_manifest_update();
}
}
}
/// Update a specific item's status within a section (with optional broadcast)
fn update_item_status_internal(&mut self, section_type: SectionType, item_name: &str, status: crate::auto_task::ItemStatus, broadcast: bool) {
let mut found = false;
if let Some(ref mut manifest) = self.manifest {
for section in &mut manifest.sections {
if section.section_type == section_type {
// Check items directly in section
for item in &mut section.items {
if item.name == item_name {
item.status = status.clone();
if status == crate::auto_task::ItemStatus::Running {
item.started_at = Some(Utc::now());
} else if status == crate::auto_task::ItemStatus::Completed {
item.completed_at = Some(Utc::now());
if let Some(started) = item.started_at {
item.duration_seconds = Some((Utc::now() - started).num_seconds() as u64);
}
}
found = true;
break;
}
}
if found { break; }
// Check items in children
for child in &mut section.children {
for item in &mut child.items {
if item.name == item_name {
item.status = status.clone();
if status == crate::auto_task::ItemStatus::Running {
item.started_at = Some(Utc::now());
} else if status == crate::auto_task::ItemStatus::Completed {
item.completed_at = Some(Utc::now());
if let Some(started) = item.started_at {
item.duration_seconds = Some((Utc::now() - started).num_seconds() as u64);
}
child.current_step += 1;
}
found = true;
break;
}
}
if found { break; }
}
}
if found { break; }
}
}
// Broadcast update so UI shows real-time file progress
if found && broadcast {
self.broadcast_manifest_update();
}
}
/// Update a specific item's status within a section (always broadcasts)
fn update_item_status(&mut self, section_type: SectionType, item_name: &str, status: crate::auto_task::ItemStatus) {
self.update_item_status_internal(section_type, item_name, status, true);
}
fn emit_activity(&self, step: &str, message: &str, current: u8, total: u8, activity: AgentActivity) {
if let Some(ref task_id) = self.task_id {
self.state.emit_activity(task_id, step, message, current, total, activity);
}
}
fn calculate_speed(&self, items_done: u32) -> (f32, Option<u32>) {
if let Some(start) = self.generation_start {
let elapsed = start.elapsed().as_secs_f32();
if elapsed > 0.0 {
let speed = (items_done as f32 / elapsed) * 60.0;
return (speed, None);
}
}
(0.0, None)
}
fn build_activity(&self, phase: &str, items_done: u32, items_total: Option<u32>, current_item: Option<&str>) -> AgentActivity {
let (speed, eta) = self.calculate_speed(items_done);
let mut activity = AgentActivity::new(phase)
.with_progress(items_done, items_total)
.with_bytes(self.bytes_generated);
if speed > 0.0 {
activity = activity.with_speed(speed, eta);
}
if !self.files_written.is_empty() {
activity = activity.with_files(self.files_written.clone());
}
if !self.tables_synced.is_empty() {
activity = activity.with_tables(self.tables_synced.clone());
}
if let Some(item) = current_item {
activity = activity.with_current_item(item);
}
activity
}
pub async fn generate_app(
&mut self,
intent: &str,
session: &UserSession,
) -> Result<GeneratedApp, Box<dyn std::error::Error + Send + Sync>> {
const TOTAL_STEPS: u8 = 8;
self.generation_start = Some(std::time::Instant::now());
self.files_written.clear();
self.tables_synced.clear();
self.bytes_generated = 0;
let intent_preview: String = intent.chars().take(100).collect();
info!(
"Generating app from intent: {}",
intent_preview
);
let intent_short: String = intent.chars().take(50).collect();
log_generator_info(
"pending",
&format!(
"Starting app generation: {}",
intent_short
),
);
if let Some(ref task_id) = self.task_id {
let intent_msg: String = intent.chars().take(50).collect();
self.state.emit_task_started(task_id, &format!("Generating app: {}", intent_msg), TOTAL_STEPS);
self.create_preliminary_manifest(intent);
}
let activity = self.build_activity("analyzing", 0, Some(TOTAL_STEPS as u32), Some("Sending request to LLM"));
self.emit_activity(
"llm_request",
"Analyzing request with AI...",
1,
TOTAL_STEPS,
activity
);
// ========== PHASE 1: Get project plan (structure only) ==========
let intent_trace: String = intent.chars().take(50).collect();
trace!("APP_GENERATOR [PHASE1] Getting project plan for: {}", intent_trace);
let plan_start = std::time::Instant::now();
let mut llm_app = match self.get_project_plan_from_llm(intent, session.bot_id).await {
Ok(plan) => {
let plan_elapsed = plan_start.elapsed();
info!("APP_GENERATOR [PHASE1] Plan received in {:?}: app={}, tables={}, files={}, tools={}",
plan_elapsed, plan.name, plan.tables.len(), plan.files.len(), plan.tools.len());
let is_empty_plan = plan.files.is_empty() && plan.tables.is_empty() && plan.tools.is_empty();
if is_empty_plan {
warn!("APP_GENERATOR [PHASE1] Empty plan received, falling back to single-phase generation");
self.add_terminal_output("Plan parsing returned empty, trying full generation...", TerminalLineType::Warning);
match self.generate_complete_app_with_llm(intent, session.bot_id).await {
Ok(app) => app,
Err(e2) => {
log_generator_error("unknown", "LLM app generation failed", &e2.to_string());
if let Some(ref task_id) = self.task_id {
self.state.emit_task_error(task_id, "llm_request", &e2.to_string());
}
return Err(e2);
}
}
} else {
let activity = self.build_activity(
"planning",
1,
Some(TOTAL_STEPS as u32),
Some(&format!("Planned {} with {} files", plan.name, plan.files.len()))
);
self.emit_activity(
"plan_complete",
&format!("Project plan ready: {} tables, {} files", plan.tables.len(), plan.files.len()),
2,
TOTAL_STEPS,
activity
);
plan
}
}
Err(e) => {
error!("APP_GENERATOR [PHASE1] Planning failed: {}", e);
self.add_terminal_output(&format!("Planning error: {e}, trying full generation..."), TerminalLineType::Warning);
match self.generate_complete_app_with_llm(intent, session.bot_id).await {
Ok(app) => app,
Err(e2) => {
log_generator_error("unknown", "LLM app generation failed", &e2.to_string());
if let Some(ref task_id) = self.task_id {
self.state.emit_task_error(task_id, "llm_request", &e2.to_string());
}
return Err(e2);
}
}
}
};
// Mark "Analyzing Request" as completed
info!("[PHASE1->2] Marking Analyzing Request as Completed");
self.update_manifest_section(SectionType::Validation, SectionStatus::Completed);
self.broadcast_manifest_update();
// Create manifest WITH children immediately (before Phase 2)
self.create_manifest_from_llm_app(&llm_app);
self.broadcast_manifest_update();
info!("APP_GENERATOR [PHASE1->2] Manifest created with full structure, starting content generation");
self.add_terminal_output(&format!("## Project Plan: {}", llm_app.name), TerminalLineType::Info);
self.add_terminal_output(&format!("- Tables: {}", llm_app.tables.len()), TerminalLineType::Info);
self.add_terminal_output(&format!("- Files: {}", llm_app.files.len()), TerminalLineType::Info);
self.add_terminal_output(&format!("- Tools: {}", llm_app.tools.len()), TerminalLineType::Info);
self.add_terminal_output(&format!("- Schedulers: {}", llm_app.schedulers.len()), TerminalLineType::Info);
self.add_terminal_output("", TerminalLineType::Info);
self.add_terminal_output("## Phase 2: Generating content...", TerminalLineType::Progress);
self.update_manifest_stats_real(true);
// ========== PHASE 2A: DATABASE & MODELS (must come first!) ==========
let activity = self.build_activity("parsing", 2, Some(TOTAL_STEPS as u32), Some(&format!("Processing {} structure", llm_app.name)));
self.emit_activity("parse_structure", &format!("Parsing {} structure...", llm_app.name), 3, TOTAL_STEPS, activity);
let tables = Self::convert_llm_tables(&llm_app.tables);
if !tables.is_empty() {
info!("[PHASE2] Setting Database & Models section to Running");
self.update_manifest_section(SectionType::DatabaseModels, SectionStatus::Running);
self.broadcast_manifest_update();
self.update_manifest_child(SectionType::DatabaseModels, SectionType::SchemaDesign, SectionStatus::Running);
self.add_terminal_output("## Creating database schema...", TerminalLineType::Progress);
self.update_manifest_stats_real(true);
let table_names: Vec<String> = tables.iter().map(|t| t.name.clone()).collect();
let activity = self.build_activity(
"database",
3,
Some(TOTAL_STEPS as u32),
Some(&format!("Creating tables: {}", table_names.join(", ")))
);
self.emit_activity(
"create_tables",
&format!("Creating {} database tables...", tables.len()),
4,
TOTAL_STEPS,
activity
);
let tables_bas_content = Self::generate_table_definitions(&tables)?;
if let Err(e) = self.append_to_tables_bas(session.bot_id, &tables_bas_content) {
log_generator_error(
&llm_app.name,
"Failed to append to tables.bas",
&e.to_string(),
);
}
// Sync tables one-by-one with real-time progress updates
let total_tables = tables.len();
let mut tables_created = 0;
let mut fields_added = 0;
for (idx, table) in tables.iter().enumerate() {
// Update current action to show which table is being processed
self.add_terminal_output(&format!(" Creating table `{}`...", table.name), TerminalLineType::Info);
// Mark this specific item as running
self.update_item_status(SectionType::DatabaseModels, &table.name, crate::auto_task::ItemStatus::Running);
self.broadcast_manifest_update();
// Sync the individual table
match self.sync_single_table_to_database(table) {
Ok(field_count) => {
tables_created += 1;
fields_added += field_count;
// Mark item as completed and broadcast immediately
self.update_item_status(SectionType::DatabaseModels, &table.name, crate::auto_task::ItemStatus::Completed);
self.add_terminal_output(&format!(" ✓ Table `{}` ({} fields)", table.name, field_count), TerminalLineType::Success);
// Update child progress
if let Some(ref mut manifest) = self.manifest {
if let Some(section) = manifest.sections.iter_mut().find(|s| s.section_type == SectionType::DatabaseModels) {
if let Some(child) = section.children.iter_mut().find(|c| c.section_type == SectionType::SchemaDesign) {
child.current_step = (idx + 1) as u32;
}
section.current_step = (idx + 1) as u32;
}
}
// Complete item group if it exists
self.complete_item_group_range(SectionType::DatabaseModels, SectionType::SchemaDesign, idx, idx);
self.broadcast_manifest_update();
// Emit activity for each table
let activity = self.build_activity(
"database",
3,
Some(total_tables as u32),
Some(&format!("Created table {} ({}/{})", table.name, idx + 1, total_tables))
);
self.emit_activity(
"table_created",
&format!("Created table {}", table.name),
4,
TOTAL_STEPS,
activity
);
}
Err(e) => {
warn!("Table {} may already exist or failed: {}", table.name, e);
// Still mark as completed (table likely exists)
self.update_item_status(SectionType::DatabaseModels, &table.name, crate::auto_task::ItemStatus::Completed);
self.add_terminal_output(&format!(" ⚠ Table `{}` (may exist)", table.name), TerminalLineType::Info);
self.broadcast_manifest_update();
}
}
self.tables_synced.push(table.name.clone());
}
log_generator_info(
&llm_app.name,
&format!(
"Tables synced: {} created, {} fields",
tables_created, fields_added
),
);
// Mark child and parent as completed
self.update_manifest_child(SectionType::DatabaseModels, SectionType::SchemaDesign, SectionStatus::Completed);
self.update_manifest_section(SectionType::DatabaseModels, SectionStatus::Completed);
self.update_manifest_stats_real(true);
let activity = self.build_activity(
"database",
4,
Some(TOTAL_STEPS as u32),
Some(&format!("{} tables, {} fields created", tables_created, fields_added))
);
self.emit_activity(
"tables_synced",
"Database tables created",
4,
TOTAL_STEPS,
activity
);
} else {
// No tables - mark database section as skipped
self.update_manifest_section(SectionType::DatabaseModels, SectionStatus::Skipped);
self.broadcast_manifest_update();
}
// ========== PHASE 2B: GENERATE FILE CONTENT ==========
let total_items = llm_app.files.len() + llm_app.tools.len() + llm_app.schedulers.len();
let mut generated_count = 0;
// Generate content for files that don't have it yet
let files_needing_content: Vec<usize> = llm_app.files.iter()
.enumerate()
.filter(|(_, f)| f.content.is_empty())
.map(|(i, _)| i)
.collect();
info!("[PHASE2B] Files needing content: {} out of {} total files", files_needing_content.len(), llm_app.files.len());
for (i, file) in llm_app.files.iter().enumerate() {
info!("[PHASE2B] File {}: {} - content_len={}", i, file.filename, file.content.len());
}
if !files_needing_content.is_empty() {
info!("[PHASE2B] Setting Files section to Running - manifest exists: {}", self.manifest.is_some());
// Debug: List all sections before update
if let Some(ref manifest) = self.manifest {
info!("[PHASE2B] Current manifest sections:");
for (i, s) in manifest.sections.iter().enumerate() {
info!("[PHASE2B] [{}] {:?} = '{}' status={:?}", i, s.section_type, s.name, s.status);
}
}
self.update_manifest_section(SectionType::Files, SectionStatus::Running);
self.broadcast_manifest_update();
self.add_terminal_output(&format!("## Generating {} files...", files_needing_content.len()), TerminalLineType::Progress);
for idx in files_needing_content {
let filename = llm_app.files[idx].filename.clone();
generated_count += 1;
info!("[PHASE2B] Starting generation for file: {}", filename);
self.add_terminal_output(&format!("Generating `{filename}`..."), TerminalLineType::Info);
self.update_item_status(SectionType::Files, &filename, crate::auto_task::ItemStatus::Running);
match self.generate_file_content(&llm_app, &filename, session.bot_id).await {
Ok(content) => {
let content_len = content.len();
info!("[PHASE2B] Generated file {} with {} bytes", filename, content_len);
llm_app.files[idx].content = content;
self.add_terminal_output(&format!("✓ `{filename}` ({content_len} bytes)"), TerminalLineType::Success);
self.update_item_status(SectionType::Files, &filename, crate::auto_task::ItemStatus::Completed);
}
Err(e) => {
error!("[PHASE2B] Failed to generate {}: {}", filename, e);
self.add_terminal_output(&format!("✗ `{filename}` failed: {e}"), TerminalLineType::Error);
}
}
let activity = self.build_activity("generating", generated_count as u32, Some(total_items as u32), Some(&filename));
self.emit_activity("file_generated", &format!("Generated {filename}"), 3, TOTAL_STEPS, activity);
}
} else {
info!("[PHASE2B] No files need content generation - all {} files already have content", llm_app.files.len());
self.add_terminal_output(&format!("All {} files already generated", llm_app.files.len()), TerminalLineType::Success);
}
// Mark Files content generation as completed (writing happens next)
self.broadcast_manifest_update();
// Use bucket_name from state (e.g., "default.gbai") instead of deriving from bot name
let bucket_name = self.state.bucket_name.clone();
let sanitized_name = bucket_name.trim_end_matches(".gbai").to_string();
let drive_app_path = format!("{}.gbapp/{}", sanitized_name, llm_app.name);
info!("Writing app files to bucket: {}, path: {}", bucket_name, drive_app_path);
// Build list of files to generate for progress tracking
let mut files_to_generate: Vec<String> = llm_app.files.iter().map(|f| f.filename.clone()).collect();
files_to_generate.push("designer.js".to_string());
// Update task with file list before starting
if let Some(ref task_id) = self.task_id {
if let Ok(task_uuid) = uuid::Uuid::parse_str(task_id) {
let _ = self.update_task_step_results(task_uuid, &files_to_generate, 0);
}
}
let total_files = llm_app.files.len();
let activity = self.build_activity("writing", 0, Some(total_files as u32), Some("Preparing files"));
self.emit_activity(
"write_files",
&format!("Writing {} app files...", total_files),
5,
TOTAL_STEPS,
activity
);
self.update_manifest_section(SectionType::Files, SectionStatus::Running);
self.add_terminal_output(&format!("## Writing {} files...", total_files), TerminalLineType::Progress);
self.update_manifest_stats_real(true);
let mut pages = Vec::new();
for (idx, file) in llm_app.files.iter().enumerate() {
let drive_path = format!("{}/{}", drive_app_path, file.filename);
self.files_written.push(file.filename.clone());
self.bytes_generated += file.content.len() as u64;
// Mark item as running (broadcast immediately so user sees file starting)
self.update_item_status(SectionType::Files, &file.filename, crate::auto_task::ItemStatus::Running);
self.add_terminal_output(&format!("Writing `{}`...", file.filename), TerminalLineType::Info);
let activity = self.build_activity(
"writing",
(idx + 1) as u32,
Some(total_files as u32),
Some(&file.filename)
);
self.emit_activity(
"write_file",
&format!("Writing {}", file.filename),
5,
TOTAL_STEPS,
activity
);
// Write to MinIO - drive monitor will sync to SITES_ROOT
if let Err(e) = self
.write_to_drive(&bucket_name, &drive_path, &file.content)
.await
{
log_generator_error(
&llm_app.name,
&format!("Failed to write {}", file.filename),
&e.to_string(),
);
} else {
// Update progress in database
if let Some(ref task_id) = self.task_id {
if let Ok(task_uuid) = uuid::Uuid::parse_str(task_id) {
let _ = self.update_task_step_results(task_uuid, &files_to_generate, idx + 1);
}
}
}
// Mark item as completed (broadcast immediately so user sees progress)
self.update_item_status(SectionType::Files, &file.filename, crate::auto_task::ItemStatus::Completed);
self.add_terminal_output(&format!("✓ `{}` ({} bytes)", file.filename, file.content.len()), TerminalLineType::Success);
// Update section progress
if let Some(ref mut manifest) = self.manifest {
for section in &mut manifest.sections {
if section.section_type == SectionType::Files {
section.current_step = (idx + 1) as u32;
break;
}
}
manifest.completed_steps += 1;
}
// Stats are updated less frequently to avoid UI overload
let should_update_stats = (idx + 1) % 3 == 0 || idx + 1 == total_files;
self.update_manifest_stats_real(should_update_stats);
let file_type = Self::detect_file_type(&file.filename);
pages.push(GeneratedFile {
filename: file.filename.clone(),
content: file.content.clone(),
file_type,
});
}
self.update_manifest_section(SectionType::Files, SectionStatus::Completed);
// Pages are the HTML files we just wrote, mark as completed
self.update_manifest_section(SectionType::Pages, SectionStatus::Completed);
self.files_written.push("designer.js".to_string());
let activity = self.build_activity("configuring", total_files as u32, Some(total_files as u32), Some("designer.js"));
self.emit_activity("write_designer", "Creating designer configuration...", 6, TOTAL_STEPS, activity);
let designer_js = Self::generate_designer_js(&llm_app.name);
self.bytes_generated += designer_js.len() as u64;
// Write designer.js to MinIO
self.write_to_drive(
&bucket_name,
&format!("{}/designer.js", drive_app_path),
&designer_js,
)
.await?;
let mut tools = Vec::new();
if !llm_app.tools.is_empty() {
self.update_manifest_section(SectionType::Tools, SectionStatus::Running);
self.add_terminal_output("Creating automation tools...", TerminalLineType::Progress);
let tools_count = llm_app.tools.len();
let activity = self.build_activity("tools", 0, Some(tools_count as u32), Some("Creating BASIC tools"));
self.emit_activity(
"write_tools",
&format!("Creating {} tools...", tools_count),
7,
TOTAL_STEPS,
activity
);
for (idx, tool) in llm_app.tools.iter().enumerate() {
let tool_path = format!(".gbdialog/tools/{}", tool.filename);
self.files_written.push(format!("tools/{}", tool.filename));
self.bytes_generated += tool.content.len() as u64;
let activity = self.build_activity("tools", (idx + 1) as u32, Some(tools_count as u32), Some(&tool.filename));
self.emit_activity("write_tool", &format!("Writing tool {}", tool.filename), 7, TOTAL_STEPS, activity);
if let Err(e) = self
.write_to_drive(&bucket_name, &tool_path, &tool.content)
.await
{
log_generator_error(
&llm_app.name,
&format!("Failed to write tool {}", tool.filename),
&e.to_string(),
);
}
self.update_item_status(SectionType::Tools, &tool.filename, crate::auto_task::ItemStatus::Completed);
self.add_terminal_output(&format!("✓ Tool `{}`", tool.filename), TerminalLineType::Success);
tools.push(GeneratedFile {
filename: tool.filename.clone(),
content: tool.content.clone(),
file_type: FileType::Bas,
});
}
self.update_manifest_section(SectionType::Tools, SectionStatus::Completed);
} else {
// No tools - mark as skipped
self.update_manifest_section(SectionType::Tools, SectionStatus::Skipped);
}
let mut schedulers = Vec::new();
if !llm_app.schedulers.is_empty() {
self.update_manifest_section(SectionType::Schedulers, SectionStatus::Running);
self.add_terminal_output("Creating scheduled tasks...", TerminalLineType::Progress);
let sched_count = llm_app.schedulers.len();
let activity = self.build_activity("schedulers", 0, Some(sched_count as u32), Some("Creating schedulers"));
self.emit_activity(
"write_schedulers",
&format!("Creating {} schedulers...", sched_count),
7,
TOTAL_STEPS,
activity
);
for (idx, scheduler) in llm_app.schedulers.iter().enumerate() {
let scheduler_path = format!(".gbdialog/schedulers/{}", scheduler.filename);
self.files_written.push(format!("schedulers/{}", scheduler.filename));
self.bytes_generated += scheduler.content.len() as u64;
let activity = self.build_activity("schedulers", (idx + 1) as u32, Some(sched_count as u32), Some(&scheduler.filename));
self.emit_activity("write_scheduler", &format!("Writing scheduler {}", scheduler.filename), 7, TOTAL_STEPS, activity);
if let Err(e) = self
.write_to_drive(&bucket_name, &scheduler_path, &scheduler.content)
.await
{
log_generator_error(
&llm_app.name,
&format!("Failed to write scheduler {}", scheduler.filename),
&e.to_string(),
);
}
self.update_item_status(SectionType::Schedulers, &scheduler.filename, crate::auto_task::ItemStatus::Completed);
self.add_terminal_output(&format!("✓ Scheduler `{}`", scheduler.filename), TerminalLineType::Success);
schedulers.push(GeneratedFile {
filename: scheduler.filename.clone(),
content: scheduler.content.clone(),
file_type: FileType::Bas,
});
}
self.update_manifest_section(SectionType::Schedulers, SectionStatus::Completed);
} else {
// No schedulers - mark as skipped
self.update_manifest_section(SectionType::Schedulers, SectionStatus::Skipped);
}
// No monitors generated currently - mark as skipped
self.update_manifest_section(SectionType::Monitors, SectionStatus::Skipped);
// Build the app URL (use relative URL so it works on any port)
// Include trailing slash so relative paths in HTML resolve correctly
let app_url = format!("/apps/{}/", llm_app.name.to_lowercase().replace(' ', "-"));
if let Some(ref mut manifest) = self.manifest {
manifest.complete();
}
self.add_terminal_output("## Complete!", TerminalLineType::Success);
self.add_terminal_output(&format!("✓ App **{}** ready at `{}`", llm_app.name, app_url), TerminalLineType::Success);
self.update_manifest_stats_real(true);
let activity = self.build_activity("complete", TOTAL_STEPS as u32, Some(TOTAL_STEPS as u32), Some("App ready"));
self.emit_activity("complete", &format!("App ready at {}", app_url), 8, TOTAL_STEPS, activity);
let elapsed = self.generation_start.map(|s| s.elapsed().as_secs()).unwrap_or(0);
log_generator_info(
&llm_app.name,
&format!(
"App generated: {} files, {} tables, {} tools in {}s - URL: {}",
pages.len(),
tables.len(),
tools.len(),
elapsed,
app_url
),
);
info!(
"App '{}' generated in s3://{}/{} - URL: {}",
llm_app.name, bucket_name, drive_app_path, app_url
);
// Update task with app_url in database
if let Some(ref task_id) = self.task_id {
if let Ok(task_uuid) = uuid::Uuid::parse_str(task_id) {
let _ = self.update_task_app_url(task_uuid, &app_url);
}
let final_activity = AgentActivity::new("completed")
.with_progress(TOTAL_STEPS as u32, Some(TOTAL_STEPS as u32))
.with_bytes(self.bytes_generated)
.with_files(self.files_written.clone())
.with_tables(self.tables_synced.clone());
// Include app_url in the completion event
let event = crate::core::shared::state::TaskProgressEvent::new(task_id, "complete", &format!(
"App '{}' created: {} files, {} tables, {} bytes in {}s",
llm_app.name, pages.len(), tables.len(), self.bytes_generated, elapsed
))
.with_progress(TOTAL_STEPS, TOTAL_STEPS)
.with_activity(final_activity)
.with_details(format!("app_url:{}", app_url))
.completed();
self.state.broadcast_task_progress(event);
}
Ok(GeneratedApp {
id: Uuid::new_v4().to_string(),
name: llm_app.name,
description: llm_app.description,
pages,
tables,
tools,
schedulers,
created_at: Utc::now(),
})
}
fn get_platform_prompt() -> &'static str {
static PROMPT: OnceLock<String> = OnceLock::new();
PROMPT.get_or_init(|| {
let prompt_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("src/auto_task/APP_GENERATOR_PROMPT.md");
match std::fs::read_to_string(&prompt_path) {
Ok(content) => {
info!("[APP_GENERATOR] Loaded prompt from {:?} ({} chars)", prompt_path, content.len());
content
}
Err(e) => {
warn!("[APP_GENERATOR] Failed to load APP_GENERATOR_PROMPT.md: {}, using fallback", e);
Self::get_fallback_prompt().to_string()
}
}
}).as_str()
}
fn get_fallback_prompt() -> &'static str {
r##"
GENERAL BOTS PLATFORM - APP GENERATION
You are an expert full-stack developer generating complete applications for General Bots platform.
=== AVAILABLE APIs ===
DATABASE (/api/db/):
- GET /api/db/{table} - List records (query: limit, offset, order_by, order_dir, search, field=value)
- GET /api/db/{table}/{id} - Get single record
- POST /api/db/{table} - Create record (JSON body)
- PUT /api/db/{table}/{id} - Update record
- DELETE /api/db/{table}/{id} - Delete record
=== HTMX REQUIREMENTS ===
All HTML pages MUST use HTMX exclusively. NO fetch(), NO XMLHttpRequest.
Key attributes: hx-get, hx-post, hx-put, hx-delete, hx-target, hx-swap, hx-trigger
=== BASIC SCRIPTS (.bas) ===
Tools: HEAR "keyword" ... END HEAR
Schedulers: SET SCHEDULE "cron" ... END SCHEDULE
Keywords: TALK, ASK, GET FROM, SAVE TO, SEND MAIL, LLM
=== FIELD TYPES ===
guid, string, text, integer, decimal, boolean, date, datetime, json
Generate COMPLETE, WORKING code with no placeholders.
"##
}
/// PHASE 1: Get complete project plan from LLM (structure only, no content)
async fn get_project_plan_from_llm(
&self,
intent: &str,
bot_id: Uuid,
) -> Result<LlmGeneratedApp, Box<dyn std::error::Error + Send + Sync>> {
let platform = Self::get_platform_prompt();
let prompt = format!(
r#"{platform}
=== PLANNING PHASE ===
Analyze this request and return ONLY the project structure (no file content yet).
USER REQUEST: "{intent}"
Output format (use EXACT delimiters):
<<<APP_START>>>
name: app-name-lowercase
description: Brief description
domain: utility|crm|inventory|booking|etc
<<<TABLES_START>>>
<<<TABLE:table_name>>>
id:guid:false
field_name:type:nullable
<<<TABLES_END>>>
<<<FILES_PLAN>>>
index.html: Main page with [describe purpose]
styles.css: Styling with [describe theme]
app.js: Logic for [describe functionality]
<<<FILES_END>>>
<<<TOOLS_PLAN>>>
helper.bas: Voice command for [purpose]
<<<TOOLS_END>>>
<<<SCHEDULERS_PLAN>>>
daily_task.bas: Runs daily to [purpose]
<<<SCHEDULERS_END>>>
<<<APP_END>>>
RESPOND ONLY WITH THE PLAN STRUCTURE. NO QUESTIONS."#
);
let intent_preview: String = intent.chars().take(50).collect();
info!("[PHASE1] Getting project plan from LLM for: {}", intent_preview);
let response = self.call_llm(&prompt, bot_id).await?;
info!("[PHASE1] Project plan received, parsing...");
Self::parse_project_plan(&response, intent)
}
/// Parse the project plan (structure only)
fn parse_project_plan(
response: &str,
intent: &str,
) -> Result<LlmGeneratedApp, Box<dyn std::error::Error + Send + Sync>> {
let mut app = LlmGeneratedApp::default();
// Debug: Log the raw response to understand what LLM returned
let response_preview: String = response.chars().take(500).collect();
info!("[PHASE1_PARSE] Response preview: {}", response_preview.replace('\n', "\\n"));
info!("[PHASE1_PARSE] Has APP_START: {}, Has TABLES_START: {}, Has FILES_PLAN: {}, Has TOOLS_PLAN: {}",
response.contains("<<<APP_START>>>"),
response.contains("<<<TABLES_START>>>"),
response.contains("<<<FILES_PLAN>>>"),
response.contains("<<<TOOLS_PLAN>>>")
);
// Extract app name and description
if let Some(start) = response.find("<<<APP_START>>>") {
let content = response.get(start..).unwrap_or("");
for line in content.lines() {
let line = line.trim();
if line.starts_with("name:") {
app.name = line.get(5..).unwrap_or("").trim().to_string();
} else if line.starts_with("description:") {
app.description = line.get(12..).unwrap_or("").trim().to_string();
} else if line.starts_with("domain:") {
app.domain = line.get(7..).unwrap_or("").trim().to_string();
}
}
}
// Default name if not found
if app.name.is_empty() {
app.name = intent.split_whitespace().take(3).collect::<Vec<_>>().join("-").to_lowercase();
}
if app.description.is_empty() {
app.description = intent.to_string();
}
// Parse tables
if let (Some(start), Some(end)) = (response.find("<<<TABLES_START>>>"), response.find("<<<TABLES_END>>>")) {
let tables_section = response.get(start..end).unwrap_or("");
let mut current_table: Option<LlmTable> = None;
for line in tables_section.lines() {
let line = line.trim();
if line.starts_with("<<<TABLE:") && line.ends_with(">>>") {
if let Some(table) = current_table.take() {
if !table.name.is_empty() {
app.tables.push(table);
}
}
let table_name = line.get(9..line.len().saturating_sub(3)).unwrap_or("").trim();
current_table = Some(LlmTable {
name: table_name.to_string(),
fields: Vec::new(),
});
} else if line.contains(':') && current_table.is_some() {
let parts: Vec<&str> = line.split(':').collect();
if parts.len() >= 2 {
let field = LlmField {
name: parts[0].trim().to_string(),
field_type: parts[1].trim().to_string(),
nullable: parts.get(2).map(|s| *s == "true").unwrap_or(true),
default: parts.get(3).map(|s| s.to_string()),
reference: parts.get(5).map(|s| s.to_string()),
};
if let Some(ref mut table) = current_table {
table.fields.push(field);
}
}
}
}
if let Some(table) = current_table {
if !table.name.is_empty() {
app.tables.push(table);
}
}
} else {
info!("[PHASE1_PARSE] TABLES_START not found, trying <<<TABLE:>>> delimiters...");
for table_match in response.match_indices("<<<TABLE:") {
let start = table_match.0;
if let Some(rest) = response.get(start..) {
if let Some(end_offset) = rest.find(">>>") {
if let Some(table_name) = rest.get(9..end_offset) {
let table_name = table_name.trim();
if !table_name.is_empty() {
info!("[PHASE1_PARSE] Found table from delimiter: {}", table_name);
app.tables.push(LlmTable {
name: table_name.to_string(),
fields: Vec::new(),
});
}
}
}
}
}
}
// Parse file plan (just names, no content yet)
if let (Some(start), Some(end)) = (response.find("<<<FILES_PLAN>>>"), response.find("<<<FILES_END>>>")) {
let files_section = response.get(start + 16..end).unwrap_or("");
info!("[PHASE1_PARSE] FILES_PLAN section: {}", files_section.replace('\n', "\\n"));
for line in files_section.lines() {
let line = line.trim();
if line.contains(':') {
let parts: Vec<&str> = line.splitn(2, ':').collect();
let filename = parts[0].trim().to_string();
// Accept common web file extensions
if !filename.is_empty() && (filename.ends_with(".html") || filename.ends_with(".css") || filename.ends_with(".js") || filename.ends_with(".bas") || filename.ends_with(".json")) {
info!("[PHASE1_PARSE] Adding file: {}", filename);
app.files.push(LlmFile {
filename,
content: String::new(), // Content will be generated in Phase 2
});
} else if !filename.is_empty() {
info!("[PHASE1_PARSE] Skipped file (unknown ext): {}", filename);
}
}
}
} else {
info!("[PHASE1_PARSE] FILES_PLAN section not found! Looking for <<<FILE: delimiters...");
// Fallback: try to find <<<FILE:xxx>>> delimiters directly
for file_match in response.match_indices("<<<FILE:") {
let start = file_match.0;
if let Some(rest) = response.get(start..) {
if let Some(end_offset) = rest.find(">>>") {
if let Some(filename) = rest.get(8..end_offset) {
let filename = filename.trim();
if !filename.is_empty() {
info!("[PHASE1_PARSE] Found file from delimiter: {}", filename);
app.files.push(LlmFile {
filename: filename.to_string(),
content: String::new(),
});
}
}
}
}
}
}
// Parse tools plan
if let (Some(start), Some(end)) = (response.find("<<<TOOLS_PLAN>>>"), response.find("<<<TOOLS_END>>>")) {
let tools_section = response.get(start + 16..end).unwrap_or("");
for line in tools_section.lines() {
let line = line.trim();
if line.contains(':') {
let parts: Vec<&str> = line.splitn(2, ':').collect();
let filename = parts[0].trim().to_string();
if !filename.is_empty() {
let filename = if filename.ends_with(".bas") { filename } else { format!("{}.bas", filename) };
app.tools.push(LlmFile {
filename,
content: String::new(),
});
}
}
}
} else {
info!("[PHASE1_PARSE] TOOLS_PLAN not found, trying <<<TOOL:>>> delimiters...");
for tool_match in response.match_indices("<<<TOOL:") {
let start = tool_match.0;
if let Some(rest) = response.get(start..) {
if let Some(end_offset) = rest.find(">>>") {
if let Some(tool_name) = rest.get(8..end_offset) {
let tool_name = tool_name.trim();
if !tool_name.is_empty() {
let filename = if tool_name.ends_with(".bas") { tool_name.to_string() } else { format!("{}.bas", tool_name) };
info!("[PHASE1_PARSE] Found tool from delimiter: {}", filename);
app.tools.push(LlmFile {
filename,
content: String::new(),
});
}
}
}
}
}
}
// Parse schedulers plan
if let (Some(start), Some(end)) = (response.find("<<<SCHEDULERS_PLAN>>>"), response.find("<<<SCHEDULERS_END>>>")) {
let sched_section = response.get(start + 21..end).unwrap_or("");
for line in sched_section.lines() {
let line = line.trim();
if line.contains(':') {
let parts: Vec<&str> = line.splitn(2, ':').collect();
let filename = parts[0].trim().to_string();
if !filename.is_empty() {
let filename = if filename.ends_with(".bas") { filename } else { format!("{}.bas", filename) };
app.schedulers.push(LlmFile {
filename,
content: String::new(),
});
}
}
}
} else {
info!("[PHASE1_PARSE] SCHEDULERS_PLAN not found, trying <<<SCHEDULER:>>> delimiters...");
for sched_match in response.match_indices("<<<SCHEDULER:") {
let start = sched_match.0;
if let Some(rest) = response.get(start..) {
if let Some(end_offset) = rest.find(">>>") {
if let Some(sched_name) = rest.get(13..end_offset) {
let sched_name = sched_name.trim();
if !sched_name.is_empty() {
let filename = if sched_name.ends_with(".bas") { sched_name.to_string() } else { format!("{}.bas", sched_name) };
info!("[PHASE1_PARSE] Found scheduler from delimiter: {}", filename);
app.schedulers.push(LlmFile {
filename,
content: String::new(),
});
}
}
}
}
}
}
info!("[PHASE1_PARSE] Final result: {} tables, {} files, {} tools, {} schedulers",
app.tables.len(), app.files.len(), app.tools.len(), app.schedulers.len());
if app.tables.is_empty() && app.files.is_empty() && app.tools.is_empty() {
warn!("[PHASE1_PARSE] Empty plan! LLM did not return expected delimiters. Full response ({} chars):", response.len());
let full_preview: String = response.chars().take(2000).collect();
warn!("[PHASE1_PARSE] {}", full_preview);
}
Ok(app)
}
/// PHASE 2: Generate content for a single file
async fn generate_file_content(
&self,
app: &LlmGeneratedApp,
filename: &str,
bot_id: Uuid,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let platform = Self::get_platform_prompt();
let tables_desc = app.tables.iter()
.map(|t| format!("- {}: {}", t.name, t.fields.iter().map(|f| f.name.clone()).collect::<Vec<_>>().join(", ")))
.collect::<Vec<_>>()
.join("\n");
let prompt = format!(
r#"{platform}
=== APP CONTEXT ===
App Name: {app_name}
Description: {description}
Tables:
{tables_desc}
=== GENERATE FILE: {filename} ===
Generate COMPLETE content for this file. No placeholders, no "...", no shortcuts.
Rules:
- Use data-app-name="{app_name}" in HTML for API calls
- Include all necessary HTML structure, CSS, and JavaScript
- For CRUD pages, implement full list/create/edit/delete functionality
- CSS should be comprehensive with dark mode support
- JavaScript should be complete and functional
RESPOND WITH ONLY THE FILE CONTENT. NO EXPLANATIONS."#,
platform = platform,
app_name = app.name,
description = app.description,
tables_desc = if tables_desc.is_empty() { "None".to_string() } else { tables_desc },
filename = filename,
);
self.call_llm(&prompt, bot_id).await
}
async fn generate_complete_app_with_llm(
&self,
intent: &str,
bot_id: Uuid,
) -> Result<LlmGeneratedApp, Box<dyn std::error::Error + Send + Sync>> {
let platform = Self::get_platform_prompt();
let prompt = format!(
r#"{platform}
=== FULL GENERATION MODE ===
USER REQUEST: "{intent}"
Generate a COMPLETE app with all files. Use the STREAMING DELIMITERS format from the platform docs.
Be creative - add features the user would love. NO placeholders, NO "...", complete code only.
NO QUESTIONS. JUST BUILD."#
);
let response = self.call_llm(&prompt, bot_id).await?;
Self::parse_streaming_response(&response)
}
/// Parse streaming delimiter format response
fn parse_streaming_response(
response: &str,
) -> Result<LlmGeneratedApp, Box<dyn std::error::Error + Send + Sync>> {
let mut app = LlmGeneratedApp::default();
// Find APP_START and APP_END
let start_idx = response.find(DELIM_APP_START);
let end_idx = response.find(DELIM_APP_END);
let content = match (start_idx, end_idx) {
(Some(s), Some(e)) => {
response.get(s + DELIM_APP_START.len()..e).unwrap_or("")
}
(Some(s), None) => {
warn!("No APP_END found, using rest of response");
response.get(s + DELIM_APP_START.len()..).unwrap_or("")
}
_ => {
// Fallback: try to parse as JSON for backwards compatibility
return Self::parse_json_fallback(response);
}
};
let lines: Vec<&str> = content.lines().collect();
let mut current_section = "header";
let mut current_table: Option<LlmTable> = None;
let mut current_file: Option<(String, String, String)> = None; // (type, filename, content)
for raw_line in lines.iter() {
let line = raw_line.trim();
// Parse header fields
if current_section == "header" {
if line.starts_with("name:") {
app.name = line.get(5..).unwrap_or("").trim().to_string();
continue;
}
if line.starts_with("description:") {
app.description = line.get(12..).unwrap_or("").trim().to_string();
continue;
}
if line.starts_with("domain:") {
app.domain = line.get(7..).unwrap_or("").trim().to_string();
continue;
}
}
// Section transitions
if line == DELIM_TABLES_START {
current_section = "tables";
continue;
}
if line == DELIM_TABLES_END {
// Save any pending table
if let Some(table) = current_table.take() {
if !table.name.is_empty() {
app.tables.push(table);
}
}
current_section = "files";
continue;
}
// Table definitions
if line.starts_with(DELIM_TABLE_PREFIX) && line.ends_with(DELIM_END) {
// Save previous table
if let Some(table) = current_table.take() {
if !table.name.is_empty() {
app.tables.push(table);
}
}
let table_name = line.get(DELIM_TABLE_PREFIX.len()..line.len().saturating_sub(DELIM_END.len())).unwrap_or("").trim();
current_table = Some(LlmTable {
name: table_name.to_string(),
fields: Vec::new(),
});
continue;
}
// Table field (when in tables section with active table)
if current_section == "tables" && current_table.is_some() && !line.is_empty() && !line.starts_with("<<<") {
if let Some(ref mut table) = current_table {
if let Some(field) = Self::parse_field_line(line) {
table.fields.push(field);
}
}
continue;
}
// File definitions
if line.starts_with(DELIM_FILE_PREFIX) && line.ends_with(DELIM_END) {
// Save previous file
if let Some((file_type, filename, content)) = current_file.take() {
Self::save_parsed_file(&mut app, &file_type, filename, content);
}
let filename = line.get(DELIM_FILE_PREFIX.len()..line.len().saturating_sub(DELIM_END.len())).unwrap_or("").trim();
current_file = Some(("file".to_string(), filename.to_string(), String::new()));
continue;
}
// Tool definitions
if line.starts_with(DELIM_TOOL_PREFIX) && line.ends_with(DELIM_END) {
if let Some((file_type, filename, content)) = current_file.take() {
Self::save_parsed_file(&mut app, &file_type, filename, content);
}
let filename = line.get(DELIM_TOOL_PREFIX.len()..line.len().saturating_sub(DELIM_END.len())).unwrap_or("").trim();
current_file = Some(("tool".to_string(), filename.to_string(), String::new()));
continue;
}
// Scheduler definitions
if line.starts_with(DELIM_SCHEDULER_PREFIX) && line.ends_with(DELIM_END) {
if let Some((file_type, filename, content)) = current_file.take() {
Self::save_parsed_file(&mut app, &file_type, filename, content);
}
let filename = line.get(DELIM_SCHEDULER_PREFIX.len()..line.len().saturating_sub(DELIM_END.len())).unwrap_or("").trim();
current_file = Some(("scheduler".to_string(), filename.to_string(), String::new()));
continue;
}
// Accumulate file content (use original line to preserve indentation)
if let Some((_, _, ref mut file_content)) = current_file {
if !file_content.is_empty() {
file_content.push('\n');
}
file_content.push_str(raw_line);
}
}
// Save any remaining file
if let Some((file_type, filename, content)) = current_file.take() {
Self::save_parsed_file(&mut app, &file_type, filename, content);
}
// Validate
if app.name.is_empty() {
return Err("No app name found in response".into());
}
if app.files.is_empty() {
return Err("No files generated".into());
}
info!(
"Parsed streaming response: name={}, tables={}, files={}, tools={}, schedulers={}",
app.name,
app.tables.len(),
app.files.len(),
app.tools.len(),
app.schedulers.len()
);
Ok(app)
}
/// Parse a table field line in format: name:type:nullable[:default][:ref:table]
fn parse_field_line(line: &str) -> Option<LlmField> {
let parts: Vec<&str> = line.split(':').collect();
if parts.len() < 3 {
return None;
}
let mut field = LlmField {
name: parts[0].trim().to_string(),
field_type: parts[1].trim().to_string(),
nullable: parts[2].trim() == "true",
reference: None,
default: None,
};
// Parse optional parts
let mut i = 3;
while i < parts.len() {
if parts[i].trim() == "ref" && i + 1 < parts.len() {
field.reference = Some(parts[i + 1].trim().to_string());
i += 2;
} else {
// It's a default value
field.default = Some(parts[i].trim().to_string());
i += 1;
}
}
Some(field)
}
/// Save a parsed file to the appropriate collection
fn save_parsed_file(app: &mut LlmGeneratedApp, file_type: &str, filename: String, content: String) {
let file = LlmFile {
filename,
content: content.trim().to_string(),
};
match file_type {
"tool" => app.tools.push(file),
"scheduler" => app.schedulers.push(file),
_ => app.files.push(file),
}
}
/// Fallback to JSON parsing for backwards compatibility
fn parse_json_fallback(
response: &str,
) -> Result<LlmGeneratedApp, Box<dyn std::error::Error + Send + Sync>> {
warn!("Falling back to JSON parsing");
let cleaned = response
.trim()
.trim_start_matches("```json")
.trim_start_matches("```")
.trim_end_matches("```")
.trim();
#[derive(Debug, Deserialize)]
struct JsonApp {
name: String,
description: String,
#[serde(default)]
domain: String,
#[serde(default)]
tables: Vec<JsonTable>,
#[serde(default)]
files: Vec<JsonFile>,
#[serde(default)]
tools: Option<Vec<JsonFile>>,
#[serde(default)]
schedulers: Option<Vec<JsonFile>>,
}
#[derive(Debug, Deserialize)]
struct JsonTable {
name: String,
fields: Vec<JsonField>,
}
#[derive(Debug, Deserialize)]
struct JsonField {
name: String,
#[serde(rename = "type")]
field_type: String,
#[serde(default)]
nullable: Option<bool>,
#[serde(default)]
reference: Option<String>,
#[serde(default, deserialize_with = "deserialize_default_value")]
default: Option<String>,
}
#[derive(Debug, Deserialize)]
struct JsonFile {
filename: String,
content: String,
}
/// Deserialize default value that can be string, bool, number, or null
fn deserialize_default_value<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: serde::Deserializer<'de>,
{
let value: Option<serde_json::Value> = Option::deserialize(deserializer)?;
match value {
None => Ok(None),
Some(serde_json::Value::Null) => Ok(None),
Some(serde_json::Value::String(s)) => Ok(Some(s)),
Some(serde_json::Value::Bool(b)) => Ok(Some(b.to_string())),
Some(serde_json::Value::Number(n)) => Ok(Some(n.to_string())),
Some(v) => Ok(Some(v.to_string())),
}
}
match serde_json::from_str::<JsonApp>(cleaned) {
Ok(json_app) => {
let app = LlmGeneratedApp {
name: json_app.name,
description: json_app.description,
domain: json_app.domain,
tables: json_app.tables.into_iter().map(|t| LlmTable {
name: t.name,
fields: t.fields.into_iter().map(|f| LlmField {
name: f.name,
field_type: f.field_type,
nullable: f.nullable.unwrap_or(true),
reference: f.reference,
default: f.default,
}).collect(),
}).collect(),
files: json_app.files.into_iter().map(|f| LlmFile {
filename: f.filename,
content: f.content,
}).collect(),
tools: json_app.tools.unwrap_or_default().into_iter().map(|f| LlmFile {
filename: f.filename,
content: f.content,
}).collect(),
schedulers: json_app.schedulers.unwrap_or_default().into_iter().map(|f| LlmFile {
filename: f.filename,
content: f.content,
}).collect(),
};
if app.files.is_empty() {
return Err("LLM generated no files".into());
}
Ok(app)
}
Err(e) => {
error!("Failed to parse LLM response: {}", e);
let preview: String = response.chars().take(500).collect();
error!("Response was: {}", preview);
Err(format!("Failed to parse LLM response: {}", e).into())
}
}
}
fn convert_llm_tables(llm_tables: &[LlmTable]) -> Vec<TableDefinition> {
llm_tables
.iter()
.map(|t| {
let fields = t
.fields
.iter()
.enumerate()
.map(|(i, f)| FieldDefinition {
name: f.name.clone(),
field_type: f.field_type.clone(),
is_key: f.name == "id",
is_nullable: f.nullable,
reference_table: f.reference.clone(),
default_value: f.default.clone(),
field_order: i as i32,
..Default::default()
})
.collect();
TableDefinition {
name: t.name.clone(),
connection_name: "default".to_string(),
fields,
..Default::default()
}
})
.collect()
}
fn detect_file_type(filename: &str) -> FileType {
let ext = filename.rsplit('.').next().unwrap_or("").to_lowercase();
match ext.as_str() {
"css" => FileType::Css,
"js" => FileType::Js,
"bas" => FileType::Bas,
"json" => FileType::Json,
_ => FileType::Html,
}
}
async fn call_llm(
&self,
prompt: &str,
bot_id: Uuid,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
#[cfg(feature = "llm")]
{
let config_manager = ConfigManager::new(self.state.conn.clone());
let model = config_manager
.get_config(&bot_id, "llm-model", None)
.unwrap_or_else(|_| {
config_manager
.get_config(&Uuid::nil(), "llm-model", None)
.unwrap_or_else(|_| "gpt-4".to_string())
});
let key = config_manager
.get_config(&bot_id, "llm-key", None)
.unwrap_or_else(|_| {
config_manager
.get_config(&Uuid::nil(), "llm-key", None)
.unwrap_or_default()
});
let llm_config = serde_json::json!({
"temperature": 0.7,
"max_tokens": 16000
});
let prompt_len = prompt.len();
trace!("APP_GENERATOR Starting LLM streaming: model={}, prompt_len={}", model, prompt_len);
let start = std::time::Instant::now();
// Use streaming to provide real-time feedback
let (tx, mut rx) = mpsc::channel::<String>(100);
let state = self.state.clone();
let task_id = self.task_id.clone();
// Spawn a task to receive stream chunks and broadcast them
let stream_task = tokio::spawn(async move {
let mut full_response = String::new();
let mut chunk_buffer = String::new();
let mut last_emit = std::time::Instant::now();
let mut chunk_count = 0u32;
let stream_start = std::time::Instant::now();
let mut last_progress_update = std::time::Instant::now();
let mut detected_tables: Vec<String> = Vec::new();
let mut detected_files: Vec<String> = Vec::new();
let mut detected_tools: Vec<String> = Vec::new();
trace!("APP_GENERATOR Stream receiver started");
while let Some(chunk) = rx.recv().await {
chunk_count += 1;
full_response.push_str(&chunk);
chunk_buffer.push_str(&chunk);
// Detect section markers using full_response (not chunk_buffer which gets trimmed)
let in_files_plan = full_response.contains("<<<FILES_PLAN>>>") && !full_response.contains("<<<FILES_END>>>");
let in_tools_plan = full_response.contains("<<<TOOLS_PLAN>>>") && !full_response.contains("<<<TOOLS_END>>>");
let in_schedulers_plan = full_response.contains("<<<SCHEDULERS_PLAN>>>") && !full_response.contains("<<<SCHEDULERS_END>>>");
// Detect items being generated in real-time (full generation format)
// Use full_response for reliable detection with safe string extraction
for table_match in full_response.match_indices("<<<TABLE:") {
let start = table_match.0;
if let Some(rest) = full_response.get(start..) {
if let Some(end_offset) = rest.find(">>>") {
if let Some(table_name) = rest.get(9..end_offset) {
let table_name = table_name.trim();
if !table_name.is_empty() && !detected_tables.contains(&table_name.to_string()) {
detected_tables.push(table_name.to_string());
info!("[LLM_STREAM] Detected table: {table_name}");
}
}
}
}
}
for file_match in full_response.match_indices("<<<FILE:") {
let start = file_match.0;
if let Some(rest) = full_response.get(start..) {
if let Some(end_offset) = rest.find(">>>") {
if let Some(file_name) = rest.get(8..end_offset) {
let file_name = file_name.trim();
if !file_name.is_empty() && !detected_files.contains(&file_name.to_string()) {
detected_files.push(file_name.to_string());
info!("[LLM_STREAM] Detected file: {file_name}");
}
}
}
}
}
for tool_match in full_response.match_indices("<<<TOOL:") {
let start = tool_match.0;
if let Some(rest) = full_response.get(start..) {
if let Some(end_offset) = rest.find(">>>") {
if let Some(tool_name) = rest.get(8..end_offset) {
let tool_name = tool_name.trim();
if !tool_name.is_empty() && !detected_tools.contains(&tool_name.to_string()) {
detected_tools.push(tool_name.to_string());
info!("[LLM_STREAM] Detected tool: {tool_name}");
}
}
}
}
}
// Detect items from plan format (filename: description lines)
// Parse from full_response for FILES_PLAN section
if in_files_plan {
if let Some(plan_start) = full_response.find("<<<FILES_PLAN>>>") {
if let Some(plan_content) = full_response.get(plan_start.saturating_add(16)..) {
for line in plan_content.lines() {
let line = line.trim();
if line.starts_with("<<<") {
break;
}
if line.contains(':') {
let parts: Vec<&str> = line.splitn(2, ':').collect();
let name = parts[0].trim();
if !name.is_empty() && (name.ends_with(".html") || name.ends_with(".css") || name.ends_with(".js") || name.ends_with(".bas")) {
if !detected_files.contains(&name.to_string()) {
detected_files.push(name.to_string());
info!("[LLM_STREAM] Detected planned file: {name}");
}
}
}
}
}
}
}
if in_tools_plan {
if let Some(plan_start) = full_response.find("<<<TOOLS_PLAN>>>") {
if let Some(plan_content) = full_response.get(plan_start.saturating_add(16)..) {
for line in plan_content.lines() {
let line = line.trim();
if line.starts_with("<<<") {
break;
}
if line.contains(':') {
let parts: Vec<&str> = line.splitn(2, ':').collect();
let name = parts[0].trim();
if !name.is_empty() {
let tool_name = if name.ends_with(".bas") { name.to_string() } else { format!("{name}.bas") };
if !detected_tools.contains(&tool_name) {
detected_tools.push(tool_name.clone());
info!("[LLM_STREAM] Detected planned tool: {tool_name}");
}
}
}
}
}
}
}
if in_schedulers_plan {
if let Some(plan_start) = full_response.find("<<<SCHEDULERS_PLAN>>>") {
if let Some(plan_content) = full_response.get(plan_start.saturating_add(21)..) {
for line in plan_content.lines() {
let line = line.trim();
if line.starts_with("<<<") {
break;
}
if line.contains(':') {
let parts: Vec<&str> = line.splitn(2, ':').collect();
let name = parts[0].trim();
if !name.is_empty() {
let sched_name = if name.ends_with(".bas") { name.to_string() } else { format!("{name}.bas") };
if !detected_tools.contains(&sched_name) {
detected_tools.push(sched_name.clone());
info!("[LLM_STREAM] Detected planned scheduler: {sched_name}");
}
}
}
}
}
}
}
// Log progress periodically
if chunk_count == 1 || chunk_count % 500 == 0 {
trace!("APP_GENERATOR Stream progress: {} chunks, {} chars, {:?}",
chunk_count, full_response.len(), stream_start.elapsed());
}
// Emit progress updates every 2 seconds
if last_progress_update.elapsed().as_secs() >= 2 {
if let Some(ref tid) = task_id {
let total_detected = detected_tables.len() + detected_files.len() + detected_tools.len();
let progress_msg = if total_detected > 0 {
format!(
"AI generating... {} tables, {} files, {} tools detected",
detected_tables.len(),
detected_files.len(),
detected_tools.len()
)
} else {
let chars_received = full_response.len();
format!("AI generating content... {} chars received", chars_received)
};
info!("[LLM_STREAM] Progress: {}", progress_msg);
let event = crate::core::shared::state::TaskProgressEvent::new(
tid,
"llm_generating",
&progress_msg,
)
.with_progress(1, 10);
state.broadcast_task_progress(event);
}
last_progress_update = std::time::Instant::now();
}
// Don't emit raw LLM stream to WebSocket - it contains HTML/code garbage
// Only clear buffer periodically to track progress
if last_emit.elapsed().as_millis() > 100 || chunk_buffer.len() > 500 {
// Keep last 200 chars for detecting split delimiters (Unicode-safe)
if chunk_buffer.chars().count() > 200 {
chunk_buffer = chunk_buffer.chars().skip(chunk_buffer.chars().count() - 200).collect();
}
last_emit = std::time::Instant::now();
}
}
// Final progress update
if let Some(ref tid) = task_id {
let total_detected = detected_tables.len() + detected_files.len() + detected_tools.len();
let final_msg = if total_detected > 0 {
format!(
"AI complete: {} tables, {} files, {} tools",
detected_tables.len(),
detected_files.len(),
detected_tools.len()
)
} else {
format!("AI complete: {} chars generated", full_response.len())
};
info!("[LLM_STREAM] {}", final_msg);
let event = crate::core::shared::state::TaskProgressEvent::new(
tid,
"llm_complete",
&final_msg,
)
.with_progress(2, 10);
state.broadcast_task_progress(event);
}
trace!("APP_GENERATOR Stream finished: {} chunks, {} chars in {:?}",
chunk_count, full_response.len(), stream_start.elapsed());
// Don't emit remaining buffer - it's raw code/HTML
if !chunk_buffer.is_empty() {
trace!("APP_GENERATOR Final buffer (not emitting): {} chars", chunk_buffer.len());
}
// Log response preview (Unicode-safe)
if !full_response.is_empty() {
let preview: String = full_response.chars().take(200).collect();
let suffix = if full_response.chars().count() > 200 { "..." } else { "" };
trace!("APP_GENERATOR Response preview: {}{}", preview.replace('\n', "\\n"), suffix);
}
full_response
});
// Start the streaming LLM call
trace!("APP_GENERATOR Starting generate_stream...");
match self
.state
.llm_provider
.generate_stream(prompt, &llm_config, tx, &model, &key)
.await
{
Ok(()) => {
trace!("APP_GENERATOR generate_stream completed, waiting for stream_task");
// Wait for the stream task to complete and get the full response
match stream_task.await {
Ok(response) => {
let elapsed = start.elapsed();
trace!("APP_GENERATOR LLM streaming succeeded: {} chars in {:?}", response.len(), elapsed);
if response.is_empty() {
error!("APP_GENERATOR Empty response from LLM");
}
return Ok(response);
}
Err(e) => {
let elapsed = start.elapsed();
error!("APP_GENERATOR LLM stream task failed after {:?}: {}", elapsed, e);
return Err(format!("Stream task failed: {}", e).into());
}
}
}
Err(e) => {
let elapsed = start.elapsed();
error!("APP_GENERATOR LLM streaming failed after {:?}: {}", elapsed, e);
// Abort the stream task
stream_task.abort();
return Err(e);
}
}
}
#[cfg(not(feature = "llm"))]
{
Err("LLM feature not enabled. App generation requires LLM.".into())
}
}
fn generate_table_definitions(
tables: &[TableDefinition],
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
use std::fmt::Write;
let mut output = String::new();
for table in tables {
let _ = writeln!(output, "\nTABLE {}", table.name);
for field in &table.fields {
let mut line = format!(" {} AS {}", field.name, field.field_type.to_uppercase());
if field.is_key {
line.push_str(" KEY");
}
if !field.is_nullable {
line.push_str(" REQUIRED");
}
if let Some(ref default) = field.default_value {
let _ = write!(line, " DEFAULT {}", default);
}
if let Some(ref refs) = field.reference_table {
let _ = write!(line, " REFERENCES {}", refs);
}
let _ = writeln!(output, "{}", line);
}
let _ = writeln!(output, "END TABLE\n");
}
Ok(output)
}
fn append_to_tables_bas(
&self,
_bot_id: Uuid,
content: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Use bucket_name from state instead of deriving from bot name
let bucket = self.state.bucket_name.clone();
let path = ".gbdata/tables.bas";
let mut conn = self.state.conn.get()?;
#[derive(QueryableByName)]
struct ContentRow {
#[diesel(sql_type = diesel::sql_types::Text)]
content: String,
}
let existing: Option<String> =
sql_query("SELECT content FROM drive_files WHERE bucket = $1 AND path = $2 LIMIT 1")
.bind::<diesel::sql_types::Text, _>(&bucket)
.bind::<diesel::sql_types::Text, _>(path)
.load::<ContentRow>(&mut conn)
.ok()
.and_then(|rows| rows.into_iter().next().map(|r| r.content));
let new_content = match existing {
Some(existing_content) => format!("{}\n{}", existing_content, content),
None => content.to_string(),
};
sql_query(
"INSERT INTO drive_files (id, bucket, path, content, content_type, created_at, updated_at)
VALUES ($1, $2, $3, $4, 'text/plain', NOW(), NOW())
ON CONFLICT (bucket, path) DO UPDATE SET content = $4, updated_at = NOW()",
)
.bind::<diesel::sql_types::Uuid, _>(Uuid::new_v4())
.bind::<diesel::sql_types::Text, _>(&bucket)
.bind::<diesel::sql_types::Text, _>(path)
.bind::<diesel::sql_types::Text, _>(&new_content)
.execute(&mut conn)?;
Ok(())
}
/// Ensure the bucket exists, creating it if necessary
async fn ensure_bucket_exists(
&self,
bucket: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if let Some(ref s3) = self.state.drive {
// Check if bucket exists
match s3.head_bucket().bucket(bucket).send().await {
Ok(_) => {
trace!("Bucket {} already exists", bucket);
return Ok(());
}
Err(_) => {
// Bucket doesn't exist, try to create it
info!("Bucket {} does not exist, creating...", bucket);
match s3.create_bucket().bucket(bucket).send().await {
Ok(_) => {
info!("Created bucket: {}", bucket);
return Ok(());
}
Err(e) => {
// Check if error is "bucket already exists" (race condition)
let err_str = format!("{:?}", e);
if err_str.contains("BucketAlreadyExists") || err_str.contains("BucketAlreadyOwnedByYou") {
trace!("Bucket {} already exists (race condition)", bucket);
return Ok(());
}
error!("Failed to create bucket {}: {}", bucket, e);
return Err(Box::new(e));
}
}
}
}
} else {
// No S3 client, we'll use DB fallback - no bucket needed
trace!("No S3 client, using DB fallback for storage");
Ok(())
}
}
async fn write_to_drive(
&self,
bucket: &str,
path: &str,
content: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!("write_to_drive: bucket={}, path={}, content_len={}", bucket, path, content.len());
if let Some(ref s3) = self.state.drive {
let body = ByteStream::from(content.as_bytes().to_vec());
let content_type = get_content_type(path);
info!("S3 client available, attempting put_object to s3://{}/{}", bucket, path);
match s3.put_object()
.bucket(bucket)
.key(path)
.body(body)
.content_type(content_type)
.send()
.await
{
Ok(_) => {
info!("Successfully wrote to S3: s3://{}/{}", bucket, path);
}
Err(e) => {
// Log detailed error info
error!("S3 put_object failed: bucket={}, path={}, error={:?}", bucket, path, e);
error!("S3 error details: {}", e);
// If bucket doesn't exist, try to create it and retry
let err_str = format!("{:?}", e);
if err_str.contains("NoSuchBucket") || err_str.contains("NotFound") {
warn!("Bucket {} not found, attempting to create...", bucket);
self.ensure_bucket_exists(bucket).await?;
// Retry the write
let body = ByteStream::from(content.as_bytes().to_vec());
s3.put_object()
.bucket(bucket)
.key(path)
.body(body)
.content_type(get_content_type(path))
.send()
.await?;
info!("Wrote to S3 after creating bucket: s3://{}/{}", bucket, path);
} else {
error!("S3 write failed (not a bucket issue): {}", err_str);
return Err(Box::new(e));
}
}
}
} else {
warn!("No S3/drive client available, using DB fallback for {}/{}", bucket, path);
self.write_to_db_fallback(bucket, path, content)?;
}
Ok(())
}
fn write_to_db_fallback(
&self,
bucket: &str,
path: &str,
content: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let mut conn = self.state.conn.get()?;
let content_type = get_content_type(path);
sql_query(
"INSERT INTO drive_files (id, bucket, path, content, content_type, size, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, NOW(), NOW())
ON CONFLICT (bucket, path) DO UPDATE SET
content = EXCLUDED.content,
content_type = EXCLUDED.content_type,
size = EXCLUDED.size,
updated_at = NOW()",
)
.bind::<diesel::sql_types::Uuid, _>(Uuid::new_v4())
.bind::<diesel::sql_types::Text, _>(bucket)
.bind::<diesel::sql_types::Text, _>(path)
.bind::<diesel::sql_types::Text, _>(content)
.bind::<diesel::sql_types::Text, _>(content_type)
.bind::<diesel::sql_types::BigInt, _>(content.len() as i64)
.execute(&mut conn)?;
trace!("Wrote to DB: {}/{}", bucket, path);
Ok(())
}
/// Sync a single table to database - used for real-time progress updates
fn sync_single_table_to_database(
&self,
table: &TableDefinition,
) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
let mut conn = self.state.conn.get()?;
let create_sql = generate_create_table_sql(table, "postgres");
sql_query(&create_sql).execute(&mut conn)?;
info!("Created table: {}", table.name);
Ok(table.fields.len())
}
fn update_task_app_url(
&self,
task_id: Uuid,
app_url: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let mut conn = self.state.conn.get()?;
let final_step_results = serde_json::json!([
{
"step_id": "file_0",
"step_order": 1,
"step_name": "Generate app structure",
"status": "Completed",
"duration_ms": 500,
"logs": [{"message": "App structure generated"}]
},
{
"step_id": "file_1",
"step_order": 2,
"step_name": "Write app files",
"status": "Completed",
"duration_ms": 300,
"logs": [{"message": "Files written to storage"}]
},
{
"step_id": "file_2",
"step_order": 3,
"step_name": "Configure app",
"status": "Completed",
"duration_ms": 200,
"logs": [{"message": format!("App ready at {}", app_url)}]
}
]);
sql_query(
"UPDATE auto_tasks SET
progress = 1.0,
current_step = 3,
total_steps = 3,
step_results = $1,
status = 'completed',
completed_at = NOW(),
updated_at = NOW()
WHERE id = $2",
)
.bind::<diesel::sql_types::Jsonb, _>(final_step_results)
.bind::<diesel::sql_types::Uuid, _>(task_id)
.execute(&mut conn)?;
info!("Updated task {} completed with app_url: {}", task_id, app_url);
Ok(())
}
fn update_task_step_results(
&self,
task_id: Uuid,
files: &[String],
completed_count: usize,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let mut conn = self.state.conn.get()?;
// Build step_results JSON with file status
let step_results: Vec<serde_json::Value> = files.iter().enumerate().map(|(idx, filename)| {
let status = if idx < completed_count {
"Completed"
} else if idx == completed_count {
"Running"
} else {
"Pending"
};
serde_json::json!({
"step_id": format!("file_{}", idx),
"step_order": idx + 1,
"step_name": format!("Write {}", filename),
"status": status,
"started_at": chrono::Utc::now().to_rfc3339(),
"duration_ms": if idx < completed_count { Some(100) } else { None::<i64> },
"logs": []
})
}).collect();
let step_results_json = serde_json::to_value(&step_results)?;
let progress = if files.is_empty() { 0.0 } else { completed_count as f64 / files.len() as f64 };
sql_query(
"UPDATE auto_tasks SET
step_results = $1,
current_step = $2,
total_steps = $3,
progress = $4,
updated_at = NOW()
WHERE id = $5",
)
.bind::<diesel::sql_types::Jsonb, _>(step_results_json)
.bind::<diesel::sql_types::Integer, _>(completed_count as i32)
.bind::<diesel::sql_types::Integer, _>(files.len() as i32)
.bind::<diesel::sql_types::Double, _>(progress)
.bind::<diesel::sql_types::Uuid, _>(task_id)
.execute(&mut conn)?;
trace!("Updated task {} step_results: {}/{} files", task_id, completed_count, files.len());
Ok(())
}
fn generate_designer_js(app_name: &str) -> String {
format!(
r#"(function() {{
const APP_NAME = '{app_name}';
const currentPage = window.location.pathname.split('/').pop() || 'index.html';
const style = document.createElement('style');
style.textContent = `
.designer-fab {{ position: fixed; bottom: 20px; right: 20px; width: 56px; height: 56px; border-radius: 50%; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border: none; cursor: pointer; box-shadow: 0 4px 12px rgba(102,126,234,0.4); font-size: 24px; z-index: 9999; transition: transform 0.2s; }}
.designer-fab:hover {{ transform: scale(1.1); }}
.designer-panel {{ position: fixed; bottom: 90px; right: 20px; width: 380px; max-height: 500px; background: white; border-radius: 16px; box-shadow: 0 10px 40px rgba(0,0,0,0.2); z-index: 9998; display: none; flex-direction: column; overflow: hidden; }}
.designer-panel.open {{ display: flex; }}
.designer-header {{ padding: 16px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; font-weight: 600; display: flex; justify-content: space-between; align-items: center; }}
.designer-close {{ background: none; border: none; color: white; font-size: 20px; cursor: pointer; }}
.designer-messages {{ flex: 1; overflow-y: auto; padding: 16px; max-height: 300px; }}
.designer-msg {{ margin: 8px 0; padding: 10px 14px; border-radius: 12px; max-width: 85%; word-wrap: break-word; }}
.designer-msg.user {{ background: #667eea; color: white; margin-left: auto; }}
.designer-msg.ai {{ background: #f0f0f0; color: #333; }}
.designer-input {{ display: flex; padding: 12px; border-top: 1px solid #eee; gap: 8px; }}
.designer-input input {{ flex: 1; padding: 10px 14px; border: 1px solid #ddd; border-radius: 20px; outline: none; }}
.designer-input button {{ padding: 10px 16px; background: #667eea; color: white; border: none; border-radius: 20px; cursor: pointer; }}
`;
document.head.appendChild(style);
const fab = document.createElement('button');
fab.className = 'designer-fab';
fab.innerHTML = '🎨';
fab.title = 'Designer AI';
document.body.appendChild(fab);
const panel = document.createElement('div');
panel.className = 'designer-panel';
panel.innerHTML = `
<div class="designer-header">
<span>🎨 Designer AI</span>
<button class="designer-close">×</button>
</div>
<div class="designer-messages">
<div class="designer-msg ai">Hi! I can help you modify this app. What would you like to change?</div>
</div>
<div class="designer-input">
<input type="text" placeholder="e.g., Add a blue header..." />
<button>Send</button>
</div>
`;
document.body.appendChild(panel);
fab.onclick = () => panel.classList.toggle('open');
panel.querySelector('.designer-close').onclick = () => panel.classList.remove('open');
const input = panel.querySelector('input');
const sendBtn = panel.querySelector('.designer-input button');
const messages = panel.querySelector('.designer-messages');
async function sendMessage() {{
const msg = input.value.trim();
if (!msg) return;
messages.innerHTML += `<div class="designer-msg user">${{msg}}</div>`;
input.value = '';
messages.scrollTop = messages.scrollHeight;
try {{
const res = await fetch('/api/designer/modify', {{
method: 'POST',
headers: {{ 'Content-Type': 'application/json' }},
body: JSON.stringify({{ app_name: APP_NAME, current_page: currentPage, message: msg }})
}});
const data = await res.json();
messages.innerHTML += `<div class="designer-msg ai">${{data.message || 'Done!'}}</div>`;
if (data.success && data.changes && data.changes.length > 0) {{
setTimeout(() => location.reload(), 1500);
}}
}} catch (e) {{
messages.innerHTML += `<div class="designer-msg ai">Sorry, something went wrong. Try again.</div>`;
if (window.AppLogger) window.AppLogger.error('Designer error', e.toString());
}}
messages.scrollTop = messages.scrollHeight;
}}
sendBtn.onclick = sendMessage;
input.onkeypress = (e) => {{ if (e.key === 'Enter') sendMessage(); }};
}})();"#,
app_name = app_name
)
}
}