From 0c11cf8d5c676b7fe43a64bae7852c6c4cfb890d Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (Pragmatismo)" Date: Thu, 4 Dec 2025 18:15:09 -0300 Subject: [PATCH] feat(email): implement email read tracking with pixel support - Add email-read-pixel config parameter to enable/disable tracking - Implement tracking pixel injection in HTML emails - Add sent_email_tracking table with migration - Create 4 new API endpoints: - GET /api/email/tracking/pixel/{id} - serve pixel & record read - GET /api/email/tracking/status/{id} - get email read status - GET /api/email/tracking/list - list all tracked emails - GET /api/email/tracking/stats - get aggregate statistics - Store tracking data: read_at, read_count, IP, user_agent - Integrate with send_email() to auto-inject pixel when enabled --- PROMPT.md | 42 ++ docs/guides/templates.md | 8 +- docs/reference/configuration.md | 32 +- migrations/6.2.1_email_tracking/down.sql | 19 + migrations/6.2.1_email_tracking/up.sql | 56 ++ src/basic/keywords/episodic_memory.rs | 66 ++- src/email/mod.rs | 498 +++++++++++++++++- src/llm/episodic_memory.rs | 14 +- .../annoucements.gbot/config.csv | 12 +- .../contacts.gbai/contacts.gbot/config.csv | 4 +- .../sales-pipeline.gbot/config.csv | 4 +- .../default.gbai/default.gbot/config.csv | 2 +- .../employees.gbai/employees.gbot/config.csv | 4 +- .../it/helpdesk.gbai/helpdesk.gbot/config.csv | 4 +- 14 files changed, 689 insertions(+), 76 deletions(-) create mode 100644 migrations/6.2.1_email_tracking/down.sql create mode 100644 migrations/6.2.1_email_tracking/up.sql diff --git a/PROMPT.md b/PROMPT.md index 560795de..961a4be0 100644 --- a/PROMPT.md +++ b/PROMPT.md @@ -5,6 +5,48 @@ --- +## Official Icons - MANDATORY + +**NEVER generate icons with LLM. ALWAYS use official SVG icons from assets.** + +Icons are stored in two locations (kept in sync): +- `botui/ui/suite/assets/icons/` - Runtime icons for UI +- `botbook/src/assets/icons/` - Documentation icons + +### Available Icons + +| Icon | File | Usage | +|------|------|-------| +| Logo | `gb-logo.svg` | Main GB branding | +| Bot | `gb-bot.svg` | Bot/assistant representation | +| Analytics | `gb-analytics.svg` | Charts, metrics, dashboards | +| Calendar | `gb-calendar.svg` | Scheduling, events | +| Chat | `gb-chat.svg` | Conversations, messaging | +| Compliance | `gb-compliance.svg` | Security, auditing | +| Designer | `gb-designer.svg` | Workflow automation | +| Drive | `gb-drive.svg` | File storage, documents | +| Mail | `gb-mail.svg` | Email functionality | +| Meet | `gb-meet.svg` | Video conferencing | +| Paper | `gb-paper.svg` | Document editing | +| Research | `gb-research.svg` | Search, investigation | +| Sources | `gb-sources.svg` | Knowledge bases | +| Tasks | `gb-tasks.svg` | Task management | + +### Icon Guidelines + +- All icons use `stroke="currentColor"` for CSS theming +- ViewBox: `0 0 24 24` +- Stroke width: `1.5` +- Rounded line caps and joins + +**DO NOT:** +- Generate new icons with AI/LLM +- Use emoji or unicode symbols as icons +- Use external icon libraries +- Create inline SVG content + +--- + ## Project Overview BotServer is the core backend for General Bots - an open-source conversational AI platform built in Rust. It provides: diff --git a/docs/guides/templates.md b/docs/guides/templates.md index 26f9bdd8..502ccc6a 100644 --- a/docs/guides/templates.md +++ b/docs/guides/templates.md @@ -79,7 +79,7 @@ name,value theme-title,My CRM Bot theme-color1,#2196F3 theme-color2,#E3F2FD -prompt-history,2 +episodic-memory-history,2 ``` ### 4. Customize Knowledge Base @@ -144,8 +144,8 @@ theme-title,My Template theme-color1,#1565C0 theme-color2,#E3F2FD theme-logo,https://example.com/logo.svg -prompt-history,2 -prompt-compact,4 +episodic-memory-history,2 +episodic-memory-threshold,4 ``` ### Step 3: Create Start Dialog @@ -239,7 +239,7 @@ Email support@example.com. - Use clear, descriptive `theme-title` - Choose accessible color combinations -- Set appropriate `prompt-history` (2-4 recommended) +- Set appropriate `episodic-memory-history` (2-4 recommended) ### Knowledge Base diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 53394fb3..25a25f94 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -50,28 +50,31 @@ theme-color2,#E3F2FD | Red | `#C62828` | `#FFEBEE` | | Dark | `#212121` | `#424242` | -## Prompt Settings +## Episodic Memory Settings | Setting | Description | Default | Range | |---------|-------------|---------|-------| -| `prompt-history` | Messages in context | `2` | 1-10 | -| `prompt-compact` | Compact mode threshold | `4` | 2-20 | -| `prompt-max-tokens` | Max response tokens | `2048` | 256-8192 | -| `prompt-temperature` | Response creativity | `0.7` | 0.0-2.0 | +| `episodic-memory-history` | Messages in context | `2` | 1-10 | +| `episodic-memory-threshold` | Compaction threshold | `4` | 2-20 | +| `episodic-memory-enabled` | Enable episodic memory | `true` | Boolean | +| `episodic-memory-model` | Model for summarization | `fast` | String | +| `episodic-memory-max-episodes` | Max episodes per user | `100` | 1-1000 | +| `episodic-memory-retention-days` | Days to retain episodes | `365` | 1-3650 | +| `episodic-memory-auto-summarize` | Auto-summarize conversations | `true` | Boolean | ```csv name,value -prompt-history,2 -prompt-compact,4 -prompt-max-tokens,2048 -prompt-temperature,0.7 +episodic-memory-history,2 +episodic-memory-threshold,4 +episodic-memory-enabled,true +episodic-memory-auto-summarize,true ``` ### History Settings -- `prompt-history=1`: Minimal context, faster responses -- `prompt-history=2`: Balanced (recommended) -- `prompt-history=5`: More context, slower responses +- `episodic-memory-history=1`: Minimal context, faster responses +- `episodic-memory-history=2`: Balanced (recommended) +- `episodic-memory-history=5`: More context, slower responses ## LLM Settings @@ -286,9 +289,8 @@ theme-title,Acme Support Bot theme-color1,#1565C0 theme-color2,#E3F2FD theme-logo,https://acme.com/logo.svg -prompt-history,2 -prompt-compact,4 -prompt-temperature,0.7 +episodic-memory-history,2 +episodic-memory-threshold,4 llm-provider,openai llm-model,gpt-4-turbo feature-voice,false diff --git a/migrations/6.2.1_email_tracking/down.sql b/migrations/6.2.1_email_tracking/down.sql new file mode 100644 index 00000000..0e8cbafe --- /dev/null +++ b/migrations/6.2.1_email_tracking/down.sql @@ -0,0 +1,19 @@ +-- Down migration: Remove email tracking table and related objects + +-- Drop trigger first +DROP TRIGGER IF EXISTS trigger_update_sent_email_tracking_updated_at ON sent_email_tracking; + +-- Drop function +DROP FUNCTION IF EXISTS update_sent_email_tracking_updated_at(); + +-- Drop indexes +DROP INDEX IF EXISTS idx_sent_email_tracking_tracking_id; +DROP INDEX IF EXISTS idx_sent_email_tracking_bot_id; +DROP INDEX IF EXISTS idx_sent_email_tracking_account_id; +DROP INDEX IF EXISTS idx_sent_email_tracking_to_email; +DROP INDEX IF EXISTS idx_sent_email_tracking_sent_at; +DROP INDEX IF EXISTS idx_sent_email_tracking_is_read; +DROP INDEX IF EXISTS idx_sent_email_tracking_read_status; + +-- Drop table +DROP TABLE IF EXISTS sent_email_tracking; diff --git a/migrations/6.2.1_email_tracking/up.sql b/migrations/6.2.1_email_tracking/up.sql new file mode 100644 index 00000000..33c73c20 --- /dev/null +++ b/migrations/6.2.1_email_tracking/up.sql @@ -0,0 +1,56 @@ +-- Email Read Tracking Table +-- Stores sent email tracking data for read receipt functionality +-- Enabled via config.csv: email-read-pixel,true + +CREATE TABLE IF NOT EXISTS sent_email_tracking ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tracking_id UUID NOT NULL UNIQUE, + bot_id UUID NOT NULL, + account_id UUID NOT NULL, + from_email VARCHAR(255) NOT NULL, + to_email VARCHAR(255) NOT NULL, + cc TEXT, + bcc TEXT, + subject TEXT NOT NULL, + sent_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + is_read BOOLEAN NOT NULL DEFAULT FALSE, + read_at TIMESTAMPTZ, + read_count INTEGER NOT NULL DEFAULT 0, + first_read_ip VARCHAR(45), + last_read_ip VARCHAR(45), + user_agent TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Indexes for efficient queries +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_tracking_id ON sent_email_tracking(tracking_id); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_bot_id ON sent_email_tracking(bot_id); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_account_id ON sent_email_tracking(account_id); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_to_email ON sent_email_tracking(to_email); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_sent_at ON sent_email_tracking(sent_at DESC); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_is_read ON sent_email_tracking(is_read); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_read_status ON sent_email_tracking(bot_id, is_read, sent_at DESC); + +-- Trigger to auto-update updated_at +CREATE OR REPLACE FUNCTION update_sent_email_tracking_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trigger_update_sent_email_tracking_updated_at ON sent_email_tracking; +CREATE TRIGGER trigger_update_sent_email_tracking_updated_at + BEFORE UPDATE ON sent_email_tracking + FOR EACH ROW + EXECUTE FUNCTION update_sent_email_tracking_updated_at(); + +-- Add comment for documentation +COMMENT ON TABLE sent_email_tracking IS 'Tracks sent emails for read receipt functionality via tracking pixel'; +COMMENT ON COLUMN sent_email_tracking.tracking_id IS 'Unique ID embedded in tracking pixel URL'; +COMMENT ON COLUMN sent_email_tracking.is_read IS 'Whether the email has been opened (pixel loaded)'; +COMMENT ON COLUMN sent_email_tracking.read_count IS 'Number of times the email was opened'; +COMMENT ON COLUMN sent_email_tracking.first_read_ip IS 'IP address of first email open'; +COMMENT ON COLUMN sent_email_tracking.last_read_ip IS 'IP address of most recent email open'; diff --git a/src/basic/keywords/episodic_memory.rs b/src/basic/keywords/episodic_memory.rs index 71df1e0b..51a3a74c 100644 --- a/src/basic/keywords/episodic_memory.rs +++ b/src/basic/keywords/episodic_memory.rs @@ -29,11 +29,12 @@ //! ```csv //! name,value //! episodic-memory-enabled,true -//! episodic-summary-threshold,20 -//! episodic-summary-model,fast -//! episodic-max-episodes,100 -//! episodic-retention-days,365 -//! episodic-auto-summarize,true +//! episodic-memory-threshold,4 +//! episodic-memory-history,2 +//! episodic-memory-model,fast +//! episodic-memory-max-episodes,100 +//! episodic-memory-retention-days,365 +//! episodic-memory-auto-summarize,true //! ``` use chrono::{DateTime, Duration, Utc}; @@ -170,9 +171,11 @@ pub struct EpisodicMemoryConfig { /// Whether episodic memory is enabled pub enabled: bool, /// Message count threshold before auto-summarization - pub summary_threshold: usize, + pub threshold: usize, + /// Number of recent exchanges to keep in full + pub history: usize, /// Model to use for summarization - pub summary_model: String, + pub model: String, /// Maximum episodes to keep per user pub max_episodes: usize, /// Days to retain episodes @@ -185,8 +188,9 @@ impl Default for EpisodicMemoryConfig { fn default() -> Self { EpisodicMemoryConfig { enabled: true, - summary_threshold: 20, - summary_model: "fast".to_string(), + threshold: 4, + history: 2, + model: "fast".to_string(), max_episodes: 100, retention_days: 365, auto_summarize: true, @@ -222,24 +226,28 @@ impl EpisodicMemoryManager { .get("episodic-memory-enabled") .map(|v| v == "true") .unwrap_or(true), - summary_threshold: config_map - .get("episodic-summary-threshold") + threshold: config_map + .get("episodic-memory-threshold") .and_then(|v| v.parse().ok()) - .unwrap_or(20), - summary_model: config_map - .get("episodic-summary-model") + .unwrap_or(4), + history: config_map + .get("episodic-memory-history") + .and_then(|v| v.parse().ok()) + .unwrap_or(2), + model: config_map + .get("episodic-memory-model") .cloned() .unwrap_or_else(|| "fast".to_string()), max_episodes: config_map - .get("episodic-max-episodes") + .get("episodic-memory-max-episodes") .and_then(|v| v.parse().ok()) .unwrap_or(100), retention_days: config_map - .get("episodic-retention-days") + .get("episodic-memory-retention-days") .and_then(|v| v.parse().ok()) .unwrap_or(365), auto_summarize: config_map - .get("episodic-auto-summarize") + .get("episodic-memory-auto-summarize") .map(|v| v == "true") .unwrap_or(true), }; @@ -248,9 +256,17 @@ impl EpisodicMemoryManager { /// Check if auto-summarization should trigger pub fn should_summarize(&self, message_count: usize) -> bool { - self.config.enabled - && self.config.auto_summarize - && message_count >= self.config.summary_threshold + self.config.enabled && self.config.auto_summarize && message_count >= self.config.threshold + } + + /// Get number of recent exchanges to keep in full + pub fn get_history_to_keep(&self) -> usize { + self.config.history + } + + /// Get the threshold value + pub fn get_threshold(&self) -> usize { + self.config.threshold } /// Generate the summarization prompt @@ -668,7 +684,8 @@ mod tests { fn test_default_config() { let config = EpisodicMemoryConfig::default(); assert!(config.enabled); - assert_eq!(config.summary_threshold, 20); + assert_eq!(config.threshold, 4); + assert_eq!(config.history, 2); assert_eq!(config.max_episodes, 100); } @@ -676,14 +693,15 @@ mod tests { fn test_should_summarize() { let manager = EpisodicMemoryManager::new(EpisodicMemoryConfig { enabled: true, - summary_threshold: 10, + threshold: 4, + history: 2, auto_summarize: true, ..Default::default() }); - assert!(!manager.should_summarize(5)); + assert!(!manager.should_summarize(2)); + assert!(manager.should_summarize(4)); assert!(manager.should_summarize(10)); - assert!(manager.should_summarize(15)); } #[test] diff --git a/src/email/mod.rs b/src/email/mod.rs index 7389f5eb..7a3fefe8 100644 --- a/src/email/mod.rs +++ b/src/email/mod.rs @@ -1,6 +1,6 @@ use crate::{config::EmailConfig, core::urls::ApiUrls, shared::state::AppState}; use axum::{ - extract::{Path, State}, + extract::{Path, Query, State}, http::StatusCode, response::{IntoResponse, Response}, Json, @@ -10,10 +10,11 @@ use axum::{ Router, }; use base64::{engine::general_purpose, Engine as _}; +use chrono::{DateTime, Utc}; use diesel::prelude::*; use imap::types::Seq; use lettre::{transport::smtp::authentication::Credentials, Message, SmtpTransport, Transport}; -use log::info; +use log::{debug, info, warn}; use mailparse::{parse_mail, MailHeaderMap}; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -64,6 +65,11 @@ pub fn configure() -> Router> { ) .route("/api/email/:id", get(get_email_content_htmx)) .route("/api/email/:id", delete(delete_email_htmx)) + // Email read tracking endpoints + .route("/api/email/tracking/pixel/{tracking_id}", get(serve_tracking_pixel)) + .route("/api/email/tracking/status/{tracking_id}", get(get_tracking_status)) + .route("/api/email/tracking/list", get(list_sent_emails_tracking)) + .route("/api/email/tracking/stats", get(get_tracking_stats)) } // Export SaveDraftRequest for other modules @@ -77,6 +83,65 @@ pub struct SaveDraftRequest { pub body: String, } +// ===== Email Tracking Structures ===== + +/// Sent email tracking record +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SentEmailTracking { + pub id: String, + pub tracking_id: String, + pub bot_id: String, + pub account_id: String, + pub from_email: String, + pub to_email: String, + pub cc: Option, + pub bcc: Option, + pub subject: String, + pub sent_at: DateTime, + pub read_at: Option>, + pub read_count: i32, + pub first_read_ip: Option, + pub last_read_ip: Option, + pub user_agent: Option, + pub is_read: bool, +} + +/// Tracking status response for UI +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TrackingStatusResponse { + pub tracking_id: String, + pub to_email: String, + pub subject: String, + pub sent_at: String, + pub is_read: bool, + pub read_at: Option, + pub read_count: i32, +} + +/// Query params for tracking pixel +#[derive(Debug, Deserialize)] +pub struct TrackingPixelQuery { + pub t: Option, // Additional tracking token +} + +/// Query params for listing tracked emails +#[derive(Debug, Deserialize)] +pub struct ListTrackingQuery { + pub account_id: Option, + pub limit: Option, + pub offset: Option, + pub filter: Option, // "all", "read", "unread" +} + +/// Tracking statistics response +#[derive(Debug, Serialize)] +pub struct TrackingStatsResponse { + pub total_sent: i64, + pub total_read: i64, + pub read_rate: f64, + pub avg_time_to_read_hours: Option, +} + // ===== Request/Response Structures ===== #[derive(Debug, Serialize, Deserialize)] @@ -598,6 +663,17 @@ pub async fn send_email( format!("{} <{}>", display_name, from_email) }; + // Check if email-read-pixel is enabled in bot config + let pixel_enabled = is_tracking_pixel_enabled(&state, None).await; + let tracking_id = Uuid::new_v4(); + + // Build email body with tracking pixel if enabled + let final_body = if pixel_enabled && request.is_html { + inject_tracking_pixel(&request.body, &tracking_id.to_string(), &state).await + } else { + request.body.clone() + }; + // Build email let mut email_builder = Message::builder() .from( @@ -609,15 +685,15 @@ pub async fn send_email( .to .parse() .map_err(|e| EmailError(format!("Invalid to address: {}", e)))?) - .subject(request.subject); + .subject(request.subject.clone()); - if let Some(cc) = request.cc { + if let Some(ref cc) = request.cc { email_builder = email_builder.cc(cc .parse() .map_err(|e| EmailError(format!("Invalid cc address: {}", e)))?); } - if let Some(bcc) = request.bcc { + if let Some(ref bcc) = request.bcc { email_builder = email_builder.bcc( bcc.parse() .map_err(|e| EmailError(format!("Invalid bcc address: {}", e)))?, @@ -625,7 +701,7 @@ pub async fn send_email( } let email = email_builder - .body(request.body) + .body(final_body) .map_err(|e| EmailError(format!("Failed to build email: {}", e)))?; // Send email @@ -640,7 +716,31 @@ pub async fn send_email( .send(&email) .map_err(|e| EmailError(format!("Failed to send email: {}", e)))?; - info!("Email sent successfully from account {}", account_uuid); + // Save tracking record if pixel tracking is enabled + if pixel_enabled { + let conn = state.conn.clone(); + let to_email = request.to.clone(); + let subject = request.subject.clone(); + let cc_clone = request.cc.clone(); + let bcc_clone = request.bcc.clone(); + + let _ = tokio::task::spawn_blocking(move || { + save_email_tracking_record( + conn, + tracking_id, + account_uuid, + Uuid::nil(), // bot_id - would come from session in production + &from_email, + &to_email, + cc_clone.as_deref(), + bcc_clone.as_deref(), + &subject, + ) + }) + .await; + } + + info!("Email sent successfully from account {} with tracking_id {}", account_uuid, tracking_id); Ok(Json(ApiResponse { success: true, @@ -793,6 +893,390 @@ pub async fn save_click( (StatusCode::OK, [("content-type", "image/gif")], pixel) } +// ===== Email Read Tracking Functions ===== + +/// 1x1 transparent GIF pixel bytes +const TRACKING_PIXEL: [u8; 43] = [ + 0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x01, 0x00, 0x01, 0x00, 0x80, 0x00, 0x00, 0xFF, 0xFF, + 0xFF, 0x00, 0x00, 0x00, 0x21, 0xF9, 0x04, 0x01, 0x00, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x02, 0x02, 0x44, 0x01, 0x00, 0x3B, +]; + +/// Check if email-read-pixel is enabled in config +async fn is_tracking_pixel_enabled(state: &Arc, bot_id: Option) -> bool { + let config_manager = crate::core::config::ConfigManager::new(state.conn.clone()); + let bot_id = bot_id.unwrap_or(Uuid::nil()); + + config_manager + .get_config(&bot_id, "email-read-pixel", Some("false")) + .map(|v| v.to_lowercase() == "true") + .unwrap_or(false) +} + +/// Inject tracking pixel into HTML email body +async fn inject_tracking_pixel(html_body: &str, tracking_id: &str, state: &Arc) -> String { + // Get base URL from config or use default + let config_manager = crate::core::config::ConfigManager::new(state.conn.clone()); + let base_url = config_manager + .get_config(&Uuid::nil(), "server-url", Some("http://localhost:8080")) + .unwrap_or_else(|| "http://localhost:8080".to_string()); + + let pixel_url = format!("{}/api/email/tracking/pixel/{}", base_url, tracking_id); + let pixel_html = format!( + r#""#, + pixel_url + ); + + // Insert pixel before closing tag, or at the end if no body tag + if html_body.to_lowercase().contains("") { + html_body.replace("", &format!("{}", pixel_html)) + .replace("", &format!("{}", pixel_html)) + } else { + format!("{}{}", html_body, pixel_html) + } +} + +/// Save email tracking record to database +fn save_email_tracking_record( + conn: crate::shared::utils::DbPool, + tracking_id: Uuid, + account_id: Uuid, + bot_id: Uuid, + from_email: &str, + to_email: &str, + cc: Option<&str>, + bcc: Option<&str>, + subject: &str, +) -> Result<(), String> { + let mut db_conn = conn.get().map_err(|e| format!("DB connection error: {}", e))?; + + let id = Uuid::new_v4(); + let now = Utc::now(); + + diesel::sql_query( + r#"INSERT INTO sent_email_tracking + (id, tracking_id, bot_id, account_id, from_email, to_email, cc, bcc, subject, sent_at, read_count, is_read) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 0, false)"# + ) + .bind::(id) + .bind::(tracking_id) + .bind::(bot_id) + .bind::(account_id) + .bind::(from_email) + .bind::(to_email) + .bind::, _>(cc) + .bind::, _>(bcc) + .bind::(subject) + .bind::(now) + .execute(&mut db_conn) + .map_err(|e| format!("Failed to save tracking record: {}", e))?; + + debug!("Saved email tracking record: tracking_id={}", tracking_id); + Ok(()) +} + +/// Serve tracking pixel and record email open +pub async fn serve_tracking_pixel( + Path(tracking_id): Path, + State(state): State>, + Query(_query): Query, + headers: axum::http::HeaderMap, +) -> impl IntoResponse { + // Extract client info from headers + let client_ip = headers + .get("x-forwarded-for") + .and_then(|v| v.to_str().ok()) + .map(|s| s.split(',').next().unwrap_or(s).trim().to_string()) + .or_else(|| { + headers + .get("x-real-ip") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + }); + + let user_agent = headers + .get("user-agent") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()); + + // Parse tracking ID + if let Ok(tracking_uuid) = Uuid::parse_str(&tracking_id) { + let conn = state.conn.clone(); + let ip_clone = client_ip.clone(); + let ua_clone = user_agent.clone(); + + // Update tracking record asynchronously + let _ = tokio::task::spawn_blocking(move || { + update_email_read_status(conn, tracking_uuid, ip_clone, ua_clone) + }) + .await; + + info!("Email read tracked: tracking_id={}, ip={:?}", tracking_id, client_ip); + } else { + warn!("Invalid tracking ID received: {}", tracking_id); + } + + // Always return the pixel, regardless of tracking success + // This prevents email clients from showing broken images + ( + StatusCode::OK, + [ + ("content-type", "image/gif"), + ("cache-control", "no-store, no-cache, must-revalidate, max-age=0"), + ("pragma", "no-cache"), + ("expires", "0"), + ], + TRACKING_PIXEL.to_vec(), + ) +} + +/// Update email read status in database +fn update_email_read_status( + conn: crate::shared::utils::DbPool, + tracking_id: Uuid, + client_ip: Option, + user_agent: Option, +) -> Result<(), String> { + let mut db_conn = conn.get().map_err(|e| format!("DB connection error: {}", e))?; + let now = Utc::now(); + + // Update tracking record - increment read count, set first/last read info + diesel::sql_query( + r#"UPDATE sent_email_tracking + SET + is_read = true, + read_count = read_count + 1, + read_at = COALESCE(read_at, $2), + first_read_ip = COALESCE(first_read_ip, $3), + last_read_ip = $3, + user_agent = COALESCE(user_agent, $4), + updated_at = $2 + WHERE tracking_id = $1"# + ) + .bind::(tracking_id) + .bind::(now) + .bind::, _>(client_ip.as_deref()) + .bind::, _>(user_agent.as_deref()) + .execute(&mut db_conn) + .map_err(|e| format!("Failed to update tracking record: {}", e))?; + + debug!("Updated email read status: tracking_id={}", tracking_id); + Ok(()) +} + +/// Get tracking status for a specific email +pub async fn get_tracking_status( + Path(tracking_id): Path, + State(state): State>, +) -> Result>, EmailError> { + let tracking_uuid = Uuid::parse_str(&tracking_id) + .map_err(|_| EmailError("Invalid tracking ID".to_string()))?; + + let conn = state.conn.clone(); + let result = tokio::task::spawn_blocking(move || { + get_tracking_record(conn, tracking_uuid) + }) + .await + .map_err(|e| EmailError(format!("Task join error: {}", e)))? + .map_err(EmailError)?; + + Ok(Json(ApiResponse { + success: true, + data: Some(result), + message: None, + })) +} + +/// Get tracking record from database +fn get_tracking_record( + conn: crate::shared::utils::DbPool, + tracking_id: Uuid, +) -> Result { + let mut db_conn = conn.get().map_err(|e| format!("DB connection error: {}", e))?; + + #[derive(QueryableByName)] + struct TrackingRow { + #[diesel(sql_type = diesel::sql_types::Uuid)] + tracking_id: Uuid, + #[diesel(sql_type = diesel::sql_types::Text)] + to_email: String, + #[diesel(sql_type = diesel::sql_types::Text)] + subject: String, + #[diesel(sql_type = diesel::sql_types::Timestamptz)] + sent_at: DateTime, + #[diesel(sql_type = diesel::sql_types::Bool)] + is_read: bool, + #[diesel(sql_type = diesel::sql_types::Nullable)] + read_at: Option>, + #[diesel(sql_type = diesel::sql_types::Integer)] + read_count: i32, + } + + let row: TrackingRow = diesel::sql_query( + r#"SELECT tracking_id, to_email, subject, sent_at, is_read, read_at, read_count + FROM sent_email_tracking WHERE tracking_id = $1"# + ) + .bind::(tracking_id) + .get_result(&mut db_conn) + .map_err(|e| format!("Tracking record not found: {}", e))?; + + Ok(TrackingStatusResponse { + tracking_id: row.tracking_id.to_string(), + to_email: row.to_email, + subject: row.subject, + sent_at: row.sent_at.to_rfc3339(), + is_read: row.is_read, + read_at: row.read_at.map(|dt| dt.to_rfc3339()), + read_count: row.read_count, + }) +} + +/// List sent emails with tracking status +pub async fn list_sent_emails_tracking( + State(state): State>, + Query(query): Query, +) -> Result>>, EmailError> { + let conn = state.conn.clone(); + let result = tokio::task::spawn_blocking(move || { + list_tracking_records(conn, query) + }) + .await + .map_err(|e| EmailError(format!("Task join error: {}", e)))? + .map_err(EmailError)?; + + Ok(Json(ApiResponse { + success: true, + data: Some(result), + message: None, + })) +} + +/// List tracking records from database +fn list_tracking_records( + conn: crate::shared::utils::DbPool, + query: ListTrackingQuery, +) -> Result, String> { + let mut db_conn = conn.get().map_err(|e| format!("DB connection error: {}", e))?; + + let limit = query.limit.unwrap_or(50); + let offset = query.offset.unwrap_or(0); + + #[derive(QueryableByName)] + struct TrackingRow { + #[diesel(sql_type = diesel::sql_types::Uuid)] + tracking_id: Uuid, + #[diesel(sql_type = diesel::sql_types::Text)] + to_email: String, + #[diesel(sql_type = diesel::sql_types::Text)] + subject: String, + #[diesel(sql_type = diesel::sql_types::Timestamptz)] + sent_at: DateTime, + #[diesel(sql_type = diesel::sql_types::Bool)] + is_read: bool, + #[diesel(sql_type = diesel::sql_types::Nullable)] + read_at: Option>, + #[diesel(sql_type = diesel::sql_types::Integer)] + read_count: i32, + } + + // Build query based on filter + let base_query = match query.filter.as_deref() { + Some("read") => { + r#"SELECT tracking_id, to_email, subject, sent_at, is_read, read_at, read_count + FROM sent_email_tracking WHERE is_read = true + ORDER BY sent_at DESC LIMIT $1 OFFSET $2"# + } + Some("unread") => { + r#"SELECT tracking_id, to_email, subject, sent_at, is_read, read_at, read_count + FROM sent_email_tracking WHERE is_read = false + ORDER BY sent_at DESC LIMIT $1 OFFSET $2"# + } + _ => { + r#"SELECT tracking_id, to_email, subject, sent_at, is_read, read_at, read_count + FROM sent_email_tracking + ORDER BY sent_at DESC LIMIT $1 OFFSET $2"# + } + }; + + let rows: Vec = diesel::sql_query(base_query) + .bind::(limit) + .bind::(offset) + .load(&mut db_conn) + .map_err(|e| format!("Query failed: {}", e))?; + + Ok(rows + .into_iter() + .map(|row| TrackingStatusResponse { + tracking_id: row.tracking_id.to_string(), + to_email: row.to_email, + subject: row.subject, + sent_at: row.sent_at.to_rfc3339(), + is_read: row.is_read, + read_at: row.read_at.map(|dt| dt.to_rfc3339()), + read_count: row.read_count, + }) + .collect()) +} + +/// Get tracking statistics +pub async fn get_tracking_stats( + State(state): State>, +) -> Result>, EmailError> { + let conn = state.conn.clone(); + let result = tokio::task::spawn_blocking(move || { + calculate_tracking_stats(conn) + }) + .await + .map_err(|e| EmailError(format!("Task join error: {}", e)))? + .map_err(EmailError)?; + + Ok(Json(ApiResponse { + success: true, + data: Some(result), + message: None, + })) +} + +/// Calculate tracking statistics from database +fn calculate_tracking_stats( + conn: crate::shared::utils::DbPool, +) -> Result { + let mut db_conn = conn.get().map_err(|e| format!("DB connection error: {}", e))?; + + #[derive(QueryableByName)] + struct StatsRow { + #[diesel(sql_type = diesel::sql_types::BigInt)] + total_sent: i64, + #[diesel(sql_type = diesel::sql_types::BigInt)] + total_read: i64, + #[diesel(sql_type = diesel::sql_types::Nullable)] + avg_time_hours: Option, + } + + let stats: StatsRow = diesel::sql_query( + r#"SELECT + COUNT(*) as total_sent, + COUNT(*) FILTER (WHERE is_read = true) as total_read, + AVG(EXTRACT(EPOCH FROM (read_at - sent_at)) / 3600) FILTER (WHERE is_read = true) as avg_time_hours + FROM sent_email_tracking"# + ) + .get_result(&mut db_conn) + .map_err(|e| format!("Stats query failed: {}", e))?; + + let read_rate = if stats.total_sent > 0 { + (stats.total_read as f64 / stats.total_sent as f64) * 100.0 + } else { + 0.0 + }; + + Ok(TrackingStatsResponse { + total_sent: stats.total_sent, + total_read: stats.total_read, + read_rate, + avg_time_to_read_hours: stats.avg_time_hours, + }) +} + pub async fn get_emails( Path(campaign_id): Path, State(_state): State>, diff --git a/src/llm/episodic_memory.rs b/src/llm/episodic_memory.rs index 5bdec12d..1f0554a4 100644 --- a/src/llm/episodic_memory.rs +++ b/src/llm/episodic_memory.rs @@ -37,20 +37,17 @@ async fn process_episodic_memory( for session in sessions { let config_manager = ConfigManager::new(state.conn.clone()); - // Support both old and new config key names for backwards compatibility let threshold = config_manager .get_config(&session.bot_id, "episodic-memory-threshold", None) - .or_else(|_| config_manager.get_config(&session.bot_id, "prompt-compact", None)) .unwrap_or_default() .parse::() - .unwrap_or(4); // Default to 4 if not configured + .unwrap_or(4); let history_to_keep = config_manager .get_config(&session.bot_id, "episodic-memory-history", None) - .or_else(|_| config_manager.get_config(&session.bot_id, "prompt-history", None)) .unwrap_or_default() .parse::() - .unwrap_or(2); // Default to 2 if not configured + .unwrap_or(2); if threshold == 0 { return Ok(()); @@ -75,7 +72,6 @@ async fn process_episodic_memory( .position(|(role, _)| role == "episodic" || role == "compact") .map(|pos| history.len() - pos - 1); - // Calculate start index: if there's a summary, start after it; otherwise start from 0 let start_index = last_summary_index.map(|idx| idx + 1).unwrap_or(0); for (_i, (role, _)) in history.iter().enumerate().skip(start_index) { @@ -112,7 +108,6 @@ async fn process_episodic_memory( history_to_keep ); - // Determine which messages to summarize and which to keep let total_messages = history.len() - start_index; let messages_to_summarize = if total_messages > history_to_keep { total_messages - history_to_keep @@ -132,7 +127,6 @@ async fn process_episodic_memory( conversation .push_str("Please summarize this conversation between user and bot: \n\n [[[***** \n"); - // Only summarize messages beyond the history_to_keep threshold for (role, content) in history.iter().skip(start_index).take(messages_to_summarize) { if role == "episodic" || role == "compact" { continue; @@ -170,7 +164,6 @@ async fn process_episodic_memory( session.id, summary.len() ); - // Use handler to filter content let handler = llm_models::get_handler( config_manager .get_config(&session.bot_id, "llm-model", None) @@ -187,7 +180,7 @@ async fn process_episodic_memory( session.id, e ); trace!("Using fallback summary for session {}", session.id); - format!("EPISODIC MEMORY: {}", filtered) // Fallback + format!("EPISODIC MEMORY: {}", filtered) } }; info!( @@ -195,7 +188,6 @@ async fn process_episodic_memory( session.id, messages_to_summarize, history_to_keep ); - // Save the episodic memory (role 9 = episodic/compact) { let mut session_manager = state.session_manager.lock().await; session_manager.save_message(session.id, session.user_id, 9, &summarized, 1)?; diff --git a/templates/announcements.gbai/annoucements.gbot/config.csv b/templates/announcements.gbai/annoucements.gbot/config.csv index 1ceb4165..4509b186 100644 --- a/templates/announcements.gbai/annoucements.gbot/config.csv +++ b/templates/announcements.gbai/annoucements.gbot/config.csv @@ -1,7 +1,7 @@ name,value -prompt-history, 2 -prompt-compact, 4 -theme-color1, #0d2b55 -theme-color2, #fff9c2 -theme-logo, https://pragmatismo.com.br/icons/general-bots.svg -theme-title, Announcements General Bots +episodic-memory-history,2 +episodic-memory-threshold,4 +theme-color1,#0d2b55 +theme-color2,#fff9c2 +theme-logo,https://pragmatismo.com.br/icons/general-bots.svg +theme-title,Announcements General Bots diff --git a/templates/crm/contacts.gbai/contacts.gbot/config.csv b/templates/crm/contacts.gbai/contacts.gbot/config.csv index ac1fb483..d7d848e8 100644 --- a/templates/crm/contacts.gbai/contacts.gbot/config.csv +++ b/templates/crm/contacts.gbai/contacts.gbot/config.csv @@ -1,6 +1,6 @@ name,value -prompt-history,2 -prompt-compact,4 +episodic-memory-history,2 +episodic-memory-threshold,4 theme-color1,#1565C0 theme-color2,#E3F2FD theme-logo,https://pragmatismo.com.br/icons/general-bots.svg diff --git a/templates/crm/sales-pipeline.gbai/sales-pipeline.gbot/config.csv b/templates/crm/sales-pipeline.gbai/sales-pipeline.gbot/config.csv index 8188ecfe..425c6fd5 100644 --- a/templates/crm/sales-pipeline.gbai/sales-pipeline.gbot/config.csv +++ b/templates/crm/sales-pipeline.gbai/sales-pipeline.gbot/config.csv @@ -1,6 +1,6 @@ name,value -prompt-history,2 -prompt-compact,4 +episodic-memory-history,2 +episodic-memory-threshold,4 theme-color1,#2E7D32 theme-color2,#E8F5E9 theme-logo,https://pragmatismo.com.br/icons/general-bots.svg diff --git a/templates/default.gbai/default.gbot/config.csv b/templates/default.gbai/default.gbot/config.csv index 135788d1..e22b7263 100644 --- a/templates/default.gbai/default.gbot/config.csv +++ b/templates/default.gbai/default.gbot/config.csv @@ -13,7 +13,7 @@ llm-cache-ttl,3600 llm-cache-semantic,true llm-cache-threshold,0.95 , -prompt-compact,4 +episodic-memory-threshold,4 , mcp-server,false , diff --git a/templates/hr/employees.gbai/employees.gbot/config.csv b/templates/hr/employees.gbai/employees.gbot/config.csv index 47443d9b..d95c9a5e 100644 --- a/templates/hr/employees.gbai/employees.gbot/config.csv +++ b/templates/hr/employees.gbai/employees.gbot/config.csv @@ -1,6 +1,6 @@ name,value -prompt-history,2 -prompt-compact,4 +episodic-memory-history,2 +episodic-memory-threshold,4 theme-color1,#2E7D32 theme-color2,#E8F5E9 theme-logo,https://pragmatismo.com.br/icons/general-bots.svg diff --git a/templates/it/helpdesk.gbai/helpdesk.gbot/config.csv b/templates/it/helpdesk.gbai/helpdesk.gbot/config.csv index 0f9f31c1..b8a97f8f 100644 --- a/templates/it/helpdesk.gbai/helpdesk.gbot/config.csv +++ b/templates/it/helpdesk.gbai/helpdesk.gbot/config.csv @@ -1,6 +1,6 @@ name,value -prompt-history,2 -prompt-compact,4 +episodic-memory-history,2 +episodic-memory-threshold,4 theme-color1,#1565C0 theme-color2,#E3F2FD theme-logo,https://pragmatismo.com.br/icons/general-bots.svg