//! Multimedia Message Handling Module //! //! This module provides support for handling various multimedia message types including //! images, videos, audio, documents, and web search results. //! //! Key features: //! - Multiple media type support (images, videos, audio, documents) //! - Media upload and download handling //! - Thumbnail generation //! - Web search integration //! - Storage abstraction for S3-compatible backends //! - URL processing and validation use crate::shared::models::{BotResponse, UserMessage}; use anyhow::Result; use async_trait::async_trait; use base64::{engine::general_purpose::STANDARD, Engine}; use serde::{Deserialize, Serialize}; use uuid::Uuid; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] pub enum MultimediaMessage { Text { content: String, }, Image { url: String, caption: Option, mime_type: String, }, Video { url: String, thumbnail_url: Option, caption: Option, duration: Option, mime_type: String, }, Audio { url: String, duration: Option, mime_type: String, }, Document { url: String, filename: String, mime_type: String, }, WebSearch { query: String, results: Vec, }, Location { latitude: f64, longitude: f64, name: Option, address: Option, }, MeetingInvite { meeting_id: String, meeting_url: String, start_time: Option, duration: Option, participants: Vec, }, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SearchResult { pub title: String, pub url: String, pub snippet: String, pub thumbnail: Option, } #[derive(Debug, Serialize, Deserialize)] pub struct MediaUploadRequest { pub file_name: String, pub content_type: String, pub data: Vec, pub user_id: String, pub session_id: String, } #[derive(Debug, Serialize, Deserialize)] pub struct MediaUploadResponse { pub media_id: String, pub url: String, pub thumbnail_url: Option, } /// Trait for handling multimedia messages #[async_trait] pub trait MultimediaHandler: Send + Sync { /// Process an incoming multimedia message async fn process_multimedia( &self, message: MultimediaMessage, user_id: &str, session_id: &str, ) -> Result; /// Upload media file to storage async fn upload_media(&self, request: MediaUploadRequest) -> Result; /// Download media file from URL async fn download_media(&self, url: &str) -> Result>; /// Perform web search async fn web_search(&self, query: &str, max_results: usize) -> Result>; /// Generate thumbnail for video/image async fn generate_thumbnail(&self, media_url: &str) -> Result; } /// Default implementation for multimedia handling #[derive(Debug)] pub struct DefaultMultimediaHandler { storage_client: Option, search_api_key: Option, } impl DefaultMultimediaHandler { pub fn new(storage_client: Option, search_api_key: Option) -> Self { Self { storage_client, search_api_key, } } pub fn storage_client(&self) -> &Option { &self.storage_client } pub fn search_api_key(&self) -> &Option { &self.search_api_key } } #[async_trait] impl MultimediaHandler for DefaultMultimediaHandler { async fn process_multimedia( &self, message: MultimediaMessage, user_id: &str, session_id: &str, ) -> Result { match message { MultimediaMessage::Text { content } => { // Process as regular text message Ok(BotResponse { bot_id: "default".to_string(), user_id: user_id.to_string(), session_id: session_id.to_string(), channel: "multimedia".to_string(), content, message_type: 0, stream_token: None, is_complete: true, suggestions: Vec::new(), context_name: None, context_length: 0, context_max_length: 0, }) } MultimediaMessage::Image { url, caption, .. } => { // Process image with optional caption log::debug!("Processing image from URL: {}", url); let response_content = format!( "I see you've shared an image from {}{}. {}", url, caption .as_ref() .map(|c| format!(" with caption: {}", c)) .unwrap_or_default(), "Let me analyze this for you." ); Ok(BotResponse { bot_id: "default".to_string(), user_id: user_id.to_string(), session_id: session_id.to_string(), channel: "multimedia".to_string(), content: response_content, message_type: 0, stream_token: None, is_complete: true, suggestions: Vec::new(), context_name: None, context_length: 0, context_max_length: 0, }) } MultimediaMessage::Video { url, caption, duration, .. } => { // Process video log::debug!("Processing video from URL: {}", url); let response_content = format!( "You've shared a video from {}{}{}. Processing video content...", url, duration.map(|d| format!(" ({}s)", d)).unwrap_or_default(), caption .as_ref() .map(|c| format!(" - {}", c)) .unwrap_or_default() ); Ok(BotResponse { bot_id: "default".to_string(), user_id: user_id.to_string(), session_id: session_id.to_string(), channel: "multimedia".to_string(), content: response_content, message_type: 0, stream_token: None, is_complete: true, suggestions: Vec::new(), context_name: None, context_length: 0, context_max_length: 0, }) } MultimediaMessage::WebSearch { query, .. } => { // Perform web search let results = self.web_search(&query, 5).await?; let response_content = if results.is_empty() { format!("No results found for: {}", query) } else { let results_text = results .iter() .enumerate() .map(|(i, r)| { format!("{}. [{}]({})\n {}", i + 1, r.title, r.url, r.snippet) }) .collect::>() .join("\n\n"); format!("Search results for \"{}\":\n\n{}", query, results_text) }; Ok(BotResponse { bot_id: "default".to_string(), user_id: user_id.to_string(), session_id: session_id.to_string(), channel: "multimedia".to_string(), content: response_content, message_type: 0, stream_token: None, is_complete: true, suggestions: Vec::new(), context_name: None, context_length: 0, context_max_length: 0, }) } MultimediaMessage::MeetingInvite { meeting_url, start_time, .. } => { let response_content = format!( "Meeting invite received. Join at: {}{}", meeting_url, start_time .as_ref() .map(|t| format!("\nScheduled for: {}", t)) .unwrap_or_default() ); Ok(BotResponse { bot_id: "default".to_string(), user_id: user_id.to_string(), session_id: session_id.to_string(), channel: "multimedia".to_string(), content: response_content, message_type: 0, stream_token: None, is_complete: true, suggestions: Vec::new(), context_name: None, context_length: 0, context_max_length: 0, }) } _ => { // Handle other message types Ok(BotResponse { bot_id: "default".to_string(), user_id: user_id.to_string(), session_id: session_id.to_string(), channel: "multimedia".to_string(), content: "Message received and processing...".to_string(), message_type: 0, stream_token: None, is_complete: true, suggestions: Vec::new(), context_name: None, context_length: 0, context_max_length: 0, }) } } } async fn upload_media(&self, request: MediaUploadRequest) -> Result { let media_id = Uuid::new_v4().to_string(); let key = format!( "media/{}/{}/{}", request.user_id, request.session_id, request.file_name ); if let Some(client) = &self.storage_client { // Upload to S3 client .put_object() .bucket("botserver-media") .key(&key) .body(request.data.into()) .content_type(&request.content_type) .send() .await?; let url = format!("https://storage.botserver.com/{}", key); Ok(MediaUploadResponse { media_id, url, thumbnail_url: None, }) } else { // Fallback to local storage let local_path = format!("./media/{}", key); std::fs::create_dir_all(std::path::Path::new(&local_path).parent().unwrap())?; std::fs::write(&local_path, request.data)?; Ok(MediaUploadResponse { media_id, url: format!("file://{}", local_path), thumbnail_url: None, }) } } async fn download_media(&self, url: &str) -> Result> { if url.starts_with("http://") || url.starts_with("https://") { let response = reqwest::get(url).await?; Ok(response.bytes().await?.to_vec()) } else if url.starts_with("file://") { let path = url.strip_prefix("file://").unwrap(); Ok(std::fs::read(path)?) } else { Err(anyhow::anyhow!("Unsupported URL scheme: {}", url)) } } async fn web_search(&self, query: &str, max_results: usize) -> Result> { // Implement web search using a search API (e.g., Bing, Google, DuckDuckGo) // For now, return mock results let mock_results = vec![ SearchResult { title: format!("Result 1 for: {}", query), url: "https://example.com/1".to_string(), snippet: "This is a sample search result snippet...".to_string(), thumbnail: None, }, SearchResult { title: format!("Result 2 for: {}", query), url: "https://example.com/2".to_string(), snippet: "Another sample search result...".to_string(), thumbnail: None, }, ]; Ok(mock_results.into_iter().take(max_results).collect()) } async fn generate_thumbnail(&self, media_url: &str) -> Result { // Generate thumbnail using image/video processing libraries // For now, return the same URL Ok(media_url.to_string()) } } /// Extension trait for UserMessage to support multimedia impl UserMessage { pub fn to_multimedia(&self) -> MultimediaMessage { // Parse message content to determine type if self.content.starts_with("http") { // Check if it's an image/video URL if self.content.contains(".jpg") || self.content.contains(".png") || self.content.contains(".gif") { MultimediaMessage::Image { url: self.content.clone(), caption: None, mime_type: "image/jpeg".to_string(), } } else if self.content.contains(".mp4") || self.content.contains(".webm") || self.content.contains(".mov") { MultimediaMessage::Video { url: self.content.clone(), thumbnail_url: None, caption: None, duration: None, mime_type: "video/mp4".to_string(), } } else { MultimediaMessage::Text { content: self.content.clone(), } } } else if self.content.starts_with("/search ") { let query = self .content .strip_prefix("/search ") .unwrap_or(&self.content); MultimediaMessage::WebSearch { query: query.to_string(), results: Vec::new(), } } else { MultimediaMessage::Text { content: self.content.clone(), } } } } // ============================================================================ // REST API Handlers // ============================================================================ use crate::shared::state::AppState; use axum::{ extract::{Path, State}, http::StatusCode, response::IntoResponse, Json, }; use std::sync::Arc; /// Upload media file pub async fn upload_media_handler( State(state): State>, Json(request): Json, ) -> impl IntoResponse { let handler = DefaultMultimediaHandler::new(state.drive.clone(), None); match handler.upload_media(request).await { Ok(response) => (StatusCode::OK, Json(serde_json::json!(response))), Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})), ), } } /// Download media file by ID pub async fn download_media_handler( State(state): State>, Path(media_id): Path, ) -> impl IntoResponse { let handler = DefaultMultimediaHandler::new(state.drive.clone(), None); // Construct URL from media_id (this would be stored in DB in production) let url = format!("https://storage.botserver.com/media/{}", media_id); match handler.download_media(&url).await { Ok(data) => ( StatusCode::OK, Json(serde_json::json!({ "media_id": media_id, "size": data.len(), "data": STANDARD.encode(&data) })), ), Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})), ), } } /// Generate thumbnail for media pub async fn generate_thumbnail_handler( State(state): State>, Path(media_id): Path, ) -> impl IntoResponse { let handler = DefaultMultimediaHandler::new(state.drive.clone(), None); // Construct URL from media_id let url = format!("https://storage.botserver.com/media/{}", media_id); match handler.generate_thumbnail(&url).await { Ok(thumbnail_url) => ( StatusCode::OK, Json(serde_json::json!({ "media_id": media_id, "thumbnail_url": thumbnail_url })), ), Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})), ), } } /// Perform web search pub async fn web_search_handler( State(state): State>, Json(payload): Json, ) -> impl IntoResponse { let query = payload.get("query").and_then(|q| q.as_str()).unwrap_or(""); let max_results = payload .get("max_results") .and_then(|m| m.as_u64()) .unwrap_or(10) as usize; let handler = DefaultMultimediaHandler::new(state.drive.clone(), None); match handler.web_search(query, max_results).await { Ok(results) => ( StatusCode::OK, Json(serde_json::json!({ "query": query, "results": results })), ), Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})), ), } }