diff --git a/beautify_all_svgs.py b/beautify_all_svgs.py new file mode 100644 index 000000000..7499c4582 --- /dev/null +++ b/beautify_all_svgs.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +""" +SVG Beautifier - Updates all SVG diagrams for perfect readability on all devices +with beautiful colors that work in both color and black/white modes. +""" + +import os +import re +from pathlib import Path + +# Beautiful color palette that works in grayscale +COLORS = { + # Primary colors with good contrast + 'primary_blue': '#2563EB', # Bright blue - appears as dark gray in B&W + 'primary_green': '#059669', # Emerald green - medium gray in B&W + 'primary_purple': '#7C3AED', # Purple - medium-dark gray in B&W + 'primary_orange': '#EA580C', # Orange - medium gray in B&W + 'primary_red': '#DC2626', # Red - dark gray in B&W + 'primary_teal': '#0891B2', # Teal - medium gray in B&W + + # Text colors for maximum readability + 'text_primary': '#1F2937', # Almost black - perfect for main text + 'text_secondary': '#4B5563', # Dark gray - for secondary text + 'text_accent': '#2563EB', # Blue for emphasis - dark in B&W + + # Background and border colors + 'bg_light': '#F9FAFB', # Very light gray background + 'border_primary': '#2563EB', # Blue borders - visible in B&W + 'border_secondary': '#9CA3AF', # Gray borders + + # Status colors + 'success': '#059669', # Green - medium gray in B&W + 'warning': '#EA580C', # Orange - medium gray in B&W + 'error': '#DC2626', # Red - dark gray in B&W + 'info': '#2563EB', # Blue - dark gray in B&W +} + +# Consistent font sizes for all devices (matching documentation) +FONT_SIZES = { + 'title': '24', # Main diagram titles + 'subtitle': '20', # Section titles + 'heading': '18', # Component headings + 'body': '16', # Main text (matches doc font size) + 'label': '14', # Labels and annotations + 'small': '12', # Small details (minimum for mobile) +} + +# Standard margins and padding +LAYOUT = { + 'margin': 40, # Outer margin + 'padding': 20, # Inner padding + 'spacing': 15, # Element spacing + 'corner_radius': 8, # Rounded corners +} + +def create_improved_svg(content, filename): + """ + Transform SVG content with improved styling for all devices. + """ + + # Extract viewBox or width/height + viewbox_match = re.search(r'viewBox="([^"]+)"', content) + width_match = re.search(r'width="(\d+)"', content) + height_match = re.search(r'height="(\d+)"', content) + + if viewbox_match: + viewbox = viewbox_match.group(1) + vb_parts = viewbox.split() + width = int(vb_parts[2]) + height = int(vb_parts[3]) + elif width_match and height_match: + width = int(width_match.group(1)) + height = int(height_match.group(1)) + else: + width, height = 800, 600 # Default size + + # Add responsive margins + new_width = width + (LAYOUT['margin'] * 2) + new_height = height + (LAYOUT['margin'] * 2) + + # Create new SVG header with responsive sizing + new_header = f''' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ''' + + # Process the content + content = re.sub(r']*>', '', content) + content = re.sub(r'', '', content) + + # Update font sizes to be mobile-friendly and consistent + content = re.sub(r'font-size="(\d+)"', lambda m: update_font_size(m), content) + content = re.sub(r'font-size:\s*(\d+)(?:px)?', lambda m: f"font-size:{update_font_size_style(m)}", content) + + # Update text colors for better contrast + content = re.sub(r'fill="#[A-Fa-f0-9]{6}"', lambda m: update_text_color(m), content) + content = re.sub(r'stroke="#[A-Fa-f0-9]{6}"', lambda m: update_stroke_color(m), content) + + # Improve rectangles with better styling + content = re.sub(r']+)>', lambda m: improve_rect(m), content) + + # Update text elements with better positioning and styling + content = re.sub(r']*)>(.*?)', lambda m: improve_text(m), content) + + # Add font family consistency + content = re.sub(r'font-family="[^"]*"', + 'font-family="-apple-system, BlinkMacSystemFont, \'Segoe UI\', Roboto, \'Helvetica Neue\', Arial, sans-serif"', + content) + + # Close the container and SVG + new_footer = ''' + +''' + + return new_header + content + new_footer + +def update_font_size(match): + """Update font sizes to be mobile-friendly.""" + size = int(match.group(1)) + if size >= 20: + return f'font-size="{FONT_SIZES["title"]}"' + elif size >= 16: + return f'font-size="{FONT_SIZES["heading"]}"' + elif size >= 14: + return f'font-size="{FONT_SIZES["body"]}"' + elif size >= 12: + return f'font-size="{FONT_SIZES["label"]}"' + else: + return f'font-size="{FONT_SIZES["small"]}"' + +def update_font_size_style(match): + """Update font sizes in style attributes.""" + size = int(match.group(1)) + if size >= 20: + return FONT_SIZES["title"] + elif size >= 16: + return FONT_SIZES["heading"] + elif size >= 14: + return FONT_SIZES["body"] + elif size >= 12: + return FONT_SIZES["label"] + else: + return FONT_SIZES["small"] + +def update_text_color(match): + """Update text fill colors for better contrast.""" + color = match.group(0) + # Check if it's a light color (rough heuristic) + if any(light in color.lower() for light in ['fff', 'fef', 'efe', 'fee', 'eee', 'ddd', 'ccc']): + return f'fill="{COLORS["text_primary"]}"' + # Keep dark colors but ensure they're dark enough + elif any(dark in color.lower() for dark in ['000', '111', '222', '333', '444']): + return f'fill="{COLORS["text_primary"]}"' + else: + # For other colors, use our palette + return f'fill="{COLORS["text_secondary"]}"' + +def update_stroke_color(match): + """Update stroke colors to use our palette.""" + color = match.group(0) + # Map to our color palette for consistency + if 'blue' in color.lower() or '4a90e2' in color.lower() or '63b3ed' in color.lower(): + return f'stroke="{COLORS["primary_blue"]}"' + elif 'green' in color.lower() or '48bb78' in color.lower() or '68d391' in color.lower(): + return f'stroke="{COLORS["primary_green"]}"' + elif 'purple' in color.lower() or 'b794f4' in color.lower() or '9f7aea' in color.lower(): + return f'stroke="{COLORS["primary_purple"]}"' + elif 'orange' in color.lower() or 'f6ad55' in color.lower() or 'ed8936' in color.lower(): + return f'stroke="{COLORS["primary_orange"]}"' + elif 'red' in color.lower() or 'e53e3e' in color.lower() or 'fc8181' in color.lower(): + return f'stroke="{COLORS["primary_red"]}"' + else: + return f'stroke="{COLORS["border_primary"]}"' + +def improve_rect(match): + """Improve rectangle elements with better styling.""" + rect = match.group(0) + # Add rounded corners if not present + if 'rx=' not in rect: + rect = rect[:-1] + f' rx="{LAYOUT["corner_radius"]}">' + # Add subtle shadow for depth + if 'filter=' not in rect: + rect = rect[:-1] + ' filter="url(#shadow)">' + # Ensure proper stroke width + rect = re.sub(r'stroke-width="[^"]*"', 'stroke-width="2"', rect) + return rect + +def improve_text(match): + """Improve text elements with better styling.""" + text_tag = match.group(1) + text_content = match.group(2) + + # Add text shadow for better readability + if 'filter=' not in text_tag: + text_tag += ' filter="url(#textShadow)"' + + # Ensure text has proper weight for readability + if 'font-weight=' not in text_tag and any(word in text_content.lower() for word in ['title', 'process', 'flow', 'system']): + text_tag += ' font-weight="600"' + + return f'{text_content}' + +def process_all_svgs(): + """Process all SVG files in the docs directory.""" + docs_dir = Path('docs') + + # Find all SVG files + svg_files = list(docs_dir.glob('**/*.svg')) + + print(f"Found {len(svg_files)} SVG files to beautify") + + for svg_file in svg_files: + # Skip font files + if 'fontawesome' in str(svg_file).lower() or 'favicon' in str(svg_file).lower(): + print(f"Skipping font/favicon file: {svg_file}") + continue + + print(f"Beautifying: {svg_file}") + + try: + # Read the original content + with open(svg_file, 'r', encoding='utf-8') as f: + content = f.read() + + # Skip if already processed + if 'Beautiful gradient definitions' in content: + print(f" Already beautified, skipping...") + continue + + # Create improved version + improved = create_improved_svg(content, svg_file.name) + + # Save the improved version + with open(svg_file, 'w', encoding='utf-8') as f: + f.write(improved) + + print(f" ✓ Successfully beautified!") + + except Exception as e: + print(f" ✗ Error processing {svg_file}: {e}") + +if __name__ == "__main__": + print("=" * 60) + print("SVG BEAUTIFIER - Making diagrams beautiful for all devices") + print("=" * 60) + print("\nFeatures:") + print("• Consistent text sizing matching documentation") + print("• Proper margins and padding for mobile") + print("• Beautiful colors that work in black & white") + print("• Responsive design for all screen sizes") + print("• Enhanced readability with shadows and gradients") + print("\nStarting beautification process...\n") + + process_all_svgs() + + print("\n" + "=" * 60) + print("✨ Beautification complete!") + print("All SVGs now have:") + print("• Mobile-friendly text sizes (min 12px)") + print("• Consistent font family") + print("• Proper margins (40px) and padding (20px)") + print("• High contrast colors readable in B&W") + print("• Responsive viewBox settings") + print("=" * 60) diff --git a/docs/src/appendix-i/assets/kb-access.svg b/docs/src/appendix-i/assets/kb-access.svg index 2b036773c..c8879893f 100644 --- a/docs/src/appendix-i/assets/kb-access.svg +++ b/docs/src/appendix-i/assets/kb-access.svg @@ -1,100 +1,96 @@ - - - - + - Knowledge Base Access + Knowledge Base Access - - - user_sessions - - - • id - • user_id - • bot_id - • session_token - - - - - Has Access + + user_sessions - - - user_kb_associations - - - • id - • session_id - • kb_id - • created_at - - - - - References + + user_kb_associations - - - kb_collections - - - • id - • bot_id - • name - • created_at - • metadata - - - - - Contains - 1:N + + kb_collections - - - kb_documents + + kb_documents - - • id - • collection_id - • content - • embeddings - • metadata + + + Vector Storage + + + + + id, user_id + bot_id, session_token + + + id, session_id + kb_id, created_at + + + id, bot_id, name + created_at, metadata + + + id, collection_id + content, embeddings + + + Embeddings stored + for semantic search - - - - Vector Storage - Embeddings stored - for semantic search + + + + + + + + + + + + + - - + + Has Access + Contains + 1:N + Indexes + References - - - - Access Pattern - - - 1. Session requests KB - 2. Association created - 3. KB documents available - 4. Vector search enabled - + + + + Access Pattern: + 1. Session requests KB access + 2. Association created in user_kb_associations + 3. KB documents become available + 4. Vector search enabled on embeddings + + + + Sessions gain access to knowledge bases through associations, enabling vector search on document embeddings + + + Each collection contains multiple documents with pre-computed embeddings for semantic search + + + Access is controlled per session, allowing dynamic knowledge base selection during conversations + diff --git a/docs/src/appendix-i/assets/kb-access.svg.backup b/docs/src/appendix-i/assets/kb-access.svg.backup new file mode 100644 index 000000000..6b988a0dd --- /dev/null +++ b/docs/src/appendix-i/assets/kb-access.svg.backup @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Knowledge Base Access + + + + + user_sessions + + + • id + • user_id + • bot_id + • session_token + + + + + Has Access + + + + + user_kb_associations + + + • id + • session_id + • kb_id + • created_at + + + + + References + + + + + kb_collections + + + • id + • bot_id + • name + • created_at + • metadata + + + + + Contains + 1:N + + + + + kb_documents + + + • id + • collection_id + • content + • embeddings + • metadata + + + + + + Vector Storage + Embeddings stored + for semantic search + + + + + + + + + Access Pattern + + + 1. Session requests KB + 2. Association created + 3. KB documents available + 4. Vector search enabled + + + + + + + \ No newline at end of file diff --git a/docs/src/appendix-i/assets/schema-detailed.svg b/docs/src/appendix-i/assets/schema-detailed.svg index 2201bad04..06d41d43d 100644 --- a/docs/src/appendix-i/assets/schema-detailed.svg +++ b/docs/src/appendix-i/assets/schema-detailed.svg @@ -1,186 +1,128 @@ - + - + - Database Entity Details + Database Entity Details - - - - - organizations + + + organizations - - id: UUID (PK) - name: VARCHAR(255) - created_at: TIMESTAMPTZ - updated_at: TIMESTAMPTZ - - + + + bots - - - - - bots + + + users - - id: UUID (PK) - organization_id: UUID (FK) - name: VARCHAR(255) - configuration: JSONB - created_at: TIMESTAMPTZ - updated_at: TIMESTAMPTZ - - + + + user_sessions - - - - - bot_memories + + + bot_memories - - id: UUID (PK) - bot_id: UUID (FK) - key: TEXT - value: TEXT - - + + + message_history - - - - - user_sessions + + + kb_collections - - id: UUID (PK) - user_id: UUID (FK) - bot_id: UUID (FK) - session_token: TEXT - created_at: TIMESTAMPTZ - expires_at: TIMESTAMPTZ - - + + + kb_documents - - - - - kb_collections + + + + id, name + created_at - - id: TEXT (PK) - bot_id: UUID (FK) - name: TEXT - description: TEXT - created_at: TIMESTAMPTZ - - + + id, org_id + name, config - - - - - users + + id, username + email, active - - id: UUID (PK) - username: TEXT - email: TEXT - password_hash: TEXT - active: BOOLEAN - created_at: TIMESTAMPTZ - - + + id, user_id, bot_id + token, expires_at - - - - - message_history + + id, bot_id + key, value - - id: UUID (PK) - session_id: UUID (FK) - user_id: UUID (FK) - bot_id: UUID (FK) - message: TEXT - sender: TEXT - - + + id, session_id + message, sender - - - - - kb_documents + + id, bot_id + name, description - - id: UUID (PK) - collection_id: TEXT (FK) - content: TEXT - embeddings: VECTOR - metadata: JSONB - created_at: TIMESTAMPTZ - + + id, collection_id + content, embeddings - + - - 1:N + - - - 1:N + + - + - - - 1:N - - - - 1:N + + - - 1:N + - - - 1:N + + + + + - - - Color Coding: + + 1:N + 1:N + 1:N + 1:N + 1:N + 1:N - - Organization + + + + Key Relationships: - - Bot Core - - - Sessions - - - Knowledge - - - Users - - - Messages + • Organizations own multiple Bots + • Users create multiple Sessions with Bots + • Sessions generate Message History + • Bots maintain Memories (key-value pairs) + • Bots have Knowledge Collections containing Documents + • Documents store embeddings for vector search + • All entities use UUIDs as primary keys (except KB collections using TEXT) + + + + Core database schema supporting multi-tenant bot operations with session management and vector search capabilities + diff --git a/docs/src/appendix-i/assets/schema-detailed.svg.backup b/docs/src/appendix-i/assets/schema-detailed.svg.backup new file mode 100644 index 000000000..c3f7b110d --- /dev/null +++ b/docs/src/appendix-i/assets/schema-detailed.svg.backup @@ -0,0 +1,217 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Database Entity Details + + + + + + organizations + + + id: UUID (PK) + name: VARCHAR(255) + created_at: TIMESTAMPTZ + updated_at: TIMESTAMPTZ + + + + + + + + bots + + + id: UUID (PK) + organization_id: UUID (FK) + name: VARCHAR(255) + configuration: JSONB + created_at: TIMESTAMPTZ + updated_at: TIMESTAMPTZ + + + + + + + + bot_memories + + + id: UUID (PK) + bot_id: UUID (FK) + key: TEXT + value: TEXT + + + + + + + + user_sessions + + + id: UUID (PK) + user_id: UUID (FK) + bot_id: UUID (FK) + session_token: TEXT + created_at: TIMESTAMPTZ + expires_at: TIMESTAMPTZ + + + + + + + + kb_collections + + + id: TEXT (PK) + bot_id: UUID (FK) + name: TEXT + description: TEXT + created_at: TIMESTAMPTZ + + + + + + + + users + + + id: UUID (PK) + username: TEXT + email: TEXT + password_hash: TEXT + active: BOOLEAN + created_at: TIMESTAMPTZ + + + + + + + + message_history + + + id: UUID (PK) + session_id: UUID (FK) + user_id: UUID (FK) + bot_id: UUID (FK) + message: TEXT + sender: TEXT + + + + + + + + kb_documents + + + id: UUID (PK) + collection_id: TEXT (FK) + content: TEXT + embeddings: VECTOR + metadata: JSONB + created_at: TIMESTAMPTZ + + + + + + + + 1:N + + + + 1:N + + + + + + + 1:N + + + + 1:N + + + + 1:N + + + + 1:N + + + + + Color Coding: + + + Organization + + + Bot Core + + + Sessions + + + Knowledge + + + Users + + + Messages + + + + + + \ No newline at end of file diff --git a/docs/src/appendix-i/assets/schema-overview.svg b/docs/src/appendix-i/assets/schema-overview.svg index 5cb678328..9792b851f 100644 --- a/docs/src/appendix-i/assets/schema-overview.svg +++ b/docs/src/appendix-i/assets/schema-overview.svg @@ -1,139 +1,131 @@ - + - + - Database Schema Overview + Database Schema Overview + - - organizations - Top Level Entity + + organizations - - bots - Bot Instances + + bots - - bot_configuration - Settings - - - - bot_memories - State Storage - - - - kb_collections - Knowledge Base - - - - kb_documents - Documents + + bot_configuration - - users - User Accounts + + users + + + + + bot_memories + + + + kb_collections - - user_sessions - Active Sessions - - - - message_history - Chat Messages - - - - user_login_tokens - Auth Tokens - - - - user_kb_associations - KB Access - - - - session_tool_associations - Tool Access + + user_sessions - - basic_tools - BASIC Scripts + + basic_tools - - + + + message_history + + + + + kb_documents + + + + user_login_tokens + + + + user_kb_associations + + + + session_tool_associations + + + - - 1:N + - - 1:N + - - 1:N + - - 1:N - - - - 1:N + - - 1:N + - - 1:N + + + + - - 1:N + - - 1:N + - - 1:N + - - - ref + + - - - ref - - - + + - - - Legend: - - 1:N Relationship + + 1:N + 1:N + 1:N + 1:N + 1:N + 1:N + 1:N + 1:N + 1:N + ref + ref - - Reference + + + + Entity Categories: + Core: organizations, bots, users • State: bot_memories, user_sessions, message_history + Knowledge: kb_collections, kb_documents • Tools: basic_tools • Auth: user_login_tokens + Associations: user_kb_associations, session_tool_associations, bot_configuration + + + + Multi-tenant database schema supporting organizations with multiple bots, user sessions, knowledge bases, and tools + diff --git a/docs/src/appendix-i/assets/schema-overview.svg.backup b/docs/src/appendix-i/assets/schema-overview.svg.backup new file mode 100644 index 000000000..70f13eb75 --- /dev/null +++ b/docs/src/appendix-i/assets/schema-overview.svg.backup @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Database Schema Overview + + + + organizations + Top Level Entity + + + + bots + Bot Instances + + + + bot_configuration + Settings + + + + bot_memories + State Storage + + + + kb_collections + Knowledge Base + + + + kb_documents + Documents + + + + users + User Accounts + + + + user_sessions + Active Sessions + + + + message_history + Chat Messages + + + + user_login_tokens + Auth Tokens + + + + user_kb_associations + KB Access + + + + session_tool_associations + Tool Access + + + + basic_tools + BASIC Scripts + + + + + + 1:N + + + + 1:N + + + + 1:N + + + + 1:N + + + + 1:N + + + + 1:N + + + + 1:N + + + + 1:N + + + + 1:N + + + + 1:N + + + + ref + + + + ref + + + + + + + + Legend: + + 1:N Relationship + + + Reference + + + + + + \ No newline at end of file diff --git a/docs/src/appendix-i/assets/session-flow.svg b/docs/src/appendix-i/assets/session-flow.svg index c138d11b9..659db6a21 100644 --- a/docs/src/appendix-i/assets/session-flow.svg +++ b/docs/src/appendix-i/assets/session-flow.svg @@ -1,95 +1,126 @@ - + - + - Session Flow Diagram + Session Flow Diagram - - User Login - Authentication - Entry Point - - - - Creates + + User Login - - - users - - - • id - • email - • zitadel_id - • created_at - • updated_at - - - - - Creates + + users - - - user_sessions - - - • id - • user_id - • bot_id - • token - • created_at - • expires_at - - - - - Generates - Messages + + user_sessions - - - message_history + + message_history - - • id - • session_id - • role - • content - • timestamp - • metadata + + + Token + + + + user_login_tokens + + + user_kb_associations + + + session_tool_associations + + + + + id, email + zitadel_id + + + id, user_id, bot_id + token, expires_at + + + id, session_id + role, content + + + JWT/UUID + Auth + + + id, user_id, token + expires_at + + + id, session_id + kb_id, created_at + + + id, session_id + tool_name, added_at - - - - 1 + + + + + + + - - 2 + + - - 3 - - - 4 + + + - - - - Process Flow + + Creates + Establishes + Logs + Returns + Stores + Links KB + Links Tools - - 1. User authenticates → 2. User record created/updated → 3. Session established → 4. Messages logged - + + + + 1 + + + 2 + + + 3 + + + 4 + + + 5 + + + + + Process Flow: + 1. User authenticates → 2. User record created/updated → 3. Session established → 4. Messages logged → 5. Token returned + + + + + Authentication flow from login through session establishment and message tracking + diff --git a/docs/src/appendix-i/assets/session-flow.svg.backup b/docs/src/appendix-i/assets/session-flow.svg.backup new file mode 100644 index 000000000..4e3126573 --- /dev/null +++ b/docs/src/appendix-i/assets/session-flow.svg.backup @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Session Flow Diagram + + + + User Login + Authentication + Entry Point + + + + Creates + + + + + users + + + • id + • email + • zitadel_id + • created_at + • updated_at + + + + + Creates + + + + + user_sessions + + + • id + • user_id + • bot_id + • token + • created_at + • expires_at + + + + + Generates + Messages + + + + + message_history + + + • id + • session_id + • role + • content + • timestamp + • metadata + + + + + + 1 + + + 2 + + + 3 + + + 4 + + + + + + Process Flow + + + 1. User authenticates → 2. User record created/updated → 3. Session established → 4. Messages logged + + + + + + + \ No newline at end of file diff --git a/docs/src/assets/architecture.svg b/docs/src/assets/architecture.svg index 6ef3e8dd7..bd8b2564d 100644 --- a/docs/src/assets/architecture.svg +++ b/docs/src/assets/architecture.svg @@ -1,79 +1,96 @@ - - - + + + + + + - - - BotServer (Single Binary) + + General Bots Architecture - - - - Web Server - (Axum) + + + Web Server - - - BASIC - Interpreter - (Rhai) + + + BASIC Interpreter - - - LLM - Integration - + + + LLM Integration - - - - - - + + + Package Manager - - - Session Manager (Tokio) + + + Console UI - - - - - - + + + Session Manager (Tokio Async Runtime) - - - - PostgreSQL - Database + + + PostgreSQL - - - Valkey - Cache + + + Valkey Cache - - - Qdrant - Vectors - + + + Qdrant Vectors - - Object Storage (SeaweedFS/S3) + + Object Storage - - - - Documents + + + Channels - - Templates + + + External API - - Assets + + + + + + + + + + + + + + + + + + + + + + + Storage Contents: + .gbkb (Documents) + .gbdialog (Scripts) + .gbot (Configs) + Templates + User Assets + + + + + Single binary with everything included - no external dependencies + diff --git a/docs/src/assets/architecture.svg.backup b/docs/src/assets/architecture.svg.backup new file mode 100644 index 000000000..6ef3e8dd7 --- /dev/null +++ b/docs/src/assets/architecture.svg.backup @@ -0,0 +1,79 @@ + + + + + + + BotServer (Single Binary) + + + + + + Web Server + (Axum) + + + + BASIC + Interpreter + (Rhai) + + + + LLM + Integration + + + + + + + + + + + + Session Manager (Tokio) + + + + + + + + + + + + + PostgreSQL + Database + + + + Valkey + Cache + + + + Qdrant + Vectors + + + + + Object Storage (SeaweedFS/S3) + + + + + Documents + + + Templates + + + Assets + + diff --git a/docs/src/assets/conversation-flow.svg b/docs/src/assets/conversation-flow.svg index 676e74e47..19aa20366 100644 --- a/docs/src/assets/conversation-flow.svg +++ b/docs/src/assets/conversation-flow.svg @@ -1,59 +1,73 @@ - + - - + + The Flow + + + - - User Input + + User Input - - BASIC Script + + BASIC Script - - LLM Decision + + LLM Decision - - - Tool Execution - - - - Save to CSV - - - - Search Knowledge + + + Bot Executor - - Bot Response + + Bot Response - - - - - - + + + + + Search Knowledge - - - - - - - - - - - - + + + Call API + + + + + + + + + + + + + + + + + + + + + + + + + The AI handles everything else - understanding intent, collecting information, executing tools, + + + answering from documents. Zero configuration. + diff --git a/docs/src/assets/conversation-flow.svg.backup b/docs/src/assets/conversation-flow.svg.backup new file mode 100644 index 000000000..676e74e47 --- /dev/null +++ b/docs/src/assets/conversation-flow.svg.backup @@ -0,0 +1,59 @@ + + + + + + + + + + + + User Input + + + + BASIC Script + + + + LLM Decision + + + + Tool Execution + + + + Save to CSV + + + + Search Knowledge + + + + Bot Response + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/assets/package-system-flow.svg b/docs/src/assets/package-system-flow.svg index ef24318f3..4541da085 100644 --- a/docs/src/assets/package-system-flow.svg +++ b/docs/src/assets/package-system-flow.svg @@ -1,95 +1,93 @@ - + - - - - + - Package System Flow + Package System Flow + - - User Request + + User Request + + + + start.bas + + + + LLM Engine - - Bot Response + + Bot Response - - - - start.bas + + + + Vector Search - - USE KB "docs" - answer=HEAR - result=LLM() - TALK result + + + .gbkb docs - - - - LLM Engine - (GPT/Local) - Processes requests - Makes decisions - - - - - Vector Search - (.gbkb docs) - Semantic search - - - - - - - - - - - - - - - - - - - - - - - - + + + + BASIC Commands + USE KB "docs" + answer = HEAR + result = LLM() + TALK result - - Commands - Results - - Query - Context - - - - - Flow Legend - - - Direct flow - - - Feedback/Return - - - - Bidirectional + + + + Package Structure + my-bot.gbai/ + ├─ .gbdialog/ + ├─ .gbkb/ + └─ .gbot/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Commands + Results + Query + Context + + + + BASIC scripts orchestrate LLM decisions, vector search, and responses with zero configuration + diff --git a/docs/src/assets/package-system-flow.svg.backup b/docs/src/assets/package-system-flow.svg.backup new file mode 100644 index 000000000..ef24318f3 --- /dev/null +++ b/docs/src/assets/package-system-flow.svg.backup @@ -0,0 +1,95 @@ + + + + + + + + + + + + Package System Flow + + + + User Request + + + + Bot Response + + + + + start.bas + + + USE KB "docs" + answer=HEAR + result=LLM() + TALK result + + + + + LLM Engine + (GPT/Local) + Processes requests + Makes decisions + + + + + Vector Search + (.gbkb docs) + Semantic search + + + + + + + + + + + + + + + + + + + + + + + + + + + + Commands + Results + + Query + Context + + + + + Flow Legend + + + Direct flow + + + Feedback/Return + + + + Bidirectional + + diff --git a/docs/src/chapter-01/assets/bootstrap-process.svg b/docs/src/chapter-01/assets/bootstrap-process.svg index b46eeee02..55bc897e4 100644 --- a/docs/src/chapter-01/assets/bootstrap-process.svg +++ b/docs/src/chapter-01/assets/bootstrap-process.svg @@ -1,98 +1,98 @@ - + - + - Auto-Bootstrap Process + Auto-Bootstrap Process - ./botserver + ./botserver - Detect System - (Linux/Mac) + Detect System + (Linux/Mac) - Auto-Bootstrap Process + Auto-Bootstrap Process - PostgreSQL - 16.2 - ✓ Install - ✓ Configure - ✓ Migrate + PostgreSQL + 16.2 + ✓ Install + ✓ Configure + ✓ Migrate - Valkey - Cache - ✓ Install - ✓ Configure - ✓ Start + Valkey + Cache + ✓ Install + ✓ Configure + ✓ Start - SeaweedFS - Storage - ✓ Install - ✓ Configure - ✓ Start + SeaweedFS + Storage + ✓ Install + ✓ Configure + ✓ Start - Qdrant - Vectors - ✓ Install - ✓ Configure - ✓ Start + Qdrant + Vectors + ✓ Install + ✓ Configure + ✓ Start - LLM Models (Optional) - ✓ Download BGE embeddings - ✓ Download Llama model (if local) + LLM Models (Optional) + ✓ Download BGE embeddings + ✓ Download Llama model (if local) - Generate .env file - • Secure passwords - • Connection URLs - • Port assignments + Generate .env file + • Secure passwords + • Connection URLs + • Port assignments - Load Templates - • Scan .gbai dirs - • Create bots - • Index documents + Load Templates + • Scan .gbai dirs + • Create bots + • Index documents - Start Web Server - localhost:8080 + Start Web Server + localhost:8080 diff --git a/docs/src/chapter-01/assets/bootstrap-process.svg.backup b/docs/src/chapter-01/assets/bootstrap-process.svg.backup new file mode 100644 index 000000000..c6d760d8e --- /dev/null +++ b/docs/src/chapter-01/assets/bootstrap-process.svg.backup @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Auto-Bootstrap Process + + + + ./botserver + + + + + + + Detect System + (Linux/Mac) + + + + + + + Auto-Bootstrap Process + + + + + + PostgreSQL + 16.2 + ✓ Install + ✓ Configure + ✓ Migrate + + + + Valkey + Cache + ✓ Install + ✓ Configure + ✓ Start + + + + SeaweedFS + Storage + ✓ Install + ✓ Configure + ✓ Start + + + + Qdrant + Vectors + ✓ Install + ✓ Configure + ✓ Start + + + + + LLM Models (Optional) + ✓ Download BGE embeddings + ✓ Download Llama model (if local) + + + + + + + Generate .env file + • Secure passwords + • Connection URLs + • Port assignments + + + + + + + Load Templates + • Scan .gbai dirs + • Create bots + • Index documents + + + + + + + Start Web Server + localhost:8080 + + + + + \ No newline at end of file diff --git a/docs/src/chapter-01/assets/quick-start-bootstrap-wide.svg b/docs/src/chapter-01/assets/quick-start-bootstrap-wide.svg new file mode 100644 index 000000000..4da3ee46e --- /dev/null +++ b/docs/src/chapter-01/assets/quick-start-bootstrap-wide.svg @@ -0,0 +1,188 @@ + + + + + + + + + + + + + + Bootstrap Flow + + + + + + + ./botserver + + + + + + + + + OS Detection + + + + + + + + + Component Installation + + + + PostgreSQL + + + Valkey + + + SeaweedFS + + + Qdrant + + + + + + + + + + + + + + + + + Configuration & Setup + + + + + + + + + Bot Deployment + + + + Scan templates/ directory + + + + + + + Load .gbai packages + + + + + + + Start Web Server + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Start + Detect + Install & Configure + Deploy + + + + + Automatic bootstrap process: detect OS, install components, configure, and deploy + + + Zero configuration required - everything runs from a single binary + + + + + + + + + + diff --git a/docs/src/chapter-01/assets/quick-start-bootstrap.svg b/docs/src/chapter-01/assets/quick-start-bootstrap.svg index 3484074b9..4da3ee46e 100644 --- a/docs/src/chapter-01/assets/quick-start-bootstrap.svg +++ b/docs/src/chapter-01/assets/quick-start-bootstrap.svg @@ -1,111 +1,188 @@ - + + + - - + + + + + - - Bootstrap Flow + + Bootstrap Flow - - - ./botserver - (first run) + - - - - - - OS Detection - Linux/Mac/Windows - - - - - - - Component Installation - - - - - - PostgreSQL - 16.2 - - - - Valkey - Cache - - - - SeaweedFS - Storage - - - - Qdrant - Vectors + + + + ./botserver - - - - - - - - + + + + + + + OS Detection - - + + - - - Configuration & Setup + + + + Component Installation - - • Generate credentials - • Create .env file - • Initialize databases - • Create storage buckets + + + PostgreSQL + + + Valkey + + + SeaweedFS + + + Qdrant + + + + + + + + - - + + - - - Bot Deployment + + + + Configuration & Setup + - - - Scan templates/ directory + + - - + + + + Bot Deployment - - Load .gbai packages - • default.gbai - • announcements.gbai + + + Scan templates/ directory - - + + - - Upload to object storage - Index documents • Register in database + + + Load .gbai packages - - + + - - - Start Web Server - http://localhost:8080 + + + Start Web Server + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Start + Detect + Install & Configure + Deploy + + + + + Automatic bootstrap process: detect OS, install components, configure, and deploy + + + Zero configuration required - everything runs from a single binary + + + + + + + + + diff --git a/docs/src/chapter-01/assets/quick-start-bootstrap.svg.backup b/docs/src/chapter-01/assets/quick-start-bootstrap.svg.backup new file mode 100644 index 000000000..b01ddcad1 --- /dev/null +++ b/docs/src/chapter-01/assets/quick-start-bootstrap.svg.backup @@ -0,0 +1,142 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Bootstrap Flow + + + + ./botserver + (first run) + + + + + + + OS Detection + Linux/Mac/Windows + + + + + + + Component Installation + + + + + + PostgreSQL + 16.2 + + + + Valkey + Cache + + + + SeaweedFS + Storage + + + + Qdrant + Vectors + + + + + + + + + + + + + + + + + + Configuration & Setup + + + • Generate credentials + • Create .env file + • Initialize databases + • Create storage buckets + + + + + + + + Bot Deployment + + + + Scan templates/ directory + + + + + + Load .gbai packages + • default.gbai + • announcements.gbai + + + + + + Upload to object storage + Index documents • Register in database + + + + + + + Start Web Server + http://localhost:8080 + + + + + \ No newline at end of file diff --git a/docs/src/chapter-01/assets/session-manager.svg b/docs/src/chapter-01/assets/session-manager.svg index c1e18c3f0..26e6bd3a4 100644 --- a/docs/src/chapter-01/assets/session-manager.svg +++ b/docs/src/chapter-01/assets/session-manager.svg @@ -1,7 +1,7 @@ - + - + @@ -10,15 +10,15 @@ - Session Manager Architecture + Session Manager Architecture - User Input + User Input - Bot Response + Bot Response @@ -26,12 +26,12 @@ - WebSocket - /HTTP + WebSocket + /HTTP - WebSocket - /HTTP + WebSocket + /HTTP @@ -39,10 +39,10 @@ - Session Manager + Session Manager - + 1. Validate Token 2. Load Session 3. Update State @@ -58,20 +58,20 @@ - Valkey - (Cache) + Valkey + (Cache) - PostgreSQL - (Persist) + PostgreSQL + (Persist) - Sync Every - Message + Sync Every + Message - + 1 @@ -91,9 +91,9 @@ - Key Features + Key Features - + • Real-time WebSocket support • Automatic session persistence • Redis-compatible caching diff --git a/docs/src/chapter-01/assets/session-manager.svg.backup b/docs/src/chapter-01/assets/session-manager.svg.backup new file mode 100644 index 000000000..f4eb2ff36 --- /dev/null +++ b/docs/src/chapter-01/assets/session-manager.svg.backup @@ -0,0 +1,130 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Session Manager Architecture + + + + User Input + + + + Bot Response + + + + + + + + WebSocket + /HTTP + + + WebSocket + /HTTP + + + + + + + + Session Manager + + + + 1. Validate Token + 2. Load Session + 3. Update State + + 4. Execute BASIC + 5. Generate Response + 6. Save History + + + + + + + + + Valkey + (Cache) + + + PostgreSQL + (Persist) + + + + Sync Every + Message + + + + + 1 + + + 2 + + + 3 + + + 4 + + + 5 + + + + + + Key Features + + + • Real-time WebSocket support + • Automatic session persistence + • Redis-compatible caching + • ACID compliance + + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-01/assets/session-states.svg b/docs/src/chapter-01/assets/session-states.svg index eeb6103ac..96fdece89 100644 --- a/docs/src/chapter-01/assets/session-states.svg +++ b/docs/src/chapter-01/assets/session-states.svg @@ -1,75 +1,75 @@ - + - + - Session State Flow + Session State Flow - Browser - Opens + Browser + Opens - CREATE - New UUID - Token Gen + CREATE + New UUID + Token Gen - ACTIVE - Chatting - Messages + ACTIVE + Chatting + Messages - IDLE - No Input - 30min Timer + IDLE + No Input + 30min Timer - User Returns + User Returns - EXPIRE - 7d Anon - Never Auth + EXPIRE + 7d Anon + Never Auth - Retry + Retry - CLEANUP - Archive - Delete + CLEANUP + Archive + Delete - + Instant Active use 30 minutes diff --git a/docs/src/chapter-01/assets/session-states.svg.backup b/docs/src/chapter-01/assets/session-states.svg.backup new file mode 100644 index 000000000..0e315e118 --- /dev/null +++ b/docs/src/chapter-01/assets/session-states.svg.backup @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Session State Flow + + + + Browser + Opens + + + + + + + CREATE + New UUID + Token Gen + + + + + + + ACTIVE + Chatting + Messages + + + + + + + IDLE + No Input + 30min Timer + + + + User Returns + + + + + + + EXPIRE + 7d Anon + Never Auth + + + + Retry + + + + + + + CLEANUP + Archive + Delete + + + + Instant + Active use + 30 minutes + 7 days / Never + Permanent + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-01/assets/tool-execution-flow.svg b/docs/src/chapter-01/assets/tool-execution-flow.svg index 0c185dd41..dd4b4224c 100644 --- a/docs/src/chapter-01/assets/tool-execution-flow.svg +++ b/docs/src/chapter-01/assets/tool-execution-flow.svg @@ -1,37 +1,37 @@ - + - + - Tool Execution Flow + Tool Execution Flow - User: "I want to enroll in - Computer Science" + User: "I want to enroll in + Computer Science" - LLM Analyzes - "enrollment need" - Intent detection + LLM Analyzes + "enrollment need" + Intent detection - Scan Available Tools + Scan Available Tools - - • enrollment.bas ✓ - • other-tools.bas + + • enrollment.bas ✓ + • other-tools.bas @@ -39,9 +39,9 @@ - Collect Parameters + Collect Parameters - + • name: (ask user) • email: (ask user) • course: "Comp Sci" @@ -52,9 +52,9 @@ - Execute enrollment.bas + Execute enrollment.bas - + • Save to CSV • Return confirmation @@ -64,11 +64,11 @@ - "Welcome to Computer Science!" - Confirmation sent to user + "Welcome to Computer Science!" + Confirmation sent to user - + ~100ms ~50ms @@ -93,9 +93,9 @@ - enrollment.bas + enrollment.bas - + ' Student enrollment tool PARAM name, email, course SAVE "enrollments.csv", name, email, course, NOW() diff --git a/docs/src/chapter-01/assets/tool-execution-flow.svg.backup b/docs/src/chapter-01/assets/tool-execution-flow.svg.backup new file mode 100644 index 000000000..2380c7437 --- /dev/null +++ b/docs/src/chapter-01/assets/tool-execution-flow.svg.backup @@ -0,0 +1,150 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tool Execution Flow + How the bot understands and executes user commands + + + + 1. User Input + "I want to enroll in Computer Science" + + + + + + + 2. LLM Analyzes Intent + Detects: "enrollment request" + Natural language understanding + + + + ~100ms + + + + + + + 3. Scan Available Tools + + + + ✓ enrollment.bas + + + + + other-tools.bas + + + + + ~50ms + + + + + + + 4. Collect Parameters + + + + • Name: + (ask user) + • Email: + (ask user) + • Course: + "Computer Science" + + + + + Interactive + + + + + + + 5. Execute enrollment.bas + + + + + Save to enrollments.csv + + + + + + Generate confirmation + + + + + ~200ms + + + + + + + 6. Bot Response + "Welcome to Computer Science, John!" + + + + Instant + diff --git a/docs/src/chapter-02/assets/package-structure.svg b/docs/src/chapter-02/assets/package-structure.svg index 692a559ff..2405ec44b 100644 --- a/docs/src/chapter-02/assets/package-structure.svg +++ b/docs/src/chapter-02/assets/package-structure.svg @@ -1,17 +1,17 @@ - + - + - Package Structure + Package Structure - my-bot.gbai/ - (Package Root) + my-bot.gbai/ + (Package Root) @@ -31,33 +31,33 @@ - .gbdialog - Dialog Scripts - Conversation Logic + .gbdialog + Dialog Scripts + Conversation Logic - .gbkb - Knowledge Base - Documents + .gbkb + Knowledge Base + Documents - .gbot - Configuration - Bot Settings + .gbot + Configuration + Bot Settings - .gbtheme - (optional) - UI Theme + .gbtheme + (optional) + UI Theme - .gbdrive - (optional) - File Storage + .gbdrive + (optional) + File Storage @@ -73,35 +73,35 @@ - Scripts - .bas files + Scripts + .bas files - Docs - PDF/TXT + Docs + PDF/TXT - Config - .csv + Config + .csv - Styles - CSS/HTML + Styles + CSS/HTML - Storage - S3 Link + Storage + S3 Link - Example Directory Structure: + Example Directory Structure: - + botname.gbai/ ├── botname.gbdialog/ │ ├── start.bas @@ -120,9 +120,9 @@ - Key Points + Key Points - + • Folder name = Bot name • Only .gbdialog is required • start.bas is the entry point diff --git a/docs/src/chapter-02/assets/package-structure.svg.backup b/docs/src/chapter-02/assets/package-structure.svg.backup new file mode 100644 index 000000000..caddd6557 --- /dev/null +++ b/docs/src/chapter-02/assets/package-structure.svg.backup @@ -0,0 +1,163 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Package Structure + + + + my-bot.gbai/ + (Package Root) + + + + + + + + + + + + + + + + + + + + .gbdialog + Dialog Scripts + Conversation Logic + + + + .gbkb + Knowledge Base + Documents + + + + .gbot + Configuration + Bot Settings + + + + .gbtheme + (optional) + UI Theme + + + + .gbdrive + (optional) + File Storage + + + + + + + + + + + + + + + + Scripts + .bas files + + + + Docs + PDF/TXT + + + + Config + .csv + + + + Styles + CSS/HTML + + + + Storage + S3 Link + + + + + Example Directory Structure: + + + botname.gbai/ + ├── botname.gbdialog/ + │ ├── start.bas + │ ├── auth.bas + │ └── tools/ + ├── botname.gbkb/ + │ ├── collection1/ + │ └── collection2/ + ├── botname.gbot/ + │ └── config.csv + └── botname.gbtheme/ + └── default.css + + + + + + + Key Points + + + • Folder name = Bot name + • Only .gbdialog is required + • start.bas is the entry point + • Deploy by copying folder + + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-02/assets/template-deployment-flow.svg b/docs/src/chapter-02/assets/template-deployment-flow.svg index 15851b18a..a74a2f55b 100644 --- a/docs/src/chapter-02/assets/template-deployment-flow.svg +++ b/docs/src/chapter-02/assets/template-deployment-flow.svg @@ -1,83 +1,83 @@ - + - + - Template Deployment Flow + Template Deployment Flow - templates/ - Source folder + templates/ + Source folder - SCAN - Find all .gbai folders + SCAN + Find all .gbai folders - VALIDATE - Check required structure - • start.bas exists? - • Folders match name? + VALIDATE + Check required structure + • start.bas exists? + • Folders match name? - UPLOAD - Copy to object storage - • Templates → S3/Drive - • Assets → CDN paths + UPLOAD + Copy to object storage + • Templates → S3/Drive + • Assets → CDN paths - INDEX - Process knowledge base - • Extract text - • Generate embeddings - • Store in Qdrant + INDEX + Process knowledge base + • Extract text + • Generate embeddings + • Store in Qdrant - REGISTER - Create in database - • Bot record - • Configuration - • URL mapping + REGISTER + Create in database + • Bot record + • Configuration + • URL mapping - ACTIVATE - Start serving - • /bot-name endpoint - • WebSocket ready - • Sessions enabled + ACTIVATE + Start serving + • /bot-name endpoint + • WebSocket ready + • Sessions enabled - 5-10s - per bot + 5-10s + per bot diff --git a/docs/src/chapter-02/assets/template-deployment-flow.svg.backup b/docs/src/chapter-02/assets/template-deployment-flow.svg.backup new file mode 100644 index 000000000..c250b5ba8 --- /dev/null +++ b/docs/src/chapter-02/assets/template-deployment-flow.svg.backup @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Template Deployment Flow + + + + templates/ + Source folder + + + + + + + SCAN + Find all .gbai folders + + + + + + + VALIDATE + Check required structure + • start.bas exists? + • Folders match name? + + + + + + + UPLOAD + Copy to object storage + • Templates → S3/Drive + • Assets → CDN paths + + + + + + + INDEX + Process knowledge base + • Extract text + • Generate embeddings + • Store in Qdrant + + + + + + + REGISTER + Create in database + • Bot record + • Configuration + • URL mapping + + + + + + + ACTIVATE + Start serving + • /bot-name endpoint + • WebSocket ready + • Sessions enabled + + + + + 5-10s + per bot + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-03/assets/caching-architecture.svg b/docs/src/chapter-03/assets/caching-architecture.svg index bae1f67df..80db903dc 100644 --- a/docs/src/chapter-03/assets/caching-architecture.svg +++ b/docs/src/chapter-03/assets/caching-architecture.svg @@ -1,4 +1,4 @@ - + @@ -90,30 +90,30 @@ - + Semantic Caching Architecture - User Query - "What's the policy?" + User Query + "What's the policy?" - Generate Key - Hash + Embed + Generate Key + Hash + Embed - Check Valkey - Memory Store + Check Valkey + Memory Store @@ -121,7 +121,7 @@ - Hit? + Hit? @@ -129,45 +129,45 @@ - Embedding Hash - 384D Vector + Embedding Hash + 384D Vector - Semantic Search - Similarity > 0.95 + Semantic Search + Similarity > 0.95 - ✓ Cache Hit + ✓ Cache Hit - Miss + Miss - Generate New - LLM Response + Generate New + LLM Response - Store in Valkey - TTL: 3600s + Store in Valkey + TTL: 3600s - + Performance Metrics - Hit Rate: ~85% - Latency: <50ms - Cost Reduction: 95% + Hit Rate: ~85% + Latency: <50ms + Cost Reduction: 95% diff --git a/docs/src/chapter-03/assets/caching-architecture.svg.backup b/docs/src/chapter-03/assets/caching-architecture.svg.backup new file mode 100644 index 000000000..af19ae918 --- /dev/null +++ b/docs/src/chapter-03/assets/caching-architecture.svg.backup @@ -0,0 +1,125 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Semantic Caching Architecture + + + + + User Query + "What's the policy?" + + + + + + + Generate Key + Hash + Embed + + + + + + + Check Valkey + Memory Store + + + + + + + + Hit? + + + + + + + + Embedding Hash + 384D Vector + + + + + + + Semantic Search + Similarity > 0.95 + + + + + ✓ Cache Hit + + + + Miss + + + + Generate New + LLM Response + + + + + + + Store in Valkey + TTL: 3600s + + + + + Performance Metrics + + Hit Rate: ~85% + Latency: <50ms + Cost Reduction: 95% + + + + + \ No newline at end of file diff --git a/docs/src/chapter-03/assets/context-window.svg b/docs/src/chapter-03/assets/context-window.svg index 793bf204a..6060dbdad 100644 --- a/docs/src/chapter-03/assets/context-window.svg +++ b/docs/src/chapter-03/assets/context-window.svg @@ -1,4 +1,4 @@ - + @@ -42,164 +42,164 @@ - + Vector Database Storage Requirements: The Real Mathematics - + Original Documents - + 1 TB Total - + PDF: 400 GB - + DOCX: 250 GB - + XLSX: 150 GB - + TXT: 100 GB - + HTML: 50 GB - + Other: 50 GB - + Processing - + Vector DB Storage - + ~3.5 TB Required - + Raw Text Extracted - + ~800 GB (cleaned) - + Deduplication reduces 20% - + Vector Embeddings - + ~1.2 TB (384-dim floats) - + 4 bytes × 384 × ~800M chunks - + = 1,228 GB - + HNSW Index - + ~600 GB - + Graph structure + links - + Metadata + Positions - + ~400 GB - + Doc refs, chunks, offsets - + Cache + Auxiliary - + ~500 GB - + Query cache, temp indices - + Storage Multiplication Factor - + Original Documents: 1.0 TB - + Vector DB Total: 3.5 TB - + Multiplication Factor: 3.5× - + With redundancy/backup: - + Production Total: 7.0 TB (2× replica) - + Reality: You need 3.5-7× your document storage - Input + Input - Storage + Storage - Factor + Factor diff --git a/docs/src/chapter-03/assets/context-window.svg.backup b/docs/src/chapter-03/assets/context-window.svg.backup new file mode 100644 index 000000000..cdea24e11 --- /dev/null +++ b/docs/src/chapter-03/assets/context-window.svg.backup @@ -0,0 +1,204 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Vector Database Storage Requirements: The Real Mathematics + + + + + + Original Documents + + + 1 TB Total + + + + + + PDF: 400 GB + + + + + DOCX: 250 GB + + + + + XLSX: 150 GB + + + + + TXT: 100 GB + + + + + HTML: 50 GB + + + + + Other: 50 GB + + + + + + Processing + + + + + + Vector DB Storage + + + ~3.5 TB Required + + + + + + Raw Text Extracted + + + ~800 GB (cleaned) + + + Deduplication reduces 20% + + + + + Vector Embeddings + + + ~1.2 TB (384-dim floats) + + + 4 bytes × 384 × ~800M chunks + + + = 1,228 GB + + + + + HNSW Index + + + ~600 GB + + + Graph structure + links + + + + + Metadata + Positions + + + ~400 GB + + + Doc refs, chunks, offsets + + + + + Cache + Auxiliary + + + ~500 GB + + + Query cache, temp indices + + + + + + Storage Multiplication Factor + + + + Original Documents: 1.0 TB + + + Vector DB Total: 3.5 TB + + + Multiplication Factor: 3.5× + + + + With redundancy/backup: + + + Production Total: 7.0 TB (2× replica) + + + + Reality: You need 3.5-7× your document storage + + + + + + Input + + + + + Storage + + + + + Factor + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-03/assets/kb-architecture-pipeline.svg b/docs/src/chapter-03/assets/kb-architecture-pipeline.svg index f412dfa77..bd49456fb 100644 --- a/docs/src/chapter-03/assets/kb-architecture-pipeline.svg +++ b/docs/src/chapter-03/assets/kb-architecture-pipeline.svg @@ -1,4 +1,4 @@ - + @@ -54,16 +54,16 @@ - + Knowledge Base Architecture Pipeline - + Document Ingestion Layer - + PDF • Word • Excel • Text • HTML • Markdown @@ -72,10 +72,10 @@ - + Preprocessing Pipeline - + Extraction • Cleaning • Normalization • Validation @@ -84,10 +84,10 @@ - + Intelligent Chunking Engine - + Semantic Segmentation • Overlap Management • Metadata Preservation @@ -96,10 +96,10 @@ - + Embedding Generation - + BGE Models • Transformer Architecture • Dimensionality: 384/768 @@ -108,10 +108,10 @@ - + Vector Index Layer - + HNSW Algorithm • Quantization • Distributed Sharding @@ -120,31 +120,31 @@ - + Semantic Retrieval Engine - + Cosine Similarity • Hybrid Search • Re-ranking • Context Injection - + Data Flow Direction - Raw Docs - Clean Text - Chunks - Vectors - Index - Results + Raw Docs + Clean Text + Chunks + Vectors + Index + Results - + Pipeline processes ~1000 documents/minute • Query latency <50ms (p99) • 95% semantic accuracy diff --git a/docs/src/chapter-03/assets/kb-architecture-pipeline.svg.backup b/docs/src/chapter-03/assets/kb-architecture-pipeline.svg.backup new file mode 100644 index 000000000..eaac4a18a --- /dev/null +++ b/docs/src/chapter-03/assets/kb-architecture-pipeline.svg.backup @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Knowledge Base Architecture Pipeline + + + + + + Document Ingestion Layer + + + PDF • Word • Excel • Text • HTML • Markdown + + + + + + + + + Preprocessing Pipeline + + + Extraction • Cleaning • Normalization • Validation + + + + + + + + + Intelligent Chunking Engine + + + Semantic Segmentation • Overlap Management • Metadata Preservation + + + + + + + + + Embedding Generation + + + BGE Models • Transformer Architecture • Dimensionality: 384/768 + + + + + + + + + Vector Index Layer + + + HNSW Algorithm • Quantization • Distributed Sharding + + + + + + + + + Semantic Retrieval Engine + + + Cosine Similarity • Hybrid Search • Re-ranking • Context Injection + + + + + + Data Flow Direction + + + + + Raw Docs + Clean Text + Chunks + Vectors + Index + Results + + + + + Pipeline processes ~1000 documents/minute • Query latency <50ms (p99) • 95% semantic accuracy + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-03/assets/search-pipeline.svg b/docs/src/chapter-03/assets/search-pipeline.svg index bae1be696..85f380f2f 100644 --- a/docs/src/chapter-03/assets/search-pipeline.svg +++ b/docs/src/chapter-03/assets/search-pipeline.svg @@ -1,4 +1,4 @@ - + @@ -69,7 +69,7 @@ - + Semantic Search Pipeline @@ -79,9 +79,9 @@ - User Query - "What's the return - policy?" + User Query + "What's the return + policy?" @@ -90,9 +90,9 @@ - Embedding - Transform to - 384D vector + Embedding + Transform to + 384D vector @@ -101,9 +101,9 @@ - Vector Search - Cosine similarity - in vector space + Vector Search + Cosine similarity + in vector space @@ -112,16 +112,16 @@ - Re-ranking - Score & sort - by relevance + Re-ranking + Score & sort + by relevance - Active Collections - policies • procedures • faq + Active Collections + policies • procedures • faq @@ -130,10 +130,10 @@ - Retrieved Context - 1. "Refund policy: 30 days..." (0.92) - 2. "Return procedures..." (0.87) - 3. "Warranty information..." (0.81) + Retrieved Context + 1. "Refund policy: 30 days..." (0.92) + 2. "Return procedures..." (0.87) + 3. "Warranty information..." (0.81) @@ -142,8 +142,8 @@ - Context Injection to LLM - Retrieved chunks provided as context for response generation + Context Injection to LLM + Retrieved chunks provided as context for response generation @@ -153,13 +153,13 @@ - Automatic - Process + Automatic + Process - + Search latency: ~20ms • Embedding: BGE-small (384D) • Similarity threshold: 0.7 • Top-K: 5 chunks diff --git a/docs/src/chapter-03/assets/search-pipeline.svg.backup b/docs/src/chapter-03/assets/search-pipeline.svg.backup new file mode 100644 index 000000000..29440df13 --- /dev/null +++ b/docs/src/chapter-03/assets/search-pipeline.svg.backup @@ -0,0 +1,138 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Semantic Search Pipeline + + + + + + + + + User Query + "What's the return + policy?" + + + + + + + + + Embedding + Transform to + 384D vector + + + + + + + + + Vector Search + Cosine similarity + in vector space + + + + + + + + + Re-ranking + Score & sort + by relevance + + + + + + Active Collections + policies • procedures • faq + + + + + + + + + Retrieved Context + 1. "Refund policy: 30 days..." (0.92) + 2. "Return procedures..." (0.87) + 3. "Warranty information..." (0.81) + + + + + + + + + Context Injection to LLM + Retrieved chunks provided as context for response generation + + + + + + + + + + Automatic + Process + + + + + + Search latency: ~20ms • Embedding: BGE-small (384D) • Similarity threshold: 0.7 • Top-K: 5 chunks + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-03/assets/storage-breakdown.svg b/docs/src/chapter-03/assets/storage-breakdown.svg index a5d854f6f..6ab636f88 100644 --- a/docs/src/chapter-03/assets/storage-breakdown.svg +++ b/docs/src/chapter-03/assets/storage-breakdown.svg @@ -1,4 +1,4 @@ - + @@ -58,7 +58,7 @@ - + LLM Context Compression Strategies @@ -66,7 +66,7 @@ - + Original Context: 10,000 tokens @@ -84,14 +84,14 @@ - + Compression Level 4 - + Compressed Context: 4,096 tokens (fits LLM window) @@ -107,27 +107,27 @@ - + Compression Techniques (Level 4) - + Semantic Deduplication - + • Remove redundant info - + • Merge similar chunks - + • Keep unique facts - + Reduction: 30-40% @@ -142,20 +142,20 @@ - + Relevance Scoring - + • Score by query match - + • Keep top-k relevant - + • Drop low scores - + Reduction: 40-50% @@ -168,20 +168,20 @@ - + Hierarchical Summary - + • Extract key points - + • Create abstracts - + • Preserve details - + Reduction: 50-60% @@ -200,28 +200,28 @@ - + Token Optimization - + • Remove stopwords - + • Compress phrases - + • Use abbreviations - + Reduction: 20-30% - + ABCDEF → ABC - + GHIJKL → GHI @@ -229,7 +229,7 @@ - + Compression Level 4 achieves 60-75% reduction while maintaining 95%+ information retention diff --git a/docs/src/chapter-03/assets/storage-breakdown.svg.backup b/docs/src/chapter-03/assets/storage-breakdown.svg.backup new file mode 100644 index 000000000..ad86654fd --- /dev/null +++ b/docs/src/chapter-03/assets/storage-breakdown.svg.backup @@ -0,0 +1,218 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + LLM Context Compression Strategies + + + + + + + + Original Context: 10,000 tokens + + + + + + + + + + + + + + + + + + Compression Level 4 + + + + + + + Compressed Context: 4,096 tokens (fits LLM window) + + + + + + + + + + + + + + + + Compression Techniques (Level 4) + + + + + + + Semantic Deduplication + + + • Remove redundant info + + + • Merge similar chunks + + + • Keep unique facts + + + + Reduction: 30-40% + + + + + + + + + + + + + + + Relevance Scoring + + + • Score by query match + + + • Keep top-k relevant + + + • Drop low scores + + + + Reduction: 40-50% + + + + + + + + + + + + + Hierarchical Summary + + + • Extract key points + + + • Create abstracts + + + • Preserve details + + + + Reduction: 50-60% + + + + + + + + + + + + + + + + + + + Token Optimization + + + • Remove stopwords + + + • Compress phrases + + + • Use abbreviations + + + + Reduction: 20-30% + + + + + ABCDEF → ABC + + + GHIJKL → GHI + + + + + + + + Compression Level 4 achieves 60-75% reduction while maintaining 95%+ information retention + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-03/assets/storage-multiplication.svg b/docs/src/chapter-03/assets/storage-multiplication.svg index bb26aed6b..6622aba9c 100644 --- a/docs/src/chapter-03/assets/storage-multiplication.svg +++ b/docs/src/chapter-03/assets/storage-multiplication.svg @@ -1,4 +1,4 @@ - + @@ -50,7 +50,7 @@ - + Storage Components per 1TB of Documents @@ -67,61 +67,61 @@ - 0 GB - 250 GB - 500 GB - 750 GB - 1000 GB + 0 GB + 250 GB + 500 GB + 750 GB + 1000 GB - + Storage Size (GB) - 1000 - Original - 1000 GB + 1000 + Original + 1000 GB - 800 - Extracted - 800 GB + 800 + Extracted + 800 GB - 1200 - Embeddings - 1200 GB + 1200 + Embeddings + 1200 GB - 600 - Index - 600 GB + 600 + Index + 600 GB - 400 - Metadata - 400 GB + 400 + Metadata + 400 GB - 500 - Cache - 500 GB + 500 + Cache + 500 GB - 3.5 TB Total + 3.5 TB Total - Components contribute to 3.5× storage multiplication factor + Components contribute to 3.5× storage multiplication factor diff --git a/docs/src/chapter-03/assets/storage-multiplication.svg.backup b/docs/src/chapter-03/assets/storage-multiplication.svg.backup new file mode 100644 index 000000000..05d664c43 --- /dev/null +++ b/docs/src/chapter-03/assets/storage-multiplication.svg.backup @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Storage Components per 1TB of Documents + + + + + + + + + + + + + + + + 0 GB + 250 GB + 500 GB + 750 GB + 1000 GB + + + + Storage Size (GB) + + + + + + 1000 + Original + 1000 GB + + + + 800 + Extracted + 800 GB + + + + 1200 + Embeddings + 1200 GB + + + + 600 + Index + 600 GB + + + + 400 + Metadata + 400 GB + + + + 500 + Cache + 500 GB + + + + + 3.5 TB Total + + + + Components contribute to 3.5× storage multiplication factor + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-03/assets/technical-specs.svg b/docs/src/chapter-03/assets/technical-specs.svg index ba7a52286..18cdb10f7 100644 --- a/docs/src/chapter-03/assets/technical-specs.svg +++ b/docs/src/chapter-03/assets/technical-specs.svg @@ -1,4 +1,4 @@ - + @@ -37,7 +37,7 @@ - + System Technical Specifications @@ -47,23 +47,23 @@ - + Embedding Configuration - + Model: bge-small-en-v1.5-f32.gguf - + • Dimensions: 384 - + • Format: GGUF (quantized) - + • Server: localhost:8082 - + • Memory: ~200MB loaded @@ -71,23 +71,23 @@ - + LLM Configuration - + Model: DeepSeek-R1-Distill-Qwen-1.5B - + • Context Size: 4096 tokens - + • Max Predict: 1024 tokens - + • Parallel Requests: 6 - + • Quantization: Q3_K_M @@ -95,54 +95,54 @@ - + Performance Characteristics - + Vector Index: HNSW Algorithm - + • M=16, ef_construction=200 - + • Distance: Cosine Similarity - + • Build: ~1000 docs/minute - + Chunking Strategy - + • Chunk Size: 512 tokens - + • Overlap: 50 tokens - + • Prompt Compact: Level 4 - + Runtime Metrics - + • Query Latency: <50ms p99 - + • Memory: ~1GB/million chunks - + • Cache TTL: 3600 seconds diff --git a/docs/src/chapter-03/assets/technical-specs.svg.backup b/docs/src/chapter-03/assets/technical-specs.svg.backup new file mode 100644 index 000000000..37fc09a30 --- /dev/null +++ b/docs/src/chapter-03/assets/technical-specs.svg.backup @@ -0,0 +1,161 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + System Technical Specifications + + + + + + + + + + Embedding Configuration + + + + Model: bge-small-en-v1.5-f32.gguf + + + • Dimensions: 384 + + + • Format: GGUF (quantized) + + + • Server: localhost:8082 + + + • Memory: ~200MB loaded + + + + + + + + LLM Configuration + + + + Model: DeepSeek-R1-Distill-Qwen-1.5B + + + • Context Size: 4096 tokens + + + • Max Predict: 1024 tokens + + + • Parallel Requests: 6 + + + • Quantization: Q3_K_M + + + + + + + + Performance Characteristics + + + + + + Vector Index: HNSW Algorithm + + + • M=16, ef_construction=200 + + + • Distance: Cosine Similarity + + + • Build: ~1000 docs/minute + + + + + + + Chunking Strategy + + + • Chunk Size: 512 tokens + + + • Overlap: 50 tokens + + + • Prompt Compact: Level 4 + + + + + + + Runtime Metrics + + + • Query Latency: <50ms p99 + + + • Memory: ~1GB/million chunks + + + • Cache TTL: 3600 seconds + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-06-gbdialog/assets/basic-execution-flow.svg b/docs/src/chapter-06-gbdialog/assets/basic-execution-flow.svg index 6d05f51fe..ff3435cdf 100644 --- a/docs/src/chapter-06-gbdialog/assets/basic-execution-flow.svg +++ b/docs/src/chapter-06-gbdialog/assets/basic-execution-flow.svg @@ -1,4 +1,4 @@ - + @@ -85,22 +85,22 @@ - + BASIC LLM Tool Execution Flow - USER - "What's the policy?" + USER + "What's the policy?" - LLM + CONTEXT - Understands intent + loaded KBs + LLM + CONTEXT + Understands intent + loaded KBs @@ -109,7 +109,7 @@ - Tool? + Tool? @@ -119,40 +119,40 @@ - Direct Answer + Direct Answer - Call Tool + Call Tool - BASIC TOOL - enrollment.bas - PARAM name, course + BASIC TOOL + enrollment.bas + PARAM name, course - RESPONSE - Generate natural answer + RESPONSE + Generate natural answer - BOT - "30-day return..." + BOT + "30-day return..." - MEMORY STORE - BOT_MEMORY • Session State • Context + MEMORY STORE + BOT_MEMORY • Session State • Context @@ -160,16 +160,16 @@ - + LLM decides tool calls • Zero IF/THEN logic • Natural conversation flow • Context-aware responses - Legend: + Legend: - Direct response + Direct response - Tool invocation + Tool invocation diff --git a/docs/src/chapter-06-gbdialog/assets/basic-execution-flow.svg.backup b/docs/src/chapter-06-gbdialog/assets/basic-execution-flow.svg.backup new file mode 100644 index 000000000..70f0ef8ae --- /dev/null +++ b/docs/src/chapter-06-gbdialog/assets/basic-execution-flow.svg.backup @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BASIC LLM Tool Execution Flow + + + + + USER + "What's the policy?" + + + + + + + LLM + CONTEXT + Understands intent + loaded KBs + + + + + + + + + Tool? + + + + + + + + Direct Answer + + + + + + Call Tool + + + + BASIC TOOL + enrollment.bas + PARAM name, course + + + + + + + RESPONSE + Generate natural answer + + + + + + + BOT + "30-day return..." + + + + MEMORY STORE + BOT_MEMORY • Session State • Context + + + + + + + + + LLM decides tool calls • Zero IF/THEN logic • Natural conversation flow • Context-aware responses + + + + + Legend: + + Direct response + + Tool invocation + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-07-gbapp/assets/data-flow.svg b/docs/src/chapter-07-gbapp/assets/data-flow.svg index ca221fa1c..afb9212d5 100644 --- a/docs/src/chapter-07-gbapp/assets/data-flow.svg +++ b/docs/src/chapter-07-gbapp/assets/data-flow.svg @@ -1,4 +1,4 @@ - + @@ -18,28 +18,28 @@ - BotServer Data Flow Architecture + BotServer Data Flow Architecture - User Input Layer + User Input Layer - Web UI + Web UI - WhatsApp + WhatsApp - Teams + Teams - Email + Email - API + API @@ -48,27 +48,27 @@ - Core Processing Engine + Core Processing Engine - Session Manager - User Context + Session Manager + User Context - BASIC Interpreter - Script Execution + BASIC Interpreter + Script Execution - LLM Integration - AI Processing + LLM Integration + AI Processing - Knowledge Base - Vector Search + Knowledge Base + Vector Search @@ -77,13 +77,13 @@ - Tool System - External APIs & Functions + Tool System + External APIs & Functions - Cache Layer - Response Optimization + Cache Layer + Response Optimization @@ -92,27 +92,27 @@ - Storage & Persistence Layer + Storage & Persistence Layer - Database - User Data + Database + User Data - Vector DB - Embeddings + Vector DB + Embeddings - Drive Storage - Files & Assets + Drive Storage + Files & Assets - Cache - Fast Access + Cache + Fast Access @@ -126,14 +126,14 @@ - Data Flow: + Data Flow: - Request/Response + Request/Response - Data Access + Data Access - All components run in async Rust for maximum performance + All components run in async Rust for maximum performance diff --git a/docs/src/chapter-07-gbapp/assets/data-flow.svg.backup b/docs/src/chapter-07-gbapp/assets/data-flow.svg.backup new file mode 100644 index 000000000..777aff725 --- /dev/null +++ b/docs/src/chapter-07-gbapp/assets/data-flow.svg.backup @@ -0,0 +1,158 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BotServer Data Flow Architecture + + + + + User Input Layer + + + + Web UI + + + WhatsApp + + + Teams + + + Email + + + API + + + + + + + + + Core Processing Engine + + + + Session Manager + User Context + + + + BASIC Interpreter + Script Execution + + + + LLM Integration + AI Processing + + + + Knowledge Base + Vector Search + + + + + + + + + Tool System + External APIs & Functions + + + + Cache Layer + Response Optimization + + + + + + + + + Storage & Persistence Layer + + + + Database + User Data + + + + Vector DB + Embeddings + + + + Drive Storage + Files & Assets + + + + Cache + Fast Access + + + + + + + + + + + + + + Data Flow: + + Request/Response + + + Data Access + + + + All components run in async Rust for maximum performance + + + + + \ No newline at end of file diff --git a/docs/src/chapter-07-gbapp/assets/module-data-flow.svg b/docs/src/chapter-07-gbapp/assets/module-data-flow.svg index 1ad9a3e41..60f47ab5e 100644 --- a/docs/src/chapter-07-gbapp/assets/module-data-flow.svg +++ b/docs/src/chapter-07-gbapp/assets/module-data-flow.svg @@ -1,4 +1,4 @@ - + @@ -26,56 +26,56 @@ - Data Flow Through Modules + Data Flow Through Modules - User Input + User Input - web_server/ | channels/ - Axum HTTP Server - Route to channel + web_server/ | channels/ + Axum HTTP Server + Route to channel - session/ - Load/Create Session - Validate Token + session/ + Load/Create Session + Validate Token - auth/ - Check Permissions - Apply RBAC + auth/ + Check Permissions + Apply RBAC - bot/ - Route to Bot Instance - Load Configuration + bot/ + Route to Bot Instance + Load Configuration - basic/ - Execute BASIC Script - Parse Keywords + basic/ + Execute BASIC Script + Parse Keywords @@ -86,23 +86,23 @@ - context/ - Load KB + context/ + Load KB - drive/ - Get Files + drive/ + Get Files - database/ - Query DB + database/ + Query DB - llm/ - Call AI + llm/ + Call AI @@ -115,21 +115,21 @@ - Bot Response + Bot Response - 1. Input Reception - 2. HTTP Routing - 3. Session Management - 4. Authorization - 5. Bot Routing - 6. Script Execution - 7. Data Processing - 8. Response Generation + 1. Input Reception + 2. HTTP Routing + 3. Session Management + 4. Authorization + 5. Bot Routing + 6. Script Execution + 7. Data Processing + 8. Response Generation - All operations are async with Tokio runtime for maximum throughput + All operations are async with Tokio runtime for maximum throughput diff --git a/docs/src/chapter-07-gbapp/assets/module-data-flow.svg.backup b/docs/src/chapter-07-gbapp/assets/module-data-flow.svg.backup new file mode 100644 index 000000000..8ec80c23d --- /dev/null +++ b/docs/src/chapter-07-gbapp/assets/module-data-flow.svg.backup @@ -0,0 +1,146 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Data Flow Through Modules + + + + User Input + + + + + + + web_server/ | channels/ + Axum HTTP Server + Route to channel + + + + + + + session/ + Load/Create Session + Validate Token + + + + + + + auth/ + Check Permissions + Apply RBAC + + + + + + + bot/ + Route to Bot Instance + Load Configuration + + + + + + + basic/ + Execute BASIC Script + Parse Keywords + + + + + + + + + + + context/ + Load KB + + + + drive/ + Get Files + + + + database/ + Query DB + + + + llm/ + Call AI + + + + + + + + + + + + + Bot Response + + + + + 1. Input Reception + 2. HTTP Routing + 3. Session Management + 4. Authorization + 5. Bot Routing + 6. Script Execution + 7. Data Processing + 8. Response Generation + + + + All operations are async with Tokio runtime for maximum throughput + + + + + \ No newline at end of file diff --git a/docs/src/chapter-07-gbapp/assets/module-dependency.svg b/docs/src/chapter-07-gbapp/assets/module-dependency.svg index 2f5f3386a..e12d67f16 100644 --- a/docs/src/chapter-07-gbapp/assets/module-dependency.svg +++ b/docs/src/chapter-07-gbapp/assets/module-dependency.svg @@ -1,4 +1,4 @@ - + @@ -22,18 +22,18 @@ - Module Dependency Graph + Module Dependency Graph - main.rs + main.rs - bootstrap/ + bootstrap/ @@ -43,15 +43,15 @@ - package_manager/ + package_manager/ - config/ + config/ - database/ + database/ @@ -59,11 +59,11 @@ - session/ + session/ - web_server/ + web_server/ @@ -80,19 +80,19 @@ - channels/ + channels/ - bot/ + bot/ - basic/ + basic/ - auth/ + auth/ @@ -102,32 +102,32 @@ - llm/ + llm/ - context/ + context/ - Layers: + Layers: - Entry Point + Entry Point - Core System + Core System - Services + Services - Features + Features - Arrows indicate compile-time dependencies + Arrows indicate compile-time dependencies diff --git a/docs/src/chapter-07-gbapp/assets/module-dependency.svg.backup b/docs/src/chapter-07-gbapp/assets/module-dependency.svg.backup new file mode 100644 index 000000000..dbda108af --- /dev/null +++ b/docs/src/chapter-07-gbapp/assets/module-dependency.svg.backup @@ -0,0 +1,148 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Module Dependency Graph + + + + main.rs + + + + + + + bootstrap/ + + + + + + + + + + package_manager/ + + + + config/ + + + + database/ + + + + + + + + session/ + + + + web_server/ + + + + + + + + + + + + + + + + + channels/ + + + + bot/ + + + + basic/ + + + + auth/ + + + + + + + + + + llm/ + + + + + + + context/ + + + + Layers: + + + Entry Point + + + Core System + + + Services + + + Features + + + + Arrows indicate compile-time dependencies + + + + + \ No newline at end of file diff --git a/docs/src/chapter-07-gbapp/assets/system-architecture.svg b/docs/src/chapter-07-gbapp/assets/system-architecture.svg index ec8cb2c91..ed3567fcd 100644 --- a/docs/src/chapter-07-gbapp/assets/system-architecture.svg +++ b/docs/src/chapter-07-gbapp/assets/system-architecture.svg @@ -1,4 +1,4 @@ - + @@ -6,49 +6,49 @@ - BotServer Architecture - Virtual Crates System + BotServer Architecture - Virtual Crates System - BotServer Binary + BotServer Binary - compiles to + compiles to - Core Engine (src/core/) + Core Engine (src/core/) - Bootstrap - System Init - Service Start + Bootstrap + System Init + Service Start - Package Manager - Component Registry - Module Loader + Package Manager + Component Registry + Module Loader - Session Manager - Context Handling - State Management + Session Manager + Context Handling + State Management - Shared State - AppState - Configuration + Shared State + AppState + Configuration - Utils - Helpers - Common + Utils + Helpers + Common @@ -58,47 +58,47 @@ - Virtual Crates (gbapp modules in src/) + Virtual Crates (gbapp modules in src/) - basic.gbapp - src/basic/ + basic.gbapp + src/basic/ - • BASIC Interpreter - • Keywords Registry - • Script Execution - • Rhai Engine + • BASIC Interpreter + • Keywords Registry + • Script Execution + • Rhai Engine - channels.gbapp - src/channels/ + channels.gbapp + src/channels/ - • WhatsApp - • Teams - • Email - • Web UI + • WhatsApp + • Teams + • Email + • Web UI - storage.gbapp - src/storage/ + storage.gbapp + src/storage/ - • Knowledge Base - • Drive Integration - • Vector DB - • Cache + • Knowledge Base + • Drive Integration + • Vector DB + • Cache - your_feature.gbapp - src/your_feature/ + your_feature.gbapp + src/your_feature/ - • Your Keywords - • Your Services - • Your Models - + Add yours! + • Your Keywords + • Your Services + • Your Models + + Add yours! @@ -107,49 +107,49 @@ - AI & LLM Integration + AI & LLM Integration - LLM Service + LLM Service - Embeddings + Embeddings - Semantic Search + Semantic Search - Persistence Layer + Persistence Layer - Database + Database - Vector DB + Vector DB - Drive + Drive - Cache + Cache - Key Concepts: + Key Concepts: - Virtual Crates = Modules in src/ + Virtual Crates = Modules in src/ - Your Contribution Space + Your Contribution Space - All compile to single optimized binary + All compile to single optimized binary - gbapp virtual crates: The bridge between old Node.js packages and new Rust modules + gbapp virtual crates: The bridge between old Node.js packages and new Rust modules diff --git a/docs/src/chapter-07-gbapp/assets/system-architecture.svg.backup b/docs/src/chapter-07-gbapp/assets/system-architecture.svg.backup new file mode 100644 index 000000000..30911ff7c --- /dev/null +++ b/docs/src/chapter-07-gbapp/assets/system-architecture.svg.backup @@ -0,0 +1,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BotServer Architecture - Virtual Crates System + + + + + + + BotServer Binary + + + + compiles to + + + + + Core Engine (src/core/) + + + + Bootstrap + System Init + Service Start + + + Package Manager + Component Registry + Module Loader + + + Session Manager + Context Handling + State Management + + + Shared State + AppState + Configuration + + + Utils + Helpers + Common + + + + + + + + + + Virtual Crates (gbapp modules in src/) + + + + basic.gbapp + src/basic/ + + • BASIC Interpreter + • Keywords Registry + • Script Execution + • Rhai Engine + + + + channels.gbapp + src/channels/ + + • WhatsApp + • Teams + • Email + • Web UI + + + + storage.gbapp + src/storage/ + + • Knowledge Base + • Drive Integration + • Vector DB + • Cache + + + + your_feature.gbapp + src/your_feature/ + + • Your Keywords + • Your Services + • Your Models + + Add yours! + + + + + + + + + AI & LLM Integration + + + LLM Service + + + Embeddings + + + Semantic Search + + + + + + Persistence Layer + + + Database + + + Vector DB + + + Drive + + + Cache + + + + + Key Concepts: + + + Virtual Crates = Modules in src/ + + + Your Contribution Space + + All compile to single optimized binary + + + + gbapp virtual crates: The bridge between old Node.js packages and new Rust modules + + + + + \ No newline at end of file diff --git a/docs/src/chapter-10-api/assets/api-architecture.svg b/docs/src/chapter-10-api/assets/api-architecture.svg index b127ce71d..7a447974c 100644 --- a/docs/src/chapter-10-api/assets/api-architecture.svg +++ b/docs/src/chapter-10-api/assets/api-architecture.svg @@ -1,4 +1,4 @@ - + @@ -61,13 +61,13 @@ - + BotServer API Architecture - + Client Applications @@ -76,10 +76,10 @@ - + HTTP/HTTPS - + Port 8080 @@ -88,10 +88,10 @@ - + API Gateway - + /api/* @@ -102,46 +102,46 @@ - + Auth Endpoints - + /auth/login - + /auth/logout - + /auth/token - + Business Endpoints - + /files/* • /users/* - + /groups/* • /tasks/* - + /sessions/* - + Admin Endpoints - + /admin/* - + /monitoring - + /analytics @@ -152,13 +152,13 @@ - + Service Layer - + • Session Manager - + • Auth Service • Bot Service @@ -169,37 +169,37 @@ - + PostgreSQL - + Database - + Sessions • Users • Config - + Valkey - + Cache - + Semantic • Session • Temp - + Qdrant - + Vectors - + Embeddings • Search diff --git a/docs/src/chapter-10-api/assets/api-architecture.svg.backup b/docs/src/chapter-10-api/assets/api-architecture.svg.backup new file mode 100644 index 000000000..52b42a1d5 --- /dev/null +++ b/docs/src/chapter-10-api/assets/api-architecture.svg.backup @@ -0,0 +1,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BotServer API Architecture + + + + + + Client Applications + + + + + + + + + HTTP/HTTPS + + + Port 8080 + + + + + + + + + API Gateway + + + /api/* + + + + + + + + + + + Auth Endpoints + + + /auth/login + + + /auth/logout + + + /auth/token + + + + + + Business Endpoints + + + /files/* • /users/* + + + /groups/* • /tasks/* + + + /sessions/* + + + + + + Admin Endpoints + + + /admin/* + + + /monitoring + + + /analytics + + + + + + + + + + + Service Layer + + + • Session Manager + + + • Auth Service • Bot Service + + + + + + + + + + + PostgreSQL + + + Database + + + Sessions • Users • Config + + + + + + Valkey + + + Cache + + + Semantic • Session • Temp + + + + + + Qdrant + + + Vectors + + + Embeddings • Search + + + + + + \ No newline at end of file diff --git a/docs/src/chapter-10-api/assets/api-request-flow.svg b/docs/src/chapter-10-api/assets/api-request-flow.svg index 9cd6bf97a..17f53718d 100644 --- a/docs/src/chapter-10-api/assets/api-request-flow.svg +++ b/docs/src/chapter-10-api/assets/api-request-flow.svg @@ -1,4 +1,4 @@ - + @@ -86,16 +86,16 @@ - + API Request Flow - + HTTP Request - + Client → Server @@ -104,42 +104,42 @@ - + Rate Limit - + Check request limits - Pass + Pass - 429 Too Many + 429 Too Many - + Auth - + Validate token/session - Valid + Valid - 401 Unauthorized + 401 Unauthorized - + Route - + Match endpoint pattern @@ -147,30 +147,30 @@ - 404 Not Found + 404 Not Found - + Validate - + Check request body - Valid + Valid - 400 Bad Request + 400 Bad Request - + Process - + Execute business logic @@ -179,10 +179,10 @@ - + Format - + JSON response @@ -191,22 +191,22 @@ - + HTTP Response - + Server → Client - + Error Response - + Request Pipeline @@ -214,11 +214,11 @@ - Success path + Success path - Error path + Error path diff --git a/docs/src/chapter-10-api/assets/api-request-flow.svg.backup b/docs/src/chapter-10-api/assets/api-request-flow.svg.backup new file mode 100644 index 000000000..583c5f778 --- /dev/null +++ b/docs/src/chapter-10-api/assets/api-request-flow.svg.backup @@ -0,0 +1,180 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + API Request Flow + + + + + + HTTP Request + + + Client → Server + + + + + + + + + Rate Limit + + + Check request limits + + + + + Pass + + + 429 Too Many + + + + + Auth + + + Validate token/session + + + + + Valid + + + 401 Unauthorized + + + + + Route + + + Match endpoint pattern + + + + + + + 404 Not Found + + + + + Validate + + + Check request body + + + + + Valid + + + 400 Bad Request + + + + + Process + + + Execute business logic + + + + + + + + + Format + + + JSON response + + + + + + + + + HTTP Response + + + Server → Client + + + + + + Error Response + + + + + + Request Pipeline + + + + + + + Success path + + + + + Error path + + + + + + \ No newline at end of file diff --git a/docs/src/introduction.md b/docs/src/introduction.md index 34b8e0a87..a1b7e9e9a 100644 --- a/docs/src/introduction.md +++ b/docs/src/introduction.md @@ -44,7 +44,7 @@ The AI handles everything else - understanding intent, collecting information, e That's it. No Kubernetes, no cloud accounts. The [bootstrap process](./chapter-01/installation.md) installs everything locally in 2-5 minutes. [PostgreSQL](./chapter-07/postgresql.md), [vector database](./chapter-03/vector-collections.md), [object storage](./chapter-07/minio.md), [cache](./chapter-03/caching.md) - all configured automatically with secure credentials. ### Real BASIC, Real Simple -Remember BASIC from the 80s? We brought it back for conversational AI. See our [complete keyword reference](./chapter-05/README.md): +We brought BASIC back for conversational AI. See our [complete keyword reference](./chapter-05/README.md): ```basic ' save-note.bas - A simple tool PARAM topic, content diff --git a/fix_all_svgs.py b/fix_all_svgs.py new file mode 100644 index 000000000..32f5ad431 --- /dev/null +++ b/fix_all_svgs.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +""" +Fix and beautify all SVG files with proper syntax and mobile-friendly design +""" + +import os +import re +import xml.etree.ElementTree as ET +from pathlib import Path + + +def fix_svg_file(filepath): + """Fix a single SVG file with proper formatting and mobile-friendly design""" + + try: + # Read the original file + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + # Skip font files and favicons + if "fontawesome" in str(filepath).lower() or "favicon" in str(filepath).lower(): + return False + + print(f"Fixing: {filepath}") + + # First, clean up any broken attributes + # Remove any malformed style attributes + content = re.sub(r'style="[^"]*"[^>]*style="[^"]*"', "", content) + + # Fix basic SVG structure + if not content.strip().startswith("\n' + content + + # Extract dimensions + width_match = re.search(r'width="(\d+)"', content) + height_match = re.search(r'height="(\d+)"', content) + viewbox_match = re.search(r'viewBox="([^"]+)"', content) + + if viewbox_match: + viewbox = viewbox_match.group(1) + elif width_match and height_match: + width = width_match.group(1) + height = height_match.group(1) + viewbox = f"0 0 {width} {height}" + else: + viewbox = "0 0 800 600" + + # Create clean SVG header + svg_header = f''' + + + + + + + + + + + + + + + + + + + + +''' + + # Extract the main content (remove old svg tags and defs) + main_content = re.sub(r"<\?xml[^>]*\?>", "", content) + main_content = re.sub(r"]*>", "", main_content) + main_content = re.sub(r"", "", main_content) + main_content = re.sub(r".*?", "", main_content, flags=re.DOTALL) + + # Fix font sizes for mobile (minimum 14px for body text) + def fix_font_size(match): + size = int(match.group(1)) + if size < 12: + return f'font-size="{14}"' + elif size < 14: + return f'font-size="{14}"' + elif size > 24: + return f'font-size="{24}"' + else: + return match.group(0) + + main_content = re.sub(r'font-size="(\d+)"', fix_font_size, main_content) + + # Fix colors for better contrast + color_map = { + # Blues + "#63B3ED": "#2563EB", + "#90CDF4": "#3B82F6", + "#4A90E2": "#2563EB", + "#CBD5E0": "#1F2937", # Light gray text to dark + "#A0AEC0": "#4B5563", # Medium gray text to darker + # Greens + "#68D391": "#059669", + "#48BB78": "#10B981", + "#38A169": "#059669", + "#9AE6B4": "#10B981", + # Purples + "#B794F4": "#7C3AED", + "#D6BCFA": "#8B5CF6", + "#9F7AEA": "#7C3AED", + "#E9D8FD": "#8B5CF6", + # Oranges + "#F6AD55": "#EA580C", + "#FBD38D": "#F97316", + "#ED8936": "#EA580C", + # Reds + "#FC8181": "#DC2626", + "#FEB2B2": "#EF4444", + "#E53E3E": "#DC2626", + # Teals + "#4FD1C5": "#0891B2", + "#81E6D9": "#06B6D4", + "#38D4B2": "#0891B2", + "#B2F5EA": "#06B6D4", + # Grays + "#4A5568": "#6B7280", + "#718096": "#6B7280", + "#888": "#6B7280", + } + + for old_color, new_color in color_map.items(): + main_content = main_content.replace( + f'fill="{old_color}"', f'fill="{new_color}"' + ) + main_content = main_content.replace( + f'stroke="{old_color}"', f'stroke="{new_color}"' + ) + main_content = main_content.replace( + f'fill="{old_color.lower()}"', f'fill="{new_color}"' + ) + main_content = main_content.replace( + f'stroke="{old_color.lower()}"', f'stroke="{new_color}"' + ) + + # Fix font families + main_content = re.sub( + r'font-family="[^"]*"', + 'font-family="system-ui, -apple-system, sans-serif"', + main_content, + ) + + # Ensure stroke widths are visible + main_content = re.sub(r'stroke-width="1"', 'stroke-width="2"', main_content) + + # Add rounded corners to rectangles + def add_rounded_corners(match): + rect = match.group(0) + if "rx=" not in rect: + rect = rect[:-1] + ' rx="6"/>' + return rect + + main_content = re.sub(r"]*/>", add_rounded_corners, main_content) + + # Combine everything + final_svg = svg_header + main_content + "\n" + + # Write the fixed file + with open(filepath, "w", encoding="utf-8") as f: + f.write(final_svg) + + print(f" ✓ Fixed successfully") + return True + + except Exception as e: + print(f" ✗ Error: {e}") + return False + + +def main(): + """Fix all SVG files in the docs directory""" + + print("=" * 60) + print("SVG FIXER - Repairing and beautifying all diagrams") + print("=" * 60) + print() + + docs_dir = Path("docs") + svg_files = list(docs_dir.glob("**/*.svg")) + + print(f"Found {len(svg_files)} SVG files") + print() + + fixed_count = 0 + skipped_count = 0 + error_count = 0 + + for svg_file in svg_files: + if "fontawesome" in str(svg_file).lower() or "favicon" in str(svg_file).lower(): + print(f"Skipping: {svg_file} (font/favicon)") + skipped_count += 1 + continue + + result = fix_svg_file(svg_file) + if result: + fixed_count += 1 + else: + error_count += 1 + + print() + print("=" * 60) + print("SUMMARY") + print("=" * 60) + print(f"✓ Fixed: {fixed_count} files") + print(f"⊘ Skipped: {skipped_count} files") + if error_count > 0: + print(f"✗ Errors: {error_count} files") + print() + print("All SVG files now have:") + print("• Mobile-friendly text sizes (min 14px)") + print("• High contrast colors") + print("• Consistent fonts") + print("• Rounded corners") + print("• Proper stroke widths") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/fix_svg_attributes.py b/fix_svg_attributes.py new file mode 100644 index 000000000..b02ef6f57 --- /dev/null +++ b/fix_svg_attributes.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +""" +Fix malformed SVG attributes in all SVG files +Specifically fixes the 'rx="5"/' issue and other attribute errors +""" + +import os +import re +from pathlib import Path + + +def fix_malformed_attributes(content): + """Fix various types of malformed attributes in SVG content""" + + # Fix malformed rx attributes (rx="5"/ should be rx="5") + content = re.sub(r'rx="([^"]+)"\s*/', r'rx="\1"', content) + + # Fix malformed ry attributes + content = re.sub(r'ry="([^"]+)"\s*/', r'ry="\1"', content) + + # Fix cases where filter appears after malformed rx + content = re.sub( + r'rx="([^"]+)"/\s*filter="([^"]+)"', r'rx="\1" filter="\2"', content + ) + + # Fix double closing brackets + content = re.sub(r"/>>", r"/>", content) + + # Fix attributes that got split incorrectly + content = re.sub(r'"\s+([a-z-]+)="', r'" \1="', content) + + # Fix rect tags with malformed endings + content = re.sub( + r']+)"\s*/\s+([a-z-]+)="([^"]+)">', r'', content + ) + + # Fix specific pattern: stroke-width="2" rx="5"/ filter="url(#shadow)"> + content = re.sub( + r'stroke-width="(\d+)"\s+rx="(\d+)"/\s*filter="([^"]+)">', + r'stroke-width="\1" rx="\2" filter="\3">', + content, + ) + + # Fix any remaining "/ patterns at the end of attributes + content = re.sub(r'="([^"]*)"\s*/', r'="\1"', content) + + # Fix rectangles that should be self-closing + lines = content.split("\n") + fixed_lines = [] + + for line in lines: + # If it's a rect element that ends with > but has no content, make it self-closing + if ( + "") + and not line.strip().endswith("/>") + ): + # Check if this rect has content after it or should be self-closing + if ( + 'fill="none"' in line + or 'fill="transparent"' in line + or 'fill="white"' in line + ): + line = line.rstrip(">") + "/>" + fixed_lines.append(line) + + content = "\n".join(fixed_lines) + + return content + + +def validate_svg_structure(content): + """Basic validation to ensure SVG structure is correct""" + + # Check for basic SVG structure + if "" not in content: + return False, "Missing closing SVG tag" + + # Count opening and closing tags for basic elements + rect_open = content.count("") + rect_self = content.count("/>") + + # Basic tag balance check (not perfect but catches major issues) + text_open = content.count("") + + if text_open != text_close: + return False, f"Text tag mismatch: {text_open} opening vs {text_close} closing" + + # Check for common malformed patterns + if "/ " in content and "filter=" in content: + malformed = re.findall(r'rx="[^"]+"/\s*filter=', content) + if malformed: + return False, f"Found malformed attribute pattern" + + return True, "OK" + + +def fix_svg_file(filepath): + """Fix a single SVG file""" + + try: + # Read the file + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + # Skip font files and favicons + if "fontawesome" in str(filepath).lower() or "favicon" in str(filepath).lower(): + return "skipped", None + + # Apply fixes + fixed_content = fix_malformed_attributes(content) + + # Validate the result + is_valid, message = validate_svg_structure(fixed_content) + + if not is_valid: + print(f" ⚠ Validation warning: {message}") + + # Write back only if content changed + if fixed_content != content: + with open(filepath, "w", encoding="utf-8") as f: + f.write(fixed_content) + return "fixed", None + else: + return "unchanged", None + + except Exception as e: + return "error", str(e) + + +def main(): + """Fix all SVG files in the docs directory""" + + print("=" * 60) + print("SVG ATTRIBUTE FIXER") + print("Fixing malformed attributes in all SVG files") + print("=" * 60) + print() + + docs_dir = Path("docs") + svg_files = list(docs_dir.glob("**/*.svg")) + + print(f"Found {len(svg_files)} SVG files") + print() + + stats = {"fixed": 0, "unchanged": 0, "skipped": 0, "error": 0} + + for svg_file in svg_files: + print(f"Processing: {svg_file}") + + status, error = fix_svg_file(svg_file) + stats[status] += 1 + + if status == "fixed": + print(f" ✓ Fixed malformed attributes") + elif status == "unchanged": + print(f" - No changes needed") + elif status == "skipped": + print(f" ⊘ Skipped (font/favicon)") + elif status == "error": + print(f" ✗ Error: {error}") + + print() + print("=" * 60) + print("SUMMARY") + print("=" * 60) + print(f"✓ Fixed: {stats['fixed']} files") + print(f"- Unchanged: {stats['unchanged']} files") + print(f"⊘ Skipped: {stats['skipped']} files") + if stats["error"] > 0: + print(f"✗ Errors: {stats['error']} files") + + print() + print("Common fixes applied:") + print('• Fixed malformed rx="5"/ attributes') + print("• Corrected filter attribute placement") + print("• Fixed self-closing tags") + print("• Cleaned up attribute spacing") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/fix_svgs_properly.py b/fix_svgs_properly.py new file mode 100644 index 000000000..1baf32e6b --- /dev/null +++ b/fix_svgs_properly.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +""" +Fix all SVG files to be properly readable on all devices +Focus on text size, contrast, and responsive design +""" + +import os +import re +from pathlib import Path + + +def fix_svg_content(content, filename): + """ + Apply comprehensive fixes to SVG content + """ + + # 1. Fix SVG header for responsiveness + # Remove any width/height attributes and ensure proper viewBox + if "]*>" + svg_replacement = f'' + content = re.sub(svg_pattern, svg_replacement, content, count=1) + + # 2. Fix all font sizes - MINIMUM 16px for readability + def increase_font_size(match): + size = int(match.group(1)) + if size < 14: + return f'font-size="16"' + elif size < 16: + return f'font-size="18"' + elif size < 18: + return f'font-size="20"' + else: + return f'font-size="{size + 4}"' # Increase all fonts slightly + + content = re.sub(r'font-size="(\d+)"', increase_font_size, content) + + # 3. Fix ALL text colors for maximum contrast + # Replace all light colors with dark, readable ones + color_replacements = { + # Light grays to dark gray/black + "#CBD5E0": "#1F2937", + "#A0AEC0": "#374151", + "#E2E8F0": "#1F2937", + "#EDF2F7": "#111827", + "#F7FAFC": "#111827", + "#9CA3AF": "#374151", + "#D1D5DB": "#4B5563", + "#718096": "#374151", + "#4A5568": "#1F2937", + # Light blues to dark blues + "#90CDF4": "#1E40AF", + "#63B3ED": "#2563EB", + "#4A90E2": "#1D4ED8", + "#81E6D9": "#0E7490", + "#4FD1C5": "#0891B2", + "#38D4B2": "#0D9488", + # Light purples to dark purples + "#E9D8FD": "#6B21A8", + "#D6BCFA": "#7C3AED", + "#B794F4": "#9333EA", + "#9F7AEA": "#7C3AED", + # Light oranges to dark oranges + "#FBD38D": "#C2410C", + "#F6AD55": "#EA580C", + "#ED8936": "#C2410C", + # Light reds to dark reds + "#FEB2B2": "#B91C1C", + "#FC8181": "#DC2626", + "#E53E3E": "#DC2626", + # Light greens to dark greens + "#9AE6B4": "#047857", + "#68D391": "#059669", + "#48BB78": "#047857", + "#38A169": "#059669", + "#B2F5EA": "#047857", + # Generic light to dark + "#888": "#374151", + "#888888": "#374151", + "#FAFAFA": "transparent", + "#fff": "#111827", + "#ffffff": "#111827", + "#FFF": "#111827", + "#FFFFFF": "#111827", + } + + for old_color, new_color in color_replacements.items(): + # Replace in fill attributes + content = re.sub( + f'fill="{old_color}"', f'fill="{new_color}"', content, flags=re.IGNORECASE + ) + # Replace in stroke attributes + content = re.sub( + f'stroke="{old_color}"', + f'stroke="{new_color}"', + content, + flags=re.IGNORECASE, + ) + # Replace in style attributes + content = re.sub( + f"fill:{old_color}", f"fill:{new_color}", content, flags=re.IGNORECASE + ) + content = re.sub( + f"stroke:{old_color}", f"stroke:{new_color}", content, flags=re.IGNORECASE + ) + + # 4. Remove white/light backgrounds + content = re.sub(r']*fill="#FAFAFA"[^>]*>', "", content) + content = re.sub(r']*fill="white"[^>]*>', "", content) + content = re.sub(r']*fill="#FFFFFF"[^>]*>', "", content) + content = re.sub(r']*fill="#ffffff"[^>]*>', "", content) + + # 5. Fix stroke widths for visibility + content = re.sub(r'stroke-width="1"', 'stroke-width="2"', content) + content = re.sub(r'stroke-width="0\.5"', 'stroke-width="2"', content) + + # 6. Fix font weights + content = re.sub(r'font-weight="bold"', 'font-weight="700"', content) + + # 7. Fix font families for better rendering + content = re.sub( + r'font-family="[^"]*"', + 'font-family="system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, sans-serif"', + content, + ) + + # 8. Ensure arrows and markers are visible + content = re.sub( + r'', + '', + content, + ) + + # 9. Add slight padding to the viewBox if needed + viewbox_match = re.search(r'viewBox="(\d+)\s+(\d+)\s+(\d+)\s+(\d+)"', content) + if viewbox_match: + x, y, width, height = map(int, viewbox_match.groups()) + # Add 20px padding + new_viewbox = f'viewBox="{x - 20} {y - 20} {width + 40} {height + 40}"' + content = re.sub(r'viewBox="[^"]*"', new_viewbox, content, count=1) + + return content + + +def process_all_svgs(): + """Process all SVG files in the docs directory""" + + docs_dir = Path("docs") + svg_files = list(docs_dir.glob("**/*.svg")) + + # Filter out font files + svg_files = [ + f + for f in svg_files + if "fontawesome" not in str(f).lower() and "favicon" not in str(f).lower() + ] + + print(f"Found {len(svg_files)} SVG files to fix") + print("=" * 60) + + fixed = 0 + errors = 0 + + for svg_file in svg_files: + try: + print(f"Processing: {svg_file.relative_to(docs_dir)}") + + # Read the file + with open(svg_file, "r", encoding="utf-8") as f: + content = f.read() + + # Apply fixes + fixed_content = fix_svg_content(content, svg_file.name) + + # Write back + with open(svg_file, "w", encoding="utf-8") as f: + f.write(fixed_content) + + print(f" ✓ Fixed successfully") + fixed += 1 + + except Exception as e: + print(f" ✗ Error: {e}") + errors += 1 + + print() + print("=" * 60) + print(f"COMPLETED: {fixed} files fixed, {errors} errors") + print() + print("Improvements applied:") + print("• All text now ≥16px (readable on mobile)") + print("• High contrast colors (dark text, no light grays)") + print("• 100% responsive width") + print("• Removed white backgrounds") + print("• Enhanced stroke widths") + print("• Added padding to prevent cutoff") + print("=" * 60) + + +if __name__ == "__main__": + print("=" * 60) + print("SVG READABILITY FIXER") + print("Making all diagrams actually readable!") + print("=" * 60) + print() + process_all_svgs() diff --git a/minimal_svg_fix.py b/minimal_svg_fix.py new file mode 100644 index 000000000..c113f25f3 --- /dev/null +++ b/minimal_svg_fix.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +Minimal SVG fix - ONLY fixes text size and contrast +No structural changes, no breaking modifications +""" + +import re +from pathlib import Path + + +def minimal_fix_svg(filepath): + """Apply minimal fixes to make SVG text readable""" + + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + # Skip font files + if "fontawesome" in str(filepath).lower() or "favicon" in str(filepath).lower(): + return False + + # 1. Increase font sizes (minimum 16px for readability) + def fix_font_size(match): + size = int(match.group(1)) + if size <= 11: + return f'font-size="16"' + elif size == 12: + return f'font-size="18"' + elif size <= 14: + return f'font-size="20"' + else: + return f'font-size="{size + 4}"' + + content = re.sub(r'font-size="(\d+)"', fix_font_size, content) + + # 2. Fix text colors for contrast + # Light grays to dark + content = content.replace('fill="#CBD5E0"', 'fill="#1F2937"') + content = content.replace('fill="#A0AEC0"', 'fill="#374151"') + content = content.replace('fill="#718096"', 'fill="#374151"') + content = content.replace('fill="#E2E8F0"', 'fill="#1F2937"') + + # Light blues to darker blues + content = content.replace('fill="#90CDF4"', 'fill="#1E40AF"') + content = content.replace('fill="#63B3ED"', 'fill="#2563EB"') + + # Light purples to darker + content = content.replace('fill="#E9D8FD"', 'fill="#7C3AED"') + content = content.replace('fill="#D6BCFA"', 'fill="#9333EA"') + content = content.replace('fill="#B794F4"', 'fill="#9333EA"') + + # Light oranges to darker + content = content.replace('fill="#FBD38D"', 'fill="#EA580C"') + content = content.replace('fill="#F6AD55"', 'fill="#D97706"') + + # Light reds to darker + content = content.replace('fill="#FEB2B2"', 'fill="#DC2626"') + content = content.replace('fill="#FC8181"', 'fill="#EF4444"') + + # Light greens stay green (they're usually OK) + # But make them slightly darker + content = content.replace('fill="#9AE6B4"', 'fill="#10B981"') + content = content.replace('fill="#68D391"', 'fill="#059669"') + content = content.replace('fill="#48BB78"', 'fill="#047857"') + + # Light teals + content = content.replace('fill="#81E6D9"', 'fill="#0891B2"') + content = content.replace('fill="#4FD1C5"', 'fill="#0891B2"') + content = content.replace('fill="#B2F5EA"', 'fill="#0E7490"') + + # Gray arrows + content = content.replace('fill="#888"', 'fill="#4B5563"') + + # 3. Make SVG responsive (add style attribute if missing) + if "")[0]: + content = re.sub( + r"(]*)(>)", + r'\1 style="max-width: 100%; height: auto;"\2', + content, + count=1, + ) + + # Write the fixed content + with open(filepath, "w", encoding="utf-8") as f: + f.write(content) + + return True + + +def main(): + """Fix all SVG files in docs/src""" + + docs_src = Path("docs/src") + svg_files = list(docs_src.rglob("*.svg")) + + print(f"Fixing {len(svg_files)} SVG files...") + + fixed = 0 + for svg_file in svg_files: + try: + if minimal_fix_svg(svg_file): + print(f"✓ {svg_file.name}") + fixed += 1 + except Exception as e: + print(f"✗ {svg_file.name}: {e}") + + print(f"\nFixed {fixed} files") + print("Changes made:") + print("• Font sizes increased (16px minimum)") + print("• Text colors darkened for contrast") + print("• SVGs made responsive") + + +if __name__ == "__main__": + main() diff --git a/prompts/dev/svg-diagram-style-guide.md b/prompts/dev/svg-diagram-style-guide.md new file mode 100644 index 000000000..0b7f4be7d --- /dev/null +++ b/prompts/dev/svg-diagram-style-guide.md @@ -0,0 +1,341 @@ +# SVG Diagram Style Guide & Prompt Template + +## Quick Prompt Template + +When creating technical diagrams or flow charts, use this prompt: + +``` +Create a clean SVG diagram with these specifications: +- Transparent background (no fill) +- Large dimensions: width="1040-1400" height="[appropriate height]" (1.3x standard size) + - For vertical flows: width="1040" height="[600-1200]" + - For horizontal flows: width="1400" height="900" (recommended) +- Simple colored borders for components (no fill, stroke-width="2.6") +- Standard Arial font (font-family="Arial, sans-serif") +- Dual-theme support with CSS classes +- Base color palette: + - Blue: #4A90E2 + - Orange: #F5A623 + - Purple: #BD10E0 + - Green: #7ED321 + - Cyan: #50E3C2 + - Gray for arrows/text: #666 +- Rounded rectangles (rx="6.5") for boxes +- Large arrow markers (13x13) with triangular heads +- Dashed lines for optional/feedback flows (stroke-dasharray="3.9,3.9") +- Subtle neon glow effects for dark themes +- Text should be centered in boxes (text-anchor="middle") +- Font sizes: 29-32px for titles, 22-24px for component labels, 18-21px for descriptions +- DUAL DIAGRAM COMPOSITION when possible (main flow + progress/legend) +- Title positioned well above content (y="45" minimum) +- Text wrapping for long labels (review box width constraints) +``` + +## Beautiful Composition Rules - THE STANDARD! + +### Dual-Diagram Approach (RECOMMENDED) +When creating process flows or pipelines, compose TWO complementary visualizations: + +1. **Main Flow Diagram** (Top Section) + - Primary process visualization with components and connections + - Positioned in upper 60-70% of canvas + - Clear phase groupings with section labels + - Components sized appropriately for their text content + +2. **Progress Indicator/Legend** (Bottom Section) + - Visual timeline or progress bar showing stages + - Positioned in lower 30-40% of canvas + - Stage markers with labels below + - Connected with subtle lines or gradient background + - Creates visual rhythm and helps navigation + +### Text Handling Rules +- **Long Text**: MUST be reviewed against box width + - If text exceeds box width, either: + - Increase box width to accommodate + - Use text wrapping with multiple elements + - Abbreviate with full text in tooltip/description +- **Component Labels**: Keep concise, max 2-3 words when possible +- **Descriptions**: Use separate text elements below main diagram + +### Spacing & Visual Hierarchy +- **Title Separation**: Position title FAR from content (y="45" minimum) +- **Phase Grouping**: Clear visual separation between logical phases +- **Vertical Rhythm**: Consistent spacing creates professional look +- **Legend Positioning**: Always at bottom with ample spacing from main diagram + +## Enhanced SVG Structure Template with Dual Composition + +```svg + + + + + + + + + + + [Title] + + + + + [Phase Label] + + + + + [Label - check width!] + + + + + [First line] + [Second line] + + + + + + + + + + + + + + + + + + + + + + + + + + + [Stage] + + + + + [Main description line] + + + [Secondary description line] + + +``` + +## Updated Component Styling Rules + +### Boxes/Rectangles (1.3x Scale) +- **Standard Dimensions**: + - Vertical flow: width="156-260" height="59-70" + - Horizontal flow: width="200-300" height="60-70" + - Compact components: width="100" height="50" + - **IMPORTANT**: Width MUST accommodate text content +- **Text Overflow Handling**: + - Review all text against box width before finalizing + - Use dynamic width sizing based on text length + - Consider multi-line text with elements +- **Border**: stroke-width="2.6" (light) / "2.8" (dark), no fill, rounded corners rx="5-6.5" +- **Colors**: Use CSS classes (neon-blue, neon-orange, etc.) for theme support +- **Spacing**: + - Vertical: minimum 35px spacing + - Horizontal: minimum 70px spacing between major phases + +### Text (1.3x Scale) +- **Title**: + - font-size="32", font-weight="600", class="main-text" + - Position FAR above content (y="45" minimum) +- **Labels**: + - font-size="22-24", font-weight="500", class="main-text" + - Centered in boxes (text-anchor="middle") + - Check width constraints! +- **Compact labels**: font-size="18", for small components in grids +- **Section headers**: font-size="21", font-weight="500", class="secondary-text" +- **Descriptions**: font-size="21", class="secondary-text" +- **Font**: Always "Arial, sans-serif" +- **Text Wrapping**: Use for multi-line text in boxes + +### Arrows (1.3x Scale) +- **Main flow**: Solid lines, stroke-width="2.6", opacity="0.7" +- **Optional/parallel**: Dashed lines, stroke-dasharray="3.9,3.9", opacity="0.5" +- **Feedback loops**: Dashed curves, stroke-dasharray="3.9,3.9", opacity="0.5" +- **Arrow heads**: Enlarged triangular marker (13x13), uses arrow-color class +- **Connection lines**: stroke-width="1.5", opacity="0.5" for component merging +- **Progress connections**: stroke-width="2", opacity="0.4" + +### Layout (1.3x Scale) +- **Canvas**: + - Vertical flows: 1040px width minimum + - Horizontal flows: 1400px width recommended + - Aspect ratio: 16:9 for horizontal, 3:4 for vertical +- **Content Zones**: + - Title zone: 0-80px + - Main diagram: 80-450px (horizontal) or 80-600px (vertical) + - Progress/Legend: 500-650px + - Descriptions: 700-800px +- **Margins**: 50px from edges minimum +- **Spacing**: + - Title to content: 50px minimum + - Main diagram to progress: 100px minimum + - Vertical flows: 52-78px between components + - Horizontal flows: 70-100px between major phases +- **Component grid**: + - Can use 2x2 grids for related components + - Merge lines with opacity="0.5" for grouped items +- **Alignment**: + - Center-align titles at x="700" (1400px width) + - Use consistent alignment within phases + +## Theme-Aware Color System + +### Light Theme (Default) +- **Blue**: #4A90E2 (Input/User/Start elements) +- **Orange**: #F5A623 (Processing/Scripts/Detection) +- **Purple**: #BD10E0 (AI/ML/Decision/Configuration) +- **Green**: #7ED321 (Execution/Action/Completion) +- **Cyan**: #50E3C2 (Output/Response/Storage) +- **Text**: #1a1a1a (main), #666 (secondary) + +### Dark Theme (Neon Effects) +- **Blue**: #00D4FF with subtle glow +- **Orange**: #FF9500 with subtle glow +- **Purple**: #E040FB with subtle glow +- **Green**: #00FF88 with subtle glow +- **Cyan**: #00E5EA with subtle glow +- **Text**: #FFFFFF (main), #B0B0B0 (secondary) + +## Example Usage + +### For a beautiful dual-diagram composition: +``` +"Create a horizontal flow SVG (1400x900) with DUAL DIAGRAM composition: + +MAIN FLOW (top section): +- Start: ./botserver (neon-blue) +- OS Detection (neon-orange) +- Component Installation (2x2 grid: PostgreSQL, Valkey, SeaweedFS, Qdrant) +- Configuration & Setup (neon-purple) +- Bot Deployment (vertical sub-flow with 3 steps) + +PROGRESS INDICATOR (bottom section): +- Gradient background bar +- 4 stage markers: Start, Detect, Install & Configure, Deploy +- Connected with subtle lines + +Position title well above content. +Check all text fits within boxes - adjust widths as needed. +Add descriptions at bottom with proper spacing. +Use CSS classes for theme support, subtle neon glow in dark mode." +``` + +### For a complex system with legend: +``` +"Create an SVG with beautiful composition (1400x900): + +MAIN ARCHITECTURE (upper 70%): +- Client requests flow horizontally through system +- API Gateway distributes to microservices +- Services connect to shared resources +- Use appropriate box widths for service names + +LEGEND/KEY (lower 30%): +- Color-coded component types +- Connection type explanations +- Status indicators + +Ensure title is well-separated from content. +Review all text against box constraints. +Include phase labels above component groups." +``` + +## Best Practices for Beautiful Compositions + +### Do's +- ✅ **ALWAYS** create dual diagrams when showing processes/flows +- ✅ Position title with generous spacing from content +- ✅ Review every text label against its container width +- ✅ Use progress indicators for multi-stage processes +- ✅ Group related components with visual phases +- ✅ Maintain consistent vertical rhythm +- ✅ Add legend/progress bar as secondary visualization +- ✅ Use gradient backgrounds for progress bars +- ✅ Keep descriptions separate and well-spaced at bottom + +### Don'ts +- ❌ Don't let text overflow boxes - adjust widths! +- ❌ Don't crowd title against diagram content +- ❌ Don't skip the progress indicator for process flows +- ❌ Don't use single diagram when dual would be clearer +- ❌ Don't forget to test text readability at different sizes +- ❌ Don't make boxes too small for their text content +- ❌ Don't position legend too close to main diagram + +## Testing Your Beautiful Composition + +Your SVG should: +1. Have TWO complementary visualizations (main + progress/legend) +2. Display title with ample separation from content +3. Fit all text comfortably within component boxes +4. Show clear visual hierarchy with phases/groupings +5. Include progress indicator for process flows +6. Position legend/progress bar with proper spacing +7. Maintain professional spacing throughout +8. Create visual rhythm with consistent element spacing +9. Work beautifully in both light and dark themes +10. Feel balanced and uncluttered + +## The Golden Rule + +**"Beautiful composition is the standard!"** - Every diagram should tell its story twice: once in the main flow, and again in the progress indicator or legend. This dual approach creates professional, scannable, and memorable visualizations. \ No newline at end of file diff --git a/rebuild_svgs.py b/rebuild_svgs.py new file mode 100644 index 000000000..caf7e0727 --- /dev/null +++ b/rebuild_svgs.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +""" +SVG Rebuilder - Converts all SVG files to match the style guide standards +Following the guidelines from botserver/prompts/dev/svg-diagram-style-guide.md +""" + +import os +import re +from pathlib import Path +from typing import Dict, List, Tuple + +# Style guide constants +COLORS = { + "blue": "#4A90E2", # Input/User elements, External/API + "orange": "#F5A623", # Processing/Scripts, Storage/Data + "purple": "#BD10E0", # AI/ML/Decision + "green": "#7ED321", # Execution/Action + "cyan": "#50E3C2", # Output/Response + "gray": "#666", # Arrows/text + "dark": "#333", # Labels +} + +SVG_TEMPLATE = """ + + + + + + + + {title} + + {content} + + + + {description} + +""" + + +def create_box(x: int, y: int, width: int, height: int, color: str, label: str) -> str: + """Create a standard box component""" + center_x = x + width // 2 + center_y = y + height // 2 + 5 + return f''' + {label}''' + + +def create_arrow( + x1: int, y1: int, x2: int, y2: int, dashed: bool = False, opacity: float = 1.0 +) -> str: + """Create an arrow between two points""" + dash_attr = ' stroke-dasharray="3,3"' if dashed else "" + opacity_attr = f' opacity="{opacity}"' if opacity < 1.0 else "" + return f'' + + +def create_curved_arrow( + points: List[Tuple[int, int]], dashed: bool = False, opacity: float = 1.0 +) -> str: + """Create a curved arrow path""" + dash_attr = ' stroke-dasharray="3,3"' if dashed else "" + opacity_attr = f' opacity="{opacity}"' if opacity < 1.0 else "" + + if len(points) < 3: + return "" + + path = f"M{points[0][0]},{points[0][1]}" + if len(points) == 3: + path += f" Q{points[1][0]},{points[1][1]} {points[2][0]},{points[2][1]}" + else: + for i in range(1, len(points)): + path += f" L{points[i][0]},{points[i][1]}" + + return f'' + + +def rebuild_conversation_flow() -> str: + """Rebuild conversation flow diagram""" + boxes = [] + arrows = [] + + # Main flow boxes + boxes.append(create_box(20, 60, 100, 40, COLORS["blue"], "User Input")) + boxes.append(create_box(160, 60, 100, 40, COLORS["orange"], "ASIC Script")) + boxes.append(create_box(300, 60, 100, 40, COLORS["purple"], "LM Decision")) + boxes.append(create_box(440, 60, 100, 40, COLORS["green"], "Bot Executor")) + boxes.append(create_box(580, 60, 100, 40, COLORS["cyan"], "Bot Response")) + + # Parallel processes + boxes.append(create_box(360, 160, 120, 40, COLORS["blue"], "Search Knowledge")) + boxes.append(create_box(500, 160, 100, 40, COLORS["orange"], "Call API")) + + # Main flow arrows + arrows.append(create_arrow(120, 80, 160, 80)) + arrows.append(create_arrow(260, 80, 300, 80)) + arrows.append(create_arrow(400, 80, 440, 80)) + arrows.append(create_arrow(540, 80, 580, 80)) + + # Branch arrows + arrows.append(create_arrow(490, 100, 420, 160, dashed=True, opacity=0.6)) + arrows.append(create_arrow(490, 100, 550, 160, dashed=True, opacity=0.6)) + + # Feedback loops + arrows.append( + create_curved_arrow( + [(420, 200), (420, 240), (630, 240), (630, 100)], dashed=True, opacity=0.4 + ) + ) + arrows.append( + create_curved_arrow( + [(550, 200), (550, 230), (620, 230), (620, 100)], dashed=True, opacity=0.4 + ) + ) + + content = ( + "\n ".join(boxes) + + '\n\n \n ' + + "\n ".join(arrows) + + "\n " + ) + + return SVG_TEMPLATE.format( + height=320, + title="The Flow", + content=content, + desc_y=300, + description="The AI handles everything else - understanding intent, collecting information, executing tools, answering from documents. Zero configuration.", + ) + + +def rebuild_architecture() -> str: + """Rebuild architecture diagram""" + boxes = [] + arrows = [] + + # Top layer + boxes.append(create_box(20, 60, 100, 40, COLORS["blue"], "Web Server")) + boxes.append(create_box(160, 60, 120, 40, COLORS["orange"], "BASIC Interpreter")) + boxes.append(create_box(320, 60, 100, 40, COLORS["purple"], "LLM Integration")) + boxes.append(create_box(460, 60, 120, 40, COLORS["green"], "Package Manager")) + boxes.append(create_box(620, 60, 100, 40, COLORS["cyan"], "Console UI")) + + # Middle layer + boxes.append( + create_box( + 250, 160, 300, 40, COLORS["blue"], "Session Manager (Tokio Async Runtime)" + ) + ) + + # Data layer + boxes.append(create_box(20, 260, 100, 40, COLORS["orange"], "PostgreSQL")) + boxes.append(create_box(160, 260, 100, 40, COLORS["purple"], "Valkey Cache")) + boxes.append(create_box(300, 260, 100, 40, COLORS["green"], "Qdrant Vectors")) + boxes.append(create_box(440, 260, 100, 40, COLORS["cyan"], "Object Storage")) + boxes.append(create_box(580, 260, 100, 40, COLORS["blue"], "Channels")) + boxes.append(create_box(700, 260, 80, 40, COLORS["orange"], "External API")) + + # Connection arrows (simplified) + for x in [70, 220, 370, 520, 670]: + arrows.append( + create_curved_arrow( + [(x, 100), (x, 130), (400, 130), (400, 160)], opacity=0.6 + ) + ) + + for x in [70, 210, 350, 490, 630]: + arrows.append(create_arrow(400, 200, x, 260, opacity=0.6)) + + # External API connection + arrows.append( + create_curved_arrow( + [(740, 260), (740, 220), (550, 180)], dashed=True, opacity=0.4 + ) + ) + + content = ( + "\n ".join(boxes) + + '\n\n \n ' + + "\n ".join(arrows) + + "\n " + ) + + # Add storage detail box + detail_box = """ + + + Storage Contents: + .gbkb (Documents) + .gbdialog (Scripts) + .gbot (Configs) + Templates + User Assets + """ + + content += detail_box + + return SVG_TEMPLATE.format( + height=400, + title="General Bots Architecture", + content=content, + desc_y=45, + description="Single binary with everything included - no external dependencies", + ) + + +def rebuild_package_system_flow() -> str: + """Rebuild package system flow diagram""" + boxes = [] + arrows = [] + + # Main flow + boxes.append(create_box(20, 60, 100, 40, COLORS["blue"], "User Request")) + boxes.append(create_box(160, 60, 100, 40, COLORS["orange"], "start.bas")) + boxes.append(create_box(300, 60, 100, 40, COLORS["purple"], "LLM Engine")) + boxes.append(create_box(440, 60, 100, 40, COLORS["cyan"], "Bot Response")) + + # Supporting components + boxes.append(create_box(240, 160, 120, 40, COLORS["blue"], "Vector Search")) + boxes.append(create_box(240, 240, 120, 40, COLORS["orange"], ".gbkb docs")) + + # Main flow arrows + arrows.append(create_arrow(120, 80, 160, 80)) + arrows.append(create_arrow(260, 80, 300, 80)) + arrows.append(create_arrow(400, 80, 440, 80)) + + # Bidirectional between start.bas and LLM + arrows.append( + create_curved_arrow( + [(210, 100), (210, 120), (300, 120), (350, 120), (350, 100)], + dashed=True, + opacity=0.6, + ) + ) + arrows.append( + create_curved_arrow( + [(350, 60), (350, 40), (260, 40), (210, 40), (210, 60)], + dashed=True, + opacity=0.6, + ) + ) + + # LLM to Vector Search + arrows.append(create_arrow(350, 100, 300, 160, opacity=0.6)) + + # Vector Search to .gbkb docs + arrows.append(create_arrow(300, 200, 300, 240, opacity=0.6)) + + # Feedback from Vector Search to LLM + arrows.append( + create_curved_arrow( + [(240, 180), (200, 140), (300, 100)], dashed=True, opacity=0.4 + ) + ) + + content = ( + "\n ".join(boxes) + + '\n\n \n ' + + "\n ".join(arrows) + + "\n " + ) + + # Add BASIC commands and package structure boxes + detail_boxes = """ + + + BASIC Commands + USE KB "docs" + answer = HEAR + result = LLM() + TALK result + + + + + Package Structure + my-bot.gbai/ + ├─ .gbdialog/ + ├─ .gbkb/ + └─ .gbot/ + """ + + content += detail_boxes + + # Add connection lines to detail boxes + content += """ + + + + """ + + # Add labels + labels = """ + Commands + Results + Query + Context""" + + content += labels + + return SVG_TEMPLATE.format( + height=400, + title="Package System Flow", + content=content, + desc_y=380, + description="BASIC scripts orchestrate LLM decisions, vector search, and responses with zero configuration", + ) + + +def main(): + """Main function to rebuild all SVGs""" + svgs_to_rebuild = { + "docs/src/assets/conversation-flow.svg": rebuild_conversation_flow(), + "docs/src/assets/architecture.svg": rebuild_architecture(), + "docs/src/assets/package-system-flow.svg": rebuild_package_system_flow(), + } + + for filepath, content in svgs_to_rebuild.items(): + full_path = Path(filepath) + if full_path.parent.exists(): + with open(full_path, "w") as f: + f.write(content) + print(f"Rebuilt: {filepath}") + else: + print(f"Skipping (directory not found): {filepath}") + + print(f"\nRebuilt {len(svgs_to_rebuild)} SVG files according to style guide") + print( + "Note: This is a demonstration script. Extend it to rebuild all 28 SVG files." + ) + + +if __name__ == "__main__": + main() diff --git a/safe_svg_improve.py b/safe_svg_improve.py new file mode 100644 index 000000000..006f994b8 --- /dev/null +++ b/safe_svg_improve.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +""" +Safe SVG Improvement Script +Enhances SVG readability for mobile devices without breaking structure +""" + +import os +import re +from pathlib import Path + + +def safe_improve_svg(filepath): + """ + Safely improve SVG file for better mobile readability + Only makes minimal, safe changes to preserve structure + """ + + try: + # Read the original file + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + # Skip font files and favicons + if "fontawesome" in str(filepath).lower() or "favicon" in str(filepath).lower(): + return False, "Skipped (font/favicon)" + + original_content = content + + # 1. Make SVG responsive by adding style attribute to svg tag if not present + if "style=" not in content.split(">")[0]: # Check only in the opening SVG tag + content = re.sub( + r"(]*)(>)", + r'\1 style="max-width: 100%; height: auto;"\2', + content, + count=1, + ) + + # 2. Increase small font sizes for mobile readability (minimum 14px) + def increase_font_size(match): + size = int(match.group(1)) + if size < 12: + return f'font-size="{14}"' + elif size == 12 or size == 13: + return f'font-size="{14}"' + else: + return match.group(0) + + content = re.sub(r'font-size="(\d+)"', increase_font_size, content) + + # 3. Improve text color contrast for better readability + # Only change very light grays to darker ones for text + text_color_improvements = { + "#CBD5E0": "#374151", # Light gray to dark gray + "#A0AEC0": "#4B5563", # Medium light gray to medium dark + "#718096": "#374151", # Another light gray to dark + "#E9D8FD": "#6B21A8", # Very light purple to dark purple + "#FBD38D": "#92400E", # Light orange to dark orange + "#90CDF4": "#1E40AF", # Light blue to dark blue + "#B2F5EA": "#047857", # Light teal to dark teal + "#9AE6B4": "#047857", # Light green to dark green + } + + for old_color, new_color in text_color_improvements.items(): + # Only replace in text elements + content = re.sub( + f'(]*fill="){old_color}(")', + f"\\1{new_color}\\2", + content, + flags=re.IGNORECASE, + ) + + # 4. Ensure stroke widths are visible (minimum 2) + content = re.sub(r'stroke-width="1"', 'stroke-width="2"', content) + content = re.sub(r'stroke-width="0\.5"', 'stroke-width="2"', content) + + # 5. Add rounded corners to rectangles if missing (but small radius) + def add_rounded_corners(match): + rect = match.group(0) + if "rx=" not in rect and 'fill="none"' in rect: + # Add small rounded corners for better aesthetics + rect = rect[:-2] + ' rx="4"/>' + return rect + + content = re.sub(r"]*/>", add_rounded_corners, content) + + # 6. Make arrow markers more visible + content = re.sub(r'fill="#888"', 'fill="#374151"', content) + + # 7. Improve font families for better cross-platform rendering + content = re.sub( + r'font-family="Arial, sans-serif"', + "font-family=\"-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif\"", + content, + ) + + # 8. Fix font weight declarations + content = re.sub(r'font-weight="bold"', 'font-weight="600"', content) + + # Only write if changes were made + if content != original_content: + # Backup original + backup_path = str(filepath) + ".backup" + if not os.path.exists(backup_path): + with open(backup_path, "w", encoding="utf-8") as f: + f.write(original_content) + + # Write improved version + with open(filepath, "w", encoding="utf-8") as f: + f.write(content) + + return True, "Improved successfully" + else: + return False, "No changes needed" + + except Exception as e: + return False, f"Error: {str(e)}" + + +def main(): + """Process all SVG files in docs directory""" + + print("=" * 60) + print("SAFE SVG IMPROVEMENT SCRIPT") + print("Enhancing readability without breaking structure") + print("=" * 60) + print() + + docs_dir = Path("docs") + svg_files = list(docs_dir.glob("**/*.svg")) + + print(f"Found {len(svg_files)} SVG files") + print() + + improved = 0 + skipped = 0 + unchanged = 0 + errors = 0 + + for svg_file in svg_files: + print(f"Processing: {svg_file}") + + success, message = safe_improve_svg(svg_file) + + if success: + print(f" ✓ {message}") + improved += 1 + elif "Skipped" in message: + print(f" ⊘ {message}") + skipped += 1 + elif "No changes" in message: + print(f" - {message}") + unchanged += 1 + else: + print(f" ✗ {message}") + errors += 1 + + print() + print("=" * 60) + print("SUMMARY") + print("=" * 60) + print(f"✓ Improved: {improved} files") + print(f"- Unchanged: {unchanged} files") + print(f"⊘ Skipped: {skipped} files") + if errors > 0: + print(f"✗ Errors: {errors} files") + + print() + print("Safe improvements applied:") + print("• Increased minimum font size to 14px") + print("• Improved text color contrast") + print("• Made SVGs responsive (100% width)") + print("• Enhanced stroke visibility") + print("• Added subtle rounded corners") + print("• Improved font families for all devices") + print() + print("Original files backed up with .backup extension") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/validate_svgs.py b/validate_svgs.py new file mode 100644 index 000000000..ba61988d4 --- /dev/null +++ b/validate_svgs.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +""" +SVG Validation and Documentation Mapping Script +Checks all SVG files for readability issues and shows where they're used in the documentation +""" + +import os +import re +from collections import defaultdict +from pathlib import Path + + +def analyze_svg(filepath): + """Analyze an SVG file for potential readability issues""" + issues = [] + info = {} + + try: + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + # Check file size + file_size = os.path.getsize(filepath) + info["size"] = f"{file_size:,} bytes" + + # Extract viewBox/dimensions + viewbox_match = re.search(r'viewBox="([^"]+)"', content) + width_match = re.search(r'width="(\d+)"', content) + height_match = re.search(r'height="(\d+)"', content) + + if viewbox_match: + info["viewBox"] = viewbox_match.group(1) + elif width_match and height_match: + info["dimensions"] = f"{width_match.group(1)}x{height_match.group(1)}" + + # Check if responsive + if 'style="max-width: 100%' in content: + info["responsive"] = "✓" + else: + info["responsive"] = "✗" + issues.append("Not responsive (missing max-width: 100%)") + + # Find all font sizes + font_sizes = re.findall(r'font-size="(\d+)"', content) + if font_sizes: + sizes = [int(s) for s in font_sizes] + info["font_sizes"] = f"min:{min(sizes)}px, max:{max(sizes)}px" + + # Check for too small fonts + small_fonts = [s for s in sizes if s < 12] + if small_fonts: + issues.append( + f"Small fonts found: {small_fonts}px (mobile needs ≥14px)" + ) + + # Check text colors for contrast + text_colors = re.findall(r']*fill="([^"]+)"', content) + light_colors = [] + for color in text_colors: + if any( + light in color.upper() + for light in [ + "#CBD5E0", + "#A0AEC0", + "#E2E8F0", + "#EDF2F7", + "#F7FAFC", + "#9CA3AF", + "#D1D5DB", + ] + ): + light_colors.append(color) + + if light_colors: + unique_colors = list(set(light_colors)) + issues.append(f"Low contrast text colors: {', '.join(unique_colors[:3])}") + + # Check for background + if ( + 'fill="#FAFAFA"' in content + or 'fill="white"' in content + or 'fill="#FFFFFF"' in content + ): + if re.search( + r']*width="[^"]*"[^>]*height="[^"]*"[^>]*fill="(white|#FAFAFA|#FFFFFF)"', + content, + ): + issues.append("Has white/light background") + + # Count elements + info["texts"] = content.count(" 5: + print(f" ... and {len(references[svg_name]) - 5} more") + else: + print(f"\n ❓ No references found in documentation") + + # Summary + print("\n" + "=" * 80) + print("SUMMARY") + print("=" * 80) + print(f"Total SVG files analyzed: {len(svg_files)}") + print(f"Total issues found: {total_issues}") + + if total_issues > 0: + print("\n🔧 RECOMMENDED FIXES:") + print("1. Increase all font sizes to minimum 14px for mobile readability") + print("2. Replace light gray text colors with darker ones for better contrast") + print("3. Remove white backgrounds or make them transparent") + print("4. Add responsive styling (max-width: 100%; height: auto)") + print("5. Consider using system fonts for better cross-platform support") + + +if __name__ == "__main__": + main()