diff --git a/beautify_all_svgs.py b/beautify_all_svgs.py deleted file mode 100644 index 7499c458..00000000 --- a/beautify_all_svgs.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env python3 -""" -SVG Beautifier - Updates all SVG diagrams for perfect readability on all devices -with beautiful colors that work in both color and black/white modes. -""" - -import os -import re -from pathlib import Path - -# Beautiful color palette that works in grayscale -COLORS = { - # Primary colors with good contrast - 'primary_blue': '#2563EB', # Bright blue - appears as dark gray in B&W - 'primary_green': '#059669', # Emerald green - medium gray in B&W - 'primary_purple': '#7C3AED', # Purple - medium-dark gray in B&W - 'primary_orange': '#EA580C', # Orange - medium gray in B&W - 'primary_red': '#DC2626', # Red - dark gray in B&W - 'primary_teal': '#0891B2', # Teal - medium gray in B&W - - # Text colors for maximum readability - 'text_primary': '#1F2937', # Almost black - perfect for main text - 'text_secondary': '#4B5563', # Dark gray - for secondary text - 'text_accent': '#2563EB', # Blue for emphasis - dark in B&W - - # Background and border colors - 'bg_light': '#F9FAFB', # Very light gray background - 'border_primary': '#2563EB', # Blue borders - visible in B&W - 'border_secondary': '#9CA3AF', # Gray borders - - # Status colors - 'success': '#059669', # Green - medium gray in B&W - 'warning': '#EA580C', # Orange - medium gray in B&W - 'error': '#DC2626', # Red - dark gray in B&W - 'info': '#2563EB', # Blue - dark gray in B&W -} - -# Consistent font sizes for all devices (matching documentation) -FONT_SIZES = { - 'title': '24', # Main diagram titles - 'subtitle': '20', # Section titles - 'heading': '18', # Component headings - 'body': '16', # Main text (matches doc font size) - 'label': '14', # Labels and annotations - 'small': '12', # Small details (minimum for mobile) -} - -# Standard margins and padding -LAYOUT = { - 'margin': 40, # Outer margin - 'padding': 20, # Inner padding - 'spacing': 15, # Element spacing - 'corner_radius': 8, # Rounded corners -} - -def create_improved_svg(content, filename): - """ - Transform SVG content with improved styling for all devices. - """ - - # Extract viewBox or width/height - viewbox_match = re.search(r'viewBox="([^"]+)"', content) - width_match = re.search(r'width="(\d+)"', content) - height_match = re.search(r'height="(\d+)"', content) - - if viewbox_match: - viewbox = viewbox_match.group(1) - vb_parts = viewbox.split() - width = int(vb_parts[2]) - height = int(vb_parts[3]) - elif width_match and height_match: - width = int(width_match.group(1)) - height = int(height_match.group(1)) - else: - width, height = 800, 600 # Default size - - # Add responsive margins - new_width = width + (LAYOUT['margin'] * 2) - new_height = height + (LAYOUT['margin'] * 2) - - # Create new SVG header with responsive sizing - new_header = f''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ''' - - # Process the content - content = re.sub(r']*>', '', content) - content = re.sub(r'', '', content) - - # Update font sizes to be mobile-friendly and consistent - content = re.sub(r'font-size="(\d+)"', lambda m: update_font_size(m), content) - content = re.sub(r'font-size:\s*(\d+)(?:px)?', lambda m: f"font-size:{update_font_size_style(m)}", content) - - # Update text colors for better contrast - content = re.sub(r'fill="#[A-Fa-f0-9]{6}"', lambda m: update_text_color(m), content) - content = re.sub(r'stroke="#[A-Fa-f0-9]{6}"', lambda m: update_stroke_color(m), content) - - # Improve rectangles with better styling - content = re.sub(r']+)>', lambda m: improve_rect(m), content) - - # Update text elements with better positioning and styling - content = re.sub(r']*)>(.*?)', lambda m: improve_text(m), content) - - # Add font family consistency - content = re.sub(r'font-family="[^"]*"', - 'font-family="-apple-system, BlinkMacSystemFont, \'Segoe UI\', Roboto, \'Helvetica Neue\', Arial, sans-serif"', - content) - - # Close the container and SVG - new_footer = ''' - -''' - - return new_header + content + new_footer - -def update_font_size(match): - """Update font sizes to be mobile-friendly.""" - size = int(match.group(1)) - if size >= 20: - return f'font-size="{FONT_SIZES["title"]}"' - elif size >= 16: - return f'font-size="{FONT_SIZES["heading"]}"' - elif size >= 14: - return f'font-size="{FONT_SIZES["body"]}"' - elif size >= 12: - return f'font-size="{FONT_SIZES["label"]}"' - else: - return f'font-size="{FONT_SIZES["small"]}"' - -def update_font_size_style(match): - """Update font sizes in style attributes.""" - size = int(match.group(1)) - if size >= 20: - return FONT_SIZES["title"] - elif size >= 16: - return FONT_SIZES["heading"] - elif size >= 14: - return FONT_SIZES["body"] - elif size >= 12: - return FONT_SIZES["label"] - else: - return FONT_SIZES["small"] - -def update_text_color(match): - """Update text fill colors for better contrast.""" - color = match.group(0) - # Check if it's a light color (rough heuristic) - if any(light in color.lower() for light in ['fff', 'fef', 'efe', 'fee', 'eee', 'ddd', 'ccc']): - return f'fill="{COLORS["text_primary"]}"' - # Keep dark colors but ensure they're dark enough - elif any(dark in color.lower() for dark in ['000', '111', '222', '333', '444']): - return f'fill="{COLORS["text_primary"]}"' - else: - # For other colors, use our palette - return f'fill="{COLORS["text_secondary"]}"' - -def update_stroke_color(match): - """Update stroke colors to use our palette.""" - color = match.group(0) - # Map to our color palette for consistency - if 'blue' in color.lower() or '4a90e2' in color.lower() or '63b3ed' in color.lower(): - return f'stroke="{COLORS["primary_blue"]}"' - elif 'green' in color.lower() or '48bb78' in color.lower() or '68d391' in color.lower(): - return f'stroke="{COLORS["primary_green"]}"' - elif 'purple' in color.lower() or 'b794f4' in color.lower() or '9f7aea' in color.lower(): - return f'stroke="{COLORS["primary_purple"]}"' - elif 'orange' in color.lower() or 'f6ad55' in color.lower() or 'ed8936' in color.lower(): - return f'stroke="{COLORS["primary_orange"]}"' - elif 'red' in color.lower() or 'e53e3e' in color.lower() or 'fc8181' in color.lower(): - return f'stroke="{COLORS["primary_red"]}"' - else: - return f'stroke="{COLORS["border_primary"]}"' - -def improve_rect(match): - """Improve rectangle elements with better styling.""" - rect = match.group(0) - # Add rounded corners if not present - if 'rx=' not in rect: - rect = rect[:-1] + f' rx="{LAYOUT["corner_radius"]}">' - # Add subtle shadow for depth - if 'filter=' not in rect: - rect = rect[:-1] + ' filter="url(#shadow)">' - # Ensure proper stroke width - rect = re.sub(r'stroke-width="[^"]*"', 'stroke-width="2"', rect) - return rect - -def improve_text(match): - """Improve text elements with better styling.""" - text_tag = match.group(1) - text_content = match.group(2) - - # Add text shadow for better readability - if 'filter=' not in text_tag: - text_tag += ' filter="url(#textShadow)"' - - # Ensure text has proper weight for readability - if 'font-weight=' not in text_tag and any(word in text_content.lower() for word in ['title', 'process', 'flow', 'system']): - text_tag += ' font-weight="600"' - - return f'{text_content}' - -def process_all_svgs(): - """Process all SVG files in the docs directory.""" - docs_dir = Path('docs') - - # Find all SVG files - svg_files = list(docs_dir.glob('**/*.svg')) - - print(f"Found {len(svg_files)} SVG files to beautify") - - for svg_file in svg_files: - # Skip font files - if 'fontawesome' in str(svg_file).lower() or 'favicon' in str(svg_file).lower(): - print(f"Skipping font/favicon file: {svg_file}") - continue - - print(f"Beautifying: {svg_file}") - - try: - # Read the original content - with open(svg_file, 'r', encoding='utf-8') as f: - content = f.read() - - # Skip if already processed - if 'Beautiful gradient definitions' in content: - print(f" Already beautified, skipping...") - continue - - # Create improved version - improved = create_improved_svg(content, svg_file.name) - - # Save the improved version - with open(svg_file, 'w', encoding='utf-8') as f: - f.write(improved) - - print(f" ✓ Successfully beautified!") - - except Exception as e: - print(f" ✗ Error processing {svg_file}: {e}") - -if __name__ == "__main__": - print("=" * 60) - print("SVG BEAUTIFIER - Making diagrams beautiful for all devices") - print("=" * 60) - print("\nFeatures:") - print("• Consistent text sizing matching documentation") - print("• Proper margins and padding for mobile") - print("• Beautiful colors that work in black & white") - print("• Responsive design for all screen sizes") - print("• Enhanced readability with shadows and gradients") - print("\nStarting beautification process...\n") - - process_all_svgs() - - print("\n" + "=" * 60) - print("✨ Beautification complete!") - print("All SVGs now have:") - print("• Mobile-friendly text sizes (min 12px)") - print("• Consistent font family") - print("• Proper margins (40px) and padding (20px)") - print("• High contrast colors readable in B&W") - print("• Responsive viewBox settings") - print("=" * 60) diff --git a/book.toml b/book.toml deleted file mode 100644 index c4a39d03..00000000 --- a/book.toml +++ /dev/null @@ -1,54 +0,0 @@ -[book] -title = "BotServer Documentation" -authors = ["BotServer Team"] -language = "en" -multilingual = false -src = "docs/src" -description = "Documentation for BotServer - A flexible bot framework" - -[build] -build-dir = "docs/book" - -[output.html] -smart-punctuation = true -mathjax-support = false -copy-fonts = true -no-section-label = false -additional-css = ["docs/src/custom.css"] -additional-js = ["docs/src/theme-sync.js"] -git-repository-url = "https://github.com/yourusername/botserver" -git-repository-icon = "fa-github" -edit-url-template = "https://github.com/yourusername/botserver/edit/main/{path}" -site-url = "/botserver/" -cname = "" -input-404 = "" - -[output.html.print] -enable = true - -[output.html.fold] -enable = false -level = 0 - -[output.html.playground] -editable = false -copyable = true -copy-js = true -line-numbers = false -runnable = false - -[output.html.search] -enable = true -limit-results = 30 -teaser-word-count = 30 -use-boolean-and = true -boost-title = 2 -boost-hierarchy = 1 -boost-paragraph = 1 -expand = true -heading-split-level = 3 -copy-js = true - -[preprocessor.index] - -[preprocessor.links] diff --git a/fix_all_svgs.py b/fix_all_svgs.py deleted file mode 100644 index 32f5ad43..00000000 --- a/fix_all_svgs.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix and beautify all SVG files with proper syntax and mobile-friendly design -""" - -import os -import re -import xml.etree.ElementTree as ET -from pathlib import Path - - -def fix_svg_file(filepath): - """Fix a single SVG file with proper formatting and mobile-friendly design""" - - try: - # Read the original file - with open(filepath, "r", encoding="utf-8") as f: - content = f.read() - - # Skip font files and favicons - if "fontawesome" in str(filepath).lower() or "favicon" in str(filepath).lower(): - return False - - print(f"Fixing: {filepath}") - - # First, clean up any broken attributes - # Remove any malformed style attributes - content = re.sub(r'style="[^"]*"[^>]*style="[^"]*"', "", content) - - # Fix basic SVG structure - if not content.strip().startswith("\n' + content - - # Extract dimensions - width_match = re.search(r'width="(\d+)"', content) - height_match = re.search(r'height="(\d+)"', content) - viewbox_match = re.search(r'viewBox="([^"]+)"', content) - - if viewbox_match: - viewbox = viewbox_match.group(1) - elif width_match and height_match: - width = width_match.group(1) - height = height_match.group(1) - viewbox = f"0 0 {width} {height}" - else: - viewbox = "0 0 800 600" - - # Create clean SVG header - svg_header = f''' - - - - - - - - - - - - - - - - - - - - -''' - - # Extract the main content (remove old svg tags and defs) - main_content = re.sub(r"<\?xml[^>]*\?>", "", content) - main_content = re.sub(r"]*>", "", main_content) - main_content = re.sub(r"", "", main_content) - main_content = re.sub(r".*?", "", main_content, flags=re.DOTALL) - - # Fix font sizes for mobile (minimum 14px for body text) - def fix_font_size(match): - size = int(match.group(1)) - if size < 12: - return f'font-size="{14}"' - elif size < 14: - return f'font-size="{14}"' - elif size > 24: - return f'font-size="{24}"' - else: - return match.group(0) - - main_content = re.sub(r'font-size="(\d+)"', fix_font_size, main_content) - - # Fix colors for better contrast - color_map = { - # Blues - "#63B3ED": "#2563EB", - "#90CDF4": "#3B82F6", - "#4A90E2": "#2563EB", - "#CBD5E0": "#1F2937", # Light gray text to dark - "#A0AEC0": "#4B5563", # Medium gray text to darker - # Greens - "#68D391": "#059669", - "#48BB78": "#10B981", - "#38A169": "#059669", - "#9AE6B4": "#10B981", - # Purples - "#B794F4": "#7C3AED", - "#D6BCFA": "#8B5CF6", - "#9F7AEA": "#7C3AED", - "#E9D8FD": "#8B5CF6", - # Oranges - "#F6AD55": "#EA580C", - "#FBD38D": "#F97316", - "#ED8936": "#EA580C", - # Reds - "#FC8181": "#DC2626", - "#FEB2B2": "#EF4444", - "#E53E3E": "#DC2626", - # Teals - "#4FD1C5": "#0891B2", - "#81E6D9": "#06B6D4", - "#38D4B2": "#0891B2", - "#B2F5EA": "#06B6D4", - # Grays - "#4A5568": "#6B7280", - "#718096": "#6B7280", - "#888": "#6B7280", - } - - for old_color, new_color in color_map.items(): - main_content = main_content.replace( - f'fill="{old_color}"', f'fill="{new_color}"' - ) - main_content = main_content.replace( - f'stroke="{old_color}"', f'stroke="{new_color}"' - ) - main_content = main_content.replace( - f'fill="{old_color.lower()}"', f'fill="{new_color}"' - ) - main_content = main_content.replace( - f'stroke="{old_color.lower()}"', f'stroke="{new_color}"' - ) - - # Fix font families - main_content = re.sub( - r'font-family="[^"]*"', - 'font-family="system-ui, -apple-system, sans-serif"', - main_content, - ) - - # Ensure stroke widths are visible - main_content = re.sub(r'stroke-width="1"', 'stroke-width="2"', main_content) - - # Add rounded corners to rectangles - def add_rounded_corners(match): - rect = match.group(0) - if "rx=" not in rect: - rect = rect[:-1] + ' rx="6"/>' - return rect - - main_content = re.sub(r"]*/>", add_rounded_corners, main_content) - - # Combine everything - final_svg = svg_header + main_content + "\n" - - # Write the fixed file - with open(filepath, "w", encoding="utf-8") as f: - f.write(final_svg) - - print(f" ✓ Fixed successfully") - return True - - except Exception as e: - print(f" ✗ Error: {e}") - return False - - -def main(): - """Fix all SVG files in the docs directory""" - - print("=" * 60) - print("SVG FIXER - Repairing and beautifying all diagrams") - print("=" * 60) - print() - - docs_dir = Path("docs") - svg_files = list(docs_dir.glob("**/*.svg")) - - print(f"Found {len(svg_files)} SVG files") - print() - - fixed_count = 0 - skipped_count = 0 - error_count = 0 - - for svg_file in svg_files: - if "fontawesome" in str(svg_file).lower() or "favicon" in str(svg_file).lower(): - print(f"Skipping: {svg_file} (font/favicon)") - skipped_count += 1 - continue - - result = fix_svg_file(svg_file) - if result: - fixed_count += 1 - else: - error_count += 1 - - print() - print("=" * 60) - print("SUMMARY") - print("=" * 60) - print(f"✓ Fixed: {fixed_count} files") - print(f"⊘ Skipped: {skipped_count} files") - if error_count > 0: - print(f"✗ Errors: {error_count} files") - print() - print("All SVG files now have:") - print("• Mobile-friendly text sizes (min 14px)") - print("• High contrast colors") - print("• Consistent fonts") - print("• Rounded corners") - print("• Proper stroke widths") - print("=" * 60) - - -if __name__ == "__main__": - main() diff --git a/fix_svg_attributes.py b/fix_svg_attributes.py deleted file mode 100644 index b02ef6f5..00000000 --- a/fix_svg_attributes.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix malformed SVG attributes in all SVG files -Specifically fixes the 'rx="5"/' issue and other attribute errors -""" - -import os -import re -from pathlib import Path - - -def fix_malformed_attributes(content): - """Fix various types of malformed attributes in SVG content""" - - # Fix malformed rx attributes (rx="5"/ should be rx="5") - content = re.sub(r'rx="([^"]+)"\s*/', r'rx="\1"', content) - - # Fix malformed ry attributes - content = re.sub(r'ry="([^"]+)"\s*/', r'ry="\1"', content) - - # Fix cases where filter appears after malformed rx - content = re.sub( - r'rx="([^"]+)"/\s*filter="([^"]+)"', r'rx="\1" filter="\2"', content - ) - - # Fix double closing brackets - content = re.sub(r"/>>", r"/>", content) - - # Fix attributes that got split incorrectly - content = re.sub(r'"\s+([a-z-]+)="', r'" \1="', content) - - # Fix rect tags with malformed endings - content = re.sub( - r']+)"\s*/\s+([a-z-]+)="([^"]+)">', r'', content - ) - - # Fix specific pattern: stroke-width="2" rx="5"/ filter="url(#shadow)"> - content = re.sub( - r'stroke-width="(\d+)"\s+rx="(\d+)"/\s*filter="([^"]+)">', - r'stroke-width="\1" rx="\2" filter="\3">', - content, - ) - - # Fix any remaining "/ patterns at the end of attributes - content = re.sub(r'="([^"]*)"\s*/', r'="\1"', content) - - # Fix rectangles that should be self-closing - lines = content.split("\n") - fixed_lines = [] - - for line in lines: - # If it's a rect element that ends with > but has no content, make it self-closing - if ( - "") - and not line.strip().endswith("/>") - ): - # Check if this rect has content after it or should be self-closing - if ( - 'fill="none"' in line - or 'fill="transparent"' in line - or 'fill="white"' in line - ): - line = line.rstrip(">") + "/>" - fixed_lines.append(line) - - content = "\n".join(fixed_lines) - - return content - - -def validate_svg_structure(content): - """Basic validation to ensure SVG structure is correct""" - - # Check for basic SVG structure - if "" not in content: - return False, "Missing closing SVG tag" - - # Count opening and closing tags for basic elements - rect_open = content.count("") - rect_self = content.count("/>") - - # Basic tag balance check (not perfect but catches major issues) - text_open = content.count("") - - if text_open != text_close: - return False, f"Text tag mismatch: {text_open} opening vs {text_close} closing" - - # Check for common malformed patterns - if "/ " in content and "filter=" in content: - malformed = re.findall(r'rx="[^"]+"/\s*filter=', content) - if malformed: - return False, f"Found malformed attribute pattern" - - return True, "OK" - - -def fix_svg_file(filepath): - """Fix a single SVG file""" - - try: - # Read the file - with open(filepath, "r", encoding="utf-8") as f: - content = f.read() - - # Skip font files and favicons - if "fontawesome" in str(filepath).lower() or "favicon" in str(filepath).lower(): - return "skipped", None - - # Apply fixes - fixed_content = fix_malformed_attributes(content) - - # Validate the result - is_valid, message = validate_svg_structure(fixed_content) - - if not is_valid: - print(f" ⚠ Validation warning: {message}") - - # Write back only if content changed - if fixed_content != content: - with open(filepath, "w", encoding="utf-8") as f: - f.write(fixed_content) - return "fixed", None - else: - return "unchanged", None - - except Exception as e: - return "error", str(e) - - -def main(): - """Fix all SVG files in the docs directory""" - - print("=" * 60) - print("SVG ATTRIBUTE FIXER") - print("Fixing malformed attributes in all SVG files") - print("=" * 60) - print() - - docs_dir = Path("docs") - svg_files = list(docs_dir.glob("**/*.svg")) - - print(f"Found {len(svg_files)} SVG files") - print() - - stats = {"fixed": 0, "unchanged": 0, "skipped": 0, "error": 0} - - for svg_file in svg_files: - print(f"Processing: {svg_file}") - - status, error = fix_svg_file(svg_file) - stats[status] += 1 - - if status == "fixed": - print(f" ✓ Fixed malformed attributes") - elif status == "unchanged": - print(f" - No changes needed") - elif status == "skipped": - print(f" ⊘ Skipped (font/favicon)") - elif status == "error": - print(f" ✗ Error: {error}") - - print() - print("=" * 60) - print("SUMMARY") - print("=" * 60) - print(f"✓ Fixed: {stats['fixed']} files") - print(f"- Unchanged: {stats['unchanged']} files") - print(f"⊘ Skipped: {stats['skipped']} files") - if stats["error"] > 0: - print(f"✗ Errors: {stats['error']} files") - - print() - print("Common fixes applied:") - print('• Fixed malformed rx="5"/ attributes') - print("• Corrected filter attribute placement") - print("• Fixed self-closing tags") - print("• Cleaned up attribute spacing") - print("=" * 60) - - -if __name__ == "__main__": - main() diff --git a/fix_svgs_properly.py b/fix_svgs_properly.py deleted file mode 100644 index 1baf32e6..00000000 --- a/fix_svgs_properly.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix all SVG files to be properly readable on all devices -Focus on text size, contrast, and responsive design -""" - -import os -import re -from pathlib import Path - - -def fix_svg_content(content, filename): - """ - Apply comprehensive fixes to SVG content - """ - - # 1. Fix SVG header for responsiveness - # Remove any width/height attributes and ensure proper viewBox - if "]*>" - svg_replacement = f'' - content = re.sub(svg_pattern, svg_replacement, content, count=1) - - # 2. Fix all font sizes - MINIMUM 16px for readability - def increase_font_size(match): - size = int(match.group(1)) - if size < 14: - return f'font-size="16"' - elif size < 16: - return f'font-size="18"' - elif size < 18: - return f'font-size="20"' - else: - return f'font-size="{size + 4}"' # Increase all fonts slightly - - content = re.sub(r'font-size="(\d+)"', increase_font_size, content) - - # 3. Fix ALL text colors for maximum contrast - # Replace all light colors with dark, readable ones - color_replacements = { - # Light grays to dark gray/black - "#CBD5E0": "#1F2937", - "#A0AEC0": "#374151", - "#E2E8F0": "#1F2937", - "#EDF2F7": "#111827", - "#F7FAFC": "#111827", - "#9CA3AF": "#374151", - "#D1D5DB": "#4B5563", - "#718096": "#374151", - "#4A5568": "#1F2937", - # Light blues to dark blues - "#90CDF4": "#1E40AF", - "#63B3ED": "#2563EB", - "#4A90E2": "#1D4ED8", - "#81E6D9": "#0E7490", - "#4FD1C5": "#0891B2", - "#38D4B2": "#0D9488", - # Light purples to dark purples - "#E9D8FD": "#6B21A8", - "#D6BCFA": "#7C3AED", - "#B794F4": "#9333EA", - "#9F7AEA": "#7C3AED", - # Light oranges to dark oranges - "#FBD38D": "#C2410C", - "#F6AD55": "#EA580C", - "#ED8936": "#C2410C", - # Light reds to dark reds - "#FEB2B2": "#B91C1C", - "#FC8181": "#DC2626", - "#E53E3E": "#DC2626", - # Light greens to dark greens - "#9AE6B4": "#047857", - "#68D391": "#059669", - "#48BB78": "#047857", - "#38A169": "#059669", - "#B2F5EA": "#047857", - # Generic light to dark - "#888": "#374151", - "#888888": "#374151", - "#FAFAFA": "transparent", - "#fff": "#111827", - "#ffffff": "#111827", - "#FFF": "#111827", - "#FFFFFF": "#111827", - } - - for old_color, new_color in color_replacements.items(): - # Replace in fill attributes - content = re.sub( - f'fill="{old_color}"', f'fill="{new_color}"', content, flags=re.IGNORECASE - ) - # Replace in stroke attributes - content = re.sub( - f'stroke="{old_color}"', - f'stroke="{new_color}"', - content, - flags=re.IGNORECASE, - ) - # Replace in style attributes - content = re.sub( - f"fill:{old_color}", f"fill:{new_color}", content, flags=re.IGNORECASE - ) - content = re.sub( - f"stroke:{old_color}", f"stroke:{new_color}", content, flags=re.IGNORECASE - ) - - # 4. Remove white/light backgrounds - content = re.sub(r']*fill="#FAFAFA"[^>]*>', "", content) - content = re.sub(r']*fill="white"[^>]*>', "", content) - content = re.sub(r']*fill="#FFFFFF"[^>]*>', "", content) - content = re.sub(r']*fill="#ffffff"[^>]*>', "", content) - - # 5. Fix stroke widths for visibility - content = re.sub(r'stroke-width="1"', 'stroke-width="2"', content) - content = re.sub(r'stroke-width="0\.5"', 'stroke-width="2"', content) - - # 6. Fix font weights - content = re.sub(r'font-weight="bold"', 'font-weight="700"', content) - - # 7. Fix font families for better rendering - content = re.sub( - r'font-family="[^"]*"', - 'font-family="system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, sans-serif"', - content, - ) - - # 8. Ensure arrows and markers are visible - content = re.sub( - r'', - '', - content, - ) - - # 9. Add slight padding to the viewBox if needed - viewbox_match = re.search(r'viewBox="(\d+)\s+(\d+)\s+(\d+)\s+(\d+)"', content) - if viewbox_match: - x, y, width, height = map(int, viewbox_match.groups()) - # Add 20px padding - new_viewbox = f'viewBox="{x - 20} {y - 20} {width + 40} {height + 40}"' - content = re.sub(r'viewBox="[^"]*"', new_viewbox, content, count=1) - - return content - - -def process_all_svgs(): - """Process all SVG files in the docs directory""" - - docs_dir = Path("docs") - svg_files = list(docs_dir.glob("**/*.svg")) - - # Filter out font files - svg_files = [ - f - for f in svg_files - if "fontawesome" not in str(f).lower() and "favicon" not in str(f).lower() - ] - - print(f"Found {len(svg_files)} SVG files to fix") - print("=" * 60) - - fixed = 0 - errors = 0 - - for svg_file in svg_files: - try: - print(f"Processing: {svg_file.relative_to(docs_dir)}") - - # Read the file - with open(svg_file, "r", encoding="utf-8") as f: - content = f.read() - - # Apply fixes - fixed_content = fix_svg_content(content, svg_file.name) - - # Write back - with open(svg_file, "w", encoding="utf-8") as f: - f.write(fixed_content) - - print(f" ✓ Fixed successfully") - fixed += 1 - - except Exception as e: - print(f" ✗ Error: {e}") - errors += 1 - - print() - print("=" * 60) - print(f"COMPLETED: {fixed} files fixed, {errors} errors") - print() - print("Improvements applied:") - print("• All text now ≥16px (readable on mobile)") - print("• High contrast colors (dark text, no light grays)") - print("• 100% responsive width") - print("• Removed white backgrounds") - print("• Enhanced stroke widths") - print("• Added padding to prevent cutoff") - print("=" * 60) - - -if __name__ == "__main__": - print("=" * 60) - print("SVG READABILITY FIXER") - print("Making all diagrams actually readable!") - print("=" * 60) - print() - process_all_svgs() diff --git a/minimal_svg_fix.py b/minimal_svg_fix.py deleted file mode 100644 index c113f25f..00000000 --- a/minimal_svg_fix.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -""" -Minimal SVG fix - ONLY fixes text size and contrast -No structural changes, no breaking modifications -""" - -import re -from pathlib import Path - - -def minimal_fix_svg(filepath): - """Apply minimal fixes to make SVG text readable""" - - with open(filepath, "r", encoding="utf-8") as f: - content = f.read() - - # Skip font files - if "fontawesome" in str(filepath).lower() or "favicon" in str(filepath).lower(): - return False - - # 1. Increase font sizes (minimum 16px for readability) - def fix_font_size(match): - size = int(match.group(1)) - if size <= 11: - return f'font-size="16"' - elif size == 12: - return f'font-size="18"' - elif size <= 14: - return f'font-size="20"' - else: - return f'font-size="{size + 4}"' - - content = re.sub(r'font-size="(\d+)"', fix_font_size, content) - - # 2. Fix text colors for contrast - # Light grays to dark - content = content.replace('fill="#CBD5E0"', 'fill="#1F2937"') - content = content.replace('fill="#A0AEC0"', 'fill="#374151"') - content = content.replace('fill="#718096"', 'fill="#374151"') - content = content.replace('fill="#E2E8F0"', 'fill="#1F2937"') - - # Light blues to darker blues - content = content.replace('fill="#90CDF4"', 'fill="#1E40AF"') - content = content.replace('fill="#63B3ED"', 'fill="#2563EB"') - - # Light purples to darker - content = content.replace('fill="#E9D8FD"', 'fill="#7C3AED"') - content = content.replace('fill="#D6BCFA"', 'fill="#9333EA"') - content = content.replace('fill="#B794F4"', 'fill="#9333EA"') - - # Light oranges to darker - content = content.replace('fill="#FBD38D"', 'fill="#EA580C"') - content = content.replace('fill="#F6AD55"', 'fill="#D97706"') - - # Light reds to darker - content = content.replace('fill="#FEB2B2"', 'fill="#DC2626"') - content = content.replace('fill="#FC8181"', 'fill="#EF4444"') - - # Light greens stay green (they're usually OK) - # But make them slightly darker - content = content.replace('fill="#9AE6B4"', 'fill="#10B981"') - content = content.replace('fill="#68D391"', 'fill="#059669"') - content = content.replace('fill="#48BB78"', 'fill="#047857"') - - # Light teals - content = content.replace('fill="#81E6D9"', 'fill="#0891B2"') - content = content.replace('fill="#4FD1C5"', 'fill="#0891B2"') - content = content.replace('fill="#B2F5EA"', 'fill="#0E7490"') - - # Gray arrows - content = content.replace('fill="#888"', 'fill="#4B5563"') - - # 3. Make SVG responsive (add style attribute if missing) - if "")[0]: - content = re.sub( - r"(]*)(>)", - r'\1 style="max-width: 100%; height: auto;"\2', - content, - count=1, - ) - - # Write the fixed content - with open(filepath, "w", encoding="utf-8") as f: - f.write(content) - - return True - - -def main(): - """Fix all SVG files in docs/src""" - - docs_src = Path("docs/src") - svg_files = list(docs_src.rglob("*.svg")) - - print(f"Fixing {len(svg_files)} SVG files...") - - fixed = 0 - for svg_file in svg_files: - try: - if minimal_fix_svg(svg_file): - print(f"✓ {svg_file.name}") - fixed += 1 - except Exception as e: - print(f"✗ {svg_file.name}: {e}") - - print(f"\nFixed {fixed} files") - print("Changes made:") - print("• Font sizes increased (16px minimum)") - print("• Text colors darkened for contrast") - print("• SVGs made responsive") - - -if __name__ == "__main__": - main() diff --git a/rebuild_svgs.py b/rebuild_svgs.py deleted file mode 100644 index caf7e072..00000000 --- a/rebuild_svgs.py +++ /dev/null @@ -1,335 +0,0 @@ -#!/usr/bin/env python3 -""" -SVG Rebuilder - Converts all SVG files to match the style guide standards -Following the guidelines from botserver/prompts/dev/svg-diagram-style-guide.md -""" - -import os -import re -from pathlib import Path -from typing import Dict, List, Tuple - -# Style guide constants -COLORS = { - "blue": "#4A90E2", # Input/User elements, External/API - "orange": "#F5A623", # Processing/Scripts, Storage/Data - "purple": "#BD10E0", # AI/ML/Decision - "green": "#7ED321", # Execution/Action - "cyan": "#50E3C2", # Output/Response - "gray": "#666", # Arrows/text - "dark": "#333", # Labels -} - -SVG_TEMPLATE = """ - - - - - - - - {title} - - {content} - - - - {description} - -""" - - -def create_box(x: int, y: int, width: int, height: int, color: str, label: str) -> str: - """Create a standard box component""" - center_x = x + width // 2 - center_y = y + height // 2 + 5 - return f''' - {label}''' - - -def create_arrow( - x1: int, y1: int, x2: int, y2: int, dashed: bool = False, opacity: float = 1.0 -) -> str: - """Create an arrow between two points""" - dash_attr = ' stroke-dasharray="3,3"' if dashed else "" - opacity_attr = f' opacity="{opacity}"' if opacity < 1.0 else "" - return f'' - - -def create_curved_arrow( - points: List[Tuple[int, int]], dashed: bool = False, opacity: float = 1.0 -) -> str: - """Create a curved arrow path""" - dash_attr = ' stroke-dasharray="3,3"' if dashed else "" - opacity_attr = f' opacity="{opacity}"' if opacity < 1.0 else "" - - if len(points) < 3: - return "" - - path = f"M{points[0][0]},{points[0][1]}" - if len(points) == 3: - path += f" Q{points[1][0]},{points[1][1]} {points[2][0]},{points[2][1]}" - else: - for i in range(1, len(points)): - path += f" L{points[i][0]},{points[i][1]}" - - return f'' - - -def rebuild_conversation_flow() -> str: - """Rebuild conversation flow diagram""" - boxes = [] - arrows = [] - - # Main flow boxes - boxes.append(create_box(20, 60, 100, 40, COLORS["blue"], "User Input")) - boxes.append(create_box(160, 60, 100, 40, COLORS["orange"], "ASIC Script")) - boxes.append(create_box(300, 60, 100, 40, COLORS["purple"], "LM Decision")) - boxes.append(create_box(440, 60, 100, 40, COLORS["green"], "Bot Executor")) - boxes.append(create_box(580, 60, 100, 40, COLORS["cyan"], "Bot Response")) - - # Parallel processes - boxes.append(create_box(360, 160, 120, 40, COLORS["blue"], "Search Knowledge")) - boxes.append(create_box(500, 160, 100, 40, COLORS["orange"], "Call API")) - - # Main flow arrows - arrows.append(create_arrow(120, 80, 160, 80)) - arrows.append(create_arrow(260, 80, 300, 80)) - arrows.append(create_arrow(400, 80, 440, 80)) - arrows.append(create_arrow(540, 80, 580, 80)) - - # Branch arrows - arrows.append(create_arrow(490, 100, 420, 160, dashed=True, opacity=0.6)) - arrows.append(create_arrow(490, 100, 550, 160, dashed=True, opacity=0.6)) - - # Feedback loops - arrows.append( - create_curved_arrow( - [(420, 200), (420, 240), (630, 240), (630, 100)], dashed=True, opacity=0.4 - ) - ) - arrows.append( - create_curved_arrow( - [(550, 200), (550, 230), (620, 230), (620, 100)], dashed=True, opacity=0.4 - ) - ) - - content = ( - "\n ".join(boxes) - + '\n\n \n ' - + "\n ".join(arrows) - + "\n " - ) - - return SVG_TEMPLATE.format( - height=320, - title="The Flow", - content=content, - desc_y=300, - description="The AI handles everything else - understanding intent, collecting information, executing tools, answering from documents. Zero configuration.", - ) - - -def rebuild_architecture() -> str: - """Rebuild architecture diagram""" - boxes = [] - arrows = [] - - # Top layer - boxes.append(create_box(20, 60, 100, 40, COLORS["blue"], "Web Server")) - boxes.append(create_box(160, 60, 120, 40, COLORS["orange"], "BASIC Interpreter")) - boxes.append(create_box(320, 60, 100, 40, COLORS["purple"], "LLM Integration")) - boxes.append(create_box(460, 60, 120, 40, COLORS["green"], "Package Manager")) - boxes.append(create_box(620, 60, 100, 40, COLORS["cyan"], "Console UI")) - - # Middle layer - boxes.append( - create_box( - 250, 160, 300, 40, COLORS["blue"], "Session Manager (Tokio Async Runtime)" - ) - ) - - # Data layer - boxes.append(create_box(20, 260, 100, 40, COLORS["orange"], "PostgreSQL")) - boxes.append(create_box(160, 260, 100, 40, COLORS["purple"], "Valkey Cache")) - boxes.append(create_box(300, 260, 100, 40, COLORS["green"], "Qdrant Vectors")) - boxes.append(create_box(440, 260, 100, 40, COLORS["cyan"], "Object Storage")) - boxes.append(create_box(580, 260, 100, 40, COLORS["blue"], "Channels")) - boxes.append(create_box(700, 260, 80, 40, COLORS["orange"], "External API")) - - # Connection arrows (simplified) - for x in [70, 220, 370, 520, 670]: - arrows.append( - create_curved_arrow( - [(x, 100), (x, 130), (400, 130), (400, 160)], opacity=0.6 - ) - ) - - for x in [70, 210, 350, 490, 630]: - arrows.append(create_arrow(400, 200, x, 260, opacity=0.6)) - - # External API connection - arrows.append( - create_curved_arrow( - [(740, 260), (740, 220), (550, 180)], dashed=True, opacity=0.4 - ) - ) - - content = ( - "\n ".join(boxes) - + '\n\n \n ' - + "\n ".join(arrows) - + "\n " - ) - - # Add storage detail box - detail_box = """ - - - Storage Contents: - .gbkb (Documents) - .gbdialog (Scripts) - .gbot (Configs) - Templates - User Assets - """ - - content += detail_box - - return SVG_TEMPLATE.format( - height=400, - title="General Bots Architecture", - content=content, - desc_y=45, - description="Single binary with everything included - no external dependencies", - ) - - -def rebuild_package_system_flow() -> str: - """Rebuild package system flow diagram""" - boxes = [] - arrows = [] - - # Main flow - boxes.append(create_box(20, 60, 100, 40, COLORS["blue"], "User Request")) - boxes.append(create_box(160, 60, 100, 40, COLORS["orange"], "start.bas")) - boxes.append(create_box(300, 60, 100, 40, COLORS["purple"], "LLM Engine")) - boxes.append(create_box(440, 60, 100, 40, COLORS["cyan"], "Bot Response")) - - # Supporting components - boxes.append(create_box(240, 160, 120, 40, COLORS["blue"], "Vector Search")) - boxes.append(create_box(240, 240, 120, 40, COLORS["orange"], ".gbkb docs")) - - # Main flow arrows - arrows.append(create_arrow(120, 80, 160, 80)) - arrows.append(create_arrow(260, 80, 300, 80)) - arrows.append(create_arrow(400, 80, 440, 80)) - - # Bidirectional between start.bas and LLM - arrows.append( - create_curved_arrow( - [(210, 100), (210, 120), (300, 120), (350, 120), (350, 100)], - dashed=True, - opacity=0.6, - ) - ) - arrows.append( - create_curved_arrow( - [(350, 60), (350, 40), (260, 40), (210, 40), (210, 60)], - dashed=True, - opacity=0.6, - ) - ) - - # LLM to Vector Search - arrows.append(create_arrow(350, 100, 300, 160, opacity=0.6)) - - # Vector Search to .gbkb docs - arrows.append(create_arrow(300, 200, 300, 240, opacity=0.6)) - - # Feedback from Vector Search to LLM - arrows.append( - create_curved_arrow( - [(240, 180), (200, 140), (300, 100)], dashed=True, opacity=0.4 - ) - ) - - content = ( - "\n ".join(boxes) - + '\n\n \n ' - + "\n ".join(arrows) - + "\n " - ) - - # Add BASIC commands and package structure boxes - detail_boxes = """ - - - BASIC Commands - USE KB "docs" - answer = HEAR - result = LLM() - TALK result - - - - - Package Structure - my-bot.gbai/ - ├─ .gbdialog/ - ├─ .gbkb/ - └─ .gbot/ - """ - - content += detail_boxes - - # Add connection lines to detail boxes - content += """ - - - - """ - - # Add labels - labels = """ - Commands - Results - Query - Context""" - - content += labels - - return SVG_TEMPLATE.format( - height=400, - title="Package System Flow", - content=content, - desc_y=380, - description="BASIC scripts orchestrate LLM decisions, vector search, and responses with zero configuration", - ) - - -def main(): - """Main function to rebuild all SVGs""" - svgs_to_rebuild = { - "docs/src/assets/conversation-flow.svg": rebuild_conversation_flow(), - "docs/src/assets/architecture.svg": rebuild_architecture(), - "docs/src/assets/package-system-flow.svg": rebuild_package_system_flow(), - } - - for filepath, content in svgs_to_rebuild.items(): - full_path = Path(filepath) - if full_path.parent.exists(): - with open(full_path, "w") as f: - f.write(content) - print(f"Rebuilt: {filepath}") - else: - print(f"Skipping (directory not found): {filepath}") - - print(f"\nRebuilt {len(svgs_to_rebuild)} SVG files according to style guide") - print( - "Note: This is a demonstration script. Extend it to rebuild all 28 SVG files." - ) - - -if __name__ == "__main__": - main() diff --git a/safe_svg_improve.py b/safe_svg_improve.py deleted file mode 100644 index 006f994b..00000000 --- a/safe_svg_improve.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3 -""" -Safe SVG Improvement Script -Enhances SVG readability for mobile devices without breaking structure -""" - -import os -import re -from pathlib import Path - - -def safe_improve_svg(filepath): - """ - Safely improve SVG file for better mobile readability - Only makes minimal, safe changes to preserve structure - """ - - try: - # Read the original file - with open(filepath, "r", encoding="utf-8") as f: - content = f.read() - - # Skip font files and favicons - if "fontawesome" in str(filepath).lower() or "favicon" in str(filepath).lower(): - return False, "Skipped (font/favicon)" - - original_content = content - - # 1. Make SVG responsive by adding style attribute to svg tag if not present - if "style=" not in content.split(">")[0]: # Check only in the opening SVG tag - content = re.sub( - r"(]*)(>)", - r'\1 style="max-width: 100%; height: auto;"\2', - content, - count=1, - ) - - # 2. Increase small font sizes for mobile readability (minimum 14px) - def increase_font_size(match): - size = int(match.group(1)) - if size < 12: - return f'font-size="{14}"' - elif size == 12 or size == 13: - return f'font-size="{14}"' - else: - return match.group(0) - - content = re.sub(r'font-size="(\d+)"', increase_font_size, content) - - # 3. Improve text color contrast for better readability - # Only change very light grays to darker ones for text - text_color_improvements = { - "#CBD5E0": "#374151", # Light gray to dark gray - "#A0AEC0": "#4B5563", # Medium light gray to medium dark - "#718096": "#374151", # Another light gray to dark - "#E9D8FD": "#6B21A8", # Very light purple to dark purple - "#FBD38D": "#92400E", # Light orange to dark orange - "#90CDF4": "#1E40AF", # Light blue to dark blue - "#B2F5EA": "#047857", # Light teal to dark teal - "#9AE6B4": "#047857", # Light green to dark green - } - - for old_color, new_color in text_color_improvements.items(): - # Only replace in text elements - content = re.sub( - f'(]*fill="){old_color}(")', - f"\\1{new_color}\\2", - content, - flags=re.IGNORECASE, - ) - - # 4. Ensure stroke widths are visible (minimum 2) - content = re.sub(r'stroke-width="1"', 'stroke-width="2"', content) - content = re.sub(r'stroke-width="0\.5"', 'stroke-width="2"', content) - - # 5. Add rounded corners to rectangles if missing (but small radius) - def add_rounded_corners(match): - rect = match.group(0) - if "rx=" not in rect and 'fill="none"' in rect: - # Add small rounded corners for better aesthetics - rect = rect[:-2] + ' rx="4"/>' - return rect - - content = re.sub(r"]*/>", add_rounded_corners, content) - - # 6. Make arrow markers more visible - content = re.sub(r'fill="#888"', 'fill="#374151"', content) - - # 7. Improve font families for better cross-platform rendering - content = re.sub( - r'font-family="Arial, sans-serif"', - "font-family=\"-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif\"", - content, - ) - - # 8. Fix font weight declarations - content = re.sub(r'font-weight="bold"', 'font-weight="600"', content) - - # Only write if changes were made - if content != original_content: - # Backup original - backup_path = str(filepath) + ".backup" - if not os.path.exists(backup_path): - with open(backup_path, "w", encoding="utf-8") as f: - f.write(original_content) - - # Write improved version - with open(filepath, "w", encoding="utf-8") as f: - f.write(content) - - return True, "Improved successfully" - else: - return False, "No changes needed" - - except Exception as e: - return False, f"Error: {str(e)}" - - -def main(): - """Process all SVG files in docs directory""" - - print("=" * 60) - print("SAFE SVG IMPROVEMENT SCRIPT") - print("Enhancing readability without breaking structure") - print("=" * 60) - print() - - docs_dir = Path("docs") - svg_files = list(docs_dir.glob("**/*.svg")) - - print(f"Found {len(svg_files)} SVG files") - print() - - improved = 0 - skipped = 0 - unchanged = 0 - errors = 0 - - for svg_file in svg_files: - print(f"Processing: {svg_file}") - - success, message = safe_improve_svg(svg_file) - - if success: - print(f" ✓ {message}") - improved += 1 - elif "Skipped" in message: - print(f" ⊘ {message}") - skipped += 1 - elif "No changes" in message: - print(f" - {message}") - unchanged += 1 - else: - print(f" ✗ {message}") - errors += 1 - - print() - print("=" * 60) - print("SUMMARY") - print("=" * 60) - print(f"✓ Improved: {improved} files") - print(f"- Unchanged: {unchanged} files") - print(f"⊘ Skipped: {skipped} files") - if errors > 0: - print(f"✗ Errors: {errors} files") - - print() - print("Safe improvements applied:") - print("• Increased minimum font size to 14px") - print("• Improved text color contrast") - print("• Made SVGs responsive (100% width)") - print("• Enhanced stroke visibility") - print("• Added subtle rounded corners") - print("• Improved font families for all devices") - print() - print("Original files backed up with .backup extension") - print("=" * 60) - - -if __name__ == "__main__": - main() diff --git a/src/core/bot/mod.rs b/src/core/bot/mod.rs index 69c94ebc..e6e75d5e 100644 --- a/src/core/bot/mod.rs +++ b/src/core/bot/mod.rs @@ -83,7 +83,7 @@ impl BotOrchestrator { let bot_id = Uuid::parse_str(&message.bot_id).unwrap_or_default(); // All database operations in one blocking section - let (session, context_data, history, model, key) = { + let (session, context_data, history, model, key, _bot_id_from_config, cache_enabled) = { let state_clone = self.state.clone(); tokio::task::spawn_blocking( move || -> Result<_, Box> { @@ -124,16 +124,43 @@ impl BotOrchestrator { .get_config(&bot_id, "llm-key", Some("")) .unwrap_or_default(); - Ok((session, context_data, history, model, key)) + // Check if llm-cache is enabled for this bot + let cache_enabled = config_manager + .get_config(&bot_id, "llm-cache", Some("true")) + .unwrap_or_else(|_| "true".to_string()); + + Ok(( + session, + context_data, + history, + model, + key, + bot_id, + cache_enabled, + )) }, ) .await?? }; - // Build messages + // Build messages with bot_id for cache let system_prompt = std::env::var("SYSTEM_PROMPT") .unwrap_or_else(|_| "You are a helpful assistant.".to_string()); - let messages = OpenAIClient::build_messages(&system_prompt, &context_data, &history); + let mut messages = OpenAIClient::build_messages(&system_prompt, &context_data, &history); + + // Add bot_id and cache config to messages for the cache layer + if let serde_json::Value::Object(ref mut map) = messages { + map.insert("bot_id".to_string(), serde_json::json!(bot_id.to_string())); + map.insert("llm_cache".to_string(), serde_json::json!(cache_enabled)); + } else if let serde_json::Value::Array(_) = messages { + // If messages is an array, wrap it in an object + let messages_array = messages.clone(); + messages = serde_json::json!({ + "messages": messages_array, + "bot_id": bot_id.to_string(), + "llm_cache": cache_enabled + }); + } // Stream from LLM let (stream_tx, mut stream_rx) = mpsc::channel::(100); diff --git a/src/llm/cache.rs b/src/llm/cache.rs index 6eb8b066..639477ce 100644 --- a/src/llm/cache.rs +++ b/src/llm/cache.rs @@ -7,9 +7,11 @@ use sha2::{Digest, Sha256}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; +use uuid::Uuid; use super::LLMProvider; -use crate::shared::utils::estimate_token_count; +use crate::config::ConfigManager; +use crate::shared::utils::{estimate_token_count, DbPool}; /// Configuration for semantic caching #[derive(Clone)] @@ -67,6 +69,8 @@ pub struct CachedLLMProvider { config: CacheConfig, /// Optional embedding service for semantic matching embedding_service: Option>, + /// Database connection pool for config + db_pool: Option, } /// Trait for embedding services @@ -97,6 +101,29 @@ impl CachedLLMProvider { cache, config, embedding_service, + db_pool: None, + } + } + + pub fn with_db_pool( + provider: Arc, + cache: Arc, + config: CacheConfig, + embedding_service: Option>, + db_pool: DbPool, + ) -> Self { + info!("Initializing CachedLLMProvider with semantic cache and DB pool"); + info!( + "Cache config: TTL={}s, Semantic={}, Threshold={}", + config.ttl, config.semantic_matching, config.similarity_threshold + ); + + Self { + provider, + cache, + config, + embedding_service, + db_pool: Some(db_pool), } } @@ -112,23 +139,98 @@ impl CachedLLMProvider { /// Check if caching is enabled based on config async fn is_cache_enabled(&self, bot_id: &str) -> bool { - // Try to get llm-cache config from bot configuration - // This would typically query the database, but for now we'll check Redis + // First check if we have a DB pool to read config + if let Some(ref db_pool) = self.db_pool { + // Parse bot_id as UUID + let bot_uuid = match Uuid::parse_str(bot_id) { + Ok(uuid) => uuid, + Err(_) => { + // If not a valid UUID, check for default bot + if bot_id == "default" { + Uuid::nil() + } else { + return self.config.semantic_matching; // Fall back to global config + } + } + }; + + // Get config from database + let config_manager = ConfigManager::new(db_pool.clone()); + let cache_enabled = config_manager + .get_config(&bot_uuid, "llm-cache", Some("true")) + .unwrap_or_else(|_| "true".to_string()); + + return cache_enabled.to_lowercase() == "true"; + } + + // Fallback: check Redis for bot-specific cache config let mut conn = match self.cache.get_multiplexed_async_connection().await { Ok(conn) => conn, Err(e) => { debug!("Cache connection failed: {}", e); - return false; + return self.config.semantic_matching; } }; let config_key = format!("bot_config:{}:llm-cache", bot_id); match conn.get::<_, String>(config_key).await { Ok(value) => value.to_lowercase() == "true", - Err(_) => { - // Default to enabled if not specified - true + Err(_) => self.config.semantic_matching, // Default to global config + } + } + + /// Get cache configuration for a specific bot + async fn get_bot_cache_config(&self, bot_id: &str) -> CacheConfig { + if let Some(ref db_pool) = self.db_pool { + let bot_uuid = match Uuid::parse_str(bot_id) { + Ok(uuid) => uuid, + Err(_) => { + if bot_id == "default" { + Uuid::nil() + } else { + return self.config.clone(); + } + } + }; + + let config_manager = ConfigManager::new(db_pool.clone()); + + // Read all cache-related configs + let ttl = config_manager + .get_config( + &bot_uuid, + "llm-cache-ttl", + Some(&self.config.ttl.to_string()), + ) + .unwrap_or_else(|_| self.config.ttl.to_string()) + .parse() + .unwrap_or(self.config.ttl); + + let semantic_enabled = config_manager + .get_config(&bot_uuid, "llm-cache-semantic", Some("true")) + .unwrap_or_else(|_| "true".to_string()) + .to_lowercase() + == "true"; + + let threshold = config_manager + .get_config( + &bot_uuid, + "llm-cache-threshold", + Some(&self.config.similarity_threshold.to_string()), + ) + .unwrap_or_else(|_| self.config.similarity_threshold.to_string()) + .parse() + .unwrap_or(self.config.similarity_threshold); + + CacheConfig { + ttl, + semantic_matching: semantic_enabled, + similarity_threshold: threshold, + max_similarity_checks: self.config.max_similarity_checks, + key_prefix: self.config.key_prefix.clone(), } + } else { + self.config.clone() } } @@ -139,8 +241,6 @@ impl CachedLLMProvider { messages: &Value, model: &str, ) -> Option { - let cache_key = self.generate_cache_key(prompt, messages, model); - let mut conn = match self.cache.get_multiplexed_async_connection().await { Ok(conn) => conn, Err(e) => { @@ -149,6 +249,14 @@ impl CachedLLMProvider { } }; + // Extract actual messages if wrapped + let actual_messages = if messages.get("messages").is_some() { + messages.get("messages").unwrap_or(messages) + } else { + messages + }; + + let cache_key = self.generate_cache_key(prompt, actual_messages, model); // Try exact match first if let Ok(cached_json) = conn.get::<_, String>(&cache_key).await { if let Ok(mut cached) = serde_json::from_str::(&cached_json) { @@ -197,8 +305,15 @@ impl CachedLLMProvider { ) -> Option { let embedding_service = self.embedding_service.as_ref()?; + // Extract actual messages if wrapped + let actual_messages = if messages.get("messages").is_some() { + messages.get("messages").unwrap_or(messages) + } else { + messages + }; + // Combine prompt with messages for more accurate matching - let combined_context = format!("{}\n{}", prompt, messages.to_string()); + let combined_context = format!("{}\n{}", prompt, actual_messages.to_string()); // Get embedding for current prompt let prompt_embedding = match embedding_service.get_embedding(&combined_context).await { @@ -269,7 +384,14 @@ impl CachedLLMProvider { /// Store a response in cache async fn cache_response(&self, prompt: &str, messages: &Value, model: &str, response: &str) { - let cache_key = self.generate_cache_key(prompt, messages, model); + // Extract actual messages if wrapped + let actual_messages = if messages.get("messages").is_some() { + messages.get("messages").unwrap_or(messages) + } else { + messages + }; + + let cache_key = self.generate_cache_key(prompt, actual_messages, model); let mut conn = match self.cache.get_multiplexed_async_connection().await { Ok(conn) => conn, @@ -281,7 +403,9 @@ impl CachedLLMProvider { // Get embedding if service is available let embedding = if let Some(ref service) = self.embedding_service { - service.get_embedding(prompt).await.ok() + // Combine prompt with messages for embedding + let combined_context = format!("{}\n{}", prompt, actual_messages.to_string()); + service.get_embedding(&combined_context).await.ok() } else { None }; @@ -289,7 +413,7 @@ impl CachedLLMProvider { let cached_response = CachedResponse { response: response.to_string(), prompt: prompt.to_string(), - messages: messages.clone(), + messages: actual_messages.clone(), model: model.to_string(), timestamp: SystemTime::now() .duration_since(UNIX_EPOCH) @@ -394,19 +518,40 @@ impl LLMProvider for CachedLLMProvider { model: &str, key: &str, ) -> Result> { + // Extract bot_id from messages if available + let bot_id = messages + .get("bot_id") + .and_then(|v| v.as_str()) + .unwrap_or("default"); + // Check if cache is enabled for this bot - let bot_id = "default"; // This should be passed from context if !self.is_cache_enabled(bot_id).await { - trace!("Cache disabled, bypassing"); + trace!("Cache disabled for bot {}, bypassing", bot_id); return self.provider.generate(prompt, messages, model, key).await; } - // Try to get from cache + // Get bot-specific cache configuration + let bot_cache_config = self.get_bot_cache_config(bot_id).await; + + // First try exact match from cache if let Some(cached) = self.get_cached_response(prompt, messages, model).await { + info!("Cache hit (exact match) for bot {}", bot_id); return Ok(cached.response); } + // Then try semantic similarity match if enabled + if bot_cache_config.semantic_matching && self.embedding_service.is_some() { + if let Some(cached) = self.find_similar_cached(prompt, messages, model).await { + info!( + "Cache hit (semantic match) for bot {} with similarity threshold {}", + bot_id, bot_cache_config.similarity_threshold + ); + return Ok(cached.response); + } + } + // Generate new response + debug!("Cache miss for bot {}, generating new response", bot_id); let response = self.provider.generate(prompt, messages, model, key).await?; // Cache the response @@ -499,6 +644,27 @@ impl LocalEmbeddingService { } } +/// Helper function to enable semantic cache for a specific bot +pub async fn enable_semantic_cache_for_bot( + cache: &redis::Client, + bot_id: &str, + enabled: bool, +) -> Result<(), Box> { + let mut conn = cache.get_multiplexed_async_connection().await?; + let config_key = format!("bot_config:{}:llm-cache", bot_id); + let value = if enabled { "true" } else { "false" }; + + conn.set_ex::<_, _, ()>(&config_key, value, 86400).await?; // 24 hour TTL + + info!( + "Semantic cache {} for bot {}", + if enabled { "enabled" } else { "disabled" }, + bot_id + ); + + Ok(()) +} + #[async_trait] impl EmbeddingService for LocalEmbeddingService { async fn get_embedding( diff --git a/src/llm/mod.rs b/src/llm/mod.rs index 1b7458cb..7ccc948a 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -3,6 +3,7 @@ use futures::StreamExt; use log::{info, trace}; use serde_json::Value; use tokio::sync::mpsc; +pub mod cache; pub mod compact_prompt; pub mod llm_models; pub mod local; diff --git a/src/main.rs b/src/main.rs index 53640af7..98f252ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -508,11 +508,52 @@ async fn main() -> std::io::Result<()> { .get_config(&default_bot_id, "llm-url", Some("http://localhost:8081")) .unwrap_or_else(|_| "http://localhost:8081".to_string()); - let llm_provider = Arc::new(botserver::llm::OpenAIClient::new( + // Create base LLM provider + let base_llm_provider = Arc::new(botserver::llm::OpenAIClient::new( "empty".to_string(), Some(llm_url.clone()), )) as Arc; + // Wrap with cache if redis is available + let llm_provider: Arc = if let Some(ref cache) = redis_client { + // Set up embedding service for semantic matching + let embedding_url = config_manager + .get_config( + &default_bot_id, + "embedding-url", + Some("http://localhost:8082"), + ) + .unwrap_or_else(|_| "http://localhost:8082".to_string()); + let embedding_model = config_manager + .get_config(&default_bot_id, "embedding-model", Some("all-MiniLM-L6-v2")) + .unwrap_or_else(|_| "all-MiniLM-L6-v2".to_string()); + + let embedding_service = Some(Arc::new(botserver::llm::cache::LocalEmbeddingService::new( + embedding_url, + embedding_model, + )) + as Arc); + + // Create cache config + let cache_config = botserver::llm::cache::CacheConfig { + ttl: 3600, // 1 hour TTL + semantic_matching: true, + similarity_threshold: 0.85, // 85% similarity threshold + max_similarity_checks: 100, + key_prefix: "llm_cache".to_string(), + }; + + Arc::new(botserver::llm::cache::CachedLLMProvider::with_db_pool( + base_llm_provider, + cache.clone(), + cache_config, + embedding_service, + pool.clone(), + )) + } else { + base_llm_provider + }; + let app_state = Arc::new(AppState { drive: Some(drive), config: Some(cfg.clone()), diff --git a/tests/semantic_cache_test.rs b/tests/semantic_cache_test.rs new file mode 100644 index 00000000..3072d8f2 --- /dev/null +++ b/tests/semantic_cache_test.rs @@ -0,0 +1,150 @@ +#[cfg(test)] +mod semantic_cache_integration_tests { + use botserver::llm::cache::{enable_semantic_cache_for_bot, CacheConfig, CachedLLMProvider}; + use botserver::llm::{LLMProvider, OpenAIClient}; + use redis::{AsyncCommands, Client}; + use serde_json::json; + use std::sync::Arc; + use uuid::Uuid; + + #[tokio::test] + async fn test_semantic_cache_with_bot_config() { + // Skip test if Redis is not available + let redis_url = + std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://127.0.0.1/".to_string()); + let cache_client = match Client::open(redis_url) { + Ok(client) => client, + Err(_) => { + println!("Skipping test - Redis not available"); + return; + } + }; + + // Test connection + let conn = match cache_client.get_multiplexed_async_connection().await { + Ok(conn) => conn, + Err(_) => { + println!("Skipping test - Cannot connect to Redis"); + return; + } + }; + + // Create a test bot ID + let bot_id = Uuid::new_v4().to_string(); + + // Enable semantic cache for this bot + if let Err(e) = enable_semantic_cache_for_bot(&cache_client, &bot_id, true).await { + println!("Failed to enable cache for bot: {}", e); + return; + } + + // Create mock LLM provider + let llm_provider = Arc::new(OpenAIClient::new( + "test-key".to_string(), + Some("http://localhost:8081".to_string()), + )); + + // Create cache configuration + let cache_config = CacheConfig { + ttl: 300, // 5 minutes for testing + semantic_matching: true, + similarity_threshold: 0.85, + max_similarity_checks: 10, + key_prefix: "test_cache".to_string(), + }; + + // Create cached provider without embedding service for basic testing + let cached_provider = CachedLLMProvider::new( + llm_provider, + Arc::new(cache_client.clone()), + cache_config, + None, // No embedding service for this basic test + ); + + // Test messages with bot_id + let messages = json!({ + "bot_id": bot_id, + "llm_cache": "true", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"} + ] + }); + + // This would normally call the LLM, but will fail without a real server + // The test is mainly to ensure the cache layer is properly initialized + let result = cached_provider + .generate("", &messages, "gpt-3.5-turbo", "test-key") + .await; + + match result { + Ok(_) => println!("Cache test succeeded (unexpected with mock server)"), + Err(e) => println!("Expected error with mock server: {}", e), + } + + // Clean up - clear test cache entries + let mut conn = cache_client + .get_multiplexed_async_connection() + .await + .unwrap(); + let _: () = conn + .del(format!("bot_config:{}:llm-cache", bot_id)) + .await + .unwrap_or(()); + } + + #[tokio::test] + async fn test_cache_key_generation() { + use botserver::llm::cache::CachedLLMProvider; + + // This test verifies that cache keys are generated consistently + let messages1 = json!({ + "bot_id": "test-bot-1", + "messages": [ + {"role": "user", "content": "Hello"} + ] + }); + + let messages2 = json!({ + "bot_id": "test-bot-2", + "messages": [ + {"role": "user", "content": "Hello"} + ] + }); + + // The messages content is the same but bot_id is different + // Cache should handle this properly by extracting actual messages + let actual_messages1 = messages1.get("messages").unwrap_or(&messages1); + let actual_messages2 = messages2.get("messages").unwrap_or(&messages2); + + // Both should have the same actual message content + assert_eq!( + actual_messages1.to_string(), + actual_messages2.to_string(), + "Actual messages should be identical" + ); + } + + #[tokio::test] + async fn test_cache_config_defaults() { + let config = CacheConfig::default(); + + assert_eq!(config.ttl, 3600, "Default TTL should be 1 hour"); + assert!( + config.semantic_matching, + "Semantic matching should be enabled by default" + ); + assert_eq!( + config.similarity_threshold, 0.95, + "Default similarity threshold should be 0.95" + ); + assert_eq!( + config.max_similarity_checks, 100, + "Default max similarity checks should be 100" + ); + assert_eq!( + config.key_prefix, "llm_cache", + "Default key prefix should be 'llm_cache'" + ); + } +} diff --git a/validate_svgs.py b/validate_svgs.py deleted file mode 100644 index ba61988d..00000000 --- a/validate_svgs.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python3 -""" -SVG Validation and Documentation Mapping Script -Checks all SVG files for readability issues and shows where they're used in the documentation -""" - -import os -import re -from collections import defaultdict -from pathlib import Path - - -def analyze_svg(filepath): - """Analyze an SVG file for potential readability issues""" - issues = [] - info = {} - - try: - with open(filepath, "r", encoding="utf-8") as f: - content = f.read() - - # Check file size - file_size = os.path.getsize(filepath) - info["size"] = f"{file_size:,} bytes" - - # Extract viewBox/dimensions - viewbox_match = re.search(r'viewBox="([^"]+)"', content) - width_match = re.search(r'width="(\d+)"', content) - height_match = re.search(r'height="(\d+)"', content) - - if viewbox_match: - info["viewBox"] = viewbox_match.group(1) - elif width_match and height_match: - info["dimensions"] = f"{width_match.group(1)}x{height_match.group(1)}" - - # Check if responsive - if 'style="max-width: 100%' in content: - info["responsive"] = "✓" - else: - info["responsive"] = "✗" - issues.append("Not responsive (missing max-width: 100%)") - - # Find all font sizes - font_sizes = re.findall(r'font-size="(\d+)"', content) - if font_sizes: - sizes = [int(s) for s in font_sizes] - info["font_sizes"] = f"min:{min(sizes)}px, max:{max(sizes)}px" - - # Check for too small fonts - small_fonts = [s for s in sizes if s < 12] - if small_fonts: - issues.append( - f"Small fonts found: {small_fonts}px (mobile needs ≥14px)" - ) - - # Check text colors for contrast - text_colors = re.findall(r']*fill="([^"]+)"', content) - light_colors = [] - for color in text_colors: - if any( - light in color.upper() - for light in [ - "#CBD5E0", - "#A0AEC0", - "#E2E8F0", - "#EDF2F7", - "#F7FAFC", - "#9CA3AF", - "#D1D5DB", - ] - ): - light_colors.append(color) - - if light_colors: - unique_colors = list(set(light_colors)) - issues.append(f"Low contrast text colors: {', '.join(unique_colors[:3])}") - - # Check for background - if ( - 'fill="#FAFAFA"' in content - or 'fill="white"' in content - or 'fill="#FFFFFF"' in content - ): - if re.search( - r']*width="[^"]*"[^>]*height="[^"]*"[^>]*fill="(white|#FAFAFA|#FFFFFF)"', - content, - ): - issues.append("Has white/light background") - - # Count elements - info["texts"] = content.count(" 5: - print(f" ... and {len(references[svg_name]) - 5} more") - else: - print(f"\n ❓ No references found in documentation") - - # Summary - print("\n" + "=" * 80) - print("SUMMARY") - print("=" * 80) - print(f"Total SVG files analyzed: {len(svg_files)}") - print(f"Total issues found: {total_issues}") - - if total_issues > 0: - print("\n🔧 RECOMMENDED FIXES:") - print("1. Increase all font sizes to minimum 14px for mobile readability") - print("2. Replace light gray text colors with darker ones for better contrast") - print("3. Remove white backgrounds or make them transparent") - print("4. Add responsive styling (max-width: 100%; height: auto)") - print("5. Consider using system fonts for better cross-platform support") - - -if __name__ == "__main__": - main()