From f7ccc95e606d150cdf9824f5720ad7b5aa05642f Mon Sep 17 00:00:00 2001 From: "Rodrigo Rodriguez (Pragmatismo)" Date: Mon, 8 Dec 2025 00:19:29 -0300 Subject: [PATCH] Fix config.csv loading on startup - Disable TLS on Vault for local development (HTTP instead of HTTPS) - Fix bot_configuration id column type mismatch (TEXT -> UUID) - Add migration 6.1.1 to convert config table id columns to UUID - Fix sync_config_csv_to_db to use UUID binding for id column - Make start_all async with proper Vault startup sequence - Sync default.gbai config.csv to existing 'Default Bot' from migrations - Add diagnostic logging for config loading - Change default LLM/embedding URLs from https to http for local dev --- config/directory_config.json | 20 + .../down.sql | 15 - .../2025-01-20-000001_multi_agent_bots/up.sql | 226 --- migrations/6.0.5_automation_updates/up.sql | 6 +- migrations/6.0.9_website_support/up.sql | 2 +- migrations/6.1.0_enterprise_suite/down.sql | 22 + migrations/6.1.0_enterprise_suite/up.sql | 1215 ++++++++++++++++- migrations/6.1.0_table_keyword/down.sql | 22 - migrations/6.1.0_table_keyword/up.sql | 120 -- migrations/6.1.1_fix_config_id_types/down.sql | 98 ++ migrations/6.1.1_fix_config_id_types/up.sql | 99 ++ migrations/6.1.1_multi_agent_memory/down.sql | 64 - migrations/6.1.1_multi_agent_memory/up.sql | 315 ----- migrations/6.1.2_phase3_phase4/down.sql | 124 -- migrations/6.1.2_phase3_phase4/up.sql | 538 -------- migrations/6.2.0_suite_apps/down.sql | 26 - migrations/6.2.0_suite_apps/up.sql | 87 -- migrations/6.2.1_email_tracking/down.sql | 19 - migrations/6.2.1_email_tracking/up.sql | 56 - src/core/bootstrap/mod.rs | 839 +++++++++--- src/core/config/mod.rs | 60 +- src/core/package_manager/facade.rs | 22 +- src/core/package_manager/installer.rs | 72 +- src/core/secrets/mod.rs | 38 +- src/core/shared/utils.rs | 10 + src/main.rs | 93 +- .../default.gbai/default.gbot/config.csv | 2 +- 27 files changed, 2206 insertions(+), 2004 deletions(-) create mode 100644 config/directory_config.json delete mode 100644 migrations/2025-01-20-000001_multi_agent_bots/down.sql delete mode 100644 migrations/2025-01-20-000001_multi_agent_bots/up.sql delete mode 100644 migrations/6.1.0_table_keyword/down.sql delete mode 100644 migrations/6.1.0_table_keyword/up.sql create mode 100644 migrations/6.1.1_fix_config_id_types/down.sql create mode 100644 migrations/6.1.1_fix_config_id_types/up.sql delete mode 100644 migrations/6.1.1_multi_agent_memory/down.sql delete mode 100644 migrations/6.1.1_multi_agent_memory/up.sql delete mode 100644 migrations/6.1.2_phase3_phase4/down.sql delete mode 100644 migrations/6.1.2_phase3_phase4/up.sql delete mode 100644 migrations/6.2.0_suite_apps/down.sql delete mode 100644 migrations/6.2.0_suite_apps/up.sql delete mode 100644 migrations/6.2.1_email_tracking/down.sql delete mode 100644 migrations/6.2.1_email_tracking/up.sql diff --git a/config/directory_config.json b/config/directory_config.json new file mode 100644 index 00000000..6870b4f8 --- /dev/null +++ b/config/directory_config.json @@ -0,0 +1,20 @@ +{ + "base_url": "http://localhost:8080", + "default_org": { + "id": "350084341642035214", + "name": "default", + "domain": "default.localhost" + }, + "default_user": { + "id": "admin", + "username": "admin", + "email": "admin@localhost", + "password": "", + "first_name": "Admin", + "last_name": "User" + }, + "admin_token": "6ToEETpAOVIPWXcuF9IclFdb4uGR0pDZvsA02rTVTUkhthzbH3MYjkJQB7OnNMHAQIFlreU", + "project_id": "", + "client_id": "350084343638589454", + "client_secret": "7rAHHUIiv04O3itDpnHbetUpH3JzG4TLP6zuL07x6TaPiUzTq1Ut3II1le8plTeG" +} \ No newline at end of file diff --git a/migrations/2025-01-20-000001_multi_agent_bots/down.sql b/migrations/2025-01-20-000001_multi_agent_bots/down.sql deleted file mode 100644 index 6b741e84..00000000 --- a/migrations/2025-01-20-000001_multi_agent_bots/down.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Rollback Multi-Agent Bots Migration - --- Drop triggers first -DROP TRIGGER IF EXISTS update_bots_updated_at ON bots; -DROP FUNCTION IF EXISTS update_updated_at_column(); - --- Drop tables in reverse order of creation (respecting foreign key dependencies) -DROP TABLE IF EXISTS play_content; -DROP TABLE IF EXISTS hear_wait_states; -DROP TABLE IF EXISTS attachments; -DROP TABLE IF EXISTS conversation_branches; -DROP TABLE IF EXISTS bot_messages; -DROP TABLE IF EXISTS session_bots; -DROP TABLE IF EXISTS bot_triggers; -DROP TABLE IF EXISTS bots; diff --git a/migrations/2025-01-20-000001_multi_agent_bots/up.sql b/migrations/2025-01-20-000001_multi_agent_bots/up.sql deleted file mode 100644 index d987acfa..00000000 --- a/migrations/2025-01-20-000001_multi_agent_bots/up.sql +++ /dev/null @@ -1,226 +0,0 @@ --- Multi-Agent Bots Migration --- Enables multiple bots to participate in conversations based on triggers - --- ============================================================================ --- BOTS TABLE - Bot definitions --- ============================================================================ - -CREATE TABLE IF NOT EXISTS bots ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name VARCHAR(255) NOT NULL UNIQUE, - description TEXT, - system_prompt TEXT, - model_config JSONB DEFAULT '{}', - tools JSONB DEFAULT '[]', - is_active BOOLEAN DEFAULT true, - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() -); - -CREATE INDEX idx_bots_name ON bots(name); -CREATE INDEX idx_bots_active ON bots(is_active) WHERE is_active = true; - --- ============================================================================ --- BOT_TRIGGERS TABLE - Trigger configurations for bots --- ============================================================================ - -CREATE TABLE IF NOT EXISTS bot_triggers ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - bot_id UUID NOT NULL REFERENCES bots(id) ON DELETE CASCADE, - trigger_type VARCHAR(50) NOT NULL, -- 'keyword', 'tool', 'schedule', 'event', 'always' - trigger_config JSONB NOT NULL DEFAULT '{}', - priority INT DEFAULT 0, - is_active BOOLEAN DEFAULT true, - created_at TIMESTAMPTZ DEFAULT NOW(), - - CONSTRAINT valid_trigger_type CHECK ( - trigger_type IN ('keyword', 'tool', 'schedule', 'event', 'always') - ) -); - -CREATE INDEX idx_bot_triggers_bot_id ON bot_triggers(bot_id); -CREATE INDEX idx_bot_triggers_type ON bot_triggers(trigger_type); - --- ============================================================================ --- SESSION_BOTS TABLE - Bots active in a session --- ============================================================================ - -CREATE TABLE IF NOT EXISTS session_bots ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - session_id UUID NOT NULL, - bot_id UUID NOT NULL REFERENCES bots(id) ON DELETE CASCADE, - bot_name VARCHAR(255) NOT NULL, - trigger_config JSONB NOT NULL DEFAULT '{}', - priority INT DEFAULT 0, - is_active BOOLEAN DEFAULT true, - joined_at TIMESTAMPTZ DEFAULT NOW(), - left_at TIMESTAMPTZ, - - CONSTRAINT unique_session_bot UNIQUE (session_id, bot_name) -); - -CREATE INDEX idx_session_bots_session ON session_bots(session_id); -CREATE INDEX idx_session_bots_active ON session_bots(session_id, is_active) WHERE is_active = true; - --- ============================================================================ --- BOT_MESSAGES TABLE - Messages from bots in conversations --- ============================================================================ - -CREATE TABLE IF NOT EXISTS bot_messages ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - session_id UUID NOT NULL, - bot_id UUID REFERENCES bots(id) ON DELETE SET NULL, - bot_name VARCHAR(255) NOT NULL, - user_message_id UUID, -- Reference to the user message this responds to - content TEXT NOT NULL, - role VARCHAR(50) DEFAULT 'assistant', - metadata JSONB DEFAULT '{}', - created_at TIMESTAMPTZ DEFAULT NOW() -); - -CREATE INDEX idx_bot_messages_session ON bot_messages(session_id); -CREATE INDEX idx_bot_messages_bot ON bot_messages(bot_id); -CREATE INDEX idx_bot_messages_created ON bot_messages(created_at); - --- ============================================================================ --- CONVERSATION_BRANCHES TABLE - Branch conversations from a point --- ============================================================================ - -CREATE TABLE IF NOT EXISTS conversation_branches ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - parent_session_id UUID NOT NULL, - branch_session_id UUID NOT NULL UNIQUE, - branch_from_message_id UUID NOT NULL, - branch_name VARCHAR(255), - created_by UUID, - created_at TIMESTAMPTZ DEFAULT NOW() -); - -CREATE INDEX idx_branches_parent ON conversation_branches(parent_session_id); -CREATE INDEX idx_branches_session ON conversation_branches(branch_session_id); - --- ============================================================================ --- ATTACHMENTS TABLE - Files attached to messages --- ============================================================================ - -CREATE TABLE IF NOT EXISTS attachments ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - message_id UUID, - session_id UUID NOT NULL, - user_id UUID NOT NULL, - file_type VARCHAR(50) NOT NULL, -- 'image', 'document', 'audio', 'video', 'code', 'archive', 'other' - file_name VARCHAR(500) NOT NULL, - file_size BIGINT NOT NULL, - mime_type VARCHAR(255), - storage_path TEXT NOT NULL, - thumbnail_path TEXT, - metadata JSONB DEFAULT '{}', - created_at TIMESTAMPTZ DEFAULT NOW(), - - CONSTRAINT valid_file_type CHECK ( - file_type IN ('image', 'document', 'audio', 'video', 'code', 'archive', 'other') - ) -); - -CREATE INDEX idx_attachments_session ON attachments(session_id); -CREATE INDEX idx_attachments_user ON attachments(user_id); -CREATE INDEX idx_attachments_message ON attachments(message_id); -CREATE INDEX idx_attachments_type ON attachments(file_type); - --- ============================================================================ --- HEAR_WAIT_STATE TABLE - Track HEAR keyword wait states --- ============================================================================ - -CREATE TABLE IF NOT EXISTS hear_wait_states ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - session_id UUID NOT NULL, - variable_name VARCHAR(255) NOT NULL, - input_type VARCHAR(50) NOT NULL DEFAULT 'any', - options JSONB, -- For menu type - retry_count INT DEFAULT 0, - max_retries INT DEFAULT 3, - is_waiting BOOLEAN DEFAULT true, - created_at TIMESTAMPTZ DEFAULT NOW(), - expires_at TIMESTAMPTZ DEFAULT NOW() + INTERVAL '1 hour', - completed_at TIMESTAMPTZ, - - CONSTRAINT unique_hear_wait UNIQUE (session_id, variable_name) -); - -CREATE INDEX idx_hear_wait_session ON hear_wait_states(session_id); -CREATE INDEX idx_hear_wait_active ON hear_wait_states(session_id, is_waiting) WHERE is_waiting = true; - --- ============================================================================ --- PLAY_CONTENT TABLE - Track content projector state --- ============================================================================ - -CREATE TABLE IF NOT EXISTS play_content ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - session_id UUID NOT NULL, - content_type VARCHAR(50) NOT NULL, - source_url TEXT NOT NULL, - title VARCHAR(500), - options JSONB DEFAULT '{}', - is_playing BOOLEAN DEFAULT true, - started_at TIMESTAMPTZ DEFAULT NOW(), - stopped_at TIMESTAMPTZ, - - CONSTRAINT valid_content_type CHECK ( - content_type IN ('video', 'audio', 'image', 'presentation', 'document', - 'code', 'spreadsheet', 'pdf', 'markdown', 'html', 'iframe', 'unknown') - ) -); - -CREATE INDEX idx_play_content_session ON play_content(session_id); -CREATE INDEX idx_play_content_active ON play_content(session_id, is_playing) WHERE is_playing = true; - --- ============================================================================ --- DEFAULT BOTS - Insert some default specialized bots --- ============================================================================ - -INSERT INTO bots (id, name, description, system_prompt, is_active) VALUES - (gen_random_uuid(), 'fraud-detector', - 'Specialized bot for detecting and handling fraud-related inquiries', - 'You are a fraud detection specialist. Help users identify suspicious activities, - report unauthorized transactions, and guide them through security procedures. - Always prioritize user security and recommend immediate action for urgent cases.', - true), - - (gen_random_uuid(), 'investment-advisor', - 'Specialized bot for investment and financial planning advice', - 'You are an investment advisor. Help users understand investment options, - analyze portfolio performance, and make informed financial decisions. - Always remind users that past performance does not guarantee future results.', - true), - - (gen_random_uuid(), 'loan-specialist', - 'Specialized bot for loan and financing inquiries', - 'You are a loan specialist. Help users understand loan options, - simulate payments, and guide them through the application process. - Always disclose interest rates and total costs clearly.', - true), - - (gen_random_uuid(), 'card-services', - 'Specialized bot for credit and debit card services', - 'You are a card services specialist. Help users manage their cards, - understand benefits, handle disputes, and manage limits. - For security, never ask for full card numbers in chat.', - true) -ON CONFLICT (name) DO NOTHING; - --- ============================================================================ --- TRIGGERS - Update timestamps automatically --- ============================================================================ - -CREATE OR REPLACE FUNCTION update_updated_at_column() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ language 'plpgsql'; - -CREATE TRIGGER update_bots_updated_at - BEFORE UPDATE ON bots - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); diff --git a/migrations/6.0.5_automation_updates/up.sql b/migrations/6.0.5_automation_updates/up.sql index 79b7979d..88586e4b 100644 --- a/migrations/6.0.5_automation_updates/up.sql +++ b/migrations/6.0.5_automation_updates/up.sql @@ -9,8 +9,8 @@ ALTER TABLE public.system_automations ADD COLUMN IF NOT EXISTS name VARCHAR(255) -- Create index on name column for faster lookups CREATE INDEX IF NOT EXISTS idx_system_automations_name ON public.system_automations(name); -ALTER TABLE bot_configuration -ADD CONSTRAINT bot_configuration_config_key_unique UNIQUE (config_key); +-- Note: bot_configuration already has UNIQUE(bot_id, config_key) from migration 6.0.4 +-- Do NOT add a global unique constraint on config_key alone as that breaks multi-bot configs -- Migration 6.0.9: Add bot_id column to system_automations -- Description: Introduces a bot_id column to associate automations with a specific bot. @@ -30,7 +30,7 @@ ADD CONSTRAINT system_automations_bot_kind_param_unique UNIQUE (bot_id, kind, param); -- Add index for the new constraint -CREATE INDEX IF NOT EXISTS idx_system_automations_bot_kind_param +CREATE INDEX IF NOT EXISTS idx_system_automations_bot_kind_param ON public.system_automations (bot_id, kind, param); diff --git a/migrations/6.0.9_website_support/up.sql b/migrations/6.0.9_website_support/up.sql index ff28c324..fc73a758 100644 --- a/migrations/6.0.9_website_support/up.sql +++ b/migrations/6.0.9_website_support/up.sql @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS session_website_associations ( -- Foreign key to sessions table CONSTRAINT fk_session_website_session FOREIGN KEY (session_id) - REFERENCES sessions(id) + REFERENCES user_sessions(id) ON DELETE CASCADE, -- Foreign key to bots table diff --git a/migrations/6.1.0_enterprise_suite/down.sql b/migrations/6.1.0_enterprise_suite/down.sql index d6294fbc..3f951e0b 100644 --- a/migrations/6.1.0_enterprise_suite/down.sql +++ b/migrations/6.1.0_enterprise_suite/down.sql @@ -49,3 +49,25 @@ DROP TABLE IF EXISTS email_templates; DROP TABLE IF EXISTS scheduled_emails; DROP TABLE IF EXISTS email_signatures; DROP TABLE IF EXISTS global_email_signatures; +-- Drop triggers and functions +DROP TRIGGER IF EXISTS external_connections_updated_at_trigger ON external_connections; +DROP FUNCTION IF EXISTS update_external_connections_updated_at(); + +DROP TRIGGER IF EXISTS dynamic_table_definitions_updated_at_trigger ON dynamic_table_definitions; +DROP FUNCTION IF EXISTS update_dynamic_table_definitions_updated_at(); + +-- Drop indexes +DROP INDEX IF EXISTS idx_external_connections_name; +DROP INDEX IF EXISTS idx_external_connections_bot_id; + +DROP INDEX IF EXISTS idx_dynamic_table_fields_name; +DROP INDEX IF EXISTS idx_dynamic_table_fields_table_id; + +DROP INDEX IF EXISTS idx_dynamic_table_definitions_connection; +DROP INDEX IF EXISTS idx_dynamic_table_definitions_name; +DROP INDEX IF EXISTS idx_dynamic_table_definitions_bot_id; + +-- Drop tables (order matters due to foreign keys) +DROP TABLE IF EXISTS external_connections; +DROP TABLE IF EXISTS dynamic_table_fields; +DROP TABLE IF EXISTS dynamic_table_definitions; diff --git a/migrations/6.1.0_enterprise_suite/up.sql b/migrations/6.1.0_enterprise_suite/up.sql index f39f3e70..62f6264e 100644 --- a/migrations/6.1.0_enterprise_suite/up.sql +++ b/migrations/6.1.0_enterprise_suite/up.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS global_email_signatures ( CONSTRAINT check_signature_position CHECK (position IN ('top', 'bottom')) ); -CREATE INDEX idx_global_signatures_bot ON global_email_signatures(bot_id) WHERE is_active = true; +CREATE INDEX IF NOT EXISTS idx_global_signatures_bot ON global_email_signatures(bot_id) WHERE is_active = true; -- ============================================================================ -- EMAIL ENTERPRISE FEATURES (Outlook/Gmail parity) @@ -43,8 +43,8 @@ CREATE TABLE IF NOT EXISTS email_signatures ( CONSTRAINT unique_user_signature_name UNIQUE (user_id, bot_id, name) ); -CREATE INDEX idx_email_signatures_user ON email_signatures(user_id); -CREATE INDEX idx_email_signatures_default ON email_signatures(user_id, bot_id) WHERE is_default = true; +CREATE INDEX IF NOT EXISTS idx_email_signatures_user ON email_signatures(user_id); +CREATE INDEX IF NOT EXISTS idx_email_signatures_default ON email_signatures(user_id, bot_id) WHERE is_default = true; -- Scheduled emails (send later) CREATE TABLE IF NOT EXISTS scheduled_emails ( @@ -67,8 +67,8 @@ CREATE TABLE IF NOT EXISTS scheduled_emails ( CONSTRAINT check_scheduled_status CHECK (status IN ('pending', 'sent', 'failed', 'cancelled')) ); -CREATE INDEX idx_scheduled_emails_pending ON scheduled_emails(scheduled_at) WHERE status = 'pending'; -CREATE INDEX idx_scheduled_emails_user ON scheduled_emails(user_id, bot_id); +CREATE INDEX IF NOT EXISTS idx_scheduled_emails_pending ON scheduled_emails(scheduled_at) WHERE status = 'pending'; +CREATE INDEX IF NOT EXISTS idx_scheduled_emails_user ON scheduled_emails(user_id, bot_id); -- Email templates CREATE TABLE IF NOT EXISTS email_templates ( @@ -88,9 +88,9 @@ CREATE TABLE IF NOT EXISTS email_templates ( updated_at TIMESTAMPTZ DEFAULT NOW() ); -CREATE INDEX idx_email_templates_bot ON email_templates(bot_id); -CREATE INDEX idx_email_templates_category ON email_templates(category); -CREATE INDEX idx_email_templates_shared ON email_templates(bot_id) WHERE is_shared = true; +CREATE INDEX IF NOT EXISTS idx_email_templates_bot ON email_templates(bot_id); +CREATE INDEX IF NOT EXISTS idx_email_templates_category ON email_templates(category); +CREATE INDEX IF NOT EXISTS idx_email_templates_shared ON email_templates(bot_id) WHERE is_shared = true; -- Auto-responders (Out of Office) - works with Stalwart Sieve CREATE TABLE IF NOT EXISTS email_auto_responders ( @@ -113,7 +113,7 @@ CREATE TABLE IF NOT EXISTS email_auto_responders ( CONSTRAINT unique_user_responder UNIQUE (user_id, bot_id, responder_type) ); -CREATE INDEX idx_auto_responders_active ON email_auto_responders(user_id, bot_id) WHERE is_active = true; +CREATE INDEX IF NOT EXISTS idx_auto_responders_active ON email_auto_responders(user_id, bot_id) WHERE is_active = true; -- Email rules/filters - synced with Stalwart Sieve CREATE TABLE IF NOT EXISTS email_rules ( @@ -131,8 +131,8 @@ CREATE TABLE IF NOT EXISTS email_rules ( updated_at TIMESTAMPTZ DEFAULT NOW() ); -CREATE INDEX idx_email_rules_user ON email_rules(user_id, bot_id); -CREATE INDEX idx_email_rules_priority ON email_rules(user_id, bot_id, priority); +CREATE INDEX IF NOT EXISTS idx_email_rules_user ON email_rules(user_id, bot_id); +CREATE INDEX IF NOT EXISTS idx_email_rules_priority ON email_rules(user_id, bot_id, priority); -- Email labels/categories CREATE TABLE IF NOT EXISTS email_labels ( @@ -147,7 +147,7 @@ CREATE TABLE IF NOT EXISTS email_labels ( CONSTRAINT unique_user_label UNIQUE (user_id, bot_id, name) ); -CREATE INDEX idx_email_labels_user ON email_labels(user_id, bot_id); +CREATE INDEX IF NOT EXISTS idx_email_labels_user ON email_labels(user_id, bot_id); -- Email-label associations CREATE TABLE IF NOT EXISTS email_label_assignments ( @@ -158,8 +158,8 @@ CREATE TABLE IF NOT EXISTS email_label_assignments ( CONSTRAINT unique_email_label UNIQUE (email_message_id, label_id) ); -CREATE INDEX idx_label_assignments_email ON email_label_assignments(email_message_id); -CREATE INDEX idx_label_assignments_label ON email_label_assignments(label_id); +CREATE INDEX IF NOT EXISTS idx_label_assignments_email ON email_label_assignments(email_message_id); +CREATE INDEX IF NOT EXISTS idx_label_assignments_label ON email_label_assignments(label_id); -- Distribution lists CREATE TABLE IF NOT EXISTS distribution_lists ( @@ -175,8 +175,8 @@ CREATE TABLE IF NOT EXISTS distribution_lists ( updated_at TIMESTAMPTZ DEFAULT NOW() ); -CREATE INDEX idx_distribution_lists_bot ON distribution_lists(bot_id); -CREATE INDEX idx_distribution_lists_owner ON distribution_lists(owner_id); +CREATE INDEX IF NOT EXISTS idx_distribution_lists_bot ON distribution_lists(bot_id); +CREATE INDEX IF NOT EXISTS idx_distribution_lists_owner ON distribution_lists(owner_id); -- Shared mailboxes - managed via Stalwart CREATE TABLE IF NOT EXISTS shared_mailboxes ( @@ -192,7 +192,7 @@ CREATE TABLE IF NOT EXISTS shared_mailboxes ( CONSTRAINT unique_shared_mailbox_email UNIQUE (bot_id, email_address) ); -CREATE INDEX idx_shared_mailboxes_bot ON shared_mailboxes(bot_id); +CREATE INDEX IF NOT EXISTS idx_shared_mailboxes_bot ON shared_mailboxes(bot_id); CREATE TABLE IF NOT EXISTS shared_mailbox_members ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -204,8 +204,8 @@ CREATE TABLE IF NOT EXISTS shared_mailbox_members ( CONSTRAINT check_permission CHECK (permission_level IN ('read', 'write', 'admin')) ); -CREATE INDEX idx_shared_mailbox_members ON shared_mailbox_members(mailbox_id); -CREATE INDEX idx_shared_mailbox_user ON shared_mailbox_members(user_id); +CREATE INDEX IF NOT EXISTS idx_shared_mailbox_members ON shared_mailbox_members(mailbox_id); +CREATE INDEX IF NOT EXISTS idx_shared_mailbox_user ON shared_mailbox_members(user_id); -- ============================================================================ -- VIDEO MEETING FEATURES (Google Meet/Zoom parity) @@ -231,8 +231,8 @@ CREATE TABLE IF NOT EXISTS meeting_recordings ( CONSTRAINT check_transcription_status CHECK (transcription_status IN ('pending', 'processing', 'completed', 'failed')) ); -CREATE INDEX idx_meeting_recordings_meeting ON meeting_recordings(meeting_id); -CREATE INDEX idx_meeting_recordings_bot ON meeting_recordings(bot_id); +CREATE INDEX IF NOT EXISTS idx_meeting_recordings_meeting ON meeting_recordings(meeting_id); +CREATE INDEX IF NOT EXISTS idx_meeting_recordings_bot ON meeting_recordings(bot_id); -- Breakout rooms CREATE TABLE IF NOT EXISTS meeting_breakout_rooms ( @@ -247,7 +247,7 @@ CREATE TABLE IF NOT EXISTS meeting_breakout_rooms ( created_at TIMESTAMPTZ DEFAULT NOW() ); -CREATE INDEX idx_breakout_rooms_meeting ON meeting_breakout_rooms(meeting_id); +CREATE INDEX IF NOT EXISTS idx_breakout_rooms_meeting ON meeting_breakout_rooms(meeting_id); -- Meeting polls CREATE TABLE IF NOT EXISTS meeting_polls ( @@ -266,7 +266,7 @@ CREATE TABLE IF NOT EXISTS meeting_polls ( CONSTRAINT check_poll_type CHECK (poll_type IN ('single', 'multiple', 'open')) ); -CREATE INDEX idx_meeting_polls_meeting ON meeting_polls(meeting_id); +CREATE INDEX IF NOT EXISTS idx_meeting_polls_meeting ON meeting_polls(meeting_id); -- Meeting Q&A CREATE TABLE IF NOT EXISTS meeting_questions ( @@ -283,8 +283,8 @@ CREATE TABLE IF NOT EXISTS meeting_questions ( created_at TIMESTAMPTZ DEFAULT NOW() ); -CREATE INDEX idx_meeting_questions_meeting ON meeting_questions(meeting_id); -CREATE INDEX idx_meeting_questions_unanswered ON meeting_questions(meeting_id) WHERE is_answered = false; +CREATE INDEX IF NOT EXISTS idx_meeting_questions_meeting ON meeting_questions(meeting_id); +CREATE INDEX IF NOT EXISTS idx_meeting_questions_unanswered ON meeting_questions(meeting_id) WHERE is_answered = false; -- Meeting waiting room CREATE TABLE IF NOT EXISTS meeting_waiting_room ( @@ -301,8 +301,8 @@ CREATE TABLE IF NOT EXISTS meeting_waiting_room ( CONSTRAINT check_waiting_status CHECK (status IN ('waiting', 'admitted', 'rejected', 'left')) ); -CREATE INDEX idx_waiting_room_meeting ON meeting_waiting_room(meeting_id); -CREATE INDEX idx_waiting_room_status ON meeting_waiting_room(meeting_id, status); +CREATE INDEX IF NOT EXISTS idx_waiting_room_meeting ON meeting_waiting_room(meeting_id); +CREATE INDEX IF NOT EXISTS idx_waiting_room_status ON meeting_waiting_room(meeting_id, status); -- Meeting live captions CREATE TABLE IF NOT EXISTS meeting_captions ( @@ -318,7 +318,7 @@ CREATE TABLE IF NOT EXISTS meeting_captions ( created_at TIMESTAMPTZ DEFAULT NOW() ); -CREATE INDEX idx_meeting_captions_meeting ON meeting_captions(meeting_id, timestamp_ms); +CREATE INDEX IF NOT EXISTS idx_meeting_captions_meeting ON meeting_captions(meeting_id, timestamp_ms); -- Virtual backgrounds CREATE TABLE IF NOT EXISTS user_virtual_backgrounds ( @@ -333,7 +333,7 @@ CREATE TABLE IF NOT EXISTS user_virtual_backgrounds ( CONSTRAINT check_bg_type CHECK (background_type IN ('image', 'blur', 'none')) ); -CREATE INDEX idx_virtual_backgrounds_user ON user_virtual_backgrounds(user_id); +CREATE INDEX IF NOT EXISTS idx_virtual_backgrounds_user ON user_virtual_backgrounds(user_id); -- ============================================================================ -- DRIVE ENTERPRISE FEATURES (Google Drive/OneDrive parity) @@ -354,8 +354,8 @@ CREATE TABLE IF NOT EXISTS file_versions ( CONSTRAINT unique_file_version UNIQUE (file_id, version_number) ); -CREATE INDEX idx_file_versions_file ON file_versions(file_id); -CREATE INDEX idx_file_versions_current ON file_versions(file_id) WHERE is_current = true; +CREATE INDEX IF NOT EXISTS idx_file_versions_file ON file_versions(file_id); +CREATE INDEX IF NOT EXISTS idx_file_versions_current ON file_versions(file_id) WHERE is_current = true; -- File comments CREATE TABLE IF NOT EXISTS file_comments ( @@ -372,8 +372,8 @@ CREATE TABLE IF NOT EXISTS file_comments ( updated_at TIMESTAMPTZ DEFAULT NOW() ); -CREATE INDEX idx_file_comments_file ON file_comments(file_id); -CREATE INDEX idx_file_comments_unresolved ON file_comments(file_id) WHERE is_resolved = false; +CREATE INDEX IF NOT EXISTS idx_file_comments_file ON file_comments(file_id); +CREATE INDEX IF NOT EXISTS idx_file_comments_unresolved ON file_comments(file_id) WHERE is_resolved = false; -- File sharing permissions CREATE TABLE IF NOT EXISTS file_shares ( @@ -394,9 +394,9 @@ CREATE TABLE IF NOT EXISTS file_shares ( CONSTRAINT check_share_permission CHECK (permission_level IN ('view', 'comment', 'edit', 'admin')) ); -CREATE INDEX idx_file_shares_file ON file_shares(file_id); -CREATE INDEX idx_file_shares_user ON file_shares(shared_with_user); -CREATE INDEX idx_file_shares_token ON file_shares(link_token) WHERE link_token IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_file_shares_file ON file_shares(file_id); +CREATE INDEX IF NOT EXISTS idx_file_shares_user ON file_shares(shared_with_user); +CREATE INDEX IF NOT EXISTS idx_file_shares_token ON file_shares(link_token) WHERE link_token IS NOT NULL; -- File activity log CREATE TABLE IF NOT EXISTS file_activities ( @@ -410,8 +410,8 @@ CREATE TABLE IF NOT EXISTS file_activities ( created_at TIMESTAMPTZ DEFAULT NOW() ); -CREATE INDEX idx_file_activities_file ON file_activities(file_id, created_at DESC); -CREATE INDEX idx_file_activities_user ON file_activities(user_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_file_activities_file ON file_activities(file_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_file_activities_user ON file_activities(user_id, created_at DESC); -- Trash bin (soft delete with restore) CREATE TABLE IF NOT EXISTS file_trash ( @@ -426,8 +426,8 @@ CREATE TABLE IF NOT EXISTS file_trash ( permanent_delete_at TIMESTAMPTZ ); -CREATE INDEX idx_file_trash_owner ON file_trash(owner_id); -CREATE INDEX idx_file_trash_expiry ON file_trash(permanent_delete_at); +CREATE INDEX IF NOT EXISTS idx_file_trash_owner ON file_trash(owner_id); +CREATE INDEX IF NOT EXISTS idx_file_trash_expiry ON file_trash(permanent_delete_at); -- Offline sync tracking CREATE TABLE IF NOT EXISTS file_sync_status ( @@ -446,8 +446,8 @@ CREATE TABLE IF NOT EXISTS file_sync_status ( CONSTRAINT unique_sync_entry UNIQUE (user_id, device_id, file_id) ); -CREATE INDEX idx_file_sync_user ON file_sync_status(user_id, device_id); -CREATE INDEX idx_file_sync_pending ON file_sync_status(user_id) WHERE sync_status = 'pending'; +CREATE INDEX IF NOT EXISTS idx_file_sync_user ON file_sync_status(user_id, device_id); +CREATE INDEX IF NOT EXISTS idx_file_sync_pending ON file_sync_status(user_id) WHERE sync_status = 'pending'; -- Storage quotas CREATE TABLE IF NOT EXISTS storage_quotas ( @@ -462,7 +462,7 @@ CREATE TABLE IF NOT EXISTS storage_quotas ( CONSTRAINT unique_bot_quota UNIQUE (bot_id) ); -CREATE INDEX idx_storage_quotas_user ON storage_quotas(user_id); +CREATE INDEX IF NOT EXISTS idx_storage_quotas_user ON storage_quotas(user_id); -- ============================================================================ -- COLLABORATION FEATURES @@ -481,7 +481,7 @@ CREATE TABLE IF NOT EXISTS document_presence ( CONSTRAINT unique_doc_user_presence UNIQUE (document_id, user_id) ); -CREATE INDEX idx_document_presence_doc ON document_presence(document_id); +CREATE INDEX IF NOT EXISTS idx_document_presence_doc ON document_presence(document_id); -- ============================================================================ -- TASK ENTERPRISE FEATURES @@ -499,8 +499,8 @@ CREATE TABLE IF NOT EXISTS task_dependencies ( CONSTRAINT unique_task_dependency UNIQUE (task_id, depends_on_task_id) ); -CREATE INDEX idx_task_dependencies_task ON task_dependencies(task_id); -CREATE INDEX idx_task_dependencies_depends ON task_dependencies(depends_on_task_id); +CREATE INDEX IF NOT EXISTS idx_task_dependencies_task ON task_dependencies(task_id); +CREATE INDEX IF NOT EXISTS idx_task_dependencies_depends ON task_dependencies(depends_on_task_id); -- Task time tracking CREATE TABLE IF NOT EXISTS task_time_entries ( @@ -515,8 +515,8 @@ CREATE TABLE IF NOT EXISTS task_time_entries ( created_at TIMESTAMPTZ DEFAULT NOW() ); -CREATE INDEX idx_task_time_task ON task_time_entries(task_id); -CREATE INDEX idx_task_time_user ON task_time_entries(user_id, started_at); +CREATE INDEX IF NOT EXISTS idx_task_time_task ON task_time_entries(task_id); +CREATE INDEX IF NOT EXISTS idx_task_time_user ON task_time_entries(user_id, started_at); -- Task recurring rules CREATE TABLE IF NOT EXISTS task_recurrence ( @@ -535,7 +535,7 @@ CREATE TABLE IF NOT EXISTS task_recurrence ( CONSTRAINT check_recurrence CHECK (recurrence_pattern IN ('daily', 'weekly', 'monthly', 'yearly', 'custom')) ); -CREATE INDEX idx_task_recurrence_next ON task_recurrence(next_occurrence) WHERE is_active = true; +CREATE INDEX IF NOT EXISTS idx_task_recurrence_next ON task_recurrence(next_occurrence) WHERE is_active = true; -- ============================================================================ -- CALENDAR ENTERPRISE FEATURES @@ -558,8 +558,8 @@ CREATE TABLE IF NOT EXISTS calendar_resources ( CONSTRAINT check_resource_type CHECK (resource_type IN ('room', 'equipment', 'vehicle', 'other')) ); -CREATE INDEX idx_calendar_resources_bot ON calendar_resources(bot_id); -CREATE INDEX idx_calendar_resources_type ON calendar_resources(bot_id, resource_type); +CREATE INDEX IF NOT EXISTS idx_calendar_resources_bot ON calendar_resources(bot_id); +CREATE INDEX IF NOT EXISTS idx_calendar_resources_type ON calendar_resources(bot_id, resource_type); CREATE TABLE IF NOT EXISTS calendar_resource_bookings ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -575,8 +575,8 @@ CREATE TABLE IF NOT EXISTS calendar_resource_bookings ( CONSTRAINT check_booking_status CHECK (status IN ('pending', 'confirmed', 'cancelled')) ); -CREATE INDEX idx_resource_bookings_resource ON calendar_resource_bookings(resource_id, start_time, end_time); -CREATE INDEX idx_resource_bookings_user ON calendar_resource_bookings(booked_by); +CREATE INDEX IF NOT EXISTS idx_resource_bookings_resource ON calendar_resource_bookings(resource_id, start_time, end_time); +CREATE INDEX IF NOT EXISTS idx_resource_bookings_user ON calendar_resource_bookings(booked_by); -- Calendar sharing CREATE TABLE IF NOT EXISTS calendar_shares ( @@ -589,8 +589,8 @@ CREATE TABLE IF NOT EXISTS calendar_shares ( CONSTRAINT check_cal_permission CHECK (permission_level IN ('free_busy', 'view', 'edit', 'admin')) ); -CREATE INDEX idx_calendar_shares_owner ON calendar_shares(owner_id); -CREATE INDEX idx_calendar_shares_shared ON calendar_shares(shared_with_user); +CREATE INDEX IF NOT EXISTS idx_calendar_shares_owner ON calendar_shares(owner_id); +CREATE INDEX IF NOT EXISTS idx_calendar_shares_shared ON calendar_shares(shared_with_user); -- ============================================================================ -- TEST SUPPORT TABLES @@ -609,7 +609,7 @@ CREATE TABLE IF NOT EXISTS test_accounts ( CONSTRAINT check_test_account_type CHECK (account_type IN ('sender', 'receiver', 'bot', 'admin')) ); -CREATE INDEX idx_test_accounts_type ON test_accounts(account_type); +CREATE INDEX IF NOT EXISTS idx_test_accounts_type ON test_accounts(account_type); -- Test execution logs CREATE TABLE IF NOT EXISTS test_execution_logs ( @@ -625,5 +625,1104 @@ CREATE TABLE IF NOT EXISTS test_execution_logs ( CONSTRAINT check_test_status CHECK (status IN ('passed', 'failed', 'skipped', 'error')) ); -CREATE INDEX idx_test_logs_suite ON test_execution_logs(test_suite, created_at DESC); -CREATE INDEX idx_test_logs_status ON test_execution_logs(status, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_test_logs_suite ON test_execution_logs(test_suite, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_test_logs_status ON test_execution_logs(status, created_at DESC); +-- Migration: 6.1.1 Multi-Agent Memory Support +-- Description: Adds tables for user memory, session preferences, and A2A protocol messaging + +-- ============================================================================ +-- User Memories Table +-- Cross-session memory that persists for users across all sessions and bots +-- ============================================================================ +CREATE TABLE IF NOT EXISTS user_memories ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL, + key VARCHAR(255) NOT NULL, + value TEXT NOT NULL, + memory_type VARCHAR(50) NOT NULL DEFAULT 'preference', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT user_memories_unique_key UNIQUE (user_id, key) +); + +CREATE INDEX IF NOT EXISTS idx_user_memories_user_id ON user_memories(user_id); +CREATE INDEX IF NOT EXISTS idx_user_memories_type ON user_memories(user_id, memory_type); + +-- ============================================================================ +-- Session Preferences Table +-- Stores per-session configuration like current model, routing strategy, etc. +-- ============================================================================ +CREATE TABLE IF NOT EXISTS session_preferences ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + session_id UUID NOT NULL, + preference_key VARCHAR(255) NOT NULL, + preference_value TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT session_preferences_unique UNIQUE (session_id, preference_key) +); + +CREATE INDEX IF NOT EXISTS idx_session_preferences_session ON session_preferences(session_id); + +-- ============================================================================ +-- A2A Messages Table +-- Agent-to-Agent protocol messages for multi-agent orchestration +-- Based on https://a2a-protocol.org/latest/ +-- ============================================================================ +CREATE TABLE IF NOT EXISTS a2a_messages ( + id UUID PRIMARY KEY, + session_id UUID NOT NULL, + from_agent VARCHAR(255) NOT NULL, + to_agent VARCHAR(255), -- NULL for broadcast messages + message_type VARCHAR(50) NOT NULL, + payload TEXT NOT NULL, + correlation_id UUID NOT NULL, + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + metadata TEXT DEFAULT '{}', + ttl_seconds INTEGER NOT NULL DEFAULT 30, + hop_count INTEGER NOT NULL DEFAULT 0, + processed BOOLEAN NOT NULL DEFAULT FALSE, + processed_at TIMESTAMPTZ, + error_message TEXT +); + +CREATE INDEX IF NOT EXISTS idx_a2a_messages_session ON a2a_messages(session_id); +CREATE INDEX IF NOT EXISTS idx_a2a_messages_to_agent ON a2a_messages(session_id, to_agent); +CREATE INDEX IF NOT EXISTS idx_a2a_messages_correlation ON a2a_messages(correlation_id); +CREATE INDEX IF NOT EXISTS idx_a2a_messages_pending ON a2a_messages(session_id, to_agent, processed) WHERE processed = FALSE; +CREATE INDEX IF NOT EXISTS idx_a2a_messages_timestamp ON a2a_messages(timestamp); + +-- ============================================================================ +-- Extended Bot Memory Table +-- Enhanced memory with TTL and different memory types +-- ============================================================================ +CREATE TABLE IF NOT EXISTS bot_memory_extended ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL, + session_id UUID, -- NULL for long-term memory + memory_type VARCHAR(20) NOT NULL CHECK (memory_type IN ('short', 'long', 'episodic')), + key VARCHAR(255) NOT NULL, + value TEXT NOT NULL, + ttl_seconds INTEGER, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ, + CONSTRAINT bot_memory_extended_unique UNIQUE (bot_id, session_id, key) +); + +CREATE INDEX IF NOT EXISTS idx_bot_memory_ext_bot ON bot_memory_extended(bot_id); +CREATE INDEX IF NOT EXISTS idx_bot_memory_ext_session ON bot_memory_extended(bot_id, session_id); +CREATE INDEX IF NOT EXISTS idx_bot_memory_ext_type ON bot_memory_extended(bot_id, memory_type); +CREATE INDEX IF NOT EXISTS idx_bot_memory_ext_expires ON bot_memory_extended(expires_at) WHERE expires_at IS NOT NULL; + +-- ============================================================================ +-- Knowledge Graph Entities Table +-- For graph-based memory and entity relationships +-- ============================================================================ +CREATE TABLE IF NOT EXISTS kg_entities ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL, + entity_type VARCHAR(100) NOT NULL, + entity_name VARCHAR(500) NOT NULL, + properties JSONB DEFAULT '{}', + embedding_vector BYTEA, -- For vector similarity search + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT kg_entities_unique UNIQUE (bot_id, entity_type, entity_name) +); + +CREATE INDEX IF NOT EXISTS idx_kg_entities_bot ON kg_entities(bot_id); +CREATE INDEX IF NOT EXISTS idx_kg_entities_type ON kg_entities(bot_id, entity_type); +CREATE INDEX IF NOT EXISTS idx_kg_entities_name ON kg_entities(entity_name); + +-- ============================================================================ +-- Knowledge Graph Relationships Table +-- For storing relationships between entities +-- ============================================================================ +CREATE TABLE IF NOT EXISTS kg_relationships ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL, + from_entity_id UUID NOT NULL REFERENCES kg_entities(id) ON DELETE CASCADE, + to_entity_id UUID NOT NULL REFERENCES kg_entities(id) ON DELETE CASCADE, + relationship_type VARCHAR(100) NOT NULL, + properties JSONB DEFAULT '{}', + weight FLOAT DEFAULT 1.0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT kg_relationships_unique UNIQUE (from_entity_id, to_entity_id, relationship_type) +); + +CREATE INDEX IF NOT EXISTS idx_kg_rel_bot ON kg_relationships(bot_id); +CREATE INDEX IF NOT EXISTS idx_kg_rel_from ON kg_relationships(from_entity_id); +CREATE INDEX IF NOT EXISTS idx_kg_rel_to ON kg_relationships(to_entity_id); +CREATE INDEX IF NOT EXISTS idx_kg_rel_type ON kg_relationships(bot_id, relationship_type); + +-- ============================================================================ +-- Episodic Memory Table +-- For storing conversation summaries and episodes +-- ============================================================================ +CREATE TABLE IF NOT EXISTS episodic_memories ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL, + user_id UUID NOT NULL, + session_id UUID, + summary TEXT NOT NULL, + key_topics JSONB DEFAULT '[]', + decisions JSONB DEFAULT '[]', + action_items JSONB DEFAULT '[]', + message_count INTEGER NOT NULL DEFAULT 0, + start_timestamp TIMESTAMPTZ NOT NULL, + end_timestamp TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_episodic_bot ON episodic_memories(bot_id); +CREATE INDEX IF NOT EXISTS idx_episodic_user ON episodic_memories(user_id); +CREATE INDEX IF NOT EXISTS idx_episodic_session ON episodic_memories(session_id); +CREATE INDEX IF NOT EXISTS idx_episodic_time ON episodic_memories(bot_id, user_id, created_at); + +-- ============================================================================ +-- Conversation Cost Tracking Table +-- For monitoring LLM usage and costs +-- ============================================================================ +CREATE TABLE IF NOT EXISTS conversation_costs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + session_id UUID NOT NULL, + user_id UUID NOT NULL, + bot_id UUID NOT NULL, + model_used VARCHAR(100), + input_tokens INTEGER NOT NULL DEFAULT 0, + output_tokens INTEGER NOT NULL DEFAULT 0, + cost_usd DECIMAL(10, 6) NOT NULL DEFAULT 0, + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_conv_costs_session ON conversation_costs(session_id); +CREATE INDEX IF NOT EXISTS idx_conv_costs_user ON conversation_costs(user_id); +CREATE INDEX IF NOT EXISTS idx_conv_costs_bot ON conversation_costs(bot_id); +CREATE INDEX IF NOT EXISTS idx_conv_costs_time ON conversation_costs(timestamp); + +-- ============================================================================ +-- Generated API Tools Table +-- For tracking tools generated from OpenAPI specs +-- ============================================================================ +CREATE TABLE IF NOT EXISTS generated_api_tools ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL, + api_name VARCHAR(255) NOT NULL, + spec_url TEXT NOT NULL, + spec_hash VARCHAR(64) NOT NULL, + tool_count INTEGER NOT NULL DEFAULT 0, + last_synced_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT generated_api_tools_unique UNIQUE (bot_id, api_name) +); + +CREATE INDEX IF NOT EXISTS idx_gen_api_tools_bot ON generated_api_tools(bot_id); + +-- ============================================================================ +-- Session Bots Junction Table (if not exists) +-- For multi-agent sessions +-- ============================================================================ +CREATE TABLE IF NOT EXISTS session_bots ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + session_id UUID NOT NULL, + bot_id UUID NOT NULL, + bot_name VARCHAR(255) NOT NULL, + trigger_config JSONB DEFAULT '{}', + priority INTEGER NOT NULL DEFAULT 0, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + added_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT session_bots_unique UNIQUE (session_id, bot_name) +); + +CREATE INDEX IF NOT EXISTS idx_session_bots_session ON session_bots(session_id); +CREATE INDEX IF NOT EXISTS idx_session_bots_active ON session_bots(session_id, is_active); + +-- ============================================================================ +-- Cleanup function for expired A2A messages +-- ============================================================================ +CREATE OR REPLACE FUNCTION cleanup_expired_a2a_messages() +RETURNS INTEGER AS $$ +DECLARE + deleted_count INTEGER; +BEGIN + DELETE FROM a2a_messages + WHERE ttl_seconds > 0 + AND timestamp + (ttl_seconds || ' seconds')::INTERVAL < NOW(); + + GET DIAGNOSTICS deleted_count = ROW_COUNT; + RETURN deleted_count; +END; +$$ LANGUAGE plpgsql; + +-- ============================================================================ +-- Cleanup function for expired bot memory +-- ============================================================================ +CREATE OR REPLACE FUNCTION cleanup_expired_bot_memory() +RETURNS INTEGER AS $$ +DECLARE + deleted_count INTEGER; +BEGIN + DELETE FROM bot_memory_extended + WHERE expires_at IS NOT NULL AND expires_at < NOW(); + + GET DIAGNOSTICS deleted_count = ROW_COUNT; + RETURN deleted_count; +END; +$$ LANGUAGE plpgsql; + +-- ============================================================================ +-- Trigger to update updated_at timestamp +-- ============================================================================ +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Apply trigger to tables with updated_at +DROP TRIGGER IF EXISTS update_user_memories_updated_at ON user_memories; +CREATE TRIGGER update_user_memories_updated_at + BEFORE UPDATE ON user_memories + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +DROP TRIGGER IF EXISTS update_bot_memory_extended_updated_at ON bot_memory_extended; +CREATE TRIGGER update_bot_memory_extended_updated_at + BEFORE UPDATE ON bot_memory_extended + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +DROP TRIGGER IF EXISTS update_kg_entities_updated_at ON kg_entities; +CREATE TRIGGER update_kg_entities_updated_at + BEFORE UPDATE ON kg_entities + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +-- ============================================================================ +-- Bot Reflections Table +-- For storing agent self-reflection analysis results +-- ============================================================================ +CREATE TABLE IF NOT EXISTS bot_reflections ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL, + session_id UUID NOT NULL, + reflection_type TEXT NOT NULL, + score FLOAT NOT NULL DEFAULT 0.0, + insights TEXT NOT NULL DEFAULT '[]', + improvements TEXT NOT NULL DEFAULT '[]', + positive_patterns TEXT NOT NULL DEFAULT '[]', + concerns TEXT NOT NULL DEFAULT '[]', + raw_response TEXT NOT NULL DEFAULT '', + messages_analyzed INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_bot_reflections_bot ON bot_reflections(bot_id); +CREATE INDEX IF NOT EXISTS idx_bot_reflections_session ON bot_reflections(session_id); +CREATE INDEX IF NOT EXISTS idx_bot_reflections_time ON bot_reflections(bot_id, created_at); + +-- ============================================================================ +-- Conversation Messages Table +-- For storing conversation history (if not already exists) +-- ============================================================================ +CREATE TABLE IF NOT EXISTS conversation_messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + session_id UUID NOT NULL, + bot_id UUID NOT NULL, + user_id UUID, + role VARCHAR(50) NOT NULL, + content TEXT NOT NULL, + metadata JSONB DEFAULT '{}', + token_count INTEGER, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_conv_messages_session ON conversation_messages(session_id); +CREATE INDEX IF NOT EXISTS idx_conv_messages_time ON conversation_messages(session_id, created_at); +CREATE INDEX IF NOT EXISTS idx_conv_messages_bot ON conversation_messages(bot_id); +-- Migration: 6.1.2_phase3_phase4 +-- Description: Phase 3 and Phase 4 multi-agent features +-- Features: +-- - Episodic memory (conversation summaries) +-- - Knowledge graphs (entity relationships) +-- - Human-in-the-loop approvals +-- - LLM observability and cost tracking + +-- ============================================ +-- EPISODIC MEMORY TABLES +-- ============================================ + +-- Conversation episodes (summaries) +CREATE TABLE IF NOT EXISTS conversation_episodes ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL, + bot_id UUID NOT NULL, + session_id UUID NOT NULL, + summary TEXT NOT NULL, + key_topics JSONB NOT NULL DEFAULT '[]', + decisions JSONB NOT NULL DEFAULT '[]', + action_items JSONB NOT NULL DEFAULT '[]', + sentiment JSONB NOT NULL DEFAULT '{"score": 0, "label": "neutral", "confidence": 0.5}', + resolution VARCHAR(50) NOT NULL DEFAULT 'unknown', + message_count INTEGER NOT NULL DEFAULT 0, + message_ids JSONB NOT NULL DEFAULT '[]', + conversation_start TIMESTAMP WITH TIME ZONE NOT NULL, + conversation_end TIMESTAMP WITH TIME ZONE NOT NULL, + metadata JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +-- Indexes for episodic memory +CREATE INDEX IF NOT EXISTS idx_episodes_user_id ON conversation_episodes(user_id); +CREATE INDEX IF NOT EXISTS idx_episodes_bot_id ON conversation_episodes(bot_id); +CREATE INDEX IF NOT EXISTS idx_episodes_session_id ON conversation_episodes(session_id); +CREATE INDEX IF NOT EXISTS idx_episodes_created_at ON conversation_episodes(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_episodes_key_topics ON conversation_episodes USING GIN(key_topics); +CREATE INDEX IF NOT EXISTS idx_episodes_resolution ON conversation_episodes(resolution); + +-- Full-text search on summaries +CREATE INDEX IF NOT EXISTS idx_episodes_summary_fts ON conversation_episodes + USING GIN(to_tsvector('english', summary)); + +-- ============================================ +-- KNOWLEDGE GRAPH TABLES - Add missing columns +-- (Tables created earlier in this migration) +-- ============================================ + +-- Add missing columns to kg_entities +ALTER TABLE kg_entities ADD COLUMN IF NOT EXISTS aliases JSONB NOT NULL DEFAULT '[]'; +ALTER TABLE kg_entities ADD COLUMN IF NOT EXISTS confidence DOUBLE PRECISION NOT NULL DEFAULT 1.0; +ALTER TABLE kg_entities ADD COLUMN IF NOT EXISTS source VARCHAR(50) NOT NULL DEFAULT 'manual'; + +-- Add missing columns to kg_relationships +ALTER TABLE kg_relationships ADD COLUMN IF NOT EXISTS confidence DOUBLE PRECISION NOT NULL DEFAULT 1.0; +ALTER TABLE kg_relationships ADD COLUMN IF NOT EXISTS bidirectional BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE kg_relationships ADD COLUMN IF NOT EXISTS source VARCHAR(50) NOT NULL DEFAULT 'manual'; + +-- Indexes for knowledge graph +CREATE INDEX IF NOT EXISTS idx_kg_entities_bot_id ON kg_entities(bot_id); +CREATE INDEX IF NOT EXISTS idx_kg_entities_type ON kg_entities(entity_type); +CREATE INDEX IF NOT EXISTS idx_kg_entities_name ON kg_entities(entity_name); +CREATE INDEX IF NOT EXISTS idx_kg_entities_name_lower ON kg_entities(LOWER(entity_name)); +CREATE INDEX IF NOT EXISTS idx_kg_entities_aliases ON kg_entities USING GIN(aliases); + +CREATE INDEX IF NOT EXISTS idx_kg_relationships_bot_id ON kg_relationships(bot_id); +CREATE INDEX IF NOT EXISTS idx_kg_relationships_from ON kg_relationships(from_entity_id); +CREATE INDEX IF NOT EXISTS idx_kg_relationships_to ON kg_relationships(to_entity_id); +CREATE INDEX IF NOT EXISTS idx_kg_relationships_type ON kg_relationships(relationship_type); + +-- Full-text search on entity names +CREATE INDEX IF NOT EXISTS idx_kg_entities_name_fts ON kg_entities + USING GIN(to_tsvector('english', entity_name)); + +-- ============================================ +-- HUMAN-IN-THE-LOOP APPROVAL TABLES +-- ============================================ + +-- Approval requests +CREATE TABLE IF NOT EXISTS approval_requests ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL, + session_id UUID NOT NULL, + initiated_by UUID NOT NULL, + approval_type VARCHAR(100) NOT NULL, + status VARCHAR(50) NOT NULL DEFAULT 'pending', + channel VARCHAR(50) NOT NULL, + recipient VARCHAR(500) NOT NULL, + context JSONB NOT NULL DEFAULT '{}', + message TEXT NOT NULL, + timeout_seconds INTEGER NOT NULL DEFAULT 3600, + default_action VARCHAR(50), + current_level INTEGER NOT NULL DEFAULT 1, + total_levels INTEGER NOT NULL DEFAULT 1, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + expires_at TIMESTAMP WITH TIME ZONE NOT NULL, + reminders_sent JSONB NOT NULL DEFAULT '[]', + decision VARCHAR(50), + decided_by VARCHAR(500), + decided_at TIMESTAMP WITH TIME ZONE, + comments TEXT +); + +-- Approval chains +CREATE TABLE IF NOT EXISTS approval_chains ( + id UUID PRIMARY KEY, + name VARCHAR(200) NOT NULL, + bot_id UUID NOT NULL, + levels JSONB NOT NULL DEFAULT '[]', + stop_on_reject BOOLEAN NOT NULL DEFAULT true, + require_all BOOLEAN NOT NULL DEFAULT false, + description TEXT, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + + UNIQUE(bot_id, name) +); + +-- Approval audit log +CREATE TABLE IF NOT EXISTS approval_audit_log ( + id UUID PRIMARY KEY, + request_id UUID NOT NULL REFERENCES approval_requests(id) ON DELETE CASCADE, + action VARCHAR(50) NOT NULL, + actor VARCHAR(500) NOT NULL, + details JSONB NOT NULL DEFAULT '{}', + timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + ip_address VARCHAR(50), + user_agent TEXT +); + +-- Approval tokens (for secure links) +CREATE TABLE IF NOT EXISTS approval_tokens ( + id UUID PRIMARY KEY, + request_id UUID NOT NULL REFERENCES approval_requests(id) ON DELETE CASCADE, + token VARCHAR(100) NOT NULL UNIQUE, + action VARCHAR(50) NOT NULL, + used BOOLEAN NOT NULL DEFAULT false, + used_at TIMESTAMP WITH TIME ZONE, + expires_at TIMESTAMP WITH TIME ZONE NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +-- Indexes for approval tables +CREATE INDEX IF NOT EXISTS idx_approval_requests_bot_id ON approval_requests(bot_id); +CREATE INDEX IF NOT EXISTS idx_approval_requests_session_id ON approval_requests(session_id); +CREATE INDEX IF NOT EXISTS idx_approval_requests_status ON approval_requests(status); +CREATE INDEX IF NOT EXISTS idx_approval_requests_expires_at ON approval_requests(expires_at); +CREATE INDEX IF NOT EXISTS idx_approval_requests_pending ON approval_requests(status, expires_at) + WHERE status = 'pending'; + +CREATE INDEX IF NOT EXISTS idx_approval_audit_request_id ON approval_audit_log(request_id); +CREATE INDEX IF NOT EXISTS idx_approval_audit_timestamp ON approval_audit_log(timestamp DESC); + +CREATE INDEX IF NOT EXISTS idx_approval_tokens_token ON approval_tokens(token); +CREATE INDEX IF NOT EXISTS idx_approval_tokens_request_id ON approval_tokens(request_id); + +-- ============================================ +-- LLM OBSERVABILITY TABLES +-- ============================================ + +-- LLM request metrics +CREATE TABLE IF NOT EXISTS llm_metrics ( + id UUID PRIMARY KEY, + request_id UUID NOT NULL, + session_id UUID NOT NULL, + bot_id UUID NOT NULL, + model VARCHAR(200) NOT NULL, + request_type VARCHAR(50) NOT NULL, + input_tokens BIGINT NOT NULL DEFAULT 0, + output_tokens BIGINT NOT NULL DEFAULT 0, + total_tokens BIGINT NOT NULL DEFAULT 0, + latency_ms BIGINT NOT NULL DEFAULT 0, + ttft_ms BIGINT, + cached BOOLEAN NOT NULL DEFAULT false, + success BOOLEAN NOT NULL DEFAULT true, + error TEXT, + estimated_cost DOUBLE PRECISION NOT NULL DEFAULT 0, + timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + metadata JSONB NOT NULL DEFAULT '{}' +); + +-- Aggregated metrics (hourly rollup) +CREATE TABLE IF NOT EXISTS llm_metrics_hourly ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL, + hour TIMESTAMP WITH TIME ZONE NOT NULL, + total_requests BIGINT NOT NULL DEFAULT 0, + successful_requests BIGINT NOT NULL DEFAULT 0, + failed_requests BIGINT NOT NULL DEFAULT 0, + cache_hits BIGINT NOT NULL DEFAULT 0, + cache_misses BIGINT NOT NULL DEFAULT 0, + total_input_tokens BIGINT NOT NULL DEFAULT 0, + total_output_tokens BIGINT NOT NULL DEFAULT 0, + total_tokens BIGINT NOT NULL DEFAULT 0, + total_cost DOUBLE PRECISION NOT NULL DEFAULT 0, + avg_latency_ms DOUBLE PRECISION NOT NULL DEFAULT 0, + p50_latency_ms DOUBLE PRECISION NOT NULL DEFAULT 0, + p95_latency_ms DOUBLE PRECISION NOT NULL DEFAULT 0, + p99_latency_ms DOUBLE PRECISION NOT NULL DEFAULT 0, + max_latency_ms BIGINT NOT NULL DEFAULT 0, + min_latency_ms BIGINT NOT NULL DEFAULT 0, + requests_by_model JSONB NOT NULL DEFAULT '{}', + tokens_by_model JSONB NOT NULL DEFAULT '{}', + cost_by_model JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + + UNIQUE(bot_id, hour) +); + +-- Budget tracking +CREATE TABLE IF NOT EXISTS llm_budget ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL UNIQUE, + daily_limit DOUBLE PRECISION NOT NULL DEFAULT 100, + monthly_limit DOUBLE PRECISION NOT NULL DEFAULT 2000, + alert_threshold DOUBLE PRECISION NOT NULL DEFAULT 0.8, + daily_spend DOUBLE PRECISION NOT NULL DEFAULT 0, + monthly_spend DOUBLE PRECISION NOT NULL DEFAULT 0, + daily_reset_date DATE NOT NULL DEFAULT CURRENT_DATE, + monthly_reset_date DATE NOT NULL DEFAULT DATE_TRUNC('month', CURRENT_DATE)::DATE, + daily_alert_sent BOOLEAN NOT NULL DEFAULT false, + monthly_alert_sent BOOLEAN NOT NULL DEFAULT false, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +-- Trace events +CREATE TABLE IF NOT EXISTS llm_traces ( + id UUID PRIMARY KEY, + parent_id UUID, + trace_id UUID NOT NULL, + name VARCHAR(200) NOT NULL, + component VARCHAR(100) NOT NULL, + event_type VARCHAR(50) NOT NULL, + duration_ms BIGINT, + start_time TIMESTAMP WITH TIME ZONE NOT NULL, + end_time TIMESTAMP WITH TIME ZONE, + attributes JSONB NOT NULL DEFAULT '{}', + status VARCHAR(50) NOT NULL DEFAULT 'in_progress', + error TEXT, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +-- Indexes for observability tables +CREATE INDEX IF NOT EXISTS idx_llm_metrics_bot_id ON llm_metrics(bot_id); +CREATE INDEX IF NOT EXISTS idx_llm_metrics_session_id ON llm_metrics(session_id); +CREATE INDEX IF NOT EXISTS idx_llm_metrics_timestamp ON llm_metrics(timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_llm_metrics_model ON llm_metrics(model); + +CREATE INDEX IF NOT EXISTS idx_llm_metrics_hourly_bot_id ON llm_metrics_hourly(bot_id); +CREATE INDEX IF NOT EXISTS idx_llm_metrics_hourly_hour ON llm_metrics_hourly(hour DESC); + +CREATE INDEX IF NOT EXISTS idx_llm_traces_trace_id ON llm_traces(trace_id); +CREATE INDEX IF NOT EXISTS idx_llm_traces_start_time ON llm_traces(start_time DESC); +CREATE INDEX IF NOT EXISTS idx_llm_traces_component ON llm_traces(component); + +-- ============================================ +-- WORKFLOW TABLES +-- ============================================ + +-- Workflow definitions +CREATE TABLE IF NOT EXISTS workflow_definitions ( + id UUID PRIMARY KEY, + bot_id UUID NOT NULL, + name VARCHAR(200) NOT NULL, + description TEXT, + steps JSONB NOT NULL DEFAULT '[]', + triggers JSONB NOT NULL DEFAULT '[]', + error_handling JSONB NOT NULL DEFAULT '{}', + enabled BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + + UNIQUE(bot_id, name) +); + +-- Workflow executions +CREATE TABLE IF NOT EXISTS workflow_executions ( + id UUID PRIMARY KEY, + workflow_id UUID NOT NULL REFERENCES workflow_definitions(id) ON DELETE CASCADE, + bot_id UUID NOT NULL, + session_id UUID, + initiated_by UUID, + status VARCHAR(50) NOT NULL DEFAULT 'pending', + current_step INTEGER NOT NULL DEFAULT 0, + input_data JSONB NOT NULL DEFAULT '{}', + output_data JSONB NOT NULL DEFAULT '{}', + step_results JSONB NOT NULL DEFAULT '[]', + error TEXT, + started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + completed_at TIMESTAMP WITH TIME ZONE, + metadata JSONB NOT NULL DEFAULT '{}' +); + +-- Workflow step executions +CREATE TABLE IF NOT EXISTS workflow_step_executions ( + id UUID PRIMARY KEY, + execution_id UUID NOT NULL REFERENCES workflow_executions(id) ON DELETE CASCADE, + step_name VARCHAR(200) NOT NULL, + step_index INTEGER NOT NULL, + status VARCHAR(50) NOT NULL DEFAULT 'pending', + input_data JSONB NOT NULL DEFAULT '{}', + output_data JSONB NOT NULL DEFAULT '{}', + error TEXT, + started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + completed_at TIMESTAMP WITH TIME ZONE, + duration_ms BIGINT +); + +-- Indexes for workflow tables +CREATE INDEX IF NOT EXISTS idx_workflow_definitions_bot_id ON workflow_definitions(bot_id); +CREATE INDEX IF NOT EXISTS idx_workflow_executions_workflow_id ON workflow_executions(workflow_id); +CREATE INDEX IF NOT EXISTS idx_workflow_executions_bot_id ON workflow_executions(bot_id); +CREATE INDEX IF NOT EXISTS idx_workflow_executions_status ON workflow_executions(status); +CREATE INDEX IF NOT EXISTS idx_workflow_step_executions_execution_id ON workflow_step_executions(execution_id); + +-- ============================================ +-- FUNCTIONS AND TRIGGERS +-- ============================================ + +-- Function to update updated_at timestamp +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ language 'plpgsql'; + +-- Triggers for updated_at +DROP TRIGGER IF EXISTS update_kg_entities_updated_at ON kg_entities; +CREATE TRIGGER update_kg_entities_updated_at + BEFORE UPDATE ON kg_entities + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +DROP TRIGGER IF EXISTS update_workflow_definitions_updated_at ON workflow_definitions; +CREATE TRIGGER update_workflow_definitions_updated_at + BEFORE UPDATE ON workflow_definitions + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +DROP TRIGGER IF EXISTS update_llm_budget_updated_at ON llm_budget; +CREATE TRIGGER update_llm_budget_updated_at + BEFORE UPDATE ON llm_budget + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +-- Function to aggregate hourly metrics +CREATE OR REPLACE FUNCTION aggregate_llm_metrics_hourly() +RETURNS void AS $$ +DECLARE + last_hour TIMESTAMP WITH TIME ZONE; +BEGIN + last_hour := DATE_TRUNC('hour', NOW() - INTERVAL '1 hour'); + + INSERT INTO llm_metrics_hourly ( + id, bot_id, hour, total_requests, successful_requests, failed_requests, + cache_hits, cache_misses, total_input_tokens, total_output_tokens, + total_tokens, total_cost, avg_latency_ms, p50_latency_ms, p95_latency_ms, + p99_latency_ms, max_latency_ms, min_latency_ms, requests_by_model, + tokens_by_model, cost_by_model + ) + SELECT + gen_random_uuid(), + bot_id, + last_hour, + COUNT(*), + COUNT(*) FILTER (WHERE success = true), + COUNT(*) FILTER (WHERE success = false), + COUNT(*) FILTER (WHERE cached = true), + COUNT(*) FILTER (WHERE cached = false), + SUM(input_tokens), + SUM(output_tokens), + SUM(total_tokens), + SUM(estimated_cost), + AVG(latency_ms), + PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY latency_ms), + PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY latency_ms), + PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY latency_ms), + MAX(latency_ms), + MIN(latency_ms), + jsonb_object_agg(model, model_count) FILTER (WHERE model IS NOT NULL), + jsonb_object_agg(model, model_tokens) FILTER (WHERE model IS NOT NULL), + jsonb_object_agg(model, model_cost) FILTER (WHERE model IS NOT NULL) + FROM ( + SELECT + bot_id, model, success, cached, input_tokens, output_tokens, + total_tokens, estimated_cost, latency_ms, + COUNT(*) OVER (PARTITION BY bot_id, model) as model_count, + SUM(total_tokens) OVER (PARTITION BY bot_id, model) as model_tokens, + SUM(estimated_cost) OVER (PARTITION BY bot_id, model) as model_cost + FROM llm_metrics + WHERE timestamp >= last_hour + AND timestamp < last_hour + INTERVAL '1 hour' + ) sub + GROUP BY bot_id + ON CONFLICT (bot_id, hour) DO UPDATE SET + total_requests = EXCLUDED.total_requests, + successful_requests = EXCLUDED.successful_requests, + failed_requests = EXCLUDED.failed_requests, + cache_hits = EXCLUDED.cache_hits, + cache_misses = EXCLUDED.cache_misses, + total_input_tokens = EXCLUDED.total_input_tokens, + total_output_tokens = EXCLUDED.total_output_tokens, + total_tokens = EXCLUDED.total_tokens, + total_cost = EXCLUDED.total_cost, + avg_latency_ms = EXCLUDED.avg_latency_ms, + p50_latency_ms = EXCLUDED.p50_latency_ms, + p95_latency_ms = EXCLUDED.p95_latency_ms, + p99_latency_ms = EXCLUDED.p99_latency_ms, + max_latency_ms = EXCLUDED.max_latency_ms, + min_latency_ms = EXCLUDED.min_latency_ms, + requests_by_model = EXCLUDED.requests_by_model, + tokens_by_model = EXCLUDED.tokens_by_model, + cost_by_model = EXCLUDED.cost_by_model; +END; +$$ LANGUAGE plpgsql; + +-- Function to reset daily budget +CREATE OR REPLACE FUNCTION reset_daily_budgets() +RETURNS void AS $$ +BEGIN + UPDATE llm_budget + SET daily_spend = 0, + daily_reset_date = CURRENT_DATE, + daily_alert_sent = false + WHERE daily_reset_date < CURRENT_DATE; +END; +$$ LANGUAGE plpgsql; + +-- Function to reset monthly budget +CREATE OR REPLACE FUNCTION reset_monthly_budgets() +RETURNS void AS $$ +BEGIN + UPDATE llm_budget + SET monthly_spend = 0, + monthly_reset_date = DATE_TRUNC('month', CURRENT_DATE)::DATE, + monthly_alert_sent = false + WHERE monthly_reset_date < DATE_TRUNC('month', CURRENT_DATE)::DATE; +END; +$$ LANGUAGE plpgsql; + +-- ============================================ +-- VIEWS +-- ============================================ + +-- View for recent episode summaries with user info +CREATE OR REPLACE VIEW v_recent_episodes AS +SELECT + e.id, + e.user_id, + e.bot_id, + e.session_id, + e.summary, + e.key_topics, + e.sentiment, + e.resolution, + e.message_count, + e.created_at, + e.conversation_start, + e.conversation_end +FROM conversation_episodes e +ORDER BY e.created_at DESC; + +-- View for knowledge graph statistics +CREATE OR REPLACE VIEW v_kg_stats AS +SELECT + bot_id, + COUNT(DISTINCT id) as total_entities, + COUNT(DISTINCT entity_type) as entity_types, + (SELECT COUNT(*) FROM kg_relationships r WHERE r.bot_id = e.bot_id) as total_relationships +FROM kg_entities e +GROUP BY bot_id; + +-- View for approval status summary +CREATE OR REPLACE VIEW v_approval_summary AS +SELECT + bot_id, + status, + COUNT(*) as count, + AVG(EXTRACT(EPOCH FROM (COALESCE(decided_at, NOW()) - created_at))) as avg_resolution_seconds +FROM approval_requests +GROUP BY bot_id, status; + +-- View for LLM usage summary (last 24 hours) +CREATE OR REPLACE VIEW v_llm_usage_24h AS +SELECT + bot_id, + model, + COUNT(*) as request_count, + SUM(total_tokens) as total_tokens, + SUM(estimated_cost) as total_cost, + AVG(latency_ms) as avg_latency_ms, + SUM(CASE WHEN cached THEN 1 ELSE 0 END)::FLOAT / COUNT(*) as cache_hit_rate, + SUM(CASE WHEN success THEN 0 ELSE 1 END)::FLOAT / COUNT(*) as error_rate +FROM llm_metrics +WHERE timestamp > NOW() - INTERVAL '24 hours' +GROUP BY bot_id, model; + +-- ============================================ +-- CLEANUP POLICIES (retention) +-- ============================================ + +-- Create a cleanup function for old data +CREATE OR REPLACE FUNCTION cleanup_old_observability_data(retention_days INTEGER DEFAULT 30) +RETURNS void AS $$ +BEGIN + -- Delete old LLM metrics (keep hourly aggregates longer) + DELETE FROM llm_metrics WHERE timestamp < NOW() - (retention_days || ' days')::INTERVAL; + + -- Delete old traces + DELETE FROM llm_traces WHERE start_time < NOW() - (retention_days || ' days')::INTERVAL; + + -- Delete old approval audit logs + DELETE FROM approval_audit_log WHERE timestamp < NOW() - (retention_days * 3 || ' days')::INTERVAL; + + -- Delete expired approval tokens + DELETE FROM approval_tokens WHERE expires_at < NOW() - INTERVAL '1 day'; +END; +$$ LANGUAGE plpgsql; +-- Suite Applications Migration +-- Adds tables for: Paper (Documents), Designer (Dialogs), and additional analytics support + +-- Paper Documents table +CREATE TABLE IF NOT EXISTS paper_documents ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT 'Untitled Document', + content TEXT NOT NULL DEFAULT '', + owner_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_paper_documents_owner ON paper_documents(owner_id); +CREATE INDEX IF NOT EXISTS idx_paper_documents_updated ON paper_documents(updated_at DESC); + +-- Designer Dialogs table +CREATE TABLE IF NOT EXISTS designer_dialogs ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + bot_id TEXT NOT NULL, + content TEXT NOT NULL DEFAULT '', + is_active BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_designer_dialogs_bot ON designer_dialogs(bot_id); +CREATE INDEX IF NOT EXISTS idx_designer_dialogs_active ON designer_dialogs(is_active); +CREATE INDEX IF NOT EXISTS idx_designer_dialogs_updated ON designer_dialogs(updated_at DESC); + +-- Sources Templates table (for template metadata caching) +CREATE TABLE IF NOT EXISTS source_templates ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT 'General', + preview_url TEXT, + file_path TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_source_templates_category ON source_templates(category); + +-- Analytics Events table (for additional event tracking) +CREATE TABLE IF NOT EXISTS analytics_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + event_type TEXT NOT NULL, + user_id UUID, + session_id UUID, + bot_id UUID, + metadata JSONB DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_analytics_events_type ON analytics_events(event_type); +CREATE INDEX IF NOT EXISTS idx_analytics_events_user ON analytics_events(user_id); +CREATE INDEX IF NOT EXISTS idx_analytics_events_session ON analytics_events(session_id); +CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at DESC); + +-- Analytics Daily Aggregates (for faster dashboard queries) +CREATE TABLE IF NOT EXISTS analytics_daily_aggregates ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + date DATE NOT NULL, + bot_id UUID, + metric_name TEXT NOT NULL, + metric_value BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(date, bot_id, metric_name) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_daily_date ON analytics_daily_aggregates(date DESC); +CREATE INDEX IF NOT EXISTS idx_analytics_daily_bot ON analytics_daily_aggregates(bot_id); + +-- Research Search History (for recent searches feature) +CREATE TABLE IF NOT EXISTS research_search_history ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL, + query TEXT NOT NULL, + collection_id TEXT, + results_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_research_history_user ON research_search_history(user_id); +CREATE INDEX IF NOT EXISTS idx_research_history_created ON research_search_history(created_at DESC); +-- Email Read Tracking Table +-- Stores sent email tracking data for read receipt functionality +-- Enabled via config.csv: email-read-pixel,true + +CREATE TABLE IF NOT EXISTS sent_email_tracking ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tracking_id UUID NOT NULL UNIQUE, + bot_id UUID NOT NULL, + account_id UUID NOT NULL, + from_email VARCHAR(255) NOT NULL, + to_email VARCHAR(255) NOT NULL, + cc TEXT, + bcc TEXT, + subject TEXT NOT NULL, + sent_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + is_read BOOLEAN NOT NULL DEFAULT FALSE, + read_at TIMESTAMPTZ, + read_count INTEGER NOT NULL DEFAULT 0, + first_read_ip VARCHAR(45), + last_read_ip VARCHAR(45), + user_agent TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Indexes for efficient queries +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_tracking_id ON sent_email_tracking(tracking_id); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_bot_id ON sent_email_tracking(bot_id); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_account_id ON sent_email_tracking(account_id); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_to_email ON sent_email_tracking(to_email); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_sent_at ON sent_email_tracking(sent_at DESC); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_is_read ON sent_email_tracking(is_read); +CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_read_status ON sent_email_tracking(bot_id, is_read, sent_at DESC); + +-- Trigger to auto-update updated_at +CREATE OR REPLACE FUNCTION update_sent_email_tracking_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trigger_update_sent_email_tracking_updated_at ON sent_email_tracking; +CREATE TRIGGER trigger_update_sent_email_tracking_updated_at + BEFORE UPDATE ON sent_email_tracking + FOR EACH ROW + EXECUTE FUNCTION update_sent_email_tracking_updated_at(); + +-- Add comment for documentation +COMMENT ON TABLE sent_email_tracking IS 'Tracks sent emails for read receipt functionality via tracking pixel'; +COMMENT ON COLUMN sent_email_tracking.tracking_id IS 'Unique ID embedded in tracking pixel URL'; +COMMENT ON COLUMN sent_email_tracking.is_read IS 'Whether the email has been opened (pixel loaded)'; +COMMENT ON COLUMN sent_email_tracking.read_count IS 'Number of times the email was opened'; +COMMENT ON COLUMN sent_email_tracking.first_read_ip IS 'IP address of first email open'; +COMMENT ON COLUMN sent_email_tracking.last_read_ip IS 'IP address of most recent email open'; +-- ============================================ +-- TABLE KEYWORD SUPPORT (from 6.1.0_table_keyword) +-- ============================================ + +-- Migration for TABLE keyword support +-- Stores dynamic table definitions created via BASIC TABLE...END TABLE syntax + +-- Table to store dynamic table definitions (metadata) +CREATE TABLE IF NOT EXISTS dynamic_table_definitions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + bot_id UUID NOT NULL, + table_name VARCHAR(255) NOT NULL, + connection_name VARCHAR(255) NOT NULL DEFAULT 'default', + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + is_active BOOLEAN DEFAULT true, + + -- Ensure unique table name per bot and connection + CONSTRAINT unique_bot_table_connection UNIQUE (bot_id, table_name, connection_name), + + -- Foreign key to bots table + CONSTRAINT fk_dynamic_table_bot + FOREIGN KEY (bot_id) + REFERENCES bots(id) + ON DELETE CASCADE +); + +-- Table to store field definitions for dynamic tables +CREATE TABLE IF NOT EXISTS dynamic_table_fields ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + table_definition_id UUID NOT NULL, + field_name VARCHAR(255) NOT NULL, + field_type VARCHAR(100) NOT NULL, + field_length INTEGER, + field_precision INTEGER, + is_key BOOLEAN DEFAULT false, + is_nullable BOOLEAN DEFAULT true, + default_value TEXT, + reference_table VARCHAR(255), + field_order INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ DEFAULT NOW(), + + -- Ensure unique field name per table definition + CONSTRAINT unique_table_field UNIQUE (table_definition_id, field_name), + + -- Foreign key to table definitions + CONSTRAINT fk_field_table_definition + FOREIGN KEY (table_definition_id) + REFERENCES dynamic_table_definitions(id) + ON DELETE CASCADE +); + +-- Table to store external database connections (from config.csv conn-* entries) +CREATE TABLE IF NOT EXISTS external_connections ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + bot_id UUID NOT NULL, + connection_name VARCHAR(255) NOT NULL, + driver VARCHAR(100) NOT NULL, + server VARCHAR(255) NOT NULL, + port INTEGER, + database_name VARCHAR(255), + username VARCHAR(255), + password_encrypted TEXT, + additional_params JSONB DEFAULT '{}'::jsonb, + is_active BOOLEAN DEFAULT true, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + last_connected_at TIMESTAMPTZ, + + -- Ensure unique connection name per bot + CONSTRAINT unique_bot_connection UNIQUE (bot_id, connection_name), + + -- Foreign key to bots table + CONSTRAINT fk_external_connection_bot + FOREIGN KEY (bot_id) + REFERENCES bots(id) + ON DELETE CASCADE +); + +-- Create indexes for efficient queries +CREATE INDEX IF NOT EXISTS idx_dynamic_table_definitions_bot_id + ON dynamic_table_definitions(bot_id); +CREATE INDEX IF NOT EXISTS idx_dynamic_table_definitions_name + ON dynamic_table_definitions(table_name); +CREATE INDEX IF NOT EXISTS idx_dynamic_table_definitions_connection + ON dynamic_table_definitions(connection_name); + +CREATE INDEX IF NOT EXISTS idx_dynamic_table_fields_table_id + ON dynamic_table_fields(table_definition_id); +CREATE INDEX IF NOT EXISTS idx_dynamic_table_fields_name + ON dynamic_table_fields(field_name); + +CREATE INDEX IF NOT EXISTS idx_external_connections_bot_id + ON external_connections(bot_id); +CREATE INDEX IF NOT EXISTS idx_external_connections_name + ON external_connections(connection_name); + +-- Create trigger to update updated_at timestamp for dynamic_table_definitions +CREATE OR REPLACE FUNCTION update_dynamic_table_definitions_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER dynamic_table_definitions_updated_at_trigger + BEFORE UPDATE ON dynamic_table_definitions + FOR EACH ROW + EXECUTE FUNCTION update_dynamic_table_definitions_updated_at(); + +-- Create trigger to update updated_at timestamp for external_connections +CREATE OR REPLACE FUNCTION update_external_connections_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER external_connections_updated_at_trigger + BEFORE UPDATE ON external_connections + FOR EACH ROW + EXECUTE FUNCTION update_external_connections_updated_at(); diff --git a/migrations/6.1.0_table_keyword/down.sql b/migrations/6.1.0_table_keyword/down.sql deleted file mode 100644 index 46f18dff..00000000 --- a/migrations/6.1.0_table_keyword/down.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Drop triggers and functions -DROP TRIGGER IF EXISTS external_connections_updated_at_trigger ON external_connections; -DROP FUNCTION IF EXISTS update_external_connections_updated_at(); - -DROP TRIGGER IF EXISTS dynamic_table_definitions_updated_at_trigger ON dynamic_table_definitions; -DROP FUNCTION IF EXISTS update_dynamic_table_definitions_updated_at(); - --- Drop indexes -DROP INDEX IF EXISTS idx_external_connections_name; -DROP INDEX IF EXISTS idx_external_connections_bot_id; - -DROP INDEX IF EXISTS idx_dynamic_table_fields_name; -DROP INDEX IF EXISTS idx_dynamic_table_fields_table_id; - -DROP INDEX IF EXISTS idx_dynamic_table_definitions_connection; -DROP INDEX IF EXISTS idx_dynamic_table_definitions_name; -DROP INDEX IF EXISTS idx_dynamic_table_definitions_bot_id; - --- Drop tables (order matters due to foreign keys) -DROP TABLE IF EXISTS external_connections; -DROP TABLE IF EXISTS dynamic_table_fields; -DROP TABLE IF EXISTS dynamic_table_definitions; diff --git a/migrations/6.1.0_table_keyword/up.sql b/migrations/6.1.0_table_keyword/up.sql deleted file mode 100644 index 80f7a38d..00000000 --- a/migrations/6.1.0_table_keyword/up.sql +++ /dev/null @@ -1,120 +0,0 @@ --- Migration for TABLE keyword support --- Stores dynamic table definitions created via BASIC TABLE...END TABLE syntax - --- Table to store dynamic table definitions (metadata) -CREATE TABLE IF NOT EXISTS dynamic_table_definitions ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - bot_id UUID NOT NULL, - table_name VARCHAR(255) NOT NULL, - connection_name VARCHAR(255) NOT NULL DEFAULT 'default', - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW(), - is_active BOOLEAN DEFAULT true, - - -- Ensure unique table name per bot and connection - CONSTRAINT unique_bot_table_connection UNIQUE (bot_id, table_name, connection_name), - - -- Foreign key to bots table - CONSTRAINT fk_dynamic_table_bot - FOREIGN KEY (bot_id) - REFERENCES bots(id) - ON DELETE CASCADE -); - --- Table to store field definitions for dynamic tables -CREATE TABLE IF NOT EXISTS dynamic_table_fields ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - table_definition_id UUID NOT NULL, - field_name VARCHAR(255) NOT NULL, - field_type VARCHAR(100) NOT NULL, - field_length INTEGER, - field_precision INTEGER, - is_key BOOLEAN DEFAULT false, - is_nullable BOOLEAN DEFAULT true, - default_value TEXT, - reference_table VARCHAR(255), - field_order INTEGER NOT NULL DEFAULT 0, - created_at TIMESTAMPTZ DEFAULT NOW(), - - -- Ensure unique field name per table definition - CONSTRAINT unique_table_field UNIQUE (table_definition_id, field_name), - - -- Foreign key to table definitions - CONSTRAINT fk_field_table_definition - FOREIGN KEY (table_definition_id) - REFERENCES dynamic_table_definitions(id) - ON DELETE CASCADE -); - --- Table to store external database connections (from config.csv conn-* entries) -CREATE TABLE IF NOT EXISTS external_connections ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - bot_id UUID NOT NULL, - connection_name VARCHAR(255) NOT NULL, - driver VARCHAR(100) NOT NULL, - server VARCHAR(255) NOT NULL, - port INTEGER, - database_name VARCHAR(255), - username VARCHAR(255), - password_encrypted TEXT, - additional_params JSONB DEFAULT '{}'::jsonb, - is_active BOOLEAN DEFAULT true, - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW(), - last_connected_at TIMESTAMPTZ, - - -- Ensure unique connection name per bot - CONSTRAINT unique_bot_connection UNIQUE (bot_id, connection_name), - - -- Foreign key to bots table - CONSTRAINT fk_external_connection_bot - FOREIGN KEY (bot_id) - REFERENCES bots(id) - ON DELETE CASCADE -); - --- Create indexes for efficient queries -CREATE INDEX IF NOT EXISTS idx_dynamic_table_definitions_bot_id - ON dynamic_table_definitions(bot_id); -CREATE INDEX IF NOT EXISTS idx_dynamic_table_definitions_name - ON dynamic_table_definitions(table_name); -CREATE INDEX IF NOT EXISTS idx_dynamic_table_definitions_connection - ON dynamic_table_definitions(connection_name); - -CREATE INDEX IF NOT EXISTS idx_dynamic_table_fields_table_id - ON dynamic_table_fields(table_definition_id); -CREATE INDEX IF NOT EXISTS idx_dynamic_table_fields_name - ON dynamic_table_fields(field_name); - -CREATE INDEX IF NOT EXISTS idx_external_connections_bot_id - ON external_connections(bot_id); -CREATE INDEX IF NOT EXISTS idx_external_connections_name - ON external_connections(connection_name); - --- Create trigger to update updated_at timestamp for dynamic_table_definitions -CREATE OR REPLACE FUNCTION update_dynamic_table_definitions_updated_at() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER dynamic_table_definitions_updated_at_trigger - BEFORE UPDATE ON dynamic_table_definitions - FOR EACH ROW - EXECUTE FUNCTION update_dynamic_table_definitions_updated_at(); - --- Create trigger to update updated_at timestamp for external_connections -CREATE OR REPLACE FUNCTION update_external_connections_updated_at() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER external_connections_updated_at_trigger - BEFORE UPDATE ON external_connections - FOR EACH ROW - EXECUTE FUNCTION update_external_connections_updated_at(); diff --git a/migrations/6.1.1_fix_config_id_types/down.sql b/migrations/6.1.1_fix_config_id_types/down.sql new file mode 100644 index 00000000..b76c5c47 --- /dev/null +++ b/migrations/6.1.1_fix_config_id_types/down.sql @@ -0,0 +1,98 @@ +-- Rollback Migration 6.1.1: Revert UUID columns back to TEXT +-- This reverts the id columns from UUID back to TEXT + +-- For bot_configuration +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'bot_configuration' + AND column_name = 'id' + AND data_type = 'uuid') THEN + ALTER TABLE bot_configuration + ALTER COLUMN id TYPE TEXT USING id::text; + END IF; +END $$; + +-- For server_configuration +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'server_configuration' + AND column_name = 'id' + AND data_type = 'uuid') THEN + ALTER TABLE server_configuration + ALTER COLUMN id TYPE TEXT USING id::text; + END IF; +END $$; + +-- For tenant_configuration +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'tenant_configuration' + AND column_name = 'id' + AND data_type = 'uuid') THEN + ALTER TABLE tenant_configuration + ALTER COLUMN id TYPE TEXT USING id::text; + END IF; +END $$; + +-- For model_configurations +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'model_configurations' + AND column_name = 'id' + AND data_type = 'uuid') THEN + ALTER TABLE model_configurations + ALTER COLUMN id TYPE TEXT USING id::text; + END IF; +END $$; + +-- For connection_configurations +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'connection_configurations' + AND column_name = 'id' + AND data_type = 'uuid') THEN + ALTER TABLE connection_configurations + ALTER COLUMN id TYPE TEXT USING id::text; + END IF; +END $$; + +-- For component_installations +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'component_installations' + AND column_name = 'id' + AND data_type = 'uuid') THEN + ALTER TABLE component_installations + ALTER COLUMN id TYPE TEXT USING id::text; + END IF; +END $$; + +-- For component_logs +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'component_logs' + AND column_name = 'id' + AND data_type = 'uuid') THEN + ALTER TABLE component_logs + ALTER COLUMN id TYPE TEXT USING id::text; + END IF; +END $$; + +-- For gbot_config_sync +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'gbot_config_sync' + AND column_name = 'id' + AND data_type = 'uuid') THEN + ALTER TABLE gbot_config_sync + ALTER COLUMN id TYPE TEXT USING id::text; + END IF; +END $$; diff --git a/migrations/6.1.1_fix_config_id_types/up.sql b/migrations/6.1.1_fix_config_id_types/up.sql new file mode 100644 index 00000000..353d3cb7 --- /dev/null +++ b/migrations/6.1.1_fix_config_id_types/up.sql @@ -0,0 +1,99 @@ +-- Migration 6.1.1: Fix bot_configuration id column type +-- The Diesel schema expects UUID but migration 6.0.4 created it as TEXT +-- This migration converts the id column from TEXT to UUID + +-- For bot_configuration (main table that needs fixing) +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'bot_configuration' + AND column_name = 'id' + AND data_type = 'text') THEN + ALTER TABLE bot_configuration + ALTER COLUMN id TYPE UUID USING id::uuid; + END IF; +END $$; + +-- Also fix server_configuration which has the same issue +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'server_configuration' + AND column_name = 'id' + AND data_type = 'text') THEN + ALTER TABLE server_configuration + ALTER COLUMN id TYPE UUID USING id::uuid; + END IF; +END $$; + +-- Also fix tenant_configuration which has the same issue +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'tenant_configuration' + AND column_name = 'id' + AND data_type = 'text') THEN + ALTER TABLE tenant_configuration + ALTER COLUMN id TYPE UUID USING id::uuid; + END IF; +END $$; + +-- Fix model_configurations +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'model_configurations' + AND column_name = 'id' + AND data_type = 'text') THEN + ALTER TABLE model_configurations + ALTER COLUMN id TYPE UUID USING id::uuid; + END IF; +END $$; + +-- Fix connection_configurations +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'connection_configurations' + AND column_name = 'id' + AND data_type = 'text') THEN + ALTER TABLE connection_configurations + ALTER COLUMN id TYPE UUID USING id::uuid; + END IF; +END $$; + +-- Fix component_installations +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'component_installations' + AND column_name = 'id' + AND data_type = 'text') THEN + ALTER TABLE component_installations + ALTER COLUMN id TYPE UUID USING id::uuid; + END IF; +END $$; + +-- Fix component_logs +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'component_logs' + AND column_name = 'id' + AND data_type = 'text') THEN + ALTER TABLE component_logs + ALTER COLUMN id TYPE UUID USING id::uuid; + END IF; +END $$; + +-- Fix gbot_config_sync +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'gbot_config_sync' + AND column_name = 'id' + AND data_type = 'text') THEN + ALTER TABLE gbot_config_sync + ALTER COLUMN id TYPE UUID USING id::uuid; + END IF; +END $$; diff --git a/migrations/6.1.1_multi_agent_memory/down.sql b/migrations/6.1.1_multi_agent_memory/down.sql deleted file mode 100644 index 1a9040b8..00000000 --- a/migrations/6.1.1_multi_agent_memory/down.sql +++ /dev/null @@ -1,64 +0,0 @@ --- Migration: 6.1.1 Multi-Agent Memory Support (DOWN) --- Description: Rollback for user memory, session preferences, and A2A protocol messaging - --- Drop triggers first -DROP TRIGGER IF EXISTS update_user_memories_updated_at ON user_memories; -DROP TRIGGER IF EXISTS update_bot_memory_extended_updated_at ON bot_memory_extended; -DROP TRIGGER IF EXISTS update_kg_entities_updated_at ON kg_entities; - --- Drop functions -DROP FUNCTION IF EXISTS update_updated_at_column(); -DROP FUNCTION IF EXISTS cleanup_expired_bot_memory(); -DROP FUNCTION IF EXISTS cleanup_expired_a2a_messages(); - --- Drop indexes (will be dropped with tables, but explicit for clarity) -DROP INDEX IF EXISTS idx_session_bots_active; -DROP INDEX IF EXISTS idx_session_bots_session; -DROP INDEX IF EXISTS idx_gen_api_tools_bot; -DROP INDEX IF EXISTS idx_conv_costs_time; -DROP INDEX IF EXISTS idx_conv_costs_bot; -DROP INDEX IF EXISTS idx_conv_costs_user; -DROP INDEX IF EXISTS idx_conv_costs_session; -DROP INDEX IF EXISTS idx_episodic_time; -DROP INDEX IF EXISTS idx_episodic_session; -DROP INDEX IF EXISTS idx_episodic_user; -DROP INDEX IF EXISTS idx_episodic_bot; -DROP INDEX IF EXISTS idx_kg_rel_type; -DROP INDEX IF EXISTS idx_kg_rel_to; -DROP INDEX IF EXISTS idx_kg_rel_from; -DROP INDEX IF EXISTS idx_kg_rel_bot; -DROP INDEX IF EXISTS idx_kg_entities_name; -DROP INDEX IF EXISTS idx_kg_entities_type; -DROP INDEX IF EXISTS idx_kg_entities_bot; -DROP INDEX IF EXISTS idx_bot_memory_ext_expires; -DROP INDEX IF EXISTS idx_bot_memory_ext_type; -DROP INDEX IF EXISTS idx_bot_memory_ext_session; -DROP INDEX IF EXISTS idx_bot_memory_ext_bot; -DROP INDEX IF EXISTS idx_a2a_messages_timestamp; -DROP INDEX IF EXISTS idx_a2a_messages_pending; -DROP INDEX IF EXISTS idx_a2a_messages_correlation; -DROP INDEX IF EXISTS idx_a2a_messages_to_agent; -DROP INDEX IF EXISTS idx_a2a_messages_session; -DROP INDEX IF EXISTS idx_session_preferences_session; -DROP INDEX IF EXISTS idx_user_memories_type; -DROP INDEX IF EXISTS idx_user_memories_user_id; -DROP INDEX IF EXISTS idx_bot_reflections_bot; -DROP INDEX IF EXISTS idx_bot_reflections_session; -DROP INDEX IF EXISTS idx_bot_reflections_time; -DROP INDEX IF EXISTS idx_conv_messages_session; -DROP INDEX IF EXISTS idx_conv_messages_time; -DROP INDEX IF EXISTS idx_conv_messages_bot; - --- Drop tables (order matters due to foreign keys) -DROP TABLE IF EXISTS conversation_messages; -DROP TABLE IF EXISTS bot_reflections; -DROP TABLE IF EXISTS session_bots; -DROP TABLE IF EXISTS generated_api_tools; -DROP TABLE IF EXISTS conversation_costs; -DROP TABLE IF EXISTS episodic_memories; -DROP TABLE IF EXISTS kg_relationships; -DROP TABLE IF EXISTS kg_entities; -DROP TABLE IF EXISTS bot_memory_extended; -DROP TABLE IF EXISTS a2a_messages; -DROP TABLE IF EXISTS session_preferences; -DROP TABLE IF EXISTS user_memories; diff --git a/migrations/6.1.1_multi_agent_memory/up.sql b/migrations/6.1.1_multi_agent_memory/up.sql deleted file mode 100644 index b7fd7ac8..00000000 --- a/migrations/6.1.1_multi_agent_memory/up.sql +++ /dev/null @@ -1,315 +0,0 @@ --- Migration: 6.1.1 Multi-Agent Memory Support --- Description: Adds tables for user memory, session preferences, and A2A protocol messaging - --- ============================================================================ --- User Memories Table --- Cross-session memory that persists for users across all sessions and bots --- ============================================================================ -CREATE TABLE IF NOT EXISTS user_memories ( - id UUID PRIMARY KEY, - user_id UUID NOT NULL, - key VARCHAR(255) NOT NULL, - value TEXT NOT NULL, - memory_type VARCHAR(50) NOT NULL DEFAULT 'preference', - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT user_memories_unique_key UNIQUE (user_id, key) -); - -CREATE INDEX IF NOT EXISTS idx_user_memories_user_id ON user_memories(user_id); -CREATE INDEX IF NOT EXISTS idx_user_memories_type ON user_memories(user_id, memory_type); - --- ============================================================================ --- Session Preferences Table --- Stores per-session configuration like current model, routing strategy, etc. --- ============================================================================ -CREATE TABLE IF NOT EXISTS session_preferences ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - session_id UUID NOT NULL, - preference_key VARCHAR(255) NOT NULL, - preference_value TEXT NOT NULL, - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT session_preferences_unique UNIQUE (session_id, preference_key) -); - -CREATE INDEX IF NOT EXISTS idx_session_preferences_session ON session_preferences(session_id); - --- ============================================================================ --- A2A Messages Table --- Agent-to-Agent protocol messages for multi-agent orchestration --- Based on https://a2a-protocol.org/latest/ --- ============================================================================ -CREATE TABLE IF NOT EXISTS a2a_messages ( - id UUID PRIMARY KEY, - session_id UUID NOT NULL, - from_agent VARCHAR(255) NOT NULL, - to_agent VARCHAR(255), -- NULL for broadcast messages - message_type VARCHAR(50) NOT NULL, - payload TEXT NOT NULL, - correlation_id UUID NOT NULL, - timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), - metadata TEXT DEFAULT '{}', - ttl_seconds INTEGER NOT NULL DEFAULT 30, - hop_count INTEGER NOT NULL DEFAULT 0, - processed BOOLEAN NOT NULL DEFAULT FALSE, - processed_at TIMESTAMPTZ, - error_message TEXT -); - -CREATE INDEX IF NOT EXISTS idx_a2a_messages_session ON a2a_messages(session_id); -CREATE INDEX IF NOT EXISTS idx_a2a_messages_to_agent ON a2a_messages(session_id, to_agent); -CREATE INDEX IF NOT EXISTS idx_a2a_messages_correlation ON a2a_messages(correlation_id); -CREATE INDEX IF NOT EXISTS idx_a2a_messages_pending ON a2a_messages(session_id, to_agent, processed) WHERE processed = FALSE; -CREATE INDEX IF NOT EXISTS idx_a2a_messages_timestamp ON a2a_messages(timestamp); - --- ============================================================================ --- Extended Bot Memory Table --- Enhanced memory with TTL and different memory types --- ============================================================================ -CREATE TABLE IF NOT EXISTS bot_memory_extended ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - session_id UUID, -- NULL for long-term memory - memory_type VARCHAR(20) NOT NULL CHECK (memory_type IN ('short', 'long', 'episodic')), - key VARCHAR(255) NOT NULL, - value TEXT NOT NULL, - ttl_seconds INTEGER, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - expires_at TIMESTAMPTZ, - CONSTRAINT bot_memory_extended_unique UNIQUE (bot_id, session_id, key) -); - -CREATE INDEX IF NOT EXISTS idx_bot_memory_ext_bot ON bot_memory_extended(bot_id); -CREATE INDEX IF NOT EXISTS idx_bot_memory_ext_session ON bot_memory_extended(bot_id, session_id); -CREATE INDEX IF NOT EXISTS idx_bot_memory_ext_type ON bot_memory_extended(bot_id, memory_type); -CREATE INDEX IF NOT EXISTS idx_bot_memory_ext_expires ON bot_memory_extended(expires_at) WHERE expires_at IS NOT NULL; - --- ============================================================================ --- Knowledge Graph Entities Table --- For graph-based memory and entity relationships --- ============================================================================ -CREATE TABLE IF NOT EXISTS kg_entities ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - entity_type VARCHAR(100) NOT NULL, - entity_name VARCHAR(500) NOT NULL, - properties JSONB DEFAULT '{}', - embedding_vector BYTEA, -- For vector similarity search - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT kg_entities_unique UNIQUE (bot_id, entity_type, entity_name) -); - -CREATE INDEX IF NOT EXISTS idx_kg_entities_bot ON kg_entities(bot_id); -CREATE INDEX IF NOT EXISTS idx_kg_entities_type ON kg_entities(bot_id, entity_type); -CREATE INDEX IF NOT EXISTS idx_kg_entities_name ON kg_entities(entity_name); - --- ============================================================================ --- Knowledge Graph Relationships Table --- For storing relationships between entities --- ============================================================================ -CREATE TABLE IF NOT EXISTS kg_relationships ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - from_entity_id UUID NOT NULL REFERENCES kg_entities(id) ON DELETE CASCADE, - to_entity_id UUID NOT NULL REFERENCES kg_entities(id) ON DELETE CASCADE, - relationship_type VARCHAR(100) NOT NULL, - properties JSONB DEFAULT '{}', - weight FLOAT DEFAULT 1.0, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT kg_relationships_unique UNIQUE (from_entity_id, to_entity_id, relationship_type) -); - -CREATE INDEX IF NOT EXISTS idx_kg_rel_bot ON kg_relationships(bot_id); -CREATE INDEX IF NOT EXISTS idx_kg_rel_from ON kg_relationships(from_entity_id); -CREATE INDEX IF NOT EXISTS idx_kg_rel_to ON kg_relationships(to_entity_id); -CREATE INDEX IF NOT EXISTS idx_kg_rel_type ON kg_relationships(bot_id, relationship_type); - --- ============================================================================ --- Episodic Memory Table --- For storing conversation summaries and episodes --- ============================================================================ -CREATE TABLE IF NOT EXISTS episodic_memories ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - user_id UUID NOT NULL, - session_id UUID, - summary TEXT NOT NULL, - key_topics JSONB DEFAULT '[]', - decisions JSONB DEFAULT '[]', - action_items JSONB DEFAULT '[]', - message_count INTEGER NOT NULL DEFAULT 0, - start_timestamp TIMESTAMPTZ NOT NULL, - end_timestamp TIMESTAMPTZ NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_episodic_bot ON episodic_memories(bot_id); -CREATE INDEX IF NOT EXISTS idx_episodic_user ON episodic_memories(user_id); -CREATE INDEX IF NOT EXISTS idx_episodic_session ON episodic_memories(session_id); -CREATE INDEX IF NOT EXISTS idx_episodic_time ON episodic_memories(bot_id, user_id, created_at); - --- ============================================================================ --- Conversation Cost Tracking Table --- For monitoring LLM usage and costs --- ============================================================================ -CREATE TABLE IF NOT EXISTS conversation_costs ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - session_id UUID NOT NULL, - user_id UUID NOT NULL, - bot_id UUID NOT NULL, - model_used VARCHAR(100), - input_tokens INTEGER NOT NULL DEFAULT 0, - output_tokens INTEGER NOT NULL DEFAULT 0, - cost_usd DECIMAL(10, 6) NOT NULL DEFAULT 0, - timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_conv_costs_session ON conversation_costs(session_id); -CREATE INDEX IF NOT EXISTS idx_conv_costs_user ON conversation_costs(user_id); -CREATE INDEX IF NOT EXISTS idx_conv_costs_bot ON conversation_costs(bot_id); -CREATE INDEX IF NOT EXISTS idx_conv_costs_time ON conversation_costs(timestamp); - --- ============================================================================ --- Generated API Tools Table --- For tracking tools generated from OpenAPI specs --- ============================================================================ -CREATE TABLE IF NOT EXISTS generated_api_tools ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - api_name VARCHAR(255) NOT NULL, - spec_url TEXT NOT NULL, - spec_hash VARCHAR(64) NOT NULL, - tool_count INTEGER NOT NULL DEFAULT 0, - last_synced_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT generated_api_tools_unique UNIQUE (bot_id, api_name) -); - -CREATE INDEX IF NOT EXISTS idx_gen_api_tools_bot ON generated_api_tools(bot_id); - --- ============================================================================ --- Session Bots Junction Table (if not exists) --- For multi-agent sessions --- ============================================================================ -CREATE TABLE IF NOT EXISTS session_bots ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - session_id UUID NOT NULL, - bot_id UUID NOT NULL, - bot_name VARCHAR(255) NOT NULL, - trigger_config JSONB DEFAULT '{}', - priority INTEGER NOT NULL DEFAULT 0, - is_active BOOLEAN NOT NULL DEFAULT TRUE, - added_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT session_bots_unique UNIQUE (session_id, bot_name) -); - -CREATE INDEX IF NOT EXISTS idx_session_bots_session ON session_bots(session_id); -CREATE INDEX IF NOT EXISTS idx_session_bots_active ON session_bots(session_id, is_active); - --- ============================================================================ --- Cleanup function for expired A2A messages --- ============================================================================ -CREATE OR REPLACE FUNCTION cleanup_expired_a2a_messages() -RETURNS INTEGER AS $$ -DECLARE - deleted_count INTEGER; -BEGIN - DELETE FROM a2a_messages - WHERE ttl_seconds > 0 - AND timestamp + (ttl_seconds || ' seconds')::INTERVAL < NOW(); - - GET DIAGNOSTICS deleted_count = ROW_COUNT; - RETURN deleted_count; -END; -$$ LANGUAGE plpgsql; - --- ============================================================================ --- Cleanup function for expired bot memory --- ============================================================================ -CREATE OR REPLACE FUNCTION cleanup_expired_bot_memory() -RETURNS INTEGER AS $$ -DECLARE - deleted_count INTEGER; -BEGIN - DELETE FROM bot_memory_extended - WHERE expires_at IS NOT NULL AND expires_at < NOW(); - - GET DIAGNOSTICS deleted_count = ROW_COUNT; - RETURN deleted_count; -END; -$$ LANGUAGE plpgsql; - --- ============================================================================ --- Trigger to update updated_at timestamp --- ============================================================================ -CREATE OR REPLACE FUNCTION update_updated_at_column() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Apply trigger to tables with updated_at -DROP TRIGGER IF EXISTS update_user_memories_updated_at ON user_memories; -CREATE TRIGGER update_user_memories_updated_at - BEFORE UPDATE ON user_memories - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -DROP TRIGGER IF EXISTS update_bot_memory_extended_updated_at ON bot_memory_extended; -CREATE TRIGGER update_bot_memory_extended_updated_at - BEFORE UPDATE ON bot_memory_extended - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -DROP TRIGGER IF EXISTS update_kg_entities_updated_at ON kg_entities; -CREATE TRIGGER update_kg_entities_updated_at - BEFORE UPDATE ON kg_entities - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - --- ============================================================================ --- Bot Reflections Table --- For storing agent self-reflection analysis results --- ============================================================================ -CREATE TABLE IF NOT EXISTS bot_reflections ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - session_id UUID NOT NULL, - reflection_type TEXT NOT NULL, - score FLOAT NOT NULL DEFAULT 0.0, - insights TEXT NOT NULL DEFAULT '[]', - improvements TEXT NOT NULL DEFAULT '[]', - positive_patterns TEXT NOT NULL DEFAULT '[]', - concerns TEXT NOT NULL DEFAULT '[]', - raw_response TEXT NOT NULL DEFAULT '', - messages_analyzed INTEGER NOT NULL DEFAULT 0, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_bot_reflections_bot ON bot_reflections(bot_id); -CREATE INDEX IF NOT EXISTS idx_bot_reflections_session ON bot_reflections(session_id); -CREATE INDEX IF NOT EXISTS idx_bot_reflections_time ON bot_reflections(bot_id, created_at); - --- ============================================================================ --- Conversation Messages Table --- For storing conversation history (if not already exists) --- ============================================================================ -CREATE TABLE IF NOT EXISTS conversation_messages ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - session_id UUID NOT NULL, - bot_id UUID NOT NULL, - user_id UUID, - role VARCHAR(50) NOT NULL, - content TEXT NOT NULL, - metadata JSONB DEFAULT '{}', - token_count INTEGER, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_conv_messages_session ON conversation_messages(session_id); -CREATE INDEX IF NOT EXISTS idx_conv_messages_time ON conversation_messages(session_id, created_at); -CREATE INDEX IF NOT EXISTS idx_conv_messages_bot ON conversation_messages(bot_id); diff --git a/migrations/6.1.2_phase3_phase4/down.sql b/migrations/6.1.2_phase3_phase4/down.sql deleted file mode 100644 index ec8b290b..00000000 --- a/migrations/6.1.2_phase3_phase4/down.sql +++ /dev/null @@ -1,124 +0,0 @@ --- Migration Rollback: 6.1.2_phase3_phase4 --- Description: Rollback Phase 3 and Phase 4 multi-agent features --- WARNING: This will delete all data in the affected tables! - --- ============================================ --- DROP VIEWS --- ============================================ - -DROP VIEW IF EXISTS v_llm_usage_24h; -DROP VIEW IF EXISTS v_approval_summary; -DROP VIEW IF EXISTS v_kg_stats; -DROP VIEW IF EXISTS v_recent_episodes; - --- ============================================ --- DROP FUNCTIONS --- ============================================ - -DROP FUNCTION IF EXISTS cleanup_old_observability_data(INTEGER); -DROP FUNCTION IF EXISTS reset_monthly_budgets(); -DROP FUNCTION IF EXISTS reset_daily_budgets(); -DROP FUNCTION IF EXISTS aggregate_llm_metrics_hourly(); - --- ============================================ --- DROP TRIGGERS --- ============================================ - -DROP TRIGGER IF EXISTS update_llm_budget_updated_at ON llm_budget; -DROP TRIGGER IF EXISTS update_workflow_definitions_updated_at ON workflow_definitions; -DROP TRIGGER IF EXISTS update_kg_entities_updated_at ON kg_entities; - --- Note: We don't drop the update_updated_at_column() function as it may be used by other tables - --- ============================================ --- DROP WORKFLOW TABLES --- ============================================ - -DROP TABLE IF EXISTS workflow_step_executions CASCADE; -DROP TABLE IF EXISTS workflow_executions CASCADE; -DROP TABLE IF EXISTS workflow_definitions CASCADE; - --- ============================================ --- DROP LLM OBSERVABILITY TABLES --- ============================================ - -DROP TABLE IF EXISTS llm_traces CASCADE; -DROP TABLE IF EXISTS llm_budget CASCADE; -DROP TABLE IF EXISTS llm_metrics_hourly CASCADE; -DROP TABLE IF EXISTS llm_metrics CASCADE; - --- ============================================ --- DROP APPROVAL TABLES --- ============================================ - -DROP TABLE IF EXISTS approval_tokens CASCADE; -DROP TABLE IF EXISTS approval_audit_log CASCADE; -DROP TABLE IF EXISTS approval_chains CASCADE; -DROP TABLE IF EXISTS approval_requests CASCADE; - --- ============================================ --- DROP KNOWLEDGE GRAPH TABLES --- ============================================ - -DROP TABLE IF EXISTS kg_relationships CASCADE; -DROP TABLE IF EXISTS kg_entities CASCADE; - --- ============================================ --- DROP EPISODIC MEMORY TABLES --- ============================================ - -DROP TABLE IF EXISTS conversation_episodes CASCADE; - --- ============================================ --- DROP INDEXES (if any remain) --- ============================================ - --- Episodic memory indexes -DROP INDEX IF EXISTS idx_episodes_user_id; -DROP INDEX IF EXISTS idx_episodes_bot_id; -DROP INDEX IF EXISTS idx_episodes_session_id; -DROP INDEX IF EXISTS idx_episodes_created_at; -DROP INDEX IF EXISTS idx_episodes_key_topics; -DROP INDEX IF EXISTS idx_episodes_resolution; -DROP INDEX IF EXISTS idx_episodes_summary_fts; - --- Knowledge graph indexes -DROP INDEX IF EXISTS idx_kg_entities_bot_id; -DROP INDEX IF EXISTS idx_kg_entities_type; -DROP INDEX IF EXISTS idx_kg_entities_name; -DROP INDEX IF EXISTS idx_kg_entities_name_lower; -DROP INDEX IF EXISTS idx_kg_entities_aliases; -DROP INDEX IF EXISTS idx_kg_entities_name_fts; -DROP INDEX IF EXISTS idx_kg_relationships_bot_id; -DROP INDEX IF EXISTS idx_kg_relationships_from; -DROP INDEX IF EXISTS idx_kg_relationships_to; -DROP INDEX IF EXISTS idx_kg_relationships_type; - --- Approval indexes -DROP INDEX IF EXISTS idx_approval_requests_bot_id; -DROP INDEX IF EXISTS idx_approval_requests_session_id; -DROP INDEX IF EXISTS idx_approval_requests_status; -DROP INDEX IF EXISTS idx_approval_requests_expires_at; -DROP INDEX IF EXISTS idx_approval_requests_pending; -DROP INDEX IF EXISTS idx_approval_audit_request_id; -DROP INDEX IF EXISTS idx_approval_audit_timestamp; -DROP INDEX IF EXISTS idx_approval_tokens_token; -DROP INDEX IF EXISTS idx_approval_tokens_request_id; - --- Observability indexes -DROP INDEX IF EXISTS idx_llm_metrics_bot_id; -DROP INDEX IF EXISTS idx_llm_metrics_session_id; -DROP INDEX IF EXISTS idx_llm_metrics_timestamp; -DROP INDEX IF EXISTS idx_llm_metrics_model; -DROP INDEX IF EXISTS idx_llm_metrics_hourly_bot_id; -DROP INDEX IF EXISTS idx_llm_metrics_hourly_hour; -DROP INDEX IF EXISTS idx_llm_traces_trace_id; -DROP INDEX IF EXISTS idx_llm_traces_start_time; -DROP INDEX IF EXISTS idx_llm_traces_component; - --- Workflow indexes -DROP INDEX IF EXISTS idx_workflow_definitions_bot_id; -DROP INDEX IF EXISTS idx_workflow_executions_workflow_id; -DROP INDEX IF EXISTS idx_workflow_executions_bot_id; -DROP INDEX IF EXISTS idx_workflow_executions_status; -DROP INDEX IF EXISTS idx_workflow_step_executions_execution_id; diff --git a/migrations/6.1.2_phase3_phase4/up.sql b/migrations/6.1.2_phase3_phase4/up.sql deleted file mode 100644 index 19a7bd3f..00000000 --- a/migrations/6.1.2_phase3_phase4/up.sql +++ /dev/null @@ -1,538 +0,0 @@ --- Migration: 6.1.2_phase3_phase4 --- Description: Phase 3 and Phase 4 multi-agent features --- Features: --- - Episodic memory (conversation summaries) --- - Knowledge graphs (entity relationships) --- - Human-in-the-loop approvals --- - LLM observability and cost tracking - --- ============================================ --- EPISODIC MEMORY TABLES --- ============================================ - --- Conversation episodes (summaries) -CREATE TABLE IF NOT EXISTS conversation_episodes ( - id UUID PRIMARY KEY, - user_id UUID NOT NULL, - bot_id UUID NOT NULL, - session_id UUID NOT NULL, - summary TEXT NOT NULL, - key_topics JSONB NOT NULL DEFAULT '[]', - decisions JSONB NOT NULL DEFAULT '[]', - action_items JSONB NOT NULL DEFAULT '[]', - sentiment JSONB NOT NULL DEFAULT '{"score": 0, "label": "neutral", "confidence": 0.5}', - resolution VARCHAR(50) NOT NULL DEFAULT 'unknown', - message_count INTEGER NOT NULL DEFAULT 0, - message_ids JSONB NOT NULL DEFAULT '[]', - conversation_start TIMESTAMP WITH TIME ZONE NOT NULL, - conversation_end TIMESTAMP WITH TIME ZONE NOT NULL, - metadata JSONB NOT NULL DEFAULT '{}', - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() -); - --- Indexes for episodic memory -CREATE INDEX IF NOT EXISTS idx_episodes_user_id ON conversation_episodes(user_id); -CREATE INDEX IF NOT EXISTS idx_episodes_bot_id ON conversation_episodes(bot_id); -CREATE INDEX IF NOT EXISTS idx_episodes_session_id ON conversation_episodes(session_id); -CREATE INDEX IF NOT EXISTS idx_episodes_created_at ON conversation_episodes(created_at DESC); -CREATE INDEX IF NOT EXISTS idx_episodes_key_topics ON conversation_episodes USING GIN(key_topics); -CREATE INDEX IF NOT EXISTS idx_episodes_resolution ON conversation_episodes(resolution); - --- Full-text search on summaries -CREATE INDEX IF NOT EXISTS idx_episodes_summary_fts ON conversation_episodes - USING GIN(to_tsvector('english', summary)); - --- ============================================ --- KNOWLEDGE GRAPH TABLES --- ============================================ - --- Knowledge graph entities -CREATE TABLE IF NOT EXISTS kg_entities ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - entity_type VARCHAR(100) NOT NULL, - entity_name VARCHAR(500) NOT NULL, - aliases JSONB NOT NULL DEFAULT '[]', - properties JSONB NOT NULL DEFAULT '{}', - confidence DOUBLE PRECISION NOT NULL DEFAULT 1.0, - source VARCHAR(50) NOT NULL DEFAULT 'manual', - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - UNIQUE(bot_id, entity_type, entity_name) -); - --- Knowledge graph relationships -CREATE TABLE IF NOT EXISTS kg_relationships ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - from_entity_id UUID NOT NULL REFERENCES kg_entities(id) ON DELETE CASCADE, - to_entity_id UUID NOT NULL REFERENCES kg_entities(id) ON DELETE CASCADE, - relationship_type VARCHAR(100) NOT NULL, - properties JSONB NOT NULL DEFAULT '{}', - confidence DOUBLE PRECISION NOT NULL DEFAULT 1.0, - bidirectional BOOLEAN NOT NULL DEFAULT false, - source VARCHAR(50) NOT NULL DEFAULT 'manual', - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - UNIQUE(bot_id, from_entity_id, to_entity_id, relationship_type) -); - --- Indexes for knowledge graph -CREATE INDEX IF NOT EXISTS idx_kg_entities_bot_id ON kg_entities(bot_id); -CREATE INDEX IF NOT EXISTS idx_kg_entities_type ON kg_entities(entity_type); -CREATE INDEX IF NOT EXISTS idx_kg_entities_name ON kg_entities(entity_name); -CREATE INDEX IF NOT EXISTS idx_kg_entities_name_lower ON kg_entities(LOWER(entity_name)); -CREATE INDEX IF NOT EXISTS idx_kg_entities_aliases ON kg_entities USING GIN(aliases); - -CREATE INDEX IF NOT EXISTS idx_kg_relationships_bot_id ON kg_relationships(bot_id); -CREATE INDEX IF NOT EXISTS idx_kg_relationships_from ON kg_relationships(from_entity_id); -CREATE INDEX IF NOT EXISTS idx_kg_relationships_to ON kg_relationships(to_entity_id); -CREATE INDEX IF NOT EXISTS idx_kg_relationships_type ON kg_relationships(relationship_type); - --- Full-text search on entity names -CREATE INDEX IF NOT EXISTS idx_kg_entities_name_fts ON kg_entities - USING GIN(to_tsvector('english', entity_name)); - --- ============================================ --- HUMAN-IN-THE-LOOP APPROVAL TABLES --- ============================================ - --- Approval requests -CREATE TABLE IF NOT EXISTS approval_requests ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - session_id UUID NOT NULL, - initiated_by UUID NOT NULL, - approval_type VARCHAR(100) NOT NULL, - status VARCHAR(50) NOT NULL DEFAULT 'pending', - channel VARCHAR(50) NOT NULL, - recipient VARCHAR(500) NOT NULL, - context JSONB NOT NULL DEFAULT '{}', - message TEXT NOT NULL, - timeout_seconds INTEGER NOT NULL DEFAULT 3600, - default_action VARCHAR(50), - current_level INTEGER NOT NULL DEFAULT 1, - total_levels INTEGER NOT NULL DEFAULT 1, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - expires_at TIMESTAMP WITH TIME ZONE NOT NULL, - reminders_sent JSONB NOT NULL DEFAULT '[]', - decision VARCHAR(50), - decided_by VARCHAR(500), - decided_at TIMESTAMP WITH TIME ZONE, - comments TEXT -); - --- Approval chains -CREATE TABLE IF NOT EXISTS approval_chains ( - id UUID PRIMARY KEY, - name VARCHAR(200) NOT NULL, - bot_id UUID NOT NULL, - levels JSONB NOT NULL DEFAULT '[]', - stop_on_reject BOOLEAN NOT NULL DEFAULT true, - require_all BOOLEAN NOT NULL DEFAULT false, - description TEXT, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - UNIQUE(bot_id, name) -); - --- Approval audit log -CREATE TABLE IF NOT EXISTS approval_audit_log ( - id UUID PRIMARY KEY, - request_id UUID NOT NULL REFERENCES approval_requests(id) ON DELETE CASCADE, - action VARCHAR(50) NOT NULL, - actor VARCHAR(500) NOT NULL, - details JSONB NOT NULL DEFAULT '{}', - timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - ip_address VARCHAR(50), - user_agent TEXT -); - --- Approval tokens (for secure links) -CREATE TABLE IF NOT EXISTS approval_tokens ( - id UUID PRIMARY KEY, - request_id UUID NOT NULL REFERENCES approval_requests(id) ON DELETE CASCADE, - token VARCHAR(100) NOT NULL UNIQUE, - action VARCHAR(50) NOT NULL, - used BOOLEAN NOT NULL DEFAULT false, - used_at TIMESTAMP WITH TIME ZONE, - expires_at TIMESTAMP WITH TIME ZONE NOT NULL, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() -); - --- Indexes for approval tables -CREATE INDEX IF NOT EXISTS idx_approval_requests_bot_id ON approval_requests(bot_id); -CREATE INDEX IF NOT EXISTS idx_approval_requests_session_id ON approval_requests(session_id); -CREATE INDEX IF NOT EXISTS idx_approval_requests_status ON approval_requests(status); -CREATE INDEX IF NOT EXISTS idx_approval_requests_expires_at ON approval_requests(expires_at); -CREATE INDEX IF NOT EXISTS idx_approval_requests_pending ON approval_requests(status, expires_at) - WHERE status = 'pending'; - -CREATE INDEX IF NOT EXISTS idx_approval_audit_request_id ON approval_audit_log(request_id); -CREATE INDEX IF NOT EXISTS idx_approval_audit_timestamp ON approval_audit_log(timestamp DESC); - -CREATE INDEX IF NOT EXISTS idx_approval_tokens_token ON approval_tokens(token); -CREATE INDEX IF NOT EXISTS idx_approval_tokens_request_id ON approval_tokens(request_id); - --- ============================================ --- LLM OBSERVABILITY TABLES --- ============================================ - --- LLM request metrics -CREATE TABLE IF NOT EXISTS llm_metrics ( - id UUID PRIMARY KEY, - request_id UUID NOT NULL, - session_id UUID NOT NULL, - bot_id UUID NOT NULL, - model VARCHAR(200) NOT NULL, - request_type VARCHAR(50) NOT NULL, - input_tokens BIGINT NOT NULL DEFAULT 0, - output_tokens BIGINT NOT NULL DEFAULT 0, - total_tokens BIGINT NOT NULL DEFAULT 0, - latency_ms BIGINT NOT NULL DEFAULT 0, - ttft_ms BIGINT, - cached BOOLEAN NOT NULL DEFAULT false, - success BOOLEAN NOT NULL DEFAULT true, - error TEXT, - estimated_cost DOUBLE PRECISION NOT NULL DEFAULT 0, - timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB NOT NULL DEFAULT '{}' -); - --- Aggregated metrics (hourly rollup) -CREATE TABLE IF NOT EXISTS llm_metrics_hourly ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - hour TIMESTAMP WITH TIME ZONE NOT NULL, - total_requests BIGINT NOT NULL DEFAULT 0, - successful_requests BIGINT NOT NULL DEFAULT 0, - failed_requests BIGINT NOT NULL DEFAULT 0, - cache_hits BIGINT NOT NULL DEFAULT 0, - cache_misses BIGINT NOT NULL DEFAULT 0, - total_input_tokens BIGINT NOT NULL DEFAULT 0, - total_output_tokens BIGINT NOT NULL DEFAULT 0, - total_tokens BIGINT NOT NULL DEFAULT 0, - total_cost DOUBLE PRECISION NOT NULL DEFAULT 0, - avg_latency_ms DOUBLE PRECISION NOT NULL DEFAULT 0, - p50_latency_ms DOUBLE PRECISION NOT NULL DEFAULT 0, - p95_latency_ms DOUBLE PRECISION NOT NULL DEFAULT 0, - p99_latency_ms DOUBLE PRECISION NOT NULL DEFAULT 0, - max_latency_ms BIGINT NOT NULL DEFAULT 0, - min_latency_ms BIGINT NOT NULL DEFAULT 0, - requests_by_model JSONB NOT NULL DEFAULT '{}', - tokens_by_model JSONB NOT NULL DEFAULT '{}', - cost_by_model JSONB NOT NULL DEFAULT '{}', - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - UNIQUE(bot_id, hour) -); - --- Budget tracking -CREATE TABLE IF NOT EXISTS llm_budget ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL UNIQUE, - daily_limit DOUBLE PRECISION NOT NULL DEFAULT 100, - monthly_limit DOUBLE PRECISION NOT NULL DEFAULT 2000, - alert_threshold DOUBLE PRECISION NOT NULL DEFAULT 0.8, - daily_spend DOUBLE PRECISION NOT NULL DEFAULT 0, - monthly_spend DOUBLE PRECISION NOT NULL DEFAULT 0, - daily_reset_date DATE NOT NULL DEFAULT CURRENT_DATE, - monthly_reset_date DATE NOT NULL DEFAULT DATE_TRUNC('month', CURRENT_DATE)::DATE, - daily_alert_sent BOOLEAN NOT NULL DEFAULT false, - monthly_alert_sent BOOLEAN NOT NULL DEFAULT false, - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() -); - --- Trace events -CREATE TABLE IF NOT EXISTS llm_traces ( - id UUID PRIMARY KEY, - parent_id UUID, - trace_id UUID NOT NULL, - name VARCHAR(200) NOT NULL, - component VARCHAR(100) NOT NULL, - event_type VARCHAR(50) NOT NULL, - duration_ms BIGINT, - start_time TIMESTAMP WITH TIME ZONE NOT NULL, - end_time TIMESTAMP WITH TIME ZONE, - attributes JSONB NOT NULL DEFAULT '{}', - status VARCHAR(50) NOT NULL DEFAULT 'in_progress', - error TEXT, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() -); - --- Indexes for observability tables -CREATE INDEX IF NOT EXISTS idx_llm_metrics_bot_id ON llm_metrics(bot_id); -CREATE INDEX IF NOT EXISTS idx_llm_metrics_session_id ON llm_metrics(session_id); -CREATE INDEX IF NOT EXISTS idx_llm_metrics_timestamp ON llm_metrics(timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_llm_metrics_model ON llm_metrics(model); - -CREATE INDEX IF NOT EXISTS idx_llm_metrics_hourly_bot_id ON llm_metrics_hourly(bot_id); -CREATE INDEX IF NOT EXISTS idx_llm_metrics_hourly_hour ON llm_metrics_hourly(hour DESC); - -CREATE INDEX IF NOT EXISTS idx_llm_traces_trace_id ON llm_traces(trace_id); -CREATE INDEX IF NOT EXISTS idx_llm_traces_start_time ON llm_traces(start_time DESC); -CREATE INDEX IF NOT EXISTS idx_llm_traces_component ON llm_traces(component); - --- ============================================ --- WORKFLOW TABLES --- ============================================ - --- Workflow definitions -CREATE TABLE IF NOT EXISTS workflow_definitions ( - id UUID PRIMARY KEY, - bot_id UUID NOT NULL, - name VARCHAR(200) NOT NULL, - description TEXT, - steps JSONB NOT NULL DEFAULT '[]', - triggers JSONB NOT NULL DEFAULT '[]', - error_handling JSONB NOT NULL DEFAULT '{}', - enabled BOOLEAN NOT NULL DEFAULT true, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - UNIQUE(bot_id, name) -); - --- Workflow executions -CREATE TABLE IF NOT EXISTS workflow_executions ( - id UUID PRIMARY KEY, - workflow_id UUID NOT NULL REFERENCES workflow_definitions(id) ON DELETE CASCADE, - bot_id UUID NOT NULL, - session_id UUID, - initiated_by UUID, - status VARCHAR(50) NOT NULL DEFAULT 'pending', - current_step INTEGER NOT NULL DEFAULT 0, - input_data JSONB NOT NULL DEFAULT '{}', - output_data JSONB NOT NULL DEFAULT '{}', - step_results JSONB NOT NULL DEFAULT '[]', - error TEXT, - started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - completed_at TIMESTAMP WITH TIME ZONE, - metadata JSONB NOT NULL DEFAULT '{}' -); - --- Workflow step executions -CREATE TABLE IF NOT EXISTS workflow_step_executions ( - id UUID PRIMARY KEY, - execution_id UUID NOT NULL REFERENCES workflow_executions(id) ON DELETE CASCADE, - step_name VARCHAR(200) NOT NULL, - step_index INTEGER NOT NULL, - status VARCHAR(50) NOT NULL DEFAULT 'pending', - input_data JSONB NOT NULL DEFAULT '{}', - output_data JSONB NOT NULL DEFAULT '{}', - error TEXT, - started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - completed_at TIMESTAMP WITH TIME ZONE, - duration_ms BIGINT -); - --- Indexes for workflow tables -CREATE INDEX IF NOT EXISTS idx_workflow_definitions_bot_id ON workflow_definitions(bot_id); -CREATE INDEX IF NOT EXISTS idx_workflow_executions_workflow_id ON workflow_executions(workflow_id); -CREATE INDEX IF NOT EXISTS idx_workflow_executions_bot_id ON workflow_executions(bot_id); -CREATE INDEX IF NOT EXISTS idx_workflow_executions_status ON workflow_executions(status); -CREATE INDEX IF NOT EXISTS idx_workflow_step_executions_execution_id ON workflow_step_executions(execution_id); - --- ============================================ --- FUNCTIONS AND TRIGGERS --- ============================================ - --- Function to update updated_at timestamp -CREATE OR REPLACE FUNCTION update_updated_at_column() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ language 'plpgsql'; - --- Triggers for updated_at -DROP TRIGGER IF EXISTS update_kg_entities_updated_at ON kg_entities; -CREATE TRIGGER update_kg_entities_updated_at - BEFORE UPDATE ON kg_entities - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - -DROP TRIGGER IF EXISTS update_workflow_definitions_updated_at ON workflow_definitions; -CREATE TRIGGER update_workflow_definitions_updated_at - BEFORE UPDATE ON workflow_definitions - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - -DROP TRIGGER IF EXISTS update_llm_budget_updated_at ON llm_budget; -CREATE TRIGGER update_llm_budget_updated_at - BEFORE UPDATE ON llm_budget - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - --- Function to aggregate hourly metrics -CREATE OR REPLACE FUNCTION aggregate_llm_metrics_hourly() -RETURNS void AS $$ -DECLARE - last_hour TIMESTAMP WITH TIME ZONE; -BEGIN - last_hour := DATE_TRUNC('hour', NOW() - INTERVAL '1 hour'); - - INSERT INTO llm_metrics_hourly ( - id, bot_id, hour, total_requests, successful_requests, failed_requests, - cache_hits, cache_misses, total_input_tokens, total_output_tokens, - total_tokens, total_cost, avg_latency_ms, p50_latency_ms, p95_latency_ms, - p99_latency_ms, max_latency_ms, min_latency_ms, requests_by_model, - tokens_by_model, cost_by_model - ) - SELECT - gen_random_uuid(), - bot_id, - last_hour, - COUNT(*), - COUNT(*) FILTER (WHERE success = true), - COUNT(*) FILTER (WHERE success = false), - COUNT(*) FILTER (WHERE cached = true), - COUNT(*) FILTER (WHERE cached = false), - SUM(input_tokens), - SUM(output_tokens), - SUM(total_tokens), - SUM(estimated_cost), - AVG(latency_ms), - PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY latency_ms), - PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY latency_ms), - PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY latency_ms), - MAX(latency_ms), - MIN(latency_ms), - jsonb_object_agg(model, model_count) FILTER (WHERE model IS NOT NULL), - jsonb_object_agg(model, model_tokens) FILTER (WHERE model IS NOT NULL), - jsonb_object_agg(model, model_cost) FILTER (WHERE model IS NOT NULL) - FROM ( - SELECT - bot_id, model, success, cached, input_tokens, output_tokens, - total_tokens, estimated_cost, latency_ms, - COUNT(*) OVER (PARTITION BY bot_id, model) as model_count, - SUM(total_tokens) OVER (PARTITION BY bot_id, model) as model_tokens, - SUM(estimated_cost) OVER (PARTITION BY bot_id, model) as model_cost - FROM llm_metrics - WHERE timestamp >= last_hour - AND timestamp < last_hour + INTERVAL '1 hour' - ) sub - GROUP BY bot_id - ON CONFLICT (bot_id, hour) DO UPDATE SET - total_requests = EXCLUDED.total_requests, - successful_requests = EXCLUDED.successful_requests, - failed_requests = EXCLUDED.failed_requests, - cache_hits = EXCLUDED.cache_hits, - cache_misses = EXCLUDED.cache_misses, - total_input_tokens = EXCLUDED.total_input_tokens, - total_output_tokens = EXCLUDED.total_output_tokens, - total_tokens = EXCLUDED.total_tokens, - total_cost = EXCLUDED.total_cost, - avg_latency_ms = EXCLUDED.avg_latency_ms, - p50_latency_ms = EXCLUDED.p50_latency_ms, - p95_latency_ms = EXCLUDED.p95_latency_ms, - p99_latency_ms = EXCLUDED.p99_latency_ms, - max_latency_ms = EXCLUDED.max_latency_ms, - min_latency_ms = EXCLUDED.min_latency_ms, - requests_by_model = EXCLUDED.requests_by_model, - tokens_by_model = EXCLUDED.tokens_by_model, - cost_by_model = EXCLUDED.cost_by_model; -END; -$$ LANGUAGE plpgsql; - --- Function to reset daily budget -CREATE OR REPLACE FUNCTION reset_daily_budgets() -RETURNS void AS $$ -BEGIN - UPDATE llm_budget - SET daily_spend = 0, - daily_reset_date = CURRENT_DATE, - daily_alert_sent = false - WHERE daily_reset_date < CURRENT_DATE; -END; -$$ LANGUAGE plpgsql; - --- Function to reset monthly budget -CREATE OR REPLACE FUNCTION reset_monthly_budgets() -RETURNS void AS $$ -BEGIN - UPDATE llm_budget - SET monthly_spend = 0, - monthly_reset_date = DATE_TRUNC('month', CURRENT_DATE)::DATE, - monthly_alert_sent = false - WHERE monthly_reset_date < DATE_TRUNC('month', CURRENT_DATE)::DATE; -END; -$$ LANGUAGE plpgsql; - --- ============================================ --- VIEWS --- ============================================ - --- View for recent episode summaries with user info -CREATE OR REPLACE VIEW v_recent_episodes AS -SELECT - e.id, - e.user_id, - e.bot_id, - e.session_id, - e.summary, - e.key_topics, - e.sentiment, - e.resolution, - e.message_count, - e.created_at, - e.conversation_start, - e.conversation_end -FROM conversation_episodes e -ORDER BY e.created_at DESC; - --- View for knowledge graph statistics -CREATE OR REPLACE VIEW v_kg_stats AS -SELECT - bot_id, - COUNT(DISTINCT id) as total_entities, - COUNT(DISTINCT entity_type) as entity_types, - (SELECT COUNT(*) FROM kg_relationships r WHERE r.bot_id = e.bot_id) as total_relationships -FROM kg_entities e -GROUP BY bot_id; - --- View for approval status summary -CREATE OR REPLACE VIEW v_approval_summary AS -SELECT - bot_id, - status, - COUNT(*) as count, - AVG(EXTRACT(EPOCH FROM (COALESCE(decided_at, NOW()) - created_at))) as avg_resolution_seconds -FROM approval_requests -GROUP BY bot_id, status; - --- View for LLM usage summary (last 24 hours) -CREATE OR REPLACE VIEW v_llm_usage_24h AS -SELECT - bot_id, - model, - COUNT(*) as request_count, - SUM(total_tokens) as total_tokens, - SUM(estimated_cost) as total_cost, - AVG(latency_ms) as avg_latency_ms, - SUM(CASE WHEN cached THEN 1 ELSE 0 END)::FLOAT / COUNT(*) as cache_hit_rate, - SUM(CASE WHEN success THEN 0 ELSE 1 END)::FLOAT / COUNT(*) as error_rate -FROM llm_metrics -WHERE timestamp > NOW() - INTERVAL '24 hours' -GROUP BY bot_id, model; - --- ============================================ --- CLEANUP POLICIES (retention) --- ============================================ - --- Create a cleanup function for old data -CREATE OR REPLACE FUNCTION cleanup_old_observability_data(retention_days INTEGER DEFAULT 30) -RETURNS void AS $$ -BEGIN - -- Delete old LLM metrics (keep hourly aggregates longer) - DELETE FROM llm_metrics WHERE timestamp < NOW() - (retention_days || ' days')::INTERVAL; - - -- Delete old traces - DELETE FROM llm_traces WHERE start_time < NOW() - (retention_days || ' days')::INTERVAL; - - -- Delete old approval audit logs - DELETE FROM approval_audit_log WHERE timestamp < NOW() - (retention_days * 3 || ' days')::INTERVAL; - - -- Delete expired approval tokens - DELETE FROM approval_tokens WHERE expires_at < NOW() - INTERVAL '1 day'; -END; -$$ LANGUAGE plpgsql; diff --git a/migrations/6.2.0_suite_apps/down.sql b/migrations/6.2.0_suite_apps/down.sql deleted file mode 100644 index 71603d95..00000000 --- a/migrations/6.2.0_suite_apps/down.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Rollback Suite Applications Migration --- Removes tables for: Paper (Documents), Designer (Dialogs), and analytics support - --- Drop indexes first -DROP INDEX IF EXISTS idx_research_history_created; -DROP INDEX IF EXISTS idx_research_history_user; -DROP INDEX IF EXISTS idx_analytics_daily_bot; -DROP INDEX IF EXISTS idx_analytics_daily_date; -DROP INDEX IF EXISTS idx_analytics_events_created; -DROP INDEX IF EXISTS idx_analytics_events_session; -DROP INDEX IF EXISTS idx_analytics_events_user; -DROP INDEX IF EXISTS idx_analytics_events_type; -DROP INDEX IF EXISTS idx_source_templates_category; -DROP INDEX IF EXISTS idx_designer_dialogs_updated; -DROP INDEX IF EXISTS idx_designer_dialogs_active; -DROP INDEX IF EXISTS idx_designer_dialogs_bot; -DROP INDEX IF EXISTS idx_paper_documents_updated; -DROP INDEX IF EXISTS idx_paper_documents_owner; - --- Drop tables -DROP TABLE IF EXISTS research_search_history; -DROP TABLE IF EXISTS analytics_daily_aggregates; -DROP TABLE IF EXISTS analytics_events; -DROP TABLE IF EXISTS source_templates; -DROP TABLE IF EXISTS designer_dialogs; -DROP TABLE IF EXISTS paper_documents; diff --git a/migrations/6.2.0_suite_apps/up.sql b/migrations/6.2.0_suite_apps/up.sql deleted file mode 100644 index 980d8bdc..00000000 --- a/migrations/6.2.0_suite_apps/up.sql +++ /dev/null @@ -1,87 +0,0 @@ --- Suite Applications Migration --- Adds tables for: Paper (Documents), Designer (Dialogs), and additional analytics support - --- Paper Documents table -CREATE TABLE IF NOT EXISTS paper_documents ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL DEFAULT 'Untitled Document', - content TEXT NOT NULL DEFAULT '', - owner_id TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_paper_documents_owner ON paper_documents(owner_id); -CREATE INDEX IF NOT EXISTS idx_paper_documents_updated ON paper_documents(updated_at DESC); - --- Designer Dialogs table -CREATE TABLE IF NOT EXISTS designer_dialogs ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - description TEXT NOT NULL DEFAULT '', - bot_id TEXT NOT NULL, - content TEXT NOT NULL DEFAULT '', - is_active BOOLEAN NOT NULL DEFAULT false, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_designer_dialogs_bot ON designer_dialogs(bot_id); -CREATE INDEX IF NOT EXISTS idx_designer_dialogs_active ON designer_dialogs(is_active); -CREATE INDEX IF NOT EXISTS idx_designer_dialogs_updated ON designer_dialogs(updated_at DESC); - --- Sources Templates table (for template metadata caching) -CREATE TABLE IF NOT EXISTS source_templates ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - description TEXT NOT NULL DEFAULT '', - category TEXT NOT NULL DEFAULT 'General', - preview_url TEXT, - file_path TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_source_templates_category ON source_templates(category); - --- Analytics Events table (for additional event tracking) -CREATE TABLE IF NOT EXISTS analytics_events ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - event_type TEXT NOT NULL, - user_id UUID, - session_id UUID, - bot_id UUID, - metadata JSONB DEFAULT '{}', - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_analytics_events_type ON analytics_events(event_type); -CREATE INDEX IF NOT EXISTS idx_analytics_events_user ON analytics_events(user_id); -CREATE INDEX IF NOT EXISTS idx_analytics_events_session ON analytics_events(session_id); -CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at DESC); - --- Analytics Daily Aggregates (for faster dashboard queries) -CREATE TABLE IF NOT EXISTS analytics_daily_aggregates ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - date DATE NOT NULL, - bot_id UUID, - metric_name TEXT NOT NULL, - metric_value BIGINT NOT NULL DEFAULT 0, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - UNIQUE(date, bot_id, metric_name) -); - -CREATE INDEX IF NOT EXISTS idx_analytics_daily_date ON analytics_daily_aggregates(date DESC); -CREATE INDEX IF NOT EXISTS idx_analytics_daily_bot ON analytics_daily_aggregates(bot_id); - --- Research Search History (for recent searches feature) -CREATE TABLE IF NOT EXISTS research_search_history ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - user_id UUID NOT NULL, - query TEXT NOT NULL, - collection_id TEXT, - results_count INTEGER NOT NULL DEFAULT 0, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS idx_research_history_user ON research_search_history(user_id); -CREATE INDEX IF NOT EXISTS idx_research_history_created ON research_search_history(created_at DESC); diff --git a/migrations/6.2.1_email_tracking/down.sql b/migrations/6.2.1_email_tracking/down.sql deleted file mode 100644 index 0e8cbafe..00000000 --- a/migrations/6.2.1_email_tracking/down.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Down migration: Remove email tracking table and related objects - --- Drop trigger first -DROP TRIGGER IF EXISTS trigger_update_sent_email_tracking_updated_at ON sent_email_tracking; - --- Drop function -DROP FUNCTION IF EXISTS update_sent_email_tracking_updated_at(); - --- Drop indexes -DROP INDEX IF EXISTS idx_sent_email_tracking_tracking_id; -DROP INDEX IF EXISTS idx_sent_email_tracking_bot_id; -DROP INDEX IF EXISTS idx_sent_email_tracking_account_id; -DROP INDEX IF EXISTS idx_sent_email_tracking_to_email; -DROP INDEX IF EXISTS idx_sent_email_tracking_sent_at; -DROP INDEX IF EXISTS idx_sent_email_tracking_is_read; -DROP INDEX IF EXISTS idx_sent_email_tracking_read_status; - --- Drop table -DROP TABLE IF EXISTS sent_email_tracking; diff --git a/migrations/6.2.1_email_tracking/up.sql b/migrations/6.2.1_email_tracking/up.sql deleted file mode 100644 index 33c73c20..00000000 --- a/migrations/6.2.1_email_tracking/up.sql +++ /dev/null @@ -1,56 +0,0 @@ --- Email Read Tracking Table --- Stores sent email tracking data for read receipt functionality --- Enabled via config.csv: email-read-pixel,true - -CREATE TABLE IF NOT EXISTS sent_email_tracking ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - tracking_id UUID NOT NULL UNIQUE, - bot_id UUID NOT NULL, - account_id UUID NOT NULL, - from_email VARCHAR(255) NOT NULL, - to_email VARCHAR(255) NOT NULL, - cc TEXT, - bcc TEXT, - subject TEXT NOT NULL, - sent_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - is_read BOOLEAN NOT NULL DEFAULT FALSE, - read_at TIMESTAMPTZ, - read_count INTEGER NOT NULL DEFAULT 0, - first_read_ip VARCHAR(45), - last_read_ip VARCHAR(45), - user_agent TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - --- Indexes for efficient queries -CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_tracking_id ON sent_email_tracking(tracking_id); -CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_bot_id ON sent_email_tracking(bot_id); -CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_account_id ON sent_email_tracking(account_id); -CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_to_email ON sent_email_tracking(to_email); -CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_sent_at ON sent_email_tracking(sent_at DESC); -CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_is_read ON sent_email_tracking(is_read); -CREATE INDEX IF NOT EXISTS idx_sent_email_tracking_read_status ON sent_email_tracking(bot_id, is_read, sent_at DESC); - --- Trigger to auto-update updated_at -CREATE OR REPLACE FUNCTION update_sent_email_tracking_updated_at() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS trigger_update_sent_email_tracking_updated_at ON sent_email_tracking; -CREATE TRIGGER trigger_update_sent_email_tracking_updated_at - BEFORE UPDATE ON sent_email_tracking - FOR EACH ROW - EXECUTE FUNCTION update_sent_email_tracking_updated_at(); - --- Add comment for documentation -COMMENT ON TABLE sent_email_tracking IS 'Tracks sent emails for read receipt functionality via tracking pixel'; -COMMENT ON COLUMN sent_email_tracking.tracking_id IS 'Unique ID embedded in tracking pixel URL'; -COMMENT ON COLUMN sent_email_tracking.is_read IS 'Whether the email has been opened (pixel loaded)'; -COMMENT ON COLUMN sent_email_tracking.read_count IS 'Number of times the email was opened'; -COMMENT ON COLUMN sent_email_tracking.first_read_ip IS 'IP address of first email open'; -COMMENT ON COLUMN sent_email_tracking.last_read_ip IS 'IP address of most recent email open'; diff --git a/src/core/bootstrap/mod.rs b/src/core/bootstrap/mod.rs index 5c1ce1eb..d3aa3648 100644 --- a/src/core/bootstrap/mod.rs +++ b/src/core/bootstrap/mod.rs @@ -1,17 +1,18 @@ use crate::config::AppConfig; use crate::package_manager::setup::{DirectorySetup, EmailSetup}; use crate::package_manager::{InstallMode, PackageManager}; -use crate::shared::utils::establish_pg_connection; +use crate::shared::utils::{establish_pg_connection, init_secrets_manager}; use anyhow::Result; use aws_config::BehaviorVersion; use aws_sdk_s3::Client; +use diesel::RunQueryDsl; +use log::debug; use log::{error, info, trace, warn}; use rand::distr::Alphanumeric; use rcgen::{ BasicConstraints, CertificateParams, DistinguishedName, DnType, IsCa, Issuer, KeyPair, }; use std::fs; -use std::io::{self, Write}; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; @@ -37,11 +38,130 @@ impl BootstrapManager { tenant, } } - pub fn start_all(&mut self) -> Result<()> { + + /// Kill all processes running from the botserver-stack directory + /// This ensures a clean startup when bootstrapping fresh + pub fn kill_stack_processes() { + info!("Killing any existing stack processes..."); + + // Kill processes by pattern matching on botserver-stack path + let patterns = vec![ + "botserver-stack/bin/vault", + "botserver-stack/bin/tables", + "botserver-stack/bin/drive", + "botserver-stack/bin/cache", + "botserver-stack/bin/directory", + "botserver-stack/bin/llm", + "botserver-stack/bin/email", + "botserver-stack/bin/proxy", + "botserver-stack/bin/dns", + "botserver-stack/bin/meeting", + "botserver-stack/bin/vector_db", + ]; + + for pattern in patterns { + let _ = Command::new("pkill").args(["-9", "-f", pattern]).output(); + } + + // Also kill by specific process names + let process_names = vec![ + "vault", + "postgres", + "minio", + "redis-server", + "zitadel", + "ollama", + "stalwart", + "caddy", + "coredns", + "livekit", + "qdrant", + ]; + + for name in process_names { + let _ = Command::new("pkill").args(["-9", "-x", name]).output(); + } + + // Give processes time to die + std::thread::sleep(std::time::Duration::from_millis(500)); + info!("Stack processes terminated"); + } + + /// Clean up the entire stack directory for a fresh bootstrap + pub fn clean_stack_directory() -> Result<()> { + let stack_dir = PathBuf::from("./botserver-stack"); + let env_file = PathBuf::from("./.env"); + + if stack_dir.exists() { + info!("Removing existing stack directory..."); + fs::remove_dir_all(&stack_dir)?; + info!("Stack directory removed"); + } + + if env_file.exists() { + info!("Removing existing .env file..."); + fs::remove_file(&env_file)?; + info!(".env file removed"); + } + + Ok(()) + } + pub async fn start_all(&mut self) -> Result<()> { let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?; - let components = vec![ - ComponentInfo { name: "vault" }, - ComponentInfo { name: "tables" }, + + // VAULT MUST START FIRST - all other services depend on it for secrets + if pm.is_installed("vault") { + info!("Starting Vault secrets service..."); + match pm.start("vault") { + Ok(_child) => { + info!("Vault process started, waiting for initialization..."); + } + Err(e) => { + warn!("Vault might already be running: {}", e); + } + } + + // Wait for Vault to be ready (up to 10 seconds) + for i in 0..10 { + let vault_ready = Command::new("sh") + .arg("-c") + .arg("curl -f -s http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 >/dev/null 2>&1") + .status() + .map(|s| s.success()) + .unwrap_or(false); + + if vault_ready { + info!("Vault is responding"); + break; + } + if i < 9 { + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } + } + + // Try to unseal Vault + if let Err(e) = self.ensure_vault_unsealed().await { + warn!("Vault unseal check: {}", e); + } + } + + // Start tables (PostgreSQL) - needed for database operations + if pm.is_installed("tables") { + info!("Starting PostgreSQL database..."); + match pm.start("tables") { + Ok(_child) => { + // Give PostgreSQL time to initialize + tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + info!("PostgreSQL started"); + } + Err(e) => { + warn!("PostgreSQL might already be running: {}", e); + } + } + } + + // Start other components (order matters less for these) + let other_components = vec![ ComponentInfo { name: "cache" }, ComponentInfo { name: "drive" }, ComponentInfo { name: "llm" }, @@ -58,63 +178,131 @@ impl BootstrapManager { ComponentInfo { name: "vector_db" }, ComponentInfo { name: "host" }, ]; - for component in components { + + for component in other_components { if pm.is_installed(component.name) { match pm.start(component.name) { Ok(_child) => { trace!("Started component: {}", component.name); } Err(e) => { - warn!( + trace!( "Component {} might already be running: {}", - component.name, e + component.name, + e ); } } } } + Ok(()) } fn generate_secure_password(&self, length: usize) -> String { let mut rng = rand::rng(); - (0..length) + let base: String = (0..length.saturating_sub(4)) .map(|_| { let byte = rand::Rng::sample(&mut rng, Alphanumeric); char::from(byte) }) - .collect() + .collect(); + // Add required symbols/complexity for Zitadel password policy + // Use ! instead of @ to avoid breaking database connection strings + format!("{}!1Aa", base) } /// Ensure critical services are running - Vault MUST be first /// Order: vault -> tables -> drive + /// If fresh_start is true, kills existing processes first pub async fn ensure_services_running(&mut self) -> Result<()> { info!("Ensuring critical services are running..."); let installer = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?; + // Check if we need to bootstrap first + let vault_installed = installer.is_installed("vault"); + let vault_initialized = PathBuf::from("./botserver-stack/conf/vault/init.json").exists(); + + if !vault_installed || !vault_initialized { + info!("Stack not fully bootstrapped, running bootstrap first..."); + // Kill any leftover processes + Self::kill_stack_processes(); + + // Run bootstrap - this will start all services + self.bootstrap().await?; + + // After bootstrap, services are already running, just ensure Vault is unsealed and env vars set + info!("Bootstrap complete, verifying Vault is ready..."); + tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + + if let Err(e) = self.ensure_vault_unsealed().await { + warn!("Failed to unseal Vault after bootstrap: {}", e); + } + + // Services were started by bootstrap, no need to restart them + return Ok(()); + } + + // If we get here, bootstrap was already done previously - just start services // VAULT MUST BE FIRST - it provides all secrets if installer.is_installed("vault") { - info!("Starting Vault secrets service..."); - match installer.start("vault") { - Ok(_child) => { - info!("Vault started successfully"); - // Give Vault time to initialize - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - - // Check if Vault needs to be unsealed - if let Err(e) = self.ensure_vault_unsealed().await { - warn!("Failed to unseal Vault: {}", e); + // Check if Vault is already running + let vault_running = Command::new("sh") + .arg("-c") + .arg("curl -f -s http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 >/dev/null 2>&1") + .status() + .map(|s| s.success()) + .unwrap_or(false); + + if !vault_running { + info!("Starting Vault secrets service..."); + match installer.start("vault") { + Ok(_child) => { + info!("Vault started successfully"); + // Give Vault time to initialize + tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + } + Err(e) => { + warn!("Vault might already be running or failed to start: {}", e); } } - Err(e) => { - warn!("Vault might already be running or failed to start: {}", e); + } else { + info!("Vault is already running"); + } + + // Always try to unseal Vault (it may have restarted) + // If unseal fails, Vault may need re-initialization (data deleted) + if let Err(e) = self.ensure_vault_unsealed().await { + warn!("Vault unseal failed: {} - running re-bootstrap", e); + + // Kill all processes and run fresh bootstrap + Self::kill_stack_processes(); + Self::clean_stack_directory()?; + + // Run bootstrap from scratch + self.bootstrap().await?; + + // After bootstrap, services are already running + info!("Re-bootstrap complete, verifying Vault is ready..."); + tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + + if let Err(e) = self.ensure_vault_unsealed().await { + return Err(anyhow::anyhow!( + "Failed to configure Vault after re-bootstrap: {}", + e + )); } + + // Services were started by bootstrap, no need to restart them + return Ok(()); } } else { // Vault not installed - cannot proceed, need to run bootstrap warn!("Vault (secrets) component not installed - run bootstrap first"); - return Err(anyhow::anyhow!("Vault not installed. Run bootstrap command first.")); + return Err(anyhow::anyhow!( + "Vault not installed. Run bootstrap command first." + )); } // Check and start PostgreSQL (after Vault is running) @@ -158,17 +346,21 @@ impl BootstrapManager { } /// Ensure Vault is unsealed (required after restart) + /// Returns Ok(()) if Vault is ready, Err if it needs re-initialization async fn ensure_vault_unsealed(&self) -> Result<()> { let vault_init_path = PathBuf::from("./botserver-stack/conf/vault/init.json"); - + let vault_addr = "http://localhost:8200"; + if !vault_init_path.exists() { - return Err(anyhow::anyhow!("Vault init.json not found")); + return Err(anyhow::anyhow!( + "Vault init.json not found - needs re-initialization" + )); } // Read unseal key from init.json let init_json = fs::read_to_string(&vault_init_path)?; let init_data: serde_json::Value = serde_json::from_str(&init_json)?; - + let unseal_key = init_data["unseal_keys_b64"] .as_array() .and_then(|arr| arr.first()) @@ -176,38 +368,78 @@ impl BootstrapManager { .unwrap_or("") .to_string(); - let root_token = init_data["root_token"] - .as_str() - .unwrap_or("") - .to_string(); + let root_token = init_data["root_token"].as_str().unwrap_or("").to_string(); if unseal_key.is_empty() || root_token.is_empty() { - return Err(anyhow::anyhow!("Invalid Vault init.json")); + return Err(anyhow::anyhow!( + "Invalid Vault init.json - needs re-initialization" + )); } - let vault_addr = "https://localhost:8200"; - - // Check if Vault is sealed + // First check if Vault is initialized (not just running) let status_output = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true ./botserver-stack/bin/vault/vault status -format=json 2>/dev/null || echo '{{\"sealed\":true}}'", + "VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>&1", vault_addr )) .output()?; - - let status_json = String::from_utf8_lossy(&status_output.stdout); - if let Ok(status) = serde_json::from_str::(&status_json) { - if status["sealed"].as_bool().unwrap_or(true) { + + let status_str = String::from_utf8_lossy(&status_output.stdout); + + // Parse status - handle both success and error cases + if let Ok(status) = serde_json::from_str::(&status_str) { + let initialized = status["initialized"].as_bool().unwrap_or(false); + let sealed = status["sealed"].as_bool().unwrap_or(true); + + if !initialized { + // Vault is running but not initialized - this means data was deleted + // We need to re-run bootstrap + warn!("Vault is running but not initialized - data may have been deleted"); + return Err(anyhow::anyhow!( + "Vault not initialized - needs re-bootstrap" + )); + } + + if sealed { info!("Unsealing Vault..."); - let _ = std::process::Command::new("sh") + let unseal_output = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true ./botserver-stack/bin/vault/vault operator unseal {}", + "VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator unseal {}", vault_addr, unseal_key )) .output()?; + + if !unseal_output.status.success() { + let stderr = String::from_utf8_lossy(&unseal_output.stderr); + warn!("Vault unseal may have failed: {}", stderr); + } + + // Verify unseal succeeded + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + let verify_output = std::process::Command::new("sh") + .arg("-c") + .arg(format!( + "VAULT_ADDR={} ./botserver-stack/bin/vault/vault status -format=json 2>&1", + vault_addr + )) + .output()?; + + let verify_str = String::from_utf8_lossy(&verify_output.stdout); + if let Ok(verify_status) = serde_json::from_str::(&verify_str) { + if verify_status["sealed"].as_bool().unwrap_or(true) { + return Err(anyhow::anyhow!( + "Failed to unseal Vault - may need re-initialization" + )); + } + } + info!("Vault unsealed successfully"); } + } else { + // Could not parse status - Vault might not be responding properly + warn!("Could not get Vault status: {}", status_str); + return Err(anyhow::anyhow!("Vault not responding properly")); } // Set environment variables for other components @@ -215,6 +447,21 @@ impl BootstrapManager { std::env::set_var("VAULT_TOKEN", &root_token); std::env::set_var("VAULT_SKIP_VERIFY", "true"); + // Also set mTLS cert paths + std::env::set_var( + "VAULT_CACERT", + "./botserver-stack/conf/system/certificates/ca/ca.crt", + ); + std::env::set_var( + "VAULT_CLIENT_CERT", + "./botserver-stack/conf/system/certificates/botserver/client.crt", + ); + std::env::set_var( + "VAULT_CLIENT_KEY", + "./botserver-stack/conf/system/certificates/botserver/client.key", + ); + + info!("Vault environment configured"); Ok(()) } @@ -242,7 +489,7 @@ impl BootstrapManager { info!("Configuring services through Vault..."); let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone()).unwrap(); - + // Vault MUST be installed first - it stores all secrets // Order: vault -> tables -> directory -> drive -> cache -> llm let required_components = vec![ @@ -253,45 +500,56 @@ impl BootstrapManager { "cache", // Redis cache "llm", // LLM service ]; + + // Special check: Vault needs setup even if binary exists but not initialized + let vault_needs_setup = !PathBuf::from("./botserver-stack/conf/vault/init.json").exists(); + for component in required_components { - if !pm.is_installed(component) { - let termination_cmd = pm + // For vault, also check if it needs initialization + let needs_install = if component == "vault" { + !pm.is_installed(component) || vault_needs_setup + } else { + !pm.is_installed(component) + }; + + if needs_install { + // Quick check if component might be running - don't hang on this + let bin_path = pm.base_path.join("bin").join(component); + let binary_name = pm .components .get(component) .and_then(|cfg| cfg.binary_name.clone()) .unwrap_or_else(|| component.to_string()); - if !termination_cmd.is_empty() { - let check = Command::new("pgrep") - .arg("-f") - .arg(&termination_cmd) - .output(); - if let Ok(output) = check { - if !output.stdout.is_empty() { - println!("Component '{}' appears to be already running from a previous install.", component); - println!("Do you want to terminate it? (y/n)"); - let mut input = String::new(); - io::stdout().flush().unwrap(); - io::stdin().read_line(&mut input).unwrap(); - if input.trim().eq_ignore_ascii_case("y") { - let _ = Command::new("pkill") - .arg("-f") - .arg(&termination_cmd) - .status(); - println!("Terminated existing '{}' process.", component); - } else { - println!( - "Skipping start of '{}' as it is already running.", - component - ); - continue; - } - } - } + + // Only terminate for services that are known to conflict + // Use simple, fast commands with timeout + if component == "vault" || component == "tables" || component == "directory" { + let _ = Command::new("sh") + .arg("-c") + .arg(format!( + "pkill -9 -f '{}/{}' 2>/dev/null; true", + bin_path.display(), + binary_name + )) + .status(); + std::thread::sleep(std::time::Duration::from_millis(200)); } _ = pm.install(component).await; - // After tables is installed, create Zitadel config files before installing directory + // After tables is installed, START PostgreSQL and create Zitadel config files before installing directory if component == "tables" { + info!("🚀 Starting PostgreSQL database..."); + match pm.start("tables") { + Ok(_) => { + info!("PostgreSQL started successfully"); + // Give PostgreSQL time to initialize + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + } + Err(e) => { + warn!("Failed to start PostgreSQL: {}", e); + } + } + info!("🔧 Creating Directory configuration files..."); if let Err(e) = self.configure_services_in_directory(&db_password).await { error!("Failed to create Directory config files: {}", e); @@ -307,12 +565,51 @@ impl BootstrapManager { } } - // After Vault is installed and running, store all secrets + // After Vault is installed, START the server then initialize it if component == "vault" { + info!("🚀 Starting Vault server..."); + match pm.start("vault") { + Ok(_) => { + info!("Vault server started"); + // Give Vault time to start + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + } + Err(e) => { + warn!("Failed to start Vault server: {}", e); + } + } + info!("🔐 Initializing Vault with secrets..."); - if let Err(e) = self.setup_vault(&db_password, &drive_accesskey, &drive_secret, &cache_password).await { + if let Err(e) = self + .setup_vault( + &db_password, + &drive_accesskey, + &drive_secret, + &cache_password, + ) + .await + { error!("Failed to setup Vault: {}", e); } + + // Initialize the global SecretsManager so other components can use Vault + info!("🔑 Initializing SecretsManager..."); + debug!( + "VAULT_ADDR={:?}, VAULT_TOKEN set={}", + std::env::var("VAULT_ADDR").ok(), + std::env::var("VAULT_TOKEN").is_ok() + ); + match init_secrets_manager().await { + Ok(_) => info!("✓ SecretsManager initialized successfully"), + Err(e) => { + error!("Failed to initialize SecretsManager: {}", e); + // Don't continue if SecretsManager fails - it's required for DB connection + return Err(anyhow::anyhow!( + "SecretsManager initialization failed: {}", + e + )); + } + } } if component == "tables" { @@ -353,14 +650,17 @@ impl BootstrapManager { let zitadel_config_path = PathBuf::from("./botserver-stack/conf/directory/zitadel.yaml"); let steps_config_path = PathBuf::from("./botserver-stack/conf/directory/steps.yaml"); - let pat_path = PathBuf::from("./botserver-stack/conf/directory/admin-pat.txt"); - + // Use absolute path for PAT file since zitadel runs from bin/directory/ + let pat_path = + std::env::current_dir()?.join("botserver-stack/conf/directory/admin-pat.txt"); + fs::create_dir_all(zitadel_config_path.parent().unwrap())?; // Generate Zitadel database password let zitadel_db_password = self.generate_secure_password(24); // Create zitadel.yaml - main configuration + // Note: Zitadel uses lowercase 'postgres' and nested User/Admin with Username field let zitadel_config = format!( r#"Log: Level: info @@ -372,10 +672,11 @@ Database: Host: localhost Port: 5432 Database: zitadel - User: zitadel - Password: "{}" - SSL: - Mode: disable + User: + Username: zitadel + Password: "{}" + SSL: + Mode: disable Admin: Username: gbuser Password: "{}" @@ -399,17 +700,27 @@ DefaultInstance: RefreshTokenExpiration: 2160h "#, zitadel_db_password, - db_password, // Use the password passed directly from bootstrap + db_password, // Use the password passed directly from bootstrap ); fs::write(&zitadel_config_path, zitadel_config)?; info!("Created zitadel.yaml configuration"); // Create steps.yaml - first instance setup that generates admin PAT + // Use Machine user with PAT for API access (Human users don't generate PAT files) let steps_config = format!( r#"FirstInstance: + InstanceName: "BotServer" + DefaultLanguage: "en" + PatPath: "{}" Org: Name: "BotServer" + Machine: + Machine: + Username: "admin-sa" + Name: "Admin Service Account" + Pat: + ExpirationDate: "2099-12-31T23:59:59Z" Human: UserName: "admin" FirstName: "Admin" @@ -418,12 +729,10 @@ DefaultInstance: Address: "admin@localhost" Verified: true Password: "{}" - PatPath: "{}" - InstanceName: "BotServer" - DefaultLanguage: "en" + PasswordChangeRequired: false "#, - self.generate_secure_password(16), pat_path.to_string_lossy(), + self.generate_secure_password(16), ); fs::write(&steps_config_path, steps_config)?; @@ -438,7 +747,7 @@ DefaultInstance: db_password )) .output(); - + if let Ok(output) = create_db_result { let stdout = String::from_utf8_lossy(&output.stdout); if !stdout.contains("already exists") { @@ -455,7 +764,7 @@ DefaultInstance: zitadel_db_password )) .output(); - + if let Ok(output) = create_user_result { let stdout = String::from_utf8_lossy(&output.stdout); if !stdout.contains("already exists") { @@ -592,20 +901,20 @@ meet IN A 127.0.0.1 info!("Waiting for Zitadel to be ready..."); let mut attempts = 0; let max_attempts = 60; // 60 seconds max wait - + while attempts < max_attempts { // Check if Zitadel is healthy let health_check = std::process::Command::new("curl") .args(["-f", "-s", "http://localhost:8080/healthz"]) .output(); - + if let Ok(output) = health_check { if output.status.success() { info!("Zitadel is healthy"); break; } } - + attempts += 1; tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; } @@ -649,7 +958,10 @@ meet IN A 127.0.0.1 // Try to create additional organization for bot users let org_name = "default"; - match setup.create_organization(org_name, "Default Organization").await { + match setup + .create_organization(org_name, "Default Organization") + .await + { Ok(org_id) => { info!("Created default organization: {}", org_name); @@ -657,15 +969,18 @@ meet IN A 127.0.0.1 let user_password = self.generate_secure_password(16); // Create user@default account for regular bot usage - match setup.create_user( - &org_id, - "user", - "user@default", - &user_password, - "User", - "Default", - false, - ).await { + match setup + .create_user( + &org_id, + "user", + "user@default", + &user_password, + "User", + "Default", + false, + ) + .await + { Ok(regular_user) => { info!("Created regular user: user@default"); info!(" Regular user ID: {}", regular_user.id); @@ -679,7 +994,7 @@ meet IN A 127.0.0.1 match setup.create_oauth_application(&org_id).await { Ok((project_id, client_id, client_secret)) => { info!("Created OAuth2 application in project: {}", project_id); - + // Save configuration let admin_user = crate::package_manager::setup::DefaultUser { id: "admin".to_string(), @@ -690,13 +1005,16 @@ meet IN A 127.0.0.1 last_name: "User".to_string(), }; - if let Ok(config) = setup.save_config( - org_id.clone(), - org_name.to_string(), - admin_user, - client_id.clone(), - client_secret, - ).await { + if let Ok(config) = setup + .save_config( + org_id.clone(), + org_name.to_string(), + admin_user, + client_id.clone(), + client_secret, + ) + .await + { info!("Directory initialized successfully!"); info!(" Organization: default"); info!(" Client ID: {}", client_id); @@ -719,39 +1037,48 @@ meet IN A 127.0.0.1 } /// Setup Vault with all service secrets and write .env file with VAULT_* variables - async fn setup_vault(&self, db_password: &str, drive_accesskey: &str, drive_secret: &str, cache_password: &str) -> Result<()> { + async fn setup_vault( + &self, + db_password: &str, + drive_accesskey: &str, + drive_secret: &str, + cache_password: &str, + ) -> Result<()> { let vault_conf_path = PathBuf::from("./botserver-stack/conf/vault"); let vault_init_path = vault_conf_path.join("init.json"); let env_file_path = PathBuf::from("./.env"); - + // Wait for Vault to be ready info!("Waiting for Vault to be ready..."); let mut attempts = 0; let max_attempts = 30; - + while attempts < max_attempts { let health_check = std::process::Command::new("curl") - .args(["-f", "-s", "-k", "https://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200"]) + .args(["-f", "-s", "http://localhost:8200/v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200"]) .output(); - + if let Ok(output) = health_check { if output.status.success() { info!("Vault is responding"); break; } } - + attempts += 1; tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; } if attempts >= max_attempts { warn!("Vault health check timed out"); - return Err(anyhow::anyhow!("Vault not ready after {} seconds", max_attempts)); + return Err(anyhow::anyhow!( + "Vault not ready after {} seconds", + max_attempts + )); } // Check if Vault is already initialized - let vault_addr = "https://localhost:8200"; + let vault_addr = "http://localhost:8200"; std::env::set_var("VAULT_ADDR", vault_addr); std::env::set_var("VAULT_SKIP_VERIFY", "true"); @@ -760,19 +1087,16 @@ meet IN A 127.0.0.1 info!("Reading Vault initialization from init.json..."); let init_json = fs::read_to_string(&vault_init_path)?; let init_data: serde_json::Value = serde_json::from_str(&init_json)?; - + let unseal_key = init_data["unseal_keys_b64"] .as_array() .and_then(|arr| arr.first()) .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); - - let root_token = init_data["root_token"] - .as_str() - .unwrap_or("") - .to_string(); - + + let root_token = init_data["root_token"].as_str().unwrap_or("").to_string(); + (unseal_key, root_token) } else { // Initialize Vault if not already done @@ -780,11 +1104,11 @@ meet IN A 127.0.0.1 let init_output = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true ./botserver-stack/bin/vault/vault operator init -key-shares=1 -key-threshold=1 -format=json", + "VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator init -key-shares=1 -key-threshold=1 -format=json", vault_addr )) .output()?; - + if !init_output.status.success() { let stderr = String::from_utf8_lossy(&init_output.stderr); if stderr.contains("already initialized") { @@ -793,11 +1117,11 @@ meet IN A 127.0.0.1 } return Err(anyhow::anyhow!("Vault init failed: {}", stderr)); } - + let init_json = String::from_utf8_lossy(&init_output.stdout); fs::write(&vault_init_path, init_json.as_ref())?; fs::set_permissions(&vault_init_path, std::fs::Permissions::from_mode(0o600))?; - + let init_data: serde_json::Value = serde_json::from_str(&init_json)?; let unseal_key = init_data["unseal_keys_b64"] .as_array() @@ -805,11 +1129,8 @@ meet IN A 127.0.0.1 .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); - let root_token = init_data["root_token"] - .as_str() - .unwrap_or("") - .to_string(); - + let root_token = init_data["root_token"].as_str().unwrap_or("").to_string(); + (unseal_key, root_token) }; @@ -822,11 +1143,11 @@ meet IN A 127.0.0.1 let unseal_output = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true ./botserver-stack/bin/vault/vault operator unseal {}", + "VAULT_ADDR={} ./botserver-stack/bin/vault/vault operator unseal {}", vault_addr, unseal_key )) .output()?; - + if !unseal_output.status.success() { let stderr = String::from_utf8_lossy(&unseal_output.stderr); if !stderr.contains("already unsealed") { @@ -842,7 +1163,7 @@ meet IN A 127.0.0.1 let _ = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault secrets enable -path=secret kv-v2 2>&1 || true", + "VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault secrets enable -path=secret kv-v2 2>&1 || true", vault_addr, root_token )) .output(); @@ -854,7 +1175,7 @@ meet IN A 127.0.0.1 let _ = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/tables host=localhost port=5432 database=botserver username=gbuser password='{}'", + "VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/tables host=localhost port=5432 database=botserver username=gbuser password='{}'", vault_addr, root_token, db_password )) .output()?; @@ -864,7 +1185,7 @@ meet IN A 127.0.0.1 let _ = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/drive accesskey='{}' secret='{}'", + "VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/drive accesskey='{}' secret='{}'", vault_addr, root_token, drive_accesskey, drive_secret )) .output()?; @@ -874,7 +1195,7 @@ meet IN A 127.0.0.1 let _ = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/cache password='{}'", + "VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/cache password='{}'", vault_addr, root_token, cache_password )) .output()?; @@ -884,7 +1205,7 @@ meet IN A 127.0.0.1 let _ = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/directory url=https://localhost:8080 project_id= client_id= client_secret=", + "VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/directory url=https://localhost:8080 project_id= client_id= client_secret=", vault_addr, root_token )) .output()?; @@ -894,7 +1215,7 @@ meet IN A 127.0.0.1 let _ = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/llm openai_key= anthropic_key= groq_key=", + "VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/llm openai_key= anthropic_key= groq_key=", vault_addr, root_token )) .output()?; @@ -904,7 +1225,7 @@ meet IN A 127.0.0.1 let _ = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/email username= password=", + "VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/email username= password=", vault_addr, root_token )) .output()?; @@ -915,7 +1236,7 @@ meet IN A 127.0.0.1 let _ = std::process::Command::new("sh") .arg("-c") .arg(format!( - "VAULT_ADDR={} VAULT_SKIP_VERIFY=true VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/encryption master_key='{}'", + "VAULT_ADDR={} VAULT_TOKEN={} ./botserver-stack/bin/vault/vault kv put secret/gbo/encryption master_key='{}'", vault_addr, root_token, encryption_key )) .output()?; @@ -924,7 +1245,7 @@ meet IN A 127.0.0.1 // Write .env file with ONLY Vault variables - NO LEGACY FALLBACK info!("Writing .env file with Vault configuration..."); let env_content = format!( -r#"# BotServer Environment Configuration + r#"# BotServer Environment Configuration # Generated by bootstrap - DO NOT ADD OTHER SECRETS HERE # All secrets are stored in Vault at the paths below: # - gbo/tables - PostgreSQL credentials @@ -939,11 +1260,8 @@ r#"# BotServer Environment Configuration VAULT_ADDR={} VAULT_TOKEN={} -# mTLS Configuration for Vault -# Set VAULT_SKIP_VERIFY=false in production with proper CA cert -VAULT_SKIP_VERIFY=true -VAULT_CACERT=./botserver-stack/conf/system/certificates/ca/ca.crt -VAULT_CLIENT_CERT=./botserver-stack/conf/system/certificates/botserver/client.crt +# Vault uses HTTP for local development (TLS disabled in config.hcl) +# In production, enable TLS and set proper certificates VAULT_CLIENT_KEY=./botserver-stack/conf/system/certificates/botserver/client.key # Cache TTL for secrets (seconds) @@ -958,7 +1276,7 @@ VAULT_CACHE_TTL=300 info!("🔐 Vault setup complete!"); info!(" Vault UI: {}/ui", vault_addr); info!(" Root token saved to: {}", vault_init_path.display()); - + Ok(()) } @@ -1016,9 +1334,15 @@ VAULT_CACHE_TTL=300 aws_sdk_s3::Client::from_conf(s3_config) } - pub async fn upload_templates_to_drive(&self, _config: &AppConfig) -> Result<()> { + /// Sync bot configurations from template config.csv files to database + /// This is separate from drive upload and does not require S3 connection + pub fn sync_templates_to_database(&self) -> Result<()> { let mut conn = establish_pg_connection()?; self.create_bots_from_templates(&mut conn)?; + Ok(()) + } + + pub async fn upload_templates_to_drive(&self, _config: &AppConfig) -> Result<()> { let templates_dir = Path::new("templates"); if !templates_dir.exists() { return Ok(()); @@ -1057,28 +1381,133 @@ VAULT_CACHE_TTL=300 fn create_bots_from_templates(&self, conn: &mut diesel::PgConnection) -> Result<()> { use crate::shared::models::schema::bots; use diesel::prelude::*; + let templates_dir = Path::new("templates"); if !templates_dir.exists() { + warn!("Templates directory does not exist"); return Ok(()); } - for entry in std::fs::read_dir(templates_dir)? { - let entry = entry?; - let path = entry.path(); - if path.is_dir() && path.extension().map(|e| e == "gbai").unwrap_or(false) { - let bot_folder = path.file_name().unwrap().to_string_lossy().to_string(); - let bot_name = bot_folder.trim_end_matches(".gbai"); - let existing: Option = bots::table - .filter(bots::name.eq(&bot_name)) - .select(bots::name) - .first(conn) - .optional()?; - if existing.is_none() { - diesel::sql_query("INSERT INTO bots (id, name, description, llm_provider, llm_config, context_provider, context_config, is_active) VALUES (gen_random_uuid(), $1, $2, 'openai', '{\"model\": \"gpt-4\", \"temperature\": 0.7}', 'database', '{}', true)").bind::(&bot_name).bind::(format!("Bot for {} template", bot_name)).execute(conn)?; - } else { - trace!("Bot {} already exists", bot_name); + + // Get the default bot (created by migrations) - we'll sync all template configs to it + let default_bot: Option<(uuid::Uuid, String)> = bots::table + .filter(bots::is_active.eq(true)) + .select((bots::id, bots::name)) + .first(conn) + .optional()?; + + let (default_bot_id, default_bot_name) = match default_bot { + Some((id, name)) => (id, name), + None => { + error!("No active bot found in database - cannot sync template configs"); + return Ok(()); + } + }; + + info!( + "Syncing template configs to bot '{}' ({})", + default_bot_name, default_bot_id + ); + + // Only sync the default.gbai template config (main config for the system) + let default_template = templates_dir.join("default.gbai"); + if default_template.exists() { + let config_path = default_template.join("default.gbot").join("config.csv"); + + if config_path.exists() { + match std::fs::read_to_string(&config_path) { + Ok(csv_content) => { + info!("Syncing config.csv from {:?}", config_path); + if let Err(e) = + self.sync_config_csv_to_db(conn, &default_bot_id, &csv_content) + { + error!("Failed to sync config.csv: {}", e); + } + } + Err(e) => { + warn!("Could not read config.csv: {}", e); + } + } + } else { + warn!("No config.csv found at {:?}", config_path); + } + } else { + warn!("default.gbai template not found"); + } + + Ok(()) + } + + /// Sync config.csv content to the bot_configuration table + /// This is critical for loading LLM settings on fresh starts + fn sync_config_csv_to_db( + &self, + conn: &mut diesel::PgConnection, + bot_id: &uuid::Uuid, + content: &str, + ) -> Result<()> { + let mut synced = 0; + let mut skipped = 0; + let lines: Vec<&str> = content.lines().collect(); + + debug!( + "Parsing config.csv with {} lines for bot {}", + lines.len(), + bot_id + ); + + for (line_num, line) in lines.iter().enumerate().skip(1) { + // Skip header line (name,value) + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + + let parts: Vec<&str> = line.splitn(2, ',').collect(); + if parts.len() >= 2 { + let key = parts[0].trim(); + let value = parts[1].trim(); + + if key.is_empty() { + skipped += 1; + continue; + } + + // Use UUID type since migration 6.1.1 converted column to UUID + let new_id = uuid::Uuid::new_v4(); + + match diesel::sql_query( + "INSERT INTO bot_configuration (id, bot_id, config_key, config_value, config_type, created_at, updated_at) \ + VALUES ($1, $2, $3, $4, 'string', NOW(), NOW()) \ + ON CONFLICT (bot_id, config_key) DO UPDATE SET config_value = EXCLUDED.config_value, updated_at = NOW()" + ) + .bind::(new_id) + .bind::(bot_id) + .bind::(key) + .bind::(value) + .execute(conn) { + Ok(_) => { + trace!(" Synced config: {} = {}", key, if key.contains("pass") || key.contains("secret") || key.contains("key") { "***" } else { value }); + synced += 1; + } + Err(e) => { + error!("Failed to sync config key '{}' at line {}: {}", key, line_num + 1, e); + // Continue with other keys instead of failing completely + } } } } + + if synced > 0 { + info!( + "✓ Synced {} config values for bot {} (skipped {} empty lines)", + synced, bot_id, skipped + ); + } else { + warn!( + "No config values synced for bot {} - check config.csv format", + bot_id + ); + } Ok(()) } fn upload_directory_recursive<'a>( @@ -1145,38 +1574,31 @@ VAULT_CACHE_TTL=300 async fn create_vault_config(&self) -> Result<()> { let vault_conf_dir = PathBuf::from("./botserver-stack/conf/vault"); let config_path = vault_conf_dir.join("config.hcl"); - + fs::create_dir_all(&vault_conf_dir)?; - - // Vault config with mTLS - requires client certificate verification - let config = r#"# Vault Configuration with mTLS + + // Vault is started from botserver-stack/bin/vault/, so paths must be relative to that + // From bin/vault/ to conf/ is ../../conf/ + // From bin/vault/ to data/ is ../../data/ + let config = r#"# Vault Configuration # Generated by BotServer bootstrap +# Note: Paths are relative to botserver-stack/bin/vault/ (Vault's working directory) # Storage backend - file-based for single instance storage "file" { - path = "./botserver-stack/data/vault" + path = "../../data/vault" } -# Listener with mTLS enabled +# Listener with TLS DISABLED for local development +# In production, enable TLS with proper certificates listener "tcp" { - address = "0.0.0.0:8200" - tls_disable = false - - # Server TLS certificate - tls_cert_file = "./botserver-stack/conf/system/certificates/vault/server.crt" - tls_key_file = "./botserver-stack/conf/system/certificates/vault/server.key" - - # mTLS - require client certificate - tls_require_and_verify_client_cert = true - tls_client_ca_file = "./botserver-stack/conf/system/certificates/ca/ca.crt" - - # TLS settings - tls_min_version = "tls12" + address = "0.0.0.0:8200" + tls_disable = true } -# API settings -api_addr = "https://localhost:8200" -cluster_addr = "https://localhost:8201" +# API settings - use HTTP for local dev +api_addr = "http://localhost:8200" +cluster_addr = "http://localhost:8201" # UI enabled for administration ui = true @@ -1192,13 +1614,16 @@ telemetry { # Log level log_level = "info" "#; - + fs::write(&config_path, config)?; - + // Create data directory for Vault storage fs::create_dir_all("./botserver-stack/data/vault")?; - - info!("Created Vault config with mTLS at {}", config_path.display()); + + info!( + "Created Vault config with mTLS at {}", + config_path.display() + ); Ok(()) } @@ -1250,40 +1675,48 @@ log_level = "info" // Generate client certificate for botserver (for mTLS to all services) let botserver_dir = cert_dir.join("botserver"); fs::create_dir_all(&botserver_dir)?; - + let client_cert_path = botserver_dir.join("client.crt"); let client_key_path = botserver_dir.join("client.key"); - + if !client_cert_path.exists() || !client_key_path.exists() { info!("Generating mTLS client certificate for botserver"); - + let mut client_params = CertificateParams::default(); client_params.not_before = time::OffsetDateTime::now_utc(); client_params.not_after = time::OffsetDateTime::now_utc() + time::Duration::days(365); - + let mut client_dn = DistinguishedName::new(); client_dn.push(DnType::CountryName, "BR"); client_dn.push(DnType::OrganizationName, "BotServer"); client_dn.push(DnType::CommonName, "botserver-client"); client_params.distinguished_name = client_dn; - + // Add client auth extended key usage - client_params.subject_alt_names.push(rcgen::SanType::DnsName("botserver".to_string().try_into()?)); - + client_params + .subject_alt_names + .push(rcgen::SanType::DnsName("botserver".to_string().try_into()?)); + let client_key = KeyPair::generate()?; let client_cert = client_params.signed_by(&client_key, &ca_issuer)?; - + fs::write(&client_cert_path, client_cert.pem())?; fs::write(&client_key_path, client_key.serialize_pem())?; fs::copy(&ca_cert_path, botserver_dir.join("ca.crt"))?; - - info!("Generated mTLS client certificate at {}", client_cert_path.display()); + + info!( + "Generated mTLS client certificate at {}", + client_cert_path.display() + ); } // Services that need certificates - Vault FIRST // Using component names: tables (postgres), drive (minio), cache (redis), vectordb (qdrant) let services = vec![ - ("vault", vec!["localhost", "127.0.0.1", "vault.botserver.local"]), + ( + "vault", + vec!["localhost", "127.0.0.1", "vault.botserver.local"], + ), ("api", vec!["localhost", "127.0.0.1", "api.botserver.local"]), ("llm", vec!["localhost", "127.0.0.1", "llm.botserver.local"]), ( diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 38bc2145..2357790e 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -257,55 +257,35 @@ impl EmailConfig { impl AppConfig { pub fn from_database(pool: &DbPool) -> Result { use crate::shared::models::schema::bot_configuration::dsl::*; + use diesel::prelude::*; + let mut conn = pool.get().map_err(|e| { diesel::result::Error::DatabaseError( diesel::result::DatabaseErrorKind::UnableToSendCommand, Box::new(e.to_string()), ) })?; - let config_map: HashMap = - bot_configuration - .select(( - id, - bot_id, - config_key, - config_value, - config_type, - is_encrypted, - )) - .load::<(Uuid, Uuid, String, String, String, bool)>(&mut conn) - .unwrap_or_default() - .into_iter() - .map(|(_, _, key, value, _, _)| { - ( - key.clone(), - (Uuid::nil(), Uuid::nil(), key, value, String::new(), false), - ) - }) - .collect(); - let mut get_str = |key: &str, default: &str| -> String { - bot_configuration - .filter(config_key.eq(key)) - .select(config_value) - .first::(&mut conn) - .unwrap_or_else(|_| default.to_string()) - }; - let _get_u32 = |key: &str, default: u32| -> u32 { + + // Load all config values into a HashMap for efficient lookup + let config_map: HashMap = bot_configuration + .select((config_key, config_value)) + .load::<(String, String)>(&mut conn) + .unwrap_or_default() + .into_iter() + .collect(); + + // Helper functions that use the pre-loaded config_map + let get_str = |key: &str, default: &str| -> String { config_map .get(key) - .and_then(|v| v.3.parse().ok()) - .unwrap_or(default) + .cloned() + .unwrap_or_else(|| default.to_string()) }; + let get_u16 = |key: &str, default: u16| -> u16 { config_map .get(key) - .and_then(|v| v.3.parse().ok()) - .unwrap_or(default) - }; - let _get_bool = |key: &str, default: bool| -> bool { - config_map - .get(key) - .map(|v| v.3.to_lowercase() == "true") + .and_then(|v| v.parse().ok()) .unwrap_or(default) }; let drive = DriveConfig { @@ -326,9 +306,9 @@ impl AppConfig { drive, email, server: ServerConfig { - host: get_str("SERVER_HOST", "127.0.0.1"), - port: get_u16("SERVER_PORT", 8080), - base_url: get_str("SERVER_BASE_URL", "http://localhost:8080"), + host: get_str("server_host", "0.0.0.0"), + port: get_u16("server_port", 8080), + base_url: get_str("server_base_url", "http://localhost:8080"), }, site_path: { ConfigManager::new(pool.clone()) diff --git a/src/core/package_manager/facade.rs b/src/core/package_manager/facade.rs index ca77c6b6..f0116544 100644 --- a/src/core/package_manager/facade.rs +++ b/src/core/package_manager/facade.rs @@ -434,8 +434,10 @@ impl PackageManager { } else { PathBuf::from("/opt/gbo/data") }; + // CONF_PATH should be the base conf directory, not component-specific + // Commands that need component subdirs include them explicitly (e.g., {{CONF_PATH}}/directory/zitadel.yaml) let conf_path = if target == "local" { - self.base_path.join("conf").join(component) + self.base_path.join("conf") } else { PathBuf::from("/opt/gbo/conf") }; @@ -444,12 +446,28 @@ impl PackageManager { } else { PathBuf::from("/opt/gbo/logs") }; + + // Get DB password from Vault for commands that need it (e.g., PostgreSQL initdb) + let db_password = match get_database_url_sync() { + Ok(url) => { + let (_, password, _, _, _) = parse_database_url(&url); + password + } + Err(_) => { + // Vault not available yet - this is OK during early bootstrap + // Commands that don't need DB_PASSWORD will still work + trace!("Vault not available for DB_PASSWORD, using empty string"); + String::new() + } + }; + for cmd in commands { let rendered_cmd = cmd .replace("{{BIN_PATH}}", &bin_path.to_string_lossy()) .replace("{{DATA_PATH}}", &data_path.to_string_lossy()) .replace("{{CONF_PATH}}", &conf_path.to_string_lossy()) - .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()); + .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy()) + .replace("{{DB_PASSWORD}}", &db_password); if target == "local" { trace!("Executing command: {}", rendered_cmd); let child = Command::new("bash") diff --git a/src/core/package_manager/installer.rs b/src/core/package_manager/installer.rs index 55a4436d..5ca96777 100644 --- a/src/core/package_manager/installer.rs +++ b/src/core/package_manager/installer.rs @@ -107,7 +107,7 @@ impl PackageManager { ("MINIO_ROOT_PASSWORD".to_string(), "$DRIVE_SECRET".to_string()), ]), data_download_list: Vec::new(), - exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 --certs-dir {{CONF_PATH}}/system/certificates/minio > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(), + exec_cmd: "nohup {{BIN_PATH}}/minio server {{DATA_PATH}} --address :9000 --console-address :9001 --certs-dir {{CONF_PATH}}/system/certificates/drive > {{LOGS_PATH}}/minio.log 2>&1 &".to_string(), check_cmd: "ps -ef | grep minio | grep -v grep | grep {{BIN_PATH}}".to_string(), }, ); @@ -124,20 +124,20 @@ impl PackageManager { macos_packages: vec![], windows_packages: vec![], download_url: Some( - "https://github.com/theseus-rs/postgresql-binaries/releases/download/18.0.0/postgresql-18.0.0-x86_64-unknown-linux-gnu.tar.gz".to_string(), + "https://github.com/theseus-rs/postgresql-binaries/releases/download/17.2.0/postgresql-17.2.0-x86_64-unknown-linux-gnu.tar.gz".to_string(), ), binary_name: Some("postgres".to_string()), pre_install_cmds_linux: vec![], post_install_cmds_linux: vec![ "chmod +x ./bin/*".to_string(), - format!("if [ ! -d \"{{{{DATA_PATH}}}}/pgdata\" ]; then PG_PASSWORD={{DB_PASSWORD}} ./bin/initdb -D {{{{DATA_PATH}}}}/pgdata -U gbuser --pwfile=<(echo $PG_PASSWORD); fi"), + "if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then PG_PASSWORD='{{DB_PASSWORD}}' ./bin/initdb -D {{DATA_PATH}}/pgdata -U gbuser --pwfile=<(echo \"$PG_PASSWORD\"); fi".to_string(), "echo \"data_directory = '{{DATA_PATH}}/pgdata'\" > {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"ident_file = '{{CONF_PATH}}/pg_ident.conf'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"port = 5432\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"listen_addresses = '*'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"ssl = on\" >> {{CONF_PATH}}/postgresql.conf".to_string(), - "echo \"ssl_cert_file = '{{CONF_PATH}}/system/certificates/postgres/server.crt'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), - "echo \"ssl_key_file = '{{CONF_PATH}}/system/certificates/postgres/server.key'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), + "echo \"ssl_cert_file = '{{CONF_PATH}}/system/certificates/tables/server.crt'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), + "echo \"ssl_key_file = '{{CONF_PATH}}/system/certificates/tables/server.key'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"ssl_ca_file = '{{CONF_PATH}}/system/certificates/ca/ca.crt'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"log_directory = '{{LOGS_PATH}}'\" >> {{CONF_PATH}}/postgresql.conf".to_string(), "echo \"logging_collector = on\" >> {{CONF_PATH}}/postgresql.conf".to_string(), @@ -147,7 +147,7 @@ impl PackageManager { "sleep 5".to_string(), "for i in $(seq 1 30); do ./bin/pg_isready -h localhost -p 5432 -U gbuser >/dev/null 2>&1 && echo 'PostgreSQL is ready' && break || echo \"Waiting for PostgreSQL... attempt $i/30\" >&2; sleep 2; done".to_string(), "./bin/pg_isready -h localhost -p 5432 -U gbuser || { echo 'ERROR: PostgreSQL failed to start properly' >&2; cat {{LOGS_PATH}}/postgres.log >&2; exit 1; }".to_string(), - format!("PGPASSWORD={{DB_PASSWORD}} ./bin/psql -h localhost -p 5432 -U gbuser -d postgres -c \"CREATE DATABASE botserver WITH OWNER gbuser\" 2>&1 | grep -v 'already exists' || true"), + "PGPASSWORD='{{DB_PASSWORD}}' ./bin/psql -h localhost -p 5432 -U gbuser -d postgres -c \"CREATE DATABASE botserver WITH OWNER gbuser\" 2>&1 | grep -v 'already exists' || true".to_string(), ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![ @@ -165,6 +165,8 @@ impl PackageManager { } fn register_cache(&mut self) { + // Using Valkey - the Redis-compatible fork with pre-built binaries + // Valkey is maintained by the Linux Foundation and provides direct binary downloads self.components.insert( "cache".to_string(), ComponentConfig { @@ -175,19 +177,23 @@ impl PackageManager { macos_packages: vec![], windows_packages: vec![], download_url: Some( - "https://download.redis.io/redis-stable.tar.gz".to_string(), + "https://github.com/valkey-io/valkey/releases/download/9.0.0/valkey-9.0.0-linux-x86_64.tar.gz".to_string(), ), - binary_name: Some("redis-server".to_string()), + binary_name: Some("valkey-server".to_string()), pre_install_cmds_linux: vec![], - post_install_cmds_linux: vec![], + post_install_cmds_linux: vec![ + // Create symlink for redis-server compatibility + "ln -sf {{BIN_PATH}}/valkey-server {{BIN_PATH}}/redis-server 2>/dev/null || true".to_string(), + "ln -sf {{BIN_PATH}}/valkey-cli {{BIN_PATH}}/redis-cli 2>/dev/null || true".to_string(), + ], pre_install_cmds_macos: vec![], post_install_cmds_macos: vec![], pre_install_cmds_windows: vec![], post_install_cmds_windows: vec![], env_vars: HashMap::new(), data_download_list: Vec::new(), - exec_cmd: "{{BIN_PATH}}/redis-server --port 0 --tls-port 6379 --tls-cert-file {{CONF_PATH}}/system/certificates/redis/server.crt --tls-key-file {{CONF_PATH}}/system/certificates/redis/server.key --tls-ca-cert-file {{CONF_PATH}}/system/certificates/ca/ca.crt".to_string(), - check_cmd: "ps -ef | grep redis-server | grep -v grep | grep {{BIN_PATH}}".to_string(), + exec_cmd: "nohup {{BIN_PATH}}/valkey-server --port 6379 --dir {{DATA_PATH}} --logfile {{LOGS_PATH}}/valkey.log --daemonize yes > {{LOGS_PATH}}/valkey-startup.log 2>&1".to_string(), + check_cmd: "{{BIN_PATH}}/valkey-cli ping 2>/dev/null | grep -q PONG".to_string(), }, ); } @@ -308,11 +314,14 @@ impl PackageManager { binary_name: Some("zitadel".to_string()), pre_install_cmds_linux: vec![ "mkdir -p {{CONF_PATH}}/directory".to_string(), + "mkdir -p {{LOGS_PATH}}".to_string(), ], post_install_cmds_linux: vec![ - // Initialize Zitadel with first instance setup to generate admin PAT - "{{BIN_PATH}}/zitadel init --config {{CONF_PATH}}/directory/zitadel.yaml".to_string(), - "{{BIN_PATH}}/zitadel setup --config {{CONF_PATH}}/directory/zitadel.yaml --init-projections --masterkeyFromEnv --steps {{CONF_PATH}}/directory/steps.yaml".to_string(), + // Use start-from-init which does init + setup + start in one command + // This properly creates the first instance with PAT + "ZITADEL_MASTERKEY=MasterkeyNeedsToHave32Characters nohup {{BIN_PATH}}/zitadel start-from-init --config {{CONF_PATH}}/directory/zitadel.yaml --masterkeyFromEnv --tlsMode disabled --steps {{CONF_PATH}}/directory/steps.yaml > {{LOGS_PATH}}/zitadel.log 2>&1 &".to_string(), + // Wait for Zitadel to be fully ready (up to 90 seconds for first instance setup) + "for i in $(seq 1 90); do curl -sf http://localhost:8080/debug/ready && break || sleep 1; done".to_string(), ], pre_install_cmds_macos: vec![ "mkdir -p {{CONF_PATH}}/directory".to_string(), @@ -718,31 +727,17 @@ impl PackageManager { macos_packages: vec![], windows_packages: vec![], download_url: Some( - "https://releases.hashicorp.com/vault/1.15.4/vault_1.15.4_linux_amd64.zip".to_string(), + "https://releases.hashicorp.com/vault/1.15.4/vault_1.15.4_linux_amd64.zip" + .to_string(), ), binary_name: Some("vault".to_string()), pre_install_cmds_linux: vec![ "mkdir -p {{DATA_PATH}}/vault".to_string(), "mkdir -p {{CONF_PATH}}/vault".to_string(), ], - post_install_cmds_linux: vec![ - // Initialize Vault and store root token - "{{BIN_PATH}}/vault operator init -key-shares=1 -key-threshold=1 -format=json > {{CONF_PATH}}/vault/init.json".to_string(), - // Extract and store unseal key and root token - "VAULT_UNSEAL_KEY=$(cat {{CONF_PATH}}/vault/init.json | grep -o '\"unseal_keys_b64\":\\[\"[^\"]*\"' | cut -d'\"' -f4)".to_string(), - "VAULT_ROOT_TOKEN=$(cat {{CONF_PATH}}/vault/init.json | grep -o '\"root_token\":\"[^\"]*\"' | cut -d'\"' -f4)".to_string(), - // Unseal vault - "{{BIN_PATH}}/vault operator unseal $VAULT_UNSEAL_KEY".to_string(), - // Enable KV secrets engine - "VAULT_TOKEN=$VAULT_ROOT_TOKEN {{BIN_PATH}}/vault secrets enable -path=gbo kv-v2".to_string(), - // Store initial secrets paths - "VAULT_TOKEN=$VAULT_ROOT_TOKEN {{BIN_PATH}}/vault kv put gbo/drive accesskey={{GENERATED_PASSWORD}} secret={{GENERATED_PASSWORD}}".to_string(), - "VAULT_TOKEN=$VAULT_ROOT_TOKEN {{BIN_PATH}}/vault kv put gbo/tables username=gbuser password={{GENERATED_PASSWORD}}".to_string(), - "VAULT_TOKEN=$VAULT_ROOT_TOKEN {{BIN_PATH}}/vault kv put gbo/cache password={{GENERATED_PASSWORD}}".to_string(), - "VAULT_TOKEN=$VAULT_ROOT_TOKEN {{BIN_PATH}}/vault kv put gbo/directory client_id= client_secret=".to_string(), - "echo 'Vault initialized. Add VAULT_ADDR=https://localhost:8200 and VAULT_TOKEN to .env'".to_string(), - "chmod 600 {{CONF_PATH}}/vault/init.json".to_string(), - ], + // Note: Vault initialization is handled in bootstrap::setup_vault() + // because it requires the Vault server to be running first + post_install_cmds_linux: vec![], pre_install_cmds_macos: vec![ "mkdir -p {{DATA_PATH}}/vault".to_string(), "mkdir -p {{CONF_PATH}}/vault".to_string(), @@ -752,13 +747,18 @@ impl PackageManager { post_install_cmds_windows: vec![], env_vars: { let mut env = HashMap::new(); - env.insert("VAULT_ADDR".to_string(), "https://localhost:8200".to_string()); + env.insert( + "VAULT_ADDR".to_string(), + "https://localhost:8200".to_string(), + ); env.insert("VAULT_SKIP_VERIFY".to_string(), "true".to_string()); env }, data_download_list: Vec::new(), - exec_cmd: "{{BIN_PATH}}/vault server -config={{CONF_PATH}}/vault/config.hcl".to_string(), - check_cmd: "curl -f -k https://localhost:8200/v1/sys/health >/dev/null 2>&1".to_string(), + exec_cmd: "{{BIN_PATH}}/vault server -config={{CONF_PATH}}/vault/config.hcl" + .to_string(), + check_cmd: "curl -f -k https://localhost:8200/v1/sys/health >/dev/null 2>&1" + .to_string(), }, ); } diff --git a/src/core/secrets/mod.rs b/src/core/secrets/mod.rs index 409cbcb5..e93c4b60 100644 --- a/src/core/secrets/mod.rs +++ b/src/core/secrets/mod.rs @@ -75,7 +75,7 @@ impl std::fmt::Debug for SecretsManager { impl SecretsManager { /// Create from environment variables with mTLS support - /// + /// /// Environment variables: /// - VAULT_ADDR - Vault server address (https://localhost:8200) /// - VAULT_TOKEN - Vault authentication token @@ -94,14 +94,16 @@ impl SecretsManager { .ok() .and_then(|v| v.parse().ok()) .unwrap_or(300); - + // mTLS certificate paths - default to botserver-stack paths let ca_cert = env::var("VAULT_CACERT") .unwrap_or_else(|_| "./botserver-stack/conf/system/certificates/ca/ca.crt".to_string()); - let client_cert = env::var("VAULT_CLIENT_CERT") - .unwrap_or_else(|_| "./botserver-stack/conf/system/certificates/botserver/client.crt".to_string()); - let client_key = env::var("VAULT_CLIENT_KEY") - .unwrap_or_else(|_| "./botserver-stack/conf/system/certificates/botserver/client.key".to_string()); + let client_cert = env::var("VAULT_CLIENT_CERT").unwrap_or_else(|_| { + "./botserver-stack/conf/system/certificates/botserver/client.crt".to_string() + }); + let client_key = env::var("VAULT_CLIENT_KEY").unwrap_or_else(|_| { + "./botserver-stack/conf/system/certificates/botserver/client.key".to_string() + }); let enabled = !token.is_empty() && !addr.is_empty(); @@ -119,40 +121,28 @@ impl SecretsManager { let ca_path = PathBuf::from(&ca_cert); let cert_path = PathBuf::from(&client_cert); let key_path = PathBuf::from(&client_key); - + let mut settings_builder = VaultClientSettingsBuilder::default(); - settings_builder - .address(&addr) - .token(&token); - + settings_builder.address(&addr).token(&token); + // Configure TLS verification if skip_verify { warn!("TLS verification disabled - NOT RECOMMENDED FOR PRODUCTION"); settings_builder.verify(false); } else { settings_builder.verify(true); - // Add CA certificate if it exists if ca_path.exists() { info!("Using CA certificate for Vault: {}", ca_cert); settings_builder.ca_certs(vec![ca_cert.clone()]); } } - + // Configure mTLS client certificates if they exist - if cert_path.exists() && key_path.exists() { + if cert_path.exists() && key_path.exists() && !skip_verify { info!("Using mTLS client certificate for Vault: {}", client_cert); - // Note: vaultrs uses the identity parameter for client certificates - // The identity is a PKCS12/PFX file or can be set via environment - // For now, we set environment variables that the underlying reqwest client will use - env::set_var("SSL_CERT_FILE", &ca_cert); - // Client certificate authentication is handled by reqwest through env vars - // or by building a custom client - vaultrs doesn't directly support client certs - // We'll document this limitation and use token auth with TLS verification - } else if !skip_verify { - info!("mTLS client certificates not found at {} - using token auth with TLS", client_cert); } - + let settings = settings_builder.build()?; let client = VaultClient::new(settings)?; diff --git a/src/core/shared/utils.rs b/src/core/shared/utils.rs index 534e85ff..c8fe5142 100644 --- a/src/core/shared/utils.rs +++ b/src/core/shared/utils.rs @@ -140,11 +140,16 @@ pub fn to_array(value: Dynamic) -> Array { /// Download a file from a URL with progress bar (when progress-bars feature is enabled) #[cfg(feature = "progress-bars")] pub async fn download_file(url: &str, output_path: &str) -> Result<(), anyhow::Error> { + use std::time::Duration; let url = url.to_string(); let output_path = output_path.to_string(); let download_handle = tokio::spawn(async move { let client = Client::builder() .user_agent("Mozilla/5.0 (compatible; BotServer/1.0)") + .connect_timeout(Duration::from_secs(30)) + .read_timeout(Duration::from_secs(300)) + .pool_idle_timeout(Duration::from_secs(90)) + .tcp_keepalive(Duration::from_secs(60)) .build()?; let response = client.get(&url).send().await?; if response.status().is_success() { @@ -176,11 +181,16 @@ pub async fn download_file(url: &str, output_path: &str) -> Result<(), anyhow::E /// Download a file from a URL (without progress bar when progress-bars feature is disabled) #[cfg(not(feature = "progress-bars"))] pub async fn download_file(url: &str, output_path: &str) -> Result<(), anyhow::Error> { + use std::time::Duration; let url = url.to_string(); let output_path = output_path.to_string(); let download_handle = tokio::spawn(async move { let client = Client::builder() .user_agent("Mozilla/5.0 (compatible; BotServer/1.0)") + .connect_timeout(Duration::from_secs(30)) + .read_timeout(Duration::from_secs(300)) + .pool_idle_timeout(Duration::from_secs(90)) + .tcp_keepalive(Duration::from_secs(60)) .build()?; let response = client.get(&url).send().await?; if response.status().is_success() { diff --git a/src/main.rs b/src/main.rs index 5936eeb6..c88e43b5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -255,7 +255,10 @@ async fn main() -> std::io::Result<()> { // Initialize SecretsManager early - this connects to Vault if configured // Only VAULT_ADDR, VAULT_TOKEN, and VAULT_SKIP_VERIFY should be in .env if let Err(e) = crate::shared::utils::init_secrets_manager().await { - warn!("Failed to initialize SecretsManager: {}. Falling back to env vars.", e); + warn!( + "Failed to initialize SecretsManager: {}. Falling back to env vars.", + e + ); } else { info!("SecretsManager initialized - fetching secrets from Vault"); } @@ -416,11 +419,20 @@ async fn main() -> std::io::Result<()> { trace!("Creating BootstrapManager..."); let mut bootstrap = BootstrapManager::new(install_mode.clone(), tenant.clone()).await; - // Check if services are already configured in Directory - let services_configured = - std::path::Path::new("./botserver-stack/conf/directory/zitadel.yaml").exists(); + // Check if bootstrap has completed by looking for: + // 1. .env with VAULT_TOKEN + // 2. Vault init.json exists (actual credentials) + // Both must exist for bootstrap to be considered complete + let env_path = std::path::Path::new("./.env"); + let vault_init_path = std::path::Path::new("./botserver-stack/conf/vault/init.json"); + let bootstrap_completed = env_path.exists() && vault_init_path.exists() && { + // Check if .env contains VAULT_TOKEN (not just exists) + std::fs::read_to_string(env_path) + .map(|content| content.contains("VAULT_TOKEN=")) + .unwrap_or(false) + }; - let cfg = if services_configured { + let cfg = if bootstrap_completed { trace!("Services already configured, ensuring all are running..."); info!("Ensuring database and drive services are running..."); progress_tx_clone @@ -437,6 +449,7 @@ async fn main() -> std::io::Result<()> { bootstrap .start_all() + .await .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; trace!("bootstrap.start_all() completed"); @@ -471,6 +484,7 @@ async fn main() -> std::io::Result<()> { .ok(); bootstrap .start_all() + .await .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; match create_conn() { @@ -480,21 +494,34 @@ async fn main() -> std::io::Result<()> { } }; - trace!("Config loaded, uploading templates..."); + trace!("Config loaded, syncing templates to database..."); progress_tx_clone .send(BootstrapProgress::UploadingTemplates) .ok(); - if let Err(e) = bootstrap.upload_templates_to_drive(&cfg).await { - trace!("Template upload error: {}", e); - progress_tx_clone - .send(BootstrapProgress::BootstrapError(format!( - "Failed to upload templates: {}", - e - ))) - .ok(); + // First sync config.csv to database (fast, no S3 needed) + if let Err(e) = bootstrap.sync_templates_to_database() { + warn!("Failed to sync templates to database: {}", e); } else { - trace!("Templates uploaded successfully"); + trace!("Templates synced to database"); + } + + // Then upload to drive with timeout to prevent blocking on MinIO issues + match tokio::time::timeout( + std::time::Duration::from_secs(30), + bootstrap.upload_templates_to_drive(&cfg), + ) + .await + { + Ok(Ok(_)) => { + trace!("Templates uploaded to drive successfully"); + } + Ok(Err(e)) => { + warn!("Template drive upload error (non-blocking): {}", e); + } + Err(_) => { + warn!("Template drive upload timed out after 30s, continuing startup..."); + } } Ok::(cfg) @@ -505,10 +532,6 @@ async fn main() -> std::io::Result<()> { trace!("Reloading dotenv..."); dotenv().ok(); - trace!("Loading refreshed config from env..."); - let refreshed_cfg = AppConfig::from_env().expect("Failed to load config from env"); - let config = std::sync::Arc::new(refreshed_cfg.clone()); - trace!("Creating database pool again..."); progress_tx.send(BootstrapProgress::ConnectingDatabase).ok(); @@ -541,6 +564,21 @@ async fn main() -> std::io::Result<()> { } }; + // Load config from database (which now has values from config.csv) + info!("Loading config from database after template sync..."); + let refreshed_cfg = AppConfig::from_database(&pool).unwrap_or_else(|e| { + warn!( + "Failed to load config from database: {}, falling back to env", + e + ); + AppConfig::from_env().expect("Failed to load config from env") + }); + let config = std::sync::Arc::new(refreshed_cfg.clone()); + info!( + "Server configured to listen on {}:{}", + config.server.host, config.server.port + ); + let cache_url = "rediss://localhost:6379".to_string(); let redis_client = match redis::Client::open(cache_url.as_str()) { Ok(client) => Some(Arc::new(client)), @@ -583,11 +621,16 @@ async fn main() -> std::io::Result<()> { let config_manager = ConfigManager::new(pool.clone()); let mut bot_conn = pool.get().expect("Failed to get database connection"); - let (default_bot_id, _default_bot_name) = crate::bot::get_default_bot(&mut bot_conn); + let (default_bot_id, default_bot_name) = crate::bot::get_default_bot(&mut bot_conn); + info!( + "Using default bot: {} (id: {})", + default_bot_name, default_bot_id + ); let llm_url = config_manager - .get_config(&default_bot_id, "llm-url", Some("https://localhost:8081")) - .unwrap_or_else(|_| "https://localhost:8081".to_string()); + .get_config(&default_bot_id, "llm-url", Some("http://localhost:8081")) + .unwrap_or_else(|_| "http://localhost:8081".to_string()); + info!("LLM URL: {}", llm_url); // Create base LLM provider let base_llm_provider = Arc::new(botserver::llm::OpenAIClient::new( @@ -602,12 +645,14 @@ async fn main() -> std::io::Result<()> { .get_config( &default_bot_id, "embedding-url", - Some("https://localhost:8082"), + Some("http://localhost:8082"), ) - .unwrap_or_else(|_| "https://localhost:8082".to_string()); + .unwrap_or_else(|_| "http://localhost:8082".to_string()); let embedding_model = config_manager .get_config(&default_bot_id, "embedding-model", Some("all-MiniLM-L6-v2")) .unwrap_or_else(|_| "all-MiniLM-L6-v2".to_string()); + info!("Embedding URL: {}", embedding_url); + info!("Embedding Model: {}", embedding_model); let embedding_service = Some(Arc::new(botserver::llm::cache::LocalEmbeddingService::new( embedding_url, diff --git a/templates/default.gbai/default.gbot/config.csv b/templates/default.gbai/default.gbot/config.csv index 9a0dfd11..89c06043 100644 --- a/templates/default.gbai/default.gbot/config.csv +++ b/templates/default.gbai/default.gbot/config.csv @@ -4,7 +4,7 @@ name,value # SERVER CONFIGURATION # ============================================================================ server_host,0.0.0.0 -server_port,8080 +server_port,8088 sites_root,/tmp , # ============================================================================