From ee273256fb9b6f0c77fae51a7ae8305777e09642 Mon Sep 17 00:00:00 2001
From: "Rodrigo Rodriguez (Pragmatismo)" <me@rodrigorodriguez.com>
Date: Sun, 12 Apr 2026 09:13:33 -0300
Subject: [PATCH] Add backoff logic to KB indexing to prevent excessive retries

- fail_count 1: wait 5 minutes before retry
- fail_count 2: wait 15 minutes before retry
- fail_count 3+: wait 1 hour before retry

This prevents the 'already being indexed, skipping duplicate task' loop.
---
 src/drive/drive_monitor/mod.rs | 43 ++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/src/drive/drive_monitor/mod.rs b/src/drive/drive_monitor/mod.rs
index 2a7bc61f..75007b23 100644
--- a/src/drive/drive_monitor/mod.rs
+++ b/src/drive/drive_monitor/mod.rs
@@ -1282,16 +1282,6 @@ impl DriveMonitor {
                         .join(&gbkb_prefix)
                         .join(kb_name);
 
-                    let kb_indexing_disabled = false;
-
-                    if kb_indexing_disabled {
-                        debug!(
-                            "KB indexing disabled via DISABLE_KB_INDEXING, skipping {}",
-                            kb_folder_path.display()
-                        );
-                        continue;
-                    }
-
                     #[cfg(any(feature = "research", feature = "llm"))]
                     {
                         if !is_embedding_server_ready() {
@@ -1301,6 +1291,39 @@ impl DriveMonitor {
                         // Create a unique key for this KB folder to track indexing state
                         let kb_key = format!("{}_{}", bot_name, kb_name);
 
+            // Check fail_count for this KB folder - implement backoff
+            {
+                let states = self.file_states.read().await;
+                let _kb_prefix = format!("{}/", gbkb_prefix);
+                
+                let max_fail_count = states.values()
+                    .map(|s| s.fail_count)
+                    .max()
+                    .unwrap_or(0);
+                
+                // Backoff: wait longer based on fail count
+                // fail_count 0: no wait, 1: 5min, 2: 15min, 3+: 1h
+                if max_fail_count > 0 {
+                    let wait_seconds = match max_fail_count {
+                        1 => 300,   // 5 min
+                        2 => 900,   // 15 min
+                        _ => 3600,  // 1 hour
+                    };
+                    
+                    if let Some(last_failed) = states.values()
+                        .filter_map(|s| s.last_failed_at)
+                        .max() 
+                    {
+                        let elapsed = chrono::Utc::now() - last_failed;
+                        if elapsed.num_seconds() < wait_seconds {
+                            trace!("[DRIVE_MONITOR] KB folder {} in backoff (fail_count={}, elapsed={}s < {}s), skipping", 
+                                kb_key, max_fail_count, elapsed.num_seconds(), wait_seconds);
+                            continue;
+                        }
+                    }
+                }
+            }
+
             // Check if this KB folder is already being indexed
             {
                 let indexing_set = self.files_being_indexed.read().await;