feat: Add retry logic for cache/Valkey connection
All checks were successful
BotServer CI / build (push) Successful in 8m19s
All checks were successful
BotServer CI / build (push) Successful in 8m19s
- Wait up to 30 seconds for cache to be ready - Retry every 1 second with progress logging - Prevents race condition during service startup - Ensures suggestions feature works when Valkey starts after botserver Fixes issue where cache connection failed during bootstrap if Valkey wasn't immediately ready. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
4e4aca0986
commit
8d0fd9da3d
1 changed files with 52 additions and 26 deletions
|
|
@ -273,7 +273,7 @@ pub async fn load_config(
|
||||||
Ok(refreshed_cfg)
|
Ok(refreshed_cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Initialize Redis/Valkey cache
|
/// Initialize Redis/Valkey cache with retry logic
|
||||||
#[cfg(feature = "cache")]
|
#[cfg(feature = "cache")]
|
||||||
pub async fn init_redis() -> Option<Arc<redis::Client>> {
|
pub async fn init_redis() -> Option<Arc<redis::Client>> {
|
||||||
let cache_url = std::env::var("CACHE_URL")
|
let cache_url = std::env::var("CACHE_URL")
|
||||||
|
|
@ -283,38 +283,64 @@ pub async fn init_redis() -> Option<Arc<redis::Client>> {
|
||||||
|
|
||||||
info!("Attempting to connect to cache at: {}", cache_url);
|
info!("Attempting to connect to cache at: {}", cache_url);
|
||||||
|
|
||||||
match redis::Client::open(cache_url.as_str()) {
|
let max_attempts = 30; // Try for up to 30 seconds
|
||||||
Ok(client) => {
|
let mut attempt = 0;
|
||||||
// Verify the connection actually works
|
|
||||||
match client.get_connection() {
|
loop {
|
||||||
Ok(mut conn) => {
|
attempt += 1;
|
||||||
// Test with PING
|
|
||||||
match redis::cmd("PING").query::<String>(&mut conn) {
|
match redis::Client::open(cache_url.as_str()) {
|
||||||
Ok(response) if response == "PONG" => {
|
Ok(client) => {
|
||||||
info!("Cache connection verified: PONG");
|
// Verify the connection actually works
|
||||||
Some(Arc::new(client))
|
match client.get_connection() {
|
||||||
|
Ok(mut conn) => {
|
||||||
|
// Test with PING
|
||||||
|
match redis::cmd("PING").query::<String>(&mut conn) {
|
||||||
|
Ok(response) if response == "PONG" => {
|
||||||
|
info!("Cache connection verified: PONG");
|
||||||
|
return Some(Arc::new(client));
|
||||||
|
}
|
||||||
|
Ok(response) => {
|
||||||
|
warn!("Cache PING returned unexpected response: {}", response);
|
||||||
|
return Some(Arc::new(client));
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
if attempt < max_attempts {
|
||||||
|
info!("Cache PING failed (attempt {}/{}): {}. Retrying in 1s...", attempt, max_attempts, e);
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
warn!("Cache PING failed after {} attempts: {}. Cache functions will be disabled.", max_attempts, e);
|
||||||
|
warn!("Suggestions and other cache-dependent features will not work.");
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(response) => {
|
}
|
||||||
warn!("Cache PING returned unexpected response: {}", response);
|
Err(e) => {
|
||||||
Some(Arc::new(client))
|
if attempt < max_attempts {
|
||||||
}
|
info!("Failed to establish cache connection (attempt {}/{}): {}. Retrying in 1s...", attempt, max_attempts, e);
|
||||||
Err(e) => {
|
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
|
||||||
warn!("Cache PING failed: {}. Cache functions will be disabled.", e);
|
continue;
|
||||||
None
|
} else {
|
||||||
|
warn!("Failed to establish cache connection after {} attempts: {}. Cache functions will be disabled.", max_attempts, e);
|
||||||
|
warn!("Suggestions and other cache-dependent features will not work.");
|
||||||
|
return None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
}
|
||||||
warn!("Failed to establish cache connection: {}. Cache functions will be disabled.", e);
|
Err(e) => {
|
||||||
warn!("Suggestions and other cache-dependent features will not work.");
|
if attempt < max_attempts {
|
||||||
None
|
info!("Failed to create cache client (attempt {}/{}): {}. Retrying in 1s...", attempt, max_attempts, e);
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
log::warn!("Failed to create cache client after {} attempts: {}. Cache functions will be disabled.", max_attempts, e);
|
||||||
|
return None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
|
||||||
log::warn!("Failed to create cache client: {}. Cache functions will be disabled.", e);
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue