feat: implement robust retry mechanism for API aggregation service
All checks were successful
Gitea Actions Demo / build (push) Successful in 4m54s
All checks were successful
Gitea Actions Demo / build (push) Successful in 4m54s
- Add unified retry strategy with exponential backoff and jitter - Implement health monitoring system for service status tracking - Enhance WebSocket connection with timeout and heartbeat detection - Improve WAN polling with connection timeout and error handling - Add comprehensive test suite for retry mechanisms - Fix connection hanging issues that prevented proper retry recovery Key improvements: - RetryConfig with fast/slow/infinite retry strategies - HealthMonitor for real-time service status tracking - Connection timeouts to prevent hanging - Automatic ping/pong handling for WebSocket - Consecutive error counting and thresholds - Detailed logging for better diagnostics All tests passing: 11 tests (3 unit + 5 integration + 3 lib tests)
This commit is contained in:
148
tests/integration_test.rs
Normal file
148
tests/integration_test.rs
Normal file
@@ -0,0 +1,148 @@
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use network_monitor::retry::{RetryConfig, retry_with_config};
|
||||
|
||||
/// 模拟网络故障的测试
|
||||
#[tokio::test]
|
||||
async fn test_network_failure_recovery() {
|
||||
// 模拟一个会失败几次然后成功的操作
|
||||
let attempt_count = Arc::new(AtomicU32::new(0));
|
||||
let max_failures = 3;
|
||||
|
||||
let config = RetryConfig {
|
||||
max_attempts: 10,
|
||||
initial_delay: Duration::from_millis(10),
|
||||
max_delay: Duration::from_millis(100),
|
||||
backoff_multiplier: 1.5,
|
||||
jitter: false, // 关闭抖动以便测试更可预测
|
||||
};
|
||||
|
||||
let attempt_count_clone = attempt_count.clone();
|
||||
let result = retry_with_config(config, move || {
|
||||
let attempt_count = attempt_count_clone.clone();
|
||||
async move {
|
||||
let current_attempt = attempt_count.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
|
||||
if current_attempt <= max_failures {
|
||||
// 模拟网络错误
|
||||
Err(format!("Network error on attempt {}", current_attempt))
|
||||
} else {
|
||||
// 模拟恢复成功
|
||||
Ok(format!("Success on attempt {}", current_attempt))
|
||||
}
|
||||
}
|
||||
}).await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(result.unwrap(), "Success on attempt 4");
|
||||
assert_eq!(attempt_count.load(Ordering::SeqCst), 4);
|
||||
}
|
||||
|
||||
/// 测试连接超时场景
|
||||
#[tokio::test]
|
||||
async fn test_connection_timeout_scenario() {
|
||||
let config = RetryConfig {
|
||||
max_attempts: 3,
|
||||
initial_delay: Duration::from_millis(5),
|
||||
max_delay: Duration::from_millis(20),
|
||||
backoff_multiplier: 2.0,
|
||||
jitter: false,
|
||||
};
|
||||
|
||||
let attempt_count = Arc::new(AtomicU32::new(0));
|
||||
|
||||
let attempt_count_clone = attempt_count.clone();
|
||||
let result: Result<String, &str> = retry_with_config(config, move || {
|
||||
let attempt_count = attempt_count_clone.clone();
|
||||
async move {
|
||||
attempt_count.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
// 模拟连接超时
|
||||
sleep(Duration::from_millis(1)).await;
|
||||
Err("Connection timeout")
|
||||
}
|
||||
}).await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(attempt_count.load(Ordering::SeqCst), 3); // 应该尝试了3次
|
||||
}
|
||||
|
||||
/// 测试快速恢复场景
|
||||
#[tokio::test]
|
||||
async fn test_fast_recovery() {
|
||||
let config = RetryConfig::fast();
|
||||
|
||||
let attempt_count = Arc::new(AtomicU32::new(0));
|
||||
|
||||
let attempt_count_clone = attempt_count.clone();
|
||||
let result = retry_with_config(config, move || {
|
||||
let attempt_count = attempt_count_clone.clone();
|
||||
async move {
|
||||
let current_attempt = attempt_count.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
|
||||
if current_attempt == 1 {
|
||||
Err("Temporary failure")
|
||||
} else {
|
||||
Ok("Quick recovery")
|
||||
}
|
||||
}
|
||||
}).await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(result.unwrap(), "Quick recovery");
|
||||
assert_eq!(attempt_count.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
/// 测试慢速重试场景
|
||||
#[tokio::test]
|
||||
async fn test_slow_retry_scenario() {
|
||||
let config = RetryConfig::slow();
|
||||
|
||||
let attempt_count = Arc::new(AtomicU32::new(0));
|
||||
|
||||
let attempt_count_clone = attempt_count.clone();
|
||||
let result = retry_with_config(config, move || {
|
||||
let attempt_count = attempt_count_clone.clone();
|
||||
async move {
|
||||
let current_attempt = attempt_count.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
|
||||
if current_attempt <= 2 {
|
||||
Err("Service unavailable")
|
||||
} else {
|
||||
Ok("Service restored")
|
||||
}
|
||||
}
|
||||
}).await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(result.unwrap(), "Service restored");
|
||||
assert_eq!(attempt_count.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
/// 测试最大重试次数限制
|
||||
#[tokio::test]
|
||||
async fn test_max_retry_limit() {
|
||||
let config = RetryConfig {
|
||||
max_attempts: 2,
|
||||
initial_delay: Duration::from_millis(1),
|
||||
max_delay: Duration::from_millis(5),
|
||||
backoff_multiplier: 2.0,
|
||||
jitter: false,
|
||||
};
|
||||
|
||||
let attempt_count = Arc::new(AtomicU32::new(0));
|
||||
|
||||
let attempt_count_clone = attempt_count.clone();
|
||||
let result = retry_with_config(config, move || {
|
||||
let attempt_count = attempt_count_clone.clone();
|
||||
async move {
|
||||
attempt_count.fetch_add(1, Ordering::SeqCst);
|
||||
Err::<String, &str>("Persistent failure")
|
||||
}
|
||||
}).await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(attempt_count.load(Ordering::SeqCst), 2);
|
||||
}
|
Reference in New Issue
Block a user