All checks were successful
Gitea Actions Demo / build (push) Successful in 4m54s
- Add unified retry strategy with exponential backoff and jitter - Implement health monitoring system for service status tracking - Enhance WebSocket connection with timeout and heartbeat detection - Improve WAN polling with connection timeout and error handling - Add comprehensive test suite for retry mechanisms - Fix connection hanging issues that prevented proper retry recovery Key improvements: - RetryConfig with fast/slow/infinite retry strategies - HealthMonitor for real-time service status tracking - Connection timeouts to prevent hanging - Automatic ping/pong handling for WebSocket - Consecutive error counting and thresholds - Detailed logging for better diagnostics All tests passing: 11 tests (3 unit + 5 integration + 3 lib tests)
149 lines
4.5 KiB
Rust
149 lines
4.5 KiB
Rust
use std::sync::atomic::{AtomicU32, Ordering};
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
use tokio::time::sleep;
|
|
use network_monitor::retry::{RetryConfig, retry_with_config};
|
|
|
|
/// 模拟网络故障的测试
|
|
#[tokio::test]
|
|
async fn test_network_failure_recovery() {
|
|
// 模拟一个会失败几次然后成功的操作
|
|
let attempt_count = Arc::new(AtomicU32::new(0));
|
|
let max_failures = 3;
|
|
|
|
let config = RetryConfig {
|
|
max_attempts: 10,
|
|
initial_delay: Duration::from_millis(10),
|
|
max_delay: Duration::from_millis(100),
|
|
backoff_multiplier: 1.5,
|
|
jitter: false, // 关闭抖动以便测试更可预测
|
|
};
|
|
|
|
let attempt_count_clone = attempt_count.clone();
|
|
let result = retry_with_config(config, move || {
|
|
let attempt_count = attempt_count_clone.clone();
|
|
async move {
|
|
let current_attempt = attempt_count.fetch_add(1, Ordering::SeqCst) + 1;
|
|
|
|
if current_attempt <= max_failures {
|
|
// 模拟网络错误
|
|
Err(format!("Network error on attempt {}", current_attempt))
|
|
} else {
|
|
// 模拟恢复成功
|
|
Ok(format!("Success on attempt {}", current_attempt))
|
|
}
|
|
}
|
|
}).await;
|
|
|
|
assert!(result.is_ok());
|
|
assert_eq!(result.unwrap(), "Success on attempt 4");
|
|
assert_eq!(attempt_count.load(Ordering::SeqCst), 4);
|
|
}
|
|
|
|
/// 测试连接超时场景
|
|
#[tokio::test]
|
|
async fn test_connection_timeout_scenario() {
|
|
let config = RetryConfig {
|
|
max_attempts: 3,
|
|
initial_delay: Duration::from_millis(5),
|
|
max_delay: Duration::from_millis(20),
|
|
backoff_multiplier: 2.0,
|
|
jitter: false,
|
|
};
|
|
|
|
let attempt_count = Arc::new(AtomicU32::new(0));
|
|
|
|
let attempt_count_clone = attempt_count.clone();
|
|
let result: Result<String, &str> = retry_with_config(config, move || {
|
|
let attempt_count = attempt_count_clone.clone();
|
|
async move {
|
|
attempt_count.fetch_add(1, Ordering::SeqCst);
|
|
|
|
// 模拟连接超时
|
|
sleep(Duration::from_millis(1)).await;
|
|
Err("Connection timeout")
|
|
}
|
|
}).await;
|
|
|
|
assert!(result.is_err());
|
|
assert_eq!(attempt_count.load(Ordering::SeqCst), 3); // 应该尝试了3次
|
|
}
|
|
|
|
/// 测试快速恢复场景
|
|
#[tokio::test]
|
|
async fn test_fast_recovery() {
|
|
let config = RetryConfig::fast();
|
|
|
|
let attempt_count = Arc::new(AtomicU32::new(0));
|
|
|
|
let attempt_count_clone = attempt_count.clone();
|
|
let result = retry_with_config(config, move || {
|
|
let attempt_count = attempt_count_clone.clone();
|
|
async move {
|
|
let current_attempt = attempt_count.fetch_add(1, Ordering::SeqCst) + 1;
|
|
|
|
if current_attempt == 1 {
|
|
Err("Temporary failure")
|
|
} else {
|
|
Ok("Quick recovery")
|
|
}
|
|
}
|
|
}).await;
|
|
|
|
assert!(result.is_ok());
|
|
assert_eq!(result.unwrap(), "Quick recovery");
|
|
assert_eq!(attempt_count.load(Ordering::SeqCst), 2);
|
|
}
|
|
|
|
/// 测试慢速重试场景
|
|
#[tokio::test]
|
|
async fn test_slow_retry_scenario() {
|
|
let config = RetryConfig::slow();
|
|
|
|
let attempt_count = Arc::new(AtomicU32::new(0));
|
|
|
|
let attempt_count_clone = attempt_count.clone();
|
|
let result = retry_with_config(config, move || {
|
|
let attempt_count = attempt_count_clone.clone();
|
|
async move {
|
|
let current_attempt = attempt_count.fetch_add(1, Ordering::SeqCst) + 1;
|
|
|
|
if current_attempt <= 2 {
|
|
Err("Service unavailable")
|
|
} else {
|
|
Ok("Service restored")
|
|
}
|
|
}
|
|
}).await;
|
|
|
|
assert!(result.is_ok());
|
|
assert_eq!(result.unwrap(), "Service restored");
|
|
assert_eq!(attempt_count.load(Ordering::SeqCst), 3);
|
|
}
|
|
|
|
/// 测试最大重试次数限制
|
|
#[tokio::test]
|
|
async fn test_max_retry_limit() {
|
|
let config = RetryConfig {
|
|
max_attempts: 2,
|
|
initial_delay: Duration::from_millis(1),
|
|
max_delay: Duration::from_millis(5),
|
|
backoff_multiplier: 2.0,
|
|
jitter: false,
|
|
};
|
|
|
|
let attempt_count = Arc::new(AtomicU32::new(0));
|
|
|
|
let attempt_count_clone = attempt_count.clone();
|
|
let result = retry_with_config(config, move || {
|
|
let attempt_count = attempt_count_clone.clone();
|
|
async move {
|
|
attempt_count.fetch_add(1, Ordering::SeqCst);
|
|
Err::<String, &str>("Persistent failure")
|
|
}
|
|
}).await;
|
|
|
|
assert!(result.is_err());
|
|
assert_eq!(attempt_count.load(Ordering::SeqCst), 2);
|
|
}
|