From 7bcbe5b7338c46350ba430d84028bed652fc6e93 Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 18 Nov 2026 11:23:18 +0800 Subject: [PATCH] feat(phase-2): add scanner cycle reset and display recovery status --- crates/cli/src/commands/admin/mod.rs | 4 +- crates/cli/src/commands/admin/scanner.rs | 76 ++++++++++++- crates/cli/tests/admin_scanner.rs | 130 +++++++++++++++++++++++ crates/cli/tests/help_contract.rs | 12 ++- crates/core/src/admin/mod.rs | 6 +- crates/core/src/admin/observability.rs | 13 +++ crates/s3/src/admin.rs | 41 +++++-- docs/reference/rc/admin.md | 7 ++ schemas/output_v3.json | 19 ++++ 9 files changed, 291 insertions(+), 17 deletions(-) diff --git a/crates/cli/src/commands/admin/mod.rs b/crates/cli/src/commands/admin/mod.rs index 35059e03..0b256ffa 100644 --- a/crates/cli/src/commands/admin/mod.rs +++ b/crates/cli/src/commands/admin/mod.rs @@ -76,7 +76,7 @@ pub enum AdminCommands { #[command(subcommand)] Kms(kms::KmsCommands), - /// Inspect scanner health and freshness + /// Inspect scanner health and manage cycle recovery #[command(subcommand)] Scanner(scanner::ScannerCommands), @@ -263,7 +263,7 @@ fn observability_error_suggestion(error: &Error) -> Option<&'static str> { Error::InvalidPath(_) | Error::Config(_) => Some("Review the command arguments and retry."), Error::Network(_) => Some("Verify the endpoint and network connectivity, then retry."), Error::Auth(_) => Some("Verify credentials and required admin permissions, then retry."), - Error::UnsupportedFeature(_) => Some("Upgrade RustFS to beta.10 or later."), + Error::UnsupportedFeature(_) => Some("Use a RustFS release that supports this operation."), Error::Interrupted(_) => Some("Retry with --yes if deletion is still intended."), _ => None, } diff --git a/crates/cli/src/commands/admin/scanner.rs b/crates/cli/src/commands/admin/scanner.rs index c011427d..b5de3dc9 100644 --- a/crates/cli/src/commands/admin/scanner.rs +++ b/crates/cli/src/commands/admin/scanner.rs @@ -1,7 +1,7 @@ -//! Scanner health and freshness diagnostics. +//! Scanner health diagnostics and cycle recovery. use clap::{Args, Subcommand}; -use rc_core::admin::{ObservabilityApi, ScannerStatus}; +use rc_core::admin::{ObservabilityApi, ScannerRecoveryApi, ScannerStatus}; use serde::Serialize; use super::{emit_observability_error, get_admin_client}; @@ -12,6 +12,25 @@ use crate::output::Formatter; pub enum ScannerCommands { /// Display scanner health, freshness, and current cycle details Status(ScannerStatusArgs), + /// Reset scanner recovery state + #[command(subcommand)] + Reset(ScannerResetCommands), +} + +#[derive(Subcommand, Debug)] +pub enum ScannerResetCommands { + /// Reset cycle recovery state for a full rescan + Cycle(ScannerCycleResetArgs), +} + +#[derive(Args, Debug)] +pub struct ScannerCycleResetArgs { + /// Alias name of the server + pub alias: String, + + /// Confirm the cycle reset (requires admin:ConfigUpdate) + #[arg(long, required = true)] + pub yes: bool, } #[derive(Args, Debug)] @@ -21,12 +40,12 @@ pub struct ScannerStatusArgs { } #[derive(Debug, Serialize)] -struct ScannerSuccessOutput<'a> { +struct ScannerSuccessOutput { schema_version: u8, #[serde(rename = "type")] output_type: &'static str, status: &'static str, - data: ScannerOutput<'a>, + data: T, } #[derive(Debug, Serialize)] @@ -39,6 +58,9 @@ struct ScannerOutput<'a> { pub async fn execute(command: ScannerCommands, formatter: &Formatter) -> ExitCode { match command { ScannerCommands::Status(args) => execute_status(args, formatter).await, + ScannerCommands::Reset(ScannerResetCommands::Cycle(args)) => { + execute_cycle_reset(args, formatter).await + } } } @@ -75,6 +97,35 @@ async fn execute_status(args: ScannerStatusArgs, formatter: &Formatter) -> ExitC } } +async fn execute_cycle_reset(args: ScannerCycleResetArgs, formatter: &Formatter) -> ExitCode { + let client = match get_admin_client(&args.alias, formatter) { + Ok(client) => client, + Err(code) => return code, + }; + match client.reset_scanner_cycle().await { + Ok(result) => { + if formatter.is_json() { + formatter.json(&ScannerSuccessOutput { + schema_version: 3, + output_type: "scanner_cycle_reset", + status: "success", + data: result, + }); + } else { + formatter.println("Scanner cycle state reset (full-rescan)."); + } + ExitCode::Success + } + Err(error) => emit_observability_error( + "scanner_cycle_reset", + "admin.scanner-cycle-reset", + "Failed to reset scanner cycle state", + &error, + formatter, + ), + } +} + fn print_status(status: &ScannerStatus, formatter: &Formatter) { formatter.println(&formatter.style_name("Scanner Status")); formatter.println(""); @@ -102,6 +153,23 @@ fn print_status(status: &ScannerStatus, formatter: &Formatter) { formatter.sanitize_text(reason) )); } + if let Some(recovery) = status.extra.get("cycle_recovery") { + if let Some(state) = recovery["state"].as_str() { + formatter.println(&format!( + "Recovery state: {}", + formatter.sanitize_text(state) + )); + } + if let Some(reason) = recovery["reason"] + .as_str() + .filter(|reason| !reason.is_empty()) + { + formatter.println(&format!( + "Recovery reason: {}", + formatter.sanitize_text(reason) + )); + } + } } fn value_or_unknown(value: &str) -> &str { diff --git a/crates/cli/tests/admin_scanner.rs b/crates/cli/tests/admin_scanner.rs index 3fe27ffe..167bae69 100644 --- a/crates/cli/tests/admin_scanner.rs +++ b/crates/cli/tests/admin_scanner.rs @@ -186,3 +186,133 @@ fn scanner_status_rejects_malformed_payload() { assert_eq!(output.status.code(), Some(1)); handle.join().expect("admin test server finished"); } + +fn run_scanner(args: &[&str], endpoint: &str) -> std::process::Output { + let config_dir = tempfile::tempdir().expect("create config dir"); + Command::new(rc_binary()) + .args(["admin", "scanner"]) + .args(args) + .env("RC_CONFIG_DIR", config_dir.path()) + .env("RC_HOST_myalias", rc_host_alias(endpoint)) + .output() + .expect("run rc scanner command") +} + +#[test] +fn scanner_status_displays_recovery_separately_from_freshness() { + let mut body: Value = serde_json::from_str(HEALTHY_SCANNER).expect("scanner fixture"); + body["freshness"]["reason"] = "last cycle is too old".into(); + body["cycle_recovery"] = serde_json::json!({ + "state": "blocked", + "reason": "invalid cycle\nmarker", + "retry_count": 3, + }); + for json in [false, true] { + let (endpoint, _receiver, handle) = + start_admin_response_test_server("200 OK", "application/json", body.to_string()); + let mut args = vec!["status", "myalias"]; + if json { + args.push("--json"); + } + let output = run_scanner(&args, &endpoint); + assert!(output.status.success()); + if json { + let value: Value = serde_json::from_slice(&output.stdout).expect("scanner status JSON"); + assert_eq!(value["data"]["cycle_recovery"], body["cycle_recovery"]); + assert_valid_v3(&value); + } else { + let stdout = String::from_utf8(output.stdout).expect("status text"); + assert!(stdout.contains("Recovery state: blocked"), "{stdout}"); + assert!( + stdout.contains(r"Recovery reason: invalid cycle\nmarker"), + "{stdout}" + ); + assert!( + stdout.contains("Reason: last cycle is too old"), + "{stdout}" + ); + } + handle.join().expect("server finished"); + } +} + +#[test] +fn scanner_cycle_reset_requires_yes_before_contacting_server() { + let output = run_scanner(&["reset", "cycle", "myalias"], "http://127.0.0.1:1"); + assert_eq!(output.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&output.stderr).contains("--yes")); +} + +#[test] +fn scanner_cycle_reset_posts_full_rescan_and_reports_acknowledgement() { + let ack = r#"{"status":"reset","mode":"full-rescan"}"#; + for json in [false, true] { + let (endpoint, receiver, handle) = + start_admin_response_test_server("200 OK", "application/json", ack.to_string()); + let mut args = vec!["reset", "cycle", "myalias", "--yes"]; + if json { + args.push("--json"); + } + let output = run_scanner(&args, &endpoint); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + if json { + let value: Value = serde_json::from_slice(&output.stdout).expect("cycle reset JSON"); + assert_eq!(value["type"], "scanner_cycle_reset"); + assert_eq!( + value["data"], + serde_json::from_str::(ack).expect("ack") + ); + assert_valid_v3(&value); + } else { + assert!(String::from_utf8_lossy(&output.stdout).contains("Scanner cycle state reset")); + } + let request = receiver + .recv_timeout(Duration::from_secs(5)) + .expect("reset request"); + assert_eq!(request.method, "POST"); + assert_eq!(request.target, "/rustfs/admin/v3/scanner/cycle-state/reset"); + assert!( + request + .headers + .to_ascii_lowercase() + .contains("authorization: aws4-hmac-sha256") + ); + assert!( + request + .headers + .to_ascii_lowercase() + .contains("content-type: application/json") + ); + assert_eq!( + serde_json::from_slice::(&request.body).expect("request JSON"), + serde_json::json!({"mode": "full-rescan"}) + ); + handle.join().expect("server finished"); + } +} + +#[test] +fn scanner_cycle_reset_rejects_denied_unsupported_and_invalid_responses() { + for (status, body, exit_code) in [ + ("403 Forbidden", "Access denied", 4), + ("404 Not Found", "Route not found", 7), + ("200 OK", "{}", 1), + ("200 OK", r#"{"status":"pending","mode":"full-rescan"}"#, 1), + ("200 OK", r#"{"status":"reset","mode":"full-rebuild"}"#, 1), + ] { + let (endpoint, _receiver, handle) = + start_admin_response_test_server(status, "application/json", body.to_string()); + let output = run_scanner(&["--json", "reset", "cycle", "myalias", "--yes"], &endpoint); + assert_eq!(output.status.code(), Some(exit_code), "{status} {body}"); + assert!(output.stdout.is_empty()); + let value: Value = serde_json::from_slice(&output.stderr).expect("reset error JSON"); + assert_eq!(value["type"], "scanner_cycle_reset"); + assert_eq!(value["status"], "error"); + assert_valid_v3(&value); + handle.join().expect("server finished"); + } +} diff --git a/crates/cli/tests/help_contract.rs b/crates/cli/tests/help_contract.rs index 3582481b..530de736 100644 --- a/crates/cli/tests/help_contract.rs +++ b/crates/cli/tests/help_contract.rs @@ -743,7 +743,17 @@ fn nested_subcommand_help_contract() { HelpCase { args: &["admin", "scanner"], usage: "Usage: rc admin scanner [OPTIONS] ", - expected_tokens: &["status"], + expected_tokens: &["status", "reset"], + }, + HelpCase { + args: &["admin", "scanner", "reset"], + usage: "Usage: rc admin scanner reset [OPTIONS] ", + expected_tokens: &["cycle"], + }, + HelpCase { + args: &["admin", "scanner", "reset", "cycle"], + usage: "Usage: rc admin scanner reset cycle [OPTIONS] --yes ", + expected_tokens: &["--yes", "full rescan"], }, HelpCase { args: &["admin", "scanner", "status"], diff --git a/crates/core/src/admin/mod.rs b/crates/core/src/admin/mod.rs index 7af2284a..492b6834 100644 --- a/crates/core/src/admin/mod.rs +++ b/crates/core/src/admin/mod.rs @@ -106,9 +106,9 @@ pub use kms_diagnostic::{ pub use observability::{ MAX_METRICS_LINE_BYTES, MAX_METRICS_RESPONSE_BYTES, MAX_METRICS_SAMPLES, MetricGroup, MetricGroups, MetricsBatch, MetricsQuery, MetricsScope, ObservabilityApi, RealtimeMetrics, - ScannerCycleSchedule, ScannerFreshness, ScannerHealth, ScannerMetrics, ScannerRuntimeConfig, - ScannerRuntimeConfigValue, ScannerStatus, StorageBackend, StorageBackendKind, StorageDisk, - StorageDiskMetrics, StorageInfo, + ScannerCycleResetResult, ScannerCycleSchedule, ScannerFreshness, ScannerHealth, ScannerMetrics, + ScannerRecoveryApi, ScannerRuntimeConfig, ScannerRuntimeConfigValue, ScannerStatus, + StorageBackend, StorageBackendKind, StorageDisk, StorageDiskMetrics, StorageInfo, }; pub use oidc::{ MAX_OIDC_RESPONSE_BYTES, OidcMutationApi, OidcMutationRequest, OidcMutationResult, diff --git a/crates/core/src/admin/observability.rs b/crates/core/src/admin/observability.rs index 443dbfb0..54edb16f 100644 --- a/crates/core/src/admin/observability.rs +++ b/crates/core/src/admin/observability.rs @@ -587,6 +587,19 @@ impl StorageInfo { } } +/// Acknowledgement of a synchronous scanner cycle-state reset. +#[derive(Debug, Serialize, Deserialize)] +pub struct ScannerCycleResetResult { + pub status: String, + pub mode: String, +} + +/// Operator-initiated recovery of scanner cycle state. +#[async_trait] +pub trait ScannerRecoveryApi: Send + Sync { + async fn reset_scanner_cycle(&self) -> Result; +} + /// Read-only scanner, storage, and metrics Admin API boundary. #[async_trait] pub trait ObservabilityApi: Send + Sync { diff --git a/crates/s3/src/admin.rs b/crates/s3/src/admin.rs index 20930001..22355116 100644 --- a/crates/s3/src/admin.rs +++ b/crates/s3/src/admin.rs @@ -49,13 +49,14 @@ use rc_core::admin::{ PolicyEntitiesResult, PolicyEntity, PolicyInfo, PoolStatus, PoolTarget, RealtimeMetrics, RebalanceStartResult, RebalanceStatus, RecoveryCodes, ReplicateEditStatus, ReplicationDiff, ReplicationDiffApi, ReplicationInspectionApi, ReplicationMetricScope, ReplicationMetrics, - ReplicationMrf, RuntimeCapabilitiesSnapshot, RuntimeCapabilityStatus, ScannerStatus, - SecretValue, ServiceAccount, ServiceAccountCreateResponse, ServiceActionResult, SiteRemoveSpec, - SiteReplicationInfo, SiteReplicationPeer, SiteReplicationRepairApi, - SiteReplicationRepairCapabilityContract, SiteReplicationRepairOperationStatus, - SiteReplicationRepairPreflight, SiteReplicationRepairRequest, SiteReplicationResyncOperation, - SiteReplicationResyncStatus, SiteStatusOptions, StorageInfo, UpdateGroupMembersRequest, - UpdateServiceAccountRequest, User, UserCredentialApi, UserMfaStatus, UserStatus, + ReplicationMrf, RuntimeCapabilitiesSnapshot, RuntimeCapabilityStatus, ScannerCycleResetResult, + ScannerRecoveryApi, ScannerStatus, SecretValue, ServiceAccount, ServiceAccountCreateResponse, + ServiceActionResult, SiteRemoveSpec, SiteReplicationInfo, SiteReplicationPeer, + SiteReplicationRepairApi, SiteReplicationRepairCapabilityContract, + SiteReplicationRepairOperationStatus, SiteReplicationRepairPreflight, + SiteReplicationRepairRequest, SiteReplicationResyncOperation, SiteReplicationResyncStatus, + SiteStatusOptions, StorageInfo, UpdateGroupMembersRequest, UpdateServiceAccountRequest, User, + UserCredentialApi, UserMfaStatus, UserStatus, }; use rc_core::{Alias, Error, Result}; use reqwest::header::{CACHE_CONTROL, CONTENT_TYPE, HOST, HeaderMap, HeaderName, HeaderValue}; @@ -4744,6 +4745,32 @@ fn config_applied(headers: &HeaderMap) -> bool { .is_some_and(|value| value.eq_ignore_ascii_case("true")) } +#[async_trait] +impl ScannerRecoveryApi for AdminClient { + async fn reset_scanner_cycle(&self) -> Result { + let result: ScannerCycleResetResult = self + .request_bounded_json( + Method::POST, + "/scanner/cycle-state/reset", + None, + Some(br#"{"mode":"full-rescan"}"#), + BoundedJsonResponse { + max_bytes: 64 * 1024, + name: "Scanner cycle reset response", + error_mapper: Self::map_error, + }, + ) + .await + .map_err(|error| observability_route_error(error, "Scanner cycle reset"))?; + if result.status != "reset" || result.mode != "full-rescan" { + return Err(Error::General( + "Server did not acknowledge a full-rescan cycle reset".to_string(), + )); + } + Ok(result) + } +} + #[async_trait] impl ObservabilityApi for AdminClient { async fn scanner_status(&self) -> Result { diff --git a/docs/reference/rc/admin.md b/docs/reference/rc/admin.md index 2ef35887..85304272 100644 --- a/docs/reference/rc/admin.md +++ b/docs/reference/rc/admin.md @@ -20,6 +20,7 @@ rc admin account mfa recovery-codes [--code CODE|--code-from-env NAME] [ rc admin diagnostics rc admin info [OPTIONS] rc admin scanner status +rc admin scanner reset cycle --yes rc admin heal mrf [OPTIONS] rc admin integrity [OPTIONS] rc admin idp openid list @@ -454,6 +455,12 @@ Admin operations use the configured alias to create a RustFS admin client. The c The diagnostic commands require capability discovery to classify the corresponding route as available. Authentication failures, unsupported routes, malformed JSON, and transport failures retain distinct exit codes. +## Scanner Cycle Recovery + +Run `rc admin scanner status ` to inspect cycle recovery on the responding node. When RustFS supplies them, the text output shows the recovery state and reason separately from scan freshness. JSON output retains the complete `data.cycle_recovery` object. + +If the scanner is blocked by invalid cycle state, run `rc admin scanner reset cycle --yes` using an identity with `admin:ConfigUpdate`. This sends a full-rescan reset request. Success confirms that RustFS handled the cycle-state reset. Use `rc admin scanner status ` to follow subsequent scan progress. + ## Observability Workflow The read-only observability commands target RustFS Admin API v3 routes introduced with the beta.10 diagnostics surface. diff --git a/schemas/output_v3.json b/schemas/output_v3.json index 9d9e4224..1f095149 100644 --- a/schemas/output_v3.json +++ b/schemas/output_v3.json @@ -39,6 +39,7 @@ "replication_operations", "bucket_operations", "scanner_status", + "scanner_cycle_reset", "storage_info", "kms", "health_snapshot", @@ -2387,6 +2388,24 @@ } ] }, + { + "allOf": [ + { "$ref": "#/definitions/successEnvelope" }, + { + "properties": { + "type": { "const": "scanner_cycle_reset" }, + "data": { + "type": "object", + "required": ["status", "mode"], + "properties": { + "status": { "const": "reset" }, + "mode": { "const": "full-rescan" } + } + } + } + } + ] + }, { "allOf": [ { "$ref": "#/definitions/successEnvelope" },