Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/cli/src/commands/admin/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ pub enum AdminCommands {
#[command(subcommand)]
Kms(kms::KmsCommands),

/// Inspect scanner health and freshness
/// Inspect scanner health and manage cycle recovery
#[command(subcommand)]
Scanner(scanner::ScannerCommands),

Expand Down Expand Up @@ -263,7 +263,7 @@ fn observability_error_suggestion(error: &Error) -> Option<&'static str> {
Error::InvalidPath(_) | Error::Config(_) => Some("Review the command arguments and retry."),
Error::Network(_) => Some("Verify the endpoint and network connectivity, then retry."),
Error::Auth(_) => Some("Verify credentials and required admin permissions, then retry."),
Error::UnsupportedFeature(_) => Some("Upgrade RustFS to beta.10 or later."),
Error::UnsupportedFeature(_) => Some("Use a RustFS release that supports this operation."),
Error::Interrupted(_) => Some("Retry with --yes if deletion is still intended."),
_ => None,
}
Expand Down
76 changes: 72 additions & 4 deletions crates/cli/src/commands/admin/scanner.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Scanner health and freshness diagnostics.
//! Scanner health diagnostics and cycle recovery.

use clap::{Args, Subcommand};
use rc_core::admin::{ObservabilityApi, ScannerStatus};
use rc_core::admin::{ObservabilityApi, ScannerRecoveryApi, ScannerStatus};
use serde::Serialize;

use super::{emit_observability_error, get_admin_client};
Expand All @@ -12,6 +12,25 @@ use crate::output::Formatter;
pub enum ScannerCommands {
/// Display scanner health, freshness, and current cycle details
Status(ScannerStatusArgs),
/// Reset scanner recovery state
#[command(subcommand)]
Reset(ScannerResetCommands),
}

#[derive(Subcommand, Debug)]
pub enum ScannerResetCommands {
/// Reset cycle recovery state for a full rescan
Cycle(ScannerCycleResetArgs),
}

#[derive(Args, Debug)]
pub struct ScannerCycleResetArgs {
/// Alias name of the server
pub alias: String,

/// Confirm the cycle reset (requires admin:ConfigUpdate)
#[arg(long, required = true)]
pub yes: bool,
}

#[derive(Args, Debug)]
Expand All @@ -21,12 +40,12 @@ pub struct ScannerStatusArgs {
}

#[derive(Debug, Serialize)]
struct ScannerSuccessOutput<'a> {
struct ScannerSuccessOutput<T> {
schema_version: u8,
#[serde(rename = "type")]
output_type: &'static str,
status: &'static str,
data: ScannerOutput<'a>,
data: T,
}

#[derive(Debug, Serialize)]
Expand All @@ -39,6 +58,9 @@ struct ScannerOutput<'a> {
pub async fn execute(command: ScannerCommands, formatter: &Formatter) -> ExitCode {
match command {
ScannerCommands::Status(args) => execute_status(args, formatter).await,
ScannerCommands::Reset(ScannerResetCommands::Cycle(args)) => {
execute_cycle_reset(args, formatter).await
}
}
}

Expand Down Expand Up @@ -75,6 +97,35 @@ async fn execute_status(args: ScannerStatusArgs, formatter: &Formatter) -> ExitC
}
}

async fn execute_cycle_reset(args: ScannerCycleResetArgs, formatter: &Formatter) -> ExitCode {
let client = match get_admin_client(&args.alias, formatter) {
Ok(client) => client,
Err(code) => return code,
};
match client.reset_scanner_cycle().await {
Ok(result) => {
if formatter.is_json() {
formatter.json(&ScannerSuccessOutput {
schema_version: 3,
output_type: "scanner_cycle_reset",
status: "success",
data: result,
});
} else {
formatter.println("Scanner cycle state reset (full-rescan).");
}
ExitCode::Success
}
Err(error) => emit_observability_error(
"scanner_cycle_reset",
"admin.scanner-cycle-reset",
"Failed to reset scanner cycle state",
&error,
formatter,
),
}
}

fn print_status(status: &ScannerStatus, formatter: &Formatter) {
formatter.println(&formatter.style_name("Scanner Status"));
formatter.println("");
Expand Down Expand Up @@ -102,6 +153,23 @@ fn print_status(status: &ScannerStatus, formatter: &Formatter) {
formatter.sanitize_text(reason)
));
}
if let Some(recovery) = status.extra.get("cycle_recovery") {
if let Some(state) = recovery["state"].as_str() {
formatter.println(&format!(
"Recovery state: {}",
formatter.sanitize_text(state)
));
}
if let Some(reason) = recovery["reason"]
.as_str()
.filter(|reason| !reason.is_empty())
{
formatter.println(&format!(
"Recovery reason: {}",
formatter.sanitize_text(reason)
));
}
}
}

fn value_or_unknown(value: &str) -> &str {
Expand Down
130 changes: 130 additions & 0 deletions crates/cli/tests/admin_scanner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -186,3 +186,133 @@ fn scanner_status_rejects_malformed_payload() {
assert_eq!(output.status.code(), Some(1));
handle.join().expect("admin test server finished");
}

fn run_scanner(args: &[&str], endpoint: &str) -> std::process::Output {
let config_dir = tempfile::tempdir().expect("create config dir");
Command::new(rc_binary())
.args(["admin", "scanner"])
.args(args)
.env("RC_CONFIG_DIR", config_dir.path())
.env("RC_HOST_myalias", rc_host_alias(endpoint))
.output()
.expect("run rc scanner command")
}

#[test]
fn scanner_status_displays_recovery_separately_from_freshness() {
let mut body: Value = serde_json::from_str(HEALTHY_SCANNER).expect("scanner fixture");
body["freshness"]["reason"] = "last cycle is too old".into();
body["cycle_recovery"] = serde_json::json!({
"state": "blocked",
"reason": "invalid cycle\nmarker",
"retry_count": 3,
});
for json in [false, true] {
let (endpoint, _receiver, handle) =
start_admin_response_test_server("200 OK", "application/json", body.to_string());
let mut args = vec!["status", "myalias"];
if json {
args.push("--json");
}
let output = run_scanner(&args, &endpoint);
assert!(output.status.success());
if json {
let value: Value = serde_json::from_slice(&output.stdout).expect("scanner status JSON");
assert_eq!(value["data"]["cycle_recovery"], body["cycle_recovery"]);
assert_valid_v3(&value);
} else {
let stdout = String::from_utf8(output.stdout).expect("status text");
assert!(stdout.contains("Recovery state: blocked"), "{stdout}");
assert!(
stdout.contains(r"Recovery reason: invalid cycle\nmarker"),
"{stdout}"
);
assert!(
stdout.contains("Reason: last cycle is too old"),
"{stdout}"
);
}
handle.join().expect("server finished");
}
}

#[test]
fn scanner_cycle_reset_requires_yes_before_contacting_server() {
let output = run_scanner(&["reset", "cycle", "myalias"], "http://127.0.0.1:1");
assert_eq!(output.status.code(), Some(2));
assert!(String::from_utf8_lossy(&output.stderr).contains("--yes"));
}

#[test]
fn scanner_cycle_reset_posts_full_rescan_and_reports_acknowledgement() {
let ack = r#"{"status":"reset","mode":"full-rescan"}"#;
for json in [false, true] {
let (endpoint, receiver, handle) =
start_admin_response_test_server("200 OK", "application/json", ack.to_string());
let mut args = vec!["reset", "cycle", "myalias", "--yes"];
if json {
args.push("--json");
}
let output = run_scanner(&args, &endpoint);
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
if json {
let value: Value = serde_json::from_slice(&output.stdout).expect("cycle reset JSON");
assert_eq!(value["type"], "scanner_cycle_reset");
assert_eq!(
value["data"],
serde_json::from_str::<Value>(ack).expect("ack")
);
assert_valid_v3(&value);
} else {
assert!(String::from_utf8_lossy(&output.stdout).contains("Scanner cycle state reset"));
}
let request = receiver
.recv_timeout(Duration::from_secs(5))
.expect("reset request");
assert_eq!(request.method, "POST");
assert_eq!(request.target, "/rustfs/admin/v3/scanner/cycle-state/reset");
assert!(
request
.headers
.to_ascii_lowercase()
.contains("authorization: aws4-hmac-sha256")
);
assert!(
request
.headers
.to_ascii_lowercase()
.contains("content-type: application/json")
);
assert_eq!(
serde_json::from_slice::<Value>(&request.body).expect("request JSON"),
serde_json::json!({"mode": "full-rescan"})
);
handle.join().expect("server finished");
}
}

#[test]
fn scanner_cycle_reset_rejects_denied_unsupported_and_invalid_responses() {
for (status, body, exit_code) in [
("403 Forbidden", "Access denied", 4),
("404 Not Found", "Route not found", 7),
("200 OK", "{}", 1),
("200 OK", r#"{"status":"pending","mode":"full-rescan"}"#, 1),
("200 OK", r#"{"status":"reset","mode":"full-rebuild"}"#, 1),
] {
let (endpoint, _receiver, handle) =
start_admin_response_test_server(status, "application/json", body.to_string());
let output = run_scanner(&["--json", "reset", "cycle", "myalias", "--yes"], &endpoint);
assert_eq!(output.status.code(), Some(exit_code), "{status} {body}");
assert!(output.stdout.is_empty());
let value: Value = serde_json::from_slice(&output.stderr).expect("reset error JSON");
assert_eq!(value["type"], "scanner_cycle_reset");
assert_eq!(value["status"], "error");
assert_valid_v3(&value);
handle.join().expect("server finished");
}
}
12 changes: 11 additions & 1 deletion crates/cli/tests/help_contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -743,7 +743,17 @@ fn nested_subcommand_help_contract() {
HelpCase {
args: &["admin", "scanner"],
usage: "Usage: rc admin scanner [OPTIONS] <COMMAND>",
expected_tokens: &["status"],
expected_tokens: &["status", "reset"],
},
HelpCase {
args: &["admin", "scanner", "reset"],
usage: "Usage: rc admin scanner reset [OPTIONS] <COMMAND>",
expected_tokens: &["cycle"],
},
HelpCase {
args: &["admin", "scanner", "reset", "cycle"],
usage: "Usage: rc admin scanner reset cycle [OPTIONS] --yes <ALIAS>",
expected_tokens: &["--yes", "full rescan"],
},
HelpCase {
args: &["admin", "scanner", "status"],
Expand Down
6 changes: 3 additions & 3 deletions crates/core/src/admin/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,9 @@ pub use kms_diagnostic::{
pub use observability::{
MAX_METRICS_LINE_BYTES, MAX_METRICS_RESPONSE_BYTES, MAX_METRICS_SAMPLES, MetricGroup,
MetricGroups, MetricsBatch, MetricsQuery, MetricsScope, ObservabilityApi, RealtimeMetrics,
ScannerCycleSchedule, ScannerFreshness, ScannerHealth, ScannerMetrics, ScannerRuntimeConfig,
ScannerRuntimeConfigValue, ScannerStatus, StorageBackend, StorageBackendKind, StorageDisk,
StorageDiskMetrics, StorageInfo,
ScannerCycleResetResult, ScannerCycleSchedule, ScannerFreshness, ScannerHealth, ScannerMetrics,
ScannerRecoveryApi, ScannerRuntimeConfig, ScannerRuntimeConfigValue, ScannerStatus,
StorageBackend, StorageBackendKind, StorageDisk, StorageDiskMetrics, StorageInfo,
};
pub use oidc::{
MAX_OIDC_RESPONSE_BYTES, OidcMutationApi, OidcMutationRequest, OidcMutationResult,
Expand Down
13 changes: 13 additions & 0 deletions crates/core/src/admin/observability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,19 @@ impl StorageInfo {
}
}

/// Acknowledgement of a synchronous scanner cycle-state reset.
#[derive(Debug, Serialize, Deserialize)]
pub struct ScannerCycleResetResult {
pub status: String,
pub mode: String,
}

/// Operator-initiated recovery of scanner cycle state.
#[async_trait]
pub trait ScannerRecoveryApi: Send + Sync {
async fn reset_scanner_cycle(&self) -> Result<ScannerCycleResetResult>;
}

/// Read-only scanner, storage, and metrics Admin API boundary.
#[async_trait]
pub trait ObservabilityApi: Send + Sync {
Expand Down
41 changes: 34 additions & 7 deletions crates/s3/src/admin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,14 @@ use rc_core::admin::{
PolicyEntitiesResult, PolicyEntity, PolicyInfo, PoolStatus, PoolTarget, RealtimeMetrics,
RebalanceStartResult, RebalanceStatus, RecoveryCodes, ReplicateEditStatus, ReplicationDiff,
ReplicationDiffApi, ReplicationInspectionApi, ReplicationMetricScope, ReplicationMetrics,
ReplicationMrf, RuntimeCapabilitiesSnapshot, RuntimeCapabilityStatus, ScannerStatus,
SecretValue, ServiceAccount, ServiceAccountCreateResponse, ServiceActionResult, SiteRemoveSpec,
SiteReplicationInfo, SiteReplicationPeer, SiteReplicationRepairApi,
SiteReplicationRepairCapabilityContract, SiteReplicationRepairOperationStatus,
SiteReplicationRepairPreflight, SiteReplicationRepairRequest, SiteReplicationResyncOperation,
SiteReplicationResyncStatus, SiteStatusOptions, StorageInfo, UpdateGroupMembersRequest,
UpdateServiceAccountRequest, User, UserCredentialApi, UserMfaStatus, UserStatus,
ReplicationMrf, RuntimeCapabilitiesSnapshot, RuntimeCapabilityStatus, ScannerCycleResetResult,
ScannerRecoveryApi, ScannerStatus, SecretValue, ServiceAccount, ServiceAccountCreateResponse,
ServiceActionResult, SiteRemoveSpec, SiteReplicationInfo, SiteReplicationPeer,
SiteReplicationRepairApi, SiteReplicationRepairCapabilityContract,
SiteReplicationRepairOperationStatus, SiteReplicationRepairPreflight,
SiteReplicationRepairRequest, SiteReplicationResyncOperation, SiteReplicationResyncStatus,
SiteStatusOptions, StorageInfo, UpdateGroupMembersRequest, UpdateServiceAccountRequest, User,
UserCredentialApi, UserMfaStatus, UserStatus,
};
use rc_core::{Alias, Error, Result};
use reqwest::header::{CACHE_CONTROL, CONTENT_TYPE, HOST, HeaderMap, HeaderName, HeaderValue};
Expand Down Expand Up @@ -4744,6 +4745,32 @@ fn config_applied(headers: &HeaderMap) -> bool {
.is_some_and(|value| value.eq_ignore_ascii_case("true"))
}

#[async_trait]
impl ScannerRecoveryApi for AdminClient {
async fn reset_scanner_cycle(&self) -> Result<ScannerCycleResetResult> {
let result: ScannerCycleResetResult = self
.request_bounded_json(
Method::POST,
"/scanner/cycle-state/reset",
None,
Some(br#"{"mode":"full-rescan"}"#),
BoundedJsonResponse {
max_bytes: 64 * 1024,
name: "Scanner cycle reset response",
error_mapper: Self::map_error,
},
)
.await
.map_err(|error| observability_route_error(error, "Scanner cycle reset"))?;
if result.status != "reset" || result.mode != "full-rescan" {
return Err(Error::General(
"Server did not acknowledge a full-rescan cycle reset".to_string(),
));
}
Ok(result)
}
}

#[async_trait]
impl ObservabilityApi for AdminClient {
async fn scanner_status(&self) -> Result<ScannerStatus> {
Expand Down
Loading
Loading