diff --git a/crates/core/src/data_transport.rs b/crates/core/src/data_transport.rs index 0e56226f..5c6b2b23 100644 --- a/crates/core/src/data_transport.rs +++ b/crates/core/src/data_transport.rs @@ -94,6 +94,37 @@ mod tests { } } + #[test] + fn transfer_error_display_includes_optional_source_without_changing_outcome() { + for state in [ + DataTransferState::NotAccepted, + DataTransferState::ConfirmedFailure, + DataTransferState::OutcomeUnknown, + ] { + let mut error = crate::Error::DataTransfer { + state, + message: "S3 request failed".to_string(), + source: None, + }; + assert_eq!( + error.to_string(), + format!("Object data transfer {state}: S3 request failed") + ); + if let crate::Error::DataTransfer { source, .. } = &mut error { + *source = Some(Box::new(crate::Error::Auth("access denied".to_string()))); + } + assert_eq!( + error.to_string(), + format!( + "Object data transfer {state}: S3 request failed: Authentication failed: access denied" + ) + ); + assert_eq!(error.exit_code(), 4); + assert!(std::error::Error::source(&error).is_some()); + assert!(!crate::is_retryable_error(&error)); + } + } + #[test] fn authoritative_error_retains_exit_code_and_cleanup_without_retry() { let error = crate::Error::DataTransfer { diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index d0b9e4c3..afb52dc2 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -104,7 +104,10 @@ pub enum Error { Network(String), /// An external data operation with explicit acceptance and outcome state. - #[error("Object data transfer {state}: {message}")] + #[error( + "Object data transfer {state}: {message}{cause}", + cause = .source.as_ref().map(|source| format!(": {source}")).unwrap_or_default() + )] DataTransfer { /// Acceptance and business outcome classification. state: crate::DataTransferState, diff --git a/crates/s3/src/client.rs b/crates/s3/src/client.rs index 53bfbeda..1c346759 100644 --- a/crates/s3/src/client.rs +++ b/crates/s3/src/client.rs @@ -4312,52 +4312,60 @@ impl S3Client { &self, path: &RemotePath, file_path: &std::path::Path, - file_size: u64, options: PathUploadOptions<'_>, + on_progress: impl Fn(u64) + Send, ) -> Result { let mut file = tokio::fs::File::open(file_path) .await .map_err(|e| Error::General(format!("open file '{}': {e}", file_path.display())))?; - let requested_checksum = if self.selected_data_transport()?.is_some() { - requested_file_sha256_checksum( + if self.selected_data_transport()?.is_some() { + // Declare the length of the opened source, which may have changed since + // the caller selected a single-part upload. + let content_length = file + .metadata() + .await + .map_err(|e| { + Error::General(format!("read metadata for '{}': {e}", file_path.display())) + })? + .len(); + let requested_checksum = requested_file_sha256_checksum( &mut file, file_path, - file_size, + content_length, options.write.checksum.as_ref(), ) - .await? - } else { - None - }; - let condition = match options.precondition { - ObjectWritePrecondition::None => WriteCondition::None, - ObjectWritePrecondition::IfAbsent => WriteCondition::IfAbsent, - ObjectWritePrecondition::IfMatch(etag) => WriteCondition::IfMatch(etag), - }; - if let Some(result) = self - .try_external_upload_reader( - UploadRequest { - path, - operation: UploadOperation::PutObject { condition }, - options: options.write, - content_length: file_size, - checksum_sha256: requested_checksum.as_deref(), - }, - &mut file, - ) - .await? - { - if let Some(checksum) = &requested_checksum { - self.verify_persisted_sha256( - path, - result.object.version_id.clone(), - checksum, - destination_sse_customer_key(options.write.encryption.as_ref()), + .await?; + let condition = match options.precondition { + ObjectWritePrecondition::None => WriteCondition::None, + ObjectWritePrecondition::IfAbsent => WriteCondition::IfAbsent, + ObjectWritePrecondition::IfMatch(etag) => WriteCondition::IfMatch(etag), + }; + if let Some(result) = self + .try_external_upload_reader( + UploadRequest { + path, + operation: UploadOperation::PutObject { condition }, + options: options.write, + content_length, + checksum_sha256: requested_checksum.as_deref(), + }, + &mut file, ) - .await - .map_err(data_transport::committed_verification_failure)?; + .await? + { + if let Some(checksum) = &requested_checksum { + self.verify_persisted_sha256( + path, + result.object.version_id.clone(), + checksum, + destination_sse_customer_key(options.write.encryption.as_ref()), + ) + .await + .map_err(data_transport::committed_verification_failure)?; + } + on_progress(content_length); + return Ok(result.object); } - return Ok(result.object); } // Auto fallback is only possible before the provider polls the file. // Retain the HTTP SDK's repeatable, immutable request body semantics. @@ -4365,6 +4373,7 @@ impl S3Client { file.read_to_end(&mut data) .await .map_err(|e| Error::General(format!("read file '{}': {e}", file_path.display())))?; + let file_size = data.len() as u64; let requested_checksum = requested_sha256_checksum(&data, options.write.checksum.as_ref())?; let body = aws_sdk_s3::primitives::ByteStream::from(data); let storage_class = rustfs_storage_class(options.write.storage_class.as_deref())?; @@ -4431,6 +4440,7 @@ impl S3Client { ) .await?; } + on_progress(file_size); Ok(info) } @@ -4954,11 +4964,8 @@ impl S3Client { self.put_object_multipart_from_path(path, file_path, file_size, options, on_progress) .await } else { - let result = self - .put_object_single_part_from_path(path, file_path, file_size, options) - .await?; - on_progress(file_size); - Ok(result) + self.put_object_single_part_from_path(path, file_path, options, on_progress) + .await } } @@ -14557,9 +14564,9 @@ mod tests { assert_eq!(third, 0); } - #[derive(Clone, Copy)] enum ExternalBehavior { Reject, + RewriteThenReject(std::path::PathBuf, &'static [u8]), PartialReject, Success, Unknown, @@ -14568,6 +14575,7 @@ mod tests { struct TestExternalTransport { behavior: ExternalBehavior, operations: std::sync::Mutex>, + uploaded_payloads: std::sync::Mutex>>, } #[async_trait] @@ -14617,7 +14625,13 @@ mod tests { .lock() .expect("record upload") .push(format!("{:?}", request.operation)); - if matches!(self.behavior, ExternalBehavior::Reject) { + if let ExternalBehavior::RewriteThenReject(path, bytes) = &self.behavior { + std::fs::write(path, bytes).expect("change source before HTTP fallback"); + } + if matches!( + self.behavior, + ExternalBehavior::Reject | ExternalBehavior::RewriteThenReject(..) + ) { return Err(data_transport::DataTransportError { state: rc_core::DataTransferState::NotAccepted, message: "provider unavailable".to_string(), @@ -14642,6 +14656,10 @@ mod tests { .await .expect("read external source"); assert_eq!(payload.len() as u64, request.content_length); + self.uploaded_payloads + .lock() + .expect("record upload payload") + .push(payload); if matches!(self.behavior, ExternalBehavior::Unknown) { return Err(data_transport::DataTransportError { state: rc_core::DataTransferState::OutcomeUnknown, @@ -14667,9 +14685,101 @@ mod tests { Arc::new(TestExternalTransport { behavior, operations: std::sync::Mutex::new(Vec::new()), + uploaded_payloads: std::sync::Mutex::new(Vec::new()), }) } + #[tokio::test] + async fn external_single_part_upload_uses_current_file_contents() { + for payload in [b"longer payload".as_slice(), b"x", b""] { + let (client, replay) = test_s3_client_with_response_sequence(Vec::new()); + let provider = external_transport(ExternalBehavior::Success); + let client = client + .with_object_data_transport(rc_core::DataTransportMode::Rdma, provider.clone()); + let source = tempfile::NamedTempFile::new().expect("file source"); + std::fs::write(source.path(), b"original").expect("initial source"); + let initial_size = std::fs::metadata(source.path()) + .expect("initial metadata") + .len(); + std::fs::write(source.path(), payload).expect("replace source"); + assert_ne!(initial_size, payload.len() as u64); + let progress = std::sync::Mutex::new(Vec::new()); + let info = client + .put_object_single_part_from_path( + &RemotePath::new("test", "bucket", "key"), + source.path(), + PathUploadOptions { + write: &ObjectWriteOptions::default(), + precondition: ObjectWritePrecondition::None, + }, + |bytes| progress.lock().expect("progress record").push(bytes), + ) + .await + .expect("upload current file contents"); + assert_eq!(info.size_bytes, Some(payload.len() as i64)); + assert_eq!( + *progress.lock().expect("progress record"), + vec![payload.len() as u64] + ); + assert_eq!( + provider + .uploaded_payloads + .lock() + .expect("uploaded payloads") + .as_slice(), + &[payload.to_vec()] + ); + assert_eq!(replay.actual_requests().count(), 0); + } + } + + #[tokio::test] + async fn external_single_part_auto_fallback_uses_current_file_contents() { + for payload in [b"longer payload".as_slice(), b"x", b""] { + let response = http::Response::builder() + .status(200) + .header("etag", "\"http-etag\"") + .body(SdkBody::empty()) + .expect("HTTP PUT response"); + let (client, receiver) = test_s3_client(Some(response)); + let source = tempfile::NamedTempFile::new().expect("file source"); + std::fs::write(source.path(), b"original").expect("initial source"); + let provider = external_transport(ExternalBehavior::RewriteThenReject( + source.path().to_owned(), + payload, + )); + let client = client + .with_object_data_transport(rc_core::DataTransportMode::Auto, provider.clone()); + let progress = std::sync::Mutex::new(Vec::new()); + let info = client + .put_object_from_path_with_options( + &RemotePath::new("test", "bucket", "key"), + source.path(), + &ObjectWriteOptions::default(), + |bytes| progress.lock().expect("progress record").push(bytes), + ) + .await + .expect("HTTP fallback uploads current file contents"); + assert_eq!(info.size_bytes, Some(payload.len() as i64)); + assert_eq!( + *progress.lock().expect("progress record"), + vec![payload.len() as u64] + ); + assert_eq!(receiver.expect_request().body().bytes(), Some(payload)); + assert!( + provider + .uploaded_payloads + .lock() + .expect("uploaded payloads") + .is_empty() + ); + assert_eq!( + provider.operations.lock().expect("upload attempts").len(), + 1 + ); + } + } + #[tokio::test] async fn external_auto_rejection_preserves_http_get_selection() { let response = http::Response::builder() @@ -14857,11 +14967,11 @@ mod tests { .put_object_single_part_from_path( &RemotePath::new("test", "bucket", "key"), source.path(), - 12, PathUploadOptions { write: &ObjectWriteOptions::default(), precondition: ObjectWritePrecondition::None, }, + |_| {}, ) .await .expect("unconsumed file can use HTTP"); @@ -14883,11 +14993,11 @@ mod tests { .put_object_single_part_from_path( &RemotePath::new("test", "bucket", "key"), source.path(), - 12, PathUploadOptions { write: &ObjectWriteOptions::default(), precondition: ObjectWritePrecondition::None, }, + |_| panic!("a failed upload cannot report completion"), ) .await .expect_err("a consumed source cannot use HTTP fallback"); @@ -14967,16 +15077,26 @@ mod tests { #[test] fn external_error_source_is_redacted_without_losing_business_class() { let (client, _) = test_s3_client(None); + let key = test_sse_customer_key(); + let headers = SseCustomerHeaders::new(&key); let error = Error::DataTransfer { state: rc_core::DataTransferState::ConfirmedFailure, message: "secret-key in response".to_string(), - source: Some(Box::new(Error::Auth("access-key secret-key".to_string()))), + source: Some(Box::new(Error::Auth(format!( + "access denied: access-key secret-key {} {} 0123456789abcdef0123456789abcdef", + headers.key.as_str(), + headers.key_md5.as_str(), + )))), }; - let error = client.redact_sensitive_error(error); + let error = client.redact_sse_customer_error(error, Some(&key)); assert_eq!(error.exit_code(), 4); - let diagnostic = format!("{error:?}"); + assert!(error.to_string().contains("access denied")); + let diagnostic = format!("{error:?} {error}"); assert!(!diagnostic.contains("secret-key")); assert!(!diagnostic.contains("access-key")); + assert!(!diagnostic.contains(headers.key.as_str())); + assert!(!diagnostic.contains(headers.key_md5.as_str())); + assert!(!diagnostic.contains("0123456789abcdef0123456789abcdef")); assert!(std::error::Error::source(&error).is_some()); assert!(!rc_core::is_retryable_error(&error)); } @@ -15040,6 +15160,7 @@ mod tests { )); assert!(!rc_core::is_retryable_error(&error)); assert_eq!(receiver.expect_request().method(), "HEAD"); + assert!(error.to_string().contains("ServiceUnavailable")); } #[tokio::test] async fn external_multipart_completion_failure_does_not_replay_the_file() { @@ -15084,6 +15205,7 @@ mod tests { )); assert!(!rc_core::is_retryable_error(&error)); assert!(error.to_string().contains("upload-id")); + assert!(error.to_string().contains("ServiceUnavailable")); assert_eq!(replay.actual_requests().count(), 3); } @@ -15131,6 +15253,7 @@ mod tests { } )); assert!(!rc_core::is_retryable_error(&error)); + assert!(error.to_string().contains("ServiceUnavailable")); assert_eq!(replay.actual_requests().count(), 3); } #[tokio::test]