From 069975814a1f24afb2e16be0daa724b9bfb4090e Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:55:30 +0100 Subject: [PATCH 1/5] fix(download): honor x-ms-range and never serve a ranged read whole The Azure SDK, and with it BuildKit's type=gha cache, resumes a dropped download with only `x-ms-range: bytes=N-` and takes any 200 body as the bytes from offset N. The server only read `Range`, so a resume got the whole blob from byte 0 and BuildKit spliced it in at N. `x-ms-range` now wins over `Range`, like Azure Get Blob. A ranged read of an unmerged entry is served as a 206 from its Parts: Parts wholly before the range cost one ranged read answered 416, and the stream stops at the end of the range. Such a read no longer starts a Merge; the next whole read does. Closes #315 Co-Authored-By: Claude Opus 5.5 --- src/routes/transfer.rs | 17 ++-- src/routes/transfer/tests.rs | 25 ++++++ src/storage.rs | 119 ++++++++++++++++++++-------- src/storage/adapter.rs | 4 +- src/storage/filesystem.rs | 6 +- src/storage/filesystem/tests.rs | 2 +- src/storage/object.rs | 6 +- src/tests/downloads.rs | 133 ++++++++++++++++++++++---------- 8 files changed, 227 insertions(+), 85 deletions(-) diff --git a/src/routes/transfer.rs b/src/routes/transfer.rs index 1d9aa62..0453522 100644 --- a/src/routes/transfer.rs +++ b/src/routes/transfer.rs @@ -23,6 +23,8 @@ use crate::storage::{ /// Clients without a request id intermittently fail with EOF (tonistiigi/go-actions-cache). const REQUEST_ID: HeaderName = HeaderName::from_static("x-ms-request-id"); +/// Azure's Range header; Get Blob prefers it over `Range`. +const X_MS_RANGE: HeaderName = HeaderName::from_static("x-ms-range"); /// Lenient like Node's `Buffer.from(value, 'base64')`. const BASE64: GeneralPurpose = GeneralPurpose::new( @@ -128,9 +130,15 @@ fn upload_not_found(upload_id: i64) -> HttpError { } /// A single range; multi-range requests and invalid headers, which includes inverted ranges, are -/// served whole. +/// served whole. `x-ms-range` wins over `Range`, like Azure Get Blob: the Azure SDK, and with it +/// `BuildKit`'s `type=gha` cache, sends only `x-ms-range` when it resumes a download. fn parse_range(headers: &HeaderMap) -> Option { - let value = headers.get(RANGE)?.to_str().ok()?.trim(); + let value = headers + .get(X_MS_RANGE) + .or_else(|| headers.get(RANGE))? + .to_str() + .ok()? + .trim(); // The unit is case-insensitive (RFC 9110 ยง14.1); the parser only knows `bytes=`. let ranges = value .get(..6)? @@ -153,7 +161,6 @@ pub(super) async fn download( headers: HeaderMap, ) -> Result { let range = parse_range(&headers); - // Unmerged entries ignore Range, so clients must key off the status, not this header. let accept_ranges = (ACCEPT_RANGES, HeaderValue::from_static("bytes")); let download = match state.storage.download(&location_id, range).await { Ok(Some(download)) => download, @@ -165,9 +172,7 @@ pub(super) async fn download( } Err(DownloadError::RangeNotSatisfiable { size }) => { let mut response = (StatusCode::RANGE_NOT_SATISFIABLE, [accept_ranges]).into_response(); - if let Some(content_range) = - size.and_then(|size| HeaderValue::from_str(&format!("bytes */{size}")).ok()) - { + if let Ok(content_range) = HeaderValue::from_str(&format!("bytes */{size}")) { response.headers_mut().insert(CONTENT_RANGE, content_range); } return Ok(response); diff --git a/src/routes/transfer/tests.rs b/src/routes/transfer/tests.rs index 4a6c779..ae306a2 100644 --- a/src/routes/transfer/tests.rs +++ b/src/routes/transfer/tests.rs @@ -52,3 +52,28 @@ fn parses_single_ranges() { assert_eq!(range("bytes=9-5"), None, "inverted"); assert_eq!(range("items=0-5"), None, "unknown unit"); } + +#[test] +fn x_ms_range_wins_over_range() { + let from = |pairs: &[(&'static str, &'static str)]| { + let mut headers = HeaderMap::new(); + for &(name, value) in pairs { + headers.insert(name, HeaderValue::from_static(value)); + } + parse_range(&headers) + }; + let from_100 = Some(RangeRequest { + start: StartPosition::Index(100), + end: EndPosition::LastByte, + }); + assert_eq!( + from(&[("x-ms-range", "bytes=100-")]), + from_100, + "x-ms-range alone, as the Azure SDK sends it" + ); + assert_eq!( + from(&[("x-ms-range", "bytes=100-"), ("range", "bytes=0-9")]), + from_100, + "x-ms-range wins, like Azure Get Blob" + ); +} diff --git a/src/storage.rs b/src/storage.rs index 4f2650c..0d5e2cd 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -15,7 +15,7 @@ use axum::body::Bytes; use base64::Engine as _; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use futures_util::future::BoxFuture; -use futures_util::{FutureExt as _, StreamExt as _, TryStreamExt as _, stream}; +use futures_util::{FutureExt as _, StreamExt as _, TryStreamExt as _, future, stream}; use sea_orm::ActiveValue::Set; use sea_orm::sea_query::{CaseStatement, Expr, ExprTrait as _, Func}; use sea_orm::{ @@ -30,11 +30,11 @@ use tokio_stream::wrappers::ReceiverStream; use tokio_util::task::{AbortOnDropHandle, TaskTracker}; use utoipa::ToSchema; -use self::adapter::ReadError; pub(crate) use self::adapter::{ Adapter, ByteStream, DownloadStream, EndPosition, Extent, RangeRequest, StartPosition, UploadKind, }; +use self::adapter::{ReadError, clamp_range}; use self::leases::{ DIRECT_DOWNLOAD_LEASE_MARGIN, LEASE_DURATION, LEASE_RENEWAL, acquire_merge_lease, create_reader_lease, deadline, release_merge_lease, release_reader_lease, renew_merge_lease, @@ -117,7 +117,7 @@ pub(crate) enum UploadAccess { #[derive(Debug, thiserror::Error)] pub(crate) enum DownloadError { #[error("range not satisfiable")] - RangeNotSatisfiable { size: Option }, + RangeNotSatisfiable { size: u64 }, #[error(transparent)] Failed(#[from] Failure), } @@ -171,8 +171,9 @@ fn entry_identity(identity: &EntryIdentity) -> Condition { .add(cache_entries::COLUMN.repo_id.eq(identity.repo_id.as_str())) } -/// Concatenates a location's Parts as they are read. -fn stream_parts(adapter: Arc, location: &StorageLocation) -> ByteStream { +/// A location's Parts as one stream, starting `skip` bytes in. A Part wholly before that offset +/// costs one ranged read answered 416, never its bytes. +fn stream_parts(adapter: Arc, location: &StorageLocation, skip: u64) -> ByteStream { if location.parts_deleted_at.is_some() { return stream::once(async { Err(io::Error::other( @@ -181,20 +182,58 @@ fn stream_parts(adapter: Arc, location: &StorageLocation) -> ByteStream }) .boxed(); } - let folder = location.folder_name.clone(); - stream::iter(0..location.part_count) - .then(move |index| { - let adapter = Arc::clone(&adapter); - let name = part_object(&folder, index); - async move { - adapter - .download_stream(&name, None) - .await - .map(|download| download.stream) - .map_err(io::Error::other) + let (folder, part_count) = (location.folder_name.clone(), location.part_count); + stream::try_unfold((0, skip), move |(first, mut skip)| { + let adapter = Arc::clone(&adapter); + let folder = folder.clone(); + async move { + for index in first..part_count { + let name = part_object(&folder, index); + let range = (skip > 0).then_some(RangeRequest { + start: StartPosition::Index(skip), + end: EndPosition::LastByte, + }); + match adapter.download_stream(&name, range).await { + // An empty Part is served whole whatever the range. + Ok(DownloadStream { + extent: Extent::Whole { size: 0 }, + .. + }) if skip > 0 => {} + Ok(download) => { + return Ok(Some((download.stream, (index.saturating_add(1), 0)))); + } + Err(ReadError::RangeNotSatisfiable { size, .. }) => { + skip = skip.saturating_sub(size); + } + Err(error) => return Err(io::Error::other(error)), + } } + Ok(None) + } + }) + .try_flatten() + .boxed() +} + +/// The first `len` bytes of a stream. +fn take_bytes(stream: ByteStream, len: u64) -> ByteStream { + stream + .scan(len, |left, chunk| { + let chunk = match chunk { + Ok(_) if *left == 0 => None, + Ok(mut bytes) => { + if let Ok(keep) = usize::try_from(*left) + && keep < bytes.len() + { + bytes.truncate(keep); + } + *left = left.saturating_sub(u64::try_from(bytes.len()).unwrap_or(u64::MAX)); + Some(Ok(bytes)) + } + Err(error) => Some(Err(error)), + }; + future::ready(chunk) }) - .try_flatten() .boxed() } @@ -379,11 +418,30 @@ impl Storage { return self.adapter.download_stream(&object, range).await; } self.ensure_parts_exist(location).await?; - // No range: an unmerged entry is concatenated from its parts as it is read, so there is - // nothing to seek into. Served whole under a 200, which Range allows. + let adapter = Arc::clone(&self.adapter); + let size = location + .size_bytes + .and_then(|size| u64::try_from(size).ok()); + let Some((range, size)) = range.zip(size.filter(|&size| size > 0)) else { + // An unmerged entry is concatenated from its Parts as it is read. Only an empty entry + // ignores a range, like an empty object does. + return Ok(DownloadStream { + stream: stream_parts(adapter, location, 0), + extent: Extent::Unsized, + }); + }; + // Never the whole entry under a 200: the Azure SDK takes any 200 body as the bytes from + // the offset it asked for. + let served = clamp_range(range, size).ok_or_else(|| ReadError::RangeNotSatisfiable { + object: parts_folder(&location.folder_name), + size, + })?; Ok(DownloadStream { - stream: stream_parts(Arc::clone(&self.adapter), location), - extent: Extent::Unsized, + stream: take_bytes(stream_parts(adapter, location, served.start), served.len()), + extent: Extent::Partial { + range: served, + size, + }, }) } @@ -509,7 +567,7 @@ impl Storage { fn merge_parts(&self, location: &StorageLocation) -> BoxFuture<'static, Result<()>> { let adapter = Arc::clone(&self.adapter); let merged = merged_object(&location.folder_name); - let part_stream = stream_parts(Arc::clone(&adapter), location); + let part_stream = stream_parts(Arc::clone(&adapter), location, 0); let kind = UploadKind::Merged { length: location .size_bytes @@ -852,11 +910,9 @@ impl Storage { tracing::warn!(location_id, object, "stale storage location"); Ok(None) } - Err(ReadError::RangeNotSatisfiable(_)) => Err(DownloadError::RangeNotSatisfiable { - size: location - .size_bytes - .and_then(|size| u64::try_from(size).ok()), - }), + Err(ReadError::RangeNotSatisfiable { size, .. }) => { + Err(DownloadError::RangeNotSatisfiable { size }) + } Err(ReadError::Failed(failure)) => Err(failure.into()), }; if let Err(error) = release_reader_lease(&self.db, &lease_id).await { @@ -871,7 +927,8 @@ impl Storage { location: &StorageLocation, range: Option, ) -> Result { - if whole_object(location).is_some() { + // A ranged read serves its slice from the Parts and leaves the Merge to a whole read. + if whole_object(location).is_some() || range.is_some() { return self.download_location(location, range).await; } self.ensure_parts_exist(location).await?; @@ -880,7 +937,7 @@ impl Storage { // the Merge copies them in the background. if let Adapter::Filesystem(_) = self.adapter.as_ref() { self.start_merge_detached(location, self.merge_parts(location)); - return self.download_location(location, range).await; + return self.download_location(location, None).await; } // Object storage is read once and teed into the response and the Merge. let (merger_sender, merger_receiver) = mpsc::channel(1); @@ -900,11 +957,11 @@ impl Storage { .boxed() }; if !self.start_merge(location, write).await? { - return self.download_location(location, range).await; + return self.download_location(location, None).await; } let (response_sender, response_receiver) = mpsc::channel(1); drop(tokio::spawn(pump_parts( - stream_parts(Arc::clone(&self.adapter), location), + stream_parts(Arc::clone(&self.adapter), location, 0), response_sender, merger_sender, ))); diff --git a/src/storage/adapter.rs b/src/storage/adapter.rs index 2f8891a..13d992f 100644 --- a/src/storage/adapter.rs +++ b/src/storage/adapter.rs @@ -94,8 +94,8 @@ impl std::fmt::Debug for DownloadStream { pub(crate) enum ReadError { #[error("object not found in storage: {0}")] NotFound(String), - #[error("range not satisfiable for {0}")] - RangeNotSatisfiable(String), + #[error("range not satisfiable for {object} of {size} bytes")] + RangeNotSatisfiable { object: String, size: u64 }, #[error(transparent)] Failed(#[from] Failure), } diff --git a/src/storage/filesystem.rs b/src/storage/filesystem.rs index 9456f9e..34763c3 100644 --- a/src/storage/filesystem.rs +++ b/src/storage/filesystem.rs @@ -124,8 +124,10 @@ impl FilesystemAdapter { extent: Extent::Whole { size }, }); }; - let served = clamp_range(range, size) - .ok_or_else(|| ReadError::RangeNotSatisfiable(name.to_owned()))?; + let served = clamp_range(range, size).ok_or_else(|| ReadError::RangeNotSatisfiable { + object: name.to_owned(), + size, + })?; file.seek(SeekFrom::Start(served.start)) .await .map_err(Failure::from)?; diff --git a/src/storage/filesystem/tests.rs b/src/storage/filesystem/tests.rs index f2265a4..dc8371f 100644 --- a/src/storage/filesystem/tests.rs +++ b/src/storage/filesystem/tests.rs @@ -62,7 +62,7 @@ async fn uploads_atomically_and_serves_ranges() { assert!( matches!( adapter.download_stream("f/parts/0", Some(past)).await, - Err(ReadError::RangeNotSatisfiable(_)) + Err(ReadError::RangeNotSatisfiable { .. }) ), "range past the end" ); diff --git a/src/storage/object.rs b/src/storage/object.rs index 73cdc0d..98e66c2 100644 --- a/src/storage/object.rs +++ b/src/storage/object.rs @@ -320,8 +320,10 @@ impl ObjectAdapter { extent: Extent::Whole { size: 0 }, }); } - let served = clamp_range(range, size) - .ok_or_else(|| ReadError::RangeNotSatisfiable(name.to_owned()))?; + let served = clamp_range(range, size).ok_or_else(|| ReadError::RangeNotSatisfiable { + object: name.to_owned(), + size, + })?; let options = GetOptions { range: Some(GetRange::Bounded( served.start..served.end.saturating_add(1), diff --git a/src/tests/downloads.rs b/src/tests/downloads.rs index d3af56f..5e4b3a6 100644 --- a/src/tests/downloads.rs +++ b/src/tests/downloads.rs @@ -43,42 +43,45 @@ async fn merged_entry(server: &TestServer) -> (StorageLocation, Vec) { } async fn get(server: &TestServer, id: &str, range: Option<&str>) -> reqwest::Response { + get_with(server, id, "range", range).await +} + +async fn get_with( + server: &TestServer, + id: &str, + header: &str, + range: Option<&str>, +) -> reqwest::Response { let mut request = reqwest::Client::new().get(format!("{}/download/{id}", server.url)); if let Some(range) = range { - request = request.header("range", range); + request = request.header(header, range); } request.send().await.unwrap() } -fn header(response: &reqwest::Response, name: &str) -> Option { +fn response_header(response: &reqwest::Response, name: &str) -> Option { response .headers() .get(name) .map(|value| value.to_str().unwrap().to_owned()) } -#[tokio::test] -async fn serves_ranges_of_merged_entries() { - let server = TestServer::start().await; - let (location, expected) = merged_entry(&server).await; +/// Every range shape, including ones crossing and starting on Part boundaries, answered 206 with +/// exactly the requested bytes, and a range past the end answered 416. +#[expect( + clippy::arithmetic_side_effects, + reason = "offsets into a fixture of a few hundred KiB" +)] +async fn assert_serves_ranges(server: &TestServer, id: &str, expected: &[u8], header: &str) { let total = expected.len(); - - let whole = get(&server, &location.id, None).await; - assert_eq!(whole.status(), 200, "no range"); - assert_eq!( - header(&whole, "accept-ranges").as_deref(), - Some("bytes"), - "ranges advertised" - ); - assert_eq!( - header(&whole, "content-length"), - Some(total.to_string()), - "length" - ); - assert_eq!(whole.bytes().await.unwrap(), expected, "whole body"); - let cases = [ ("bytes=4096-8191".to_owned(), 4096, 8191), + ( + format!("bytes={}-{}", PART - 6, PART + 9), + PART - 6, + PART + 9, + ), + (format!("bytes={PART}-"), PART, total - 1), (format!("bytes={}-", total - 100), total - 100, total - 1), ( format!("bytes={}-{}", total - 10, total + 5000), @@ -88,32 +91,61 @@ async fn serves_ranges_of_merged_entries() { ("bytes=-100".to_owned(), total - 100, total - 1), ]; for (range, start, end) in cases { - let response = get(&server, &location.id, Some(&range)).await; - assert_eq!(response.status(), 206, "{range}"); + let response = get_with(server, id, header, Some(&range)).await; + assert_eq!(response.status(), 206, "{header}: {range}"); assert_eq!( - header(&response, "content-range"), + response_header(&response, "content-range"), Some(format!("bytes {start}-{end}/{total}")), - "{range}" + "{header}: {range}" ); assert_eq!( - header(&response, "content-length"), + response_header(&response, "content-length"), Some((end - start + 1).to_string()), - "{range}" + "{header}: {range}" ); assert_eq!( response.bytes().await.unwrap(), expected[start..=end], - "{range}" + "{header}: {range}" ); } - let past_end = get(&server, &location.id, Some(&format!("bytes={total}-"))).await; - assert_eq!(past_end.status(), 416, "range starting at the end"); + let past_end = get_with(server, id, header, Some(&format!("bytes={total}-"))).await; + assert_eq!( + past_end.status(), + 416, + "{header}: range starting at the end" + ); assert_eq!( - header(&past_end, "content-range"), + response_header(&past_end, "content-range"), Some(format!("bytes */{total}")), - "size reported" + "{header}: size reported" + ); +} + +#[tokio::test] +async fn serves_ranges_of_merged_entries() { + let server = TestServer::start().await; + let (location, expected) = merged_entry(&server).await; + let total = expected.len(); + + let whole = get(&server, &location.id, None).await; + assert_eq!(whole.status(), 200, "no range"); + assert_eq!( + response_header(&whole, "accept-ranges").as_deref(), + Some("bytes"), + "ranges advertised" + ); + assert_eq!( + response_header(&whole, "content-length"), + Some(total.to_string()), + "length" ); + assert_eq!(whole.bytes().await.unwrap(), expected, "whole body"); + + for header in ["range", "x-ms-range"] { + assert_serves_ranges(&server, &location.id, &expected, header).await; + } let unparseable = get(&server, &location.id, Some("bytes=abc")).await; assert_eq!(unparseable.status(), 200, "unparseable range falls through"); @@ -121,7 +153,7 @@ async fn serves_ranges_of_merged_entries() { } #[tokio::test] -async fn ignores_ranges_of_unmerged_entries_and_pins_bytes_to_a_location() { +async fn serves_ranges_of_unmerged_entries_and_pins_bytes_to_a_location() { let server = TestServer::start().await; let key = uuid::Uuid::new_v4().to_string(); let first = save_parts(server.storage(), &key, &parts()).await; @@ -134,13 +166,32 @@ async fn ignores_ranges_of_unmerged_entries_and_pins_bytes_to_a_location() { 404, "the replaced location is no longer served" ); - let response = get(&server, &resaved.id, Some("bytes=0-1")).await; - assert_eq!(response.status(), 200, "range is ignored while unmerged"); - assert_eq!(header(&response, "content-range"), None, "no content-range"); + let response = get(&server, &resaved.id, Some("bytes=1-3")).await; + assert_eq!(response.status(), 206, "a range across two Parts"); + assert_eq!( + response_header(&response, "content-range").as_deref(), + Some("bytes 1-3/7"), + "content-range" + ); assert_eq!( response.bytes().await.unwrap().as_ref(), - b"resaved", - "whole body from parts" + b"esa", + "ranged body" + ); + + // The Azure SDK (BuildKit's `type=gha`) resumes with `x-ms-range` and takes any 200 body as the + // bytes from its offset, so an unmerged entry must never fall back to serving itself whole. + let unmerged = save_parts(server.storage(), "unmerged", &parts()).await; + for header in ["range", "x-ms-range"] { + assert_serves_ranges(&server, &unmerged.id, &parts().concat(), header).await; + } + assert_eq!( + find_location(server.db(), &unmerged.id) + .await + .unwrap() + .merged_at, + None, + "ranges were served from the Parts" ); let single = save_parts(server.storage(), "single", &[b"one part".to_vec()]).await; @@ -151,7 +202,7 @@ async fn ignores_ranges_of_unmerged_entries_and_pins_bytes_to_a_location() { "a single Part needs no Merge to serve ranges" ); assert_eq!( - header(&ranged, "content-range"), + response_header(&ranged, "content-range"), Some("bytes 4-7/8".to_owned()), "content-range" ); @@ -169,9 +220,9 @@ async fn an_empty_entry_ignores_ranges_on_every_backend() { for range in ["bytes=0-", "bytes=5-"] { let response = get(&server, &location.id, Some(range)).await; assert_eq!(response.status(), 200, "{range}"); - assert_eq!(header(&response, "content-range"), None, "{range}"); + assert_eq!(response_header(&response, "content-range"), None, "{range}"); assert_eq!( - header(&response, "content-length").as_deref(), + response_header(&response, "content-length").as_deref(), Some("0"), "{range}" ); From 0f6c4173103d6556356f6fbea275769082fad165 Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:01:25 +0100 Subject: [PATCH 2/5] perf(download): seek ranged reads of unmerged entries from one part listing Skipping Parts with a 416 probe each cost one storage round trip per skipped Part, and BuildKit reopens a download on every read once it has resumed. `list_folder` now returns sizes by name, so the Part sizes from the existing listing locate the start Part directly. Co-Authored-By: Claude Opus 5.5 --- src/routes/transfer.rs | 4 +- src/storage.rs | 126 ++++++++++++++++---------------- src/storage/adapter.rs | 9 ++- src/storage/filesystem.rs | 16 ++-- src/storage/filesystem/tests.rs | 2 +- src/storage/object.rs | 17 +++-- src/tests/storage_backends.rs | 7 +- 7 files changed, 97 insertions(+), 84 deletions(-) diff --git a/src/routes/transfer.rs b/src/routes/transfer.rs index 0453522..b478f46 100644 --- a/src/routes/transfer.rs +++ b/src/routes/transfer.rs @@ -172,7 +172,9 @@ pub(super) async fn download( } Err(DownloadError::RangeNotSatisfiable { size }) => { let mut response = (StatusCode::RANGE_NOT_SATISFIABLE, [accept_ranges]).into_response(); - if let Ok(content_range) = HeaderValue::from_str(&format!("bytes */{size}")) { + if let Some(content_range) = + size.and_then(|size| HeaderValue::from_str(&format!("bytes */{size}")).ok()) + { response.headers_mut().insert(CONTENT_RANGE, content_range); } return Ok(response); diff --git a/src/storage.rs b/src/storage.rs index 0d5e2cd..13862d9 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -117,7 +117,7 @@ pub(crate) enum UploadAccess { #[derive(Debug, thiserror::Error)] pub(crate) enum DownloadError { #[error("range not satisfiable")] - RangeNotSatisfiable { size: u64 }, + RangeNotSatisfiable { size: Option }, #[error(transparent)] Failed(#[from] Failure), } @@ -171,9 +171,13 @@ fn entry_identity(identity: &EntryIdentity) -> Condition { .add(cache_entries::COLUMN.repo_id.eq(identity.repo_id.as_str())) } -/// A location's Parts as one stream, starting `skip` bytes in. A Part wholly before that offset -/// costs one ranged read answered 416, never its bytes. -fn stream_parts(adapter: Arc, location: &StorageLocation, skip: u64) -> ByteStream { +/// Concatenates a location's Parts as they are read, from `skip` bytes into Part `first`. +fn stream_parts( + adapter: Arc, + location: &StorageLocation, + first: i32, + skip: u64, +) -> ByteStream { if location.parts_deleted_at.is_some() { return stream::once(async { Err(io::Error::other( @@ -182,37 +186,25 @@ fn stream_parts(adapter: Arc, location: &StorageLocation, skip: u64) -> }) .boxed(); } - let (folder, part_count) = (location.folder_name.clone(), location.part_count); - stream::try_unfold((0, skip), move |(first, mut skip)| { - let adapter = Arc::clone(&adapter); - let folder = folder.clone(); - async move { - for index in first..part_count { - let name = part_object(&folder, index); - let range = (skip > 0).then_some(RangeRequest { - start: StartPosition::Index(skip), - end: EndPosition::LastByte, - }); - match adapter.download_stream(&name, range).await { - // An empty Part is served whole whatever the range. - Ok(DownloadStream { - extent: Extent::Whole { size: 0 }, - .. - }) if skip > 0 => {} - Ok(download) => { - return Ok(Some((download.stream, (index.saturating_add(1), 0)))); - } - Err(ReadError::RangeNotSatisfiable { size, .. }) => { - skip = skip.saturating_sub(size); - } - Err(error) => return Err(io::Error::other(error)), - } + let folder = location.folder_name.clone(); + stream::iter(first..location.part_count) + .then(move |index| { + let adapter = Arc::clone(&adapter); + let name = part_object(&folder, index); + let range = (index == first && skip > 0).then_some(RangeRequest { + start: StartPosition::Index(skip), + end: EndPosition::LastByte, + }); + async move { + adapter + .download_stream(&name, range) + .await + .map(|download| download.stream) + .map_err(io::Error::other) } - Ok(None) - } - }) - .try_flatten() - .boxed() + }) + .try_flatten() + .boxed() } /// The first `len` bytes of a stream. @@ -222,11 +214,7 @@ fn take_bytes(stream: ByteStream, len: u64) -> ByteStream { let chunk = match chunk { Ok(_) if *left == 0 => None, Ok(mut bytes) => { - if let Ok(keep) = usize::try_from(*left) - && keep < bytes.len() - { - bytes.truncate(keep); - } + bytes.truncate(usize::try_from(*left).unwrap_or(usize::MAX)); *left = left.saturating_sub(u64::try_from(bytes.len()).unwrap_or(u64::MAX)); Some(Ok(bytes)) } @@ -400,13 +388,18 @@ impl Storage { .boxed() } - async fn ensure_parts_exist(&self, location: &StorageLocation) -> Result<(), ReadError> { + /// The sizes of a location's Parts in order, from one listing; `NotFound` when one is gone. + async fn part_sizes(&self, location: &StorageLocation) -> Result, ReadError> { let folder = parts_folder(&location.folder_name); let parts = self.adapter.list_folder(&folder).await?; - if parts.len() < usize::try_from(location.part_count).unwrap_or(0) { - return Err(ReadError::NotFound(folder)); - } - Ok(()) + (0..location.part_count) + .map(|index| { + parts + .get(&index.to_string()) + .copied() + .ok_or_else(|| ReadError::NotFound(folder.clone())) + }) + .collect() } async fn download_location( @@ -417,27 +410,34 @@ impl Storage { if let Some((object, _)) = whole_object(location) { return self.adapter.download_stream(&object, range).await; } - self.ensure_parts_exist(location).await?; + let part_sizes = self.part_sizes(location).await?; let adapter = Arc::clone(&self.adapter); - let size = location - .size_bytes - .and_then(|size| u64::try_from(size).ok()); - let Some((range, size)) = range.zip(size.filter(|&size| size > 0)) else { + let size = part_sizes + .iter() + .fold(0_u64, |total, &part| total.saturating_add(part)); + let Some(range) = range.filter(|_| size > 0) else { // An unmerged entry is concatenated from its Parts as it is read. Only an empty entry // ignores a range, like an empty object does. return Ok(DownloadStream { - stream: stream_parts(adapter, location, 0), + stream: stream_parts(adapter, location, 0, 0), extent: Extent::Unsized, }); }; // Never the whole entry under a 200: the Azure SDK takes any 200 body as the bytes from // the offset it asked for. - let served = clamp_range(range, size).ok_or_else(|| ReadError::RangeNotSatisfiable { - object: parts_folder(&location.folder_name), - size, - })?; + let served = clamp_range(range, size) + .ok_or_else(|| ReadError::RangeNotSatisfiable(parts_folder(&location.folder_name)))?; + // The Part the range starts in, and the offset into it. + let (mut first, mut skip) = (0_i32, served.start); + for &part in &part_sizes { + if skip < part { + break; + } + skip = skip.saturating_sub(part); + first = first.saturating_add(1); + } Ok(DownloadStream { - stream: take_bytes(stream_parts(adapter, location, served.start), served.len()), + stream: take_bytes(stream_parts(adapter, location, first, skip), served.len()), extent: Extent::Partial { range: served, size, @@ -567,7 +567,7 @@ impl Storage { fn merge_parts(&self, location: &StorageLocation) -> BoxFuture<'static, Result<()>> { let adapter = Arc::clone(&self.adapter); let merged = merged_object(&location.folder_name); - let part_stream = stream_parts(Arc::clone(&adapter), location, 0); + let part_stream = stream_parts(Arc::clone(&adapter), location, 0, 0); let kind = UploadKind::Merged { length: location .size_bytes @@ -726,7 +726,7 @@ impl Storage { return Err(self.abandon_upload(&upload, reason).await); } let size_bytes = parts - .iter() + .values() .fold(0_u64, |total, bytes| total.saturating_add(*bytes)); let location = StorageLocation { id: uuid::Uuid::new_v4().to_string(), @@ -910,9 +910,11 @@ impl Storage { tracing::warn!(location_id, object, "stale storage location"); Ok(None) } - Err(ReadError::RangeNotSatisfiable { size, .. }) => { - Err(DownloadError::RangeNotSatisfiable { size }) - } + Err(ReadError::RangeNotSatisfiable(_)) => Err(DownloadError::RangeNotSatisfiable { + size: location + .size_bytes + .and_then(|size| u64::try_from(size).ok()), + }), Err(ReadError::Failed(failure)) => Err(failure.into()), }; if let Err(error) = release_reader_lease(&self.db, &lease_id).await { @@ -931,7 +933,7 @@ impl Storage { if whole_object(location).is_some() || range.is_some() { return self.download_location(location, range).await; } - self.ensure_parts_exist(location).await?; + self.part_sizes(location).await?; // Local Parts are cheap to read twice, so the client reads them at its own pace while // the Merge copies them in the background. @@ -961,7 +963,7 @@ impl Storage { } let (response_sender, response_receiver) = mpsc::channel(1); drop(tokio::spawn(pump_parts( - stream_parts(Arc::clone(&self.adapter), location, 0), + stream_parts(Arc::clone(&self.adapter), location, 0, 0), response_sender, merger_sender, ))); diff --git a/src/storage/adapter.rs b/src/storage/adapter.rs index 13d992f..c749223 100644 --- a/src/storage/adapter.rs +++ b/src/storage/adapter.rs @@ -1,5 +1,6 @@ //! Storage backends behind one interface. +use std::collections::HashMap; use std::io; use axum::body::Bytes; @@ -94,8 +95,8 @@ impl std::fmt::Debug for DownloadStream { pub(crate) enum ReadError { #[error("object not found in storage: {0}")] NotFound(String), - #[error("range not satisfiable for {object} of {size} bytes")] - RangeNotSatisfiable { object: String, size: u64 }, + #[error("range not satisfiable for {0}")] + RangeNotSatisfiable(String), #[error(transparent)] Failed(#[from] Failure), } @@ -215,8 +216,8 @@ impl Adapter { } } - /// Sizes of the objects under a folder. - pub(crate) async fn list_folder(&self, folder: &str) -> Result> { + /// Sizes of the objects under a folder, by name relative to it. + pub(crate) async fn list_folder(&self, folder: &str) -> Result> { match self { Self::Filesystem(adapter) => adapter.list_folder(folder).await, Self::Object(adapter) => adapter.list_folder(folder).await, diff --git a/src/storage/filesystem.rs b/src/storage/filesystem.rs index 34763c3..f7fb624 100644 --- a/src/storage/filesystem.rs +++ b/src/storage/filesystem.rs @@ -1,5 +1,6 @@ //! Storage on a local (or mounted) filesystem. +use std::collections::HashMap; use std::io::{self, SeekFrom}; use std::path::{Component, Path, PathBuf}; use std::time::UNIX_EPOCH; @@ -124,10 +125,8 @@ impl FilesystemAdapter { extent: Extent::Whole { size }, }); }; - let served = clamp_range(range, size).ok_or_else(|| ReadError::RangeNotSatisfiable { - object: name.to_owned(), - size, - })?; + let served = clamp_range(range, size) + .ok_or_else(|| ReadError::RangeNotSatisfiable(name.to_owned()))?; file.seek(SeekFrom::Start(served.start)) .await .map_err(Failure::from)?; @@ -223,18 +222,19 @@ impl FilesystemAdapter { }) } - pub(crate) async fn list_folder(&self, folder: &str) -> Result> { + pub(crate) async fn list_folder(&self, folder: &str) -> Result> { let mut entries = match fs::read_dir(self.safe_path(folder)?).await { Ok(entries) => entries, - Err(error) if is_not_found(&error) => return Ok(Vec::new()), + Err(error) if is_not_found(&error) => return Ok(HashMap::new()), Err(error) => return Err(error.into()), }; - let mut objects = Vec::new(); + let mut objects = HashMap::new(); while let Some(entry) = entries.next_entry().await? { if entry.file_type().await?.is_dir() { continue; } - objects.push(fs::metadata(entry.path()).await?.len()); + let size = fs::metadata(entry.path()).await?.len(); + objects.insert(entry.file_name().to_string_lossy().into_owned(), size); } Ok(objects) } diff --git a/src/storage/filesystem/tests.rs b/src/storage/filesystem/tests.rs index dc8371f..f2265a4 100644 --- a/src/storage/filesystem/tests.rs +++ b/src/storage/filesystem/tests.rs @@ -62,7 +62,7 @@ async fn uploads_atomically_and_serves_ranges() { assert!( matches!( adapter.download_stream("f/parts/0", Some(past)).await, - Err(ReadError::RangeNotSatisfiable { .. }) + Err(ReadError::RangeNotSatisfiable(_)) ), "range past the end" ); diff --git a/src/storage/object.rs b/src/storage/object.rs index 98e66c2..76fe3de 100644 --- a/src/storage/object.rs +++ b/src/storage/object.rs @@ -320,10 +320,8 @@ impl ObjectAdapter { extent: Extent::Whole { size: 0 }, }); } - let served = clamp_range(range, size).ok_or_else(|| ReadError::RangeNotSatisfiable { - object: name.to_owned(), - size, - })?; + let served = clamp_range(range, size) + .ok_or_else(|| ReadError::RangeNotSatisfiable(name.to_owned()))?; let options = GetOptions { range: Some(GetRange::Bounded( served.start..served.end.saturating_add(1), @@ -478,11 +476,16 @@ impl ObjectAdapter { self.delete_prefix(&self.path(folder)).await } - pub(crate) async fn list_folder(&self, folder: &str) -> Result> { + pub(crate) async fn list_folder(&self, folder: &str) -> Result> { + let folder = self.path(folder); self.store - .list(Some(&self.path(folder))) + .list(Some(&folder)) .map_err(Failure::from) - .map_ok(|object| object.size) + .map_ok(|object| { + let name = object.location.as_ref().strip_prefix(folder.as_ref()); + let name = name.unwrap_or_default().trim_start_matches('/'); + (name.to_owned(), object.size) + }) .try_collect() .await } diff --git a/src/tests/storage_backends.rs b/src/tests/storage_backends.rs index 9fc4873..1e41797 100644 --- a/src/tests/storage_backends.rs +++ b/src/tests/storage_backends.rs @@ -1,5 +1,6 @@ //! Adapter behaviour every backend must share, plus backend-specific edges. +use std::collections::HashMap; use std::time::Duration; use super::harness::{TestServer, put, storage_driver}; @@ -13,7 +14,11 @@ async fn folder_operations_do_not_match_sibling_prefixes() { put(adapter, "abc123/merged", b"bb").await; let objects = adapter.list_folder("abc").await.unwrap(); - assert_eq!(objects, [1], "only the folder's own object"); + assert_eq!( + objects, + HashMap::from([("merged".to_owned(), 1)]), + "only the folder's own object" + ); let deleted = adapter.delete_folder("abc").await.unwrap(); assert_eq!( (deleted.objects, deleted.bytes), From 273e2ab5458253abf6959a202826e2b061509323 Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:01:25 +0100 Subject: [PATCH 3/5] test(download): one header pass for merged ranges Co-Authored-By: Claude Opus 5.5 --- src/tests/downloads.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tests/downloads.rs b/src/tests/downloads.rs index 5e4b3a6..168c3ed 100644 --- a/src/tests/downloads.rs +++ b/src/tests/downloads.rs @@ -143,9 +143,7 @@ async fn serves_ranges_of_merged_entries() { ); assert_eq!(whole.bytes().await.unwrap(), expected, "whole body"); - for header in ["range", "x-ms-range"] { - assert_serves_ranges(&server, &location.id, &expected, header).await; - } + assert_serves_ranges(&server, &location.id, &expected, "range").await; let unparseable = get(&server, &location.id, Some("bytes=abc")).await; assert_eq!(unparseable.status(), 200, "unparseable range falls through"); From 40cdd88c9e2d80fd35871d61e8c726f8e8ba05bc Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:01:25 +0100 Subject: [PATCH 4/5] docs(how-it-works): ranged downloads and x-ms-range Co-Authored-By: Claude Opus 5.5 --- website/src/content/docs/how-it-works.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/src/content/docs/how-it-works.md b/website/src/content/docs/how-it-works.md index 1ae120f..72f7209 100644 --- a/website/src/content/docs/how-it-works.md +++ b/website/src/content/docs/how-it-works.md @@ -22,8 +22,8 @@ The server implements the same cache API that GitHub's hosted cache service expo 1. **Lookup.** The runner asks for a key and a list of restore keys. The server answers with a download URL for the best match in the job's scope, or a miss. 2. **Upload.** On a miss, the step reserves an entry and uploads the archive in blocks to the signed upload URL the server hands out. On object storage the blocks go straight to the bucket through presigned URLs. 3. **Finalize.** The runner commits the upload. The entry becomes visible to the next lookup. -4. **Merge.** Blocks are merged into one object on the first download, or right after upload with [`EAGER_MERGE`](/configuration#eager_merge). -5. **Download.** The server streams the object, or hands out a signed storage URL with [`ENABLE_DIRECT_DOWNLOADS`](/configuration#enable_direct_downloads). +4. **Merge.** Blocks are merged into one object on the first full download, or right after upload with [`EAGER_MERGE`](/configuration#eager_merge). +5. **Download.** The server streams the object, or hands out a signed storage URL with [`ENABLE_DIRECT_DOWNLOADS`](/configuration#enable_direct_downloads). It serves byte ranges from `Range` or, as the Azure SDK in BuildKit's `type=gha` cache sends them, `x-ms-range`. Before the merge, a range is read from the blocks it covers. Requests the server does not handle, such as artifact uploads, are forwarded to GitHub's results service, so they keep working. From f3796d21443d0073d56b2c42834e6b79d3233422 Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:37:09 +0100 Subject: [PATCH 5/5] docs(configuration): warn that S3 and GCS direct downloads break BuildKit resumes Co-Authored-By: Claude Opus 5.5 --- website/src/content/docs/configuration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/src/content/docs/configuration.md b/website/src/content/docs/configuration.md index bc54b79..e958acb 100644 --- a/website/src/content/docs/configuration.md +++ b/website/src/content/docs/configuration.md @@ -94,6 +94,10 @@ The actions runner needs to be able to reach the storage provider directly to us A signed URL is a storage credential, so the server refuses to start with both `ENABLE_DIRECT_DOWNLOADS=true` and [`SKIP_TOKEN_VALIDATION=true`](#skip_token_validation), unless [`UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION`](#unsafe_allow_direct_downloads_without_token_validation) is set. The filesystem storage driver has no direct downloads. +:::caution[BuildKit on S3 or GCS] +BuildKit's `type=gha` cache resumes a dropped download with an `x-ms-range` header. Azure Blob Storage honors it, but S3 and GCS ignore it and send the whole object, which BuildKit writes at the wrong offset. The build then fails on a corrupted layer. With BuildKit on S3 or GCS, leave direct downloads off. The server itself honors `x-ms-range`. +::: + ### `DIRECT_DOWNLOAD_URL_TTL_SECONDS` - Default: `600`