diff --git a/src/routes/transfer.rs b/src/routes/transfer.rs index 1d9aa62..b478f46 100644 --- a/src/routes/transfer.rs +++ b/src/routes/transfer.rs @@ -23,6 +23,8 @@ use crate::storage::{ /// Clients without a request id intermittently fail with EOF (tonistiigi/go-actions-cache). const REQUEST_ID: HeaderName = HeaderName::from_static("x-ms-request-id"); +/// Azure's Range header; Get Blob prefers it over `Range`. +const X_MS_RANGE: HeaderName = HeaderName::from_static("x-ms-range"); /// Lenient like Node's `Buffer.from(value, 'base64')`. const BASE64: GeneralPurpose = GeneralPurpose::new( @@ -128,9 +130,15 @@ fn upload_not_found(upload_id: i64) -> HttpError { } /// A single range; multi-range requests and invalid headers, which includes inverted ranges, are -/// served whole. +/// served whole. `x-ms-range` wins over `Range`, like Azure Get Blob: the Azure SDK, and with it +/// `BuildKit`'s `type=gha` cache, sends only `x-ms-range` when it resumes a download. fn parse_range(headers: &HeaderMap) -> Option { - let value = headers.get(RANGE)?.to_str().ok()?.trim(); + let value = headers + .get(X_MS_RANGE) + .or_else(|| headers.get(RANGE))? + .to_str() + .ok()? + .trim(); // The unit is case-insensitive (RFC 9110 ยง14.1); the parser only knows `bytes=`. let ranges = value .get(..6)? @@ -153,7 +161,6 @@ pub(super) async fn download( headers: HeaderMap, ) -> Result { let range = parse_range(&headers); - // Unmerged entries ignore Range, so clients must key off the status, not this header. let accept_ranges = (ACCEPT_RANGES, HeaderValue::from_static("bytes")); let download = match state.storage.download(&location_id, range).await { Ok(Some(download)) => download, diff --git a/src/routes/transfer/tests.rs b/src/routes/transfer/tests.rs index 4a6c779..ae306a2 100644 --- a/src/routes/transfer/tests.rs +++ b/src/routes/transfer/tests.rs @@ -52,3 +52,28 @@ fn parses_single_ranges() { assert_eq!(range("bytes=9-5"), None, "inverted"); assert_eq!(range("items=0-5"), None, "unknown unit"); } + +#[test] +fn x_ms_range_wins_over_range() { + let from = |pairs: &[(&'static str, &'static str)]| { + let mut headers = HeaderMap::new(); + for &(name, value) in pairs { + headers.insert(name, HeaderValue::from_static(value)); + } + parse_range(&headers) + }; + let from_100 = Some(RangeRequest { + start: StartPosition::Index(100), + end: EndPosition::LastByte, + }); + assert_eq!( + from(&[("x-ms-range", "bytes=100-")]), + from_100, + "x-ms-range alone, as the Azure SDK sends it" + ); + assert_eq!( + from(&[("x-ms-range", "bytes=100-"), ("range", "bytes=0-9")]), + from_100, + "x-ms-range wins, like Azure Get Blob" + ); +} diff --git a/src/storage.rs b/src/storage.rs index 4f2650c..13862d9 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -15,7 +15,7 @@ use axum::body::Bytes; use base64::Engine as _; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use futures_util::future::BoxFuture; -use futures_util::{FutureExt as _, StreamExt as _, TryStreamExt as _, stream}; +use futures_util::{FutureExt as _, StreamExt as _, TryStreamExt as _, future, stream}; use sea_orm::ActiveValue::Set; use sea_orm::sea_query::{CaseStatement, Expr, ExprTrait as _, Func}; use sea_orm::{ @@ -30,11 +30,11 @@ use tokio_stream::wrappers::ReceiverStream; use tokio_util::task::{AbortOnDropHandle, TaskTracker}; use utoipa::ToSchema; -use self::adapter::ReadError; pub(crate) use self::adapter::{ Adapter, ByteStream, DownloadStream, EndPosition, Extent, RangeRequest, StartPosition, UploadKind, }; +use self::adapter::{ReadError, clamp_range}; use self::leases::{ DIRECT_DOWNLOAD_LEASE_MARGIN, LEASE_DURATION, LEASE_RENEWAL, acquire_merge_lease, create_reader_lease, deadline, release_merge_lease, release_reader_lease, renew_merge_lease, @@ -171,8 +171,13 @@ fn entry_identity(identity: &EntryIdentity) -> Condition { .add(cache_entries::COLUMN.repo_id.eq(identity.repo_id.as_str())) } -/// Concatenates a location's Parts as they are read. -fn stream_parts(adapter: Arc, location: &StorageLocation) -> ByteStream { +/// Concatenates a location's Parts as they are read, from `skip` bytes into Part `first`. +fn stream_parts( + adapter: Arc, + location: &StorageLocation, + first: i32, + skip: u64, +) -> ByteStream { if location.parts_deleted_at.is_some() { return stream::once(async { Err(io::Error::other( @@ -182,13 +187,17 @@ fn stream_parts(adapter: Arc, location: &StorageLocation) -> ByteStream .boxed(); } let folder = location.folder_name.clone(); - stream::iter(0..location.part_count) + stream::iter(first..location.part_count) .then(move |index| { let adapter = Arc::clone(&adapter); let name = part_object(&folder, index); + let range = (index == first && skip > 0).then_some(RangeRequest { + start: StartPosition::Index(skip), + end: EndPosition::LastByte, + }); async move { adapter - .download_stream(&name, None) + .download_stream(&name, range) .await .map(|download| download.stream) .map_err(io::Error::other) @@ -198,6 +207,24 @@ fn stream_parts(adapter: Arc, location: &StorageLocation) -> ByteStream .boxed() } +/// The first `len` bytes of a stream. +fn take_bytes(stream: ByteStream, len: u64) -> ByteStream { + stream + .scan(len, |left, chunk| { + let chunk = match chunk { + Ok(_) if *left == 0 => None, + Ok(mut bytes) => { + bytes.truncate(usize::try_from(*left).unwrap_or(usize::MAX)); + *left = left.saturating_sub(u64::try_from(bytes.len()).unwrap_or(u64::MAX)); + Some(Ok(bytes)) + } + Err(error) => Some(Err(error)), + }; + future::ready(chunk) + }) + .boxed() +} + /// Releases a Storage Reader Lease and stops its renewal once the protected stream is dropped. struct LeaseGuard { db: Db, @@ -361,13 +388,18 @@ impl Storage { .boxed() } - async fn ensure_parts_exist(&self, location: &StorageLocation) -> Result<(), ReadError> { + /// The sizes of a location's Parts in order, from one listing; `NotFound` when one is gone. + async fn part_sizes(&self, location: &StorageLocation) -> Result, ReadError> { let folder = parts_folder(&location.folder_name); let parts = self.adapter.list_folder(&folder).await?; - if parts.len() < usize::try_from(location.part_count).unwrap_or(0) { - return Err(ReadError::NotFound(folder)); - } - Ok(()) + (0..location.part_count) + .map(|index| { + parts + .get(&index.to_string()) + .copied() + .ok_or_else(|| ReadError::NotFound(folder.clone())) + }) + .collect() } async fn download_location( @@ -378,12 +410,38 @@ impl Storage { if let Some((object, _)) = whole_object(location) { return self.adapter.download_stream(&object, range).await; } - self.ensure_parts_exist(location).await?; - // No range: an unmerged entry is concatenated from its parts as it is read, so there is - // nothing to seek into. Served whole under a 200, which Range allows. + let part_sizes = self.part_sizes(location).await?; + let adapter = Arc::clone(&self.adapter); + let size = part_sizes + .iter() + .fold(0_u64, |total, &part| total.saturating_add(part)); + let Some(range) = range.filter(|_| size > 0) else { + // An unmerged entry is concatenated from its Parts as it is read. Only an empty entry + // ignores a range, like an empty object does. + return Ok(DownloadStream { + stream: stream_parts(adapter, location, 0, 0), + extent: Extent::Unsized, + }); + }; + // Never the whole entry under a 200: the Azure SDK takes any 200 body as the bytes from + // the offset it asked for. + let served = clamp_range(range, size) + .ok_or_else(|| ReadError::RangeNotSatisfiable(parts_folder(&location.folder_name)))?; + // The Part the range starts in, and the offset into it. + let (mut first, mut skip) = (0_i32, served.start); + for &part in &part_sizes { + if skip < part { + break; + } + skip = skip.saturating_sub(part); + first = first.saturating_add(1); + } Ok(DownloadStream { - stream: stream_parts(Arc::clone(&self.adapter), location), - extent: Extent::Unsized, + stream: take_bytes(stream_parts(adapter, location, first, skip), served.len()), + extent: Extent::Partial { + range: served, + size, + }, }) } @@ -509,7 +567,7 @@ impl Storage { fn merge_parts(&self, location: &StorageLocation) -> BoxFuture<'static, Result<()>> { let adapter = Arc::clone(&self.adapter); let merged = merged_object(&location.folder_name); - let part_stream = stream_parts(Arc::clone(&adapter), location); + let part_stream = stream_parts(Arc::clone(&adapter), location, 0, 0); let kind = UploadKind::Merged { length: location .size_bytes @@ -668,7 +726,7 @@ impl Storage { return Err(self.abandon_upload(&upload, reason).await); } let size_bytes = parts - .iter() + .values() .fold(0_u64, |total, bytes| total.saturating_add(*bytes)); let location = StorageLocation { id: uuid::Uuid::new_v4().to_string(), @@ -871,16 +929,17 @@ impl Storage { location: &StorageLocation, range: Option, ) -> Result { - if whole_object(location).is_some() { + // A ranged read serves its slice from the Parts and leaves the Merge to a whole read. + if whole_object(location).is_some() || range.is_some() { return self.download_location(location, range).await; } - self.ensure_parts_exist(location).await?; + self.part_sizes(location).await?; // Local Parts are cheap to read twice, so the client reads them at its own pace while // the Merge copies them in the background. if let Adapter::Filesystem(_) = self.adapter.as_ref() { self.start_merge_detached(location, self.merge_parts(location)); - return self.download_location(location, range).await; + return self.download_location(location, None).await; } // Object storage is read once and teed into the response and the Merge. let (merger_sender, merger_receiver) = mpsc::channel(1); @@ -900,11 +959,11 @@ impl Storage { .boxed() }; if !self.start_merge(location, write).await? { - return self.download_location(location, range).await; + return self.download_location(location, None).await; } let (response_sender, response_receiver) = mpsc::channel(1); drop(tokio::spawn(pump_parts( - stream_parts(Arc::clone(&self.adapter), location), + stream_parts(Arc::clone(&self.adapter), location, 0, 0), response_sender, merger_sender, ))); diff --git a/src/storage/adapter.rs b/src/storage/adapter.rs index 2f8891a..c749223 100644 --- a/src/storage/adapter.rs +++ b/src/storage/adapter.rs @@ -1,5 +1,6 @@ //! Storage backends behind one interface. +use std::collections::HashMap; use std::io; use axum::body::Bytes; @@ -215,8 +216,8 @@ impl Adapter { } } - /// Sizes of the objects under a folder. - pub(crate) async fn list_folder(&self, folder: &str) -> Result> { + /// Sizes of the objects under a folder, by name relative to it. + pub(crate) async fn list_folder(&self, folder: &str) -> Result> { match self { Self::Filesystem(adapter) => adapter.list_folder(folder).await, Self::Object(adapter) => adapter.list_folder(folder).await, diff --git a/src/storage/filesystem.rs b/src/storage/filesystem.rs index 9456f9e..f7fb624 100644 --- a/src/storage/filesystem.rs +++ b/src/storage/filesystem.rs @@ -1,5 +1,6 @@ //! Storage on a local (or mounted) filesystem. +use std::collections::HashMap; use std::io::{self, SeekFrom}; use std::path::{Component, Path, PathBuf}; use std::time::UNIX_EPOCH; @@ -221,18 +222,19 @@ impl FilesystemAdapter { }) } - pub(crate) async fn list_folder(&self, folder: &str) -> Result> { + pub(crate) async fn list_folder(&self, folder: &str) -> Result> { let mut entries = match fs::read_dir(self.safe_path(folder)?).await { Ok(entries) => entries, - Err(error) if is_not_found(&error) => return Ok(Vec::new()), + Err(error) if is_not_found(&error) => return Ok(HashMap::new()), Err(error) => return Err(error.into()), }; - let mut objects = Vec::new(); + let mut objects = HashMap::new(); while let Some(entry) = entries.next_entry().await? { if entry.file_type().await?.is_dir() { continue; } - objects.push(fs::metadata(entry.path()).await?.len()); + let size = fs::metadata(entry.path()).await?.len(); + objects.insert(entry.file_name().to_string_lossy().into_owned(), size); } Ok(objects) } diff --git a/src/storage/object.rs b/src/storage/object.rs index 73cdc0d..76fe3de 100644 --- a/src/storage/object.rs +++ b/src/storage/object.rs @@ -476,11 +476,16 @@ impl ObjectAdapter { self.delete_prefix(&self.path(folder)).await } - pub(crate) async fn list_folder(&self, folder: &str) -> Result> { + pub(crate) async fn list_folder(&self, folder: &str) -> Result> { + let folder = self.path(folder); self.store - .list(Some(&self.path(folder))) + .list(Some(&folder)) .map_err(Failure::from) - .map_ok(|object| object.size) + .map_ok(|object| { + let name = object.location.as_ref().strip_prefix(folder.as_ref()); + let name = name.unwrap_or_default().trim_start_matches('/'); + (name.to_owned(), object.size) + }) .try_collect() .await } diff --git a/src/tests/downloads.rs b/src/tests/downloads.rs index d3af56f..168c3ed 100644 --- a/src/tests/downloads.rs +++ b/src/tests/downloads.rs @@ -43,42 +43,45 @@ async fn merged_entry(server: &TestServer) -> (StorageLocation, Vec) { } async fn get(server: &TestServer, id: &str, range: Option<&str>) -> reqwest::Response { + get_with(server, id, "range", range).await +} + +async fn get_with( + server: &TestServer, + id: &str, + header: &str, + range: Option<&str>, +) -> reqwest::Response { let mut request = reqwest::Client::new().get(format!("{}/download/{id}", server.url)); if let Some(range) = range { - request = request.header("range", range); + request = request.header(header, range); } request.send().await.unwrap() } -fn header(response: &reqwest::Response, name: &str) -> Option { +fn response_header(response: &reqwest::Response, name: &str) -> Option { response .headers() .get(name) .map(|value| value.to_str().unwrap().to_owned()) } -#[tokio::test] -async fn serves_ranges_of_merged_entries() { - let server = TestServer::start().await; - let (location, expected) = merged_entry(&server).await; +/// Every range shape, including ones crossing and starting on Part boundaries, answered 206 with +/// exactly the requested bytes, and a range past the end answered 416. +#[expect( + clippy::arithmetic_side_effects, + reason = "offsets into a fixture of a few hundred KiB" +)] +async fn assert_serves_ranges(server: &TestServer, id: &str, expected: &[u8], header: &str) { let total = expected.len(); - - let whole = get(&server, &location.id, None).await; - assert_eq!(whole.status(), 200, "no range"); - assert_eq!( - header(&whole, "accept-ranges").as_deref(), - Some("bytes"), - "ranges advertised" - ); - assert_eq!( - header(&whole, "content-length"), - Some(total.to_string()), - "length" - ); - assert_eq!(whole.bytes().await.unwrap(), expected, "whole body"); - let cases = [ ("bytes=4096-8191".to_owned(), 4096, 8191), + ( + format!("bytes={}-{}", PART - 6, PART + 9), + PART - 6, + PART + 9, + ), + (format!("bytes={PART}-"), PART, total - 1), (format!("bytes={}-", total - 100), total - 100, total - 1), ( format!("bytes={}-{}", total - 10, total + 5000), @@ -88,32 +91,59 @@ async fn serves_ranges_of_merged_entries() { ("bytes=-100".to_owned(), total - 100, total - 1), ]; for (range, start, end) in cases { - let response = get(&server, &location.id, Some(&range)).await; - assert_eq!(response.status(), 206, "{range}"); + let response = get_with(server, id, header, Some(&range)).await; + assert_eq!(response.status(), 206, "{header}: {range}"); assert_eq!( - header(&response, "content-range"), + response_header(&response, "content-range"), Some(format!("bytes {start}-{end}/{total}")), - "{range}" + "{header}: {range}" ); assert_eq!( - header(&response, "content-length"), + response_header(&response, "content-length"), Some((end - start + 1).to_string()), - "{range}" + "{header}: {range}" ); assert_eq!( response.bytes().await.unwrap(), expected[start..=end], - "{range}" + "{header}: {range}" ); } - let past_end = get(&server, &location.id, Some(&format!("bytes={total}-"))).await; - assert_eq!(past_end.status(), 416, "range starting at the end"); + let past_end = get_with(server, id, header, Some(&format!("bytes={total}-"))).await; + assert_eq!( + past_end.status(), + 416, + "{header}: range starting at the end" + ); assert_eq!( - header(&past_end, "content-range"), + response_header(&past_end, "content-range"), Some(format!("bytes */{total}")), - "size reported" + "{header}: size reported" ); +} + +#[tokio::test] +async fn serves_ranges_of_merged_entries() { + let server = TestServer::start().await; + let (location, expected) = merged_entry(&server).await; + let total = expected.len(); + + let whole = get(&server, &location.id, None).await; + assert_eq!(whole.status(), 200, "no range"); + assert_eq!( + response_header(&whole, "accept-ranges").as_deref(), + Some("bytes"), + "ranges advertised" + ); + assert_eq!( + response_header(&whole, "content-length"), + Some(total.to_string()), + "length" + ); + assert_eq!(whole.bytes().await.unwrap(), expected, "whole body"); + + assert_serves_ranges(&server, &location.id, &expected, "range").await; let unparseable = get(&server, &location.id, Some("bytes=abc")).await; assert_eq!(unparseable.status(), 200, "unparseable range falls through"); @@ -121,7 +151,7 @@ async fn serves_ranges_of_merged_entries() { } #[tokio::test] -async fn ignores_ranges_of_unmerged_entries_and_pins_bytes_to_a_location() { +async fn serves_ranges_of_unmerged_entries_and_pins_bytes_to_a_location() { let server = TestServer::start().await; let key = uuid::Uuid::new_v4().to_string(); let first = save_parts(server.storage(), &key, &parts()).await; @@ -134,13 +164,32 @@ async fn ignores_ranges_of_unmerged_entries_and_pins_bytes_to_a_location() { 404, "the replaced location is no longer served" ); - let response = get(&server, &resaved.id, Some("bytes=0-1")).await; - assert_eq!(response.status(), 200, "range is ignored while unmerged"); - assert_eq!(header(&response, "content-range"), None, "no content-range"); + let response = get(&server, &resaved.id, Some("bytes=1-3")).await; + assert_eq!(response.status(), 206, "a range across two Parts"); + assert_eq!( + response_header(&response, "content-range").as_deref(), + Some("bytes 1-3/7"), + "content-range" + ); assert_eq!( response.bytes().await.unwrap().as_ref(), - b"resaved", - "whole body from parts" + b"esa", + "ranged body" + ); + + // The Azure SDK (BuildKit's `type=gha`) resumes with `x-ms-range` and takes any 200 body as the + // bytes from its offset, so an unmerged entry must never fall back to serving itself whole. + let unmerged = save_parts(server.storage(), "unmerged", &parts()).await; + for header in ["range", "x-ms-range"] { + assert_serves_ranges(&server, &unmerged.id, &parts().concat(), header).await; + } + assert_eq!( + find_location(server.db(), &unmerged.id) + .await + .unwrap() + .merged_at, + None, + "ranges were served from the Parts" ); let single = save_parts(server.storage(), "single", &[b"one part".to_vec()]).await; @@ -151,7 +200,7 @@ async fn ignores_ranges_of_unmerged_entries_and_pins_bytes_to_a_location() { "a single Part needs no Merge to serve ranges" ); assert_eq!( - header(&ranged, "content-range"), + response_header(&ranged, "content-range"), Some("bytes 4-7/8".to_owned()), "content-range" ); @@ -169,9 +218,9 @@ async fn an_empty_entry_ignores_ranges_on_every_backend() { for range in ["bytes=0-", "bytes=5-"] { let response = get(&server, &location.id, Some(range)).await; assert_eq!(response.status(), 200, "{range}"); - assert_eq!(header(&response, "content-range"), None, "{range}"); + assert_eq!(response_header(&response, "content-range"), None, "{range}"); assert_eq!( - header(&response, "content-length").as_deref(), + response_header(&response, "content-length").as_deref(), Some("0"), "{range}" ); diff --git a/src/tests/storage_backends.rs b/src/tests/storage_backends.rs index 9fc4873..1e41797 100644 --- a/src/tests/storage_backends.rs +++ b/src/tests/storage_backends.rs @@ -1,5 +1,6 @@ //! Adapter behaviour every backend must share, plus backend-specific edges. +use std::collections::HashMap; use std::time::Duration; use super::harness::{TestServer, put, storage_driver}; @@ -13,7 +14,11 @@ async fn folder_operations_do_not_match_sibling_prefixes() { put(adapter, "abc123/merged", b"bb").await; let objects = adapter.list_folder("abc").await.unwrap(); - assert_eq!(objects, [1], "only the folder's own object"); + assert_eq!( + objects, + HashMap::from([("merged".to_owned(), 1)]), + "only the folder's own object" + ); let deleted = adapter.delete_folder("abc").await.unwrap(); assert_eq!( (deleted.objects, deleted.bytes), diff --git a/website/src/content/docs/configuration.md b/website/src/content/docs/configuration.md index bc54b79..e958acb 100644 --- a/website/src/content/docs/configuration.md +++ b/website/src/content/docs/configuration.md @@ -94,6 +94,10 @@ The actions runner needs to be able to reach the storage provider directly to us A signed URL is a storage credential, so the server refuses to start with both `ENABLE_DIRECT_DOWNLOADS=true` and [`SKIP_TOKEN_VALIDATION=true`](#skip_token_validation), unless [`UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION`](#unsafe_allow_direct_downloads_without_token_validation) is set. The filesystem storage driver has no direct downloads. +:::caution[BuildKit on S3 or GCS] +BuildKit's `type=gha` cache resumes a dropped download with an `x-ms-range` header. Azure Blob Storage honors it, but S3 and GCS ignore it and send the whole object, which BuildKit writes at the wrong offset. The build then fails on a corrupted layer. With BuildKit on S3 or GCS, leave direct downloads off. The server itself honors `x-ms-range`. +::: + ### `DIRECT_DOWNLOAD_URL_TTL_SECONDS` - Default: `600` diff --git a/website/src/content/docs/how-it-works.md b/website/src/content/docs/how-it-works.md index 1ae120f..72f7209 100644 --- a/website/src/content/docs/how-it-works.md +++ b/website/src/content/docs/how-it-works.md @@ -22,8 +22,8 @@ The server implements the same cache API that GitHub's hosted cache service expo 1. **Lookup.** The runner asks for a key and a list of restore keys. The server answers with a download URL for the best match in the job's scope, or a miss. 2. **Upload.** On a miss, the step reserves an entry and uploads the archive in blocks to the signed upload URL the server hands out. On object storage the blocks go straight to the bucket through presigned URLs. 3. **Finalize.** The runner commits the upload. The entry becomes visible to the next lookup. -4. **Merge.** Blocks are merged into one object on the first download, or right after upload with [`EAGER_MERGE`](/configuration#eager_merge). -5. **Download.** The server streams the object, or hands out a signed storage URL with [`ENABLE_DIRECT_DOWNLOADS`](/configuration#enable_direct_downloads). +4. **Merge.** Blocks are merged into one object on the first full download, or right after upload with [`EAGER_MERGE`](/configuration#eager_merge). +5. **Download.** The server streams the object, or hands out a signed storage URL with [`ENABLE_DIRECT_DOWNLOADS`](/configuration#enable_direct_downloads). It serves byte ranges from `Range` or, as the Azure SDK in BuildKit's `type=gha` cache sends them, `x-ms-range`. Before the merge, a range is read from the blocks it covers. Requests the server does not handle, such as artifact uploads, are forwarded to GitHub's results service, so they keep working.