Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions src/routes/transfer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ use crate::storage::{

/// Clients without a request id intermittently fail with EOF (tonistiigi/go-actions-cache).
const REQUEST_ID: HeaderName = HeaderName::from_static("x-ms-request-id");
/// Azure's Range header; Get Blob prefers it over `Range`.
const X_MS_RANGE: HeaderName = HeaderName::from_static("x-ms-range");

/// Lenient like Node's `Buffer.from(value, 'base64')`.
const BASE64: GeneralPurpose = GeneralPurpose::new(
Expand Down Expand Up @@ -128,9 +130,15 @@ fn upload_not_found(upload_id: i64) -> HttpError {
}

/// A single range; multi-range requests and invalid headers, which includes inverted ranges, are
/// served whole.
/// served whole. `x-ms-range` wins over `Range`, like Azure Get Blob: the Azure SDK, and with it
/// `BuildKit`'s `type=gha` cache, sends only `x-ms-range` when it resumes a download.
fn parse_range(headers: &HeaderMap) -> Option<RangeRequest> {
let value = headers.get(RANGE)?.to_str().ok()?.trim();
let value = headers
.get(X_MS_RANGE)
.or_else(|| headers.get(RANGE))?
.to_str()
.ok()?
.trim();
// The unit is case-insensitive (RFC 9110 Β§14.1); the parser only knows `bytes=`.
let ranges = value
.get(..6)?
Expand All @@ -153,7 +161,6 @@ pub(super) async fn download(
headers: HeaderMap,
) -> Result<Response, HttpError> {
let range = parse_range(&headers);
// Unmerged entries ignore Range, so clients must key off the status, not this header.
let accept_ranges = (ACCEPT_RANGES, HeaderValue::from_static("bytes"));
let download = match state.storage.download(&location_id, range).await {
Ok(Some(download)) => download,
Expand Down
25 changes: 25 additions & 0 deletions src/routes/transfer/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,28 @@ fn parses_single_ranges() {
assert_eq!(range("bytes=9-5"), None, "inverted");
assert_eq!(range("items=0-5"), None, "unknown unit");
}

#[test]
fn x_ms_range_wins_over_range() {
let from = |pairs: &[(&'static str, &'static str)]| {
let mut headers = HeaderMap::new();
for &(name, value) in pairs {
headers.insert(name, HeaderValue::from_static(value));
}
parse_range(&headers)
};
let from_100 = Some(RangeRequest {
start: StartPosition::Index(100),
end: EndPosition::LastByte,
});
assert_eq!(
from(&[("x-ms-range", "bytes=100-")]),
from_100,
"x-ms-range alone, as the Azure SDK sends it"
);
assert_eq!(
from(&[("x-ms-range", "bytes=100-"), ("range", "bytes=0-9")]),
from_100,
"x-ms-range wins, like Azure Get Blob"
);
}
105 changes: 82 additions & 23 deletions src/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use axum::body::Bytes;
use base64::Engine as _;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use futures_util::future::BoxFuture;
use futures_util::{FutureExt as _, StreamExt as _, TryStreamExt as _, stream};
use futures_util::{FutureExt as _, StreamExt as _, TryStreamExt as _, future, stream};
use sea_orm::ActiveValue::Set;
use sea_orm::sea_query::{CaseStatement, Expr, ExprTrait as _, Func};
use sea_orm::{
Expand All @@ -30,11 +30,11 @@ use tokio_stream::wrappers::ReceiverStream;
use tokio_util::task::{AbortOnDropHandle, TaskTracker};
use utoipa::ToSchema;

use self::adapter::ReadError;
pub(crate) use self::adapter::{
Adapter, ByteStream, DownloadStream, EndPosition, Extent, RangeRequest, StartPosition,
UploadKind,
};
use self::adapter::{ReadError, clamp_range};
use self::leases::{
DIRECT_DOWNLOAD_LEASE_MARGIN, LEASE_DURATION, LEASE_RENEWAL, acquire_merge_lease,
create_reader_lease, deadline, release_merge_lease, release_reader_lease, renew_merge_lease,
Expand Down Expand Up @@ -171,8 +171,13 @@ fn entry_identity(identity: &EntryIdentity) -> Condition {
.add(cache_entries::COLUMN.repo_id.eq(identity.repo_id.as_str()))
}

/// Concatenates a location's Parts as they are read.
fn stream_parts(adapter: Arc<Adapter>, location: &StorageLocation) -> ByteStream {
/// Concatenates a location's Parts as they are read, from `skip` bytes into Part `first`.
fn stream_parts(
adapter: Arc<Adapter>,
location: &StorageLocation,
first: i32,
skip: u64,
) -> ByteStream {
if location.parts_deleted_at.is_some() {
return stream::once(async {
Err(io::Error::other(
Expand All @@ -182,13 +187,17 @@ fn stream_parts(adapter: Arc<Adapter>, location: &StorageLocation) -> ByteStream
.boxed();
}
let folder = location.folder_name.clone();
stream::iter(0..location.part_count)
stream::iter(first..location.part_count)
.then(move |index| {
let adapter = Arc::clone(&adapter);
let name = part_object(&folder, index);
let range = (index == first && skip > 0).then_some(RangeRequest {
start: StartPosition::Index(skip),
end: EndPosition::LastByte,
});
async move {
adapter
.download_stream(&name, None)
.download_stream(&name, range)
.await
.map(|download| download.stream)
.map_err(io::Error::other)
Expand All @@ -198,6 +207,24 @@ fn stream_parts(adapter: Arc<Adapter>, location: &StorageLocation) -> ByteStream
.boxed()
}

/// The first `len` bytes of a stream.
fn take_bytes(stream: ByteStream, len: u64) -> ByteStream {
stream
.scan(len, |left, chunk| {
let chunk = match chunk {
Ok(_) if *left == 0 => None,
Ok(mut bytes) => {
bytes.truncate(usize::try_from(*left).unwrap_or(usize::MAX));
*left = left.saturating_sub(u64::try_from(bytes.len()).unwrap_or(u64::MAX));
Some(Ok(bytes))
}
Err(error) => Some(Err(error)),
};
future::ready(chunk)
})
.boxed()
}

/// Releases a Storage Reader Lease and stops its renewal once the protected stream is dropped.
struct LeaseGuard {
db: Db,
Expand Down Expand Up @@ -361,13 +388,18 @@ impl Storage {
.boxed()
}

async fn ensure_parts_exist(&self, location: &StorageLocation) -> Result<(), ReadError> {
/// The sizes of a location's Parts in order, from one listing; `NotFound` when one is gone.
async fn part_sizes(&self, location: &StorageLocation) -> Result<Vec<u64>, ReadError> {
let folder = parts_folder(&location.folder_name);
let parts = self.adapter.list_folder(&folder).await?;
if parts.len() < usize::try_from(location.part_count).unwrap_or(0) {
return Err(ReadError::NotFound(folder));
}
Ok(())
(0..location.part_count)
.map(|index| {
parts
.get(&index.to_string())
.copied()
.ok_or_else(|| ReadError::NotFound(folder.clone()))
})
.collect()
}

async fn download_location(
Expand All @@ -378,12 +410,38 @@ impl Storage {
if let Some((object, _)) = whole_object(location) {
return self.adapter.download_stream(&object, range).await;
}
self.ensure_parts_exist(location).await?;
// No range: an unmerged entry is concatenated from its parts as it is read, so there is
// nothing to seek into. Served whole under a 200, which Range allows.
let part_sizes = self.part_sizes(location).await?;
let adapter = Arc::clone(&self.adapter);
let size = part_sizes
.iter()
.fold(0_u64, |total, &part| total.saturating_add(part));
let Some(range) = range.filter(|_| size > 0) else {
// An unmerged entry is concatenated from its Parts as it is read. Only an empty entry
// ignores a range, like an empty object does.
return Ok(DownloadStream {
stream: stream_parts(adapter, location, 0, 0),
extent: Extent::Unsized,
});
};
// Never the whole entry under a 200: the Azure SDK takes any 200 body as the bytes from
// the offset it asked for.
let served = clamp_range(range, size)
.ok_or_else(|| ReadError::RangeNotSatisfiable(parts_folder(&location.folder_name)))?;
// The Part the range starts in, and the offset into it.
let (mut first, mut skip) = (0_i32, served.start);
for &part in &part_sizes {
if skip < part {
break;
}
skip = skip.saturating_sub(part);
first = first.saturating_add(1);
}
Ok(DownloadStream {
stream: stream_parts(Arc::clone(&self.adapter), location),
extent: Extent::Unsized,
stream: take_bytes(stream_parts(adapter, location, first, skip), served.len()),
extent: Extent::Partial {
range: served,
size,
},
})
}

Expand Down Expand Up @@ -509,7 +567,7 @@ impl Storage {
fn merge_parts(&self, location: &StorageLocation) -> BoxFuture<'static, Result<()>> {
let adapter = Arc::clone(&self.adapter);
let merged = merged_object(&location.folder_name);
let part_stream = stream_parts(Arc::clone(&adapter), location);
let part_stream = stream_parts(Arc::clone(&adapter), location, 0, 0);
let kind = UploadKind::Merged {
length: location
.size_bytes
Expand Down Expand Up @@ -668,7 +726,7 @@ impl Storage {
return Err(self.abandon_upload(&upload, reason).await);
}
let size_bytes = parts
.iter()
.values()
.fold(0_u64, |total, bytes| total.saturating_add(*bytes));
let location = StorageLocation {
id: uuid::Uuid::new_v4().to_string(),
Expand Down Expand Up @@ -871,16 +929,17 @@ impl Storage {
location: &StorageLocation,
range: Option<RangeRequest>,
) -> Result<DownloadStream, ReadError> {
if whole_object(location).is_some() {
// A ranged read serves its slice from the Parts and leaves the Merge to a whole read.
if whole_object(location).is_some() || range.is_some() {
return self.download_location(location, range).await;
}
self.ensure_parts_exist(location).await?;
self.part_sizes(location).await?;

// Local Parts are cheap to read twice, so the client reads them at its own pace while
// the Merge copies them in the background.
if let Adapter::Filesystem(_) = self.adapter.as_ref() {
self.start_merge_detached(location, self.merge_parts(location));
return self.download_location(location, range).await;
return self.download_location(location, None).await;
}
// Object storage is read once and teed into the response and the Merge.
let (merger_sender, merger_receiver) = mpsc::channel(1);
Expand All @@ -900,11 +959,11 @@ impl Storage {
.boxed()
};
if !self.start_merge(location, write).await? {
return self.download_location(location, range).await;
return self.download_location(location, None).await;
}
let (response_sender, response_receiver) = mpsc::channel(1);
drop(tokio::spawn(pump_parts(
stream_parts(Arc::clone(&self.adapter), location),
stream_parts(Arc::clone(&self.adapter), location, 0, 0),
response_sender,
merger_sender,
)));
Expand Down
5 changes: 3 additions & 2 deletions src/storage/adapter.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! Storage backends behind one interface.

use std::collections::HashMap;
use std::io;

use axum::body::Bytes;
Expand Down Expand Up @@ -215,8 +216,8 @@ impl Adapter {
}
}

/// Sizes of the objects under a folder.
pub(crate) async fn list_folder(&self, folder: &str) -> Result<Vec<u64>> {
/// Sizes of the objects under a folder, by name relative to it.
pub(crate) async fn list_folder(&self, folder: &str) -> Result<HashMap<String, u64>> {
match self {
Self::Filesystem(adapter) => adapter.list_folder(folder).await,
Self::Object(adapter) => adapter.list_folder(folder).await,
Expand Down
10 changes: 6 additions & 4 deletions src/storage/filesystem.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! Storage on a local (or mounted) filesystem.

use std::collections::HashMap;
use std::io::{self, SeekFrom};
use std::path::{Component, Path, PathBuf};
use std::time::UNIX_EPOCH;
Expand Down Expand Up @@ -221,18 +222,19 @@ impl FilesystemAdapter {
})
}

pub(crate) async fn list_folder(&self, folder: &str) -> Result<Vec<u64>> {
pub(crate) async fn list_folder(&self, folder: &str) -> Result<HashMap<String, u64>> {
let mut entries = match fs::read_dir(self.safe_path(folder)?).await {
Ok(entries) => entries,
Err(error) if is_not_found(&error) => return Ok(Vec::new()),
Err(error) if is_not_found(&error) => return Ok(HashMap::new()),
Err(error) => return Err(error.into()),
};
let mut objects = Vec::new();
let mut objects = HashMap::new();
while let Some(entry) = entries.next_entry().await? {
if entry.file_type().await?.is_dir() {
continue;
}
objects.push(fs::metadata(entry.path()).await?.len());
let size = fs::metadata(entry.path()).await?.len();
objects.insert(entry.file_name().to_string_lossy().into_owned(), size);
}
Ok(objects)
}
Expand Down
11 changes: 8 additions & 3 deletions src/storage/object.rs
Original file line number Diff line number Diff line change
Expand Up @@ -476,11 +476,16 @@ impl ObjectAdapter {
self.delete_prefix(&self.path(folder)).await
}

pub(crate) async fn list_folder(&self, folder: &str) -> Result<Vec<u64>> {
pub(crate) async fn list_folder(&self, folder: &str) -> Result<HashMap<String, u64>> {
let folder = self.path(folder);
self.store
.list(Some(&self.path(folder)))
.list(Some(&folder))
.map_err(Failure::from)
.map_ok(|object| object.size)
.map_ok(|object| {
let name = object.location.as_ref().strip_prefix(folder.as_ref());
let name = name.unwrap_or_default().trim_start_matches('/');
(name.to_owned(), object.size)
})
.try_collect()
.await
}
Expand Down
Loading
Loading