Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions python/changelog.d/870.feat.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added `ArtifactVersion.file_sha256`. Artifact uploads now send the SHA-256 of the file, and each version returns it, so a caller can skip uploading an unchanged file.
1 change: 1 addition & 0 deletions python/changelog.d/870.fix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fixed uploads from an open file object that was already read, which sent only the bytes after its position, or an empty file. A file object now uploads whole, whatever its position.
31 changes: 29 additions & 2 deletions python/lib/sift_client/_internal/low_level_wrappers/upload.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
from __future__ import annotations

import hashlib
import logging
import mimetypes
from contextlib import nullcontext
from pathlib import Path
from typing import TYPE_CHECKING, Any, BinaryIO, ContextManager
from typing import TYPE_CHECKING, Any, BinaryIO, ContextManager, cast

from requests_toolbelt import MultipartEncoder

Expand All @@ -14,6 +15,8 @@
from sift_client.transport import WithRestClient

if TYPE_CHECKING:
import io

from sift_client.transport.rest_transport import RestClient

# Configure logging
Expand All @@ -23,6 +26,8 @@
mimetypes.add_type("application/vnd.apache.parquet", ".pqt")
mimetypes.add_type("application/vnd.apache.parquet", ".parquet")

_HASH_CHUNK_SIZE = 1024 * 1024


class UploadLowLevelClient(LowLevelClientBase, WithRestClient):
"""Low-level client for file upload operations.
Expand Down Expand Up @@ -70,18 +75,21 @@ async def upload_attachment(
description: str | None = None,
organization_id: str | None = None,
file_name: str | None = None,
sha256: bool = False,
) -> str:
"""Upload a file attachment to an entity.

Args:
source: Path to the file to upload, or an open binary file such as `io.BytesIO`.
source: Path to the file to upload, or an open seekable binary file such as
`io.BytesIO`. A file object is sent whole, from its start, whatever its position.
entity_id: The ID of the entity to attach the file to.
entity_type: The type of entity (e.g., "runs", "annotations", "annotation_logs").
metadata: Optional metadata for the file (e.g., video/image metadata).
description: Optional description of the file.
organization_id: Optional organization ID.
file_name: The stored file name when `source` is a file object; its extension sets
the MIME type. A path upload uses the file's own name.
sha256: Send the SHA-256 of the file with it.

Returns:
The remote file ID of the uploaded file.
Expand Down Expand Up @@ -115,6 +123,7 @@ async def upload_attachment(
metadata,
description,
organization_id,
sha256,
)

def _upload_file_sync(
Expand All @@ -128,6 +137,7 @@ def _upload_file_sync(
metadata: dict[str, Any] | None,
description: str | None,
organization_id: str | None,
sha256: bool,
) -> str:
"""Synchronous helper to upload the file.

Expand All @@ -139,6 +149,8 @@ def _upload_file_sync(
else:
opened = nullcontext(source)
with opened as file:
# MultipartEncoder sends an io.BytesIO whole but an open file from its position.
file.seek(0)
form_fields: dict[str, Any] = {
"entityId": entity_id,
"entityType": entity_type,
Expand Down Expand Up @@ -169,6 +181,9 @@ def _upload_file_sync(
if description:
form_fields["description"] = description

if sha256:
form_fields["sha256"] = _sha256(file)

form_data = MultipartEncoder(fields=form_fields)

# Use the RestClient to make the POST request
Expand Down Expand Up @@ -211,3 +226,15 @@ def _mime_and_content_type_from_path(path: Path) -> tuple[str, str | None, str |
file_name = path.name
mime, encoding = mimetypes.guess_type(path)
return file_name, mime, encoding


def _sha256(file: BinaryIO) -> str:
"""Hash `file`, which must be at its start, in chunks, then seek back to the start."""
digest = hashlib.sha256()
buffer = bytearray(_HASH_CHUNK_SIZE)
view = memoryview(buffer)
readinto = cast("io.BufferedIOBase", file).readinto
while size := readinto(buffer):
digest.update(view[:size])
file.seek(0)
return digest.hexdigest()
Original file line number Diff line number Diff line change
@@ -1,15 +1,53 @@
"""Tests for UploadLowLevelClient functionality."""

import hashlib
import io
from pathlib import Path
from typing import BinaryIO, cast
from unittest.mock import MagicMock, patch

import pytest
from requests import HTTPError, Request, Response
from requests_toolbelt import MultipartEncoder

from sift_client._internal.low_level_wrappers.upload import UploadLowLevelClient


def _posting_client() -> UploadLowLevelClient:
client = UploadLowLevelClient.__new__(UploadLowLevelClient)
client._rest_client = MagicMock()
client._rest_client.post.return_value.status_code = 200
client._rest_client.post.return_value.json.return_value = {
"remoteFile": {"remoteFileId": "rf1"}
}
return client


def _upload(
client: UploadLowLevelClient,
file: BinaryIO,
*,
sha256: bool = False,
entity_type: str = "artifact_versions",
) -> str:
return client._upload_file_sync(
file,
"x.bin",
"application/octet-stream",
None,
"v1",
entity_type,
None,
None,
None,
sha256,
)


def _posted_form(client: UploadLowLevelClient) -> MultipartEncoder:
return cast("MagicMock", client._rest_client.post).call_args.kwargs["data"]


class TestUploadAttachment:
@pytest.mark.asyncio
async def test_known_mime_type(self, tmp_path):
Expand Down Expand Up @@ -94,27 +132,12 @@ async def test_file_object_needs_file_name(self):
)

def test_posts_file_object_without_opening_a_path(self):
client = UploadLowLevelClient.__new__(UploadLowLevelClient)
client._rest_client = MagicMock()
client._rest_client.post.return_value.status_code = 200
client._rest_client.post.return_value.json.return_value = {
"remoteFile": {"remoteFileId": "rf1"}
}

remote_file_id = client._upload_file_sync(
io.BytesIO(b"a,b\n1,2\n"),
"t.csv",
"text/csv",
None,
"v1",
"artifact_versions",
None,
None,
None,
)
client = _posting_client()

remote_file_id = _upload(client, io.BytesIO(b"a,b\n1,2\n"))

assert remote_file_id == "rf1"
client._rest_client.post.assert_called_once()
cast("MagicMock", client._rest_client.post).assert_called_once()

def test_upload_error_includes_server_body(self):
response = Response()
Expand All @@ -127,14 +150,76 @@ def test_upload_error_includes_server_body(self):
client._rest_client.post.return_value = response

with pytest.raises(HTTPError, match="Server response: invalid entity_type"):
client._upload_file_sync(
io.BytesIO(b"x"),
"t.csv",
"text/csv",
None,
"v1",
"artifact_versions",
None,
None,
None,
_upload(client, io.BytesIO(b"x"))


class TestUploadSha256:
@pytest.mark.asyncio
async def test_path(self, tmp_path):
path = tmp_path / "table.csv"
path.write_bytes(b"a,b\n1,2\n")
client = _posting_client()

with patch(
"sift_client._internal.low_level_wrappers.upload.run_sync_function",
side_effect=lambda fn, *args: fn(*args),
):
await client.upload_attachment(
source=path, entity_id="v1", entity_type="artifact_versions", sha256=True
)

assert _posted_form(client).fields["sha256"] == hashlib.sha256(b"a,b\n1,2\n").hexdigest()

@pytest.mark.parametrize(
"data",
[b"", b"hello", bytes(range(256)) * 10_000],
ids=["empty", "small", "multi_chunk"],
)
def test_bytes(self, data):
client = _posting_client()

_upload(client, io.BytesIO(data), sha256=True)

assert _posted_form(client).fields["sha256"] == hashlib.sha256(data).hexdigest()

@pytest.mark.parametrize("sha256", [True, False], ids=["hashed", "not_hashed"])
@pytest.mark.parametrize(
"position", [len(b"header|"), len(b"header|payload")], ids=["middle", "end"]
)
@pytest.mark.parametrize(
"open_file",
[lambda path: open(path, "rb"), lambda path: io.BytesIO(path.read_bytes())],
ids=["file", "bytes_io"],
)
def test_file_object_sends_whole_file(self, tmp_path, open_file, position, sha256):
path = tmp_path / "data.bin"
path.write_bytes(b"header|payload")
client = _posting_client()

with open_file(path) as file:
file.seek(position)
_upload(client, file, sha256=sha256)
form = _posted_form(client)
body = form.to_string()

assert b"\r\n\r\nheader|payload\r\n" in body
if sha256:
assert form.fields["sha256"] == hashlib.sha256(b"header|payload").hexdigest()

def test_written_bytes_io_sends_whole_buffer(self):
buffer = io.BytesIO()
buffer.write(b"payload")
client = _posting_client()

_upload(client, buffer, sha256=True)

form = _posted_form(client)
assert form.fields["sha256"] == hashlib.sha256(b"payload").hexdigest()
assert b"\r\n\r\npayload\r\n" in form.to_string()

def test_off_by_default(self):
client = _posting_client()

_upload(client, io.BytesIO(b"x"), entity_type="runs")

assert "sha256" not in _posted_form(client).fields
14 changes: 12 additions & 2 deletions python/lib/sift_client/_tests/resources/test_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,11 @@ async def test_uploads_path_to_new_version(self, artifacts_api, tmp_path):
assert create.links == links
assert create.created_via is ArtifactCreatedVia.UPLOAD
artifacts_api._upload_client.upload_attachment.assert_awaited_once_with(
source=path, entity_id="v1", entity_type="artifact_versions", file_name=None
source=path,
entity_id="v1",
entity_type="artifact_versions",
file_name=None,
sha256=True,
)
artifacts_api._low_level_client.get_artifact.assert_awaited_once_with(
artifact_id="a1", artifact_version_id="v1"
Expand All @@ -254,6 +258,7 @@ async def test_uploads_bytes_under_file_name(self, artifacts_api):
assert kwargs["source"].read() == b"hello"
assert kwargs["file_name"] == "notes.txt"
assert kwargs["entity_id"] == "v1"
assert kwargs["sha256"] is True

@pytest.mark.asyncio
async def test_uploads_file_object(self, artifacts_api):
Expand All @@ -264,6 +269,7 @@ async def test_uploads_file_object(self, artifacts_api):
kwargs = artifacts_api._upload_client.upload_attachment.call_args.kwargs
assert kwargs["source"] is data
assert kwargs["file_name"] == "table.csv"
assert kwargs["sha256"] is True

@pytest.mark.asyncio
async def test_file_object_needs_file_name(self, artifacts_api):
Expand Down Expand Up @@ -322,7 +328,11 @@ async def test_new_file(self, artifacts_api, tmp_path):

assert artifacts_api._low_level_client.update_artifact.call_args.kwargs["new_file"] is True
artifacts_api._upload_client.upload_attachment.assert_awaited_once_with(
source=path, entity_id="v2", entity_type="artifact_versions", file_name=None
source=path,
entity_id="v2",
entity_type="artifact_versions",
file_name=None,
sha256=True,
)
artifacts_api._low_level_client.get_artifact.assert_awaited_once_with(
artifact_id="a1", artifact_version_id="v2"
Expand Down
3 changes: 3 additions & 0 deletions python/lib/sift_client/_tests/sift_types/test_artifact.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def _version_proto(**overrides) -> artifacts_pb2.ArtifactVersion:
created_date=to_timestamp_pb(NOW),
metadata=metadata_dict_to_proto({"campaign": "x"}),
is_current=True,
file_sha256="ab" * 32,
)
fields.update(overrides)
return artifacts_pb2.ArtifactVersion(**fields)
Expand Down Expand Up @@ -82,6 +83,7 @@ def test_from_proto(self):
assert version.summary is None
assert version.remote_file_id == "rf1"
assert version.file_name == "summary.png"
assert version.file_sha256 == "ab" * 32
assert version.metadata == {"campaign": "x"}
assert version.created_date == NOW
assert version.is_current is True
Expand All @@ -97,6 +99,7 @@ def test_unset_optionals_are_none(self):
assert version.remote_file_id is None
assert version.file_name is None
assert version.file_mime_type is None
assert version.file_sha256 is None
assert version.is_current is False


Expand Down
8 changes: 5 additions & 3 deletions python/lib/sift_client/resources/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,8 +393,8 @@ async def create(

Args:
create: The artifact's title, summary, metadata, links, and surface.
file: The file to upload: a path, bytes, or a binary file object such as
`io.BytesIO`.
file: The file to upload: a path, bytes, or a seekable binary file object such as
`io.BytesIO`. A file object is sent whole, whatever its position.
file_name: The stored file name when `file` is bytes or a file object. A path
upload uses the file's own name.

Expand Down Expand Up @@ -438,7 +438,8 @@ async def update(
Args:
artifact: The Artifact or artifact ID to update.
update: The fields to change.
file: New bytes for the new version: a path, bytes, or a binary file object.
file: New bytes for the new version: a path, bytes, or a seekable binary file
object. A file object is sent whole, whatever its position.
file_name: The stored file name when `file` is bytes or a file object. A path
upload uses the file's own name.

Expand Down Expand Up @@ -569,6 +570,7 @@ async def _upload_and_get(
entity_id=version_id,
entity_type=ARTIFACT_VERSION_ENTITY_TYPE,
file_name=file_name,
sha256=True,
)
return await self.get(artifact_id=artifact._id_or_error, artifact_version_id=version_id)

Expand Down
7 changes: 4 additions & 3 deletions python/lib/sift_client/resources/sync_stubs/__init__.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -574,8 +574,8 @@ class ArtifactsAPI:

Args:
create: The artifact's title, summary, metadata, links, and surface.
file: The file to upload: a path, bytes, or a binary file object such as
`io.BytesIO`.
file: The file to upload: a path, bytes, or a seekable binary file object such as
`io.BytesIO`. A file object is sent whole, whatever its position.
file_name: The stored file name when `file` is bytes or a file object. A path
upload uses the file's own name.

Expand Down Expand Up @@ -770,7 +770,8 @@ class ArtifactsAPI:
Args:
artifact: The Artifact or artifact ID to update.
update: The fields to change.
file: New bytes for the new version: a path, bytes, or a binary file object.
file: New bytes for the new version: a path, bytes, or a seekable binary file
object. A file object is sent whole, whatever its position.
file_name: The stored file name when `file` is bytes or a file object. A path
upload uses the file's own name.

Expand Down
Loading
Loading