diff --git a/python/changelog.d/870.feat.md b/python/changelog.d/870.feat.md new file mode 100644 index 000000000..a97f3697a --- /dev/null +++ b/python/changelog.d/870.feat.md @@ -0,0 +1 @@ +Added `ArtifactVersion.file_sha256`. Artifact uploads now send the SHA-256 of the file, and each version returns it, so a caller can skip uploading an unchanged file. diff --git a/python/changelog.d/870.fix.md b/python/changelog.d/870.fix.md new file mode 100644 index 000000000..2ebd2ac1d --- /dev/null +++ b/python/changelog.d/870.fix.md @@ -0,0 +1 @@ +Fixed uploads from an open file object that was already read, which sent only the bytes after its position, or an empty file. A file object now uploads whole, whatever its position. diff --git a/python/lib/sift_client/_internal/low_level_wrappers/upload.py b/python/lib/sift_client/_internal/low_level_wrappers/upload.py index 292f83ecd..2c176d75a 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/upload.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/upload.py @@ -1,10 +1,11 @@ from __future__ import annotations +import hashlib import logging import mimetypes from contextlib import nullcontext from pathlib import Path -from typing import TYPE_CHECKING, Any, BinaryIO, ContextManager +from typing import TYPE_CHECKING, Any, BinaryIO, ContextManager, cast from requests_toolbelt import MultipartEncoder @@ -14,6 +15,8 @@ from sift_client.transport import WithRestClient if TYPE_CHECKING: + import io + from sift_client.transport.rest_transport import RestClient # Configure logging @@ -23,6 +26,8 @@ mimetypes.add_type("application/vnd.apache.parquet", ".pqt") mimetypes.add_type("application/vnd.apache.parquet", ".parquet") +_HASH_CHUNK_SIZE = 1024 * 1024 + class UploadLowLevelClient(LowLevelClientBase, WithRestClient): """Low-level client for file upload operations. @@ -70,11 +75,13 @@ async def upload_attachment( description: str | None = None, organization_id: str | None = None, file_name: str | None = None, + sha256: bool = False, ) -> str: """Upload a file attachment to an entity. Args: - source: Path to the file to upload, or an open binary file such as `io.BytesIO`. + source: Path to the file to upload, or an open seekable binary file such as + `io.BytesIO`. A file object is sent whole, from its start, whatever its position. entity_id: The ID of the entity to attach the file to. entity_type: The type of entity (e.g., "runs", "annotations", "annotation_logs"). metadata: Optional metadata for the file (e.g., video/image metadata). @@ -82,6 +89,7 @@ async def upload_attachment( organization_id: Optional organization ID. file_name: The stored file name when `source` is a file object; its extension sets the MIME type. A path upload uses the file's own name. + sha256: Send the SHA-256 of the file with it. Returns: The remote file ID of the uploaded file. @@ -115,6 +123,7 @@ async def upload_attachment( metadata, description, organization_id, + sha256, ) def _upload_file_sync( @@ -128,6 +137,7 @@ def _upload_file_sync( metadata: dict[str, Any] | None, description: str | None, organization_id: str | None, + sha256: bool, ) -> str: """Synchronous helper to upload the file. @@ -139,6 +149,8 @@ def _upload_file_sync( else: opened = nullcontext(source) with opened as file: + # MultipartEncoder sends an io.BytesIO whole but an open file from its position. + file.seek(0) form_fields: dict[str, Any] = { "entityId": entity_id, "entityType": entity_type, @@ -169,6 +181,9 @@ def _upload_file_sync( if description: form_fields["description"] = description + if sha256: + form_fields["sha256"] = _sha256(file) + form_data = MultipartEncoder(fields=form_fields) # Use the RestClient to make the POST request @@ -211,3 +226,15 @@ def _mime_and_content_type_from_path(path: Path) -> tuple[str, str | None, str | file_name = path.name mime, encoding = mimetypes.guess_type(path) return file_name, mime, encoding + + +def _sha256(file: BinaryIO) -> str: + """Hash `file`, which must be at its start, in chunks, then seek back to the start.""" + digest = hashlib.sha256() + buffer = bytearray(_HASH_CHUNK_SIZE) + view = memoryview(buffer) + readinto = cast("io.BufferedIOBase", file).readinto + while size := readinto(buffer): + digest.update(view[:size]) + file.seek(0) + return digest.hexdigest() diff --git a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py index ce5ad2d44..6e366655b 100644 --- a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py +++ b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py @@ -1,15 +1,53 @@ """Tests for UploadLowLevelClient functionality.""" +import hashlib import io from pathlib import Path +from typing import BinaryIO, cast from unittest.mock import MagicMock, patch import pytest from requests import HTTPError, Request, Response +from requests_toolbelt import MultipartEncoder from sift_client._internal.low_level_wrappers.upload import UploadLowLevelClient +def _posting_client() -> UploadLowLevelClient: + client = UploadLowLevelClient.__new__(UploadLowLevelClient) + client._rest_client = MagicMock() + client._rest_client.post.return_value.status_code = 200 + client._rest_client.post.return_value.json.return_value = { + "remoteFile": {"remoteFileId": "rf1"} + } + return client + + +def _upload( + client: UploadLowLevelClient, + file: BinaryIO, + *, + sha256: bool = False, + entity_type: str = "artifact_versions", +) -> str: + return client._upload_file_sync( + file, + "x.bin", + "application/octet-stream", + None, + "v1", + entity_type, + None, + None, + None, + sha256, + ) + + +def _posted_form(client: UploadLowLevelClient) -> MultipartEncoder: + return cast("MagicMock", client._rest_client.post).call_args.kwargs["data"] + + class TestUploadAttachment: @pytest.mark.asyncio async def test_known_mime_type(self, tmp_path): @@ -94,27 +132,12 @@ async def test_file_object_needs_file_name(self): ) def test_posts_file_object_without_opening_a_path(self): - client = UploadLowLevelClient.__new__(UploadLowLevelClient) - client._rest_client = MagicMock() - client._rest_client.post.return_value.status_code = 200 - client._rest_client.post.return_value.json.return_value = { - "remoteFile": {"remoteFileId": "rf1"} - } - - remote_file_id = client._upload_file_sync( - io.BytesIO(b"a,b\n1,2\n"), - "t.csv", - "text/csv", - None, - "v1", - "artifact_versions", - None, - None, - None, - ) + client = _posting_client() + + remote_file_id = _upload(client, io.BytesIO(b"a,b\n1,2\n")) assert remote_file_id == "rf1" - client._rest_client.post.assert_called_once() + cast("MagicMock", client._rest_client.post).assert_called_once() def test_upload_error_includes_server_body(self): response = Response() @@ -127,14 +150,76 @@ def test_upload_error_includes_server_body(self): client._rest_client.post.return_value = response with pytest.raises(HTTPError, match="Server response: invalid entity_type"): - client._upload_file_sync( - io.BytesIO(b"x"), - "t.csv", - "text/csv", - None, - "v1", - "artifact_versions", - None, - None, - None, + _upload(client, io.BytesIO(b"x")) + + +class TestUploadSha256: + @pytest.mark.asyncio + async def test_path(self, tmp_path): + path = tmp_path / "table.csv" + path.write_bytes(b"a,b\n1,2\n") + client = _posting_client() + + with patch( + "sift_client._internal.low_level_wrappers.upload.run_sync_function", + side_effect=lambda fn, *args: fn(*args), + ): + await client.upload_attachment( + source=path, entity_id="v1", entity_type="artifact_versions", sha256=True ) + + assert _posted_form(client).fields["sha256"] == hashlib.sha256(b"a,b\n1,2\n").hexdigest() + + @pytest.mark.parametrize( + "data", + [b"", b"hello", bytes(range(256)) * 10_000], + ids=["empty", "small", "multi_chunk"], + ) + def test_bytes(self, data): + client = _posting_client() + + _upload(client, io.BytesIO(data), sha256=True) + + assert _posted_form(client).fields["sha256"] == hashlib.sha256(data).hexdigest() + + @pytest.mark.parametrize("sha256", [True, False], ids=["hashed", "not_hashed"]) + @pytest.mark.parametrize( + "position", [len(b"header|"), len(b"header|payload")], ids=["middle", "end"] + ) + @pytest.mark.parametrize( + "open_file", + [lambda path: open(path, "rb"), lambda path: io.BytesIO(path.read_bytes())], + ids=["file", "bytes_io"], + ) + def test_file_object_sends_whole_file(self, tmp_path, open_file, position, sha256): + path = tmp_path / "data.bin" + path.write_bytes(b"header|payload") + client = _posting_client() + + with open_file(path) as file: + file.seek(position) + _upload(client, file, sha256=sha256) + form = _posted_form(client) + body = form.to_string() + + assert b"\r\n\r\nheader|payload\r\n" in body + if sha256: + assert form.fields["sha256"] == hashlib.sha256(b"header|payload").hexdigest() + + def test_written_bytes_io_sends_whole_buffer(self): + buffer = io.BytesIO() + buffer.write(b"payload") + client = _posting_client() + + _upload(client, buffer, sha256=True) + + form = _posted_form(client) + assert form.fields["sha256"] == hashlib.sha256(b"payload").hexdigest() + assert b"\r\n\r\npayload\r\n" in form.to_string() + + def test_off_by_default(self): + client = _posting_client() + + _upload(client, io.BytesIO(b"x"), entity_type="runs") + + assert "sha256" not in _posted_form(client).fields diff --git a/python/lib/sift_client/_tests/resources/test_artifacts.py b/python/lib/sift_client/_tests/resources/test_artifacts.py index 717250d74..2583122d5 100644 --- a/python/lib/sift_client/_tests/resources/test_artifacts.py +++ b/python/lib/sift_client/_tests/resources/test_artifacts.py @@ -239,7 +239,11 @@ async def test_uploads_path_to_new_version(self, artifacts_api, tmp_path): assert create.links == links assert create.created_via is ArtifactCreatedVia.UPLOAD artifacts_api._upload_client.upload_attachment.assert_awaited_once_with( - source=path, entity_id="v1", entity_type="artifact_versions", file_name=None + source=path, + entity_id="v1", + entity_type="artifact_versions", + file_name=None, + sha256=True, ) artifacts_api._low_level_client.get_artifact.assert_awaited_once_with( artifact_id="a1", artifact_version_id="v1" @@ -254,6 +258,7 @@ async def test_uploads_bytes_under_file_name(self, artifacts_api): assert kwargs["source"].read() == b"hello" assert kwargs["file_name"] == "notes.txt" assert kwargs["entity_id"] == "v1" + assert kwargs["sha256"] is True @pytest.mark.asyncio async def test_uploads_file_object(self, artifacts_api): @@ -264,6 +269,7 @@ async def test_uploads_file_object(self, artifacts_api): kwargs = artifacts_api._upload_client.upload_attachment.call_args.kwargs assert kwargs["source"] is data assert kwargs["file_name"] == "table.csv" + assert kwargs["sha256"] is True @pytest.mark.asyncio async def test_file_object_needs_file_name(self, artifacts_api): @@ -322,7 +328,11 @@ async def test_new_file(self, artifacts_api, tmp_path): assert artifacts_api._low_level_client.update_artifact.call_args.kwargs["new_file"] is True artifacts_api._upload_client.upload_attachment.assert_awaited_once_with( - source=path, entity_id="v2", entity_type="artifact_versions", file_name=None + source=path, + entity_id="v2", + entity_type="artifact_versions", + file_name=None, + sha256=True, ) artifacts_api._low_level_client.get_artifact.assert_awaited_once_with( artifact_id="a1", artifact_version_id="v2" diff --git a/python/lib/sift_client/_tests/sift_types/test_artifact.py b/python/lib/sift_client/_tests/sift_types/test_artifact.py index 1728fbce6..457791ab3 100644 --- a/python/lib/sift_client/_tests/sift_types/test_artifact.py +++ b/python/lib/sift_client/_tests/sift_types/test_artifact.py @@ -38,6 +38,7 @@ def _version_proto(**overrides) -> artifacts_pb2.ArtifactVersion: created_date=to_timestamp_pb(NOW), metadata=metadata_dict_to_proto({"campaign": "x"}), is_current=True, + file_sha256="ab" * 32, ) fields.update(overrides) return artifacts_pb2.ArtifactVersion(**fields) @@ -82,6 +83,7 @@ def test_from_proto(self): assert version.summary is None assert version.remote_file_id == "rf1" assert version.file_name == "summary.png" + assert version.file_sha256 == "ab" * 32 assert version.metadata == {"campaign": "x"} assert version.created_date == NOW assert version.is_current is True @@ -97,6 +99,7 @@ def test_unset_optionals_are_none(self): assert version.remote_file_id is None assert version.file_name is None assert version.file_mime_type is None + assert version.file_sha256 is None assert version.is_current is False diff --git a/python/lib/sift_client/resources/artifacts.py b/python/lib/sift_client/resources/artifacts.py index 2b43cda32..1c5cfc218 100644 --- a/python/lib/sift_client/resources/artifacts.py +++ b/python/lib/sift_client/resources/artifacts.py @@ -393,8 +393,8 @@ async def create( Args: create: The artifact's title, summary, metadata, links, and surface. - file: The file to upload: a path, bytes, or a binary file object such as - `io.BytesIO`. + file: The file to upload: a path, bytes, or a seekable binary file object such as + `io.BytesIO`. A file object is sent whole, whatever its position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. @@ -438,7 +438,8 @@ async def update( Args: artifact: The Artifact or artifact ID to update. update: The fields to change. - file: New bytes for the new version: a path, bytes, or a binary file object. + file: New bytes for the new version: a path, bytes, or a seekable binary file + object. A file object is sent whole, whatever its position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. @@ -569,6 +570,7 @@ async def _upload_and_get( entity_id=version_id, entity_type=ARTIFACT_VERSION_ENTITY_TYPE, file_name=file_name, + sha256=True, ) return await self.get(artifact_id=artifact._id_or_error, artifact_version_id=version_id) diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 80094023a..6923f3908 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -574,8 +574,8 @@ class ArtifactsAPI: Args: create: The artifact's title, summary, metadata, links, and surface. - file: The file to upload: a path, bytes, or a binary file object such as - `io.BytesIO`. + file: The file to upload: a path, bytes, or a seekable binary file object such as + `io.BytesIO`. A file object is sent whole, whatever its position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. @@ -770,7 +770,8 @@ class ArtifactsAPI: Args: artifact: The Artifact or artifact ID to update. update: The fields to change. - file: New bytes for the new version: a path, bytes, or a binary file object. + file: New bytes for the new version: a path, bytes, or a seekable binary file + object. A file object is sent whole, whatever its position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. diff --git a/python/lib/sift_client/sift_types/artifact.py b/python/lib/sift_client/sift_types/artifact.py index ae9e176be..9fc7f631d 100644 --- a/python/lib/sift_client/sift_types/artifact.py +++ b/python/lib/sift_client/sift_types/artifact.py @@ -143,6 +143,8 @@ class ArtifactVersion(BaseType[ArtifactVersionProto, "ArtifactVersion"]): remote_file_id: str | None file_name: str | None file_mime_type: str | None + file_sha256: str | None + """Lowercase hex SHA-256 that the upload sent with the file. None if it sent none.""" metadata: dict[str, str | float | bool] created_date: datetime is_current: bool @@ -162,6 +164,7 @@ def _from_proto( remote_file_id=proto.remote_file_id if proto.HasField("remote_file_id") else None, file_name=proto.file_name if proto.HasField("file_name") else None, file_mime_type=proto.file_mime_type if proto.HasField("file_mime_type") else None, + file_sha256=proto.file_sha256 if proto.HasField("file_sha256") else None, metadata=metadata_proto_to_dict(list(proto.metadata)), created_date=proto.created_date.ToDatetime(tzinfo=timezone.utc), is_current=proto.is_current,