From fe380d6791ace9799efc0f0cddefc534951fb52b Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 4 Nov 2026 14:10:50 -0800 Subject: [PATCH 1/6] python(feat): send and expose the artifact file SHA-256 --- python/changelog.d/PRNUMBER.feat.md | 1 + python/changelog.d/PRNUMBER.fix.md | 1 + .../_internal/low_level_wrappers/upload.py | 37 +++++ .../low_level_wrappers/test_upload.py | 135 +++++++++++++++++- .../_tests/resources/test_artifacts.py | 14 +- .../_tests/sift_types/test_artifact.py | 3 + python/lib/sift_client/resources/artifacts.py | 4 +- python/lib/sift_client/sift_types/artifact.py | 3 + 8 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 python/changelog.d/PRNUMBER.feat.md create mode 100644 python/changelog.d/PRNUMBER.fix.md diff --git a/python/changelog.d/PRNUMBER.feat.md b/python/changelog.d/PRNUMBER.feat.md new file mode 100644 index 000000000..a97f3697a --- /dev/null +++ b/python/changelog.d/PRNUMBER.feat.md @@ -0,0 +1 @@ +Added `ArtifactVersion.file_sha256`. Artifact uploads now send the SHA-256 of the file, and each version returns it, so a caller can skip uploading an unchanged file. diff --git a/python/changelog.d/PRNUMBER.fix.md b/python/changelog.d/PRNUMBER.fix.md new file mode 100644 index 000000000..41eae8775 --- /dev/null +++ b/python/changelog.d/PRNUMBER.fix.md @@ -0,0 +1 @@ +Fixed file uploads from an `io.BytesIO` that is not at position 0, so they send the bytes from the current position, as uploads from an open file do. diff --git a/python/lib/sift_client/_internal/low_level_wrappers/upload.py b/python/lib/sift_client/_internal/low_level_wrappers/upload.py index 292f83ecd..3843e1fc0 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/upload.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/upload.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import io import logging import mimetypes from contextlib import nullcontext @@ -23,6 +25,8 @@ mimetypes.add_type("application/vnd.apache.parquet", ".pqt") mimetypes.add_type("application/vnd.apache.parquet", ".parquet") +_HASH_CHUNK_SIZE = 1024 * 1024 + class UploadLowLevelClient(LowLevelClientBase, WithRestClient): """Low-level client for file upload operations. @@ -70,6 +74,7 @@ async def upload_attachment( description: str | None = None, organization_id: str | None = None, file_name: str | None = None, + sha256: bool = False, ) -> str: """Upload a file attachment to an entity. @@ -82,6 +87,9 @@ async def upload_attachment( organization_id: Optional organization ID. file_name: The stored file name when `source` is a file object; its extension sets the MIME type. A path upload uses the file's own name. + sha256: Send the SHA-256 of the uploaded bytes with the file. A file object is + hashed from its current position. An unseekable file object is sent without a + hash. Returns: The remote file ID of the uploaded file. @@ -115,6 +123,7 @@ async def upload_attachment( metadata, description, organization_id, + sha256, ) def _upload_file_sync( @@ -128,6 +137,7 @@ def _upload_file_sync( metadata: dict[str, Any] | None, description: str | None, organization_id: str | None, + sha256: bool = False, ) -> str: """Synchronous helper to upload the file. @@ -139,6 +149,10 @@ def _upload_file_sync( else: opened = nullcontext(source) with opened as file: + if hasattr(file, "getvalue") and file.tell(): + # MultipartEncoder sends getvalue(), which ignores the position. + file = io.BytesIO(file.read()) + form_fields: dict[str, Any] = { "entityId": entity_id, "entityType": entity_type, @@ -169,6 +183,11 @@ def _upload_file_sync( if description: form_fields["description"] = description + if sha256: + digest = _sha256_from_position(file) + if digest: + form_fields["sha256"] = digest + form_data = MultipartEncoder(fields=form_fields) # Use the RestClient to make the POST request @@ -211,3 +230,21 @@ def _mime_and_content_type_from_path(path: Path) -> tuple[str, str | None, str | file_name = path.name mime, encoding = mimetypes.guess_type(path) return file_name, mime, encoding + + +def _sha256_from_position(file: BinaryIO) -> str | None: + """Hash `file` from its current position in chunks, then seek back to that position. + + Returns None when the file cannot seek back, so the upload still reads every byte. + """ + try: + if not file.seekable(): + return None + start = file.tell() + except (AttributeError, OSError): + return None + digest = hashlib.sha256() + for chunk in iter(lambda: file.read(_HASH_CHUNK_SIZE), b""): + digest.update(chunk) + file.seek(start) + return digest.hexdigest() diff --git a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py index ce5ad2d44..1081a3562 100644 --- a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py +++ b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py @@ -1,13 +1,19 @@ """Tests for UploadLowLevelClient functionality.""" +import hashlib import io from pathlib import Path +from typing import cast from unittest.mock import MagicMock, patch import pytest from requests import HTTPError, Request, Response +from requests_toolbelt import MultipartEncoder -from sift_client._internal.low_level_wrappers.upload import UploadLowLevelClient +from sift_client._internal.low_level_wrappers.upload import ( + UploadLowLevelClient, + _sha256_from_position, +) class TestUploadAttachment: @@ -138,3 +144,130 @@ def test_upload_error_includes_server_body(self): None, None, ) + + +def _posting_client() -> UploadLowLevelClient: + client = UploadLowLevelClient.__new__(UploadLowLevelClient) + client._rest_client = MagicMock() + client._rest_client.post.return_value.status_code = 200 + client._rest_client.post.return_value.json.return_value = { + "remoteFile": {"remoteFileId": "rf1"} + } + return client + + +def _posted_form(client: UploadLowLevelClient) -> MultipartEncoder: + return cast("MagicMock", client._rest_client.post).call_args.kwargs["data"] + + +def _posted_fields(client: UploadLowLevelClient) -> dict: + return _posted_form(client).fields + + +def _posted_body(client: UploadLowLevelClient) -> bytes: + return _posted_form(client).to_string() + + +class _Unseekable(io.RawIOBase): + def __init__(self, data: bytes): + self._data = io.BytesIO(data) + + def readable(self) -> bool: + return True + + def readinto(self, buffer) -> int: + chunk = self._data.read(len(buffer)) + buffer[: len(chunk)] = chunk + return len(chunk) + + +class TestUploadSha256: + @pytest.mark.asyncio + async def test_path(self, tmp_path): + path = tmp_path / "table.csv" + path.write_bytes(b"a,b\n1,2\n") + client = _posting_client() + + with patch( + "sift_client._internal.low_level_wrappers.upload.run_sync_function", + side_effect=lambda fn, *args: fn(*args), + ): + await client.upload_attachment( + source=path, entity_id="v1", entity_type="artifact_versions", sha256=True + ) + + assert _posted_fields(client)["sha256"] == hashlib.sha256(b"a,b\n1,2\n").hexdigest() + + @pytest.mark.parametrize( + "data", + [b"", b"hello", bytes(range(256)) * 10_000], + ids=["empty", "small", "multi_chunk"], + ) + def test_bytes(self, data): + client = _posting_client() + + client._upload_file_sync( + io.BytesIO(data), + "x.bin", + "application/octet-stream", + None, + "v1", + "artifact_versions", + None, + None, + None, + True, + ) + + digest = _posted_fields(client)["sha256"] + assert digest == hashlib.sha256(data).hexdigest() + assert digest == digest.lower() + + @pytest.mark.parametrize("kind", ["file", "bytes_io"]) + def test_file_object_hashes_from_its_position(self, tmp_path, kind): + path = tmp_path / "data.bin" + path.write_bytes(b"header|payload") + client = _posting_client() + + with open(path, "rb") if kind == "file" else io.BytesIO(b"header|payload") as file: + file.seek(len(b"header|")) + client._upload_file_sync( + file, + "payload.bin", + "application/octet-stream", + None, + "v1", + "artifact_versions", + None, + None, + None, + True, + ) + assert _posted_fields(client)["sha256"] == hashlib.sha256(b"payload").hexdigest() + body = _posted_body(client) + + assert b"\r\n\r\npayload\r\n" in body + assert b"header|" not in body + + def test_unseekable_file_object_is_not_hashed(self): + stream = io.BufferedReader(_Unseekable(b"stream")) + + assert _sha256_from_position(stream) is None + assert stream.read() == b"stream" + + def test_off_by_default(self): + client = _posting_client() + + client._upload_file_sync( + io.BytesIO(b"x"), + "x.bin", + "application/octet-stream", + None, + "v1", + "runs", + None, + None, + None, + ) + + assert "sha256" not in _posted_fields(client) diff --git a/python/lib/sift_client/_tests/resources/test_artifacts.py b/python/lib/sift_client/_tests/resources/test_artifacts.py index 717250d74..2583122d5 100644 --- a/python/lib/sift_client/_tests/resources/test_artifacts.py +++ b/python/lib/sift_client/_tests/resources/test_artifacts.py @@ -239,7 +239,11 @@ async def test_uploads_path_to_new_version(self, artifacts_api, tmp_path): assert create.links == links assert create.created_via is ArtifactCreatedVia.UPLOAD artifacts_api._upload_client.upload_attachment.assert_awaited_once_with( - source=path, entity_id="v1", entity_type="artifact_versions", file_name=None + source=path, + entity_id="v1", + entity_type="artifact_versions", + file_name=None, + sha256=True, ) artifacts_api._low_level_client.get_artifact.assert_awaited_once_with( artifact_id="a1", artifact_version_id="v1" @@ -254,6 +258,7 @@ async def test_uploads_bytes_under_file_name(self, artifacts_api): assert kwargs["source"].read() == b"hello" assert kwargs["file_name"] == "notes.txt" assert kwargs["entity_id"] == "v1" + assert kwargs["sha256"] is True @pytest.mark.asyncio async def test_uploads_file_object(self, artifacts_api): @@ -264,6 +269,7 @@ async def test_uploads_file_object(self, artifacts_api): kwargs = artifacts_api._upload_client.upload_attachment.call_args.kwargs assert kwargs["source"] is data assert kwargs["file_name"] == "table.csv" + assert kwargs["sha256"] is True @pytest.mark.asyncio async def test_file_object_needs_file_name(self, artifacts_api): @@ -322,7 +328,11 @@ async def test_new_file(self, artifacts_api, tmp_path): assert artifacts_api._low_level_client.update_artifact.call_args.kwargs["new_file"] is True artifacts_api._upload_client.upload_attachment.assert_awaited_once_with( - source=path, entity_id="v2", entity_type="artifact_versions", file_name=None + source=path, + entity_id="v2", + entity_type="artifact_versions", + file_name=None, + sha256=True, ) artifacts_api._low_level_client.get_artifact.assert_awaited_once_with( artifact_id="a1", artifact_version_id="v2" diff --git a/python/lib/sift_client/_tests/sift_types/test_artifact.py b/python/lib/sift_client/_tests/sift_types/test_artifact.py index 1728fbce6..457791ab3 100644 --- a/python/lib/sift_client/_tests/sift_types/test_artifact.py +++ b/python/lib/sift_client/_tests/sift_types/test_artifact.py @@ -38,6 +38,7 @@ def _version_proto(**overrides) -> artifacts_pb2.ArtifactVersion: created_date=to_timestamp_pb(NOW), metadata=metadata_dict_to_proto({"campaign": "x"}), is_current=True, + file_sha256="ab" * 32, ) fields.update(overrides) return artifacts_pb2.ArtifactVersion(**fields) @@ -82,6 +83,7 @@ def test_from_proto(self): assert version.summary is None assert version.remote_file_id == "rf1" assert version.file_name == "summary.png" + assert version.file_sha256 == "ab" * 32 assert version.metadata == {"campaign": "x"} assert version.created_date == NOW assert version.is_current is True @@ -97,6 +99,7 @@ def test_unset_optionals_are_none(self): assert version.remote_file_id is None assert version.file_name is None assert version.file_mime_type is None + assert version.file_sha256 is None assert version.is_current is False diff --git a/python/lib/sift_client/resources/artifacts.py b/python/lib/sift_client/resources/artifacts.py index 2b43cda32..3eaeeca1e 100644 --- a/python/lib/sift_client/resources/artifacts.py +++ b/python/lib/sift_client/resources/artifacts.py @@ -561,7 +561,8 @@ async def _upload_and_get( ) -> Artifact: """Upload the file to the version `artifact` resolves to, then fetch that version. - The version becomes the artifact's current version once the upload commits. + The upload sends the file's SHA-256, which the version returns as `file_sha256`. The + version becomes the artifact's current version once the upload commits. """ version_id = artifact.artifact_version._id_or_error await self._upload_client.upload_attachment( @@ -569,6 +570,7 @@ async def _upload_and_get( entity_id=version_id, entity_type=ARTIFACT_VERSION_ENTITY_TYPE, file_name=file_name, + sha256=True, ) return await self.get(artifact_id=artifact._id_or_error, artifact_version_id=version_id) diff --git a/python/lib/sift_client/sift_types/artifact.py b/python/lib/sift_client/sift_types/artifact.py index ae9e176be..73797105e 100644 --- a/python/lib/sift_client/sift_types/artifact.py +++ b/python/lib/sift_client/sift_types/artifact.py @@ -143,6 +143,8 @@ class ArtifactVersion(BaseType[ArtifactVersionProto, "ArtifactVersion"]): remote_file_id: str | None file_name: str | None file_mime_type: str | None + file_sha256: str | None = None + """Lowercase hex SHA-256 that the upload sent with the file. None if it sent none.""" metadata: dict[str, str | float | bool] created_date: datetime is_current: bool @@ -162,6 +164,7 @@ def _from_proto( remote_file_id=proto.remote_file_id if proto.HasField("remote_file_id") else None, file_name=proto.file_name if proto.HasField("file_name") else None, file_mime_type=proto.file_mime_type if proto.HasField("file_mime_type") else None, + file_sha256=proto.file_sha256 if proto.HasField("file_sha256") else None, metadata=metadata_proto_to_dict(list(proto.metadata)), created_date=proto.created_date.ToDatetime(tzinfo=timezone.utc), is_current=proto.is_current, From a19112ca1449352a83c3730e0a55871c67844035 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 4 Nov 2026 14:32:04 -0800 Subject: [PATCH 2/6] name the changelog fragments after the PR --- python/changelog.d/{PRNUMBER.feat.md => 870.feat.md} | 0 python/changelog.d/{PRNUMBER.fix.md => 870.fix.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/changelog.d/{PRNUMBER.feat.md => 870.feat.md} (100%) rename python/changelog.d/{PRNUMBER.fix.md => 870.fix.md} (100%) diff --git a/python/changelog.d/PRNUMBER.feat.md b/python/changelog.d/870.feat.md similarity index 100% rename from python/changelog.d/PRNUMBER.feat.md rename to python/changelog.d/870.feat.md diff --git a/python/changelog.d/PRNUMBER.fix.md b/python/changelog.d/870.fix.md similarity index 100% rename from python/changelog.d/PRNUMBER.fix.md rename to python/changelog.d/870.fix.md From 8bc2761a8a1ebd913317a3bb9e71505feffbbc12 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 4 Nov 2026 16:23:15 -0800 Subject: [PATCH 3/6] send uploads from the file position without copying, and simplify the hash tests --- .../_internal/low_level_wrappers/upload.py | 55 +++--- .../low_level_wrappers/test_upload.py | 183 ++++++------------ python/lib/sift_client/resources/artifacts.py | 8 +- python/lib/sift_client/sift_types/artifact.py | 2 +- 4 files changed, 97 insertions(+), 151 deletions(-) diff --git a/python/lib/sift_client/_internal/low_level_wrappers/upload.py b/python/lib/sift_client/_internal/low_level_wrappers/upload.py index 3843e1fc0..ffb8f462b 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/upload.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/upload.py @@ -6,7 +6,7 @@ import mimetypes from contextlib import nullcontext from pathlib import Path -from typing import TYPE_CHECKING, Any, BinaryIO, ContextManager +from typing import TYPE_CHECKING, Any, BinaryIO, ContextManager, cast from requests_toolbelt import MultipartEncoder @@ -88,8 +88,7 @@ async def upload_attachment( file_name: The stored file name when `source` is a file object; its extension sets the MIME type. A path upload uses the file's own name. sha256: Send the SHA-256 of the uploaded bytes with the file. A file object is - hashed from its current position. An unseekable file object is sent without a - hash. + hashed from its current position. Returns: The remote file ID of the uploaded file. @@ -137,7 +136,7 @@ def _upload_file_sync( metadata: dict[str, Any] | None, description: str | None, organization_id: str | None, - sha256: bool = False, + sha256: bool, ) -> str: """Synchronous helper to upload the file. @@ -149,10 +148,6 @@ def _upload_file_sync( else: opened = nullcontext(source) with opened as file: - if hasattr(file, "getvalue") and file.tell(): - # MultipartEncoder sends getvalue(), which ignores the position. - file = io.BytesIO(file.read()) - form_fields: dict[str, Any] = { "entityId": entity_id, "entityType": entity_type, @@ -161,14 +156,14 @@ def _upload_file_sync( if content_encoding: form_fields["file"] = ( file_name, - file, + _FromPosition(file), mimetype, { "Content-Encoding": content_encoding, }, ) else: - form_fields["file"] = (file_name, file, mimetype) + form_fields["file"] = (file_name, _FromPosition(file), mimetype) if metadata: import json @@ -184,9 +179,7 @@ def _upload_file_sync( form_fields["description"] = description if sha256: - digest = _sha256_from_position(file) - if digest: - form_fields["sha256"] = digest + form_fields["sha256"] = _sha256_from_position(file) form_data = MultipartEncoder(fields=form_fields) @@ -232,19 +225,35 @@ def _mime_and_content_type_from_path(path: Path) -> tuple[str, str | None, str | return file_name, mime, encoding -def _sha256_from_position(file: BinaryIO) -> str | None: - """Hash `file` from its current position in chunks, then seek back to that position. +class _FromPosition: + """Sends a file object from its current position. - Returns None when the file cannot seek back, so the upload still reads every byte. + MultipartEncoder sends all of `getvalue()` for an `io.BytesIO`, ignoring the position. It + sends an object with `read` and `len` as is, and reads until `len` is 0. """ - try: - if not file.seekable(): - return None + + def __init__(self, file: BinaryIO): + self._file = file start = file.tell() - except (AttributeError, OSError): - return None + self._end = file.seek(0, io.SEEK_END) + file.seek(start) + + @property + def len(self) -> int: + return self._end - self._file.tell() + + def read(self, size: int = -1) -> bytes: + return self._file.read(size) + + +def _sha256_from_position(file: BinaryIO) -> str: + """Hash `file` from its current position in chunks, then seek back to that position.""" + start = file.tell() digest = hashlib.sha256() - for chunk in iter(lambda: file.read(_HASH_CHUNK_SIZE), b""): - digest.update(chunk) + buffer = bytearray(_HASH_CHUNK_SIZE) + view = memoryview(buffer) + readinto = cast("io.BufferedIOBase", file).readinto + while size := readinto(buffer): + digest.update(view[:size]) file.seek(start) return digest.hexdigest() diff --git a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py index 1081a3562..e663a4b89 100644 --- a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py +++ b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py @@ -3,17 +3,49 @@ import hashlib import io from pathlib import Path -from typing import cast +from typing import BinaryIO, cast from unittest.mock import MagicMock, patch import pytest from requests import HTTPError, Request, Response from requests_toolbelt import MultipartEncoder -from sift_client._internal.low_level_wrappers.upload import ( - UploadLowLevelClient, - _sha256_from_position, -) +from sift_client._internal.low_level_wrappers.upload import UploadLowLevelClient + + +def _posting_client() -> UploadLowLevelClient: + client = UploadLowLevelClient.__new__(UploadLowLevelClient) + client._rest_client = MagicMock() + client._rest_client.post.return_value.status_code = 200 + client._rest_client.post.return_value.json.return_value = { + "remoteFile": {"remoteFileId": "rf1"} + } + return client + + +def _upload( + client: UploadLowLevelClient, + file: BinaryIO, + *, + sha256: bool = False, + entity_type: str = "artifact_versions", +) -> str: + return client._upload_file_sync( + file, + "x.bin", + "application/octet-stream", + None, + "v1", + entity_type, + None, + None, + None, + sha256, + ) + + +def _posted_form(client: UploadLowLevelClient) -> MultipartEncoder: + return cast("MagicMock", client._rest_client.post).call_args.kwargs["data"] class TestUploadAttachment: @@ -100,24 +132,9 @@ async def test_file_object_needs_file_name(self): ) def test_posts_file_object_without_opening_a_path(self): - client = UploadLowLevelClient.__new__(UploadLowLevelClient) - client._rest_client = MagicMock() - client._rest_client.post.return_value.status_code = 200 - client._rest_client.post.return_value.json.return_value = { - "remoteFile": {"remoteFileId": "rf1"} - } - - remote_file_id = client._upload_file_sync( - io.BytesIO(b"a,b\n1,2\n"), - "t.csv", - "text/csv", - None, - "v1", - "artifact_versions", - None, - None, - None, - ) + client = _posting_client() + + remote_file_id = _upload(client, io.BytesIO(b"a,b\n1,2\n")) assert remote_file_id == "rf1" client._rest_client.post.assert_called_once() @@ -133,52 +150,7 @@ def test_upload_error_includes_server_body(self): client._rest_client.post.return_value = response with pytest.raises(HTTPError, match="Server response: invalid entity_type"): - client._upload_file_sync( - io.BytesIO(b"x"), - "t.csv", - "text/csv", - None, - "v1", - "artifact_versions", - None, - None, - None, - ) - - -def _posting_client() -> UploadLowLevelClient: - client = UploadLowLevelClient.__new__(UploadLowLevelClient) - client._rest_client = MagicMock() - client._rest_client.post.return_value.status_code = 200 - client._rest_client.post.return_value.json.return_value = { - "remoteFile": {"remoteFileId": "rf1"} - } - return client - - -def _posted_form(client: UploadLowLevelClient) -> MultipartEncoder: - return cast("MagicMock", client._rest_client.post).call_args.kwargs["data"] - - -def _posted_fields(client: UploadLowLevelClient) -> dict: - return _posted_form(client).fields - - -def _posted_body(client: UploadLowLevelClient) -> bytes: - return _posted_form(client).to_string() - - -class _Unseekable(io.RawIOBase): - def __init__(self, data: bytes): - self._data = io.BytesIO(data) - - def readable(self) -> bool: - return True - - def readinto(self, buffer) -> int: - chunk = self._data.read(len(buffer)) - buffer[: len(chunk)] = chunk - return len(chunk) + _upload(client, io.BytesIO(b"x")) class TestUploadSha256: @@ -196,7 +168,7 @@ async def test_path(self, tmp_path): source=path, entity_id="v1", entity_type="artifact_versions", sha256=True ) - assert _posted_fields(client)["sha256"] == hashlib.sha256(b"a,b\n1,2\n").hexdigest() + assert _posted_form(client).fields["sha256"] == hashlib.sha256(b"a,b\n1,2\n").hexdigest() @pytest.mark.parametrize( "data", @@ -206,68 +178,33 @@ async def test_path(self, tmp_path): def test_bytes(self, data): client = _posting_client() - client._upload_file_sync( - io.BytesIO(data), - "x.bin", - "application/octet-stream", - None, - "v1", - "artifact_versions", - None, - None, - None, - True, - ) - - digest = _posted_fields(client)["sha256"] - assert digest == hashlib.sha256(data).hexdigest() - assert digest == digest.lower() - - @pytest.mark.parametrize("kind", ["file", "bytes_io"]) - def test_file_object_hashes_from_its_position(self, tmp_path, kind): + _upload(client, io.BytesIO(data), sha256=True) + + assert _posted_form(client).fields["sha256"] == hashlib.sha256(data).hexdigest() + + @pytest.mark.parametrize( + "open_file", + [lambda path: open(path, "rb"), lambda path: io.BytesIO(path.read_bytes())], + ids=["file", "bytes_io"], + ) + def test_file_object_hashes_from_its_position(self, tmp_path, open_file): path = tmp_path / "data.bin" path.write_bytes(b"header|payload") client = _posting_client() - with open(path, "rb") if kind == "file" else io.BytesIO(b"header|payload") as file: + with open_file(path) as file: file.seek(len(b"header|")) - client._upload_file_sync( - file, - "payload.bin", - "application/octet-stream", - None, - "v1", - "artifact_versions", - None, - None, - None, - True, - ) - assert _posted_fields(client)["sha256"] == hashlib.sha256(b"payload").hexdigest() - body = _posted_body(client) + _upload(client, file, sha256=True) + form = _posted_form(client) + assert form.fields["sha256"] == hashlib.sha256(b"payload").hexdigest() + body = form.to_string() assert b"\r\n\r\npayload\r\n" in body assert b"header|" not in body - def test_unseekable_file_object_is_not_hashed(self): - stream = io.BufferedReader(_Unseekable(b"stream")) - - assert _sha256_from_position(stream) is None - assert stream.read() == b"stream" - def test_off_by_default(self): client = _posting_client() - client._upload_file_sync( - io.BytesIO(b"x"), - "x.bin", - "application/octet-stream", - None, - "v1", - "runs", - None, - None, - None, - ) - - assert "sha256" not in _posted_fields(client) + _upload(client, io.BytesIO(b"x"), entity_type="runs") + + assert "sha256" not in _posted_form(client).fields diff --git a/python/lib/sift_client/resources/artifacts.py b/python/lib/sift_client/resources/artifacts.py index 3eaeeca1e..a87e2e5ed 100644 --- a/python/lib/sift_client/resources/artifacts.py +++ b/python/lib/sift_client/resources/artifacts.py @@ -394,7 +394,7 @@ async def create( Args: create: The artifact's title, summary, metadata, links, and surface. file: The file to upload: a path, bytes, or a binary file object such as - `io.BytesIO`. + `io.BytesIO`, sent from its current position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. @@ -438,7 +438,8 @@ async def update( Args: artifact: The Artifact or artifact ID to update. update: The fields to change. - file: New bytes for the new version: a path, bytes, or a binary file object. + file: New bytes for the new version: a path, bytes, or a binary file object, sent + from its current position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. @@ -561,8 +562,7 @@ async def _upload_and_get( ) -> Artifact: """Upload the file to the version `artifact` resolves to, then fetch that version. - The upload sends the file's SHA-256, which the version returns as `file_sha256`. The - version becomes the artifact's current version once the upload commits. + The version becomes the artifact's current version once the upload commits. """ version_id = artifact.artifact_version._id_or_error await self._upload_client.upload_attachment( diff --git a/python/lib/sift_client/sift_types/artifact.py b/python/lib/sift_client/sift_types/artifact.py index 73797105e..9fc7f631d 100644 --- a/python/lib/sift_client/sift_types/artifact.py +++ b/python/lib/sift_client/sift_types/artifact.py @@ -143,7 +143,7 @@ class ArtifactVersion(BaseType[ArtifactVersionProto, "ArtifactVersion"]): remote_file_id: str | None file_name: str | None file_mime_type: str | None - file_sha256: str | None = None + file_sha256: str | None """Lowercase hex SHA-256 that the upload sent with the file. None if it sent none.""" metadata: dict[str, str | float | bool] created_date: datetime From 36700a64a4de6d3b7045a52f27e9ec97b1e26a02 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 4 Nov 2026 16:33:43 -0800 Subject: [PATCH 4/6] cast the upload mock for pyright --- .../_tests/_internal/low_level_wrappers/test_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py index e663a4b89..149de1cec 100644 --- a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py +++ b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py @@ -137,7 +137,7 @@ def test_posts_file_object_without_opening_a_path(self): remote_file_id = _upload(client, io.BytesIO(b"a,b\n1,2\n")) assert remote_file_id == "rf1" - client._rest_client.post.assert_called_once() + cast("MagicMock", client._rest_client.post).assert_called_once() def test_upload_error_includes_server_body(self): response = Response() From 7849162b374bd013c887797b8b54cd18815a1581 Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Wed, 4 Nov 2026 16:43:22 -0800 Subject: [PATCH 5/6] regenerate the sync stubs --- python/lib/sift_client/resources/sync_stubs/__init__.pyi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 80094023a..4e71cc2bb 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -575,7 +575,7 @@ class ArtifactsAPI: Args: create: The artifact's title, summary, metadata, links, and surface. file: The file to upload: a path, bytes, or a binary file object such as - `io.BytesIO`. + `io.BytesIO`, sent from its current position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. @@ -770,7 +770,8 @@ class ArtifactsAPI: Args: artifact: The Artifact or artifact ID to update. update: The fields to change. - file: New bytes for the new version: a path, bytes, or a binary file object. + file: New bytes for the new version: a path, bytes, or a binary file object, sent + from its current position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. From 826aa89d8486a86c288b42fff3369c52cebd730d Mon Sep 17 00:00:00 2001 From: Alex Luck Date: Thu, 5 Nov 2026 16:59:59 -0800 Subject: [PATCH 6/6] upload a file object whole, whatever its position --- python/changelog.d/870.fix.md | 2 +- .../_internal/low_level_wrappers/upload.py | 45 ++++++------------- .../low_level_wrappers/test_upload.py | 27 ++++++++--- python/lib/sift_client/resources/artifacts.py | 8 ++-- .../resources/sync_stubs/__init__.pyi | 8 ++-- 5 files changed, 43 insertions(+), 47 deletions(-) diff --git a/python/changelog.d/870.fix.md b/python/changelog.d/870.fix.md index 41eae8775..2ebd2ac1d 100644 --- a/python/changelog.d/870.fix.md +++ b/python/changelog.d/870.fix.md @@ -1 +1 @@ -Fixed file uploads from an `io.BytesIO` that is not at position 0, so they send the bytes from the current position, as uploads from an open file do. +Fixed uploads from an open file object that was already read, which sent only the bytes after its position, or an empty file. A file object now uploads whole, whatever its position. diff --git a/python/lib/sift_client/_internal/low_level_wrappers/upload.py b/python/lib/sift_client/_internal/low_level_wrappers/upload.py index ffb8f462b..2c176d75a 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/upload.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/upload.py @@ -1,7 +1,6 @@ from __future__ import annotations import hashlib -import io import logging import mimetypes from contextlib import nullcontext @@ -16,6 +15,8 @@ from sift_client.transport import WithRestClient if TYPE_CHECKING: + import io + from sift_client.transport.rest_transport import RestClient # Configure logging @@ -79,7 +80,8 @@ async def upload_attachment( """Upload a file attachment to an entity. Args: - source: Path to the file to upload, or an open binary file such as `io.BytesIO`. + source: Path to the file to upload, or an open seekable binary file such as + `io.BytesIO`. A file object is sent whole, from its start, whatever its position. entity_id: The ID of the entity to attach the file to. entity_type: The type of entity (e.g., "runs", "annotations", "annotation_logs"). metadata: Optional metadata for the file (e.g., video/image metadata). @@ -87,8 +89,7 @@ async def upload_attachment( organization_id: Optional organization ID. file_name: The stored file name when `source` is a file object; its extension sets the MIME type. A path upload uses the file's own name. - sha256: Send the SHA-256 of the uploaded bytes with the file. A file object is - hashed from its current position. + sha256: Send the SHA-256 of the file with it. Returns: The remote file ID of the uploaded file. @@ -148,6 +149,8 @@ def _upload_file_sync( else: opened = nullcontext(source) with opened as file: + # MultipartEncoder sends an io.BytesIO whole but an open file from its position. + file.seek(0) form_fields: dict[str, Any] = { "entityId": entity_id, "entityType": entity_type, @@ -156,14 +159,14 @@ def _upload_file_sync( if content_encoding: form_fields["file"] = ( file_name, - _FromPosition(file), + file, mimetype, { "Content-Encoding": content_encoding, }, ) else: - form_fields["file"] = (file_name, _FromPosition(file), mimetype) + form_fields["file"] = (file_name, file, mimetype) if metadata: import json @@ -179,7 +182,7 @@ def _upload_file_sync( form_fields["description"] = description if sha256: - form_fields["sha256"] = _sha256_from_position(file) + form_fields["sha256"] = _sha256(file) form_data = MultipartEncoder(fields=form_fields) @@ -225,35 +228,13 @@ def _mime_and_content_type_from_path(path: Path) -> tuple[str, str | None, str | return file_name, mime, encoding -class _FromPosition: - """Sends a file object from its current position. - - MultipartEncoder sends all of `getvalue()` for an `io.BytesIO`, ignoring the position. It - sends an object with `read` and `len` as is, and reads until `len` is 0. - """ - - def __init__(self, file: BinaryIO): - self._file = file - start = file.tell() - self._end = file.seek(0, io.SEEK_END) - file.seek(start) - - @property - def len(self) -> int: - return self._end - self._file.tell() - - def read(self, size: int = -1) -> bytes: - return self._file.read(size) - - -def _sha256_from_position(file: BinaryIO) -> str: - """Hash `file` from its current position in chunks, then seek back to that position.""" - start = file.tell() +def _sha256(file: BinaryIO) -> str: + """Hash `file`, which must be at its start, in chunks, then seek back to the start.""" digest = hashlib.sha256() buffer = bytearray(_HASH_CHUNK_SIZE) view = memoryview(buffer) readinto = cast("io.BufferedIOBase", file).readinto while size := readinto(buffer): digest.update(view[:size]) - file.seek(start) + file.seek(0) return digest.hexdigest() diff --git a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py index 149de1cec..6e366655b 100644 --- a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py +++ b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_upload.py @@ -182,25 +182,40 @@ def test_bytes(self, data): assert _posted_form(client).fields["sha256"] == hashlib.sha256(data).hexdigest() + @pytest.mark.parametrize("sha256", [True, False], ids=["hashed", "not_hashed"]) + @pytest.mark.parametrize( + "position", [len(b"header|"), len(b"header|payload")], ids=["middle", "end"] + ) @pytest.mark.parametrize( "open_file", [lambda path: open(path, "rb"), lambda path: io.BytesIO(path.read_bytes())], ids=["file", "bytes_io"], ) - def test_file_object_hashes_from_its_position(self, tmp_path, open_file): + def test_file_object_sends_whole_file(self, tmp_path, open_file, position, sha256): path = tmp_path / "data.bin" path.write_bytes(b"header|payload") client = _posting_client() with open_file(path) as file: - file.seek(len(b"header|")) - _upload(client, file, sha256=True) + file.seek(position) + _upload(client, file, sha256=sha256) form = _posted_form(client) - assert form.fields["sha256"] == hashlib.sha256(b"payload").hexdigest() body = form.to_string() - assert b"\r\n\r\npayload\r\n" in body - assert b"header|" not in body + assert b"\r\n\r\nheader|payload\r\n" in body + if sha256: + assert form.fields["sha256"] == hashlib.sha256(b"header|payload").hexdigest() + + def test_written_bytes_io_sends_whole_buffer(self): + buffer = io.BytesIO() + buffer.write(b"payload") + client = _posting_client() + + _upload(client, buffer, sha256=True) + + form = _posted_form(client) + assert form.fields["sha256"] == hashlib.sha256(b"payload").hexdigest() + assert b"\r\n\r\npayload\r\n" in form.to_string() def test_off_by_default(self): client = _posting_client() diff --git a/python/lib/sift_client/resources/artifacts.py b/python/lib/sift_client/resources/artifacts.py index a87e2e5ed..1c5cfc218 100644 --- a/python/lib/sift_client/resources/artifacts.py +++ b/python/lib/sift_client/resources/artifacts.py @@ -393,8 +393,8 @@ async def create( Args: create: The artifact's title, summary, metadata, links, and surface. - file: The file to upload: a path, bytes, or a binary file object such as - `io.BytesIO`, sent from its current position. + file: The file to upload: a path, bytes, or a seekable binary file object such as + `io.BytesIO`. A file object is sent whole, whatever its position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. @@ -438,8 +438,8 @@ async def update( Args: artifact: The Artifact or artifact ID to update. update: The fields to change. - file: New bytes for the new version: a path, bytes, or a binary file object, sent - from its current position. + file: New bytes for the new version: a path, bytes, or a seekable binary file + object. A file object is sent whole, whatever its position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 4e71cc2bb..6923f3908 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -574,8 +574,8 @@ class ArtifactsAPI: Args: create: The artifact's title, summary, metadata, links, and surface. - file: The file to upload: a path, bytes, or a binary file object such as - `io.BytesIO`, sent from its current position. + file: The file to upload: a path, bytes, or a seekable binary file object such as + `io.BytesIO`. A file object is sent whole, whatever its position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name. @@ -770,8 +770,8 @@ class ArtifactsAPI: Args: artifact: The Artifact or artifact ID to update. update: The fields to change. - file: New bytes for the new version: a path, bytes, or a binary file object, sent - from its current position. + file: New bytes for the new version: a path, bytes, or a seekable binary file + object. A file object is sent whole, whatever its position. file_name: The stored file name when `file` is bytes or a file object. A path upload uses the file's own name.