From 6c37f9f8bdebd9b9eb1d79e4c498fdf35e586990 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Mon, 24 Aug 2026 23:58:09 -0700 Subject: [PATCH 01/28] fix: Stream issues --- pr312-followups.md | 108 +++++++++++++++++++++++++++++++++++++++++++++ src/c2pa/c2pa.py | 23 ++++++++-- 2 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 pr312-followups.md diff --git a/pr312-followups.md b/pr312-followups.md new file mode 100644 index 00000000..9dceafee --- /dev/null +++ b/pr312-followups.md @@ -0,0 +1,108 @@ +# c2pa-python pull request 312: additional fixes + +Sentinel in the native thread-local error slot. Scratch note, not for the +repository. + +The change is correct as written. Three items, the first of which quietly +disables the diagnostic the sentinel exists to provide. + +--- + +## 1. Match on a substring, not on exact equality + +**The problem.** The comparison is + +```python +if error == ManagedResource._NO_NATIVE_ERROR.decode('utf-8'): +``` + +`c2pa_error_set_last` does not store the string verbatim. It runs it through +`Error::from` and then `CimplError::from`, and its own documentation states that +a missing or invalid error type is replaced with `Other` and the message includes +the original string. The sentinel already carries an `Other: ` prefix, so it may +round-trip unchanged, or it may come back re-prefixed or otherwise normalised. + +**Why it matters, and why it is easy to miss.** The failure is silent rather than +dangerous. If the comparison never matches, control falls through to the final +branch, which now performs the identical `_teardown(free_handle=False)`. Same +action, no crash, all tests that check behaviour still pass. + +What is lost is the distinct log line. That log line is the entire reason for +planting a sentinel rather than simply clearing the slot: it separates "the +native side reported nothing" from "the native side reported a real error", and +it is the only field evidence available for how often the ambiguous case occurs. +Losing it costs nothing today and costs the whole diagnostic tomorrow. + +**Fix.** Match on the distinctive part only: + +```python +_NO_NATIVE_ERROR_MARKER = "c2pa-python-no-native-error" +... +if _NO_NATIVE_ERROR_MARKER in error: +``` + +**And pin the round-trip in a test regardless**, since it is a property of the +native side that can change without notice: + +```python +def test_sentinel_round_trips_through_native_error_slot(self): + c2pa_module._lib.c2pa_error_set_last(ManagedResource._NO_NATIVE_ERROR) + self.assertIn(_NO_NATIVE_ERROR_MARKER, c2pa_module._read_native_error()) +``` + +That test fails loudly if the normalisation ever changes, which is exactly the +kind of upstream invariant worth pinning rather than assuming. + +## 2. Restore the ordering rationale that was deleted + +The removed paragraph explained that `c2pa_free` on a handle the registry no +longer tracks returns minus one and overwrites the slot with its own +untracked-pointer message, so the error must be read before any free or the +substitute carries a pre-consume tag and inverts the retain decision. + +That constraint is still true, and the current code still depends on it. The +replacement text explains the sentinel but says nothing about why the read comes +first. A later edit that moves the read after a free would reintroduce the +inversion with no warning anywhere in the file. + +One sentence is enough: + +> The read must precede any free: `c2pa_free` on an untracked handle overwrites +> the slot with its own untracked-pointer message, which carries a pre-consume +> tag and would invert the decision below. + +## 3. Confirm the changed test is green for the right reason + +`test_context_build_null_return_frees_builder` loses its explicit +`c2pa_error_set_last(b"UntrackedPointer: ...")` line. That test needs the +retained branch to fire, which needs a pre-consume tag present at the moment the +failure is read. + +With the sentinel now planted inside `_invoke_consume`, a mock that merely +returns `None` leaves the sentinel in place, the sentinel branch fires, +`_teardown(free_handle=False)` runs, and no free happens. The assertion should +then fail. + +Presumably the mock is now built with `_fail_with_native_error(b"UntrackedPointer: ...")`, +which restores the tag from inside the call rather than before it. Worth +confirming that is what landed, because a test that passes for the wrong reason +here is worse than one that fails: it would be asserting the retained branch +while actually exercising the consumed one. + +--- + +## What already holds + +The sentinel is planted immediately before `ffi_call`, inside `_invoke_consume`, +with nothing between them, on the thread that makes the call. That is the correct +placement and the thread-local slot means it cannot disturb any other worker. + +`_setup_function(_lib.c2pa_error_set_last, [ctypes.c_char_p], ctypes.c_int)` +supplies the explicit argument and return types, which was the one open check. +The return value itself needs no guard: minus one is returned only for a null +pointer, so any non-null sentinel returns zero. + +Changing the final fallback from `_release_handle()` to +`_teardown(free_handle=False)` removes the guarded free from the ambiguous path +entirely. That is the more important half of this pull request, and it holds even +if the sentinel comparison in item 1 never matches. diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 5f3dfa61..4ef1ee6d 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -2507,9 +2507,7 @@ def __init__( else: # format_or_path is a format string, stream is a stream object - with Stream(stream) as stream_obj: - self._create_reader( - format_bytes, stream_obj, manifest_data) + self._init_from_stream(stream, format_bytes, manifest_data) @staticmethod def _resolve_format_bytes(format_or_path, stream) -> Optional[bytes]: @@ -2580,6 +2578,25 @@ def _init_from_file(self, path, format_bytes, raise C2paError.Io( Reader._ERROR_MESSAGES['io_error'].format(str(e))) + def _init_from_stream(self, stream, format_bytes, + manifest_data=None): + """Create a reader from a caller-supplied stream object. + The native reader reads through this stream for as long as it is + alive, so the wrapper is stored on the instance and released by + _release(). + + Args: + stream: A stream-like object owned by the caller + format_bytes: UTF-8 encoded format/MIME type + manifest_data: Optional manifest bytes + """ + try: + self._own_stream = Stream(stream) + self._create_reader(format_bytes, self._own_stream, manifest_data) + except Exception: + self._close_streams() + raise + def _init_from_context(self, context, format_or_path, stream, manifest_data=None): """Initialize Reader from a Context object implementing From a41ddadf3407bc23189b7f1f06715b9854990a6b Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:25:38 -0700 Subject: [PATCH 02/28] fix: Lock managed resource --- src/c2pa/c2pa.py | 232 ++++++++++------- tests/test_unit_tests_threaded.py | 411 ++++++++++++++++++++++++++++++ 2 files changed, 553 insertions(+), 90 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 4ef1ee6d..0255e6db 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -19,6 +19,7 @@ import logging import sys import os +import threading import warnings import weakref from abc import ABC, abstractmethod @@ -264,8 +265,37 @@ def _init_attrs(self): def __init__(self): self._lifecycle_state = LifecycleState.UNINITIALIZED self._handle = None + self._op_lock = threading.RLock() record_owner_pid(self) + def _lock(self): + """Return this resource's operation lock. + + Reentrant because CPython can run a finalizer at any bytecode + boundary, including inside a region this thread has already locked, + and because a consuming call tears the handle down from inside the + locked region (_invoke_consume, _raise_consume_failure). + + Falls back to a fresh lock when the attribute is missing: an object + whose __init__ raised before the assignment is still finalized, and + __del__ must not raise. + + Never hold this across a native call that drives stream callbacks + (construction, resource_to_stream, the Builder stream methods, + signing). Those calls release the GIL and re-enter caller-supplied + Python, which may call back into this API on another thread; holding + the lock across them deadlocks. Only calls that touch no callbacks + are serialized here, and no path holds two of these locks at once. + """ + lock = getattr(self, '_op_lock', None) + if lock is None: + lock = threading.RLock() + try: + self._op_lock = lock + except Exception: + pass + return lock + @staticmethod def _free_native_ptr(ptr): """Free a native pointer by passing it to c2pa_free. @@ -321,22 +351,26 @@ def _safe_release(self): def _teardown(self, free_handle: bool): """Close the object: run _release, optionally free the handle, null it. free_handle=False (consumed) frees nothing, the new owner needs to free. + + Holds the operation lock so the free cannot land between another + thread's state check and its use of the handle in a native call. """ - if is_foreign_process(self): - self._handle = None - self._lifecycle_state = LifecycleState.CLOSED - return + with self._lock(): + if is_foreign_process(self): + self._handle = None + self._lifecycle_state = LifecycleState.CLOSED + return - self._lifecycle_state = LifecycleState.CLOSED - self._safe_release() + self._lifecycle_state = LifecycleState.CLOSED + self._safe_release() - handle, self._handle = self._handle, None - if free_handle and handle: - try: - ManagedResource._free_native_ptr(handle) - except Exception: - logger.error("Failed to free native %s resources", - type(self).__name__, exc_info=True) + handle, self._handle = self._handle, None + if free_handle and handle: + try: + ManagedResource._free_native_ptr(handle) + except Exception: + logger.error("Failed to free native %s resources", + type(self).__name__, exc_info=True) def _release_handle(self): """Free this handle, then close the object. Used only where ownership is @@ -1548,16 +1582,17 @@ def set(self, path: str, value: str) -> 'Settings': Returns: self, for method chaining. """ - self._ensure_valid_state() - path_bytes = _to_utf8_bytes(path, "settings path") value_bytes = _to_utf8_bytes(value, "settings value") - _check_ffi_operation_result( - _lib.c2pa_settings_set_value( - self._handle, path_bytes, value_bytes), - "Failed to set settings value", - check=lambda r: r != 0) + with self._lock(): + self._ensure_valid_state() + + _check_ffi_operation_result( + _lib.c2pa_settings_set_value( + self._handle, path_bytes, value_bytes), + "Failed to set settings value", + check=lambda r: r != 0) return self @@ -1574,15 +1609,16 @@ def update( Returns: self, for method chaining. """ - self._ensure_valid_state() - data_bytes = _to_utf8_bytes(data, "settings data") - _check_ffi_operation_result( - _lib.c2pa_settings_update_from_string( - self._handle, data_bytes, b"json"), - "Failed to update settings", - check=lambda r: r != 0) + with self._lock(): + self._ensure_valid_state() + + _check_ffi_operation_result( + _lib.c2pa_settings_update_from_string( + self._handle, data_bytes, b"json"), + "Failed to update settings", + check=lambda r: r != 0) return self @@ -2784,19 +2820,24 @@ def json(self) -> str: C2paError: If there was an error getting the JSON """ - self._ensure_valid_state() + # The state check and the handle read are one critical section: a + # finalizer on another thread frees the handle while it is still + # non-null, so a check made outside the lock says nothing about the + # handle this call goes on to pass to native code. + with self._lock(): + self._ensure_valid_state() - # Return cached result if available - if self._manifest_json_str_cache is not None: - return self._manifest_json_str_cache + # Return cached result if available + if self._manifest_json_str_cache is not None: + return self._manifest_json_str_cache - result = _lib.c2pa_reader_json(self._handle) - _check_ffi_operation_result(result, - "Error during manifest parsing in Reader") + result = _lib.c2pa_reader_json(self._handle) + _check_ffi_operation_result( + result, "Error during manifest parsing in Reader") - # Cache the result and return it - self._manifest_json_str_cache = _convert_to_py_string(result) - return self._manifest_json_str_cache + # Cache the result and return it + self._manifest_json_str_cache = _convert_to_py_string(result) + return self._manifest_json_str_cache def detailed_json(self) -> str: """Get the detailed JSON representation of the C2PA manifest store. @@ -2814,13 +2855,14 @@ def detailed_json(self) -> str: the Reader has been closed. """ - self._ensure_valid_state() + with self._lock(): + self._ensure_valid_state() - result = _lib.c2pa_reader_detailed_json(self._handle) - _check_ffi_operation_result( - result, "Error during detailed manifest parsing in Reader") + result = _lib.c2pa_reader_detailed_json(self._handle) + _check_ffi_operation_result( + result, "Error during detailed manifest parsing in Reader") - return _convert_to_py_string(result) + return _convert_to_py_string(result) def crjson(self) -> str: """Get the manifest store as a crJSON string. @@ -2836,12 +2878,13 @@ def crjson(self) -> str: call returns null. """ - self._ensure_valid_state() + with self._lock(): + self._ensure_valid_state() - result = _lib.c2pa_reader_crjson(self._handle) - _check_ffi_operation_result(result, "Error parsing crJSON") + result = _lib.c2pa_reader_crjson(self._handle) + _check_ffi_operation_result(result, "Error parsing crJSON") - return _convert_to_py_string(result) + return _convert_to_py_string(result) def _get_manifest_field(self, extractor): """Extract a field from (cached) manifest data, or None if unavailable. @@ -2981,11 +3024,12 @@ def is_embedded(self) -> bool: Raises: C2paError: If there was an error checking the embedded status """ - self._ensure_valid_state() + with self._lock(): + self._ensure_valid_state() - result = _lib.c2pa_reader_is_embedded(self._handle) + result = _lib.c2pa_reader_is_embedded(self._handle) - return bool(result) + return bool(result) def get_remote_url(self) -> Optional[str]: """Get the remote URL of the manifest if it was obtained remotely. @@ -2998,17 +3042,18 @@ def get_remote_url(self) -> Optional[str]: Raises: C2paError: If there was an error getting the remote URL """ - self._ensure_valid_state() + with self._lock(): + self._ensure_valid_state() - result = _lib.c2pa_reader_remote_url(self._handle) + result = _lib.c2pa_reader_remote_url(self._handle) - if result is None: - # No remote URL set (manifest is embedded) - return None + if result is None: + # No remote URL set (manifest is embedded) + return None - # Convert the C string to Python string - url_str = _convert_to_py_string(result) - return url_str + # Convert the C string to Python string + url_str = _convert_to_py_string(result) + return url_str class Signer(ManagedResource): @@ -3226,16 +3271,17 @@ def reserve_size(self) -> int: Raises: C2paError: If there was an error getting the size """ - self._ensure_valid_state() + with self._lock(): + self._ensure_valid_state() - result = _lib.c2pa_signer_reserve_size(self._handle) + result = _lib.c2pa_signer_reserve_size(self._handle) - _check_ffi_operation_result( - result, - "Failed to get reserve size", - check=lambda r: r < 0) + _check_ffi_operation_result( + result, + "Failed to get reserve size", + check=lambda r: r < 0) - return result + return result class Builder(ManagedResource): @@ -3433,8 +3479,9 @@ def set_no_embed(self): into the asset when signing. This is useful when creating cloud or sidecar manifests. """ - self._ensure_valid_state() - _lib.c2pa_builder_set_no_embed(self._handle) + with self._lock(): + self._ensure_valid_state() + _lib.c2pa_builder_set_no_embed(self._handle) def set_remote_url(self, remote_url: str): """Set the remote URL. @@ -3448,15 +3495,17 @@ def set_remote_url(self, remote_url: str): Raises: C2paError: If there was an error setting the remote URL """ - self._ensure_valid_state() - url_bytes = _to_utf8_bytes(remote_url, "remote URL") - result = _lib.c2pa_builder_set_remote_url(self._handle, url_bytes) - _check_ffi_operation_result( - result, - Builder._ERROR_MESSAGES['url_error'], - check=lambda r: r != 0) + with self._lock(): + self._ensure_valid_state() + + result = _lib.c2pa_builder_set_remote_url(self._handle, url_bytes) + + _check_ffi_operation_result( + result, + Builder._ERROR_MESSAGES['url_error'], + check=lambda r: r != 0) def set_intent( self, @@ -3484,18 +3533,19 @@ def set_intent( Raises: C2paError: If there was an error setting the intent """ - self._ensure_valid_state() + with self._lock(): + self._ensure_valid_state() - result = _lib.c2pa_builder_set_intent( - self._handle, - ctypes.c_uint(intent), - ctypes.c_uint(digital_source_type), - ) + result = _lib.c2pa_builder_set_intent( + self._handle, + ctypes.c_uint(intent), + ctypes.c_uint(digital_source_type), + ) - _check_ffi_operation_result( - result, - Builder._ERROR_MESSAGES['intent_error'], - check=lambda r: r != 0) + _check_ffi_operation_result( + result, + Builder._ERROR_MESSAGES['intent_error'], + check=lambda r: r != 0) def add_resource(self, uri: str, stream: Any): """Add a resource to the builder. @@ -3599,15 +3649,17 @@ def add_action(self, action_json: Union[str, dict]) -> None: C2paError: If there was an error adding the action C2paError.Encoding: If the action JSON contains invalid UTF-8 chars """ - self._ensure_valid_state() - action_str = _to_utf8_bytes(action_json, "action JSON") - result = _lib.c2pa_builder_add_action(self._handle, action_str) - _check_ffi_operation_result( - result, - Builder._ERROR_MESSAGES['action_error'], - check=lambda r: r != 0) + with self._lock(): + self._ensure_valid_state() + + result = _lib.c2pa_builder_add_action(self._handle, action_str) + + _check_ffi_operation_result( + result, + Builder._ERROR_MESSAGES['action_error'], + check=lambda r: r != 0) def to_archive(self, stream: Any) -> None: """Write an archive of the builder to a stream. diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index d0d0b2c1..f2b2a643 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -16,6 +16,9 @@ import os import io import json +import subprocess +import sys +import textwrap import unittest import threading import concurrent.futures @@ -3035,5 +3038,413 @@ def build_context_and_builder(): self.assertEqual(settings._owner_pid, pid) +class TestManagedResourceLockDeadlock(unittest.TestCase): + """Tests for the operation lock that serializes native calls against + teardown. + + Every join here is bounded: a deadlock must fail the test, not hang the + suite. + """ + + JOIN_TIMEOUT = 30 + + def _join_all(self, threads, what): + for thread in threads: + thread.join(self.JOIN_TIMEOUT) + stuck = [t for t in threads if t.is_alive()] + self.assertEqual( + stuck, [], + "{} did not finish within {}s: deadlock".format( + what, self.JOIN_TIMEOUT)) + + def _run_isolated(self, body, timeout=180): + """Run body in a subprocess and return it. + + A segfault kills the interpreter, so a crash cannot be asserted on + in-process: it would take the test runner with it. + """ + source = textwrap.dedent(body) + return subprocess.run( + [sys.executable, "-c", source], + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + capture_output=True, + timeout=timeout, + ) + + def test_json_racing_finalizer_does_not_crash(self): + """Readers used on one thread while others are collected. + + Without the lock this segfaults inside c2pa_reader_json: the + finalizer frees the handle between the state check and the call. + """ + result = self._run_isolated(""" + import sys, io, gc, random, threading, time + sys.path.insert(0, "src") + from c2pa import Reader + + data = open("tests/fixtures/C.jpg", "rb").read() + stop = threading.Event() + pool, lock = [], threading.Lock() + + def worker(): + while not stop.is_set(): + choice = random.random() + try: + if choice < 0.40: + reader = Reader("image/jpeg", io.BytesIO(data)) + with lock: + pool.append(reader) + elif choice < 0.75: + with lock: + snapshot = list(pool) + if snapshot: + reader = random.choice(snapshot) + reader._manifest_json_str_cache = None + reader.json() + elif choice < 0.90: + with lock: + reader = pool.pop(0) if pool else None + if reader: + reader.close() + else: + with lock: + if len(pool) > 20: + del pool[0:5] + gc.collect() + except Exception: + pass + + threads = [threading.Thread(target=worker) for _ in range(12)] + for thread in threads: + thread.start() + deadline = time.time() + 10 + while time.time() < deadline: + time.sleep(0.05) + stop.set() + for thread in threads: + thread.join(30) + """) + self.assertEqual( + result.returncode, 0, + "reader churn crashed with {} " + "(139=SIGSEGV, 134=SIGABRT): {}".format( + result.returncode, result.stderr.decode()[-800:])) + + def test_finalizer_inside_locked_operation(self): + """A finalizer can run at any bytecode boundary, including inside a + region this same thread has locked. A non-reentrant lock deadlocks + here; RLock does not. + """ + resource = _ConcreteResource() + resource._activate(0x51000) + observed = [] + + class Dropped: + def __del__(self): + # Runs on this thread, inside the locked region below. + with resource._lock(): + observed.append(True) + + def body(): + with resource._lock(): + dropped = Dropped() + del dropped + gc.collect() + + thread = threading.Thread(target=body) + thread.start() + self._join_all([thread], "finalizer inside locked region") + self.assertEqual(observed, [True], + "finalizer did not re-enter the lock") + resource.close() + + def test_close_racing_json_does_not_deadlock(self): + """close() on one thread against json() on another.""" + data = open(DEFAULT_TEST_FILE, 'rb').read() + errors = [] + + def rounds(): + try: + for _ in range(40): + reader = Reader("image/jpeg", io.BytesIO(data)) + closer = threading.Thread(target=reader.close) + closer.start() + try: + reader._manifest_json_str_cache = None + reader.json() + except Error: + pass + closer.join(self.JOIN_TIMEOUT) + if closer.is_alive(): + errors.append("closer stuck") + return + except Exception as exc: + errors.append(repr(exc)) + + threads = [threading.Thread(target=rounds) for _ in range(4)] + for thread in threads: + thread.start() + self._join_all(threads, "close/json race") + self.assertEqual(errors, []) + + def test_context_manager_exit_racing_json_does_not_deadlock(self): + """__exit__ closes while another thread is calling json().""" + data = open(DEFAULT_TEST_FILE, 'rb').read() + errors = [] + + def body(): + try: + for _ in range(40): + reader = Reader("image/jpeg", io.BytesIO(data)) + + def use(): + for _ in range(5): + try: + reader._manifest_json_str_cache = None + reader.json() + except Error: + pass + + user = threading.Thread(target=use) + user.start() + with reader: + pass + user.join(self.JOIN_TIMEOUT) + if user.is_alive(): + errors.append("user stuck") + return + except Exception as exc: + errors.append(repr(exc)) + + thread = threading.Thread(target=body) + thread.start() + self._join_all([thread], "__exit__/json race") + self.assertEqual(errors, []) + + def test_consume_failure_teardown_does_not_deadlock(self): + """A failing consuming call tears the handle down from inside the + operation, re-entering the lock on the same thread. + + with_fragment on a JPEG returns NotSupported, which routes through + _raise_consume_failure. + """ + data = open(DEFAULT_TEST_FILE, 'rb').read() + errors = [] + + def body(): + try: + for _ in range(20): + reader = Reader("image/jpeg", io.BytesIO(data)) + try: + reader.with_fragment( + "image/jpeg", io.BytesIO(data), io.BytesIO(data)) + except Error: + pass + reader.close() + except Exception as exc: + errors.append(repr(exc)) + + thread = threading.Thread(target=body) + thread.start() + self._join_all([thread], "consume-failure teardown") + self.assertEqual(errors, []) + + def test_close_during_sign_does_not_deadlock(self): + """_sign_internal calls self.close() inside its own try block, so + signing re-enters the lock on the signing thread. + """ + certs = open(os.path.join(FIXTURES_FOLDER, + "es256_certs.pem"), 'rb').read() + key = open(os.path.join(FIXTURES_FOLDER, + "es256_private.key"), 'rb').read() + data = open(DEFAULT_TEST_FILE, 'rb').read() + signer_info = C2paSignerInfo( + alg=b"es256", + sign_cert=certs, + private_key=key, + ta_url=b"http://timestamp.digicert.com", + ) + manifest = { + "claim_generator": "python_test", + "claim_generator_info": [ + {"name": "python_test", "version": "0.0.1"}], + "format": "image/jpeg", + "assertions": [], + } + errors = [] + + def body(): + try: + for _ in range(3): + signer = Signer.from_info(signer_info) + builder = Builder(manifest) + builder.sign(signer, "image/jpeg", + io.BytesIO(data), io.BytesIO()) + except Exception as exc: + errors.append(repr(exc)) + + threads = [threading.Thread(target=body) for _ in range(4)] + for thread in threads: + thread.start() + self._join_all(threads, "sign with internal close") + self.assertEqual(errors, []) + + def test_stream_callback_reentering_api_does_not_deadlock(self): + """Construction drives caller-supplied stream callbacks, and a caller + may legitimately call back into the API from one. + + This passes only because construction does not hold the lock. + """ + data = open(DEFAULT_TEST_FILE, 'rb').read() + other = Reader("image/jpeg", io.BytesIO(data)) + errors = [] + + class ReentrantStream(io.BytesIO): + def readinto(self, buffer): + try: + other.json() + except Exception: + pass + return super().readinto(buffer) + + def body(): + try: + for _ in range(10): + Reader("image/jpeg", ReentrantStream(data)) + except Exception as exc: + errors.append(repr(exc)) + + thread = threading.Thread(target=body) + thread.start() + self._join_all([thread], "callback re-entering API") + self.assertEqual(errors, []) + other.close() + + def test_stream_callback_blocking_on_other_thread_does_not_deadlock(self): + """The adversarial case: a stream callback that blocks on another + thread which touches the same object. + + A lock held across construction deadlocks here, whether it is global + or per-object. This is the test that pins the scoping decision. + """ + data = open(DEFAULT_TEST_FILE, 'rb').read() + target = Reader("image/jpeg", io.BytesIO(data)) + errors = [] + + class BlockingStream(io.BytesIO): + def readinto(self, buffer): + def use(): + try: + target._manifest_json_str_cache = None + target.json() + except Exception: + pass + + helper = threading.Thread(target=use) + helper.start() + helper.join(10) + if helper.is_alive(): + errors.append("helper stuck inside stream callback") + return super().readinto(buffer) + + def body(): + try: + for _ in range(5): + Reader("image/jpeg", BlockingStream(data)) + except Exception as exc: + errors.append(repr(exc)) + + thread = threading.Thread(target=body) + thread.start() + self._join_all([thread], "callback blocking on another thread") + self.assertEqual(errors, []) + target.close() + + def test_no_nested_op_locks(self): + """No code path may hold two resources' operation locks at once. + + That property, not the tests above, is what makes the design + deadlock-free: with only one lock ever held, no cycle can form. + """ + data = open(DEFAULT_TEST_FILE, 'rb').read() + held = threading.local() + violations = [] + real_lock = ManagedResource._lock + + def tracking_lock(resource): + lock = real_lock(resource) + depth = getattr(held, 'stack', None) + if depth is None: + depth = held.stack = [] + + class Tracked: + def __enter__(self): + others = [r for r in depth if r is not resource] + if others: + violations.append( + "{} while holding {}".format( + type(resource).__name__, + [type(o).__name__ for o in others])) + depth.append(resource) + return lock.__enter__() + + def __exit__(self, *exc): + depth.pop() + return lock.__exit__(*exc) + + return Tracked() + + ManagedResource._lock = tracking_lock + try: + reader = Reader("image/jpeg", io.BytesIO(data)) + reader.json() + reader.detailed_json() + reader.is_embedded() + reader.get_remote_url() + reader.close() + finally: + ManagedResource._lock = real_lock + + self.assertEqual(violations, [], + "a thread held two operation locks at once") + + def test_concurrent_storm_terminates(self): + """Readers, closers and collection running together must all finish.""" + data = open(DEFAULT_TEST_FILE, 'rb').read() + stop = threading.Event() + shared = [Reader("image/jpeg", io.BytesIO(data))] + errors = [] + + def reader_worker(): + while not stop.is_set(): + try: + current = shared[0] + current._manifest_json_str_cache = None + current.json() + except Exception: + pass + + def closer_worker(): + while not stop.is_set(): + try: + shared[0].close() + shared[0] = Reader("image/jpeg", io.BytesIO(data)) + gc.collect() + except Exception as exc: + errors.append(repr(exc)) + return + + threads = [threading.Thread(target=reader_worker) for _ in range(6)] + threads += [threading.Thread(target=closer_worker) for _ in range(2)] + for thread in threads: + thread.start() + deadline = time.time() + 5 + while time.time() < deadline: + time.sleep(0.05) + stop.set() + self._join_all(threads, "concurrent storm") + self.assertEqual(errors, []) + + if __name__ == '__main__': unittest.main() From 101655c6471b74f8d56fb01194ddbf2a82593eba Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:36:47 -0700 Subject: [PATCH 03/28] fix: Warnings in tests --- tests/test_unit_tests_threaded.py | 33 ++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index f2b2a643..c1118534 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -3048,6 +3048,17 @@ class TestManagedResourceLockDeadlock(unittest.TestCase): JOIN_TIMEOUT = 30 + @classmethod + def setUpClass(cls): + with open(DEFAULT_TEST_FILE, 'rb') as handle: + cls.image_bytes = handle.read() + with open(os.path.join(FIXTURES_FOLDER, + "es256_certs.pem"), 'rb') as handle: + cls.certs = handle.read() + with open(os.path.join(FIXTURES_FOLDER, + "es256_private.key"), 'rb') as handle: + cls.private_key = handle.read() + def _join_all(self, threads, what): for thread in threads: thread.join(self.JOIN_TIMEOUT) @@ -3160,7 +3171,7 @@ def body(): def test_close_racing_json_does_not_deadlock(self): """close() on one thread against json() on another.""" - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes errors = [] def rounds(): @@ -3189,7 +3200,7 @@ def rounds(): def test_context_manager_exit_racing_json_does_not_deadlock(self): """__exit__ closes while another thread is calling json().""" - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes errors = [] def body(): @@ -3228,7 +3239,7 @@ def test_consume_failure_teardown_does_not_deadlock(self): with_fragment on a JPEG returns NotSupported, which routes through _raise_consume_failure. """ - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes errors = [] def body(): @@ -3253,11 +3264,9 @@ def test_close_during_sign_does_not_deadlock(self): """_sign_internal calls self.close() inside its own try block, so signing re-enters the lock on the signing thread. """ - certs = open(os.path.join(FIXTURES_FOLDER, - "es256_certs.pem"), 'rb').read() - key = open(os.path.join(FIXTURES_FOLDER, - "es256_private.key"), 'rb').read() - data = open(DEFAULT_TEST_FILE, 'rb').read() + certs = self.certs + key = self.private_key + data = self.image_bytes signer_info = C2paSignerInfo( alg=b"es256", sign_cert=certs, @@ -3295,7 +3304,7 @@ def test_stream_callback_reentering_api_does_not_deadlock(self): This passes only because construction does not hold the lock. """ - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes other = Reader("image/jpeg", io.BytesIO(data)) errors = [] @@ -3327,7 +3336,7 @@ def test_stream_callback_blocking_on_other_thread_does_not_deadlock(self): A lock held across construction deadlocks here, whether it is global or per-object. This is the test that pins the scoping decision. """ - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes target = Reader("image/jpeg", io.BytesIO(data)) errors = [] @@ -3366,7 +3375,7 @@ def test_no_nested_op_locks(self): That property, not the tests above, is what makes the design deadlock-free: with only one lock ever held, no cycle can form. """ - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes held = threading.local() violations = [] real_lock = ManagedResource._lock @@ -3410,7 +3419,7 @@ def __exit__(self, *exc): def test_concurrent_storm_terminates(self): """Readers, closers and collection running together must all finish.""" - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes stop = threading.Event() shared = [Reader("image/jpeg", io.BytesIO(data))] errors = [] From 78e5d862b8b457078ef402f3f7917d3840dd9f3e Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:32:45 -0700 Subject: [PATCH 04/28] WIP 2 (#313) * fix: Warnings in tests * fix: Deferred teardown * fix: Protect signer * fix: Borrow test * fix: Borrow test 2 --- src/c2pa/c2pa.py | 224 ++++++++---- tests/test_unit_tests_threaded.py | 581 +++++++++++++++++++++++++++++- 2 files changed, 732 insertions(+), 73 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 0255e6db..6e0fe836 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -13,6 +13,7 @@ # Version: 0.37.8 +import contextlib import ctypes import enum import json @@ -266,6 +267,8 @@ def __init__(self): self._lifecycle_state = LifecycleState.UNINITIALIZED self._handle = None self._op_lock = threading.RLock() + self._inflight = 0 + self._pending_teardown = None record_owner_pid(self) def _lock(self): @@ -296,6 +299,39 @@ def _lock(self): pass return lock + @contextlib.contextmanager + def _native_call(self): + """Hold the handle valid across a native call that goes back + and forth to native layers. + + Calls that pass a Stream to the native library run caller-supplied + callbacks, so the lock cannot be held across them: the callback may + re-enter this API on another thread and deadlock. Instead the call is + counted as in flight, and a teardown arriving meanwhile records its + intent rather than freeing. The last caller out performs the free. + + The resource is marked closed as soon as the teardown is recorded, so + a caller that closed it cannot keep using it while the free is + pending. + """ + with self._lock(): + self._ensure_valid_state() + self._inflight = getattr(self, '_inflight', 0) + 1 + try: + yield + finally: + with self._lock(): + self._inflight -= 1 + pending = (self._pending_teardown + if self._inflight == 0 else None) + if pending is not None: + self._pending_teardown = None + # Released the lock before the free: _teardown takes it again, + # and keeping the two acquisitions separate means the counter + # update is never held across the release work. + if pending is not None: + self._teardown(pending) + @staticmethod def _free_native_ptr(ptr): """Free a native pointer by passing it to c2pa_free. @@ -356,6 +392,16 @@ def _teardown(self, free_handle: bool): thread's state check and its use of the handle in a native call. """ with self._lock(): + if getattr(self, '_inflight', 0) > 0: + # A native call is running that re-enters caller Python and + # is still using this handle. Record the intent; whichever + # caller leaves _native_call last performs the free. Mark the + # resource closed now so it cannot be used while the free is + # pending. + self._pending_teardown = free_handle + self._lifecycle_state = LifecycleState.CLOSED + return + if is_foreign_process(self): self._handle = None self._lifecycle_state = LifecycleState.CLOSED @@ -1735,13 +1781,22 @@ def __init__( check=lambda r: r != 0) if signer is not None: - signer._ensure_valid_state() - # A rejected signer is retained, not closed and leaked. - self._signer_callback_cb = signer._callback_cb - signer._consume_no_replacement( - lambda h: _lib.c2pa_context_builder_set_signer( - nb._handle, h), - "Failed to set signer on Context: {}") + # The signer's own in-flight guard: this hands its handle + # to native, so a signer.close() on another thread must + # not free it between the state check and the call. The + # guard also makes the check and the consume atomic. + # + # _consume_no_replacement tears the signer down from + # inside this region. A teardown recorded while the guard + # is held is deferred and performed as the guard unwinds, + # which is still before __init__ returns. + with signer._native_call(): + # A rejected signer is retained, not closed and leaked. + self._signer_callback_cb = signer._callback_cb + signer._consume_no_replacement( + lambda h: _lib.c2pa_context_builder_set_signer( + nb._handle, h), + "Failed to set signer on Context: {}") self._has_signer = True context_ptr = nb._consume_into( @@ -2658,12 +2713,19 @@ def _init_from_context(self, context, format_or_path, self._own_stream = Stream(stream) try: - # Adopt before the consuming call: _consume_and_swap needs an - # active resource, and cleanup then owns the pointer either way. - self._create_and_activate( - lambda: _lib.c2pa_reader_from_context( - context.execution_context), - Reader._ERROR_MESSAGES['reader_error']) + # The Context is caller-supplied and may be shared, so its handle + # needs its own in-flight guard across the native call: the + # execution_context property validates and returns the handle, and + # without the guard a context.close() on another thread could free + # it before c2pa_reader_from_context reads it. + with context._native_call(): + # Adopt before the consuming call: _consume_and_swap needs an + # active resource, and cleanup then owns the pointer either + # way. + self._create_and_activate( + lambda: _lib.c2pa_reader_from_context( + context.execution_context), + Reader._ERROR_MESSAGES['reader_error']) if manifest_data is not None: manifest_array = ( @@ -2702,6 +2764,10 @@ def _init_attrs(self): # Tracks a file we opened ourselves and must close later. self._backing_file = None + # Fragment streams handed to the native reader by with_fragment, + # which it keeps reading from for the rest of its life. + self._fragment_streams = [] + # Caches for manifest JSON string and parsed data. # These are invalidated when with_fragment() is called. self._manifest_json_str_cache = None @@ -2725,6 +2791,12 @@ def _close_streams(self): logger.warning("Failed to close Reader backing file") finally: self._backing_file = None + for fragment in getattr(self, '_fragment_streams', []): + try: + fragment.close() + except Exception: + logger.warning("Failed to close Reader fragment stream") + self._fragment_streams = [] def _release(self): """Release Reader-specific resources (caches, stream, backing file). @@ -2787,19 +2859,38 @@ def with_fragment(self, format: Optional[str], stream, cannot be retried: create a new one instead of reusing this instance. """ - self._ensure_valid_state() - format_arg = _format_ffi_arg(_encode_format(format, "Reader")) - with Stream(stream) as main_obj, Stream(fragment_stream) as frag_obj: - self._consume_and_swap( - lambda handle: _lib.c2pa_reader_with_fragment( - handle, - format_arg, - main_obj._stream, - frag_obj._stream, - ), - Reader._ERROR_MESSAGES['fragment_error']) + # The native reader keeps reading through both streams after this + # returns, so they are owned here and released by _release() rather + # than at the end of a with block. + main_obj = Stream(stream) + frag_obj = Stream(fragment_stream) + try: + with self._native_call(): + self._consume_and_swap( + lambda handle: _lib.c2pa_reader_with_fragment( + handle, + format_arg, + main_obj._stream, + frag_obj._stream, + ), + Reader._ERROR_MESSAGES['fragment_error']) + except Exception: + main_obj.close() + frag_obj.close() + raise + + # Replace the streams this reader owned, closing the previous ones so + # repeated calls do not accumulate them. + previous = self._own_stream + self._own_stream = main_obj + self._fragment_streams.append(frag_obj) + if previous is not None and previous is not main_obj: + try: + previous.close() + except Exception: + logger.warning("Failed to close previous Reader stream") # Invalidate caches: processing a new BMFF fragment updates the native # reader's state, which can change the manifest data it returns. @@ -3000,10 +3091,8 @@ def resource_to_stream(self, uri: str, stream: Any) -> int: Raises: C2paError: If there was an error writing the resource to stream """ - self._ensure_valid_state() - uri_str = uri.encode('utf-8') - with Stream(stream) as stream_obj: + with self._native_call(), Stream(stream) as stream_obj: result = _lib.c2pa_reader_resource_to_stream( self._handle, uri_str, stream_obj._stream) @@ -3451,11 +3540,17 @@ def _init_from_context(self, context, json_str): if not context.is_valid: raise C2paError("Context is not valid") - # Adopt before the consuming call: _consume_and_swap needs an - # active resource, and cleanup then owns the pointer either way. - self._create_and_activate( - lambda: _lib.c2pa_builder_from_context(context.execution_context), - Builder._ERROR_MESSAGES['builder_error']) + # The Context is caller-supplied and may be shared, so its handle + # needs its own in-flight guard across the native call: without it a + # context.close() on another thread frees the handle between the + # is_valid check and c2pa_builder_from_context reading it. + with context._native_call(): + # Adopt before the consuming call: _consume_and_swap needs an + # active resource, and cleanup then owns the pointer either way. + self._create_and_activate( + lambda: _lib.c2pa_builder_from_context( + context.execution_context), + Builder._ERROR_MESSAGES['builder_error']) self._consume_and_swap( lambda handle: _lib.c2pa_builder_with_definition( @@ -3558,10 +3653,8 @@ def add_resource(self, uri: str, stream: Any): Raises: C2paError: If there was an error adding the resource """ - self._ensure_valid_state() - uri_bytes = _to_utf8_bytes(uri, "resource URI") - with Stream(stream) as stream_obj: + with self._native_call(), Stream(stream) as stream_obj: result = _lib.c2pa_builder_add_resource( self._handle, uri_bytes, stream_obj._stream) @@ -3622,7 +3715,7 @@ def add_ingredient_from_stream( ingredient_str = _to_utf8_bytes(ingredient_json, "ingredient JSON") format_str = _to_utf8_bytes(format, "ingredient format") - with Stream(source) as source_stream: + with self._native_call(), Stream(source) as source_stream: result = ( _lib.c2pa_builder_add_ingredient_from_stream( self._handle, @@ -3671,9 +3764,7 @@ def to_archive(self, stream: Any) -> None: Raises: C2paError: If there was an error writing the archive """ - self._ensure_valid_state() - - with Stream(stream) as stream_obj: + with self._native_call(), Stream(stream) as stream_obj: result = _lib.c2pa_builder_to_archive( self._handle, stream_obj._stream) @@ -3698,7 +3789,7 @@ def write_ingredient_archive(self, ingredient_id: str, stream: Any) -> None: ingredient_id_str = _to_utf8_bytes(ingredient_id, "ingredient_id") - with Stream(stream) as stream_obj: + with self._native_call(), Stream(stream) as stream_obj: result = _lib.c2pa_builder_write_ingredient_archive( self._handle, ingredient_id_str, stream_obj._stream) @@ -3718,9 +3809,7 @@ def add_ingredient_from_archive(self, stream: Any) -> None: Raises: C2paError: If there was an error reading the archive """ - self._ensure_valid_state() - - with Stream(stream) as stream_obj: + with self._native_call(), Stream(stream) as stream_obj: result = _lib.c2pa_builder_add_ingredient_from_archive( self._handle, stream_obj._stream) @@ -3749,7 +3838,7 @@ def with_archive(self, stream: Any) -> 'Builder': """ self._ensure_valid_state() - with Stream(stream) as stream_obj: + with self._native_call(), Stream(stream) as stream_obj: self._consume_and_swap( lambda handle: _lib.c2pa_builder_with_archive( handle, stream_obj._stream), @@ -3797,23 +3886,36 @@ def _sign_internal( manifest_bytes_ptr = ctypes.POINTER(ctypes.c_ubyte)() try: - if signer is not None: - result = _lib.c2pa_builder_sign( - self._handle, - format_arg, - source_stream._stream, - dest_stream._stream, - signer._handle, - ctypes.byref(manifest_bytes_ptr) - ) - else: - result = _lib.c2pa_builder_sign_context( - self._handle, - format_arg, - source_stream._stream, - dest_stream._stream, - ctypes.byref(manifest_bytes_ptr), - ) + # _native_call covers the signing call only. The close() below is + # deliberately outside it, so the deferred teardown it records is + # performed on the way out rather than being deferred forever. + with self._native_call(): + if signer is not None: + # c2pa_builder_sign borrows the signer's handle, so the + # signer needs its own in-flight guard: the Builder's + # guard holds only the Builder's handle valid, and a + # signer.close() on another thread would otherwise free + # this handle mid-call. Entered inside self's guard so + # concurrent signs sharing objects acquire in one order. + # The check above is a fast-fail; this re-check inside + # the guard is the one that makes check-then-use atomic. + with signer._native_call(): + result = _lib.c2pa_builder_sign( + self._handle, + format_arg, + source_stream._stream, + dest_stream._stream, + signer._handle, + ctypes.byref(manifest_bytes_ptr) + ) + else: + result = _lib.c2pa_builder_sign_context( + self._handle, + format_arg, + source_stream._stream, + dest_stream._stream, + ctypes.byref(manifest_bytes_ptr), + ) # Sign borrows the Builder without taking ownership. # Closing here ensures resources clean up, # and single use/single sign done by a Builder. diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index f2b2a643..0b9b4f95 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -11,9 +11,12 @@ # specific language governing permissions and limitations under # each license. +import ast import ctypes import gc import os +import re +import inspect import io import json import subprocess @@ -3048,6 +3051,17 @@ class TestManagedResourceLockDeadlock(unittest.TestCase): JOIN_TIMEOUT = 30 + @classmethod + def setUpClass(cls): + with open(DEFAULT_TEST_FILE, 'rb') as handle: + cls.image_bytes = handle.read() + with open(os.path.join(FIXTURES_FOLDER, + "es256_certs.pem"), 'rb') as handle: + cls.certs = handle.read() + with open(os.path.join(FIXTURES_FOLDER, + "es256_private.key"), 'rb') as handle: + cls.private_key = handle.read() + def _join_all(self, threads, what): for thread in threads: thread.join(self.JOIN_TIMEOUT) @@ -3160,7 +3174,7 @@ def body(): def test_close_racing_json_does_not_deadlock(self): """close() on one thread against json() on another.""" - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes errors = [] def rounds(): @@ -3189,7 +3203,7 @@ def rounds(): def test_context_manager_exit_racing_json_does_not_deadlock(self): """__exit__ closes while another thread is calling json().""" - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes errors = [] def body(): @@ -3228,7 +3242,7 @@ def test_consume_failure_teardown_does_not_deadlock(self): with_fragment on a JPEG returns NotSupported, which routes through _raise_consume_failure. """ - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes errors = [] def body(): @@ -3253,11 +3267,9 @@ def test_close_during_sign_does_not_deadlock(self): """_sign_internal calls self.close() inside its own try block, so signing re-enters the lock on the signing thread. """ - certs = open(os.path.join(FIXTURES_FOLDER, - "es256_certs.pem"), 'rb').read() - key = open(os.path.join(FIXTURES_FOLDER, - "es256_private.key"), 'rb').read() - data = open(DEFAULT_TEST_FILE, 'rb').read() + certs = self.certs + key = self.private_key + data = self.image_bytes signer_info = C2paSignerInfo( alg=b"es256", sign_cert=certs, @@ -3295,7 +3307,7 @@ def test_stream_callback_reentering_api_does_not_deadlock(self): This passes only because construction does not hold the lock. """ - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes other = Reader("image/jpeg", io.BytesIO(data)) errors = [] @@ -3327,7 +3339,7 @@ def test_stream_callback_blocking_on_other_thread_does_not_deadlock(self): A lock held across construction deadlocks here, whether it is global or per-object. This is the test that pins the scoping decision. """ - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes target = Reader("image/jpeg", io.BytesIO(data)) errors = [] @@ -3366,7 +3378,7 @@ def test_no_nested_op_locks(self): That property, not the tests above, is what makes the design deadlock-free: with only one lock ever held, no cycle can form. """ - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes held = threading.local() violations = [] real_lock = ManagedResource._lock @@ -3410,7 +3422,7 @@ def __exit__(self, *exc): def test_concurrent_storm_terminates(self): """Readers, closers and collection running together must all finish.""" - data = open(DEFAULT_TEST_FILE, 'rb').read() + data = self.image_bytes stop = threading.Event() shared = [Reader("image/jpeg", io.BytesIO(data))] errors = [] @@ -3445,6 +3457,551 @@ def closer_worker(): self._join_all(threads, "concurrent storm") self.assertEqual(errors, []) + def _counted_free(self): + """Patch _free_native_ptr to count frees; returns the list.""" + freed = [] + real = ManagedResource._free_native_ptr + + def counting(ptr): + freed.append(ptr) + return real(ptr) + + ManagedResource._free_native_ptr = staticmethod(counting) + self.addCleanup( + lambda: setattr(ManagedResource, '_free_native_ptr', real)) + return freed + + def _thumbnail_uri(self, reader): + manifests = json.loads(reader.json()).get("manifests", {}) + for manifest in manifests.values(): + thumbnail = manifest.get("thumbnail") + if thumbnail and thumbnail.get("identifier"): + return thumbnail["identifier"] + self.skipTest("fixture has no thumbnail resource to stream") + + def test_close_inside_callback_defers_free(self): + """A close() from inside a stream callback must not free the handle + the native call is still using.""" + freed = self._counted_free() + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + uri = self._thumbnail_uri(reader) + during = [] + + class Closer(io.BytesIO): + def write(self, buffer): + reader.close() + during.append(len(freed)) + return super().write(buffer) + + try: + reader.resource_to_stream(uri, Closer()) + except Error: + pass + + self.assertEqual(during, [0], "handle was freed mid-call") + self.assertEqual(len(freed), 1, "deferred free did not run once") + self.assertEqual(reader._inflight, 0) + self.assertIsNone(reader._pending_teardown) + self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) + + def test_cross_thread_close_during_callback_defers_free(self): + """Same race, with the close arriving from another thread.""" + freed = self._counted_free() + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + uri = self._thumbnail_uri(reader) + during = [] + started = threading.Event() + + class Slow(io.BytesIO): + def write(self, buffer): + started.set() + time.sleep(0.3) + during.append(len(freed)) + return super().write(buffer) + + def closer(): + started.wait(self.JOIN_TIMEOUT) + reader.close() + + thread = threading.Thread(target=closer) + thread.start() + try: + reader.resource_to_stream(uri, Slow()) + except Error: + pass + self._join_all([thread], "cross-thread closer") + + self.assertEqual(during, [0], "handle was freed mid-call") + self.assertEqual(len(freed), 1) + self.assertEqual(reader._inflight, 0) + + def test_deferred_teardown_still_closes(self): + """After a deferred free the resource is closed and a later close() + is a no-op rather than a second free.""" + freed = self._counted_free() + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + uri = self._thumbnail_uri(reader) + + class Closer(io.BytesIO): + def write(self, buffer): + reader.close() + return super().write(buffer) + + try: + reader.resource_to_stream(uri, Closer()) + except Error: + pass + + self.assertEqual(len(freed), 1) + reader.close() + self.assertEqual(len(freed), 1, "second close() freed again") + self.assertIsNone(reader._handle) + + def test_use_after_deferred_close_is_rejected(self): + """Deferring must not leave the resource usable: the free is pending, + so the handle is about to go away.""" + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + uri = self._thumbnail_uri(reader) + states = [] + + class Closer(io.BytesIO): + def write(self, buffer): + reader.close() + states.append(reader._lifecycle_state) + try: + reader.json() + states.append("json succeeded") + except Error: + states.append("json rejected") + return super().write(buffer) + + try: + reader.resource_to_stream(uri, Closer()) + except Error: + pass + + self.assertEqual(states[0], LifecycleState.CLOSED) + self.assertEqual(states[1], "json rejected") + + def test_exception_from_callback_still_frees(self): + """An exception unwinding through the native call must not strand the + in-flight counter, or the handle is never freed.""" + freed = self._counted_free() + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + uri = self._thumbnail_uri(reader) + + class Exploding(io.BytesIO): + def write(self, buffer): + reader.close() + raise RuntimeError("callback failure") + + try: + reader.resource_to_stream(uri, Exploding()) + except Exception: + pass + + self.assertEqual(reader._inflight, 0, "in-flight counter stranded") + self.assertEqual(len(freed), 1, "deferred free did not run") + + def test_inflight_cleared_before_deferred_free(self): + """The counter must reach zero before the deferred free runs. + + _teardown defers whenever _inflight is above zero, so performing the + free while the counter is still raised would defer it a second time + and the handle would never be released. + """ + seen = [] + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + uri = self._thumbnail_uri(reader) + real_release = Reader._release + + def probing_release(self): + seen.append(self._inflight) + return real_release(self) + + class Closer(io.BytesIO): + def write(self, buffer): + reader.close() + return super().write(buffer) + + with patch.object(Reader, '_release', probing_release): + try: + reader.resource_to_stream(uri, Closer()) + except Error: + pass + + self.assertEqual(seen, [0], + "deferred free ran while still counted in flight") + self.assertIsNone(reader._handle) + + def test_release_raising_during_deferred_teardown_does_not_leak(self): + """The deferred free survives a failing _release: the handle must + still be freed.""" + freed = self._counted_free() + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + uri = self._thumbnail_uri(reader) + + def boom(self): + raise RuntimeError("release failure") + + class Closer(io.BytesIO): + def write(self, buffer): + reader.close() + return super().write(buffer) + + with patch.object(Reader, '_release', boom): + try: + reader.resource_to_stream(uri, Closer()) + except Error: + pass + + self.assertEqual(reader._inflight, 0) + self.assertEqual(len(freed), 1, "handle leaked when _release raised") + + def test_concurrent_closes_during_callback_free_once(self): + """Many threads closing while one native call is in flight must + produce exactly one free.""" + freed = self._counted_free() + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + uri = self._thumbnail_uri(reader) + started = threading.Event() + closers = [] + + class Slow(io.BytesIO): + def write(self, buffer): + started.set() + time.sleep(0.3) + return super().write(buffer) + + def closer(): + started.wait(self.JOIN_TIMEOUT) + reader.close() + + for _ in range(8): + thread = threading.Thread(target=closer) + closers.append(thread) + thread.start() + try: + reader.resource_to_stream(uri, Slow()) + except Error: + pass + self._join_all(closers, "concurrent closers") + + self.assertEqual(len(freed), 1, + "racing closers freed {} times".format(len(freed))) + self.assertEqual(reader._inflight, 0) + + def test_sign_with_internal_close_frees_once(self): + """_sign_internal closes the Builder inside its own try, so the close + defers and the free happens on the way out.""" + freed = self._counted_free() + signer_info = C2paSignerInfo( + alg=b"es256", + sign_cert=self.certs, + private_key=self.private_key, + ta_url=b"http://timestamp.digicert.com", + ) + manifest = { + "claim_generator": "python_test", + "claim_generator_info": [ + {"name": "python_test", "version": "0.0.1"}], + "format": "image/jpeg", + "assertions": [], + } + signer = Signer.from_info(signer_info) + builder = Builder(manifest) + builder.sign(signer, "image/jpeg", + io.BytesIO(self.image_bytes), io.BytesIO()) + + self.assertEqual(builder._lifecycle_state, LifecycleState.CLOSED) + self.assertEqual(builder._inflight, 0) + builder_frees = [f for f in freed if f is not None] + self.assertGreaterEqual(len(builder_frees), 1) + with self.assertRaises(Error): + builder.sign(signer, "image/jpeg", + io.BytesIO(self.image_bytes), io.BytesIO()) + + def test_class_a_construction_is_not_guarded(self): + """Construction is deliberately unguarded: no external caller holds a + reference yet, and guarding it would reintroduce the deadlock where a + stream callback re-enters the API.""" + entered = [] + real = ManagedResource._native_call + + def recording(resource): + entered.append(type(resource).__name__) + return real(resource) + + ManagedResource._native_call = recording + try: + Reader("image/jpeg", io.BytesIO(self.image_bytes)) + finally: + ManagedResource._native_call = real + + self.assertEqual(entered, [], + "construction entered _native_call: guarding it " + "reintroduces the callback deadlock") + + def test_every_callback_running_method_is_guarded(self): + """Coverage check: every method that hands a Stream to the native + library must be guarded, except the three construction paths. + + A method missed here keeps the use-after-free, and the symptom is a + rare segfault rather than a failing test, so this is checked + mechanically rather than by eye. + """ + source = inspect.getsource(sys.modules[Reader.__module__]) + lines = source.split("\n") + class_a = { + ("Reader", "_create_reader"), + ("Reader", "_init_from_context"), + ("Builder", "from_archive"), + } + stream_use = re.compile( + r"(_stream|stream_obj|source_stream|dest_stream|main_obj" + r"|frag_obj)\._stream") + + bodies = {} + current_class = current_method = None + start = None + for index, line in enumerate(lines): + if re.match(r"^class ", line): + current_class = line.split("(")[0].replace( + "class ", "").strip(":") + if re.match(r"^def ", line): + current_class = None + match = re.match(r"^ def (\w+)", line) + if match: + if current_class and current_method and start is not None: + bodies[(current_class, current_method)] = "\n".join( + lines[start:index]) + current_method = match.group(1) + start = index + if current_class and current_method and start is not None: + bodies[(current_class, current_method)] = "\n".join(lines[start:]) + + unguarded = [] + checked = 0 + for key, body in bodies.items(): + if not stream_use.search(body): + continue + checked += 1 + if key in class_a: + continue + if "_native_call()" not in body: + unguarded.append("{}.{}".format(*key)) + + self.assertGreater(checked, 0, "coverage scan found no methods") + self.assertEqual( + unguarded, [], + "these hand a Stream to native without _native_call(): {}".format( + unguarded)) + + def test_every_borrowed_handle_is_guarded(self): + """Coverage check: when a method hands a *second* object's handle to + the native library, that object needs its own _native_call() guard. + + test_every_callback_running_method_is_guarded only asks whether the + string "_native_call()" appears in the method body, which cannot + express *whose* handle is guarded. A method that guards self while + passing signer._handle to native passes that check and still has the + use-after-free, so the ownership is checked structurally here. + """ + module = sys.modules[Reader.__module__] + tree = ast.parse(inspect.getsource(module)) + + # Attributes that carry a native handle out of an object. + handle_attrs = {"_handle", "execution_context"} + + def guarded_names(node): + """Names X with an active `with X._native_call():` at this node.""" + found = set() + for item in getattr(node, "items", []): + call = item.context_expr + if (isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "_native_call" + and isinstance(call.func.value, ast.Name)): + found.add(call.func.value.id) + return found + + def borrowed_in_call(call): + """Names X whose handle this _lib.* call receives, X not self.""" + if not (isinstance(call.func, ast.Attribute) + and isinstance(call.func.value, ast.Name) + and call.func.value.id == "_lib"): + return set() + names = set() + for arg in ast.walk(call): + if (isinstance(arg, ast.Attribute) + and arg.attr in handle_attrs + and isinstance(arg.value, ast.Name) + and arg.value.id != "self"): + names.add(arg.value.id) + return names + + def locally_owned(method): + """Names bound to an object this method itself constructed. + + A resource created inside the method never escapes to another + thread, so nothing can close it mid-call and it needs no guard. + Only handles reaching the method from outside (parameters, + attributes) are exposed to a concurrent teardown. + """ + owned = set() + for node in ast.walk(method): + # `with self._NativeBuilder() as nb:` / `x = Foo()` + if isinstance(node, (ast.With, ast.AsyncWith)): + for item in node.items: + if (isinstance(item.context_expr, ast.Call) + and isinstance(item.optional_vars, ast.Name)): + owned.add(item.optional_vars.id) + elif isinstance(node, ast.Assign): + if isinstance(node.value, ast.Call): + for target in node.targets: + if isinstance(target, ast.Name): + owned.add(target.id) + return owned + + unguarded = [] + checked = 0 + + for cls in ast.walk(tree): + if not isinstance(cls, ast.ClassDef): + continue + for method in cls.body: + if not isinstance(method, (ast.FunctionDef, + ast.AsyncFunctionDef)): + continue + owned = locally_owned(method) + + # Walk the body tracking which guards are open, so a borrowed + # handle is only accepted when its own guard encloses the use. + def visit(node, active): + nonlocal checked + if isinstance(node, (ast.With, ast.AsyncWith)): + active = active | guarded_names(node) + if isinstance(node, ast.Call): + for name in borrowed_in_call(node) - owned: + checked += 1 + if name not in active: + unguarded.append( + "{}.{} passes {}._handle to native " + "without {}._native_call()".format( + cls.name, method.name, name, name)) + for child in ast.iter_child_nodes(node): + visit(child, active) + + visit(method, frozenset()) + + self.assertGreater( + checked, 0, + "ownership scan found no borrowed handles: the scan is broken") + self.assertEqual( + unguarded, [], + "borrowed handles used without their own guard:\n " + + "\n ".join(unguarded)) + + +class TestSharedSignerTeardownRace(unittest.TestCase): + """A Signer shared across threads must not be freed mid-sign. + + Builder.sign borrows the signer's handle for the duration of the native + call. Without a guard on the signer itself, a close() on another thread + frees that handle while c2pa_builder_sign is using it, and the process + dies with SIGSEGV instead of raising. + """ + + def setUp(self): + self.data_dir = os.path.join(os.path.dirname(__file__), "fixtures") + with open(os.path.join(self.data_dir, "C.jpg"), "rb") as f: + self.image_bytes = f.read() + with open(os.path.join(self.data_dir, "es256_certs.pem"), "rb") as f: + self.certs = f.read() + with open(os.path.join(self.data_dir, "es256_private.key"), "rb") as f: + self.key = f.read() + self.manifest = { + "claim_generator_info": [{"name": "test", "version": "0.1"}], + "assertions": [], + } + + def _make_signer(self): + return Signer.from_info(C2paSignerInfo( + SigningAlg.ES256, self.certs, self.key, None)) + + def test_close_during_concurrent_sign_does_not_crash(self): + """Rotate a shared signer while other threads sign with it. + + Runs in a subprocess: the failure mode is a segfault, which would + take the test runner down with it rather than reporting a failure. + """ + source = textwrap.dedent(""" + import io, os, sys, threading + from c2pa import (Builder, Signer, C2paSignerInfo, + C2paSigningAlg as SigningAlg) + + data_dir = sys.argv[1] + certs = open(os.path.join(data_dir, "es256_certs.pem"), "rb").read() + key = open(os.path.join(data_dir, "es256_private.key"), "rb").read() + img = open(os.path.join(data_dir, "C.jpg"), "rb").read() + manifest = {"claim_generator_info": + [{"name": "test", "version": "0.1"}], + "assertions": []} + + def make(): + return Signer.from_info(C2paSignerInfo( + SigningAlg.ES256, certs, key, None)) + + box = {"signer": make(), "stop": False} + + def rotate(): + while not box["stop"]: + old = box["signer"] + try: + box["signer"] = make() + old.close() + except Exception: + pass + + def sign(): + for _ in range(120): + if box["stop"]: + return + try: + b = Builder(manifest) + b.sign(box["signer"], "image/jpeg", + io.BytesIO(img), io.BytesIO()) + b.close() + except Exception: + # A closed signer may legitimately be rejected; + # only a crash is a failure here. + pass + + rot = threading.Thread(target=rotate, daemon=True) + rot.start() + threads = [threading.Thread(target=sign) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + box["stop"] = True + rot.join(timeout=5) + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", source, self.data_dir], + capture_output=True, text=True, timeout=300) + + self.assertNotEqual( + result.returncode, -11, + "SIGSEGV: a signer was freed while a sign was using its handle") + self.assertEqual( + result.returncode, 0, + "shared-signer teardown race failed (rc={}):\n{}".format( + result.returncode, result.stderr[-2000:])) + self.assertIn("OK", result.stdout) + if __name__ == '__main__': unittest.main() From 2eaeec0e9390d980eb28a22fec436ea2abd3d8b0 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 25 Aug 2026 09:43:36 -0700 Subject: [PATCH 05/28] fix: Notes clean up --- pr312-followups.md | 108 --------------------------------------------- 1 file changed, 108 deletions(-) delete mode 100644 pr312-followups.md diff --git a/pr312-followups.md b/pr312-followups.md deleted file mode 100644 index 9dceafee..00000000 --- a/pr312-followups.md +++ /dev/null @@ -1,108 +0,0 @@ -# c2pa-python pull request 312: additional fixes - -Sentinel in the native thread-local error slot. Scratch note, not for the -repository. - -The change is correct as written. Three items, the first of which quietly -disables the diagnostic the sentinel exists to provide. - ---- - -## 1. Match on a substring, not on exact equality - -**The problem.** The comparison is - -```python -if error == ManagedResource._NO_NATIVE_ERROR.decode('utf-8'): -``` - -`c2pa_error_set_last` does not store the string verbatim. It runs it through -`Error::from` and then `CimplError::from`, and its own documentation states that -a missing or invalid error type is replaced with `Other` and the message includes -the original string. The sentinel already carries an `Other: ` prefix, so it may -round-trip unchanged, or it may come back re-prefixed or otherwise normalised. - -**Why it matters, and why it is easy to miss.** The failure is silent rather than -dangerous. If the comparison never matches, control falls through to the final -branch, which now performs the identical `_teardown(free_handle=False)`. Same -action, no crash, all tests that check behaviour still pass. - -What is lost is the distinct log line. That log line is the entire reason for -planting a sentinel rather than simply clearing the slot: it separates "the -native side reported nothing" from "the native side reported a real error", and -it is the only field evidence available for how often the ambiguous case occurs. -Losing it costs nothing today and costs the whole diagnostic tomorrow. - -**Fix.** Match on the distinctive part only: - -```python -_NO_NATIVE_ERROR_MARKER = "c2pa-python-no-native-error" -... -if _NO_NATIVE_ERROR_MARKER in error: -``` - -**And pin the round-trip in a test regardless**, since it is a property of the -native side that can change without notice: - -```python -def test_sentinel_round_trips_through_native_error_slot(self): - c2pa_module._lib.c2pa_error_set_last(ManagedResource._NO_NATIVE_ERROR) - self.assertIn(_NO_NATIVE_ERROR_MARKER, c2pa_module._read_native_error()) -``` - -That test fails loudly if the normalisation ever changes, which is exactly the -kind of upstream invariant worth pinning rather than assuming. - -## 2. Restore the ordering rationale that was deleted - -The removed paragraph explained that `c2pa_free` on a handle the registry no -longer tracks returns minus one and overwrites the slot with its own -untracked-pointer message, so the error must be read before any free or the -substitute carries a pre-consume tag and inverts the retain decision. - -That constraint is still true, and the current code still depends on it. The -replacement text explains the sentinel but says nothing about why the read comes -first. A later edit that moves the read after a free would reintroduce the -inversion with no warning anywhere in the file. - -One sentence is enough: - -> The read must precede any free: `c2pa_free` on an untracked handle overwrites -> the slot with its own untracked-pointer message, which carries a pre-consume -> tag and would invert the decision below. - -## 3. Confirm the changed test is green for the right reason - -`test_context_build_null_return_frees_builder` loses its explicit -`c2pa_error_set_last(b"UntrackedPointer: ...")` line. That test needs the -retained branch to fire, which needs a pre-consume tag present at the moment the -failure is read. - -With the sentinel now planted inside `_invoke_consume`, a mock that merely -returns `None` leaves the sentinel in place, the sentinel branch fires, -`_teardown(free_handle=False)` runs, and no free happens. The assertion should -then fail. - -Presumably the mock is now built with `_fail_with_native_error(b"UntrackedPointer: ...")`, -which restores the tag from inside the call rather than before it. Worth -confirming that is what landed, because a test that passes for the wrong reason -here is worse than one that fails: it would be asserting the retained branch -while actually exercising the consumed one. - ---- - -## What already holds - -The sentinel is planted immediately before `ffi_call`, inside `_invoke_consume`, -with nothing between them, on the thread that makes the call. That is the correct -placement and the thread-local slot means it cannot disturb any other worker. - -`_setup_function(_lib.c2pa_error_set_last, [ctypes.c_char_p], ctypes.c_int)` -supplies the explicit argument and return types, which was the one open check. -The return value itself needs no guard: minus one is returned only for a null -pointer, so any non-null sentinel returns zero. - -Changing the final fallback from `_release_handle()` to -`_teardown(free_handle=False)` removes the guarded free from the ambiguous path -entirely. That is the more important half of this pull request, and it holds even -if the sentinel comparison in item 1 never matches. From 0edcb35fff348089fc7564d815d22084caaadc99 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 25 Aug 2026 10:35:44 -0700 Subject: [PATCH 06/28] fix: Clean up comemnts --- src/c2pa/c2pa.py | 98 ++++++++++++++++++++---------------------------- 1 file changed, 41 insertions(+), 57 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 6e0fe836..035335c9 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -274,21 +274,18 @@ def __init__(self): def _lock(self): """Return this resource's operation lock. - Reentrant because CPython can run a finalizer at any bytecode + Reentrant because it is possible to run a finalizer at any bytecode boundary, including inside a region this thread has already locked, and because a consuming call tears the handle down from inside the - locked region (_invoke_consume, _raise_consume_failure). + locked region. - Falls back to a fresh lock when the attribute is missing: an object - whose __init__ raised before the assignment is still finalized, and - __del__ must not raise. + Falls back to a fresh lock when the attribute is missing. Never hold this across a native call that drives stream callbacks (construction, resource_to_stream, the Builder stream methods, signing). Those calls release the GIL and re-enter caller-supplied - Python, which may call back into this API on another thread; holding - the lock across them deadlocks. Only calls that touch no callbacks - are serialized here, and no path holds two of these locks at once. + Python, which may call back into this API on another thread. + Only calls that touch no callbacks are serialized here. """ lock = getattr(self, '_op_lock', None) if lock is None: @@ -305,10 +302,9 @@ def _native_call(self): and forth to native layers. Calls that pass a Stream to the native library run caller-supplied - callbacks, so the lock cannot be held across them: the callback may - re-enter this API on another thread and deadlock. Instead the call is - counted as in flight, and a teardown arriving meanwhile records its - intent rather than freeing. The last caller out performs the free. + callbacks, so the lock cannot be held across them. Instead the call + is counted as in flight, and a teardown arriving meanwhile records + its intent rather than freeing. The last caller out performs the free. The resource is marked closed as soon as the teardown is recorded, so a caller that closed it cannot keep using it while the free is @@ -326,9 +322,10 @@ def _native_call(self): if self._inflight == 0 else None) if pending is not None: self._pending_teardown = None - # Released the lock before the free: _teardown takes it again, - # and keeping the two acquisitions separate means the counter - # update is never held across the release work. + # Released the lock before the free: + # _teardown takes it again, and keeping the two acquisitions + # separate means the counter update is never held across + # the release work. if pending is not None: self._teardown(pending) @@ -388,16 +385,16 @@ def _teardown(self, free_handle: bool): """Close the object: run _release, optionally free the handle, null it. free_handle=False (consumed) frees nothing, the new owner needs to free. - Holds the operation lock so the free cannot land between another + Holds the operation lock so the free cannot happen between another thread's state check and its use of the handle in a native call. """ with self._lock(): if getattr(self, '_inflight', 0) > 0: # A native call is running that re-enters caller Python and - # is still using this handle. Record the intent; whichever - # caller leaves _native_call last performs the free. Mark the - # resource closed now so it cannot be used while the free is - # pending. + # is still using this handle. Record the intent and whichever + # caller leaves _native_call last performs the free. + # Mark the resource closed now so it cannot be used + # while the free is pending. self._pending_teardown = free_handle self._lifecycle_state = LifecycleState.CLOSED return @@ -1781,10 +1778,10 @@ def __init__( check=lambda r: r != 0) if signer is not None: - # The signer's own in-flight guard: this hands its handle - # to native, so a signer.close() on another thread must - # not free it between the state check and the call. The - # guard also makes the check and the consume atomic. + # The signer's in-flight guard: + # this hands its handle to native, + # so a signer.close() on another thread must not + # free it between the state check and the call. # # _consume_no_replacement tears the signer down from # inside this region. A teardown recorded while the guard @@ -2672,9 +2669,8 @@ def _init_from_file(self, path, format_bytes, def _init_from_stream(self, stream, format_bytes, manifest_data=None): """Create a reader from a caller-supplied stream object. - The native reader reads through this stream for as long as it is - alive, so the wrapper is stored on the instance and released by - _release(). + The native reader reads through this stream as long as it's alive, + so the wrapper is stored on the instance and released by _release(). Args: stream: A stream-like object owned by the caller @@ -2714,10 +2710,7 @@ def _init_from_context(self, context, format_or_path, try: # The Context is caller-supplied and may be shared, so its handle - # needs its own in-flight guard across the native call: the - # execution_context property validates and returns the handle, and - # without the guard a context.close() on another thread could free - # it before c2pa_reader_from_context reads it. + # needs its own in-flight guard across the native call. with context._native_call(): # Adopt before the consuming call: _consume_and_swap needs an # active resource, and cleanup then owns the pointer either @@ -2765,7 +2758,7 @@ def _init_attrs(self): self._backing_file = None # Fragment streams handed to the native reader by with_fragment, - # which it keeps reading from for the rest of its life. + # which it keeps reading from for the rest of its lifecycle. self._fragment_streams = [] # Caches for manifest JSON string and parsed data. @@ -2861,8 +2854,8 @@ def with_fragment(self, format: Optional[str], stream, """ format_arg = _format_ffi_arg(_encode_format(format, "Reader")) - # The native reader keeps reading through both streams after this - # returns, so they are owned here and released by _release() rather + # The native reader keeps reading through both streams after this returns, + # so they are owned here and released by _release() rather # than at the end of a with block. main_obj = Stream(stream) frag_obj = Stream(fragment_stream) @@ -2881,8 +2874,8 @@ def with_fragment(self, format: Optional[str], stream, frag_obj.close() raise - # Replace the streams this reader owned, closing the previous ones so - # repeated calls do not accumulate them. + # Replace the streams this reader owned, + # closing the previous ones so repeated calls do not accumulate them. previous = self._own_stream self._own_stream = main_obj self._fragment_streams.append(frag_obj) @@ -2911,10 +2904,7 @@ def json(self) -> str: C2paError: If there was an error getting the JSON """ - # The state check and the handle read are one critical section: a - # finalizer on another thread frees the handle while it is still - # non-null, so a check made outside the lock says nothing about the - # handle this call goes on to pass to native code. + # Lock due to checks on native handles. with self._lock(): self._ensure_valid_state() @@ -3540,13 +3530,11 @@ def _init_from_context(self, context, json_str): if not context.is_valid: raise C2paError("Context is not valid") - # The Context is caller-supplied and may be shared, so its handle - # needs its own in-flight guard across the native call: without it a - # context.close() on another thread frees the handle between the - # is_valid check and c2pa_builder_from_context reading it. + # The Context is caller-supplied and may be shared, + # so its handle needs its own in-flight guard across + # the native call, especially for state checks. with context._native_call(): - # Adopt before the consuming call: _consume_and_swap needs an - # active resource, and cleanup then owns the pointer either way. + # Adopt before the consuming call. self._create_and_activate( lambda: _lib.c2pa_builder_from_context( context.execution_context), @@ -3886,19 +3874,15 @@ def _sign_internal( manifest_bytes_ptr = ctypes.POINTER(ctypes.c_ubyte)() try: - # _native_call covers the signing call only. The close() below is - # deliberately outside it, so the deferred teardown it records is - # performed on the way out rather than being deferred forever. + # _native_call covers the signing call only. + # The close() below is deliberately outside it, + # so the deferred teardown it records is performed + # on the way out rather than being deferred forever. with self._native_call(): if signer is not None: - # c2pa_builder_sign borrows the signer's handle, so the - # signer needs its own in-flight guard: the Builder's - # guard holds only the Builder's handle valid, and a - # signer.close() on another thread would otherwise free - # this handle mid-call. Entered inside self's guard so - # concurrent signs sharing objects acquire in one order. - # The check above is a fast-fail; this re-check inside - # the guard is the one that makes check-then-use atomic. + # Signer needs its own in-flight guard. + # Entered inside self's guard so concurrent signs + # sharing objects (Signers) acquire in one order. with signer._native_call(): result = _lib.c2pa_builder_sign( self._handle, From 9dfaf878f554b06544addd26359c28cf6b2d7c98 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 25 Aug 2026 11:19:09 -0700 Subject: [PATCH 07/28] fix: clean up tests --- tests/test_unit_tests_threaded.py | 97 +++++++++++++------------------ 1 file changed, 40 insertions(+), 57 deletions(-) diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 0b9b4f95..5cbccea2 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -3045,8 +3045,8 @@ class TestManagedResourceLockDeadlock(unittest.TestCase): """Tests for the operation lock that serializes native calls against teardown. - Every join here is bounded: a deadlock must fail the test, not hang the - suite. + Every join here is bounded: + A deadlock must fail the test when timing out, not hang the suite. """ JOIN_TIMEOUT = 30 @@ -3072,10 +3072,8 @@ def _join_all(self, threads, what): what, self.JOIN_TIMEOUT)) def _run_isolated(self, body, timeout=180): - """Run body in a subprocess and return it. - - A segfault kills the interpreter, so a crash cannot be asserted on - in-process: it would take the test runner with it. + """Run body in a subprocess and return it, + so that crashes can be caught and do not crash the suite itself. """ source = textwrap.dedent(body) return subprocess.run( @@ -3087,9 +3085,6 @@ def _run_isolated(self, body, timeout=180): def test_json_racing_finalizer_does_not_crash(self): """Readers used on one thread while others are collected. - - Without the lock this segfaults inside c2pa_reader_json: the - finalizer frees the handle between the state check and the call. """ result = self._run_isolated(""" import sys, io, gc, random, threading, time @@ -3146,8 +3141,8 @@ def worker(): def test_finalizer_inside_locked_operation(self): """A finalizer can run at any bytecode boundary, including inside a - region this same thread has locked. A non-reentrant lock deadlocks - here; RLock does not. + region this same thread has locked. + A non-reentrant lock deadlocks here, but RLock does not. """ resource = _ConcreteResource() resource._activate(0x51000) @@ -3240,7 +3235,7 @@ def test_consume_failure_teardown_does_not_deadlock(self): operation, re-entering the lock on the same thread. with_fragment on a JPEG returns NotSupported, which routes through - _raise_consume_failure. + _raise_consume_failure (on purpose). """ data = self.image_bytes errors = [] @@ -3264,8 +3259,8 @@ def body(): self.assertEqual(errors, []) def test_close_during_sign_does_not_deadlock(self): - """_sign_internal calls self.close() inside its own try block, so - signing re-enters the lock on the signing thread. + """_sign_internal calls self.close() inside its own try block, + so signing re-enters the lock on the signing thread. """ certs = self.certs key = self.private_key @@ -3302,10 +3297,10 @@ def body(): self.assertEqual(errors, []) def test_stream_callback_reentering_api_does_not_deadlock(self): - """Construction drives caller-supplied stream callbacks, and a caller - may legitimately call back into the API from one. + """Construction drives caller-supplied stream callbacks, + and a caller may call back into the API from one. - This passes only because construction does not hold the lock. + This passes because construction does not hold the lock. """ data = self.image_bytes other = Reader("image/jpeg", io.BytesIO(data)) @@ -3333,11 +3328,11 @@ def body(): other.close() def test_stream_callback_blocking_on_other_thread_does_not_deadlock(self): - """The adversarial case: a stream callback that blocks on another - thread which touches the same object. + """A stream callback that blocks on another thread + which touches the same object. A lock held across construction deadlocks here, whether it is global - or per-object. This is the test that pins the scoping decision. + or per-object. """ data = self.image_bytes target = Reader("image/jpeg", io.BytesIO(data)) @@ -3374,9 +3369,7 @@ def body(): def test_no_nested_op_locks(self): """No code path may hold two resources' operation locks at once. - - That property, not the tests above, is what makes the design - deadlock-free: with only one lock ever held, no cycle can form. + With only one lock ever held, no cycle can form here. """ data = self.image_bytes held = threading.local() @@ -3505,7 +3498,8 @@ def write(self, buffer): self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) def test_cross_thread_close_during_callback_defers_free(self): - """Same race, with the close arriving from another thread.""" + """A close() from inside a stream callback must not free the handle + the native call is still using.""" freed = self._counted_free() reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) uri = self._thumbnail_uri(reader) @@ -3558,8 +3552,8 @@ def write(self, buffer): self.assertIsNone(reader._handle) def test_use_after_deferred_close_is_rejected(self): - """Deferring must not leave the resource usable: the free is pending, - so the handle is about to go away.""" + """Deferring must not leave the resource usable: + the free is pending, so the handle is about to go away.""" reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) uri = self._thumbnail_uri(reader) states = [] @@ -3584,8 +3578,8 @@ def write(self, buffer): self.assertEqual(states[1], "json rejected") def test_exception_from_callback_still_frees(self): - """An exception unwinding through the native call must not strand the - in-flight counter, or the handle is never freed.""" + """An exception unwinding through the native call must not + leave the inflight-handler hanging.""" freed = self._counted_free() reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) uri = self._thumbnail_uri(reader) @@ -3635,8 +3629,8 @@ def write(self, buffer): self.assertIsNone(reader._handle) def test_release_raising_during_deferred_teardown_does_not_leak(self): - """The deferred free survives a failing _release: the handle must - still be freed.""" + """The deferred free survives a failing _release: + the handle must still be freed.""" freed = self._counted_free() reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) uri = self._thumbnail_uri(reader) @@ -3659,8 +3653,9 @@ def write(self, buffer): self.assertEqual(len(freed), 1, "handle leaked when _release raised") def test_concurrent_closes_during_callback_free_once(self): - """Many threads closing while one native call is in flight must - produce exactly one free.""" + """Many threads closing while one native call is in flight + must produce exactly one free (avoid double-frees, + or freeing something the object wouldn't own).""" freed = self._counted_free() reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) uri = self._thumbnail_uri(reader) @@ -3692,8 +3687,8 @@ def closer(): self.assertEqual(reader._inflight, 0) def test_sign_with_internal_close_frees_once(self): - """_sign_internal closes the Builder inside its own try, so the close - defers and the free happens on the way out.""" + """_sign_internal closes the Builder inside its own try, + so the close defers and the free happens on the way out.""" freed = self._counted_free() signer_info = C2paSignerInfo( alg=b"es256", @@ -3722,9 +3717,8 @@ def test_sign_with_internal_close_frees_once(self): io.BytesIO(self.image_bytes), io.BytesIO()) def test_class_a_construction_is_not_guarded(self): - """Construction is deliberately unguarded: no external caller holds a - reference yet, and guarding it would reintroduce the deadlock where a - stream callback re-enters the API.""" + """Construction is unguarded: no external caller holds a reference yet. + """ entered = [] real = ManagedResource._native_call @@ -3743,12 +3737,8 @@ def recording(resource): "reintroduces the callback deadlock") def test_every_callback_running_method_is_guarded(self): - """Coverage check: every method that hands a Stream to the native - library must be guarded, except the three construction paths. - - A method missed here keeps the use-after-free, and the symptom is a - rare segfault rather than a failing test, so this is checked - mechanically rather than by eye. + """Every method that hands a Stream to the native lib must be guarded, + except the construction paths. """ source = inspect.getsource(sys.modules[Reader.__module__]) lines = source.split("\n") @@ -3798,14 +3788,11 @@ def test_every_callback_running_method_is_guarded(self): unguarded)) def test_every_borrowed_handle_is_guarded(self): - """Coverage check: when a method hands a *second* object's handle to - the native library, that object needs its own _native_call() guard. - - test_every_callback_running_method_is_guarded only asks whether the - string "_native_call()" appears in the method body, which cannot - express *whose* handle is guarded. A method that guards self while - passing signer._handle to native passes that check and still has the - use-after-free, so the ownership is checked structurally here. + """When a method hands a second object's handle to the native library, + that object needs its own _native_call() guard. + + This can happen in callbacks, where you can't express whose handle + is the one needing attention. """ module = sys.modules[Reader.__module__] tree = ast.parse(inspect.getsource(module)) @@ -3841,12 +3828,8 @@ def borrowed_in_call(call): return names def locally_owned(method): - """Names bound to an object this method itself constructed. - - A resource created inside the method never escapes to another + """A resource created inside the method never escapes to another thread, so nothing can close it mid-call and it needs no guard. - Only handles reaching the method from outside (parameters, - attributes) are exposed to a concurrent teardown. """ owned = set() for node in ast.walk(method): @@ -3933,7 +3916,7 @@ def test_close_during_concurrent_sign_does_not_crash(self): """Rotate a shared signer while other threads sign with it. Runs in a subprocess: the failure mode is a segfault, which would - take the test runner down with it rather than reporting a failure. + take the test runner down with it otherwise. """ source = textwrap.dedent(""" import io, os, sys, threading From e80b0397c4ba56829618284f6fafba7cb0ff20a5 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:40:53 -0700 Subject: [PATCH 08/28] fix: with_fragment has issues too --- src/c2pa/c2pa.py | 13 ++++++++++++- tests/perf/baseline.json | 9 +++++++-- tests/perf/scenarios.py | 25 +++++++++++++++++++++++++ tests/test_unit_tests.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 3 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 035335c9..d762e285 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -2876,14 +2876,25 @@ def with_fragment(self, format: Optional[str], stream, # Replace the streams this reader owned, # closing the previous ones so repeated calls do not accumulate them. + # Only the current fragment is retained: the native reader does not + # read a superseded one back, and each wrapper held open pins a native + # stream, its callbacks and the caller's buffer. previous = self._own_stream + previous_fragments = self._fragment_streams self._own_stream = main_obj - self._fragment_streams.append(frag_obj) + self._fragment_streams = [frag_obj] if previous is not None and previous is not main_obj: try: previous.close() except Exception: logger.warning("Failed to close previous Reader stream") + for fragment in previous_fragments: + if fragment is frag_obj: + continue + try: + fragment.close() + except Exception: + logger.warning("Failed to close Reader fragment stream") # Invalidate caches: processing a new BMFF fragment updates the native # reader's state, which can change the manifest data it returns. diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index c151efe5..feb51bd0 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -2,8 +2,8 @@ "_meta": { "memray_version": "1.19.3", "python_version": "3.12.13", - "c2pa_native_version": "c2pa-v0.90.0", - "iterations": 200, + "c2pa_native_version": "c2pa-v0.90.15", + "iterations": 100, "perf_env": "python-3.12-slim", "arch": "aarch64" }, @@ -296,5 +296,10 @@ "peak_bytes": 3681161, "leaked_bytes": 3350287, "total_allocations": 672537 + }, + "reader_with_fragment_repeated": { + "peak_bytes": 3803564, + "leaked_bytes": 3381191, + "total_allocations": 966288 } } \ No newline at end of file diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py index 23300aed..518c8f97 100644 --- a/tests/perf/scenarios.py +++ b/tests/perf/scenarios.py @@ -524,6 +524,30 @@ def scenario_reader_with_fragment_swap(iterations: int = 100) -> None: reader.close() +def scenario_reader_with_fragment_repeated(iterations: int = 100) -> None: + """Loop Reader.with_fragment() against a SINGLE long-lived Reader. + + The Reader is built outside the loop on purpose. Every other fragment + scenario constructs one per iteration and closes it, which releases the + streams each time round and so cannot show anything retained across calls. + Only repeated calls on one instance expose a fragment stream that is kept + instead of released, and each one held open pins a native C2paStream, its + four ctypes callbacks and the caller's buffer. + """ + init_bytes = DASH_INIT_MP4.read_bytes() + fragment_bytes = DASH_FRAGMENT.read_bytes() + reader = Reader("video/mp4", io.BytesIO(init_bytes)) + try: + for _ in _iterate(iterations): + reader.with_fragment( + "video/mp4", + io.BytesIO(init_bytes), + io.BytesIO(fragment_bytes), + ) + finally: + reader.close() + + def scenario_builder_from_archive_roundtrip(iterations: int = 100) -> None: """Loop Builder.from_archive() itself (context-less alternate constructor), then sign. Regression guard for the classmethod's native-handle wrapping. @@ -1370,6 +1394,7 @@ def scenario_fork_stream_cleanup(iterations: int = 100) -> None: "builder_from_archive_roundtrip": scenario_builder_from_archive_roundtrip, "builder_with_archive_swap": scenario_builder_with_archive_swap, "reader_with_fragment_swap": scenario_reader_with_fragment_swap, + "reader_with_fragment_repeated": scenario_reader_with_fragment_repeated, "with_fragment_pre_consume_rejection": scenario_reader_with_fragment_pre_consume_rejection, "with_archive_post_consume_failure": diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 4bff6dbb..5bf091d5 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -8786,6 +8786,40 @@ def test_with_fragment_pre_consume_rejection_does_not_leak(self): self.assertTrue(reader.json()) reader.close() + def test_repeated_with_fragment_does_not_accumulate_streams(self): + """Repeated calls on one Reader must not pile up fragment streams. + + Each retained wrapper pins a native C2paStream, four ctypes callback + trampolines and the caller's buffer, so an unbounded list grows the + process by tens of megabytes over a long-lived Reader. Every other + fragment test builds a fresh Reader per call, which never accumulates. + """ + init_path = os.path.join(FIXTURES_DIR, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_DIR, "dash1.m4s") + + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + self.addCleanup(reader.close) + + superseded = [] + for _ in range(25): + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + reader.with_fragment("video/mp4", init, frag) + self.assertLessEqual( + len(reader._fragment_streams), 1, + "fragment streams accumulated across repeated calls") + superseded.append(reader._fragment_streams[-1]) + + # Dropping the reference is not enough: the native stream is only + # released by close(), so every superseded wrapper must be closed. + self.assertTrue( + all(s.closed for s in superseded[:-1]), + "a superseded fragment stream was dropped without being closed") + + # The reader still works on the fragment it currently holds. + self.assertTrue(reader.json()) + def test_with_archive_post_consume_failure_consumes_handle(self): # Ownership taken, then the operation failed: # The handle is gone, so close() must not free it again. From 099c82ec2dee50a4a888d12acf7a05b742977b22 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:09:01 -0700 Subject: [PATCH 09/28] fix: Reorder to avoid potential deadlock --- src/c2pa/c2pa.py | 33 ++++++--- tests/test_unit_tests_threaded.py | 108 ++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 8 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index d762e285..4ef44d12 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -286,7 +286,17 @@ def _lock(self): signing). Those calls release the GIL and re-enter caller-supplied Python, which may call back into this API on another thread. Only calls that touch no callbacks are serialized here. + + Raises in a forked child rather than returning the lock. + A child inherits this lock in whatever state it had at fork(), + and a thread holding it does not exist in the child to release it, + so acquiring it there waits and waits and waits. + The child's copy is unusable for the same reason a closed resource is, + and reports the same error. """ + if is_foreign_process(self): + raise C2paError(f"{type(self).__name__} is closed") + lock = getattr(self, '_op_lock', None) if lock is None: lock = threading.RLock() @@ -387,23 +397,30 @@ def _teardown(self, free_handle: bool): Holds the operation lock so the free cannot happen between another thread's state check and its use of the handle in a native call. + + The forked-child case is handled before the lock is taken, because + _lock() refuses in a child: this path has to finish rather than report + an error, so it cannot rely on acquiring. """ + if is_foreign_process(self): + # The parent owns the handle and frees its own copy. Mark this one + # closed and drop the pointer so the child cannot use or free it. + self._handle = None + self._lifecycle_state = LifecycleState.CLOSED + return + with self._lock(): if getattr(self, '_inflight', 0) > 0: - # A native call is running that re-enters caller Python and - # is still using this handle. Record the intent and whichever - # caller leaves _native_call last performs the free. + # A native call is running that re-enters calling non-native code + # and is still using this handle. + # Record the intent and whichever caller leaves + # _native_call last performs the free. # Mark the resource closed now so it cannot be used # while the free is pending. self._pending_teardown = free_handle self._lifecycle_state = LifecycleState.CLOSED return - if is_foreign_process(self): - self._handle = None - self._lifecycle_state = LifecycleState.CLOSED - return - self._lifecycle_state = LifecycleState.CLOSED self._safe_release() diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 5cbccea2..c7dca601 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -184,6 +184,114 @@ def test_foreign_pid_close_marks_closed(self): self.assertFalse(obj._initialized) +class TestForkedChildDoesNotDeadlock(unittest.TestCase): + """A forked child must never block on a lock the parent held at fork(). + + Locking a resource for the duration of an operation means a child that + forks while some thread holds that lock inherits it locked, with the owner + thread gone. Anything in the child that acquires it waits forever. + + The failure mode is a hang: each operation runs on a worker thread and + is joined with a timeout: a test that called it directly would hang the + runner instead of failing. + """ + + _TIMEOUT = 5.0 + + def _foreign_reader_with_lock_held(self): + """A Reader in the state a forked child inherits: + lock held by another thread, and stamped with a PID other than this process's. + """ + with open(DEFAULT_TEST_FILE, "rb") as asset: + reader = Reader("image/jpeg", asset) + holding = threading.Event() + release = threading.Event() + + def hold_the_lock(): + with reader._lock(): + holding.set() + release.wait(30) + + holder = threading.Thread(target=hold_the_lock, daemon=True) + holder.start() + self.assertTrue(holding.wait(self._TIMEOUT), + "helper thread never acquired the lock") + self.addCleanup(holder.join, self._TIMEOUT) + self.addCleanup(release.set) + + reader._owner_pid = os.getpid() + 1 + return reader + + def _run_with_timeout(self, operation): + """Run operation on a worker; return 'ok', the exception, or None if it + was still running when the timeout expired.""" + result = {} + + def run(): + try: + operation() + result["outcome"] = "ok" + except BaseException as e: # noqa: BLE001 - asserted on below + result["outcome"] = e + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(self._TIMEOUT) + return result.get("outcome") + + def test_locked_read_raises_instead_of_blocking(self): + reader = self._foreign_reader_with_lock_held() + outcome = self._run_with_timeout(reader.json) + self.assertIsNotNone( + outcome, "json() blocked on a lock inherited from the parent") + self.assertIsInstance(outcome, Error) + + def test_native_call_path_raises_instead_of_blocking(self): + reader = self._foreign_reader_with_lock_held() + outcome = self._run_with_timeout( + lambda: reader.resource_to_stream("any-uri", io.BytesIO())) + self.assertIsNotNone( + outcome, + "resource_to_stream() blocked on a lock inherited from the parent") + self.assertIsInstance(outcome, Error) + + def test_close_still_completes(self): + reader = self._foreign_reader_with_lock_held() + self.assertEqual(self._run_with_timeout(reader.close), "ok", + "close() must neither block nor raise") + + def test_teardown_still_completes(self): + # Cleanup has to finish, not report an error. + reader = self._foreign_reader_with_lock_held() + self.assertEqual( + self._run_with_timeout( + lambda: reader._teardown(free_handle=True)), "ok", + "_teardown() must neither block nor raise") + self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) + self.assertIsNone(reader._handle) + + def test_parent_copy_unaffected(self): + """The child closing its copy must leave the parent's usable. + """ + with open(DEFAULT_TEST_FILE, "rb") as asset: + reader = Reader("image/jpeg", asset) + self.addCleanup(reader.close) + before = reader.json() + + pid = os.fork() + if pid == 0: + try: + reader.close() + os._exit(0) + except BaseException: + os._exit(1) + _, status = os.waitpid(pid, 0) + + self.assertEqual(status >> 8, 0, "child could not close its own copy") + self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) + self.assertEqual(reader.json(), before) + + class TestHelpers(unittest.TestCase): def test_record_and_detect_own_pid(self): From a59f06bb4d17e14caabb2c683a286eabc23e7bbd Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:19:19 -0700 Subject: [PATCH 10/28] Update iterations count in baseline.json --- tests/perf/baseline.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index feb51bd0..7ce38df4 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -3,7 +3,7 @@ "memray_version": "1.19.3", "python_version": "3.12.13", "c2pa_native_version": "c2pa-v0.90.15", - "iterations": 100, + "iterations": 200, "perf_env": "python-3.12-slim", "arch": "aarch64" }, @@ -302,4 +302,4 @@ "leaked_bytes": 3381191, "total_allocations": 966288 } -} \ No newline at end of file +} From aa605af0c6c929d016f4d9fae466ed7247b76845 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:24:03 -0700 Subject: [PATCH 11/28] fix: Error handling --- src/c2pa/c2pa.py | 7 ++- tests/test_unit_tests.py | 93 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 4ef44d12..83d9593b 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -517,7 +517,12 @@ def _swap_handle(self, new_handle): # Errors set by native lib, hinting at the cause of the error # These errors here means the pointer got somehow rejected by the lib, # so it is still ours to deal with. - _PRE_CONSUME_ERROR_TAGS = ("UntrackedPointer:", "WrongPointerType:") + _PRE_CONSUME_ERROR_TAGS = ( + "UntrackedPointer:", + "WrongPointerType:", + "NullParameter:", + "InvalidBufferSize:", + ) def _invoke_consume(self, ffi_call, error_message): """Run an FFI call that consumes this handle, returning its raw result. diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index 5bf091d5..d8c09bdd 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -8786,6 +8786,99 @@ def test_with_fragment_pre_consume_rejection_does_not_leak(self): self.assertTrue(reader.json()) reader.close() + def _reader_from_context(self): + """A Reader holding a fresh native handle and nothing else. + + Built through the FFI so the consuming call can be + set up with one deliberately invalid argument. + """ + context = Context() + self.addCleanup(context.close) + reader = Reader.__new__(Reader) + ManagedResource.__init__(reader) + reader._init_attrs() + with context._native_call(): + reader._create_and_activate( + lambda: c2pa_module._lib.c2pa_reader_from_context( + context.execution_context), + "Failed to create reader: {}") + return reader + + def test_null_parameter_rejection_retains_the_handle(self): + """A null argument is rejected before the reader is untracked. + Ownership never transferred, so the handle is still ours to free. + Treating it as consumed leaks one reader per call. + """ + reader = self._reader_from_context() + handle = reader._handle + freed = self._instrument_frees() + + with self.assertRaises(Error) as caught: + with reader._native_call(): + reader._consume_and_swap( + lambda h: c2pa_module._lib.c2pa_reader_with_stream( + h, b"image/jpeg", None), + "Failed to configure reader: {}") + + self.assertIn("NullParameter", str(caught.exception)) + self.assertIsNotNone(reader._handle, "the retained handle was dropped") + self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) + + reader.close() + self.assertEqual( + self._free_count(freed, handle), 1, + "a handle the native side never took was leaked") + + def test_invalid_buffer_size_rejection_retains_the_handle(self): + """A zero-length manifest buffer is rejected before the untrack.. + """ + reader = self._reader_from_context() + handle = reader._handle + freed = self._instrument_frees() + empty = (ctypes.c_ubyte * 4)() + + with Stream(io.BytesIO(b"abc")) as stream_obj: + with self.assertRaises(Error) as caught: + with reader._native_call(): + reader._consume_and_swap( + lambda h: ( + c2pa_module._lib + .c2pa_reader_with_manifest_data_and_stream( + h, b"image/jpeg", stream_obj._stream, + empty, 0) + ), + "Failed to configure reader: {}") + + self.assertIn("InvalidBufferSize", str(caught.exception)) + self.assertIsNotNone(reader._handle, "the retained handle was dropped") + self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) + + reader.close() + self.assertEqual( + self._free_count(freed, handle), 1, + "a handle the native side never took was leaked") + + def test_repeated_rejections_do_not_accumulate_handles(self): + """Every rejected call must give its handle back, not just the first. + """ + handles = [] + freed = self._instrument_frees() + + for _ in range(10): + reader = self._reader_from_context() + handles.append(reader._handle) + with self.assertRaises(Error): + with reader._native_call(): + reader._consume_and_swap( + lambda h: c2pa_module._lib.c2pa_reader_with_stream( + h, b"image/jpeg", None), + "Failed to configure reader: {}") + reader.close() + + leaked = [h for h in handles if self._free_count(freed, h) == 0] + self.assertEqual( + leaked, [], f"{len(leaked)} of {len(handles)} handles leaked") + def test_repeated_with_fragment_does_not_accumulate_streams(self): """Repeated calls on one Reader must not pile up fragment streams. From ab110f73386a1883273d9676ba33ccf49bd07f13 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:49:04 -0700 Subject: [PATCH 12/28] fix: Rewrite some threaded tests to avoid multifork issues --- tests/test_unit_tests_threaded.py | 63 +++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index c7dca601..85097321 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -272,24 +272,49 @@ def test_teardown_still_completes(self): def test_parent_copy_unaffected(self): """The child closing its copy must leave the parent's usable. + + Runs in a subprocess so that the fork happens in a single-threaded + process. Operations that reach the network, such as reading an asset + with a remote manifest, start background native threads that outlive + the object that triggered them, and forking a multi-threaded process + can lead to issues. """ - with open(DEFAULT_TEST_FILE, "rb") as asset: - reader = Reader("image/jpeg", asset) - self.addCleanup(reader.close) - before = reader.json() + source = textwrap.dedent(""" + import os, sys + from c2pa import Reader + from c2pa.c2pa import LifecycleState - pid = os.fork() - if pid == 0: - try: - reader.close() - os._exit(0) - except BaseException: - os._exit(1) - _, status = os.waitpid(pid, 0) + asset_path = sys.argv[1] + with open(asset_path, "rb") as asset: + reader = Reader("image/jpeg", asset) + before = reader.json() - self.assertEqual(status >> 8, 0, "child could not close its own copy") - self.assertEqual(reader._lifecycle_state, LifecycleState.ACTIVE) - self.assertEqual(reader.json(), before) + pid = os.fork() + if pid == 0: + try: + reader.close() + os._exit(0) + except BaseException: + os._exit(1) + _, status = os.waitpid(pid, 0) + + assert status >> 8 == 0, "child could not close its own copy" + assert reader._lifecycle_state == LifecycleState.ACTIVE + assert reader.json() == before + reader.close() + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", source, DEFAULT_TEST_FILE], + capture_output=True, text=True, timeout=120) + + self.assertEqual( + result.returncode, 0, + "parent copy was affected by the child (rc={}):\n{}".format( + result.returncode, result.stderr[-2000:])) + self.assertIn("OK", result.stdout) + self.assertNotIn("DeprecationWarning", result.stderr) class TestHelpers(unittest.TestCase): @@ -813,12 +838,12 @@ def setUp(self): with open(os.path.join(self.data_dir, "es256_private.key"), "rb") as key_file: self.key = key_file.read() - # Create a local Es256 signer with certs and a timestamp server + # Create a local Es256 signer with certs and no timestamp server. self.signer_info = C2paSignerInfo( alg=b"es256", sign_cert=self.certs, private_key=self.key, - ta_url=b"http://timestamp.digicert.com" + ta_url=None ) self.signer = Signer.from_info(self.signer_info) @@ -3377,7 +3402,7 @@ def test_close_during_sign_does_not_deadlock(self): alg=b"es256", sign_cert=certs, private_key=key, - ta_url=b"http://timestamp.digicert.com", + ta_url=None, ) manifest = { "claim_generator": "python_test", @@ -3802,7 +3827,7 @@ def test_sign_with_internal_close_frees_once(self): alg=b"es256", sign_cert=self.certs, private_key=self.private_key, - ta_url=b"http://timestamp.digicert.com", + ta_url=None, ) manifest = { "claim_generator": "python_test", From fc630b80c5cf7c9cae7292f40a70dabd78a4523b Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:59:26 -0700 Subject: [PATCH 13/28] fix: Docs --- docs/native-resources-management.md | 44 +++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 1cf057f0..2ab550a6 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -106,15 +106,27 @@ Therefore, the managed resources have the following principles: ### Double-free risk mitigations -Three distinct risks. Two have a mechanism in this layer; the third is the caller's to synchronize: +Three distinct risks, each with its own mechanism in this layer: | Hazard | Covered by | How | | --- | --- | --- | -| Freeing a pointer a consuming call already took (single flow) | `_swap_handle` / `_teardown(free_handle=False)` triage | The consumed pointer is abandoned, never freed. The retained-vs-consumed decision reads the native error tag (`UntrackedPointer:` / `WrongPointerType:` mean not taken). | +| Freeing a pointer a consuming call already took (single flow) | `_swap_handle` / `_teardown(free_handle=False)` triage | The consumed pointer is abandoned, never freed. The retained-vs-consumed decision reads the native error tag (`UntrackedPointer:` / `WrongPointerType:` / `NullParameter:` / `InvalidBufferSize:` mean not taken). | | A forked child freeing a pointer its parent owns | PID stamp (`record_owner_pid` / `is_foreign_process`) | Cleanup in a process that did not allocate the pointer nulls the handle and marks `CLOSED` without freeing (see [Fork safety](#fork-safety)). | -| Two **threads** in one process racing frees on distinct objects, where the allocator recycles a just-freed address | Not covered here | `ManagedResource` has no lock and no thread stamping. The PID stamp cannot see it: sibling threads share a PID. Safety for genuinely shared handles must come from the caller's own synchronization or from the native registry, not this layer. | +| Two **threads** racing a `close()` against an in-flight native call on the same object, where the allocator recycles a just-freed address | `_op_lock` / `_native_call()` / `_pending_teardown` | A close arriving while a native call is in flight is recorded rather than applied. The last caller to leave `_native_call()` performs the deferred free (see [Locking and in-flight tracking](#locking-and-in-flight-tracking)). | -The PID stamp is fork-only: it compares process IDs, and two threads in the same process always match. Sharing one `ManagedResource` instance across threads without external synchronization is outside what this layer protects against. +The PID stamp is fork-only: it compares process IDs, and two threads in the same process always match on that PID. Sharing one `ManagedResource` instance across threads still needs locks: nothing here protects two threads racing on genuinely distinct objects that happen to share an allocator. + +## Locking and in-flight tracking + +Each `ManagedResource` holds a reentrant lock, `_op_lock`, and a counter, `_inflight`, that together serialize teardown against concurrent use from other threads. + +A lock (Python's `threading.Lock`) can be acquired once, and a second `acquire()` from the same thread on that lock blocks forever, waiting on a lock that thread itself is holding. A reentrant lock (`threading.RLock`) tracks which thread holds it and how many times: the owning thread can acquire it again without blocking, and the lock is only released once that thread has released it the same number of times it acquired it. A different thread still blocks until the owner releases fully. + +`_op_lock` is an `RLock` rather than a plain `Lock` for two reasons specific to this code. First, a finalizer (`__del__`) can run at any bytecode boundary — including one in the middle of a method that has already acquired the lock on this same thread — so `__del__` calling back into locked code must not deadlock against itself. Second, a consuming call tears the handle down from inside the locked region it is already holding: `_teardown()` is called while `_op_lock` is held, and it needs to acquire the same lock again rather than re-entering as a different, blocked acquisition. `_lock()` returns it, except in a forked child: there it raises `C2paError` immediately rather than blocking, because the thread that might hold the lock at fork time does not exist in the child to release it, and waiting on it would hang forever (see [Fork safety](#fork-safety)). + +The lock is never held across a native call that drives a stream callback: construction, `resource_to_stream`, the Builder stream methods, and signing all release the GIL and call back into caller-supplied Python, which may itself call into this API on another thread. Holding `_op_lock` there would deadlock against that reentry. Those calls go through `_native_call()` instead: a context manager that increments `_inflight` under the lock, yields to run the native call unlocked, then decrements `_inflight` on the way out. If `_teardown()` runs while a call is in flight, it records the requested `free_handle` value in `_pending_teardown` and marks the resource `CLOSED` immediately, so no other caller can start using it, but defers the actual free. The last `_native_call()` to exit picks up `_pending_teardown` and runs `_teardown()` for real. + +`Context.__init__` wraps the signer hand-off in `signer._native_call()`, so a `signer.close()` on another thread cannot free the handle between the state check and the consuming call. `Builder._sign_internal` wraps the sign call in `self._native_call()` and, when an explicit `Signer` is passed, nests `signer._native_call()` inside it in that fixed order, so two concurrent `sign()` calls sharing one `Signer` cannot deadlock by acquiring the two locks in opposite orders. The Builder's `close()` after signing runs outside its own `_native_call()` block, so a teardown deferred during the call still executes once the call returns. ## Guarantees provided by ManagedResource @@ -317,6 +329,8 @@ While `ACTIVE`, callers can use `.add_ingredient()`, `.add_action()`, etc. repea The native sign call borrows the builder's pointer rather than taking ownership of it, so `Builder` never marks it consumed and the pointer is freed normally through `c2pa_free`. The close enforces single use; it is not a memory-management requirement. +The sign call runs inside `self._native_call()`, nesting `signer._native_call()` when the caller passes an explicit `Signer`, and `close()` runs after that block exits (see [Locking and in-flight tracking](#locking-and-in-flight-tracking) for why the order matters and what it protects against). + ## Ownership transfer Some operations transfer a native pointer from one object to another. When this happens, the original object must stop managing the pointer (e.g. so it is not freed twice). @@ -338,6 +352,8 @@ sequenceDiagram C->>X: Context(settings, signer) X->>B: with _NativeBuilder() (owns the builder, close() frees it on any failure) X->>S: _ensure_valid_state() + X->>S: enter _native_call() + Note right of S: Pins the Signer active for the duration:
a close() on another thread now waits
instead of freeing the handle mid-transfer X->>X: copy signer._callback_cb to _signer_callback_cb Note right of X: Pin the callback first:
the Signer is about to be consumed X->>S: _consume_no_replacement(set_signer) @@ -346,12 +362,13 @@ sequenceDiagram alt status 0 (success) S->>S: _teardown(free_handle=False) Note right of S: Consumed: native took the signer - else pre-consume rejection (UntrackedPointer / WrongPointerType) + else pre-consume rejection (one of _PRE_CONSUME_ERROR_TAGS) Note right of S: Rejected before ownership moved:
Signer retained, typed error raised else other error S->>S: _teardown(free_handle=False) Note right of S: Native took it then failed and dropped it end + X->>S: exit _native_call() X->>B: _consume_into(build) B->>N: c2pa_context_builder_build(builder_ptr) @@ -362,7 +379,8 @@ sequenceDiagram Details in that sequence that are easy to get wrong: - The callback is copied to the Context *before* the transfer. A successful consume runs `_release()`, which drops the Signer's reference to the callback; a Context that copied it afterwards would be pointing at a callback nothing keeps alive. -- `set_signer` does not always take the pointer. A pre-consume rejection (`UntrackedPointer:` / `WrongPointerType:`) leaves the Signer `ACTIVE` and retained, so the triage must read the native error before deciding to close it. Treating every failure as "consumed" would close a signer the native side never took. +- The state check and the consuming call both run inside `signer._native_call()`, so a `signer.close()` racing on another thread cannot free the handle in the gap between them. If a close does arrive while the transfer is in flight, it is recorded as a pending teardown and applied once the transfer finishes (see [Locking and in-flight tracking](#locking-and-in-flight-tracking)). +- `set_signer` does not always take the pointer. A pre-consume rejection (one of `_PRE_CONSUME_ERROR_TAGS`) leaves the Signer `ACTIVE` and retained, so the triage must read the native error before deciding to close it. Treating every failure as "consumed" would close a signer the native side never took. - A `ctypes.ArgumentError` from `set_signer` is re-raised untouched by `_invoke_consume`: marshalling failed, the native function never ran, and the Signer still owns its handle. Only calls that reached native go through the consumed/retained triage. - The builder is never held as a raw local across the signer and build calls. `_NativeBuilder`'s `with` block owns it: a settings error, a retained-signer error, a build rejection, or an async interrupt all free it through `close()`, and a successful build consumes it so `close()` is then a no-op. The old raw-pointer recovery block that used to free `builder_ptr` on the un-reached-build path is gone. @@ -396,6 +414,8 @@ stateDiagram-v2 On success the object stays `ACTIVE` because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from a consumed teardown (`_teardown(free_handle=False)`), where the object transitions to `CLOSED` and becomes unusable. On the success path the old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. The failure path is different and is covered by the triage in [`_consume_and_swap()`](#_consume_and_swap). +`Reader.with_fragment()` runs the swap inside `self._native_call()`, and keeps a `_fragment_streams` list holding the `Stream` wrapper for the current fragment. Each call to `with_fragment()` replaces that list rather than appending to it, closing the previous fragment's wrapper immediately: the native reader never reads a superseded fragment back, and each open wrapper pins a native stream, its callbacks, and the caller's buffer. + ### `_consume_and_swap()` Every call of this shape goes through one helper, which takes the FFI call as a callable and handles the outcomes: @@ -428,7 +448,7 @@ The two failure paths are indistinguishable from the return value alone. Only th | Native error | Who owns the handle | What the helper does | | --- | --- | --- | -| `UntrackedPointer:` or `WrongPointerType:` | Still ours: rejected before ownership moved | Handle kept, resource stays `ACTIVE`, typed error raised. Normal cleanup frees it later. | +| One of `_PRE_CONSUME_ERROR_TAGS` | Still ours: rejected before ownership moved | Handle kept, resource stays `ACTIVE`, typed error raised. Normal cleanup frees it later. | | Any other error | Taken, then the operation failed | `_teardown(free_handle=False)`: the native side already dropped the value, so nothing is freed here. Resource goes `CLOSED`, error typed from the native message. | | No error at all | Unknown | `_release_handle()` guarded free, the caller's message is raised with `"Unknown error"` filled in. | @@ -448,17 +468,17 @@ Three consume helpers share this triage; they differ only in what the FFI call r A consuming FFI call can fail. It may reject the borrowed pointer before taking it, or it may take ownership first and then, on a later failure, drop the value itself. -The native error message indicates which of the errors happened. A rejection carries one of the `_PRE_CONSUME_ERROR_TAGS` (`UntrackedPointer:` or `WrongPointerType:`), which means the handle was never taken and is retained. Any other error message means the native side may have taken ownership and already dropped the value. On top of those, preparing the call's own arguments can fail in Python before the native function ever runs (for example, encoding a bad value or a ctypes marshalling error other than `ArgumentError`), and that outcome is handled separately. +The native error message indicates which of the errors happened. A rejection carries one of the `_PRE_CONSUME_ERROR_TAGS`, which means the handle was never taken and is retained. Any other error message means the native side may have taken ownership and already dropped the value. On top of those, preparing the call's own arguments can fail in Python before the native function ever runs (for example, encoding a bad value or a ctypes marshalling error other than `ArgumentError`), and that outcome is handled separately. -The two settled branches each take the exact action their ownership implies. A pre-consume rejection (an error prefixed `UntrackedPointer:` or `WrongPointerType:`) means the handle is still the caller's, so it is retained and freed later by normal cleanup. Any other native error means the value is already gone, so `_teardown(free_handle=False)` runs the Python-side cleanup without freeing anything. +The two settled branches each take the exact action their ownership implies. A pre-consume rejection (one of the `_PRE_CONSUME_ERROR_TAGS`) means the handle is still the caller's, so it is retained and freed later by normal cleanup. Any other native error means the value is already gone, so `_teardown(free_handle=False)` runs the Python-side cleanup without freeing anything. Always calling the guarded free instead, even where the value is known to be gone, is tempting because a stale free looks like a harmless `-1` no-op. It is only harmless while the freed address stays unclaimed. The native registry rejects an address it no longer tracks, but once another thread allocates a fresh tracked object at that recycled address, the registry does track it again — and a stale free aimed at the old value would now find a live entry and destroy a different thread's object. The scenario is unlikely, but not unreachable: it needs a second thread inside its own FFI call, an allocator that hands back the exact address just freed, and that reuse to happen during the (narrow) window between the native drop and this free. But the window is real under concurrent use. The failure is a silent cross-thread corruption rather than a clean error, and the free is not needed in the first place on this branch. So where the value is known to be consumed, the free is skipped rather than issued and left to the registry to reject. The native error slot stays sticky: it holds whatever it last held until the next error overwrites it, and nothing clears it in between. Issuing an unneeded free would set an untracked-pointer error there that a later caller could mistake for the failure it actually asked about, so skipping the free keeps the slot free for the next real error. `_release_handle()` (a guarded free) is reserved for the two branches where ownership is not known for certain: a Python exception raised before native reports anything, and a failure that leaves the error slot empty (which no defined native failure is expected to produce). In both, a guarded free is a good default, since it is a real free when the handle is still ours and a `-1` no-op when the native side already took it. -None of this is protected by a lock on the Python side: `ManagedResource` has no thread-safety mechanism of its own, and the retained-vs-consumed guarantee comes entirely from the native pointer registry and its thread-local error slot. As noted under [Which double-free risks this layer guards](#double-free-risk-mitigations), sharing one instance across threads without external synchronization is the caller's responsibility. This is a different hazard from [Fork safety](#fork-safety), which concerns a forked child process, not a thread within the same process. +The retained-vs-consumed decision itself comes entirely from the native pointer registry and its thread-local error slot, not from a Python-side lock: `_op_lock` guards concurrent teardown against a call still in flight (see [Locking and in-flight tracking](#locking-and-in-flight-tracking)), but it plays no part in reading which rejection prefix the native side set. That is a different hazard from [Fork safety](#fork-safety), which concerns a forked child process, not a thread within the same process. -A consuming C FFI function first removes the pointer from its registry, then reconstructs the owned value from it. `untrack_or_return!` runs ahead of `Box::from_raw` in `c2pa_c_ffi`. If the address is unknown or the wrong type, the untrack step fails before ownership is taken and sets an error whose prefix (`UntrackedPointer:` or `WrongPointerType:`) identifies it as a pre-consume rejection. Once the value has been reconstructed, a later failure simply drops it, the same as any owned value going out of scope. The Python side stays defensive (and as generic as possible) rather than assuming any exact behavior: it retains the handle when it recognizes one of those rejection prefixes, and where the outcome is unclear it falls back to the guarded free. A native side that behaved differently would degrade in one of two bounded ways: If it kept a pointer the Python side treated as consumed, nothing would free that pointer and it would leak. If it had already released a pointer the Python side then tried to free, the registry would not find the address and the free would return `-1` without touching memory. +A consuming C FFI function first removes the pointer from its registry, then reconstructs the owned value from it. `untrack_or_return!` runs ahead of `Box::from_raw` in `c2pa_c_ffi`. If the address is unknown or the wrong type, the untrack step fails before ownership is taken and sets an error whose prefix (one of the `_PRE_CONSUME_ERROR_TAGS`) identifies it as a pre-consume rejection. Once the value has been reconstructed, a later failure simply drops it, the same as any owned value going out of scope. The Python side stays defensive (and as generic as possible) rather than assuming any exact behavior: it retains the handle when it recognizes one of those rejection prefixes, and where the outcome is unclear it falls back to the guarded free. A native side that behaved differently would degrade in one of two bounded ways: a pointer the native side still held but Python treated as consumed would leak, since nothing would free it; a pointer the native side had already released but Python then tried to free would return `-1` from the registry without touching memory. ### Adopting the handle before giving it away @@ -531,6 +551,8 @@ sequenceDiagram Both `_cleanup_resources()` and the consumed teardown take this branch. Neither simply skips the work: they null the handle and mark the object `CLOSED` so the child cannot go on to use it or try to free it later. Mutating the child's copy has no effect on the parent's, which is untouched and still valid. +`_teardown()` checks `is_foreign_process()` before taking `_op_lock`, not after, so the foreign-process branch above never tries to acquire a lock in the child. The lock itself would raise there anyway (see [Locking and in-flight tracking](#locking-and-in-flight-tracking)), but `_teardown()` needs to finish its cleanup rather than raise. Therefore, it settles the fork case first and only reaches for the lock once it knows this process owns the pointer. + The memory the child skips is not lost for good. A child that calls `exec()` replaces its address space; a child that exits has its memory reclaimed by the OS. Even a long-lived child (a `multiprocessing` worker using the fork start method) retains at most the objects it inherited at fork time, which is a bounded, one-off amount rather than a growing leak. Anything the child allocates itself carries the child's own PID and is freed normally. > [!NOTE] From e657a236f985eb865dc18ebc2dea8283fab3fac2 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:33:27 -0700 Subject: [PATCH 14/28] fix: Rebaseline --- tests/perf/baseline.json | 360 ++++++++++++++++++------------------ tests/perf/reports/.gitkeep | 0 2 files changed, 180 insertions(+), 180 deletions(-) delete mode 100644 tests/perf/reports/.gitkeep diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index 7ce38df4..74a431a9 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -8,298 +8,298 @@ "arch": "aarch64" }, "reader_jpeg_legacy": { - "peak_bytes": 3851610, - "leaked_bytes": 3351823, - "total_allocations": 1362322 + "peak_bytes": 3878176, + "leaked_bytes": 3381657, + "total_allocations": 1307172 }, "reader_jpeg_with_context": { - "peak_bytes": 3845367, - "leaked_bytes": 3345097, - "total_allocations": 1349879 + "peak_bytes": 3872284, + "leaked_bytes": 3374437, + "total_allocations": 1299545 }, "reader_manifest_data_context": { - "peak_bytes": 7636730, - "leaked_bytes": 3468040, - "total_allocations": 1147359 + "peak_bytes": 7658137, + "leaked_bytes": 3491877, + "total_allocations": 1098306 }, "reader_mp4": { - "peak_bytes": 4222601, - "leaked_bytes": 3345724, - "total_allocations": 4095915 + "peak_bytes": 4238670, + "leaked_bytes": 3374080, + "total_allocations": 3984581 }, "reader_wav": { - "peak_bytes": 4523095, - "leaked_bytes": 3355666, - "total_allocations": 742391 + "peak_bytes": 4539135, + "leaked_bytes": 3384038, + "total_allocations": 739057 }, "builder_sign_jpeg_legacy": { - "peak_bytes": 7785129, - "leaked_bytes": 3468507, - "total_allocations": 1041412 + "peak_bytes": 7810402, + "leaked_bytes": 3498186, + "total_allocations": 1019644 }, "builder_sign_jpeg_with_context": { - "peak_bytes": 7779538, - "leaked_bytes": 3463042, - "total_allocations": 1027485 + "peak_bytes": 7802790, + "leaked_bytes": 3490850, + "total_allocations": 1005722 }, "builder_sign_png_legacy": { - "peak_bytes": 8023081, - "leaked_bytes": 3468300, - "total_allocations": 3883115 + "peak_bytes": 8048349, + "leaked_bytes": 3498022, + "total_allocations": 3861729 }, "builder_sign_png_with_context": { - "peak_bytes": 8017008, - "leaked_bytes": 3462829, - "total_allocations": 3869515 + "peak_bytes": 8041266, + "leaked_bytes": 3491795, + "total_allocations": 3847713 }, "builder_sign_jpeg_parallel_split_pool": { - "peak_bytes": 45854797, - "leaked_bytes": 3840928, - "total_allocations": 1035646 + "peak_bytes": 45869711, + "leaked_bytes": 3860295, + "total_allocations": 1009812 }, "builder_sign_jpeg_parallel_split_barrier": { - "peak_bytes": 45844809, - "leaked_bytes": 3861014, - "total_allocations": 1037741 + "peak_bytes": 45838322, + "leaked_bytes": 3859113, + "total_allocations": 1008528 }, "builder_sign_png_parallel_split_pool": { - "peak_bytes": 46586728, - "leaked_bytes": 3868054, - "total_allocations": 3877696 + "peak_bytes": 46107474, + "leaked_bytes": 3877946, + "total_allocations": 3851810 }, "builder_sign_png_parallel_split_barrier": { - "peak_bytes": 46082548, - "leaked_bytes": 3879161, - "total_allocations": 3879780 + "peak_bytes": 46075853, + "leaked_bytes": 3877260, + "total_allocations": 3850542 }, "builder_sign_gif": { - "peak_bytes": 14635465, - "leaked_bytes": 3461270, - "total_allocations": 17017654 + "peak_bytes": 14660656, + "leaked_bytes": 3491475, + "total_allocations": 16995947 }, "builder_sign_heic": { - "peak_bytes": 4698434, - "leaked_bytes": 3469086, - "total_allocations": 1563419 + "peak_bytes": 4723711, + "leaked_bytes": 3499336, + "total_allocations": 1529895 }, "builder_sign_m4a": { - "peak_bytes": 18833496, - "leaked_bytes": 3469085, - "total_allocations": 5194205 + "peak_bytes": 18859208, + "leaked_bytes": 3499290, + "total_allocations": 5160957 }, "builder_sign_webp": { - "peak_bytes": 8991237, - "leaked_bytes": 3461271, - "total_allocations": 916145 + "peak_bytes": 9016473, + "leaked_bytes": 3491521, + "total_allocations": 898326 }, "builder_sign_avi": { - "peak_bytes": 7130933, - "leaked_bytes": 3461270, - "total_allocations": 89982012 + "peak_bytes": 7156127, + "leaked_bytes": 3491475, + "total_allocations": 89959516 }, "builder_sign_mp4": { - "peak_bytes": 6245379, - "leaked_bytes": 3469085, - "total_allocations": 3788717 + "peak_bytes": 6270688, + "leaked_bytes": 3499335, + "total_allocations": 3753347 }, "builder_sign_tiff": { - "peak_bytes": 13213169, - "leaked_bytes": 3461271, - "total_allocations": 10862700 + "peak_bytes": 13238405, + "leaked_bytes": 3491521, + "total_allocations": 10845456 }, "builder_sign_jpeg_parent_of": { - "peak_bytes": 14265295, - "leaked_bytes": 3461665, - "total_allocations": 2506107 + "peak_bytes": 14290485, + "leaked_bytes": 3492132, + "total_allocations": 2434129 }, "builder_sign_jpeg_component_of": { - "peak_bytes": 14266996, - "leaked_bytes": 3462012, - "total_allocations": 2551180 + "peak_bytes": 14291315, + "leaked_bytes": 3491288, + "total_allocations": 2477796 }, "builder_sign_jpeg_parent_and_component": { - "peak_bytes": 14665241, - "leaked_bytes": 3614613, - "total_allocations": 4523960 + "peak_bytes": 14638503, + "leaked_bytes": 3636596, + "total_allocations": 4394837 }, "builder_sign_jpeg_parent_and_component_mixed_mime": { - "peak_bytes": 14568780, - "leaked_bytes": 3462718, - "total_allocations": 5517180 + "peak_bytes": 14593417, + "leaked_bytes": 3492387, + "total_allocations": 5447516 }, "builder_sign_jpeg_two_components_same_mime": { - "peak_bytes": 14559274, - "leaked_bytes": 3564233, - "total_allocations": 4497379 + "peak_bytes": 14631537, + "leaked_bytes": 3636600, + "total_allocations": 4367414 }, "builder_sign_jpeg_two_components_mixed_mime": { - "peak_bytes": 14564839, - "leaked_bytes": 3461873, - "total_allocations": 5490592 + "peak_bytes": 14589983, + "leaked_bytes": 3492082, + "total_allocations": 5419842 }, "builder_sign_jpeg_archive_roundtrip": { - "peak_bytes": 14297571, - "leaked_bytes": 3481212, - "total_allocations": 3467149 + "peak_bytes": 14321971, + "leaked_bytes": 3512017, + "total_allocations": 3343806 }, "builder_from_archive_roundtrip": { - "peak_bytes": 14297349, - "leaked_bytes": 3480475, - "total_allocations": 3101030 + "peak_bytes": 14320730, + "leaked_bytes": 3510869, + "total_allocations": 2987299 }, "builder_with_archive_swap": { - "peak_bytes": 3681081, - "leaked_bytes": 3350198, - "total_allocations": 704373 + "peak_bytes": 3720558, + "leaked_bytes": 3389404, + "total_allocations": 708591 }, "reader_with_fragment_swap": { - "peak_bytes": 3778159, - "leaked_bytes": 3353205, - "total_allocations": 3787587 + "peak_bytes": 3805398, + "leaked_bytes": 3382233, + "total_allocations": 3769246 + }, + "reader_with_fragment_repeated": { + "peak_bytes": 3803023, + "leaked_bytes": 3380589, + "total_allocations": 1842543 }, "with_fragment_pre_consume_rejection": { - "peak_bytes": 3778057, - "leaked_bytes": 3354795, - "total_allocations": 2094004 + "peak_bytes": 3805255, + "leaked_bytes": 3384325, + "total_allocations": 2093470 }, "with_archive_post_consume_failure": { - "peak_bytes": 3350600, - "leaked_bytes": 3308056, - "total_allocations": 175290 + "peak_bytes": 3388641, + "leaked_bytes": 3346670, + "total_allocations": 185458 }, "with_fragment_marshalling_error": { - "peak_bytes": 3708068, - "leaked_bytes": 3352335, - "total_allocations": 2077090 + "peak_bytes": 3734139, + "leaked_bytes": 3381598, + "total_allocations": 2072540 }, "with_fragment_mixed_outcomes": { - "peak_bytes": 3779175, - "leaked_bytes": 3356294, - "total_allocations": 2656787 + "peak_bytes": 3803836, + "leaked_bytes": 3382987, + "total_allocations": 2650818 }, "builder_to_archive_with_ingredient": { - "peak_bytes": 14069232, - "leaked_bytes": 3337316, - "total_allocations": 1830896 + "peak_bytes": 14107950, + "leaked_bytes": 3375874, + "total_allocations": 1766289 }, "builder_sign_jpeg_archive_roundtrip_ingredient_in_archive": { - "peak_bytes": 14287046, - "leaked_bytes": 3481977, - "total_allocations": 5879957 + "peak_bytes": 14311785, + "leaked_bytes": 3511500, + "total_allocations": 5681125 }, "builder_write_ingredient_archive": { - "peak_bytes": 14069289, - "leaked_bytes": 3337377, - "total_allocations": 1805304 + "peak_bytes": 14107944, + "leaked_bytes": 3375872, + "total_allocations": 1742859 }, "builder_sign_jpeg_add_ingredient_from_archive": { - "peak_bytes": 14133742, - "leaked_bytes": 3480831, - "total_allocations": 3415920 + "peak_bytes": 14174086, + "leaked_bytes": 3512155, + "total_allocations": 3320279 }, "builder_ingredient_archive_roundtrip": { - "peak_bytes": 14284443, - "leaked_bytes": 3480809, - "total_allocations": 5132060 + "peak_bytes": 14310577, + "leaked_bytes": 3512018, + "total_allocations": 4973895 }, "builder_sign_jpeg_two_ingredient_archives": { - "peak_bytes": 14134560, - "leaked_bytes": 3481604, - "total_allocations": 4215728 + "peak_bytes": 14173988, + "leaked_bytes": 3512433, + "total_allocations": 4113195 }, "reader_error_no_manifest": { - "peak_bytes": 3564471, - "leaked_bytes": 3323629, - "total_allocations": 276175 + "peak_bytes": 3588164, + "leaked_bytes": 3352030, + "total_allocations": 276214 }, "builder_error_invalid_manifest": { - "peak_bytes": 3352053, - "leaked_bytes": 3297079, - "total_allocations": 113926 + "peak_bytes": 3388827, + "leaked_bytes": 3333835, + "total_allocations": 115678 }, "reader_string_apis": { - "peak_bytes": 3978113, - "leaked_bytes": 3346111, - "total_allocations": 2287335 + "peak_bytes": 4005286, + "leaked_bytes": 3375590, + "total_allocations": 2183974 }, "signer_construction": { - "peak_bytes": 3350893, - "leaked_bytes": 3288137, - "total_allocations": 153245 + "peak_bytes": 3388872, + "leaked_bytes": 3325939, + "total_allocations": 155796 }, "builder_from_context_construction": { - "peak_bytes": 3350600, - "leaked_bytes": 3288582, - "total_allocations": 112688 + "peak_bytes": 3388641, + "leaked_bytes": 3327073, + "total_allocations": 120460 }, "fork_reader_collect": { - "peak_bytes": 3850530, - "leaked_bytes": 3353063, - "total_allocations": 1328122 + "peak_bytes": 3877630, + "leaked_bytes": 3381450, + "total_allocations": 1271971 }, "fork_contended_mutex": { - "peak_bytes": 7679019, - "leaked_bytes": 3482128, - "total_allocations": 67472694 + "peak_bytes": 7646946, + "leaked_bytes": 3477655, + "total_allocations": 65668554 }, "fork_thread_local_orphan": { - "peak_bytes": 3936170, - "leaked_bytes": 3439733, - "total_allocations": 1381055 + "peak_bytes": 3960026, + "leaked_bytes": 3468161, + "total_allocations": 1324309 }, "fork_gc_cycle": { - "peak_bytes": 3850434, - "leaked_bytes": 3353160, - "total_allocations": 1332098 + "peak_bytes": 3875869, + "leaked_bytes": 3379622, + "total_allocations": 1276946 }, "fork_parent_frees_after_fork": { - "peak_bytes": 5447584, - "leaked_bytes": 3350400, - "total_allocations": 24829257 + "peak_bytes": 5561869, + "leaked_bytes": 3389085, + "total_allocations": 23724547 }, "fork_child_closes_then_parent_frees": { - "peak_bytes": 5446620, - "leaked_bytes": 3350407, - "total_allocations": 24829254 + "peak_bytes": 5563403, + "leaked_bytes": 3390349, + "total_allocations": 23724544 }, "fork_child_sys_exit": { - "peak_bytes": 3850546, - "leaked_bytes": 3353234, - "total_allocations": 1335925 + "peak_bytes": 3877646, + "leaked_bytes": 3381666, + "total_allocations": 1282172 }, "fork_stream_cleanup": { - "peak_bytes": 3464063, - "leaked_bytes": 3291969, - "total_allocations": 105340 + "peak_bytes": 3500857, + "leaked_bytes": 3329555, + "total_allocations": 105687 }, "fork_swap_cleanup": { - "peak_bytes": 3681171, - "leaked_bytes": 3350696, - "total_allocations": 714376 + "peak_bytes": 3720613, + "leaked_bytes": 3389867, + "total_allocations": 718596 }, "fork_contended_mutex_swap": { - "peak_bytes": 7302379, - "leaked_bytes": 3475147, - "total_allocations": 35948516 + "peak_bytes": 7306199, + "leaked_bytes": 3492867, + "total_allocations": 35863039 }, "fork_contended_mutex_wrap": { - "peak_bytes": 7288748, - "leaked_bytes": 3463411, - "total_allocations": 34847186 + "peak_bytes": 7295518, + "leaked_bytes": 3491027, + "total_allocations": 35014720 }, "fork_consumed_signer": { - "peak_bytes": 3350894, - "leaked_bytes": 3288906, - "total_allocations": 175055 + "peak_bytes": 3388873, + "leaked_bytes": 3327740, + "total_allocations": 196221 }, "swap_chain_churn": { - "peak_bytes": 3681161, - "leaked_bytes": 3350287, - "total_allocations": 672537 - }, - "reader_with_fragment_repeated": { - "peak_bytes": 3803564, - "leaked_bytes": 3381191, - "total_allocations": 966288 + "peak_bytes": 3720603, + "leaked_bytes": 3389458, + "total_allocations": 669590 } -} +} \ No newline at end of file diff --git a/tests/perf/reports/.gitkeep b/tests/perf/reports/.gitkeep deleted file mode 100644 index e69de29b..00000000 From 63fc5a7ad0ccc1a8e658aec7d922e80111ebc519 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:57:23 -0700 Subject: [PATCH 15/28] fix: Reorder locking --- src/c2pa/c2pa.py | 150 ++++++++++++++++-------------- tests/test_unit_tests_threaded.py | 60 ++++++++++++ 2 files changed, 141 insertions(+), 69 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 83d9593b..1ad8dce1 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -1923,6 +1923,8 @@ def __init__(self, file_like_stream): self._closed = False self._initialized = False self._stream = None + # Serializes close() and __del__ against a concurrent double-free. + self._close_lock = threading.Lock() # Generate unique stream ID using object ID and counter stream_counter = next(Stream._stream_id_counter) @@ -2144,22 +2146,22 @@ def __del__(self): try: if is_foreign_process(self): return - # Only cleanup if not already closed and we have a valid stream - if hasattr(self, '_closed') and not self._closed: - stream = self._stream - if hasattr(self, '_stream') and stream: - # Use internal cleanup to avoid calling close() which could - # cause issues - try: - _lib.c2pa_release_stream(stream) - except Exception: - # Destructors shouldn't raise exceptions - logger.error("Failed to release Stream") - pass - finally: - self._stream = None - self._closed = True - self._initialized = False + lock = getattr(self, '_close_lock', None) + with lock if lock is not None else contextlib.nullcontext(): + # Only cleanup if not already closed and we have a valid stream + if hasattr(self, '_closed') and not self._closed: + stream = self._stream + if hasattr(self, '_stream') and stream: + try: + _lib.c2pa_release_stream(stream) + except Exception: + # Destructors shouldn't raise exceptions + logger.error("Failed to release Stream") + pass + finally: + self._stream = None + self._closed = True + self._initialized = False except Exception: # Destructors must not raise exceptions pass @@ -2172,45 +2174,48 @@ def close(self): Errors during cleanup are logged but not raised to ensure cleanup. Multiple calls to close() are handled gracefully. """ - if self._closed: - return - if is_foreign_process(self): - self._closed = True - self._initialized = False - return + # Serializes against __del__ / a concurrent close(). + with self._close_lock: + if self._closed: + return + if is_foreign_process(self): + self._closed = True + self._initialized = False + return - try: - # Clean up stream first as it depends on callbacks - # Note: We don't close self._file_like_stream as we don't own it, - # the opener owns it. - stream = self._stream - if stream: - try: - _lib.c2pa_release_stream(stream) - except Exception as e: - logger.error( - Stream._ERROR_MESSAGES['stream_error'].format( - str(e))) - finally: - self._stream = None - - # Clean up callbacks - for attr in ['_read_cb', '_seek_cb', '_write_cb', '_flush_cb']: - if hasattr(self, attr): + try: + # Clean up stream first as it depends on callbacks + # Note: We don't close self._file_like_stream as we don't + # own it, the opener owns it. + stream = self._stream + if stream: try: - setattr(self, attr, None) + _lib.c2pa_release_stream(stream) except Exception as e: logger.error( - Stream._ERROR_MESSAGES['callback_error'].format( - attr, str(e))) + Stream._ERROR_MESSAGES['stream_error'].format( + str(e))) + finally: + self._stream = None - except Exception as e: - logger.error( - Stream._ERROR_MESSAGES['cleanup_error'].format( - str(e))) - finally: - self._closed = True - self._initialized = False + # Clean up callbacks + for attr in [ + '_read_cb', '_seek_cb', '_write_cb', '_flush_cb']: + if hasattr(self, attr): + try: + setattr(self, attr, None) + except Exception as e: + logger.error( + Stream._ERROR_MESSAGES['callback_error'] + .format(attr, str(e))) + + except Exception as e: + logger.error( + Stream._ERROR_MESSAGES['cleanup_error'].format( + str(e))) + finally: + self._closed = True + self._initialized = False def write_to_target(self, dest_stream): self._file_like_stream.seek(0) @@ -2896,27 +2901,34 @@ def with_fragment(self, format: Optional[str], stream, frag_obj.close() raise - # Replace the streams this reader owned, - # closing the previous ones so repeated calls do not accumulate them. - # Only the current fragment is retained: the native reader does not - # read a superseded one back, and each wrapper held open pins a native - # stream, its callbacks and the caller's buffer. - previous = self._own_stream - previous_fragments = self._fragment_streams - self._own_stream = main_obj - self._fragment_streams = [frag_obj] - if previous is not None and previous is not main_obj: - try: - previous.close() - except Exception: - logger.warning("Failed to close previous Reader stream") - for fragment in previous_fragments: - if fragment is frag_obj: - continue + # Locked so a concurrent close() cannot run _release() + # between the check and the field swap. + with self._lock(): try: - fragment.close() + self._ensure_valid_state() except Exception: - logger.warning("Failed to close Reader fragment stream") + main_obj.close() + frag_obj.close() + raise + + # Replace the streams this reader owned, closing the previous + # ones (only the current fragment is retained). + previous = self._own_stream + previous_fragments = self._fragment_streams + self._own_stream = main_obj + self._fragment_streams = [frag_obj] + if previous is not None and previous is not main_obj: + try: + previous.close() + except Exception: + logger.warning("Failed to close previous Reader stream") + for fragment in previous_fragments: + if fragment is frag_obj: + continue + try: + fragment.close() + except Exception: + logger.warning("Failed to close Reader fragment stream") # Invalidate caches: processing a new BMFF fragment updates the native # reader's state, which can change the manifest data it returns. diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 85097321..001ab50e 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -12,6 +12,7 @@ # each license. import ast +import contextlib import ctypes import gc import os @@ -64,6 +65,7 @@ def _make_stream(pid_offset): obj._closed = False obj._initialized = True obj._stream = MagicMock() # non-None stream handle + obj._close_lock = threading.Lock() if pid_offset is not None: obj._owner_pid = os.getpid() + pid_offset return obj @@ -317,6 +319,64 @@ def test_parent_copy_unaffected(self): self.assertNotIn("DeprecationWarning", result.stderr) +class TestReaderWithFragmentConcurrentClose(unittest.TestCase): + """with_fragment's post-native-call bookkeeping must not race close().""" + + def test_close_during_with_fragment_does_not_double_close_stream(self): + init_path = os.path.join(FIXTURES_FOLDER, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_FOLDER, "dash1.m4s") + + with open(init_path, "rb") as init: + reader = Reader("video/mp4", init) + + entered_gap = threading.Event() + release_gap = threading.Event() + + real_native_call = reader._native_call + + @contextlib.contextmanager + def gated_native_call(): + with real_native_call(): + yield + # Pauses right in with_fragment's unlocked window before it reassigns _own_stream/_fragment_streams. + entered_gap.set() + release_gap.wait(5) + + reader._native_call = gated_native_call + + result = {} + + def run_with_fragment(): + try: + with open(init_path, "rb") as init, \ + open(fragment_path, "rb") as frag: + reader.with_fragment("video/mp4", init, frag) + result["outcome"] = "ok" + except BaseException as e: # noqa: BLE001 - asserted below + result["outcome"] = e + + worker = threading.Thread(target=run_with_fragment, daemon=True) + worker.start() + self.assertTrue( + entered_gap.wait(5), + "with_fragment never reached the post-native-call gap") + + # close() must win the race cleanly, not leave with_fragment hung, crashed, or silently successful. + reader.close() + release_gap.set() + worker.join(5) + self.assertFalse(worker.is_alive(), "with_fragment hung") + self.assertIsInstance( + result.get("outcome"), Error, + "with_fragment must raise C2paError when it loses the race, " + "not hang, crash, or silently succeed") + + self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) + # with_fragment must not resurrect these fields on a reader close() already tore down. + self.assertIsNone(reader._own_stream) + self.assertEqual(reader._fragment_streams, []) + + class TestHelpers(unittest.TestCase): def test_record_and_detect_own_pid(self): From cbda0808b675b1bebe6d51e84a5cbbfcde7c2f4e Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:02:27 -0700 Subject: [PATCH 16/28] fix: Handle fragments better --- src/c2pa/c2pa.py | 89 ++++++++++-------- tests/test_unit_tests_threaded.py | 148 +++++++++++++++++++++++++++--- 2 files changed, 187 insertions(+), 50 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 1ad8dce1..843bdd3d 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -2788,6 +2788,10 @@ def _init_attrs(self): # which it keeps reading from for the rest of its lifecycle. self._fragment_streams = [] + # Serializes with_fragment against itself. + # Held across the native call, unlike _op_lock. Only with_fragment takes it. + self._fragment_lock = threading.RLock() + # Caches for manifest JSON string and parsed data. # These are invalidated when with_fragment() is called. self._manifest_json_str_cache = None @@ -2881,54 +2885,61 @@ def with_fragment(self, format: Optional[str], stream, """ format_arg = _format_ffi_arg(_encode_format(format, "Reader")) - # The native reader keeps reading through both streams after this returns, - # so they are owned here and released by _release() rather - # than at the end of a with block. - main_obj = Stream(stream) - frag_obj = Stream(fragment_stream) - try: - with self._native_call(): - self._consume_and_swap( - lambda handle: _lib.c2pa_reader_with_fragment( - handle, - format_arg, - main_obj._stream, - frag_obj._stream, - ), - Reader._ERROR_MESSAGES['fragment_error']) - except Exception: - main_obj.close() - frag_obj.close() - raise + # A forked child cannot wait on a lock no surviving thread will + # release, so it reports the same error _lock() does. + if is_foreign_process(self): + raise C2paError(f"{type(self).__name__} is closed") - # Locked so a concurrent close() cannot run _release() - # between the check and the field swap. - with self._lock(): + # The native call and the ownership transfer are one unit. + with self._fragment_lock: + # The native reader keeps reading through both streams. + main_obj = Stream(stream) + frag_obj = Stream(fragment_stream) try: - self._ensure_valid_state() + with self._native_call(): + self._consume_and_swap( + lambda handle: _lib.c2pa_reader_with_fragment( + handle, + format_arg, + main_obj._stream, + frag_obj._stream, + ), + Reader._ERROR_MESSAGES['fragment_error']) except Exception: main_obj.close() frag_obj.close() raise - # Replace the streams this reader owned, closing the previous - # ones (only the current fragment is retained). - previous = self._own_stream - previous_fragments = self._fragment_streams - self._own_stream = main_obj - self._fragment_streams = [frag_obj] - if previous is not None and previous is not main_obj: - try: - previous.close() - except Exception: - logger.warning("Failed to close previous Reader stream") - for fragment in previous_fragments: - if fragment is frag_obj: - continue + # Locked so a concurrent close() cannot run _release() + # between the check and the field swap. + with self._lock(): try: - fragment.close() + self._ensure_valid_state() except Exception: - logger.warning("Failed to close Reader fragment stream") + main_obj.close() + frag_obj.close() + raise + + # Replace the streams this reader owned, closing the previous + # ones (only the current fragment is retained). + previous = self._own_stream + previous_fragments = self._fragment_streams + self._own_stream = main_obj + self._fragment_streams = [frag_obj] + if previous is not None and previous is not main_obj: + try: + previous.close() + except Exception: + logger.warning( + "Failed to close previous Reader stream") + for fragment in previous_fragments: + if fragment is frag_obj: + continue + try: + fragment.close() + except Exception: + logger.warning( + "Failed to close Reader fragment stream") # Invalidate caches: processing a new BMFF fragment updates the native # reader's state, which can change the manifest data it returns. diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 001ab50e..940df8e3 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -200,7 +200,7 @@ class TestForkedChildDoesNotDeadlock(unittest.TestCase): _TIMEOUT = 5.0 - def _foreign_reader_with_lock_held(self): + def _foreign_reader_with_lock_held(self, fragment_lock=False): """A Reader in the state a forked child inherits: lock held by another thread, and stamped with a PID other than this process's. """ @@ -210,7 +210,9 @@ def _foreign_reader_with_lock_held(self): release = threading.Event() def hold_the_lock(): - with reader._lock(): + held = (reader._fragment_lock if fragment_lock + else reader._lock()) + with held: holding.set() release.wait(30) @@ -257,6 +259,16 @@ def test_native_call_path_raises_instead_of_blocking(self): "resource_to_stream() blocked on a lock inherited from the parent") self.assertIsInstance(outcome, Error) + def test_fragment_lock_path_raises_instead_of_blocking(self): + reader = self._foreign_reader_with_lock_held(fragment_lock=True) + outcome = self._run_with_timeout( + lambda: reader.with_fragment( + "video/mp4", io.BytesIO(b""), io.BytesIO(b""))) + self.assertIsNotNone( + outcome, + "with_fragment() blocked on a lock inherited from the parent") + self.assertIsInstance(outcome, Error) + def test_close_still_completes(self): reader = self._foreign_reader_with_lock_held() self.assertEqual(self._run_with_timeout(reader.close), "ok", @@ -319,14 +331,27 @@ def test_parent_copy_unaffected(self): self.assertNotIn("DeprecationWarning", result.stderr) -class TestReaderWithFragmentConcurrentClose(unittest.TestCase): - """with_fragment's post-native-call bookkeeping must not race close().""" +class TestReaderWithFragmentConcurrency(unittest.TestCase): + """with_fragment's native call and its stream-ownership transfer + must must not interleave with another with_fragment on the same Reader. + """ - def test_close_during_with_fragment_does_not_double_close_stream(self): - init_path = os.path.join(FIXTURES_FOLDER, "dashinit.mp4") - fragment_path = os.path.join(FIXTURES_FOLDER, "dash1.m4s") + def setUp(self): + self.init_path = os.path.join(FIXTURES_FOLDER, "dashinit.mp4") + self.fragment_path = os.path.join(FIXTURES_FOLDER, "dash1.m4s") + with open(self.init_path, "rb") as f: + self.init_bytes = f.read() + with open(self.fragment_path, "rb") as f: + self.fragment_bytes = f.read() + + def _advance(self, reader): + reader.with_fragment( + "video/mp4", + io.BytesIO(self.init_bytes), + io.BytesIO(self.fragment_bytes)) - with open(init_path, "rb") as init: + def test_close_during_with_fragment_does_not_double_close_stream(self): + with open(self.init_path, "rb") as init: reader = Reader("video/mp4", init) entered_gap = threading.Event() @@ -338,7 +363,7 @@ def test_close_during_with_fragment_does_not_double_close_stream(self): def gated_native_call(): with real_native_call(): yield - # Pauses right in with_fragment's unlocked window before it reassigns _own_stream/_fragment_streams. + # Pauses in with_fragment's window before it reassigns _own_stream/_fragment_streams. entered_gap.set() release_gap.wait(5) @@ -348,8 +373,8 @@ def gated_native_call(): def run_with_fragment(): try: - with open(init_path, "rb") as init, \ - open(fragment_path, "rb") as frag: + with open(self.init_path, "rb") as init, \ + open(self.fragment_path, "rb") as frag: reader.with_fragment("video/mp4", init, frag) result["outcome"] = "ok" except BaseException as e: # noqa: BLE001 - asserted below @@ -376,6 +401,107 @@ def run_with_fragment(): self.assertIsNone(reader._own_stream) self.assertEqual(reader._fragment_streams, []) + def test_interleaved_with_fragment_leaves_reader_consistent(self): + reader = Reader("video/mp4", io.BytesIO(self.init_bytes)) + + # Parks one call between its native call + # and its native handle bookkeeping. + real_native_call = reader._native_call + in_gap = threading.Event() + contended = threading.Event() + leave_gap = threading.Event() + + @contextlib.contextmanager + def gated_native_call(): + with real_native_call(): + yield + if not in_gap.is_set(): + in_gap.set() + leave_gap.wait(10) + + reader._native_call = gated_native_call + + class ContentionReportingLock: + """Flags when a caller has to wait for the lock it wraps.""" + + def __init__(self, inner): + self._inner = inner + + def __enter__(self): + if not self._inner.acquire(blocking=False): + contended.set() + self._inner.acquire() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._inner.release() + return False + + real_fragment_lock = reader._fragment_lock + reader._fragment_lock = ContentionReportingLock(real_fragment_lock) + + outcomes = {} + installed_by_second = {} + + def first(): + try: + self._advance(reader) + outcomes["first"] = "ok" + except Error as e: + outcomes["first"] = e + + def second(): + try: + self._advance(reader) + outcomes["second"] = "ok" + # The streams matching the handle this call swapped in. + installed_by_second["own"] = reader._own_stream + installed_by_second["fragments"] = list( + reader._fragment_streams) + except Error as e: + outcomes["second"] = e + + t1 = threading.Thread(target=first, daemon=True) + t1.start() + self.assertTrue(in_gap.wait(10), "never reached the bookkeeping gap") + + t2 = threading.Thread(target=second, daemon=True) + t2.start() + # Unset when the lock is bypassed, which is the case this test guards against. + contended.wait(5) + + leave_gap.set() + t1.join(10) + self.assertFalse(t1.is_alive(), "first with_fragment hung") + t2.join(10) + self.assertFalse(t2.is_alive(), "second with_fragment hung") + + try: + if outcomes.get("second") != "ok": + # A refused second call never swapped, + # so the first call's streams are the right ones. + self.assertIsInstance(outcomes["second"], Error) + else: + # Both swapped, so the reader must retain one call's streams. + self.assertIs( + reader._own_stream, installed_by_second["own"], + "reader retains a different call's stream than the one " + "its live native handle reads through") + self.assertEqual( + list(reader._fragment_streams), + installed_by_second["fragments"]) + + retained = [reader._own_stream] + list(reader._fragment_streams) + for wrapper in retained: + self.assertIsNotNone(wrapper) + self.assertFalse( + wrapper._closed, + "reader retained a released stream wrapper") + finally: + reader._native_call = real_native_call + reader._fragment_lock = real_fragment_lock + reader.close() + class TestHelpers(unittest.TestCase): From dfaaac169672b72afd486fac00200aff77aef954 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:41:54 -0700 Subject: [PATCH 17/28] fix: Handle fragments better 2 --- src/c2pa/c2pa.py | 41 ++++---- tests/test_unit_tests_threaded.py | 151 ++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 20 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 843bdd3d..3c92c633 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -2842,22 +2842,25 @@ def _get_cached_manifest_data(self) -> Optional[dict]: Raises: C2paError: If there was an error getting the JSON """ - if self._manifest_data_cache is None: - if self._manifest_json_str_cache is None: - self._manifest_json_str_cache = self.json() + # Locked so the cache fields can't be read and written + # across concurrent handle swaps. + with self._lock(): + if self._manifest_data_cache is None: + if self._manifest_json_str_cache is None: + self._manifest_json_str_cache = self.json() - try: - self._manifest_data_cache = json.loads( - self._manifest_json_str_cache - ) - except json.JSONDecodeError: - # Reset cache to reattempt read, possibly - self._manifest_data_cache = None - self._manifest_json_str_cache = None - # Failed to parse manifest JSON - return None + try: + self._manifest_data_cache = json.loads( + self._manifest_json_str_cache + ) + except json.JSONDecodeError: + # Reset cache to reattempt read, possibly + self._manifest_data_cache = None + self._manifest_json_str_cache = None + # Failed to parse manifest JSON + return None - return self._manifest_data_cache + return self._manifest_data_cache def with_fragment(self, format: Optional[str], stream, fragment_stream) -> "Reader": @@ -2941,12 +2944,10 @@ def with_fragment(self, format: Optional[str], stream, logger.warning( "Failed to close Reader fragment stream") - # Invalidate caches: processing a new BMFF fragment updates the native - # reader's state, which can change the manifest data it returns. - # The cached JSON string and parsed dict may now be stale, so clear - # them to force a fresh read from the native layer on next access. - self._manifest_json_str_cache = None - self._manifest_data_cache = None + # Cleared here because these describe the replaced handle, + # and a reader must never be served them. + self._manifest_json_str_cache = None + self._manifest_data_cache = None return self diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 940df8e3..a6bd6c1b 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -401,6 +401,157 @@ def run_with_fragment(): self.assertIsNone(reader._own_stream) self.assertEqual(reader._fragment_streams, []) + def _manifest_before_and_after_fragment(self): + """Tests the manifest a fresh Reader reports, + and the one it reports once a fragment has been processed. + """ + reader = Reader("video/mp4", io.BytesIO(self.init_bytes)) + try: + before = reader.json() + finally: + reader.close() + + reader = Reader("video/mp4", io.BytesIO(self.init_bytes)) + try: + self._advance(reader) + after = reader.json() + finally: + reader.close() + return before, after + + def test_read_during_swap_never_serves_the_previous_handles_manifest(self): + before, after = self._manifest_before_and_after_fragment() + self.assertNotEqual( + before, after, + "fixtures must differ before and after the fragment for this " + "test to mean anything") + + reader = Reader("video/mp4", io.BytesIO(self.init_bytes)) + # Populates the cache with the soon to be replaced handle. + self.assertEqual(reader.json(), before) + + real_lock = reader._lock + at_gap = threading.Event() + leave_gap = threading.Event() + # _native_call takes this lock before the swap does, + # so park on the acquisition that actually performed the swap. + swapped = [] + + class GatedLock: + """Parks once after the swap's locked region releases.""" + + def __init__(self, inner): + self._inner = inner + + def __enter__(self): + return self._inner.__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + performed_swap = reader._own_stream is not None and ( + reader._own_stream not in swapped) + result = self._inner.__exit__(exc_type, exc_val, exc_tb) + if performed_swap and not at_gap.is_set(): + at_gap.set() + leave_gap.wait(10) + return result + + swapped.append(reader._own_stream) + reader._lock = lambda: GatedLock(real_lock()) + + served = {} + + def advance(): + try: + self._advance(reader) + except Error as e: + served["advance"] = e + + def read_in_gap(): + try: + served["json"] = reader.json() + except Error as e: + served["json"] = e + + advancer = threading.Thread(target=advance, daemon=True) + advancer.start() + self.assertTrue(at_gap.wait(10), "never reached the post-swap gap") + + gap_reader = threading.Thread(target=read_in_gap, daemon=True) + gap_reader.start() + gap_reader.join(10) + + leave_gap.set() + advancer.join(10) + + try: + self.assertFalse(gap_reader.is_alive(), "json() hung in the gap") + # Smoke test comparison. + names = {before: "the replaced handle's manifest", + after: "the current handle's manifest"} + self.assertEqual( + names.get(served.get("json"), "something else"), + "the current handle's manifest", + "json() must not be served a manifest cached from the " + "handle with_fragment already replaced") + finally: + reader._lock = real_lock + reader.close() + + def test_manifest_accessors_stay_consistent_while_fragments_advance(self): + """get_active_manifest() parses the cached JSON, + so its read and write of the cache must not prevent a clean handle swap. + """ + before, after = self._manifest_before_and_after_fragment() + valid = {before, after} + + reader = Reader("video/mp4", io.BytesIO(self.init_bytes)) + stop = threading.Event() + unexpected = [] + served = [] + + def read_manifest(): + while not stop.is_set(): + try: + if reader.get_active_manifest() is not None: + served.append(reader.json()) + except Error: + pass + except BaseException as e: # noqa: BLE001 - asserted below + unexpected.append(repr(e)) + + def advance(): + while not stop.is_set(): + try: + self._advance(reader) + except Error: + pass + except BaseException as e: # noqa: BLE001 - asserted below + unexpected.append(repr(e)) + + workers = ([threading.Thread(target=read_manifest, daemon=True) + for _ in range(3)] + + [threading.Thread(target=advance, daemon=True) + for _ in range(2)]) + for t in workers: + t.start() + time.sleep(0.3) + stop.set() + for t in workers: + t.join(10) + + try: + self.assertFalse( + [t for t in workers if t.is_alive()], + "a manifest accessor or fragment advance hung") + self.assertEqual(unexpected, []) + self.assertTrue(served, "no manifest was ever read") + self.assertTrue( + set(served) <= valid, + "a manifest was served that matches neither the pre- nor the " + "post-fragment state") + finally: + reader.close() + def test_interleaved_with_fragment_leaves_reader_consistent(self): reader = Reader("video/mp4", io.BytesIO(self.init_bytes)) From ea1dac5d8be1af71057e18f8cd088c092b754c7e Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:47:08 -0700 Subject: [PATCH 18/28] fix: Docs --- docs/native-resources-management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 2ab550a6..c1900bf1 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -414,7 +414,7 @@ stateDiagram-v2 On success the object stays `ACTIVE` because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from a consumed teardown (`_teardown(free_handle=False)`), where the object transitions to `CLOSED` and becomes unusable. On the success path the old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. The failure path is different and is covered by the triage in [`_consume_and_swap()`](#_consume_and_swap). -`Reader.with_fragment()` runs the swap inside `self._native_call()`, and keeps a `_fragment_streams` list holding the `Stream` wrapper for the current fragment. Each call to `with_fragment()` replaces that list rather than appending to it, closing the previous fragment's wrapper immediately: the native reader never reads a superseded fragment back, and each open wrapper pins a native stream, its callbacks, and the caller's buffer. +`Reader.with_fragment()` runs the native call inside `self._native_call()`, and keeps a `_fragment_streams` list holding the `Stream` wrapper for the current fragment. Each call to `with_fragment()` replaces that list rather than appending to it, closing the previous fragment's wrapper immediately: the native reader never reads a superseded fragment back, and each open wrapper pins a native stream, its callbacks, and the caller's buffer. The native call and the field swap that follows it are both covered by `_fragment_lock`, described in [`Reader._fragment_lock`](#readerfragment_lock). ### `_consume_and_swap()` From 5ebe99a2d7f046be9870545994010d5bd65f4983 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:42:33 -0700 Subject: [PATCH 19/28] fix: The test that checks lock ordering --- tests/test_unit_tests_threaded.py | 110 ++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index a6bd6c1b..579bc7e5 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -4355,6 +4355,116 @@ def visit(node, active): "borrowed handles used without their own guard:\n " + "\n ".join(unguarded)) + def test_no_conflicting_lock_acquisition_order(self): + """No two locks may be nested in opposite orders by different methods. + + Two methods nesting the same pair of locks in opposite order is a + AB/BA deadlock shape: thread 1 holds A and waits for B while + thread 2 holds B and waits for A. + + This scans the code to check. + """ + module = sys.modules[Reader.__module__] + tree = ast.parse(inspect.getsource(module)) + + # Every self._X = threading.Lock()/RLock()/Condition() assignment, + # grouped by the class that owns it. + lock_attrs_by_class = {} + for cls in ast.walk(tree): + if not isinstance(cls, ast.ClassDef): + continue + found = set() + for node in ast.walk(cls): + if not (isinstance(node, ast.Assign) + and len(node.targets) == 1): + continue + target = node.targets[0] + if not (isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "self"): + continue + value = node.value + if (isinstance(value, ast.Call) + and isinstance(value.func, ast.Attribute) + and value.func.attr in + ("Lock", "RLock", "Condition")): + found.add(target.attr) + if found: + lock_attrs_by_class[cls.name] = found + + def lock_name_for_with(item): + """The lock attribute a `with` item enters, or None.""" + ctx = item.context_expr + # with self._fragment_lock: + if (isinstance(ctx, ast.Attribute) + and isinstance(ctx.value, ast.Name) + and ctx.value.id == "self" + and any(ctx.attr in attrs + for attrs in lock_attrs_by_class.values())): + return ctx.attr + # with self._lock(): returns _op_lock itself. + if (isinstance(ctx, ast.Call) + and isinstance(ctx.func, ast.Attribute) + and ctx.func.attr == "_lock" + and isinstance(ctx.func.value, ast.Name) + and ctx.func.value.id == "self"): + return "_op_lock" + return None + + def orders_in(node, stack, pairs): + """Record (outer, inner) for every nesting this node contains.""" + if isinstance(node, (ast.With, ast.AsyncWith)): + names = [n for n in (lock_name_for_with(i) + for i in node.items) if n] + for name in names: + if stack: + pairs.add((stack[-1], name)) + stack.append(name) + for child in node.body: + orders_in(child, stack, pairs) + for _ in names: + stack.pop() + return + for child in ast.iter_child_nodes(node): + orders_in(child, stack, pairs) + + pairs_by_method = {} + for cls in ast.walk(tree): + if not isinstance(cls, ast.ClassDef): + continue + for method in cls.body: + if not isinstance(method, (ast.FunctionDef, + ast.AsyncFunctionDef)): + continue + pairs = set() + orders_in(method, [], pairs) + if pairs: + pairs_by_method[(cls.name, method.name)] = pairs + + all_pairs = set().union(*pairs_by_method.values()) \ + if pairs_by_method else set() + conflicts = [] + for (outer, inner) in all_pairs: + # Each conflicting pair appears twice in all_pairs, once per direction. + if outer >= inner or (inner, outer) not in all_pairs: + continue + forward = [k for k, v in pairs_by_method.items() + if (outer, inner) in v] + backward = [k for k, v in pairs_by_method.items() + if (inner, outer) in v] + conflicts.append( + "{} nests {} inside {}, but {} nests {} inside {}".format( + forward[0], inner, outer, backward[0], outer, inner)) + + self.assertGreater( + len(pairs_by_method), 0, + "lock nesting scan found no nested lock acquisitions: " + "the scan is broken") + self.assertEqual( + conflicts, [], + "conflicting lock acquisition order:\n " + + "\n ".join(sorted(set(conflicts)))) + class TestSharedSignerTeardownRace(unittest.TestCase): """A Signer shared across threads must not be freed mid-sign. From bd2de491ae5bbb14941f3cb62bbdd4c0f5dcd41a Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:54:26 -0700 Subject: [PATCH 20/28] fix: The test that checks lock ordering --- tests/test_unit_tests.py | 114 ++++++++++++++ tests/test_unit_tests_threaded.py | 240 +++++++----------------------- 2 files changed, 170 insertions(+), 184 deletions(-) diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index d8c09bdd..e7af0c05 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -11,6 +11,7 @@ # specific language governing permissions and limitations under # each license. +import ast import gc import inspect import os @@ -9676,5 +9677,118 @@ def test_ed25519_sign_with_empty_data_raises(self): c2pa_module.ed25519_sign(b"", "not a key") +class TestLockOrderStaticAnalysis(unittest.TestCase): + """Static analysis over the source, not runtime behavior: + no threads are spawned here. + """ + + def test_no_conflicting_lock_acquisition_order(self): + """No two locks may be nested in opposite orders by different methods. + + Two methods nesting the same pair of locks in opposite order is a + AB/BA deadlock shape: thread 1 holds A and waits for B while + thread 2 holds B and waits for A. + """ + tree = ast.parse(inspect.getsource(c2pa_module)) + + # Every self._X = threading.Lock()/RLock()/Condition() assignment, + # grouped by the class that owns it. + lock_attrs_by_class = {} + for cls in ast.walk(tree): + if not isinstance(cls, ast.ClassDef): + continue + found = set() + for node in ast.walk(cls): + if not (isinstance(node, ast.Assign) + and len(node.targets) == 1): + continue + target = node.targets[0] + if not (isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "self"): + continue + value = node.value + if (isinstance(value, ast.Call) + and isinstance(value.func, ast.Attribute) + and value.func.attr in + ("Lock", "RLock", "Condition")): + found.add(target.attr) + if found: + lock_attrs_by_class[cls.name] = found + + def lock_name_for_with(item): + """The lock attribute a `with` item enters, or None.""" + ctx = item.context_expr + # with self._fragment_lock: + if (isinstance(ctx, ast.Attribute) + and isinstance(ctx.value, ast.Name) + and ctx.value.id == "self" + and any(ctx.attr in attrs + for attrs in lock_attrs_by_class.values())): + return ctx.attr + # with self._lock(): returns _op_lock itself. + if (isinstance(ctx, ast.Call) + and isinstance(ctx.func, ast.Attribute) + and ctx.func.attr == "_lock" + and isinstance(ctx.func.value, ast.Name) + and ctx.func.value.id == "self"): + return "_op_lock" + return None + + def orders_in(node, stack, pairs): + """Record (outer, inner) for every nesting this node contains.""" + if isinstance(node, (ast.With, ast.AsyncWith)): + names = [n for n in (lock_name_for_with(i) + for i in node.items) if n] + for name in names: + if stack: + pairs.add((stack[-1], name)) + stack.append(name) + for child in node.body: + orders_in(child, stack, pairs) + for _ in names: + stack.pop() + return + for child in ast.iter_child_nodes(node): + orders_in(child, stack, pairs) + + pairs_by_method = {} + for cls in ast.walk(tree): + if not isinstance(cls, ast.ClassDef): + continue + for method in cls.body: + if not isinstance(method, (ast.FunctionDef, + ast.AsyncFunctionDef)): + continue + pairs = set() + orders_in(method, [], pairs) + if pairs: + pairs_by_method[(cls.name, method.name)] = pairs + + all_pairs = set().union(*pairs_by_method.values()) \ + if pairs_by_method else set() + conflicts = [] + for (outer, inner) in all_pairs: + # Each conflicting pair appears twice in all_pairs, once per direction. + if outer >= inner or (inner, outer) not in all_pairs: + continue + forward = [k for k, v in pairs_by_method.items() + if (outer, inner) in v] + backward = [k for k, v in pairs_by_method.items() + if (inner, outer) in v] + conflicts.append( + "{} nests {} inside {}, but {} nests {} inside {}".format( + forward[0], inner, outer, backward[0], outer, inner)) + + self.assertGreater( + len(pairs_by_method), 0, + "lock nesting scan found no nested lock acquisitions: " + "the scan is broken") + self.assertEqual( + conflicts, [], + "conflicting lock acquisition order:\n " + + "\n ".join(sorted(set(conflicts)))) + + if __name__ == '__main__': unittest.main(warnings='ignore') diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 579bc7e5..85c9aefc 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -3393,10 +3393,30 @@ def thread_work(thread_id): self.assertNotEqual(current_manifest["active_manifest"], thread_manifest_data[other_thread_id]["active_manifest"]) -class TestManagedResourceCrossThread(unittest.TestCase): - """Tests cross-thread resources handling, especially closing/releasind. +class TestLocking(unittest.TestCase): + """Tests for the locks that guard native resources: + - the per-object operation lock that serializes native calls against teardown, + - the fragment lock, + - cross-thread creation/closing/releasing. + + Every join here is bounded: + A deadlock must fail the test when timing out, not hang the suite. """ + JOIN_TIMEOUT = 30 + + @classmethod + def setUpClass(cls): + cls.data_dir = FIXTURES_FOLDER + with open(DEFAULT_TEST_FILE, 'rb') as handle: + cls.image_bytes = handle.read() + with open(os.path.join(FIXTURES_FOLDER, + "es256_certs.pem"), 'rb') as handle: + cls.certs = handle.read() + with open(os.path.join(FIXTURES_FOLDER, + "es256_private.key"), 'rb') as handle: + cls.private_key = handle.read() + def setUp(self): # Flush pending finalizers through the real free first. gc.collect() @@ -3407,6 +3427,31 @@ def setUp(self): def tearDown(self): ManagedResource._free_native_ptr = self._real_free + def _join_all(self, threads, what): + for thread in threads: + thread.join(self.JOIN_TIMEOUT) + stuck = [t for t in threads if t.is_alive()] + self.assertEqual( + stuck, [], + "{} did not finish within {}s: deadlock".format( + what, self.JOIN_TIMEOUT)) + + def _run_isolated(self, body, timeout=180): + """Run body in a subprocess and return it, + so that crashes can be caught and do not crash the suite itself. + """ + source = textwrap.dedent(body) + return subprocess.run( + [sys.executable, "-c", source], + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + capture_output=True, + timeout=timeout, + ) + + def _make_signer(self): + return Signer.from_info(C2paSignerInfo( + SigningAlg.ES256, self.certs, self.private_key, None)) + def _free_counts(self): counts = {} for handle in self.freed: @@ -3510,49 +3555,6 @@ def build_context_and_builder(): self.assertTrue(valid) self.assertEqual(settings._owner_pid, pid) - -class TestManagedResourceLockDeadlock(unittest.TestCase): - """Tests for the operation lock that serializes native calls against - teardown. - - Every join here is bounded: - A deadlock must fail the test when timing out, not hang the suite. - """ - - JOIN_TIMEOUT = 30 - - @classmethod - def setUpClass(cls): - with open(DEFAULT_TEST_FILE, 'rb') as handle: - cls.image_bytes = handle.read() - with open(os.path.join(FIXTURES_FOLDER, - "es256_certs.pem"), 'rb') as handle: - cls.certs = handle.read() - with open(os.path.join(FIXTURES_FOLDER, - "es256_private.key"), 'rb') as handle: - cls.private_key = handle.read() - - def _join_all(self, threads, what): - for thread in threads: - thread.join(self.JOIN_TIMEOUT) - stuck = [t for t in threads if t.is_alive()] - self.assertEqual( - stuck, [], - "{} did not finish within {}s: deadlock".format( - what, self.JOIN_TIMEOUT)) - - def _run_isolated(self, body, timeout=180): - """Run body in a subprocess and return it, - so that crashes can be caught and do not crash the suite itself. - """ - source = textwrap.dedent(body) - return subprocess.run( - [sys.executable, "-c", source], - cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - capture_output=True, - timeout=timeout, - ) - def test_json_racing_finalizer_does_not_crash(self): """Readers used on one thread while others are collected. """ @@ -4355,148 +4357,18 @@ def visit(node, active): "borrowed handles used without their own guard:\n " + "\n ".join(unguarded)) - def test_no_conflicting_lock_acquisition_order(self): - """No two locks may be nested in opposite orders by different methods. - - Two methods nesting the same pair of locks in opposite order is a - AB/BA deadlock shape: thread 1 holds A and waits for B while - thread 2 holds B and waits for A. - - This scans the code to check. - """ - module = sys.modules[Reader.__module__] - tree = ast.parse(inspect.getsource(module)) - - # Every self._X = threading.Lock()/RLock()/Condition() assignment, - # grouped by the class that owns it. - lock_attrs_by_class = {} - for cls in ast.walk(tree): - if not isinstance(cls, ast.ClassDef): - continue - found = set() - for node in ast.walk(cls): - if not (isinstance(node, ast.Assign) - and len(node.targets) == 1): - continue - target = node.targets[0] - if not (isinstance(target, ast.Attribute) - and isinstance(target.value, ast.Name) - and target.value.id == "self"): - continue - value = node.value - if (isinstance(value, ast.Call) - and isinstance(value.func, ast.Attribute) - and value.func.attr in - ("Lock", "RLock", "Condition")): - found.add(target.attr) - if found: - lock_attrs_by_class[cls.name] = found - - def lock_name_for_with(item): - """The lock attribute a `with` item enters, or None.""" - ctx = item.context_expr - # with self._fragment_lock: - if (isinstance(ctx, ast.Attribute) - and isinstance(ctx.value, ast.Name) - and ctx.value.id == "self" - and any(ctx.attr in attrs - for attrs in lock_attrs_by_class.values())): - return ctx.attr - # with self._lock(): returns _op_lock itself. - if (isinstance(ctx, ast.Call) - and isinstance(ctx.func, ast.Attribute) - and ctx.func.attr == "_lock" - and isinstance(ctx.func.value, ast.Name) - and ctx.func.value.id == "self"): - return "_op_lock" - return None - - def orders_in(node, stack, pairs): - """Record (outer, inner) for every nesting this node contains.""" - if isinstance(node, (ast.With, ast.AsyncWith)): - names = [n for n in (lock_name_for_with(i) - for i in node.items) if n] - for name in names: - if stack: - pairs.add((stack[-1], name)) - stack.append(name) - for child in node.body: - orders_in(child, stack, pairs) - for _ in names: - stack.pop() - return - for child in ast.iter_child_nodes(node): - orders_in(child, stack, pairs) - - pairs_by_method = {} - for cls in ast.walk(tree): - if not isinstance(cls, ast.ClassDef): - continue - for method in cls.body: - if not isinstance(method, (ast.FunctionDef, - ast.AsyncFunctionDef)): - continue - pairs = set() - orders_in(method, [], pairs) - if pairs: - pairs_by_method[(cls.name, method.name)] = pairs - - all_pairs = set().union(*pairs_by_method.values()) \ - if pairs_by_method else set() - conflicts = [] - for (outer, inner) in all_pairs: - # Each conflicting pair appears twice in all_pairs, once per direction. - if outer >= inner or (inner, outer) not in all_pairs: - continue - forward = [k for k, v in pairs_by_method.items() - if (outer, inner) in v] - backward = [k for k, v in pairs_by_method.items() - if (inner, outer) in v] - conflicts.append( - "{} nests {} inside {}, but {} nests {} inside {}".format( - forward[0], inner, outer, backward[0], outer, inner)) - - self.assertGreater( - len(pairs_by_method), 0, - "lock nesting scan found no nested lock acquisitions: " - "the scan is broken") - self.assertEqual( - conflicts, [], - "conflicting lock acquisition order:\n " - + "\n ".join(sorted(set(conflicts)))) - - -class TestSharedSignerTeardownRace(unittest.TestCase): - """A Signer shared across threads must not be freed mid-sign. - - Builder.sign borrows the signer's handle for the duration of the native - call. Without a guard on the signer itself, a close() on another thread - frees that handle while c2pa_builder_sign is using it, and the process - dies with SIGSEGV instead of raising. - """ - - def setUp(self): - self.data_dir = os.path.join(os.path.dirname(__file__), "fixtures") - with open(os.path.join(self.data_dir, "C.jpg"), "rb") as f: - self.image_bytes = f.read() - with open(os.path.join(self.data_dir, "es256_certs.pem"), "rb") as f: - self.certs = f.read() - with open(os.path.join(self.data_dir, "es256_private.key"), "rb") as f: - self.key = f.read() - self.manifest = { - "claim_generator_info": [{"name": "test", "version": "0.1"}], - "assertions": [], - } - - def _make_signer(self): - return Signer.from_info(C2paSignerInfo( - SigningAlg.ES256, self.certs, self.key, None)) def test_close_during_concurrent_sign_does_not_crash(self): - """Rotate a shared signer while other threads sign with it. + """A Signer shared across threads must not be freed mid-sign. + + Builder.sign borrows the signer's handle for the duration of the + native call. Without a guard on the signer itself, a close() on + another thread frees that handle while c2pa_builder_sign is using + it, and the process dies with SIGSEGV instead of raising. - Runs in a subprocess: the failure mode is a segfault, which would - take the test runner down with it otherwise. + Rotates a shared signer while other threads sign with it. Runs in a + subprocess: the failure mode is a segfault, which would take the + test runner down with it otherwise. """ source = textwrap.dedent(""" import io, os, sys, threading From 39acabd30b3fc6ce3102e1b858174c9f8f5ee86d Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Wed, 26 Aug 2026 14:21:46 -0700 Subject: [PATCH 21/28] fix: Double free scan --- src/c2pa/c2pa.py | 9 ++++ tests/test_unit_tests_threaded.py | 69 +++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 3c92c633..9e635d1b 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -269,6 +269,7 @@ def __init__(self): self._op_lock = threading.RLock() self._inflight = 0 self._pending_teardown = None + self._released = False record_owner_pid(self) def _lock(self): @@ -410,6 +411,13 @@ def _teardown(self, free_handle: bool): return with self._lock(): + if getattr(self, '_released', False): + # A racing close()/__del__ already ran the release branch + # under this lock. Idempotent: nothing left to release or free. + # Keyed on the release having happened, not on CLOSED: the + # deferred path below sets CLOSED without releasing, and still + # owes a release performed by _native_call()'s finally. + return if getattr(self, '_inflight', 0) > 0: # A native call is running that re-enters calling non-native code # and is still using this handle. @@ -421,6 +429,7 @@ def _teardown(self, free_handle: bool): self._lifecycle_state = LifecycleState.CLOSED return + self._released = True self._lifecycle_state = LifecycleState.CLOSED self._safe_release() diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 85c9aefc..3983a112 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -4158,6 +4158,75 @@ def closer(): "racing closers freed {} times".format(len(freed))) self.assertEqual(reader._inflight, 0) + def test_concurrent_close_runs_release_once(self): + """Two racing close() calls on one instance must run _release() + exactly once. + + The native free is already single (the handle is nulled after the + first teardown), so a free-counting test cannot see this: it is + _release() -- the Python-side stream/cache cleanup a subclass + overrides -- that must not run twice. _teardown() has to be + idempotent under its own lock. + + Gate _teardown so the first close() pauses on entry, before taking + the lock; the second then runs a full teardown (release + free + + mark closed); the first resumes and must find the resource already + released and do nothing. + """ + join_timeout = self.JOIN_TIMEOUT + orig_teardown = ManagedResource._teardown + + for _ in range(20): + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + release_calls = [] + orig_release = reader._release + + def counting_release(_orig=orig_release, _calls=release_calls): + _calls.append(1) + _orig() + + reader._release = counting_release + + call_count = {"n": 0} + count_lock = threading.Lock() + first_arrived = threading.Event() + release_first = threading.Event() + + def gated_teardown(self, free_handle, _target=reader, + _timeout=join_timeout): + if self is _target: + with count_lock: + call_count["n"] += 1 + is_first = call_count["n"] == 1 + if is_first: + first_arrived.set() + release_first.wait(_timeout) + return orig_teardown(self, free_handle) + + with patch.object(ManagedResource, '_teardown', gated_teardown): + t1 = threading.Thread(target=reader.close) + t1.start() + self.assertTrue( + first_arrived.wait(join_timeout), + "first close() never reached _teardown()") + + t2 = threading.Thread(target=reader.close) + t2.start() + t2.join(join_timeout) + self.assertFalse( + t2.is_alive(), + "second close() should complete unblocked while the " + "first is paused") + + release_first.set() + self._join_all([t1], "paused close() resuming") + + self.assertEqual( + len(release_calls), 1, + "_release() ran {} times for one instance across racing " + "close() calls; _teardown() must be idempotent under its " + "own lock".format(len(release_calls))) + def test_sign_with_internal_close_frees_once(self): """_sign_internal closes the Builder inside its own try, so the close defers and the free happens on the way out.""" From 95cc46518c6a12e22abf5998aeca3b7479e38b25 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:11:35 -0700 Subject: [PATCH 22/28] fix: Fix a crash --- src/c2pa/c2pa.py | 21 +++++++++++++--- tests/test_unit_tests_threaded.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 9e635d1b..49f66b77 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -413,19 +413,32 @@ def _teardown(self, free_handle: bool): with self._lock(): if getattr(self, '_released', False): # A racing close()/__del__ already ran the release branch - # under this lock. Idempotent: nothing left to release or free. + # under this lock. + # Idempotent: nothing left to release or free. # Keyed on the release having happened, not on CLOSED: the # deferred path below sets CLOSED without releasing, and still # owes a release performed by _native_call()'s finally. return if getattr(self, '_inflight', 0) > 0: - # A native call is running that re-enters calling non-native code - # and is still using this handle. + # A native call is running that re-enters calling non-native + # code and is still using this handle. # Record the intent and whichever caller leaves # _native_call last performs the free. # Mark the resource closed now so it cannot be used # while the free is pending. - self._pending_teardown = free_handle + # + # free_handle=False records that a consuming call handed + # ownership to the native library. Ownership does not come + # back, so a later teardown cannot restore the right to free: + # the recorded value only ever moves True -> False, never the + # reverse. Without this, a _teardown(True) arriving second + # (from _release_handle, whose state check is read outside + # this lock and can go stale) frees a pointer native owns. + if self._pending_teardown is None: + self._pending_teardown = free_handle + else: + self._pending_teardown = ( + self._pending_teardown and free_handle) self._lifecycle_state = LifecycleState.CLOSED return diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 3983a112..1b9b65b5 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -4158,6 +4158,48 @@ def closer(): "racing closers freed {} times".format(len(freed))) self.assertEqual(reader._inflight, 0) + def test_deferred_consume_is_not_upgraded_to_free(self): + """A deferred consuming teardown must not be overwritten by a later + free intent arriving while the same call is still in flight. + + Scenario: a Signer shared across concurrent signs: sign borrows + the handle (holding the in-flight guard) while Context.__init__ + consumes it. + """ + freed = self._counted_free() + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + releases = [] + orig_release = reader._release + + def counting_release(): + releases.append(1) + orig_release() + + reader._release = counting_release + + with reader._native_call(): + # The consuming call: native took ownership, so nothing here frees. + reader._teardown(free_handle=False) + self.assertFalse( + reader._pending_teardown, + "consuming teardown did not record free_handle=False") + + # A free intent arriving behind it, past a stale state check. + reader._teardown(free_handle=True) + self.assertFalse( + reader._pending_teardown, + "recorded consume was upgraded back to a free") + + self.assertEqual( + freed, [], + "freed a handle the native library already owns") + self.assertEqual( + len(releases), 1, + "_release() ran {} times, expected once".format(len(releases))) + self.assertEqual(reader._inflight, 0) + self.assertIsNone(reader._pending_teardown) + self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) + def test_concurrent_close_runs_release_once(self): """Two racing close() calls on one instance must run _release() exactly once. From 93832b2fb9ba1c214260ebbd20fa5674d54d317f Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 19:51:36 -0700 Subject: [PATCH 23/28] fix: Additional crashes handling (#315) * fix: Fix a crash * fix: Fix a crash * fix: Fix a crash --- .../README.md | 24 + .../faulthandler-output.txt | 16 + .../repro.py | 60 + docs/native-resources-management.md | 18 +- src/c2pa/c2pa.py | 143 +- .../python-3.10-slim-perf-Dockerfile | 22 - .../python-3.12-slim-perf-Dockerfile | 22 - .../Dockerfiles/ubuntu-22.04-perf-Dockerfile | 31 - .../Dockerfiles/ubuntu-24.04-perf-Dockerfile | 31 - tests/perf/README.md | 265 --- tests/perf/__init__.py | 1 - tests/perf/baseline.json | 305 ---- tests/perf/entrypoint.sh | 44 - tests/perf/run_profile.py | 388 ----- tests/perf/scenarios.py | 1436 ----------------- tests/test_unit_tests_threaded.py | 361 +++++ 16 files changed, 589 insertions(+), 2578 deletions(-) create mode 100644 crashes/context-close-drops-signer-callback-mid-sign/README.md create mode 100644 crashes/context-close-drops-signer-callback-mid-sign/faulthandler-output.txt create mode 100644 crashes/context-close-drops-signer-callback-mid-sign/repro.py delete mode 100644 tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile delete mode 100644 tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile delete mode 100644 tests/perf/Dockerfiles/ubuntu-22.04-perf-Dockerfile delete mode 100644 tests/perf/Dockerfiles/ubuntu-24.04-perf-Dockerfile delete mode 100644 tests/perf/README.md delete mode 100644 tests/perf/__init__.py delete mode 100644 tests/perf/baseline.json delete mode 100644 tests/perf/entrypoint.sh delete mode 100644 tests/perf/run_profile.py delete mode 100644 tests/perf/scenarios.py diff --git a/crashes/context-close-drops-signer-callback-mid-sign/README.md b/crashes/context-close-drops-signer-callback-mid-sign/README.md new file mode 100644 index 00000000..1d480a24 --- /dev/null +++ b/crashes/context-close-drops-signer-callback-mid-sign/README.md @@ -0,0 +1,24 @@ +The process dies with SIGSEGV (exit code 139, no Python exception) when a +`Context` built from a callback signer is closed on one thread while another +thread runs a context-sign (`Builder(manifest, context=ctx)` followed by +`builder.sign(format, source, dest)`) through it. + +40-120 trials: + +| Variant | Result | +|---|---| +| Close during concurrent context-sign, callback signer | SIGSEGV, reproducible | +| Same race, `Context._release` patched to keep the callback reference alive | 80/80 clean | +| Same race, context's native free suppressed (release still runs) | still SIGSEGV | +| Same race, info signer (`Signer.from_info`, no Python callback) | 120/120 clean | +| Single-threaded close-then-sign | clean (errors, no crash) | +| Dropping the last `ctx` reference mid-sign (finalizer close) | 80/80 clean | + +``` +python3 crashes/context-close-drops-signer-callback-mid-sign/repro.py +``` + +Exit code 139 within a few trials. The script: build a `Context` from +`Signer.from_callback(...)`, start a thread running a context-sign, sleep +~2 ms after the sign begins, call `ctx.close()` from the main thread, join, +repeat. diff --git a/crashes/context-close-drops-signer-callback-mid-sign/faulthandler-output.txt b/crashes/context-close-drops-signer-callback-mid-sign/faulthandler-output.txt new file mode 100644 index 00000000..fa1c6173 --- /dev/null +++ b/crashes/context-close-drops-signer-callback-mid-sign/faulthandler-output.txt @@ -0,0 +1,16 @@ +Fatal Python error: Segmentation fault + +Current thread 0x000000016e3ab000 (most recent call first): + File "/Users/taniamathern/Desktop/code/c2pa-python/src/c2pa/c2pa.py", line 4032 in _sign_internal + File "/Users/taniamathern/Desktop/code/c2pa-python/src/c2pa/c2pa.py", line 4110 in _sign_common + File "/Users/taniamathern/Desktop/code/c2pa-python/src/c2pa/c2pa.py", line 4186 in sign + File "/private/tmp/claude-501/-Users-taniamathern-Desktop-code-c2pa-python/a1e731b8-8f71-4e3d-a858-5b5d2dfe2dfb/scratchpad/crash/min.py", line 24 in w + File "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/threading.py", line 994 in run + File "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/threading.py", line 1043 in _bootstrap_inner + File "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/threading.py", line 1014 in _bootstrap + +Thread 0x00000001efdc1d80 (most recent call first): + File "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/threading.py", line 1094 in join + File "/private/tmp/claude-501/-Users-taniamathern-Desktop-code-c2pa-python/a1e731b8-8f71-4e3d-a858-5b5d2dfe2dfb/scratchpad/crash/min.py", line 31 in + +Extension modules: _cffi_backend (total: 1) diff --git a/crashes/context-close-drops-signer-callback-mid-sign/repro.py b/crashes/context-close-drops-signer-callback-mid-sign/repro.py new file mode 100644 index 00000000..61f6f195 --- /dev/null +++ b/crashes/context-close-drops-signer-callback-mid-sign/repro.py @@ -0,0 +1,60 @@ +"""SIGSEGV reproduction: Context.close() racing a context-sign that uses a +callback signer. Run from the repository root: + + python3 crashes/context-close-drops-signer-callback-mid-sign/repro.py + +Expected: the process dies with SIGSEGV (exit 139) within a few trials. +The crash needs the `cryptography` package for the ES256 callback. +""" +import sys, io, os, threading, time, faulthandler + +sys.path.insert(0, "src") +faulthandler.enable() + +from c2pa import Builder, Signer, Context, C2paSigningAlg as Alg +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec + +FIXTURES = "tests/fixtures" +certs = open(os.path.join(FIXTURES, "es256_certs.pem"), "rb").read().decode() +key_bytes = open(os.path.join(FIXTURES, "es256_private.key"), "rb").read() +image = open(os.path.join(FIXTURES, "C.jpg"), "rb").read() +MANIFEST = {"claim_generator_info": [{"name": "repro", "version": "0.1"}], + "assertions": []} + +private_key = serialization.load_pem_private_key(key_bytes, password=None) + + +def sign_callback(data: bytes) -> bytes: + return private_key.sign(data, ec.ECDSA(hashes.SHA256())) + + +def make_context() -> Context: + signer = Signer.from_callback(sign_callback, Alg.ES256, certs, + "http://timestamp.digicert.com") + return Context(signer=signer) # consumes the signer + + +for trial in range(80): + ctx = make_context() + entered = threading.Event() + + def worker(): + try: + builder = Builder(dict(MANIFEST), context=ctx) + entered.set() + builder.sign("image/jpeg", io.BytesIO(image), io.BytesIO()) + builder.close() + except Exception: + entered.set() + + t = threading.Thread(target=worker) + t.start() + entered.wait(5) + time.sleep(0.002) # let the sign enter the native call + ctx.close() # drops _signer_callback_cb mid-invocation + t.join(20) + if trial % 20 == 0: + print("trial", trial, "still alive") + +print("survived 80 trials (crash did not reproduce this run)") diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index c1900bf1..71fba3d2 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -124,9 +124,23 @@ A lock (Python's `threading.Lock`) can be acquired once, and a second `acquire() `_op_lock` is an `RLock` rather than a plain `Lock` for two reasons specific to this code. First, a finalizer (`__del__`) can run at any bytecode boundary — including one in the middle of a method that has already acquired the lock on this same thread — so `__del__` calling back into locked code must not deadlock against itself. Second, a consuming call tears the handle down from inside the locked region it is already holding: `_teardown()` is called while `_op_lock` is held, and it needs to acquire the same lock again rather than re-entering as a different, blocked acquisition. `_lock()` returns it, except in a forked child: there it raises `C2paError` immediately rather than blocking, because the thread that might hold the lock at fork time does not exist in the child to release it, and waiting on it would hang forever (see [Fork safety](#fork-safety)). -The lock is never held across a native call that drives a stream callback: construction, `resource_to_stream`, the Builder stream methods, and signing all release the GIL and call back into caller-supplied Python, which may itself call into this API on another thread. Holding `_op_lock` there would deadlock against that reentry. Those calls go through `_native_call()` instead: a context manager that increments `_inflight` under the lock, yields to run the native call unlocked, then decrements `_inflight` on the way out. If `_teardown()` runs while a call is in flight, it records the requested `free_handle` value in `_pending_teardown` and marks the resource `CLOSED` immediately, so no other caller can start using it, but defers the actual free. The last `_native_call()` to exit picks up `_pending_teardown` and runs `_teardown()` for real. +The lock is never held across a native call that drives a stream callback: construction, `resource_to_stream`, the Builder stream methods, and signing all release the Global Interpreter Lock (GIL) and call back into caller-supplied Python, which may itself call into this API on another thread. Holding `_op_lock` there would deadlock against that reentry. Those calls go through `_native_call()` instead: a context manager that increments `_inflight` under the lock, yields to run the native call unlocked, then decrements `_inflight` on the way out. If `_teardown()` runs while a call is in flight, it records the requested `free_handle` value in `_pending_teardown` and marks the resource `CLOSED` immediately, so no other caller can start using it, but defers the actual free. The last `_native_call()` to exit picks up `_pending_teardown` and runs `_teardown()` for real. -`Context.__init__` wraps the signer hand-off in `signer._native_call()`, so a `signer.close()` on another thread cannot free the handle between the state check and the consuming call. `Builder._sign_internal` wraps the sign call in `self._native_call()` and, when an explicit `Signer` is passed, nests `signer._native_call()` inside it in that fixed order, so two concurrent `sign()` calls sharing one `Signer` cannot deadlock by acquiring the two locks in opposite orders. The Builder's `close()` after signing runs outside its own `_native_call()` block, so a teardown deferred during the call still executes once the call returns. +`Context.__init__` does not wrap the signer hand-off in `signer._native_call()`: the consuming call marks the signer `CLOSED` under its own lock before calling native, which is what stops a `signer.close()` on another thread from freeing the handle mid-transfer (see [Borrowing versus consuming](#borrowing-versus-consuming)). `Builder._sign_internal` wraps the sign call in `self._native_call()` and, when an explicit `Signer` is passed, nests `signer._native_call()` inside it in that fixed order, so two concurrent `sign()` calls sharing one `Signer` cannot deadlock by acquiring the two locks in opposite orders. The Builder's `close()` after signing runs outside its own `_native_call()` block, so a teardown deferred during the call still executes once the call returns. When the Builder was created from a `Context` and signs through its context signer, `self._context._native_call()` is nested in that same position instead, for the reason described in [Context lifetime during a context-sign](#context-lifetime-during-a-context-sign). + +### Borrowing versus consuming + +Deferring a teardown protects a consuming call against a racing `close()`. It does not protect a borrowing call against a racing consume, which is a different risk with a different mitigation. + +A borrowing call passes the handle to native and gets it back unchanged. A consuming call hands ownership over, and the native side frees the pointer during the call. A borrowing call validates the pointer once on entry, then holds it for the duration of the operation. The pointer registry is never consulted again. So a consume starting midway through a borrow frees memory the borrowing call is still reading, and the usual `-1` rejection never happens because validation already succeeded. + +`ManagedResource` therefore refuses the consume rather than allowing it to start. `_begin_consume()` runs `_ensure_not_borrowed()`, which rejects the call when `_inflight` is nonzero, and then marks the resource `CLOSED` before releasing `_op_lock`. + +The check catches a borrow already in flight. The `CLOSED` mark catches one arriving afterwards: `_native_call()` calls `_ensure_valid_state()` under the same lock, so a borrow that starts later is refused instead of reaching a pointer about to be freed. The lock cannot simply be held across the native call, because those calls run caller-supplied stream callbacks that re-enter this API. + +The mark is provisional. `_abort_consume()` restores the previous state when the native call turns out not to have taken the handle, which keeps the retained branch of the [ownership-taken triage](#why-an-ownership-taken-failure-does-not-free) handing back a usable object. + +`_consume_and_swap()` is excluded. `_swap_handle()` requires the resource to stay `ACTIVE` and the object remains usable with its replacement pointer, so there is no `CLOSED` mark to make and no check. Its callers (`Reader.with_fragment`, `Builder.with_archive`) pass streams whose callbacks re-enter this API, so they hold their own `_native_call()`, and they act on resources the caller is required to serialize. ## Guarantees provided by ManagedResource diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 9e635d1b..2a9b30ca 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -284,8 +284,9 @@ def _lock(self): Never hold this across a native call that drives stream callbacks (construction, resource_to_stream, the Builder stream methods, - signing). Those calls release the GIL and re-enter caller-supplied - Python, which may call back into this API on another thread. + signing). Those calls release the Global Interpreter Lock (GIL) + and re-enter caller-supplied Python code, which may call back into + this API on another thread. Only calls that touch no callbacks are serialized here. Raises in a forked child rather than returning the lock. @@ -307,6 +308,18 @@ def _lock(self): pass return lock + def _ensure_not_borrowed(self): + """Raise if a native call is in flight on this handle. + + Raises: + C2paError: If a native call is in flight on this resource. + """ + if getattr(self, '_inflight', 0) > 0: + name = type(self).__name__ + raise C2paError( + f"{name} is in use by another operation and " + f"cannot be consumed") + @contextlib.contextmanager def _native_call(self): """Hold the handle valid across a native call that goes back @@ -400,7 +413,7 @@ def _teardown(self, free_handle: bool): thread's state check and its use of the handle in a native call. The forked-child case is handled before the lock is taken, because - _lock() refuses in a child: this path has to finish rather than report + _lock() raises in a child: this path has to finish rather than report an error, so it cannot rely on acquiring. """ if is_foreign_process(self): @@ -413,19 +426,32 @@ def _teardown(self, free_handle: bool): with self._lock(): if getattr(self, '_released', False): # A racing close()/__del__ already ran the release branch - # under this lock. Idempotent: nothing left to release or free. + # under this lock. + # Idempotent: nothing left to release or free. # Keyed on the release having happened, not on CLOSED: the # deferred path below sets CLOSED without releasing, and still # owes a release performed by _native_call()'s finally. return if getattr(self, '_inflight', 0) > 0: - # A native call is running that re-enters calling non-native code - # and is still using this handle. + # A native call is running that re-enters calling non-native + # code and is still using this handle. # Record the intent and whichever caller leaves # _native_call last performs the free. # Mark the resource closed now so it cannot be used # while the free is pending. - self._pending_teardown = free_handle + # + # free_handle=False records that a consuming call handed + # ownership to the native library. Ownership does not come + # back, so a later teardown cannot restore the right to free: + # the recorded value only ever moves True -> False, never the + # reverse. Without this, a _teardown(True) arriving second + # (from _release_handle, whose state check is read outside + # this lock and can go stale) frees a pointer native owns. + if self._pending_teardown is None: + self._pending_teardown = free_handle + else: + self._pending_teardown = ( + self._pending_teardown and free_handle) self._lifecycle_state = LifecycleState.CLOSED return @@ -606,10 +632,44 @@ def _raise_consume_failure(self, error_message): self._release_handle() raise C2paError(error_message.format("Unknown error")) + def _begin_consume(self): + """Reserve this handle for a consuming call, or raise. + + Returns: + The lifecycle state to restore if the call turns out not to have + consumed the handle. + + Raises: + C2paError: If a native call is in flight on this resource. + """ + with self._lock(): + # A consumed or closed resource has no handle left to hand over; + # without this the call would pass a null pointer to native. + self._ensure_valid_state() + self._ensure_not_borrowed() + previous = self._lifecycle_state + self._lifecycle_state = LifecycleState.CLOSED + return previous + + def _abort_consume(self, previous_state): + """Undo _begin_consume() after a call that did not take the handle. + + A pre-consume rejection leaves the handle ours, + so the resource has to become usable again. + """ + with self._lock(): + if self._lifecycle_state == LifecycleState.CLOSED and self._handle: + self._lifecycle_state = previous_state + def _consume_and_swap(self, ffi_call, error_message): """Run an FFI call that consumes this handle and returns a replacement. On success the native lib consumed the handle and returned a new one, which we swap in. A null return is a failure. + + Unlike the consuming teardown paths this neither refuses a borrowed + handle nor pre-marks the resource CLOSED: _swap_handle() requires it to + stay ACTIVE, and the object remains usable afterwards with its new + pointer. """ new_ptr = self._invoke_consume(ffi_call, error_message) if new_ptr: @@ -623,10 +683,16 @@ def _consume_no_replacement(self, ffi_call, error_message): handle. A non-zero status is a failure routed to _raise_consume_failure. """ - result = self._invoke_consume(ffi_call, error_message) + previous_state = self._begin_consume() + try: + result = self._invoke_consume(ffi_call, error_message) + except Exception: + self._abort_consume(previous_state) + raise if result == 0: self._teardown(free_handle=False) return + self._abort_consume(previous_state) self._raise_consume_failure(error_message) def _consume_into(self, ffi_call, error_message): @@ -635,10 +701,16 @@ def _consume_into(self, ffi_call, error_message): and the new pointer is returned for the caller to own. A null return is a failure routed to _raise_consume_failure. """ - result = self._invoke_consume(ffi_call, error_message) + previous_state = self._begin_consume() + try: + result = self._invoke_consume(ffi_call, error_message) + except Exception: + self._abort_consume(previous_state) + raise if result: self._teardown(free_handle=False) return result + self._abort_consume(previous_state) self._raise_consume_failure(error_message) @classmethod @@ -1809,22 +1881,21 @@ def __init__( check=lambda r: r != 0) if signer is not None: - # The signer's in-flight guard: - # this hands its handle to native, - # so a signer.close() on another thread must not - # free it between the state check and the call. + # No in-flight guard around the hand-off: the consume + # marks the signer CLOSED under its lock before calling + # native, which is what stops a signer.close() on another + # thread from freeing the handle mid-transfer. That mark + # also makes the consume refuse to start while another + # thread is borrowing the handle to sign with. # - # _consume_no_replacement tears the signer down from - # inside this region. A teardown recorded while the guard - # is held is deferred and performed as the guard unwinds, - # which is still before __init__ returns. - with signer._native_call(): - # A rejected signer is retained, not closed and leaked. - self._signer_callback_cb = signer._callback_cb - signer._consume_no_replacement( - lambda h: _lib.c2pa_context_builder_set_signer( - nb._handle, h), - "Failed to set signer on Context: {}") + # Pin the callback first: a rejected signer is retained, + # not closed and leaked, and _release() nulls _callback_cb + # once the signer is torn down. + self._signer_callback_cb = signer._callback_cb + signer._consume_no_replacement( + lambda h: _lib.c2pa_context_builder_set_signer( + nb._handle, h), + "Failed to set signer on Context: {}") self._has_signer = True context_ptr = nb._consume_into( @@ -3959,13 +4030,23 @@ def _sign_internal( ctypes.byref(manifest_bytes_ptr) ) else: - result = _lib.c2pa_builder_sign_context( - self._handle, - format_arg, - source_stream._stream, - dest_stream._stream, - ctypes.byref(manifest_bytes_ptr), - ) + # The Context pins the consumed signer's callback, which + # native invokes during this call. + # Its in-flight guard defers a close() arriving on another + # thread, the same way the signer branch above defers one + # for a borrowed Signer. + # + # Entered inside self's guard, matching the Builder to + # Signer order, so the two acquisitions are always + # taken in one direction. + with self._context._native_call(): + result = _lib.c2pa_builder_sign_context( + self._handle, + format_arg, + source_stream._stream, + dest_stream._stream, + ctypes.byref(manifest_bytes_ptr), + ) # Sign borrows the Builder without taking ownership. # Closing here ensures resources clean up, # and single use/single sign done by a Builder. diff --git a/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile b/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile deleted file mode 100644 index 100a082f..00000000 --- a/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM python:3.10.20-slim-bookworm - -WORKDIR /workspace - -# libunwind-dev for memray native stack unwinding. -RUN apt-get update && apt-get install -y --no-install-recommends \ - libunwind-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# Pre-install Python deps using only the requirements files (layer-cached). -# The full project arrives via the -v mount at runtime. -COPY requirements.txt requirements-dev.txt ./ -RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt - -RUN pip install --no-cache-dir memray==1.19.3 - -COPY tests/perf/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["python", "-m", "tests.perf.run_profile"] diff --git a/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile b/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile deleted file mode 100644 index 03968dbc..00000000 --- a/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM python:3.12.13-slim-bookworm - -WORKDIR /workspace - -# libunwind-dev for memray native stack unwinding. -RUN apt-get update && apt-get install -y --no-install-recommends \ - libunwind-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# Pre-install Python deps using only the requirements files (layer-cached). -# The full project arrives via the -v mount at runtime. -COPY requirements.txt requirements-dev.txt ./ -RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt - -RUN pip install --no-cache-dir memray==1.19.3 - -COPY tests/perf/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["python", "-m", "tests.perf.run_profile"] diff --git a/tests/perf/Dockerfiles/ubuntu-22.04-perf-Dockerfile b/tests/perf/Dockerfiles/ubuntu-22.04-perf-Dockerfile deleted file mode 100644 index 649422ac..00000000 --- a/tests/perf/Dockerfiles/ubuntu-22.04-perf-Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM ubuntu:22.04 - -ENV DEBIAN_FRONTEND=noninteractive - -WORKDIR /workspace - -# Ubuntu 22.04 ships Python 3.10 as python3 by default. -# libunwind for memray native stack unwinding. -# python3-dbg supplies the interpreter's debug symbols so memray can resolve -# file names + line numbers for native (C) frames in the flamegraphs. -RUN apt-get update && apt-get install -y --no-install-recommends \ - python3 \ - python3-pip \ - python3-venv \ - python3-dbg \ - libunwind-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* \ - && ln -s /usr/bin/python3 /usr/bin/python - -# Pre-install runtime deps only. Project arrives via -v mount. -COPY requirements.txt ./ -RUN pip3 install --no-cache-dir -r requirements.txt - -RUN pip3 install --no-cache-dir memray==1.19.3 requests==2.34.2 - -COPY tests/perf/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["python", "-m", "tests.perf.run_profile"] diff --git a/tests/perf/Dockerfiles/ubuntu-24.04-perf-Dockerfile b/tests/perf/Dockerfiles/ubuntu-24.04-perf-Dockerfile deleted file mode 100644 index 0fd3a523..00000000 --- a/tests/perf/Dockerfiles/ubuntu-24.04-perf-Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM ubuntu:24.04 - -ENV DEBIAN_FRONTEND=noninteractive - -WORKDIR /workspace - -# Ubuntu 24.04 ships Python 3.12 as python3 by default. -# libunwind used for memray native stack unwinding. -# python3-dbg supplies the interpreter's debug symbols so memray can resolve -# file names + line numbers for native (C) frames in the flamegraphs. -RUN apt-get update && apt-get install -y --no-install-recommends \ - python3 \ - python3-pip \ - python3-venv \ - python3-dbg \ - libunwind-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* \ - && ln -s /usr/bin/python3 /usr/bin/python - -# Pre-install runtime deps only. Project arrives via -v mount. -COPY requirements.txt ./ -RUN pip3 install --no-cache-dir --break-system-packages -r requirements.txt - -RUN pip3 install --no-cache-dir --break-system-packages memray==1.19.3 requests==2.34.2 - -COPY tests/perf/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["python", "-m", "tests.perf.run_profile"] diff --git a/tests/perf/README.md b/tests/perf/README.md deleted file mode 100644 index 1e2baf41..00000000 --- a/tests/perf/README.md +++ /dev/null @@ -1,265 +0,0 @@ -# Memory profiling framework - -Uses [memray](https://github.com/bloomberg/memray) to track peak memory, allocation patterns, -and memory leaks across c2pa-python SDK operations. - -## Files - -| File | Purpose | -| --- | --- | -| `scenarios.py` | Functions that exercise each profiling scenario. Imported by `run_profile.py`. | -| `run_profile.py` | Memory performance/usage analysis. Runs each scenario under `memray`, generates HTML reports, reads metrics, and compares against `baseline.json`. | -| `Dockerfiles/` | One Dockerfile per target environment. Selected via `PERF_ENV` at `make` time when running the memory analysis. | -| `entrypoint.sh` | Container entrypoint. Downloads the Linux native `libc2pa_c.so` at startup into the volume-mounted workspace so it sticks around even through the `-v` mount. | -| `reports/` | Generated HTML reports (gitignored). Three files per scenario: `-peak.html` (peak/high-water view), `-leaks.html` (leak view), and `-temporary.html` (temporary-allocations view). | - -## Scenarios - -Each scenario loops multiple times so leaks accumulate and become visible in the leaks flamegraph and the memory use graph (defaults to 100). Change the count of iterations when running by setting the `MEMRAY_ITERATIONS` variable (the Makefile forwards it into the container): - -```bash -make memory-use-bench MEMRAY_ITERATIONS=1000 -``` - -Most scenarios use the Context API: they build a `Context` once and reuse it across iterations, so its settings are parsed a single time. The jpeg and png cases also keep a `_legacy` variant that builds the `Reader`/`Builder` without a `Context`, which re-reads the thread-local settings on each construction. Running a pair (for example `builder_sign_jpeg_legacy` and `builder_sign_jpeg_with_context`) compares the two paths. - -The `builder_sign_{jpeg,png}_parallel_*` scenarios build one `Context` and share it across 10 threads that sign concurrently, each with its own streams and `Builder`. The name encodes two axes. `split` divides the iteration budget across the threads, so total work matches a single-threaded scenario; `full` runs the full loop on each of the 10 threads, so total work is 10x (use these with `SCENARIO=` rather than the whole suite). `pool` runs the threads through a `ThreadPoolExecutor`; `barrier` starts all 10 at once with a `threading.Barrier`. - -## Environments - -Select the target environment with `PERF_ENV` (default: `python-3.12-slim`): - -| `PERF_ENV` value | Base image | Python | Native symbols | -| --- | --- | --- | --- | -| `python-3.12-slim` | `python:3.12-slim` | 3.12 | interpreter frames unresolved | -| `python-3.10-slim` | `python:3.10-slim` | 3.10 | interpreter frames unresolved | -| `ubuntu-22.04` | `ubuntu:22.04` | 3.10 (apt default) | resolved (`python3-dbg`) | -| `ubuntu-24.04` | `ubuntu:24.04` | 3.12 (apt default) | resolved (`python3-dbg`) | - -The slim images run a source-built `/usr/local/bin/python` that ships stripped, and Debian's `python3-dbg` targets a different binary (build-id mismatch), so memray cannot resolve the interpreter's native (C) frames there. You will see a "No debug information was found for the Python interpreter" warning, and native traces may lack file names and line numbers. The ubuntu images install `python3-dbg` for the matching apt interpreter, so their native flamegraphs are fully symbolized. Use an `ubuntu-*` `PERF_ENV` when you need resolved native traces. - -## Running (via Docker) - -```bash -# First run (if there is no baseline.json): establishes baseline.json -make memory-use-bench - -# Subsequent runs: compares against baseline, fails if >10% regression -make memory-use-bench - -# Refresh baseline after an intentional memory change -make memory-use-bench PERF_ARGS=--update-baseline - -# Run against a different runner environment -make memory-use-bench PERF_ENV=ubuntu-24.04 - -# Run a single scenario instead of the whole suite -make memory-use-bench SCENARIO=builder_sign_gif - -# Refresh just one scenario's baseline entry (others are preserved) -make memory-use-bench SCENARIO=builder_sign_gif PERF_ARGS=--update-baseline - -# Remove all generated HTML reports -make clean-memory-perf-reports -``` - -The trailing `VAR=value` arguments (e.g. `PERF_ENV=ubuntu-24.04`, `PERF_ARGS=--update-baseline`) are `make` variable overrides, not shell env vars. `make` parses `word=value` argument as a variable assignment. Each overrides a `?=` default in the Makefile, and the recipe interpolates them into the `docker build`/`docker run` commands. See [Configuration](#configuration) for the full list and what each forwards to. - -Reports are written to `tests/perf/reports/` on the local machine. Three HTML files per scenario, one per suffix (described below). Open any in a browser. After a run, the run also reports if the scenarios were or were not all within baseline threshold (baseline +10% memory use tolerance). - -## Running in CI - -The `.github/workflows/memory-benchmark.yml` workflow runs the Docker-based benchmarks on a PR, but only when the PR has the `check-memory-benchmark` label. This runs `make memory-use-bench`, so: - -- A regression (peak or leaked > baseline +10%) makes the benchmark job exit non-zero. -- A values report table is written to the job's Step Summary. -- All three flamegraph HTML views per scenario are uploaded as the `memray-flamegraphs` artifact. - -The gate only acts as regression test once a `tests/perf/baseline.json` is committed on the branch. Without one, `run_profile.py` treats the run as baseline creation (exits 0, no gating). - -## Report views - -Each scenario produces three [memray flamegraphs](https://bloomberg.github.io/memray/flamegraph.html). All three are flamegraphs of the same run. They differ only in which allocations they count. - -### `-peak.html`: peak/high-water view - -What it shows: allocations that were simultaneously alive at the moment the process used the most memory (the high-water mark). - -Why it's useful: tells you what drives the largest memory footprint, the working set you must hold at once. Consult this view when you care about peak RSS or OOM headroom. - -How to read it: the widest frames are the biggest contributors to peak. Walk up a wide column to the top frame to find the call site holding that memory at the high-water instant. - -### `-leaks.html`: leak view - -What it shows: memory that was allocated but never freed before tracking stopped (`memray --leaks`). - -Why it's useful: finds memory leaks, meaning memory that grows with work done. It is never zero, because one-time static setup (the native `libc2pa_c` library loading global structures that live for the whole process) shows as "never freed." A real leak is one that scales with iterations. Profile at `MEMRAY_ITERATIONS=100` and `=1000` and compare: flat means static overhead, growing means a leak. See [Why is leaked_bytes not zero?](#why-is-leaked_bytes-not-zero). - -How to read it: a wide frame here is unfreed memory. If its width grows when you raise the iteration count, that top frame is the leaking call site. - -### `-temporary.html`: temporary-allocations view - -What it shows: short-lived churn, meaning memory allocated and then freed almost immediately (memray's threshold: freed before more than one other allocation happens). - -Why it's useful: temporary allocations are not leaks, since the memory is returned, but high allocation and free turnover costs CPU and can fragment the heap. This view surfaces hot per-call churn that the peak and leak views hide, because those objects are freed between iterations and so barely register at the high-water mark. Use it when a loop allocates too much. - -How to read it: wide frames are the biggest sources of throwaway allocations. The view may be sparse or empty for a scenario that does little churn, which is itself a valid result. See [Temporary allocations](#temporary-allocations). - -The temporary view is the heaviest to render: memray holds every allocation and free to decide which are short-lived. On a very large capture (a long run, a high `MEMRAY_ITERATIONS`, or a churn-heavy scenario) the render can run out of memory and fail. The run does not abort in that case; it records what failed and keeps going. See [Troubleshooting](#troubleshooting). - -## Running without Docker (if memray is supported and installed locally) - -```bash -pip install memray -python -m tests.perf.run_profile -``` - -Run a single scenario (useful for generating data for one operation without the full suite): - -```bash -python -m tests.perf.run_profile --scenario builder_sign_gif -``` - -With `--update-baseline`, a single-scenario run only rewrites that scenario's entry in `baseline.json`; the other scenarios' entries are preserved. - -```bash -python -m tests.perf.run_profile --scenario builder_sign_gif --update-baseline -``` - -## Configuration - -With `make memory-use-bench VAR=value` you set the **`make` variable** and the Makefile forwards it as shown in the "Forwarded as" column. Running `run_profile.py` without Docker, you set the **env var** (or pass the CLI arg) directly. - -| `make` variable | Forwarded as | Default | Description | -| --- | --- | --- | --- | -| `PERF_ENV` | `PERF_ENV` env var | `python-3.12-slim` | Target environment; selects the Dockerfile, tags report filenames (`--.html`), recorded in `baseline.json` `_meta`. See [Environments](#environments). | -| `MEMRAY_ITERATIONS` | `MEMRAY_ITERATIONS` env var | `100` | Loop count per scenario. | -| `MEMRAY_THRESHOLD` | `MEMRAY_THRESHOLD` env var | `1.1` | Regression multiplier (1.1 = 10% tolerance). | -| `SCENARIO` | `--scenario` CLI arg | _(all)_ | Run a single scenario (e.g. `SCENARIO=builder_sign_jpeg`). | -| `PERF_ARGS` | passed straight through | _(none)_ | Extra `run_profile.py` args (e.g. `PERF_ARGS=--update-baseline`). | - -`PERF_SCENARIO` is an additional env var, but internal: the runner sets it per scenario so the loop can label its progress. Not user-configurable. - -Example to override iteration count: - -```bash -make memory-use-bench MEMRAY_ITERATIONS=1000 -``` - -## Reading baseline.json - -`baseline.json` is committed to the repo and reports following data for each scenario: - -```json -{ - "_meta": { - "memray_version": "1.19.3", - "python_version": "3.12.13", - "c2pa_native_version": "c2pa-v0.85.0", - "iterations": 100, - "perf_env": "python-3.12-slim", - "arch": "x86_64" - }, - "scenario_name": { - "peak_bytes": 62914560, - "leaked_bytes": 3271766, - "total_allocations": 12840 - }, - ... -} -``` - -The `_meta` block records which toolchain produced the baseline so the numbers are reproducible. It is provenance only and is never compared against. The regression check only looks at the per-scenario entries. - -| `_meta` field | Meaning | -| --- | --- | -| `memray_version` | memray version that generated the metrics | -| `python_version` | Python version that ran the test framework | -| `c2pa_native_version` | native `libc2pa_c` version (from `c2pa-native-version.txt`) | -| `iterations` | `MEMRAY_ITERATIONS` used for the run | -| `perf_env` | `PERF_ENV` (target environment) | -| `arch` | machine architecture (`platform.machine()`) | - -`peak_bytes`, `total_allocations` and the `arch`/`python`/`memray` versions are all environment-sensitive: a baseline is most meaningful when compared against a run from the same `_meta`. - -`peak_bytes` is the highest amount of memory in use at any single point during the scenario. - -`leaked_bytes` is memory that was allocated during the run but never freed before the process exited. Static allocations persist, since there are one-time loads such as the native library. - -`total_allocations` is the total number of individual memory allocation calls made. - -### Why is leaked_bytes not zero? - -You might expect the baseline to show `leaked_bytes: 0`. In practice it never does. When the c2pa native library (`libc2pa_c.so`) is first loaded, Rust sets up global data structures designed to live for the entire lifetime of the process. They get cleaned up when the process exits, which is after memray stops watching, so memray sees them as "never freed" even though they are not leaking. - -A memory leak grows proportionally with work done. If you sign 50 images and get 3.2 MB leaked, then sign 1000 images and still get 3.2 MB leaked, that 3.2 MB is static one-time overhead rather than a leak, since it does not grow with the work that ran. If signing 1000 images gave you 64 MB leaked, that would be a leak, as the leaked memory grows with the work executed. - -The baseline captures this expected static overhead. Future runs compare against it: if `leaked_bytes` grows beyond the baseline by more than 10%, the run fails. - -The framework runs `gc.collect()` twice after the scenario finishes, while memray is still tracking. Without that sweep, objects sitting in not-yet-collected reference cycles would be counted in `leaked_bytes` and the number would depend on garbage collector timing rather than on actual leaks. With it, `leaked_bytes` means memory that is still allocated even though nothing in Python can reach it: true leaks plus the one-time static overhead described above. - -### How to confirm no leak exists? - -Run with a higher iteration count than default (100) and compare: - -```bash -make memory-use-bench MEMRAY_ITERATIONS=1000 PERF_ARGS=--update-baseline -``` - -If `leaked_bytes` stays flat compared to a baseline run or in a larger run (more iterations), there is no leak. If it scales with iterations, open `tests/perf/reports/-leaks.html` in a browser to see which function is responsible. - -### Reading the "Resident set size over time" graph (why memory looks like it climbs) - -The "Resident set size over time" plot (chart icon, top-right of the report) draws two lines. "Resident size" (RSS) is every page the OS counts as resident: interpreter and pages the allocator holds but has not returned. "Heap size" is only the live tracked allocations. - -On the parallel scenarios the RSS line steps up and stays high. The threads each hold their own source, output, and `Builder` live at once, so RSS rises to cover that combined working set (the steps line up with the moments all threads overlap). The allocator then keeps those arena pages for reuse instead of returning them, so RSS plateaus at the high-water mark. - -Judge leaks by the heap line. The heap rises early and then settles or falls, the same shape as the single-threaded baseline. A within-run heap rise is not by itself proof of a leak (the allocator high-water can climb and settle within a bounded run). - -### Temporary allocations - -`-temporary.html` shows temporary allocations, meaning memory that is allocated and then freed almost immediately (memray's threshold is one allocation: a block is temporary if it is freed before more than one other allocation happens). The memory is returned, so these are not leaks, but they are churn: high allocation and free turnover that costs CPU and can fragment the heap. A scenario doing lots of short-lived work can show heavy temporary allocations while `leaked_bytes` stays flat. - -### When to update the baseline - -Update `baseline.json` after any intentional change that affects memory use: - -```bash -make memory-use-bench PERF_ARGS=--update-baseline -``` - -Commit the updated `baseline.json` alongside the code change, so it becomes the new reference to compare against. - -## Troubleshooting - -### A flamegraph render fails with `exit -9` - -You may see a message like `flamegraph render failed for reader_mp4-...-temporary.html (killed (likely OOM))`. The `-9` is SIGKILL: the operating system's out-of-memory killer terminated the `memray flamegraph` subprocess. The temporary view is the heaviest to render, and on a large capture (a long run, a high `MEMRAY_ITERATIONS`, or a churn-heavy scenario such as `reader_mp4`) it can exhaust available memory. - -The run does not abort. The capture and the metrics (`peak_bytes`, `leaked_bytes`, `total_allocations`) are read separately and are still recorded, the baseline is still written, and the run lists every failed render at the end. Only the HTML render is missing, and you have two ways to regenerate it. - -#### Option A: rerun the one scenario - -A single-scenario run renders one capture at a time with nothing else resident, so it often fits where the full suite did not: - -```bash -make memory-use-bench SCENARIO=reader_mp4 -``` - -If it still runs out of memory, lower the iteration count to shrink the capture: - -```bash -make memory-use-bench SCENARIO=reader_mp4 MEMRAY_ITERATIONS=20 -``` - -A lower iteration count makes that scenario's absolute allocation numbers no longer directly comparable to a full 100-iteration run. - -#### Option B: re-render the kept capture (no re-profiling) - -When a render fails, the run keeps that scenario's capture as `reports/-.bin`. Re-render just the failed view from that file with a higher temporary-allocation threshold, which cuts how much memray holds in memory so the render fits. This uses the original run's data, so the result stays comparable to the rest of the run: - -```bash -python3 -m memray flamegraph reports/reader_mp4-python-3.12-slim.bin \ - -o reports/reader_mp4-python-3.12-slim-temporary.html \ - --temporary-allocations --temporary-allocation-threshold=10 --force -``` diff --git a/tests/perf/__init__.py b/tests/perf/__init__.py deleted file mode 100644 index a56982a7..00000000 --- a/tests/perf/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Empty placeholder file to facilitate imports \ No newline at end of file diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json deleted file mode 100644 index 74a431a9..00000000 --- a/tests/perf/baseline.json +++ /dev/null @@ -1,305 +0,0 @@ -{ - "_meta": { - "memray_version": "1.19.3", - "python_version": "3.12.13", - "c2pa_native_version": "c2pa-v0.90.15", - "iterations": 200, - "perf_env": "python-3.12-slim", - "arch": "aarch64" - }, - "reader_jpeg_legacy": { - "peak_bytes": 3878176, - "leaked_bytes": 3381657, - "total_allocations": 1307172 - }, - "reader_jpeg_with_context": { - "peak_bytes": 3872284, - "leaked_bytes": 3374437, - "total_allocations": 1299545 - }, - "reader_manifest_data_context": { - "peak_bytes": 7658137, - "leaked_bytes": 3491877, - "total_allocations": 1098306 - }, - "reader_mp4": { - "peak_bytes": 4238670, - "leaked_bytes": 3374080, - "total_allocations": 3984581 - }, - "reader_wav": { - "peak_bytes": 4539135, - "leaked_bytes": 3384038, - "total_allocations": 739057 - }, - "builder_sign_jpeg_legacy": { - "peak_bytes": 7810402, - "leaked_bytes": 3498186, - "total_allocations": 1019644 - }, - "builder_sign_jpeg_with_context": { - "peak_bytes": 7802790, - "leaked_bytes": 3490850, - "total_allocations": 1005722 - }, - "builder_sign_png_legacy": { - "peak_bytes": 8048349, - "leaked_bytes": 3498022, - "total_allocations": 3861729 - }, - "builder_sign_png_with_context": { - "peak_bytes": 8041266, - "leaked_bytes": 3491795, - "total_allocations": 3847713 - }, - "builder_sign_jpeg_parallel_split_pool": { - "peak_bytes": 45869711, - "leaked_bytes": 3860295, - "total_allocations": 1009812 - }, - "builder_sign_jpeg_parallel_split_barrier": { - "peak_bytes": 45838322, - "leaked_bytes": 3859113, - "total_allocations": 1008528 - }, - "builder_sign_png_parallel_split_pool": { - "peak_bytes": 46107474, - "leaked_bytes": 3877946, - "total_allocations": 3851810 - }, - "builder_sign_png_parallel_split_barrier": { - "peak_bytes": 46075853, - "leaked_bytes": 3877260, - "total_allocations": 3850542 - }, - "builder_sign_gif": { - "peak_bytes": 14660656, - "leaked_bytes": 3491475, - "total_allocations": 16995947 - }, - "builder_sign_heic": { - "peak_bytes": 4723711, - "leaked_bytes": 3499336, - "total_allocations": 1529895 - }, - "builder_sign_m4a": { - "peak_bytes": 18859208, - "leaked_bytes": 3499290, - "total_allocations": 5160957 - }, - "builder_sign_webp": { - "peak_bytes": 9016473, - "leaked_bytes": 3491521, - "total_allocations": 898326 - }, - "builder_sign_avi": { - "peak_bytes": 7156127, - "leaked_bytes": 3491475, - "total_allocations": 89959516 - }, - "builder_sign_mp4": { - "peak_bytes": 6270688, - "leaked_bytes": 3499335, - "total_allocations": 3753347 - }, - "builder_sign_tiff": { - "peak_bytes": 13238405, - "leaked_bytes": 3491521, - "total_allocations": 10845456 - }, - "builder_sign_jpeg_parent_of": { - "peak_bytes": 14290485, - "leaked_bytes": 3492132, - "total_allocations": 2434129 - }, - "builder_sign_jpeg_component_of": { - "peak_bytes": 14291315, - "leaked_bytes": 3491288, - "total_allocations": 2477796 - }, - "builder_sign_jpeg_parent_and_component": { - "peak_bytes": 14638503, - "leaked_bytes": 3636596, - "total_allocations": 4394837 - }, - "builder_sign_jpeg_parent_and_component_mixed_mime": { - "peak_bytes": 14593417, - "leaked_bytes": 3492387, - "total_allocations": 5447516 - }, - "builder_sign_jpeg_two_components_same_mime": { - "peak_bytes": 14631537, - "leaked_bytes": 3636600, - "total_allocations": 4367414 - }, - "builder_sign_jpeg_two_components_mixed_mime": { - "peak_bytes": 14589983, - "leaked_bytes": 3492082, - "total_allocations": 5419842 - }, - "builder_sign_jpeg_archive_roundtrip": { - "peak_bytes": 14321971, - "leaked_bytes": 3512017, - "total_allocations": 3343806 - }, - "builder_from_archive_roundtrip": { - "peak_bytes": 14320730, - "leaked_bytes": 3510869, - "total_allocations": 2987299 - }, - "builder_with_archive_swap": { - "peak_bytes": 3720558, - "leaked_bytes": 3389404, - "total_allocations": 708591 - }, - "reader_with_fragment_swap": { - "peak_bytes": 3805398, - "leaked_bytes": 3382233, - "total_allocations": 3769246 - }, - "reader_with_fragment_repeated": { - "peak_bytes": 3803023, - "leaked_bytes": 3380589, - "total_allocations": 1842543 - }, - "with_fragment_pre_consume_rejection": { - "peak_bytes": 3805255, - "leaked_bytes": 3384325, - "total_allocations": 2093470 - }, - "with_archive_post_consume_failure": { - "peak_bytes": 3388641, - "leaked_bytes": 3346670, - "total_allocations": 185458 - }, - "with_fragment_marshalling_error": { - "peak_bytes": 3734139, - "leaked_bytes": 3381598, - "total_allocations": 2072540 - }, - "with_fragment_mixed_outcomes": { - "peak_bytes": 3803836, - "leaked_bytes": 3382987, - "total_allocations": 2650818 - }, - "builder_to_archive_with_ingredient": { - "peak_bytes": 14107950, - "leaked_bytes": 3375874, - "total_allocations": 1766289 - }, - "builder_sign_jpeg_archive_roundtrip_ingredient_in_archive": { - "peak_bytes": 14311785, - "leaked_bytes": 3511500, - "total_allocations": 5681125 - }, - "builder_write_ingredient_archive": { - "peak_bytes": 14107944, - "leaked_bytes": 3375872, - "total_allocations": 1742859 - }, - "builder_sign_jpeg_add_ingredient_from_archive": { - "peak_bytes": 14174086, - "leaked_bytes": 3512155, - "total_allocations": 3320279 - }, - "builder_ingredient_archive_roundtrip": { - "peak_bytes": 14310577, - "leaked_bytes": 3512018, - "total_allocations": 4973895 - }, - "builder_sign_jpeg_two_ingredient_archives": { - "peak_bytes": 14173988, - "leaked_bytes": 3512433, - "total_allocations": 4113195 - }, - "reader_error_no_manifest": { - "peak_bytes": 3588164, - "leaked_bytes": 3352030, - "total_allocations": 276214 - }, - "builder_error_invalid_manifest": { - "peak_bytes": 3388827, - "leaked_bytes": 3333835, - "total_allocations": 115678 - }, - "reader_string_apis": { - "peak_bytes": 4005286, - "leaked_bytes": 3375590, - "total_allocations": 2183974 - }, - "signer_construction": { - "peak_bytes": 3388872, - "leaked_bytes": 3325939, - "total_allocations": 155796 - }, - "builder_from_context_construction": { - "peak_bytes": 3388641, - "leaked_bytes": 3327073, - "total_allocations": 120460 - }, - "fork_reader_collect": { - "peak_bytes": 3877630, - "leaked_bytes": 3381450, - "total_allocations": 1271971 - }, - "fork_contended_mutex": { - "peak_bytes": 7646946, - "leaked_bytes": 3477655, - "total_allocations": 65668554 - }, - "fork_thread_local_orphan": { - "peak_bytes": 3960026, - "leaked_bytes": 3468161, - "total_allocations": 1324309 - }, - "fork_gc_cycle": { - "peak_bytes": 3875869, - "leaked_bytes": 3379622, - "total_allocations": 1276946 - }, - "fork_parent_frees_after_fork": { - "peak_bytes": 5561869, - "leaked_bytes": 3389085, - "total_allocations": 23724547 - }, - "fork_child_closes_then_parent_frees": { - "peak_bytes": 5563403, - "leaked_bytes": 3390349, - "total_allocations": 23724544 - }, - "fork_child_sys_exit": { - "peak_bytes": 3877646, - "leaked_bytes": 3381666, - "total_allocations": 1282172 - }, - "fork_stream_cleanup": { - "peak_bytes": 3500857, - "leaked_bytes": 3329555, - "total_allocations": 105687 - }, - "fork_swap_cleanup": { - "peak_bytes": 3720613, - "leaked_bytes": 3389867, - "total_allocations": 718596 - }, - "fork_contended_mutex_swap": { - "peak_bytes": 7306199, - "leaked_bytes": 3492867, - "total_allocations": 35863039 - }, - "fork_contended_mutex_wrap": { - "peak_bytes": 7295518, - "leaked_bytes": 3491027, - "total_allocations": 35014720 - }, - "fork_consumed_signer": { - "peak_bytes": 3388873, - "leaked_bytes": 3327740, - "total_allocations": 196221 - }, - "swap_chain_churn": { - "peak_bytes": 3720603, - "leaked_bytes": 3389458, - "total_allocations": 669590 - } -} \ No newline at end of file diff --git a/tests/perf/entrypoint.sh b/tests/perf/entrypoint.sh deleted file mode 100644 index e0cbf737..00000000 --- a/tests/perf/entrypoint.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -set -e - -cd /workspace -export PYTHONPATH=/workspace/src - -# Download the Linux native library into the volume-mounted workspace. -# Runs at container start so libs land in the host-mounted tree, -# not in a build layer that gets shadowed by the -v mount. -C2PA_VERSION=$(cat c2pa-native-version.txt) -ARCH=$(uname -m) - -if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - PLATFORM="aarch64-unknown-linux-gnu" -else - PLATFORM="x86_64-unknown-linux-gnu" -fi - -# Skip the GitHub API round-trip when the lib is already on disk -# Set C2PA_FORCE_DOWNLOAD=1 to override. -if [ -z "$C2PA_FORCE_DOWNLOAD" ] && [ -f "artifacts/$PLATFORM/libc2pa_c.so" ]; then - echo "Using cached c2pa native lib: artifacts/$PLATFORM (set C2PA_FORCE_DOWNLOAD=1 to re-download)" -else - echo "Downloading c2pa native lib: $C2PA_VERSION / $PLATFORM" - C2PA_LIBS_PLATFORM=$PLATFORM python scripts/download_artifacts.py "$C2PA_VERSION" -fi - -# Replicate what setup.py copy_platform_libraries() does: -# So the correct Linux library is here for the Dockerfile -python - < .bin -- Generates three flamegraph views: -peak.html (high-water), - -leaks.html (--leaks), -temporary.html (--temporary-allocations) -- Reads peak_bytes and leaked_bytes from the .bin via memray.FileReader -- Compares against baseline.json (creates it on first run) -- Exits non-zero only if leaked_bytes exceeds baseline * threshold. peak_bytes - is reported (and any over-threshold drift noted) but never fails the run: it - is a high-water mark that swings with allocation timing on alloc-heavy - scenarios, so it is informational, not a gate. - -Usage: - python -m tests.perf.run_profile [--update-baseline] - -Environment variables: -- MEMRAY_ITERATIONS: number of times each scenario loops (default: 100) -- MEMRAY_THRESHOLD: regression multiplier, e.g. 1.1 for 10% (default: 1.1) -""" - -import argparse -import json -import os -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path - -import platform - -import memray - -# Scenario name list -from tests.perf.scenarios import SCENARIO_NAMES - -HERE = Path(__file__).parent -REPORTS_DIR = HERE / "reports" -BASELINE_FILE = HERE / "baseline.json" - -ITERATIONS = int(os.environ.get("MEMRAY_ITERATIONS", "100")) -THRESHOLD = float(os.environ.get("MEMRAY_THRESHOLD", "1.1")) -PERF_ENV = os.environ.get("PERF_ENV", "") - - -def _run_scenario_under_memray(name: str, bin_path: Path) -> None: - """Spawn a subprocess that runs one scenario under memray --native.""" - repo_root = HERE.parent.parent - script = f""" -import sys -sys.path.insert(0, "{repo_root}") -sys.path.insert(0, "{repo_root / 'src'}") -from tests.perf.scenarios import SCENARIOS -SCENARIOS["{name}"]({ITERATIONS}) -# Collect cycle garbage before tracking ends so leaked_bytes means -# "still allocated but unreachable" (true leaks + one-time statics). -import gc -gc.collect() -gc.collect() -""" - cmd = [ - sys.executable, "-m", "memray", "run", - "--native", - "--trace-python-allocators", - "--force", - "-o", str(bin_path), - "-c", script, - ] - # Pass the scenario name so the loop can label its progress - env = {**os.environ, "PERF_SCENARIO": name} - result = subprocess.run(cmd, text=True, env=env) - if result.returncode != 0: - print(f" memray run failed for {name} (exit {result.returncode})", file=sys.stderr) - sys.exit(1) - - -def _generate_flamegraph(bin_path: Path, out_path: Path, mode: str = "peak") -> bool: - """Render one flamegraph view of a capture file. - - mode: - - 'peak': high-water-mark view (the default flamegraph render). - - 'leaks': memory still live when tracking stopped (--leaks). - - 'temporary': allocations freed before more than one other allocation - occurs, i.e. short-lived churn (--temporary-allocations). - These are mutually exclusive views, so each is a separate render. - """ - cmd = [sys.executable, "-m", "memray", "flamegraph", str(bin_path), "-o", str(out_path), "--force"] - if mode == "leaks": - cmd.append("--leaks") - elif mode == "temporary": - # --temporary-allocations == --temporary-allocation-threshold=1 - cmd.append("--temporary-allocations") - # Stream memray's output instead of capturing it, so run does not look stuck - print(f" flamegraph ({mode})...", flush=True) - result = subprocess.run(cmd, text=True) - if result.returncode != 0: - # -9 is SIGKILL, almost always the OOM killer reaping the heavy - # temporary render on a large capture. Do not abort the whole run: - # the capture and metrics are recorded separately and still good. - reason = "killed (likely OOM)" if result.returncode == -9 else f"exit {result.returncode}" - print(f" flamegraph {mode} render failed for {out_path.name} ({reason})", file=sys.stderr) - return False - return True - - -# get_allocation_records() yields deallocation records too... -# They carry size 0, so they don't affect byte sums, but they -# inflate record count, so we filter them out when counting alloc calls. -_DEALLOCATORS = { - memray.AllocatorType.FREE, - memray.AllocatorType.MUNMAP, - memray.AllocatorType.PYMALLOC_FREE, -} - - -def _read_metrics(bin_path: Path) -> dict: - """Extract peak_bytes, leaked_bytes and total_allocations from a memray .bin file.""" - with memray.FileReader(str(bin_path)) as reader: - # peak_bytes: the high-water mark of live memory, i.e. the most memory - # in use at any single instant. - peak_bytes = reader.metadata.peak_memory - - # total_allocations: number of allocation calls. - # We exclude deallocator records to count just allocations. - total_allocations = sum( - 1 - for record in reader.get_allocation_records() - if record.allocator not in _DEALLOCATORS - ) - - # leaked_bytes: memory still reachable when tracking ended (never freed). - leaked_bytes = sum( - record.size - for record in reader.get_leaked_allocation_records(merge_threads=True) - ) - - return { - "peak_bytes": peak_bytes, - "leaked_bytes": leaked_bytes, - "total_allocations": total_allocations, - } - - -def _build_meta() -> dict: - """Provenance for the baseline: which toolchain produced these numbers. - Recorded so a committed baseline is reproducible under same conditions. - """ - native_version = "" - try: - native_version = (HERE.parent.parent / "c2pa-native-version.txt").read_text().strip() - except OSError: - pass - return { - "memray_version": getattr(memray, "__version__", ""), - "python_version": platform.python_version(), - "c2pa_native_version": native_version, - "iterations": ITERATIONS, - "perf_env": PERF_ENV, - "arch": platform.machine(), - } - - -def _fmt(n: int) -> str: - if n >= 1024 ** 2: - return f"{n / 1024**2:.1f} MiB" - if n >= 1024: - return f"{n / 1024:.1f} KiB" - return f"{n} B" - - -def _delta_pct(current: int, base: int) -> str: - """Signed percentage change vs baseline, or '-' when no baseline.""" - if not base: - return "-" - return f"{(current - base) / base * 100:+.1f}%" - - -def _write_github_summary(results: dict, baseline: dict) -> None: - """Append a values table to $GITHUB_STEP_SUMMARY when running in CI. - """ - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if not summary_path or not results: - return - - lines = [ - "## Memory benchmark (memray)", - "", - f"Iterations: {ITERATIONS} · threshold: +{(THRESHOLD - 1) * 100:.0f}%" - f"{f' · env: {PERF_ENV}' if PERF_ENV else ''}", - "", - "| scenario | peak | allocs | peak Δ% | memory used Δ% | status |", - "|----------|------|--------|---------|----------------|--------|", - ] - for name, m in results.items(): - b = baseline.get(name, {}) if baseline else {} - peak_base = b.get("peak_bytes", 0) - leaked_base = b.get("leaked_bytes", 0) - regressed = ( - (peak_base and m["peak_bytes"] > peak_base * THRESHOLD) - or (leaked_base and m["leaked_bytes"] > leaked_base * THRESHOLD) - ) - status = "REGRESSED" if regressed else "ok" - lines.append( - f"| {name} | {_fmt(m['peak_bytes'])} " - f"| {m['total_allocations']} | {_delta_pct(m['peak_bytes'], peak_base)} " - f"| {_delta_pct(m['leaked_bytes'], leaked_base)} | {status} |" - ) - lines.append("") - - with open(summary_path, "a", encoding="utf-8") as fh: - fh.write("\n".join(lines) + "\n") - - -def main() -> None: - parser = argparse.ArgumentParser(description="c2pa-python memory profiler") - parser.add_argument( - "--update-baseline", - action="store_true", - help="Overwrite baseline.json with current measurements and exit 0", - ) - parser.add_argument( - "--scenario", - choices=SCENARIO_NAMES, - default=None, - help="Run a single scenario instead of all of them. With --update-baseline, " - "only that scenario's entry in baseline.json is updated; the rest are kept.", - ) - args = parser.parse_args() - - scenarios_to_run = (args.scenario,) if args.scenario else SCENARIO_NAMES - - REPORTS_DIR.mkdir(parents=True, exist_ok=True) - - # prior_baseline: the existing file, always loaded so a single-scenario - # update can preserve the other scenarios' entries when it rewrites the file. - prior_baseline: dict = {} - - # baseline: the subset used for the regression comparison below, which is - # suppressed when --update-baseline is set (because we are re-baselining). - if BASELINE_FILE.exists(): - prior_baseline = json.loads(BASELINE_FILE.read_text()) - baseline: dict = {} if args.update_baseline else prior_baseline - - results: dict = {} - failures: list[str] = [] - render_failures: list[dict] = [] - - total = len(scenarios_to_run) - for idx, name in enumerate(scenarios_to_run, 1): - print(f"\n=== [{idx}/{total}] {name} (iterations={ITERATIONS}) ===") - - with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as tmp: - bin_path = Path(tmp.name) - - env_tag = f"-{PERF_ENV}" if PERF_ENV else "" - scenario_render_failed = False - failed_modes: list[dict] = [] - try: - print(f" profiling...") - _run_scenario_under_memray(name, bin_path) - - peak_html = REPORTS_DIR / f"{name}{env_tag}-peak.html" - leaks_html = REPORTS_DIR / f"{name}{env_tag}-leaks.html" - temporary_html = REPORTS_DIR / f"{name}{env_tag}-temporary.html" - print(f" generating flamegraphs (peak + leaks + temporary)...") - scenario_render_failed = False - failed_modes: list[dict] = [] - for mode, html in (("peak", peak_html), - ("leaks", leaks_html), - ("temporary", temporary_html)): - if not _generate_flamegraph(bin_path, html, mode=mode): - scenario_render_failed = True - failed_modes.append({"name": name, "mode": mode, "html": html.name}) - - print(f" reading metrics...", flush=True) - metrics = _read_metrics(bin_path) - results[name] = metrics - - print(f" peak: {_fmt(metrics['peak_bytes'])}") - print(f" leaked: {_fmt(metrics['leaked_bytes'])}") - print(f" allocs: {metrics['total_allocations']}") - print(f" peak report: {peak_html}") - print(f" leaks report: {leaks_html}") - print(f" temporary report: {temporary_html}") - - if baseline and name in baseline: - b = baseline[name] - # Only leaked_bytes gates the run. It is the leak signal and is - # stable run-to-run. peak_bytes is a high-water mark that swings - # with transient-allocation timing on alloc-heavy scenarios, - # so it is reported for visibility but doesn't fail the run. - for metric in ("peak_bytes", "leaked_bytes"): - current = metrics[metric] - base = b.get(metric, 0) - limit = base * THRESHOLD - if current <= limit: - continue - diff_pct = (current - base) / base * 100 if base else float("inf") - msg = ( - f"{name}.{metric}: {_fmt(current)} > baseline {_fmt(base)}" - f" (+{diff_pct:.1f}%, threshold {(THRESHOLD-1)*100:.0f}%)" - ) - if metric == "leaked_bytes": - failures.append(msg) - else: - print(f" note (informational): {msg}", flush=True) - finally: - if scenario_render_failed: - # Keep the capture so the failed view can be re-rendered - # offline (with a higher --temporary-allocation-threshold) - # instead of re-profiling the whole scenario. - kept = REPORTS_DIR / f"{name}{env_tag}.bin" - shutil.move(str(bin_path), str(kept)) - for fm in failed_modes: - fm["bin"] = str(kept) - render_failures.extend(failed_modes) - else: - bin_path.unlink(missing_ok=True) - - if args.update_baseline or not prior_baseline: - # When running a single scenario, merge its result into the existing - # baseline so the other scenarios' entries are preserved. A full run - # replaces the file wholesale. - if args.scenario and prior_baseline: - output = dict(prior_baseline) - else: - output = {} - new_meta = _build_meta() - # On a single-scenario merge the new entry must come from the same - # toolchain as the entries it is being merged next to, or the numbers - # are not comparable. Warn if _meta would change (e.g. wrong PERF_ENV, - # iteration count, or native version) instead of silently overwriting it. - if args.scenario and prior_baseline: - old_meta = prior_baseline.get("_meta", {}) - if old_meta and old_meta != new_meta: - diffs = sorted( - set(old_meta) | set(new_meta), - key=str, - ) - changed = [ - f"{k}: {old_meta.get(k)!r} -> {new_meta.get(k)!r}" - for k in diffs if old_meta.get(k) != new_meta.get(k) - ] - print( - "\nWARNING: this run's environment differs from the existing " - "baseline's _meta; the merged entry will NOT be comparable to " - "the other scenarios:\n " + "\n ".join(changed), - file=sys.stderr, - ) - output["_meta"] = new_meta - output.update(results) - BASELINE_FILE.write_text(json.dumps(output, indent=2)) - verb = "Updated" if prior_baseline else "Created" - print(f"\n{verb} baseline: {BASELINE_FILE}") - - # Emit the report table to the PR's Step Summary in CI. - _write_github_summary(results, baseline) - - if render_failures: - print("\nFLAMEGRAPH RENDERS FAILED (capture + metrics still recorded):", file=sys.stderr) - for r in render_failures: - print(f" {r['name']} [{r['mode']}] -> {r['html']} (capture kept: {r['bin']})", file=sys.stderr) - print(" Recover without re-profiling, e.g.:", file=sys.stderr) - print(" python3 -m memray flamegraph -o " - "--temporary-allocations --temporary-allocation-threshold=10 --force", file=sys.stderr) - - if failures: - print("\nLEAK REGRESSIONS DETECTED (leaked_bytes over baseline):", - file=sys.stderr) - for f in failures: - print(f" {f}", file=sys.stderr) - sys.exit(1) - - print("\nAll scenarios within baseline leaked_bytes thresholds " - "(peak_bytes is informational only).") - - -if __name__ == "__main__": - main() diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py deleted file mode 100644 index 518c8f97..00000000 --- a/tests/perf/scenarios.py +++ /dev/null @@ -1,1436 +0,0 @@ -# Copyright 2026 Adobe. All rights reserved. -# This file is licensed to you under the Apache License, -# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) -# or the MIT license (http://opensource.org/licenses/MIT), -# at your option. - -""" -Plain functions (no pytest dependencies) that exercise the profiling scenarios. -Each function is called N times by run_profile.py. -""" - -import ctypes -import gc -import io -import json -import os -import sys -import threading -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from types import SimpleNamespace -from c2pa import ( - Builder, - C2paError, - C2paSignerInfo, - Context, - Reader, - Signer, - Stream, -) -import c2pa.c2pa as c2pa_module - -FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" -READING_FIXTURES_DIR = FIXTURES_DIR / "files-for-reading-tests" -SIGNING_FIXTURES_DIR = FIXTURES_DIR / "files-for-signing-tests" - -SIGNED_JPEG = FIXTURES_DIR / "C.jpg" -CLOUD_JPEG = FIXTURES_DIR / "cloud.jpg" -SOURCE_JPEG = FIXTURES_DIR / "A.jpg" -SIGNING_PNG = SIGNING_FIXTURES_DIR / "sample1.png" -DASH_INIT_MP4 = FIXTURES_DIR / "dashinit.mp4" -DASH_FRAGMENT = FIXTURES_DIR / "dash1.m4s" - -_DST_COMPOSITE = "http://cv.iptc.org/newscodes/digitalsourcetype/compositeWithTrainedAlgorithmicMedia" - -_PARENT_ID = "xmp:iid:aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" -_PLACED_ID = "xmp:iid:bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" -_PARENT_ID2 = "xmp:iid:cccccccc-0003-0003-0003-cccccccccccc" -_PLACED_ID2 = "xmp:iid:dddddddd-0004-0004-0004-dddddddddddd" -_PARENT_ID3 = "xmp:iid:eeeeeeee-0005-0005-0005-eeeeeeeeeeee" -_PLACED_ID3 = "xmp:iid:ffffffff-0006-0006-0006-ffffffffffff" -_PLACED_ID4 = "xmp:iid:11111111-0007-0007-0007-111111111111" -_PLACED_ID5 = "xmp:iid:22222222-0008-0008-0008-222222222222" -_ARCH_PARENT_ID = "xmp:iid:33333333-0009-0009-0009-333333333333" -_ARCH_COMP_ID = "xmp:iid:44444444-0010-0010-0010-444444444444" -_ARCH_COMP_ID2 = "xmp:iid:55555555-0011-0011-0011-555555555555" - -MANIFEST_BASE = { - "claim_generator": "perf_test", - "claim_generator_info": [{"name": "perf_test", "version": "0.0.1"}], - "format": "image/jpeg", - "title": "Perf Test Image", - "ingredients": [], - "assertions": [ - { - "label": "c2pa.actions", - "data": { - "actions": [ - { - "action": "c2pa.created", - "digitalSourceType": "http://cv.iptc.org/newscodes/digitalsourcetype/digitalCreation", - } - ] - }, - } - ], -} - - -# Scenario name for progress output, set per-run by run_profile.py via the env. -_SCENARIO = os.environ.get("PERF_SCENARIO", "") - - -def _iterate(n: int): - """Yield range(n), printing a progress line to stderr ~every 10%. - - The memray run phase is otherwise silent for the whole scenario, which at - high iteration counts looks hung. The print is gated to ~10 lines total so - it stays readable at N=100 and N=100000 alike, and writes to stderr so it - never lands in the captured/parsed metrics output. - """ - step = max(1, n // 10) - label = f"{_SCENARIO}: " if _SCENARIO else "" - for i in range(n): - if i % step == 0: - print(f" {label}iter {i}/{n} ({i * 100 // n if n else 100}%)", - file=sys.stderr, flush=True) - yield i - print(f" {label}iter {n}/{n} (100%)", file=sys.stderr, flush=True) - - -def _make_signer() -> Signer: - certs = (FIXTURES_DIR / "es256_certs.pem").read_bytes() - key = (FIXTURES_DIR / "es256_private.key").read_bytes() - info = C2paSignerInfo( - alg=b"es256", - sign_cert=certs, - private_key=key, - ta_url=b"http://timestamp.digicert.com", - ) - return Signer.from_info(info) - - -def _sign_file(path: Path, mime: str, iterations: int) -> None: - signer = _make_signer() - source_bytes = path.read_bytes() - manifest = {**MANIFEST_BASE, "format": mime} - for _ in _iterate(iterations): - source = io.BytesIO(source_bytes) - output = io.BytesIO() - builder = Builder(manifest) - builder.sign(signer, mime, source, output) - - -def _read_file(path: Path, mime: str, iterations: int) -> None: - for _ in _iterate(iterations): - with open(path, "rb") as f: - reader = Reader(mime, f) - reader.json() - reader.close() - - -# Context-API helpers: the Context is built once before the loop and reused on -# every iteration, so its settings are parsed a single time. Most scenarios use -# these. The `_legacy` jpeg/png scenarios build the Reader/Builder without a -# Context, which re-reads thread-local settings on each construction; running a -# legacy scenario against its `_with_context` pair isolates the settings cost. - -def _sign_file_context(path: Path, mime: str, iterations: int) -> None: - signer = _make_signer() - context = Context(signer=signer) # signer is consumed into the context - source_bytes = path.read_bytes() - manifest = {**MANIFEST_BASE, "format": mime} - for _ in _iterate(iterations): - source = io.BytesIO(source_bytes) - output = io.BytesIO() - builder = Builder(manifest, context=context) - # str first arg selects the context signer (c2pa_builder_sign_context). - builder.sign(mime, source, output) - - -def _read_file_context(path: Path, mime: str, iterations: int) -> None: - context = Context() - for _ in _iterate(iterations): - with open(path, "rb") as f: - reader = Reader(mime, f, manifest_data=None, context=context) - reader.json() - reader.close() - - -# Parallel signing: one Context built once and shared across threads. Each -# thread uses its own BytesIO source/dest and its own Builder per sign; the -# Context (and its signer) is only read. This exercises Context thread-safety -# under concurrent signing. - -_PARALLEL_THREADS = 10 - - -def _sign_parallel(path: Path, mime: str, iterations: int, *, - per_thread_full: bool, launch: str) -> None: - """Sign from `_PARALLEL_THREADS` threads sharing one Context. - - per_thread_full=False: the iteration budget is split across threads (each - does iterations // _PARALLEL_THREADS), so total work matches the - single-threaded scenarios. - per_thread_full=True: each thread runs the full `iterations` loop, so total - work is _PARALLEL_THREADS x iterations (aggregate concurrent load). - launch="pool": ThreadPoolExecutor(max_workers=_PARALLEL_THREADS). - launch="barrier": threads released together by a Barrier so all signs run - simultaneously (peak Context contention). - """ - signer = _make_signer() - context = Context(signer=signer) # built once, shared, kept open - source_bytes = path.read_bytes() - manifest = {**MANIFEST_BASE, "format": mime} - - per_thread = ( - iterations if per_thread_full - else max(1, iterations // _PARALLEL_THREADS) - ) - - def work(barrier=None): - if barrier is not None: - barrier.wait() # release all threads at once - for _ in range(per_thread): - source = io.BytesIO(source_bytes) # per-thread, never shared - output = io.BytesIO() - builder = Builder(manifest, context=context) - # str first arg selects the context signer. - builder.sign(mime, source, output) - - if launch == "pool": - with ThreadPoolExecutor(max_workers=_PARALLEL_THREADS) as ex: - futures = [ex.submit(work) for _ in range(_PARALLEL_THREADS)] - for f in futures: - f.result() # surface exceptions from worker threads - else: # barrier - barrier = threading.Barrier(_PARALLEL_THREADS) - threads = [ - threading.Thread(target=work, args=(barrier,)) - for _ in range(_PARALLEL_THREADS) - ] - for t in threads: - t.start() - for t in threads: - t.join() - - -# Reader scenarios: read manifests from files with manifests - -def scenario_reader_jpeg_legacy(iterations: int = 100) -> None: - _read_file(SIGNED_JPEG, "image/jpeg", iterations) - - -def scenario_reader_mp4(iterations: int = 100) -> None: - _read_file_context(READING_FIXTURES_DIR / "video1.mp4", "video/mp4", iterations) - - -def scenario_reader_wav(iterations: int = 100) -> None: - _read_file_context(READING_FIXTURES_DIR / "sample1_signed.wav", "audio/wav", iterations) - - -# Builder.sign (without ingredients)) - -def scenario_builder_sign_jpeg_legacy(iterations: int = 100) -> None: - _sign_file(SOURCE_JPEG, "image/jpeg", iterations) - - -def scenario_builder_sign_gif(iterations: int = 100) -> None: - _sign_file_context(SIGNING_FIXTURES_DIR / "sample1.gif", "image/gif", iterations) - - -def scenario_builder_sign_heic(iterations: int = 100) -> None: - _sign_file_context(SIGNING_FIXTURES_DIR / "sample1.heic", "image/heic", iterations) - - -def scenario_builder_sign_m4a(iterations: int = 100) -> None: - _sign_file_context(SIGNING_FIXTURES_DIR / "sample1.m4a", "audio/mp4", iterations) - - -def scenario_builder_sign_png_legacy(iterations: int = 100) -> None: - _sign_file(SIGNING_FIXTURES_DIR / "sample1.png", "image/png", iterations) - - -def scenario_builder_sign_webp(iterations: int = 100) -> None: - _sign_file_context(SIGNING_FIXTURES_DIR / "sample1.webp", "image/webp", iterations) - - -def scenario_builder_sign_avi(iterations: int = 100) -> None: - _sign_file_context(SIGNING_FIXTURES_DIR / "test.avi", "video/x-msvideo", iterations) - - -def scenario_builder_sign_mp4(iterations: int = 100) -> None: - _sign_file_context(SIGNING_FIXTURES_DIR / "video1.mp4", "video/mp4", iterations) - - -def scenario_builder_sign_tiff(iterations: int = 100) -> None: - _sign_file_context(SIGNING_FIXTURES_DIR / "TUSCANY.TIF", "image/tiff", iterations) - - -# Builder.sign scenarios with ingredient linking - -def scenario_builder_sign_jpeg_parent_of(iterations: int = 100) -> None: - """One parentOf ingredient linked to c2pa.opened action.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - ingredient_bytes = SIGNED_JPEG.read_bytes() - manifest = { - **MANIFEST_BASE, - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [{ - "action": "c2pa.opened", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_PARENT_ID]}, - "digitalSourceType": _DST_COMPOSITE, - }]}, - }], - } - for _ in _iterate(iterations): - builder = Builder(manifest, context=context) - with io.BytesIO(ingredient_bytes) as ing: - builder.add_ingredient( - {"relationship": "parentOf", "instance_id": _PARENT_ID}, - "image/jpeg", ing, - ) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_sign_jpeg_component_of(iterations: int = 100) -> None: - """One componentOf ingredient linked to c2pa.placed action.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - ingredient_bytes = SIGNED_JPEG.read_bytes() - manifest = { - **MANIFEST_BASE, - "ingredients": [{"format": "image/jpeg", "relationship": "componentOf", "instance_id": _PLACED_ID}], - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [{ - "action": "c2pa.placed", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_PLACED_ID]}, - "digitalSourceType": _DST_COMPOSITE, - }]}, - }], - } - for _ in _iterate(iterations): - builder = Builder(manifest, context=context) - with io.BytesIO(ingredient_bytes) as ing: - builder.add_ingredient( - {"relationship": "componentOf", "instance_id": _PLACED_ID}, - "image/jpeg", ing, - ) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_sign_jpeg_parent_and_component(iterations: int = 100) -> None: - """parentOf + componentOf ingredients (both JPEG) linked to opened + placed actions.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - parent_bytes = SIGNED_JPEG.read_bytes() - placed_bytes = CLOUD_JPEG.read_bytes() - manifest = { - **MANIFEST_BASE, - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [ - { - "action": "c2pa.opened", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_PARENT_ID2]}, - "digitalSourceType": _DST_COMPOSITE, - }, - { - "action": "c2pa.placed", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_PLACED_ID2]}, - "digitalSourceType": _DST_COMPOSITE, - }, - ]}, - }], - } - for _ in _iterate(iterations): - builder = Builder(manifest, context=context) - with io.BytesIO(parent_bytes) as ing1, io.BytesIO(placed_bytes) as ing2: - builder.add_ingredient( - {"relationship": "parentOf", "instance_id": _PARENT_ID2}, "image/jpeg", ing1, - ) - builder.add_ingredient( - {"relationship": "componentOf", "instance_id": _PLACED_ID2}, "image/jpeg", ing2, - ) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_sign_jpeg_parent_and_component_mixed_mime(iterations: int = 100) -> None: - """parentOf JPEG + componentOf PNG linked to opened + placed actions.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - parent_bytes = SIGNED_JPEG.read_bytes() - placed_bytes = SIGNING_PNG.read_bytes() - manifest = { - **MANIFEST_BASE, - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [ - { - "action": "c2pa.opened", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_PARENT_ID3]}, - "digitalSourceType": _DST_COMPOSITE, - }, - { - "action": "c2pa.placed", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_PLACED_ID3]}, - "digitalSourceType": _DST_COMPOSITE, - }, - ]}, - }], - } - for _ in _iterate(iterations): - builder = Builder(manifest, context=context) - with io.BytesIO(parent_bytes) as ing1, io.BytesIO(placed_bytes) as ing2: - builder.add_ingredient( - {"relationship": "parentOf", "instance_id": _PARENT_ID3}, "image/jpeg", ing1, - ) - builder.add_ingredient( - {"relationship": "componentOf", "instance_id": _PLACED_ID3}, "image/png", ing2, - ) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_sign_jpeg_two_components_same_mime(iterations: int = 100) -> None: - """Two componentOf JPEG ingredients in a single c2pa.placed action.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - comp1_bytes = SIGNED_JPEG.read_bytes() - comp2_bytes = CLOUD_JPEG.read_bytes() - manifest = { - **MANIFEST_BASE, - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [{ - "action": "c2pa.placed", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_PLACED_ID4, _PLACED_ID5]}, - "digitalSourceType": _DST_COMPOSITE, - }]}, - }], - } - for _ in _iterate(iterations): - builder = Builder(manifest, context=context) - with io.BytesIO(comp1_bytes) as ing1, io.BytesIO(comp2_bytes) as ing2: - builder.add_ingredient( - {"relationship": "componentOf", "instance_id": _PLACED_ID4}, "image/jpeg", ing1, - ) - builder.add_ingredient( - {"relationship": "componentOf", "instance_id": _PLACED_ID5}, "image/jpeg", ing2, - ) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_sign_jpeg_two_components_mixed_mime(iterations: int = 100) -> None: - """componentOf JPEG + componentOf PNG in a single c2pa.placed action.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - comp1_bytes = SIGNED_JPEG.read_bytes() - comp2_bytes = SIGNING_PNG.read_bytes() - manifest = { - **MANIFEST_BASE, - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [{ - "action": "c2pa.placed", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_PLACED_ID4, _PLACED_ID5]}, - "digitalSourceType": _DST_COMPOSITE, - }]}, - }], - } - for _ in _iterate(iterations): - builder = Builder(manifest, context=context) - with io.BytesIO(comp1_bytes) as ing1, io.BytesIO(comp2_bytes) as ing2: - builder.add_ingredient( - {"relationship": "componentOf", "instance_id": _PLACED_ID4}, "image/jpeg", ing1, - ) - builder.add_ingredient( - {"relationship": "componentOf", "instance_id": _PLACED_ID5}, "image/png", ing2, - ) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_sign_jpeg_archive_roundtrip(iterations: int = 100) -> None: - """Serialize builder to archive, reload, add ingredient, sign.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - ingredient_bytes = SIGNED_JPEG.read_bytes() - for _ in _iterate(iterations): - archive = io.BytesIO() - Builder(MANIFEST_BASE).to_archive(archive) - archive.seek(0) - # from_archive() yields a context-less Builder. To keep the Context - # (and its signer), build with the context first, then load the archive. - builder = Builder(MANIFEST_BASE, context=context).with_archive(archive) - with io.BytesIO(ingredient_bytes) as ing: - builder.add_ingredient( - {"relationship": "parentOf", "instance_id": _PARENT_ID}, - "image/jpeg", ing, - ) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_with_archive_swap(iterations: int = 100) -> None: - """Loop Builder.with_archive(), the consume-and-return FFI path. - - c2pa_builder_with_archive consumes the old native handle and returns a - replacement, so the Python side swaps the pointer without freeing the - consumed one. Freeing it would be a double-free, and failing to adopt the - replacement would leak. The other builder scenarios never swap a live - handle, so neither mistake would show up there. - """ - context = Context(signer=_make_signer()) - archive = io.BytesIO() - Builder(MANIFEST_BASE).to_archive(archive) - archive_bytes = archive.getvalue() - for _ in _iterate(iterations): - builder = Builder(MANIFEST_BASE, context=context) - builder.with_archive(io.BytesIO(archive_bytes)) - builder.close() - - -def scenario_reader_with_fragment_swap(iterations: int = 100) -> None: - """Loop Reader.with_fragment(), the other consume-and-return FFI path. - - Same ownership hand-off as with_archive: c2pa_reader_with_fragment eats - the old reader handle and returns a new one. - """ - init_bytes = DASH_INIT_MP4.read_bytes() - fragment_bytes = DASH_FRAGMENT.read_bytes() - for _ in _iterate(iterations): - reader = Reader("video/mp4", io.BytesIO(init_bytes)) - try: - reader.with_fragment( - "video/mp4", - io.BytesIO(init_bytes), - io.BytesIO(fragment_bytes), - ) - except C2paError: - # A failed call consumed the old handle just as a successful one - # would, so the scenario measures both outcomes. - pass - finally: - reader.close() - - -def scenario_reader_with_fragment_repeated(iterations: int = 100) -> None: - """Loop Reader.with_fragment() against a SINGLE long-lived Reader. - - The Reader is built outside the loop on purpose. Every other fragment - scenario constructs one per iteration and closes it, which releases the - streams each time round and so cannot show anything retained across calls. - Only repeated calls on one instance expose a fragment stream that is kept - instead of released, and each one held open pins a native C2paStream, its - four ctypes callbacks and the caller's buffer. - """ - init_bytes = DASH_INIT_MP4.read_bytes() - fragment_bytes = DASH_FRAGMENT.read_bytes() - reader = Reader("video/mp4", io.BytesIO(init_bytes)) - try: - for _ in _iterate(iterations): - reader.with_fragment( - "video/mp4", - io.BytesIO(init_bytes), - io.BytesIO(fragment_bytes), - ) - finally: - reader.close() - - -def scenario_builder_from_archive_roundtrip(iterations: int = 100) -> None: - """Loop Builder.from_archive() itself (context-less alternate constructor), - then sign. Regression guard for the classmethod's native-handle wrapping. - """ - signer = _make_signer() - source_bytes = SOURCE_JPEG.read_bytes() - ingredient_bytes = SIGNED_JPEG.read_bytes() - archive_bytes = io.BytesIO() - Builder(MANIFEST_BASE).to_archive(archive_bytes) - archive_bytes = archive_bytes.getvalue() - for _ in _iterate(iterations): - # from_archive() yields a context-less Builder, so sign() needs an - # explicit signer (no Context to pull one from). - builder = Builder.from_archive(io.BytesIO(archive_bytes)) - with io.BytesIO(ingredient_bytes) as ing: - builder.add_ingredient( - {"relationship": "parentOf", "instance_id": _PARENT_ID}, - "image/jpeg", ing, - ) - builder.sign(signer, "image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -# Consume-and-return failure paths. -# -# These calls take ownership partway through their body, so a null return is -# ambiguous and getting it wrong leaks one handle per call. The success paths -# are covered above; these loop the failure paths, where the leak would be. - -def _untracked_reader_handle(): - """A pointer the native registry does not know about. - - A never-allocated buffer, so it is rejected like a stale handle without - allocating a real Reader per call, which would swamp the measurement. - Freed handles are unusable here: recycled addresses become tracked again. - """ - buf = ctypes.create_string_buffer(64) - return ctypes.cast(buf, ctypes.POINTER(c2pa_module.C2paReader)), buf - - -def scenario_reader_with_fragment_pre_consume_rejection( - iterations: int = 100) -> None: - """Loop the rejection that precedes the ownership transfer. - - The handle is still ours, so treating this as consumed drops a pointer - the registry still holds and leaked_bytes climbs with iterations. - """ - init_bytes = DASH_INIT_MP4.read_bytes() - fragment_bytes = DASH_FRAGMENT.read_bytes() - for _ in _iterate(iterations): - reader = Reader("video/mp4", io.BytesIO(init_bytes)) - real_handle = reader._handle - # Keep the buffer alive: the cast pointer does not own it, and an - # early collection would hand the FFI a dangling address. - bogus, _buf = _untracked_reader_handle() - reader._handle = bogus - try: - reader.with_fragment("video/mp4", io.BytesIO(init_bytes), - io.BytesIO(fragment_bytes)) - raise AssertionError("pre-consume rejection did not raise") - except C2paError as e: - # Fail loudly: without these the scenario still runs when the - # ownership logic regresses, and a rejection that stops being - # recognised looks identical to a pass. - if not any(tag in str(e) for tag in - c2pa_module.ManagedResource._PRE_CONSUME_ERROR_TAGS): - raise AssertionError( - f"expected a pre-consume rejection, got: {e}") from e - if reader._handle is None: - raise AssertionError( - "handle was dropped on a pre-consume rejection; the " - "native side never took ownership, so this leaks") from e - finally: - # Restore before close() so the real handle is freed exactly once. - reader._handle = real_handle - reader.close() - - -def scenario_builder_with_archive_post_consume_failure( - iterations: int = 100) -> None: - """Loop a failure after the ownership transfer. - - The control: if the fix over-corrected into retaining handles the native - side already dropped, this scenario double-frees or leaks. - """ - for _ in _iterate(iterations): - builder = Builder(MANIFEST_BASE) - try: - builder.with_archive(io.BytesIO(b"not a valid archive")) - raise AssertionError("post-consume failure did not raise") - except C2paError: - pass - finally: - builder.close() - - -def scenario_with_fragment_marshalling_error(iterations: int = 100) -> None: - """Loop a failure that never reaches native code. - - Nothing was consumed, so the reader must stay usable. The old blanket - except marked it consumed here, leaking on what is only a type error. - """ - init_bytes = DASH_INIT_MP4.read_bytes() - for _ in _iterate(iterations): - reader = Reader("video/mp4", io.BytesIO(init_bytes)) - try: - reader.with_fragment("video/mp4", object(), object()) - raise AssertionError("marshalling error did not raise") - except (C2paError, TypeError, ctypes.ArgumentError): - pass - finally: - # Must still be usable: nothing was handed over. - reader.json() - reader.close() - - -def scenario_with_fragment_mixed_outcomes(iterations: int = 100) -> None: - """Interleave success, pre-consume rejection and post-consume failure. - - Each path leaves a different state behind, so running them in sequence - catches a stale error being read as the current call's. - """ - init_bytes = DASH_INIT_MP4.read_bytes() - fragment_bytes = DASH_FRAGMENT.read_bytes() - for i in _iterate(iterations): - phase = i % 3 - reader = Reader("video/mp4", io.BytesIO(init_bytes)) - try: - if phase == 0: - reader.with_fragment("video/mp4", io.BytesIO(init_bytes), - io.BytesIO(fragment_bytes)) - elif phase == 1: - real_handle = reader._handle - bogus, _buf = _untracked_reader_handle() - reader._handle = bogus - try: - reader.with_fragment("video/mp4", io.BytesIO(init_bytes), - io.BytesIO(fragment_bytes)) - raise AssertionError( - "pre-consume rejection did not raise") - except C2paError as e: - # A stale error from the phase before must not be read as - # this call's; the rejection would stop being recognised. - if not any( - tag in str(e) for tag in c2pa_module - .ManagedResource._PRE_CONSUME_ERROR_TAGS): - raise AssertionError( - f"expected a pre-consume rejection, got: {e}" - ) from e - if reader._handle is None: - raise AssertionError( - "handle dropped on a pre-consume rejection" - ) from e - finally: - reader._handle = real_handle - else: - try: - reader.with_fragment("video/mp4", io.BytesIO(b"garbage"), - io.BytesIO(b"garbage")) - except C2paError: - pass - finally: - reader.close() - - -# Archive scenarios: builder as working store (to_archive/with_archive) and -# per-ingredient archives (write_ingredient_archive/add_ingredient_from_archive). - -def _ingredient_archive_bytes(ingredient_json: dict, mime: str, asset_bytes: bytes) -> bytes: - """Build a per-ingredient archive once, for reuse inside scenario loops.""" - builder = Builder(MANIFEST_BASE) - with io.BytesIO(asset_bytes) as ing: - builder.add_ingredient(ingredient_json, mime, ing) - archive = io.BytesIO() - builder.write_ingredient_archive(ingredient_json["instance_id"], archive) - return archive.getvalue() - - -def scenario_builder_to_archive_with_ingredient(iterations: int = 100) -> None: - """Serialize a builder holding one ingredient to an archive (no signing).""" - ingredient_bytes = SIGNED_JPEG.read_bytes() - for _ in _iterate(iterations): - builder = Builder(MANIFEST_BASE) - with io.BytesIO(ingredient_bytes) as ing: - builder.add_ingredient( - {"relationship": "parentOf", "instance_id": _ARCH_PARENT_ID}, - "image/jpeg", ing, - ) - builder.to_archive(io.BytesIO()) - - -def scenario_builder_sign_jpeg_archive_roundtrip_ingredient_in_archive(iterations: int = 100) -> None: - """Add ingredient, serialize to archive, reload, sign. - - Unlike scenario_builder_sign_jpeg_archive_roundtrip, the ingredient is - added before to_archive, so its resources travel through the archive. - """ - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - ingredient_bytes = SIGNED_JPEG.read_bytes() - manifest = { - **MANIFEST_BASE, - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [{ - "action": "c2pa.opened", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_ARCH_PARENT_ID]}, - "digitalSourceType": _DST_COMPOSITE, - }]}, - }], - } - for _ in _iterate(iterations): - archive = io.BytesIO() - src_builder = Builder(manifest) - with io.BytesIO(ingredient_bytes) as ing: - src_builder.add_ingredient( - {"relationship": "parentOf", "instance_id": _ARCH_PARENT_ID}, - "image/jpeg", ing, - ) - src_builder.to_archive(archive) - archive.seek(0) - builder = Builder(manifest, context=context).with_archive(archive) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_write_ingredient_archive(iterations: int = 100) -> None: - """Add one ingredient and write it out as a per-ingredient archive.""" - ingredient_bytes = SIGNED_JPEG.read_bytes() - for _ in _iterate(iterations): - builder = Builder(MANIFEST_BASE) - with io.BytesIO(ingredient_bytes) as ing: - builder.add_ingredient( - {"relationship": "parentOf", "instance_id": _ARCH_PARENT_ID}, - "image/jpeg", ing, - ) - builder.write_ingredient_archive(_ARCH_PARENT_ID, io.BytesIO()) - - -def scenario_builder_sign_jpeg_add_ingredient_from_archive(iterations: int = 100) -> None: - """Restore one ingredient from a prebuilt archive and sign.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - archive_bytes = _ingredient_archive_bytes( - {"relationship": "parentOf", "instance_id": _ARCH_PARENT_ID}, - "image/jpeg", SIGNED_JPEG.read_bytes(), - ) - manifest = { - **MANIFEST_BASE, - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [{ - "action": "c2pa.opened", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_ARCH_PARENT_ID]}, - "digitalSourceType": _DST_COMPOSITE, - }]}, - }], - } - for _ in _iterate(iterations): - builder = Builder(manifest, context=context) - builder.add_ingredient_from_archive(io.BytesIO(archive_bytes)) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_ingredient_archive_roundtrip(iterations: int = 100) -> None: - """Write a per-ingredient archive from one builder, load into another, sign.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - ingredient_bytes = SIGNED_JPEG.read_bytes() - manifest = { - **MANIFEST_BASE, - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [{ - "action": "c2pa.opened", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_ARCH_PARENT_ID]}, - "digitalSourceType": _DST_COMPOSITE, - }]}, - }], - } - for _ in _iterate(iterations): - archive = io.BytesIO() - src_builder = Builder(MANIFEST_BASE) - with io.BytesIO(ingredient_bytes) as ing: - src_builder.add_ingredient( - {"relationship": "parentOf", "instance_id": _ARCH_PARENT_ID}, - "image/jpeg", ing, - ) - src_builder.write_ingredient_archive(_ARCH_PARENT_ID, archive) - archive.seek(0) - builder = Builder(manifest, context=context) - builder.add_ingredient_from_archive(archive) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - - -def scenario_builder_sign_jpeg_two_ingredient_archives(iterations: int = 100) -> None: - """Restore two ingredients (JPEG + PNG) from prebuilt archives and sign.""" - context = Context(signer=_make_signer()) - source_bytes = SOURCE_JPEG.read_bytes() - archive1_bytes = _ingredient_archive_bytes( - {"relationship": "componentOf", "instance_id": _ARCH_COMP_ID}, - "image/jpeg", SIGNED_JPEG.read_bytes(), - ) - archive2_bytes = _ingredient_archive_bytes( - {"relationship": "componentOf", "instance_id": _ARCH_COMP_ID2}, - "image/png", SIGNING_PNG.read_bytes(), - ) - manifest = { - **MANIFEST_BASE, - "assertions": [{ - "label": "c2pa.actions.v2", - "data": {"actions": [{ - "action": "c2pa.placed", - "softwareAgent": {"name": "perf_test"}, - "parameters": {"ingredientIds": [_ARCH_COMP_ID, _ARCH_COMP_ID2]}, - "digitalSourceType": _DST_COMPOSITE, - }]}, - }], - } - for _ in _iterate(iterations): - builder = Builder(manifest, context=context) - builder.add_ingredient_from_archive(io.BytesIO(archive1_bytes)) - builder.add_ingredient_from_archive(io.BytesIO(archive2_bytes)) - builder.sign("image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) -def scenario_reader_error_no_manifest(iterations: int = 100) -> None: - """Reader on an unsigned asset: partial-init cleanup.""" - source_bytes = SOURCE_JPEG.read_bytes() # A.jpg carries no manifest - for _ in _iterate(iterations): - try: - Reader("image/jpeg", io.BytesIO(source_bytes)).json() - except C2paError: - pass - - -def scenario_builder_error_invalid_manifest(iterations: int = 100) -> None: - """Error case: Builder with malformed manifest JSON.""" - for _ in _iterate(iterations): - try: - Builder('{"not valid json') - except C2paError: - pass - - -def scenario_reader_string_apis(iterations: int = 100) -> None: - """Uncached string returns: detailed_json/crjson/remote_url/resource_to_stream.""" - source_bytes = SIGNED_JPEG.read_bytes() - context = Context() - # Resolve a real resource URI once, outside the measured loop. - probe = Reader("image/jpeg", io.BytesIO(source_bytes), - manifest_data=None, context=context) - manifests = json.loads(probe.json()) - active = manifests["manifests"][manifests["active_manifest"]] - thumb_uri = active["thumbnail"]["identifier"] - probe.close() - for _ in _iterate(iterations): - reader = Reader("image/jpeg", io.BytesIO(source_bytes), - manifest_data=None, context=context) - reader.detailed_json() - reader.crjson() - reader.get_remote_url() - reader.resource_to_stream(thumb_uri, io.BytesIO()) - reader.close() - - -def scenario_builder_from_context_construction(iterations: int = 100) -> None: - """Loop Builder(context=...) construction, the consume-and-swap path. - - c2pa_builder_from_context hands back a handle that - c2pa_builder_with_definition then consumes and replaces. The Builder - adopts the first handle before that call so _consume_and_swap can own the - swap, which means a mis-sequenced swap leaks the replacement or frees the - consumed pointer twice. The other context builder scenarios sign a full - asset per iteration, so a one-handle regression here would sit under their - noise. This one only constructs and closes. - - scenario_reader_manifest_data_context is the Reader-side equivalent. - """ - context = Context() - for _ in _iterate(iterations): - builder = Builder(MANIFEST_BASE, context=context) - builder.close() - - -def scenario_signer_construction(iterations: int = 100) -> None: - """Loop Signer.from_info()/__init__ construction and teardown. - - Every other scenario calls _make_signer() once outside its loop, so - repeated Signer construction/destruction has no coverage elsewhere. - Regression guard for Signer.__init__'s native-handle activation. - """ - for _ in _iterate(iterations): - signer = _make_signer() - signer.close() - - -# jpeg + png context variants, paired with the `_legacy` scenarios above for -# side-by-side comparison. - -def scenario_builder_sign_jpeg_with_context(iterations: int = 100) -> None: - _sign_file_context(SOURCE_JPEG, "image/jpeg", iterations) - - -def scenario_builder_sign_png_with_context(iterations: int = 100) -> None: - _sign_file_context(SIGNING_PNG, "image/png", iterations) - - -def scenario_reader_jpeg_with_context(iterations: int = 100) -> None: - _read_file_context(SIGNED_JPEG, "image/jpeg", iterations) - - -def scenario_reader_manifest_data_context(iterations: int = 100) -> None: - """Reader over a detached (sidecar) manifest with a Context. - - Exercises c2pa_reader_with_manifest_data_and_stream, the consume-and-swap - FFI path (reader_from_context handle is consumed and replaced each call). - The manifest is signed once outside the loop; each iteration re-reads the - same asset + detached manifest, so flat RSS confirms no per-iteration leak - in the consume-and-swap path. - """ - source_bytes = SOURCE_JPEG.read_bytes() - signer = _make_signer() - builder = Builder({**MANIFEST_BASE, "format": "image/jpeg"}) - builder.set_no_embed() - manifest_bytes = builder.sign( - signer, "image/jpeg", io.BytesIO(source_bytes), io.BytesIO()) - builder.close() - signer.close() - - context = Context() - for _ in _iterate(iterations): - reader = Reader("image/jpeg", io.BytesIO(source_bytes), - manifest_data=manifest_bytes, context=context) - reader.json() - reader.close() - - -# Parallel signing variants: one shared Context across 10 threads. -# {split, full} x {pool, barrier} x {jpeg, png}. - -def scenario_builder_sign_jpeg_parallel_split_pool(iterations: int = 100) -> None: - _sign_parallel(SOURCE_JPEG, "image/jpeg", iterations, per_thread_full=False, launch="pool") - - -def scenario_builder_sign_jpeg_parallel_split_barrier(iterations: int = 100) -> None: - _sign_parallel(SOURCE_JPEG, "image/jpeg", iterations, per_thread_full=False, launch="barrier") - - -def scenario_builder_sign_png_parallel_split_pool(iterations: int = 100) -> None: - _sign_parallel(SIGNING_PNG, "image/png", iterations, per_thread_full=False, launch="pool") - - -def scenario_builder_sign_png_parallel_split_barrier(iterations: int = 100) -> None: - _sign_parallel(SIGNING_PNG, "image/png", iterations, per_thread_full=False, launch="barrier") - - -def _fork_wait(child_fn) -> None: - """Fork; run child_fn() in child then _exit(0); parent waits up to 5 s.""" - import signal - - def _on_alarm(signum, frame): - raise TimeoutError("fork child deadlocked — 5 s alarm fired") - - pid = os.fork() - if pid == 0: - child_fn() - os._exit(0) - - old = signal.signal(signal.SIGALRM, _on_alarm) - try: - signal.alarm(5) - _, status = os.waitpid(pid, 0) - signal.alarm(0) - finally: - signal.signal(signal.SIGALRM, old) - assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0, ( - f"child exited abnormally: status={status}" - ) - - -def scenario_fork_reader_collect(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - Baseline: create Reader, fork, child gc.collect() + _exit, parent closes. - Guard fires in child (no deadlock); parent frees normally (no leak). - """ - if not hasattr(os, "fork"): - return - for _ in _iterate(iterations): - with open(SIGNED_JPEG, "rb") as f: - reader = Reader("image/jpeg", f) - _fork_wait(lambda: gc.collect()) - reader.close() - - -def scenario_fork_contended_mutex(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - 8 threads create/close Readers in a tight loop while the main thread - forks 5× per iteration (500 total forks). Maximises the probability that - the registry Mutex is held at the instant of fork(). Each fork inherits - a Reader created by the main thread; the child explicitly closes it - (then runs GC), so the PID guard is exercised on every fork — without - the guard the close would call into the native library and could - deadlock on a mutex left locked by a vanished worker thread. The parent - closes the same Reader after the child exits (its own PID: real free). - - Note: the workers' Readers are pinned by frozen thread frames in the - child, so child gc.collect() alone would free nothing — hence the - explicit close of an inherited object. - """ - if not hasattr(os, "fork"): - return - stop = threading.Event() - - def _worker(): - while not stop.is_set(): - with open(SIGNED_JPEG, "rb") as f: - r = Reader("image/jpeg", f) - r.close() - - threads = [threading.Thread(target=_worker, daemon=True) - for _ in range(8)] - for t in threads: - t.start() - try: - for _ in _iterate(iterations): - for _ in range(5): - with open(SIGNED_JPEG, "rb") as f: - reader = Reader("image/jpeg", f) - - def _child(r=reader): - r.close() - gc.collect() - - _fork_wait(_child) - reader.close() - finally: - stop.set() - for t in threads: - t.join(timeout=5) - - -def scenario_fork_thread_local_orphan(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - A thread stores Reader in threading.local, joins, then main forks. - """ - if not hasattr(os, "fork"): - return - for _ in _iterate(iterations): - tl = threading.local() - - def _create(): - with open(SIGNED_JPEG, "rb") as f: - tl.reader = Reader("image/jpeg", f) - - t = threading.Thread(target=_create) - t.start() - t.join() - _fork_wait(lambda: gc.collect()) - - -def scenario_fork_gc_cycle(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - Reader in a reference cycle, freed only by cyclic GC, not refcounting. - Child calls gc.collect(), which triggers __del__ on the Reader. - """ - if not hasattr(os, "fork"): - return - for _ in _iterate(iterations): - with open(SIGNED_JPEG, "rb") as f: - reader = Reader("image/jpeg", f) - container = SimpleNamespace(reader=reader) - reader.container = container # cycle: reader ↔ container - del reader, container # refcount > 0; cycle survives until GC - - _fork_wait(lambda: gc.collect()) - gc.collect() # parent cleans up - - -def scenario_fork_parent_frees_after_fork(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - 20 Readers created, fork, child exits immediately, parent closes all 20. - Primary false-positive test: if is_foreign_process() wrongly fires in the - parent, all 20 native frees are skipped and leaked_bytes spikes ~20x. - """ - if not hasattr(os, "fork"): - return - for _ in _iterate(iterations): - readers = [] - for _ in range(20): - with open(SIGNED_JPEG, "rb") as f: - readers.append(Reader("image/jpeg", f)) - _fork_wait(lambda: None) # child does nothing, exits 0 - for r in readers: - r.close() - - -def scenario_fork_child_closes_then_parent_frees(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - 20 Readers created, fork, the CHILD closes all 20 inherited Readers (and - runs GC so any __del__ fires) before exiting, then the parent closes its - own 20 copies. Exercises the child-side path where _cleanup_resources marks - the child's copy CLOSED and nulls the handle while skipping the native free - — the branch scenario_fork_parent_frees_after_fork never hits (its child - does nothing). Two invariants: the child must exit cleanly (no deadlock via - the 5 s alarm, no crash from a child-side double-free), and the parent must - still free all 20 (leaked_bytes stays at baseline — the child's state - mutation does not suppress the parent's frees, since the copies are - independent post-fork). - """ - if not hasattr(os, "fork"): - return - for _ in _iterate(iterations): - readers = [] - for _ in range(20): - with open(SIGNED_JPEG, "rb") as f: - readers.append(Reader("image/jpeg", f)) - - def _child(): - for r in readers: - r.close() # foreign teardown: mark closed, skip native free - gc.collect() - - _fork_wait(_child) - for r in readers: - r.close() # parent's own copies: real free - - -def scenario_fork_child_sys_exit(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - Child calls sys.exit(0), full Python shutdown: atexit, finalizers, GC. - Every native-handle wrapper's __del__ fires in the child. Guard must - survive Py_Finalize() without deadlocking. - """ - if not hasattr(os, "fork"): - return - for _ in _iterate(iterations): - with open(SIGNED_JPEG, "rb") as f: - reader = Reader("image/jpeg", f) - context = Context() - - def _child(): - import sys as _sys - _sys.exit(0) # full Python shutdown, not _exit - - _fork_wait(_child) - reader.close() - context.close() - - -def _fork_contended_over(make_object, iterations): - """Fork over an object built by make_object() while 8 threads churn - Readers, so the registry Mutex is likely held at the instant of fork(). - - The child closes the inherited object. Without the PID guard that close - calls into the native library and can block forever on a mutex left - locked by a thread that fork() did not clone, which _fork_wait's alarm - reports as a timeout. The parent closes afterwards for the real free. - """ - if not hasattr(os, "fork"): - return - stop = threading.Event() - - def _worker(): - while not stop.is_set(): - with open(SIGNED_JPEG, "rb") as f: - r = Reader("image/jpeg", f) - r.close() - - threads = [threading.Thread(target=_worker, daemon=True) - for _ in range(8)] - for t in threads: - t.start() - try: - for _ in _iterate(iterations): - for _ in range(5): - obj = make_object() - - def _child(o=obj): - o.close() - gc.collect() - - _fork_wait(_child) - obj.close() - finally: - stop.set() - for t in threads: - t.join(timeout=5) - - -def scenario_fork_contended_mutex_swap(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - fork over a Builder whose handle came from with_archive(), under the same - thread contention as fork_contended_mutex. That scenario only ever forks - over handles that came straight from a constructor, so a swapped-in - handle losing its stamp would go unnoticed there. - """ - if not hasattr(os, "fork"): - return - context = Context(signer=_make_signer()) - archive = io.BytesIO() - Builder(MANIFEST_BASE).to_archive(archive) - archive_bytes = archive.getvalue() - - def _make(): - builder = Builder(MANIFEST_BASE, context=context) - builder.with_archive(io.BytesIO(archive_bytes)) - return builder - - _fork_contended_over(_make, iterations) - - -def scenario_fork_contended_mutex_wrap(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - fork over a Builder built by from_archive(), under thread contention. - from_archive is the only path that bypasses __init__, so it is the one - most likely to be missing the PID stamp the child's close() depends on. - """ - if not hasattr(os, "fork"): - return - archive = io.BytesIO() - Builder(MANIFEST_BASE).to_archive(archive) - archive_bytes = archive.getvalue() - - _fork_contended_over( - lambda: Builder.from_archive(io.BytesIO(archive_bytes)), iterations) - - -def scenario_fork_consumed_signer(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - the parent builds a Context that consumed a Signer, then forks. The child - closes both. The consumed Signer holds no handle, so it must be inert in - either process, and the Context must be skipped by the PID guard. - """ - if not hasattr(os, "fork"): - return - for _ in _iterate(iterations): - signer = _make_signer() - context = Context(signer=signer) - - def _child(c=context, s=signer): - s.close() - c.close() - gc.collect() - - _fork_wait(_child) - signer.close() - context.close() - - -def scenario_swap_chain_churn(iterations: int = 100) -> None: - """Loop with_archive() repeatedly on one Builder, so a chain of handles - is consumed and replaced on a single live object. Every other scenario - swaps a given object at most once. - - This one is a crash and allocation-churn guard rather than a leak gate. - Only one Builder is closed however many times the loop runs, so a - close-path leak here is O(1) and invisible against the interpreter's - allocation floor. What a broken swap does instead is fail loudly: keeping - the consumed pointer makes the next call raise UntrackedPointer from the - native registry, and freeing it makes the free itself fail. total_allocations - still tracks the churn. - """ - context = Context(signer=_make_signer()) - archive = io.BytesIO() - Builder(MANIFEST_BASE).to_archive(archive) - archive_bytes = archive.getvalue() - builder = Builder(MANIFEST_BASE, context=context) - for _ in _iterate(iterations): - builder.with_archive(io.BytesIO(archive_bytes)) - builder.close() - context.close() - - -def scenario_fork_swap_cleanup(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - the handle a Builder owns at fork time came from with_archive(), which - consumed the original and returned a replacement. The child must skip the - free on the swapped-in handle just as it would on an original one, and the - parent must still free it exactly once afterwards. The other fork - scenarios only ever fork over handles that came straight from a - constructor. - """ - if not hasattr(os, "fork"): - return - context = Context(signer=_make_signer()) - archive = io.BytesIO() - Builder(MANIFEST_BASE).to_archive(archive) - archive_bytes = archive.getvalue() - for _ in _iterate(iterations): - builder = Builder(MANIFEST_BASE, context=context) - builder.with_archive(io.BytesIO(archive_bytes)) - - def _child(b=builder): - b.close() - gc.collect() - - _fork_wait(_child) - builder.close() - - -def scenario_fork_stream_cleanup(iterations: int = 100) -> None: - """Fork safety benchmark scenario: - Stream wraps a BytesIO with ctypes callbacks stored as instance attributes. - Both Stream.__del__ and Stream.close carry fork guards. This tests the - stream-specific path (separate from ManagedResource). - """ - if not hasattr(os, "fork"): - return - source_bytes = SIGNED_JPEG.read_bytes() - for _ in _iterate(iterations): - stream = Stream(io.BytesIO(source_bytes)) - _fork_wait(lambda: gc.collect()) - stream.close() - - -SCENARIOS = { - "reader_jpeg_legacy": scenario_reader_jpeg_legacy, - "reader_jpeg_with_context": scenario_reader_jpeg_with_context, - "reader_manifest_data_context": scenario_reader_manifest_data_context, - "reader_mp4": scenario_reader_mp4, - "reader_wav": scenario_reader_wav, - "builder_sign_jpeg_legacy": scenario_builder_sign_jpeg_legacy, - "builder_sign_jpeg_with_context": scenario_builder_sign_jpeg_with_context, - "builder_sign_png_legacy": scenario_builder_sign_png_legacy, - "builder_sign_png_with_context": scenario_builder_sign_png_with_context, - "builder_sign_jpeg_parallel_split_pool": scenario_builder_sign_jpeg_parallel_split_pool, - "builder_sign_jpeg_parallel_split_barrier": scenario_builder_sign_jpeg_parallel_split_barrier, - "builder_sign_png_parallel_split_pool": scenario_builder_sign_png_parallel_split_pool, - "builder_sign_png_parallel_split_barrier": scenario_builder_sign_png_parallel_split_barrier, - "builder_sign_gif": scenario_builder_sign_gif, - "builder_sign_heic": scenario_builder_sign_heic, - "builder_sign_m4a": scenario_builder_sign_m4a, - "builder_sign_webp": scenario_builder_sign_webp, - "builder_sign_avi": scenario_builder_sign_avi, - "builder_sign_mp4": scenario_builder_sign_mp4, - "builder_sign_tiff": scenario_builder_sign_tiff, - "builder_sign_jpeg_parent_of": scenario_builder_sign_jpeg_parent_of, - "builder_sign_jpeg_component_of": scenario_builder_sign_jpeg_component_of, - "builder_sign_jpeg_parent_and_component": scenario_builder_sign_jpeg_parent_and_component, - "builder_sign_jpeg_parent_and_component_mixed_mime": scenario_builder_sign_jpeg_parent_and_component_mixed_mime, - "builder_sign_jpeg_two_components_same_mime": scenario_builder_sign_jpeg_two_components_same_mime, - "builder_sign_jpeg_two_components_mixed_mime": scenario_builder_sign_jpeg_two_components_mixed_mime, - "builder_sign_jpeg_archive_roundtrip": scenario_builder_sign_jpeg_archive_roundtrip, - "builder_from_archive_roundtrip": scenario_builder_from_archive_roundtrip, - "builder_with_archive_swap": scenario_builder_with_archive_swap, - "reader_with_fragment_swap": scenario_reader_with_fragment_swap, - "reader_with_fragment_repeated": scenario_reader_with_fragment_repeated, - "with_fragment_pre_consume_rejection": - scenario_reader_with_fragment_pre_consume_rejection, - "with_archive_post_consume_failure": - scenario_builder_with_archive_post_consume_failure, - "with_fragment_marshalling_error": - scenario_with_fragment_marshalling_error, - "with_fragment_mixed_outcomes": scenario_with_fragment_mixed_outcomes, - "builder_to_archive_with_ingredient": scenario_builder_to_archive_with_ingredient, - "builder_sign_jpeg_archive_roundtrip_ingredient_in_archive": scenario_builder_sign_jpeg_archive_roundtrip_ingredient_in_archive, - "builder_write_ingredient_archive": scenario_builder_write_ingredient_archive, - "builder_sign_jpeg_add_ingredient_from_archive": scenario_builder_sign_jpeg_add_ingredient_from_archive, - "builder_ingredient_archive_roundtrip": scenario_builder_ingredient_archive_roundtrip, - "builder_sign_jpeg_two_ingredient_archives": scenario_builder_sign_jpeg_two_ingredient_archives, - "reader_error_no_manifest": scenario_reader_error_no_manifest, - "builder_error_invalid_manifest": scenario_builder_error_invalid_manifest, - "reader_string_apis": scenario_reader_string_apis, - "signer_construction": scenario_signer_construction, - "builder_from_context_construction": - scenario_builder_from_context_construction, - "fork_reader_collect": scenario_fork_reader_collect, - "fork_contended_mutex": scenario_fork_contended_mutex, - "fork_thread_local_orphan": scenario_fork_thread_local_orphan, - "fork_gc_cycle": scenario_fork_gc_cycle, - "fork_parent_frees_after_fork": scenario_fork_parent_frees_after_fork, - "fork_child_closes_then_parent_frees": - scenario_fork_child_closes_then_parent_frees, - "fork_child_sys_exit": scenario_fork_child_sys_exit, - "fork_stream_cleanup": scenario_fork_stream_cleanup, - "fork_swap_cleanup": scenario_fork_swap_cleanup, - "fork_contended_mutex_swap": scenario_fork_contended_mutex_swap, - "fork_contended_mutex_wrap": scenario_fork_contended_mutex_wrap, - "fork_consumed_signer": scenario_fork_consumed_signer, - "swap_chain_churn": scenario_swap_chain_churn, -} - - -# Canonical scenario name list, derived from SCENARIOS so the two cannot drift. -# (dict preserves insertion order, so this matches the dict's declaration order.) -SCENARIO_NAMES = tuple(SCENARIOS) diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index 3983a112..ffff10c5 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -4158,6 +4158,153 @@ def closer(): "racing closers freed {} times".format(len(freed))) self.assertEqual(reader._inflight, 0) + def _borrow_resource(self): + """An ACTIVE resource with no native handle behind it.""" + res = _ConcreteResource() + res._lifecycle_state = LifecycleState.ACTIVE + res._handle = ctypes.c_void_p(1) + return res + + def test_consume_during_foreign_borrow_raises(self): + """A consume must refuse to start while another thread borrows. + """ + res = self._borrow_resource() + borrowing = threading.Event() + release = threading.Event() + + def borrower(): + with res._native_call(): + borrowing.set() + release.wait(self.JOIN_TIMEOUT) + + thread = threading.Thread(target=borrower) + thread.start() + try: + self.assertTrue(borrowing.wait(self.JOIN_TIMEOUT), + "borrower never entered the native call") + with self.assertRaises(Error) as caught: + res._consume_no_replacement(lambda h: 0, "unused: {}") + self.assertIn("in use", str(caught.exception)) + self.assertEqual( + res._lifecycle_state, LifecycleState.ACTIVE, + "a refused consume must leave the resource usable") + self.assertIsNotNone(res._handle) + finally: + release.set() + self._join_all([thread], "borrower") + + def test_unborrowed_consume_proceeds(self): + """A consume with nothing in flight runs and closes the resource. + + The guard rejects on any in-flight count, so a consuming call must not + wrap itself in _native_call(): the callers pin the handle by marking + the resource CLOSED under the lock instead. + """ + res = self._borrow_resource() + res._consume_no_replacement(lambda h: 0, "unused: {}") + self.assertEqual(res._lifecycle_state, LifecycleState.CLOSED) + + def test_consume_inside_own_borrow_is_refused(self): + """A consume is refused even when this thread owns the borrow. + + The guard counts frames, not threads. A consuming call nested in a + _native_call() would hand a pointer to native while that same frame + still expects it back, so no such nesting is allowed. + """ + res = self._borrow_resource() + with res._native_call(): + with self.assertRaises(Error): + res._consume_no_replacement(lambda h: 0, "unused: {}") + self.assertEqual(res._lifecycle_state, LifecycleState.ACTIVE) + + def test_refused_consume_leaves_borrow_counts_intact(self): + """A refused consume must not disturb the in-flight bookkeeping.""" + res = self._borrow_resource() + borrowing = threading.Event() + release = threading.Event() + + def borrower(): + with res._native_call(): + borrowing.set() + release.wait(self.JOIN_TIMEOUT) + + thread = threading.Thread(target=borrower) + thread.start() + try: + self.assertTrue(borrowing.wait(self.JOIN_TIMEOUT)) + with self.assertRaises(Error): + res._consume_no_replacement(lambda h: 0, "unused: {}") + self.assertEqual(res._inflight, 1, "the real borrow was lost") + finally: + release.set() + self._join_all([thread], "borrower") + self.assertEqual(res._inflight, 0) + + def test_failed_consume_restores_active_state(self): + """A call that did not take the handle must leave it usable. + """ + res = self._borrow_resource() + with patch('c2pa.c2pa._read_native_error', + return_value="Other: UntrackedPointer: 0x1"): + with self.assertRaises(Exception): + res._consume_no_replacement(lambda h: -1, "rejected: {}") + self.assertEqual(res._lifecycle_state, LifecycleState.ACTIVE, + "a retained handle was left marked closed") + self.assertIsNotNone(res._handle) + + def test_consume_raising_restores_active_state(self): + """An exception from the native call must not leave a stale mark.""" + res = self._borrow_resource() + + def boom(handle): + raise ctypes.ArgumentError("marshalling failed") + + with self.assertRaises(ctypes.ArgumentError): + res._consume_no_replacement(boom, "unused: {}") + self.assertEqual(res._lifecycle_state, LifecycleState.ACTIVE) + + def test_deferred_consume_is_not_upgraded_to_free(self): + """A deferred consuming teardown must not be overwritten by a later + free intent arriving while the same call is still in flight. + + Scenario: a Signer shared across concurrent signs: sign borrows + the handle (holding the in-flight guard) while Context.__init__ + consumes it. + """ + freed = self._counted_free() + reader = Reader("image/jpeg", io.BytesIO(self.image_bytes)) + releases = [] + orig_release = reader._release + + def counting_release(): + releases.append(1) + orig_release() + + reader._release = counting_release + + with reader._native_call(): + # The consuming call: native took ownership, so nothing here frees. + reader._teardown(free_handle=False) + self.assertFalse( + reader._pending_teardown, + "consuming teardown did not record free_handle=False") + + # A free intent arriving behind it, past a stale state check. + reader._teardown(free_handle=True) + self.assertFalse( + reader._pending_teardown, + "recorded consume was upgraded back to a free") + + self.assertEqual( + freed, [], + "freed a handle the native library already owns") + self.assertEqual( + len(releases), 1, + "_release() ran {} times, expected once".format(len(releases))) + self.assertEqual(reader._inflight, 0) + self.assertIsNone(reader._pending_teardown) + self.assertEqual(reader._lifecycle_state, LifecycleState.CLOSED) + def test_concurrent_close_runs_release_once(self): """Two racing close() calls on one instance must run _release() exactly once. @@ -4427,6 +4574,220 @@ def visit(node, active): + "\n ".join(unguarded)) + def test_consume_during_concurrent_sign_does_not_crash(self): + """Consuming a shared Signer must not free it under a live sign. + + Runs in a subprocess: the failure mode is a segfault, which would take + the test runner down with it otherwise. + """ + source = textwrap.dedent(""" + import io, os, sys, threading, time + from c2pa import (Builder, Context, Signer, C2paSignerInfo, + C2paSigningAlg as SigningAlg) + + data_dir = sys.argv[1] + certs_path = os.path.join(data_dir, "es256_certs.pem") + key_path = os.path.join(data_dir, "es256_private.key") + certs = open(certs_path, "rb").read() + key = open(key_path, "rb").read() + img = open(os.path.join(data_dir, "C.jpg"), "rb").read() + manifest = {"claim_generator_info": + [{"name": "test", "version": "0.1"}], + "assertions": []} + + signer = Signer.from_info(C2paSignerInfo( + SigningAlg.ES256, certs, key, None)) + stop = threading.Event() + + def sign(): + while not stop.is_set(): + try: + builder = Builder(manifest) + builder.sign(signer, "image/jpeg", + io.BytesIO(img), io.BytesIO()) + builder.close() + except Exception: + # A consumed signer may legitimately be rejected; + # only a crash is a failure here. + pass + + threads = [threading.Thread(target=sign) for _ in range(6)] + for t in threads: + t.start() + time.sleep(0.4) + try: + Context(signer=signer) + except Exception: + # Refusing the consume while borrows are live is the fix. + pass + stop.set() + for t in threads: + t.join() + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", source, self.data_dir], + capture_output=True, text=True, timeout=300) + + self.assertNotEqual( + result.returncode, -11, + "SIGSEGV: a signer was consumed while a sign was using its handle") + self.assertEqual( + result.returncode, 0, + "shared-signer consume race failed (rc={}):\n{}".format( + result.returncode, result.stderr[-2000:])) + self.assertIn("OK", result.stdout) + + def _callback_signer_source(self): + """Shared subprocess preamble: an ES256 callback signer.""" + return """ + import io, os, sys, threading, time + from c2pa import (Builder, Context, Signer, + C2paSigningAlg as SigningAlg) + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + + data_dir = sys.argv[1] + certs = open(os.path.join(data_dir, + "es256_certs.pem"), "rb").read().decode() + key_path = os.path.join(data_dir, "es256_private.key") + key = open(key_path, "rb").read() + img = open(os.path.join(data_dir, "C.jpg"), "rb").read() + manifest = {"claim_generator_info": + [{"name": "test", "version": "0.1"}], + "assertions": []} + private_key = serialization.load_pem_private_key( + key, password=None) + + def sign_callback(data): + return private_key.sign(data, ec.ECDSA(hashes.SHA256())) + + def make_context(): + return Context(signer=Signer.from_callback( + sign_callback, SigningAlg.ES256, certs, + "http://timestamp.digicert.com")) +""" + + def test_context_close_during_context_sign_does_not_crash(self): + """Closing a Context must not free the signer callback mid-sign. + + Context.__init__ pins the consumed signer's ctypes callback so it + outlives the Signer object, and Context._release() drops that pin. + Without an in-flight guard on the Context, a close() on another thread + runs _release() while c2pa_builder_sign_context is calling through the + trampoline, and the process dies with SIGSEGV. + + Runs in a subprocess: the failure mode is a segfault, which would take + the test runner down with it otherwise. + """ + source = textwrap.dedent(self._callback_signer_source() + """ + for trial in range(60): + ctx = make_context() + entered = threading.Event() + + def worker(): + try: + builder = Builder(dict(manifest), context=ctx) + entered.set() + builder.sign("image/jpeg", io.BytesIO(img), + io.BytesIO()) + builder.close() + except Exception: + # A closed context may legitimately be rejected; + # only a crash is a failure here. + entered.set() + + t = threading.Thread(target=worker) + t.start() + entered.wait(5) + time.sleep(0.002) + ctx.close() + t.join(20) + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", source, self.data_dir], + capture_output=True, text=True, timeout=300) + + self.assertNotEqual( + result.returncode, -11, + "SIGSEGV: the signer callback was freed while native was " + "calling it") + self.assertEqual( + result.returncode, 0, + "context-close-during-sign race failed (rc={}):\n{}".format( + result.returncode, result.stderr[-2000:])) + self.assertIn("OK", result.stdout) + + def test_context_close_during_sign_defers_teardown(self): + """A close() arriving mid-sign defers instead of releasing. + + The callback pin and the native handle both have to survive until the + call in flight finishes, so a sign already running is never cut short. + """ + context = Context() + with context._native_call(): + context.close() + self.assertEqual(context._lifecycle_state, LifecycleState.CLOSED, + "close() must mark the context closed at once") + self.assertIsNotNone( + context._pending_teardown, + "the teardown should be recorded, not performed") + self.assertFalse( + context._released, + "_release() ran while a native call was still in flight") + self.assertTrue(context._handle, + "the handle was freed mid-call") + + self.assertTrue(context._released, + "the deferred teardown never ran") + self.assertIsNone(context._pending_teardown) + + def test_context_sign_after_close_raises_rather_than_skipping_signer(self): + """Signing through a closed Context must raise, not silently succeed. + + Context._release() has already dropped the pinned callback, so the + native side signs without ever invoking it: the call returns a + manifest of the same size while the caller's signing callback runs + zero times. Refusing the call is what makes that visible. + + Runs in a subprocess because the callback signer needs the + cryptography package, which this module does not otherwise import. + """ + source = textwrap.dedent(self._callback_signer_source() + """ + calls = [] + + def counting_callback(data): + calls.append(1) + return private_key.sign(data, ec.ECDSA(hashes.SHA256())) + + signer = Signer.from_callback( + counting_callback, SigningAlg.ES256, certs, + "http://timestamp.digicert.com") + ctx = Context(signer=signer) + builder = Builder(dict(manifest), context=ctx) + ctx.close() + + try: + builder.sign("image/jpeg", io.BytesIO(img), io.BytesIO()) + print("SIGNED_WITH_CALLS", len(calls)) + except Exception as exc: + print("RAISED", type(exc).__name__, len(calls)) + """) + + result = subprocess.run( + [sys.executable, "-c", source, self.data_dir], + capture_output=True, text=True, timeout=300) + + self.assertEqual(result.returncode, 0, result.stderr[-2000:]) + self.assertIn( + "RAISED", result.stdout, + "signing through a closed context returned a manifest its " + "signer callback never produced: {}".format(result.stdout.strip())) + self.assertIn("0", result.stdout.split()[-1]) + def test_close_during_concurrent_sign_does_not_crash(self): """A Signer shared across threads must not be freed mid-sign. From f8b9deca22ffe41b9a4c56e2b0fae5cdd9618307 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:17:53 -0700 Subject: [PATCH 24/28] fix: Update docs --- .../README.md | 24 --- .../faulthandler-output.txt | 16 -- .../repro.py | 60 ------ docs/native-resources-management.md | 196 ++++++++++++++++-- 4 files changed, 175 insertions(+), 121 deletions(-) delete mode 100644 crashes/context-close-drops-signer-callback-mid-sign/README.md delete mode 100644 crashes/context-close-drops-signer-callback-mid-sign/faulthandler-output.txt delete mode 100644 crashes/context-close-drops-signer-callback-mid-sign/repro.py diff --git a/crashes/context-close-drops-signer-callback-mid-sign/README.md b/crashes/context-close-drops-signer-callback-mid-sign/README.md deleted file mode 100644 index 1d480a24..00000000 --- a/crashes/context-close-drops-signer-callback-mid-sign/README.md +++ /dev/null @@ -1,24 +0,0 @@ -The process dies with SIGSEGV (exit code 139, no Python exception) when a -`Context` built from a callback signer is closed on one thread while another -thread runs a context-sign (`Builder(manifest, context=ctx)` followed by -`builder.sign(format, source, dest)`) through it. - -40-120 trials: - -| Variant | Result | -|---|---| -| Close during concurrent context-sign, callback signer | SIGSEGV, reproducible | -| Same race, `Context._release` patched to keep the callback reference alive | 80/80 clean | -| Same race, context's native free suppressed (release still runs) | still SIGSEGV | -| Same race, info signer (`Signer.from_info`, no Python callback) | 120/120 clean | -| Single-threaded close-then-sign | clean (errors, no crash) | -| Dropping the last `ctx` reference mid-sign (finalizer close) | 80/80 clean | - -``` -python3 crashes/context-close-drops-signer-callback-mid-sign/repro.py -``` - -Exit code 139 within a few trials. The script: build a `Context` from -`Signer.from_callback(...)`, start a thread running a context-sign, sleep -~2 ms after the sign begins, call `ctx.close()` from the main thread, join, -repeat. diff --git a/crashes/context-close-drops-signer-callback-mid-sign/faulthandler-output.txt b/crashes/context-close-drops-signer-callback-mid-sign/faulthandler-output.txt deleted file mode 100644 index fa1c6173..00000000 --- a/crashes/context-close-drops-signer-callback-mid-sign/faulthandler-output.txt +++ /dev/null @@ -1,16 +0,0 @@ -Fatal Python error: Segmentation fault - -Current thread 0x000000016e3ab000 (most recent call first): - File "/Users/taniamathern/Desktop/code/c2pa-python/src/c2pa/c2pa.py", line 4032 in _sign_internal - File "/Users/taniamathern/Desktop/code/c2pa-python/src/c2pa/c2pa.py", line 4110 in _sign_common - File "/Users/taniamathern/Desktop/code/c2pa-python/src/c2pa/c2pa.py", line 4186 in sign - File "/private/tmp/claude-501/-Users-taniamathern-Desktop-code-c2pa-python/a1e731b8-8f71-4e3d-a858-5b5d2dfe2dfb/scratchpad/crash/min.py", line 24 in w - File "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/threading.py", line 994 in run - File "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/threading.py", line 1043 in _bootstrap_inner - File "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/threading.py", line 1014 in _bootstrap - -Thread 0x00000001efdc1d80 (most recent call first): - File "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/threading.py", line 1094 in join - File "/private/tmp/claude-501/-Users-taniamathern-Desktop-code-c2pa-python/a1e731b8-8f71-4e3d-a858-5b5d2dfe2dfb/scratchpad/crash/min.py", line 31 in - -Extension modules: _cffi_backend (total: 1) diff --git a/crashes/context-close-drops-signer-callback-mid-sign/repro.py b/crashes/context-close-drops-signer-callback-mid-sign/repro.py deleted file mode 100644 index 61f6f195..00000000 --- a/crashes/context-close-drops-signer-callback-mid-sign/repro.py +++ /dev/null @@ -1,60 +0,0 @@ -"""SIGSEGV reproduction: Context.close() racing a context-sign that uses a -callback signer. Run from the repository root: - - python3 crashes/context-close-drops-signer-callback-mid-sign/repro.py - -Expected: the process dies with SIGSEGV (exit 139) within a few trials. -The crash needs the `cryptography` package for the ES256 callback. -""" -import sys, io, os, threading, time, faulthandler - -sys.path.insert(0, "src") -faulthandler.enable() - -from c2pa import Builder, Signer, Context, C2paSigningAlg as Alg -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import ec - -FIXTURES = "tests/fixtures" -certs = open(os.path.join(FIXTURES, "es256_certs.pem"), "rb").read().decode() -key_bytes = open(os.path.join(FIXTURES, "es256_private.key"), "rb").read() -image = open(os.path.join(FIXTURES, "C.jpg"), "rb").read() -MANIFEST = {"claim_generator_info": [{"name": "repro", "version": "0.1"}], - "assertions": []} - -private_key = serialization.load_pem_private_key(key_bytes, password=None) - - -def sign_callback(data: bytes) -> bytes: - return private_key.sign(data, ec.ECDSA(hashes.SHA256())) - - -def make_context() -> Context: - signer = Signer.from_callback(sign_callback, Alg.ES256, certs, - "http://timestamp.digicert.com") - return Context(signer=signer) # consumes the signer - - -for trial in range(80): - ctx = make_context() - entered = threading.Event() - - def worker(): - try: - builder = Builder(dict(MANIFEST), context=ctx) - entered.set() - builder.sign("image/jpeg", io.BytesIO(image), io.BytesIO()) - builder.close() - except Exception: - entered.set() - - t = threading.Thread(target=worker) - t.start() - entered.wait(5) - time.sleep(0.002) # let the sign enter the native call - ctx.close() # drops _signer_callback_cb mid-invocation - t.join(20) - if trial % 20 == 0: - print("trial", trial, "still alive") - -print("survived 80 trials (crash did not reproduce this run)") diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 71fba3d2..8e83df28 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -106,7 +106,9 @@ Therefore, the managed resources have the following principles: ### Double-free risk mitigations -Three distinct risks, each with its own mechanism in this layer: +Freeing the same native pointer twice corrupts the allocator's bookkeeping. Allocators that detect it stop the process (see [How a native memory bug reports itself](#how-a-native-memory-bug-reports-itself)). Where they do not, the damage surfaces later, somewhere unrelated to the code responsible. Separate situations lead here: a single flow misreading who owns a pointer, a forked child freeing its parent's memory, and two threads racing. + +Each risk and its mechanism: | Hazard | Covered by | How | | --- | --- | --- | @@ -116,6 +118,109 @@ Three distinct risks, each with its own mechanism in this layer: The PID stamp is fork-only: it compares process IDs, and two threads in the same process always match on that PID. Sharing one `ManagedResource` instance across threads still needs locks: nothing here protects two threads racing on genuinely distinct objects that happen to share an allocator. +## Streams + +Bytes reach the native library through a `Stream`. + +`Stream` wraps a Python stream-like object (file stream or memory stream) so the native library can read from and write to it via callbacks. It does not inherit from `ManagedResource`, and it uses `c2pa_release_stream()` instead of `c2pa_free()` for cleanup. + +### Why is `Stream` not a `ManagedResource`? + +A `Reader` or `Builder` holds a native resource that Python code calls methods on. A `Stream` holds a native handle that the native library calls back into (read, seek, write, flush). The native library needs a different release function to tear down the callback machinery, since ownership has a different meaning here (due to the callbacks). + +`Stream` tracks its own state with `_closed` and `_initialized` flags rather than `LifecycleState`, but it supports the same three cleanup paths: context manager, explicit `.close()`, and `__del__` fallback. + +### Callbacks re-enter Python + +A `Stream` registers four ctypes callbacks (`_read_cb`, `_seek_cb`, `_write_cb`, `_flush_cb`). When the native library reads from a stream, it calls one of them, which runs caller-supplied Python. + +A native call driving a stream callback cannot hold `_op_lock`. The callback may re-enter this API on the same thread and deadlock against the lock already taken (see [Locking and in-flight tracking](#locking-and-in-flight-tracking)). + +The callback objects must also outlive the native side's use of them, covered in [Preventing garbage collection of live references](#preventing-garbage-collection-of-live-references). + +Each callback checks `_initialized` and `_closed` before touching the underlying Python stream, and returns `-1` if the stream is not initialized or already closed. A callback arriving after teardown reports an error instead of reading through dropped state. + +### `Stream` cleanup + +`Stream` holds `_close_lock`, a plain `Lock` rather than an `RLock`. It serializes the three cleanup paths: `close()`, `__del__`, and a `close()` on another thread. Without it, two of them reach the same stream and call `c2pa_release_stream` twice on one native handle. `Stream` needs its own lock since it does not inherit the `_op_lock` machinery. + +Cleanup runs in the direction of dependency: whatever can still invoke or reach the other is torn down first. Because callbacks run the opposite way for a `Stream`, its close order is the reverse of `ManagedResource`'s: + +| | Order | Why | +| --- | --- | --- | +| `ManagedResource` | `_release()` first (dropping streams and callbacks), then `c2pa_free` | The native pointer depends on the Python-side resources, so those are torn down while the pointer is still valid. | +| `Stream` | `c2pa_release_stream` first, then drop the callbacks | The native stream invokes the callbacks. Releasing it first guarantees none can fire, and the callback objects are dropped after that. | + +`Stream` does not own the Python object it wraps and never closes it. The caller that opened a file owns that file. A `Reader` that opened a file itself tracks it as `_backing_file` and closes it during its own `_release()`. + +Both `close()` and `__del__` take the foreign-process branch, marking the stream closed without calling into the native library. [Fork safety](#fork-safety) covers why. + +### Reference cycles in the callbacks + +Each ctypes callback closes over the `Stream` it belongs to. Captured directly, that forms a cycle: the `Stream` holds the callback, the callback's closure holds the `Stream`. Nothing in that loop reaches a refcount of zero, so cleanup falls to the cycle collector. [Why `__del__` is not reliable enough](#why-__del__-is-not-reliable-enough) covers why that timing cannot be relied on. + +The closures capture a `weakref` to the `Stream` instead, resolving it on each call. The reference count can then reach zero, keeping cleanup on the deterministic path. + +## Threads, the GIL, and locks + +### How a native memory bug reports itself + +A Python bug raises an exception with a traceback. A native memory bug terminates the process, and the operating system reports it as a signal. + +SIGSEGV (segmentation violation) comes from the hardware. Every process has a map of which address ranges it may touch and how. The CPU traps when asked to read or write an unmapped address, or to write one mapped read-only. The kernel then sends SIGSEGV. A segmentation fault, or segfault, is that event. SIGSEGV is the signal delivered for it. It fires whenever a dereferenced pointer does not point at accessible memory. A null pointer does it. So does a pointer into a stack frame that has already returned, or into memory that was freed and unmapped. Freeing memory does not always unmap it. Allocators keep the pages and reuse them. Reading through a freed pointer therefore often succeeds, returning whatever occupies that address now. The read returns another object's bytes, corrupting state that surfaces far from the code responsible. SIGSEGV is another outcome when the pages happen to be gone. + +SIGABRT (abort) has a different origin. No hardware raises it. The program raises it against itself by calling `abort()`, on detecting that its own invariants are broken. Allocators do this. A `free()` handed a pointer it never issued, or the same pointer twice, or a heap whose bookkeeping a stray write has damaged, stops the process rather than continuing on a corrupt heap. + +Which signal arrives depends on the allocator. glibc can print a diagnostic and raises SIGABRT; the macOS allocator traps instead, giving SIGTRAP. A use-after-free that reaches unmapped pages gives SIGSEGV on both. + +Both terminate the process from inside native code. No exception is raised, no `finally` block runs, no traceback is printed. + +### A close arriving mid-call + +A race here is two threads, one object, one native pointer, and operations that take several steps. Failure follow roughly these steps: + +1. Thread A calls `reader.json()` (for instance), which validates the handle and enters the native call. +2. While that call is running, thread B calls `reader.close()` (for instance, close on same object), which frees the native pointer. +3. Thread A's native code, still running, reads through the pointer it was given. + +Step 3 reads freed memory. The process takes SIGSEGV if those pages are gone. If the allocator kept them and reissued that address, the read succeeds and returns another object's bytes. Both happen inside native code. + +### What the GIL can guarantee + +CPython compiles source into bytecode: a sequence of small instructions the interpreter executes one at a time. A single line of Python becomes several of them. `self._ensure_valid_state()` compiles to four (load `self`, load the attribute, call, discard the result). + +CPython has a Global Interpreter Lock, a single interpreter-wide lock. Only one thread executes bytecode at a time. One instruction cannot interleave with another, so built-in containers do not corrupt structurally under concurrent access. + +Anything longer than one instruction gets no such guarantee. The interpreter can switch threads between any two instructions, so a line of Python can be interrupted partway. + +### Why the GIL is not enough here + +ctypes releases the GIL for the duration of a foreign function call. Other Python threads then run in parallel with the native code. Every native call in this layer opens that window, and a `close()` can arrive inside it. [A close arriving mid-call](#a-close-arriving-mid-call) walks through that race. + +The check-then-use sequences here span many instructions. `_ensure_valid_state()` validates the handle; a later instruction loads `self._handle` and passes it to native. The interpreter can switch threads in between, so a passing check does not promise the handle is still live at the point of use. `_op_lock` makes those pairs one unit. + +`_native_call()` closes the window by making the free wait for the call to finish: + +```mermaid +flowchart TD + subgraph guarded [Guarded by _native_call] + direction TB + C1["Thread A: enters guard
_inflight becomes 1"] --> C2["Thread A: native call runs"] + D1["Thread B: close()"] --> D2{"_inflight nonzero?"} + D2 -->|yes| D3["Record _pending_teardown,
mark CLOSED, free nothing"] + C2 --> C3["Thread A: leaves guard
_inflight becomes 0"] + D3 -.->|"free deferred to
whoever leaves last"| C3 + C3 --> C4["Deferred free runs
pointer no longer in use"] + end +``` + +### Why races are observed + +A race on a native pointer produces a use-after-free or a double-free, which is a crash or silent memory corruption, and the corruption can surface arbitrarily far from its cause. [Double-free risk mitigations](#double-free-risk-mitigations) covers the three specific hazards and the mechanism for each. + +> [!NOTE] +> Free-threaded CPython builds remove the GIL entirely, so code that was accidentally relying on it for atomicity loses that protection anyway. + ## Locking and in-flight tracking Each `ManagedResource` holds a reentrant lock, `_op_lock`, and a counter, `_inflight`, that together serialize teardown against concurrent use from other threads. @@ -124,9 +229,31 @@ A lock (Python's `threading.Lock`) can be acquired once, and a second `acquire() `_op_lock` is an `RLock` rather than a plain `Lock` for two reasons specific to this code. First, a finalizer (`__del__`) can run at any bytecode boundary — including one in the middle of a method that has already acquired the lock on this same thread — so `__del__` calling back into locked code must not deadlock against itself. Second, a consuming call tears the handle down from inside the locked region it is already holding: `_teardown()` is called while `_op_lock` is held, and it needs to acquire the same lock again rather than re-entering as a different, blocked acquisition. `_lock()` returns it, except in a forked child: there it raises `C2paError` immediately rather than blocking, because the thread that might hold the lock at fork time does not exist in the child to release it, and waiting on it would hang forever (see [Fork safety](#fork-safety)). -The lock is never held across a native call that drives a stream callback: construction, `resource_to_stream`, the Builder stream methods, and signing all release the Global Interpreter Lock (GIL) and call back into caller-supplied Python, which may itself call into this API on another thread. Holding `_op_lock` there would deadlock against that reentry. Those calls go through `_native_call()` instead: a context manager that increments `_inflight` under the lock, yields to run the native call unlocked, then decrements `_inflight` on the way out. If `_teardown()` runs while a call is in flight, it records the requested `free_handle` value in `_pending_teardown` and marks the resource `CLOSED` immediately, so no other caller can start using it, but defers the actual free. The last `_native_call()` to exit picks up `_pending_teardown` and runs `_teardown()` for real. +The lock is never held across a native call that drives a stream callback: construction, `resource_to_stream`, the Builder stream methods, and signing all release the GIL and re-enter caller-supplied Python, which may itself call into this API on another thread (see [Callbacks re-enter Python](#callbacks-re-enter-python)). Holding `_op_lock` there would deadlock against that reentry. Those calls go through `_native_call()` instead: a context manager that increments `_inflight` under the lock, yields to run the native call unlocked, then decrements `_inflight` on the way out. If `_teardown()` runs while a call is in flight, it records the requested `free_handle` value in `_pending_teardown` and marks the resource `CLOSED` immediately, so no other caller can start using it, but defers the actual free. The last `_native_call()` to exit picks up `_pending_teardown` and runs `_teardown()` for real. + +A recorded `_pending_teardown` only ever moves from `True` to `False`, never back. When one caller records a teardown that frees and another records one that does not, the one that does not wins. A pointer the native side already took is never freed on the way out. -`Context.__init__` does not wrap the signer hand-off in `signer._native_call()`: the consuming call marks the signer `CLOSED` under its own lock before calling native, which is what stops a `signer.close()` on another thread from freeing the handle mid-transfer (see [Borrowing versus consuming](#borrowing-versus-consuming)). `Builder._sign_internal` wraps the sign call in `self._native_call()` and, when an explicit `Signer` is passed, nests `signer._native_call()` inside it in that fixed order, so two concurrent `sign()` calls sharing one `Signer` cannot deadlock by acquiring the two locks in opposite orders. The Builder's `close()` after signing runs outside its own `_native_call()` block, so a teardown deferred during the call still executes once the call returns. When the Builder was created from a `Context` and signs through its context signer, `self._context._native_call()` is nested in that same position instead, for the reason described in [Context lifetime during a context-sign](#context-lifetime-during-a-context-sign). +Cleanup idempotency keys on a `_released` flag rather than on the `CLOSED` state. The deferred path marks the resource `CLOSED` when it records the teardown, while still owing the release that runs later. + +`Context.__init__` does not wrap the signer hand-off in `signer._native_call()`: the consuming call marks the signer `CLOSED` under its own lock before calling native, which is what stops a `signer.close()` on another thread from freeing the handle mid-transfer (see [Borrowing versus consuming](#borrowing-versus-consuming)). The Builder's `close()` after signing runs outside its own `_native_call()` block, so a teardown deferred during the call still executes once the call returns. + +### Lock ordering + +Two threads acquiring the same pair of locks in opposite orders deadlock: each holds what the other waits for. A single order of lock acquisition avoids the issue. + +An operation holds guards by role, from outermost to innermost: + +1. A method-specific lock, where a method serializes itself against other calls to itself. +2. The guard of the object the method is called on. +3. The guard of any object it borrows for the duration of the call. + +Rules that follow from the order: + +- A borrowed object's guard is always taken inside the operating object's guard, never the reverse. Two concurrent operations sharing a borrowed object then acquire in one direction. +- A lock held across a native call must be one no callback path acquires. +- `Stream._close_lock` is a leaf: nothing is acquired while holding it, so it cannot join a cycle. + +No code path holds two resources' `_op_lock` at once. `_native_call()` counts a call as in flight instead of holding the lock across it, which is what keeps that true. `test_no_nested_op_locks` asserts it. ### Borrowing versus consuming @@ -140,7 +267,17 @@ The check catches a borrow already in flight. The `CLOSED` mark catches one arri The mark is provisional. `_abort_consume()` restores the previous state when the native call turns out not to have taken the handle, which keeps the retained branch of the [ownership-taken triage](#why-an-ownership-taken-failure-does-not-free) handing back a usable object. -`_consume_and_swap()` is excluded. `_swap_handle()` requires the resource to stay `ACTIVE` and the object remains usable with its replacement pointer, so there is no `CLOSED` mark to make and no check. Its callers (`Reader.with_fragment`, `Builder.with_archive`) pass streams whose callbacks re-enter this API, so they hold their own `_native_call()`, and they act on resources the caller is required to serialize. +`_consume_and_swap()` is excluded. `_swap_handle()` requires the resource to stay `ACTIVE` and the object remains usable with its replacement pointer, so there is no `CLOSED` mark to make and no check. Its callers (`Reader.with_fragment`, `Builder.with_archive`) pass streams whose callbacks re-enter this API, so they hold their own `_native_call()`. `Reader.with_fragment()` additionally serializes itself with a lock of its own, described in [`Reader.with_fragment()`](#readerwith_fragment). + +### Context lifetime during a context-sign + +A `Context` built with a signer keeps that signer alive after consuming it. `Context.__init__` copies the signer's ctypes callback into `_signer_callback_cb`, because the `Signer` object is closed by the consuming call while the native side still needs the trampoline to invoke. `Context._release()` drops that reference. + +`c2pa_builder_sign_context` calls back into that trampoline for the duration of the sign, so the Context has to stay alive across it just as a borrowed `Signer` does. `Builder._sign_internal` therefore wraps the call in `self._context._native_call()`. A `close()` arriving on another thread then takes the deferred branch: it records the teardown, marks the Context `CLOSED`, and leaves `_release()` for whichever caller leaves the guard last. The callback stays pinned and the handle stays valid until the sign finishes. + +A sign already in flight is never cut short. Closing a Context mid-sign is safe; the close takes effect once the call returns. Without the guard, `_release()` frees the ctypes trampoline while native is calling through it, and the process takes SIGSEGV rather than raising. + +A sign cannot start once the Context is closed, and raises `C2paError` instead. A released Context has already dropped the callback, so the native side signed without ever invoking it. An error is the only way that is visible to the caller. ## Guarantees provided by ManagedResource @@ -160,10 +297,12 @@ The mark is provisional. `_abort_consume()` restores the previous state when the When a Python object passes a callback or pointer to the native library, that reference must stay alive for as long as the native side might use it. Python's garbage collector has no way to know that native code is still holding a reference to a Python callback. -The SDK solves this by storing these references as instance attributes on the owning object. For example, `Stream` stores its four callback objects (`_read_cb`, `_seek_cb`, `_write_cb`, `_flush_cb`) as instance attributes. As long as the `Stream` object is alive, its callbacks have a nonzero reference count and will not be collected. Similarly, when a `Signer` is consumed by a `Context`, the Context copies the signer's `_callback_cb` to its own `_signer_callback_cb` attribute so the callback survives even though the Signer object is now closed. +The SDK solves this by storing these references as instance attributes on the owning object. For example, `Stream` stores its four callback objects (`_read_cb`, `_seek_cb`, `_write_cb`, `_flush_cb`) as instance attributes. As long as the `Stream` object is alive, its callbacks have a nonzero reference count and will not be collected (see [Streams](#streams) for how those callbacks avoid forming a reference cycle with the `Stream` itself). Similarly, when a `Signer` is consumed by a `Context`, the Context copies the signer's `_callback_cb` to its own `_signer_callback_cb` attribute so the callback survives even though the Signer object is now closed. During cleanup, `_release()` sets these attributes to `None`, which drops the reference count on the callback objects and allows them to be collected. In the cleanup sequence, `_release()` runs first, then `c2pa_free` frees the native pointer. `_release()` goes first so that subclass-specific resources (open file handles, stream wrappers) are torn down before the native pointer they depend on is freed. +This ordering applies to `ManagedResource`. `Stream` releases in the opposite order, and [`Stream` cleanup](#stream-cleanup) explains why each side is correct for the direction its callbacks run. + ## How native memory is freed The native Rust library exposes a single C FFI function, `c2pa_free`, that deallocates memory it previously allocated. `ManagedResource` wraps this in a static method: @@ -302,7 +441,7 @@ with open("photo.jpg", "rb") as file: manifest = reader.json() ``` -The order matters because resources often depend on each other. In both examples, the `Reader` holds a native pointer that references the file's data through a `Stream` wrapper. If the file handle were closed first, the native library would still hold a pointer into the stream's read callbacks, and any subsequent access (including cleanup) could read freed memory or trigger a segfault. By closing the Reader first, the native pointer is freed while the underlying file is still open and valid. Python's `with` statement guarantees this ordering: resources listed later (or nested deeper) are torn down first. +The order matters because resources often depend on each other. In both examples, the `Reader` holds a native pointer that references the file's data through a [`Stream`](#streams) wrapper, and the native library reads that file by calling back into the stream's callbacks. If the file handle were closed first, those callbacks would still be reachable from native code but would be reading a closed file, and any subsequent access (including cleanup) could read freed memory or segfault. By closing the Reader first, the native pointer is freed while the underlying file is still open and valid. Python's `with` statement guarantees this ordering: resources listed later (or nested deeper) are torn down first. ## Reader lifecycle @@ -366,23 +505,23 @@ sequenceDiagram C->>X: Context(settings, signer) X->>B: with _NativeBuilder() (owns the builder, close() frees it on any failure) X->>S: _ensure_valid_state() - X->>S: enter _native_call() - Note right of S: Pins the Signer active for the duration:
a close() on another thread now waits
instead of freeing the handle mid-transfer X->>X: copy signer._callback_cb to _signer_callback_cb Note right of X: Pin the callback first:
the Signer is about to be consumed X->>S: _consume_no_replacement(set_signer) + S->>S: _begin_consume(): under _op_lock,
refuse if borrowed, then mark CLOSED + Note right of S: The CLOSED mark is the protection:
a close() on another thread finds it
already closed and frees nothing S->>N: c2pa_context_builder_set_signer(builder_ptr, handle) alt status 0 (success) S->>S: _teardown(free_handle=False) Note right of S: Consumed: native took the signer else pre-consume rejection (one of _PRE_CONSUME_ERROR_TAGS) - Note right of S: Rejected before ownership moved:
Signer retained, typed error raised + S->>S: _abort_consume(): restore the previous state + Note right of S: Rejected before ownership moved:
Signer retained and usable, typed error raised else other error S->>S: _teardown(free_handle=False) Note right of S: Native took it then failed and dropped it end - X->>S: exit _native_call() X->>B: _consume_into(build) B->>N: c2pa_context_builder_build(builder_ptr) @@ -393,7 +532,7 @@ sequenceDiagram Details in that sequence that are easy to get wrong: - The callback is copied to the Context *before* the transfer. A successful consume runs `_release()`, which drops the Signer's reference to the callback; a Context that copied it afterwards would be pointing at a callback nothing keeps alive. -- The state check and the consuming call both run inside `signer._native_call()`, so a `signer.close()` racing on another thread cannot free the handle in the gap between them. If a close does arrive while the transfer is in flight, it is recorded as a pending teardown and applied once the transfer finishes (see [Locking and in-flight tracking](#locking-and-in-flight-tracking)). +- The transfer is not wrapped in `signer._native_call()`. It is protected by the `CLOSED` mark that `_begin_consume()` makes under `_op_lock` before the native call starts. A racing `signer.close()` finds the resource already closed and frees nothing, and a later borrow is refused for the same reason. That check also refuses the consume outright when another thread is already borrowing the handle to sign with (see [Borrowing versus consuming](#borrowing-versus-consuming)). - `set_signer` does not always take the pointer. A pre-consume rejection (one of `_PRE_CONSUME_ERROR_TAGS`) leaves the Signer `ACTIVE` and retained, so the triage must read the native error before deciding to close it. Treating every failure as "consumed" would close a signer the native side never took. - A `ctypes.ArgumentError` from `set_signer` is re-raised untouched by `_invoke_consume`: marshalling failed, the native function never ran, and the Signer still owns its handle. Only calls that reached native go through the consumed/retained triage. - The builder is never held as a raw local across the signer and build calls. `_NativeBuilder`'s `with` block owns it: a settings error, a retained-signer error, a build rejection, or an async interrupt all free it through `close()`, and a successful build consumes it so `close()` is then a no-op. The old raw-pointer recovery block that used to free `builder_ptr` on the un-reached-build path is gone. @@ -428,7 +567,30 @@ stateDiagram-v2 On success the object stays `ACTIVE` because the Python-side object is still valid: it has a live native pointer, its public methods still work, and callers may continue using it (e.g. reading the updated manifest or feeding in another fragment). The lifecycle state does not change because from `ManagedResource`'s perspective nothing has closed. Only the underlying native pointer has been swapped. This is different from a consumed teardown (`_teardown(free_handle=False)`), where the object transitions to `CLOSED` and becomes unusable. On the success path the old pointer must not be freed by `ManagedResource` because the native library already consumed it as part of the FFI call. The failure path is different and is covered by the triage in [`_consume_and_swap()`](#_consume_and_swap). -`Reader.with_fragment()` runs the native call inside `self._native_call()`, and keeps a `_fragment_streams` list holding the `Stream` wrapper for the current fragment. Each call to `with_fragment()` replaces that list rather than appending to it, closing the previous fragment's wrapper immediately: the native reader never reads a superseded fragment back, and each open wrapper pins a native stream, its callbacks, and the caller's buffer. The native call and the field swap that follows it are both covered by `_fragment_lock`, described in [`Reader._fragment_lock`](#readerfragment_lock). +### `Reader.with_fragment()` + +One `with_fragment()` call does two things: + +1. The FFI call consumes the Reader's current handle and returns a replacement, which `_swap_handle()` stores. +2. The Reader updates its own Python-side fields: the `Stream` wrappers it owns and the manifest caches. Both still describe the consumed handle. + +`_fragment_streams` holds the `Stream` wrapper for the current fragment. Each call replaces that list rather than appending to it, closing the previous wrapper immediately. The native reader never reads a superseded fragment back, and each open wrapper pins a native stream, its callbacks, and the caller's buffer. + +#### Serializing `with_fragment()` against itself + +Those two steps have to run as one unit. Two threads interleaving them can close a `Stream` the native reader is still reading through. They can also leave the Reader holding wrappers and caches that belong to a handle another thread has already replaced. + +`Reader._fragment_lock` covers both steps. It is an `RLock`, and `with_fragment()` is the only method that takes it. + +Unlike `_op_lock`, this lock *is* held across the native call. The rule against that exists to stop a re-entering callback from blocking on a lock its own thread holds. Nothing on the callback path takes `_fragment_lock`, so a re-entering callback cannot block on it. + +The two locks nest in a [fixed order](#lock-ordering): `_fragment_lock` outside, then `_native_call()` and `_op_lock` inside it. + +The fork check runs before `_fragment_lock` is taken, for the same reason `_teardown()` checks first. A forked child cannot wait on a lock no surviving thread will release, so it reports the error `_lock()` reports rather than hanging (see [Fork safety](#fork-safety)). + +#### Cache invalidation + +A `Reader` caches the manifest data. The caches describe the handle they were read from, so a successful swap invalidates them. The invalidation happens inside the locked region, and the cache reads sit under `_op_lock` too. A concurrent `json()` therefore sees either the caches from before the swap or the empty ones after it, never a manifest belonging to a replaced handle. ### `_consume_and_swap()` @@ -565,21 +727,13 @@ sequenceDiagram Both `_cleanup_resources()` and the consumed teardown take this branch. Neither simply skips the work: they null the handle and mark the object `CLOSED` so the child cannot go on to use it or try to free it later. Mutating the child's copy has no effect on the parent's, which is untouched and still valid. -`_teardown()` checks `is_foreign_process()` before taking `_op_lock`, not after, so the foreign-process branch above never tries to acquire a lock in the child. The lock itself would raise there anyway (see [Locking and in-flight tracking](#locking-and-in-flight-tracking)), but `_teardown()` needs to finish its cleanup rather than raise. Therefore, it settles the fork case first and only reaches for the lock once it knows this process owns the pointer. +`_teardown()` checks `is_foreign_process()` before taking `_op_lock`, not after, so the foreign-process branch never tries to acquire a lock in the child. The lock itself would raise there anyway (see [Locking and in-flight tracking](#locking-and-in-flight-tracking)), but `_teardown()` needs to finish its cleanup rather than raise. Therefore, it settles the fork case first and only reaches for the lock once it knows this process owns the pointer. The memory the child skips is not lost for good. A child that calls `exec()` replaces its address space; a child that exits has its memory reclaimed by the OS. Even a long-lived child (a `multiprocessing` worker using the fork start method) retains at most the objects it inherited at fork time, which is a bounded, one-off amount rather than a growing leak. Anything the child allocates itself carries the child's own PID and is freed normally. > [!NOTE] > `is_foreign_process()` returns `False` when no owner PID was ever recorded, so an object that somehow missed the stamp is cleaned up as before rather than leaking silently. -## Why is `Stream` not a `ManagedResource`? - -`Stream` wraps a Python stream-like object (file stream or memory stream) so the native library can read from and write to it via callbacks. It does not inherit from `ManagedResource`, and it uses `c2pa_release_stream()` instead of `c2pa_free()` for cleanup. - -The reason is that ownership runs in the opposite direction. A `Reader` or `Builder` holds a native resource that Python code calls methods on. A `Stream` holds a native handle that the native library calls *back into* (read, seek, write, flush). The native library needs a different release function to tear down the callback machinery. - -`Stream` tracks its own state with `_closed` and `_initialized` flags rather than `LifecycleState`, but it supports the same three cleanup paths: context manager, explicit `.close()`, and `__del__` fallback. - ## Which method to use when? `_create_and_activate`, `_consume_and_swap`, `_consume_no_replacement`, From 4ac0a10a8c0c92e6a557f69689d23079de8e9b69 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:42:41 -0700 Subject: [PATCH 25/28] fix: Update docs 3 --- docs/native-resources-management.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 8e83df28..c92836bc 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -151,6 +151,10 @@ Cleanup runs in the direction of dependency: whatever can still invoke or reach | `ManagedResource` | `_release()` first (dropping streams and callbacks), then `c2pa_free` | The native pointer depends on the Python-side resources, so those are torn down while the pointer is still valid. | | `Stream` | `c2pa_release_stream` first, then drop the callbacks | The native stream invokes the callbacks. Releasing it first guarantees none can fire, and the callback objects are dropped after that. | +`close()` performs both steps: it calls `c2pa_release_stream`, then sets `_read_cb`, `_seek_cb`, `_write_cb`, and `_flush_cb` to `None`. + +`__del__` performs only the first. It calls `c2pa_release_stream` and leaves the four attributes pointing at their callbacks. Nothing leaks, because `__del__` runs when the `Stream` is being collected: the attributes are freed with the object that holds them. Explicit cleanup drops them itself rather than waiting for collection. + `Stream` does not own the Python object it wraps and never closes it. The caller that opened a file owns that file. A `Reader` that opened a file itself tracks it as `_backing_file` and closes it during its own `_release()`. Both `close()` and `__del__` take the foreign-process branch, marking the stream closed without calling into the native library. [Fork safety](#fork-safety) covers why. @@ -253,7 +257,7 @@ Rules that follow from the order: - A lock held across a native call must be one no callback path acquires. - `Stream._close_lock` is a leaf: nothing is acquired while holding it, so it cannot join a cycle. -No code path holds two resources' `_op_lock` at once. `_native_call()` counts a call as in flight instead of holding the lock across it, which is what keeps that true. `test_no_nested_op_locks` asserts it. +No code path holds two resources' `_op_lock` at once. `_native_call()` counts a call as in flight instead of holding the lock across it, which is what keeps that true. `test_no_nested_op_locks` checks it over the Reader read path by intercepting `_lock()` and recording any acquisition made while another is held. ### Borrowing versus consuming @@ -291,7 +295,7 @@ A sign cannot start once the Context is closed, and raises `C2paError` instead. | **State transitions are one-way** | Lifecycle moves only from UNINITIALIZED to ACTIVE to CLOSED. A closed resource cannot be reactivated. | | **Transitions go through helper methods** | Subclasses call `_activate()`, `_swap_handle()` or `_teardown()` and never assign `_handle` or `_lifecycle_state` directly. `_activate()` and `_swap_handle()` validate before mutating, so an object cannot end up active with a null handle. | | **Ownership transfer is safe** | When a pointer is transferred elsewhere (e.g. via `_teardown(free_handle=False)`), the object stops managing it and does not call `c2pa_free` on it. | -| **Public methods validate lifecycle state** | Every public API calls `_ensure_valid_state()` before use; closed or invalid state yields `C2paError` instead of undefined behavior or crashes. | +| **Public methods validate lifecycle state** | Every public method that uses the handle calls `_ensure_valid_state()` before doing so; closed or invalid state yields `C2paError` instead of undefined behavior or crashes. The exceptions touch no handle: `is_valid` reports the state rather than requiring it, and the `get_supported_mime_types` classmethods query the library itself. | ## Preventing garbage collection of live references @@ -345,7 +349,7 @@ Each transition has one method that performs it, and subclasses must go through | --- | --- | --- | | `_activate(handle)` | UNINITIALIZED to ACTIVE | Rejects a null handle, and refuses to run on an already-activated resource. A rejected activation leaves the object exactly as it was. | | `_swap_handle(new_handle)` | ACTIVE to ACTIVE | Requires the resource to already be active and the replacement to be non-null. Used when an FFI call consumed the old handle and returned a new one. | -| `_teardown(free_handle=False)` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side (e.g. `Signer` into `Context`). Runs `_release()` first, so subclass cleanup still happens. Unlike the other two, it validates nothing. | +| `_teardown(free_handle=False)` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side (e.g. `Signer` into `Context`). Runs `_release()` first, so subclass cleanup still happens. Unlike the other two it enforces no precondition on the current state: it closes whatever it is given. | | `_release_handle()` | ACTIVE to CLOSED | Frees the handle (guarded, via `_teardown(free_handle=True)`) and closes the object. Same post-state as the consumed teardown. | Because activation is the only way in, no code path can leave an object ACTIVE while holding a null handle. @@ -359,7 +363,7 @@ Two terms recur throughout this document. An **owned free** calls `c2pa_free` on | `True` | Either the pointer is still provably ours (normal `close()`, `__del__`) — an owned free — or ownership is unknown after a failure (`_release_handle()`) — a guarded free. | Calls `c2pa_free`. On the owned paths the pointer is really freed; on the unknown-ownership path the registry returns `-1` without touching memory if the native side already took it. | | `False` | The native side already took ownership: a consuming FFI call swallowed the pointer, or it passed to another object. | Frees nothing; the new owner does. A `c2pa_free` here would double-free (or hit the guarded `-1` no-op that dirties the error slot and risks racing a recycled address). | -Every public method calls `_ensure_valid_state()` before doing any work, which raises `C2paError` unless the resource is ACTIVE with a non-null handle. +Every public method that uses the handle calls `_ensure_valid_state()` before doing any work, which raises `C2paError` unless the resource is ACTIVE with a non-null handle. ## Ways to clean up @@ -590,7 +594,16 @@ The fork check runs before `_fragment_lock` is taken, for the same reason `_tear #### Cache invalidation -A `Reader` caches the manifest data. The caches describe the handle they were read from, so a successful swap invalidates them. The invalidation happens inside the locked region, and the cache reads sit under `_op_lock` too. A concurrent `json()` therefore sees either the caches from before the swap or the empty ones after it, never a manifest belonging to a replaced handle. +A `Reader` caches the manifest data. The caches describe the handle they were read from, so a successful swap invalidates them. Both the invalidation and the cache reads in `json()` run under `_op_lock`, so a reader never observes a half-updated cache. + +Replacing the handle and clearing the caches are two separate steps, and `_op_lock` is not held between them: + +1. `_consume_and_swap()` replaces the handle. This runs inside `_native_call()`, which counts the call as in flight rather than holding `_op_lock`. +2. `with_fragment()` acquires `_op_lock` and clears the caches. + +Between the two, the Reader has the new handle and the old manifest. A `json()` on another thread acquires `_op_lock` in that gap, finds the cache populated, and returns it without consulting the handle at all. The manifest it serves belongs to the fragment that was just replaced. + +`_fragment_lock` does not prevent this. It serializes `with_fragment()` against other calls to `with_fragment()`, and readers never take it. Callers sharing a `Reader` across threads have to serialize `with_fragment()` against their own reads. ### `_consume_and_swap()` From fce5327fe9fcfbbac787ff0d43ed5e446cc6dacd Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:02:32 -0700 Subject: [PATCH 26/28] fix: Update docs --- docs/native-resources-management.md | 58 +++++++++++++++++------------ 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index c92836bc..17b01d92 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -101,7 +101,7 @@ Therefore, the managed resources have the following principles: - Each `ManagedResource` holds exactly one `_handle`. `_swap_handle()` replaces it with the pointer a consuming call returned and does not free the old value, since the native side took it (see [Consume-and-swap](#consume-and-swap)). - `_teardown(free_handle=False)`, `_consume_no_replacement()`, and `_consume_into()` all close or advance the object without calling `c2pa_free`, because ownership moved to the native side. -- Only a few sites free a live handle. Two of them free a pointer this layer still provably owns: normal teardown (`_teardown(free_handle=True)`), and the create-then-validate path, which frees a freshly created pointer if activation fails. The third, `_release_handle()`, is a *guarded* free used only when ownership is genuinely unknown (a consuming call failed without setting an error, or a Python exception was raised before the native side reported anything): if the native side already took the pointer, its address is no longer in the registry and `c2pa_free` is a `-1` no-op, so the free touches no memory. No path frees a pointer known to have been consumed and reallocated (see [Why an ownership-taken failure does not free](#why-an-ownership-taken-failure-does-not-free)). +- Only a few sites free a live handle, and most free a pointer this layer still provably owns: normal teardown (`_teardown(free_handle=True)`); the create-then-validate path, which frees a freshly created pointer if activation fails; and the constructors that free a raw pointer when wrapping it raises, since no instance took ownership (`Signer.from_info`, `Signer.from_callback`, `Builder.from_archive`). The exception is `_release_handle()`, a *guarded* free used only when ownership is unknown (a consuming call failed without setting an error, or a Python exception was raised before the native side reported anything): if the native side already took the pointer, its address is no longer in the registry and `c2pa_free` is a `-1` no-op, so the free touches no memory. No path frees a pointer known to have been consumed and reallocated (see [Why an ownership-taken failure does not free](#why-an-ownership-taken-failure-does-not-free)). - `_release()` drops stream wrappers, callbacks, and caches before the native pointer is freed (see [Subclass-specific cleanup with `_release()`](#subclass-specific-cleanup)). ### Double-free risk mitigations @@ -401,10 +401,10 @@ If neither the context manager nor an explicit `.close()` is used, `__del__` att Cleanup must not raise an *ordinary* exception. A failure during cleanup (for example, the native library crashing on free) should not mask the original exception that caused the `with` block to exit. `ManagedResource` enforces this: -- `close()` delegates to `_cleanup_resources()`, which wraps the entire cleanup sequence in a try/except that catches and silences `Exception`. +- `close()` delegates to `_cleanup_resources()`, which wraps the entire cleanup sequence in a try/except that catches and silences `Exception`. It performs no teardown itself: after the foreign-process and already-closed checks it calls `_teardown(free_handle=True)`, which does the remaining work. - `_release()` is never called directly during cleanup. It runs inside `_safe_release()`, which logs any `Exception` with a traceback and returns normally, so a subclass whose `_release()` raises an ordinary error cannot stop the native pointer from being freed afterwards. - If freeing the native pointer fails, the error is logged via Python's `logging` module but not re-raised. -- The state is set to `CLOSED` as the very first step, before attempting to free anything. If cleanup fails halfway, the object is still marked closed, preventing a second attempt from doing further damage. +- `_teardown()` sets `CLOSED` before running `_release()` or freeing anything. If cleanup fails halfway, the object is still marked closed, preventing a second attempt from doing further damage. It then moves the pointer into a local and nulls `_handle` before calling `c2pa_free`, so no other caller can read a handle that is about to be freed. - Cleanup is idempotent. Calling `close()` on an already-closed object returns immediately. These handlers catch `Exception`, not `BaseException`. The signals the interpreter raises to unwind a process (a cancellation request, or an exit already in progress) are `BaseException`, so they pass through cleanup untouched and the remaining free may not run. That is intentional: the signal means the whole process is going away, and its address space, native allocations included, is reclaimed on exit. Holding the interpreter in cleanup to finish a free that is about to become irrelevant would only delay the shutdown the caller asked for. @@ -418,11 +418,13 @@ flowchart TD FP -->|yes| N["null the handle, set CLOSED,
do not free"] --> DONE([return]) FP -->|no| ST{"already CLOSED?"} ST -->|yes| DONE - ST -->|no| SET["set CLOSED first"] + ST -->|no| TD["_teardown(free_handle=True)"] + TD --> SET["mark _released, set CLOSED"] SET --> REL["_safe_release()
logs and swallows"] - REL --> H{"handle set?"} + REL --> NULL["take the pointer into a local,
set _handle = None"] + NULL --> H{"pointer was set?"} H -->|no| DONE - H -->|yes| FREE["_free_native_ptr()
logs on failure"] --> NULL["_handle = None"] --> DONE + H -->|yes| FREE["_free_native_ptr()
logs on failure"] --> DONE ``` The `foreign process` branch is explained under [Fork safety](#fork-safety). @@ -482,7 +484,7 @@ stateDiagram-v2 end note ``` -While `ACTIVE`, callers can use `.add_ingredient()`, `.add_action()`, etc. repeatedly. `.sign()` closes the Builder when it returns, on both the success and the failure path. Closing without signing frees the pointer the same way. +While `ACTIVE`, callers can use `.add_ingredient()`, `.add_action()`, etc. repeatedly. `.sign()` closes the Builder when it returns, whether the signing succeeded or failed. A call rejected on its arguments before signing starts, such as a first argument that is neither a `Signer` nor a format string, raises without closing: nothing was signed, so the Builder is still usable. Closing without signing frees the pointer the same way. The native sign call borrows the builder's pointer rather than taking ownership of it, so `Builder` never marks it consumed and the pointer is freed normally through `c2pa_free`. The close enforces single use; it is not a memory-management requirement. @@ -519,12 +521,19 @@ sequenceDiagram alt status 0 (success) S->>S: _teardown(free_handle=False) Note right of S: Consumed: native took the signer - else pre-consume rejection (one of _PRE_CONSUME_ERROR_TAGS) + else non-zero status S->>S: _abort_consume(): restore the previous state - Note right of S: Rejected before ownership moved:
Signer retained and usable, typed error raised - else other error - S->>S: _teardown(free_handle=False) - Note right of S: Native took it then failed and dropped it + S->>S: _raise_consume_failure() reads the native error + Note right of S: The error is read before any free,
so a free's own error cannot overwrite it + alt error carries a pre-consume tag + Note right of S: Rejected before ownership moved:
Signer stays ACTIVE, typed error raised + else any other error + S->>S: _teardown(free_handle=False) + Note right of S: Native took it, then failed and
dropped the value itself: free nothing + else error slot empty + S->>S: _release_handle() guarded free + Note right of S: Ownership unknown: a real free if still
ours, a -1 no-op if native took it + end end X->>B: _consume_into(build) @@ -612,8 +621,10 @@ Every call of this shape goes through one helper, which takes the FFI call as a ```python # Reader.with_fragment() internally does: self._consume_and_swap( - lambda handle: _lib.c2pa_reader_with_fragment(handle, format_bytes, stream), - Reader._ERROR_MESSAGES['reader_error']) + lambda handle: _lib.c2pa_reader_with_fragment( + handle, format_arg, main_obj._stream, frag_obj._stream, + ), + Reader._ERROR_MESSAGES['fragment_error']) ``` The call is passed as a lambda because the helper supplies the handle and, on success, replaces it via `_swap_handle()`. @@ -622,8 +633,8 @@ The helper exists because a failed return can be ambiguous. The native functions ```mermaid flowchart TD - CALL["FFI call(handle)"] --> V{"validate borrowed handle"} - V -->|invalid| R["reject: handle NOT taken
sets UntrackedPointer / WrongPointerType"] --> F1["returns a failure value
(null, or non-zero status)"] + CALL["FFI call(handle)"] --> V{"validate arguments,
then the borrowed handle"} + V -->|invalid| R["reject: handle NOT taken
sets one of _PRE_CONSUME_ERROR_TAGS"] --> F1["returns a failure value
(null, or non-zero status)"] V -->|valid| TAKE["take ownership of handle"] TAKE --> WORK{"execute function logic"} WORK -->|fails| DROP["native drops the value itself
sets some other error"] --> F2["returns a failure value
(null, or non-zero status)"] @@ -674,13 +685,14 @@ A consuming C FFI function first removes the pointer from its registry, then rec `Reader._init_from_context` and `Builder._init_from_context` both create a native object, immediately activate it, and only then make the consuming call. `_create_and_activate()` handles the create-then-activate half: it calls the FFI constructor, validates the result with `_check_ffi_operation_result`, and `_activate()`s it, freeing the pointer if either step fails so a rejected creation leaks nothing. Reduced to its shape: ```python -self._create_and_activate( - lambda: _lib.c2pa_reader_from_context(context.execution_context), - Reader._ERROR_MESSAGES['reader_error']) +with context._native_call(): + self._create_and_activate( + lambda: _lib.c2pa_reader_from_context(context.execution_context), + Reader._ERROR_MESSAGES['reader_error']) self._consume_and_swap( lambda handle: _lib.c2pa_reader_with_stream( - handle, format_bytes, self._own_stream._stream, + handle, format_arg, self._own_stream._stream, ), Reader._ERROR_MESSAGES['reader_error']) ``` @@ -800,7 +812,7 @@ class NativeResource(ManagedResource): "Failed to create MyResource: {}") def _release(self): - # 4. Clean up class-specific resources. + # 3. Clean up class-specific resources. # Never let this method raise. Must be idempotent. # # Consider defining a simple lifecycle for native resources @@ -818,7 +830,7 @@ class NativeResource(ManagedResource): self._my_stream = None def do_something(self): - # 5. Check state at the start of every public method. + # 4. Check state at the start of every public method. # This raises C2paError if the resource is closed. self._ensure_valid_state() return _lib.c2pa_my_resource_do_something(self._handle) @@ -826,7 +838,7 @@ class NativeResource(ManagedResource): ### Troubleshooting -- An attribute set only in `__init__` is missing on an instance built by `_wrap_native_handle()`, because that path never runs `__init__`. The failure shows up later as an `AttributeError` from whichever method reads the attribute, often `_release()` during cleanup. Attributes belong in `_init_attrs()`, which `__init__` calls. +- An attribute set only in `__init__` is missing on an instance built by `_wrap_native_handle()`, because that path never runs `__init__`. The failure shows up later as an `AttributeError` from whichever method reads the attribute, often `_release()` during cleanup. Attributes belong in `_init_attrs()`, which each subclass `__init__` calls and which `_wrap_native_handle()` calls in its place. `ManagedResource.__init__` does not call it, so a subclass that omits the call gets neither path. - `_init_attrs()` called after an FFI call that can raise leaves `_release()` accessing attributes that do not exist yet when that call fails, crashing with `AttributeError`. It belongs immediately after `super().__init__()`, before anything that can fail. From 205ef5f66ec3a23cacb7a2695198b420489a692a Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Fri, 28 Aug 2026 07:11:20 -0700 Subject: [PATCH 27/28] fix: Additional hardening and docs (#317) * fix: Additional hardening * fix: And here the docs --- docs/context-settings.md | 2 + docs/native-resources-management.md | 24 +- src/c2pa/c2pa.py | 121 ++++++++-- tests/test_unit_tests.py | 244 +++++++++++++++++++- tests/test_unit_tests_threaded.py | 334 +++++++++++++++++++++++++++- 5 files changed, 690 insertions(+), 35 deletions(-) diff --git a/docs/context-settings.md b/docs/context-settings.md index 35692a82..bd950fb9 100644 --- a/docs/context-settings.md +++ b/docs/context-settings.md @@ -752,6 +752,8 @@ ctx = Context() assert isinstance(ctx, ContextProvider) # True ``` +A provider that does not derive from `ManagedResource` runs without in-flight teardown protection. `Reader` and `Builder` check `is_valid` before use, but nothing defers a teardown that arrives mid-construction, so closing such a provider on another thread while a `Reader` or `Builder` is being built from it can free the native context while that construction is still using it. The built-in `Context` carries that protection. Custom providers that share a context across threads should keep it alive for the duration of any construction that uses it. + ## Migrating from load_settings The `load_settings()` function is deprecated. Replace it with `Settings` and `Context` APIs: diff --git a/docs/native-resources-management.md b/docs/native-resources-management.md index 17b01d92..9d4de693 100644 --- a/docs/native-resources-management.md +++ b/docs/native-resources-management.md @@ -142,7 +142,9 @@ Each callback checks `_initialized` and `_closed` before touching the underlying ### `Stream` cleanup -`Stream` holds `_close_lock`, a plain `Lock` rather than an `RLock`. It serializes the three cleanup paths: `close()`, `__del__`, and a `close()` on another thread. Without it, two of them reach the same stream and call `c2pa_release_stream` twice on one native handle. `Stream` needs its own lock since it does not inherit the `_op_lock` machinery. +`Stream` holds `_close_lock`, an `RLock`. It serializes the three cleanup paths: `close()`, `__del__`, and a `close()` on another thread. Without it, two of them reach the same stream and call `c2pa_release_stream` twice on one native handle. `Stream` needs its own lock since it does not inherit the `_op_lock` machinery. + +The lock is reentrant for the same reason `_op_lock` is. `close()` sets the four callback attributes to `None` inside the locked region, which can drop the last reference to an object whose finalizer runs at that bytecode boundary. `__del__` takes the same lock. A plain `Lock` deadlocks against itself when that finalizer belongs to the stream being closed. Cleanup runs in the direction of dependency: whatever can still invoke or reach the other is torn down first. Because callbacks run the opposite way for a `Stream`, its close order is the reverse of `ManagedResource`'s: @@ -159,6 +161,8 @@ Cleanup runs in the direction of dependency: whatever can still invoke or reach Both `close()` and `__del__` take the foreign-process branch, marking the stream closed without calling into the native library. [Fork safety](#fork-safety) covers why. +Both check `is_foreign_process()` before acquiring `_close_lock`, as `_teardown()` and `_lock()` do. A child inherits the lock in whatever state it had at `fork()`, and the thread holding it does not exist there to release it, so a child that acquired first would wait on it forever. + ### Reference cycles in the callbacks Each ctypes callback closes over the `Stream` it belongs to. Captured directly, that forms a cycle: the `Stream` holds the callback, the callback's closure holds the `Stream`. Nothing in that loop reaches a refcount of zero, so cleanup falls to the cycle collector. [Why `__del__` is not reliable enough](#why-__del__-is-not-reliable-enough) covers why that timing cannot be relied on. @@ -271,6 +275,8 @@ The check catches a borrow already in flight. The `CLOSED` mark catches one arri The mark is provisional. `_abort_consume()` restores the previous state when the native call turns out not to have taken the handle, which keeps the retained branch of the [ownership-taken triage](#why-an-ownership-taken-failure-does-not-free) handing back a usable object. +`_raise_consume_failure()` performs that restore, on the pre-consume branch only. The reservation is held until the branch is known. `_read_native_error()` is itself a native call and releases the GIL, so a resource restored to `ACTIVE` before the error is classified is visible as usable to another thread while the native side may already own its handle. + `_consume_and_swap()` is excluded. `_swap_handle()` requires the resource to stay `ACTIVE` and the object remains usable with its replacement pointer, so there is no `CLOSED` mark to make and no check. Its callers (`Reader.with_fragment`, `Builder.with_archive`) pass streams whose callbacks re-enter this API, so they hold their own `_native_call()`. `Reader.with_fragment()` additionally serializes itself with a lock of its own, described in [`Reader.with_fragment()`](#readerwith_fragment). ### Context lifetime during a context-sign @@ -350,7 +356,7 @@ Each transition has one method that performs it, and subclasses must go through | `_activate(handle)` | UNINITIALIZED to ACTIVE | Rejects a null handle, and refuses to run on an already-activated resource. A rejected activation leaves the object exactly as it was. | | `_swap_handle(new_handle)` | ACTIVE to ACTIVE | Requires the resource to already be active and the replacement to be non-null. Used when an FFI call consumed the old handle and returned a new one. | | `_teardown(free_handle=False)` | ACTIVE to CLOSED | Drops the handle without freeing it, for when ownership passed to the native side (e.g. `Signer` into `Context`). Runs `_release()` first, so subclass cleanup still happens. Unlike the other two it enforces no precondition on the current state: it closes whatever it is given. | -| `_release_handle()` | ACTIVE to CLOSED | Frees the handle (guarded, via `_teardown(free_handle=True)`) and closes the object. Same post-state as the consumed teardown. | +| `_release_handle()` | ACTIVE to CLOSED | Frees the handle (guarded, via `_teardown(free_handle=True)`) and closes the object. Same post-state as the consumed teardown. A resource that is already non-ACTIVE takes the other branch, which clears the handle without freeing it; the reserved consume paths call `_teardown()` directly for that reason. | Because activation is the only way in, no code path can leave an object ACTIVE while holding a null handle. @@ -522,16 +528,16 @@ sequenceDiagram S->>S: _teardown(free_handle=False) Note right of S: Consumed: native took the signer else non-zero status - S->>S: _abort_consume(): restore the previous state S->>S: _raise_consume_failure() reads the native error - Note right of S: The error is read before any free,
so a free's own error cannot overwrite it + Note right of S: The error is read before any free,
so a free's own error cannot overwrite it.
The reservation is held until the branch is known alt error carries a pre-consume tag + S->>S: _abort_consume(): restore the previous state Note right of S: Rejected before ownership moved:
Signer stays ACTIVE, typed error raised else any other error S->>S: _teardown(free_handle=False) Note right of S: Native took it, then failed and
dropped the value itself: free nothing else error slot empty - S->>S: _release_handle() guarded free + S->>S: _teardown(free_handle=True) guarded free Note right of S: Ownership unknown: a real free if still
ours, a -1 no-op if native took it end end @@ -595,7 +601,11 @@ Those two steps have to run as one unit. Two threads interleaving them can close `Reader._fragment_lock` covers both steps. It is an `RLock`, and `with_fragment()` is the only method that takes it. -Unlike `_op_lock`, this lock *is* held across the native call. The rule against that exists to stop a re-entering callback from blocking on a lock its own thread holds. Nothing on the callback path takes `_fragment_lock`, so a re-entering callback cannot block on it. +The guard spans the native call, which drives caller-supplied stream callbacks. `with_fragment()` therefore takes it with `acquire(blocking=False)` and releases it in a `finally`. A second thread finding it held is refused with `C2paError` rather than parked behind a native call that is waiting on a callback to return. A callback that starts a thread of its own and waits for it would otherwise deadlock: the new thread waits for the guard, and the call holding the guard waits for the callback. + +A refusal leaves the Reader untouched. No stream is built and no handle is consumed, so the call succeeds once the other thread returns. + +Reentrancy applies to the owning thread only. A callback that calls `with_fragment()` synchronously passes the guard and reaches the native call, which rejects the handle it has already consumed. The two locks nest in a [fixed order](#lock-ordering): `_fragment_lock` outside, then `_native_call()` and `_op_lock` inside it. @@ -650,7 +660,7 @@ The two failure paths are indistinguishable from the return value alone. Only th | --- | --- | --- | | One of `_PRE_CONSUME_ERROR_TAGS` | Still ours: rejected before ownership moved | Handle kept, resource stays `ACTIVE`, typed error raised. Normal cleanup frees it later. | | Any other error | Taken, then the operation failed | `_teardown(free_handle=False)`: the native side already dropped the value, so nothing is freed here. Resource goes `CLOSED`, error typed from the native message. | -| No error at all | Unknown | `_release_handle()` guarded free, the caller's message is raised with `"Unknown error"` filled in. | +| No error at all | Unknown | Guarded free, the caller's message is raised with `"Unknown error"` filled in. A reserved consume frees through `_teardown(free_handle=True)`, because `_release_handle()` treats a reserved resource as one it does not own. | This error and ownership triage relies on the native error still being readable (and correctly being the last error encountered) after the call returns. Reading an error copies the message out and frees the copy, but leaves the native slot set until the next error overwrites it. diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 2a9b30ca..9209d580 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -559,7 +559,7 @@ def _swap_handle(self, new_handle): "InvalidBufferSize:", ) - def _invoke_consume(self, ffi_call, error_message): + def _invoke_consume(self, ffi_call, error_message, *, reserved=False): """Run an FFI call that consumes this handle, returning its raw result. A marshalling ArgumentError is re-raised untouched (call never reached @@ -573,6 +573,9 @@ def _invoke_consume(self, ffi_call, error_message): result (a replacement pointer, a status code, ...). error_message: Format string with one placeholder, used to wrap a callback exception. + reserved: True when the caller reserved the handle with + _begin_consume(), which frees here rather than through + _release_handle(). Raises: ctypes.ArgumentError: If marshalling failed; handle untouched. @@ -585,10 +588,15 @@ def _invoke_consume(self, ffi_call, error_message): # is untouched and still ours. Re-raise as-is. raise except Exception as e: - self._release_handle() + if reserved: + # A reservation leaves the resource CLOSED with the handle set, + # which _release_handle() nulls without freeing. + self._teardown(free_handle=True) + else: + self._release_handle() raise C2paError(error_message.format(e)) from e - def _raise_consume_failure(self, error_message): + def _raise_consume_failure(self, error_message, previous_state=None): """Raise the error from an FFI handler consuming call. The native error is read before any free so a free's own @@ -603,9 +611,19 @@ def _raise_consume_failure(self, error_message): with another one and, because that substitute carries a pre-consume tag, invert the retain/consume decision made below. + A caller that reserved the handle with _begin_consume() passes + previous_state and stays reserved until this classification finishes. + _read_native_error() is a native call and releases the GIL, so a + resource restored to ACTIVE before the tags are examined is visible as + usable to another thread while native may already own its handle. Only + the pre-consume branch hands the resource back. + Args: error_message: Format string with one placeholder, used when the native layer offers no error of its own. + previous_state: Lifecycle state to restore if the handle turns out + to have been rejected before native took ownership. None when + the caller holds no reservation. Raises: C2paError: Always; typed by the native error when there is one. @@ -619,6 +637,8 @@ def _raise_consume_failure(self, error_message): "ownership (%s); handle retained", type(self).__name__, error) + if previous_state is not None: + self._abort_consume(previous_state) _raise_typed_c2pa_error(error) # A non-tag error means the native side took ownership then failed, @@ -629,7 +649,12 @@ def _raise_consume_failure(self, error_message): _raise_typed_c2pa_error(error) # No error in the slot: ownership is unknown, so free defensively. - self._release_handle() + # A reservation leaves the resource CLOSED with the handle set, + # which _release_handle() nulls without freeing. + if previous_state is not None: + self._teardown(free_handle=True) + else: + self._release_handle() raise C2paError(error_message.format("Unknown error")) def _begin_consume(self): @@ -673,7 +698,19 @@ def _consume_and_swap(self, ffi_call, error_message): """ new_ptr = self._invoke_consume(ffi_call, error_message) if new_ptr: - self._swap_handle(new_ptr) + try: + self._swap_handle(new_ptr) + except Exception: + # _swap_handle refuses a resource a concurrent close() left + # CLOSED. Native consumed the old pointer and returned this + # one, so nothing else holds it. + try: + ManagedResource._free_native_ptr(new_ptr) + except Exception: + logger.error( + "Failed to free the replacement %s handle", + type(self).__name__, exc_info=True) + raise return self._raise_consume_failure(error_message) @@ -685,15 +722,15 @@ def _consume_no_replacement(self, ffi_call, error_message): """ previous_state = self._begin_consume() try: - result = self._invoke_consume(ffi_call, error_message) + result = self._invoke_consume( + ffi_call, error_message, reserved=True) except Exception: self._abort_consume(previous_state) raise if result == 0: self._teardown(free_handle=False) return - self._abort_consume(previous_state) - self._raise_consume_failure(error_message) + self._raise_consume_failure(error_message, previous_state) def _consume_into(self, ffi_call, error_message): """Run an FFI call that consumes this handle and returns a *different* @@ -703,15 +740,15 @@ def _consume_into(self, ffi_call, error_message): """ previous_state = self._begin_consume() try: - result = self._invoke_consume(ffi_call, error_message) + result = self._invoke_consume( + ffi_call, error_message, reserved=True) except Exception: self._abort_consume(previous_state) raise if result: self._teardown(free_handle=False) return result - self._abort_consume(previous_state) - self._raise_consume_failure(error_message) + self._raise_consume_failure(error_message, previous_state) @classmethod def _wrap_native_handle(cls, handle): @@ -1644,11 +1681,35 @@ def load_settings(settings: Union[str, dict], format: str = "json") -> None: check=lambda r: r != 0) +@contextlib.contextmanager +def _context_guard(context): + """Hold a caller-supplied context valid across a native call. + + ContextProvider requires only is_valid and execution_context. + A provider that also manages a native handle, + such as the built-in Context, offers _native_call, + which counts the call in flight so a concurrent close() records + its intent and defers the free until the call returns. A provider + implementing just the two required properties runs without that guard. + """ + native_call = getattr(context, "_native_call", None) + if native_call is None: + yield + return + with native_call(): + yield + + class ContextProvider(ABC): """Abstract base class for types that provide a C2PA context. Subclass to implement a custom context provider. The built-in Context class is the standard implementation. + + A provider that does not derive from ManagedResource is used without + in-flight teardown protection: closing it on another thread while a Reader + or Builder is being constructed from it can free the native context while + that construction is still using it. """ @property @@ -2004,7 +2065,7 @@ def __init__(self, file_like_stream): self._initialized = False self._stream = None # Serializes close() and __del__ against a concurrent double-free. - self._close_lock = threading.Lock() + self._close_lock = threading.RLock() # Generate unique stream ID using object ID and counter stream_counter = next(Stream._stream_id_counter) @@ -2254,14 +2315,18 @@ def close(self): Errors during cleanup are logged but not raised to ensure cleanup. Multiple calls to close() are handled gracefully. """ + # Checked before the lock, as _lock() and __del__ do: + # a child inherits _close_lock in whatever state it had at fork(), + # and the thread holding it does not exist there to release it. + if is_foreign_process(self): + self._closed = True + self._initialized = False + return + # Serializes against __del__ / a concurrent close(). with self._close_lock: if self._closed: return - if is_foreign_process(self): - self._closed = True - self._initialized = False - return try: # Clean up stream first as it depends on callbacks @@ -2818,7 +2883,7 @@ def _init_from_context(self, context, format_or_path, try: # The Context is caller-supplied and may be shared, so its handle # needs its own in-flight guard across the native call. - with context._native_call(): + with _context_guard(context): # Adopt before the consuming call: _consume_and_swap needs an # active resource, and cleanup then owns the pointer either # way. @@ -2965,6 +3030,9 @@ def with_fragment(self, format: Optional[str], stream, underlying object, in which case this Reader is closed and cannot be retried: create a new one instead of reusing this instance. + C2paError: If another thread is inside this method on the same + Reader. This one leaves the Reader untouched, so the call can + be retried once that thread returns. """ format_arg = _format_ffi_arg(_encode_format(format, "Reader")) @@ -2974,7 +3042,18 @@ def with_fragment(self, format: Optional[str], stream, raise C2paError(f"{type(self).__name__} is closed") # The native call and the ownership transfer are one unit. - with self._fragment_lock: + # Taken without blocking because the call drives caller-supplied stream + # callbacks: a second thread, including one a callback starts, would + # otherwise wait here for a native call that is itself waiting on that + # callback to return. + # + # Reentrant, so the thread already inside this region passes through + # and re-enters the native call, which rejects the handle it consumed. + if not self._fragment_lock.acquire(blocking=False): + raise C2paError( + f"{type(self).__name__} is already processing a fragment " + f"on another thread") + try: # The native reader keeps reading through both streams. main_obj = Stream(stream) frag_obj = Stream(fragment_stream) @@ -3028,6 +3107,8 @@ def with_fragment(self, format: Optional[str], stream, # and a reader must never be served them. self._manifest_json_str_cache = None self._manifest_data_cache = None + finally: + self._fragment_lock.release() return self @@ -3670,7 +3751,7 @@ def _init_from_context(self, context, json_str): # The Context is caller-supplied and may be shared, # so its handle needs its own in-flight guard across # the native call, especially for state checks. - with context._native_call(): + with _context_guard(context): # Adopt before the consuming call. self._create_and_activate( lambda: _lib.c2pa_builder_from_context( @@ -4039,7 +4120,7 @@ def _sign_internal( # Entered inside self's guard, matching the Builder to # Signer order, so the two acquisitions are always # taken in one direction. - with self._context._native_call(): + with _context_guard(self._context): result = _lib.c2pa_builder_sign_context( self._handle, format_arg, diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index e7af0c05..2c1fb42e 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -9677,6 +9677,200 @@ def test_ed25519_sign_with_empty_data_raises(self): c2pa_module.ed25519_sign(b"", "not a key") +class TestConsumeOwnership(unittest.TestCase): + """Ownership of the native handle across the consuming call paths.""" + + def setUp(self): + self.freed = [] + self._real_free = ManagedResource._free_native_ptr + + def counting_free(ptr): + self.freed.append(ptr) + return self._real_free(ptr) + + ManagedResource._free_native_ptr = staticmethod(counting_free) + + def tearDown(self): + ManagedResource._free_native_ptr = staticmethod(self._real_free) + + def test_generic_exception_frees_the_reserved_handle(self): + """A reserved consume that raises must free, not drop, the handle. + + _begin_consume() leaves the resource CLOSED with the handle still set, + which _release_handle() reads as "not ours" and nulls without freeing, + while _abort_consume() can no longer restore it. + """ + def boom(handle): + raise RuntimeError("callback failed after the reservation") + + for name in ("_consume_no_replacement", "_consume_into"): + with self.subTest(helper=name): + resource = Settings() + self.freed.clear() + + with self.assertRaises(Error): + getattr(resource, name)(boom, "consume failed: {}") + + self.assertEqual( + len(self.freed), 1, + "{} dropped the handle without freeing it".format(name)) + self.assertIsNone(resource._handle) + + def test_marshalling_error_retains_the_handle(self): + """Positive control for the free counter. + + An ArgumentError means the call never reached native, so the handle is + untouched and must NOT be freed. Without this, a zero-free assertion + could pass simply because the counter never fires. + """ + def bad_marshal(handle): + raise ctypes.ArgumentError("marshalling failed") + + resource = Settings() + self.freed.clear() + + with self.assertRaises(ctypes.ArgumentError): + resource._consume_no_replacement(bad_marshal, "consume: {}") + + self.assertEqual(self.freed, []) + self.assertIsNotNone(resource._handle) + self.assertEqual(resource._lifecycle_state, LifecycleState.ACTIVE) + + def test_pre_consume_rejection_restores_the_resource(self): + """A handle native rejected before taking ownership stays usable. + + The reservation is held until _raise_consume_failure classifies the + error, so no other thread sees the resource as ACTIVE while its + ownership is still undetermined. + """ + resource = Settings() + self.freed.clear() + real_read = c2pa_module._read_native_error + c2pa_module._read_native_error = ( + lambda: "Other: UntrackedPointer: 0x1234") + try: + with self.assertRaises(Error): + resource._consume_no_replacement(lambda h: 1, "consume: {}") + finally: + c2pa_module._read_native_error = real_read + + self.assertEqual(resource._lifecycle_state, LifecycleState.ACTIVE) + self.assertIsNotNone(resource._handle) + self.assertEqual(self.freed, []) + + def test_post_consume_failure_keeps_the_resource_closed(self): + """An error without a pre-consume tag means native took ownership. + + The value is native's to drop, so the resource stays closed and frees + nothing. + """ + resource = Settings() + self.freed.clear() + real_read = c2pa_module._read_native_error + c2pa_module._read_native_error = lambda: "Other: operation failed" + try: + with self.assertRaises(Error): + resource._consume_no_replacement(lambda h: 1, "consume: {}") + finally: + c2pa_module._read_native_error = real_read + + self.assertEqual(resource._lifecycle_state, LifecycleState.CLOSED) + self.assertEqual(self.freed, []) + + def test_failure_without_a_native_error_frees_the_handle(self): + """An empty error slot leaves ownership unknown, so the handle is + freed defensively rather than dropped. + """ + resource = Settings() + self.freed.clear() + real_read = c2pa_module._read_native_error + c2pa_module._read_native_error = lambda: None + try: + with self.assertRaises(Error): + resource._consume_no_replacement(lambda h: 1, "consume: {}") + finally: + c2pa_module._read_native_error = real_read + + self.assertEqual( + len(self.freed), 1, + "an unknown-ownership failure dropped the handle without freeing") + self.assertIsNone(resource._handle) + + def test_rejected_replacement_is_freed(self): + """A replacement _swap_handle refuses must not be left unowned. + + Native consumed the old pointer and returned this one, so nothing else + holds it. + """ + resource = Settings() + spare = Settings() + replacement = spare._handle + # Detach so only the code under test can free it. + spare._handle = None + spare._lifecycle_state = LifecycleState.CLOSED + self.freed.clear() + + # A close() arriving mid-call leaves the resource CLOSED. + resource._lifecycle_state = LifecycleState.CLOSED + + with self.assertRaises(Error): + resource._consume_and_swap(lambda h: replacement, "swap: {}") + + self.assertIn(replacement, self.freed) + + +class TestContextProviderContract(unittest.TestCase): + """The published ContextProvider contract is is_valid plus + execution_context, and nothing more. + """ + + class _MinimalProvider(ContextProvider): + """Implements exactly what the abstract base class declares.""" + + def __init__(self): + self._inner = Context(Settings()) + + @property + def is_valid(self): + return self._inner.is_valid + + @property + def execution_context(self): + return self._inner.execution_context + + def test_reader_accepts_a_minimal_provider(self): + provider = self._MinimalProvider() + try: + Reader("image/jpeg", io.BytesIO(b"not a real jpeg"), + context=provider) + except AttributeError as e: + self.fail("Reader requires more than the documented " + "ContextProvider contract: {}".format(e)) + except Error: + # Rejecting the bytes is the native library doing its job. + pass + + def test_builder_accepts_a_minimal_provider(self): + provider = self._MinimalProvider() + try: + Builder({"claim_generator": "test"}, context=provider) + except AttributeError as e: + self.fail("Builder requires more than the documented " + "ContextProvider contract: {}".format(e)) + + def test_built_in_context_still_gets_in_flight_protection(self): + """The compatibility shim must not silently drop the guard for the + provider that does implement it. + """ + context = Context(Settings()) + self.assertEqual(context._inflight, 0) + with c2pa_module._context_guard(context): + self.assertGreater( + context._inflight, 0, + "built-in Context lost its in-flight guard") + self.assertEqual(context._inflight, 0) + + class TestLockOrderStaticAnalysis(unittest.TestCase): """Static analysis over the source, not runtime behavior: no threads are spawned here. @@ -9735,6 +9929,33 @@ def lock_name_for_with(item): return "_op_lock" return None + def lock_name_for_acquire(node): + """A lock taken with acquire() and released in a finally nests just + as a `with` does, so the scan has to follow it or it silently stops + seeing whole regions. + """ + call = node.value if isinstance(node, ast.Expr) else node + if isinstance(call, ast.UnaryOp) and isinstance(call.op, ast.Not): + call = call.operand + if not (isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "acquire"): + return None + owner = call.func.value + if (isinstance(owner, ast.Attribute) + and isinstance(owner.value, ast.Name) + and owner.value.id == "self" + and any(owner.attr in attrs + for attrs in lock_attrs_by_class.values())): + return owner.attr + return None + + def acquires_in_test(node): + """Lock taken by `if not self._X.acquire(...)`-style guards.""" + if isinstance(node, ast.If): + return lock_name_for_acquire(node.test) + return None + def orders_in(node, stack, pairs): """Record (outer, inner) for every nesting this node contains.""" if isinstance(node, (ast.With, ast.AsyncWith)): @@ -9749,8 +9970,29 @@ def orders_in(node, stack, pairs): for _ in names: stack.pop() return + # A statement list can open a lock partway through via acquire(); + # everything after it in that list is nested inside. + for field, value in ast.iter_fields(node): + if not isinstance(value, list): + continue + held = [] + for child in value: + if not isinstance(child, ast.stmt): + continue + name = (lock_name_for_acquire(child) + or acquires_in_test(child)) + if name: + if stack: + pairs.add((stack[-1], name)) + stack.append(name) + held.append(name) + continue + orders_in(child, stack, pairs) + for _ in held: + stack.pop() for child in ast.iter_child_nodes(node): - orders_in(child, stack, pairs) + if not isinstance(child, ast.stmt): + orders_in(child, stack, pairs) pairs_by_method = {} for cls in ast.walk(tree): diff --git a/tests/test_unit_tests_threaded.py b/tests/test_unit_tests_threaded.py index ffff10c5..20537e46 100644 --- a/tests/test_unit_tests_threaded.py +++ b/tests/test_unit_tests_threaded.py @@ -27,6 +27,7 @@ import threading import concurrent.futures import time +import signal import asyncio import random from unittest.mock import MagicMock, patch @@ -34,6 +35,7 @@ from c2pa import Builder, C2paError as Error, Reader, C2paSigningAlg as SigningAlg, C2paSignerInfo, Signer, sdk_version # noqa: E501 from c2pa import Context, Settings from c2pa.c2pa import ManagedResource, Stream, LifecycleState +import c2pa.c2pa as c2pa_module from c2pa.lib import is_foreign_process, record_owner_pid PROJECT_PATH = os.getcwd() @@ -573,19 +575,37 @@ def gated_native_call(): reader._native_call = gated_native_call class ContentionReportingLock: - """Flags when a caller has to wait for the lock it wraps.""" + """Flags when a caller finds the lock it wraps already held. + + with_fragment takes this lock with acquire(blocking=False) and + releases it in a finally, so those are the methods wrapped here. + """ def __init__(self, inner): self._inner = inner - def __enter__(self): + def acquire(self, blocking=True, timeout=-1): + if not blocking: + acquired = self._inner.acquire(blocking=False) + if not acquired: + # The second caller is refused rather than parked, + # which is the mutual exclusion this test checks for. + contended.set() + return acquired if not self._inner.acquire(blocking=False): contended.set() - self._inner.acquire() + return self._inner.acquire(blocking, timeout) + return True + + def release(self): + self._inner.release() + + def __enter__(self): + self.acquire() return self def __exit__(self, exc_type, exc_val, exc_tb): - self._inner.release() + self.release() return False real_fragment_lock = reader._fragment_lock @@ -3393,6 +3413,296 @@ def thread_work(thread_id): self.assertNotEqual(current_manifest["active_manifest"], thread_manifest_data[other_thread_id]["active_manifest"]) +class TestWithFragmentReentrancy(unittest.TestCase): + """with_fragment drives caller-supplied stream callbacks, so it must not + hold a lock a callback-spawned thread would wait on. + """ + + def test_reentrant_call_is_refused_rather_than_blocked(self): + init_path = os.path.join(FIXTURES_FOLDER, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_FOLDER, "dash1.m4s") + with open(init_path, "rb") as handle: + init_bytes = handle.read() + with open(fragment_path, "rb") as handle: + fragment_bytes = handle.read() + + reader = Reader("video/mp4", io.BytesIO(init_bytes)) + state = {"fired": False, "result": None, "hung": None} + + class ReentrantStream(io.BytesIO): + """Re-enters the API from another thread, from inside a callback, + and waits for it: the shape that deadlocks a lock held across the + native call. + """ + + def _reenter_once(self): + if state["fired"]: + return + state["fired"] = True + + def second_call(): + try: + reader.with_fragment( + "video/mp4", + io.BytesIO(init_bytes), + io.BytesIO(fragment_bytes)) + state["result"] = "completed" + except Error as e: + state["result"] = e + + thread = threading.Thread(target=second_call, daemon=True) + thread.start() + thread.join(10) + state["hung"] = thread.is_alive() + + def read(self, size=-1): + self._reenter_once() + return super().read(size) + + def seek(self, offset, whence=0): + self._reenter_once() + return super().seek(offset, whence) + + reader.with_fragment("video/mp4", + ReentrantStream(init_bytes), + io.BytesIO(fragment_bytes)) + + self.assertTrue(state["fired"], "the callback never re-entered") + self.assertFalse( + state["hung"], + "a with_fragment call started from a stream callback blocked on " + "the lock the running call holds") + self.assertIsInstance( + state["result"], Error, + "the re-entrant call must be refused, not silently interleaved") + + def test_same_thread_reentry_does_not_corrupt_the_reader(self): + """_fragment_lock is reentrant, so a callback calling with_fragment + synchronously passes the guard. The native layer rejects the handle it + already consumed, and the Reader survives. + """ + init_path = os.path.join(FIXTURES_FOLDER, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_FOLDER, "dash1.m4s") + with open(init_path, "rb") as handle: + init_bytes = handle.read() + with open(fragment_path, "rb") as handle: + fragment_bytes = handle.read() + + reader = Reader("video/mp4", io.BytesIO(init_bytes)) + state = {"fired": False, "inner": None} + + class SelfReentrantStream(io.BytesIO): + def _reenter_once(self): + if state["fired"]: + return + state["fired"] = True + try: + reader.with_fragment("video/mp4", + io.BytesIO(init_bytes), + io.BytesIO(fragment_bytes)) + state["inner"] = "completed" + except Error as e: + state["inner"] = e + + def read(self, size=-1): + self._reenter_once() + return super().read(size) + + def seek(self, offset, whence=0): + self._reenter_once() + return super().seek(offset, whence) + + reader.with_fragment("video/mp4", + SelfReentrantStream(init_bytes), + io.BytesIO(fragment_bytes)) + + self.assertTrue(state["fired"], "the callback never re-entered") + self.assertIsInstance( + state["inner"], Error, + "a nested consume on the same handle must be rejected") + # The outer call still owns a live handle. + self.assertTrue(reader.is_valid) + self.assertIsInstance(reader.json(), str) + + def test_refused_call_leaves_the_reader_usable(self): + """The refusal reports contention without touching the Reader, so the + caller can retry once the other thread returns. + """ + init_path = os.path.join(FIXTURES_FOLDER, "dashinit.mp4") + fragment_path = os.path.join(FIXTURES_FOLDER, "dash1.m4s") + with open(init_path, "rb") as handle: + init_bytes = handle.read() + with open(fragment_path, "rb") as handle: + fragment_bytes = handle.read() + + reader = Reader("video/mp4", io.BytesIO(init_bytes)) + + holding = threading.Event() + release = threading.Event() + + def hold_the_guard(): + reader._fragment_lock.acquire() + holding.set() + release.wait(10) + reader._fragment_lock.release() + + holder = threading.Thread(target=hold_the_guard, daemon=True) + holder.start() + self.assertTrue(holding.wait(5), "the guard was never taken") + + with self.assertRaises(Error): + reader.with_fragment("video/mp4", + io.BytesIO(init_bytes), + io.BytesIO(fragment_bytes)) + + # Refused before any stream was built or handle consumed. + self.assertTrue(reader.is_valid) + + release.set() + holder.join(5) + + # The same call succeeds once the other thread is out. + reader.with_fragment("video/mp4", + io.BytesIO(init_bytes), + io.BytesIO(fragment_bytes)) + self.assertTrue(reader.is_valid) + + +class TestStreamCloseReentrancy(unittest.TestCase): + """close() clears the callback references inside _close_lock, which can run + a finalizer at that bytecode boundary, and __del__ takes the same lock. + """ + + def test_close_can_be_reentered_on_the_same_thread(self): + stream = Stream(io.BytesIO(b"payload")) + finished = threading.Event() + + def hold_then_reenter(): + with stream._close_lock: + # A finalizer running here re-takes the lock this thread holds. + stream.close() + finished.set() + + worker = threading.Thread(target=hold_then_reenter, daemon=True) + worker.start() + + self.assertTrue( + finished.wait(10), + "close() blocked re-entering _close_lock from the thread that " + "already holds it") + self.assertTrue(stream._closed) + + +@unittest.skipUnless(hasattr(os, "fork"), "requires fork()") +class TestStreamCloseAfterFork(unittest.TestCase): + """A forked child must not wait on a lock no surviving thread will + release. + """ + + def test_close_in_child_does_not_block_on_an_inherited_lock(self): + stream = Stream(io.BytesIO(b"payload")) + + holding = threading.Event() + release = threading.Event() + + def hold_the_lock(): + with stream._close_lock: + holding.set() + release.wait(30) + + holder = threading.Thread(target=hold_the_lock, daemon=True) + holder.start() + self.assertTrue(holding.wait(5), "lock was never taken") + + # The child inherits _close_lock held by a thread that does not exist + # there, so close() has to take the foreign-process path without + # acquiring it. + pid = os.fork() + if pid == 0: + try: + stream.close() + # Exit 3 rather than 0 if close() returned without marking the + # stream closed, so a silent no-op cannot pass as success. + marked = stream._closed and not stream._initialized + os._exit(0 if marked else 3) + except BaseException: + os._exit(2) + + deadline = time.time() + 15 + status = None + while time.time() < deadline: + done, wait_status = os.waitpid(pid, os.WNOHANG) + if done: + status = wait_status + break + time.sleep(0.05) + + if status is None: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + release.set() + holder.join(5) + self.fail("close() in the forked child blocked on the inherited " + "lock instead of taking the foreign-process path") + + release.set() + holder.join(5) + self.assertEqual( + os.WEXITSTATUS(status), 0, + "close() in the forked child raised (2) or returned without " + "closing the stream (3)") + + +class TestConsumeReservationWindow(unittest.TestCase): + """The consume reservation must outlast ownership classification. + + _read_native_error() is a native call that releases the GIL, so a resource + restored to ACTIVE before the error is classified is visible as usable to + another thread while native may already own its handle. + """ + + def test_no_thread_sees_a_consumed_handle_as_valid(self): + resource = Settings() + + reading = threading.Event() + may_finish = threading.Event() + seen_valid = [] + + real_read = c2pa_module._read_native_error + + def gated_read(): + # Stand in for the GIL release inside the real native call. + reading.set() + may_finish.wait(10) + # No pre-consume tag: native took ownership and then failed. + return "Other: operation failed after taking ownership" + + def observer(): + if not reading.wait(10): + return + # The consuming call is mid-classification right now. + seen_valid.append(resource.is_valid) + may_finish.set() + + watcher = threading.Thread(target=observer, daemon=True) + watcher.start() + + c2pa_module._read_native_error = gated_read + try: + with self.assertRaises(Error): + resource._consume_no_replacement(lambda h: 1, "consume: {}") + finally: + c2pa_module._read_native_error = real_read + may_finish.set() + watcher.join(10) + + self.assertTrue(seen_valid, "observer never sampled the resource") + self.assertFalse( + seen_valid[0], + "another thread saw a resource whose handle native may already " + "own as valid") + + class TestLocking(unittest.TestCase): """Tests for the locks that guard native resources: - the per-object operation lock that serializes native calls against teardown, @@ -4489,15 +4799,25 @@ def test_every_borrowed_handle_is_guarded(self): handle_attrs = {"_handle", "execution_context"} def guarded_names(node): - """Names X with an active `with X._native_call():` at this node.""" + """Names X guarded at this node, by either form: + `with X._native_call():`, or `with _context_guard(X):` for a + caller-supplied ContextProvider, which enters X._native_call() + when X offers it. + """ found = set() for item in getattr(node, "items", []): call = item.context_expr - if (isinstance(call, ast.Call) - and isinstance(call.func, ast.Attribute) + if not isinstance(call, ast.Call): + continue + if (isinstance(call.func, ast.Attribute) and call.func.attr == "_native_call" and isinstance(call.func.value, ast.Name)): found.add(call.func.value.id) + elif (isinstance(call.func, ast.Name) + and call.func.id == "_context_guard" + and call.args + and isinstance(call.args[0], ast.Name)): + found.add(call.args[0].id) return found def borrowed_in_call(call): From 121d963d56d7018aef0300ee7883f5039b88899b Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Fri, 28 Aug 2026 07:12:50 -0700 Subject: [PATCH 28/28] fix: Make free function configurable (#318) * fix: Additional hardening * fix: And here the docs * fix: Parameterize deallocator --- src/c2pa/c2pa.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/c2pa/c2pa.py b/src/c2pa/c2pa.py index 9209d580..1f66226f 100644 --- a/src/c2pa/c2pa.py +++ b/src/c2pa/c2pa.py @@ -462,7 +462,8 @@ def _teardown(self, free_handle: bool): handle, self._handle = self._handle, None if free_handle and handle: try: - ManagedResource._free_native_ptr(handle) + # Subclasses may override the deallocator. + type(self)._free_native_ptr(handle) except Exception: logger.error("Failed to free native %s resources", type(self).__name__, exc_info=True)