dmihalcik-virtru · Nov 9, 2026
@@ -44,17 +44,15 @@ jobs:
       # No platform and no SDK builds required, so the part that has to be
       # *correct* is checked on every PR rather than only when the nightly runs.
       #
+      # -m units still collects every module, so conftest.py's tier check
+      # fails this step for any test the xtest.yml steps would not select.
+      #
       # --frozen --no-build: resolve nothing and build nothing, so a
       # dependency cannot slip in an unlocked version or a setup script on a
       # runner that already has everything installed from the step above.
       - name: Test xtest offline harnesses
         run: >-
-          uv run --frozen --no-build pytest --no-header -q
-          test_bench_stats.py test_bench_measure.py test_bench_runner.py
-          test_bench_arms.py test_bench_config.py test_bench_report.py
-          test_sdk_commands.py test_tdfs_units.py test_encryption_units.py
-          test_sizes_units.py test_zip64_units.py test_assertions_units.py
-          test_zip_conformance_units.py
+          uv run --frozen --no-build pytest --no-header -q -m units
         working-directory: xtest
       - name: Lint and test otdf-local
         run: |

@@ -713,7 +713,7 @@ jobs:
       - name: Validate xtest helper library (tests of the test harness and its utilities)
         if: ${{ !inputs }}
         run: |-
-          uv run pytest --html=test-results/helper-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" test_self.py test_audit_logs.py
+          uv run pytest --html=test-results/helper-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -m harness
         working-directory: otdftests/xtest
         env:
           PLATFORM_TAG: ${{ matrix.platform-tag }}
@@ -732,7 +732,7 @@ jobs:
       - name: Run legacy decryption tests
         run: |-
           skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "")
-          uv run pytest -n auto --dist worksteal --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-decrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" $skip_flag test_legacy.py
+          uv run pytest -n auto --dist worksteal --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-decrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" $skip_flag -m legacy
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -743,7 +743,7 @@ jobs:
         if: ${{ env.FOCUS_SDK == 'all' }}
         run: |-
           skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "")
-          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v $skip_flag test_tdfs.py test_tdf_assertions.py test_policytypes.py test_zip_conformance.py
+          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v $skip_flag -m core
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -755,7 +755,7 @@ jobs:
         if: ${{ env.FOCUS_SDK != 'all' }}
         run: |-
           skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "")
-          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" $skip_flag test_tdfs.py test_tdf_assertions.py test_policytypes.py test_zip_conformance.py
+          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" $skip_flag -m core
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -888,7 +888,7 @@ jobs:
             --sdks-encrypt "${ENCRYPT_SDK}" \
             --focus "$FOCUS_SDK" \
             $skip_flag \
-            test_abac.py test_pqc.py test_dpop.py
+            -m multikas
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -1173,7 +1173,7 @@ jobs:
             --bench-out test-results/benchmarks \
             --html "test-results/bench-${BENCH_SDK}.html" \
             --self-contained-html \
-            test_benchmarks.py
+            -m benchmark
         working-directory: otdftests/xtest
         env:
           BENCH_SDK: ${{ matrix.sdk }}
@@ -1489,7 +1489,7 @@ jobs:
             --junitxml "test-results/zip64-${ZIP64_SDK}.xml" \
             --html "test-results/zip64-${ZIP64_SDK}.html" \
             --self-contained-html \
-            test_zip64.py
+            -m zip64
         working-directory: otdftests/xtest
         env:
           ZIP64_SDK: ${{ matrix.sdk }}

@@ -19,6 +19,29 @@ fixture system.
 | `sdk/{go,java,js}/dist/<version>/` | SDK CLI builds. Installed by `otdf-sdk-mgr install` (see `../otdf-sdk-mgr/AGENTS.md`). |
 | `test.env` | Default endpoint and client-credential env vars. Source with `set -a && source test.env && set +a`. |
 
+## Test Tiers
+
+CI selects tests by marker, not by file name, so adding a module never means
+editing a workflow. Every test needs exactly one tier; collection fails
+otherwise, because an untiered test would match no CI step and silently never
+run. Declare it once per module:
+
+```python
+pytestmark = pytest.mark.core
+```
+
+| Tier | Needs | Run by |
+|------|-------|--------|
+| `units` | Nothing: no platform, no SDK builds. Implied by a `*_units.py` file name. | `check.yml`, every PR |
+| `harness` | Platform; tests the harness itself. | `xtest.yml` helper step |
+| `core` | Platform + SDKs, platform's own KAS. | `xtest.yml` standard step |
+| `legacy` | Golden TDFs and their keys. Decrypt-only. | `xtest.yml` legacy step |
+| `multikas` | KAS alpha..delta and km1..km3. | `xtest.yml` ABAC step |
+| `benchmark`, `zip64` | `--bench` / `--sizes medium`. | nightly jobs |
+
+Pick the tier by what the test needs, not by topic: `test_pqc.py` is
+`multikas` because of its KAS fixtures, not because PQC is about ABAC.
+
 ## Custom pytest Options (defined in `conftest.py`)
 
 | Option | Purpose |

@@ -425,7 +425,7 @@ def _parametrize_bench_cells(metafunc: pytest.Metafunc):
     """Fan the benchmark module out over its cells.
 
     Without --bench there is nothing to fan out over, and the items are
-    dropped wholesale in :func:`pytest_collection_modifyitems` rather than
+    dropped wholesale in :func:`_drop_unrequested` rather than
     parametrized here. Parametrizing over an empty list would *not* collect
     zero items: pytest's default ``empty_parameter_set_mark`` turns an empty
     set into one skipped item per test, so every ordinary run would carry
@@ -541,9 +541,44 @@ def _item_exercises_zip64_window(item: pytest.Item, session_sizes: list[str]) ->
     return any(sizes.exercises_zip64_window(size) for size in session_sizes)
 
 
+# CI steps select tests by tier (``-m core``), not by file name, so a new
+# module is picked up by declaring ``pytestmark = pytest.mark.<tier>``. The
+# tiers are registered in pyproject.toml.
+TIERS = frozenset(
+    {"units", "harness", "core", "legacy", "multikas", "benchmark", "zip64"}
+)
+
+
+def _require_one_tier(items: list[pytest.Item]) -> None:
+    """Fail collection on any test without exactly one tier.
+
+    An untiered test is matched by no CI step's ``-m`` and would never run,
+    which looks exactly like passing. ``*_units.py`` is ``units`` by name.
+    """
+    bad: dict[str, int] = {}
+    for item in items:
+        if item.path.name.endswith("_units.py"):
+            item.add_marker(pytest.mark.units)
+        n = len({m.name for m in item.iter_markers()} & TIERS)
+        if n != 1:
+            bad.setdefault(item.nodeid.split("::")[0], n)
+    if bad:
+        found = ", ".join(f"{mod} ({n})" for mod, n in sorted(bad.items()))
+        raise pytest.UsageError(
+            f"need exactly one tier marker of {sorted(TIERS)}; found: {found}"
+        )
+
+
+# tryfirst: the tier check must see every item, before -m deselects any.
+@pytest.hookimpl(tryfirst=True)
 def pytest_collection_modifyitems(
     config: pytest.Config, items: list[pytest.Item]
 ) -> None:
+    _require_one_tier(items)
+    _drop_unrequested(config, items)
+
+
+def _drop_unrequested(config: pytest.Config, items: list[pytest.Item]) -> None:
     """Drop cells the session did not ask for.
 
     Two groups, deselected rather than skipped for the same reason: neither a

@@ -90,6 +90,14 @@ python_files = ["test_*.py"]
 python_functions = ["test_*"]
 addopts = "-ra -v"
 markers = [
+    # Tiers: each test carries exactly one (conftest.py enforces it), and CI
+    # selects by tier with -m rather than by file name. benchmark and zip64
+    # below are tiers too.
+    "units: offline harness tests; no platform, no SDK builds (implied by *_units.py)",
+    "harness: harness self-tests against a running platform",
+    "core: cross-SDK tests against the platform's own KAS",
+    "legacy: golden TDF decryption; needs the golden keys loaded",
+    "multikas: needs the alpha..delta and km1..km3 KAS instances",
     "benchmark: paired A/B performance cell; only collected under --bench",
     "no_audit_logs: opt this test out of the default audit-log assertions",
     "zip64: multi-GiB ZIP64 boundary cell; only collected when --sizes reaches 2**31",

@@ -21,6 +21,8 @@
 from otdfctl import OpentdfCommandLineTool
 from test_policytypes import skip_rts_as_needed
 
+pytestmark = pytest.mark.multikas
+
 rewrap_403_pattern = (
     "tdf: rewrap request 403|403 for \\[https?://[^\\]]+\\]; rewrap permission denied"
 )

@@ -20,6 +20,8 @@
     ParsedAuditEvent,  # noqa: F401 - Used in TestParsedAuditEvent class
 )
 
+pytestmark = pytest.mark.units
+
 
 class TestLogEntry:
     """Tests for the LogEntry class."""

@@ -29,6 +29,8 @@
 from audit_logs import AuditLogAsserter
 from otdfctl import OpentdfCommandLineTool
 
+pytestmark = pytest.mark.multikas
+
 logger = logging.getLogger(__name__)
 
 
@@ -408,8 +410,13 @@ def tamper_policy_binding(manifest: tdfs.Manifest) -> tdfs.Manifest:
         except subprocess.CalledProcessError:
             pass  # Expected
 
-        # Verify rewrap error was audited
-        audit_logs.assert_rewrap_error(min_count=1, since_mark=mark)
+        # Verify rewrap error was audited. Only this part is expected to fail
+        # (opentdf/tests#709): the decrypt rejection above must stay blocking.
+        try:
+            audit_logs.assert_rewrap_error(min_count=1, since_mark=mark)
+        except AssertionError:
+            pytest.xfail("no rewrap error audit event; see opentdf/tests#709")
+        pytest.fail("rewrap error now audited: drop the xfail (opentdf/tests#709)")
 
     @pytest.mark.slow
     def test_audit_under_sequential_load(

@@ -33,6 +33,8 @@
 )
 from perf.runner import BenchConfig
 
+pytestmark = pytest.mark.units
+
 
 def install(root: Path, sdk: str, *versions: str) -> None:
     """Lay down a stub build tree, as ``otdf-sdk-mgr install`` would."""

@@ -31,6 +31,8 @@
     validate_sdk_name,
 )
 
+pytestmark = pytest.mark.units
+
 
 class TestArmRequest:
     def test_str_with_spec(self):

@@ -19,10 +19,13 @@
 from perf import measure
 from perf.measure import MeasurementError, Sample
 
-pytestmark = pytest.mark.skipif(
-    not hasattr(os, "wait4"),
-    reason="per-process rusage requires os.wait4",
-)
+pytestmark = [
+    pytest.mark.units,
+    pytest.mark.skipif(
+        not hasattr(os, "wait4"),
+        reason="per-process rusage requires os.wait4",
+    ),
+]
 
 
 def python_child(body: str) -> list[str]:

@@ -11,6 +11,8 @@
 from perf.report import _p_cell
 from perf.stats import Verdict
 
+pytestmark = pytest.mark.units
+
 
 def _comparison(
     ratio: float,

@@ -30,6 +30,8 @@
     run_cell,
 )
 
+pytestmark = pytest.mark.units
+
 BASELINE_WALL_S = 1.0
 BASELINE_RSS = 100_000_000
 BASELINE_CPU = 0.8

@@ -19,6 +19,8 @@
 from perf import stats
 from perf.stats import Verdict
 
+pytestmark = pytest.mark.units
+
 # Typical CI-runner dispersion for a CLI invocation: roughly +/-8% round to
 # round. Large enough to be realistic, small enough that 30 rounds can resolve
 # a 15% effect.

@@ -36,6 +36,8 @@
 from audit_logs import AuditLogAsserter
 from fixtures.encryption import EncryptFactory
 
+pytestmark = pytest.mark.multikas
+
 
 def _token_endpoint() -> str:
     if endpoint := os.getenv("TOKENENDPOINT"):

@@ -5,6 +5,8 @@
 
 import tdfs
 
+pytestmark = pytest.mark.legacy
+
 
 def get_golden_file(golden_file_name: str) -> Path:
     xtest_dir = Path(__file__).parent

@@ -9,6 +9,8 @@
 from abac import Attribute
 from fixtures.encryption import EncryptFactory
 
+pytestmark = pytest.mark.core
+
 
 def skip_rts_as_needed(
     encrypt_sdk: tdfs.SDK,

@@ -16,6 +16,8 @@
 from fixtures.encryption import EncryptFactory
 from tdfs import KeyAccessObject
 
+pytestmark = pytest.mark.multikas
+
 # Rewrap session-key algorithms, as passed to SDK.decrypt(session_key_algorithm=...)
 # / the CLI wrappers, mapped to the features that must be present to try them.
 # Every case also needs the platform's "audit-session-key-type". RSA is every SDK's

@@ -42,6 +42,8 @@
 from abac import Attribute
 from fixtures.encryption import EncryptFactory
 
+pytestmark = pytest.mark.core
+
 ManifestChange = Callable[[tdfs.Manifest], tdfs.Manifest]
 PayloadChange = Callable[[bytes], bytes]
 

@@ -17,6 +17,8 @@
 
 import tdfs
 
+pytestmark = pytest.mark.units
+
 
 @pytest.fixture
 def sdk(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tdfs.SDK:

@@ -1,10 +1,14 @@
 import random
 import string
 
+import pytest
+
 import abac
 from audit_logs import AuditLogAsserter
 from otdfctl import OpentdfCommandLineTool
 
+pytestmark = pytest.mark.harness
+
 otdfctl = OpentdfCommandLineTool()
 
 

@@ -10,6 +10,8 @@
 from fixtures.encryption import EncryptFactory
 from test_tdfs import assert_tamper_error, looks_like_422
 
+pytestmark = pytest.mark.core
+
 #### MANIFEST VALIDITY TESTS
 
 

@@ -13,6 +13,8 @@
 from audit_logs import AuditLogAsserter
 from fixtures.encryption import EncryptFactory
 
+pytestmark = pytest.mark.core
+
 dspx1153Fails = []
 
 try:

@@ -3,7 +3,7 @@
 Every test here is marked ``zip64`` and is deselected unless the session asks
 for a payload size that can reach ``2**31`` -- see the size table in
 ``sizes.py`` for why 2.1 GiB and not something rounder, and
-``pytest_collection_modifyitems`` in ``conftest.py`` for the deselection.
+``_drop_unrequested`` in ``conftest.py`` for the deselection.
 
 Run it with::