From a0806f614be271b5eae3b7d0cd57b97374d5432f Mon Sep 17 00:00:00 2001 From: Jay Qi <2721979+jayqi@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:16:22 -0500 Subject: [PATCH 1/2] Report unit and integration test coverage separately Assisted-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 64 ++++++++++++++++++++++++++++++++++-- CONTRIBUTING.md | 10 ++++++ integration_tests/pytest.ini | 1 + pyproject.toml | 3 +- 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3da260c..69cd063 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,13 @@ jobs: PYTHON: ${{ matrix.python }} LOWEST_DIRECT: ${{ matrix.lowest_direct && 'true' || '' }} UPGRADE: ${{ (github.event_name == 'schedule' || inputs.upgrade) && 'true' || '' }} + - name: Upload unit test coverage data + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-data-unit-py${{ matrix.python }}${{ matrix.lowest_direct && '-lowest-direct' || '' }} + path: .coverage + include-hidden-files: true + if-no-files-found: error - run: just typecheck --python="$PYTHON" ${LOWEST_DIRECT:+--lowest-direct} if: ${{ !cancelled() }} env: @@ -120,11 +127,64 @@ jobs: timeout-minutes: 5 env: UPGRADE: ${{ (github.event_name == 'schedule' || inputs.upgrade) && 'true' || '' }} + - name: Upload integration test coverage data + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-data-integration + path: .coverage + include-hidden-files: true + if-no-files-found: error + + coverage: + name: Coverage + needs: [test, integration] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + cache-suffix: coverage + - name: Download coverage data + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: coverage-data-* + - name: Report unit and integration coverage separately + run: | + coverage() { uv run --no-dev --group tests -- coverage "$@"; } + for suite in unit integration; do + coverage combine --data-file=".coverage.$suite" coverage-data-"$suite"*/.coverage + coverage report --data-file=".coverage.$suite" --show-missing + coverage html --data-file=".coverage.$suite" --directory="htmlcov/$suite" --title="${suite^} test coverage" + coverage json --data-file=".coverage.$suite" -o "coverage-json/$suite.json" + { echo "## ${suite^} test coverage"; coverage report --data-file=".coverage.$suite" --format=markdown; } >> "$GITHUB_STEP_SUMMARY" + done + - name: Upload HTML coverage reports + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-html + path: htmlcov/ + - name: Upload JSON coverage reports + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-json + path: coverage-json/ + - name: Merge coverage data into one artifact + uses: actions/upload-artifact/merge@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-data + pattern: coverage-data-* + separate-directories: true + include-hidden-files: true + delete-merged: true all-green: name: All green if: always() - needs: [lint, zizmor, test, integration] + needs: [lint, zizmor, test, integration, coverage] runs-on: ubuntu-latest permissions: {} steps: @@ -134,7 +194,7 @@ jobs: notify: name: Notify failed build - needs: [lint, zizmor, test, integration] + needs: [lint, zizmor, test, integration, coverage] if: failure() && (github.event_name == 'push' || github.event_name == 'schedule') runs-on: ubuntu-latest permissions: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6f2a94e..097c6d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,16 @@ The integration tests read their settings from environment variables (`PDC_API_U The unit tests mock every HTTP call, so they can't catch a change in PDC's behavior. The integration tests can, and CI runs both on every pull request. +Both suites print a coverage report. CI reports each suite's coverage separately in the run summary, with unit coverage combined across the Python versions it tests. Each run uploads three artifacts: + +- `coverage-html`: an HTML report for each suite. +- `coverage-json`: `unit.json` and `integration.json`, from `coverage json`. +- `coverage-data`: the raw coverage data from each test job, one directory per job. + +```bash +gh run download -n coverage-json +``` + ## Updating to a new PDC spec ```bash diff --git a/integration_tests/pytest.ini b/integration_tests/pytest.ini index 706b0ff..e524d3c 100644 --- a/integration_tests/pytest.ini +++ b/integration_tests/pytest.ini @@ -1,3 +1,4 @@ [pytest] # A separate config so the unit suite's global `responses` interception does not apply. testpaths = integration_tests +addopts = --cov=pdc_api_client --cov-report=term-missing diff --git a/pyproject.toml b/pyproject.toml index ce44fa1..759bed6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ source-include = [ wheel-exclude = ["pdc_api_client/openapi.json"] [tool.coverage.run] +branch = true relative_files = true [tool.coverage.paths] @@ -78,7 +79,7 @@ pdc_api_client = ["src/pdc_api_client/", "*/site-packages/pdc_api_client/"] [tool.pytest.ini_options] testpaths = ["tests"] -addopts = ["--cov=pdc_api_client", "--cov-report=term-missing", "--cov-branch"] +addopts = ["--cov=pdc_api_client", "--cov-report=term-missing"] [tool.mypy] files = ["src", "tests", "integration_tests"] From 6000e4c610043808639ce4db322b4981f4a331b7 Mon Sep 17 00:00:00 2001 From: Jay Qi <2721979+jayqi@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:52:35 -0500 Subject: [PATCH 2/2] Run integration tests with the pytest config in pyproject.toml Assisted-by: Claude Opus 5.5 --- integration_tests/pytest.ini | 4 ---- justfile | 2 +- pyproject.toml | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 integration_tests/pytest.ini diff --git a/integration_tests/pytest.ini b/integration_tests/pytest.ini deleted file mode 100644 index 2bd34d0..0000000 --- a/integration_tests/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -# A separate config so the unit suite's testpaths and coverage options in pyproject.toml do not apply. -testpaths = integration_tests -addopts = --cov=pdc_api_client --cov-report=term-missing diff --git a/justfile b/justfile index 3ce944b..0ec6bce 100644 --- a/justfile +++ b/justfile @@ -73,7 +73,7 @@ test python=python lowest_direct="false" upgrade="false" *args: (_pytest python [arg("python", long)] [arg("lowest_direct", long="lowest-direct", value="true")] [arg("upgrade", long, value="true")] -integration python=python lowest_direct="false" upgrade="false" *args: (_pytest python lowest_direct upgrade "-c integration_tests/pytest.ini integration_tests/ " + args) +integration python=python lowest_direct="false" upgrade="false" *args: (_pytest python lowest_direct upgrade "integration_tests/ " + args) _typecheck python lowest_direct tool *args: uv run --python={{python}} {{ if lowest_direct == "true" { "--resolution=lowest-direct" } else { "" } }} --isolated --no-dev --group tests --group typecheck -- \ diff --git a/pyproject.toml b/pyproject.toml index 759bed6..d217d40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,6 @@ source-include = [ "CHANGELOG.md", "codegen/*.py", "integration_tests/*.py", - "integration_tests/pytest.ini", "just/*.just", "justfile", "tests/*.py",