diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4145b0..91a2b8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,13 @@ jobs: PYTHON: ${{ matrix.python }} LOWEST_DIRECT: ${{ matrix.lowest_direct && 'true' || '' }} UPGRADE: ${{ (github.event_name == 'schedule' || inputs.upgrade) && 'true' || '' }} + - name: Upload unit test coverage data + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-data-unit-py${{ matrix.python }}${{ matrix.lowest_direct && '-lowest-direct' || '' }} + path: .coverage + include-hidden-files: true + if-no-files-found: error - name: Run the type checks run: just typecheck --python="$PYTHON" ${LOWEST_DIRECT:+--lowest-direct} if: ${{ !cancelled() }} @@ -137,11 +144,66 @@ jobs: timeout-minutes: 5 env: UPGRADE: ${{ (github.event_name == 'schedule' || inputs.upgrade) && 'true' || '' }} + - name: Upload integration test coverage data + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-data-integration + path: .coverage + include-hidden-files: true + if-no-files-found: error + + coverage: + name: Coverage + needs: [unit-tests-and-type-checks, integration] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + cache-suffix: coverage + - name: Download coverage data + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: coverage-data-* + - name: Report unit and integration coverage separately + run: | + coverage() { uv run --no-dev --group tests -- coverage "$@"; } + for suite in unit integration; do + coverage combine --data-file=".coverage.$suite" coverage-data-"$suite"*/.coverage + coverage report --data-file=".coverage.$suite" --show-missing + coverage html --data-file=".coverage.$suite" --directory="htmlcov/$suite" --title="${suite^} test coverage" + coverage json --data-file=".coverage.$suite" -o "coverage-json/$suite.json" + { echo "## ${suite^} test coverage"; coverage report --data-file=".coverage.$suite" --format=markdown; } >> "$GITHUB_STEP_SUMMARY" + done + - name: Upload HTML coverage reports + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-html + path: htmlcov/ + - name: Upload JSON coverage reports + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-json + path: coverage-json/ + - name: Merge coverage data into one artifact + uses: actions/upload-artifact/merge@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-data + pattern: coverage-data-* + separate-directories: true + include-hidden-files: true + delete-merged: true all-green: name: All green if: always() - needs: [lint, zizmor, unit-tests-and-type-checks, integration] + needs: [lint, zizmor, unit-tests-and-type-checks, integration, coverage] runs-on: ubuntu-latest permissions: {} steps: @@ -152,7 +214,7 @@ jobs: notify: name: Notify failed build - needs: [lint, zizmor, unit-tests-and-type-checks, integration] + needs: [lint, zizmor, unit-tests-and-type-checks, integration, coverage] if: failure() && (github.event_name == 'push' || github.event_name == 'schedule') runs-on: ubuntu-latest permissions: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f79b7c3..6130110 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,16 @@ The integration tests read their settings from environment variables (`PDC_API_U The unit tests mock every HTTP call, so they can't catch a change in PDC's behavior. The integration tests can, and CI runs both on every pull request. +Both suites print a coverage report. CI reports each suite's coverage separately in the run summary, with unit coverage combined across the Python versions it tests. Each run uploads three artifacts: + +- `coverage-html`: an HTML report for each suite. +- `coverage-json`: `unit.json` and `integration.json`, from `coverage json`. +- `coverage-data`: the raw coverage data from each test job, one directory per job. + +```bash +gh run download -n coverage-json +``` + ## Updating to a new PDC spec ```bash diff --git a/integration_tests/pytest.ini b/integration_tests/pytest.ini deleted file mode 100644 index 6443df0..0000000 --- a/integration_tests/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -# A separate config so the unit suite's testpaths and coverage options in pyproject.toml do not apply. -testpaths = integration_tests diff --git a/justfile b/justfile index 3ce944b..0ec6bce 100644 --- a/justfile +++ b/justfile @@ -73,7 +73,7 @@ test python=python lowest_direct="false" upgrade="false" *args: (_pytest python [arg("python", long)] [arg("lowest_direct", long="lowest-direct", value="true")] [arg("upgrade", long, value="true")] -integration python=python lowest_direct="false" upgrade="false" *args: (_pytest python lowest_direct upgrade "-c integration_tests/pytest.ini integration_tests/ " + args) +integration python=python lowest_direct="false" upgrade="false" *args: (_pytest python lowest_direct upgrade "integration_tests/ " + args) _typecheck python lowest_direct tool *args: uv run --python={{python}} {{ if lowest_direct == "true" { "--resolution=lowest-direct" } else { "" } }} --isolated --no-dev --group tests --group typecheck -- \ diff --git a/pyproject.toml b/pyproject.toml index ce44fa1..d217d40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,6 @@ source-include = [ "CHANGELOG.md", "codegen/*.py", "integration_tests/*.py", - "integration_tests/pytest.ini", "just/*.just", "justfile", "tests/*.py", @@ -71,6 +70,7 @@ source-include = [ wheel-exclude = ["pdc_api_client/openapi.json"] [tool.coverage.run] +branch = true relative_files = true [tool.coverage.paths] @@ -78,7 +78,7 @@ pdc_api_client = ["src/pdc_api_client/", "*/site-packages/pdc_api_client/"] [tool.pytest.ini_options] testpaths = ["tests"] -addopts = ["--cov=pdc_api_client", "--cov-report=term-missing", "--cov-branch"] +addopts = ["--cov=pdc_api_client", "--cov-report=term-missing"] [tool.mypy] files = ["src", "tests", "integration_tests"]