diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b43284b7..50aac042e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,33 +1,83 @@ name: CI +# The required check is the Verify job, which passes only when every job the change needs passed. +# A pull request runs what its changed files need (scripts/replay_scope decides): +# - documentation and website changes build and check the site only; +# - code changes run check-ci, the Rust and Python tests, and the replay of the recorded cases, +# which reuses main's judgements for every case whose conversion and recording are unchanged; +# - a change under a full-replay path replays every case. +# The nightly run on main replays every case, refreshes the replay cache pull requests reuse, +# converts DuckShop within its time budget and fuzzes. Release Please dispatches a full run on +# each release pull request's branch, so every release replays every case first. on: pull_request: - push: - branches: [main] + schedule: + - cron: "17 3 * * *" workflow_dispatch: permissions: contents: read concurrency: - group: ci-${{ github.ref }} + group: ci-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + jobs: - verify: - name: Verify + changes: + name: Classify the change + runs-on: ubuntu-24.04 + outputs: + replay: ${{ steps.scope.outputs.replay }} + code: ${{ steps.scope.outputs.code }} + site: ${{ steps.scope.outputs.site }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # The repository's scripts need Python 3.12. + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + # Every pull request, documentation only or not, keeps agent files and private paths out. + - name: Run the clutter guard + run: python3 -m scripts.check_clutter_guard + - name: Select the checks the change needs + id: scope + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [[ "${EVENT_NAME}" == "pull_request" ]]; then + python3 -m scripts.select_replay scope --base "${BASE_SHA}" >> "${GITHUB_OUTPUT}" + else + printf 'replay=full\ncode=true\nsite=true\n' >> "${GITHUB_OUTPUT}" + fi + + site: + name: Website + needs: changes + if: needs.changes.outputs.site == 'true' + uses: ./.github/workflows/website-build.yml + + checks: + name: Checks, tests and replay + needs: changes + if: needs.changes.outputs.code == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 20 + # A cold build of the workspace takes most of this; the cached dependency builds cut it. + timeout-minutes: 120 env: - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: "0" # The Snowflake recorder is never built or run in CI. RECORDER: "0" # `uv run` would otherwise sync the default groups, recorder included. UV_NO_SYNC: "1" - # The evidence pin check and the result baselines compare with the pull request base or the - # previous main commit. - EVIDENCE_BASE: ${{ github.event.pull_request.base.sha || github.event.before || 'origin/main' }} + # The evidence pin check and the result baselines compare with the pull request's base. + EVIDENCE_BASE: ${{ github.event.pull_request.base.sha || 'origin/main' }} + REPLAY: ${{ needs.changes.outputs.replay }} steps: - uses: actions/checkout@v4 with: @@ -39,10 +89,21 @@ jobs: cache-dependency-glob: uv.lock - name: Install the pinned Rust toolchain run: rustup toolchain install - # Caches the dependency builds, polyglot above all, keyed by Cargo.lock and the toolchain. + # The gate build and its tests outgrow the runner's free disk with the preinstalled SDKs. + - name: Free disk space + run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + # The dependency builds, keyed by Cargo.lock and the toolchain; pull requests only read them. - uses: Swatinem/rust-cache@v2 with: - shared-key: verify + shared-key: gate + save-if: ${{ github.event_name != 'pull_request' }} + - name: Download the DuckDB library again if the restored cache lacks it + run: scripts/refetch_duckdb_library.sh + # The evidence snapshot evidence.toml pins; scripts/evidence.sh verifies it after a restore. + - uses: actions/cache@v4 + with: + path: .evidence/pinned + key: evidence-${{ hashFiles('evidence.toml') }} - name: Install Python run: uv python install 3.12 - name: Install development tools without the recorder @@ -50,34 +111,71 @@ jobs: # The instruction-count guard in check-ci runs its workloads under callgrind. - name: Install valgrind run: sudo apt-get update -q && sudo apt-get install -y -q --no-install-recommends valgrind - - name: Run static checks - run: make check-ci - - name: Run tests - run: make test - # `make verify-recorded`, with the same verify flags as `make gate`, then the measure and SQL - # size checks. + # One gate build serves every step: check-ci, the Rust tests, the Python tests and the replay. + - name: Run static checks and tests + run: make check-merge test-python + - name: Find the replay cache's scope + id: replay + if: env.REPLAY != 'none' + run: echo "scope=$(uv run python -m scripts.select_replay digest)" >> "${GITHUB_OUTPUT}" + # Judgements made under the same full-replay paths, most recently by main's nightly run. + - name: Restore the replay cache + if: env.REPLAY == 'changed' + uses: actions/cache/restore@v4 + with: + path: target/replay-cache.json + key: replay-${{ steps.replay.outputs.scope }}-${{ github.run_id }} + restore-keys: replay-${{ steps.replay.outputs.scope }}- - name: Replay recorded translations on DuckDB and check their baselines - run: >- - make check-convert DIALECT=snowflake VERIFY_PROFILE=dev RESULTS_BASE="$EVIDENCE_BASE" + if: env.REPLAY != 'none' + run: make verify-recorded RESULTS_BASE="${EVIDENCE_BASE}" REPLAY_CACHE=target/replay-cache.json + - name: Report the replay + if: env.REPLAY == 'none' + run: echo "No changed file is in a crate the sqlquack binary is built from, so nothing replays." + # Pull requests restore the cache saved on their base branch, main's from the nightly run. + - name: Save the replay cache + if: env.REPLAY != 'none' && github.event_name != 'pull_request' + uses: actions/cache/save@v4 + with: + path: target/replay-cache.json + key: replay-${{ steps.replay.outputs.scope }}-${{ github.run_id }} + + verify: + name: Verify + if: always() + needs: [changes, site, checks] + runs-on: ubuntu-24.04 + steps: + - name: Confirm every job the change needs passed + env: + CHANGES: ${{ needs.changes.result }} + SITE: ${{ needs.site.result }} + SITE_NEEDED: ${{ needs.changes.outputs.site }} + CHECKS: ${{ needs.checks.result }} + CHECKS_NEEDED: ${{ needs.changes.outputs.code }} + run: | + failed=0 + expect() { + local name=$1 result=$2 needed=$3 + if [[ "${result}" == "success" || ( "${needed}" != "true" && "${result}" == "skipped" ) ]]; then + echo "${name}: ${result}" + else + echo "${name}: ${result}, but the change needs it" >&2 + failed=1 + fi + } + expect "Classify the change" "${CHANGES}" true + expect "Website" "${SITE}" "${SITE_NEEDED}" + expect "Checks, tests and replay" "${CHECKS}" "${CHECKS_NEEDED}" + exit "${failed}" - polyglot-check: - name: Exact parser check on DuckShop + duckshop: + name: Convert DuckShop within the time budget + if: github.event_name != 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 30 - env: - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: "0" - # The DuckShop benchmarks and the SQLBuild release that compiles them, pinned together. - SQLBUILD_REF: a7bc60e0abee5bb19d67e5bdc948e048a1df82e0 - SQLBUILD_VERSION: 0.187.2 steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: sqlstack-io/sqlbuild - ref: ${{ env.SQLBUILD_REF }} - path: sqlbuild - sparse-checkout: benchmarks - uses: astral-sh/setup-uv@v5 with: version: "0.12.3" @@ -85,31 +183,21 @@ jobs: run: rustup toolchain install - uses: Swatinem/rust-cache@v2 with: - shared-key: verify - save-if: false + shared-key: duckshop + save-if: ${{ github.event_name != 'pull_request' }} + - name: Download the DuckDB library again if the restored cache lacks it + run: scripts/refetch_duckdb_library.sh - name: Compile DuckShop and DuckShop 10K - run: | - uv venv -p 3.12 "$RUNNER_TEMP/sqlbuild" - VIRTUAL_ENV="$RUNNER_TEMP/sqlbuild" uv pip install "sqlbuild==$SQLBUILD_VERSION" - python3 sqlbuild/benchmarks/duckshop-10k/generate.py --out "$RUNNER_TEMP/duckshop-10k" - (cd sqlbuild/benchmarks/duckshop/project && "$RUNNER_TEMP/sqlbuild/bin/sqb" compile) - (cd "$RUNNER_TEMP/duckshop-10k" && "$RUNNER_TEMP/sqlbuild/bin/sqb" compile) - - name: Compare with polyglot - run: | - cargo run --locked --release -p sqlquack-polyglot-check -- \ - directory sqlbuild/benchmarks/duckshop/project/target/compiled --dialect snowflake - cargo run --locked --release -p sqlquack-polyglot-check -- \ - directory "$RUNNER_TEMP/duckshop-10k/target/compiled" --dialect snowflake - - name: Convert DuckShop within the time budget - run: | - make census-duckshop DUCKSHOP_COMPILED="sqlbuild/benchmarks/duckshop/project/target/compiled $RUNNER_TEMP/duckshop-10k/target/compiled" + run: make duckshop-compiled + - name: Convert them + run: make census-duckshop fuzz: name: Fuzz tolerant parsing + if: github.event_name != 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 15 env: - CARGO_TERM_COLOR: always # Seconds of fuzzing after the build. FUZZ_SECONDS: "90" steps: diff --git a/.github/workflows/distributions.yml b/.github/workflows/distributions.yml index 8ae861b68..d4a8d3cb7 100644 --- a/.github/workflows/distributions.yml +++ b/.github/workflows/distributions.yml @@ -1,9 +1,9 @@ name: Distributions -# Builds and smoke tests every wheel and sdist when a pull request changes what goes into them, and -# on every push to main the linux-x86_64 wheels only, so a distribution-only failure shows at the -# landing rather than at the next release. Both also run the command line's convert tests with -# DuckDB compiled in. Neither blocks a landing. +# Builds and smoke tests the linux-x86_64 wheels when a pull request changes what goes into them, +# and on every push to main, so a distribution-only failure shows at the landing rather than at the +# next release. Both also run the command line's convert tests with DuckDB compiled in. Every +# platform and the sdists build nightly and for each release. Neither blocks a landing. on: push: branches: [main] @@ -31,8 +31,8 @@ jobs: name: Distributions uses: ./.github/workflows/build-distributions.yml with: - platforms: ${{ github.event_name == 'push' && 'linux-x86_64' || '' }} - sdist: ${{ github.event_name != 'push' }} + platforms: ${{ github.event_name == 'workflow_dispatch' && '' || 'linux-x86_64' }} + sdist: ${{ github.event_name == 'workflow_dispatch' }} # The command line built with `bundled` alone, as the sqlquack wheel compiles DuckDB in: no system # DuckDB. diff --git a/.github/workflows/pull-request-metadata.yml b/.github/workflows/pull-request-metadata.yml new file mode 100644 index 000000000..53a31ac6c --- /dev/null +++ b/.github/workflows/pull-request-metadata.yml @@ -0,0 +1,57 @@ +name: Pull Request Metadata +run-name: >- + ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) + || github.event.pull_request.title || github.event.head_commit.message + || github.event.merge_group.head_commit.message || github.workflow }} + +on: + pull_request: + types: [opened, edited, synchronize, reopened] + merge_group: + workflow_dispatch: + inputs: + pr_number: + description: Pull request number to validate + required: true + type: number + request_id: + description: "Release Please attempt that dispatched this run; leave empty for manual runs" + required: false + type: string + +permissions: + contents: read + pull-requests: read + +jobs: + validate: + name: Pull Request Metadata + runs-on: ubuntu-24.04 + steps: + # The queue only admits pull requests whose metadata check already passed, so a merge group + # reports success instead of skipping: a required check must report for the queue to merge. + - name: Accept metadata validated on the pull request + if: github.event_name == 'merge_group' + run: echo "Pull request metadata was validated on the pull request before it was queued." + + - uses: actions/checkout@v4 + if: github.event_name != 'merge_group' + - name: Resolve pull request metadata + if: github.event_name != 'merge_group' + env: + EVENT_NAME: ${{ github.event_name }} + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + if [[ "${EVENT_NAME}" == "pull_request" ]]; then + cp "${GITHUB_EVENT_PATH}" "${RUNNER_TEMP}/pull-request.json" + else + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" \ + > "${RUNNER_TEMP}/pull-request.json" + fi + + - name: Validate branch, title, and description + if: github.event_name != 'merge_group' + env: + PR_METADATA_PATH: ${{ runner.temp }}/pull-request.json + run: python -m scripts.validate_pr_metadata --metadata-file "${PR_METADATA_PATH}" diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index ac9e4b6b8..0416ae7e2 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -1,8 +1,8 @@ name: Release Please # On every push to main, opens or updates the release pull request from the Conventional Commits -# since the last release and dispatches Release Distributions to build its distributions. Once the -# release pull request merges, it tags the release and dispatches Publish for that tag. +# since the last release, and dispatches its required checks and Release Distributions on its +# branch. Once the release pull request merges, it tags the release and dispatches Publish for it. on: push: branches: @@ -37,16 +37,23 @@ jobs: gh workflow run publish.yml --repo "${GITHUB_REPOSITORY}" --ref main \ -f "tag=${RELEASE_TAG}" - - name: Build the release pull request's distributions + # A pull request this workflow opens starts no pull_request runs, so the release pull + # request's required checks are dispatched on its branch: CI replays every recorded case + # before the release, alongside the distribution build. + - name: Check the release pull request and build its distributions if: steps.release.outputs.prs_created == 'true' env: GH_TOKEN: ${{ github.token }} RELEASE_PRS: ${{ steps.release.outputs.prs }} run: | release_branch="$(jq -r '.[0].headBranchName // empty' <<< "${RELEASE_PRS}")" - if [[ -z "${release_branch}" ]]; then - echo "Release Please reported a created pull request without a head branch" >&2 + release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")" + if [[ -z "${release_branch}" || -z "${release_pr}" ]]; then + echo "Release Please reported a created pull request without a head branch or number" >&2 exit 1 fi - gh workflow run release-distributions.yml --repo "${GITHUB_REPOSITORY}" \ - --ref "${release_branch}" + run() { gh workflow run "$1" --repo "${GITHUB_REPOSITORY}" --ref "${release_branch}" "${@:2}"; } + run release-distributions.yml + run ci.yml + run version-guard.yml + run pull-request-metadata.yml -f "pr_number=${release_pr}" diff --git a/.github/workflows/version-guard.yml b/.github/workflows/version-guard.yml new file mode 100644 index 000000000..c5fa2a09e --- /dev/null +++ b/.github/workflows/version-guard.yml @@ -0,0 +1,59 @@ +name: Version Guard +run-name: >- + ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) + || github.event.pull_request.title || github.event.head_commit.message + || github.event.merge_group.head_commit.message || github.workflow }} + +on: + pull_request: + branches: + - main + push: + branches: + - main + # Checks out the merge group's head, so the version that would land on main is the one checked. + merge_group: + workflow_dispatch: + inputs: + request_id: + description: "Release Please attempt that dispatched this run; leave empty for manual runs" + required: false + type: string + +permissions: + contents: read + +jobs: + block-1-0: + name: Block automatic 1.0 releases + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Reject versions at or above 1.0.0 + run: | + python - <<'PY' + import sys + import tomllib + + # Release Please bumps the workspace version; the Python package takes it from Cargo. + with open("Cargo.toml", "rb") as manifest: + version = tomllib.load(manifest)["workspace"]["package"]["version"] + + parts = version.split(".") + try: + major = int(parts[0]) + except (IndexError, ValueError): + print(f"Invalid project version: {version}", file=sys.stderr) + sys.exit(1) + + if major >= 1: + print( + f"Refusing automatic release version {version}. " + "Crossing to 1.0.0 requires an explicit manual decision.", + file=sys.stderr, + ) + sys.exit(1) + + print(f"Project version {version} is below 1.0.0.") + PY diff --git a/.github/workflows/website-build.yml b/.github/workflows/website-build.yml new file mode 100644 index 000000000..bffac99b9 --- /dev/null +++ b/.github/workflows/website-build.yml @@ -0,0 +1,78 @@ +name: Website Build + +# Builds the playground engine and the site and checks them: the generated references, the engine's +# lint and tests, the build and its links. Website deploys what it uploads; CI runs it on pull +# requests that touch the site or the crates the engine is built from. +on: + workflow_call: + inputs: + upload: + description: Upload the built site for GitHub Pages + type: boolean + default: false + +permissions: + contents: read + +jobs: + build: + name: Build the engine and the website + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: website + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: website/package-lock.json + + - name: Install the pinned Rust toolchain with the wasm32 target + working-directory: . + run: | + rustup toolchain install + rustup target add wasm32-unknown-unknown + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: website/wasm + + - uses: taiki-e/install-action@v2 + with: + tool: wasm-bindgen-cli@0.2.130 + + - name: Check the diagnostics reference matches the registry + run: python3 scripts/generate_diagnostics.py --check + + - name: Check the known differences table matches the entries + run: python3 scripts/generate_known_differences.py --check + + - name: Lint the engine + working-directory: website/wasm + run: | + cargo fmt --check + cargo clippy --locked --target wasm32-unknown-unknown --release -- -D warnings + + - name: Install dependencies + run: npm ci + + - name: Build the engine + run: ./scripts/build_engine.sh + + - name: Test the engine and its spans + run: npm run -s test:engine + + - name: Build + run: npm run build + + - name: Check links + run: python3 scripts/check_links.py dist + + - name: Upload site + if: inputs.upload + uses: actions/upload-pages-artifact@v3 + with: + path: website/dist diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index c7d86538c..a3d2a7e78 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -1,9 +1,7 @@ name: Website -# Builds the playground engine and the site, checks them, and deploys sqlquack.io to GitHub Pages -# from main. While the repository is private, only pull requests and manual runs build, and nothing -# deploys. The site goes live from the `public` event when the repository is made public, or from a -# manual run on main once Pages is enabled. +# Builds and checks the playground engine and the site with Website Build, and deploys sqlquack.io to +# GitHub Pages from main. Pull requests build the site in CI. on: public: push: @@ -20,19 +18,6 @@ on: - crates/sqlquack-syntax/** - crates/sqlquack-syntax-snowflake/** - .github/workflows/website.yml - pull_request: - paths: - - website/** - - crates/sqlquack-diagnostics/** - - crates/sqlquack-parse/** - - crates/sqlquack-blocks/** - - crates/sqlquack-convert/** - - crates/sqlquack-derive/** - - crates/sqlquack-policy/** - - crates/sqlquack-semantics/** - - crates/sqlquack-syntax/** - - crates/sqlquack-syntax-snowflake/** - - .github/workflows/website.yml workflow_dispatch: permissions: @@ -44,67 +29,11 @@ concurrency: jobs: build: - name: Build the engine and the website - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || !github.event.repository.private - runs-on: ubuntu-24.04 - defaults: - run: - working-directory: website - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: npm - cache-dependency-path: website/package-lock.json - - - name: Install the pinned Rust toolchain with the wasm32 target - working-directory: . - run: | - rustup toolchain install - rustup target add wasm32-unknown-unknown - - - uses: Swatinem/rust-cache@v2 - with: - workspaces: website/wasm - - - uses: taiki-e/install-action@v2 - with: - tool: wasm-bindgen-cli@0.2.130 - - - name: Check the diagnostics reference matches the registry - run: python3 scripts/generate_diagnostics.py --check - - - name: Check the known differences table matches the entries - run: python3 scripts/generate_known_differences.py --check - - - name: Lint the engine - working-directory: website/wasm - run: | - cargo fmt --check - cargo clippy --locked --target wasm32-unknown-unknown --release -- -D warnings - - - name: Install dependencies - run: npm ci - - - name: Build the engine - run: ./scripts/build_engine.sh - - - name: Test the engine and its spans - run: npm run -s test:engine - - - name: Build - run: npm run build - - - name: Check links - run: python3 scripts/check_links.py dist - - - name: Upload site - if: github.event_name != 'pull_request' - uses: actions/upload-pages-artifact@v3 - with: - path: website/dist + name: Build + if: github.event_name == 'workflow_dispatch' || !github.event.repository.private + uses: ./.github/workflows/website-build.yml + with: + upload: true deploy: name: Deploy to GitHub Pages diff --git a/Cargo.lock b/Cargo.lock index e69f7098a..32afd0a52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1491,6 +1491,7 @@ version = "0.4.1" dependencies = [ "arrow", "duckdb", + "rayon", "serde", "serde_json", "sha2", diff --git a/Makefile b/Makefile index e5cce379d..650d15b4a 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS) PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS) endif -.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build polyglot-guard engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard polyglot-duckshop census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure +.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build polyglot-guard engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard polyglot-duckshop census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure format: cargo fmt --all @@ -220,7 +220,7 @@ referee-pending: uv run python -m scripts.write_referee_pending $(REFEREE_ARGS) # The static gate, the first step of `gate`, and the Python API's tests. -check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts evidence-pin-check clutter-guard polyglot-guard engine-free bundled-extensions dialect-builds generated-cases syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices ## The static gate CI runs. +check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts evidence-pin-check clutter-guard polyglot-guard engine-free bundled-extensions dialect-builds generated-cases syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs. # Each distribution ships the licence and third-party notices; its copies must match the root files. distribution-notices: @@ -251,6 +251,11 @@ test-rust: gate-build $(TEST_EVIDENCE) RUST_TEST_JOBS=$(RUST_TEST_JOBS) RUST_TEST_FEATURES=$(GATE_FEATURES) \ scripts/run_rust_tests.sh $(RUST_TEST_PROFILE) +# One crate's tests in the gate profile, for the edit-test loop; it builds only that crate's graph. +test-crate: ## One crate's tests: make test-crate CRATE=sqlquack-evidence. + @test -n "$(CRATE)" || { echo "test-crate: name the crate, as in make test-crate CRATE=sqlquack-evidence" >&2; exit 2; } + $(TEST_EVIDENCE) cargo test --locked --profile $(GATE_PROFILE) -p $(CRATE) + # What every merge to main must pass, on the exact head: the static gate and the Rust tests. It does # not replay the recorded cases; `make verify-recorded` does, and CI runs it on every pull request. check-merge: check-ci test-rust ## The merge gate: check-ci and the Rust tests; it replays no recordings. @@ -268,7 +273,7 @@ python-module: gate-build $(PYTHON_PACKAGE)/ # The API tests compare the module's reports with the command line's, both built with GATE_PROFILE. -python-api: python-module +python-api: python-module ## Build the Python module and run its API tests. SQLQUACK_BIN=$(abspath $(GATE_DIR)/sqlquack) \ LD_LIBRARY_PATH=$(abspath $(PYTHON_PACKAGE)) uv run pytest $(PYTHON_API_TESTS) -n auto --dist loadfile @@ -280,8 +285,8 @@ python-wheel: | .venv/bin/python # `make gate` both run verify-recorded, so these are the one set of verify flags. # Backup pushes may drop --require-verified; failed cases and error findings still block. VERIFY_FLAGS ?= --require-verified --strict-errors -# The cargo profile verify is built with: release locally, dev in CI, where the tests' debug build -# is already cached. +# The cargo profile `pending` and a narrowed check-convert build with; verify-recorded runs the gate +# build's binary. VERIFY_PROFILE ?= release # Verify reports evidence coverage (docs/coverage.md) and writes it to COVERAGE_REPORT as JSON. # REQUIRE_COVERAGE=1 fails on any gap; it stays 0 until the generated hazard cases are recorded. @@ -308,14 +313,35 @@ SQL_SIZE_BASELINE := coverage/sql-size/baseline.json SQL_SIZE_BASE_FILE := target/sql-size-base.json SQL_SIZE_ALLOWANCES := coverage/sql-size/allowances.toml -verify-recorded: ## Replay every recorded translation on DuckDB; check-convert DIALECT=snowflake runs it. +# With REPLAY_CACHE set, verify reuses the judgements a run left there for every case whose +# conversion and recording are unchanged, and replays the rest. The cache is scoped to a digest of +# the full-replay paths in scripts/replay_scope, so a change under them replays every case. +REPLAY_CACHE ?= +REPLAY_FLAGS = $(if $(REPLAY_CACHE),--replay-cache $(REPLAY_CACHE) --replay-scope $(shell uv run --no-sync python -m scripts.select_replay digest)) + +# verify runs the gate build's binary, with the DuckDB library python-module puts beside the module. +verify-recorded: python-module ## Replay every recorded translation on DuckDB; check-convert DIALECT=snowflake runs it. mkdir -p $(dir $(COVERAGE_REPORT)) - cargo run --profile $(VERIFY_PROFILE) --locked -p sqlquack -- verify $(VERIFY_FLAGS) $(COVERAGE_FLAGS) --python $(VERIFY_PYTHON) \ + LD_LIBRARY_PATH=$(abspath $(PYTHON_PACKAGE)) $(GATE_DIR)/sqlquack verify $(VERIFY_FLAGS) $(COVERAGE_FLAGS) \ + --python $(VERIFY_PYTHON) $(REPLAY_FLAGS) \ $(call convert_paths,snowflake) --audit-results $(call convert_dir,snowflake,repeats) \ --translation-statements $(TRANSLATION_STATEMENTS) --sql-sizes $(SQL_SIZES) $(MAKE) translation-results $(MAKE) sql-sizes +verify-changed: ## Replay only the recorded cases whose conversion changed since the last verify-changed. + $(MAKE) verify-recorded REPLAY_CACHE=$(or $(REPLAY_CACHE),target/replay-cache.json) + +# Validates a pull request's branch, title and description as the Pull Request Metadata check does. +check-pr-metadata: ## Check PR metadata: make check-pr-metadata PR_TITLE='fix: …' PR_BODY_FILE=body.md. + @test -n "$(PR_TITLE)" || { echo "PR_TITLE is required" >&2; exit 2; } + @test -n "$(PR_BODY_FILE)" || { echo "PR_BODY_FILE is required" >&2; exit 2; } + uv run python -m scripts.validate_pr_metadata --title "$(PR_TITLE)" --body-file "$(PR_BODY_FILE)" + +# Fails when a declared full-replay or replay-input path matches no tracked file. +replay-scope: + uv run python -m scripts.select_replay check + # The suites, one pattern for every dialect; README.md, "Running the suites", walks through them. # DIALECT is always named, so no dialect is the unnamed default. The check- targets replay committed # evidence offline and need no account; the record- targets need the dialect's engine or account, @@ -435,11 +461,11 @@ translation-files: gate-build $(GATE_DIR)/syntax-coverage results --files-only $(RESULTS_ROOTS) # The result-matching measure over the statements verify wrote; see verify-recorded. -translation-results: +translation-results: gate-build @if git show $(RESULTS_BASE):coverage/translation/results-baseline.json > $(RESULTS_BASE_FILE) 2>/dev/null; \ then against="--against $(RESULTS_BASE_FILE)"; \ else echo "translation-results: $(RESULTS_BASE) has no baseline; checking the committed one only"; against=; fi; \ - cargo run --locked -q -p sqlquack-coverage-check --bin syntax-coverage -- results \ + $(GATE_DIR)/syntax-coverage results \ $(TRANSLATION_STATEMENTS) $(RESULTS_ROOTS) --report $(TRANSLATION_REPORT) $$against # Compiles DuckShop and DuckShop 10K at SQLBUILD_REF with sqlbuild SQLBUILD_VERSION, as CI does. diff --git a/README.md b/README.md index ef72cb8c5..6abf25855 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,12 @@ make check-convert DIALECT=snowflake # replay recorded translations on DuckDB | `make record-syntax DIALECT=duckdb` | Record evidence for new syntax cases | | `make check-convert DIALECT=snowflake [CASES=temporal]` | Replay recorded translations on DuckDB (offline) | | `make record-convert DIALECT=snowflake` | Record new translation cases on Snowflake | +| `make verify-changed` | Replay only the cases whose conversion or recording changed since its last run | +| `make check-merge` | The static checks and the Rust tests every change must pass | +| `make test-crate CRATE=sqlquack-evidence` | One crate's tests | +| `make python-api` | Build the Python module and run its API tests | + +A pull request runs what its changed files need. Documentation and website changes build and check the site. Other changes run `make check-merge`, the Python tests and the replay. The replay reuses main's judgement for every case whose conversion and recording are unchanged, and replays only the rest. A change under one of the full-replay paths in `scripts/replay_scope/constants.py` replays every case: the DuckDB pin, the lock files, the replay, comparison and policy crates, the error kinds, CI's toolchain and the evidence pin. A change outside every crate the `sqlquack` binary is built from replays nothing. Recording needs the dialect's engine: DuckDB and SQLite record locally, while Snowflake, BigQuery and Databricks need a connection. Recording Snowflake syntax evidence isn't public yet. `make help` lists every target. diff --git a/crates/sqlquack-diagnostics/src/failures/models.rs b/crates/sqlquack-diagnostics/src/failures/models.rs index 7bbc26101..8eee33e2a 100644 --- a/crates/sqlquack-diagnostics/src/failures/models.rs +++ b/crates/sqlquack-diagnostics/src/failures/models.rs @@ -2,10 +2,10 @@ use std::fmt::{self, Display}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; /// What kind of failure an error is, the same for Snowflake and DuckDB. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ErrorKind { /// A number beyond its type's range. diff --git a/crates/sqlquack-evidence/Cargo.toml b/crates/sqlquack-evidence/Cargo.toml index b58f1b6cc..c5578643e 100644 --- a/crates/sqlquack-evidence/Cargo.toml +++ b/crates/sqlquack-evidence/Cargo.toml @@ -12,6 +12,7 @@ rust-version.workspace = true [dependencies] arrow.workspace = true duckdb.workspace = true +rayon.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/sqlquack-evidence/src/constants.rs b/crates/sqlquack-evidence/src/constants.rs index f8fd3e89c..b0b7184f0 100644 --- a/crates/sqlquack-evidence/src/constants.rs +++ b/crates/sqlquack-evidence/src/constants.rs @@ -121,3 +121,6 @@ pub const COMPILE_ERROR_SQLSTATE_CLASSES: &[&str] = &["0A", "42"]; /// The sidecar outcome of a recording that holds rows. pub(crate) const ROWS_OUTCOME: &str = "rows"; + +/// The version of the replay cache file; a file of another version is ignored. +pub const REPLAY_CACHE_FORMAT: &str = "sqlquack-replay-cache/1"; diff --git a/crates/sqlquack-evidence/src/verify/_helpers/judge.rs b/crates/sqlquack-evidence/src/verify/_helpers/judge.rs index ac4a3c593..d6fcd8bf0 100644 --- a/crates/sqlquack-evidence/src/verify/_helpers/judge.rs +++ b/crates/sqlquack-evidence/src/verify/_helpers/judge.rs @@ -21,9 +21,12 @@ use crate::verify::_helpers::compile_errors::undeclared_compile_error; use crate::verify::_helpers::error_check::error_check; use crate::verify::_helpers::layout_check::{layout_failure, reads_clock}; use crate::verify::_helpers::microsecond_evidence::differs_below_microseconds; +use crate::verify::_helpers::replay_cache::judgement_key; use crate::verify::_helpers::row_checks::{RowCheck, row_check}; use crate::verify::_helpers::session::{SessionCheck, session_check}; -use crate::verify::models::{Conversion, ErrorCheck, Verdict}; +use crate::verify::models::{ + Conversion, ConvertedCase, ErrorCheck, Judgement, ReplayCache, Verdict, +}; use crate::verify::types::{CaseConverter, FailureModes}; /// A case's verdict, what its conversion relied on, and how the two errors compare. @@ -38,7 +41,7 @@ type Replayed = Result; pub(crate) fn judge( case: &LoadedCase, (results, defaults): (&Path, Option<&SessionDefaults>), - policy: &Policy, + (policy, cache): (&Policy, Option<(&ReplayCache, &str)>), checks: (&dyn CaseConverter, &FailureModes), ) -> JudgedCase { let stored = match load_result(results, &case.hash) { @@ -53,15 +56,21 @@ pub(crate) fn judge( }; match session_check(case, &stored.metadata, defaults) { SessionCheck::Mismatch(reason) => ((Verdict::Failed { reason }, None, None), false), - SessionCheck::Matches => (judge_stored(case, (stored, results), policy, checks), false), - SessionCheck::Unproven => (judge_stored(case, (stored, results), policy, checks), true), + SessionCheck::Matches => ( + judge_stored(case, (stored, results), (policy, cache), checks), + false, + ), + SessionCheck::Unproven => ( + judge_stored(case, (stored, results), (policy, cache), checks), + true, + ), } } fn judge_stored( case: &LoadedCase, (stored, results): (StoredResult, &Path), - policy: &Policy, + (policy, cache): (&Policy, Option<(&ReplayCache, &str)>), (converter, failure_modes): (&dyn CaseConverter, &FailureModes), ) -> Judged { if case.role == CaseRole::Typing { @@ -122,6 +131,75 @@ fn judge_stored( ); } }; + let converted = ConvertedCase { + laid_out: converter.laid_out(&duckdb_sql), + sql: duckdb_sql, + setup, + provenance, + raises, + python, + }; + let keyed = cache + .filter(|_| case.expected_divergence.is_none()) + .map(|(cache, context)| { + let key = judgement_key(context, (case, &stored.metadata), &converted); + (cache, key) + }); + if let Some((cache, key)) = &keyed + && let Some(known) = cache.known.get(key) + { + remember(cache, key.clone(), known.clone()); + return ( + known.verdict.clone(), + known.provenance.clone(), + known.error_check.clone(), + ); + } + let judged = judge_converted( + case, + (&stored, policy), + (converter, failure_modes), + converted, + ); + if let Some((cache, key)) = keyed.filter(|_| !matches!(judged.0, Verdict::Failed { .. })) { + let (verdict, provenance, error_check) = judged.clone(); + remember( + cache, + key, + Judgement { + verdict, + provenance, + error_check, + }, + ); + } + judged +} + +/// Records `judgement` under `key` for the next run's cache. +fn remember(cache: &ReplayCache, key: String, judgement: Judgement) { + cache + .made + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(key, judgement); +} + +/// Replays a converted case on DuckDB and judges the replay against its recording. +fn judge_converted( + case: &LoadedCase, + (stored, policy): (&StoredResult, &Policy), + (converter, failure_modes): (&dyn CaseConverter, &FailureModes), + converted: ConvertedCase, +) -> Judged { + let ConvertedCase { + sql: duckdb_sql, + setup, + provenance, + raises, + python, + laid_out, + } = converted; let settings = ReplaySettings { time_zone: policy .session_profiles @@ -131,7 +209,7 @@ fn judge_stored( }; let replayed = replay_on_engine((&setup, &duckdb_sql), python.as_deref(), &settings); let layout = layout_failure( - &converter.laid_out(&duckdb_sql), + &laid_out, (&setup, &duckdb_sql), (&replayed, reads_clock(provenance.divergences.keys())), (python.as_deref(), &settings), @@ -140,11 +218,11 @@ fn judge_stored( let verdict = match &case.expected_divergence { Some(divergence) => divergence_verdict( case, - (&stored, replayed), + (stored, replayed), policy, (divergence, &provenance, &raises), ), - None => replayed_verdict(case, (&stored, replayed), policy, &raises), + None => replayed_verdict(case, (stored, replayed), policy, &raises), }; let verdict = match (verdict, &stored.rows) { ( diff --git a/crates/sqlquack-evidence/src/verify/_helpers/layout_check.rs b/crates/sqlquack-evidence/src/verify/_helpers/layout_check.rs index c4f45bb28..6e1401a21 100644 --- a/crates/sqlquack-evidence/src/verify/_helpers/layout_check.rs +++ b/crates/sqlquack-evidence/src/verify/_helpers/layout_check.rs @@ -1,5 +1,7 @@ //! The layout check: the laid-out SQL returns exactly what the single-line SQL returns. +use duckdb::Connection; +use serde_json::Value; use sqlquack_compare::values::models::ResultSet; use crate::replay::main::replay_on_engine::replay_on_engine; @@ -8,6 +10,9 @@ use crate::replay::models::{PythonReplay, ReplaySettings}; /// The divergence whose values read the clock, so two replays may differ by the time between them. const CLOCK_DIVERGENCE: &str = "current_time"; +/// The fields of DuckDB's parse tree that hold where a node starts in the SQL text. +const POSITION_FIELDS: [&str; 2] = ["query_location", "query_location_length"]; + /// Why the laid-out SQL returns another result or error than the single-line `sql`, if it does. pub(crate) fn layout_failure( laid_out: &Result, @@ -17,6 +22,7 @@ pub(crate) fn layout_failure( ) -> Option { let laid_out = match laid_out { Ok(laid_out) if laid_out == sql => return None, + Ok(laid_out) if parsed_alike(sql, laid_out) => return None, Ok(laid_out) => laid_out, Err(reason) => return Some(format!("the layout changed more than whitespace: {reason}")), }; @@ -51,3 +57,42 @@ fn same_result(left: &ResultSet, right: &ResultSet, reads_clock: bool) -> bool { fn first_line(error: &str) -> &str { error.lines().next().unwrap_or_default() } + +/// Whether DuckDB parses both texts into the same statements, apart from source positions. +fn parsed_alike(left: &str, right: &str) -> bool { + let Ok(connection) = Connection::open_in_memory() else { + return false; + }; + match ( + parse_tree(&connection, left), + parse_tree(&connection, right), + ) { + (Ok(left), Ok(right)) => left == right, + _ => false, + } +} + +/// DuckDB's parse tree of `sql` without source positions, or why DuckDB cannot serialize it. +fn parse_tree(connection: &Connection, sql: &str) -> Result { + let text: String = connection + .query_row("SELECT json_serialize_sql(?)", [sql], |row| row.get(0)) + .map_err(|error| error.to_string())?; + let mut tree: Value = serde_json::from_str(&text).map_err(|error| error.to_string())?; + if tree.get("error") != Some(&Value::Bool(false)) { + return Err(text); + } + drop_positions(&mut tree); + Ok(tree) +} + +/// Removes every source position field from a parse tree, at every depth. +fn drop_positions(value: &mut Value) { + match value { + Value::Object(fields) => { + fields.retain(|name, _| !POSITION_FIELDS.contains(&name.as_str())); + fields.values_mut().for_each(drop_positions); + } + Value::Array(items) => items.iter_mut().for_each(drop_positions), + _ => {} + } +} diff --git a/crates/sqlquack-evidence/src/verify/_helpers/mod.rs b/crates/sqlquack-evidence/src/verify/_helpers/mod.rs index 3913899e9..eab206237 100644 --- a/crates/sqlquack-evidence/src/verify/_helpers/mod.rs +++ b/crates/sqlquack-evidence/src/verify/_helpers/mod.rs @@ -5,5 +5,6 @@ pub(crate) mod judge; pub(crate) mod layout_check; pub(crate) mod messages; pub(crate) mod microsecond_evidence; +pub(crate) mod replay_cache; pub(crate) mod row_checks; pub(crate) mod session; diff --git a/crates/sqlquack-evidence/src/verify/_helpers/replay_cache.rs b/crates/sqlquack-evidence/src/verify/_helpers/replay_cache.rs new file mode 100644 index 000000000..e3a93b61f --- /dev/null +++ b/crates/sqlquack-evidence/src/verify/_helpers/replay_cache.rs @@ -0,0 +1,30 @@ +//! The replay cache's keys: digests of everything a converted case's judgement is computed from. + +use std::fmt::Debug; + +use sha2::{Digest, Sha256}; +use sqlquack_policy::policy::models::Policy; + +use crate::cases::models::LoadedCase; +use crate::constants::REPLAY_CACHE_FORMAT; +use crate::results::models::ResultMetadata; +use crate::verify::types::FailureModes; + +/// The digest every key shares: the cache format, the caller's scope, the policy and failure modes. +pub(crate) fn replay_context(scope: &str, policy: &Policy, failure_modes: &FailureModes) -> String { + digest(format!("{REPLAY_CACHE_FORMAT}\n{scope}\n{policy:?}\n{failure_modes:?}").as_bytes()) +} + +/// The key of one converted case's judgement. +pub(crate) fn judgement_key( + context: &str, + (case, metadata): (&LoadedCase, &ResultMetadata), + conversion: &dyn Debug, +) -> String { + digest(format!("{context}\n{case:?}\n{metadata:?}\n{conversion:?}").as_bytes()) +} + +/// The SHA-256 of `text`, as lower-case hex. +fn digest(text: &[u8]) -> String { + format!("{:x}", Sha256::digest(text)) +} diff --git a/crates/sqlquack-evidence/src/verify/main/load_replay_cache.rs b/crates/sqlquack-evidence/src/verify/main/load_replay_cache.rs new file mode 100644 index 000000000..c08185e20 --- /dev/null +++ b/crates/sqlquack-evidence/src/verify/main/load_replay_cache.rs @@ -0,0 +1,38 @@ +//! `load_replay_cache`: an earlier run's judgements, ready to reuse. + +use std::path::Path; + +use serde_json::Value; + +use crate::constants::REPLAY_CACHE_FORMAT; +use crate::errors::EvidenceError; +use crate::verify::models::ReplayCache; + +/// The judgements in the cache at `path` under `scope`; none when it is absent, unreadable or stale. +pub fn load_replay_cache(path: &Path, scope: &str) -> Result { + let empty = ReplayCache { + scope: scope.to_owned(), + ..ReplayCache::default() + }; + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(empty), + Err(error) => return Err(EvidenceError::Io(format!("{}: {error}", path.display()))), + }; + let invalid = + |error: serde_json::Error| EvidenceError::Io(format!("{}: {error}", path.display())); + let mut file: Value = match serde_json::from_str(&text) { + Ok(file) => file, + Err(error) if !error.is_io() => return Ok(empty), + Err(error) => return Err(invalid(error)), + }; + if file.get("format").and_then(Value::as_str) != Some(REPLAY_CACHE_FORMAT) { + return Ok(empty); + } + let known = match serde_json::from_value(file["judgements"].take()) { + Ok(known) => known, + Err(error) if !error.is_io() => return Ok(empty), + Err(error) => return Err(invalid(error)), + }; + Ok(ReplayCache { known, ..empty }) +} diff --git a/crates/sqlquack-evidence/src/verify/main/mod.rs b/crates/sqlquack-evidence/src/verify/main/mod.rs index 085c96603..54ccf1428 100644 --- a/crates/sqlquack-evidence/src/verify/main/mod.rs +++ b/crates/sqlquack-evidence/src/verify/main/mod.rs @@ -1,2 +1,4 @@ pub mod judge_recording; +pub mod load_replay_cache; +pub mod save_replay_cache; pub mod verify_cases; diff --git a/crates/sqlquack-evidence/src/verify/main/save_replay_cache.rs b/crates/sqlquack-evidence/src/verify/main/save_replay_cache.rs new file mode 100644 index 000000000..340262fb3 --- /dev/null +++ b/crates/sqlquack-evidence/src/verify/main/save_replay_cache.rs @@ -0,0 +1,21 @@ +//! `save_replay_cache`: this run's judgements, for the next run to reuse. + +use std::path::Path; + +use crate::constants::REPLAY_CACHE_FORMAT; +use crate::errors::EvidenceError; +use crate::verify::models::ReplayCache; + +/// Writes the judgements `cache` collected to `path`, replacing the file; returns how many. +pub fn save_replay_cache(path: &Path, cache: &ReplayCache) -> Result { + let made = cache + .made + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let file = serde_json::json!({ "format": REPLAY_CACHE_FORMAT, "judgements": *made }); + let text = serde_json::to_string(&file) + .map_err(|error| EvidenceError::Io(format!("{}: {error}", path.display())))?; + std::fs::write(path, text) + .map_err(|error| EvidenceError::Io(format!("{}: {error}", path.display())))?; + Ok(made.len()) +} diff --git a/crates/sqlquack-evidence/src/verify/main/verify_cases.rs b/crates/sqlquack-evidence/src/verify/main/verify_cases.rs index a440eef82..b7230d571 100644 --- a/crates/sqlquack-evidence/src/verify/main/verify_cases.rs +++ b/crates/sqlquack-evidence/src/verify/main/verify_cases.rs @@ -1,16 +1,23 @@ //! `verify_cases`: every case's verdict against its recorded result, and every claim's support. use std::collections::BTreeSet; +use std::path::Path; + +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use sqlquack_diagnostics::provenance::main::merge_provenance::merge_provenance; -use crate::cases::models::CaseRole; +use crate::cases::models::{CaseRole, LoadedCase}; use crate::errors::EvidenceError; use crate::results::main::recorded_hashes::recorded_hashes; +use crate::results::models::SessionDefaults; use crate::verify::_helpers::claims::claim_verdicts; use crate::verify::_helpers::judge::judge; use crate::verify::_helpers::messages::recorded_message; -use crate::verify::models::{CaseVerdict, Verdict, VerifyInputs, VerifyReport}; +use crate::verify::_helpers::replay_cache::replay_context; +use crate::verify::models::{CaseVerdict, ReplayCache, Verdict, VerifyInputs, VerifyReport}; +use crate::verify::types::{CaseConverter, FailureModes}; +use sqlquack_policy::policy::models::Policy; /// Replays recorded cases, checks equivalence pairs, and counts the passed cases behind claims. pub fn verify_cases(inputs: VerifyInputs<'_>) -> Result { @@ -23,38 +30,29 @@ pub fn verify_cases(inputs: VerifyInputs<'_>) -> Result = cases.iter().map(|case| case.hash.as_str()).collect(); let unreferenced_results: Vec = recorded_hashes(results)? .into_iter() .filter(|hash| !referenced.contains(hash.as_str())) .collect(); - let mut verdicts: Vec = Vec::new(); - let mut unproven_sessions: usize = 0; - for case in cases { - if matches!(case.role, CaseRole::EquivalenceCanonical { .. }) { - continue; - } - let ((verdict, provenance, error_check), unproven) = judge( - case, - (results, session_defaults), - policy, - (converter, failure_modes), - ); - unproven_sessions += usize::from(unproven); - let snowflake_message = match error_check { - Some(_) => recorded_message(&case.hash, results, audit_results)?, - None => None, - }; - verdicts.push(CaseVerdict { - reference: case.reference.clone(), - hash: case.hash.clone(), - verdict, - provenance, - error_check, - snowflake_message, - }); - } + let judged: Vec<(CaseVerdict, bool)> = cases + .par_iter() + .filter(|case| !matches!(case.role, CaseRole::EquivalenceCanonical { .. })) + .map(|case| { + judged_case( + case, + (results, audit_results, session_defaults), + (policy, cache), + (converter, failure_modes), + ) + }) + .collect::>()?; + let unproven_sessions: usize = judged.iter().filter(|(_, unproven)| *unproven).count(); + let verdicts: Vec = judged.into_iter().map(|(verdict, _)| verdict).collect(); let verified = merge_provenance( verdicts .iter() @@ -69,3 +67,27 @@ pub fn verify_cases(inputs: VerifyInputs<'_>) -> Result, Option<&SessionDefaults>), + (policy, cache): (&Policy, Option<(&ReplayCache, &str)>), + checks: (&dyn CaseConverter, &FailureModes), +) -> Result<(CaseVerdict, bool), EvidenceError> { + let ((verdict, provenance, error_check), unproven) = + judge(case, (results, session_defaults), (policy, cache), checks); + let snowflake_message = match error_check { + Some(_) => recorded_message(&case.hash, results, audit_results)?, + None => None, + }; + let verdict = CaseVerdict { + reference: case.reference.clone(), + hash: case.hash.clone(), + verdict, + provenance, + error_check, + snowflake_message, + }; + Ok((verdict, unproven)) +} diff --git a/crates/sqlquack-evidence/src/verify/models.rs b/crates/sqlquack-evidence/src/verify/models.rs index 5690ea0fd..cc3c52777 100644 --- a/crates/sqlquack-evidence/src/verify/models.rs +++ b/crates/sqlquack-evidence/src/verify/models.rs @@ -2,8 +2,9 @@ use std::collections::BTreeMap; use std::path::Path; +use std::sync::Mutex; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use sqlquack_compare::comparison::models::Mismatch; use sqlquack_diagnostics::failures::models::ErrorKind; use sqlquack_diagnostics::provenance::models::Provenance; @@ -41,7 +42,7 @@ pub enum Conversion { } /// The verdict for one case. -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "verdict", rename_all = "snake_case")] pub enum Verdict { /// No result is recorded yet. @@ -53,7 +54,7 @@ pub enum Verdict { /// The refused construct keys, sorted. constructs: Vec, /// Whether the recording shows Snowflake refusing to compile the case, as it declares. - #[serde(skip_serializing_if = "std::ops::Not::not")] + #[serde(default, skip_serializing_if = "std::ops::Not::not")] snowflake_rejected: bool, }, /// The replay matched the recorded result, or an equivalence's spellings recorded one result. @@ -105,7 +106,7 @@ pub struct CaseVerdict { } /// How DuckDB's error compares with the recorded Snowflake error, when both raised. -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "outcome", rename_all = "snake_case")] pub enum ErrorCheck { /// The same kind, and a rule the conversion used declares the Snowflake error. @@ -188,6 +189,47 @@ pub struct VerifyInputs<'a> { pub audit_results: Option<&'a Path>, /// The account's recorded session defaults, vouching for results recorded before pinning. pub session_defaults: Option<&'a SessionDefaults>, + /// Earlier judgements to reuse, and where this run's are collected; `None` replays every case. + pub replay_cache: Option<&'a ReplayCache>, +} + +/// A converted case's DuckDB SQL, with everything replaying and judging it needs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConvertedCase { + /// The DuckDB SQL. + pub sql: String, + /// Statements run first, such as the macros of the case's project functions. + pub setup: Vec, + /// The equivalences, rules, building blocks and typing functions used. + pub provenance: Provenance, + /// Where each raise ID in the SQL comes from in the case's source SQL. + pub raises: BTreeMap>, + /// The Python functions the SQL calls, registered before it runs. + pub python: Option>, + /// The SQL as users see it, or why laying it out changed more than whitespace. + pub laid_out: Result, +} + +/// A converted case's judgement: its verdict, what its conversion relied on, how the errors compare. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Judgement { + /// The verdict. + pub verdict: Verdict, + /// What the conversion relied on. + pub provenance: Option, + /// How DuckDB's error compares with Snowflake's. + pub error_check: Option, +} + +/// Judgements keyed by a digest of the case, recording, policy and conversion they came from. +#[derive(Debug, Default)] +pub struct ReplayCache { + /// What else the judgements depend on, such as a digest of the replay engine's source. + pub scope: String, + /// Judgements read from an earlier run. + pub known: BTreeMap, + /// Judgements this run made or reused, failures apart, for the next run. + pub made: Mutex>, } /// Verification of every case. diff --git a/crates/sqlquack-evidence/src/verify/tests/helpers.rs b/crates/sqlquack-evidence/src/verify/tests/helpers.rs index bea84574e..e3ae015a5 100644 --- a/crates/sqlquack-evidence/src/verify/tests/helpers.rs +++ b/crates/sqlquack-evidence/src/verify/tests/helpers.rs @@ -403,6 +403,7 @@ pub(crate) fn round_trip_with_defaults( failure_modes: &FailureModes::new(), audit_results: None, session_defaults: defaults.as_ref(), + replay_cache: None, }) .expect("verify runs"); RoundTrip { diff --git a/crates/sqlquack-evidence/src/verify/types.rs b/crates/sqlquack-evidence/src/verify/types.rs index 228f81f26..2accaea68 100644 --- a/crates/sqlquack-evidence/src/verify/types.rs +++ b/crates/sqlquack-evidence/src/verify/types.rs @@ -8,7 +8,7 @@ use crate::cases::models::LoadedCase; use crate::verify::models::Conversion; /// Turns a case's source SQL into DuckDB SQL, or refuses. -pub trait CaseConverter: std::fmt::Debug { +pub trait CaseConverter: std::fmt::Debug + Sync { /// The conversion of `case`. fn convert(&self, case: &LoadedCase) -> Conversion; diff --git a/crates/sqlquack/src/cli/_helpers/review_command.rs b/crates/sqlquack/src/cli/_helpers/review_command.rs index b3d6db676..4e29b3306 100644 --- a/crates/sqlquack/src/cli/_helpers/review_command.rs +++ b/crates/sqlquack/src/cli/_helpers/review_command.rs @@ -32,7 +32,7 @@ pub(crate) fn review_command(arguments: &ReviewArguments, stderr: &mut dyn Write } fn review(arguments: &ReviewArguments) -> Result { - let (evidence, report) = run_verification(&arguments.paths, None, None)?; + let (evidence, report) = run_verification(&arguments.paths, (None, None), None)?; let converter = PipelineConverter { options: conversion_options(&evidence.policy, false), python: None, diff --git a/crates/sqlquack/src/cli/_helpers/verify_command.rs b/crates/sqlquack/src/cli/_helpers/verify_command.rs index 595fc83db..1a2e8ca6b 100644 --- a/crates/sqlquack/src/cli/_helpers/verify_command.rs +++ b/crates/sqlquack/src/cli/_helpers/verify_command.rs @@ -22,10 +22,12 @@ use sqlquack_evidence::replay::models::ReplaySettings; use sqlquack_evidence::results::main::load_recording::load_recording; use sqlquack_evidence::results::main::load_session_defaults::load_session_defaults; use sqlquack_evidence::verify::main::judge_recording::judge_recording; +use sqlquack_evidence::verify::main::load_replay_cache::load_replay_cache; +use sqlquack_evidence::verify::main::save_replay_cache::save_replay_cache; use sqlquack_evidence::verify::main::verify_cases::verify_cases; use sqlquack_evidence::verify::models::RecordingMatch; use sqlquack_evidence::verify::models::{ - CaseVerdict, Conversion, ErrorCheck, Verdict, VerifyInputs, VerifyReport, + CaseVerdict, Conversion, ErrorCheck, ReplayCache, Verdict, VerifyInputs, VerifyReport, }; use sqlquack_evidence::verify::types::CaseConverter; use sqlquack_policy::policy::constants::{HASH_DIVERGENCE, PROJECT_UDF}; @@ -58,7 +60,12 @@ pub(crate) fn verify_command( "sqlquack verify: loading cases and replaying recorded ones on DuckDB…" ); let (evidence, report) = match verify(arguments) { - Ok(verified) => verified, + Ok((evidence, report, reuse)) => { + if let Some(reuse) = reuse { + let _ = writeln!(stderr, "sqlquack verify: {reuse}"); + } + (evidence, report) + } Err(error) => { let _ = writeln!(stderr, "sqlquack verify: failed: {error}"); return ExitCode::FAILURE; @@ -196,12 +203,36 @@ pub(crate) fn verify_command( } } -fn verify(arguments: &VerifyArguments) -> Result<(Evidence, VerifyReport), String> { - run_verification( +/// The evidence and report, and when a replay cache was used, how many judgements it reused. +fn verify(arguments: &VerifyArguments) -> Result<(Evidence, VerifyReport, Option), String> { + let Some(path) = &arguments.replay_cache else { + let (evidence, report) = run_verification( + &arguments.paths, + (Some(&arguments.audit_results), None), + arguments.python.as_deref(), + )?; + return Ok((evidence, report, None)); + }; + let cache = + load_replay_cache(path, &arguments.replay_scope).map_err(|error| error.to_string())?; + let (evidence, report) = run_verification( &arguments.paths, - Some(&arguments.audit_results), + (Some(&arguments.audit_results), Some(&cache)), arguments.python.as_deref(), - ) + )?; + let made = save_replay_cache(path, &cache).map_err(|error| error.to_string())?; + let reused = cache + .made + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .keys() + .filter(|key| cache.known.contains_key(*key)) + .count(); + let reuse = format!( + "reused {reused} of {made} judgements from {}, replaying the rest", + path.display() + ); + Ok((evidence, report, Some(reuse))) } /// The coverage report, or why it could not be made. @@ -351,7 +382,7 @@ fn render_coverage(coverage: &CoverageReport) -> String { /// Loads the evidence and verifies every case with the production converter. pub(crate) fn run_verification( paths: &EvidencePaths, - audit_results: Option<&Path>, + (audit_results, replay_cache): (Option<&Path>, Option<&ReplayCache>), python: Option<&Path>, ) -> Result<(Evidence, VerifyReport), String> { let evidence = load_evidence(paths)?; @@ -376,6 +407,7 @@ pub(crate) fn run_verification( failure_modes: &failure_modes, audit_results, session_defaults: session_defaults.as_ref(), + replay_cache, }) .map_err(|error| error.to_string())?; Ok((evidence, report)) diff --git a/crates/sqlquack/src/cli/models.rs b/crates/sqlquack/src/cli/models.rs index df38651e9..ac4f24667 100644 --- a/crates/sqlquack/src/cli/models.rs +++ b/crates/sqlquack/src/cli/models.rs @@ -254,6 +254,12 @@ pub struct VerifyArguments { /// Write the bytes of DuckDB SQL each converted case emits as JSON, for the size ratchet. #[arg(long)] pub sql_sizes: Option, + /// Reuse this file's judgements of unchanged cases, replay the rest, and rewrite the file. + #[arg(long)] + pub replay_cache: Option, + /// What else judgements depend on; a cache written under another scope is not reused. + #[arg(long, default_value = "", requires = "replay_cache")] + pub replay_scope: String, /// Where coverage is checked against. #[command(flatten)] pub coverage: CoverageArguments, diff --git a/crates/sqlquack/tests/cli/helpers.rs b/crates/sqlquack/tests/cli/helpers.rs index 8ba0e0b63..faecca08a 100644 --- a/crates/sqlquack/tests/cli/helpers.rs +++ b/crates/sqlquack/tests/cli/helpers.rs @@ -11,7 +11,7 @@ use walkdir::WalkDir; use crate::test_types::{ CommandOutcome, CompareTestCase, ConvertOutcome, CoverageOutcome, DivergenceOutcome, ErrorCheckOutcome, FunctionMacroOutcome, FunctionStdinOutcome, ProjectFile, ProjectOutcome, - ProjectTestCase, ReviewOutcome, ReviewedRepository, + ProjectTestCase, ReplayCacheOutcome, ReviewOutcome, ReviewedRepository, }; fn repository_root() -> PathBuf { @@ -1094,6 +1094,86 @@ pub(crate) fn compare_outcome(test_case: &CompareTestCase) -> String { /// Each statement `verify --translation-statements` writes for the batch-2 aggregates and fidelity. pub(crate) fn translation_statements() -> Vec { + translation_statements_in(aggregate_evidence().path()) +} + +/// The statements `verify --translation-statements` writes for the evidence folder `directory`. +fn translation_statements_in(directory: &Path) -> Vec { + let status = Command::new(env!("CARGO_BIN_EXE_sqlquack")) + .args(["verify", "--translation-statements", "statements.json"]) + .current_dir(directory) + .status() + .expect("sqlquack runs"); + assert!(status.success(), "verify fails: {status}"); + let written: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(directory.join("statements.json")).expect("written"), + ) + .expect("statements parse"); + assert_eq!(written["format"], "sqlquack-translation-statements/1"); + written["statements"] + .as_array() + .cloned() + .unwrap_or_default() +} + +/// Two `verify --json --replay-cache` runs over the aggregate cases, `between` run between them. +pub(crate) fn replay_cache_runs(second_scope: &str, between: fn(&Path)) -> ReplayCacheOutcome { + let directory = aggregate_evidence(); + let verify = |scope: &str| { + let output = Command::new(env!("CARGO_BIN_EXE_sqlquack")) + .args(["verify", "--json", "--replay-cache", "cache.json"]) + .args(["--replay-scope", scope]) + .current_dir(directory.path()) + .output() + .expect("sqlquack runs"); + assert!(output.status.success(), "verify fails: {output:?}"); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + (output.stdout, reuse_counts(&stderr)) + }; + let (first_report, (first_reused, _)) = verify(""); + between(directory.path()); + let (second_report, (second_reused, second_made)) = verify(second_scope); + ReplayCacheOutcome { + first_reused, + second_replayed: (second_made - second_reused).min(2), + second_reused_any: second_reused > 0, + same_report: first_report == second_report, + } +} + +/// Leaves the evidence folder as it is. +pub(crate) fn unchanged_evidence(_: &Path) {} + +/// Truncates the replay cache the first run wrote in the folder `directory`. +pub(crate) fn truncate_cache(directory: &Path) { + std::fs::write( + directory.join("cache.json"), + "{\"format\": \"sqlquack-replay", + ) + .expect("cut"); +} + +/// Re-dates the recording of the aggregate case `any_value_all_null` in the folder `directory`. +pub(crate) fn redate_one_recording(directory: &Path) { + let statements = translation_statements_in(directory); + let hash = statements + .iter() + .find(|statement| { + statement["reference"] == "aggregate/aggregates_batch2.toml#any_value_all_null" + }) + .and_then(|statement| statement["hash"].as_str()) + .expect("the case's hash") + .to_owned(); + let path = directory.join(format!("evidence/results/{hash}.json")); + let mut metadata: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).expect("metadata read")) + .expect("metadata parses"); + metadata["recorded_at"] = "2001-02-03T04:05:06Z".into(); + std::fs::write(&path, metadata.to_string()).expect("metadata written"); +} + +/// A folder holding the evidence pin, the fidelity cases, one aggregate case file and every result. +fn aggregate_evidence() -> tempfile::TempDir { let directory = tempfile::tempdir().expect("temporary directory"); copy_evidence(directory.path()); copy_files( @@ -1108,21 +1188,27 @@ pub(crate) fn translation_statements() -> Vec { ) .expect("case copied"); copy_files(&results_dir(), &directory.path().join("evidence/results")); - let status = Command::new(env!("CARGO_BIN_EXE_sqlquack")) - .args(["verify", "--translation-statements", "statements.json"]) - .current_dir(directory.path()) - .status() - .expect("sqlquack runs"); - assert!(status.success(), "verify fails: {status}"); - let written: serde_json::Value = serde_json::from_str( - &std::fs::read_to_string(directory.path().join("statements.json")).expect("written"), - ) - .expect("statements parse"); - assert_eq!(written["format"], "sqlquack-translation-statements/1"); - written["statements"] - .as_array() - .cloned() - .unwrap_or_default() + directory +} + +/// The reused and total judgements `verify`'s replay cache line reports. +fn reuse_counts(stderr: &str) -> (usize, usize) { + let line = stderr + .lines() + .find_map(|line| line.strip_prefix("sqlquack verify: reused ")) + .expect("the replay cache line"); + let mut numbers = line + .split(' ') + .filter_map(|word| word.parse::().ok()); + let counts = ( + numbers.next().expect("reused"), + numbers.next().expect("made"), + ); + assert!( + counts.1 > 2, + "too few judgements to tell one from many: {line}" + ); + counts } /// The first refusal's help converting `sql` under the repository policy, its path as ``. diff --git a/crates/sqlquack/tests/cli/test_evidence_commands.rs b/crates/sqlquack/tests/cli/test_evidence_commands.rs index a0759b1c7..b5da53f7e 100644 --- a/crates/sqlquack/tests/cli/test_evidence_commands.rs +++ b/crates/sqlquack/tests/cli/test_evidence_commands.rs @@ -1,6 +1,10 @@ -use crate::helpers::{pending_flags, run_on_repository_cases, translation_statements}; +use crate::helpers::{ + pending_flags, redate_one_recording, replay_cache_runs, run_on_repository_cases, + translation_statements, truncate_cache, unchanged_evidence, +}; use crate::test_types::{ - CommandOutcome, CommandTestCase, PendingFlagsTestCase, TranslationStatementTestCase, + CommandOutcome, CommandTestCase, PendingFlagsTestCase, ReplayCacheOutcome, ReplayCacheTestCase, + TranslationStatementTestCase, }; #[test] @@ -192,3 +196,61 @@ fn given_recorded_cases_when_verify_writes_translation_statements_then_each_outc .count(); assert_eq!(aggregates, 50); } + +#[test] +fn given_a_replay_cache_when_verifying_again_then_only_cases_whose_inputs_changed_replay() { + let test_cases = [ + ReplayCacheTestCase { + description: "an unchanged rerun reuses every judgement and prints the same report", + second_scope: "", + between_runs: unchanged_evidence, + expected_outcome: ReplayCacheOutcome { + first_reused: 0, + second_replayed: 0, + second_reused_any: true, + same_report: true, + }, + }, + ReplayCacheTestCase { + description: "a cache written under another scope is not reused", + second_scope: "another engine", + between_runs: unchanged_evidence, + expected_outcome: ReplayCacheOutcome { + first_reused: 0, + second_replayed: 2, + second_reused_any: false, + same_report: true, + }, + }, + ReplayCacheTestCase { + description: "a truncated cache replays every case instead of failing", + second_scope: "", + between_runs: truncate_cache, + expected_outcome: ReplayCacheOutcome { + first_reused: 0, + second_replayed: 2, + second_reused_any: false, + same_report: true, + }, + }, + ReplayCacheTestCase { + description: "a re-recorded case replays alone", + second_scope: "", + between_runs: redate_one_recording, + expected_outcome: ReplayCacheOutcome { + first_reused: 0, + second_replayed: 1, + second_reused_any: true, + same_report: true, + }, + }, + ]; + for test_case in test_cases { + assert_eq!( + replay_cache_runs(test_case.second_scope, test_case.between_runs), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack/tests/cli/test_types.rs b/crates/sqlquack/tests/cli/test_types.rs index a597abd14..68a9d628c 100644 --- a/crates/sqlquack/tests/cli/test_types.rs +++ b/crates/sqlquack/tests/cli/test_types.rs @@ -379,3 +379,20 @@ pub(crate) struct ParquetRunTestCase { pub(crate) expected_exit: Option, pub(crate) expected_stdout: &'static str, } + +/// Two `verify --replay-cache` runs: the first's reuses, the second's replays (2 is two or more). +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct ReplayCacheOutcome { + pub(crate) first_reused: usize, + pub(crate) second_replayed: usize, + pub(crate) second_reused_any: bool, + pub(crate) same_report: bool, +} + +/// The second run's scope, and what changes in the evidence folder before it. +pub(crate) struct ReplayCacheTestCase { + pub(crate) description: &'static str, + pub(crate) second_scope: &'static str, + pub(crate) between_runs: fn(&std::path::Path), + pub(crate) expected_outcome: ReplayCacheOutcome, +} diff --git a/scripts/pr_metadata/__init__.py b/scripts/pr_metadata/__init__.py new file mode 100644 index 000000000..4a2695b86 --- /dev/null +++ b/scripts/pr_metadata/__init__.py @@ -0,0 +1 @@ +"""Pull request metadata validation tooling.""" diff --git a/scripts/pr_metadata/_helpers/__init__.py b/scripts/pr_metadata/_helpers/__init__.py new file mode 100644 index 000000000..64c5983b6 --- /dev/null +++ b/scripts/pr_metadata/_helpers/__init__.py @@ -0,0 +1 @@ +"""Pull request metadata validation helpers.""" diff --git a/scripts/pr_metadata/_helpers/inputs.py b/scripts/pr_metadata/_helpers/inputs.py new file mode 100644 index 000000000..2a204a45a --- /dev/null +++ b/scripts/pr_metadata/_helpers/inputs.py @@ -0,0 +1,29 @@ +"""Pull request metadata input resolution.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + + +def load_event_metadata(path: Path) -> tuple[str, str, str]: + """Load branch, title, and body from a GitHub pull request payload.""" + payload: dict[str, Any] = json.loads(path.read_text(encoding="utf-8")) + pull_request: dict[str, Any] = payload.get("pull_request", payload) + return ( + pull_request["head"]["ref"], + pull_request["title"], + pull_request.get("body") or "", + ) + + +def get_current_branch() -> str: + """Return the current Git branch name.""" + return subprocess.run( + ["git", "branch", "--show-current"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() diff --git a/scripts/pr_metadata/_helpers/validation.py b/scripts/pr_metadata/_helpers/validation.py new file mode 100644 index 000000000..2bb947e9d --- /dev/null +++ b/scripts/pr_metadata/_helpers/validation.py @@ -0,0 +1,43 @@ +"""Pull request metadata policy validation.""" + +from __future__ import annotations + +import re + +from scripts.pr_metadata.constants import ( + AUTOMATION_PATTERN, + BRANCH_PATTERN, + EXPECTED_SECTIONS, + MAX_BODY_LENGTH, + SECTION_PATTERN, + TITLE_PATTERN, +) + + +def get_pr_metadata_errors(*, branch: str, title: str, body: str) -> tuple[str, ...]: + """Return every pull request metadata policy violation.""" + errors: list[str] = [] + automated: bool = AUTOMATION_PATTERN.match(branch) is not None + if not automated and BRANCH_PATTERN.fullmatch(branch) is None: + errors.append("Branch must use /.") + if TITLE_PATTERN.fullmatch(title) is None: + errors.append("PR title must follow Conventional Commits.") + + if automated: + return tuple(errors) + if len(body) > MAX_BODY_LENGTH: + errors.append("PR description must not exceed 2,000 characters.") + + sections: list[re.Match[str]] = list(SECTION_PATTERN.finditer(body)) + if tuple(match.group(1) for match in sections) != EXPECTED_SECTIONS: + errors.append( + "PR description must contain Why, Changes, and Verification sections in that order." + ) + return tuple(errors) + + for index, match in enumerate(sections): + end: int = sections[index + 1].start() if index + 1 < len(sections) else len(body) + content: str = re.sub(r"", "", body[match.end() : end], flags=re.DOTALL) + if not content.strip(): + errors.append(f"PR description section {match.group(1)} is empty.") + return tuple(errors) diff --git a/scripts/pr_metadata/constants.py b/scripts/pr_metadata/constants.py new file mode 100644 index 000000000..bd16cd6a1 --- /dev/null +++ b/scripts/pr_metadata/constants.py @@ -0,0 +1,18 @@ +"""Stable pull request metadata policy constants.""" + +import re + +BRANCH_PATTERN: re.Pattern[str] = re.compile( + r"^(feat|fix|perf|refactor|test|docs|build|ci|chore|revert)/" + r"[a-z0-9]+(?:-[a-z0-9]+)*$" +) +AUTOMATION_PATTERN: re.Pattern[str] = re.compile( + r"^(release-please--|dependabot/|renovate/|github-actions/|blacksmith-migration-)" +) +TITLE_PATTERN: re.Pattern[str] = re.compile( + r"^(feat|fix|perf|refactor|test|docs|build|ci|chore|revert)" + r"(?:\([a-z0-9][a-z0-9-]*\))?!?: [^ ].+$" +) +SECTION_PATTERN: re.Pattern[str] = re.compile(r"^## (Why|Changes|Verification)\s*$", re.MULTILINE) +EXPECTED_SECTIONS: tuple[str, ...] = ("Why", "Changes", "Verification") +MAX_BODY_LENGTH: int = 2_000 diff --git a/scripts/pr_metadata/main/__init__.py b/scripts/pr_metadata/main/__init__.py new file mode 100644 index 000000000..077a62a97 --- /dev/null +++ b/scripts/pr_metadata/main/__init__.py @@ -0,0 +1 @@ +"""Pull request metadata command entrypoints.""" diff --git a/scripts/pr_metadata/main/validate.py b/scripts/pr_metadata/main/validate.py new file mode 100644 index 000000000..b09befc00 --- /dev/null +++ b/scripts/pr_metadata/main/validate.py @@ -0,0 +1,46 @@ +"""Pull request metadata command entrypoint.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from scripts.pr_metadata._helpers.inputs import get_current_branch, load_event_metadata +from scripts.pr_metadata._helpers.validation import get_pr_metadata_errors + + +def run_pr_metadata_validation(argv: list[str] | None = None) -> int: + """Validate metadata supplied directly or through a GitHub event payload.""" + parser: argparse.ArgumentParser = _build_parser() + arguments: argparse.Namespace = parser.parse_args(argv) + branch: str + title: str + body: str + if arguments.metadata_file is not None: + branch, title, body = load_event_metadata(arguments.metadata_file) + else: + if arguments.title is None or arguments.body_file is None: + parser.error("--title and --body-file are required without --metadata-file") + branch = arguments.branch or get_current_branch() + title = arguments.title + body = arguments.body_file.read_text(encoding="utf-8") + + errors: tuple[str, ...] = get_pr_metadata_errors(branch=branch, title=title, body=body) + if errors: + print(f"Invalid pull request metadata for {branch}:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + print(f"Pull request metadata is valid for {branch}.") + return 0 + + +def _build_parser() -> argparse.ArgumentParser: + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--metadata-file", type=Path) + parser.add_argument("--branch") + parser.add_argument("--title") + parser.add_argument("--body-file", type=Path) + return parser diff --git a/scripts/refetch_duckdb_library.sh b/scripts/refetch_duckdb_library.sh new file mode 100755 index 000000000..adf4ae827 --- /dev/null +++ b/scripts/refetch_duckdb_library.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Makes Cargo download the DuckDB library again when a restored build cache lacks it. +# +# Usage: scripts/refetch_duckdb_library.sh; CARGO_TARGET_DIR, when set, is the target directory. +# libduckdb-sys' build script downloads libduckdb.so into /duckdb-download, beside the +# profile directories, where a build cache such as Swatinem/rust-cache does not keep it. The cache +# does keep the build script's recorded output, so Cargo would not run it again, and the build links +# and copies a library that is not there. For each recorded run whose library is missing, this +# removes its fingerprint, so the next build runs the build script and downloads the library. +set -euo pipefail +target=${CARGO_TARGET_DIR:-target} +for output in "$target"/*/build/libduckdb-sys-*/output; do + [ -f "$output" ] || continue + library="$(sed -n 's/^cargo:rustc-link-search=native=\(.*duckdb-download.*\)$/\1/p' "$output" | head -1)" + if [ -n "$library" ] && [ ! -f "$library/libduckdb.so" ]; then + unit="$(dirname "$output")" + echo "refetch DuckDB library: $(basename "$unit") recorded $library/libduckdb.so, which is missing" + rm -rf "$(dirname "$(dirname "$unit")")/.fingerprint/$(basename "$unit")" + fi +done diff --git a/scripts/replay_scope/__init__.py b/scripts/replay_scope/__init__.py new file mode 100644 index 000000000..559950168 --- /dev/null +++ b/scripts/replay_scope/__init__.py @@ -0,0 +1 @@ +"""Which checks and replay a change needs, and the scope a replay cache is valid in.""" diff --git a/scripts/replay_scope/_helpers/__init__.py b/scripts/replay_scope/_helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/replay_scope/_helpers/crates.py b/scripts/replay_scope/_helpers/crates.py new file mode 100644 index 000000000..61fc80694 --- /dev/null +++ b/scripts/replay_scope/_helpers/crates.py @@ -0,0 +1,60 @@ +"""The workspace crates a package is built from, read from the Cargo manifests alone.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import Any + +from scripts.replay_scope.constants import BUILD_TABLES, DIRECTORY_SUFFIX, PATH_KEY + + +def crate_dirs(*, root: Path, packages: tuple[str, ...]) -> frozenset[str]: + """The directories, relative to `root` and ending in `/`, of `packages` and their path deps.""" + paths: dict[str, str] = workspace_paths(root=root) + found: set[str] = set() + pending: list[str] = list(packages) + while pending: + name: str = pending.pop() + if name in found or name not in paths: + continue + found.add(name) + pending.extend(manifest_dependencies(path=root / paths[name] / "Cargo.toml")) + return frozenset(paths[name].rstrip(DIRECTORY_SUFFIX) + DIRECTORY_SUFFIX for name in found) + + +def workspace_paths(*, root: Path) -> dict[str, str]: + """Each workspace dependency declared by path, and its directory.""" + manifest: dict[str, Any] = tomllib.loads((root / "Cargo.toml").read_text()) + declared: dict[str, Any] = manifest["workspace"].get("dependencies", {}) + return { + name: entry[PATH_KEY] + for name, entry in declared.items() + if isinstance(entry, dict) and PATH_KEY in entry + } + + +def manifest_dependencies(*, path: Path) -> list[str]: + """The names a manifest's build dependency tables list, its target tables' included.""" + manifest: dict[str, Any] = tomllib.loads(path.read_text()) + tables: list[dict[str, Any]] = [manifest, *manifest.get("target", {}).values()] + names: list[str] = [] + for table in tables: + names.extend(table_dependencies(table=table)) + return names + + +def table_dependencies(*, table: dict[str, Any]) -> list[str]: + """The names one manifest table's build dependency sections list.""" + names: list[str] = [] + for section in BUILD_TABLES: + names.extend(table.get(section, {})) + return names + + +def under(*, path: str, prefixes: tuple[str, ...] | frozenset[str]) -> bool: + """Whether `path` is one of `prefixes` or lies in one ending in `/`.""" + return any( + path == prefix or (prefix.endswith(DIRECTORY_SUFFIX) and path.startswith(prefix)) + for prefix in prefixes + ) diff --git a/scripts/replay_scope/_helpers/decisions.py b/scripts/replay_scope/_helpers/decisions.py new file mode 100644 index 000000000..1f3397e9a --- /dev/null +++ b/scripts/replay_scope/_helpers/decisions.py @@ -0,0 +1,87 @@ +"""What a change replays and checks, the replay cache's scope, and the declared paths' check.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from scripts.replay_scope._helpers.crates import crate_dirs, manifest_dependencies, under +from scripts.replay_scope._helpers.git import changed, tracked +from scripts.replay_scope.constants import ( + BINARY_PACKAGE, + DIRECTORY_SUFFIX, + DOCUMENTATION_PATHS, + FULL_REPLAY_PATHS, + MARKDOWN_SUFFIX, + REPLAY_INPUT_PATHS, + SITE_MANIFEST, + SITE_PATHS, +) +from scripts.replay_scope.types import Scope + + +def replay_scope(*, paths: list[str], binary_dirs: frozenset[str]) -> Scope: + """`full` when a changed path forces it, `changed` when one feeds the replay, else `none`.""" + if any(under(path=path, prefixes=FULL_REPLAY_PATHS) for path in paths): + return "full" + if any(feeds_replay(path=path, binary_dirs=binary_dirs) for path in paths): + return "changed" + return "none" + + +def feeds_replay(*, path: str, binary_dirs: frozenset[str]) -> bool: + """Whether `path` is in a crate the binary is built from or among the replay's inputs.""" + return under(path=path, prefixes=binary_dirs) or under(path=path, prefixes=REPLAY_INPUT_PATHS) + + +def needs_code_checks(*, paths: list[str]) -> bool: + """Whether a changed path lies outside the documentation and the top-level Markdown files.""" + return any(not documentation(path=path) for path in paths) + + +def documentation(*, path: str) -> bool: + """Whether `path` is documentation: under a documentation path, or top-level Markdown.""" + top_level_markdown: bool = path.endswith(MARKDOWN_SUFFIX) and DIRECTORY_SUFFIX not in path + return top_level_markdown or under(path=path, prefixes=DOCUMENTATION_PATHS) + + +def needs_site(*, paths: list[str], site_dirs: frozenset[str]) -> bool: + """Whether a changed path is in the site or a crate the playground engine is built from.""" + return any( + under(path=path, prefixes=SITE_PATHS) or under(path=path, prefixes=site_dirs) + for path in paths + ) + + +def change_plan(*, root: Path, base: str) -> dict[str, str]: + """The replay scope and whether the code and site checks run, as GitHub step outputs.""" + paths: list[str] = changed(root=root, base=base) + binary_dirs: frozenset[str] = crate_dirs(root=root, packages=(BINARY_PACKAGE,)) + engine: tuple[str, ...] = tuple(manifest_dependencies(path=root / SITE_MANIFEST)) + site_dirs: frozenset[str] = crate_dirs(root=root, packages=engine) + return { + "replay": replay_scope(paths=paths, binary_dirs=binary_dirs), + "code": str(needs_code_checks(paths=paths)).lower(), + "site": str(needs_site(paths=paths, site_dirs=site_dirs)).lower(), + } + + +def scope_digest(*, root: Path) -> str: + """The SHA-256 of every tracked file under the full-replay paths, by path and content.""" + parts: list[bytes] = [] + for path in tracked(root=root, prefixes=FULL_REPLAY_PATHS): + content: bytes = (root / path).read_bytes() + parts.extend((f"{path}\0{len(content)}\0".encode(), content)) + return hashlib.sha256(b"".join(parts)).hexdigest() + + +def unmatched(*, root: Path) -> list[str]: + """The full-replay and replay-input paths no tracked file matches.""" + files: list[str] = tracked(root=root, prefixes=()) + declared: tuple[str, ...] = (*FULL_REPLAY_PATHS, *REPLAY_INPUT_PATHS) + return [prefix for prefix in declared if not matches_any(prefix=prefix, files=files)] + + +def matches_any(*, prefix: str, files: list[str]) -> bool: + """Whether any of `files` is `prefix` or lies under it.""" + return any(under(path=path, prefixes=(prefix,)) for path in files) diff --git a/scripts/replay_scope/_helpers/git.py b/scripts/replay_scope/_helpers/git.py new file mode 100644 index 000000000..1b37b9012 --- /dev/null +++ b/scripts/replay_scope/_helpers/git.py @@ -0,0 +1,30 @@ +"""Tracked and changed files, from Git.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +def tracked(*, root: Path, prefixes: tuple[str, ...]) -> list[str]: + """The tracked files under `prefixes`, every tracked file when there are none, sorted.""" + return sorted(git_paths(root=root, arguments=("ls-files", "-z", "--", *prefixes))) + + +def changed(*, root: Path, base: str) -> list[str]: + """The files changed since the merge base with `base`, committed or not, renames as two.""" + committed: set[str] = git_paths( + root=root, arguments=("diff", "-z", "--name-only", "--no-renames", f"{base}...") + ) + working: set[str] = git_paths( + root=root, arguments=("diff", "-z", "--name-only", "--no-renames", "HEAD") + ) + return sorted(committed | working) + + +def git_paths(*, root: Path, arguments: tuple[str, ...]) -> set[str]: + """The NUL-separated paths a Git command prints.""" + output: bytes = subprocess.run( + ("git", *arguments), cwd=root, check=True, capture_output=True + ).stdout + return {path for path in output.decode("utf-8").split("\0") if path} diff --git a/scripts/replay_scope/constants.py b/scripts/replay_scope/constants.py new file mode 100644 index 000000000..cf5a14b4b --- /dev/null +++ b/scripts/replay_scope/constants.py @@ -0,0 +1,44 @@ +"""The paths that decide what a change replays and checks; README.md explains each list.""" + +from __future__ import annotations + +FULL_REPLAY_PATHS: tuple[str, ...] = ( + "Cargo.lock", + "rust-toolchain.toml", + "evidence.toml", + "uv.lock", + ".cargo/config.toml", + ".github/workflows/ci.yml", + "crates/sqlquack-compare/", + "crates/sqlquack-diagnostics/src/failures/", + "crates/sqlquack-diagnostics/src/provenance/", + "crates/sqlquack-diagnostics/src/snowflake/failures/", + "crates/sqlquack-evidence/", + "crates/sqlquack-policy/", + "crates/sqlquack/src/cli/_helpers/verify_command.rs", + "scripts/replay_scope/", +) +REPLAY_INPUT_PATHS: tuple[str, ...] = ( + "Cargo.toml", + "Makefile", + "coverage/sql-size/", + "coverage/translation/", + "scripts/check_sql_sizes.py", + "scripts/sql_sizes/", + "tools/sqlquack-coverage-check/", +) +DOCUMENTATION_PATHS: tuple[str, ...] = ("website/", "docs/") +SITE_PATHS: tuple[str, ...] = ( + "website/", + ".github/workflows/website-build.yml", + "rust-toolchain.toml", +) +SITE_MANIFEST: str = "website/wasm/Cargo.toml" +BINARY_PACKAGE: str = "sqlquack" +BUILD_TABLES: tuple[str, ...] = ("dependencies", "build-dependencies") +PATH_KEY: str = "path" +DIRECTORY_SUFFIX: str = "/" +MARKDOWN_SUFFIX: str = ".md" +SCOPE_COMMAND: str = "scope" +DIGEST_COMMAND: str = "digest" +CHECK_COMMAND: str = "check" diff --git a/scripts/replay_scope/main/__init__.py b/scripts/replay_scope/main/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/replay_scope/main/scope.py b/scripts/replay_scope/main/scope.py new file mode 100644 index 000000000..deb82d241 --- /dev/null +++ b/scripts/replay_scope/main/scope.py @@ -0,0 +1,42 @@ +"""What a change replays and checks, the replay cache's scope, or the declared paths' check.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from scripts.replay_scope._helpers.decisions import change_plan, scope_digest, unmatched +from scripts.replay_scope.constants import CHECK_COMMAND, DIGEST_COMMAND, SCOPE_COMMAND + + +def run_replay_scope(argv: list[str] | None = None) -> int: + """Run one of the scope, digest and check commands.""" + arguments: argparse.Namespace = _parser().parse_args(argv) + root: Path = arguments.root + if arguments.command == DIGEST_COMMAND: + print(scope_digest(root=root)) + return 0 + if arguments.command == CHECK_COMMAND: + missing: list[str] = unmatched(root=root) + for prefix in missing: + print(f"replay scope: {prefix} matches no tracked file", file=sys.stderr) + print(f"replay scope: {'FAILED' if missing else 'passed'}", file=sys.stderr) + return 1 if missing else 0 + plan: dict[str, str] = change_plan(root=root, base=arguments.base) + print(f"replay scope: {plan} against {arguments.base}", file=sys.stderr) + print("\n".join(f"{key}={value}" for key, value in plan.items())) + return 0 + + +def _parser() -> argparse.ArgumentParser: + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "command", + choices=(SCOPE_COMMAND, DIGEST_COMMAND, CHECK_COMMAND), + help="scope: replay=none|changed|full, code= and site=true|false; digest: the replay " + "cache's scope; check: fail when a declared path matches no tracked file", + ) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--base", default="origin/main", help="the commit the change merges into") + return parser diff --git a/scripts/replay_scope/types.py b/scripts/replay_scope/types.py new file mode 100644 index 000000000..5fb72cfa0 --- /dev/null +++ b/scripts/replay_scope/types.py @@ -0,0 +1,7 @@ +"""The replay a change needs: none, the cases whose judgement inputs changed, or every case.""" + +from __future__ import annotations + +from typing import Literal + +type Scope = Literal["none", "changed", "full"] diff --git a/scripts/select_replay.py b/scripts/select_replay.py new file mode 100644 index 000000000..6d54a81e1 --- /dev/null +++ b/scripts/select_replay.py @@ -0,0 +1,14 @@ +"""Direct wrapper for the replay scope: the replay a change needs, and the cache scope's digest.""" + +from __future__ import annotations + +from scripts.replay_scope.main.scope import run_replay_scope + + +def main(argv: list[str] | None = None) -> int: + """Print the replay a change needs or the replay cache's scope, or check the declared paths.""" + return run_replay_scope(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_pr_metadata.py b/scripts/validate_pr_metadata.py new file mode 100644 index 000000000..6ee42ac03 --- /dev/null +++ b/scripts/validate_pr_metadata.py @@ -0,0 +1,14 @@ +"""Direct wrapper for pull request metadata validation.""" + +from __future__ import annotations + +from scripts.pr_metadata.main.validate import run_pr_metadata_validation + + +def main(argv: list[str] | None = None) -> int: + """Validate pull request metadata.""" + return run_pr_metadata_validation(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/scripts/case_generators/helpers.py b/tests/unit/scripts/case_generators/helpers.py index cc987819d..63d3450a8 100644 --- a/tests/unit/scripts/case_generators/helpers.py +++ b/tests/unit/scripts/case_generators/helpers.py @@ -8,7 +8,12 @@ from scripts.case_generators.constants import GENERATED_DIRECTORY from scripts.case_generators.models import Case -from scripts.hazard_cases.constants import COVERAGE_DOMAINS, COVERAGE_HAZARDS, REPOSITORY +from scripts.hazard_cases.constants import ( + COVERAGE_DOMAINS, + COVERAGE_HAZARDS, + POLICY_FILE, + REPOSITORY, +) from scripts.hazard_cases.main.hazard_files import hazard_files @@ -167,7 +172,8 @@ def hazard_cases(repository: Path) -> dict[str, Case]: (repository / "coverage/domains").mkdir(parents=True) (repository / "coverage/domains/get.toml").write_text(HAZARD_DOMAIN) (repository / "coverage/hazards.toml").write_text(HAZARD_CATALOGUE) - (repository / "sqlquack.toml").write_text(HAZARD_POLICY) + (repository / POLICY_FILE).parent.mkdir(parents=True) + (repository / POLICY_FILE).write_text(HAZARD_POLICY) cases: dict[str, Case] = {} for case_file in hazard_files(repository): for case in case_file.cases: diff --git a/tests/unit/scripts/pr_metadata/__init__.py b/tests/unit/scripts/pr_metadata/__init__.py new file mode 100644 index 000000000..9571618cf --- /dev/null +++ b/tests/unit/scripts/pr_metadata/__init__.py @@ -0,0 +1 @@ +"""Pull request metadata tooling tests.""" diff --git a/tests/unit/scripts/pr_metadata/_helpers/__init__.py b/tests/unit/scripts/pr_metadata/_helpers/__init__.py new file mode 100644 index 000000000..a1b1fc8ed --- /dev/null +++ b/tests/unit/scripts/pr_metadata/_helpers/__init__.py @@ -0,0 +1 @@ +"""Pull request metadata helper tests.""" diff --git a/tests/unit/scripts/pr_metadata/_helpers/_test_types.py b/tests/unit/scripts/pr_metadata/_helpers/_test_types.py new file mode 100644 index 000000000..d9581f567 --- /dev/null +++ b/tests/unit/scripts/pr_metadata/_helpers/_test_types.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class PrMetadataValidationTestCase: + description: str + branch: str + title: str + body: str + expected_errors: tuple[str, ...] diff --git a/tests/unit/scripts/pr_metadata/_helpers/test_validation.py b/tests/unit/scripts/pr_metadata/_helpers/test_validation.py new file mode 100644 index 000000000..d672ac02e --- /dev/null +++ b/tests/unit/scripts/pr_metadata/_helpers/test_validation.py @@ -0,0 +1,104 @@ +import pytest + +from scripts.pr_metadata._helpers.validation import get_pr_metadata_errors +from tests.unit.scripts.pr_metadata._helpers._test_types import PrMetadataValidationTestCase + +VALID_BODY: str = "## Why\nReason\n\n## Changes\nChange\n\n## Verification\nTested\n" + + +@pytest.mark.parametrize( + "test_case", + [ + PrMetadataValidationTestCase( + description="valid branch and complete body", + branch="feat/named-connections", + title="feat: add named connections", + body=VALID_BODY, + expected_errors=(), + ), + PrMetadataValidationTestCase( + description="body exactly at maximum length", + branch="ci/local-pr-check", + title="ci: add local PR check", + body=VALID_BODY + "x" * (2_000 - len(VALID_BODY)), + expected_errors=(), + ), + PrMetadataValidationTestCase( + description="automation branch skips human body requirements", + branch="dependabot/pip/ruff-1.0", + title="chore(deps): update ruff", + body="", + expected_errors=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_valid_metadata_when_validating_then_returns_no_errors( + test_case: PrMetadataValidationTestCase, +) -> None: + errors: tuple[str, ...] = get_pr_metadata_errors( + branch=test_case.branch, + title=test_case.title, + body=test_case.body, + ) + + assert errors == test_case.expected_errors + + +@pytest.mark.parametrize( + "test_case", + [ + PrMetadataValidationTestCase( + description="invalid branch", + branch="someone/named-connections", + title="feat: add named connections", + body=VALID_BODY, + expected_errors=("Branch must use /.",), + ), + PrMetadataValidationTestCase( + description="non-conventional title", + branch="feat/named-connections", + title="Add named connections", + body=VALID_BODY, + expected_errors=("PR title must follow Conventional Commits.",), + ), + PrMetadataValidationTestCase( + description="body exceeds maximum length", + branch="feat/named-connections", + title="feat: add named connections", + body=VALID_BODY + "x" * (2_001 - len(VALID_BODY)), + expected_errors=("PR description must not exceed 2,000 characters.",), + ), + PrMetadataValidationTestCase( + description="sections are out of order", + branch="feat/named-connections", + title="feat: add named connections", + body="## Changes\nChange\n## Why\nReason\n## Verification\nTested\n", + expected_errors=( + "PR description must contain Why, Changes, and Verification sections in that order.", + ), + ), + PrMetadataValidationTestCase( + description="comment-only section is empty", + branch="feat/named-connections", + title="feat: add named connections", + body=("## Why\n\n\n## Changes\nChange\n\n## Verification\nTested\n"), + expected_errors=("PR description section Why is empty.",), + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_metadata_when_validating_then_returns_policy_errors( + test_case: PrMetadataValidationTestCase, +) -> None: + errors: tuple[str, ...] = get_pr_metadata_errors( + branch=test_case.branch, + title=test_case.title, + body=test_case.body, + ) + + assert errors == test_case.expected_errors + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/replay_scope/__init__.py b/tests/unit/scripts/replay_scope/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/scripts/replay_scope/_test_types.py b/tests/unit/scripts/replay_scope/_test_types.py new file mode 100644 index 000000000..bd7004d34 --- /dev/null +++ b/tests/unit/scripts/replay_scope/_test_types.py @@ -0,0 +1,45 @@ +"""Test types for the replay scope.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.replay_scope.types import Scope + + +@dataclass(frozen=True) +class ScopeTestCase: + description: str + changed: tuple[str, ...] + expected_scope: Scope + + +@dataclass(frozen=True) +class ChecksTestCase: + description: str + changed: tuple[str, ...] + expected_code: bool + expected_site: bool + + +@dataclass(frozen=True) +class DigestTestCase: + description: str + edited: str + expected_changed: bool + + +@dataclass(frozen=True) +class RepositoryTestCase: + description: str + expected_unmatched: tuple[str, ...] + expected_forced_scopes: frozenset[str] + expected_binary_crates: tuple[bool, bool, bool] + + +@dataclass(frozen=True) +class RenameTestCase: + description: str + source: str + destination: str + expected_changed: tuple[str, ...] diff --git a/tests/unit/scripts/replay_scope/helpers.py b/tests/unit/scripts/replay_scope/helpers.py new file mode 100644 index 000000000..2c33d40f2 --- /dev/null +++ b/tests/unit/scripts/replay_scope/helpers.py @@ -0,0 +1,36 @@ +"""A small git repository for the replay scope's tests.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +def committed_tree(*, root: Path, files: dict[str, str]) -> Path: + """`root` as a git repository with `files` committed.""" + for name, text in files.items(): + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_text(text) + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + return commit_all(root=root) + + +def commit_all(*, root: Path) -> Path: + """`root` with every change in it committed.""" + subprocess.run(["git", "add", "-A", "."], cwd=root, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=t", + "-c", + "user.email=t@example.com", + "commit", + "-q", + "-m", + "files", + ], + cwd=root, + check=True, + ) + return root diff --git a/tests/unit/scripts/replay_scope/test_scope.py b/tests/unit/scripts/replay_scope/test_scope.py new file mode 100644 index 000000000..651bd583b --- /dev/null +++ b/tests/unit/scripts/replay_scope/test_scope.py @@ -0,0 +1,229 @@ +"""What a change replays and checks, the cache scope's digest, and the declared paths.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.replay_scope._helpers.crates import crate_dirs +from scripts.replay_scope._helpers.decisions import ( + needs_code_checks, + needs_site, + replay_scope, + scope_digest, + unmatched, +) +from scripts.replay_scope._helpers.git import changed, tracked +from scripts.replay_scope.constants import FULL_REPLAY_PATHS +from tests.unit.scripts.replay_scope._test_types import ( + ChecksTestCase, + DigestTestCase, + RenameTestCase, + RepositoryTestCase, + ScopeTestCase, +) +from tests.unit.scripts.replay_scope.helpers import commit_all, committed_tree + +_BINARY_DIRS: frozenset[str] = frozenset( + {"crates/sqlquack/", "crates/sqlquack-convert/", "crates/sqlquack-diagnostics/"} +) +_SITE_DIRS: frozenset[str] = frozenset({"crates/sqlquack-convert/"}) +_REPOSITORY: Path = Path(__file__).resolve().parents[4] + + +@pytest.mark.parametrize( + "test_case", + ( + ScopeTestCase( + description="documentation and the website replay nothing", + changed=("README.md", "website/src/content/docs/index.mdx"), + expected_scope="none", + ), + ScopeTestCase( + description="a crate the binary is not built from replays nothing", + changed=("crates/sqlquack-lint/src/lib.rs",), + expected_scope="none", + ), + ScopeTestCase( + description="a rule in the converter replays the changed cases", + changed=("README.md", "crates/sqlquack-convert/src/snowflake/rules/cast.rs"), + expected_scope="changed", + ), + ScopeTestCase( + description="a directory's name alone is not inside it", + changed=("crates/sqlquack-convert-extra/src/lib.rs",), + expected_scope="none", + ), + ScopeTestCase( + description="the size ratchet's baseline reruns the replay's checks", + changed=("coverage/sql-size/baseline.json",), + expected_scope="changed", + ), + ScopeTestCase( + description="the replay engine replays every case", + changed=("crates/sqlquack-evidence/src/replay/main/replay_on_engine.rs",), + expected_scope="full", + ), + ScopeTestCase( + description="a refusal message replays the changed cases", + changed=("crates/sqlquack-diagnostics/src/refusals/constants.rs",), + expected_scope="changed", + ), + ScopeTestCase( + description="an error kind, which error checks read, replays every case", + changed=("crates/sqlquack-diagnostics/src/failures/constants.rs",), + expected_scope="full", + ), + ScopeTestCase( + description="a forced file wins over a rule change", + changed=("crates/sqlquack-convert/src/lib.rs", "Cargo.lock"), + expected_scope="full", + ), + ), + ids=lambda case: case.description, +) +def test_given_changed_paths_when_scoping_then_the_replay_matches_what_they_feed( + test_case: ScopeTestCase, +) -> None: + scope: str = replay_scope(paths=list(test_case.changed), binary_dirs=_BINARY_DIRS) + assert scope == test_case.expected_scope + + +@pytest.mark.parametrize( + "test_case", + ( + ChecksTestCase( + description="a docs page needs only the site", + changed=("website/src/content/docs/index.mdx",), + expected_code=False, + expected_site=True, + ), + ChecksTestCase( + description="the README needs neither the code checks nor the site", + changed=("README.md",), + expected_code=False, + expected_site=False, + ), + ChecksTestCase( + description="a Markdown file inside a crate is code", + changed=("crates/sqlquack/README.md",), + expected_code=True, + expected_site=False, + ), + ChecksTestCase( + description="a crate the engine is built from needs both", + changed=("crates/sqlquack-convert/src/lib.rs",), + expected_code=True, + expected_site=True, + ), + ChecksTestCase( + description="a workflow is code", + changed=(".github/workflows/ci.yml",), + expected_code=True, + expected_site=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_changed_paths_when_planning_checks_then_only_what_they_need_runs( + test_case: ChecksTestCase, +) -> None: + paths: list[str] = list(test_case.changed) + planned: tuple[bool, bool] = ( + needs_code_checks(paths=paths), + needs_site(paths=paths, site_dirs=_SITE_DIRS), + ) + assert planned == (test_case.expected_code, test_case.expected_site) + + +@pytest.mark.parametrize( + "test_case", + ( + DigestTestCase( + description="a forced file's content", edited="Cargo.lock", expected_changed=True + ), + DigestTestCase( + description="a file in a forced directory", + edited="crates/sqlquack-evidence/src/lib.rs", + expected_changed=True, + ), + DigestTestCase( + description="a converter file", + edited="crates/sqlquack-convert/src/lib.rs", + expected_changed=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_an_edit_when_digesting_the_scope_then_only_forced_paths_change_it( + tmp_path: Path, test_case: DigestTestCase +) -> None: + root: Path = committed_tree( + root=tmp_path, + files={ + "Cargo.lock": "a", + "crates/sqlquack-evidence/src/lib.rs": "b", + "crates/sqlquack-convert/src/lib.rs": "c", + }, + ) + before: str = scope_digest(root=root) + (root / test_case.edited).write_text("edited") + assert (scope_digest(root=root) != before) == test_case.expected_changed + + +@pytest.mark.parametrize( + "test_case", + ( + RepositoryTestCase( + description="every declared path is tracked, a file under each forces a full replay, " + "and the binary is built from the converter and the replay but not the linter", + expected_unmatched=(), + expected_forced_scopes=frozenset({"full"}), + expected_binary_crates=(True, True, False), + ), + ), + ids=lambda case: case.description, +) +def test_given_the_repository_when_checking_the_declared_paths_then_each_is_honoured( + test_case: RepositoryTestCase, +) -> None: + binary_dirs: frozenset[str] = crate_dirs(root=_REPOSITORY, packages=("sqlquack",)) + forced_files: list[str] = [ + tracked(root=_REPOSITORY, prefixes=(prefix,))[0] for prefix in FULL_REPLAY_PATHS + ] + forced_scopes: frozenset[str] = frozenset( + replay_scope(paths=[path], binary_dirs=binary_dirs) for path in forced_files + ) + binary_crates: tuple[bool, bool, bool] = ( + "crates/sqlquack-convert/" in binary_dirs, + "crates/sqlquack-evidence/" in binary_dirs, + "crates/sqlquack-lint/" in binary_dirs, + ) + assert (tuple(unmatched(root=_REPOSITORY)), forced_scopes, binary_crates) == ( + test_case.expected_unmatched, + test_case.expected_forced_scopes, + test_case.expected_binary_crates, + ) + + +@pytest.mark.parametrize( + "test_case", + ( + RenameTestCase( + description="a file moved out of a crate into the docs lists both of its paths", + source="crates/sqlquack-evidence/notes.md", + destination="docs/notes.md", + expected_changed=("crates/sqlquack-evidence/notes.md", "docs/notes.md"), + ), + ), + ids=lambda case: case.description, +) +def test_given_a_rename_when_listing_changes_then_both_paths_are_listed( + tmp_path: Path, test_case: RenameTestCase +) -> None: + root: Path = committed_tree(root=tmp_path, files={test_case.source: "notes"}) + (root / test_case.destination).parent.mkdir(parents=True, exist_ok=True) + (root / test_case.source).rename(root / test_case.destination) + commit_all(root=root) + assert tuple(changed(root=root, base="HEAD~1")) == test_case.expected_changed