Skip to content
Verify

fix: skip target retention defaults for views #955

Sign in to view logs

fix: skip target retention defaults for views

fix: skip target retention defaults for views #955

Workflow file for this run

.github/workflows/verify.yml at 178b941
name: Verify
on:
pull_request:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: verify-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
changes:
name: Classify verification scope
runs-on: ubuntu-24.04
outputs:
duckdb: ${{ steps.scope.outputs.duckdb }}
postgres: ${{ steps.scope.outputs.postgres }}
performance: ${{ steps.scope.outputs.performance }}
format_performance: ${{ steps.scope.outputs.format_performance }}
rules_performance: ${{ steps.scope.outputs.rules_performance }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Select relevant suites
id: scope
env:
EVENT_NAME: ${{ github.event_name }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
write_scope() {
{
echo "duckdb=$1"
echo "postgres=$2"
echo "performance=$3"
echo "format_performance=$4"
echo "rules_performance=$5"
} >> "${GITHUB_OUTPUT}"
}
if [[ "${EVENT_NAME}" != "pull_request" ]]; then
write_scope true true true true true
exit 0
fi
duckdb=false
postgres=false
performance=false
format_performance=false
rules_performance=false
broad=false
changed=false
while IFS= read -r -d '' path; do
changed=true
case "${path}" in
crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock)
broad=true
performance=true
rules_performance=true
;;
src/sqlbuild/compiler/*|src/sqlbuild/adapter/type_system/*)
broad=true
performance=true
rules_performance=true
;;
src/sqlbuild/cli/commands/_helpers/compile/*|src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|tests/e2e/src/sqlbuild/cli/commands/main/compile/*)
broad=true
performance=true
;;
src/sqlbuild/rule_engine/*|src/sqlbuild/lint/*|scripts/rules_benchmark/*|scripts/benchmark_rules_ci.py|tests/performance/*)
broad=true
performance=true
format_performance=true
rules_performance=true
;;
tests/e2e/src/sqlbuild/cli/commands/main/format/*)
duckdb=true
format_performance=true
;;
tests/e2e/src/sqlbuild/cli/commands/main/postgres/*|tests/integration/src/sqlbuild/adapters/postgres/*|tests/integration/src/sqlbuild/virtual/state/classes/postgres/*|tests/integration/src/sqlbuild/runtime/execution_history/*)
postgres=true
;;
tests/e2e/*)
duckdb=true
;;
tests/unit/*|tests/integration/*)
;;
.github/workflows/verify.yml|Makefile)
performance=true
format_performance=true
rules_performance=true
;;
.github/*|docs/*|*.md|.gitignore|LICENSE)
;;
*)
broad=true
;;
esac
done < <(git diff --name-only -z "${BASE_SHA}" "${HEAD_SHA}")
if [[ "${changed}" != "true" ]]; then
write_scope true true true true true
exit 0
fi
if [[ "${broad}" == "true" ]]; then
duckdb=true
postgres=true
format_performance=true
fi
write_scope \
"${duckdb}" \
"${postgres}" \
"${performance}" \
"${format_performance}" \
"${rules_performance}"
lockfile:
name: Lockfile consistency
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Verify lockfile
run: uv lock --check --python 3.12
checks:
name: Static checks and unit + integration tests
needs: lockfile
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras
- name: Run static checks
run: make check-ci
- name: Run native checks
run: make rust-check
- name: Run unit and integration tests
run: make test
e2e_shards:
name: DuckDB E2E (${{ matrix.name }})
needs: [changes, lockfile]
if: needs.changes.outputs.duckdb == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- name: build core
target: test-e2e-duckdb-build-core
- name: build incremental
target: test-e2e-duckdb-build-incremental
- name: build virtual
target: test-e2e-duckdb-build-virtual
- name: CLI data
target: test-e2e-duckdb-cli-data
- name: CLI commands
target: test-e2e-duckdb-cli
- name: virtual environments
target: test-e2e-duckdb-virtual
- name: integrations
target: test-e2e-duckdb-integrations
env:
PYTEST_XDIST_AUTO_NUM_WORKERS: "4"
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras
- name: Report runner CPU topology
run: |
echo "nproc=$(nproc)"
echo "online_processors=$(getconf _NPROCESSORS_ONLN)"
lscpu
uv run python - <<'PY'
import os
print(f"os.cpu_count={os.cpu_count()}")
print(f"sched_getaffinity={sorted(os.sched_getaffinity(0))}")
print(
"PYTEST_XDIST_AUTO_NUM_WORKERS="
f"{os.environ.get('PYTEST_XDIST_AUTO_NUM_WORKERS', '<unset>')}"
)
PY
- name: Run DuckDB E2E shard
run: make ${{ matrix.target }}
format_performance:
name: Formatter performance guards
needs: [changes, lockfile]
if: needs.changes.outputs.format_performance == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras --python 3.12
- name: Run formatter performance guards
run: >-
uv run pytest
tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_performance.py
-n auto --dist loadfile -m performance
performance:
name: Performance guards
needs: [changes, lockfile]
# Temporarily disabled while the Blacksmith runner allowance is unavailable.
# These budgets are calibrated for the 8-vCPU Blacksmith runner.
if: false
runs-on: ubuntu-24.04
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras
- name: Run performance guards
run: make test-e2e-performance
cold_compile_performance:
name: Fresh-process compile (${{ matrix.models }} models)
needs: [changes, lockfile]
if: needs.changes.outputs.performance == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
models: [3000, 5000, 10000]
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras
- name: Run fresh-process compile guard under 4 GiB
env:
BENCHMARK_MODELS: ${{ matrix.models }}
MEMORY_LIMIT_BYTES: "4294967296"
run: |
sudo systemd-run \
--unit="sqlbuild-cold-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \
--uid="$(id -u)" \
--gid="$(id -g)" \
--same-dir \
--setenv="HOME=${HOME}" \
--setenv="PATH=${PATH}" \
--setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \
--property="MemoryMax=${MEMORY_LIMIT_BYTES}" \
--property="MemorySwapMax=0" \
--wait \
--collect \
--pipe \
make test-e2e-cold-compile-performance \
SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
dense_compile_performance:
name: Dense all-rules compile (${{ matrix.models }} models)
needs: [changes, lockfile]
if: needs.changes.outputs.performance == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
models: [1000, 3000, 5000, 10000]
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras
- name: Run dense all-rules guard under 4 GiB
env:
BENCHMARK_MODELS: ${{ matrix.models }}
MEMORY_LIMIT_BYTES: "4294967296"
run: |
sudo systemd-run \
--unit="sqlbuild-dense-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \
--uid="$(id -u)" \
--gid="$(id -g)" \
--same-dir \
--setenv="HOME=${HOME}" \
--setenv="PATH=${PATH}" \
--setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \
--property="MemoryMax=${MEMORY_LIMIT_BYTES}" \
--property="MemorySwapMax=0" \
--wait \
--collect \
--pipe \
make test-e2e-dense-compile-performance \
SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
cache_compile_performance:
name: Fresh-process compile cache (${{ matrix.models }} models)
needs: [changes, lockfile]
if: needs.changes.outputs.performance == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
models: [3000, 5000, 10000]
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras
- name: Run fresh-process compile cache guard under 4 GiB
env:
BENCHMARK_MODELS: ${{ matrix.models }}
MEMORY_LIMIT_BYTES: "4294967296"
run: |
sudo systemd-run \
--unit="sqlbuild-cache-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \
--uid="$(id -u)" \
--gid="$(id -g)" \
--same-dir \
--setenv="HOME=${HOME}" \
--setenv="PATH=${PATH}" \
--setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \
--property="MemoryMax=${MEMORY_LIMIT_BYTES}" \
--property="MemorySwapMax=0" \
--wait \
--collect \
--pipe \
make test-e2e-cache-compile-performance \
SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
- name: Run varied 1k warm-cache guard under 4 GiB
if: matrix.models == 3000
env:
MEMORY_LIMIT_BYTES: "4294967296"
run: |
sudo systemd-run \
--unit="sqlbuild-varied-cache-${GITHUB_RUN_ID}" \
--uid="$(id -u)" \
--gid="$(id -g)" \
--same-dir \
--setenv="HOME=${HOME}" \
--setenv="PATH=${PATH}" \
--setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \
--property="MemoryMax=${MEMORY_LIMIT_BYTES}" \
--property="MemorySwapMax=0" \
--wait \
--collect \
--pipe \
make test-e2e-varied-cache-performance
rules_performance:
name: Rules performance guards
needs: [changes, lockfile]
# Temporarily disabled while the Blacksmith runner allowance is unavailable.
# These budgets are calibrated for the 8-vCPU Blacksmith runner.
if: false
runs-on: ubuntu-24.04
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras
- name: Run required Rules performance profile
timeout-minutes: 10
run: >-
uv run python -m scripts.benchmark_rules_ci
--output "${RUNNER_TEMP}/rules-performance-ci.json"
--summary-output "${RUNNER_TEMP}/rules-performance-ci.md"
--max-seconds 420
- name: Publish benchmark summary
if: always()
run: |
if [[ -f "${RUNNER_TEMP}/rules-performance-ci.md" ]]; then
cat "${RUNNER_TEMP}/rules-performance-ci.md" >> "${GITHUB_STEP_SUMMARY}"
fi
- name: Upload benchmark evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: rules-performance-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/rules-performance-ci.json
${{ runner.temp }}/rules-performance-ci.md
if-no-files-found: error
retention-days: 90
postgres:
name: PostgreSQL integration and E2E tests
needs: [changes, lockfile]
if: needs.changes.outputs.postgres == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras
- name: Verify Docker
run: docker info
- name: Run PostgreSQL tests
env:
PYTHONUNBUFFERED: "1"
TESTCONTAINERS_RYUK_DISABLED: "true"
run: >-
uv run pytest tests
-m "real_warehouse and postgres"
-vv --color=yes -n auto --dist loadfile
verify:
name: Verify
if: always()
needs:
[
changes,
lockfile,
checks,
e2e_shards,
format_performance,
performance,
cold_compile_performance,
dense_compile_performance,
cache_compile_performance,
rules_performance,
postgres,
]
runs-on: ubuntu-24.04
steps:
- name: Confirm all required verification jobs passed
env:
CHANGES_RESULT: ${{ needs.changes.result }}
LOCKFILE_RESULT: ${{ needs.lockfile.result }}
CHECKS_RESULT: ${{ needs.checks.result }}
E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }}
FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }}
PERFORMANCE_RESULT: ${{ needs.performance.result }}
COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
DENSE_COMPILE_PERFORMANCE_RESULT: ${{ needs.dense_compile_performance.result }}
CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }}
RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }}
POSTGRES_RESULT: ${{ needs.postgres.result }}
run: |
if [[ "${CHANGES_RESULT}" != "success" || \
"${LOCKFILE_RESULT}" != "success" || \
"${CHECKS_RESULT}" != "success" ]]; then
echo "Required classification, lockfile, or static checks did not pass" >&2
exit 1
fi
for result in \
"${E2E_SHARDS_RESULT}" \
"${FORMAT_PERFORMANCE_RESULT}" \
"${PERFORMANCE_RESULT}" \
"${COLD_COMPILE_PERFORMANCE_RESULT}" \
"${DENSE_COMPILE_PERFORMANCE_RESULT}" \
"${CACHE_COMPILE_PERFORMANCE_RESULT}" \
"${RULES_PERFORMANCE_RESULT}" \
"${POSTGRES_RESULT}"; do
if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then
echo "A selected verification suite did not pass: ${result}" >&2
exit 1
fi
done
echo "All path-relevant verification jobs passed"