Skip to content

Commit 518cdb9

Browse files
authored
Merge branch 'main' into perf/chi-607-orchestration
2 parents 1ef47ad + 93da704 commit 518cdb9

8 files changed

Lines changed: 562 additions & 192 deletions

File tree

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
name: Build Distributions
2+
3+
# The single owner of the wheel platform matrix, the maturin settings and the distribution smoke
4+
# tests. Verify builds the wheels for pull requests that touch the native build, Release
5+
# Distributions builds and keeps every distribution for a release pull request, and Publish either
6+
# smoke tests and reuses those distributions or builds them here from the release tag.
7+
on:
8+
workflow_call:
9+
inputs:
10+
ref:
11+
description: "Commit, tag or branch to build; empty builds the triggering commit"
12+
required: false
13+
type: string
14+
default: ""
15+
sdist:
16+
description: "Also build the source distributions"
17+
required: false
18+
type: boolean
19+
default: true
20+
upload:
21+
description: "Upload the distributions and a manifest recording their source commit and tree"
22+
required: false
23+
type: boolean
24+
default: false
25+
prebuilt-run-id:
26+
description: >-
27+
Release Distributions run whose distributions are smoke tested and uploaded instead of
28+
building; its manifest must record the tree of ref. The caller grants actions: read.
29+
required: false
30+
type: string
31+
default: ""
32+
retention-days:
33+
description: "Retention of uploaded artifacts in days; 0 uses the repository default"
34+
required: false
35+
type: number
36+
default: 0
37+
outputs:
38+
sha:
39+
description: "Commit the distributions were built from"
40+
value: ${{ jobs.source.outputs.sha }}
41+
tree:
42+
description: "Tree of the commit the distributions were built from"
43+
value: ${{ jobs.source.outputs.tree }}
44+
45+
jobs:
46+
source:
47+
name: Resolve source
48+
runs-on: ubuntu-24.04
49+
outputs:
50+
sha: ${{ steps.source.outputs.sha }}
51+
tree: ${{ steps.source.outputs.tree }}
52+
steps:
53+
- uses: actions/checkout@v4
54+
with:
55+
ref: ${{ inputs.ref }}
56+
- name: Pin the source commit
57+
id: source
58+
run: |
59+
sha="$(git rev-parse HEAD)"
60+
tree="$(git rev-parse 'HEAD^{tree}')"
61+
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
62+
echo "tree=${tree}" >> "${GITHUB_OUTPUT}"
63+
echo "Distributions for commit ${sha} with tree ${tree}" >> "${GITHUB_STEP_SUMMARY}"
64+
- name: Download the prebuilt manifest
65+
if: inputs.prebuilt-run-id != ''
66+
uses: actions/download-artifact@v4
67+
with:
68+
name: release-manifest
69+
path: ${{ runner.temp }}/prebuilt
70+
run-id: ${{ inputs.prebuilt-run-id }}
71+
github-token: ${{ github.token }}
72+
- name: Require prebuilt distributions from this exact tree
73+
if: inputs.prebuilt-run-id != ''
74+
env:
75+
MANIFEST: ${{ runner.temp }}/prebuilt/release-manifest.json
76+
PREBUILT_RUN_ID: ${{ inputs.prebuilt-run-id }}
77+
TREE: ${{ steps.source.outputs.tree }}
78+
run: |
79+
recorded="$(jq -r '.tree // empty' "${MANIFEST}")"
80+
if [[ "${recorded}" != "${TREE}" ]]; then
81+
echo "Run ${PREBUILT_RUN_ID} built tree ${recorded:-<none>}, not ${TREE}" >&2
82+
exit 1
83+
fi
84+
echo "Reusing distributions of run ${PREBUILT_RUN_ID}, built from tree ${TREE}" \
85+
>> "${GITHUB_STEP_SUMMARY}"
86+
87+
wheels:
88+
name: Build wheel (${{ matrix.platform.name }})
89+
needs: source
90+
runs-on: ${{ matrix.platform.runner }}
91+
timeout-minutes: 45
92+
strategy:
93+
fail-fast: false
94+
matrix:
95+
platform:
96+
- name: linux-x86_64
97+
runner: ubuntu-24.04
98+
target: x86_64
99+
- name: linux-aarch64
100+
runner: ubuntu-24.04-arm
101+
target: aarch64
102+
- name: macos-x86_64
103+
runner: macos-15-intel
104+
target: x86_64
105+
- name: macos-arm64
106+
runner: macos-15
107+
target: aarch64
108+
- name: windows-x86_64
109+
runner: windows-2025
110+
target: x86_64
111+
steps:
112+
- uses: actions/checkout@v4
113+
if: inputs.prebuilt-run-id == ''
114+
with:
115+
ref: ${{ needs.source.outputs.sha }}
116+
- uses: actions/setup-python@v5
117+
with:
118+
python-version: "3.12"
119+
- uses: dtolnay/rust-toolchain@master
120+
if: inputs.prebuilt-run-id == ''
121+
with:
122+
toolchain: stable
123+
- uses: PyO3/maturin-action@v1
124+
if: inputs.prebuilt-run-id == ''
125+
with:
126+
command: build
127+
args: --release --out dist --features extension-module
128+
target: ${{ matrix.platform.target }}
129+
manylinux: auto
130+
- uses: PyO3/maturin-action@v1
131+
if: inputs.prebuilt-run-id == ''
132+
with:
133+
command: build
134+
args: --release --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml
135+
target: ${{ matrix.platform.target }}
136+
manylinux: auto
137+
- name: Download prebuilt wheels
138+
if: inputs.prebuilt-run-id != ''
139+
uses: actions/download-artifact@v4
140+
with:
141+
name: dist-wheel-${{ matrix.platform.name }}
142+
path: dist
143+
run-id: ${{ inputs.prebuilt-run-id }}
144+
github-token: ${{ github.token }}
145+
- name: Smoke test installed wheels and native console entry points
146+
shell: bash
147+
run: |
148+
python -m venv "${RUNNER_TEMP}/wheel-smoke"
149+
if [[ -x "${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" ]]; then
150+
smoke_python="${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe"
151+
else
152+
smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python"
153+
fi
154+
"${smoke_python}" -m pip install --find-links dist dist/*.whl
155+
cd "${RUNNER_TEMP}"
156+
"${smoke_python}" -c "import sqlbuild._native"
157+
smoke_scripts="$(dirname "${smoke_python}")"
158+
"${smoke_scripts}/sqb" --version
159+
"${smoke_scripts}/sqb" --help
160+
"${smoke_scripts}/sqlbuild" --help
161+
- name: Check wheel contents and run a native command
162+
if: matrix.platform.name == 'linux-x86_64'
163+
run: |
164+
wheel="$(ls dist/sqlbuild-[0-9]*.whl)"
165+
if unzip -l "${wheel}" | grep -E '(__pycache__/|\.pyc$|\.tmp$)'; then
166+
echo "Wheel contains generated or temporary script files"
167+
exit 1
168+
fi
169+
unzip -l "${wheel}" | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md'
170+
smoke="${RUNNER_TEMP}/wheel-smoke/bin"
171+
project="${RUNNER_TEMP}/sqlbuild-scope-smoke"
172+
cd "${RUNNER_TEMP}"
173+
"${smoke}/python" -c \
174+
"import sqlbuild._native as native; assert native.semantic_help_catalogue()"
175+
mkdir -p "${project}/models"
176+
printf 'name = "scope_smoke"\nadapter = "duckdb"\n' > "${project}/sqlbuild_project.toml"
177+
printf 'MODEL();\nSELECT 1 AS id\n' > "${project}/models/orders.sql"
178+
"${smoke}/sqb" --project-dir "${project}" scope model:orders --json
179+
"${smoke}/sqlbuild" --project-dir "${project}" scope model:orders
180+
- uses: actions/upload-artifact@v4
181+
if: inputs.upload
182+
with:
183+
name: dist-wheel-${{ matrix.platform.name }}
184+
path: dist
185+
if-no-files-found: error
186+
retention-days: ${{ inputs.retention-days }}
187+
188+
sdist:
189+
name: Build sdist
190+
needs: source
191+
if: inputs.sdist
192+
runs-on: ubuntu-24.04
193+
steps:
194+
- uses: actions/checkout@v4
195+
if: inputs.prebuilt-run-id == ''
196+
with:
197+
ref: ${{ needs.source.outputs.sha }}
198+
- uses: dtolnay/rust-toolchain@master
199+
if: inputs.prebuilt-run-id == ''
200+
with:
201+
toolchain: stable
202+
- uses: PyO3/maturin-action@v1
203+
if: inputs.prebuilt-run-id == ''
204+
with:
205+
command: sdist
206+
args: --out dist
207+
- uses: PyO3/maturin-action@v1
208+
if: inputs.prebuilt-run-id == ''
209+
with:
210+
command: sdist
211+
args: --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml
212+
- name: Download prebuilt source distributions
213+
if: inputs.prebuilt-run-id != ''
214+
uses: actions/download-artifact@v4
215+
with:
216+
name: dist-sdist
217+
path: dist
218+
run-id: ${{ inputs.prebuilt-run-id }}
219+
github-token: ${{ github.token }}
220+
- name: Inspect source distribution contents
221+
run: |
222+
for archive in dist/*.tar.gz; do
223+
tar -tzf "$archive" > "${RUNNER_TEMP}/archive-contents.txt"
224+
if grep -E '(__pycache__/|\.pyc$|\.tmp$)' "${RUNNER_TEMP}/archive-contents.txt"; then
225+
echo "Source distribution contains generated or temporary script files"
226+
exit 1
227+
fi
228+
done
229+
- uses: actions/upload-artifact@v4
230+
if: inputs.upload
231+
with:
232+
name: dist-sdist
233+
path: dist
234+
if-no-files-found: error
235+
retention-days: ${{ inputs.retention-days }}
236+
237+
manifest:
238+
name: Record distribution source
239+
needs: [source, wheels, sdist]
240+
if: >-
241+
${{ !cancelled() && inputs.upload && needs.wheels.result == 'success'
242+
&& (needs.sdist.result == 'success' || needs.sdist.result == 'skipped') }}
243+
runs-on: ubuntu-24.04
244+
steps:
245+
- name: Write the manifest
246+
env:
247+
SHA: ${{ needs.source.outputs.sha }}
248+
TREE: ${{ needs.source.outputs.tree }}
249+
run: |
250+
jq -n --arg sha "${SHA}" --arg tree "${TREE}" --arg run_id "${GITHUB_RUN_ID}" \
251+
'{sha: $sha, tree: $tree, run_id: $run_id}' \
252+
> "${RUNNER_TEMP}/release-manifest.json"
253+
cat "${RUNNER_TEMP}/release-manifest.json"
254+
- uses: actions/upload-artifact@v4
255+
with:
256+
name: release-manifest
257+
path: ${{ runner.temp }}/release-manifest.json
258+
if-no-files-found: error
259+
retention-days: ${{ inputs.retention-days }}

0 commit comments

Comments
 (0)