Stop committing Cargo.lock #22
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bench | |
| # Build tract on GitHub-hosted runners, run the bench suite on the self-hosted bench | |
| # machines, and either (nightly) append one row per run to the bench-data branch, or | |
| # (PR) compare against that reference and post a single fan-in comment. | |
| # The Jetson (aarch64 + CUDA 12) can't be built on a hosted runner natively, so it | |
| # reuses cross.sh's debian-stretch cross-build. | |
| on: | |
| schedule: | |
| - cron: '0 4 * * *' # after large-models (3am) so they don't fight for the self-hosted boxes | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: "Optional PR number to bench (from fork ok). Leave empty to run on selected branch." | |
| required: false | |
| type: number | |
| mt_ladder: | |
| description: "Thread-count ladder (comma list, 0=physical cores) for a scaling run; empty = normal vs-main bench." | |
| required: false | |
| type: string | |
| default: '' | |
| pull_request: | |
| paths-ignore: | |
| - '**/*.md' | |
| - 'doc/**' | |
| - '.github/**' | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # cancel superseded PR pushes (escape hatch); nightly keys on run_id so it never cancels | |
| group: bench-${{ github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event.inputs.pr_number != '' }} | |
| jobs: | |
| prepare: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| enabled: ${{ steps.gate.outputs.enabled }} | |
| ref: ${{ steps.gate.outputs.ref }} | |
| day: ${{ steps.gate.outputs.day }} | |
| pr_number: ${{ steps.gate.outputs.pr_number }} | |
| is_pr: ${{ steps.gate.outputs.is_pr }} | |
| same_repo: ${{ steps.gate.outputs.same_repo }} | |
| mt_ladder: ${{ steps.gate.outputs.mt_ladder }} | |
| steps: | |
| # Resolves this run's target ref and whether it should behave like a PR run | |
| # (compare + comment) or a reference run (append to bench-data): a manual | |
| # `/ci bench` dispatch on a PR must be treated as a PR run even though its | |
| # event_name is workflow_dispatch, or its numbers would get pushed to | |
| # bench-data as if it were a nightly/main reference point. | |
| - id: gate | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| BENCH_ENABLED: ${{ vars.BENCH_ENABLED }} | |
| with: | |
| script: | | |
| if (process.env.BENCH_ENABLED === 'false') { | |
| core.notice('bench disabled via BENCH_ENABLED repo variable'); | |
| core.setOutput('enabled', 'false'); | |
| } else { | |
| core.setOutput('enabled', 'true'); | |
| } | |
| core.setOutput('day', new Date().toISOString().slice(0, 10)); | |
| // Set on any event; a pull_request trigger has no inputs, so it stays empty | |
| // (normal vs-main bench). Only a workflow_dispatch can request a scaling run. | |
| core.setOutput('mt_ladder', context.payload.inputs?.mt_ladder || ''); | |
| const baseRepo = `${context.repo.owner}/${context.repo.repo}`; | |
| if (context.eventName === 'pull_request') { | |
| const pr = context.payload.pull_request; | |
| core.setOutput('is_pr', 'true'); | |
| core.setOutput('pr_number', String(pr.number)); | |
| core.setOutput('same_repo', String(pr.head.repo.full_name === baseRepo)); | |
| core.setOutput('ref', pr.head.sha); | |
| return; | |
| } | |
| const prInput = context.payload.inputs?.pr_number; | |
| if (!prInput) { | |
| core.setOutput('is_pr', 'false'); | |
| core.setOutput('pr_number', ''); | |
| core.setOutput('same_repo', 'true'); | |
| core.setOutput('ref', process.env.GITHUB_SHA); | |
| return; | |
| } | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: Number(prInput), | |
| }); | |
| core.setOutput('is_pr', 'true'); | |
| core.setOutput('pr_number', String(prInput)); | |
| core.setOutput('same_repo', String(pr.data.head.repo.full_name === baseRepo)); | |
| core.setOutput('ref', pr.data.head.sha); | |
| build: | |
| needs: prepare | |
| if: needs.prepare.outputs.enabled == 'true' | |
| runs-on: ${{ matrix.host }} | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| packages: read # pull the prebuilt cross-build image (ghcr, private) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - triple: aarch64-apple-darwin | |
| host: macos-latest | |
| build: ROOT=. ./.travis/ci-system-setup.sh && cargo build -p tract-cli --release --features bench-suite | |
| bin: target/release/tract | |
| - triple: x86_64-unknown-linux-gnu | |
| host: ubuntu-latest | |
| build: ROOT=. ./.travis/ci-system-setup.sh && cargo build -p tract-cli --release --features bench-suite | |
| bin: target/release/tract | |
| - triple: aarch64-unknown-linux-gnu-stretch | |
| host: ubuntu-latest | |
| build: CARGO_TARGET_DIR=.cross-cache/target PLATFORM=aarch64-unknown-linux-gnu-stretch ./.travis/cross.sh | |
| bin: .cross-cache/target/aarch64-unknown-linux-gnu/release/tract | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.prepare.outputs.ref }} | |
| persist-credentials: false | |
| - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 | |
| with: | |
| # per-PR build on the developer feedback path; PR runs restore but only main saves. | |
| # The stretch leg cross-builds in a debian-stretch container into .cross-cache/target | |
| # (CARGO_TARGET_DIR); cache that verbatim so the container's aarch64 artifacts survive — | |
| # rust-cache's own target/ cleanup is host-target-only and would drop them. The container | |
| # also bind-mounts the host cargo registry (see cross.sh) so crate downloads persist. | |
| # -gpu: keep the cuda build's cache off the bare-triple key the no-cuda builds share. | |
| key: ${{ matrix.triple }}-gpu | |
| cache-directories: .cross-cache/target | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - run: ${{ matrix.build }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # ghcr login for the cross-build image | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: tract-cli-${{ matrix.triple }} | |
| path: ${{ matrix.bin }} | |
| bench: | |
| needs: [ prepare, build ] | |
| if: needs.prepare.outputs.enabled == 'true' | |
| runs-on: ${{ matrix.runner }} | |
| permissions: | |
| contents: write # push to bench-data (nightly only) | |
| # No concurrency group: each box is a single self-hosted agent, so the agent | |
| # serialises bench/hwbench/smoke itself (queues, never cancels). A shared | |
| # group would instead cancel one PR's queued bench when another PR's lands. | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - target: apple-m1-max | |
| runner: macOS | |
| triple: aarch64-apple-darwin | |
| - target: i9-11900kb_rtx-4060 | |
| runner: cuda-lovelace | |
| triple: x86_64-unknown-linux-gnu | |
| - target: jetson-orin-nx | |
| runner: orin-nx-16g | |
| triple: aarch64-unknown-linux-gnu-stretch | |
| # GPU-only box; skip the CPU net suite (redundant with apple-m1-max). | |
| bench_args: --skip-cpu | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.prepare.outputs.ref }} | |
| persist-credentials: false | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: tract-cli-${{ matrix.triple }} | |
| path: tract-cli | |
| # Read-only bench-data checkout: the orchestrator builds this device's retry | |
| # expectations from it inline (no separate job, no Python on the runner). | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: bench-data | |
| path: bench-ref | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| - name: Run bench suite | |
| env: | |
| TRIPLE: ${{ matrix.triple }} | |
| DEVICE: ${{ matrix.target }} | |
| MT_LADDER: ${{ needs.prepare.outputs.mt_ladder }} | |
| run: | | |
| chmod +x tract-cli/tract | |
| tract-cli/tract bench-suite \ | |
| --manifest .travis/benches.toml \ | |
| --bench-data bench-ref \ | |
| --thresholds .travis/bench-thresholds.toml \ | |
| --triple "$TRIPLE" --device "$DEVICE" \ | |
| ${MT_LADDER:+--threads-ladder "$MT_LADDER"} \ | |
| ${{ needs.prepare.outputs.is_pr != 'true' && '--samples 5' || '' }} \ | |
| ${{ matrix.bench_args }} # produces ./metrics | |
| # --- nightly / dispatch: append the row to bench-data --- | |
| - name: Checkout bench-data | |
| if: needs.prepare.outputs.is_pr != 'true' | |
| # zizmor: ignore[artipacked] credentials are needed to push the row back to | |
| # bench-data; this checkout is never uploaded as an artifact. | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: bench-data | |
| path: bench-data | |
| fetch-depth: 1 | |
| - name: Append and push | |
| if: needs.prepare.outputs.is_pr != 'true' | |
| env: | |
| DEVICE: ${{ matrix.target }} | |
| TRIPLE: ${{ matrix.triple }} | |
| REF: ${{ needs.prepare.outputs.ref }} | |
| DAY: ${{ needs.prepare.outputs.day }} | |
| run: | | |
| tract-cli/tract bench-append --metrics metrics --out bench-data \ | |
| --triple "$TRIPLE" --device "$DEVICE" --day "$DAY" | |
| cd bench-data | |
| git config user.name "tract-ci" | |
| git config user.email "tract-ci@users.noreply.github.com" | |
| git add -A | |
| git commit -m "bench: $DEVICE $DAY ${REF:0:9}" | |
| for i in 1 2 3 4 5; do | |
| git push origin HEAD:bench-data && exit 0 | |
| git fetch origin bench-data | |
| git rebase origin/bench-data || git rebase --abort | |
| done | |
| echo "push failed after retries" >&2; exit 1 | |
| # --- PR: emit this device's result for the fan-in report job --- | |
| - name: Stage PR result | |
| if: needs.prepare.outputs.is_pr == 'true' | |
| env: | |
| DEVICE: ${{ matrix.target }} | |
| TRIPLE: ${{ matrix.triple }} | |
| run: | | |
| mkdir -p result | |
| cp metrics result/metrics | |
| printf '{"device":"%s","triple":"%s"}\n' "$DEVICE" "$TRIPLE" > result/meta.json | |
| - name: Upload PR result | |
| if: needs.prepare.outputs.is_pr == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: bench-result-${{ matrix.target }} | |
| path: result | |
| # --- dinghy target fleet (kill switch: BENCH_DINGHY_ENABLED=false). --- | |
| # A sidekick runner cross-runs the CLI on each target over dinghy's ssh transport; per-target | |
| # config lives in the sidekick's .dinghy.toml. On by default; opt-out only, so fork PRs (which | |
| # don't inherit the repo variable) still run the ARM boards. | |
| build-dinghy: | |
| needs: prepare | |
| if: ${{ needs.prepare.outputs.enabled == 'true' && vars.BENCH_DINGHY_ENABLED != 'false' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - triple: aarch64-unknown-linux-gnu-stretch | |
| rustc_triple: aarch64-unknown-linux-gnu | |
| - triple: armv7-unknown-linux-gnueabihf-stretch | |
| rustc_triple: armv7-unknown-linux-gnueabihf | |
| # RISC-V boards run a fully-static musl CLI (built directly by cross.sh, no stretch | |
| # container); one self-tuning build serves every rv64gc board. The RVV kernels are | |
| # picked at runtime off the hart's VLEN, so this same binary runs them on an RVV 1.0 | |
| # board and the generic scalar path where the vector unit is absent or the 0.7.1 draft. | |
| - triple: riscv64gc-unknown-linux-musl | |
| rustc_triple: riscv64gc-unknown-linux-musl | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.prepare.outputs.ref }} | |
| persist-credentials: false | |
| - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 | |
| with: | |
| key: ${{ matrix.triple }} | |
| cache-directories: .cross-cache/target | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| # glibc-stretch CLI for the board (matches the production devices), cross-built in the | |
| # debian-stretch container: bench-suite on, cuda/tflite off; SKIP_QEMU_TEST builds only. | |
| # TRACT_CLI_FEATURES suppresses the stretch leg's default cuda forcing (see cross.sh). | |
| - run: >- | |
| SKIP_QEMU_TEST=skip SUDO=sudo CARGO_TARGET_DIR=.cross-cache/target | |
| TRACT_CLI_FEATURES=onnx,tf,pulse,pulse-opl,transformers,extra,bench-suite | |
| PLATFORM=${{ matrix.triple }} ./.travis/cross.sh | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: tract-cli-${{ matrix.triple }}-dinghy | |
| path: .cross-cache/target/${{ matrix.rustc_triple }}/release/tract | |
| bench-dinghy: | |
| needs: [ prepare, build-dinghy ] | |
| if: ${{ needs.prepare.outputs.enabled == 'true' && vars.BENCH_DINGHY_ENABLED != 'false' }} | |
| # One sidekick runner per target (label per board), so boards bench in parallel. | |
| runs-on: ${{ matrix.runner }} | |
| # Bound the run so an offline/stuck target can't hang the PR report (which fans this in). | |
| timeout-minutes: 120 | |
| # No concurrency group: one sidekick agent per device serialises its own work. | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - device: cortex-a55 | |
| triple: aarch64-unknown-linux-gnu-stretch | |
| runner: dinghy-sidekick-a55 | |
| dinghy_target: dinghy-a55 | |
| - device: cortex-a53 | |
| triple: aarch64-unknown-linux-gnu-stretch | |
| runner: dinghy-sidekick-a53 | |
| dinghy_target: dinghy-a53 | |
| - device: cortex-a7 | |
| triple: armv7-unknown-linux-gnueabihf-stretch | |
| runner: dinghy-sidekick-a7 | |
| dinghy_target: dinghy-a7 | |
| - device: cortex-a9 | |
| triple: armv7-unknown-linux-gnueabihf-stretch | |
| runner: dinghy-sidekick-a9 | |
| dinghy_target: dinghy-a9 | |
| - device: beaglev-ahead | |
| triple: riscv64gc-unknown-linux-musl | |
| runner: dinghy-sidekick-rv-beaglev | |
| dinghy_target: dinghy-rv-beaglev | |
| # Orange Pi RV2 (SpacemiT K1, RVV 1.0), same riscv64 musl build. | |
| - device: orangepi-rv2 | |
| triple: riscv64gc-unknown-linux-musl | |
| runner: dinghy-sidekick-rv-rv2 | |
| dinghy_target: dinghy-rv-rv2 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.prepare.outputs.ref }} | |
| persist-credentials: false | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: tract-cli-${{ matrix.triple }}-dinghy | |
| path: tract-cli | |
| - name: Bench on the dinghy target | |
| env: | |
| DINGHY_TARGET: ${{ matrix.dinghy_target }} | |
| DEVICE: ${{ matrix.device }} | |
| TRIPLE: ${{ matrix.triple }} | |
| SAMPLES: ${{ needs.prepare.outputs.is_pr != 'true' && '--samples 5' || '' }} | |
| MT_LADDER: ${{ needs.prepare.outputs.mt_ladder }} | |
| run: | | |
| chmod +x tract-cli/tract | |
| mkdir -p result | |
| # dinghy otherwise copies the whole checkout to the board (source + every committed | |
| # test model), which overflows the small armv7 boards and fails the rsync deploy. | |
| # Scope the copy to just the bench manifest with a gitignore-style allowlist; the exe | |
| # is copied separately. Written into the ephemeral work folder, never committed: a | |
| # repo .dinghyignore would strip fixtures from everyone else's `cargo dinghy test`. | |
| printf '%s\n' '/*' '!/.travis/' '/.travis/*' '!/.travis/benches.toml' > .dinghyignore | |
| # --cleanup also tears the deploy down afterwards. It must precede -d: after -d | |
| # consumes its value cargo-dinghy misparses a later -c and bails with -Zscript. | |
| if CARGO_PKG_NAME=tract cargo-dinghy --cleanup -d "$DINGHY_TARGET" runner \ | |
| ./tract-cli/tract -- bench-suite \ | |
| --manifest .travis/benches.toml --skip-runtimes --no-cache --output - $SAMPLES \ | |
| ${MT_LADDER:+--threads-ladder "$MT_LADDER"} \ | |
| --base-url '${TRACT_BENCH_BASE_URL}' --cpu-governor '${TRACT_BENCH_CPU_GOVERNOR}' \ | |
| >captured.out 2>captured.err | |
| then | |
| grep '"metric"' captured.out > result/metrics || true | |
| else | |
| echo "::error::bench-dinghy: run failed" | |
| exit 1 | |
| fi | |
| test -s result/metrics || { echo "::error::bench-dinghy: no metrics captured"; exit 1; } | |
| printf '{"device":"%s","triple":"%s"}\n' "$DEVICE" "$TRIPLE" > result/meta.json | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: bench-result-${{ matrix.device }} | |
| path: result | |
| # Nightly only: append the dinghy target's row to bench-data. The board can't run the append tool, | |
| # so a hosted job does it with the x86 CLI (bench-append is arch-agnostic JSON) reading the JSONL result. | |
| append-dinghy: | |
| needs: [ prepare, build, bench-dinghy ] | |
| # always() (like the report job) so a cancelled/failed device leg (e.g. one board that | |
| # timed out) still lets the boards that did finish append; each leg no-ops when its own | |
| # result artifact is absent. | |
| if: ${{ always() && needs.prepare.outputs.is_pr != 'true' && needs.prepare.outputs.enabled == 'true' && vars.BENCH_DINGHY_ENABLED != 'false' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - device: cortex-a55 | |
| triple: aarch64-unknown-linux-gnu-stretch | |
| - device: cortex-a53 | |
| triple: aarch64-unknown-linux-gnu-stretch | |
| - device: cortex-a7 | |
| triple: armv7-unknown-linux-gnueabihf-stretch | |
| - device: cortex-a9 | |
| triple: armv7-unknown-linux-gnueabihf-stretch | |
| - device: beaglev-ahead | |
| triple: riscv64gc-unknown-linux-musl | |
| - device: orangepi-rv2 | |
| triple: riscv64gc-unknown-linux-musl | |
| steps: | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: tract-cli-x86_64-unknown-linux-gnu | |
| path: tract-cli | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| # A leg whose bench never produced a result (timed out / failed) has no artifact; | |
| # tolerate that here and skip the append below rather than failing the whole job. | |
| continue-on-error: true | |
| with: | |
| name: bench-result-${{ matrix.device }} | |
| path: result | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| # zizmor: ignore[artipacked] credentials are needed to push the row back to bench-data; | |
| # this checkout is never uploaded as an artifact. | |
| if: ${{ hashFiles('result/metrics') != '' }} | |
| with: | |
| ref: bench-data | |
| path: bench-data | |
| fetch-depth: 1 | |
| - name: Append and push | |
| if: ${{ hashFiles('result/metrics') != '' }} | |
| env: | |
| DEVICE: ${{ matrix.device }} | |
| TRIPLE: ${{ matrix.triple }} | |
| REF: ${{ needs.prepare.outputs.ref }} | |
| DAY: ${{ needs.prepare.outputs.day }} | |
| run: | | |
| chmod +x tract-cli/tract | |
| tract-cli/tract bench-append --metrics result/metrics --out bench-data \ | |
| --triple "$TRIPLE" --device "$DEVICE" --day "$DAY" | |
| cd bench-data | |
| git config user.name "tract-ci" | |
| git config user.email "tract-ci@users.noreply.github.com" | |
| git add -A | |
| git commit -m "bench: $DEVICE $DAY ${REF:0:9}" | |
| for i in 1 2 3 4 5; do | |
| git push origin HEAD:bench-data && exit 0 | |
| git fetch origin bench-data | |
| git rebase origin/bench-data || git rebase --abort | |
| done | |
| echo "push failed after retries" >&2; exit 1 | |
| report: | |
| needs: [ prepare, build, bench, bench-dinghy ] | |
| # always() (not !cancelled()) so a cancelled/failed device leg still reports from the rest; | |
| # bench-report writes no file with nothing to compare, so a fully-cancelled run posts nothing. | |
| if: ${{ always() && needs.prepare.outputs.is_pr == 'true' && needs.prepare.outputs.enabled == 'true' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: write # post the comment directly on same-repo PRs | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.prepare.outputs.ref }} | |
| persist-credentials: false | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: bench-result-* | |
| path: results | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: tract-cli-x86_64-unknown-linux-gnu | |
| path: tract-cli | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: bench-data | |
| path: bench-data | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| # An mt-ladder run renders a self-contained thread-scaling table (speed + speedup | |
| # vs serial, no bench-data reference); a normal run renders the vs-main comparison. | |
| - name: Render comparison | |
| env: | |
| PR_SHA: ${{ needs.prepare.outputs.ref }} | |
| MT_LADDER: ${{ needs.prepare.outputs.mt_ladder }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| chmod +x tract-cli/tract | |
| if [ -n "$MT_LADDER" ]; then | |
| tract-cli/tract bench-mt-report --results results --pr-sha "$PR_SHA" --out pr-comment.md | |
| else | |
| tract-cli/tract bench-report --results results --bench-data bench-data \ | |
| --thresholds .travis/bench-thresholds.toml --pr-sha "$PR_SHA" --out pr-comment.md | |
| fi | |
| # Only an automatic `pull_request` event on a fork gets a read-only token; every other | |
| # trigger (same-repo pull_request, or any workflow_dispatch — including /ci bench, | |
| # which always runs with a write token regardless of the target PR's fork status) can | |
| # post directly. A fresh comment per run (like the fork trampoline) so the PR thread | |
| # keeps the full history in chronological order instead of overwriting the previous result. | |
| - name: Comment on PR (direct) | |
| if: ${{ hashFiles('pr-comment.md') != '' && (github.event_name != 'pull_request' || needs.prepare.outputs.same_repo == 'true') }} | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| PR_NUMBER: ${{ needs.prepare.outputs.pr_number }} | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const body = fs.readFileSync('pr-comment.md', 'utf8'); | |
| const issue_number = parseInt(process.env.PR_NUMBER, 10); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, issue_number, body, | |
| }); | |
| # Fork PRs on the automatic pull_request event get a read-only token and can't comment, | |
| # so hand the rendered comment + target PR to the bench-comment trampoline (workflow_run, | |
| # base context, write token) to post instead. | |
| - name: Stage comment for the trampoline (fork PRs) | |
| if: ${{ hashFiles('pr-comment.md') != '' && github.event_name == 'pull_request' && needs.prepare.outputs.same_repo != 'true' }} | |
| env: | |
| PR_NUMBER: ${{ needs.prepare.outputs.pr_number }} | |
| run: | | |
| mkdir -p bench-comment | |
| mv pr-comment.md bench-comment/ | |
| printf '%s\n' "$PR_NUMBER" > bench-comment/pr-number | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: ${{ hashFiles('bench-comment/pr-comment.md') != '' }} | |
| with: | |
| name: bench-comment | |
| path: bench-comment | |
| # --- daily kernel-pick gate: fail if the dispatcher's matmul pick lags the fastest | |
| # available kernel for a shape by more than the tolerance. Reuses the bench binaries; | |
| # self-referential (picked vs best-in-pool), so no golden numbers to maintain. Nightly | |
| # only for now. Shares the per-box concurrency group so it never fights a running bench. | |
| hwbench: | |
| needs: [ prepare, build ] | |
| if: ${{ needs.prepare.outputs.enabled == 'true' && github.event_name == 'schedule' }} | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - target: apple-m1-max | |
| runner: macOS | |
| triple: aarch64-apple-darwin | |
| - target: i9-11900kb_rtx-4060 | |
| runner: cuda-lovelace | |
| triple: x86_64-unknown-linux-gnu | |
| - target: jetson-orin-nx | |
| runner: orin-nx-16g | |
| triple: aarch64-unknown-linux-gnu-stretch | |
| steps: | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: tract-cli-${{ matrix.triple }} | |
| path: tract-cli | |
| - name: Assert kernel picks | |
| run: | | |
| chmod +x tract-cli/tract | |
| tract-cli/tract hwbench --no-cache --no-memory --assert --tolerance 20 | |
| # --- daily wasm kernel-pick gate. The wasm kernels and their packings are picked at | |
| # compile time on simd128 / relaxed-simd, so no native cohort exercises that half of | |
| # the dispatcher, and without either the pool is portable Rust alone. Build the CLI for | |
| # wasm32-wasip1 in each configuration on a hosted runner, then assert the picks under | |
| # wasmtime on the bench box. Self-referential (picked vs best in pool), so there is | |
| # nothing to keep in bench-data. | |
| build-wasm: | |
| needs: prepare | |
| if: ${{ needs.prepare.outputs.enabled == 'true' && github.event_name == 'schedule' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - config: no-simd | |
| rustflags: "" | |
| - config: simd128 | |
| rustflags: -C target-feature=+simd128 | |
| - config: relaxed-simd | |
| rustflags: -C target-feature=+simd128,+relaxed-simd | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.prepare.outputs.ref }} | |
| persist-credentials: false | |
| - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 | |
| with: | |
| key: wasm32-wasip1-${{ matrix.config }} | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| # hwbench needs no model loader, so the featureless CLI keeps the build small. | |
| - env: | |
| RUSTFLAGS: ${{ matrix.rustflags }} | |
| run: | | |
| rustup target add wasm32-wasip1 | |
| cargo build -p tract-cli --release --target wasm32-wasip1 --no-default-features | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: tract-wasm-${{ matrix.config }} | |
| path: target/wasm32-wasip1/release/tract.wasm | |
| hwbench-wasm: | |
| needs: [ prepare, build-wasm ] | |
| if: ${{ needs.prepare.outputs.enabled == 'true' && github.event_name == 'schedule' }} | |
| # The x86 bench box, not a hosted runner: the pick ratios are stable to ~2 points | |
| # there and swing by 2x when something else is compiling on the machine. | |
| runs-on: cuda-lovelace | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: [ no-simd, simd128, relaxed-simd ] | |
| steps: | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: tract-wasm-${{ matrix.config }} | |
| path: tract-wasm | |
| # Pinned: a wasmtime bump changes Cranelift's codegen and can reorder the pool, | |
| # so it is a deliberate step, not whatever the box picked up last. | |
| - name: Setup wasmtime | |
| env: | |
| WASMTIME_VERSION: v48.0.0 | |
| run: | | |
| curl -sSL "https://github.com/bytecodealliance/wasmtime/releases/download/$WASMTIME_VERSION/wasmtime-$WASMTIME_VERSION-x86_64-linux.tar.xz" | tar xJ | |
| echo "$PWD/wasmtime-$WASMTIME_VERSION-x86_64-linux" >> "$GITHUB_PATH" | |
| # The bandwidth probes are threaded and wasip1 has no threads, hence --no-cache | |
| # --no-memory: the matmul battery is the whole point here anyway. --include-generic | |
| # because autovectorised Rust is a real contender here, and the only pool at all | |
| # without simd128. | |
| - name: Report the whole battery | |
| run: | | |
| wasmtime run tract-wasm/tract.wasm hwbench --no-cache --no-memory --include-generic | |
| # Gated on the f32 shapes only. wasm has no real f16 kernel, so f16 runs on the | |
| # portable pool, where the dispatcher takes generic_f32_4x4 at 63-76% of | |
| # generic_f32_4x1 — a live mispick, reported by the step above but not gated until | |
| # it is fixed. Then this list goes away and the whole battery gates. | |
| - name: Assert kernel picks under wasmtime | |
| run: | | |
| wasmtime run tract-wasm/tract.wasm hwbench \ | |
| 512,512,120,f32 256,256,256,f32 192,288,1225,f32 64,64,64,f32 \ | |
| --no-cache --no-memory --include-generic --assert --tolerance 20 | |
| # --- daily load-and-run gate: every model in the bench manifest must load, | |
| # optimize and run once on each runtime the box offers (CPU + accelerator). | |
| # Reuses the bench binaries; no golden numbers, just exit status. Nightly only, | |
| # sharing the per-box concurrency group so it never fights a running bench. | |
| smoke: | |
| needs: [ prepare, build ] | |
| if: ${{ needs.prepare.outputs.enabled == 'true' && github.event_name == 'schedule' }} | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 90 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - target: apple-m1-max | |
| runner: macOS | |
| triple: aarch64-apple-darwin | |
| - target: i9-11900kb_rtx-4060 | |
| runner: cuda-lovelace | |
| triple: x86_64-unknown-linux-gnu | |
| - target: jetson-orin-nx | |
| runner: orin-nx-16g | |
| triple: aarch64-unknown-linux-gnu-stretch | |
| # GPU-only box: its arm64 CPU is redundant with apple-m1-max, and an 8B | |
| # LLM on it would be pointlessly slow, so smoke only the Orin GPU. | |
| smoke_args: --skip-cpu | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.prepare.outputs.ref }} | |
| persist-credentials: false | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: tract-cli-${{ matrix.triple }} | |
| path: tract-cli | |
| - name: Load-and-run every bench model | |
| run: | | |
| chmod +x tract-cli/tract | |
| tract-cli/tract bench-suite --smoke --manifest .travis/benches.toml ${{ matrix.smoke_args }} | |
| hwbench-dinghy: | |
| needs: [ prepare, build-dinghy ] | |
| if: ${{ needs.prepare.outputs.enabled == 'true' && vars.BENCH_DINGHY_ENABLED != 'false' && github.event_name == 'schedule' }} | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - device: cortex-a55 | |
| triple: aarch64-unknown-linux-gnu-stretch | |
| runner: dinghy-sidekick-a55 | |
| dinghy_target: dinghy-a55 | |
| - device: cortex-a53 | |
| triple: aarch64-unknown-linux-gnu-stretch | |
| runner: dinghy-sidekick-a53 | |
| dinghy_target: dinghy-a53 | |
| - device: cortex-a7 | |
| triple: armv7-unknown-linux-gnueabihf-stretch | |
| runner: dinghy-sidekick-a7 | |
| dinghy_target: dinghy-a7 | |
| - device: cortex-a9 | |
| triple: armv7-unknown-linux-gnueabihf-stretch | |
| runner: dinghy-sidekick-a9 | |
| dinghy_target: dinghy-a9 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.prepare.outputs.ref }} | |
| persist-credentials: false | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: tract-cli-${{ matrix.triple }}-dinghy | |
| path: tract-cli | |
| - name: Assert kernel picks on the dinghy target | |
| env: | |
| DINGHY_TARGET: ${{ matrix.dinghy_target }} | |
| run: | | |
| chmod +x tract-cli/tract | |
| CARGO_PKG_NAME=tract cargo-dinghy -d "$DINGHY_TARGET" runner \ | |
| ./tract-cli/tract -- hwbench --no-cache --no-memory --assert --tolerance 20 </dev/null |