From b932d3bfb46713e704b3c6684d368fd6f98fe694 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 25 Aug 2026 02:11:55 -0700 Subject: [PATCH] ci(faiss): require the storage-kind rejection, not just a nonzero exit The faiss LVQ/LeanVec tutorials carry no capability gate, so they only run successfully with an LVQ/LeanVec-enabled runtime on Intel hardware. test-faiss.sh inferred that from the CPU vendor alone, which is wrong for the public-only build: the formats are compiled out there, so the examples abort on every runner and that leg was green only when GitHub allocated a non-Intel one. Derive the expectation from ENABLE_LVQ_LEANVEC as well, plumbed into the test job from the same value the artifact was built with, and name the actual reason in the log. The examples still run in that case and must still fail, but now they have to fail with the runtime's storage-kind rejection: a bare nonzero exit also covers a missing library or a segfault, which is not what is under test. Supersedes #314, which keyed off the -public-only artifact name. Co-Authored-By: Claude Opus 5 --- .github/scripts/test-faiss.sh | 71 +++++++++++-------- .../workflows/build-cpp-runtime-bindings.yml | 6 +- 2 files changed, 47 insertions(+), 30 deletions(-) diff --git a/.github/scripts/test-faiss.sh b/.github/scripts/test-faiss.sh index ec22f6fde..849addf19 100644 --- a/.github/scripts/test-faiss.sh +++ b/.github/scripts/test-faiss.sh @@ -51,25 +51,46 @@ echo " FAISS C++ tests: " echo "-----------------------------------------------" echo " FAISS-SVS C++ examples: " make 10-SVS-Vamana-LVQ 11-SVS-Vamana-LeanVec -# Check if running on Intel hardware (LVQ/LeanVec require Intel-specific instructions) -if grep -q "GenuineIntel" /proc/cpuinfo; then +# The examples request LVQ/LeanVec unconditionally, which needs an enabled runtime and +# Intel hardware; public-only compiles the formats out, so the CPU vendor alone mispredicts. +lvq_leanvec_missing="" +if [ "${ENABLE_LVQ_LEANVEC:-ON}" != "ON" ]; then + lvq_leanvec_missing="the runtime is built without LVQ/LeanVec support" +elif ! grep -q "GenuineIntel" /proc/cpuinfo; then + lvq_leanvec_missing="the CPU is not GenuineIntel" +fi + +# The trailing class name differs between bindings/cpp/src/vamana_index.cpp and +# dynamic_vamana_index.cpp, so matching the full message would pin the examples to one index kind. +storage_kind_rejection="The specified storage kind is not compatible with the" + +# A nonzero exit alone would let a broken example, a missing library or a segfault pass as expected. +expect_storage_kind_rejection() { + local label="$1" + shift + local output status + output=$("$@" 2>&1) && status=0 || status=$? + echo "$output" + if [ "$status" -eq 0 ]; then + echo "UNEXPECTED: $label succeeded although $lvq_leanvec_missing" + return 1 + fi + if ! printf '%s\n' "$output" | grep -qF "$storage_kind_rejection"; then + echo "UNEXPECTED: $label exited $status without rejecting the storage kind" + return 1 + fi + echo "XFAIL: $label rejected the storage kind as expected ($lvq_leanvec_missing)" +} + +if [ -z "$lvq_leanvec_missing" ]; then $RUN_PREFIX ./tutorial/cpp/10-SVS-Vamana-LVQ $RUN_PREFIX ./tutorial/cpp/11-SVS-Vamana-LeanVec else - echo "Non-Intel CPU detected - LVQ/LeanVec examples expected to fail" - set +e - ./tutorial/cpp/10-SVS-Vamana-LVQ - exit_code_10=$? - ./tutorial/cpp/11-SVS-Vamana-LeanVec - exit_code_11=$? - set -e - - if [ $exit_code_10 -ne 0 ] && [ $exit_code_11 -ne 0 ]; then - echo "XFAIL: Examples failed as expected on non-Intel hardware" - else - echo "UNEXPECTED: One or more tests passed on non-Intel hardware (exit codes: $exit_code_10, $exit_code_11)" - exit 1 - fi + echo "LVQ/LeanVec examples expected to reject the storage kind: $lvq_leanvec_missing" + expect_storage_kind_rejection 10-SVS-Vamana-LVQ \ + $RUN_PREFIX ./tutorial/cpp/10-SVS-Vamana-LVQ + expect_storage_kind_rejection 11-SVS-Vamana-LeanVec \ + $RUN_PREFIX ./tutorial/cpp/11-SVS-Vamana-LeanVec fi echo "-----------------------------------------------" echo " FAISS python bindings: " @@ -82,19 +103,11 @@ PYTHONPATH=../build/faiss/python/build/lib/ OMP_NUM_THREADS=4 python -m unittest echo "-----------------------------------------------" echo " FAISS-SVS python examples: " cd ../tutorial/python/ -if grep -q "GenuineIntel" /proc/cpuinfo; then +if [ -z "$lvq_leanvec_missing" ]; then PYTHONPATH=../../build/faiss/python/build/lib/ OMP_NUM_THREADS=4 $RUN_PREFIX python 11-SVS.py else - echo "Non-Intel CPU detected - SVS python example expected to fail" - set +e - PYTHONPATH=../../build/faiss/python/build/lib/ OMP_NUM_THREADS=4 python 11-SVS.py - exit_code=$? - set -e - - if [ $exit_code -ne 0 ]; then - echo "XFAIL: Python example failed as expected on non-Intel hardware" - else - echo "UNEXPECTED: Python example passed on non-Intel hardware" - exit 1 - fi + echo "SVS python example expected to reject the storage kind: $lvq_leanvec_missing" + expect_storage_kind_rejection 11-SVS.py \ + env PYTHONPATH=../../build/faiss/python/build/lib/ OMP_NUM_THREADS=4 \ + $RUN_PREFIX python 11-SVS.py fi diff --git a/.github/workflows/build-cpp-runtime-bindings.yml b/.github/workflows/build-cpp-runtime-bindings.yml index f13a540e5..b87cd0ab5 100644 --- a/.github/workflows/build-cpp-runtime-bindings.yml +++ b/.github/workflows/build-cpp-runtime-bindings.yml @@ -95,10 +95,14 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: + # enable_lvq_leanvec must match the build job's value for this suffix; if they + # disagree, test-faiss.sh expects the wrong outcome from the downloaded artifact. include: - name: "with static library" + enable_lvq_leanvec: "ON" suffix: "" - name: "public only" + enable_lvq_leanvec: "OFF" suffix: "-public-only" fail-fast: false @@ -126,6 +130,6 @@ jobs: -v ${{ github.workspace }}:/workspace \ -v ${{ github.workspace }}/runtime_conda:/runtime_conda \ -w /workspace \ - -e SUFFIX=${{ matrix.suffix }} \ + -e ENABLE_LVQ_LEANVEC=${{ matrix.enable_lvq_leanvec }} \ svs-manylinux228:latest \ /bin/bash .github/scripts/test-faiss.sh