Generate and serve API data for EESSI #315
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate and serve API data for EESSI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 0 * * *" # runs once every 24 hours at 00:00 UTC | |
| permissions: | |
| contents: read | |
| pages: write | |
| id-token: write | |
| jobs: | |
| deploy: | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/configure-pages@v5 | |
| - uses: actions/checkout@v5 | |
| - uses: eessi/github-action-eessi@v3 | |
| with: | |
| use_eessi_module: true | |
| eessi_stack_version: "2025.06" | |
| - name: Create a virtualenv to install zensical | |
| run: | | |
| python -m venv /tmp/venv_docs | |
| source /tmp/venv_docs/bin/activate | |
| pip install zensical | |
| - name: Generate API data | |
| run: | | |
| echo "Generating data files..." | |
| module --force purge | |
| # Set CPU override for the stack to query | |
| export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="x86_64/intel/icelake" | |
| # Set GPU overrides for the stack to query (we have no GPU, so also override the GPU check) | |
| export EESSI_OVERRIDE_GPU_CHECK=1 | |
| # First do 2023.06 for EB 4 | |
| export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc80" # Stick to 8.0 for EB4 as options for 9.0 set by EESSI-extend are not understood there | |
| ( module load EESSI/2023.06 && module load EasyBuild/4 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2023.06 ) & | |
| pid1=$! | |
| # then 2023.06 for EB 5 | |
| export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc90" | |
| ( module load EESSI/2023.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2023.06 ) & | |
| pid2=$! | |
| # then 2025.06 for EB 5 (does not have EB4) | |
| export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc90" | |
| ( module load EESSI/2025.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2025.06 ) & | |
| pid3=$! | |
| fail=0 # initialise | |
| wait $pid1 || fail=1 | |
| wait $pid2 || fail=2 | |
| wait $pid3 || fail=3 | |
| if [ $fail -ne 0 ]; then | |
| echo "One or more jobs failed (at least job $fail)" | |
| exit 1 | |
| fi | |
| # Merge all these results together | |
| module reset | |
| module load EESSI-extend # Loads EasyBuild and we need that for our merge scripts | |
| python scripts/merge_data_files.py out.yaml eessi*.yaml | |
| mv out.yaml docs/data/eessi_software_metadata.yaml | |
| # Also do RISC-V repository | |
| module --force purge | |
| rm eessi*.yaml | |
| # Need to set environment variables to load EESSI module for RISC-V | |
| ( | |
| export EESSI_DEBUG_INIT=1 | |
| export EESSI_VERSION_OVERRIDE="2025.06-001" | |
| export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" | |
| export EESSI_SOFTWARE_SUBDIR_OVERRIDE="riscv64/generic" | |
| export EESSI_ACCELERATOR_TARGET_OVERRIDE="does/not/exist" | |
| export EESSI_CPU_FAMILY_OVERRIDE="x86_64" | |
| module load EESSI/2025.06 | |
| module load EasyBuild/5 | |
| module load EESSI-extend | |
| echo $MODULEPATH | |
| python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" | |
| ) | |
| # Merge all these results together | |
| module reset | |
| module load EESSI-extend # Loads EasyBuild and we need that for our merge scripts | |
| python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml | |
| mv out_riscv.yaml docs/data/eessi_software_metadata-riscv.yaml | |
| # Generate json data files and markdown index/description for them | |
| cd docs/data | |
| python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata.yaml eessi_api_metadata | |
| python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata-riscv.yaml eessi_api_metadata-riscv | |
| python ../../scripts/calculate_hashes.py | |
| for json_file in eessi_api_metadata_*.json; do | |
| python ../../scripts/generate_schema_md.py $json_file >> index.md | |
| done | |
| echo "" >> index.md | |
| echo "### RISC-V datafiles" >> index.md | |
| echo "" >> index.md | |
| for json_file in eessi_api_metadata-riscv_*.json; do | |
| python ../../scripts/generate_schema_md.py $json_file >> index.md | |
| done | |
| - run: | | |
| source /tmp/venv_docs/bin/activate | |
| zensical build --clean | |
| - uses: actions/upload-pages-artifact@v4 | |
| with: | |
| path: site | |
| - uses: actions/deploy-pages@v4 |