api_data/.github/workflows/docs.yml at main · EESSI/api_data · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
name: Generate and serve API data for EESSI
on:
  push:
    branches:
      - main
  schedule:
    - cron: "0 0 * * *"  # runs once every 24 hours at 00:00 UTC
permissions:
  contents: read
  pages: write
  id-token: write
jobs:
  deploy:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/configure-pages@v5
      - uses: actions/checkout@v5
      - uses: eessi/github-action-eessi@v3
        with:
          use_eessi_module: true
          eessi_stack_version: "2025.06"
      - name: Create a virtualenv to install zensical
        run: |
          python -m venv /tmp/venv_docs
          source /tmp/venv_docs/bin/activate
          pip install zensical
      - name: Generate API data
        run: |
          echo "Generating data files..."
          module --force purge

          # Set CPU override for the stack to query
          export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="x86_64/intel/icelake"
          # Set GPU overrides for the stack to query (we have no GPU, so also override the GPU check)
          export EESSI_OVERRIDE_GPU_CHECK=1

          ## CPU only ##
          # First do 2023.06 for EB 4
          ( module load EESSI/2023.06 && module load EasyBuild/4 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2023.06 ) &
          pid1=$!
          # then 2023.06 for EB 5
          ( module load EESSI/2023.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2023.06 ) &
          pid2=$!
          # then 2025.06 for EB 5 (does not have EB4)
          ( module load EESSI/2025.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2025.06 ) &
          pid3=$!
          # then 2026.06 for EB 5 (does not have EB4)
          ( module load EESSI/2026.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2026.06 ) &
          pid4=$!
          fail=0  # initialise
          wait $pid1 || fail=1
          wait $pid2 || fail=2
          wait $pid3 || fail=3
          wait $pid4 || fail=4
          if [ $fail -ne 0 ]; then
            echo "One or more jobs failed for CPU only (at least job $fail)"
            exit 1
          fi

          ## NVIDIA GPU ##
          # First do 2023.06 for EB 4
          export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc80"  # Stick to 8.0 for EB4 as options for 9.0 set by EESSI-extend are not understood there
          ( module load EESSI/2023.06 && module load EasyBuild/4 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2023.06 ) &
          pid1=$!
          # then 2023.06 for EB 5
          export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc90"
          ( module load EESSI/2023.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2023.06 ) &
          pid2=$!
          # then 2025.06 for EB 5 (does not have EB4)
          export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc90"
          ( module load EESSI/2025.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2025.06 ) &
          pid3=$!
          # then 2026.06 for EB 5 (does not have EB4)
          export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc90"
          ( module load EESSI/2026.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2026.06 ) &
          pid4=$!
          fail=0  # initialise
          wait $pid1 || fail=1
          wait $pid2 || fail=2
          wait $pid3 || fail=3
          wait $pid4 || fail=4
          if [ $fail -ne 0 ]; then
            echo "One or more jobs failed for NVIDIA GPU (at least job $fail)"
            exit 1
          fi

          ## AMD GPU (2025.06+) ##
          # 2025.06 for EB 5 (does not have EB4)
          export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/amd/gfx1200"
          ( module load EESSI/2025.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2025.06 ) &
          pid1=$!
          # then 2026.06 for EB 5 (does not have EB4)
          #export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/amd/gfx1200"
          #( module load EESSI/2026.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2026.06 ) &
          #pid2=$!
          fail=0  # initialise
          wait $pid1 || fail=1
          #wait $pid2 || fail=2
          if [ $fail -ne 0 ]; then
            echo "One or more jobs failed (at least job $fail)"
            exit 1
          fi

          # Merge all these results together
          unset EESSI_ACCELERATOR_TARGET_OVERRIDE
          module reset
          module load EESSI-extend  # Loads EasyBuild and we need that for our merge scripts
          python scripts/merge_data_files.py out.yaml eessi*.yaml
          mv out.yaml docs/data/eessi_software_metadata.yaml

          # Also do RISC-V repository
          module --force purge
          rm eessi*.yaml
          # Need to set environment variables to load EESSI module for RISC-V
          (
            export EESSI_DEBUG_INIT=1
            export EESSI_VERSION_OVERRIDE="2025.06-001"
            export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic"
            export EESSI_SOFTWARE_SUBDIR_OVERRIDE="riscv64/generic"
            export EESSI_CPU_FAMILY_OVERRIDE="x86_64"
            module load EESSI/2025.06
            module load EasyBuild/5
            module load EESSI-extend
            echo $MODULEPATH
            python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv"
          )
          # Merge all these results together
          module reset
          module load EESSI-extend  # Loads EasyBuild and we need that for our merge scripts
          python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml
          mv out_riscv.yaml docs/data/eessi_software_metadata-riscv.yaml

          # Generate json data files and markdown index/description for them
          cd docs/data
          python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata.yaml eessi_api_metadata
          python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata-riscv.yaml eessi_api_metadata-riscv
          python ../../scripts/calculate_hashes.py
          for json_file in eessi_api_metadata_*.json; do
            python ../../scripts/generate_schema_md.py $json_file >> index.md
          done
          echo "" >> index.md
          echo "### RISC-V datafiles"  >> index.md
          echo "" >> index.md
          for json_file in eessi_api_metadata-riscv_*.json; do
            python ../../scripts/generate_schema_md.py $json_file >> index.md
          done
      - run: |
          source /tmp/venv_docs/bin/activate
          zensical build --clean
      - uses: actions/upload-pages-artifact@v4
        with:
          path: site
      - uses: actions/deploy-pages@v4