From baa2d60beaab83f545830ad7e3d4c1b08e8ffcc3 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Wed, 28 Oct 2026 21:01:48 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20site=20layer=20=E2=80=94=20site.yaml=20?= =?UTF-8?q?schema,=20Config.push=5Fsite,=20shipped=20site/ral.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hpc-campaign phase 1 (PyAutoLabs/PyAutoNerves#206). - autonerves/site_config.py: site.yaml schema (partitions with gpu_only, cores_per_node, mem_per_node_mb, max_time, max_array_size, tdp_w_per_cpu, cpu_is_thread, gpu_model, gpu_tdp_w; pue as a range; grid_region; ci_basis; energy_accounting) + validation. Any value may be plain, {value: ...} or a {low, high[, central]} range and may carry `unverified: true`. Site.cpu_partitions / Site.check_cpu_job keep CPU-only jobs off gpu_only partitions. - Config.push_site(name): config-dir site/.yaml overrides the shipped autonerves/site/.yaml; result kept on Config.site. - autonerves/site/ral.yaml (package data): `ral` CPU partition and `gpu` (gpu_only: true, A100) encoding the 2026-09-30 rule; every unconfirmed constant unverified (PUE 1.14-1.45, TDP 2.8-4.4 W/CPU, A100 250-400 W). - Nerves imports nothing from autofit (tested). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016nhQ8N4RdvbZz9cstLYtNF --- autonerves/conf.py | 23 ++ autonerves/site/ral.yaml | 32 +++ autonerves/site_config.py | 408 ++++++++++++++++++++++++++++ pyproject.toml | 3 + test_autonerves/test_site_config.py | 148 ++++++++++ 5 files changed, 614 insertions(+) create mode 100644 autonerves/site/ral.yaml create mode 100644 autonerves/site_config.py create mode 100644 test_autonerves/test_site_config.py diff --git a/autonerves/conf.py b/autonerves/conf.py index cb30d1b..550c206 100644 --- a/autonerves/conf.py +++ b/autonerves/conf.py @@ -117,6 +117,9 @@ def __init__(self, *config_paths, output_path: Union[str, Path] = "output"): self.output_path = output_path + # The site layer (`push_site`): facts about the computing site, or None. + self.site = None + @property def dict(self): """ @@ -275,6 +278,26 @@ def push( self.configure_logging() + def push_site(self, name: Union[str, Path]): + """ + Load the site layer ``name`` (``autonerves.site_config``): the partitions and + energy constants of a computing site, from ``site/.yaml`` in the first + config directory that has one, else the copy shipped with autonerves + (``autonerves/site/``). ``name`` may also be a path to a site file. + + The validated ``Site`` is kept as ``self.site`` and returned. + + Raises + ------ + SiteConfigException + If no site file is found or it is invalid. + """ + from autonerves.site_config import load_site, site_path + + self.site = load_site(site_path(str(name), config_paths=self.paths)) + logger.debug(f"Pushed site {self.site.name} from {self.site.source}") + return self.site + def register(self, file: str): """ Add defaults for a given project diff --git a/autonerves/site/ral.yaml b/autonerves/site/ral.yaml new file mode 100644 index 0000000..200e493 --- /dev/null +++ b/autonerves/site/ral.yaml @@ -0,0 +1,32 @@ +# RAL (Rutherford Appleton Laboratory) Slurm cluster used for the Euclid strong-lens work. +# Loaded by `conf.instance.push_site("ral")`; see autonerves/site_config.py for the schema. +# +# Values marked `unverified: true` have not been confirmed against the cluster +# (`scontrol show config`, `scontrol show partition`, `lscpu` on a compute node) and are +# used as ranges until a human confirms them; `value: null` means not yet known. +name: ral +partitions: + ral: + gpu_only: false + cores_per_node: {value: null, unverified: true} + mem_per_node_mb: {value: null, unverified: true} + max_time: {value: null, unverified: true} + max_array_size: {value: 1001, unverified: true} # Slurm's default MaxArraySize; DR1 runs 1000-task parts. + tdp_w_per_cpu: {low: 2.8, central: 3.5, high: 4.4, unverified: true} # EPYC 9754 / 9654 / 7763 candidates, per Slurm CPU. + cpu_is_thread: {value: true, unverified: true} + note: Bulk CPU arrays go here, even when it is busy. + gpu: + gpu_only: true # Human rule 2026-09-30: CPU-only arrays never go on gpu (nor ral,gpu / gpu,ral); they starved the A100s. + cores_per_node: {value: 62, unverified: true} # 124 CPUs across euclid-ral-gpu-1/-2. + mem_per_node_mb: {value: null, unverified: true} + max_time: {value: null, unverified: true} + max_array_size: {value: 1001, unverified: true} + tdp_w_per_cpu: {low: 2.8, central: 3.5, high: 4.4, unverified: true} + cpu_is_thread: {value: true, unverified: true} + gpu_model: A100 + gpu_tdp_w: {low: 250, central: 300, high: 400, unverified: true} # PCIe 250/300 W or SXM 400 W; measure with NVML. + note: 8 A100s on two nodes (4 per node, unverified). +pue: {low: 1.14, central: 1.31, high: 1.45, unverified: true} # R89 measured 1.28-1.45 (mean 1.31); new centre ~1.14; hosting building unverified. +grid_region: GB +ci_basis: {value: location_annual_average, unverified: true} +energy_accounting: {value: false, unverified: true} # Slurm AcctGatherEnergyType not confirmed. diff --git a/autonerves/site_config.py b/autonerves/site_config.py new file mode 100644 index 0000000..dd8c79a --- /dev/null +++ b/autonerves/site_config.py @@ -0,0 +1,408 @@ +""" +The site layer: facts about the computing site a campaign runs on, loaded from a +``site/.yaml`` file by ``Config.push_site(name)``. + +A site file names the scheduler partitions jobs may use and the constants an energy +and carbon estimate needs, in the vocabulary of GA4HPC's ``cluster_info.yaml``: + +.. code-block:: yaml + + name: ral + partitions: + ral: + gpu_only: false + cores_per_node: {value: null, unverified: true} # not yet known + mem_per_node_mb: {value: null, unverified: true} + max_time: {value: "2-00:00:00", unverified: true} + max_array_size: {value: 1001, unverified: true} + tdp_w_per_cpu: {low: 2.8, central: 3.5, high: 4.4, unverified: true} + cpu_is_thread: {value: true, unverified: true} + gpu: + gpu_only: true # CPU-only jobs must never be sent here + gpu_model: A100 + gpu_tdp_w: {low: 250, high: 400, unverified: true} + pue: {low: 1.14, central: 1.31, high: 1.45, unverified: true} + grid_region: GB + ci_basis: location_annual_average + energy_accounting: false + +Any value may be written plainly (``gpu_only: true``), as ``{value: ...}`` or as a +range ``{low: ..., high: ...}`` (with an optional ``central``), and any of these may +carry ``unverified: true``: a constant nobody has confirmed against the site is kept, +used as a range and reported as unverified, never silently trusted. ``pue`` is always +a range. + +A partition with ``gpu_only: true`` takes GPU jobs only: a planner must never place a +CPU-only job there (``Site.cpu_partitions`` leaves it out, ``Site.check_cpu_job`` +raises). + +The site files shipped with autonerves are in ``autonerves/site/``; a +``site/.yaml`` in any pushed config directory overrides the shipped one. +""" +import re +from pathlib import Path +from typing import Dict, List, Optional, Union + +import yaml + +from autonerves.exc import ConfigException + +PACKAGE_SITE_DIRECTORY = Path(__file__).parent / "site" + +PARTITION_FIELDS = { + "gpu_only": "bool", + "cores_per_node": "int", + "mem_per_node_mb": "int", + "max_time": "time", + "max_array_size": "int", + "tdp_w_per_cpu": "number", + "cpu_is_thread": "bool", + "gpu_model": "str", + "gpu_tdp_w": "number", + "note": "str", +} + +SITE_FIELDS = { + "name": "str", + "partitions": "partitions", + "pue": "range", + "grid_region": "str", + "ci_basis": "str", + "energy_accounting": "bool", + "note": "str", +} + +# Slurm's time formats: "minutes", "HH:MM:SS", "D-HH", "D-HH:MM", "D-HH:MM:SS", +# "MM:SS"; or "UNLIMITED". +_TIME = re.compile(r"^(\d+-)?\d+(:\d{2}){0,2}$|^UNLIMITED$", re.IGNORECASE) + + +class SiteConfigException(ConfigException): + pass + + +class SiteValue: + def __init__( + self, + value=None, + low=None, + high=None, + central=None, + unverified: bool = False, + ): + """ + One site constant: a plain ``value`` or a ``low``-``high`` range with an + optional ``central`` value, and whether it is ``unverified``. + """ + self.value = value + self.low = low + self.high = high + self._central = central + self.unverified = bool(unverified) + + @classmethod + def from_yaml(cls, raw) -> "SiteValue": + if isinstance(raw, dict): + return cls( + value=raw.get("value"), + low=raw.get("low"), + high=raw.get("high"), + central=raw.get("central"), + unverified=raw.get("unverified", False), + ) + return cls(value=raw) + + @property + def is_range(self) -> bool: + return self.low is not None or self.high is not None + + @property + def central(self): + """ + The value to use when one number is needed: ``value``, else ``central``, + else the middle of the range. + """ + if self.value is not None: + return self.value + if self._central is not None: + return self._central + if self.low is not None and self.high is not None: + return (self.low + self.high) / 2 + return self.low if self.low is not None else self.high + + @property + def range(self): + """ + ``(low, high)``; a plain value is the range of itself. + """ + if self.is_range: + return self.low, self.high + return self.value, self.value + + def __repr__(self): + flag = ", unverified" if self.unverified else "" + if self.is_range: + return f"SiteValue({self.low}-{self.high}{flag})" + return f"SiteValue({self.value!r}{flag})" + + +class Partition: + def __init__(self, name: str, raw: Dict): + self.name = name + self.values = { + key: SiteValue.from_yaml(value) + for key, value in raw.items() + if key in PARTITION_FIELDS + } + + def __getattr__(self, item): + if item in PARTITION_FIELDS: + value = self.__dict__["values"].get(item) + return None if value is None else value.central + raise AttributeError(item) + + @property + def gpu_only(self) -> bool: + value = self.values.get("gpu_only") + return bool(value.central) if value is not None else False + + @property + def accepts_cpu_jobs(self) -> bool: + return not self.gpu_only + + def __repr__(self): + return f"Partition({self.name!r}, gpu_only={self.gpu_only})" + + +class Site: + def __init__(self, raw: Dict, source: Optional[Path] = None): + """ + A validated site file (``load_site``). + """ + self.raw = raw + self.source = source + self.name = raw.get("name") + self.partitions = { + name: Partition(name, partition or {}) + for name, partition in (raw.get("partitions") or {}).items() + } + self.values = { + key: SiteValue.from_yaml(raw[key]) + for key in ("pue", "grid_region", "ci_basis", "energy_accounting") + if key in raw + } + + def partition(self, name: str) -> Partition: + try: + return self.partitions[name] + except KeyError: + raise SiteConfigException( + f"Site {self.name!r} has no partition {name!r} " + f"(it has {sorted(self.partitions)})." + ) + + @property + def cpu_partitions(self) -> List[Partition]: + """ + The partitions a CPU-only job may use (every partition not ``gpu_only``). + """ + return [p for p in self.partitions.values() if p.accepts_cpu_jobs] + + def check_cpu_job(self, partitions: Union[str, List[str]]): + """ + Raise if a CPU-only job would be sent to a ``gpu_only`` partition. + ``partitions`` is a partition name, a Slurm list (``"ral,gpu"``) or a list. + """ + if isinstance(partitions, str): + partitions = partitions.split(",") + for name in partitions: + if self.partition(name.strip()).gpu_only: + raise SiteConfigException( + f"Partition {name.strip()!r} of site {self.name!r} is gpu_only: " + f"CPU-only jobs must use {[p.name for p in self.cpu_partitions]}." + ) + + @property + def pue(self) -> Optional[SiteValue]: + return self.values.get("pue") + + @property + def grid_region(self): + value = self.values.get("grid_region") + return None if value is None else value.central + + @property + def ci_basis(self): + value = self.values.get("ci_basis") + return None if value is None else value.central + + @property + def energy_accounting(self) -> bool: + value = self.values.get("energy_accounting") + return bool(value.central) if value is not None else False + + def unverified(self) -> List[str]: + """ + The dotted names of every value marked ``unverified``. + """ + names = [key for key, value in self.values.items() if value.unverified] + for partition in self.partitions.values(): + names += [ + f"partitions.{partition.name}.{key}" + for key, value in partition.values.items() + if value.unverified + ] + return names + + def __repr__(self): + return f"Site({self.name!r}, partitions={sorted(self.partitions)})" + + +def _check_type(kind: str, value) -> bool: + if kind == "bool": + return isinstance(value, bool) + if kind == "int": + return isinstance(value, int) and not isinstance(value, bool) + if kind == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + if kind == "str": + return isinstance(value, str) + if kind == "time": + return isinstance(value, (str, int)) and bool(_TIME.match(str(value))) + return True + + +def _check_value(where: str, kind: str, raw, errors: List[str], range_only=False): + if isinstance(raw, dict): + unknown = set(raw) - {"value", "low", "high", "central", "unverified"} + if unknown: + errors.append(f"{where}: unknown keys {sorted(unknown)}") + if "unverified" in raw and not isinstance(raw["unverified"], bool): + errors.append(f"{where}.unverified must be true or false") + has_value = raw.get("value") is not None + has_range = raw.get("low") is not None or raw.get("high") is not None + if has_value and has_range: + errors.append(f"{where}: give value or low/high, not both") + if range_only and not (raw.get("low") is not None and raw.get("high") is not None): + errors.append(f"{where}: must be a range with low and high") + if has_range and kind not in ("number", "int", "range"): + errors.append(f"{where}: a {kind} cannot be a range") + numeric = "number" if kind == "range" else kind + for key in ("value", "low", "high", "central"): + if raw.get(key) is not None and not _check_type(numeric, raw[key]): + errors.append(f"{where}.{key} must be a {numeric}: {raw[key]!r}") + low, high = raw.get("low"), raw.get("high") + if ( + isinstance(low, (int, float)) + and isinstance(high, (int, float)) + and low > high + ): + errors.append(f"{where}: low {low} is above high {high}") + return + if range_only: + errors.append(f"{where}: must be a range with low and high") + return + if raw is not None and not _check_type(kind, raw): + errors.append(f"{where} must be a {kind}: {raw!r}") + + +def validate(raw) -> List[str]: + """ + Every problem with a site file's contents, as messages (empty when valid). + """ + errors = [] + if not isinstance(raw, dict): + return ["a site file must be a mapping"] + + unknown = set(raw) - set(SITE_FIELDS) + if unknown: + errors.append(f"unknown site keys {sorted(unknown)}") + + partitions = raw.get("partitions") + if not isinstance(partitions, dict) or not partitions: + errors.append("partitions: at least one partition is required") + partitions = {} + + for name, partition in partitions.items(): + where = f"partitions.{name}" + if partition is None: + partition = {} + if not isinstance(partition, dict): + errors.append(f"{where} must be a mapping") + continue + unknown = set(partition) - set(PARTITION_FIELDS) + if unknown: + errors.append(f"{where}: unknown keys {sorted(unknown)}") + for key, kind in PARTITION_FIELDS.items(): + if key in partition: + _check_value(f"{where}.{key}", kind, partition[key], errors) + gpu_only = SiteValue.from_yaml(partition.get("gpu_only")).central + if gpu_only and partition.get("gpu_model") is None: + errors.append(f"{where}: a gpu_only partition must name its gpu_model") + + for key in ("grid_region", "ci_basis", "name", "note"): + if key in raw: + _check_value(key, "str", raw[key], errors) + if "energy_accounting" in raw: + _check_value("energy_accounting", "bool", raw["energy_accounting"], errors) + if "pue" in raw: + _check_value("pue", "range", raw["pue"], errors, range_only=True) + pue = raw["pue"] if isinstance(raw["pue"], dict) else {} + if isinstance(pue.get("low"), (int, float)) and pue["low"] < 1: + errors.append("pue: a PUE is at least 1") + + return errors + + +def load_site(path: Union[str, Path]) -> Site: + """ + Load and validate the site file at ``path``. + + Raises + ------ + SiteConfigException + If the file is missing, is not YAML or fails ``validate``. + """ + path = Path(path) + try: + raw = yaml.safe_load(path.read_text()) + except OSError as exception: + raise SiteConfigException(f"Site file {path} cannot be read: {exception}") + except yaml.YAMLError as exception: + raise SiteConfigException(f"Site file {path} is not valid YAML: {exception}") + + errors = validate(raw) + if errors: + raise SiteConfigException( + f"Site file {path} is invalid:\n " + "\n ".join(errors) + ) + raw.setdefault("name", path.stem) + return Site(raw, source=path) + + +def site_path(name: str, config_paths=()) -> Path: + """ + The file of site ``name``: ``site/.yaml`` in the first config directory + that has one, else the copy shipped in ``autonerves/site/``. ``name`` may also be a + path to a site file. + + Raises + ------ + SiteConfigException + If no such site file exists. + """ + direct = Path(name) + if direct.suffix in (".yaml", ".yml") and direct.is_file(): + return direct + + candidates = [Path(path) / "site" / f"{name}.yaml" for path in config_paths] + candidates.append(PACKAGE_SITE_DIRECTORY / f"{name}.yaml") + for candidate in candidates: + if candidate.is_file(): + return candidate + + shipped = sorted(p.stem for p in PACKAGE_SITE_DIRECTORY.glob("*.yaml")) + raise SiteConfigException( + f"No site file for {name!r} (looked in {[str(c) for c in candidates]}; " + f"shipped sites: {shipped})." + ) diff --git a/pyproject.toml b/pyproject.toml index 9ccc997..126f51f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,9 @@ Homepage = "https://github.com/PyAutoLabs/PyAutoNerves" [tool.setuptools] include-package-data = true +[tool.setuptools.package-data] +autonerves = ["site/*.yaml"] + [tool.setuptools.packages.find] include = ["autonerves*"] exclude = ["docs", "test_autonerves", "test_autonerves*", "scripts", "scripts*"] diff --git a/test_autonerves/test_site_config.py b/test_autonerves/test_site_config.py new file mode 100644 index 0000000..d98d802 --- /dev/null +++ b/test_autonerves/test_site_config.py @@ -0,0 +1,148 @@ +""" +The site layer (``autonerves.site_config``, ``Config.push_site``): schema validation, +values with ``unverified`` flags and ranges, the shipped RAL site and its GPU-only rule. +""" +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from autonerves import conf +from autonerves.site_config import ( + PACKAGE_SITE_DIRECTORY, + SiteConfigException, + SiteValue, + load_site, + validate, +) + +VALID = { + "name": "toy", + "partitions": { + "cpu": {"gpu_only": False, "cores_per_node": 64, "max_time": "2-00:00:00"}, + "gpu": {"gpu_only": True, "gpu_model": "A100", "gpu_tdp_w": {"low": 250, "high": 400}}, + }, + "pue": {"low": 1.1, "high": 1.5}, + "grid_region": "GB", + "ci_basis": "location_annual_average", + "energy_accounting": False, +} + + +def test_push_site_loads_the_shipped_ral_site(config): + site = config.push_site("ral") + + assert config.site is site + assert site.name == "ral" + assert site.source == PACKAGE_SITE_DIRECTORY / "ral.yaml" + assert set(site.partitions) == {"ral", "gpu"} + + +def test_ral_gpu_partition_is_gpu_only(config): + site = config.push_site("ral") + + assert site.partition("gpu").gpu_only + assert site.partition("gpu").gpu_model == "A100" + assert not site.partition("ral").gpu_only + assert [p.name for p in site.cpu_partitions] == ["ral"] + + site.check_cpu_job("ral") + for partitions in ("gpu", "ral,gpu", "gpu,ral", ["ral", "gpu"]): + with pytest.raises(SiteConfigException, match="gpu_only"): + site.check_cpu_job(partitions) + + +def test_ral_unconfirmed_constants_are_unverified_ranges(config): + site = config.push_site("ral") + + unverified = site.unverified() + assert "pue" in unverified + assert "partitions.ral.tdp_w_per_cpu" in unverified + assert "partitions.gpu.gpu_tdp_w" in unverified + # The GPU-only rule is a human ruling, not an unconfirmed constant. + assert "partitions.gpu.gpu_only" not in unverified + + assert site.pue.is_range and site.pue.range == (1.14, 1.45) + assert site.pue.central == 1.31 + assert site.partition("ral").tdp_w_per_cpu == 3.5 + + +def test_shipped_site_files_are_all_valid(): + files = list(PACKAGE_SITE_DIRECTORY.glob("*.yaml")) + assert files + for path in files: + assert validate(yaml.safe_load(path.read_text())) == [], path + + +def test_config_directory_site_overrides_the_shipped_one(tmp_path, files_directory): + (tmp_path / "site").mkdir() + (tmp_path / "site" / "ral.yaml").write_text(yaml.safe_dump(VALID)) + config = conf.Config(tmp_path, files_directory / "default") + + site = config.push_site("ral") + + assert site.source == tmp_path / "site" / "ral.yaml" + assert site.name == "toy" + + +def test_push_site_by_path_and_unknown_name(tmp_path, config): + path = tmp_path / "mine.yaml" + path.write_text(yaml.safe_dump({k: v for k, v in VALID.items() if k != "name"})) + + assert config.push_site(path).name == "mine" + + with pytest.raises(SiteConfigException, match="No site file"): + config.push_site("no_such_site") + + +def test_site_value_forms(): + assert SiteValue.from_yaml(3).central == 3 + assert SiteValue.from_yaml({"value": 3, "unverified": True}).unverified + ranged = SiteValue.from_yaml({"low": 2, "high": 4}) + assert ranged.is_range and ranged.central == 3 and ranged.range == (2, 4) + assert not ranged.unverified + + +@pytest.mark.parametrize( + "change, message", + [ + ({"partitions": {}}, "at least one partition"), + ({"surprise": 1}, "unknown site keys"), + ({"pue": 1.3}, "must be a range"), + ({"pue": {"low": 1.5, "high": 1.2}}, "is above high"), + ({"pue": {"low": 0.9, "high": 1.2}}, "at least 1"), + ({"energy_accounting": "no"}, "must be a bool"), + ({"partitions": {"gpu": {"gpu_only": True}}}, "must name its gpu_model"), + ({"partitions": {"cpu": {"cores_per_node": "many"}}}, "must be a int"), + ({"partitions": {"cpu": {"max_time": "a week"}}}, "must be a time"), + ({"partitions": {"cpu": {"cpu_is_thread": {"low": 1, "high": 2}}}}, "cannot be a range"), + ({"partitions": {"cpu": {"cores_per_node": {"value": 1, "unverified": "yes"}}}}, "unverified must be"), + ({"partitions": {"cpu": {"cores": 1}}}, "unknown keys"), + ], +) +def test_validation_errors(change, message, tmp_path): + raw = {**VALID, **change} + assert any(message in error for error in validate(raw)), validate(raw) + + path = tmp_path / "bad.yaml" + path.write_text(yaml.safe_dump(raw)) + with pytest.raises(SiteConfigException, match=message): + load_site(path) + + +def test_valid_toy_site(): + assert validate(VALID) == [] + + +def test_nerves_imports_no_autofit(): + """ + Loading a site imports nothing from autofit (Nerves sits below the libraries). + """ + code = ( + "import sys; from autonerves import conf; " + "conf.Config().push_site('ral'); " + "assert not [m for m in sys.modules if m.split('.')[0] == 'autofit'], 'autofit'" + ) + subprocess.run([sys.executable, "-c", code], check=True, cwd=Path(conf.__file__).parents[1])