diff --git a/CHANGELOG.md b/CHANGELOG.md index 040251e..6b59e63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ ## [0.2.0] — 2026-10-22 Every deterministic model now gives the same doubles as TerraME at every -step (checked against `terrame-goldens` v0.1.0). The trajectories of +step, except Daisyworld, which agrees to about 15 significant digits (checked +against `terrame-goldens` v0.1.0). The trajectories of `Daisyworld` and `MonoLake` change. - **Fix:** `Daisyworld` now follows TerraME: after updating the white area, @@ -20,9 +21,13 @@ step (checked against `terrame-goldens` v0.1.0). The trajectories of - `PredatorPrey`, `RoomTemperature`, `SIR` and `MonoLake` do their arithmetic in TerraME's order, so that the results are the same doubles. - `tests/test_goldens.py` (marker `golden`): every model against TerraME - 2.0.1 runs from `terrame-goldens` v0.1.0. All 16 deterministic cases - (13 models, some with two parameter sets) give **the same doubles as - TerraME at every step**; RandomWalk is compared by distribution. + 2.0.1 runs from `terrame-goldens` v0.1.0, generated in the + `terrame-docker` image. 15 of the 16 deterministic cases (13 models, some + with two parameter sets) give **the same doubles as TerraME at every + step**. Daisyworld is compared with a relative tolerance of 10⁻¹³: its + temperatures use the C library's `pow`, whose last bit differs between + glibc 2.27 (the image) and newer versions. RandomWalk is compared by + distribution. `tests/goldens/` holds the goldens, their SHA-256 provenance and the script that copies them. The strict `xfail`s of `Daisyworld` and `MonoLake` in `test_terrame_parity.py` are removed. diff --git a/README.md b/README.md index f66aad3..5c59ba7 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,11 @@ Every model was run in TerraME 2.0.1 with the `sysdyn` package ([`terrame-goldens`](https://github.com/LambdaGeo/terrame-goldens) v0.1.0). For the 16 deterministic cases (13 models, some with two parameter sets), the Python models give **the same doubles as TerraME at every step**, bit for -bit, for every variable TerraME records; RandomWalk is compared by -distribution. `tests/test_goldens.py` runs these comparisons +bit, for every variable TerraME records, except Daisyworld, whose fourth +roots go through the C library's `pow`: its last bit depends on the system +(the goldens come from the `terrame-docker` image, with glibc 2.27), so +Daisyworld agrees to about 15 significant digits and is compared with a +relative tolerance of 10⁻¹³. RandomWalk is compared by distribution. `tests/test_goldens.py` runs these comparisons (`pytest -m golden`, under a second). ## Examples diff --git a/tests/goldens/daisyworld.csv.gz b/tests/goldens/daisyworld.csv.gz index f9a7383..4a6995a 100644 Binary files a/tests/goldens/daisyworld.csv.gz and b/tests/goldens/daisyworld.csv.gz differ diff --git a/tests/goldens/reference.json b/tests/goldens/reference.json index a7cbaf6..17f0b5d 100644 --- a/tests/goldens/reference.json +++ b/tests/goldens/reference.json @@ -17,7 +17,7 @@ "files": { "goldens/sysdyn/chaotic_growth.csv.gz": "6328b3248918b88ec4adfc1002a99ec6c4d252fa1e6a3beaa0c1477faa528c75", "goldens/sysdyn/coffee.csv.gz": "cf32769b784339a21489a54ef05f1e59c403da6857fa7421dd28e05f1581a4c1", - "goldens/sysdyn/daisyworld.csv.gz": "da394371bf4dc84114ae5c5815797d20aaa7fd85b6e7b182446e180f7d747c6c", + "goldens/sysdyn/daisyworld.csv.gz": "1b8668e090f51550b12562d5af517d2fe4b2845e1ec68381c4eb21e9bee516b4", "goldens/sysdyn/homeostasis.csv.gz": "2cb49a621d7153e98632f7f0e11bfdc47ea04a105a2f659d6976bfa3dfc0fd4f", "goldens/sysdyn/homeostasis_100.csv.gz": "6896a8faed843dc0c42f58194b284381fad971e793bed8de531ef26282ac280b", "goldens/sysdyn/limited_growth.csv.gz": "9972047016a135b12c2e70e26209d438e1b911dd44ac927c47f4b8eab5f28b87", diff --git a/tests/test_goldens.py b/tests/test_goldens.py index 376950c..d22d2ef 100644 --- a/tests/test_goldens.py +++ b/tests/test_goldens.py @@ -8,6 +8,14 @@ **the same doubles** as TerraME at every step (bit for bit), for every variable TerraME records. +The exception is Daisyworld, whose temperatures take a fourth root +(``x ** 0.25`` in Python, ``x ^ 0.25`` in Lua). Both languages call the C +library's ``pow``, and its last bit depends on the library: the goldens come +from the ``terrame-docker`` image (glibc 2.27), and the same TerraME binary +on glibc 2.39 differs from the 16th significant digit on (see the README of +``terrame-goldens``). Python on any recent system behaves like the latter, so +Daisyworld is compared with a relative tolerance of ``POW_RTOL``. + RandomWalk draws random numbers, so it is compared by distribution with the 50 TerraME runs. """ @@ -28,6 +36,9 @@ GOLDENS = Path(__file__).resolve().parent / "goldens" +#: Cases that use the C library's pow, and their relative tolerance (see above). +POW_RTOL = {"daisyworld": 1e-13} + with open(GOLDENS / "cases.csv", newline="") as _f: CASES = [c for c in csv.DictReader(_f) if not c["seeds"]] @@ -63,7 +74,7 @@ def execute(self) -> None: @pytest.mark.parametrize("case", CASES, ids=[c["case"] for c in CASES]) -def test_trajectory_is_bit_identical_to_terrame(case: dict[str, str]) -> None: +def test_trajectory_matches_terrame(case: dict[str, str]) -> None: variables, ref = load(case["case"]) attrs = [snake(v) for v in variables] env = Environment(start_time=1, end_time=int(case["final_time"])) @@ -75,7 +86,12 @@ def test_trajectory_is_bit_identical_to_terrame(case: dict[str, str]) -> None: for var, e, g in zip(variables, expected, got, strict=True): if np.isnan(e): # not defined yet in TerraME (Daisyworld at step 0) continue - assert g == e, f"{case['case']}.{var} at step {t}: {g!r} != TerraME {e!r}" + rtol = POW_RTOL.get(case["case"]) + if rtol is None: + assert g == e, f"{case['case']}.{var} at step {t}: {g!r} != TerraME {e!r}" + else: + assert abs(g - e) <= rtol * abs(e), ( + f"{case['case']}.{var} at step {t}: {g!r} vs TerraME {e!r}") def test_random_walk_matches_terrame_distribution() -> None: