diff --git a/app/nullius/catalogue/benchmarks/open-telco/benchmark.json b/app/nullius/catalogue/benchmarks/open-telco/benchmark.json new file mode 100644 index 00000000..829563e6 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/open-telco/benchmark.json @@ -0,0 +1,206 @@ +{ + "key": "open-telco", + "title": "Open Telco", + "category": "Telecom", + "domain": "GSMA's eight telecom tests, from 3GPP standards and maths to 5G root-cause analysis and 6G reasoning.", + "scoring": "Answer accuracy", + "harness": [ + "standard" + ], + "origin": "official", + "good_to_know": [ + "Lite is GSMA's sample of each test, the one its leaderboard ranks; Full is the whole test.", + "Each test is a dataset scored on its own. The leaderboard's average is the mean of the tests' accuracies.", + "An answer counts only in the form its prompt asks for: 'ANSWER: ', a \\boxed{} value, or a working group's code." + ], + "upstream_limitations": [ + "3GPP TSG's scorer cannot read RAN_AH1: its 6 Lite and 129 Full questions always fail, so 94% and 93.6% are the ceilings (gsma-labs/evals#81).", + "TeleMath's scorer accepts any number within 0.01 of the answer, looser than its 1% for answers between -1 and 1 (44 Lite, 245 Full): 0 passes on 10 Lite and 47 Full questions whose answer is not 0.", + "The multiple-choice prompt asks for 'ANSWER: $LETTER': a reply that keeps the dollar sign is scored wrong, on all five multiple-choice tests (gsma-labs/evals#73)." + ], + "limitations": [ + "An agent gets TeleMath's system prompt and the question as one prompt: ACP has no system message.", + "TeleLogs' majority vote across epochs (maj@k) is not computed; several attempts give pass@k instead." + ], + "datasets": [ + { + "key": "teleqna-lite", + "title": "TeleQnA · Lite", + "tasks": 1000, + "official": "https://huggingface.co/datasets/GSMA/ot-lite", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "Telecom standards and research questions: GSMA's sample, the leaderboard's, 1,000 questions. MIT License." + }, + { + "key": "teleqna-full", + "title": "TeleQnA · Full", + "tasks": 10000, + "official": "https://huggingface.co/datasets/GSMA/ot-full", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "Telecom standards and research questions: the whole test, 10,000 questions. MIT License." + }, + { + "key": "teletables-lite", + "title": "TeleTables · Lite", + "tasks": 100, + "official": "https://huggingface.co/datasets/GSMA/ot-lite", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "Questions on tables of 3GPP specifications: GSMA's sample, the leaderboard's, 100 questions. MIT License." + }, + { + "key": "teletables-full", + "title": "TeleTables · Full", + "tasks": 500, + "official": "https://huggingface.co/datasets/GSMA/ot-full", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "Questions on tables of 3GPP specifications: the whole test, 500 questions. MIT License." + }, + { + "key": "telemath-lite", + "title": "TeleMath · Lite", + "tasks": 100, + "official": "https://huggingface.co/datasets/GSMA/ot-lite", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "GSMA's telemath_scorer: the last \\boxed{} number, right within 1% or 0.01 of the answer.", + "notes": "Telecom maths problems: GSMA's sample, the leaderboard's, 100 questions. MIT License." + }, + { + "key": "telemath-full", + "title": "TeleMath · Full", + "tasks": 500, + "official": "https://huggingface.co/datasets/GSMA/ot-full", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "GSMA's telemath_scorer: the last \\boxed{} number, right within 1% or 0.01 of the answer.", + "notes": "Telecom maths problems: the whole test, 500 questions. MIT License." + }, + { + "key": "telelogs-lite", + "title": "TeleLogs · Lite", + "tasks": 100, + "official": "https://huggingface.co/datasets/GSMA/ot-lite", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "GSMA's telelogs_scorer: the last \\boxed{} answer, its first number against the root cause's (soft, the default).", + "notes": "5G root-cause analysis from drive-test data: GSMA's sample, the leaderboard's, 100 questions. MIT License." + }, + { + "key": "telelogs-full", + "title": "TeleLogs · Full", + "tasks": 864, + "official": "https://huggingface.co/datasets/GSMA/ot-full", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "GSMA's telelogs_scorer: the last \\boxed{} answer, its first number against the root cause's (soft, the default).", + "notes": "5G root-cause analysis from drive-test data: the whole test, 864 questions. MIT License." + }, + { + "key": "3gpp_tsg-lite", + "title": "3GPP TSG · Lite", + "tasks": 100, + "official": "https://huggingface.co/datasets/GSMA/ot-lite", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "GSMA's pattern scorer: the first working-group code in the reply, case aside.", + "notes": "3GPP documents to classify by working group: GSMA's sample, the leaderboard's, 100 questions. MIT License." + }, + { + "key": "3gpp_tsg-full", + "title": "3GPP TSG · Full", + "tasks": 2000, + "official": "https://huggingface.co/datasets/GSMA/ot-full", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "GSMA's pattern scorer: the first working-group code in the reply, case aside.", + "notes": "3GPP documents to classify by working group: the whole test, 2,000 questions. MIT License." + }, + { + "key": "oranbench-lite", + "title": "ORANBench · Lite", + "tasks": 150, + "official": "https://huggingface.co/datasets/GSMA/ot-lite", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "Questions on O-RAN specifications: GSMA's sample, the leaderboard's, 150 questions. MIT License." + }, + { + "key": "oranbench-full", + "title": "ORANBench · Full", + "tasks": 1500, + "official": "https://huggingface.co/datasets/GSMA/ot-full", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "Questions on O-RAN specifications: the whole test, 1,500 questions. MIT License." + }, + { + "key": "srsranbench-lite", + "title": "srsRANBench · Lite", + "tasks": 150, + "official": "https://huggingface.co/datasets/GSMA/ot-lite", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "Questions on the srsRAN 5G codebase: GSMA's sample, the leaderboard's, 150 questions. MIT License." + }, + { + "key": "srsranbench-full", + "title": "srsRANBench · Full", + "tasks": 1502, + "official": "https://huggingface.co/datasets/GSMA/ot-full", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "Questions on the srsRAN 5G codebase: the whole test, 1,502 questions. MIT License." + }, + { + "key": "sixg_bench-lite", + "title": "6G-Bench · Lite", + "tasks": 150, + "official": "https://huggingface.co/datasets/GSMA/ot-lite", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "6G network reasoning across 30 task families: GSMA's sample, the leaderboard's, 150 questions. MIT License." + }, + { + "key": "sixg_bench-full", + "title": "6G-Bench · Full", + "tasks": 3722, + "official": "https://huggingface.co/datasets/GSMA/ot-full", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them.", + "notes": "6G network reasoning across 30 task families: the whole test, 3,722 questions. MIT License." + } + ], + "task_count": 20588, + "citation": "@misc{maatouk2023teleqna,\n title={TeleQnA: A Benchmark Dataset to Assess Large Language Models Telecommunications Knowledge},\n author={Maatouk, Ali and Ayed, Fadhel and Piovesan, Nicola and De Domenico, Antonio and Debbah, Merouane and Luo, Zhi-Quan},\n year={2023}, eprint={2310.15051}, archivePrefix={arXiv}\n}\n\n@misc{ezzakri2025teletables,\n title={TeleTables: A Benchmark for Large Language Models in Telecom Table Interpretation},\n author={Ezzakri, Anas and Piovesan, Nicola and Sana, Mohamed and De Domenico, Antonio and Ayed, Fadhel and Zhang, Haozhe},\n year={2025}, eprint={2601.04202}, archivePrefix={arXiv}\n}\n\n@misc{colle2025telemath,\n title={TeleMath: A Benchmark for Large Language Models in Telecom Mathematical Problem Solving},\n author={Colle, Vincenzo and Sana, Mohamed and Piovesan, Nicola and De Domenico, Antonio and Ayed, Fadhel and Debbah, Merouane},\n year={2025}, eprint={2506.10674}, archivePrefix={arXiv}\n}\n\n@misc{sana2025telelogs,\n title={Reasoning Language Models for Root Cause Analysis in 5G Wireless Networks},\n author={Sana, Mohamed and Piovesan, Nicola and De Domenico, Antonio and Kang, Yibin and Zhang, Haozhe and Debbah, Merouane and Ayed, Fadhel},\n year={2025}, eprint={2507.21974}, archivePrefix={arXiv}\n}\n\n@misc{zou2024telecomgpt,\n title={TelecomGPT: A Framework to Build Telecom-Specific Large Language Models},\n author={Zou, Hang and Zhao, Qiyang and Tian, Yu and Bariah, Lina and Bader, Faouzi and Lestable, Thierry and Debbah, Merouane},\n year={2024}, eprint={2407.09424}, archivePrefix={arXiv}\n}\n\n@misc{gajjar2024oranbench,\n title={ORAN-Bench-13K: An Open Source Benchmark for Assessing LLMs in Open Radio Access Networks},\n author={Gajjar, Pranshav and Shah, Vijay K.},\n year={2024}, eprint={2407.06245}, archivePrefix={arXiv}\n}\n\n@misc{gajjar2025oransight2,\n title={ORANSight-2.0: Foundational LLMs for O-RAN},\n author={Gajjar, Pranshav and Shah, Vijay K.},\n year={2025}, eprint={2503.05200}, archivePrefix={arXiv}\n}\n\n@misc{ferrag2026sixgbench,\n title={6G-Bench: An Open Benchmark for Semantic Communication and Network-Level Reasoning with Foundation Models in AI-Native 6G Networks},\n author={Ferrag, Mohamed Amine and Lakas, Abderrahmane and Debbah, Merouane},\n year={2026}, eprint={2602.08675}, archivePrefix={arXiv}\n}", + "citation_source": "https://huggingface.co/datasets/GSMA/ot-full" +} diff --git a/app/nullius/catalogue/benchmarks/open-telco/data-source.json b/app/nullius/catalogue/benchmarks/open-telco/data-source.json new file mode 100644 index 00000000..9946bfe6 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/open-telco/data-source.json @@ -0,0 +1,104 @@ +{ + "cards": { + "GSMA/ot-lite": "91c38cb7fc86f7f93a06705fec69bf454542540969a0e5cc0635dbcd1e5e70de", + "GSMA/ot-full": "fe85b5a82350552bd4cc87d48c7a667d90ef6d563f00c786dfa9319fcf8fe46d" + }, + "datasets": { + "teleqna-lite": { + "repository": "GSMA/ot-lite", + "revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7", + "file": "test_teleqna.json", + "sha256": "3bbb71301dd7a2228a8f42d4ca48c100fd57cbbb80b2e22749d5efe86ae31b5a" + }, + "teleqna-full": { + "repository": "GSMA/ot-full", + "revision": "6319806f04783eafe04d9facf755d379c66b7664", + "file": "teleqna/test-00000-of-00001.parquet", + "sha256": "73f3cd724d8ff3c1071fbb135e80d69a54b0773acde590006949a67956338e57" + }, + "teletables-lite": { + "repository": "GSMA/ot-lite", + "revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7", + "file": "test_teletables.json", + "sha256": "d08065834a41cf295242816594963a9ec3f59699c0d26e8bfaabbe6445647b73" + }, + "teletables-full": { + "repository": "GSMA/ot-full", + "revision": "6319806f04783eafe04d9facf755d379c66b7664", + "file": "teletables/test-00000-of-00001.parquet", + "sha256": "6b4d56dc644e9999519349202a80d2f9b525322ce228b5ed35387757c31b72df" + }, + "telemath-lite": { + "repository": "GSMA/ot-lite", + "revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7", + "file": "test_telemath.json", + "sha256": "5d4c64ae79d3cdec50c28a4d5cb36b098ae982bd5301aec0337358ec9db4453a" + }, + "telemath-full": { + "repository": "GSMA/ot-full", + "revision": "6319806f04783eafe04d9facf755d379c66b7664", + "file": "telemath/test-00000-of-00001.parquet", + "sha256": "c9b0523f7c6ae2ee83478f810a30fd8fbb3e320c615373a3f7e71c1e0f4cd9be" + }, + "telelogs-lite": { + "repository": "GSMA/ot-lite", + "revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7", + "file": "test_telelogs.json", + "sha256": "9d1eea3a86d39cc9ab792bca0f69edba8703e5681583e70ae82f6ef49b531bf9" + }, + "telelogs-full": { + "repository": "GSMA/ot-full", + "revision": "6319806f04783eafe04d9facf755d379c66b7664", + "file": "telelogs/test-00000-of-00001.parquet", + "sha256": "6ee5861ad6ec21f14d3debecff7c37575b0eed35efd55a2df808edf9b2b553b1" + }, + "3gpp_tsg-lite": { + "repository": "GSMA/ot-lite", + "revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7", + "file": "test_3gpp_tsg.json", + "sha256": "a7ae12665448bc35dd656629daea640a02fb96a7fe32e400f6a95055c65be20b" + }, + "3gpp_tsg-full": { + "repository": "GSMA/ot-full", + "revision": "6319806f04783eafe04d9facf755d379c66b7664", + "file": "3gpp_tsg/test-00000-of-00001.parquet", + "sha256": "1a8607099873455d9079bfbb19dfcfc6fd796973f2242eb44e1a3232a1943104" + }, + "oranbench-lite": { + "repository": "GSMA/ot-lite", + "revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7", + "file": "test_oranbench.json", + "sha256": "458be8c1962041196908aabb94838eae82633efac56c56d114c6982883e81fe8" + }, + "oranbench-full": { + "repository": "GSMA/ot-full", + "revision": "6319806f04783eafe04d9facf755d379c66b7664", + "file": "oranbench/test-00000-of-00001.parquet", + "sha256": "1dafd3e4606c9e08add68797cdb0f202679dca4e4f6136c99a30bd1939f62778" + }, + "srsranbench-lite": { + "repository": "GSMA/ot-lite", + "revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7", + "file": "test_srsranbench.json", + "sha256": "e7e43cb23c76970a8881c1f5e83319df4dcdc817cedd3841f4dd6144dd57eb88" + }, + "srsranbench-full": { + "repository": "GSMA/ot-full", + "revision": "6319806f04783eafe04d9facf755d379c66b7664", + "file": "srsranbench/test-00000-of-00001.parquet", + "sha256": "f59c40de70add5a4cce8513f7652ab4f95cdd15864b148e173f326403be48198" + }, + "sixg_bench-lite": { + "repository": "GSMA/ot-lite", + "revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7", + "file": "test_sixg_bench.json", + "sha256": "b86cff25bb490a5c10a8e22b34d0473fbe6e6b04566f8187942bd6302ba5a5da" + }, + "sixg_bench-full": { + "repository": "GSMA/ot-full", + "revision": "6319806f04783eafe04d9facf755d379c66b7664", + "file": "sixg_bench/test-00000-of-00001.parquet", + "sha256": "1e5995e1248a8f69f514a6d7afaac74fd8f82e8e05514778aecdaff873ed9371" + } + } +} diff --git a/app/nullius/catalogue/benchmarks/open-telco/driver.py b/app/nullius/catalogue/benchmarks/open-telco/driver.py new file mode 100644 index 00000000..067804ea --- /dev/null +++ b/app/nullius/catalogue/benchmarks/open-telco/driver.py @@ -0,0 +1,256 @@ +"""Open Telco's driver for the Nullius standard (JSON lines on stdin and stdout). + +The upstream is GSMA's evals repository ($NULLIUS_UPSTREAM): one Inspect AI task +per benchmark, with its loader (hf_dataset over GSMA/ot-lite or GSMA/ot-full), +its solver (the prompt, then the reading of the reply) and its scorer. Every op +runs that task as published, in Inspect's own eval loop, on one sample; only +the model in it differs: + +units - the task's samples by the id Inspect gives them: TeleTables' own ids, + else the sample's place in the file (1, 2, ...), zero-padded. A filter + option (TeleQnA's subject, ORANBench's difficulty, 6G-Bench's task) + runs the task with that argument; its samples keep their numbers +setup - the task on the unit with a model that replies nothing: what the + solver asks it (TeleMath's system prompt, then the question; the + multiple-choice template with its lettered choices) is the prompt +answer - the original setup: the task on the unit with the run's model through + the unit's relay (Inspect's OpenAI-compatible provider), as + `inspect eval` runs it; the temperature option is its --temperature +score - the task on the unit with the agent's last message as the model's + reply: the task's solver reads it and its scorer scores it +oracle - the dataset's answer, written as the task's prompt asks for it + +The data are the two datasets' Hugging Face repositories at pinned revisions +(data-source.json), each file checked by its sha256 and kept beside the +upstream; the task's own dataset_path argument points it at that copy, so +its hf_dataset call reads it with the dataset card's configuration. The +reference answers never reach the workspace. +""" +import contextlib +import fcntl +import hashlib +import importlib +import inspect +import json +import os +import shutil +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +UPSTREAM = os.environ["NULLIUS_UPSTREAM"] +DATASET = os.environ.get("NULLIUS_DATASET_ID", "") +with open(os.path.join(HERE, "data-source.json")) as f: + SOURCE = json.load(f) +WORK = os.getcwd() # the job's driver folder, shared by its drivers +FILTERS = ("subject", "difficulty", "task_id") # a task argument that picks samples +SCORING = ("eval_type",) # a task argument of the scorer +# The reply each task's prompt asks for; a multiple-choice task's is 'ANSWER: '. +FORMATS = {"telemath": "\\boxed{%s}", "telelogs": "\\boxed{%s}", "three_gpp": '{"WORKING GROUP": "%s"}'} + + +def quiet(): + return contextlib.redirect_stdout(sys.stderr) # stdout carries the replies + + +@contextlib.contextmanager +def locked(path): + with open(path, "a") as f: + fcntl.flock(f, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(f, fcntl.LOCK_UN) + + +with quiet(): + from inspect_ai import eval as inspect_eval + from inspect_ai.model import ModelOutput, get_model + from inspect_ai.scorer import value_to_float + from evals import _registry # the upstream's tasks, as Inspect registers them + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def snapshot(dataset): + """The dataset's repository at its pinned revision: its card and every file + of it a dataset names (the card configures them all), each checked.""" + own = SOURCE["datasets"][dataset] + repo, rev = own["repository"], own["revision"] + files = {"README.md": SOURCE["cards"][repo], + **{d["file"]: d["sha256"] for d in SOURCE["datasets"].values() if (d["repository"], d["revision"]) == (repo, rev)}} + root = os.path.join(UPSTREAM, ".open-telco-data", repo.replace("/", "__") + "@" + rev) + good = lambda f: os.path.isfile(os.path.join(root, f)) and sha256(os.path.join(root, f)) == files[f] + if all(good(f) for f in files): + return root + os.makedirs(os.path.dirname(root), exist_ok=True) + with locked(root + ".lock"): + missing = [f for f in files if not good(f)] + if missing: + from huggingface_hub import hf_hub_download # reads HF_ENDPOINT, the run's mirror + for f in missing: + with quiet(): + hf_hub_download(repo, f, repo_type="dataset", revision=rev, local_dir=root) + if not good(f): + os.remove(os.path.join(root, f)) + raise ValueError(f"{repo}@{rev[:12]}: {f} is not the file its sha256 names") + return root + + +def task_function(): + """The upstream's task for the dataset: the one whose module loads its configuration.""" + if DATASET not in SOURCE["datasets"]: + raise ValueError(f"no dataset {DATASET}") + config = DATASET.rsplit("-", 1)[0] + for name in _registry.__all__: + fn = getattr(_registry, name) + if getattr(importlib.import_module(fn.__module__), "DEFAULT_DATASET_NAME", None) == config: + return name, fn + raise ValueError(f"no upstream task loads {config}") + + +NAME, FN = task_function() +ROOT = snapshot(DATASET) +PARAMS = inspect.signature(FN).parameters +TASKS = {} + + +def task(**args): + key = json.dumps(args, sort_keys=True) + if key not in TASKS: + # one driver at a time: Inspect caches the dataset it loads, and another + # driver would read a cache still being written + with locked(os.path.join(WORK, "open-telco.lock")), quiet(): + TASKS[key] = FN(dataset_path=ROOT, **args) + return TASKS[key] + + +def option(name): + return os.environ.get("NULLIUS_OPTION_" + name.upper()) + + +def filters(): + """The filter options this task takes, as its arguments: a value that is + the upstream's default (or 'all', for none) picks nothing.""" + out = {} + for name in FILTERS: + value = option(name) + if name not in PARAMS or value is None or value in ("all", PARAMS[name].default): + continue + if name == "subject": # the subject as the data write it: research-overview is 'Research overview' + subjects = {str((s.metadata or {}).get("subject")) for s in task().dataset} + value = {s.lower().replace(" ", "-"): s for s in subjects}.get(value, value) + out[name] = value + return out + + +def scoring(): + return {n: option(n) for n in SCORING if n in PARAMS and option(n) is not None} + + +NUMBERED = {} + + +def numbered(): + """Each unit's id and sample, in the file's order: the sample's own id, else its place.""" + if not NUMBERED: + samples = task().dataset + width = len(str(len(samples))) + NUMBERED.update({(s.id if isinstance(s.id, str) else f"{i:0{width}d}"): s for i, s in enumerate(samples, 1)}) + return NUMBERED + + +def same(a, b): + return (a.input, a.choices, a.target, a.metadata) == (b.input, b.choices, b.target, b.metadata) + + +def units(dataset, setup=None): + every = list(numbered().items()) + picked = filters() + if not picked: + return {"units": [u for u, _ in every]} + out, i = [], 0 + for s in task(**picked).dataset: # the upstream's filter keeps the file's order + while not same(every[i][1], s): + i += 1 + out.append(every[i][0]) + i += 1 + return {"units": out} + + +def sample_id(unit): + return unit if isinstance(numbered()[unit].id, str) else int(unit) + + +def run(unit, model, **config): + """The task on one sample, in Inspect's eval loop: the sample as its log records it.""" + logs = tempfile.mkdtemp(prefix="inspect-", dir=WORK) + try: + with quiet(): + log, = inspect_eval(task(**scoring()), model=model, sample_id=sample_id(unit), log_dir=logs, + display="none", score_display=False, **config) + finally: + shutil.rmtree(logs, ignore_errors=True) + if not log.samples: + raise RuntimeError(f"Inspect's eval ended {log.status}: {log.error.message if log.error else 'no sample'}") + s = log.samples[0] + if s.error: + raise RuntimeError(s.error.message) + return s + + +def canned(text): + return get_model("mockllm/model", custom_outputs=[ModelOutput.from_content(model="mockllm/model", content=text)]) + + +def setup(unit, dir): + asked = next(e for e in run(unit, canned("")).events if e.event == "model") + return {"prompt": "\n\n".join(m.text for m in asked.input)} + + +def answer(unit, dir, setup, model, base_url, key, **_): + if setup != "original": + raise ValueError(f"Open Telco has no {setup} setup") + temperature = option("temperature") or "default" + config = {} if temperature == "default" else {"temperature": float(temperature)} + with quiet(): # each unit has a relay of its own: not kept for the next + served = get_model("openai-api/nullius/" + model, base_url=base_url, api_key=key, memoize=False) + return {"answer": run(unit, served, **config).output.completion, "model": model} + + +def score(unit, dir, transcript, answer, setup=None, **_): + s = run(unit, canned(answer or "")) + (scorer, verdict), = s.scores.items() + reward = float(value_to_float()(verdict.value)) + return {"reward": reward, "passed": reward == 1.0, + "detail": {"scorer": scorer, "value": verdict.value, "extracted": verdict.answer, "target": s.target, + "answer": (answer or "")[-4000:]}} + + +def oracle(unit, dir): + s = numbered()[unit] + return {"answer": f"ANSWER: {s.target}" if s.choices else FORMATS[NAME] % s.target} + + +HANDLERS = {"units": units, "setup": setup, "answer": answer, "score": score, "oracle": oracle} + + +def main(): + for line in sys.stdin: + try: + req = json.loads(line) + with quiet(): + out = HANDLERS[req.pop("op")](**req) + except Exception as e: + out = {"error": f"{type(e).__name__}: {e}"} + print(json.dumps(out), flush=True) + + +if __name__ == "__main__": + main() diff --git a/app/nullius/catalogue/benchmarks/open-telco/standard.json b/app/nullius/catalogue/benchmarks/open-telco/standard.json new file mode 100644 index 00000000..4d20ac5d --- /dev/null +++ b/app/nullius/catalogue/benchmarks/open-telco/standard.json @@ -0,0 +1,901 @@ +{ + "schema": "nullius.benchmark/v1", + "key": "open-telco", + "label": "Open Telco", + "summary": "Each question is one of GSMA's telecom tests, asked as its Inspect task asks it; the task's own scorer reads the last message.", + "family": "agent", + "needs": [ + "agent.task/v1" + ], + "upstream": { + "repository": "https://github.com/gsma-labs/evals", + "tag": "v1.1.0", + "tested": "708f1c5498793536824eae9919ae003f13c910ea", + "tested_date": "2026-03-07", + "license": "MIT" + }, + "driver": { + "command": [ + "{install}/.venv/bin/python", + "{here}/driver.py" + ], + "install": { + "python": "3.12", + "project": ".", + "locked": true, + "pip": [ + "openai==2.26.0" + ] + } + }, + "datasets": [ + { + "id": "teleqna-lite", + "label": "TeleQnA · Lite", + "units": 1000 + }, + { + "id": "teleqna-full", + "label": "TeleQnA · Full", + "units": 10000 + }, + { + "id": "teletables-lite", + "label": "TeleTables · Lite", + "units": 100 + }, + { + "id": "teletables-full", + "label": "TeleTables · Full", + "units": 500 + }, + { + "id": "telemath-lite", + "label": "TeleMath · Lite", + "units": 100 + }, + { + "id": "telemath-full", + "label": "TeleMath · Full", + "units": 500 + }, + { + "id": "telelogs-lite", + "label": "TeleLogs · Lite", + "units": 100 + }, + { + "id": "telelogs-full", + "label": "TeleLogs · Full", + "units": 864 + }, + { + "id": "3gpp_tsg-lite", + "label": "3GPP TSG · Lite", + "units": 100 + }, + { + "id": "3gpp_tsg-full", + "label": "3GPP TSG · Full", + "units": 2000 + }, + { + "id": "oranbench-lite", + "label": "ORANBench · Lite", + "units": 150 + }, + { + "id": "oranbench-full", + "label": "ORANBench · Full", + "units": 1500 + }, + { + "id": "srsranbench-lite", + "label": "srsRANBench · Lite", + "units": 150 + }, + { + "id": "srsranbench-full", + "label": "srsRANBench · Full", + "units": 1502 + }, + { + "id": "sixg_bench-lite", + "label": "6G-Bench · Lite", + "units": 150 + }, + { + "id": "sixg_bench-full", + "label": "6G-Bench · Full", + "units": 3722 + } + ], + "units": { + "noun": "question", + "nouns": "questions" + }, + "score": { + "kind": "pass", + "says": "GSMA's own task for the question, run in Inspect AI's eval loop with the answer as the model's reply: the multiple-choice solver reads the 'ANSWER: ' line and choice() checks the letter (TeleQnA, TeleTables, ORANBench, srsRANBench, 6G-Bench); TeleMath's scorer takes the last \\boxed{} and accepts a number within 1% or 0.01; TeleLogs' takes the last \\boxed{} and compares its first number (soft, the default) or all of it (hard); 3GPP TSG's pattern takes the first working-group code, case aside" + }, + "controls": { + "nop": { + "source": "nullius", + "from": "Nullius: no agent runs; the answer is empty." + }, + "oracle": { + "source": "nullius", + "from": "Nullius, from the authors' answers: the dataset's answer, written as the task's prompt asks for it ('ANSWER: ', \\boxed{}, {\"WORKING GROUP\": \"\"}). 3GPP TSG's pattern cannot read RAN_AH1, so those questions fail: 6 of 100 Lite, 129 of 2,000 Full.", + "expect": { + "at_least": 0.93 + } + } + }, + "setups": { + "original": { + "summary": "GSMA's own Inspect task on the question, as `inspect eval` runs it: its prompt (the multiple-choice template, TeleMath's system prompt), one call to the run's model through Inspect's OpenAI-compatible client, and its scorer. Any model, as GSMA's leaderboard ranks them.", + "providers": { + "openai": "", + "openrouter": "", + "custom": "" + }, + "turn_seconds": 900, + "paper": "GSMA's Open Telco leaderboard (GSMA/leaderboard): each model's accuracy per test on the Lite sets" + } + }, + "options": { + "temperature": { + "summary": "Sampling randomness: Inspect's --temperature. Provider default sends none, as GSMA's leaderboard runs do; enter a non-negative decimal to set one. Accepted values depend on the model.", + "default": "default", + "setups": [ + "original" + ], + "pattern": "default|(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?", + "example": "0.5" + }, + "subject": { + "summary": "TeleQnA's question category: every one, or one alone (the task's subject argument).", + "datasets": [ + "teleqna-lite", + "teleqna-full" + ], + "values": [ + "all", + "lexicon", + "research-overview", + "research-publications", + "standards-overview", + "standards-specifications" + ], + "default": "all", + "about": { + "all": "Every category. The upstream's default.", + "lexicon": "Lexicon: 50 Lite, 500 Full.", + "research-overview": "Research overview: 200 Lite, 2,000 Full.", + "research-publications": "Research publications: 450 Lite, 4,500 Full.", + "standards-overview": "Standards overview: 100 Lite, 1,000 Full.", + "standards-specifications": "Standards specifications: 200 Lite, 2,000 Full." + }, + "setups": [ + "original", + "yours" + ] + }, + "difficulty": { + "summary": "ORANBench's difficulty level: every one, or one alone (the task's difficulty argument).", + "datasets": [ + "oranbench-lite", + "oranbench-full" + ], + "values": [ + "all", + "easy", + "medium", + "hard" + ], + "default": "all", + "about": { + "all": "Every level. The upstream's default.", + "easy": "50 Lite, 500 Full.", + "medium": "45 Lite, 500 Full.", + "hard": "55 Lite, 500 Full." + }, + "setups": [ + "original", + "yours" + ] + }, + "task_id": { + "label": "Task family", + "summary": "6G-Bench's task family: all 30, or one alone (the task's task_id argument).", + "datasets": [ + "sixg_bench-lite", + "sixg_bench-full" + ], + "values": [ + "all", + "T1", + "T2", + "T3", + "T4", + "T5", + "T6", + "T7", + "T8", + "T9", + "T10", + "T11", + "T12", + "T13", + "T14", + "T15", + "T16", + "T17", + "T18", + "T19", + "T20", + "T21", + "T22", + "T23", + "T24", + "T25", + "T26", + "T27", + "T28", + "T29", + "T30" + ], + "default": "all", + "about": { + "all": "Every family. The upstream's default.", + "T1": "Intent Feasibility Assessment", + "T2": "Intent Conflict Resolution", + "T3": "Intent Drift Detection", + "T4": "Slice Selection Reasoning", + "T5": "Slice Switching Decision", + "T6": "Slice Fairness vs Safety", + "T7": "Compute Placement Decision", + "T8": "Graceful Degradation under Edge Overload", + "T9": "Trust-Aware Offloading", + "T10": "SLA Violation Prediction", + "T11": "Preemptive Autonomy Downgrade", + "T12": "Conservative Continuation Decision", + "T13": "Swarm-Level Slice Negotiation", + "T14": "Scheduler Reconfiguration Adaptation", + "T15": "Decision Consistency under Replanning", + "T16": "Network-Exposed Compute Marketplace", + "T17": "Network-Knowledge RAG Augmentation", + "T18": "AI Agent Identity & Onboarding", + "T19": "AI Agent Interoperability & Federation", + "T20": "Agent-to-Agent Communication Management", + "T21": "Device-Network Task Offload Arbitration", + "T22": "Federated / Collaborative Learning Orchestration", + "T23": "Network-Assisted Digital Twin Control", + "T24": "Sensing-Enhanced Decisioning (ISAC)", + "T25": "AI-Agent-based Disaster / Public-Safety Coordination", + "T26": "Trust-Aware Third-Party Agent Exposure", + "T27": "Agent Lifecycle & Management", + "T28": "6G Model Training-as-a-Service Decision", + "T29": "Immersive/AR Resource Prioritization", + "T30": "Network Security Detection & Response Automation" + }, + "setups": [ + "original", + "yours" + ] + }, + "eval_type": { + "label": "Answer match", + "summary": "How TeleLogs' scorer compares the boxed root cause with the answer (its eval_type).", + "datasets": [ + "telelogs-lite", + "telelogs-full" + ], + "values": [ + "soft", + "hard" + ], + "default": "soft", + "about": { + "soft": "The first number in the box: 3 and C3 both match C3. The upstream's default.", + "hard": "The whole box, case aside: only C3 matches C3." + }, + "setups": [ + "original", + "yours" + ] + } + }, + "proven": { + "teleqna-lite": { + "units": { + "units": 1000, + "passed": "1000 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 1000, + "looked_for": 0, + "short": 1000, + "passed": "1000/1000", + "note": "1000 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/1000", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 1000, + "job": "2026-11-18__13-25-23__open-telco__teleqna-lite__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "1000/1000", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 1000, + "job": "2026-11-18__13-25-23__open-telco__teleqna-lite__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "teletables-lite": { + "units": { + "units": 100, + "passed": "100 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 100, + "looked_for": 0, + "short": 100, + "passed": "100/100", + "note": "100 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/100", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 100, + "job": "2026-11-18__13-28-18__open-telco__teletables-lite__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "100/100", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 100, + "job": "2026-11-18__13-28-18__open-telco__teletables-lite__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "telemath-lite": { + "units": { + "units": 100, + "passed": "100 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 100, + "looked_for": 81, + "short": 19, + "passed": "100/100", + "note": "19 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/100", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 100, + "job": "2026-11-18__13-30-18__open-telco__telemath-lite__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "100/100", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 100, + "job": "2026-11-18__13-30-18__open-telco__telemath-lite__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "telelogs-lite": { + "units": { + "units": 100, + "passed": "100 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 100, + "looked_for": 0, + "short": 100, + "passed": "100/100", + "note": "100 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/100", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 100, + "job": "2026-11-18__13-32-19__open-telco__telelogs-lite__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "100/100", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 100, + "job": "2026-11-18__13-32-19__open-telco__telelogs-lite__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "3gpp_tsg-lite": { + "units": { + "units": 100, + "passed": "100 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 100, + "looked_for": 100, + "short": 0, + "passed": "100/100", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/100", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 100, + "job": "2026-11-18__13-34-19__open-telco__3gpp_tsg-lite__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "94/100", + "value": 0.94, + "expect": { + "at_least": 0.93 + }, + "units": 100, + "job": "2026-11-18__13-34-19__open-telco__3gpp_tsg-lite__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "oranbench-lite": { + "units": { + "units": 150, + "passed": "150 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 150, + "looked_for": 0, + "short": 150, + "passed": "150/150", + "note": "150 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/150", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 150, + "job": "2026-11-18__13-36-17__open-telco__oranbench-lite__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "150/150", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 150, + "job": "2026-11-18__13-36-17__open-telco__oranbench-lite__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "srsranbench-lite": { + "units": { + "units": 150, + "passed": "150 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 150, + "looked_for": 0, + "short": 150, + "passed": "150/150", + "note": "150 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/150", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 150, + "job": "2026-11-18__13-38-20__open-telco__srsranbench-lite__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "150/150", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 150, + "job": "2026-11-18__13-38-20__open-telco__srsranbench-lite__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "sixg_bench-lite": { + "units": { + "units": 150, + "passed": "150 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 150, + "looked_for": 0, + "short": 150, + "passed": "150/150", + "note": "150 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/150", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 150, + "job": "2026-11-18__13-40-24__open-telco__sixg_bench-lite__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "150/150", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 150, + "job": "2026-11-18__13-40-24__open-telco__sixg_bench-lite__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "telemath-full": { + "units": { + "units": 500, + "passed": "500 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 500, + "looked_for": 425, + "short": 75, + "passed": "500/500", + "note": "75 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/500", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 500, + "job": "2026-11-18__13-42-24__open-telco__telemath-full__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "500/500", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 500, + "job": "2026-11-18__13-42-24__open-telco__telemath-full__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "teletables-full": { + "units": { + "units": 500, + "passed": "500 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 500, + "looked_for": 0, + "short": 500, + "passed": "500/500", + "note": "500 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/500", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 500, + "job": "2026-11-18__13-44-54__open-telco__teletables-full__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "500/500", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 500, + "job": "2026-11-18__13-44-54__open-telco__teletables-full__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "telelogs-full": { + "units": { + "units": 864, + "passed": "864 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 864, + "looked_for": 0, + "short": 864, + "passed": "864/864", + "note": "864 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/864", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 864, + "job": "2026-11-18__13-47-16__open-telco__telelogs-full__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "864/864", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 864, + "job": "2026-11-18__13-47-16__open-telco__telelogs-full__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "oranbench-full": { + "units": { + "units": 1500, + "passed": "1500 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 1500, + "looked_for": 0, + "short": 1500, + "passed": "1500/1500", + "note": "1500 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/1500", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 1500, + "job": "2026-11-18__13-50-08__open-telco__oranbench-full__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "1500/1500", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 1500, + "job": "2026-11-18__13-50-08__open-telco__oranbench-full__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "srsranbench-full": { + "units": { + "units": 1502, + "passed": "1502 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 1502, + "looked_for": 0, + "short": 1502, + "passed": "1502/1502", + "note": "1502 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/1502", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 1502, + "job": "2026-11-18__13-53-33__open-telco__srsranbench-full__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "1502/1502", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 1502, + "job": "2026-11-18__13-53-33__open-telco__srsranbench-full__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "3gpp_tsg-full": { + "units": { + "units": 2000, + "passed": "2000 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 2000, + "looked_for": 2000, + "short": 0, + "passed": "2000/2000", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/2000", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 2000, + "job": "2026-11-18__13-56-50__open-telco__3gpp_tsg-full__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "1871/2000", + "value": 0.9355, + "expect": { + "at_least": 0.93 + }, + "units": 2000, + "job": "2026-11-18__13-56-50__open-telco__3gpp_tsg-full__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "sixg_bench-full": { + "units": { + "units": 3722, + "passed": "3722 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 3722, + "looked_for": 0, + "short": 3722, + "passed": "3722/3722", + "note": "3722 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/3722", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 3722, + "job": "2026-11-18__14-00-45__open-telco__sixg_bench-full__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "3722/3722", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 3722, + "job": "2026-11-18__14-00-45__open-telco__sixg_bench-full__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + }, + "teleqna-full": { + "units": { + "units": 10000, + "passed": "10000 ids, the same twice, unique", + "on": "2026-11-18" + }, + "leak": { + "units": 10000, + "looked_for": 0, + "short": 10000, + "passed": "10000/10000", + "note": "10000 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-18" + }, + "nop": { + "passed": "0/10000", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 10000, + "job": "2026-11-18__14-07-49__open-telco__teleqna-full__prove-nop", + "source": "nullius", + "on": "2026-11-18" + }, + "oracle": { + "passed": "10000/10000", + "value": 1.0, + "expect": { + "at_least": 0.93 + }, + "units": 10000, + "job": "2026-11-18__14-07-49__open-telco__teleqna-full__prove-oracle", + "source": "nullius", + "on": "2026-11-18" + } + } + } +} diff --git a/app/nullius/catalogue/benchmarks/open-telco/upstream-settings.json b/app/nullius/catalogue/benchmarks/open-telco/upstream-settings.json new file mode 100644 index 00000000..78e4892a --- /dev/null +++ b/app/nullius/catalogue/benchmarks/open-telco/upstream-settings.json @@ -0,0 +1,1073 @@ +[ + { + "entry": "src/evals/teleqna/teleqna.py (@task teleqna, inspect eval -T)", + "setting": "subject", + "values": "full, Lexicon, Research overview, Research publications, Standards overview, Standards specifications", + "default": "full", + "nullius": "option:subject" + }, + { + "entry": "src/evals/teleqna/teleqna.py (@task teleqna, inspect eval -T)", + "setting": "dataset_path", + "values": "any Hugging Face dataset", + "default": "GSMA/ot-lite", + "nullius": "fixed:GSMA/ot-lite or GSMA/ot-full at a pinned revision - the upstream's own datasets: a checked copy of each repository is passed as dataset_path, and the task's hf_dataset call loads it with the dataset card's configuration" + }, + { + "entry": "src/evals/teleqna/teleqna.py (@task teleqna, inspect eval -T)", + "setting": "split", + "values": "test", + "default": "test", + "nullius": "fixed:test - the only split GSMA publishes" + }, + { + "entry": "src/evals/teleqna/teleqna.py (@task teleqna, inspect eval -T)", + "setting": "full=false", + "values": "GSMA/ot-lite, 1,000 questions", + "default": "false", + "nullius": "dataset:teleqna-lite" + }, + { + "entry": "src/evals/teleqna/teleqna.py (@task teleqna, inspect eval -T)", + "setting": "full=true", + "values": "GSMA/ot-full, 10,000 questions", + "default": "", + "nullius": "dataset:teleqna-full" + }, + { + "entry": "src/evals/teletables/teletables.py (@task teletables, inspect eval -T)", + "setting": "dataset_path", + "values": "any Hugging Face dataset", + "default": "GSMA/ot-lite", + "nullius": "fixed:GSMA/ot-lite or GSMA/ot-full at a pinned revision - the upstream's own datasets: a checked copy of each repository is passed as dataset_path, and the task's hf_dataset call loads it with the dataset card's configuration" + }, + { + "entry": "src/evals/teletables/teletables.py (@task teletables, inspect eval -T)", + "setting": "split", + "values": "test", + "default": "test", + "nullius": "fixed:test - the only split GSMA publishes" + }, + { + "entry": "src/evals/teletables/teletables.py (@task teletables, inspect eval -T)", + "setting": "full=false", + "values": "GSMA/ot-lite, 100 questions", + "default": "false", + "nullius": "dataset:teletables-lite" + }, + { + "entry": "src/evals/teletables/teletables.py (@task teletables, inspect eval -T)", + "setting": "full=true", + "values": "GSMA/ot-full, 500 questions", + "default": "", + "nullius": "dataset:teletables-full" + }, + { + "entry": "src/evals/telemath/telemath.py (@task telemath, inspect eval -T)", + "setting": "dataset_path", + "values": "any Hugging Face dataset", + "default": "GSMA/ot-lite", + "nullius": "fixed:GSMA/ot-lite or GSMA/ot-full at a pinned revision - the upstream's own datasets: a checked copy of each repository is passed as dataset_path, and the task's hf_dataset call loads it with the dataset card's configuration" + }, + { + "entry": "src/evals/telemath/telemath.py (@task telemath, inspect eval -T)", + "setting": "split", + "values": "test", + "default": "test", + "nullius": "fixed:test - the only split GSMA publishes" + }, + { + "entry": "src/evals/telemath/telemath.py (@task telemath, inspect eval -T)", + "setting": "full=false", + "values": "GSMA/ot-lite, 100 questions", + "default": "false", + "nullius": "dataset:telemath-lite" + }, + { + "entry": "src/evals/telemath/telemath.py (@task telemath, inspect eval -T)", + "setting": "full=true", + "values": "GSMA/ot-full, 500 questions", + "default": "", + "nullius": "dataset:telemath-full" + }, + { + "entry": "src/evals/telelogs/telelogs.py (@task telelogs, inspect eval -T)", + "setting": "eval_type", + "values": "soft, hard", + "default": "soft", + "nullius": "option:eval_type" + }, + { + "entry": "src/evals/telelogs/telelogs.py (@task telelogs, inspect eval -T)", + "setting": "dataset_path", + "values": "any Hugging Face dataset", + "default": "GSMA/ot-lite", + "nullius": "fixed:GSMA/ot-lite or GSMA/ot-full at a pinned revision - the upstream's own datasets: a checked copy of each repository is passed as dataset_path, and the task's hf_dataset call loads it with the dataset card's configuration" + }, + { + "entry": "src/evals/telelogs/telelogs.py (@task telelogs, inspect eval -T)", + "setting": "split", + "values": "test", + "default": "test", + "nullius": "fixed:test - the only split GSMA publishes" + }, + { + "entry": "src/evals/telelogs/telelogs.py (@task telelogs, inspect eval -T)", + "setting": "full=false", + "values": "GSMA/ot-lite, 100 questions", + "default": "false", + "nullius": "dataset:telelogs-lite" + }, + { + "entry": "src/evals/telelogs/telelogs.py (@task telelogs, inspect eval -T)", + "setting": "full=true", + "values": "GSMA/ot-full, 864 questions", + "default": "", + "nullius": "dataset:telelogs-full" + }, + { + "entry": "src/evals/three_gpp/three_gpp.py (@task three_gpp, inspect eval -T)", + "setting": "dataset_path", + "values": "any Hugging Face dataset", + "default": "GSMA/ot-lite", + "nullius": "fixed:GSMA/ot-lite or GSMA/ot-full at a pinned revision - the upstream's own datasets: a checked copy of each repository is passed as dataset_path, and the task's hf_dataset call loads it with the dataset card's configuration" + }, + { + "entry": "src/evals/three_gpp/three_gpp.py (@task three_gpp, inspect eval -T)", + "setting": "split", + "values": "test", + "default": "test", + "nullius": "fixed:test - the only split GSMA publishes" + }, + { + "entry": "src/evals/three_gpp/three_gpp.py (@task three_gpp, inspect eval -T)", + "setting": "full=false", + "values": "GSMA/ot-lite, 100 questions", + "default": "false", + "nullius": "dataset:3gpp_tsg-lite" + }, + { + "entry": "src/evals/three_gpp/three_gpp.py (@task three_gpp, inspect eval -T)", + "setting": "full=true", + "values": "GSMA/ot-full, 2,000 questions", + "default": "", + "nullius": "dataset:3gpp_tsg-full" + }, + { + "entry": "src/evals/oranbench/oranbench.py (@task oranbench, inspect eval -T)", + "setting": "difficulty", + "values": "full, easy, medium, hard", + "default": "full", + "nullius": "option:difficulty" + }, + { + "entry": "src/evals/oranbench/oranbench.py (@task oranbench, inspect eval -T)", + "setting": "dataset_path", + "values": "any Hugging Face dataset", + "default": "GSMA/ot-lite", + "nullius": "fixed:GSMA/ot-lite or GSMA/ot-full at a pinned revision - the upstream's own datasets: a checked copy of each repository is passed as dataset_path, and the task's hf_dataset call loads it with the dataset card's configuration" + }, + { + "entry": "src/evals/oranbench/oranbench.py (@task oranbench, inspect eval -T)", + "setting": "split", + "values": "test", + "default": "test", + "nullius": "fixed:test - the only split GSMA publishes" + }, + { + "entry": "src/evals/oranbench/oranbench.py (@task oranbench, inspect eval -T)", + "setting": "full=false", + "values": "GSMA/ot-lite, 150 questions", + "default": "false", + "nullius": "dataset:oranbench-lite" + }, + { + "entry": "src/evals/oranbench/oranbench.py (@task oranbench, inspect eval -T)", + "setting": "full=true", + "values": "GSMA/ot-full, 1,500 questions", + "default": "", + "nullius": "dataset:oranbench-full" + }, + { + "entry": "src/evals/srsranbench/srsranbench.py (@task srsranbench, inspect eval -T)", + "setting": "dataset_path", + "values": "any Hugging Face dataset", + "default": "GSMA/ot-lite", + "nullius": "fixed:GSMA/ot-lite or GSMA/ot-full at a pinned revision - the upstream's own datasets: a checked copy of each repository is passed as dataset_path, and the task's hf_dataset call loads it with the dataset card's configuration" + }, + { + "entry": "src/evals/srsranbench/srsranbench.py (@task srsranbench, inspect eval -T)", + "setting": "split", + "values": "test", + "default": "test", + "nullius": "fixed:test - the only split GSMA publishes" + }, + { + "entry": "src/evals/srsranbench/srsranbench.py (@task srsranbench, inspect eval -T)", + "setting": "full=false", + "values": "GSMA/ot-lite, 150 questions", + "default": "false", + "nullius": "dataset:srsranbench-lite" + }, + { + "entry": "src/evals/srsranbench/srsranbench.py (@task srsranbench, inspect eval -T)", + "setting": "full=true", + "values": "GSMA/ot-full, 1,502 questions", + "default": "", + "nullius": "dataset:srsranbench-full" + }, + { + "entry": "src/evals/sixg_bench/sixg_bench.py (@task sixg_bench, inspect eval -T)", + "setting": "task_id", + "values": "None, T1 to T30", + "default": "None", + "nullius": "option:task_id" + }, + { + "entry": "src/evals/sixg_bench/sixg_bench.py (@task sixg_bench, inspect eval -T)", + "setting": "dataset_path", + "values": "any Hugging Face dataset", + "default": "GSMA/ot-lite", + "nullius": "fixed:GSMA/ot-lite or GSMA/ot-full at a pinned revision - the upstream's own datasets: a checked copy of each repository is passed as dataset_path, and the task's hf_dataset call loads it with the dataset card's configuration" + }, + { + "entry": "src/evals/sixg_bench/sixg_bench.py (@task sixg_bench, inspect eval -T)", + "setting": "split", + "values": "test", + "default": "test", + "nullius": "fixed:test - the only split GSMA publishes" + }, + { + "entry": "src/evals/sixg_bench/sixg_bench.py (@task sixg_bench, inspect eval -T)", + "setting": "full=false", + "values": "GSMA/ot-lite, 150 questions", + "default": "false", + "nullius": "dataset:sixg_bench-lite" + }, + { + "entry": "src/evals/sixg_bench/sixg_bench.py (@task sixg_bench, inspect eval -T)", + "setting": "full=true", + "values": "GSMA/ot-full, 3,722 questions", + "default": "", + "nullius": "dataset:sixg_bench-full" + }, + { + "entry": "src/evals/teleqna/teleqna.py (hard-coded)", + "setting": "solver multiple_choice(cot)", + "values": "cot=False, cot=True", + "default": "cot=False", + "nullius": "fixed:cot=False - the upstream's: the template asks for 'ANSWER: ' alone" + }, + { + "entry": "src/evals/teleqna/teleqna.py (hard-coded)", + "setting": "scorer", + "values": "choice()", + "default": "choice()", + "nullius": "fixed:choice() - the upstream's, run as published" + }, + { + "entry": "src/evals/teletables/teletables.py (hard-coded)", + "setting": "solver multiple_choice(cot)", + "values": "cot=False, cot=True", + "default": "cot=False", + "nullius": "fixed:cot=False - the upstream's: the template asks for 'ANSWER: ' alone" + }, + { + "entry": "src/evals/teletables/teletables.py (hard-coded)", + "setting": "scorer", + "values": "choice()", + "default": "choice()", + "nullius": "fixed:choice() - the upstream's, run as published" + }, + { + "entry": "src/evals/oranbench/oranbench.py (hard-coded)", + "setting": "solver multiple_choice(cot)", + "values": "cot=False, cot=True", + "default": "cot=False", + "nullius": "fixed:cot=False - the upstream's: the template asks for 'ANSWER: ' alone" + }, + { + "entry": "src/evals/oranbench/oranbench.py (hard-coded)", + "setting": "scorer", + "values": "choice()", + "default": "choice()", + "nullius": "fixed:choice() - the upstream's, run as published" + }, + { + "entry": "src/evals/srsranbench/srsranbench.py (hard-coded)", + "setting": "solver multiple_choice(cot)", + "values": "cot=False, cot=True", + "default": "cot=False", + "nullius": "fixed:cot=False - the upstream's: the template asks for 'ANSWER: ' alone" + }, + { + "entry": "src/evals/srsranbench/srsranbench.py (hard-coded)", + "setting": "scorer", + "values": "choice()", + "default": "choice()", + "nullius": "fixed:choice() - the upstream's, run as published" + }, + { + "entry": "src/evals/sixg_bench/sixg_bench.py (hard-coded)", + "setting": "solver multiple_choice(cot)", + "values": "cot=False, cot=True", + "default": "cot=False", + "nullius": "fixed:cot=False - the upstream's: the template asks for 'ANSWER: ' alone" + }, + { + "entry": "src/evals/sixg_bench/sixg_bench.py (hard-coded)", + "setting": "scorer", + "values": "choice()", + "default": "choice()", + "nullius": "fixed:choice() - the upstream's, run as published" + }, + { + "entry": "src/evals/telemath/telemath.py (hard-coded)", + "setting": "SYSTEM_PROMPT", + "values": "the solving guidelines, ending in the \\boxed{} format", + "default": "", + "nullius": "fixed:the upstream's - the Original's model gets it as the system message, an agent ahead of the question (limitations)" + }, + { + "entry": "src/evals/telemath/telemath.py (hard-coded)", + "setting": "telemath_scorer tolerance", + "values": "math.isclose rel_tol=0.01, abs_tol=0.01", + "default": "0.01, 0.01", + "nullius": "fixed:the upstream's - 0 passes on the questions whose answer is within 0.01 of it (upstream limitations)" + }, + { + "entry": "src/evals/telelogs/telelogs.py (hard-coded)", + "setting": "metric maj_at_k", + "values": "majority vote over epochs", + "default": "", + "nullius": "not offered - a metric across epochs; a run's attempts give pass@k instead (limitations)" + }, + { + "entry": "src/evals/three_gpp/three_gpp.py (hard-coded)", + "setting": "WG_PATTERN", + "values": "([A-Z]+\\d+(?:-[A-Z]+)?), ignore_case", + "default": "", + "nullius": "fixed:the upstream's - it cannot read RAN_AH1, whose questions always fail (upstream limitations)" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--model", + "values": "any Inspect model", + "default": "", + "nullius": "setup:original" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--model-base-url", + "values": "a URL", + "default": "", + "nullius": "fixed:the unit's relay - every call goes through it, the key never leaves it" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--model-config", + "values": "a file", + "default": "", + "nullius": "not offered - model arguments; GSMA's runs set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--model-role", + "values": "role=model", + "default": "", + "nullius": "not offered - GSMA's tasks ask one model" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--task-config", + "values": "a file", + "default": "", + "nullius": "fixed:the task arguments above - from the run's options and dataset" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--solver", + "values": "a solver", + "default": "the task's", + "nullius": "not offered - the task's own solver is the benchmark" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--solver-config", + "values": "a file", + "default": "", + "nullius": "not offered - the task's own solver is the benchmark" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--tags", + "values": "text", + "default": "", + "nullius": "not offered - a log label, not what is measured" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--metadata", + "values": "key=value", + "default": "", + "nullius": "not offered - a log label, not what is measured" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--approval", + "values": "a file", + "default": "", + "nullius": "not offered - GSMA's tasks give the model no tools" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--sandbox", + "values": "docker, local, ...", + "default": "", + "nullius": "not offered - GSMA's tasks use no sandbox" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--no-sandbox-cleanup", + "values": "flag", + "default": "", + "nullius": "not offered - GSMA's tasks use no sandbox" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--limit", + "values": "N or N-M", + "default": "every sample", + "nullius": "fixed:the run's --limit - the first N questions of the file, or of an option's subset" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--sample-id", + "values": "ids", + "default": "every sample", + "nullius": "fixed:one question per unit - the driver runs Inspect on the unit's own sample id" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--sample-shuffle", + "values": "a seed", + "default": "off", + "nullius": "not offered - the order of questions, not what is measured" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--epochs", + "values": "N", + "default": "1", + "nullius": "fixed:1 per attempt - a run's --attempts K asks each question K times (pass@k); the leaderboard runs 1" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--epochs-reducer", + "values": "mean, median, mode, max, pass_at_k", + "default": "mean", + "nullius": "fixed:mean - Inspect's default; one epoch a unit, nothing to reduce" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--no-epochs-reducer", + "values": "flag", + "default": "off", + "nullius": "fixed:off - one epoch a unit" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--max-connections", + "values": "N", + "default": "model specific", + "nullius": "fixed:the run's concurrency (-j) - how many questions at once, not what is measured" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--max-retries", + "values": "N", + "default": "unlimited", + "nullius": "fixed:unlimited - Inspect's default: a rate-limited call waits and retries until the unit's time ends (limitations)" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--timeout", + "values": "seconds", + "default": "none", + "nullius": "fixed:none - Inspect's default; the unit's limit (900 s) ends a call that hangs" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--attempt-timeout", + "values": "seconds", + "default": "none", + "nullius": "fixed:none - Inspect's default" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--max-samples", + "values": "N", + "default": "max-connections + 1", + "nullius": "fixed:the run's concurrency (-j) - how many questions at once, not what is measured" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--max-tasks", + "values": "N", + "default": "1", + "nullius": "fixed:the run's concurrency (-j) - how many questions at once, not what is measured" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--max-subprocesses", + "values": "N", + "default": "CPU count", + "nullius": "not offered - GSMA's tasks start no subprocess" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--max-sandboxes", + "values": "N", + "default": "", + "nullius": "not offered - GSMA's tasks use no sandbox" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--message-limit", + "values": "N", + "default": "none", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--token-limit", + "values": "N", + "default": "none", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--cost-limit", + "values": "dollars", + "default": "none", + "nullius": "fixed:none - a run's --max-usd caps spending instead" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--model-cost-config", + "values": "a file", + "default": "", + "nullius": "not offered - a cost comes from the run's price list" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--time-limit", + "values": "seconds", + "default": "none", + "nullius": "fixed:none - the unit's limit (900 s) instead" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--working-limit", + "values": "seconds", + "default": "none", + "nullius": "fixed:none - the unit's limit (900 s) instead" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--fail-on-error", + "values": "a threshold", + "default": "every error fails", + "nullius": "fixed:a sample's error is the unit's error - a run's --retries runs it again" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--no-fail-on-error", + "values": "flag", + "default": "off", + "nullius": "fixed:off - a sample's error is the unit's error" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--continue-on-fail", + "values": "flag", + "default": "off", + "nullius": "fixed:off - one sample per eval" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--retry-on-error", + "values": "N", + "default": "none", + "nullius": "fixed:none - a run's --retries instead" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--no-log-samples", + "values": "flag", + "default": "off", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--no-log-realtime", + "values": "flag", + "default": "off", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--log-images", + "values": "flag", + "default": "on", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--log-model-api", + "values": "flag", + "default": "off", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--log-refusals", + "values": "flag", + "default": "off", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--log-buffer", + "values": "N", + "default": "", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--log-shared", + "values": "seconds", + "default": "", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--log-format", + "values": "eval, json", + "default": "eval", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--log-level-transcript", + "values": "a level", + "default": "info", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--log-level", + "values": "a level", + "default": "warning", + "nullius": "fixed:warning - Inspect's default, into the driver's log" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--log-dir", + "values": "a folder", + "default": "./logs", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--no-score", + "values": "flag", + "default": "off", + "nullius": "fixed:scored - the score is the unit's verdict" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--no-score-display", + "values": "flag", + "default": "off", + "nullius": "fixed:no display - the driver runs headless" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--generate-config", + "values": "a file", + "default": "", + "nullius": "fixed:none - temperature is an option, the rest Inspect's defaults" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--max-tokens", + "values": "N", + "default": "the provider's", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--system-message", + "values": "text", + "default": "", + "nullius": "not offered - it would replace the task's own prompt" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--best-of", + "values": "N", + "default": "1", + "nullius": "not offered - one reply per call, as GSMA runs" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--frequency-penalty", + "values": "-2 to 2", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--presence-penalty", + "values": "-2 to 2", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--logit-bias", + "values": "token=bias", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--seed", + "values": "N", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--stop-seqs", + "values": "text", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--temperature", + "values": "0 to 2", + "default": "the provider's", + "nullius": "option:temperature" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--top-p", + "values": "0 to 1", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--top-k", + "values": "N", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--num-choices", + "values": "N", + "default": "1", + "nullius": "not offered - one reply per call, as GSMA runs" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--logprobs", + "values": "flag", + "default": "off", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--top-logprobs", + "values": "0 to 20", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--parallel-tool-calls", + "values": "flag", + "default": "", + "nullius": "not offered - GSMA's tasks give the model no tools" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--internal-tools", + "values": "flag", + "default": "", + "nullius": "not offered - GSMA's tasks give the model no tools" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--max-tool-output", + "values": "bytes", + "default": "", + "nullius": "not offered - GSMA's tasks give the model no tools" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--cache-prompt", + "values": "auto, true, false", + "default": "auto", + "nullius": "fixed:auto - Inspect's default" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--verbosity", + "values": "low, medium, high", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--effort", + "values": "low, medium, high, max", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--reasoning-effort", + "values": "none to xhigh", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--reasoning-tokens", + "values": "N", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--reasoning-summary", + "values": "none, concise, detailed, auto", + "default": "", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--reasoning-history", + "values": "none, all, last, auto", + "default": "auto", + "nullius": "fixed:auto - Inspect's default; one call a question" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--response-schema", + "values": "a JSON schema", + "default": "", + "nullius": "not offered - it would change the answer the task's solver reads" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--cache", + "values": "a policy", + "default": "off", + "nullius": "fixed:off - Inspect's default: every call is made" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--batch", + "values": "a batch size", + "default": "off", + "nullius": "fixed:off - Inspect's default: one call a question" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--display", + "values": "full, conversation, rich, plain, log, none", + "default": "full", + "nullius": "fixed:none - the driver runs headless" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--traceback-locals", + "values": "flag", + "default": "off", + "nullius": "not offered - debugging" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--env", + "values": "NAME=VALUE", + "default": "", + "nullius": "not offered - the driver's environment is the run's" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--debug", + "values": "flag", + "default": "off", + "nullius": "not offered - debugging" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--debug-port", + "values": "N", + "default": "", + "nullius": "not offered - debugging" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "--debug-errors", + "values": "flag", + "default": "off", + "nullius": "not offered - debugging" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "INSPECT_EVAL_* variables", + "values": "each flag above", + "default": "unset", + "nullius": "fixed:unset - the driver passes its settings as arguments" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "OPENAI_API_KEY, OPENROUTER_API_KEY, ...", + "values": "a provider's key", + "default": "", + "nullius": "fixed:the relay's word - the key stays in the relay" + }, + { + "entry": "inspect eval (the README's and docs' command)", + "setting": "HF_TOKEN", + "values": "a Hugging Face token", + "default": "unset", + "nullius": "not offered - GSMA's datasets are public" + }, + { + "entry": "src/evals/run_evals.py (python -m evals.run_evals)", + "setting": "--full", + "values": "off (GSMA/ot-lite), on (GSMA/ot-full)", + "default": "off", + "nullius": "fixed:a dataset per test and size - each test's · Lite or · Full dataset; a launch may select several" + }, + { + "entry": "src/evals/run_evals.py (python -m evals.run_evals)", + "setting": "tasks", + "values": "telelogs, three_gpp, teleqna, telemath, teletables, oranbench, srsranbench", + "default": "these seven", + "nullius": "fixed:one test per dataset - 6G-Bench is a dataset too" + }, + { + "entry": "src/evals/run_evals.py (python -m evals.run_evals)", + "setting": "model", + "values": "openrouter/openai/gpt-5.2, openrouter/anthropic/claude-opus-4.5, openrouter/google/gemini-3-flash-preview, openrouter/mistralai/mistral-large-3-2512, openrouter/deepseek/deepseek-v3.2", + "default": "these five", + "nullius": "setup:original" + }, + { + "entry": "src/evals/run_evals.py (python -m evals.run_evals)", + "setting": "epochs", + "values": "1", + "default": "1", + "nullius": "fixed:1 - a run's --attempts asks again" + }, + { + "entry": "src/evals/run_evals.py (python -m evals.run_evals)", + "setting": "temperature", + "values": "0.0", + "default": "0.0", + "nullius": "option:temperature" + }, + { + "entry": "src/evals/run_evals.py (python -m evals.run_evals)", + "setting": "log_dir", + "values": "logs/leaderboard", + "default": "logs/leaderboard", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "gsma-labs/satellite (open-telco CLI, run-evals: the leaderboard's submissions)", + "setting": "full", + "values": "False, always", + "default": "False", + "nullius": "fixed:the · Lite datasets are the leaderboard's - Full is a dataset of its own" + }, + { + "entry": "gsma-labs/satellite (open-telco CLI, run-evals: the leaderboard's submissions)", + "setting": "epochs", + "values": "N", + "default": "1 (DEFAULT_EPOCHS)", + "nullius": "fixed:1 - a run's --attempts asks again" + }, + { + "entry": "gsma-labs/satellite (open-telco CLI, run-evals: the leaderboard's submissions)", + "setting": "max_connections", + "values": "N", + "default": "10 (DEFAULT_MAX_CONNECTIONS)", + "nullius": "fixed:the run's concurrency (-j) - how many questions at once, not what is measured" + }, + { + "entry": "gsma-labs/satellite (open-telco CLI, run-evals: the leaderboard's submissions)", + "setting": "limit", + "values": "N", + "default": "none", + "nullius": "fixed:the run's --limit - how many questions, not how they are scored" + }, + { + "entry": "gsma-labs/satellite (open-telco CLI, run-evals: the leaderboard's submissions)", + "setting": "token_limit", + "values": "N", + "default": "none", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "gsma-labs/satellite (open-telco CLI, run-evals: the leaderboard's submissions)", + "setting": "message_limit", + "values": "N", + "default": "none", + "nullius": "fixed:unset - Inspect's default; GSMA's docs, run_evals.py and leaderboard CLI set none" + }, + { + "entry": "gsma-labs/satellite (open-telco CLI, run-evals: the leaderboard's submissions)", + "setting": "log_format, display, log_samples, log_realtime", + "values": "json, none, true, true", + "default": "", + "nullius": "fixed:a temporary log per question - the relay records each model call, the trial the verdict" + }, + { + "entry": "driver (ours)", + "setting": "model client", + "values": "openai-api/nullius/", + "default": "", + "nullius": "fixed:Inspect's OpenAI-compatible client to the unit's relay, whatever the provider - GSMA's runs used openrouter/, the same request at their defaults" + }, + { + "entry": "driver (ours)", + "setting": "agent prompt", + "values": "the messages the task's solver sends, joined by a blank line", + "default": "", + "nullius": "fixed:Nullius's - an agent has no system message, so TeleMath's comes ahead of the question (limitations)" + }, + { + "entry": "driver (ours)", + "setting": "oracle reply", + "values": "'ANSWER: ', \\boxed{}, {\"WORKING GROUP\": \"\"}", + "default": "", + "nullius": "fixed:Nullius's - the dataset's answer written as the task's prompt asks for it" + }, + { + "entry": "driver (ours)", + "setting": "data copy", + "values": "the two repositories at pinned revisions, each file checked by its sha256", + "default": "", + "nullius": "fixed:a copy beside the upstream (.open-telco-data) - downloaded once through the run's Hugging Face endpoint" + } +] diff --git a/app/nullius/catalogue/sources.py b/app/nullius/catalogue/sources.py index 0ede402f..fa765fe3 100644 --- a/app/nullius/catalogue/sources.py +++ b/app/nullius/catalogue/sources.py @@ -52,16 +52,20 @@ def _files(defined: List[dict]) -> List[dict]: return out -def _data_source(here) -> Optional[tuple]: +def _data_source(here, key: str = "") -> Optional[tuple]: """The Hugging Face dataset a driver downloads itself, ``data-source.json`` - beside its definition: (repository, revision), or None.""" + beside its definition: (repository, revision), or None. A dataset's own + entry there (``datasets.``) may name its repository and revision, + for a driver that reads several.""" if not here: return None try: with open(os.path.join(here, "data-source.json")) as f: src = json.load(f) - repo, rev = src["repository"], src["revision"] - except (OSError, ValueError, TypeError, KeyError): + norm = key.replace("-", "_") + own = next((v for k, v in (src.get("datasets") or {}).items() if k.replace("-", "_") == norm), None) or {} + repo, rev = own.get("repository", src.get("repository")), own.get("revision", src.get("revision")) + except (OSError, ValueError, TypeError, AttributeError): return None ok = isinstance(repo, str) and _HF_REPO.match(repo) and isinstance(rev, str) and _REVISION.match(rev) return (repo, rev) if ok else None @@ -98,7 +102,7 @@ def of(dataset, harness: List[str], standard: Optional[dict] = None, only: bool rows = [] if standard: rows.append(dict(_own(standard.get("upstream") or {}, _matching(standard.get("datasets") or [], dataset.key, only), "Nullius", - _data_source(standard.get("_here"))), + _data_source(standard.get("_here"), dataset.key)), way="standard", label="Nullius")) if "harbor" in harness and (dataset.hub or dataset.registry): rows.append({"way": "harbor", "label": "Harbor", "from": "Harbor hub" if dataset.hub else "Harbor registry", diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 8e3570b2..9129248a 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -503,11 +503,37 @@ def _merged(proven: dict, got: Dict[str, dict]) -> dict: return proven +def _pct(v: float) -> str: + return f"{v * 100:g}%" + + +def must_score(expect: dict) -> str: + """A control's bound as a page says it: "nothing" and "everything" at the + full mark, else the bound ("at least 93%").""" + (bound, v), = expect.items() + if bound == "at_most": + return "nothing" if v <= 0 else f"at most {_pct(v)}" + return "everything" if v >= 1 else f"at least {_pct(v)}" + + +def short_of(rec: Optional[dict], expect: dict) -> str: + """A proof that met a bound short of the full mark - an oracle under every + unit, a nop over none - as a page says it: what it reached, and the bound + ("93.55%, at least 93% needed"). "" at the full mark, or with no record.""" + if not rec or not isinstance(rec.get("value"), (int, float)): + return "" + (bound, v), = (rec.get("expect") or expect).items() + if (rec["value"] <= 0) if bound == "at_most" else (rec["value"] >= 1): + return "" + return f"{_pct(rec['value'])}, " + (f"at most {_pct(v)} allowed" if bound == "at_most" else f"at least {_pct(v)} needed") + + def status(bench: dict) -> Dict[str, Dict[str, dict]]: """Per dataset, each control the benchmark declares - its source, where it - comes from, its bound - and whether it is proven on that dataset, with the - record; then the units and leak checks the same way. ``failed``: the last - attempt that did not pass, while none has since. What a page shows.""" + comes from, its bound and what it must score - and whether it is proven on + that dataset, with the record and how far short of the full mark it is + (``short_of``); then the units and leak checks the same way. ``failed``: + the last attempt that did not pass, while none has since. What a page shows.""" out = {} controls = bench.get("controls") or {} for ds in bench["datasets"]: @@ -517,7 +543,8 @@ def status(bench: dict) -> Dict[str, Dict[str, dict]]: for c in definitions.CONTROLS: if c in controls: d = definitions.control(bench, c) - row[c] = {"source": d["source"], "from": d["from"], "expect": d["expect"], "proven": rec.get(c), "failed": failed.get(c)} + row[c] = {"source": d["source"], "from": d["from"], "expect": d["expect"], "must": must_score(d["expect"]), + "proven": rec.get(c), "short": short_of(rec.get(c), d["expect"]), "failed": failed.get(c)} for c in ("units", "leak"): row[c] = {"proven": rec.get(c), "failed": failed.get(c)} out[ds["id"]] = row diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 85b62fa9..018cba1b 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -116,7 +116,7 @@ def _bench_page(b: dict) -> dict: ``proofs``, the units and leak checks the same way (``prove.status``).""" kinds = sorted({p.split(".", 1)[1].split("/")[0] for p in b.get("needs") or []}) if b["family"] == "capability" else [] status = prove.status(b) - controls = [{k: c[k] for k in ("key", "label", "source", "from", "expect")} + controls = [{**{k: c[k] for k in ("key", "label", "source", "from", "expect")}, "must": prove.must_score(c["expect"])} for c in (definitions.control(b, w) for w in definitions.CONTROLS) if c] versioned = definitions.version_of(b) is not None return {"key": b["key"], "label": b["label"], "summary": b["summary"], "family": b["family"], "kind": kinds[0] if kinds else "", @@ -163,7 +163,7 @@ def _bench_page(b: dict) -> dict: for n, r in hostreq.all_of(b).items()}, "datasets": [{"id": d["id"], "label": d.get("label") or d["id"], "summary": d.get("summary", ""), "units": d.get("units"), "environment": d.get("environment") or "", - "controls": [dict(c, proven=status[d["id"]][c["key"]]["proven"], failed=status[d["id"]][c["key"]]["failed"]) + "controls": [dict(c, **{k: status[d["id"]][c["key"]][k] for k in ("proven", "short", "failed")}) for c in controls], "proofs": {k: status[d["id"]][k]["proven"] for k in ("units", "leak")}, "failed": {k: status[d["id"]][k]["failed"] for k in ("units", "leak") if status[d["id"]][k]["failed"]}} diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 6b8f81d0..7f8d0045 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -442,16 +442,17 @@ // reached against its bound, or not run. Its source (nullius/standard/definitions.py, SOURCES) is in its title. const SOURCE_SAYS = {official: 'from the authors', adapter: 'from Harbor\'s adapter', nullius: 'made by Nullius'}; const bound = e => { const [k, v] = Object.entries(e || {})[0] || ['at_least', 1]; return [k === 'at_most' ? '>' : '<', v]; }; - function proofOf(proven, failed, expect) { - if (proven) return {state: 'ok', text: '✓ ' + (proven.passed || '') + (proven.limit ? ' · first ' + proven.limit : ''), title: 'proven on ' + (proven.on || '?') + (proven.shard ? ', shard ' + proven.shard : '')}; + // short: a proof that met a bound short of the full mark (prove.short_of) - its tick is not green, its bound on hover + function proofOf(proven, failed, expect, short) { + if (proven) return {state: short ? 'part' : 'ok', text: '✓ ' + (proven.passed || '') + (proven.limit ? ' · first ' + proven.limit : ''), title: (short ? short + ' · ' : '') + 'proven on ' + (proven.on || '?') + (proven.shard ? ', shard ' + proven.shard : '')}; if (failed && failed.value !== undefined) { const [op, v] = bound(failed.expect || expect); return {state: 'no', text: '✕ ' + failed.value + ' ' + op + ' ' + v, title: 'not proven: ' + [failed.passed, failed.on, ...(failed.problems || [])].filter(Boolean).join(' · ')}; } if (failed) return {state: 'no', text: '✕ not proven', title: (failed.problems || []).join(' · ')}; return {state: 'none', text: 'not run'}; } const proofSpan = p => `${esc(p.text)}`; // A control or a check as a plain chip: its name and its proof; where the control comes from is in its title. - const controlChip = (c, proof = true) => `${esc(c.key)}` - + (proof ? proofSpan(proofOf(c.proven, c.failed, c.expect)) : '') + ''; + const controlChip = (c, proof = true) => `${esc(c.key)}` + + (proof ? proofSpan(proofOf(c.proven, c.failed, c.expect, c.short)) : '') + ''; const checkChip = (name, proven, failed, says) => { const p = proofOf(proven, failed); if (p.state === 'ok') { p.title = [says, proven.passed, p.title].filter(Boolean).join(' · '); p.text = '✓'; } return `${esc(name)}${proofSpan(p)}`; }; // The yes to an agent running as you: asked only for the local backend, where it is no sandbox. diff --git a/app/static/catalogue/marks.svg b/app/static/catalogue/marks.svg index 1576916d..dce84e25 100644 --- a/app/static/catalogue/marks.svg +++ b/app/static/catalogue/marks.svg @@ -33,4 +33,6 @@ + + diff --git a/app/static/style.css b/app/static/style.css index 9eb9201f..5f33e414 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -1753,6 +1753,7 @@ pre.instruction { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 32 .std-checkchip { border-color: var(--border-soft); color: var(--text-dim); } .std-proof { font: 11px/1.6 var(--sans); overflow: hidden; text-overflow: ellipsis; } .std-proof-ok { color: var(--ok); } +.std-proof-part { color: var(--patch_failed_tests); } .std-proof-no { color: var(--bad); } .std-proof-none { color: var(--text-faint); } .std-fidelity.ok { color: var(--ok) !important; } diff --git a/app/templates/catalogue/benchmark.html b/app/templates/catalogue/benchmark.html index 898b3ec2..4a13f240 100644 --- a/app/templates/catalogue/benchmark.html +++ b/app/templates/catalogue/benchmark.html @@ -36,14 +36,14 @@ {% if std %} {#- The standard's proofs of it, per dataset (standard/prove.py): its controls as chips, their source in the title - the builder's dataset step draws the same - then its units and leak checks; and Prove it, cli prove as a run. -#} - {% macro proof(rec, failed, expect) -%} - {%- if rec %}✓ {{ rec.passed }}{% if rec.limit %} · first {{ rec.limit }}{% endif %} + {% macro proof(rec, failed, expect, short='') -%} + {%- if rec %}✓ {{ rec.passed }}{% if rec.limit %} · first {{ rec.limit }}{% endif %} {%- elif failed and failed.value is defined %}{% set e = (failed.expect or expect or {'at_least': 1}) %}{% set k = e|list|first %}✕ {{ failed.value }} {{ '>' if k == 'at_most' else '<' }} {{ e[k] }} {%- elif failed %}✕ not proven {%- else %}not run{% endif -%} {%- endmacro %} {% macro chip(c) -%} - {{ c.key }}{{ proof(c.proven, c.failed, c.expect) }} + {{ c.key }}{{ proof(c.proven, c.failed, c.expect, c.short) }} {%- endmacro %} {% macro check(name, says, rec, failed) -%} {{ name }}{% if rec %}✓{% else %}{{ proof(rec, failed) }}{% endif %} diff --git a/app/tests/test_catalog.py b/app/tests/test_catalog.py index 6b670997..4a60efe2 100644 --- a/app/tests/test_catalog.py +++ b/app/tests/test_catalog.py @@ -44,7 +44,7 @@ def test_tabs_are_alphabetical_and_lanes_are_not_benchmarks(self): self.assertEqual(titles, sorted(titles, key=catalog.sort_key)) # what has a usable route is listed, whoever made it and whether or not it has run listed = {x.key for x in cat.listed_benchmarks} - self.assertEqual(sorted(listed), ["gaia", "gsm8k", "humaneval", "locomo", "longmemeval", "osworld1", "osworld2", "swe-bench-verified", "tau3-bench", "terminal-bench"]) + self.assertEqual(sorted(listed), ["gaia", "gsm8k", "humaneval", "locomo", "longmemeval", "open-telco", "osworld1", "osworld2", "swe-bench-verified", "tau3-bench", "terminal-bench"]) self.assertEqual((cat.benchmark("longmemeval").origin, cat.benchmark("longmemeval").label), ("official", "Official")) self.assertEqual(cat.benchmark("tau3-bench").label, "Official") # τ³ files as "t3…", before "te…": a reader looking under T finds it diff --git a/app/tests/test_catalogue_sources.py b/app/tests/test_catalogue_sources.py index 224e7fab..dd33de45 100644 --- a/app/tests/test_catalogue_sources.py +++ b/app/tests/test_catalogue_sources.py @@ -48,6 +48,21 @@ def test_a_dataset_the_driver_downloads_from_hugging_face_names_it_not_the_upstr self.assertEqual((row["from"], row["name"], row["pin"]), ("Hugging Face", "org/data", "0123456")) self.assertEqual(row["href"], f"https://huggingface.co/datasets/org/data/tree/{COMMIT}") + def test_a_driver_that_downloads_from_two_repositories_names_each_datasets_own(self): + lite, full = catalog.Dataset(key="qa-lite", title="Lite"), catalog.Dataset(key="qa-full", title="Full") + other = "f" * 40 + with tempfile.TemporaryDirectory() as here: + std = {"_here": here, "upstream": {"repository": "https://github.com/org/evals", "tested": COMMIT}, + "datasets": [{"id": "qa-lite", "units": 3}, {"id": "qa-full", "units": 9}]} + with open(os.path.join(here, "data-source.json"), "w") as f: + json.dump({"datasets": {"qa-lite": {"repository": "org/lite", "revision": COMMIT}, + "qa_full": {"repository": "org/full", "revision": other}}}, f) + (a,), (b,) = sources.of(lite, ["standard"], std, only=False), sources.of(full, ["standard"], std, only=False) + self.assertEqual((a["from"], a["name"], a["pin"]), ("Hugging Face", "org/lite", "0123456")) + self.assertEqual((b["name"], b["href"]), ("org/full", f"https://huggingface.co/datasets/org/full/tree/{other}")) + unnamed = catalog.Dataset(key="qa-other", title="Other") # no repository of its own, none for all + self.assertEqual(sources.of(unnamed, ["standard"], std, only=False)[0]["from"], "GitHub") + def test_no_harbor_row_without_a_harbor_name(self): d = catalog.Dataset(key="main", title="Main", source="GitHub") self.assertEqual(sources.of(d, ["harbor"]), []) diff --git a/app/tests/test_open_telco_driver.py b/app/tests/test_open_telco_driver.py new file mode 100644 index 00000000..c26ea881 --- /dev/null +++ b/app/tests/test_open_telco_driver.py @@ -0,0 +1,244 @@ +"""Open Telco's driver glue, with GSMA's tasks and Inspect AI's eval loop stubbed. + +No download, no Inspect install, no model call: the stubs stand for the +upstream's task functions and Inspect's eval, so these tests check what the +driver asks of them and what it makes of their reply. GSMA's prompts and +scorers are not reimplemented here; a real run calls them unchanged. +""" +import copy +import hashlib +import importlib.util +import inspect as pyinspect +import json +import os +import re +import shutil +import sys +import tempfile +import unittest +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import patch + +FOLDER = Path(__file__).resolve().parents[1] / "nullius" / "catalogue" / "benchmarks" / "open-telco" +REV = "a" * 40 + + +def sample(text, target, choices=None, id=None, **metadata): + return SimpleNamespace(id=id, input=text, target=target, choices=choices, metadata=metadata or None) + + +SUBJECTS = ["Lexicon", "Research overview"] * 6 +DATA = { + "teleqna": [sample(f"q{i}", "AB"[i % 2], ["x", "y"], subject=s) for i, s in enumerate(SUBJECTS, 1)], + "teletables": [sample("t1", "C", ["a", "b", "c"], id="t_7_abc123"), sample("t2", "A", ["a", "b", "c"], id="t_9_def456")], + "telemath": [sample("m1", "0.5"), sample("m2", "12.0")], + "telelogs": [sample("l1", "C3")], + "3gpp_tsg": [sample("d1", "SA5")], +} +FILES = {"teleqna": "test_teleqna.json", "teletables": "test_teletables.json", "telemath": "test_telemath.json", + "telelogs": "test_telelogs.json", "3gpp_tsg": "test_3gpp_tsg.json"} + + +class Upstream: + """GSMA's task functions and Inspect's eval loop, as far as the driver sees them.""" + + def __init__(self): + self.built, self.evals, self.models = [], [], [] + + def tasks(self): + def make(name, config, system=None, **params): + def fn(dataset_path="GSMA/ot-lite", split="test", full=False, **args): + self.built.append((name, dataset_path, args)) + rows = [copy.copy(s) for s in DATA[config]] # each load is new objects, as hf_dataset's + for k, v in args.items(): + if k in ("subject", "difficulty", "task_id") and v not in ("full", None): + rows = [s for s in rows if (s.metadata or {}).get(k) == v] + return SimpleNamespace(dataset=rows, system=system, args=args) + fn.__name__ = name + fn.__module__ = f"evals.{name}.{name}" + fn.__signature__ = pyinspect.Signature( + [pyinspect.Parameter(p, pyinspect.Parameter.KEYWORD_ONLY, default=d) + for p, d in {**params, "dataset_path": "GSMA/ot-lite", "split": "test", "full": False}.items()]) + module = ModuleType(fn.__module__) + module.DEFAULT_DATASET_NAME = config + return fn, module + + return {"teleqna": make("teleqna", "teleqna", subject="full"), "teletables": make("teletables", "teletables"), + "telemath": make("telemath", "telemath", system="SOLVE, THEN \\boxed{}"), + "telelogs": make("telelogs", "telelogs", eval_type="soft"), "three_gpp": make("three_gpp", "3gpp_tsg")} + + def eval(self, task, model, sample_id, log_dir, display, score_display, **config): + self.evals.append({"task": task, "model": model, "sample_id": sample_id, "config": config}) + s = next(x for x in task.dataset if x.id == sample_id) if isinstance(sample_id, str) else task.dataset[sample_id - 1] + asked = ([SimpleNamespace(text=task.system)] if task.system else []) + [SimpleNamespace(text="ASK " + s.input)] + reply = model.kw["custom_outputs"][0].content if model.name == "mockllm/model" else "ANSWER: B" + right = bool(reply) and str(s.target) in reply + verdict = SimpleNamespace(value="C" if right else "I", answer=reply[-3:]) + logged = SimpleNamespace(events=[SimpleNamespace(event="sample_init"), SimpleNamespace(event="model", input=asked)], + output=SimpleNamespace(completion=reply), scores={"stub_scorer": verdict}, + target=s.target, error=None) + return [SimpleNamespace(samples=[logged], status="success", error=None)] + + def get_model(self, name, **kw): + self.models.append((name, kw)) + return SimpleNamespace(name=name, kw=kw) + + def modules(self, fetch=None): + inspect_ai = ModuleType("inspect_ai") + inspect_ai.eval = self.eval + model = ModuleType("inspect_ai.model") + model.get_model = self.get_model + model.ModelOutput = SimpleNamespace(from_content=lambda model, content: SimpleNamespace(content=content)) + scorer = ModuleType("inspect_ai.scorer") + scorer.value_to_float = lambda: (lambda v: 1.0 if v == "C" else 0.0) + registry = ModuleType("evals._registry") + evals = ModuleType("evals") + evals._registry = registry + mods = {"inspect_ai": inspect_ai, "inspect_ai.model": model, "inspect_ai.scorer": scorer, + "evals": evals, "evals._registry": registry} + registry.__all__ = [] + for name, (fn, module) in self.tasks().items(): + registry.__all__.append(name) + setattr(registry, name, fn) + mods[fn.__module__] = module + hub = ModuleType("huggingface_hub") + hub.hf_hub_download = fetch or (lambda *a, **k: (_ for _ in ()).throw(AssertionError("no download expected"))) + mods["huggingface_hub"] = hub + return mods + + +class OpenTelcoDriver(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory(prefix="nullius-open-telco-") + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.here = self.root / "definition" + self.here.mkdir() + shutil.copy(FOLDER / "driver.py", self.here / "driver.py") + self.upstream = self.root / "upstream" + self.snap = self.upstream / ".open-telco-data" / f"org__lite@{REV}" + self.snap.mkdir(parents=True) + files = {"README.md": b"configs: every test"} | {f: f.encode() for f in FILES.values()} + for name, data in files.items(): + (self.snap / name).write_bytes(data) + digest = {name: hashlib.sha256(data).hexdigest() for name, data in files.items()} + source = {"cards": {"org/lite": digest["README.md"]}, + "datasets": {f"{c}-lite": {"repository": "org/lite", "revision": REV, "file": f, "sha256": digest[f]} + for c, f in FILES.items()}} + (self.here / "data-source.json").write_text(json.dumps(source)) + self.work = self.root / "work" + self.work.mkdir() + cwd = os.getcwd() + os.chdir(self.work) + self.addCleanup(os.chdir, cwd) + self.up = Upstream() + + def load(self, dataset, fetch=None, **options): + env = {"NULLIUS_UPSTREAM": str(self.upstream), "NULLIUS_DATASET_ID": dataset, + **{"NULLIUS_OPTION_" + k.upper(): v for k, v in options.items()}} + stubs = patch.dict(sys.modules, self.up.modules(fetch)) + environment = patch.dict(os.environ, env) + stubs.start() + environment.start() + self.addCleanup(stubs.stop) + self.addCleanup(environment.stop) + spec = importlib.util.spec_from_file_location(f"open_telco_driver_{dataset.replace('-', '_')}", self.here / "driver.py") + driver = importlib.util.module_from_spec(spec) + spec.loader.exec_module(driver) + return driver + + def test_units_are_the_datas_own_ids_else_the_place_in_the_file(self): + self.assertEqual(self.load("teletables-lite").units("teletables-lite")["units"], ["t_7_abc123", "t_9_def456"]) + units = self.load("teleqna-lite").units("teleqna-lite")["units"] + self.assertEqual(units, [f"{i:02d}" for i in range(1, 13)]) # padded to the file's width + self.assertEqual(self.up.built[-1][1], str(self.snap)) # the pinned copy, as dataset_path + + def test_a_filter_runs_the_tasks_own_argument_and_keeps_each_questions_number(self): + driver = self.load("teleqna-lite", subject="research-overview") + self.assertEqual(driver.units("teleqna-lite")["units"], ["02", "04", "06", "08", "10", "12"]) + self.assertIn(("teleqna", str(self.snap), {"subject": "Research overview"}), self.up.built) # as the data write it + everything = self.load("teleqna-lite", subject="all") + self.assertEqual(len(everything.units("teleqna-lite")["units"]), 12) + + def test_the_prompt_is_what_the_task_asks_its_model_and_nothing_is_written(self): + driver = self.load("telemath-lite") + prompt = driver.setup("2", str(self.work))["prompt"] + self.assertEqual(prompt, "SOLVE, THEN \\boxed{}\n\nASK m2") # the system message first, then the question + (call,) = self.up.evals + self.assertEqual((call["model"].name, call["model"].kw["custom_outputs"][0].content, call["sample_id"]), + ("mockllm/model", "", 2)) + self.assertEqual([p.name for p in self.work.iterdir() if p.name != "open-telco.lock"], []) + + def test_the_answer_is_scored_as_the_tasks_model_reply(self): + driver = self.load("teleqna-lite") + right = driver.score("01", str(self.work), [], "ANSWER: B") + self.assertEqual((right["reward"], right["passed"], right["detail"]["target"]), (1.0, True, "B")) + self.assertEqual(self.up.evals[-1]["model"].kw["custom_outputs"][0].content, "ANSWER: B") + nothing = driver.score("01", str(self.work), [], None) + self.assertEqual((nothing["reward"], nothing["passed"]), (0.0, False)) + self.assertEqual(self.up.evals[-1]["model"].kw["custom_outputs"][0].content, "") + + def test_the_scorers_own_argument_reaches_the_task(self): + driver = self.load("telelogs-lite", eval_type="hard") + driver.score("1", str(self.work), [], "\\boxed{C3}") + self.assertEqual(self.up.evals[-1]["task"].args, {"eval_type": "hard"}) + + def test_the_reference_is_written_as_each_prompt_asks(self): + self.assertEqual(self.load("teleqna-lite").oracle("02", "")["answer"], "ANSWER: A") + self.assertEqual(self.load("teletables-lite").oracle("t_7_abc123", "")["answer"], "ANSWER: C") + self.assertEqual(self.load("telemath-lite").oracle("1", "")["answer"], "\\boxed{0.5}") + self.assertEqual(self.load("telelogs-lite").oracle("1", "")["answer"], "\\boxed{C3}") + self.assertEqual(self.load("3gpp_tsg-lite").oracle("1", "")["answer"], '{"WORKING GROUP": "SA5"}') + + def test_the_original_reaches_the_model_through_the_units_relay_only(self): + driver = self.load("teleqna-lite") + with self.assertRaisesRegex(ValueError, "no model setup"): + driver.answer("01", "", "model", "m", "http://relay/u1/v1", "word") + out = driver.answer("01", "", "original", "org/m-1", "http://relay/u1/v1", "word") + self.assertEqual(out, {"answer": "ANSWER: B", "model": "org/m-1"}) + self.assertEqual(self.up.models[-1], ("openai-api/nullius/org/m-1", + {"base_url": "http://relay/u1/v1", "api_key": "word", "memoize": False})) + self.assertEqual(self.up.evals[-1]["config"], {}) # provider default: no temperature sent + warm = self.load("teleqna-lite", temperature="0.5") + warm.answer("01", "", "original", "m", "http://relay/u2/v1", "word") + self.assertEqual(self.up.evals[-1]["config"], {"temperature": 0.5}) + + def test_a_file_that_is_not_its_pin_is_fetched_again_and_refused_if_still_wrong(self): + (self.snap / "test_telemath.json").write_bytes(b"changed upstream") + fetched = [] + + def wrong(repo, name, repo_type, revision, local_dir): + fetched.append((repo, name, repo_type, revision)) + Path(local_dir, name).write_bytes(b"still not it") + + with self.assertRaisesRegex(ValueError, "test_telemath.json is not the file its sha256 names"): + self.load("telemath-lite", fetch=wrong) + self.assertEqual(fetched, [("org/lite", "test_telemath.json", "dataset", REV)]) + self.assertFalse((self.snap / "test_telemath.json").exists()) + + def right(repo, name, repo_type, revision, local_dir): + Path(local_dir, name).write_bytes(name.encode()) + + self.assertEqual(self.load("telemath-lite", fetch=right).ROOT, str(self.snap)) + + +class BundledDataSource(unittest.TestCase): + def test_every_dataset_has_a_pinned_file_and_its_card(self): + standard = json.loads((FOLDER / "standard.json").read_text()) + source = json.loads((FOLDER / "data-source.json").read_text()) + ids = [d["id"] for d in standard["datasets"]] + self.assertEqual(sorted(source["datasets"]), sorted(ids)) + for key, d in source["datasets"].items(): + with self.subTest(key): + self.assertEqual(d["repository"], "GSMA/ot-lite" if key.endswith("-lite") else "GSMA/ot-full") + self.assertRegex(d["revision"], r"^[0-9a-f]{40}$") + self.assertRegex(d["sha256"], r"^[0-9a-f]{64}$") + self.assertIn(key.rsplit("-", 1)[0], d["file"]) + self.assertRegex(source["cards"][d["repository"]], r"^[0-9a-f]{64}$") + self.assertEqual(len({(d["repository"], d["revision"]) for d in source["datasets"].values()}), 2) + self.assertTrue(all(re.fullmatch(r"[a-z0-9_]+-(lite|full)", i) for i in ids)) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_ui.py b/app/tests/test_standard_ui.py index 17bb32f7..a3de3af5 100644 --- a/app/tests/test_standard_ui.py +++ b/app/tests/test_standard_ui.py @@ -124,6 +124,28 @@ def test_a_failure_is_kept_with_the_value_it_reached_and_a_pass_clears_it(self): self.assertNotIn("failed", self.b["proven"]["all"]) self.assertEqual(prove.status(self.b)["all"]["oracle"]["proven"]["passed"], "4/4") + def test_a_proof_within_a_bound_short_of_the_full_mark_says_so_and_the_bound(self): + # an oracle lowered for answers its scorer cannot read, a nop a judge sometimes passes: proven, and short of the mark + self.b["controls"] = {"nop": dict(CONTROLS["nop"], expect={"at_most": 0.07}), + "oracle": dict(CONTROLS["oracle"], expect={"at_least": 0.93})} + prove.record(self.b, {"all": {"oracle": {"ok": True, "passed": "1871/2000", "value": 0.9355, "expect": {"at_least": 0.93}, "units": 2000}, + "nop": {"ok": True, "passed": "20/500", "value": 0.04, "expect": {"at_most": 0.07}, "units": 500}}}) + row = prove.status(self.b)["all"] + self.assertEqual((row["oracle"]["must"], row["oracle"]["short"]), ("at least 93%", "93.55%, at least 93% needed")) + self.assertEqual((row["nop"]["must"], row["nop"]["short"]), ("at most 7%", "4%, at most 7% allowed")) + with patch.object(recipe, "_all", lambda: {"benchmark": [self.b], "agent": [], "capability": []}): + b = recipe.choices()["benchmarks"][0] + self.assertEqual([(c["key"], c["must"]) for c in b["controls"]], [("nop", "at most 7%"), ("oracle", "at least 93%")]) + self.assertEqual([c["short"] for c in b["datasets"][0]["controls"]], ["4%, at most 7% allowed", "93.55%, at least 93% needed"]) + # at the full mark, under the same lowered bound, it is a plain pass; at the default bound, it must score everything + prove.record(self.b, {"all": {"oracle": {"ok": True, "passed": "2000/2000", "value": 1.0, "expect": {"at_least": 0.93}, "units": 2000}, + "nop": {"ok": True, "passed": "0/500", "value": 0.0, "expect": {"at_most": 0.07}, "units": 500}}}) + self.assertEqual([prove.status(self.b)["all"][c]["short"] for c in ("nop", "oracle")], ["", ""]) + self.b["controls"] = CONTROLS + self.assertEqual([prove.status(self.b)["all"][c]["must"] for c in ("nop", "oracle")], ["nothing", "everything"]) + self.assertEqual(prove.short_of(None, {"at_least": 1.0}), "") + self.assertEqual(prove.short_of({"passed": "3/3"}, {"at_least": 1.0}), "") # an older record, with no value + class AgentPage(unittest.TestCase): def test_each_proof_carries_its_kit_and_an_older_kit_says_so(self): diff --git a/app/ui_tests/test_catalogue_routes.py b/app/ui_tests/test_catalogue_routes.py index 874cebe7..9b3de520 100644 --- a/app/ui_tests/test_catalogue_routes.py +++ b/app/ui_tests/test_catalogue_routes.py @@ -147,7 +147,7 @@ def test_every_listed_benchmark_is_a_card_opening_its_page(self): cat = catalog.load() keys = re.findall(r'', 1)[0] for b in cat.listed_benchmarks: @@ -186,6 +186,18 @@ def test_a_page_exists_only_for_a_listed_benchmark(self): class BenchmarkPage(PageCase): + def test_a_proof_short_of_the_full_mark_has_no_green_tick_and_says_its_bound(self): + # Open Telco's oracle must score at least 93%: 3GPP TSG's scorer cannot read RAN_AH1, so its rows stop short + html = self.page_html("open-telco") + row = lambda label: re.search(r'%s<.*?' % re.escape(label), html, re.S).group(0) + self.assertIn('", row("3GPP TSG · Full")) + self.assertIn(' -GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files +GAIAGSM8KHumanEvalOpen TelcoOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0One task, Workspace files, SearchOne taskWorkspace filesOne taskComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files -codex@2.2.2✓✓✓✓✓✓✓✓ -codex@0.16.0✓✓✓✗✗✓✓✓ -goose✓✓✓✗✗✓✓✓ -jiuwenswarm@0.2.7✗✓✓✗✗✓✓✓ -jiuwenswarm@0.2.8✓✓✓✗✗✓✓✓ -mini-swe-agent✗✗✗✗✗✓✗✗ -opencode✓✓✓✗✗✓✓✓ -reference✗✓✓✗✗✓✓✓ -terminus-2✗✗✗✗✗✗✗✓ +codex@2.2.2✓✓✓✓✓✓✓✓✓ +codex@0.16.0✓✓✓✓✗✗✓✓✓ +goose✓✓✓✓✗✗✓✓✓ +jiuwenswarm@0.2.7✗✓✓✓✗✗✓✓✓ +jiuwenswarm@0.2.8✓✓✓✓✗✗✓✓✓ +mini-swe-agent✗✗✗✗✗✗✓✗✗ +opencode✓✓✓✓✗✗✓✓✓ +reference✗✓✓✓✗✗✓✓✓ +terminus-2✗✗✗✗✗✗✗✗✓ - ✗: the missing port, on hover. diff --git a/docs/agents/supported/codex.md b/docs/agents/supported/codex.md index e14e23dd..fa271132 100644 --- a/docs/agents/supported/codex.md +++ b/docs/agents/supported/codex.md @@ -34,10 +34,10 @@ OpenAI's coding agent. ## Benchmarks it fits - + - - + +
GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
GAIAGSM8KHumanEvalOpen TelcoOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesOne taskComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
codex@2.2.2✓✓✓✓✓✓✓✓
codex@0.16.0✓✓✓✗✗✓✓✓
codex@2.2.2✓✓✓✓✓✓✓✓✓
codex@0.16.0✓✓✓✓✗✗✓✓✓
- ✗: the missing port, on hover. diff --git a/docs/agents/supported/goose.md b/docs/agents/supported/goose.md index 6f0975e1..907d3452 100644 --- a/docs/agents/supported/goose.md +++ b/docs/agents/supported/goose.md @@ -30,9 +30,9 @@ The AAIF's open-source agent, formerly Block's. ## Benchmarks it fits - + - +
GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
GAIAGSM8KHumanEvalOpen TelcoOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesOne taskComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
goose✓✓✓✗✗✓✓✓
goose✓✓✓✓✗✗✓✓✓
- ✗: the missing port, on hover. diff --git a/docs/agents/supported/jiuwenswarm.md b/docs/agents/supported/jiuwenswarm.md index 578f7b6a..4de45f23 100644 --- a/docs/agents/supported/jiuwenswarm.md +++ b/docs/agents/supported/jiuwenswarm.md @@ -34,10 +34,10 @@ openJiuwen's agent. ## Benchmarks it fits - + - - + +
GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
GAIAGSM8KHumanEvalOpen TelcoOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesOne taskComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
jiuwenswarm@0.2.7✗✓✓✗✗✓✓✓
jiuwenswarm@0.2.8✓✓✓✗✗✓✓✓
jiuwenswarm@0.2.7✗✓✓✓✗✗✓✓✓
jiuwenswarm@0.2.8✓✓✓✓✗✗✓✓✓
- ✗: the missing port, on hover. diff --git a/docs/agents/supported/mini-swe-agent.md b/docs/agents/supported/mini-swe-agent.md index 52aabde2..f43378ed 100644 --- a/docs/agents/supported/mini-swe-agent.md +++ b/docs/agents/supported/mini-swe-agent.md @@ -30,9 +30,9 @@ Princeton's 100-line agent, bash only. ## Benchmarks it fits - + - +
GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
GAIAGSM8KHumanEvalOpen TelcoOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesOne taskComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
mini-swe-agent✗✗✗✗✗✓✗✗
mini-swe-agent✗✗✗✗✗✗✓✗✗
- ✗: the missing port, on hover. diff --git a/docs/agents/supported/opencode.md b/docs/agents/supported/opencode.md index c9863857..c288131d 100644 --- a/docs/agents/supported/opencode.md +++ b/docs/agents/supported/opencode.md @@ -30,9 +30,9 @@ The open-source terminal coding agent. ## Benchmarks it fits - + - +
GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
GAIAGSM8KHumanEvalOpen TelcoOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesOne taskComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
opencode✓✓✓✗✗✓✓✓
opencode✓✓✓✓✗✗✓✓✓
- ✗: the missing port, on hover. diff --git a/docs/agents/supported/terminus-2.md b/docs/agents/supported/terminus-2.md index f51a7620..fcc73382 100644 --- a/docs/agents/supported/terminus-2.md +++ b/docs/agents/supported/terminus-2.md @@ -30,9 +30,9 @@ Harbor's terminal agent, used by Terminal-Bench. ## Benchmarks it fits - + - +
GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
GAIAGSM8KHumanEvalOpen TelcoOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesOne taskComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
terminus-2✗✗✗✗✗✗✗✓
terminus-2✗✗✗✗✗✗✗✗✓
- ✗: the missing port, on hover. diff --git a/docs/benchmarks/supported/README.md b/docs/benchmarks/supported/README.md index 68b87061..679245af 100644 --- a/docs/benchmarks/supported/README.md +++ b/docs/benchmarks/supported/README.md @@ -17,6 +17,7 @@ Every benchmark in the catalogue: where it runs, what it needs, the ways it runs HumanEval
humanevalPython functions written from their signature and docstring, each checked by its unit tests.Coding✓–164Official LoCoMo
locomoVery long-term conversational memory over conversations of 19 to 32 sessions.Memory✓–10Official LongMemEval
longmemevalLong-term chat memory: questions over timestamped chat histories.Memory✓–1500Official +Open Telco
open-telcoGSMA's eight telecom tests, from 3GPP standards and maths to 5G root-cause analysis and 6G reasoning.Telecom✓–20588Official OSWorld1
osworld1Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of its own.Computer use✓–369Official OSWorld2
osworld2Long-horizon, real-world tasks on an Ubuntu desktop, across its apps, each in a VM of its own.Computer use✓–108Official SWE-bench Verified
swe-bench-verifiedSoftware engineering: fix a real GitHub issue so the failing tests pass.Coding✓✓500Official @@ -34,6 +35,7 @@ Every benchmark in the catalogue: where it runs, what it needs, the ways it runs HumanEvalagent.workspace/v1Nulliusthe authors✓✓✓ LoCoMocap.memory/v1NulliusNullius✓✓✓ LongMemEvalcap.memory/v1NulliusNullius✓✓✓ +Open Telcoagent.task/v1NulliusNullius✓✓✓ OSWorld1agent.computer/v1Nullius–✓✓✓ OSWorld2agent.computer/v1Nullius–✓✓– SWE-bench Verifiedagent.workspace/v1Nulliusthe authors✓✓✓ diff --git a/docs/benchmarks/supported/open-telco.md b/docs/benchmarks/supported/open-telco.md new file mode 100644 index 00000000..f33bdede --- /dev/null +++ b/docs/benchmarks/supported/open-telco.md @@ -0,0 +1,131 @@ + + +# Open Telco + +GSMA's eight telecom tests, from 3GPP standards and maths to 5G root-cause analysis and 6G reasoning. + + + + + + + + + + +
Keyopen-telco
CategoryTelecom
OriginOfficial
Runs onNullius
Tasks20588
PaperTeleQnA: A Benchmark Dataset to Assess Large Language Models Telecommunications Knowledge, arXiv · 2023
Needs, ports provenagent.task/v1
+ +## Datasets + +| Dataset | Key | Tasks | From | Verifier | +|---|---|---|---|---| +| TeleQnA · Lite | `teleqna-lite` | 1000 | Hugging Face [GSMA/ot-lite](https://huggingface.co/datasets/GSMA/ot-lite/tree/1c0f2eac3ad0baa29704b147a95fea283b2906c7) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | +| TeleQnA · Full | `teleqna-full` | 10000 | Hugging Face [GSMA/ot-full](https://huggingface.co/datasets/GSMA/ot-full/tree/6319806f04783eafe04d9facf755d379c66b7664) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | +| TeleTables · Lite | `teletables-lite` | 100 | Hugging Face [GSMA/ot-lite](https://huggingface.co/datasets/GSMA/ot-lite/tree/1c0f2eac3ad0baa29704b147a95fea283b2906c7) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | +| TeleTables · Full | `teletables-full` | 500 | Hugging Face [GSMA/ot-full](https://huggingface.co/datasets/GSMA/ot-full/tree/6319806f04783eafe04d9facf755d379c66b7664) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | +| TeleMath · Lite | `telemath-lite` | 100 | Hugging Face [GSMA/ot-lite](https://huggingface.co/datasets/GSMA/ot-lite/tree/1c0f2eac3ad0baa29704b147a95fea283b2906c7) | GSMA's telemath_scorer: the last \boxed{} number, right within 1% or 0.01 of the answer. | +| TeleMath · Full | `telemath-full` | 500 | Hugging Face [GSMA/ot-full](https://huggingface.co/datasets/GSMA/ot-full/tree/6319806f04783eafe04d9facf755d379c66b7664) | GSMA's telemath_scorer: the last \boxed{} number, right within 1% or 0.01 of the answer. | +| TeleLogs · Lite | `telelogs-lite` | 100 | Hugging Face [GSMA/ot-lite](https://huggingface.co/datasets/GSMA/ot-lite/tree/1c0f2eac3ad0baa29704b147a95fea283b2906c7) | GSMA's telelogs_scorer: the last \boxed{} answer, its first number against the root cause's (soft, the default). | +| TeleLogs · Full | `telelogs-full` | 864 | Hugging Face [GSMA/ot-full](https://huggingface.co/datasets/GSMA/ot-full/tree/6319806f04783eafe04d9facf755d379c66b7664) | GSMA's telelogs_scorer: the last \boxed{} answer, its first number against the root cause's (soft, the default). | +| 3GPP TSG · Lite | `3gpp_tsg-lite` | 100 | Hugging Face [GSMA/ot-lite](https://huggingface.co/datasets/GSMA/ot-lite/tree/1c0f2eac3ad0baa29704b147a95fea283b2906c7) | GSMA's pattern scorer: the first working-group code in the reply, case aside. | +| 3GPP TSG · Full | `3gpp_tsg-full` | 2000 | Hugging Face [GSMA/ot-full](https://huggingface.co/datasets/GSMA/ot-full/tree/6319806f04783eafe04d9facf755d379c66b7664) | GSMA's pattern scorer: the first working-group code in the reply, case aside. | +| ORANBench · Lite | `oranbench-lite` | 150 | Hugging Face [GSMA/ot-lite](https://huggingface.co/datasets/GSMA/ot-lite/tree/1c0f2eac3ad0baa29704b147a95fea283b2906c7) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | +| ORANBench · Full | `oranbench-full` | 1500 | Hugging Face [GSMA/ot-full](https://huggingface.co/datasets/GSMA/ot-full/tree/6319806f04783eafe04d9facf755d379c66b7664) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | +| srsRANBench · Lite | `srsranbench-lite` | 150 | Hugging Face [GSMA/ot-lite](https://huggingface.co/datasets/GSMA/ot-lite/tree/1c0f2eac3ad0baa29704b147a95fea283b2906c7) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | +| srsRANBench · Full | `srsranbench-full` | 1502 | Hugging Face [GSMA/ot-full](https://huggingface.co/datasets/GSMA/ot-full/tree/6319806f04783eafe04d9facf755d379c66b7664) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | +| 6G-Bench · Lite | `sixg_bench-lite` | 150 | Hugging Face [GSMA/ot-lite](https://huggingface.co/datasets/GSMA/ot-lite/tree/1c0f2eac3ad0baa29704b147a95fea283b2906c7) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | +| 6G-Bench · Full | `sixg_bench-full` | 3722 | Hugging Face [GSMA/ot-full](https://huggingface.co/datasets/GSMA/ot-full/tree/6319806f04783eafe04d9facf755d379c66b7664) | Inspect's multiple_choice solver reads the 'ANSWER: ' line and choice() checks the letter, as GSMA's task runs them. | + +## On the standard + +| Control | Written by | +|---|---| +| `nop` | Nullius | +| `oracle` | Nullius | + +### Versions + +| Version | Setups as published, whose model | Options | +|---|---|---| +| `open-telco` | Original · No agent · Any of your choice | `difficulty`, `eval_type`, `subject`, `task_id`, `temperature` | + +## Options + +| Option | Values, default in bold | What it sets | Setups | +|---|---|---|---| +| `difficulty` | **`all`**, `easy`, `medium`, `hard` | ORANBench's difficulty level: every one, or one alone (the task's difficulty argument). | `original`, `yours` | +| `eval_type` | **`soft`**, `hard` | How TeleLogs' scorer compares the boxed root cause with the answer (its eval_type). | `original`, `yours` | +| `subject` | **`all`**, `lexicon`, `research-overview`, `research-publications`, `standards-overview`, `standards-specifications` | TeleQnA's question category: every one, or one alone (the task's subject argument). | `original`, `yours` | +| `task_id` | **`all`**, `T1`, `T2`, `T3`, `T4`, `T5`, `T6`, `T7`, `T8`, `T9`, `T10`, `T11`, `T12`, `T13`, `T14`, `T15`, `T16`, `T17`, `T18`, `T19`, `T20`, `T21`, `T22`, `T23`, `T24`, `T25`, `T26`, `T27`, `T28`, `T29`, `T30` | 6G-Bench's task family: all 30, or one alone (the task's task_id argument). | `original`, `yours` | +| `temperature` | e.g. `0.5`, default `default` | Sampling randomness: Inspect's --temperature. Provider default sends none, as GSMA's leaderboard runs do; enter a non-negative decimal to set one. Accepted values depend on the model. | `original` | + +## Good to know + +- Lite is GSMA's sample of each test, the one its leaderboard ranks; Full is the whole test. +- Each test is a dataset scored on its own. The leaderboard's average is the mean of the tests' accuracies. +- An answer counts only in the form its prompt asks for: 'ANSWER: ', a \boxed{} value, or a working group's code. + +## Upstream limitations + +- 3GPP TSG's scorer cannot read RAN_AH1: its 6 Lite and 129 Full questions always fail, so 94% and 93.6% are the ceilings (gsma-labs/evals#81). +- TeleMath's scorer accepts any number within 0.01 of the answer, looser than its 1% for answers between -1 and 1 (44 Lite, 245 Full): 0 passes on 10 Lite and 47 Full questions whose answer is not 0. +- The multiple-choice prompt asks for 'ANSWER: $LETTER': a reply that keeps the dollar sign is scored wrong, on all five multiple-choice tests (gsma-labs/evals#73). + +## Nullius limitations + +- An agent gets TeleMath's system prompt and the question as one prompt: ACP has no system message. +- TeleLogs' majority vote across epochs (maj@k) is not computed; several attempts give pass@k instead. + +## Cite + +??? citation "BibTeX" + + ```bibtex + @misc{maatouk2023teleqna, + title={TeleQnA: A Benchmark Dataset to Assess Large Language Models Telecommunications Knowledge}, + author={Maatouk, Ali and Ayed, Fadhel and Piovesan, Nicola and De Domenico, Antonio and Debbah, Merouane and Luo, Zhi-Quan}, + year={2023}, eprint={2310.15051}, archivePrefix={arXiv} + } + + @misc{ezzakri2025teletables, + title={TeleTables: A Benchmark for Large Language Models in Telecom Table Interpretation}, + author={Ezzakri, Anas and Piovesan, Nicola and Sana, Mohamed and De Domenico, Antonio and Ayed, Fadhel and Zhang, Haozhe}, + year={2025}, eprint={2601.04202}, archivePrefix={arXiv} + } + + @misc{colle2025telemath, + title={TeleMath: A Benchmark for Large Language Models in Telecom Mathematical Problem Solving}, + author={Colle, Vincenzo and Sana, Mohamed and Piovesan, Nicola and De Domenico, Antonio and Ayed, Fadhel and Debbah, Merouane}, + year={2025}, eprint={2506.10674}, archivePrefix={arXiv} + } + + @misc{sana2025telelogs, + title={Reasoning Language Models for Root Cause Analysis in 5G Wireless Networks}, + author={Sana, Mohamed and Piovesan, Nicola and De Domenico, Antonio and Kang, Yibin and Zhang, Haozhe and Debbah, Merouane and Ayed, Fadhel}, + year={2025}, eprint={2507.21974}, archivePrefix={arXiv} + } + + @misc{zou2024telecomgpt, + title={TelecomGPT: A Framework to Build Telecom-Specific Large Language Models}, + author={Zou, Hang and Zhao, Qiyang and Tian, Yu and Bariah, Lina and Bader, Faouzi and Lestable, Thierry and Debbah, Merouane}, + year={2024}, eprint={2407.09424}, archivePrefix={arXiv} + } + + @misc{gajjar2024oranbench, + title={ORAN-Bench-13K: An Open Source Benchmark for Assessing LLMs in Open Radio Access Networks}, + author={Gajjar, Pranshav and Shah, Vijay K.}, + year={2024}, eprint={2407.06245}, archivePrefix={arXiv} + } + + @misc{gajjar2025oransight2, + title={ORANSight-2.0: Foundational LLMs for O-RAN}, + author={Gajjar, Pranshav and Shah, Vijay K.}, + year={2025}, eprint={2503.05200}, archivePrefix={arXiv} + } + + @misc{ferrag2026sixgbench, + title={6G-Bench: An Open Benchmark for Semantic Communication and Network-Level Reasoning with Foundation Models in AI-Native 6G Networks}, + author={Ferrag, Mohamed Amine and Lakas, Abderrahmane and Debbah, Merouane}, + year={2026}, eprint={2602.08675}, archivePrefix={arXiv} + } + ```