Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 206 additions & 0 deletions app/nullius/catalogue/benchmarks/open-telco/benchmark.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
{
"key": "open-telco",
"title": "Open Telco",
"category": "Telecom",
"domain": "GSMA's eight telecom tests, from 3GPP standards and maths to 5G root-cause analysis and 6G reasoning.",
"scoring": "Answer accuracy",
"harness": [
"standard"
],
"origin": "official",
"good_to_know": [
"Lite is GSMA's sample of each test, the one its leaderboard ranks; Full is the whole test.",
"Each test is a dataset scored on its own. The leaderboard's average is the mean of the tests' accuracies.",
"An answer counts only in the form its prompt asks for: 'ANSWER: <letter>', a \\boxed{} value, or a working group's code."
],
"upstream_limitations": [
"3GPP TSG's scorer cannot read RAN_AH1: its 6 Lite and 129 Full questions always fail, so 94% and 93.6% are the ceilings (gsma-labs/evals#81).",
"TeleMath's scorer accepts any number within 0.01 of the answer, looser than its 1% for answers between -1 and 1 (44 Lite, 245 Full): 0 passes on 10 Lite and 47 Full questions whose answer is not 0.",
"The multiple-choice prompt asks for 'ANSWER: $LETTER': a reply that keeps the dollar sign is scored wrong, on all five multiple-choice tests (gsma-labs/evals#73)."
],
"limitations": [
"An agent gets TeleMath's system prompt and the question as one prompt: ACP has no system message.",
"TeleLogs' majority vote across epochs (maj@k) is not computed; several attempts give pass@k instead."
],
"datasets": [
{
"key": "teleqna-lite",
"title": "TeleQnA · Lite",
"tasks": 1000,
"official": "https://huggingface.co/datasets/GSMA/ot-lite",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "Telecom standards and research questions: GSMA's sample, the leaderboard's, 1,000 questions. MIT License."
},
{
"key": "teleqna-full",
"title": "TeleQnA · Full",
"tasks": 10000,
"official": "https://huggingface.co/datasets/GSMA/ot-full",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "Telecom standards and research questions: the whole test, 10,000 questions. MIT License."
},
{
"key": "teletables-lite",
"title": "TeleTables · Lite",
"tasks": 100,
"official": "https://huggingface.co/datasets/GSMA/ot-lite",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "Questions on tables of 3GPP specifications: GSMA's sample, the leaderboard's, 100 questions. MIT License."
},
{
"key": "teletables-full",
"title": "TeleTables · Full",
"tasks": 500,
"official": "https://huggingface.co/datasets/GSMA/ot-full",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "Questions on tables of 3GPP specifications: the whole test, 500 questions. MIT License."
},
{
"key": "telemath-lite",
"title": "TeleMath · Lite",
"tasks": 100,
"official": "https://huggingface.co/datasets/GSMA/ot-lite",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "GSMA's telemath_scorer: the last \\boxed{} number, right within 1% or 0.01 of the answer.",
"notes": "Telecom maths problems: GSMA's sample, the leaderboard's, 100 questions. MIT License."
},
{
"key": "telemath-full",
"title": "TeleMath · Full",
"tasks": 500,
"official": "https://huggingface.co/datasets/GSMA/ot-full",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "GSMA's telemath_scorer: the last \\boxed{} number, right within 1% or 0.01 of the answer.",
"notes": "Telecom maths problems: the whole test, 500 questions. MIT License."
},
{
"key": "telelogs-lite",
"title": "TeleLogs · Lite",
"tasks": 100,
"official": "https://huggingface.co/datasets/GSMA/ot-lite",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "GSMA's telelogs_scorer: the last \\boxed{} answer, its first number against the root cause's (soft, the default).",
"notes": "5G root-cause analysis from drive-test data: GSMA's sample, the leaderboard's, 100 questions. MIT License."
},
{
"key": "telelogs-full",
"title": "TeleLogs · Full",
"tasks": 864,
"official": "https://huggingface.co/datasets/GSMA/ot-full",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "GSMA's telelogs_scorer: the last \\boxed{} answer, its first number against the root cause's (soft, the default).",
"notes": "5G root-cause analysis from drive-test data: the whole test, 864 questions. MIT License."
},
{
"key": "3gpp_tsg-lite",
"title": "3GPP TSG · Lite",
"tasks": 100,
"official": "https://huggingface.co/datasets/GSMA/ot-lite",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "GSMA's pattern scorer: the first working-group code in the reply, case aside.",
"notes": "3GPP documents to classify by working group: GSMA's sample, the leaderboard's, 100 questions. MIT License."
},
{
"key": "3gpp_tsg-full",
"title": "3GPP TSG · Full",
"tasks": 2000,
"official": "https://huggingface.co/datasets/GSMA/ot-full",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "GSMA's pattern scorer: the first working-group code in the reply, case aside.",
"notes": "3GPP documents to classify by working group: the whole test, 2,000 questions. MIT License."
},
{
"key": "oranbench-lite",
"title": "ORANBench · Lite",
"tasks": 150,
"official": "https://huggingface.co/datasets/GSMA/ot-lite",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "Questions on O-RAN specifications: GSMA's sample, the leaderboard's, 150 questions. MIT License."
},
{
"key": "oranbench-full",
"title": "ORANBench · Full",
"tasks": 1500,
"official": "https://huggingface.co/datasets/GSMA/ot-full",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "Questions on O-RAN specifications: the whole test, 1,500 questions. MIT License."
},
{
"key": "srsranbench-lite",
"title": "srsRANBench · Lite",
"tasks": 150,
"official": "https://huggingface.co/datasets/GSMA/ot-lite",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "Questions on the srsRAN 5G codebase: GSMA's sample, the leaderboard's, 150 questions. MIT License."
},
{
"key": "srsranbench-full",
"title": "srsRANBench · Full",
"tasks": 1502,
"official": "https://huggingface.co/datasets/GSMA/ot-full",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "Questions on the srsRAN 5G codebase: the whole test, 1,502 questions. MIT License."
},
{
"key": "sixg_bench-lite",
"title": "6G-Bench · Lite",
"tasks": 150,
"official": "https://huggingface.co/datasets/GSMA/ot-lite",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "6G network reasoning across 30 task families: GSMA's sample, the leaderboard's, 150 questions. MIT License."
},
{
"key": "sixg_bench-full",
"title": "6G-Bench · Full",
"tasks": 3722,
"official": "https://huggingface.co/datasets/GSMA/ot-full",
"source": "Hugging Face",
"registry": "",
"hub": "",
"verifier": "Inspect's multiple_choice solver reads the 'ANSWER: <letter>' line and choice() checks the letter, as GSMA's task runs them.",
"notes": "6G network reasoning across 30 task families: the whole test, 3,722 questions. MIT License."
}
],
"task_count": 20588,
"citation": "@misc{maatouk2023teleqna,\n title={TeleQnA: A Benchmark Dataset to Assess Large Language Models Telecommunications Knowledge},\n author={Maatouk, Ali and Ayed, Fadhel and Piovesan, Nicola and De Domenico, Antonio and Debbah, Merouane and Luo, Zhi-Quan},\n year={2023}, eprint={2310.15051}, archivePrefix={arXiv}\n}\n\n@misc{ezzakri2025teletables,\n title={TeleTables: A Benchmark for Large Language Models in Telecom Table Interpretation},\n author={Ezzakri, Anas and Piovesan, Nicola and Sana, Mohamed and De Domenico, Antonio and Ayed, Fadhel and Zhang, Haozhe},\n year={2025}, eprint={2601.04202}, archivePrefix={arXiv}\n}\n\n@misc{colle2025telemath,\n title={TeleMath: A Benchmark for Large Language Models in Telecom Mathematical Problem Solving},\n author={Colle, Vincenzo and Sana, Mohamed and Piovesan, Nicola and De Domenico, Antonio and Ayed, Fadhel and Debbah, Merouane},\n year={2025}, eprint={2506.10674}, archivePrefix={arXiv}\n}\n\n@misc{sana2025telelogs,\n title={Reasoning Language Models for Root Cause Analysis in 5G Wireless Networks},\n author={Sana, Mohamed and Piovesan, Nicola and De Domenico, Antonio and Kang, Yibin and Zhang, Haozhe and Debbah, Merouane and Ayed, Fadhel},\n year={2025}, eprint={2507.21974}, archivePrefix={arXiv}\n}\n\n@misc{zou2024telecomgpt,\n title={TelecomGPT: A Framework to Build Telecom-Specific Large Language Models},\n author={Zou, Hang and Zhao, Qiyang and Tian, Yu and Bariah, Lina and Bader, Faouzi and Lestable, Thierry and Debbah, Merouane},\n year={2024}, eprint={2407.09424}, archivePrefix={arXiv}\n}\n\n@misc{gajjar2024oranbench,\n title={ORAN-Bench-13K: An Open Source Benchmark for Assessing LLMs in Open Radio Access Networks},\n author={Gajjar, Pranshav and Shah, Vijay K.},\n year={2024}, eprint={2407.06245}, archivePrefix={arXiv}\n}\n\n@misc{gajjar2025oransight2,\n title={ORANSight-2.0: Foundational LLMs for O-RAN},\n author={Gajjar, Pranshav and Shah, Vijay K.},\n year={2025}, eprint={2503.05200}, archivePrefix={arXiv}\n}\n\n@misc{ferrag2026sixgbench,\n title={6G-Bench: An Open Benchmark for Semantic Communication and Network-Level Reasoning with Foundation Models in AI-Native 6G Networks},\n author={Ferrag, Mohamed Amine and Lakas, Abderrahmane and Debbah, Merouane},\n year={2026}, eprint={2602.08675}, archivePrefix={arXiv}\n}",
"citation_source": "https://huggingface.co/datasets/GSMA/ot-full"
}
104 changes: 104 additions & 0 deletions app/nullius/catalogue/benchmarks/open-telco/data-source.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
{
"cards": {
"GSMA/ot-lite": "91c38cb7fc86f7f93a06705fec69bf454542540969a0e5cc0635dbcd1e5e70de",
"GSMA/ot-full": "fe85b5a82350552bd4cc87d48c7a667d90ef6d563f00c786dfa9319fcf8fe46d"
},
"datasets": {
"teleqna-lite": {
"repository": "GSMA/ot-lite",
"revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7",
"file": "test_teleqna.json",
"sha256": "3bbb71301dd7a2228a8f42d4ca48c100fd57cbbb80b2e22749d5efe86ae31b5a"
},
"teleqna-full": {
"repository": "GSMA/ot-full",
"revision": "6319806f04783eafe04d9facf755d379c66b7664",
"file": "teleqna/test-00000-of-00001.parquet",
"sha256": "73f3cd724d8ff3c1071fbb135e80d69a54b0773acde590006949a67956338e57"
},
"teletables-lite": {
"repository": "GSMA/ot-lite",
"revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7",
"file": "test_teletables.json",
"sha256": "d08065834a41cf295242816594963a9ec3f59699c0d26e8bfaabbe6445647b73"
},
"teletables-full": {
"repository": "GSMA/ot-full",
"revision": "6319806f04783eafe04d9facf755d379c66b7664",
"file": "teletables/test-00000-of-00001.parquet",
"sha256": "6b4d56dc644e9999519349202a80d2f9b525322ce228b5ed35387757c31b72df"
},
"telemath-lite": {
"repository": "GSMA/ot-lite",
"revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7",
"file": "test_telemath.json",
"sha256": "5d4c64ae79d3cdec50c28a4d5cb36b098ae982bd5301aec0337358ec9db4453a"
},
"telemath-full": {
"repository": "GSMA/ot-full",
"revision": "6319806f04783eafe04d9facf755d379c66b7664",
"file": "telemath/test-00000-of-00001.parquet",
"sha256": "c9b0523f7c6ae2ee83478f810a30fd8fbb3e320c615373a3f7e71c1e0f4cd9be"
},
"telelogs-lite": {
"repository": "GSMA/ot-lite",
"revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7",
"file": "test_telelogs.json",
"sha256": "9d1eea3a86d39cc9ab792bca0f69edba8703e5681583e70ae82f6ef49b531bf9"
},
"telelogs-full": {
"repository": "GSMA/ot-full",
"revision": "6319806f04783eafe04d9facf755d379c66b7664",
"file": "telelogs/test-00000-of-00001.parquet",
"sha256": "6ee5861ad6ec21f14d3debecff7c37575b0eed35efd55a2df808edf9b2b553b1"
},
"3gpp_tsg-lite": {
"repository": "GSMA/ot-lite",
"revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7",
"file": "test_3gpp_tsg.json",
"sha256": "a7ae12665448bc35dd656629daea640a02fb96a7fe32e400f6a95055c65be20b"
},
"3gpp_tsg-full": {
"repository": "GSMA/ot-full",
"revision": "6319806f04783eafe04d9facf755d379c66b7664",
"file": "3gpp_tsg/test-00000-of-00001.parquet",
"sha256": "1a8607099873455d9079bfbb19dfcfc6fd796973f2242eb44e1a3232a1943104"
},
"oranbench-lite": {
"repository": "GSMA/ot-lite",
"revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7",
"file": "test_oranbench.json",
"sha256": "458be8c1962041196908aabb94838eae82633efac56c56d114c6982883e81fe8"
},
"oranbench-full": {
"repository": "GSMA/ot-full",
"revision": "6319806f04783eafe04d9facf755d379c66b7664",
"file": "oranbench/test-00000-of-00001.parquet",
"sha256": "1dafd3e4606c9e08add68797cdb0f202679dca4e4f6136c99a30bd1939f62778"
},
"srsranbench-lite": {
"repository": "GSMA/ot-lite",
"revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7",
"file": "test_srsranbench.json",
"sha256": "e7e43cb23c76970a8881c1f5e83319df4dcdc817cedd3841f4dd6144dd57eb88"
},
"srsranbench-full": {
"repository": "GSMA/ot-full",
"revision": "6319806f04783eafe04d9facf755d379c66b7664",
"file": "srsranbench/test-00000-of-00001.parquet",
"sha256": "f59c40de70add5a4cce8513f7652ab4f95cdd15864b148e173f326403be48198"
},
"sixg_bench-lite": {
"repository": "GSMA/ot-lite",
"revision": "1c0f2eac3ad0baa29704b147a95fea283b2906c7",
"file": "test_sixg_bench.json",
"sha256": "b86cff25bb490a5c10a8e22b34d0473fbe6e6b04566f8187942bd6302ba5a5da"
},
"sixg_bench-full": {
"repository": "GSMA/ot-full",
"revision": "6319806f04783eafe04d9facf755d379c66b7664",
"file": "sixg_bench/test-00000-of-00001.parquet",
"sha256": "1e5995e1248a8f69f514a6d7afaac74fd8f82e8e05514778aecdaff873ed9371"
}
}
}
Loading