From 7a4d70195807d9bdfb5ae3f770c75ab64f456243 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Sat, 15 Aug 2026 00:35:22 +0000 Subject: [PATCH 1/4] feat: add NucleusClient.merge_model_runs() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A benchmark evaluation names a single model run, and a benchmark's items may span several datasets. A model whose predictions were uploaded as separate runs — one per dataset, or one per inference batch — therefore had no single run covering the benchmark, and every uncovered item scored as a false negative. Merging the runs produces one run that does cover it, which can then be passed to create_benchmark_evaluation_v2(). The merge is a full union: predictions are copied, never deduplicated. Colliding annotation_ids are rewritten rather than dropped, and the response reports predictions_copied, predictions_ignored and annotation_ids_rewritten so nothing is lost silently. Wraps POST /v1/nucleus/modelRun/merge. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 7 +++++ nucleus/__init__.py | 62 ++++++++++++++++++++++++++++++++++++++++++++ nucleus/constants.py | 1 + pyproject.toml | 2 +- 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5e93ba8..ecf68d00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.21.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.0) - 2026-08-15 + +### Added +- **`NucleusClient.merge_model_runs()`.** Merges two or more model runs into one new run holding the union of their predictions, leaving the sources untouched. A benchmark evaluation names a single model run and a benchmark's items may span datasets, so a model uploaded as several runs previously had no single run covering the benchmark — every uncovered item scored as a false negative. Merge first, then pass the new run to `create_benchmark_evaluation_v2()`. + + The merge is a full union: predictions are copied, never deduplicated. Colliding `annotation_id`s are rewritten rather than dropped, and the response reports `predictions_copied`, `predictions_ignored` and `annotation_ids_rewritten`. + ## [0.20.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.20.0) - 2026-08-11 ### Added diff --git a/nucleus/__init__.py b/nucleus/__init__.py index 2fe38883..0f3a6297 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -145,8 +145,10 @@ MESSAGE_KEY, METADATA_KEY, METRIC_TYPE_KEY, + MODEL_ID_KEY, MODEL_IDS_KEY, MODEL_RUN_ID_KEY, + MODEL_RUN_IDS_KEY, MODEL_TAGS_KEY, MODEL_TRAINED_SLICE_IDS_KEY, NAME_KEY, @@ -526,6 +528,66 @@ def delete_model_run(self, model_run_id: str): {}, f"modelRun/{model_run_id}", requests.delete ) + def merge_model_runs( + self, + model_run_ids: List[str], + name: str, + *, + model_id: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """Merge several model runs into one new run holding all their predictions. + + A benchmark evaluation names a single model run, and a benchmark's items may + span several datasets. A model whose predictions were uploaded as separate runs + — one per dataset, or one per inference batch — therefore has no single run + covering the benchmark, and every uncovered item scores as a false negative. + Merging the runs produces one run that does cover it, which you can then pass to + :meth:`create_benchmark_evaluation_v2`. + + The merge is a full union: predictions are copied, never deduplicated. If two + source runs predict on the same item with the same ``annotation_id``, the + colliding id is rewritten rather than dropped, and the count of rewrites comes + back in ``annotation_ids_rewritten``. + + The source runs are left untouched. + + Parameters: + model_run_ids: Two or more model run ids (``run_*``) to merge. + name: Display name for the merged run. + model_id: Model the merged run belongs to (``prj_*``). Defaults to the + sources' shared model; required when they belong to different models. + metadata: Optional metadata for the merged run. The merge always records + ``merged_from_model_run_ids`` alongside whatever you pass. + + Returns: + Dict describing the merge:: + + { + "model_run_id": str, # the new run + "source_model_run_ids": List[str], + "dataset_ids": List[str], # datasets the new run spans + "predictions_copied": int, + "predictions_ignored": int, # already present in the target + "annotation_ids_rewritten": int, + "errors": List[str], + } + """ + if len(set(model_run_ids)) < 2: + raise ValueError( + "merge_model_runs needs at least two distinct model run ids, got " + f"{sorted(set(model_run_ids))}" + ) + payload: Dict[str, Any] = { + MODEL_RUN_IDS_KEY: model_run_ids, + NAME_KEY: name, + } + if model_id is not None: + payload[MODEL_ID_KEY] = model_id + if metadata is not None: + payload[METADATA_KEY] = metadata + return self.make_request(payload, "modelRun/merge") + def create_dataset_from_project( self, project_id: str, diff --git a/nucleus/constants.py b/nucleus/constants.py index 23556815..a2bf46b0 100644 --- a/nucleus/constants.py +++ b/nucleus/constants.py @@ -111,6 +111,7 @@ MODEL_TRAINED_SLICE_IDS_KEY = "trained_slice_ids" MODEL_ID_KEY = "model_id" MODEL_RUN_ID_KEY = "model_run_id" +MODEL_RUN_IDS_KEY = "model_run_ids" MODEL_PREDICTION_ID_KEY = "model_prediction_id" MODEL_PREDICTION_LABEL_KEY = "model_prediction_label" NAME_KEY = "name" diff --git a/pyproject.toml b/pyproject.toml index 6f6de6a0..23b970ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.20.0" +version = "0.21.0" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] From c28d21c8e462a7f3c3ace8d1b7f7722d696379c9 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Fri, 21 Aug 2026 08:24:23 -0500 Subject: [PATCH 2/4] fix(nucleus): align merge_model_runs() with async backend contract Drop the unsupported model_id parameter (the backend Joi schema rejects unknown keys and forbids cross-model merges), handle the async 202 response by returning an AsyncJob so callers wait before evaluating, make name optional to match the server default, and correct the docstring and CHANGELOG to describe the real return shape. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 +-- nucleus/__init__.py | 60 ++++++++++++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d0c8a0..c78ea788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.21.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.0) - 2026-08-15 ### Added -- **`NucleusClient.merge_model_runs()`.** Merges two or more model runs into one new run holding the union of their predictions, leaving the sources untouched. A benchmark evaluation names a single model run and a benchmark's items may span datasets, so a model uploaded as several runs previously had no single run covering the benchmark — every uncovered item scored as a false negative. Merge first, then pass the new run to `create_benchmark_evaluation_v2()`. +- **`NucleusClient.merge_model_runs()`.** Merges two or more model runs into one new run holding the union of their predictions, leaving the sources untouched. A benchmark evaluation names a single model run and a benchmark's items may span datasets, so a model uploaded as several runs previously had no single run covering the benchmark — every uncovered item scored as a false negative. Merge first, wait for the copy to finish, then pass the new run to `create_benchmark_evaluation_v2()`. All source runs must belong to the same model. - The merge is a full union: predictions are copied, never deduplicated. Colliding `annotation_id`s are rewritten rather than dropped, and the response reports `predictions_copied`, `predictions_ignored` and `annotation_ids_rewritten`. + The copy runs asynchronously: the call returns `{"model_run_id", "dataset_ids", "job"}` immediately, but the new run is empty until the `job` completes — call `job.sleep_until_complete()` before evaluating. The merge is a full union: predictions are copied, never deduplicated, and colliding `annotation_id`s are rewritten rather than dropped. Copy counts (`predictions_copied`, `predictions_ignored`, `annotation_ids_rewritten`) are reported on the job. ## [0.20.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.20.2) - 2026-08-18 diff --git a/nucleus/__init__.py b/nucleus/__init__.py index ff095aec..011487bf 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -147,7 +147,6 @@ MESSAGE_KEY, METADATA_KEY, METRIC_TYPE_KEY, - MODEL_ID_KEY, MODEL_IDS_KEY, MODEL_RUN_ID_KEY, MODEL_RUN_IDS_KEY, @@ -544,9 +543,8 @@ def delete_model_run(self, model_run_id: str): def merge_model_runs( self, model_run_ids: List[str], - name: str, + name: Optional[str] = None, *, - model_id: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Merge several model runs into one new run holding all their predictions. @@ -558,33 +556,45 @@ def merge_model_runs( Merging the runs produces one run that does cover it, which you can then pass to :meth:`create_benchmark_evaluation_v2`. + All source runs must belong to the same model; merging across models is rejected + server-side (a run's model is its provenance, read by eval, leaderboards and the + model page). + The merge is a full union: predictions are copied, never deduplicated. If two source runs predict on the same item with the same ``annotation_id``, the - colliding id is rewritten rather than dropped, and the count of rewrites comes - back in ``annotation_ids_rewritten``. - - The source runs are left untouched. + colliding id is rewritten rather than dropped. The source runs are left + untouched. + + **Asynchronous.** The new run is created and returned immediately, but its + predictions are copied by a background job. The run is *empty until the job + completes*, so wait on the returned job before evaluating — otherwise the + evaluation scores uncopied items as false negatives, the very failure this is + meant to fix:: + + result = client.merge_model_runs(["run_abc", "run_def"]) + result["job"].sleep_until_complete() + client.create_benchmark_evaluation_v2( + benchmark_id, result["model_run_id"] + ) Parameters: - model_run_ids: Two or more model run ids (``run_*``) to merge. - name: Display name for the merged run. - model_id: Model the merged run belongs to (``prj_*``). Defaults to the - sources' shared model; required when they belong to different models. + model_run_ids: Two or more distinct model run ids (``run_*``) to merge. + name: Display name for the merged run. Defaults server-side to the model's + own name when omitted. metadata: Optional metadata for the merged run. The merge always records ``merged_from_model_run_ids`` alongside whatever you pass. Returns: - Dict describing the merge:: + Dict describing the newly created (still-populating) run:: { - "model_run_id": str, # the new run - "source_model_run_ids": List[str], - "dataset_ids": List[str], # datasets the new run spans - "predictions_copied": int, - "predictions_ignored": int, # already present in the target - "annotation_ids_rewritten": int, - "errors": List[str], + "model_run_id": str, # the new run, usable once the job finishes + "dataset_ids": List[str], # datasets the new run spans + "job": AsyncJob, # copy progress; poll or sleep_until_complete() } + + The copy's counts (``predictions_copied``, ``predictions_ignored``, + ``annotation_ids_rewritten``, errors) are reported on the job, not here. """ if len(set(model_run_ids)) < 2: raise ValueError( @@ -593,13 +603,17 @@ def merge_model_runs( ) payload: Dict[str, Any] = { MODEL_RUN_IDS_KEY: model_run_ids, - NAME_KEY: name, } - if model_id is not None: - payload[MODEL_ID_KEY] = model_id + if name is not None: + payload[NAME_KEY] = name if metadata is not None: payload[METADATA_KEY] = metadata - return self.make_request(payload, "modelRun/merge") + response = self.make_request(payload, "modelRun/merge") + return { + MODEL_RUN_ID_KEY: response[MODEL_RUN_ID_KEY], + DATASET_IDS_KEY: response[DATASET_IDS_KEY], + "job": AsyncJob.from_id(response[JOB_ID_KEY], self), + } def create_dataset_from_project( self, From 154b9cfbd899fc9bfbce66467420d6907b0109c4 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Fri, 21 Aug 2026 09:06:39 -0500 Subject: [PATCH 3/4] update nits --- nucleus/__init__.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/nucleus/__init__.py b/nucleus/__init__.py index 011487bf..1f3755cc 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -543,8 +543,8 @@ def delete_model_run(self, model_run_id: str): def merge_model_runs( self, model_run_ids: List[str], - name: Optional[str] = None, *, + name: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Merge several model runs into one new run holding all their predictions. @@ -556,20 +556,16 @@ def merge_model_runs( Merging the runs produces one run that does cover it, which you can then pass to :meth:`create_benchmark_evaluation_v2`. - All source runs must belong to the same model; merging across models is rejected - server-side (a run's model is its provenance, read by eval, leaderboards and the - model page). + All source runs must belong to the same model. - The merge is a full union: predictions are copied, never deduplicated. If two + The merge is a full union of all predictions. If two source runs predict on the same item with the same ``annotation_id``, the colliding id is rewritten rather than dropped. The source runs are left untouched. **Asynchronous.** The new run is created and returned immediately, but its predictions are copied by a background job. The run is *empty until the job - completes*, so wait on the returned job before evaluating — otherwise the - evaluation scores uncopied items as false negatives, the very failure this is - meant to fix:: + completes*, so wait on the returned job before evaluating:: result = client.merge_model_runs(["run_abc", "run_def"]) result["job"].sleep_until_complete() @@ -596,13 +592,14 @@ def merge_model_runs( The copy's counts (``predictions_copied``, ``predictions_ignored``, ``annotation_ids_rewritten``, errors) are reported on the job, not here. """ - if len(set(model_run_ids)) < 2: + unique_model_run_ids = list(dict.fromkeys(model_run_ids)) + if len(unique_model_run_ids) < 2: raise ValueError( "merge_model_runs needs at least two distinct model run ids, got " - f"{sorted(set(model_run_ids))}" + f"{sorted(unique_model_run_ids)}" ) payload: Dict[str, Any] = { - MODEL_RUN_IDS_KEY: model_run_ids, + MODEL_RUN_IDS_KEY: unique_model_run_ids, } if name is not None: payload[NAME_KEY] = name @@ -612,7 +609,7 @@ def merge_model_runs( return { MODEL_RUN_ID_KEY: response[MODEL_RUN_ID_KEY], DATASET_IDS_KEY: response[DATASET_IDS_KEY], - "job": AsyncJob.from_id(response[JOB_ID_KEY], self), + "job": AsyncJob.from_json(response, self), } def create_dataset_from_project( From d022de27526da80da64821347835aa337e50861a Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Fri, 21 Aug 2026 14:18:20 +0000 Subject: [PATCH 4/4] style: strip trailing whitespace in model_run.py (pylint C0303) Co-Authored-By: Claude Opus 4.8 --- nucleus/model_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nucleus/model_run.py b/nucleus/model_run.py index d78b7336..922ffb7c 100644 --- a/nucleus/model_run.py +++ b/nucleus/model_run.py @@ -173,7 +173,7 @@ def predict( "predictions_ignored": int, } """ - + uploader = PredictionUploader( client=self._client, dataset_id=self.dataset_id,