From ac0ebb857532a5e17bf8f5dbef76e3abe5da4538 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Tue, 25 Aug 2026 18:57:29 +0000 Subject: [PATCH 1/7] feat(model): add Model.model_runs() to list a model's run ids The model-scoped counterpart to Dataset.model_runs(): lists every run under a model via GET /nucleus/model/:modelId/modelRun. include_versions=True unions runs across the model's version lineage (?family=true). Results are dataset-scoped server-side. Adds a mock unit test, CHANGELOG entry, and version bump to 0.21.3. Requires the matching scaleapi route (scaleapi#158386); live calls 404 until it deploys. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 11 +++++++++ nucleus/model.py | 19 +++++++++++++++ pyproject.toml | 2 +- tests/test_model_runs_listing.py | 40 ++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/test_model_runs_listing.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 021de916..555f7f07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.21.3](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.3) - 2026-08-25 + +### Added +- **`Model.model_runs()`.** Lists the ids of every model run for a model — the model-scoped counterpart to `Dataset.model_runs()`, which only lists a single dataset's runs. Pass `include_versions=True` to union runs across the model's version lineage (its version root and all descendants). Results are scoped server-side to runs on datasets you can read. + + ```python + run_ids = model.model_runs() + ``` + +> **Server dependency:** requires the `GET /nucleus/model/:modelId/modelRun` route in scaleapi. Unit tests pass regardless; live calls 404 until that deploys. + ## [0.21.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.2) - 2026-08-17 ### Added diff --git a/nucleus/model.py b/nucleus/model.py index 18340571..219ef5eb 100644 --- a/nucleus/model.py +++ b/nucleus/model.py @@ -229,6 +229,25 @@ def create_run( run.add_predictions(predictions) return run + def model_runs(self, include_versions: bool = False) -> List[str]: + """List the ids of every model run for this model. :: + + run_ids = model.model_runs() + + Args: + include_versions: Also include runs from other versions in this + model's lineage — its version root and all descendants. Defaults + to False, returning only runs whose ``model_id`` is this model. + + Returns: + The model run ids (``run_*``). Scoped server-side to runs on datasets + you can read, so a run on a dataset you can't access is omitted. + """ + route = f"model/{self.id}/modelRun" + if include_versions: + route += "?family=true" + return self._client.make_request({}, route, requests.get) + def evaluate(self, scenario_test_names: List[str]) -> AsyncJob: """Evaluates this on the specified Unit Tests. :: diff --git a/pyproject.toml b/pyproject.toml index 4901f914..68651aab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.21.2" +version = "0.21.3" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] diff --git a/tests/test_model_runs_listing.py b/tests/test_model_runs_listing.py new file mode 100644 index 00000000..50cb29b7 --- /dev/null +++ b/tests/test_model_runs_listing.py @@ -0,0 +1,40 @@ +from unittest.mock import MagicMock + +import requests + +from nucleus import Model + + +def _model_with_client(): + client = MagicMock() + model = Model( + model_id="prj_123", + name="my-model", + reference_id="my-ref", + metadata=None, + client=client, + ) + return model, client + + +def test_model_runs_returns_ids_and_hits_the_route(): + model, client = _model_with_client() + client.make_request.return_value = ["run_a", "run_b"] + + result = model.model_runs() + + assert result == ["run_a", "run_b"] + payload, route, requests_command = client.make_request.call_args.args + assert payload == {} + assert route == "model/prj_123/modelRun" + assert requests_command is requests.get + + +def test_model_runs_include_versions_appends_family_query(): + model, client = _model_with_client() + client.make_request.return_value = [] + + model.model_runs(include_versions=True) + + _, route, _ = client.make_request.call_args.args + assert route == "model/prj_123/modelRun?family=true" From 726000b22f68eb0a7f0f8b5956177101d29da01b Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Wed, 26 Aug 2026 23:09:35 +0000 Subject: [PATCH 2/7] feat(model): add run-free ("model v2") predictions [DE-8678] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a run-free prediction concept where predictions are tied directly to a Model as (model, dataset_item) -> prediction, with no ModelRun or Dataset. Purely additive: all existing model-run prediction paths are unchanged. - Model.upload_predictions(...) — upsert predictions onto model/{id}/predictions (sync + async), reusing the PredictionUploader batching machinery. - Model.predictions_loc / predictions_refloc / predictions_iloc — model-scoped reads. - Model.copy_predictions_from_run(model_run_id) — backfill from a v1 run (AsyncJob). - create_benchmark_evaluation_v2 gains an optional model_id anchor (accepts a prj_* id or a Model) as an alternative to model_run_id; exactly one required. - EvaluationV2 gains an optional model_id field; model_run_id now optional. - serialize_and_write_to_presigned_url gains route_prefix for the model route. - Docs + CHANGELOG; minor version bump to 0.22.0. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 12 +++ nucleus/__init__.py | 26 ++++- nucleus/evaluation_v2.py | 17 +++- nucleus/model.py | 202 ++++++++++++++++++++++++++++++++++++++- nucleus/model_run.py | 17 +++- nucleus/utils.py | 28 +++++- pyproject.toml | 2 +- 7 files changed, 290 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 021de916..d6b64c36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.22.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.22.0) - 2026-08-26 + +### Added +- **Run-free ("model v2") predictions.** Predictions can now be uploaded and read directly against a `Model`, with no `ModelRun` or `Dataset` involved — the concept is `(model, dataset_item) -> prediction`. New methods on `Model`: + - `Model.upload_predictions(predictions, update=False, asynchronous=False, batch_size=5000, ...)` — upserts predictions onto the model (`box` / `polygon` / `cuboid` only). Reuses the existing `PredictionUploader` batching machinery, targeting `model/{id}/predictions` (async posts to `model/{id}/predictions?async=1` and returns an `AsyncJob`). + - `Model.predictions_loc(dataset_item_id)`, `Model.predictions_refloc(reference_id)`, `Model.predictions_iloc(i)` — model-scoped reads returning the same shape as their `Dataset` equivalents. + - `Model.copy_predictions_from_run(model_run_id, asynchronous=True)` — backfills the run-free store from an existing model run, returning an `AsyncJob`. +- **Model-anchored benchmark evaluations.** `NucleusClient.create_benchmark_evaluation_v2()` accepts a `model_id` (a `prj_*` id or a `Model`) as an alternative to `model_run_id`; the model-anchored flow evaluates the model's run-free predictions and ignores model runs. Provide exactly one of the two. `EvaluationV2` now exposes an optional `model_id` field alongside `model_run_id`. + +### Changed +- The existing run-based prediction paths (`Dataset.upload_predictions`, `ModelRun.add_predictions`, `create_benchmark_evaluation_v2(model_run_id=...)`) are unchanged and continue to work; the model-centric methods are purely additive. + ## [0.21.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.2) - 2026-08-17 ### Added diff --git a/nucleus/__init__.py b/nucleus/__init__.py index e5a94c5a..b9bab5b9 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -149,6 +149,7 @@ MESSAGE_KEY, METADATA_KEY, METRIC_TYPE_KEY, + MODEL_ID_KEY, MODEL_IDS_KEY, MODEL_RUN_ID_KEY, MODEL_RUN_IDS_KEY, @@ -1562,8 +1563,9 @@ def finalize_benchmark(self, benchmark_id: str) -> Benchmark: def create_benchmark_evaluation_v2( self, benchmark_id: str, - model_run_id: str, + model_run_id: Optional[str] = None, *, + model_id: Optional[Union[str, Model]] = None, name: Optional[str] = None, rollup_groups: Optional[List[RollupGroup]] = None, allowed_label_matches: Optional[List[AllowedLabelMatch]] = None, @@ -1587,10 +1589,19 @@ def create_benchmark_evaluation_v2( being rejected. To give a run predictions across several datasets, use :meth:`Dataset.upload_predictions_for_model_run`. + The evaluation can be anchored on either a legacy model run + (``model_run_id``) or, for the run-free "model v2" flow, a model + (``model_id``). Provide exactly one; the model-anchored flow evaluates + the model's run-free predictions and ignores model runs entirely. + Parameters: benchmark_id: Benchmark id (``bm_*``). model_run_id: Model run id (``run_*``). It need not cover the benchmark's datasets — coverage may be partial, or empty. + Mutually exclusive with ``model_id``. + model_id: Model id (``prj_*``) or :class:`Model` to anchor the + evaluation on the model's run-free predictions instead of a + model run. Mutually exclusive with ``model_run_id``. name: Optional display name. rollup_groups: Optional rollup classes (the primary label configuration); each :class:`RollupGroup` maps raw labels @@ -1610,6 +1621,13 @@ def create_benchmark_evaluation_v2( Returns: :class:`EvaluationV2`: The created evaluation. """ + resolved_model_id = ( + model_id.id if isinstance(model_id, Model) else model_id + ) + if (resolved_model_id is None) == (model_run_id is None): + raise ValueError( + "Provide exactly one of model_run_id or model_id." + ) if preset is not None: if ( rollup_groups is None @@ -1635,7 +1653,11 @@ def create_benchmark_evaluation_v2( "Set at most one of rollup_groups, allowed_label_matches, " "or allowed_label_matches_id" ) - payload: Dict[str, Any] = {MODEL_RUN_ID_KEY: model_run_id} + payload: Dict[str, Any] = ( + {MODEL_ID_KEY: resolved_model_id} + if resolved_model_id is not None + else {MODEL_RUN_ID_KEY: model_run_id} + ) if name is not None: payload[NAME_KEY] = name if rollup_groups is not None: diff --git a/nucleus/evaluation_v2.py b/nucleus/evaluation_v2.py index 0d4847af..a70e8bd3 100644 --- a/nucleus/evaluation_v2.py +++ b/nucleus/evaluation_v2.py @@ -31,6 +31,7 @@ LABELS_KEY, LIMIT_KEY, MATCH_TYPE_KEY, + MODEL_ID_KEY, MODEL_PREDICTION_LABEL_CAMEL_KEY, MODEL_PREDICTION_LABEL_KEY, MODEL_RUN_ID_KEY, @@ -170,12 +171,13 @@ def _parse_rollup_groups(raw_groups: Any) -> Optional[List[RollupGroup]]: @dataclass class EvaluationV2: - """An Evaluation V2 run for a model run.""" + """An Evaluation V2 run for a model run or a run-free model.""" id: str - model_run_id: str + model_run_id: Optional[str] dataset_id: str status: str + model_id: Optional[str] = None name: Optional[str] = None temporal_workflow_id: Optional[str] = None error_message: Optional[str] = None @@ -202,9 +204,18 @@ def from_json( return cls( id=str(payload[ID_KEY]), - model_run_id=str(payload[MODEL_RUN_ID_KEY]), + model_run_id=( + str(payload[MODEL_RUN_ID_KEY]) + if payload.get(MODEL_RUN_ID_KEY) is not None + else None + ), dataset_id=str(payload[DATASET_ID_KEY]), status=str(payload[STATUS_KEY]), + model_id=( + str(payload[MODEL_ID_KEY]) + if payload.get(MODEL_ID_KEY) is not None + else None + ), name=payload.get(NAME_KEY), temporal_workflow_id=payload.get(TEMPORAL_WORKFLOW_ID_KEY), error_message=payload.get(ERROR_MESSAGE_KEY), diff --git a/nucleus/model.py b/nucleus/model.py index 18340571..75317521 100644 --- a/nucleus/model.py +++ b/nucleus/model.py @@ -1,14 +1,24 @@ -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union import requests +from nucleus.annotation import check_all_mask_paths_remote +from nucleus.annotation_uploader import PredictionUploader +from nucleus.utils import ( + format_prediction_response, + serialize_and_write_to_presigned_url, +) + from .async_job import AsyncJob from .constants import ( METADATA_KEY, + MODEL_RUN_ID_KEY, MODEL_TAGS_KEY, MODEL_TRAINED_SLICE_IDS_KEY, NAME_KEY, REFERENCE_ID_KEY, + REQUEST_ID_KEY, + UPDATE_KEY, ) from .dataset import Dataset from .model_run import ModelRun @@ -17,6 +27,7 @@ BoxPrediction, CuboidPrediction, PolygonPrediction, + Prediction, SegmentationPrediction, ) @@ -229,6 +240,195 @@ def create_run( run.add_predictions(predictions) return run + def upload_predictions( + self, + predictions: List[Prediction], + update: bool = False, + asynchronous: bool = False, + batch_size: int = 5000, + remote_files_per_upload_request: int = 20, + local_files_per_upload_request: int = 10, + ) -> Union[Dict[str, Any], AsyncJob]: + """Uploads predictions directly to this model, with no model run. + + This is the run-free ("model v2") prediction path: predictions are tied + to the model itself as ``(model, dataset_item) -> prediction`` and are + upserted server-side. Each prediction identifies its target item by + ``dataset_item_id`` (the ``di_*`` id returned on exported items) or by + ``reference_id``, so a single model can hold predictions for items that + live in different datasets — no :class:`Dataset` or :class:`ModelRun` is + needed. Reads go through :meth:`predictions_loc`, + :meth:`predictions_refloc`, and :meth:`predictions_iloc`. + + Only ``box``, ``polygon``, and ``cuboid`` predictions are accepted on + this path. + + The legacy run-based path (:meth:`Dataset.upload_predictions` / + :meth:`ModelRun.add_predictions`) continues to work unchanged. + + Args: + predictions: List of prediction objects to upload. + update: If True, existing predictions for the same + (reference_id, annotation_id) are overwritten. If False, they + are skipped. Default is False. + asynchronous: Whether or not to process the upload asynchronously + (and return an :class:`AsyncJob` object). Default is False. + batch_size: Number of predictions processed in each concurrent + batch. Default is 5000. If you get timeouts when uploading + geometric predictions, you can try lowering this batch size. + This is only relevant for asynchronous=False. + remote_files_per_upload_request: Number of remote files to upload in + each request. Only relevant for asynchronous=False. + local_files_per_upload_request: Number of local files to upload in + each request. The maximum is 10. Only relevant for + asynchronous=False. + + Returns: + Payload describing the synchronous upload, or an :class:`AsyncJob` + when ``asynchronous=True``:: + + { + "model_id": str, + "predictions_processed": int, + "predictions_ignored": int, + } + """ + uploader = PredictionUploader( + client=self._client, + route=f"model/{self.id}/predictions", + ) + uploader.check_for_duplicate_ids(predictions) + + if asynchronous: + check_all_mask_paths_remote(predictions) + request_id = serialize_and_write_to_presigned_url( + predictions, + dataset_id=None, + client=self._client, + route_prefix=f"model/{self.id}", + ) + response = self._client.make_request( + payload={REQUEST_ID_KEY: request_id, UPDATE_KEY: update}, + route=f"model/{self.id}/predictions?async=1", + ) + return AsyncJob.from_json(response, self._client) + + return uploader.upload( + annotations=predictions, + batch_size=batch_size, + update=update, + remote_files_per_upload_request=remote_files_per_upload_request, + local_files_per_upload_request=local_files_per_upload_request, + ) + + def predictions_loc(self, dataset_item_id: str): + """Fetches all of this model's predictions for a dataset item by its id. + + Model-scoped counterpart of :meth:`Dataset.prediction_loc` for the + run-free prediction path. + + Parameters: + dataset_item_id: Internally controlled id for the dataset item + (``di_*``). + + Returns: + Dictionary mapping prediction type to a list of prediction objects + for this model:: + + { + "box": List[BoxPrediction], + "polygon": List[PolygonPrediction], + "cuboid": List[CuboidPrediction], + } + """ + return format_prediction_response( + self._client.make_request( + payload=None, + route=f"model/{self.id}/predictions/loc/{dataset_item_id}", + requests_command=requests.get, + ) + ) + + def predictions_refloc(self, reference_id: str): + """Fetches all of this model's predictions for a dataset item by its reference id. + + Model-scoped counterpart of :meth:`Dataset.predictions_refloc` for the + run-free prediction path. + + Parameters: + reference_id: User-defined reference id of the dataset item. + + Returns: + Dictionary mapping prediction type to a list of prediction objects + for this model:: + + { + "box": List[BoxPrediction], + "polygon": List[PolygonPrediction], + "cuboid": List[CuboidPrediction], + } + """ + return format_prediction_response( + self._client.make_request( + payload=None, + route=f"model/{self.id}/predictions/refloc/{reference_id}", + requests_command=requests.get, + ) + ) + + def predictions_iloc(self, i: int): + """Fetches all of this model's predictions for a dataset item by its index. + + Model-scoped counterpart of :meth:`Dataset.predictions_iloc` for the + run-free prediction path. + + Parameters: + i: Absolute index of the dataset item. + + Returns: + Dictionary mapping prediction type to a list of prediction objects + for this model:: + + { + "box": List[BoxPrediction], + "polygon": List[PolygonPrediction], + "cuboid": List[CuboidPrediction], + } + """ + return format_prediction_response( + self._client.make_request( + payload=None, + route=f"model/{self.id}/predictions/iloc/{i}", + requests_command=requests.get, + ) + ) + + def copy_predictions_from_run( + self, model_run_id: str, asynchronous: bool = True + ) -> AsyncJob: + """Copies predictions from a legacy v1 model run onto this model. + + Backfills the run-free ("model v2") prediction store for this model from + an existing :class:`ModelRun`, so predictions previously uploaded via the + run-based path become readable through :meth:`predictions_loc` and + friends. The source run is left untouched. + + Args: + model_run_id: Source model run id (``run_*``) to copy predictions + from. + asynchronous: Retained for forward compatibility; the copy always + runs server-side as an async job. Default is True. + + Returns: + An :class:`AsyncJob` tracking the copy. + """ + response = self._client.make_request( + {MODEL_RUN_ID_KEY: model_run_id}, + route=f"model/{self.id}/predictions/copyFromRun", + requests_command=requests.post, + ) + return AsyncJob.from_json(response, self._client) + def evaluate(self, scenario_test_names: List[str]) -> AsyncJob: """Evaluates this on the specified Unit Tests. :: diff --git a/nucleus/model_run.py b/nucleus/model_run.py index aa626ca8..3f20d7ef 100644 --- a/nucleus/model_run.py +++ b/nucleus/model_run.py @@ -1,18 +1,29 @@ """ Model Runs are deprecated and will be removed in a future version of the python client. -It is now possible to upload model predictions without a need for creating a model run +It is now possible to upload model predictions without a need for creating a model run. -For example:: +The recommended run-free ("model v2") path uploads and reads predictions +directly on a :class:`~nucleus.model.Model` — the concept is +``(model, dataset_item) -> prediction``, with no model run or dataset:: import nucleus client = nucleus.NucleusClient(YOUR_SCALE_API_KEY) prediction_1 = nucleus.BoxPrediction(label="label", x=0, y=0, width=10, height=10, reference_id="1", confidence=0.9, class_pdf={'label': 0.9, 'other_label': 0.1}) prediction_2 = nucleus.BoxPrediction(label="label", x=0, y=0, width=10, height=10, reference_id="2", confidence=0.2, class_pdf={'label': 0.2, 'other_label': 0.8}) model = client.create_model(name="My Model", reference_id="My-CNN", metadata={"timestamp": "121012401"}) + + # Run-free upload / read, tied to the model itself: + model.upload_predictions([prediction_1, prediction_2]) + model.predictions_refloc("1") + +Benchmark evaluations can likewise anchor on the model directly, ignoring model +runs, via ``client.create_benchmark_evaluation_v2(benchmark_id, model_id=model.id)``. + +The older per-dataset path also remains available:: + response = dataset.upload_predictions(model, [prediction_1, prediction_2]) """ - from typing import List, Optional, Union import requests diff --git a/nucleus/utils.py b/nucleus/utils.py index d4db15dc..f8861934 100644 --- a/nucleus/utils.py +++ b/nucleus/utils.py @@ -6,7 +6,17 @@ import urllib.request import uuid from collections import defaultdict -from typing import IO, TYPE_CHECKING, Dict, List, Sequence, Tuple, Type, Union +from typing import ( + IO, + TYPE_CHECKING, + Dict, + List, + Optional, + Sequence, + Tuple, + Type, + Union, +) import requests from PIL import Image @@ -396,12 +406,22 @@ def serialize_and_write_to_presigned_url( upload_units: Sequence[ Union[DatasetItem, Annotation, LidarScene, VideoScene] ], - dataset_id: str, + dataset_id: Optional[str], client, + route_prefix: Optional[str] = None, ): - """This helper function can be used to serialize a list of API objects to NDJSON.""" + """This helper function can be used to serialize a list of API objects to NDJSON. + + By default the presigned URL is requested from the dataset-scoped route + ``dataset/{dataset_id}/signedUrl/{request_id}``. Pass ``route_prefix`` (e.g. + ``model/{model_id}``) to target a different signed-URL route — used by the + model-scoped prediction upload, which has no owning dataset. + """ request_id = uuid.uuid4().hex - route = f"dataset/{dataset_id}/signedUrl/{request_id}" + prefix = ( + route_prefix if route_prefix is not None else f"dataset/{dataset_id}" + ) + route = f"{prefix}/signedUrl/{request_id}" if os.environ.get("S3_ENDPOINT") is not None: route += "?s3Endpoint=" + urllib.request.pathname2url( os.environ["S3_ENDPOINT"] diff --git a/pyproject.toml b/pyproject.toml index 4901f914..5b47c0b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.21.2" +version = "0.22.0" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] From 3aaf66366890ee307c91de288a19e1b105f0c1de Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Wed, 26 Aug 2026 23:38:27 +0000 Subject: [PATCH 3/7] fix(model): make v2 prediction upload + copy sync-only to match backend [DE-8678] The live scaleapi backend for DE-8678 is synchronous-only for these routes: - Model.upload_predictions: the model route has no async/signed-URL endpoint (?async=1 returns HTTP 400). Remove the assumed signed-URL async flow and raise NotImplementedError when asynchronous=True; keep the sync path as-is. Revert the now-unused route_prefix param added to serialize_and_write_to_presigned_url in nucleus/utils.py. - Model.copy_predictions_from_run: the backend runs synchronously and returns {model_id, model_run_ids, predictions_copied, predictions_skipped_unsupported}. Return that dict directly (drop the AsyncJob wrapping and the unused asynchronous param); note it's synchronous in the docstring. CHANGELOG updated to match; still additive, still v0.22.0. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 4 +-- nucleus/model.py | 64 ++++++++++++++++++++---------------------------- nucleus/utils.py | 28 +++------------------ 3 files changed, 32 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6b64c36..8e9ac4e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **Run-free ("model v2") predictions.** Predictions can now be uploaded and read directly against a `Model`, with no `ModelRun` or `Dataset` involved — the concept is `(model, dataset_item) -> prediction`. New methods on `Model`: - - `Model.upload_predictions(predictions, update=False, asynchronous=False, batch_size=5000, ...)` — upserts predictions onto the model (`box` / `polygon` / `cuboid` only). Reuses the existing `PredictionUploader` batching machinery, targeting `model/{id}/predictions` (async posts to `model/{id}/predictions?async=1` and returns an `AsyncJob`). + - `Model.upload_predictions(predictions, update=False, batch_size=5000, ...)` — upserts predictions onto the model (`box` / `polygon` / `cuboid` only), targeting `model/{id}/predictions`, and reusing the existing `PredictionUploader` batching machinery. Synchronous only for now: `asynchronous=True` raises `NotImplementedError`. - `Model.predictions_loc(dataset_item_id)`, `Model.predictions_refloc(reference_id)`, `Model.predictions_iloc(i)` — model-scoped reads returning the same shape as their `Dataset` equivalents. - - `Model.copy_predictions_from_run(model_run_id, asynchronous=True)` — backfills the run-free store from an existing model run, returning an `AsyncJob`. + - `Model.copy_predictions_from_run(model_run_id)` — synchronously backfills the run-free store from an existing model run, returning a dict `{model_id, model_run_ids, predictions_copied, predictions_skipped_unsupported}`. - **Model-anchored benchmark evaluations.** `NucleusClient.create_benchmark_evaluation_v2()` accepts a `model_id` (a `prj_*` id or a `Model`) as an alternative to `model_run_id`; the model-anchored flow evaluates the model's run-free predictions and ignores model runs. Provide exactly one of the two. `EvaluationV2` now exposes an optional `model_id` field alongside `model_run_id`. ### Changed diff --git a/nucleus/model.py b/nucleus/model.py index 75317521..d6bab803 100644 --- a/nucleus/model.py +++ b/nucleus/model.py @@ -2,12 +2,8 @@ import requests -from nucleus.annotation import check_all_mask_paths_remote from nucleus.annotation_uploader import PredictionUploader -from nucleus.utils import ( - format_prediction_response, - serialize_and_write_to_presigned_url, -) +from nucleus.utils import format_prediction_response from .async_job import AsyncJob from .constants import ( @@ -17,8 +13,6 @@ MODEL_TRAINED_SLICE_IDS_KEY, NAME_KEY, REFERENCE_ID_KEY, - REQUEST_ID_KEY, - UPDATE_KEY, ) from .dataset import Dataset from .model_run import ModelRun @@ -248,7 +242,7 @@ def upload_predictions( batch_size: int = 5000, remote_files_per_upload_request: int = 20, local_files_per_upload_request: int = 10, - ) -> Union[Dict[str, Any], AsyncJob]: + ) -> Dict[str, Any]: """Uploads predictions directly to this model, with no model run. This is the run-free ("model v2") prediction path: predictions are tied @@ -271,21 +265,19 @@ def upload_predictions( update: If True, existing predictions for the same (reference_id, annotation_id) are overwritten. If False, they are skipped. Default is False. - asynchronous: Whether or not to process the upload asynchronously - (and return an :class:`AsyncJob` object). Default is False. + asynchronous: Not yet supported for this path — passing True raises + :class:`NotImplementedError`. The upload always runs + synchronously. batch_size: Number of predictions processed in each concurrent batch. Default is 5000. If you get timeouts when uploading geometric predictions, you can try lowering this batch size. - This is only relevant for asynchronous=False. remote_files_per_upload_request: Number of remote files to upload in - each request. Only relevant for asynchronous=False. + each request. local_files_per_upload_request: Number of local files to upload in - each request. The maximum is 10. Only relevant for - asynchronous=False. + each request. The maximum is 10. Returns: - Payload describing the synchronous upload, or an :class:`AsyncJob` - when ``asynchronous=True``:: + Payload describing the synchronous upload:: { "model_id": str, @@ -293,26 +285,18 @@ def upload_predictions( "predictions_ignored": int, } """ + if asynchronous: + raise NotImplementedError( + "async is not yet supported for model prediction v2 uploads; " + "use asynchronous=False" + ) + uploader = PredictionUploader( client=self._client, route=f"model/{self.id}/predictions", ) uploader.check_for_duplicate_ids(predictions) - if asynchronous: - check_all_mask_paths_remote(predictions) - request_id = serialize_and_write_to_presigned_url( - predictions, - dataset_id=None, - client=self._client, - route_prefix=f"model/{self.id}", - ) - response = self._client.make_request( - payload={REQUEST_ID_KEY: request_id, UPDATE_KEY: update}, - route=f"model/{self.id}/predictions?async=1", - ) - return AsyncJob.from_json(response, self._client) - return uploader.upload( annotations=predictions, batch_size=batch_size, @@ -403,9 +387,7 @@ def predictions_iloc(self, i: int): ) ) - def copy_predictions_from_run( - self, model_run_id: str, asynchronous: bool = True - ) -> AsyncJob: + def copy_predictions_from_run(self, model_run_id: str) -> Dict[str, Any]: """Copies predictions from a legacy v1 model run onto this model. Backfills the run-free ("model v2") prediction store for this model from @@ -413,21 +395,27 @@ def copy_predictions_from_run( run-based path become readable through :meth:`predictions_loc` and friends. The source run is left untouched. + Runs synchronously server-side and returns once the copy completes. + Args: model_run_id: Source model run id (``run_*``) to copy predictions from. - asynchronous: Retained for forward compatibility; the copy always - runs server-side as an async job. Default is True. Returns: - An :class:`AsyncJob` tracking the copy. + Payload describing the copy:: + + { + "model_id": str, + "model_run_ids": List[str], + "predictions_copied": int, + "predictions_skipped_unsupported": int, + } """ - response = self._client.make_request( + return self._client.make_request( {MODEL_RUN_ID_KEY: model_run_id}, route=f"model/{self.id}/predictions/copyFromRun", requests_command=requests.post, ) - return AsyncJob.from_json(response, self._client) def evaluate(self, scenario_test_names: List[str]) -> AsyncJob: """Evaluates this on the specified Unit Tests. :: diff --git a/nucleus/utils.py b/nucleus/utils.py index f8861934..d4db15dc 100644 --- a/nucleus/utils.py +++ b/nucleus/utils.py @@ -6,17 +6,7 @@ import urllib.request import uuid from collections import defaultdict -from typing import ( - IO, - TYPE_CHECKING, - Dict, - List, - Optional, - Sequence, - Tuple, - Type, - Union, -) +from typing import IO, TYPE_CHECKING, Dict, List, Sequence, Tuple, Type, Union import requests from PIL import Image @@ -406,22 +396,12 @@ def serialize_and_write_to_presigned_url( upload_units: Sequence[ Union[DatasetItem, Annotation, LidarScene, VideoScene] ], - dataset_id: Optional[str], + dataset_id: str, client, - route_prefix: Optional[str] = None, ): - """This helper function can be used to serialize a list of API objects to NDJSON. - - By default the presigned URL is requested from the dataset-scoped route - ``dataset/{dataset_id}/signedUrl/{request_id}``. Pass ``route_prefix`` (e.g. - ``model/{model_id}``) to target a different signed-URL route — used by the - model-scoped prediction upload, which has no owning dataset. - """ + """This helper function can be used to serialize a list of API objects to NDJSON.""" request_id = uuid.uuid4().hex - prefix = ( - route_prefix if route_prefix is not None else f"dataset/{dataset_id}" - ) - route = f"{prefix}/signedUrl/{request_id}" + route = f"dataset/{dataset_id}/signedUrl/{request_id}" if os.environ.get("S3_ENDPOINT") is not None: route += "?s3Endpoint=" + urllib.request.pathname2url( os.environ["S3_ENDPOINT"] From e5a8cb886af95c830eb27f767d675f1dccfc220d Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Fri, 28 Aug 2026 12:31:24 -0500 Subject: [PATCH 4/7] fix(model): parse run-free prediction reads in format_prediction_response [DE-8678] Model.predictions_loc / predictions_refloc / predictions_iloc were shipped non-functional: the run-free read endpoints return a flat {"predictions": [...]} list (each element carrying its own "type"), but format_prediction_response only understood the legacy type-keyed {"annotations": {"box": [...]}} shape. It fell through to the "an error occurred" branch and returned the raw payload unparsed, so these reads yielded the raw dict instead of the documented {"box": [...], "polygon": [...], "cuboid": [...]}. Add a flat-list branch that groups predictions by their per-element "type" into that same shape. Legacy type-keyed reads and the error/empty case are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 +++ nucleus/utils.py | 24 ++++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e9ac4e8..1cf17652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - The existing run-based prediction paths (`Dataset.upload_predictions`, `ModelRun.add_predictions`, `create_benchmark_evaluation_v2(model_run_id=...)`) are unchanged and continue to work; the model-centric methods are purely additive. +### Fixed +- `Model.predictions_loc` / `predictions_refloc` / `predictions_iloc` now actually parse their responses. The run-free read endpoints return a flat `{"predictions": [...]}` list (each element carrying its own `"type"`), but `format_prediction_response` only understood the legacy type-keyed `{"annotations": {"box": [...]}}` shape, so these methods returned the raw payload unparsed instead of the documented `{"box": [...], "polygon": [...], "cuboid": [...]}` dict. + ## [0.21.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.2) - 2026-08-17 ### Added diff --git a/nucleus/utils.py b/nucleus/utils.py index d4db15dc..6fd870d7 100644 --- a/nucleus/utils.py +++ b/nucleus/utils.py @@ -47,6 +47,7 @@ SCALE_TASK_INFO_KEY, SCENE_KEY, SEGMENTATION_TYPE, + TYPE_KEY, ) from .dataset_item import DatasetItem from .prediction import ( @@ -131,10 +132,6 @@ def format_prediction_response( keyed by the type name. """ annotation_payload = response.get(ANNOTATIONS_KEY, None) - if not annotation_payload: - # An error occurred - return response - annotation_response = {} type_key_to_class: Dict[ str, Union[ @@ -156,6 +153,24 @@ def format_prediction_response( KEYPOINTS_TYPE: KeypointsPrediction, SEGMENTATION_TYPE: SegmentationPrediction, } + if not annotation_payload: + # Run-free ("model v2") reads (Model.predictions_loc / _refloc / + # _iloc) return a flat list under "predictions", each element carrying + # its own "type", rather than the type-keyed "annotations" dict. Group + # it into the same {type: [obj, ...]} shape those methods promise. + prediction_payload = response.get(PREDICTIONS_KEY, None) + if not prediction_payload: + # An error occurred, or there are no predictions. + return response + annotation_response: Dict[str, list] = {} + for prediction in prediction_payload: + type_key = prediction[TYPE_KEY] + type_class = type_key_to_class[type_key] + annotation_response.setdefault(type_key, []).append( + type_class.from_json(prediction) + ) + return annotation_response + annotation_response = {} for type_key in annotation_payload: type_class = type_key_to_class[type_key] annotation_response[type_key] = [ @@ -230,6 +245,7 @@ def format_scale_task_info_response(response: dict) -> Union[Dict, List[Dict]]: ret.append(row) return ret + # pylint: disable=too-many-branches,too-many-statements def convert_export_payload(api_payload, has_predictions: bool = False): """Helper function to convert raw JSON to API objects From 7d94ade45bba15bfe298e982a4a15754f83a007b Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Mon, 31 Aug 2026 14:32:04 -0500 Subject: [PATCH 5/7] remove dataset id refs --- CHANGELOG.md | 9 ++ nucleus/__init__.py | 70 +++++++++----- nucleus/benchmark.py | 3 +- nucleus/data_transfer_object/evaluation_v2.py | 4 - nucleus/evaluation_v2.py | 32 ++++++- nucleus/evaluation_v2_preset.py | 13 ++- pyproject.toml | 2 +- tests/test_benchmarks.py | 92 ++++++++++++++----- tests/test_evaluation_v2.py | 52 ++++++++--- tests/test_evaluation_v2_presets.py | 73 +++++++++++---- tests/test_leaderboard.py | 3 - 11 files changed, 255 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cf17652..bc3107f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.22.1](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.22.1) - 2026-08-31 + +### Deprecated +- **`allowed_label_matches` on Evaluation V2.** `create_evaluation_v2_preset()`, `update_evaluation_v2_preset()`, `create_benchmark_evaluation_v2()`, and `Benchmark.create_evaluation_v2()` still accept `allowed_label_matches` / `allowed_label_matches_id` for backwards compatibility, but they now emit a `DeprecationWarning`. Use `rollup_groups` instead. `AllowedLabelMatch` and the corresponding fields on `EvaluationV2` / `EvaluationV2Preset` are likewise marked deprecated. + ## [0.22.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.22.0) - 2026-08-26 ### Added @@ -13,10 +18,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `Model.predictions_loc(dataset_item_id)`, `Model.predictions_refloc(reference_id)`, `Model.predictions_iloc(i)` — model-scoped reads returning the same shape as their `Dataset` equivalents. - `Model.copy_predictions_from_run(model_run_id)` — synchronously backfills the run-free store from an existing model run, returning a dict `{model_id, model_run_ids, predictions_copied, predictions_skipped_unsupported}`. - **Model-anchored benchmark evaluations.** `NucleusClient.create_benchmark_evaluation_v2()` accepts a `model_id` (a `prj_*` id or a `Model`) as an alternative to `model_run_id`; the model-anchored flow evaluates the model's run-free predictions and ignores model runs. Provide exactly one of the two. `EvaluationV2` now exposes an optional `model_id` field alongside `model_run_id`. +- **`list_evaluations_v2` accepts a model.** `NucleusClient.list_evaluations_v2()` takes exactly one of `model_run_id` (`run_*`) or `model_id` (`prj_*` or a `Model`). The model-anchored path hits `GET model/{id}/evaluationsV2` and returns that model's run-free evaluations. ### Changed - The existing run-based prediction paths (`Dataset.upload_predictions`, `ModelRun.add_predictions`, `create_benchmark_evaluation_v2(model_run_id=...)`) are unchanged and continue to work; the model-centric methods are purely additive. +### Removed +- **`dataset_id` dropped from the EvaluationV2 surface** (breaking). An evaluation is no longer anchored on a single dataset — a model run now carries a *set* of datasets and a benchmark's items may span several — so the backend no longer returns a denormalized dataset on evaluations or leaderboards. Removed `EvaluationV2.dataset_id`, and `dataset_id` / `dataset_name` from `LeaderboardRankingEntry` and `LeaderboardF1CurveEntry`, matching the current backend responses. Without this, `EvaluationV2.from_json` raised `KeyError: 'dataset_id'` on every model-anchored (run-free) benchmark evaluation, since those payloads never carry a `dataset_id`. + ### Fixed - `Model.predictions_loc` / `predictions_refloc` / `predictions_iloc` now actually parse their responses. The run-free read endpoints return a flat `{"predictions": [...]}` list (each element carrying its own `"type"`), but `format_prediction_response` only understood the legacy type-keyed `{"annotations": {"box": [...]}}` shape, so these methods returned the raw payload unparsed instead of the documented `{"box": [...], "polygon": [...], "cuboid": [...]}` dict. diff --git a/nucleus/__init__.py b/nucleus/__init__.py index b9bab5b9..8bdd0ad7 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -212,6 +212,7 @@ EvaluationV2, EvaluationV2Status, RollupGroup, + _warn_allowed_label_matches_deprecated, ) from .evaluation_v2_exclusions import ( BoxAreaExclusionRule, @@ -558,13 +559,6 @@ def merge_model_runs( ) -> Dict[str, Any]: """Merge several model runs into one new run holding all their predictions. - A benchmark evaluation names a single model run, and a benchmark's items may - span several datasets. A model whose predictions were uploaded as separate runs - — one per dataset, or one per inference batch — therefore has no single run - covering the benchmark, and every uncovered item scores as a false negative. - Merging the runs produces one run that does cover it, which you can then pass to - :meth:`create_benchmark_evaluation_v2`. - All source runs must belong to the same model. The merge is a full union of all predictions. If two @@ -1035,16 +1029,40 @@ def get_evaluation_v2(self, evaluation_id: str) -> EvaluationV2: data = self.get(f"evaluationsV2/{evaluation_id}") return EvaluationV2.from_json(data, self) - def list_evaluations_v2(self, model_run_id: str) -> List[EvaluationV2]: - """List evaluations for a model run (newest first). + def list_evaluations_v2( + self, + model_run_id: Optional[str] = None, + *, + model_id: Optional[Union[str, Model]] = None, + ) -> List[EvaluationV2]: + """List evaluations for a model run or a run-free model (newest first). + + Provide exactly one of ``model_run_id`` or ``model_id``. The + model-anchored path lists run-free evaluations for that model + (``model_run_id`` is null on those rows). Parameters: - model_run_id: Model run id (``run_*``). + model_run_id: Model run id (``run_*``). Mutually exclusive with + ``model_id``. + model_id: Model id (``prj_*``) or :class:`Model` to list run-free + evaluations. Mutually exclusive with ``model_run_id``. Returns: List of :class:`EvaluationV2`. """ - rows = self.get(f"modelRun/{model_run_id}/evaluationsV2") + resolved_model_id = ( + model_id.id if isinstance(model_id, Model) else model_id + ) + if (resolved_model_id is None) == (model_run_id is None): + raise ValueError( + "Provide exactly one of model_run_id or model_id." + ) + route = ( + f"model/{resolved_model_id}/evaluationsV2" + if resolved_model_id is not None + else f"modelRun/{model_run_id}/evaluationsV2" + ) + rows = self.get(route) if not isinstance(rows, list): raise RuntimeError( f"Unexpected list evaluations V2 response: {rows!r}" @@ -1083,14 +1101,16 @@ def create_evaluation_v2_preset( configuration); each :class:`RollupGroup` maps raw labels onto one class name. Mutually exclusive with ``allowed_label_matches``. - allowed_label_matches: Optional legacy label pairs to treat as - matches. Prefer ``rollup_groups``. + allowed_label_matches: Deprecated. Use ``rollup_groups``. Optional + legacy label pairs to treat as matches. exclusion_rules: Optional rules that drop items/annotations (same types accepted by :meth:`create_benchmark_evaluation_v2`). Returns: :class:`EvaluationV2Preset`: The created preset. """ + if allowed_label_matches is not None: + _warn_allowed_label_matches_deprecated() if rollup_groups is not None and allowed_label_matches is not None: raise ValueError( "rollup_groups and allowed_label_matches cannot both be set" @@ -1132,12 +1152,15 @@ def update_evaluation_v2_preset( name: Optional new name. rollup_groups: Optional new rollup classes, or ``None`` to clear. Mutually exclusive with ``allowed_label_matches``. - allowed_label_matches: Optional new legacy label-match list. + allowed_label_matches: Deprecated. Use ``rollup_groups``. Optional + new legacy label-match list, or ``None`` to clear. exclusion_rules: Optional new exclusion rules, or ``None`` to clear. Returns: :class:`EvaluationV2Preset`: The updated preset. """ + if allowed_label_matches is not _UNSET: + _warn_allowed_label_matches_deprecated() if ( rollup_groups is not _UNSET and rollup_groups is not None @@ -1583,12 +1606,6 @@ def create_benchmark_evaluation_v2( background — call :meth:`EvaluationV2.wait_for_completion`, then :meth:`EvaluationV2.charts` or :meth:`EvaluationV2.examples`. - The benchmark may span datasets the model run has no predictions in at - all. Those members are scored as false negatives like any other - uncovered item, so a partial run still ranks comparably rather than - being rejected. To give a run predictions across several datasets, use - :meth:`Dataset.upload_predictions_for_model_run`. - The evaluation can be anchored on either a legacy model run (``model_run_id``) or, for the run-free "model v2" flow, a model (``model_id``). Provide exactly one; the model-anchored flow evaluates @@ -1607,10 +1624,10 @@ def create_benchmark_evaluation_v2( configuration); each :class:`RollupGroup` maps raw labels onto one class name. Mutually exclusive with the ``allowed_label_matches*`` arguments. - allowed_label_matches: Optional legacy label pairs to treat as - matches. Prefer ``rollup_groups``. - allowed_label_matches_id: Optional id of a saved label-match - configuration. + allowed_label_matches: Deprecated. Use ``rollup_groups``. Optional + legacy label pairs to treat as matches. + allowed_label_matches_id: Deprecated. Use ``rollup_groups``. + Optional id of a saved label-match configuration. exclusion_rules: Optional rules that drop items/annotations before metrics are computed (see :mod:`nucleus.evaluation_v2_exclusions`). @@ -1621,6 +1638,11 @@ def create_benchmark_evaluation_v2( Returns: :class:`EvaluationV2`: The created evaluation. """ + if ( + allowed_label_matches is not None + or allowed_label_matches_id is not None + ): + _warn_allowed_label_matches_deprecated() resolved_model_id = ( model_id.id if isinstance(model_id, Model) else model_id ) diff --git a/nucleus/benchmark.py b/nucleus/benchmark.py index 1028078b..f54fceb7 100644 --- a/nucleus/benchmark.py +++ b/nucleus/benchmark.py @@ -200,7 +200,8 @@ def create_evaluation_v2( :meth:`Dataset.upload_predictions_for_model_run`. See :meth:`NucleusClient.create_benchmark_evaluation_v2` for parameter - details. + details. ``allowed_label_matches`` / ``allowed_label_matches_id`` are + deprecated; use ``rollup_groups``. Returns: :class:`~nucleus.evaluation_v2.EvaluationV2`: The created evaluation. diff --git a/nucleus/data_transfer_object/evaluation_v2.py b/nucleus/data_transfer_object/evaluation_v2.py index 91e9dbdf..a6c11c53 100644 --- a/nucleus/data_transfer_object/evaluation_v2.py +++ b/nucleus/data_transfer_object/evaluation_v2.py @@ -197,8 +197,6 @@ class LeaderboardRankingEntry(DictCompatibleModel): model_version_minor: Optional[int] = None model_version_label: Optional[str] = None parent_model_project_id: Optional[str] = None - dataset_id: Optional[str] = None - dataset_name: Optional[str] = None score: float rank: int @@ -217,8 +215,6 @@ class LeaderboardF1CurveEntry(DictCompatibleModel): model_run_name: Optional[str] = None model_id: Optional[str] = None model_name: Optional[str] = None - dataset_id: Optional[str] = None - dataset_name: Optional[str] = None best_f1: Optional[float] = None points: List[LeaderboardF1CurvePoint] rank: int diff --git a/nucleus/evaluation_v2.py b/nucleus/evaluation_v2.py index a70e8bd3..f6c5acdd 100644 --- a/nucleus/evaluation_v2.py +++ b/nucleus/evaluation_v2.py @@ -4,6 +4,7 @@ import json import time +import warnings from dataclasses import dataclass, field from enum import Enum from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union @@ -18,7 +19,6 @@ CLASS_NAME_CAMEL_KEY, CLASS_NAME_KEY, CREATED_AT_KEY, - DATASET_ID_KEY, ERROR_MESSAGE_KEY, EVALUATION_ID_KEY, EXCLUSION_RULES_KEY, @@ -71,6 +71,24 @@ class EvaluationV2Status(str, Enum): EvaluationV2Status.CANCELLED, } +_ALLOWED_LABEL_MATCHES_DEPRECATION = ( + "allowed_label_matches is deprecated and will be removed in a future " + "release. Use rollup_groups instead." +) + + +def _warn_allowed_label_matches_deprecated() -> None: + """Emit the Evaluation V2 ``allowed_label_matches`` deprecation warning. + + ``stacklevel=3`` points at the public method's caller (this helper → + the client/wrapper method → user code). + """ + warnings.warn( + _ALLOWED_LABEL_MATCHES_DEPRECATION, + DeprecationWarning, + stacklevel=3, + ) + def _parse_json_field(value: Any) -> Optional[Any]: """Normalize a field that may arrive already decoded or as a JSON string.""" @@ -86,7 +104,10 @@ def _parse_json_field(value: Any) -> Optional[Any]: @dataclass class AllowedLabelMatch: - """Ground-truth and prediction label pair that counts as a match.""" + """Deprecated. Use :class:`RollupGroup` instead. + + Ground-truth and prediction label pair that counts as a match. + """ ground_truth_label: str model_prediction_label: str @@ -175,15 +196,17 @@ class EvaluationV2: id: str model_run_id: Optional[str] - dataset_id: str status: str model_id: Optional[str] = None name: Optional[str] = None temporal_workflow_id: Optional[str] = None error_message: Optional[str] = None created_at: Optional[str] = None + #: Deprecated. Prefer :attr:`rollup_groups`. allowed_label_matches_id: Optional[str] = None + #: Deprecated. Prefer :attr:`rollup_groups`. allowed_label_matches: Optional[List[AllowedLabelMatch]] = None + #: Deprecated. Prefer :attr:`rollup_groups`. allowed_label_matches_name: Optional[str] = None rollup_groups: Optional[List[RollupGroup]] = None benchmark_id: Optional[str] = None @@ -209,7 +232,6 @@ def from_json( if payload.get(MODEL_RUN_ID_KEY) is not None else None ), - dataset_id=str(payload[DATASET_ID_KEY]), status=str(payload[STATUS_KEY]), model_id=( str(payload[MODEL_ID_KEY]) @@ -320,7 +342,7 @@ def retry(self) -> "EvaluationV2": """Retry this evaluation if it failed. Creates a new evaluation for the same model run, reusing this - evaluation's slice, allowed-label-matches, and exclusion rules. Only + evaluation's slice, rollup groups, and exclusion rules. Only ``failed`` evaluations can be retried. Returns: diff --git a/nucleus/evaluation_v2_preset.py b/nucleus/evaluation_v2_preset.py index 3821e0be..c56c699d 100644 --- a/nucleus/evaluation_v2_preset.py +++ b/nucleus/evaluation_v2_preset.py @@ -1,9 +1,11 @@ """Evaluation V2 presets — saved, reusable evaluation configurations. -A preset bundles a ``name`` with a label configuration (``rollup_groups``, -or legacy ``allowed_label_matches``) and ``exclusion_rules`` so the same -configuration can be applied across many evaluations. Presets are private to -the creating user. +A preset bundles a ``name`` with a label configuration (``rollup_groups``) +and ``exclusion_rules`` so the same configuration can be applied across many +evaluations. Presets are private to the creating user. + +``allowed_label_matches`` is still accepted for backwards compatibility but +is deprecated; use ``rollup_groups``. Create and manage presets via :class:`~nucleus.NucleusClient`:: @@ -63,6 +65,7 @@ class EvaluationV2Preset: id: str name: str rollup_groups: Optional[List[RollupGroup]] = None + #: Deprecated. Prefer :attr:`rollup_groups`. allowed_label_matches: Optional[List[AllowedLabelMatch]] = None exclusion_rules: Optional[List[Dict[str, Any]]] = None created_by_user_id: Optional[str] = None @@ -117,6 +120,8 @@ def update( ``rollup_groups=None`` / ``exclusion_rules=None`` clears that field; omitting an argument leaves it unchanged. + ``allowed_label_matches`` is deprecated; use ``rollup_groups``. + Returns: self, with updated fields. """ diff --git a/pyproject.toml b/pyproject.toml index 5b47c0b8..a495dd8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.22.0" +version = "0.22.1" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index d380ded7..096afd66 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -84,7 +84,9 @@ def _mock_async_create(client, *, benchmark_row=None): client.connection.post = MagicMock( return_value={"benchmark_id": "bm_1", "job_id": "job_1"} ) - client.get_job = MagicMock() # .sleep_until_complete() is a no-op MagicMock + client.get_job = ( + MagicMock() + ) # .sleep_until_complete() is a no-op MagicMock client.get_benchmark = MagicMock( return_value=Benchmark.from_json( benchmark_row or {**_BENCHMARK_ROW, "status": "ready"}, client @@ -128,7 +130,11 @@ def test_create_benchmark_from_item_ids_and_metadata(): def test_create_benchmark_no_wait_returns_building_without_polling(): client = _mock_async_create( NucleusClient(api_key="test"), - benchmark_row={**_BENCHMARK_ROW, "status": "building", "item_count": 0}, + benchmark_row={ + **_BENCHMARK_ROW, + "status": "building", + "item_count": 0, + }, ) benchmark = client.create_benchmark( "city-streets", slice_id="slc_1", wait_for_completion=False @@ -272,9 +278,7 @@ def test_add_benchmark_items_posts_sources_and_polls(): client = NucleusClient(api_key="test") client.connection.post = MagicMock(return_value={"job_id": "job_add"}) client.get_job = MagicMock() - client.add_benchmark_items( - "bm_1", item_ids=["di_1"], slice_ids=["slc_1"] - ) + client.add_benchmark_items("bm_1", item_ids=["di_1"], slice_ids=["slc_1"]) payload, route = client.connection.post.call_args[0] assert route == "benchmarks/bm_1/items" assert payload["item_ids"] == ["di_1"] @@ -317,9 +321,7 @@ def test_benchmark_finalize_updates_self_in_place(): client.connection.post = MagicMock( return_value={**_BENCHMARK_ROW, "status": "ready"} ) - draft = Benchmark.from_json( - {**_BENCHMARK_ROW, "status": "draft"}, client - ) + draft = Benchmark.from_json({**_BENCHMARK_ROW, "status": "draft"}, client) result = draft.finalize() assert result is draft assert draft.status == "ready" @@ -425,20 +427,22 @@ def test_create_benchmark_evaluation_v2_with_rollup_groups(): def test_create_benchmark_evaluation_v2_label_config_mutual_exclusion(): client = NucleusClient(api_key="test") - with pytest.raises(ValueError, match="at most one"): - client.create_benchmark_evaluation_v2( - "bm_1", - "run_1", - rollup_groups=[RollupGroup("vehicle", ["car"])], - allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], - ) - with pytest.raises(ValueError, match="at most one"): - client.create_benchmark_evaluation_v2( - "bm_1", - "run_1", - rollup_groups=[RollupGroup("vehicle", ["car"])], - allowed_label_matches_id="alm_1", - ) + with pytest.warns(DeprecationWarning, match="allowed_label_matches"): + with pytest.raises(ValueError, match="at most one"): + client.create_benchmark_evaluation_v2( + "bm_1", + "run_1", + rollup_groups=[RollupGroup("vehicle", ["car"])], + allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], + ) + with pytest.warns(DeprecationWarning, match="allowed_label_matches"): + with pytest.raises(ValueError, match="at most one"): + client.create_benchmark_evaluation_v2( + "bm_1", + "run_1", + rollup_groups=[RollupGroup("vehicle", ["car"])], + allowed_label_matches_id="alm_1", + ) def test_create_benchmark_evaluation_v2_preset_seeds_rollup_groups(): @@ -467,6 +471,8 @@ def test_create_benchmark_evaluation_v2_preset_seeds_legacy_matches(): name="p", allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], ) + # Seeding from a stored preset does not itself warn — the deprecated + # kwargs were not passed by the caller. client.create_benchmark_evaluation_v2("bm_1", "run_1", preset=preset) payload = client.connection.post.call_args[0][0] assert payload["allowed_label_matches"] == [ @@ -475,6 +481,48 @@ def test_create_benchmark_evaluation_v2_preset_seeds_legacy_matches(): assert "rollupGroups" not in payload +def test_create_benchmark_evaluation_v2_allowed_label_matches_deprecated(): + client = NucleusClient(api_key="test") + _mock_create_eval(client) + with pytest.warns(DeprecationWarning, match="allowed_label_matches"): + client.create_benchmark_evaluation_v2( + "bm_1", + "run_1", + allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], + ) + payload = client.connection.post.call_args[0][0] + assert payload["allowed_label_matches"] == [ + {"ground_truth_label": "car", "model_prediction_label": "vehicle"} + ] + + +def test_create_benchmark_evaluation_v2_allowed_label_matches_id_deprecated(): + client = NucleusClient(api_key="test") + _mock_create_eval(client) + with pytest.warns(DeprecationWarning, match="allowed_label_matches"): + client.create_benchmark_evaluation_v2( + "bm_1", + "run_1", + allowed_label_matches_id="alm_1", + ) + payload = client.connection.post.call_args[0][0] + assert payload["allowed_label_matches_id"] == "alm_1" + + +def test_create_benchmark_evaluation_v2_rollup_groups_does_not_warn(): + import warnings + + client = NucleusClient(api_key="test") + _mock_create_eval(client) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + client.create_benchmark_evaluation_v2( + "bm_1", + "run_1", + rollup_groups=[RollupGroup("vehicle", ["car"])], + ) + + def test_create_benchmark_evaluation_v2_explicit_args_override_preset(): client = NucleusClient(api_key="test") _mock_create_eval(client) diff --git a/tests/test_evaluation_v2.py b/tests/test_evaluation_v2.py index 0b5a3f5e..3ce3f2f2 100644 --- a/tests/test_evaluation_v2.py +++ b/tests/test_evaluation_v2.py @@ -11,6 +11,7 @@ EvaluationV2, LabelExclusionRule, MetadataExclusionRule, + Model, NucleusClient, ) from nucleus.data_transfer_object.evaluation_v2 import ( @@ -85,7 +86,6 @@ def test_evaluation_v2_from_json_with_matches(): payload = { "id": "evalv2_1", "model_run_id": "run_1", - "dataset_id": "ds_1", "status": "pending", "allowed_label_matches": [ {"groundTruthLabel": "x", "modelPredictionLabel": "y"}, @@ -115,7 +115,6 @@ def test_list_evaluations_v2_returns_rows(): { "id": "evalv2_1", "model_run_id": "run_1", - "dataset_id": "ds_1", "status": "succeeded", }, ] @@ -133,6 +132,43 @@ def test_list_evaluations_v2_invalid_response(): client.list_evaluations_v2("run_1") +def test_list_evaluations_v2_by_model_id(): + client = NucleusClient(api_key="test") + client.connection.get = MagicMock( + return_value=[ + { + "id": "evalv2_1", + "model_run_id": None, + "model_id": "prj_1", + "dataset_id": "ds_1", + "status": "succeeded", + }, + ] + ) + result = client.list_evaluations_v2(model_id="prj_1") + assert len(result) == 1 + assert result[0].id == "evalv2_1" + assert result[0].model_id == "prj_1" + client.connection.get.assert_called_once_with("model/prj_1/evaluationsV2") + + +def test_list_evaluations_v2_by_model_object(): + client = NucleusClient(api_key="test") + client.connection.get = MagicMock(return_value=[]) + model = Model("prj_1", "My CNN", "My-CNN", {}, client) + result = client.list_evaluations_v2(model_id=model) + assert result == [] + client.connection.get.assert_called_once_with("model/prj_1/evaluationsV2") + + +def test_list_evaluations_v2_requires_exactly_one_id(): + client = NucleusClient(api_key="test") + with pytest.raises(ValueError, match="exactly one"): + client.list_evaluations_v2() + with pytest.raises(ValueError, match="exactly one"): + client.list_evaluations_v2("run_1", model_id="prj_1") + + def test_evaluation_v2_filter_args_gt_area_and_slices(): filters = EvaluationV2FilterArgs( gt_area_range=RangeNum(min=1024, max=9216), @@ -150,7 +186,6 @@ def test_evaluation_v2_from_json_slice_and_exclusions(): { "id": "evalv2_1", "model_run_id": "run_1", - "dataset_id": "ds_1", "status": "succeeded", "slice_id": "slc_x", "exclusion_rules": '[{"type":"labels","scope":"item","target":"prediction","labels":["ignore"]}]', @@ -174,7 +209,6 @@ def test_evaluation_v2_from_json_exclusions_absent(): { "id": "evalv2_1", "model_run_id": "run_1", - "dataset_id": "ds_1", "status": "succeeded", } ) @@ -189,7 +223,6 @@ def test_charts_post_body(): ev = EvaluationV2( id="evalv2_1", model_run_id="run_1", - dataset_id="ds_1", status="succeeded", _client=client, ) @@ -207,7 +240,6 @@ def test_charts_with_filter_args(): ev = EvaluationV2( id="evalv2_1", model_run_id="run_1", - dataset_id="ds_1", status="succeeded", _client=client, ) @@ -232,7 +264,6 @@ def test_examples_post_body(): ev = EvaluationV2( id="evalv2_1", model_run_id="run_1", - dataset_id="ds_1", status="succeeded", _client=client, ) @@ -253,7 +284,6 @@ def test_examples_with_filter_args(): ev = EvaluationV2( id="evalv2_1", model_run_id="run_1", - dataset_id="ds_1", status="succeeded", _client=client, ) @@ -278,13 +308,11 @@ def test_wait_for_completion(): { "id": "evalv2_1", "model_run_id": "run_1", - "dataset_id": "ds_1", "status": "pending", }, { "id": "evalv2_1", "model_run_id": "run_1", - "dataset_id": "ds_1", "status": "succeeded", }, ] @@ -292,7 +320,6 @@ def test_wait_for_completion(): ev = EvaluationV2( id="evalv2_1", model_run_id="run_1", - dataset_id="ds_1", status="pending", _client=client, ) @@ -309,7 +336,6 @@ def test_delete_success(status_code): ev = EvaluationV2( id="evalv2_1", model_run_id="run_1", - dataset_id="ds_1", status="succeeded", _client=client, ) @@ -360,7 +386,6 @@ def test_evaluation_v2_from_json_benchmark_id_and_rollup_groups(): { "id": "evalv2_1", "model_run_id": "run_1", - "dataset_id": "ds_1", "status": "pending", "benchmark_id": "bm_1", "rollup_groups": [{"className": "vehicle", "labels": ["car"]}], @@ -376,7 +401,6 @@ def test_evaluation_v2_from_json_benchmark_fields_absent(): { "id": "evalv2_1", "model_run_id": "run_1", - "dataset_id": "ds_1", "status": "pending", } ) diff --git a/tests/test_evaluation_v2_presets.py b/tests/test_evaluation_v2_presets.py index 09927927..cd55f0f7 100644 --- a/tests/test_evaluation_v2_presets.py +++ b/tests/test_evaluation_v2_presets.py @@ -1,8 +1,10 @@ """Unit tests for Evaluation V2 presets, cancel/retry, and label-schema discovery (no live API).""" +import warnings from unittest.mock import MagicMock +import pytest import requests from nucleus import ( @@ -56,15 +58,16 @@ def test_create_evaluation_v2_preset_payload(): "exclusion_rules": None, } ) - preset = client.create_evaluation_v2_preset( - "vehicles", - allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], - exclusion_rules=[ - LabelExclusionRule( - scope="item", target="prediction", labels=["ignore"] - ) - ], - ) + with pytest.warns(DeprecationWarning, match="allowed_label_matches"): + preset = client.create_evaluation_v2_preset( + "vehicles", + allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], + exclusion_rules=[ + LabelExclusionRule( + scope="item", target="prediction", labels=["ignore"] + ) + ], + ) payload, route = client.connection.post.call_args[0] assert route == "evaluationV2Presets" assert payload["name"] == "vehicles" @@ -135,7 +138,6 @@ def _eval(client, status="computing"): return EvaluationV2( id="evalv2_1", model_run_id="run_1", - dataset_id="ds_1", status=status, _client=client, ) @@ -163,7 +165,6 @@ def test_evaluation_retry_resolves_new_evaluation(): client.get_evaluation_v2.return_value = EvaluationV2( id="evalv2_retry", model_run_id="run_1", - dataset_id="ds_1", status="pending", _client=client, ) @@ -237,15 +238,16 @@ def test_create_evaluation_v2_preset_rollup_and_matches_mutually_exclusive(): from nucleus import RollupGroup client = NucleusClient(api_key="test") - try: - client.create_evaluation_v2_preset( - "p", - rollup_groups=[RollupGroup("vehicle", ["car"])], - allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], - ) - raise AssertionError("expected ValueError") - except ValueError as e: - assert "cannot both be set" in str(e) + with pytest.warns(DeprecationWarning, match="allowed_label_matches"): + try: + client.create_evaluation_v2_preset( + "p", + rollup_groups=[RollupGroup("vehicle", ["car"])], + allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], + ) + raise AssertionError("expected ValueError") + except ValueError as e: + assert "cannot both be set" in str(e) def test_update_evaluation_v2_preset_rollup_groups_and_clear(): @@ -280,3 +282,34 @@ def test_preset_from_json_parses_rollup_groups_both_casings(): ) assert preset.rollup_groups is not None assert preset.rollup_groups[0].labels == ["car", "truck"] + + +def test_create_evaluation_v2_preset_rollup_groups_does_not_warn(): + from nucleus import RollupGroup + + client = NucleusClient(api_key="test") + client.connection.post = MagicMock( + return_value={"id": "prev_1", "name": "vehicles", "rollup_groups": []} + ) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + client.create_evaluation_v2_preset( + "vehicles", + rollup_groups=[RollupGroup("vehicle", ["car"])], + ) + + +def test_update_evaluation_v2_preset_allowed_label_matches_warns(): + client = NucleusClient(api_key="test") + client.connection.patch = MagicMock( + return_value={"id": "prev_1", "name": "p"} + ) + with pytest.warns(DeprecationWarning, match="allowed_label_matches"): + client.update_evaluation_v2_preset( + "prev_1", + allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], + ) + payload = client.connection.patch.call_args[0][0] + assert payload["allowedLabelMatches"] == [ + {"ground_truth_label": "car", "model_prediction_label": "vehicle"} + ] diff --git a/tests/test_leaderboard.py b/tests/test_leaderboard.py index 538e6255..5a8b404a 100644 --- a/tests/test_leaderboard.py +++ b/tests/test_leaderboard.py @@ -29,8 +29,6 @@ "model_version_minor": 2, "model_version_label": "1.2", "parent_model_project_id": None, - "dataset_id": "ds_1", - "dataset_name": "dataset", "score": 0.42, "rank": 1, } @@ -127,7 +125,6 @@ def test_evaluation_filter_schema_instance_method(): evaluation = EvaluationV2( id="evalv2_1", model_run_id="run_1", - dataset_id="ds_1", status="succeeded", _client=client, ) From 71409598b0b231a9c0ca633bf66877c159f51242 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Mon, 31 Aug 2026 14:43:18 -0500 Subject: [PATCH 6/7] deprecate model runs --- CHANGELOG.md | 3 + nucleus/__init__.py | 56 ++++++++++--------- nucleus/data_transfer_object/evaluation_v2.py | 8 ++- nucleus/evaluation_v2.py | 32 ++++++++++- tests/test_benchmarks.py | 21 ++++++- tests/test_evaluation_v2.py | 21 ++++++- tests/test_leaderboard.py | 18 ++++++ 7 files changed, 128 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc3107f2..c3d0be42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - The existing run-based prediction paths (`Dataset.upload_predictions`, `ModelRun.add_predictions`, `create_benchmark_evaluation_v2(model_run_id=...)`) are unchanged and continue to work; the model-centric methods are purely additive. +### Deprecated +- **Model-run-anchored Evaluation V2 is deprecated** in favor of the run-free (`model_id`) path. Passing `model_run_id` to `create_benchmark_evaluation_v2()` or `list_evaluations_v2()` now emits a `DeprecationWarning`; both keep working. `EvaluationV2.model_run_id` is documented as deprecated (it is `None` on run-free evaluations). On the leaderboard, `LeaderboardRankingEntry` / `LeaderboardF1CurveEntry` `model_run_id` and `model_run_name` are deprecated and now `Optional` (they are `None` for run-free evaluations — previously `model_run_id` was a required field and would fail to parse), and `collapse="allRuns"` on `leaderboard_ranking()` is discouraged. Prefer anchoring on and identifying evaluations by `model_id`. + ### Removed - **`dataset_id` dropped from the EvaluationV2 surface** (breaking). An evaluation is no longer anchored on a single dataset — a model run now carries a *set* of datasets and a benchmark's items may span several — so the backend no longer returns a denormalized dataset on evaluations or leaderboards. Removed `EvaluationV2.dataset_id`, and `dataset_id` / `dataset_name` from `LeaderboardRankingEntry` and `LeaderboardF1CurveEntry`, matching the current backend responses. Without this, `EvaluationV2.from_json` raised `KeyError: 'dataset_id'` on every model-anchored (run-free) benchmark evaluation, since those payloads never carry a `dataset_id`. diff --git a/nucleus/__init__.py b/nucleus/__init__.py index 8bdd0ad7..8f44b8b7 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -213,6 +213,7 @@ EvaluationV2Status, RollupGroup, _warn_allowed_label_matches_deprecated, + _warn_model_run_deprecated, ) from .evaluation_v2_exclusions import ( BoxAreaExclusionRule, @@ -1035,21 +1036,23 @@ def list_evaluations_v2( *, model_id: Optional[Union[str, Model]] = None, ) -> List[EvaluationV2]: - """List evaluations for a model run or a run-free model (newest first). + """List evaluations for a run-free model (newest first). - Provide exactly one of ``model_run_id`` or ``model_id``. The - model-anchored path lists run-free evaluations for that model - (``model_run_id`` is null on those rows). + Provide exactly one of ``model_id`` or ``model_run_id``. The + model-anchored path lists run-free evaluations for that model; the + ``model_run_id`` path is deprecated. Parameters: - model_run_id: Model run id (``run_*``). Mutually exclusive with - ``model_id``. + model_run_id: Deprecated. Legacy model run id (``run_*``); prefer + ``model_id``. Mutually exclusive with ``model_id``. model_id: Model id (``prj_*``) or :class:`Model` to list run-free evaluations. Mutually exclusive with ``model_run_id``. Returns: List of :class:`EvaluationV2`. """ + if model_run_id is not None: + _warn_model_run_deprecated() resolved_model_id = ( model_id.id if isinstance(model_id, Model) else model_id ) @@ -1598,27 +1601,27 @@ def create_benchmark_evaluation_v2( ] = None, preset: Optional[EvaluationV2Preset] = None, ) -> EvaluationV2: - """Evaluate a model run against a benchmark. + """Evaluate a model against a benchmark. - Every benchmark item is scored: items the model run has no - predictions for count as false negatives, keeping scores comparable - across runs with different coverage. The evaluation runs in the - background — call :meth:`EvaluationV2.wait_for_completion`, then + Every benchmark item is scored: items the model has no predictions for + count as false negatives, keeping scores comparable across models with + different coverage. The evaluation runs in the background — call + :meth:`EvaluationV2.wait_for_completion`, then :meth:`EvaluationV2.charts` or :meth:`EvaluationV2.examples`. - The evaluation can be anchored on either a legacy model run - (``model_run_id``) or, for the run-free "model v2" flow, a model - (``model_id``). Provide exactly one; the model-anchored flow evaluates - the model's run-free predictions and ignores model runs entirely. + Anchor the evaluation on a model (``model_id``) for the run-free + "model v2" flow, which evaluates the model's run-free predictions. + The legacy ``model_run_id`` anchor is deprecated. Provide exactly one. Parameters: benchmark_id: Benchmark id (``bm_*``). - model_run_id: Model run id (``run_*``). It need not cover the - benchmark's datasets — coverage may be partial, or empty. - Mutually exclusive with ``model_id``. model_id: Model id (``prj_*``) or :class:`Model` to anchor the - evaluation on the model's run-free predictions instead of a - model run. Mutually exclusive with ``model_run_id``. + evaluation on the model's run-free predictions. Mutually + exclusive with ``model_run_id``. + model_run_id: Deprecated. Legacy model run id (``run_*``); prefer + ``model_id``. It need not cover the benchmark's datasets — + coverage may be partial, or empty. Mutually exclusive with + ``model_id``. name: Optional display name. rollup_groups: Optional rollup classes (the primary label configuration); each :class:`RollupGroup` maps raw labels @@ -1643,6 +1646,8 @@ def create_benchmark_evaluation_v2( or allowed_label_matches_id is not None ): _warn_allowed_label_matches_deprecated() + if model_run_id is not None: + _warn_model_run_deprecated() resolved_model_id = ( model_id.id if isinstance(model_id, Model) else model_id ) @@ -1715,7 +1720,7 @@ def leaderboard_ranking( scope: Optional[str] = None, collapse: Optional[str] = None, ) -> List[LeaderboardRankingEntry]: - """Rank model runs on one or more benchmarks by a metric. + """Rank models on one or more benchmarks by a metric. Parameters: metric_type: Metric to rank by — one of ``"MAP_50"``, @@ -1727,8 +1732,8 @@ def leaderboard_ranking( model_ids: Optional model ids to restrict the ranking to. scope: ``"mine"`` (only the caller's evaluations) or ``"all"`` (default). - collapse: ``"bestPerModel"`` (default), ``"allRuns"``, or - ``"allEvaluations"``. + collapse: ``"bestPerModel"`` (default) or ``"allEvaluations"``. + ``"allRuns"`` is deprecated (model runs are being phased out). Returns: List of :class:`LeaderboardRankingEntry`, best score first. @@ -1759,12 +1764,13 @@ def leaderboard_f1_curve( model_ids: Optional[List[str]] = None, top_n: int = 5, ) -> List[LeaderboardF1CurveEntry]: - """Return F1-vs-confidence curves for the top runs on benchmarks. + """Return F1-vs-confidence curves for the top models on benchmarks. Parameters: benchmark_ids: Benchmark ids (``bm_*``). model_ids: Optional model ids to restrict the curves to. - top_n: Number of top-ranked runs to return curves for (default 5). + top_n: Number of top-ranked models to return curves for + (default 5). Returns: List of :class:`LeaderboardF1CurveEntry`, best F1 first. diff --git a/nucleus/data_transfer_object/evaluation_v2.py b/nucleus/data_transfer_object/evaluation_v2.py index a6c11c53..4e876588 100644 --- a/nucleus/data_transfer_object/evaluation_v2.py +++ b/nucleus/data_transfer_object/evaluation_v2.py @@ -189,7 +189,9 @@ class LeaderboardRankingEntry(DictCompatibleModel): evaluation_id: str evaluation_name: Optional[str] = None - model_run_id: str + #: Deprecated. ``None`` for run-free evaluations; prefer :attr:`model_id`. + model_run_id: Optional[str] = None + #: Deprecated. ``None`` for run-free evaluations; prefer :attr:`model_name`. model_run_name: Optional[str] = None model_id: Optional[str] = None model_name: Optional[str] = None @@ -211,7 +213,9 @@ class LeaderboardF1CurveEntry(DictCompatibleModel): evaluation_id: str evaluation_name: Optional[str] = None - model_run_id: str + #: Deprecated. ``None`` for run-free evaluations; prefer :attr:`model_id`. + model_run_id: Optional[str] = None + #: Deprecated. ``None`` for run-free evaluations; prefer :attr:`model_name`. model_run_name: Optional[str] = None model_id: Optional[str] = None model_name: Optional[str] = None diff --git a/nucleus/evaluation_v2.py b/nucleus/evaluation_v2.py index f6c5acdd..7b75c278 100644 --- a/nucleus/evaluation_v2.py +++ b/nucleus/evaluation_v2.py @@ -90,6 +90,27 @@ def _warn_allowed_label_matches_deprecated() -> None: ) +_MODEL_RUN_DEPRECATION = ( + "Model-run-anchored Evaluation V2 is deprecated and will be removed in a " + "future release. Upload run-free predictions to a Model " + "(Model.upload_predictions / Model.copy_predictions_from_run) and pass " + "model_id instead of model_run_id." +) + + +def _warn_model_run_deprecated() -> None: + """Emit the Evaluation V2 model-run deprecation warning. + + ``stacklevel=3`` points at the public method's caller (this helper → + the client/wrapper method → user code). + """ + warnings.warn( + _MODEL_RUN_DEPRECATION, + DeprecationWarning, + stacklevel=3, + ) + + def _parse_json_field(value: Any) -> Optional[Any]: """Normalize a field that may arrive already decoded or as a JSON string.""" if value is None or isinstance(value, (dict, list)): @@ -192,9 +213,15 @@ def _parse_rollup_groups(raw_groups: Any) -> Optional[List[RollupGroup]]: @dataclass class EvaluationV2: - """An Evaluation V2 run for a model run or a run-free model.""" + """An Evaluation V2 run for a run-free model. + + The model-run-anchored flow is deprecated; new evaluations should be + anchored on a :class:`~nucleus.model.Model` via ``model_id``. + """ id: str + #: Deprecated. Model-run-anchored evaluations are being phased out; this is + #: ``None`` for run-free evaluations. Prefer :attr:`model_id`. model_run_id: Optional[str] status: str model_id: Optional[str] = None @@ -341,7 +368,8 @@ def cancel(self) -> "EvaluationV2": def retry(self) -> "EvaluationV2": """Retry this evaluation if it failed. - Creates a new evaluation for the same model run, reusing this + Creates a new evaluation for the same anchor (the model for a run-free + evaluation, or the model run for a legacy one), reusing this evaluation's slice, rollup groups, and exclusion rules. Only ``failed`` evaluations can be retried. diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 096afd66..522c115b 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -518,11 +518,30 @@ def test_create_benchmark_evaluation_v2_rollup_groups_does_not_warn(): warnings.simplefilter("error", DeprecationWarning) client.create_benchmark_evaluation_v2( "bm_1", - "run_1", + model_id="prj_1", rollup_groups=[RollupGroup("vehicle", ["car"])], ) +def test_create_benchmark_evaluation_v2_model_run_id_deprecated(): + client = NucleusClient(api_key="test") + _mock_create_eval(client) + with pytest.warns(DeprecationWarning, match="model_run_id"): + client.create_benchmark_evaluation_v2("bm_1", "run_1") + payload = client.connection.post.call_args[0][0] + assert payload["model_run_id"] == "run_1" + + +def test_create_benchmark_evaluation_v2_model_id_does_not_warn(): + import warnings + + client = NucleusClient(api_key="test") + _mock_create_eval(client) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + client.create_benchmark_evaluation_v2("bm_1", model_id="prj_1") + + def test_create_benchmark_evaluation_v2_explicit_args_override_preset(): client = NucleusClient(api_key="test") _mock_create_eval(client) diff --git a/tests/test_evaluation_v2.py b/tests/test_evaluation_v2.py index 3ce3f2f2..75cdfa84 100644 --- a/tests/test_evaluation_v2.py +++ b/tests/test_evaluation_v2.py @@ -140,7 +140,6 @@ def test_list_evaluations_v2_by_model_id(): "id": "evalv2_1", "model_run_id": None, "model_id": "prj_1", - "dataset_id": "ds_1", "status": "succeeded", }, ] @@ -169,6 +168,26 @@ def test_list_evaluations_v2_requires_exactly_one_id(): client.list_evaluations_v2("run_1", model_id="prj_1") +def test_list_evaluations_v2_by_model_run_id_deprecated(): + client = NucleusClient(api_key="test") + client.connection.get = MagicMock(return_value=[]) + with pytest.warns(DeprecationWarning, match="model_run_id"): + client.list_evaluations_v2("run_1") + client.connection.get.assert_called_once_with( + "modelRun/run_1/evaluationsV2" + ) + + +def test_list_evaluations_v2_by_model_id_does_not_warn(): + import warnings + + client = NucleusClient(api_key="test") + client.connection.get = MagicMock(return_value=[]) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + client.list_evaluations_v2(model_id="prj_1") + + def test_evaluation_v2_filter_args_gt_area_and_slices(): filters = EvaluationV2FilterArgs( gt_area_range=RangeNum(min=1024, max=9216), diff --git a/tests/test_leaderboard.py b/tests/test_leaderboard.py index 5a8b404a..aff16aeb 100644 --- a/tests/test_leaderboard.py +++ b/tests/test_leaderboard.py @@ -80,6 +80,24 @@ def test_leaderboard_ranking_payload_and_parsing(): assert rows[0].rank == 1 +def test_leaderboard_ranking_parses_run_free_row(): + # Run-free evaluations have no model run: model_run_id / model_run_name + # arrive absent (or null) and must parse to None, not raise. + client = NucleusClient(api_key="test") + run_free_row = { + "evaluation_id": "evalv2_1", + "model_id": "prj_1", + "model_name": "model", + "score": 0.5, + "rank": 1, + } + client.connection.post = MagicMock(return_value=[run_free_row]) + rows = client.leaderboard_ranking("MAP_50", ["bm_1"]) + assert rows[0].model_run_id is None + assert rows[0].model_run_name is None + assert rows[0].model_id == "prj_1" + + def test_leaderboard_ranking_minimal_payload(): client = NucleusClient(api_key="test") client.connection.post = MagicMock(return_value=[]) From d56530b6b61ddec6e4c79c0df469ee0318fb6eae Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Mon, 31 Aug 2026 15:34:36 -0500 Subject: [PATCH 7/7] depracte allowed label matches --- CHANGELOG.md | 3 +- nucleus/__init__.py | 134 ++++++++-------------------- nucleus/benchmark.py | 25 +++--- nucleus/evaluation_v2.py | 87 ------------------ nucleus/evaluation_v2_preset.py | 17 ---- tests/test_benchmarks.py | 85 ++++-------------- tests/test_evaluation_v2.py | 19 ++-- tests/test_evaluation_v2_presets.py | 70 ++++----------- 8 files changed, 86 insertions(+), 354 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3d0be42..52574631 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,9 +24,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The existing run-based prediction paths (`Dataset.upload_predictions`, `ModelRun.add_predictions`, `create_benchmark_evaluation_v2(model_run_id=...)`) are unchanged and continue to work; the model-centric methods are purely additive. ### Deprecated -- **Model-run-anchored Evaluation V2 is deprecated** in favor of the run-free (`model_id`) path. Passing `model_run_id` to `create_benchmark_evaluation_v2()` or `list_evaluations_v2()` now emits a `DeprecationWarning`; both keep working. `EvaluationV2.model_run_id` is documented as deprecated (it is `None` on run-free evaluations). On the leaderboard, `LeaderboardRankingEntry` / `LeaderboardF1CurveEntry` `model_run_id` and `model_run_name` are deprecated and now `Optional` (they are `None` for run-free evaluations — previously `model_run_id` was a required field and would fail to parse), and `collapse="allRuns"` on `leaderboard_ranking()` is discouraged. Prefer anchoring on and identifying evaluations by `model_id`. +- **Model-run-anchored Evaluation V2 is deprecated** in favor of the run-free (`model_id`) path. `Benchmark.create_evaluation_v2()` gains a `model_id` argument (run-free anchor) to match `create_benchmark_evaluation_v2()`. Passing `model_run_id` to `create_benchmark_evaluation_v2()`, `Benchmark.create_evaluation_v2()`, or `list_evaluations_v2()` now emits a `DeprecationWarning`; all keep working. `EvaluationV2.model_run_id` is documented as deprecated (it is `None` on run-free evaluations). On the leaderboard, `LeaderboardRankingEntry` / `LeaderboardF1CurveEntry` `model_run_id` and `model_run_name` are deprecated and now `Optional` (they are `None` for run-free evaluations — previously `model_run_id` was a required field and would fail to parse), and `collapse="allRuns"` on `leaderboard_ranking()` is discouraged. Prefer anchoring on and identifying evaluations by `model_id`. ### Removed +- **`allowed_label_matches` removed from the EvaluationV2 surface** (breaking). The run-free (model-source) eval path — the one this SDK now steers toward — rejects `allowedLabelMatches` server-side (400, "use rollupGroups"); it only survives as a legacy fallback on the deprecated model-run path, where `rollupGroups` wins anyway. Removed the `AllowedLabelMatch` class (and its top-level export), the `allowed_label_matches` / `allowed_label_matches_id` arguments from `create_benchmark_evaluation_v2()`, `Benchmark.create_evaluation_v2()`, `create_evaluation_v2_preset()`, and `update_evaluation_v2_preset()`, and the `allowed_label_matches*` fields from `EvaluationV2` and `EvaluationV2Preset`. Use `rollup_groups` (:class:`RollupGroup`) exclusively. - **`dataset_id` dropped from the EvaluationV2 surface** (breaking). An evaluation is no longer anchored on a single dataset — a model run now carries a *set* of datasets and a benchmark's items may span several — so the backend no longer returns a denormalized dataset on evaluations or leaderboards. Removed `EvaluationV2.dataset_id`, and `dataset_id` / `dataset_name` from `LeaderboardRankingEntry` and `LeaderboardF1CurveEntry`, matching the current backend responses. Without this, `EvaluationV2.from_json` raised `KeyError: 'dataset_id'` on every model-anchored (run-free) benchmark evaluation, since those payloads never carry a `dataset_id`. ### Fixed diff --git a/nucleus/__init__.py b/nucleus/__init__.py index 8f44b8b7..ead7ec12 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -2,7 +2,6 @@ __all__ = [ "AsyncJob", - "AllowedLabelMatch", "Benchmark", "BenchmarkItemsPage", "EmbeddingsExportJob", @@ -108,7 +107,6 @@ from .camera_params import CameraParams from .connection import Connection from .constants import ( - ALLOWED_LABEL_MATCHES_CAMEL_KEY, ANNOTATION_METADATA_SCHEMA_KEY, ANNOTATIONS_IGNORED_KEY, ANNOTATIONS_PROCESSED_KEY, @@ -208,11 +206,9 @@ NucleusAPIError, ) from .evaluation_v2 import ( - AllowedLabelMatch, EvaluationV2, EvaluationV2Status, RollupGroup, - _warn_allowed_label_matches_deprecated, _warn_model_run_deprecated, ) from .evaluation_v2_exclusions import ( @@ -266,6 +262,27 @@ # pylint: disable=C0302 +def _evaluation_v2_config_payload( + name: Optional[str], + rollup_groups: Optional[List[RollupGroup]], + exclusion_rules: Optional[List[Union[EvaluationV2ExclusionRule, Dict]]], +) -> Dict[str, Any]: + """Build the optional name/label/exclusion fields of an eval-V2 payload.""" + payload: Dict[str, Any] = {} + if name is not None: + payload[NAME_KEY] = name + if rollup_groups is not None: + payload[ROLLUP_GROUPS_CAMEL_KEY] = [ + g.to_api_dict() for g in rollup_groups + ] + if exclusion_rules is not None: + payload[EXCLUSION_RULES_CAMEL_KEY] = [ + rule.to_api_dict() if hasattr(rule, "to_api_dict") else rule + for rule in exclusion_rules + ] + return payload + + class NucleusClient: """Client to interact with the Nucleus API via Python SDK. @@ -1090,7 +1107,6 @@ def create_evaluation_v2_preset( name: str, *, rollup_groups: Optional[List[RollupGroup]] = None, - allowed_label_matches: Optional[List[AllowedLabelMatch]] = None, exclusion_rules: Optional[ List[Union[EvaluationV2ExclusionRule, Dict[str, Any]]] ] = None, @@ -1100,33 +1116,19 @@ def create_evaluation_v2_preset( Parameters: name: Preset name. Must be non-empty and unique among the user's presets. - rollup_groups: Optional rollup classes (the primary label - configuration); each :class:`RollupGroup` maps raw labels onto - one class name. Mutually exclusive with - ``allowed_label_matches``. - allowed_label_matches: Deprecated. Use ``rollup_groups``. Optional - legacy label pairs to treat as matches. + rollup_groups: Optional rollup classes (the label configuration); + each :class:`RollupGroup` maps raw labels onto one class name. exclusion_rules: Optional rules that drop items/annotations (same types accepted by :meth:`create_benchmark_evaluation_v2`). Returns: :class:`EvaluationV2Preset`: The created preset. """ - if allowed_label_matches is not None: - _warn_allowed_label_matches_deprecated() - if rollup_groups is not None and allowed_label_matches is not None: - raise ValueError( - "rollup_groups and allowed_label_matches cannot both be set" - ) payload: Dict[str, Any] = {NAME_KEY: name} if rollup_groups is not None: payload[ROLLUP_GROUPS_CAMEL_KEY] = [ g.to_api_dict() for g in rollup_groups ] - if allowed_label_matches is not None: - payload[ALLOWED_LABEL_MATCHES_CAMEL_KEY] = [ - m.to_api_dict() for m in allowed_label_matches - ] if exclusion_rules is not None: payload[EXCLUSION_RULES_CAMEL_KEY] = [ rule.to_api_dict() if hasattr(rule, "to_api_dict") else rule @@ -1141,7 +1143,6 @@ def update_evaluation_v2_preset( *, name: Any = _UNSET, rollup_groups: Any = _UNSET, - allowed_label_matches: Any = _UNSET, exclusion_rules: Any = _UNSET, ) -> EvaluationV2Preset: """Update a saved Evaluation V2 preset. @@ -1154,25 +1155,11 @@ def update_evaluation_v2_preset( preset_id: Preset id (``prev_*``). Must be owned by the caller. name: Optional new name. rollup_groups: Optional new rollup classes, or ``None`` to clear. - Mutually exclusive with ``allowed_label_matches``. - allowed_label_matches: Deprecated. Use ``rollup_groups``. Optional - new legacy label-match list, or ``None`` to clear. exclusion_rules: Optional new exclusion rules, or ``None`` to clear. Returns: :class:`EvaluationV2Preset`: The updated preset. """ - if allowed_label_matches is not _UNSET: - _warn_allowed_label_matches_deprecated() - if ( - rollup_groups is not _UNSET - and rollup_groups is not None - and allowed_label_matches is not _UNSET - and allowed_label_matches is not None - ): - raise ValueError( - "rollup_groups and allowed_label_matches cannot both be set" - ) payload: Dict[str, Any] = {} if name is not _UNSET: payload[NAME_KEY] = name @@ -1182,12 +1169,6 @@ def update_evaluation_v2_preset( if rollup_groups is None else [g.to_api_dict() for g in rollup_groups] ) - if allowed_label_matches is not _UNSET: - payload[ALLOWED_LABEL_MATCHES_CAMEL_KEY] = ( - None - if allowed_label_matches is None - else [m.to_api_dict() for m in allowed_label_matches] - ) if exclusion_rules is not _UNSET: payload[EXCLUSION_RULES_CAMEL_KEY] = ( None @@ -1594,8 +1575,6 @@ def create_benchmark_evaluation_v2( model_id: Optional[Union[str, Model]] = None, name: Optional[str] = None, rollup_groups: Optional[List[RollupGroup]] = None, - allowed_label_matches: Optional[List[AllowedLabelMatch]] = None, - allowed_label_matches_id: Optional[str] = None, exclusion_rules: Optional[ List[Union[EvaluationV2ExclusionRule, Dict[str, Any]]] ] = None, @@ -1623,29 +1602,18 @@ def create_benchmark_evaluation_v2( coverage may be partial, or empty. Mutually exclusive with ``model_id``. name: Optional display name. - rollup_groups: Optional rollup classes (the primary label - configuration); each :class:`RollupGroup` maps raw labels - onto one class name. Mutually exclusive with the - ``allowed_label_matches*`` arguments. - allowed_label_matches: Deprecated. Use ``rollup_groups``. Optional - legacy label pairs to treat as matches. - allowed_label_matches_id: Deprecated. Use ``rollup_groups``. - Optional id of a saved label-match configuration. + rollup_groups: Optional rollup classes (the label configuration); + each :class:`RollupGroup` maps raw labels onto one class name. exclusion_rules: Optional rules that drop items/annotations before metrics are computed (see :mod:`nucleus.evaluation_v2_exclusions`). - preset: Optional :class:`EvaluationV2Preset` whose label - configuration and ``exclusion_rules`` seed this evaluation. - Explicit arguments take precedence over the preset's values. + preset: Optional :class:`EvaluationV2Preset` whose ``rollup_groups`` + and ``exclusion_rules`` seed this evaluation. Explicit + arguments take precedence over the preset's values. Returns: :class:`EvaluationV2`: The created evaluation. """ - if ( - allowed_label_matches is not None - or allowed_label_matches_id is not None - ): - _warn_allowed_label_matches_deprecated() if model_run_id is not None: _warn_model_run_deprecated() resolved_model_id = ( @@ -1656,52 +1624,22 @@ def create_benchmark_evaluation_v2( "Provide exactly one of model_run_id or model_id." ) if preset is not None: - if ( - rollup_groups is None - and allowed_label_matches is None - and allowed_label_matches_id is None - ): + if rollup_groups is None: rollup_groups = preset.rollup_groups - if rollup_groups is None: - allowed_label_matches = preset.allowed_label_matches if exclusion_rules is None and preset.exclusion_rules is not None: exclusion_rules = list(preset.exclusion_rules) - label_configs = [ - config - for config in ( - rollup_groups, - allowed_label_matches, - allowed_label_matches_id, - ) - if config is not None - ] - if len(label_configs) > 1: - raise ValueError( - "Set at most one of rollup_groups, allowed_label_matches, " - "or allowed_label_matches_id" - ) payload: Dict[str, Any] = ( {MODEL_ID_KEY: resolved_model_id} if resolved_model_id is not None else {MODEL_RUN_ID_KEY: model_run_id} ) - if name is not None: - payload[NAME_KEY] = name - if rollup_groups is not None: - payload[ROLLUP_GROUPS_CAMEL_KEY] = [ - g.to_api_dict() for g in rollup_groups - ] - if allowed_label_matches is not None: - payload["allowed_label_matches"] = [ - m.to_api_dict() for m in allowed_label_matches - ] - if allowed_label_matches_id is not None: - payload["allowed_label_matches_id"] = allowed_label_matches_id - if exclusion_rules is not None: - payload[EXCLUSION_RULES_CAMEL_KEY] = [ - rule.to_api_dict() if hasattr(rule, "to_api_dict") else rule - for rule in exclusion_rules - ] + payload.update( + _evaluation_v2_config_payload( + name, + rollup_groups, + exclusion_rules, + ) + ) result = self.post(payload, f"benchmarks/{benchmark_id}/evaluationsV2") eval_id = result.get(EVALUATION_ID_KEY) if not eval_id: diff --git a/nucleus/benchmark.py b/nucleus/benchmark.py index f54fceb7..1dd9a598 100644 --- a/nucleus/benchmark.py +++ b/nucleus/benchmark.py @@ -9,7 +9,7 @@ benchmark = client.create_benchmark("city-streets-v1", slice_id="slc_...") evaluation = benchmark.create_evaluation_v2( - model_run_id, + model_id=model.id, rollup_groups=[RollupGroup("vehicle", ["car", "truck"])], ) evaluation.wait_for_completion() @@ -48,7 +48,6 @@ ) from nucleus.data_transfer_object.evaluation_v2 import BenchmarkItemsPage from nucleus.evaluation_v2 import ( - AllowedLabelMatch, EvaluationV2, RollupGroup, ) @@ -57,6 +56,7 @@ if TYPE_CHECKING: from nucleus import NucleusClient + from nucleus.model import Model @dataclass @@ -181,27 +181,25 @@ def items( def create_evaluation_v2( self, - model_run_id: str, + model_run_id: Optional[str] = None, *, + model_id: Optional[Union[str, "Model"]] = None, name: Optional[str] = None, rollup_groups: Optional[List[RollupGroup]] = None, - allowed_label_matches: Optional[List[AllowedLabelMatch]] = None, - allowed_label_matches_id: Optional[str] = None, exclusion_rules: Optional[ List[Union[EvaluationV2ExclusionRule, Dict[str, Any]]] ] = None, preset: Optional[EvaluationV2Preset] = None, ) -> EvaluationV2: - """Evaluate a model run against this benchmark. + """Evaluate a model against this benchmark. - The run need not cover this benchmark's datasets — uncovered members are - scored as false negatives, so a partial run still ranks comparably. To - give a run predictions across several datasets, use - :meth:`Dataset.upload_predictions_for_model_run`. + Anchor the evaluation on a model (``model_id``) for the run-free + "model v2" flow. The legacy ``model_run_id`` anchor is deprecated. + Provide exactly one. Uncovered benchmark members are scored as false + negatives, so partial coverage still ranks comparably. See :meth:`NucleusClient.create_benchmark_evaluation_v2` for parameter - details. ``allowed_label_matches`` / ``allowed_label_matches_id`` are - deprecated; use ``rollup_groups``. + details. Returns: :class:`~nucleus.evaluation_v2.EvaluationV2`: The created evaluation. @@ -211,10 +209,9 @@ def create_evaluation_v2( return self._client.create_benchmark_evaluation_v2( self.id, model_run_id, + model_id=model_id, name=name, rollup_groups=rollup_groups, - allowed_label_matches=allowed_label_matches, - allowed_label_matches_id=allowed_label_matches_id, exclusion_rules=exclusion_rules, preset=preset, ) diff --git a/nucleus/evaluation_v2.py b/nucleus/evaluation_v2.py index 7b75c278..054aa3b0 100644 --- a/nucleus/evaluation_v2.py +++ b/nucleus/evaluation_v2.py @@ -12,9 +12,6 @@ import requests from nucleus.constants import ( - ALLOWED_LABEL_MATCHES_ID_KEY, - ALLOWED_LABEL_MATCHES_KEY, - ALLOWED_LABEL_MATCHES_NAME_KEY, BENCHMARK_ID_KEY, CLASS_NAME_CAMEL_KEY, CLASS_NAME_KEY, @@ -24,16 +21,12 @@ EXCLUSION_RULES_KEY, EXCLUSION_STATS_KEY, FILTERS_KEY, - GROUND_TRUTH_LABEL_CAMEL_KEY, - GROUND_TRUTH_LABEL_KEY, ID_KEY, IOU_THRESHOLD_KEY, LABELS_KEY, LIMIT_KEY, MATCH_TYPE_KEY, MODEL_ID_KEY, - MODEL_PREDICTION_LABEL_CAMEL_KEY, - MODEL_PREDICTION_LABEL_KEY, MODEL_RUN_ID_KEY, NAME_KEY, OFFSET_KEY, @@ -71,25 +64,6 @@ class EvaluationV2Status(str, Enum): EvaluationV2Status.CANCELLED, } -_ALLOWED_LABEL_MATCHES_DEPRECATION = ( - "allowed_label_matches is deprecated and will be removed in a future " - "release. Use rollup_groups instead." -) - - -def _warn_allowed_label_matches_deprecated() -> None: - """Emit the Evaluation V2 ``allowed_label_matches`` deprecation warning. - - ``stacklevel=3`` points at the public method's caller (this helper → - the client/wrapper method → user code). - """ - warnings.warn( - _ALLOWED_LABEL_MATCHES_DEPRECATION, - DeprecationWarning, - stacklevel=3, - ) - - _MODEL_RUN_DEPRECATION = ( "Model-run-anchored Evaluation V2 is deprecated and will be removed in a " "future release. Upload run-free predictions to a Model " @@ -123,52 +97,6 @@ def _parse_json_field(value: Any) -> Optional[Any]: return value -@dataclass -class AllowedLabelMatch: - """Deprecated. Use :class:`RollupGroup` instead. - - Ground-truth and prediction label pair that counts as a match. - """ - - ground_truth_label: str - model_prediction_label: str - - def to_api_dict(self) -> Dict[str, str]: - return { - GROUND_TRUTH_LABEL_KEY: self.ground_truth_label, - MODEL_PREDICTION_LABEL_KEY: self.model_prediction_label, - } - - -def _parse_allowed_label_matches( - raw_matches: Any, -) -> Optional[List[AllowedLabelMatch]]: - """Parse an ``allowed_label_matches`` array from an API payload. - - Tolerates either key casing and drops malformed entries. - """ - if not isinstance(raw_matches, list): - return None - matches: List[AllowedLabelMatch] = [] - for m in raw_matches: - if not isinstance(m, dict): - continue - gt = m.get(GROUND_TRUTH_LABEL_CAMEL_KEY) - if gt is None: - gt = m.get(GROUND_TRUTH_LABEL_KEY) - mp = m.get(MODEL_PREDICTION_LABEL_CAMEL_KEY) - if mp is None: - mp = m.get(MODEL_PREDICTION_LABEL_KEY) - if gt is not None and mp is not None: - matches.append( - AllowedLabelMatch( - ground_truth_label=str(gt), - model_prediction_label=str(mp), - ) - ) - return matches - - @dataclass class RollupGroup: """A rollup class: raw labels evaluated together under one class name. @@ -229,12 +157,6 @@ class EvaluationV2: temporal_workflow_id: Optional[str] = None error_message: Optional[str] = None created_at: Optional[str] = None - #: Deprecated. Prefer :attr:`rollup_groups`. - allowed_label_matches_id: Optional[str] = None - #: Deprecated. Prefer :attr:`rollup_groups`. - allowed_label_matches: Optional[List[AllowedLabelMatch]] = None - #: Deprecated. Prefer :attr:`rollup_groups`. - allowed_label_matches_name: Optional[str] = None rollup_groups: Optional[List[RollupGroup]] = None benchmark_id: Optional[str] = None slice_id: Optional[str] = None @@ -248,10 +170,6 @@ def from_json( payload: Dict[str, Any], client: Optional["NucleusClient"] = None, ) -> "EvaluationV2": - matches = _parse_allowed_label_matches( - payload.get(ALLOWED_LABEL_MATCHES_KEY) - ) - return cls( id=str(payload[ID_KEY]), model_run_id=( @@ -269,11 +187,6 @@ def from_json( temporal_workflow_id=payload.get(TEMPORAL_WORKFLOW_ID_KEY), error_message=payload.get(ERROR_MESSAGE_KEY), created_at=payload.get(CREATED_AT_KEY), - allowed_label_matches_id=payload.get(ALLOWED_LABEL_MATCHES_ID_KEY), - allowed_label_matches=matches, - allowed_label_matches_name=payload.get( - ALLOWED_LABEL_MATCHES_NAME_KEY - ), rollup_groups=_parse_rollup_groups( _parse_json_field(payload.get(ROLLUP_GROUPS_KEY)) ), diff --git a/nucleus/evaluation_v2_preset.py b/nucleus/evaluation_v2_preset.py index c56c699d..f3f0ae0f 100644 --- a/nucleus/evaluation_v2_preset.py +++ b/nucleus/evaluation_v2_preset.py @@ -4,9 +4,6 @@ and ``exclusion_rules`` so the same configuration can be applied across many evaluations. Presets are private to the creating user. -``allowed_label_matches`` is still accepted for backwards compatibility but -is deprecated; use ``rollup_groups``. - Create and manage presets via :class:`~nucleus.NucleusClient`:: preset = client.create_evaluation_v2_preset( @@ -23,8 +20,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional from nucleus.constants import ( - ALLOWED_LABEL_MATCHES_CAMEL_KEY, - ALLOWED_LABEL_MATCHES_KEY, CREATED_AT_KEY, CREATED_BY_USER_ID_KEY, DELETED_AT_KEY, @@ -37,9 +32,7 @@ UPDATED_AT_KEY, ) from nucleus.evaluation_v2 import ( - AllowedLabelMatch, RollupGroup, - _parse_allowed_label_matches, _parse_json_field, _parse_rollup_groups, ) @@ -65,8 +58,6 @@ class EvaluationV2Preset: id: str name: str rollup_groups: Optional[List[RollupGroup]] = None - #: Deprecated. Prefer :attr:`rollup_groups`. - allowed_label_matches: Optional[List[AllowedLabelMatch]] = None exclusion_rules: Optional[List[Dict[str, Any]]] = None created_by_user_id: Optional[str] = None created_at: Optional[str] = None @@ -90,10 +81,6 @@ def from_json( else payload.get(ROLLUP_GROUPS_CAMEL_KEY) ) ), - allowed_label_matches=_parse_allowed_label_matches( - payload.get(ALLOWED_LABEL_MATCHES_KEY) - or payload.get(ALLOWED_LABEL_MATCHES_CAMEL_KEY) - ), exclusion_rules=_parse_json_field( payload.get(EXCLUSION_RULES_KEY) if payload.get(EXCLUSION_RULES_KEY) is not None @@ -111,7 +98,6 @@ def update( *, name: Any = _UNSET, rollup_groups: Any = _UNSET, - allowed_label_matches: Any = _UNSET, exclusion_rules: Any = _UNSET, ) -> "EvaluationV2Preset": """Update this preset in place. @@ -120,8 +106,6 @@ def update( ``rollup_groups=None`` / ``exclusion_rules=None`` clears that field; omitting an argument leaves it unchanged. - ``allowed_label_matches`` is deprecated; use ``rollup_groups``. - Returns: self, with updated fields. """ @@ -134,7 +118,6 @@ def update( self.id, name=name, rollup_groups=rollup_groups, - allowed_label_matches=allowed_label_matches, exclusion_rules=exclusion_rules, ) self.__dict__.update(updated.__dict__) diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 522c115b..2aef6934 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -6,7 +6,6 @@ import requests from nucleus import ( - AllowedLabelMatch, Benchmark, EvaluationV2Preset, LabelExclusionRule, @@ -28,7 +27,6 @@ _EVAL_ROW = { "id": "evalv2_1", "model_run_id": "run_1", - "dataset_id": "ds_1", "benchmark_id": "bm_1", "status": "pending", } @@ -382,6 +380,18 @@ def test_benchmark_instance_methods_delegate_to_client(): assert kwargs["name"] == "e" +def test_benchmark_create_evaluation_v2_run_free_passes_model_id(): + client = MagicMock(spec=NucleusClient) + benchmark = Benchmark(id="bm_1", name="b", _client=client) + benchmark.create_evaluation_v2( + model_id="prj_1", rollup_groups=[RollupGroup("vehicle", ["car"])] + ) + args, kwargs = client.create_benchmark_evaluation_v2.call_args + # benchmark id positional, model_run_id positional None (run-free). + assert args == ("bm_1", None) + assert kwargs["model_id"] == "prj_1" + + def test_benchmark_without_client_raises(): benchmark = Benchmark(id="bm_1", name="b") with pytest.raises(RuntimeError, match="no client"): @@ -425,26 +435,6 @@ def test_create_benchmark_evaluation_v2_with_rollup_groups(): assert evaluation.benchmark_id == "bm_1" -def test_create_benchmark_evaluation_v2_label_config_mutual_exclusion(): - client = NucleusClient(api_key="test") - with pytest.warns(DeprecationWarning, match="allowed_label_matches"): - with pytest.raises(ValueError, match="at most one"): - client.create_benchmark_evaluation_v2( - "bm_1", - "run_1", - rollup_groups=[RollupGroup("vehicle", ["car"])], - allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], - ) - with pytest.warns(DeprecationWarning, match="allowed_label_matches"): - with pytest.raises(ValueError, match="at most one"): - client.create_benchmark_evaluation_v2( - "bm_1", - "run_1", - rollup_groups=[RollupGroup("vehicle", ["car"])], - allowed_label_matches_id="alm_1", - ) - - def test_create_benchmark_evaluation_v2_preset_seeds_rollup_groups(): client = NucleusClient(api_key="test") _mock_create_eval(client) @@ -454,59 +444,14 @@ def test_create_benchmark_evaluation_v2_preset_seeds_rollup_groups(): rollup_groups=[RollupGroup("vehicle", ["car"])], exclusion_rules=[{"type": "labels", "scope": "item"}], ) - client.create_benchmark_evaluation_v2("bm_1", "run_1", preset=preset) + client.create_benchmark_evaluation_v2( + "bm_1", model_id="prj_1", preset=preset + ) payload = client.connection.post.call_args[0][0] assert payload["rollupGroups"] == [ {"class_name": "vehicle", "labels": ["car"]} ] assert payload["exclusionRules"] == [{"type": "labels", "scope": "item"}] - assert "allowed_label_matches" not in payload - - -def test_create_benchmark_evaluation_v2_preset_seeds_legacy_matches(): - client = NucleusClient(api_key="test") - _mock_create_eval(client) - preset = EvaluationV2Preset( - id="prev_1", - name="p", - allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], - ) - # Seeding from a stored preset does not itself warn — the deprecated - # kwargs were not passed by the caller. - client.create_benchmark_evaluation_v2("bm_1", "run_1", preset=preset) - payload = client.connection.post.call_args[0][0] - assert payload["allowed_label_matches"] == [ - {"ground_truth_label": "car", "model_prediction_label": "vehicle"} - ] - assert "rollupGroups" not in payload - - -def test_create_benchmark_evaluation_v2_allowed_label_matches_deprecated(): - client = NucleusClient(api_key="test") - _mock_create_eval(client) - with pytest.warns(DeprecationWarning, match="allowed_label_matches"): - client.create_benchmark_evaluation_v2( - "bm_1", - "run_1", - allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], - ) - payload = client.connection.post.call_args[0][0] - assert payload["allowed_label_matches"] == [ - {"ground_truth_label": "car", "model_prediction_label": "vehicle"} - ] - - -def test_create_benchmark_evaluation_v2_allowed_label_matches_id_deprecated(): - client = NucleusClient(api_key="test") - _mock_create_eval(client) - with pytest.warns(DeprecationWarning, match="allowed_label_matches"): - client.create_benchmark_evaluation_v2( - "bm_1", - "run_1", - allowed_label_matches_id="alm_1", - ) - payload = client.connection.post.call_args[0][0] - assert payload["allowed_label_matches_id"] == "alm_1" def test_create_benchmark_evaluation_v2_rollup_groups_does_not_warn(): diff --git a/tests/test_evaluation_v2.py b/tests/test_evaluation_v2.py index 75cdfa84..b1989c4a 100644 --- a/tests/test_evaluation_v2.py +++ b/tests/test_evaluation_v2.py @@ -6,7 +6,6 @@ import requests from nucleus import ( - AllowedLabelMatch, BoxAreaExclusionRule, EvaluationV2, LabelExclusionRule, @@ -73,19 +72,13 @@ def test_camelize_filter_value_preserves_predicate_value(): ) == {"key": "k", "op": "EQ", "value": {"keep_snake": 1}} -def test_allowed_label_match_to_api_dict(): - m = AllowedLabelMatch(ground_truth_label="a", model_prediction_label="b") - assert m.to_api_dict() == { - "ground_truth_label": "a", - "model_prediction_label": "b", - } - - -def test_evaluation_v2_from_json_with_matches(): +def test_evaluation_v2_from_json_ignores_legacy_allowed_label_matches(): + # allowed_label_matches was removed from the eval-V2 surface; a payload + # that still carries it (legacy backend rows) parses fine and drops it. client = NucleusClient(api_key="k") payload = { "id": "evalv2_1", - "model_run_id": "run_1", + "model_id": "prj_1", "status": "pending", "allowed_label_matches": [ {"groundTruthLabel": "x", "modelPredictionLabel": "y"}, @@ -93,9 +86,7 @@ def test_evaluation_v2_from_json_with_matches(): } ev = EvaluationV2.from_json(payload, client) assert ev.id == "evalv2_1" - assert ev.allowed_label_matches is not None - assert len(ev.allowed_label_matches) == 1 - assert ev.allowed_label_matches[0].ground_truth_label == "x" + assert not hasattr(ev, "allowed_label_matches") def test_list_evaluations_v2_empty(): diff --git a/tests/test_evaluation_v2_presets.py b/tests/test_evaluation_v2_presets.py index cd55f0f7..cb85e3be 100644 --- a/tests/test_evaluation_v2_presets.py +++ b/tests/test_evaluation_v2_presets.py @@ -8,11 +8,11 @@ import requests from nucleus import ( - AllowedLabelMatch, EvaluationV2, EvaluationV2Preset, LabelExclusionRule, NucleusClient, + RollupGroup, ) from nucleus.dataset import Dataset @@ -27,11 +27,8 @@ def test_list_evaluation_v2_presets(): { "id": "prev_1", "name": "vehicles", - "allowed_label_matches": [ - { - "groundTruthLabel": "car", - "modelPredictionLabel": "vehicle", - } + "rollup_groups": [ + {"class_name": "vehicle", "labels": ["car", "truck"]} ], "exclusion_rules": None, "created_by_user_id": "u_1", @@ -43,8 +40,8 @@ def test_list_evaluation_v2_presets(): assert len(presets) == 1 assert presets[0].id == "prev_1" assert presets[0].name == "vehicles" - assert presets[0].allowed_label_matches[0] == AllowedLabelMatch( - ground_truth_label="car", model_prediction_label="vehicle" + assert presets[0].rollup_groups[0] == RollupGroup( + class_name="vehicle", labels=["car", "truck"] ) @@ -54,25 +51,24 @@ def test_create_evaluation_v2_preset_payload(): return_value={ "id": "prev_1", "name": "vehicles", - "allowed_label_matches": [], + "rollup_groups": [], "exclusion_rules": None, } ) - with pytest.warns(DeprecationWarning, match="allowed_label_matches"): - preset = client.create_evaluation_v2_preset( - "vehicles", - allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], - exclusion_rules=[ - LabelExclusionRule( - scope="item", target="prediction", labels=["ignore"] - ) - ], - ) + preset = client.create_evaluation_v2_preset( + "vehicles", + rollup_groups=[RollupGroup("vehicle", ["car", "truck"])], + exclusion_rules=[ + LabelExclusionRule( + scope="item", target="prediction", labels=["ignore"] + ) + ], + ) payload, route = client.connection.post.call_args[0] assert route == "evaluationV2Presets" assert payload["name"] == "vehicles" - assert payload["allowedLabelMatches"] == [ - {"ground_truth_label": "car", "model_prediction_label": "vehicle"} + assert payload["rollupGroups"] == [ + {"class_name": "vehicle", "labels": ["car", "truck"]} ] assert payload["exclusionRules"] == [ { @@ -234,22 +230,6 @@ def test_create_evaluation_v2_preset_with_rollup_groups(): assert preset.rollup_groups[0].class_name == "vehicle" -def test_create_evaluation_v2_preset_rollup_and_matches_mutually_exclusive(): - from nucleus import RollupGroup - - client = NucleusClient(api_key="test") - with pytest.warns(DeprecationWarning, match="allowed_label_matches"): - try: - client.create_evaluation_v2_preset( - "p", - rollup_groups=[RollupGroup("vehicle", ["car"])], - allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], - ) - raise AssertionError("expected ValueError") - except ValueError as e: - assert "cannot both be set" in str(e) - - def test_update_evaluation_v2_preset_rollup_groups_and_clear(): from nucleus import RollupGroup @@ -297,19 +277,3 @@ def test_create_evaluation_v2_preset_rollup_groups_does_not_warn(): "vehicles", rollup_groups=[RollupGroup("vehicle", ["car"])], ) - - -def test_update_evaluation_v2_preset_allowed_label_matches_warns(): - client = NucleusClient(api_key="test") - client.connection.patch = MagicMock( - return_value={"id": "prev_1", "name": "p"} - ) - with pytest.warns(DeprecationWarning, match="allowed_label_matches"): - client.update_evaluation_v2_preset( - "prev_1", - allowed_label_matches=[AllowedLabelMatch("car", "vehicle")], - ) - payload = client.connection.patch.call_args[0][0] - assert payload["allowedLabelMatches"] == [ - {"ground_truth_label": "car", "model_prediction_label": "vehicle"} - ]