From be268f116153465f18308ccb3169c499012e81d2 Mon Sep 17 00:00:00 2001 From: hieuddo Date: Thu, 22 Oct 2026 15:50:24 +0800 Subject: [PATCH 1/5] Add Collision-Corrected Evaluation and item-level resolvers --- cornac/eval_methods/__init__.py | 18 + cornac/eval_methods/semantic_id_evaluation.py | 170 +++++++ cornac/eval_methods/sid_evaluation.py | 130 +++++ cornac/eval_methods/sid_resolvers.py | 171 +++++++ cornac/models/__init__.py | 8 + cornac/models/diffgrm/diffgrm.py | 36 +- cornac/models/diffgrm/recom_diffgrm.py | 50 +- cornac/models/letter/recom_letter.py | 26 +- cornac/models/recommender.py | 18 + cornac/models/rpg/recom_rpg.py | 56 ++- cornac/models/semantic_id.py | 384 ++++++++++++++ cornac/models/tiger/recom_tiger.py | 60 ++- docs/source/api_ref/eval_methods.rst | 55 ++ docs/source/api_ref/models.rst | 8 + examples/README.md | 2 + examples/semantic_id_evaluation_example.py | 56 +++ .../test_semantic_id_evaluation.py | 330 ++++++++++++ .../models/test_semantic_id_adapters.py | 476 ++++++++++++++++++ 18 files changed, 2029 insertions(+), 25 deletions(-) create mode 100644 cornac/eval_methods/semantic_id_evaluation.py create mode 100644 cornac/eval_methods/sid_evaluation.py create mode 100644 cornac/eval_methods/sid_resolvers.py create mode 100644 cornac/models/semantic_id.py create mode 100644 examples/semantic_id_evaluation_example.py create mode 100644 tests/cornac/eval_methods/test_semantic_id_evaluation.py create mode 100644 tests/cornac/models/test_semantic_id_adapters.py diff --git a/cornac/eval_methods/__init__.py b/cornac/eval_methods/__init__.py index 74979c804..4522b2329 100644 --- a/cornac/eval_methods/__init__.py +++ b/cornac/eval_methods/__init__.py @@ -25,6 +25,16 @@ from .next_item_evaluation import NextItemEvaluation from .propensity_stratified_evaluation import PropensityStratifiedEvaluation +from .semantic_id_evaluation import SemanticIDEvaluation, collect_semantic_id_trace +from .sid_evaluation import evaluate_semantic_id_trace +from .sid_resolvers import ( + SemanticIDResolver, + TrainingPopularity, + HistoryCosine, + CustomScorer, + resolve_semantic_ids, +) + __all__ = [ "BaseMethod", "RatioSplit", @@ -34,4 +44,12 @@ "NextBasketEvaluation", "NextItemEvaluation", "PropensityStratifiedEvaluation", + "SemanticIDEvaluation", + "collect_semantic_id_trace", + "evaluate_semantic_id_trace", + "SemanticIDResolver", + "TrainingPopularity", + "HistoryCosine", + "CustomScorer", + "resolve_semantic_ids", ] diff --git a/cornac/eval_methods/semantic_id_evaluation.py b/cornac/eval_methods/semantic_id_evaluation.py new file mode 100644 index 000000000..cb5332a65 --- /dev/null +++ b/cornac/eval_methods/semantic_id_evaluation.py @@ -0,0 +1,170 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Opt-in Semantic ID evaluation for Cornac Experiment.""" + +import time +from collections import OrderedDict + +import numpy as np + +from ..experiment.result import Result +from ..metrics import Recall, NDCG +from ..models.recommender import SemanticIDRecommender +from ..models.semantic_id import ( + SemanticIDCatalog, + SemanticIDPrediction, + SemanticIDTrace, + SemanticIDTraceRecord, +) +from .next_item_evaluation import NextItemEvaluation +from .sid_evaluation import evaluate_semantic_id_trace, _cutoffs +from .sid_resolvers import HistoryCosine, TrainingPopularity + + +def collect_semantic_id_trace(model, dataset, *, mode="last", n_candidates=20, + catalog=None, exclude_unknowns=False, + num_train_items=None): + """Collect raw predictions once per query, with no target in model context. + + ``n_candidates`` requests a minimum budget; legitimately shorter decoder + output stays shorter. ``catalog`` may restrict the model's complete catalog + to the evaluator's admitted universe. Excluded addresses retain empty slots. + """ + if mode not in ("last", "next"): + raise ValueError("mode must be last or next") + _cutoffs((n_candidates,)) + catalog = model.semantic_id_catalog() if catalog is None else catalog + records = [] + for [session], [mapped], [items] in dataset.si_iter(batch_size=1, shuffle=False): + if len(items) < 2: + continue + user = int(dataset.uir_tuple[0][mapped[0]]) + start = 1 if mode == "next" else len(items) - 1 + for position in range(start, len(items)): + target = int(items[position]) + if exclude_unknowns and target >= num_train_items: + continue + query = len(records) + prediction = model.predict_semantic_ids( + history_items=list(items[:position]), user_index=user, + query_index=query, session_index=int(session), + evaluation_position=position, n_candidates=n_candidates) + # Candidate filtering changes admission, never rank positions. + validity = np.asarray([tuple(row) in catalog.address_to_items + for row in prediction.address_sids]) + if not np.array_equal(validity, prediction.catalog_validity): + # Verify original model validity before applying an explicit + # restricted universe; otherwise a malformed adapter is hidden. + from .sid_resolvers import validate_prediction + validate_prediction(model.semantic_id_catalog(), prediction) + prediction = SemanticIDPrediction(prediction.address_sids, + prediction.scores, validity, prediction.decoder_diagnostics) + try: + address = catalog.address_sid_for_item(target) + tokenizer = catalog.tokenizer_sid_for_item(target) + except KeyError as error: + raise ValueError("evaluation target is absent from the admitted model catalog") from error + records.append(SemanticIDTraceRecord(query, user, int(session), position, + 0, position, target, tokenizer, address, prediction)) + return SemanticIDTrace(catalog.stable_hash, tuple(records), + {"mode": mode, "requested_candidates": n_candidates}) + + +class SemanticIDEvaluation(NextItemEvaluation): + """Evaluate raw Semantic IDs through SID, CCE and explicit item views. + + Accepts NextItemEvaluation dataset/splitting arguments. ``resolvers`` is an + optional mapping of names to fitted SemanticIDResolver instances; None uses + training popularity. ``item_features`` explicitly enables training-history + cosine and must use global Cornac item rows. ``feature_source`` describes + their provenance. Only Recall/NDCG with positive finite k are supported. + Existing model ranking/training and checkpoint selection remain unchanged. + """ + + def __init__(self, *args, resolvers=None, item_features=None, + feature_source="provided", **kwargs): + self.resolvers = resolvers + self.item_features = item_features + self.feature_source = feature_source + super().__init__(*args, **kwargs) + + def _evaluate_split(self, model, dataset, metrics, resolvers, user_based): + native = model.semantic_id_catalog() + catalog = native + if self.exclude_unknowns: + mask = native.item_indices < self.train_set.num_items + if not np.all(mask): + catalog = SemanticIDCatalog(native.item_indices[mask], + native.tokenizer_sids[mask], native.address_sids[mask], + native.tokenizer_codebook_cardinalities, + native.address_codebook_cardinalities, + metadata={"source_catalog_hash": native.stable_hash, + "admission": "training_items"}) + cutoffs = tuple(dict.fromkeys(int(metric.k) for metric in metrics)) + trace = collect_semantic_id_trace(model, dataset, mode=self.mode, + n_candidates=max(cutoffs), catalog=catalog, + exclude_unknowns=self.exclude_unknowns, + num_train_items=self.train_set.num_items) + replay = evaluate_semantic_id_trace(catalog, trace, cutoffs, resolvers) + averages, per_user = OrderedDict(), OrderedDict() + users = np.asarray([r.user_index for r in trace.records]) + for view, rows in replay["per_query"].items(): + for metric in metrics: + field = "recall" if isinstance(metric, Recall) else "ndcg" + values = rows[field][:, cutoffs.index(metric.k)] + name = f"{view}/{metric.name}" + grouped = {int(user): values[users == user].tolist() for user in np.unique(users)} + per_user[name] = grouped + averages[name] = (float(np.mean([np.mean(v) for v in grouped.values()])) + if user_based and grouped else + float(values.mean()) if len(values) else 0.) + result = Result(model.name, averages, per_user) + result.semantic_id_trace = trace + result.semantic_id_evaluation = replay + return result + + def evaluate(self, model, metrics, user_based=False, show_validation=True): + if not isinstance(model, SemanticIDRecommender): + raise ValueError("model must implement SemanticIDRecommender") + if not metrics or any(type(metric) not in (Recall, NDCG) for metric in metrics): + raise ValueError("SemanticIDEvaluation supports Recall and NDCG only") + _cutoffs(tuple(dict.fromkeys(metric.k for metric in metrics))) + if self.train_set is None or self.test_set is None: + raise ValueError("train_set and test_set are required") + self._reset() + started = time.time() + model.fit(self.train_set, self.val_set) + train_time = time.time() - started + users, items = self.train_set.uir_tuple[:2] + resolvers = (dict(self.resolvers) if self.resolvers is not None else + {"popularity": TrainingPopularity(items, self.total_items)}) + if self.item_features is not None: + if "history_cosine" in resolvers: + raise ValueError("history_cosine resolver name is already configured") + if len(self.item_features) < self.total_items: + raise ValueError("item_features must cover all global Cornac item indices") + resolvers["history_cosine"] = HistoryCosine(self.item_features, users, + items, self.total_users, feature_source=self.feature_source) + started = time.time() + model.transform(self.test_set) + result = self._evaluate_split(model, self.test_set, metrics, resolvers, user_based) + result.metric_avg_results["Train (s)"] = train_time + result.metric_avg_results["Test (s)"] = time.time() - started + validation = None + if show_validation and self.val_set is not None: + model.transform(self.val_set) + validation = self._evaluate_split(model, self.val_set, metrics, resolvers, user_based) + return result, validation diff --git a/cornac/eval_methods/sid_evaluation.py b/cornac/eval_methods/sid_evaluation.py new file mode 100644 index 000000000..b67f833bb --- /dev/null +++ b/cornac/eval_methods/sid_evaluation.py @@ -0,0 +1,130 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Paired SID, expanded CCE and exact-item evaluation from raw traces.""" + +from collections.abc import Mapping + +import numpy as np + +from ..models.semantic_id import SemanticIDCatalog, SemanticIDTrace +from .sid_resolvers import resolve_semantic_ids, validate_prediction + + +def _cutoffs(values): + values = tuple(values) + if (not values or any(isinstance(k, (bool, np.bool_)) or + not isinstance(k, (int, np.integer)) or k <= 0 for k in values) or + len(set(values)) != len(values)): + raise ValueError("cutoffs must be distinct positive finite integers") + return tuple(int(k) for k in values) + + +def _rank_rows(rank, cutoffs): + hit = np.asarray([float(rank is not None and rank < k) for k in cutoffs]) + ndcg = hit / np.log2(rank + 2) if rank is not None else hit.copy() + return hit, ndcg + + +def evaluate_semantic_id_trace(catalog, trace, cutoffs=(5, 10, 20), resolvers=None): + """Evaluate every view on the same ordered raw predictions. + + CCE is the analytical expectation under uniform permutations *within full + expanded buckets*. Earlier buckets consume their full widths. Invalid and + duplicate addresses consume one empty rank slot in every view. A member + resolver chooses one item per bucket without changing address order. + + ``resolvers`` maps display names to SemanticIDResolver instances. Callbacks + receive query/user identity and bucket members only. Results contain scalar + aggregates, per-query Recall/NDCG arrays and protocol/diagnostic sidecars. + This function averages over queries; SemanticIDEvaluation also supports the + framework's per-user aggregation. + """ + cutoffs = _cutoffs(cutoffs) + if not isinstance(catalog, SemanticIDCatalog) or not isinstance(trace, SemanticIDTrace): + raise ValueError("expected SemanticIDCatalog and SemanticIDTrace") + if trace.catalog_hash != catalog.stable_hash: + raise ValueError("trace catalog_hash differs from catalog") + if resolvers is None: + resolvers = {} + if not isinstance(resolvers, Mapping) or any( + not isinstance(name, str) or not name or any(c in name for c in "[]/") + for name in resolvers): + raise ValueError("resolvers must map nonempty plain names to resolvers") + from .sid_resolvers import SemanticIDResolver + if any(not isinstance(r, SemanticIDResolver) for r in resolvers.values()): + raise ValueError("resolvers must contain SemanticIDResolver instances") + views = ("SID", "CCE") + tuple(f"Item[{name}]" for name in resolvers) + rows = {view: {metric: np.zeros((len(trace.records), len(cutoffs))) + for metric in ("recall", "ndcg")} for view in views} + invalid = duplicate = total = 0 + sizes = [] + for index, record in enumerate(trace.records): + try: + address = catalog.address_sid_for_item(record.target_item_index) + tokenizer = catalog.tokenizer_sid_for_item(record.target_item_index) + except KeyError as error: + raise ValueError("trace target item is outside catalog") from error + if address != record.target_address_sid or tokenizer != record.target_tokenizer_sid: + raise ValueError("trace target codes differ from catalog") + prediction = record.prediction + validate_prediction(catalog, prediction) + addresses = [tuple(int(v) for v in row) for row in prediction.address_sids] + rank = next((i for i, value in enumerate(addresses) if value == address), None) + rows["SID"]["recall"][index], rows["SID"]["ndcg"][index] = _rank_rows(rank, cutoffs) + before, seen, reached = 0, set(), False + for value, valid in zip(addresses, prediction.catalog_validity): + if value == address: + reached = True + break + if not valid or value in seen: + before += 1 + else: + seen.add(value) + before += len(catalog.address_to_items[value]) + size = len(catalog.address_to_items[address]) + sizes.append(size) + if reached: + for col, cutoff in enumerate(cutoffs): + included = min(size, max(0, cutoff - before)) + rows["CCE"]["recall"][index, col] = included / size + rows["CCE"]["ndcg"][index, col] = ( + np.sum(1 / np.log2(before + np.arange(included) + 2)) / size) + for name, resolver in resolvers.items(): + slots = resolve_semantic_ids(catalog, prediction, resolver, + user_index=record.user_index, + query_index=record.query_index) + item_rank = next((i for i, item in enumerate(slots) + if item == record.target_item_index), None) + rows[f"Item[{name}]"]["recall"][index], rows[f"Item[{name}]"]["ndcg"][index] = _rank_rows(item_rank, cutoffs) + total += len(addresses) + invalid += int(np.count_nonzero(~prediction.catalog_validity)) + duplicate += len(addresses) - len(set(addresses)) + aggregate = {} + for view, metrics in rows.items(): + aggregate[view] = {} + for metric, values in metrics.items(): + values.setflags(write=False) + for col, cutoff in enumerate(cutoffs): + aggregate[view][f"{metric.upper() if metric == 'ndcg' else 'Recall'}@{cutoff}"] = ( + float(values[:, col].mean()) if len(values) else 0.) + return {"aggregate": aggregate, "per_query": rows, + "diagnostics": {"invalid_rate": invalid / total if total else 0., + "duplicate_rate": duplicate / total if total else 0., + "target_collision_group_size": np.asarray(sizes, dtype="int64")}, + "protocol": {"catalog_hash": catalog.stable_hash, "cutoffs": cutoffs, + "invalid_and_duplicate_slots": "preserved", + "order": "decoder_order", "CCE": "uniform_expanded_bucket_expectation", + "resolvers": {name: resolver.metadata for name, resolver in resolvers.items()}}} diff --git a/cornac/eval_methods/sid_resolvers.py b/cornac/eval_methods/sid_resolvers.py new file mode 100644 index 000000000..77226fe4f --- /dev/null +++ b/cornac/eval_methods/sid_resolvers.py @@ -0,0 +1,171 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Target-free selection of one item within each generated complete SID.""" + +import hashlib +from abc import ABC, abstractmethod + +import numpy as np + +from ..models.semantic_id import SemanticIDCatalog, SemanticIDPrediction + + +def _indices(values, name, upper=None): + values = np.asarray(values) + if (values.ndim != 1 or (values.size and + not np.issubdtype(values.dtype, np.integer)) or + np.any(values < 0) or (upper is not None and np.any(values >= upper))): + raise ValueError(f"{name} must contain covered nonnegative integer indices") + return values.astype("int64", copy=False) + + +def _unit_rows(features): + features = np.array(features, dtype="float64", copy=True) + if (features.ndim != 2 or not all(features.shape) or + not np.all(np.isfinite(features))): + raise ValueError("features must be a nonempty finite item matrix") + scales = np.max(np.abs(features), axis=1, keepdims=True) + np.divide(features, scales, out=features, where=scales != 0) + norms = np.linalg.norm(features, axis=1, keepdims=True) + np.divide(features, norms, out=features, where=norms != 0) + return np.frombuffer(features.tobytes(), dtype=features.dtype).reshape(features.shape) + + +class SemanticIDResolver(ABC): + """Score candidate members without targets; highest score wins. + + Implement ``score(query_index, user_index, candidate_items)``. The shared + operation checks scores and breaks ties by smallest global item index. + Scores never reorder generated addresses or admit outside candidates. + """ + + @abstractmethod + def score(self, query_index, user_index, candidate_items): + """Return one finite score per candidate, in candidate order.""" + raise NotImplementedError + + @property + def metadata(self): + return {"resolver": type(self).__name__} + + +class TrainingPopularity(SemanticIDResolver): + """Count explicitly supplied training events, retaining repeated events.""" + + def __init__(self, training_items, num_items): + if not isinstance(num_items, (int, np.integer)) or num_items <= 0: + raise ValueError("num_items must be a positive integer") + items = _indices(training_items, "training_items", num_items) + counts = np.bincount(items, minlength=num_items) + self.counts = np.frombuffer(counts.tobytes(), dtype=counts.dtype) + + def score(self, query_index, user_index, candidate_items): + return self.counts[list(candidate_items)] + + @property + def metadata(self): + return {"resolver": "training_popularity", "source": "training_events", + "counts_sha256": hashlib.sha256(self.counts.tobytes()).hexdigest()} + + +class HistoryCosine(SemanticIDResolver): + """Cosine to normalized sums of normalized training-event item features. + + Rows are global Cornac item indices. Supply training events only; repeats + contribute repeatedly. A zero user profile ties all members. ``feature_source`` + labels the explicit scoring feature space, which need not be the model's + tokenizer feature space. No evaluation history or target enters the profile. + """ + + def __init__(self, features, training_users, training_items, num_users, + feature_source="provided"): + self.features = _unit_rows(features) + if not isinstance(num_users, (int, np.integer)) or num_users <= 0: + raise ValueError("num_users must be a positive integer") + users = _indices(training_users, "training_users", num_users) + items = _indices(training_items, "training_items", len(self.features)) + if users.shape != items.shape: + raise ValueError("training user/item events must align") + if not isinstance(feature_source, str) or not feature_source: + raise ValueError("feature_source must be a nonempty string") + profiles = np.zeros((num_users, self.features.shape[1]), dtype="float64") + np.add.at(profiles, users, self.features[items]) + self.profiles = _unit_rows(profiles) + self.feature_source = feature_source + + def score(self, query_index, user_index, candidate_items): + if user_index is None or not 0 <= user_index < len(self.profiles): + raise ValueError("user_index is outside the training profile space") + return self.features[list(candidate_items)] @ self.profiles[user_index] + + @property + def metadata(self): + return {"resolver": "training_history_cosine", "source": "training_events", + "feature_source": self.feature_source, + "feature_join": "global_item_index", "zero_profile": "item_index_tie", + "normalized_features_sha256": hashlib.sha256(self.features.tobytes()).hexdigest()} + + +class CustomScorer(SemanticIDResolver): + """Adapt ``callback(query_index, user_index, candidate_items)`` to a resolver.""" + + def __init__(self, callback): + if not callable(callback): + raise ValueError("callback must be callable") + self.callback = callback + + def score(self, query_index, user_index, candidate_items): + return self.callback(query_index, user_index, candidate_items) + + +def validate_prediction(catalog, prediction): + """Check raw prediction validity against the admitted frozen catalog.""" + if not isinstance(catalog, SemanticIDCatalog) or not isinstance(prediction, SemanticIDPrediction): + raise ValueError("expected SemanticIDCatalog and SemanticIDPrediction") + if prediction.address_sids.shape[1] != catalog.address_sids.shape[1]: + raise ValueError("prediction address width differs from catalog") + expected = [tuple(row) in catalog.address_to_items for row in prediction.address_sids] + if not np.array_equal(prediction.catalog_validity, expected): + raise ValueError("prediction catalog_validity differs from catalog") + + +def resolve_semantic_ids(catalog, prediction, resolver, *, user_index=None, + query_index=None): + """Return item/None rank slots from raw predictions for inference or replay. + + Invalid and repeated addresses consume empty slots. Singleton buckets bypass + scoring. This operation returns no fillers, changes no input, and exposes no + target to the resolver. Choose a complete-address catalog matching the model. + """ + validate_prediction(catalog, prediction) + if not isinstance(resolver, SemanticIDResolver): + raise ValueError("resolver must be a SemanticIDResolver") + slots, seen = [], set() + for row, valid in zip(prediction.address_sids, prediction.catalog_validity): + address = tuple(int(value) for value in row) + if not valid or address in seen: + slots.append(None) + continue + seen.add(address) + items = catalog.address_to_items[address] + if len(items) == 1: + slots.append(items[0]) + continue + scores = np.asarray(resolver.score(query_index, user_index, items), dtype="float64") + if scores.shape != (len(items),) or not np.all(np.isfinite(scores)): + raise ValueError("resolver must return one finite score per candidate") + slots.append(min(zip(items, scores), key=lambda pair: (-pair[1], pair[0]))[0]) + return tuple(slots) diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index 5b95a2aae..cfe8b7ba6 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -16,6 +16,7 @@ from .recommender import Recommender from .recommender import NextBasketRecommender from .recommender import NextItemRecommender +from .recommender import SemanticIDRecommender from .amr import AMR from .ann import AnnoyANN @@ -94,3 +95,10 @@ from .vbpr import VBPR from .vmf import VMF from .wmf import WMF + +from .semantic_id import ( + SemanticIDCatalog, + SemanticIDPrediction, + SemanticIDTrace, + SemanticIDTraceRecord, +) diff --git a/cornac/models/diffgrm/diffgrm.py b/cornac/models/diffgrm/diffgrm.py index 2734a33ef..5a3cd493b 100644 --- a/cornac/models/diffgrm/diffgrm.py +++ b/cornac/models/diffgrm/diffgrm.py @@ -360,12 +360,15 @@ def cpd_decode_batch( digit_order=None, greedy_final=False, return_diagnostics=False, + return_raw=False, ): """Batched confidence-prioritized decoding over digit/code assignments. At every step, every still-masked digit competes globally. ``constrained`` additionally removes partial assignments that cannot lead to a catalog semantic ID. Complete duplicate IDs are collapsed by maximum path score. + When ``return_raw`` is true, the additional result retains every final + beam path, including invalid and duplicate addresses, before that collapse. """ device = memory.device batch_size = memory.size(0) @@ -477,14 +480,24 @@ def cpd_decode_batch( catalog_set = { tuple(row) for row in np.asarray(catalog_codes).tolist() } - batch_codes, batch_scores, batch_diagnostics = [], [], [] + batch_codes, batch_scores, batch_diagnostics, batch_raw = [], [], [], [] for row in range(batch_size): complete = {} + raw_codes = [] + raw_scores = [] + raw_validity = [] for codes, score in zip( beams[row].cpu().tolist(), beam_scores[row].cpu().tolist() ): key = tuple(codes) - if -1 in key or not np.isfinite(score): + is_complete = -1 not in key and np.isfinite(score) + is_valid = is_complete and ( + catalog_set is None or key in catalog_set + ) + raw_codes.append(key) + raw_scores.append(float(score)) + raw_validity.append(bool(is_valid)) + if not is_complete: continue complete[key] = max(complete.get(key, -float("inf")), score) best = { @@ -504,9 +517,20 @@ def cpd_decode_batch( "duplicate_paths": len(beams[row]) - len(complete), } ) + batch_raw.append( + { + "address_sids": raw_codes, + "scores": raw_scores, + "catalog_validity": raw_validity, + } + ) + if return_diagnostics and return_raw: + return batch_codes, batch_scores, batch_diagnostics, batch_raw if return_diagnostics: return batch_codes, batch_scores, batch_diagnostics + if return_raw: + return batch_codes, batch_scores, batch_raw return batch_codes, batch_scores @@ -522,6 +546,7 @@ def cpd_decode( digit_order=None, greedy_final=False, return_diagnostics=False, + return_raw=False, ): """Single-history wrapper around :func:`cpd_decode_batch`.""" if memory.size(0) != 1: @@ -537,9 +562,16 @@ def cpd_decode( digit_order=digit_order, greedy_final=greedy_final, return_diagnostics=return_diagnostics, + return_raw=return_raw, ) + if return_diagnostics and return_raw: + codes, scores, diagnostics, raw = result + return codes[0], scores[0], diagnostics[0], raw[0] if return_diagnostics: codes, scores, diagnostics = result return codes[0], scores[0], diagnostics[0] + if return_raw: + codes, scores, raw = result + return codes[0], scores[0], raw[0] codes, scores = result return codes[0], scores[0] diff --git a/cornac/models/diffgrm/recom_diffgrm.py b/cornac/models/diffgrm/recom_diffgrm.py index 1841d891e..2b0cf50ec 100644 --- a/cornac/models/diffgrm/recom_diffgrm.py +++ b/cornac/models/diffgrm/recom_diffgrm.py @@ -20,7 +20,11 @@ import numpy as np from tqdm.auto import trange -from cornac.models.recommender import NextItemRecommender +from ..recommender import SemanticIDRecommender +from ..semantic_id import ( + SemanticIDCatalog, + SemanticIDPrediction, +) from ...utils import get_rng @@ -82,7 +86,7 @@ def _diffgrm_session_iter( ) -class DiffGRM(NextItemRecommender): +class DiffGRM(SemanticIDRecommender): """DiffGRM: Diffusion-based Generative Recommendation Model. DiffGRM combines parallel OPQ semantic IDs (PSE), on-policy @@ -618,6 +622,7 @@ def _fit_model(self, torch, val_set): self.model.load_state_dict(best_state) def fit(self, train_set, val_set=None): + self.__dict__.pop("semantic_id_catalog_", None) super().fit(train_set, val_set) if not self.trainable: return self @@ -688,12 +693,12 @@ def _ensure_device(self, torch): if next(self.model.parameters()).device != requested: self.model.to(requested) - def _score_history(self, history_items): + def _score_history(self, history_items, n_candidates=None, return_raw=False): import torch from .diffgrm import cpd_decode - if not history_items: + if len(history_items) == 0: return np.ones(self.total_items, dtype="float") self._ensure_device(torch) history = list(history_items)[-self.max_len :] @@ -709,11 +714,11 @@ def _score_history(self, history_items): decode_start = time.perf_counter() with torch.no_grad(): memory, padding_mask = self.model.encode_history(inputs, masks) - codes, path_scores, diagnostics = cpd_decode( + decoded = cpd_decode( self.model, memory, padding_mask, - beam_size=self.beam_size, + beam_size=max(self.beam_size, n_candidates or 0), catalog_codes=self.sid_table, valid_code_set=self.sid_to_items_, constrained=self.scoring == "catalog", @@ -722,9 +727,19 @@ def _score_history(self, history_items): else None, greedy_final=self.scoring == "released", return_diagnostics=True, + return_raw=return_raw, ) + codes, path_scores, diagnostics = decoded[:3] self.last_decode_time_ = time.perf_counter() - decode_start self.last_decode_diagnostics_ = diagnostics + if return_raw: + raw = decoded[3] + return SemanticIDPrediction( + np.asarray(raw["address_sids"], dtype="int64").reshape(-1, self.n_digit), + raw["scores"], raw["catalog_validity"], + dict(diagnostics, scoring=self.scoring, + beam_size=max(self.beam_size, n_candidates or 0)), + ) return self._decoded_item_scores(codes, path_scores) def _decoded_item_scores(self, codes, path_scores): @@ -740,3 +755,26 @@ def _decoded_item_scores(self, codes, path_scores): def score(self, user_idx, history_items, **kwargs): return self._score_history(history_items) + + def semantic_id_catalog(self): + """Export all complete IDs independently of native collision policy.""" + if not hasattr(self, "sid_table"): + raise ValueError("DiffGRM must be fitted before exporting its SID catalog") + cardinalities = (self.codebook_size,) * self.n_digit + if not hasattr(self, "semantic_id_catalog_"): + self.semantic_id_catalog_ = SemanticIDCatalog( + np.arange(len(self.sid_table)), self.sid_table, self.sid_table, + cardinalities, cardinalities, + ) + return self.semantic_id_catalog_ + + def predict_semantic_ids(self, history_items, **context): + """Return final CPD paths before validity filtering or deduplication.""" + if len(history_items) == 0: + return SemanticIDPrediction( + np.empty((0, self.semantic_id_catalog().address_sids.shape[1]), dtype="int64"), + [], [], {"empty_history": True}, + ) + return self._score_history( + history_items, context.get("n_candidates"), return_raw=True + ) diff --git a/cornac/models/letter/recom_letter.py b/cornac/models/letter/recom_letter.py index b32cde156..fe9772216 100644 --- a/cornac/models/letter/recom_letter.py +++ b/cornac/models/letter/recom_letter.py @@ -21,6 +21,7 @@ from tqdm.auto import trange from ..tiger.recom_tiger import TIGER +from ..semantic_id import SemanticIDCatalog, prediction_from_catalog_scores def _generator_optimizer_groups(model, weight_decay): @@ -325,6 +326,7 @@ def _tokenize(self, torch, feats_t): def _build_semantic_ids(self, codes): """Build the official fixed-length IDs without TIGER's dedup level.""" + self.__dict__.pop("semantic_id_catalog_", None) self.sid_table = np.asarray(codes, dtype="int64") self.level_sizes = [self.rqvae_codebook_size] * self.rqvae_num_levels children = [defaultdict(set) for _ in self.level_sizes] @@ -521,11 +523,9 @@ def lr_lambda(step): if best_state is not None: self.model.load_state_dict(best_state) - def score(self, user_idx, history_items, **kwargs): + def _predict_history(self, history_items, n_candidates=None, return_raw=False): import torch - if len(history_items) == 0: - return np.ones(self.total_items, dtype="float") self._ensure_device(torch) enc_tokens, enc_mask = self._encoder_batch( torch, [list(history_items)[-self.max_len :]] @@ -534,8 +534,10 @@ def score(self, user_idx, history_items, **kwargs): with torch.no_grad(): if self.scoring == "beam": beams, log_probs = self.model.generate_beam( - enc_tokens, enc_mask, self.n_beams, self.prefix_children + enc_tokens, enc_mask, max(self.n_beams, n_candidates or 0), self.prefix_children ) + if return_raw: + return self._beam_prediction(beams, log_probs, n_candidates) scores = np.full(self.total_items, -1e10, dtype="float") for sid, log_prob in zip(beams, log_probs): scores[self.sid_to_item[sid]] = log_prob @@ -546,4 +548,20 @@ def score(self, user_idx, history_items, **kwargs): scores = self.model.score_all_items( enc_tokens, enc_mask, sid_table, self.scoring_batch_size ).astype("float") + if return_raw: + return prediction_from_catalog_scores( + self.semantic_id_catalog(), scores, + {"scoring": "exact", "address_score_aggregation": "max_member"}, + ) return scores + + def semantic_id_catalog(self): + """Export LETTER's suffix-free IDs, preserving collision buckets.""" + if not hasattr(self, "sid_table"): + raise ValueError("LETTER must be fitted before exporting its SID catalog") + if not hasattr(self, "semantic_id_catalog_"): + self.semantic_id_catalog_ = SemanticIDCatalog( + np.arange(len(self.sid_table)), self.sid_table, self.sid_table, + tuple(self.level_sizes), tuple(self.level_sizes), + ) + return self.semantic_id_catalog_ diff --git a/cornac/models/recommender.py b/cornac/models/recommender.py index 5d8514e95..73136c09d 100644 --- a/cornac/models/recommender.py +++ b/cornac/models/recommender.py @@ -764,3 +764,21 @@ def score(self, user_idx, history_items, **kwargs): """ raise NotImplementedError("The algorithm is not able to make score prediction!") + + +class SemanticIDRecommender(NextItemRecommender): + """Optional interface for recommenders that generate Semantic IDs.""" + + def semantic_id_catalog(self): + """Return the fitted :class:`~cornac.models.semantic_id.SemanticIDCatalog`.""" + raise NotImplementedError + + def predict_semantic_ids(self, history_items, **context): + """Return raw predictions before resolution, preserving decoder order. + + ``context`` may contain ``user_index``, ``query_index``, session/history + positions, and ``n_candidates`` (a requested minimum output budget). + Adapters should honor the budget where supported and may return fewer + candidates when decoding terminates. Never fill from ungenerated items. + """ + raise NotImplementedError diff --git a/cornac/models/rpg/recom_rpg.py b/cornac/models/rpg/recom_rpg.py index c66f0f12a..6e20dbb0a 100644 --- a/cornac/models/rpg/recom_rpg.py +++ b/cornac/models/rpg/recom_rpg.py @@ -18,7 +18,12 @@ import numpy as np from tqdm.auto import trange -from cornac.models.recommender import NextItemRecommender +from ..recommender import SemanticIDRecommender +from ..semantic_id import ( + SemanticIDCatalog, + SemanticIDPrediction, + prediction_from_catalog_scores, +) from ...utils import get_rng @@ -98,7 +103,7 @@ def _rpg_session_iter( ) -class RPG(NextItemRecommender): +class RPG(SemanticIDRecommender): """RPG: Generating Long Semantic IDs in Parallel for Recommendation. RPG is the architectural counterpoint to TIGER. Instead of short *ordered* @@ -602,6 +607,7 @@ def _fit_backbone(self, torch, val_set): self.model.load_state_dict(best_state) def fit(self, train_set, val_set=None): + self.__dict__.pop("semantic_id_catalog_", None) super().fit(train_set, val_set) if not self.trainable: return self @@ -646,7 +652,7 @@ def _ensure_device(self, torch): if next(self.model.parameters()).device.type != torch.device(self.device_).type: self.model.to(self.device_) - def score(self, user_idx, history_items, **kwargs): + def _predict_history(self, history_items, n_candidates=None, return_raw=False): import torch from .rpg import graph_decode @@ -663,9 +669,13 @@ def score(self, user_idx, history_items, **kwargs): logits = self.model.next_item_logits(input_ids, attn_mask)[0] # (M, K) if self.scoring == "exact": - return ( - self._exact_item_scores(logits)[0].cpu().numpy().astype("float") - ) + scores = self._exact_item_scores(logits)[0].cpu().numpy().astype("float") + if return_raw: + return prediction_from_catalog_scores( + self.semantic_id_catalog(), scores, + {"scoring": "exact", "address_score_aggregation": "max_member"}, + ) + return scores # graph-guided beam decoding over the final propagated beam flat = logits.reshape(-1).cpu().numpy() @@ -673,10 +683,42 @@ def score(self, user_idx, history_items, **kwargs): flat, self.adjacency, self.item_tokens, - self.n_beams, + max(self.n_beams, n_candidates or 0), self.propagation_steps, self.graph_rng, ) + if return_raw: + return SemanticIDPrediction( + self.sid_table[candidates], candidate_scores, + np.ones(len(candidates), dtype=bool), + {"scoring": "graph", "beam_size": max(self.n_beams, n_candidates or 0)}, + ) scores = np.full(self.total_items, -1e10, dtype="float") scores[candidates] = candidate_scores return scores + + def semantic_id_catalog(self): + """Export un-offset OPQ codes with complete collision buckets.""" + if not hasattr(self, "sid_table"): + raise ValueError("RPG must be fitted before exporting its SID catalog") + cardinalities = (self.codebook_size,) * self.n_codebook + if not hasattr(self, "semantic_id_catalog_"): + self.semantic_id_catalog_ = SemanticIDCatalog( + np.arange(len(self.sid_table)), self.sid_table, self.sid_table, + cardinalities, cardinalities, + ) + return self.semantic_id_catalog_ + + def predict_semantic_ids(self, history_items, **context): + """Retain final graph candidate order, using the native mutable RNG.""" + if len(history_items) == 0: + return SemanticIDPrediction( + np.empty((0, self.semantic_id_catalog().address_sids.shape[1]), dtype="int64"), + [], [], {"empty_history": True}, + ) + return self._predict_history( + history_items, context.get("n_candidates"), return_raw=True + ) + + def score(self, user_idx, history_items, **kwargs): + return self._predict_history(history_items) diff --git a/cornac/models/semantic_id.py b/cornac/models/semantic_id.py new file mode 100644 index 000000000..f5cac0cfc --- /dev/null +++ b/cornac/models/semantic_id.py @@ -0,0 +1,384 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Model-neutral contracts for Semantic-ID recommendation.""" + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType + +import numpy as np + + +def _readonly_array(values, dtype, ndim, name): + original = np.asarray(values) + if dtype == "int64" and original.size and not np.issubdtype(original.dtype, np.integer): + raise ValueError(f"{name} must contain integers") + array = np.array(values, dtype=dtype, order="C", copy=True) + if array.ndim != ndim: + raise ValueError(f"{name} must be {ndim}-dimensional") + return np.frombuffer(array.tobytes(), dtype=array.dtype).reshape(array.shape) + + +def _freeze_native(value, name="metadata"): + if isinstance(value, np.generic): + return value.item() + if isinstance(value, Mapping): + frozen = {} + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError(f"{name} keys must be strings") + frozen[key] = _freeze_native(item, name) + return MappingProxyType(frozen) + if isinstance(value, (list, tuple)): + return tuple(_freeze_native(item, name) for item in value) + if value is None or isinstance(value, (str, int, float, bool)): + return value + raise ValueError(f"{name} must contain only Python-native values") + + +def _thaw_native(value): + if isinstance(value, Mapping): + return {key: _thaw_native(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thaw_native(item) for item in value] + return value + + +def _validate_cardinalities(cardinalities, width, name): + if any(isinstance(value, (bool, np.bool_)) or not isinstance(value, (int, np.integer)) + for value in cardinalities): + raise ValueError(f"{name} must contain integers") + values = tuple(int(value) for value in cardinalities) + if len(values) != width: + raise ValueError(f"{name} must contain one value per SID position") + if any(value <= 0 for value in values): + raise ValueError(f"{name} values must be positive") + return values + + +def _validate_sid_range(sids, cardinalities, name): + if sids.size == 0: + return + if sids.min() < 0: + raise ValueError(f"{name} values must be non-negative") + for position, cardinality in enumerate(cardinalities): + if sids[:, position].max() >= cardinality: + raise ValueError( + f"{name} position {position} contains a value outside " + f"[0, {cardinality})" + ) + + +@dataclass(frozen=True, eq=False) +class SemanticIDCatalog: + """Immutable item-to-SID catalog with an explicit one-to-many reverse map. + + ``tokenizer_sids`` stores the representation before collision suffixing or + reassignment. ``address_sids`` stores the complete codes generated by the model. + The two arrays may have different widths. + """ + + item_indices: np.ndarray + tokenizer_sids: np.ndarray + address_sids: np.ndarray + tokenizer_codebook_cardinalities: tuple + address_codebook_cardinalities: tuple + mapping_policy: str = "preserve_all" + metadata: Mapping = field(default_factory=dict) + _address_to_items: Mapping = field(init=False, repr=False, compare=False) + _item_to_row: Mapping = field(init=False, repr=False, compare=False) + _stable_hash: str = field(init=False, repr=False, compare=False) + + def __post_init__(self): + item_indices = _readonly_array( + self.item_indices, dtype="int64", ndim=1, name="item_indices" + ) + tokenizer_sids = _readonly_array( + self.tokenizer_sids, dtype="int64", ndim=2, name="tokenizer_sids" + ) + address_sids = _readonly_array( + self.address_sids, dtype="int64", ndim=2, name="address_sids" + ) + if tokenizer_sids.shape[0] != len(item_indices): + raise ValueError("tokenizer_sids must contain one row per item") + if address_sids.shape[0] != len(item_indices): + raise ValueError("address_sids must contain one row per item") + if len(item_indices) == 0: + raise ValueError("a Semantic-ID catalog must contain at least one item") + if tokenizer_sids.shape[1] == 0 or address_sids.shape[1] == 0: + raise ValueError("SID arrays must contain at least one position") + if len(np.unique(item_indices)) != len(item_indices): + raise ValueError("item_indices must not contain duplicates") + if item_indices.size and item_indices.min() < 0: + raise ValueError("item_indices must be non-negative") + + tokenizer_cardinalities = _validate_cardinalities( + self.tokenizer_codebook_cardinalities, + tokenizer_sids.shape[1], + "tokenizer_codebook_cardinalities", + ) + address_cardinalities = _validate_cardinalities( + self.address_codebook_cardinalities, + address_sids.shape[1], + "address_codebook_cardinalities", + ) + _validate_sid_range(tokenizer_sids, tokenizer_cardinalities, "tokenizer_sids") + _validate_sid_range(address_sids, address_cardinalities, "address_sids") + if not isinstance(self.mapping_policy, str) or not self.mapping_policy: + raise ValueError("mapping_policy must be a non-empty string") + metadata = _freeze_native(self.metadata) + + address_to_items = {} + item_to_row = {} + for row, (item_index, address) in enumerate(zip(item_indices, address_sids)): + key = tuple(int(value) for value in address) + address_to_items.setdefault(key, []).append(int(item_index)) + item_to_row[int(item_index)] = row + address_to_items = MappingProxyType( + { + address: tuple(sorted(items)) + for address, items in address_to_items.items() + } + ) + + object.__setattr__(self, "item_indices", item_indices) + object.__setattr__(self, "tokenizer_sids", tokenizer_sids) + object.__setattr__(self, "address_sids", address_sids) + object.__setattr__( + self, "tokenizer_codebook_cardinalities", tokenizer_cardinalities + ) + object.__setattr__( + self, "address_codebook_cardinalities", address_cardinalities + ) + object.__setattr__(self, "metadata", metadata) + object.__setattr__(self, "_address_to_items", address_to_items) + object.__setattr__(self, "_item_to_row", MappingProxyType(item_to_row)) + object.__setattr__(self, "_stable_hash", self._compute_hash()) + + def __deepcopy__(self, memo): + """Reuse this immutable value when a fitted model is copied for saving.""" + memo[id(self)] = self + return self + + def __reduce__(self): + """Rebuild proxy-backed state through the validated public constructor.""" + return ( + type(self), + ( + self.item_indices, + self.tokenizer_sids, + self.address_sids, + self.tokenizer_codebook_cardinalities, + self.address_codebook_cardinalities, + self.mapping_policy, + _thaw_native(self.metadata), + ), + ) + + @property + def address_to_items(self): + """Mapping from generated address SID to sorted global item indices.""" + return self._address_to_items + + @property + def stable_hash(self): + """Stable SHA-256 identity of the arrays and catalog policy.""" + return self._stable_hash + + def tokenizer_sid_for_item(self, item_index): + row = self._item_to_row[int(item_index)] + return tuple(int(value) for value in self.tokenizer_sids[row]) + + def address_sid_for_item(self, item_index): + row = self._item_to_row[int(item_index)] + return tuple(int(value) for value in self.address_sids[row]) + + def _compute_hash(self): + digest = hashlib.sha256(b"cornac-semantic-id-catalog-v1\0") + for name, array in ( + ("item_indices", self.item_indices), + ("tokenizer_sids", self.tokenizer_sids), + ("address_sids", self.address_sids), + ): + canonical = np.ascontiguousarray(array, dtype="= len(scores): + raise ValueError("item_scores does not cover every catalog item") + address_scores = {} + for item_index, address_row in zip(catalog.item_indices, catalog.address_sids): + address = tuple(int(value) for value in address_row) + score = float(scores[item_index]) + address_scores[address] = max(address_scores.get(address, -np.inf), score) + ranked = sorted(address_scores.items(), key=lambda pair: (-pair[1], pair[0])) + diagnostics = dict(decoder_diagnostics or {}) + diagnostics.update( + { + "scored_catalog_items": len(catalog.item_indices), + "unique_complete_sids": len(ranked), + "duplicate_catalog_addresses": len(catalog.item_indices) - len(ranked), + "invalid_sids": 0, + } + ) + return SemanticIDPrediction( + address_sids=np.asarray([address for address, _ in ranked], dtype="int64"), + scores=np.asarray([score for _, score in ranked], dtype="float64"), + catalog_validity=np.ones(len(ranked), dtype="bool"), + decoder_diagnostics=diagnostics, + ) + + +@dataclass(frozen=True, eq=False) +class SemanticIDTraceRecord: + """One evaluation query paired with its immutable raw prediction.""" + + query_index: int + user_index: int + session_index: int + evaluation_position: int + history_start_position: int + history_end_position: int + target_item_index: int + target_tokenizer_sid: tuple + target_address_sid: tuple + prediction: SemanticIDPrediction + + def __post_init__(self): + integer_fields = ( + "query_index", + "user_index", + "session_index", + "evaluation_position", + "history_start_position", + "history_end_position", + "target_item_index", + ) + for name in integer_fields: + value = getattr(self, name) + if (isinstance(value, (bool, np.bool_)) or + not isinstance(value, (int, np.integer)) or value < 0): + raise ValueError(f"{name} must be a nonnegative integer") + object.__setattr__(self, name, int(value)) + if self.query_index < 0 or self.target_item_index < 0: + raise ValueError("query_index and target_item_index must be non-negative") + if not 0 <= self.history_start_position <= self.history_end_position: + raise ValueError("history positions must define a non-negative interval") + if self.history_end_position != self.evaluation_position: + raise ValueError( + "history_end_position must equal the target evaluation_position" + ) + tokenizer_sid = tuple(int(value) for value in self.target_tokenizer_sid) + address_sid = tuple(int(value) for value in self.target_address_sid) + if not tokenizer_sid or not address_sid: + raise ValueError("target SIDs must not be empty") + if not isinstance(self.prediction, SemanticIDPrediction): + raise ValueError("prediction must be a SemanticIDPrediction") + object.__setattr__(self, "target_tokenizer_sid", tokenizer_sid) + object.__setattr__(self, "target_address_sid", address_sid) + + +@dataclass(frozen=True, eq=False) +class SemanticIDTrace: + """Immutable raw predictions tied to one frozen catalog.""" + + catalog_hash: str + records: tuple + metadata: Mapping = field(default_factory=dict) + + def __reduce__(self): + return (type(self), (self.catalog_hash, self.records, + _thaw_native(self.metadata))) + + def __post_init__(self): + records = tuple(self.records) + if any(not isinstance(record, SemanticIDTraceRecord) for record in records): + raise ValueError("records must contain SemanticIDTraceRecord instances") + query_indices = [record.query_index for record in records] + if len(set(query_indices)) != len(query_indices): + raise ValueError("query_index must be unique within a trace") + if not isinstance(self.catalog_hash, str) or not self.catalog_hash: + raise ValueError("catalog_hash must be a non-empty string") + object.__setattr__(self, "records", records) + object.__setattr__(self, "metadata", _freeze_native(self.metadata)) diff --git a/cornac/models/tiger/recom_tiger.py b/cornac/models/tiger/recom_tiger.py index 1ec31bbcd..ea5f3c2f8 100644 --- a/cornac/models/tiger/recom_tiger.py +++ b/cornac/models/tiger/recom_tiger.py @@ -19,7 +19,12 @@ import numpy as np from tqdm.auto import trange -from cornac.models.recommender import NextItemRecommender +from ..recommender import SemanticIDRecommender +from ..semantic_id import ( + SemanticIDCatalog, + SemanticIDPrediction, + prediction_from_catalog_scores, +) from ...utils import get_rng from ..seq_utils import session_seq_iter @@ -29,7 +34,7 @@ SUPPORTED_LR_SCHEDULES = ("constant", "cosine") -class TIGER(NextItemRecommender): +class TIGER(SemanticIDRecommender): """TIGER: Recommender Systems with Generative Retrieval. Two-stage generative retrieval with semantic IDs: (1) an RQ-VAE quantizes @@ -361,6 +366,7 @@ def _tokenize(self, torch, feats_t): ).astype("int64") def _build_semantic_ids(self, codes): + self.__dict__.pop("semantic_id_catalog_", None) # extra level disambiguating items that share the same code tuple counters = defaultdict(int) dedup = np.zeros(len(codes), dtype="int64") @@ -531,6 +537,7 @@ def _validate(self, val_sessions, metric): return float(np.mean(results)) if results else 0.0 def fit(self, train_set, val_set=None): + self.__dict__.pop("semantic_id_catalog_", None) super().fit(train_set, val_set) if not self.trainable: return self @@ -564,11 +571,9 @@ def _ensure_device(self, torch): if next(self.model.parameters()).device.type != torch.device(self.device_).type: self.model.to(self.device_) - def score(self, user_idx, history_items, **kwargs): + def _predict_history(self, history_items, n_candidates=None, return_raw=False): import torch - if len(history_items) == 0: - return np.ones(self.total_items, dtype="float") self._ensure_device(torch) hist = list(history_items)[-self.max_len :] hist = [self.pad_idx] * (self.max_len - len(hist)) + hist @@ -581,7 +586,9 @@ def score(self, user_idx, history_items, **kwargs): self.model.eval() with torch.no_grad(): if self.scoring == "beam": - beams, logps = self.model.generate_beam(enc_tokens_t, enc_mask_t, self.n_beams, self.prefix_children) + beams, logps = self.model.generate_beam(enc_tokens_t, enc_mask_t, max(self.n_beams, n_candidates or 0), self.prefix_children) + if return_raw: + return self._beam_prediction(beams, logps, n_candidates) scores = np.full(self.total_items, -1e10, dtype="float") for sid, lp in zip(beams, logps): scores[self.sid_to_item[sid]] = lp @@ -590,4 +597,45 @@ def score(self, user_idx, history_items, **kwargs): scores = self.model.score_all_items( enc_tokens_t, enc_mask_t, sid_table_t, self.scoring_batch_size ).astype("float") + if return_raw: + return prediction_from_catalog_scores( + self.semantic_id_catalog(), scores, + {"scoring": "exact", "address_score_aggregation": "max_member"}, + ) return scores + + def semantic_id_catalog(self): + """Export tokenizer prefixes and complete, suffix-bearing addresses.""" + if not hasattr(self, "sid_table"): + raise ValueError("TIGER must be fitted before exporting its SID catalog") + if not hasattr(self, "semantic_id_catalog_"): + self.semantic_id_catalog_ = SemanticIDCatalog( + np.arange(len(self.sid_table)), self.sid_table[:, :-1], self.sid_table, + tuple(self.level_sizes[:-1]), tuple(self.level_sizes), + mapping_policy="disambiguation_suffix", + ) + return self.semantic_id_catalog_ + + def _beam_prediction(self, beams, scores, n_candidates): + catalog = self.semantic_id_catalog() + addresses = np.asarray(beams, dtype="int64").reshape( + -1, catalog.address_sids.shape[1] + ) + return SemanticIDPrediction( + addresses, scores, + [tuple(address) in catalog.address_to_items for address in addresses], + {"scoring": "beam", "beam_size": max(self.n_beams, n_candidates or 0)}, + ) + + def predict_semantic_ids(self, history_items, **context): + """Decode once before item conversion, retaining invalid/duplicate slots.""" + if len(history_items) == 0: + return self._beam_prediction([], [], context.get("n_candidates")) + return self._predict_history( + history_items, context.get("n_candidates"), return_raw=True + ) + + def score(self, user_idx, history_items, **kwargs): + if len(history_items) == 0: + return np.ones(self.total_items, dtype="float") + return self._predict_history(history_items) diff --git a/docs/source/api_ref/eval_methods.rst b/docs/source/api_ref/eval_methods.rst index d482f66da..57afec443 100644 --- a/docs/source/api_ref/eval_methods.rst +++ b/docs/source/api_ref/eval_methods.rst @@ -32,3 +32,58 @@ Timestamp Split --------------- .. automodule:: cornac.eval_methods.timestamp_split :members: + +Semantic ID Evaluation +---------------------- + +``SemanticIDEvaluation`` evaluates TIGER, LETTER, RPG and DiffGRM through the same +raw Semantic ID predictions. It reports SID accuracy, analytical +Collision-Corrected Evaluation (CCE), and exact-item accuracy under explicit +resolvers. Supported metrics are ``Recall(k)`` and ``NDCG(k)`` with positive +integer cutoffs; result names include ``SID/Recall@10``, ``CCE/NDCG@10`` and +``Item[popularity]/Recall@10``. ``user_based=True`` averages queries per user +before averaging users. + +CCE expands each complete ID into its full item bucket and averages over uniform +member permutations. Earlier buckets consume their full widths. Invalid and +duplicate addresses preserve empty rank slots. A resolver selects one item per +bucket while preserving address order. Complete IDs include any disambiguating +suffix; injective catalogs give identical views under the same slot rules. + +Popularity uses training events only. Supplying ``item_features`` adds history +cosine: normalize item vectors, sum each user's training-event vectors (including +repeats), normalize that profile and score bucket members by cosine. Features +must align with global Cornac item indices. Ties, including zero profiles, choose +the smallest item index. Pass fitted resolvers through ``resolvers`` or implement +``SemanticIDResolver.score(query_index, user_index, candidate_items)``; custom +state must also use training-only information. ``resolve_semantic_ids`` exposes +the same operation for inference. + +For an existing Semantic ID model and item-feature modality:: + + evaluation = SemanticIDEvaluation.from_splits( + train, test, fmt="USIT", item_feature=features) + Experiment(evaluation, [TIGER(device="cpu")], [Recall(10), NDCG(10)]).run() + +See ``examples/semantic_id_evaluation_example.py`` for a standalone CPU example. +``Result.semantic_id_trace`` holds the predictions, and +``Result.semantic_id_evaluation`` holds per-query arrays and protocol metadata. + +Adapter authors subclass ``cornac.models.recommender.SemanticIDRecommender`` and +implement ``semantic_id_catalog()`` and ``predict_semantic_ids(history_items, +**context)``. Export complete addresses separately from tokenizer prefixes and +preserve raw decoder order, invalid slots and duplicates before item conversion. +The evaluator requests ``n_candidates=max(k)``; honor it where supported and keep +shorter output without fillers. ``prediction_from_catalog_scores`` supports exact +scoring by maximum member score per address, with lexicographic address ties. +The contract covers fixed-width integer addresses, a frozen catalog and one +next-item target per query. + +.. automodule:: cornac.eval_methods.semantic_id_evaluation + :members: + +.. automodule:: cornac.eval_methods.sid_evaluation + :members: + +.. automodule:: cornac.eval_methods.sid_resolvers + :members: diff --git a/docs/source/api_ref/models.rst b/docs/source/api_ref/models.rst index 030b88912..e7593e53f 100644 --- a/docs/source/api_ref/models.rst +++ b/docs/source/api_ref/models.rst @@ -333,3 +333,11 @@ Weighted Matrix Factorization (WMF) ----------------------------------- .. automodule:: cornac.models.wmf.recom_wmf :members: + +Semantic ID Model Contracts +-------------------------- +.. autoclass:: cornac.models.recommender.SemanticIDRecommender + :members: + +.. automodule:: cornac.models.semantic_id + :members: diff --git a/examples/README.md b/examples/README.md index 44a1be0bb..da2f09dbd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -122,6 +122,8 @@ ## Next-Item Algorithms +[semantic_id_evaluation_example.py](semantic_id_evaluation_example.py) - Synthetic example of shared SID, CCE and explicit item-resolver evaluation through Experiment. + [spop_yoochoose.py](spop_yoochoose.py) - Next-item recommendation based on item popularity. [gru4rec_yoochoose.py](gru4rec_yoochoose.py) - Example of Session-based Recommendations with Recurrent Neural Networks (GRU4Rec). diff --git a/examples/semantic_id_evaluation_example.py b/examples/semantic_id_evaluation_example.py new file mode 100644 index 000000000..f47638f14 --- /dev/null +++ b/examples/semantic_id_evaluation_example.py @@ -0,0 +1,56 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Backend-free, synthetic demonstration of shared Semantic ID evaluation.""" +import numpy as np + +import cornac +from cornac.eval_methods import SemanticIDEvaluation +from cornac.metrics import Recall, NDCG +from cornac.models import SemanticIDCatalog, SemanticIDPrediction, SemanticIDRecommender + + +class FrozenSIDExample(SemanticIDRecommender): + """A fixed generator for demonstrating the interface, not a trained model.""" + + def __init__(self): + super().__init__("FrozenSIDExample") + + def fit(self, train_set, val_set=None): + super().fit(train_set, val_set) + codes = np.arange(self.total_items).reshape(-1, 1) // 2 + cardinalities = (int(codes.max()) + 1,) + self.catalog = SemanticIDCatalog(np.arange(self.total_items), codes, + codes, cardinalities, cardinalities) + return self + + def semantic_id_catalog(self): + return self.catalog + + def predict_semantic_ids(self, history_items, **context): + addresses = np.asarray(sorted(self.catalog.address_to_items), dtype="int64") + return SemanticIDPrediction(addresses, np.zeros(len(addresses)), + np.ones(len(addresses), dtype=bool)) + + +if __name__ == "__main__": + train = [("u", "train", "a", 0), ("u", "train", "b", 1), + ("u", "train", "c", 2)] + test = [("u", "test", "a", 3), ("u", "test", "b", 4)] + evaluation = SemanticIDEvaluation.from_splits( + train, test, fmt="USIT", item_features=np.eye(3), + feature_source="synthetic identity vectors") + cornac.Experiment(evaluation, [FrozenSIDExample()], [Recall(2), NDCG(2)], + show_validation=False).run() diff --git a/tests/cornac/eval_methods/test_semantic_id_evaluation.py b/tests/cornac/eval_methods/test_semantic_id_evaluation.py new file mode 100644 index 000000000..c9706cafa --- /dev/null +++ b/tests/cornac/eval_methods/test_semantic_id_evaluation.py @@ -0,0 +1,330 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import itertools +import pickle +import subprocess +import sys + +import numpy as np +import pytest + +from cornac import Experiment +from cornac.eval_methods import (SemanticIDEvaluation, evaluate_semantic_id_trace, + TrainingPopularity, HistoryCosine, CustomScorer, resolve_semantic_ids) +from cornac.metrics import Recall, NDCG, HitRatio +from cornac.models import (SemanticIDCatalog, SemanticIDPrediction, + SemanticIDRecommender, SemanticIDTrace, SemanticIDTraceRecord) +from cornac.models.semantic_id import prediction_from_catalog_scores + + +def catalog(codes=((0,), (0,), (1,), (1,), (2,))): + codes = np.asarray(codes) + cardinalities = tuple(int(codes[:, i].max()) + 1 for i in range(codes.shape[1])) + return SemanticIDCatalog(np.arange(len(codes)), codes, codes, cardinalities, cardinalities) + + +def prediction(cat, addresses): + values = np.asarray(addresses, dtype="int64").reshape(-1, cat.address_sids.shape[1]) + return SemanticIDPrediction(values, np.arange(len(values), dtype=float), + [tuple(row) in cat.address_to_items for row in values]) + + +def trace(cat, target, addresses, user=0, query=0): + return SemanticIDTrace(cat.stable_hash, (SemanticIDTraceRecord(query, user, + 0, 1, 0, 1, target, cat.tokenizer_sid_for_item(target), + cat.address_sid_for_item(target), prediction(cat, addresses)),)) + + +def test_catalog_freezes_copies_hash_and_pickle(): + codes = np.array([[0], [0], [1]]) + cat = catalog(codes) + codes[0, 0] = 9 + assert cat.address_to_items[(0,)] == (0, 1) + with pytest.raises(ValueError): + cat.address_sids.setflags(write=True) + loaded = pickle.loads(pickle.dumps(cat)) + assert loaded.stable_hash == cat.stable_hash + assert loaded.address_to_items == cat.address_to_items + with pytest.raises(TypeError): + cat.address_to_items[(0,)] = (1,) + with pytest.raises(ValueError, match="integers"): + catalog(((0.5,), (1.,))) + + +def test_resolution_preserves_slots_members_ties_and_target_free_context(): + cat = catalog() + pred = prediction(cat, ((8,), (1,), (1,), (0,), (2,))) + calls = [] + def score(query, user, candidates): + calls.append((query, user, candidates)) + return np.zeros(len(candidates)) + slots = resolve_semantic_ids(cat, pred, CustomScorer(score), user_index=2, query_index=7) + assert slots == (None, 2, None, 0, 4) + assert calls == [(7, 2, (2, 3)), (7, 2, (0, 1))] + assert pred.address_sids.tolist() == [[8], [1], [1], [0], [2]] + for bad in (lambda *args: [1], lambda *args: [np.nan, 0]): + with pytest.raises(ValueError, match="finite score"): + resolve_semantic_ids(cat, pred, CustomScorer(bad)) + + +def test_training_popularity_and_cosine_multiplicity_and_zero_profile(): + cat = catalog() + pop = TrainingPopularity([1, 1, 0, 3], 5) + assert resolve_semantic_ids(cat, prediction(cat, [[0], [1]]), pop) == (1, 3) + features = np.array([[1, 0], [0, 1], [1, 0], [0, 1], [0, 0]], dtype=float) + scorer = HistoryCosine(features, [0, 0, 0], [0, 1, 1], 2, "fixed-test-features") + assert resolve_semantic_ids(cat, prediction(cat, [[0], [1]]), scorer, user_index=0) == (1, 3) + assert resolve_semantic_ids(cat, prediction(cat, [[0], [1]]), scorer, user_index=1) == (0, 2) + features[:] = 0 + assert scorer.features[0, 0] == 1 + assert scorer.metadata["feature_source"] == "fixed-test-features" + with pytest.raises(ValueError, match="align"): + HistoryCosine(np.ones((5, 2)), [0], [0, 1], 2) + with pytest.raises(ValueError): + TrainingPopularity([5], 5) + with pytest.raises(ValueError): + HistoryCosine([[np.nan]], [0], [0], 1) + + +@pytest.mark.parametrize("addresses", [((0,), (1,), (2,)), + ((9,), (0,), (0,), (1,), (2,)), ((1,), (0,)), (), ((2,),)]) +def test_cce_matches_exhaustive_expansion_oracle(addresses): + cat = catalog() + buckets, seen = [], set() + for address in addresses: + if address not in cat.address_to_items or address in seen: + buckets.append(((None,),)) + else: + buckets.append(tuple(itertools.permutations(cat.address_to_items[address]))) + seen.add(address) + lists = [sum(parts, ()) for parts in itertools.product(*buckets)] + cutoffs = tuple(range(1, 9)) + for target in cat.item_indices: + result = evaluate_semantic_id_trace(cat, trace(cat, int(target), addresses), cutoffs) + for column, cutoff in enumerate(cutoffs): + hits, ndcgs = [], [] + for items in lists: + rank = items.index(target) if target in items else None + hit = float(rank is not None and rank < cutoff) + hits.append(hit) + ndcgs.append(hit / np.log2(rank + 2) if rank is not None else 0.) + assert result["per_query"]["CCE"]["recall"][0, column] == pytest.approx(np.mean(hits)) + assert result["per_query"]["CCE"]["ndcg"][0, column] == pytest.approx(np.mean(ndcgs)) + + +def test_injective_complete_ids_despite_prefix_collisions(): + cat = SemanticIDCatalog([0, 1, 2], [[0], [0], [1]], [[0, 0], [0, 1], [1, 0]], (2,), (2, 2)) + result = evaluate_semantic_id_trace(cat, trace(cat, 1, ((9, 9), (0, 0), (0, 0), (0, 1))), + (1, 3, 4), {"popularity": TrainingPopularity([0, 0], 3)}) + for metric in ("recall", "ndcg"): + np.testing.assert_array_equal(result["per_query"]["SID"][metric], result["per_query"]["CCE"][metric]) + np.testing.assert_array_equal(result["per_query"]["SID"][metric], result["per_query"]["Item[popularity]"][metric]) + + +def test_replay_rejects_mismatches_and_bad_cutoffs(): + cat = catalog() + saved = trace(cat, 1, ((0,),)) + for cutoffs in ((0,), (1.2,), (True,), (), (1, 1), (np.inf,)): + with pytest.raises(ValueError): + evaluate_semantic_id_trace(cat, saved, cutoffs) + other = catalog(((0,), (1,), (1,), (1,), (2,))) + with pytest.raises(ValueError, match="catalog_hash"): + evaluate_semantic_id_trace(other, saved) + pred = SemanticIDPrediction([[0]], [1.], [False]) + with pytest.raises(ValueError, match="validity"): + resolve_semantic_ids(cat, pred, TrainingPopularity([], 5)) + + +def test_catalog_score_conversion_declares_max_member_aggregation(): + cat = catalog() + pred = prediction_from_catalog_scores(cat, [1, 3, 2, 0, 2]) + assert pred.address_sids.tolist() == [[0], [1], [2]] + assert pred.scores.tolist() == [3, 2, 2] + + +class FakeSID(SemanticIDRecommender): + def __init__(self): + super().__init__("FakeSID") + self.calls = [] + + def fit(self, train_set, val_set=None): + super().fit(train_set, val_set) + self.catalog_ = catalog(tuple((i // 2,) for i in range(self.total_items))) + return self + + def semantic_id_catalog(self): + return self.catalog_ + + def predict_semantic_ids(self, history_items, **context): + assert set(context) == {"user_index", "query_index", "session_index", + "evaluation_position", "n_candidates"} + self.calls.append((tuple(history_items), context)) + return prediction(self.catalog_, ((9,), (0,), (0,), (1,))) + + def score(self, *args, **kwargs): + raise AssertionError("raw evaluation must never call score/rank") + + +def data(): + train = [("u0", "t0", "a", 0), ("u0", "t0", "b", 1), + ("u1", "t1", "c", 0), ("u1", "t1", "d", 1)] + test = [("u0", "s0", "a", 2), ("u0", "s0", "b", 3), ("u0", "s0", "a", 4), + ("u0", "s1", "b", 2), ("u0", "s1", "c", 3), + ("u1", "s2", "c", 2), ("u1", "s2", "d", 3)] + return train, test + + +@pytest.mark.parametrize("mode, count", [("last", 3), ("next", 4)]) +@pytest.mark.parametrize("user_based", [False, True]) +def test_experiment_collects_once_per_query_with_modes_and_user_means(mode, count, user_based, tmp_path): + train, test = data() + evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", mode=mode, + item_features=np.eye(4), feature_source="identity-features") + model = FakeSID() + experiment = Experiment(evaluation, [model], [Recall(4), NDCG(4)], + user_based=user_based, show_validation=False, verbose=False, + save_dir=str(tmp_path)) + experiment.run() + assert len(model.calls) == count + assert all(context["n_candidates"] == 4 for _, context in model.calls) + result = experiment.result[0] + assert "Item[history_cosine]/Recall@4" in result.metric_avg_results + replay = result.semantic_id_evaluation + for view, rows in replay["per_query"].items(): + for field, metric in [("recall", "Recall"), ("ndcg", "NDCG")]: + values = rows[field][:, 0] + users = [record.user_index for record in result.semantic_id_trace.records] + expected = np.mean([np.mean(values[np.array(users) == user]) for user in set(users)]) if user_based else values.mean() + assert result.metric_avg_results[f"{view}/{metric}@4"] == pytest.approx(expected) + loaded = FakeSID.load(next(tmp_path.rglob("*.pkl"))) + assert loaded.semantic_id_catalog().stable_hash == model.semantic_id_catalog().stable_hash + + +def test_evaluation_uses_training_only_and_separates_validation(): + train, test = data() + val = [("u0", "v0", "c", 5), ("u0", "v0", "b", 6)] + evaluation = SemanticIDEvaluation.from_splits(train, test, val, fmt="USIT") + model = FakeSID() + result, validation = evaluation.evaluate(model, [Recall(4)], show_validation=True) + expected = TrainingPopularity([0, 1, 2, 3], 4).metadata + assert result.semantic_id_evaluation["protocol"]["resolvers"]["popularity"] == expected + assert validation.semantic_id_evaluation["protocol"]["resolvers"]["popularity"] == expected + assert len(model.calls) == 4 + assert len(validation.semantic_id_trace.records) == 1 + + +@pytest.mark.parametrize("exclude", [False, True]) +def test_unknown_item_handling_and_short_predictions(exclude): + train, test = data() + test += [("u1", "s3", "a", 2), ("u1", "s3", "new", 3)] + evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", exclude_unknowns=exclude) + model = FakeSID() + result, _ = evaluation.evaluate(model, [Recall(8)], show_validation=False) + assert len(result.semantic_id_trace.records) == (3 if exclude else 4) + assert all(len(record.prediction.address_sids) == 4 for record in result.semantic_id_trace.records) + assert result.semantic_id_evaluation["diagnostics"]["invalid_rate"] > 0 + + +def test_unsupported_metric_fails_before_fit(): + train, test = data() + evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT") + for metrics in ([HitRatio(3)], [Recall(-1)], [Recall(1.5)]): + model = FakeSID() + with pytest.raises(ValueError): + evaluation.evaluate(model, metrics) + assert not hasattr(model, "catalog_") + + +def test_lazy_imports(): + subprocess.run([sys.executable, "-c", "import sys; from cornac.eval_methods import SemanticIDEvaluation; from cornac.models import SemanticIDRecommender; assert 'torch' not in sys.modules; assert 'transformers' not in sys.modules"], check=True) + + +def test_trace_pickle_keeps_identity_and_readonly_predictions(): + cat = catalog() + saved = trace(cat, 1, ((0,), (9,))) + restored = pickle.loads(pickle.dumps(saved)) + assert restored.catalog_hash == saved.catalog_hash + with pytest.raises(ValueError): + restored.records[0].prediction.address_sids.setflags(write=True) + result = evaluate_semantic_id_trace(cat, restored, (1, 2)) + assert result["aggregate"]["CCE"]["Recall@1"] == .5 + + +def test_custom_resolver_in_experiment_and_empty_output(): + train, test = data() + custom = CustomScorer(lambda query, user, items: np.arange(len(items))) + evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", + resolvers={"custom": custom}) + result, _ = evaluation.evaluate(FakeSID(), [Recall(4)], show_validation=False) + assert "Item[custom]/Recall@4" in result.metric_avg_results + assert "Item[popularity]/Recall@4" not in result.metric_avg_results + class EmptySID(FakeSID): + def predict_semantic_ids(self, history_items, **context): + return prediction(self.catalog_, ()) + result, _ = evaluation.evaluate(EmptySID(), [Recall(4)], show_validation=False) + assert all(value == 0 for key, value in result.metric_avg_results.items() + if key.endswith("@4")) + + +def test_feature_coverage_rejected_and_trace_query_identity_unique(): + train, test = data() + evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", + item_features=np.eye(2)) + with pytest.raises(ValueError, match="cover"): + evaluation.evaluate(FakeSID(), [Recall(4)]) + cat = catalog() + saved = trace(cat, 1, ((0,),)) + with pytest.raises(ValueError, match="unique"): + SemanticIDTrace(cat.stable_hash, saved.records * 2) + + +def test_restricted_catalog_keeps_excluded_address_as_empty_slot(): + train, test = data() + class ExtraCatalogSID(FakeSID): + def fit(self, train_set, val_set=None): + super().fit(train_set, val_set) + self.catalog_ = catalog(((0,), (1,), (2,), (3,), (4,))) + return self + + def predict_semantic_ids(self, history_items, **context): + return prediction(self.catalog_, ((4,), (0,))) + evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", exclude_unknowns=True) + result, _ = evaluation.evaluate(ExtraCatalogSID(), [Recall(1), Recall(2)], show_validation=False) + records = result.semantic_id_trace.records + assert records[0].prediction.address_sids.tolist() == [[4], [0]] + assert records[0].prediction.catalog_validity.tolist() == [False, True] + assert result.semantic_id_evaluation["per_query"]["CCE"]["recall"][0].tolist() == [0., 1.] + for values in (records[0].prediction.address_sids, records[0].prediction.scores, + records[0].prediction.catalog_validity): + with pytest.raises(ValueError): + values.setflags(write=True) + + +def test_wrong_target_codes_rejected_and_same_user_queries_are_distinct(): + from dataclasses import replace + cat = catalog() + saved = trace(cat, 1, ((0,),)) + record = replace(saved.records[0], target_address_sid=(1,)) + with pytest.raises(ValueError, match="target codes"): + evaluate_semantic_id_trace(cat, SemanticIDTrace(cat.stable_hash, (record,))) + seen = [] + resolver = CustomScorer(lambda query, user, candidates: + seen.append((query, user)) or ([0, 1] if query == 0 else [1, 0])) + records = saved.records + (replace(saved.records[0], query_index=1),) + result = evaluate_semantic_id_trace(cat, + SemanticIDTrace(cat.stable_hash, records), (1,), {"custom": resolver}) + assert seen == [(0, 0), (1, 0)] + assert result["per_query"]["Item[custom]"]["recall"].tolist() == [[1.], [0.]] diff --git a/tests/cornac/models/test_semantic_id_adapters.py b/tests/cornac/models/test_semantic_id_adapters.py new file mode 100644 index 000000000..8272fd71b --- /dev/null +++ b/tests/cornac/models/test_semantic_id_adapters.py @@ -0,0 +1,476 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Raw SID adapters retain native scoring and decoder slot semantics.""" + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("transformers") + +from cornac.models import TIGER +from cornac.models.semantic_id import prediction_from_catalog_scores + + +def tiny_tiger(model_class=TIGER): + torch.manual_seed(7) + model = model_class( + rqvae_num_levels=2, rqvae_codebook_size=2, n_epochs=0, + d_model=8, d_ff=16, num_heads=2, d_kv=4, + num_enc_layers=1, num_dec_layers=1, max_len=2, + dropout=0, n_beams=2, device="cpu", seed=7, + ) + model.num_items = 3 + model.device_ = "cpu" + model._build_semantic_ids(np.array([[0, 0], [0, 0], [1, 1]])) + model._fit_seq2seq(torch, None) + return model + + +def native_tiger_prediction(model, history, budget): + """Direct decoder invocation with master's padding and encoder layout.""" + hist = list(history)[-model.max_len:] + hist = [model.pad_idx] * (model.max_len - len(hist)) + hist + tokens = torch.tensor(model.enc_token_table[hist].reshape(1, -1)) + mask = (tokens != 0).float() + model.model.eval() + with torch.no_grad(): + if model.scoring == "beam": + return model.model.generate_beam(tokens, mask, budget, model.prefix_children) + return model.model.score_all_items( + tokens, mask, torch.tensor(model.sid_table), model.scoring_batch_size + ) + + +def test_tiger_catalog_and_raw_slots(monkeypatch): + model = tiny_tiger() + cat = model.semantic_id_catalog() + assert cat.tokenizer_sid_for_item(0) == cat.tokenizer_sid_for_item(1) + assert cat.address_sid_for_item(0) != cat.address_sid_for_item(1) + a = cat.address_sid_for_item(0) + outputs = [a, (9, 9, 9), a] + seen = [] + + def decode(tokens, mask, budget, children): + seen.append(budget) + return outputs, [-1., -2., -3.] + + monkeypatch.setattr(model.model, "generate_beam", decode) + pred = model.predict_semantic_ids(np.array([0]), n_candidates=4) + assert seen == [4] + assert model.n_beams == 2 + assert pred.address_sids.tolist() == [list(row) for row in outputs] + assert pred.catalog_validity.tolist() == [True, False, True] + assert len(model.predict_semantic_ids([]).scores) == 0 + np.testing.assert_equal(model.score(0, []), np.ones(3)) + + +@pytest.mark.parametrize("scoring", ["beam", "exact"]) +def test_tiger_native_parity_and_persistence(scoring, tmp_path): + model = tiny_tiger() + model.scoring = scoring + native = native_tiger_prediction(model, [0], model.n_beams) + if scoring == "beam": + addresses, logps = native + expected = np.full(3, -1e10) + for sid, score in zip(addresses, logps): + expected[model.sid_to_item[sid]] = score + raw = model.predict_semantic_ids([0]) + np.testing.assert_array_equal(raw.address_sids, addresses) + np.testing.assert_allclose(raw.scores, logps) + enlarged = model.predict_semantic_ids([0], n_candidates=3) + assert len(enlarged.scores) == 3 + else: + expected = native + oracle = prediction_from_catalog_scores(model.semantic_id_catalog(), expected) + raw = model.predict_semantic_ids([0]) + np.testing.assert_array_equal(raw.address_sids, oracle.address_sids) + np.testing.assert_allclose(raw.scores, oracle.scores) + np.testing.assert_allclose(model.score(0, [0]), expected) + restored = model.load(model.save(str(tmp_path))) + assert restored.semantic_id_catalog().stable_hash == model.semantic_id_catalog().stable_hash + np.testing.assert_allclose(restored.score(0, [0]), expected) + np.testing.assert_array_equal(restored.predict_semantic_ids([0]).address_sids, raw.address_sids) + + +def test_tiger_cpu_fit_and_experiment(tmp_path): + from cornac.data import FeatureModality + from cornac.eval_methods import SemanticIDEvaluation + from cornac.metrics import Recall + + evaluation = SemanticIDEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT", + item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"]), + ) + model = TIGER( + rqvae_num_levels=2, rqvae_codebook_size=2, rqvae_n_epochs=1, + rqvae_hidden_dims=(4,), rqvae_latent_dim=2, n_epochs=1, + d_model=8, d_ff=16, num_heads=2, d_kv=4, + num_enc_layers=1, num_dec_layers=1, max_len=2, + dropout=0, n_beams=2, device="cpu", seed=7, + ) + result, _ = evaluation.evaluate(model, [Recall(2)], user_based=False) + assert {"SID/Recall@2", "CCE/Recall@2", "Item[popularity]/Recall@2"} <= set(result.metric_avg_results) + restored = model.load(model.save(str(tmp_path))) + np.testing.assert_array_equal(model.predict_semantic_ids([0]).address_sids, + restored.predict_semantic_ids([0]).address_sids) + + +def tiny_letter(): + from cornac.data import FeatureModality + from cornac.eval_methods import SemanticIDEvaluation + from cornac.models import LETTER + + evaluation = SemanticIDEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT", + item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"]), + ) + model = LETTER( + precomputed_semantic_ids=np.array([[0, 0], [0, 0], [1, 1]]), + rqvae_num_levels=2, rqvae_codebook_size=2, n_epochs=1, + d_model=8, d_ff=16, num_heads=2, d_kv=4, letter_base_vocab_size=8, + num_enc_layers=1, num_dec_layers=1, max_len=2, + dropout=0, n_beams=2, device="cpu", seed=7, + ) + model.fit(evaluation.train_set) + return model, evaluation + + +@pytest.mark.parametrize("scoring", ["beam", "exact"]) +def test_letter_native_parity_collisions_and_cpu_fit(scoring, tmp_path): + from cornac.metrics import Recall + + model, evaluation = tiny_letter() + model.scoring = scoring + cat = model.semantic_id_catalog() + assert cat.address_to_items[(0, 0)] == (0, 1) + assert cat.address_sids.shape == cat.tokenizer_sids.shape == (3, 2) + inputs, mask = model._encoder_batch(torch, [[0]]) + model.model.eval() + with torch.no_grad(): + if scoring == "beam": + beams, scores = model.model.generate_beam(inputs, mask, 2, model.prefix_children) + expected = np.full(3, -1e10) + for sid, score in zip(beams, scores): + expected[model.sid_to_item[sid]] = score + raw = model.predict_semantic_ids([0], n_candidates=2) + np.testing.assert_array_equal(raw.address_sids, beams) + np.testing.assert_allclose(raw.scores, scores) + else: + expected = model.model.score_all_items(inputs, mask, torch.tensor(model.sid_table), model.scoring_batch_size) + raw = model.predict_semantic_ids([0]) + oracle = prediction_from_catalog_scores(cat, expected) + np.testing.assert_array_equal(raw.address_sids, oracle.address_sids) + np.testing.assert_allclose(raw.scores, oracle.scores) + np.testing.assert_allclose(model.score(0, [0]), expected) + restored = model.load(model.save(str(tmp_path))) + np.testing.assert_array_equal(restored.predict_semantic_ids([0]).address_sids, raw.address_sids) + assert restored.semantic_id_catalog().stable_hash == cat.stable_hash + # Avoid retraining here: the fit smoke above is sufficient; evaluate frozen model. + model.trainable = False + result, _ = evaluation.evaluate(model, [Recall(2)], user_based=False) + assert "CCE/Recall@2" in result.metric_avg_results + + +def test_letter_raw_slots_and_budget(monkeypatch): + model, _ = tiny_letter() + outputs = [(0, 0), (8, 8), (0, 0)] + budgets = [] + + def decode(tokens, mask, budget, children): + budgets.append(budget) + return outputs, [-1., -2., -3.] + + monkeypatch.setattr(model.model, "generate_beam", decode) + raw = model.predict_semantic_ids([0], n_candidates=5) + assert budgets == [5] + assert model.n_beams == 2 + assert raw.address_sids.tolist() == [list(sid) for sid in outputs] + assert raw.catalog_validity.tolist() == [True, False, True] + + +def tiny_rpg(monkeypatch): + from cornac.data import FeatureModality + from cornac.eval_methods import SemanticIDEvaluation + from cornac.models import RPG + + evaluation = SemanticIDEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT", + item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"]), + ) + model = RPG( + n_codebook=2, codebook_size=2, n_epochs=1, d_model=8, + n_layer=1, n_head=2, n_inner=16, max_len=2, + resid_dropout=0, embd_dropout=0, attn_dropout=0, + n_beams=2, graph_edges=2, propagation_steps=1, device="cpu", seed=7, + ) + # faiss OPQ uses 8-bit PQ and needs >=256 training items; freeze only + # tokenization to keep this real training/graph-building CPU smoke tiny. + with monkeypatch.context() as patch: + patch.setattr(RPG, "_opq_tokenize", lambda self, feats, train_mask: np.array([[0, 0], [0, 0], [1, 1]])) + model.fit(evaluation.train_set) + return model, evaluation + + +@pytest.mark.parametrize("scoring", ["graph", "exact"]) +def test_rpg_native_parity_rng_and_persistence(monkeypatch, scoring, tmp_path): + from cornac.models.rpg.rpg import graph_decode + + model, _ = tiny_rpg(monkeypatch) + model.scoring = scoring + state = model.graph_rng.get_state() + model.model.eval() + with torch.no_grad(): + tokens = torch.tensor([[model.pad_idx, 0]]) + logits = model.model.next_item_logits(tokens, (tokens != model.pad_idx).float())[0] + if scoring == "graph": + candidates, scores = graph_decode(logits.reshape(-1).numpy(), model.adjacency, + model.item_tokens, 2, model.propagation_steps, model.graph_rng) + after_native = model.graph_rng.get_state() + expected = np.full(3, -1e10) + expected[candidates] = scores + model.graph_rng.set_state(state) + raw = model.predict_semantic_ids([0]) + np.testing.assert_array_equal(raw.address_sids, model.sid_table[candidates]) + np.testing.assert_allclose(raw.scores, scores) + assert all(np.array_equal(a, b) for a, b in zip(after_native, model.graph_rng.get_state())) + assert not all(np.array_equal(a, b) for a, b in zip(state, after_native)) + else: + expected = model._exact_item_scores(logits)[0].numpy().astype(float) + raw = model.predict_semantic_ids([0]) + oracle = prediction_from_catalog_scores(model.semantic_id_catalog(), expected) + np.testing.assert_array_equal(raw.address_sids, oracle.address_sids) + np.testing.assert_allclose(raw.scores, oracle.scores) + assert len(raw.scores) == 2 # maximum member score collapses the collision + model.graph_rng.set_state(state) + np.testing.assert_allclose(model.score(0, [0]), expected) + restored = model.load(model.save(str(tmp_path))) + # Save copied the RNG state, so compare the same next stochastic call. + np.testing.assert_array_equal(restored.predict_semantic_ids([0]).address_sids, + model.predict_semantic_ids([0]).address_sids) + assert model.semantic_id_catalog().address_to_items[(0, 0)] == (0, 1) + assert len(model.predict_semantic_ids([]).scores) == 0 + np.testing.assert_array_equal(model.score(0, []), np.ones(3)) + + +def test_rpg_graph_order_collisions_and_budget(monkeypatch): + import cornac.models.rpg.rpg as backend + model, evaluation = tiny_rpg(monkeypatch) + seen = [] + + def decode(flat, adjacency, tokens, budget, steps, rng): + assert rng is model.graph_rng + seen.append(budget) + return np.array([2, 0, 1]), np.array([-1., -2., -3.]) + + with monkeypatch.context() as patch: + patch.setattr(backend, "graph_decode", decode) + raw = model.predict_semantic_ids([0], n_candidates=5) + assert seen == [5] + assert model.n_beams == 2 + assert raw.address_sids.tolist() == [[1, 1], [0, 0], [0, 0]] + from cornac.metrics import Recall + model.trainable = False + result, _ = evaluation.evaluate(model, [Recall(2)], user_based=False) + assert "CCE/Recall@2" in result.metric_avg_results + + +class ConstantCPD: + n_digit = 2 + codebook_size = 2 + + def decode_logits(self, memory, padding, partial): + return torch.tensor([[[4., 0.], [3., 0.]]]).expand(len(partial), -1, -1) + + +@pytest.mark.parametrize("diagnostics", [False, True]) +@pytest.mark.parametrize("constrained", [False, True]) +def test_cpd_raw_single_batch_default_parity(diagnostics, constrained): + from cornac.models.diffgrm.diffgrm import cpd_decode, cpd_decode_batch + + memory, padding = torch.zeros(2, 1, 4), torch.zeros(2, 1, dtype=torch.bool) + kwargs = dict(beam_size=4, catalog_codes=np.array([[0, 0], [1, 1]]), + constrained=constrained, return_diagnostics=diagnostics) + default = cpd_decode_batch(ConstantCPD(), memory, padding, **kwargs) + extra = cpd_decode_batch(ConstantCPD(), memory, padding, return_raw=True, **kwargs) + assert extra[:-1] == default + raw = extra[-1] + assert len(raw) == 2 + for entry in raw: + assert len(entry["address_sids"]) == len(entry["scores"]) == len(entry["catalog_validity"]) == 4 + # Parallel assignment orders produce repeated complete paths. + assert len(set(entry["address_sids"])) < len(entry["address_sids"]) + assert entry["catalog_validity"] == [ + sid in {(0, 0), (1, 1)} and np.isfinite(score) + for sid, score in zip(entry["address_sids"], entry["scores"]) + ] + single = cpd_decode(ConstantCPD(), memory[:1], padding[:1], return_raw=True, **kwargs) + assert single == tuple(part[0] for part in extra) + legacy_single = cpd_decode(ConstantCPD(), memory[:1], padding[:1], **kwargs) + assert legacy_single == tuple(part[0] for part in default) + with pytest.raises(ValueError, match="one encoded history"): + cpd_decode(ConstantCPD(), memory, padding, **kwargs) + + +def test_cpd_raw_marks_nonfinite_and_invalid_slots(): + from cornac.models.diffgrm.diffgrm import cpd_decode + memory, padding = torch.zeros(1, 1, 4), torch.zeros(1, 1, dtype=torch.bool) + codes, scores, raw = cpd_decode(ConstantCPD(), memory, padding, + beam_size=16, catalog_codes=np.array([[1, 1]]), return_raw=True) + assert len(raw["address_sids"]) > len(codes) + assert any(not valid for valid in raw["catalog_validity"]) + for sid, score, valid in zip(raw["address_sids"], raw["scores"], raw["catalog_validity"]): + assert valid == (sid == (1, 1) and -1 not in sid and np.isfinite(score)) + with pytest.raises(ValueError, match="permutation"): + cpd_decode(ConstantCPD(), memory, padding, beam_size=2, digit_order=[0, 0]) + + +def tiny_diffgrm(): + from cornac.eval_methods import SemanticIDEvaluation + from cornac.models import DiffGRM + evaluation = SemanticIDEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT") + model = DiffGRM( + item_sids=np.array([[0, 0], [0, 0], [1, 1]]), n_digit=2, + codebook_size=2, d_model=8, encoder_n_layer=1, decoder_n_layer=1, + n_head=2, n_inner=16, dropout=0, max_len=2, min_history=1, + n_epochs=1, n_views=1, beam_size=4, lr_schedule="constant", + device="cpu", seed=7, + ) + model.fit(evaluation.train_set) + return model, evaluation + + +@pytest.mark.parametrize("scoring", ["released", "paper", "catalog", "fixed"]) +@pytest.mark.parametrize("policy", ["all", "first", "last"]) +def test_diffgrm_native_parity_raw_and_persistence(scoring, policy, tmp_path): + from cornac.models.diffgrm.diffgrm import cpd_decode + + model, _ = tiny_diffgrm() + model.scoring = scoring + model.collision_policy = policy + model.model.eval() + # Original master's left-aligned history input and decode options. + with torch.no_grad(): + inputs = torch.tensor([[0, model.pad_idx]]) + memory, padding = model.model.encode_history(inputs, torch.tensor([[1., 0.]])) + codes, scores, diagnostics, raw = cpd_decode(model.model, memory, padding, + beam_size=model.beam_size, catalog_codes=model.sid_table, + valid_code_set=model.sid_to_items_, constrained=scoring == "catalog", + digit_order=model.fixed_decode_order_ if scoring == "fixed" else None, + greedy_final=scoring == "released", return_diagnostics=True, return_raw=True) + expected = np.full(3, -1e10) + for sid, score in zip(codes, scores): + members = model.sid_to_items_[sid] + members = members[:1] if policy == "first" else members[-1:] if policy == "last" else members + expected[members] = score + np.testing.assert_allclose(model.score(0, [0]), expected) + pred = model.predict_semantic_ids(np.array([0])) + np.testing.assert_array_equal(pred.address_sids, raw["address_sids"]) + np.testing.assert_allclose(pred.scores, raw["scores"]) + assert pred.catalog_validity.tolist() == raw["catalog_validity"] + assert model.semantic_id_catalog().address_to_items[(0, 0)] == (0, 1) + restored = model.load(model.save(str(tmp_path))) + np.testing.assert_allclose(restored.score(0, [0]), expected) + np.testing.assert_array_equal(restored.predict_semantic_ids([0]).address_sids, pred.address_sids) + assert len(model.predict_semantic_ids([]).scores) == 0 + + +def test_diffgrm_requested_budget_and_evaluation(monkeypatch): + import cornac.models.diffgrm.diffgrm as backend + from cornac.metrics import Recall + + model, evaluation = tiny_diffgrm() + original = backend.cpd_decode + seen = [] + + def decode(*args, **kwargs): + seen.append((kwargs["beam_size"], kwargs["return_raw"])) + return original(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(backend, "cpd_decode", decode) + raw = model.predict_semantic_ids([0], n_candidates=8) + assert seen == [(8, True)] + assert model.beam_size == 4 + assert len(raw.scores) == 8 + model.trainable = False + result, _ = evaluation.evaluate(model, [Recall(2)], user_based=False) + assert "CCE/Recall@2" in result.metric_avg_results + + +@pytest.mark.parametrize("kind", ["tiger", "letter", "rpg", "diffgrm"]) +def test_fitted_catalog_cached_refit_and_old_pickle(kind, monkeypatch, tmp_path): + if kind == "tiger": + # Full fitted dataset is needed below; reuse the known tiny fixture. + from cornac.data import FeatureModality + from cornac.eval_methods import SemanticIDEvaluation + evaluation = SemanticIDEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT", + item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"])) + model = tiny_tiger() + model.rqvae_n_epochs = 1 + model.rqvae_hidden_dims = (4,) + model.rqvae_latent_dim = 2 + model.fit(evaluation.train_set) + elif kind == "letter": + model, evaluation = tiny_letter() + elif kind == "rpg": + model, evaluation = tiny_rpg(monkeypatch) + else: + model, evaluation = tiny_diffgrm() + first = model.semantic_id_catalog() + assert model.semantic_id_catalog() is first + model.predict_semantic_ids([0]) + assert model.semantic_id_catalog() is first + model.trainable = False + with pytest.warns(UserWarning, match="already fitted"): + model.fit(evaluation.train_set) + second = model.semantic_id_catalog() + assert second is not first + assert second.stable_hash == first.stable_hash + new_codes = np.array([[1, 0], [0, 1], [1, 0]]) + model.trainable = True + with monkeypatch.context() as patch: + if kind in ("tiger", "letter"): + patch.setattr(type(model), "_tokenize", lambda self, torch, feats: new_codes.copy()) + elif kind == "rpg": + patch.setattr(type(model), "_opq_tokenize", lambda self, feats, train_mask: new_codes.copy()) + else: + model.item_sids = new_codes + with pytest.warns(UserWarning, match="already fitted"): + model.fit(evaluation.train_set) + third = model.semantic_id_catalog() + assert third is model.semantic_id_catalog() + assert third.stable_hash != second.stable_hash + np.testing.assert_array_equal(third.tokenizer_sids, new_codes) + if kind in ("tiger", "letter"): + model._build_semantic_ids(np.array([[0, 1], [0, 0], [1, 1]])) + rebuilt = model.semantic_id_catalog() + assert rebuilt is not third + assert rebuilt.stable_hash != third.stable_hash + expected_hash = model.semantic_id_catalog().stable_hash + # Simulate an older model pickle that has SID state but no cached contract. + del model.semantic_id_catalog_ + restored = model.load(model.save(str(tmp_path))) + old_state_catalog = restored.semantic_id_catalog() + assert old_state_catalog.stable_hash == expected_hash + assert restored.semantic_id_catalog() is old_state_catalog From 6f3be74ce1bcc78a621a78514cb39cdc5f991966 Mon Sep 17 00:00:00 2001 From: hieuddo Date: Thu, 22 Oct 2026 18:47:27 +0800 Subject: [PATCH 2/5] Consolidate Semantic ID utilities and evaluation --- cornac/eval_methods/__init__.py | 17 +- cornac/eval_methods/semantic_id_evaluation.py | 134 +++++++++++++- cornac/eval_methods/sid_evaluation.py | 130 ------------- cornac/eval_methods/sid_resolvers.py | 171 ------------------ cornac/models/__init__.py | 7 - cornac/models/diffgrm/recom_diffgrm.py | 2 +- cornac/models/letter/recom_letter.py | 2 +- cornac/models/recommender.py | 2 +- cornac/models/rpg/recom_rpg.py | 2 +- cornac/models/tiger/recom_tiger.py | 2 +- cornac/{models => utils}/semantic_id.py | 162 ++++++++++++++++- docs/source/api_ref/eval_methods.rst | 56 +++++- docs/source/api_ref/models.rst | 3 - examples/semantic_id_evaluation_example.py | 9 +- .../test_semantic_id_evaluation.py | 29 ++- .../models/test_semantic_id_adapters.py | 2 +- 16 files changed, 376 insertions(+), 354 deletions(-) delete mode 100644 cornac/eval_methods/sid_evaluation.py delete mode 100644 cornac/eval_methods/sid_resolvers.py rename cornac/{models => utils}/semantic_id.py (67%) diff --git a/cornac/eval_methods/__init__.py b/cornac/eval_methods/__init__.py index 4522b2329..ee15c41b3 100644 --- a/cornac/eval_methods/__init__.py +++ b/cornac/eval_methods/__init__.py @@ -25,14 +25,10 @@ from .next_item_evaluation import NextItemEvaluation from .propensity_stratified_evaluation import PropensityStratifiedEvaluation -from .semantic_id_evaluation import SemanticIDEvaluation, collect_semantic_id_trace -from .sid_evaluation import evaluate_semantic_id_trace -from .sid_resolvers import ( - SemanticIDResolver, - TrainingPopularity, - HistoryCosine, - CustomScorer, - resolve_semantic_ids, +from .semantic_id_evaluation import ( + SemanticIDEvaluation, + collect_semantic_id_trace, + evaluate_semantic_id_trace, ) __all__ = [ @@ -47,9 +43,4 @@ "SemanticIDEvaluation", "collect_semantic_id_trace", "evaluate_semantic_id_trace", - "SemanticIDResolver", - "TrainingPopularity", - "HistoryCosine", - "CustomScorer", - "resolve_semantic_ids", ] diff --git a/cornac/eval_methods/semantic_id_evaluation.py b/cornac/eval_methods/semantic_id_evaluation.py index cb5332a65..2a704576f 100644 --- a/cornac/eval_methods/semantic_id_evaluation.py +++ b/cornac/eval_methods/semantic_id_evaluation.py @@ -13,25 +13,137 @@ # limitations under the License. # ============================================================================ -"""Opt-in Semantic ID evaluation for Cornac Experiment.""" +"""Next-item recommendation evaluation from raw Semantic ID predictions.""" import time from collections import OrderedDict +from collections.abc import Mapping import numpy as np from ..experiment.result import Result from ..metrics import Recall, NDCG from ..models.recommender import SemanticIDRecommender -from ..models.semantic_id import ( +from ..utils.semantic_id import ( SemanticIDCatalog, SemanticIDPrediction, SemanticIDTrace, SemanticIDTraceRecord, + SemanticIDResolver, + HistoryCosine, + TrainingPopularity, + resolve_semantic_ids, + validate_prediction, ) from .next_item_evaluation import NextItemEvaluation -from .sid_evaluation import evaluate_semantic_id_trace, _cutoffs -from .sid_resolvers import HistoryCosine, TrainingPopularity + + +def _cutoffs(values): + values = tuple(values) + if (not values or any(isinstance(k, (bool, np.bool_)) or + not isinstance(k, (int, np.integer)) or k <= 0 for k in values) or + len(set(values)) != len(values)): + raise ValueError("cutoffs must be distinct positive finite integers") + return tuple(int(k) for k in values) + + +def _rank_rows(rank, cutoffs): + hit = np.asarray([float(rank is not None and rank < k) for k in cutoffs]) + ndcg = hit / np.log2(rank + 2) if rank is not None else hit.copy() + return hit, ndcg + + +def evaluate_semantic_id_trace(catalog, trace, cutoffs=(5, 10, 20), resolvers=None): + """Evaluate every view on the same ordered raw predictions. + + CCE is the analytical expectation under uniform permutations *within full + expanded buckets*. Earlier buckets consume their full widths. Invalid and + duplicate addresses consume one empty rank slot in every view. A member + resolver chooses one item per bucket without changing address order. + + ``resolvers`` maps display names to SemanticIDResolver instances. Callbacks + receive query/user identity and bucket members only. Results contain scalar + aggregates, per-query Recall/NDCG arrays and protocol/diagnostic sidecars. + This function averages over queries; SemanticIDEvaluation also supports the + framework's per-user aggregation. CCE follows Zhang et al., + "Faithful Evaluation of Semantic-ID Tokenizers for Generative Recommendation" + (CIKM 2026): https://arxiv.org/abs/2605.25330v2. + """ + cutoffs = _cutoffs(cutoffs) + if not isinstance(catalog, SemanticIDCatalog) or not isinstance(trace, SemanticIDTrace): + raise ValueError("expected SemanticIDCatalog and SemanticIDTrace") + if trace.catalog_hash != catalog.stable_hash: + raise ValueError("trace catalog_hash differs from catalog") + if resolvers is None: + resolvers = {} + if not isinstance(resolvers, Mapping) or any( + not isinstance(name, str) or not name or any(c in name for c in "[]/") + for name in resolvers): + raise ValueError("resolvers must map nonempty plain names to resolvers") + if any(not isinstance(r, SemanticIDResolver) for r in resolvers.values()): + raise ValueError("resolvers must contain SemanticIDResolver instances") + views = ("SID", "CCE") + tuple(f"Item[{name}]" for name in resolvers) + rows = {view: {metric: np.zeros((len(trace.records), len(cutoffs))) + for metric in ("recall", "ndcg")} for view in views} + invalid = duplicate = total = 0 + sizes = [] + for index, record in enumerate(trace.records): + try: + address = catalog.address_sid_for_item(record.target_item_index) + tokenizer = catalog.tokenizer_sid_for_item(record.target_item_index) + except KeyError as error: + raise ValueError("trace target item is outside catalog") from error + if address != record.target_address_sid or tokenizer != record.target_tokenizer_sid: + raise ValueError("trace target codes differ from catalog") + prediction = record.prediction + validate_prediction(catalog, prediction) + addresses = [tuple(int(v) for v in row) for row in prediction.address_sids] + rank = next((i for i, value in enumerate(addresses) if value == address), None) + rows["SID"]["recall"][index], rows["SID"]["ndcg"][index] = _rank_rows(rank, cutoffs) + before, seen, reached = 0, set(), False + for value, valid in zip(addresses, prediction.catalog_validity): + if value == address: + reached = True + break + if not valid or value in seen: + before += 1 + else: + seen.add(value) + before += len(catalog.address_to_items[value]) + size = len(catalog.address_to_items[address]) + sizes.append(size) + if reached: + for col, cutoff in enumerate(cutoffs): + included = min(size, max(0, cutoff - before)) + rows["CCE"]["recall"][index, col] = included / size + rows["CCE"]["ndcg"][index, col] = ( + np.sum(1 / np.log2(before + np.arange(included) + 2)) / size) + for name, resolver in resolvers.items(): + slots = resolve_semantic_ids(catalog, prediction, resolver, + user_index=record.user_index, + query_index=record.query_index) + item_rank = next((i for i, item in enumerate(slots) + if item == record.target_item_index), None) + rows[f"Item[{name}]"]["recall"][index], rows[f"Item[{name}]"]["ndcg"][index] = _rank_rows(item_rank, cutoffs) + total += len(addresses) + invalid += int(np.count_nonzero(~prediction.catalog_validity)) + duplicate += len(addresses) - len(set(addresses)) + aggregate = {} + for view, metrics in rows.items(): + aggregate[view] = {} + for metric, values in metrics.items(): + values.setflags(write=False) + for col, cutoff in enumerate(cutoffs): + aggregate[view][f"{metric.upper() if metric == 'ndcg' else 'Recall'}@{cutoff}"] = ( + float(values[:, col].mean()) if len(values) else 0.) + return {"aggregate": aggregate, "per_query": rows, + "diagnostics": {"invalid_rate": invalid / total if total else 0., + "duplicate_rate": duplicate / total if total else 0., + "target_collision_group_size": np.asarray(sizes, dtype="int64")}, + "protocol": {"catalog_hash": catalog.stable_hash, "cutoffs": cutoffs, + "invalid_and_duplicate_slots": "preserved", + "order": "decoder_order", "CCE": "uniform_expanded_bucket_expectation", + "resolvers": {name: resolver.metadata for name, resolver in resolvers.items()}}} def collect_semantic_id_trace(model, dataset, *, mode="last", n_candidates=20, @@ -68,7 +180,6 @@ def collect_semantic_id_trace(model, dataset, *, mode="last", n_candidates=20, if not np.array_equal(validity, prediction.catalog_validity): # Verify original model validity before applying an explicit # restricted universe; otherwise a malformed adapter is hidden. - from .sid_resolvers import validate_prediction validate_prediction(model.semantic_id_catalog(), prediction) prediction = SemanticIDPrediction(prediction.address_sids, prediction.scores, validity, prediction.decoder_diagnostics) @@ -84,7 +195,18 @@ def collect_semantic_id_trace(model, dataset, *, mode="last", n_candidates=20, class SemanticIDEvaluation(NextItemEvaluation): - """Evaluate raw Semantic IDs through SID, CCE and explicit item views. + """Evaluate next-item recommendation through SID, CCE and item views. + + Use this instead of NextItemEvaluation when raw Semantic ID predictions + can refer to multiple items. NextItemEvaluation scores item rankings from + ``model.rank()``; this evaluator preserves raw decoder slots and measures + recommendation performance under CCE or explicit item resolution. It does + not evaluate the intrinsic quality of the IDs. + + Models must implement SemanticIDRecommender. Evaluate ordinary next-item + models separately with NextItemEvaluation on identical data splits and + cutoffs; a single mixed-model Experiment is not supported. + Accepts NextItemEvaluation dataset/splitting arguments. ``resolvers`` is an optional mapping of names to fitted SemanticIDResolver instances; None uses diff --git a/cornac/eval_methods/sid_evaluation.py b/cornac/eval_methods/sid_evaluation.py deleted file mode 100644 index b67f833bb..000000000 --- a/cornac/eval_methods/sid_evaluation.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -"""Paired SID, expanded CCE and exact-item evaluation from raw traces.""" - -from collections.abc import Mapping - -import numpy as np - -from ..models.semantic_id import SemanticIDCatalog, SemanticIDTrace -from .sid_resolvers import resolve_semantic_ids, validate_prediction - - -def _cutoffs(values): - values = tuple(values) - if (not values or any(isinstance(k, (bool, np.bool_)) or - not isinstance(k, (int, np.integer)) or k <= 0 for k in values) or - len(set(values)) != len(values)): - raise ValueError("cutoffs must be distinct positive finite integers") - return tuple(int(k) for k in values) - - -def _rank_rows(rank, cutoffs): - hit = np.asarray([float(rank is not None and rank < k) for k in cutoffs]) - ndcg = hit / np.log2(rank + 2) if rank is not None else hit.copy() - return hit, ndcg - - -def evaluate_semantic_id_trace(catalog, trace, cutoffs=(5, 10, 20), resolvers=None): - """Evaluate every view on the same ordered raw predictions. - - CCE is the analytical expectation under uniform permutations *within full - expanded buckets*. Earlier buckets consume their full widths. Invalid and - duplicate addresses consume one empty rank slot in every view. A member - resolver chooses one item per bucket without changing address order. - - ``resolvers`` maps display names to SemanticIDResolver instances. Callbacks - receive query/user identity and bucket members only. Results contain scalar - aggregates, per-query Recall/NDCG arrays and protocol/diagnostic sidecars. - This function averages over queries; SemanticIDEvaluation also supports the - framework's per-user aggregation. - """ - cutoffs = _cutoffs(cutoffs) - if not isinstance(catalog, SemanticIDCatalog) or not isinstance(trace, SemanticIDTrace): - raise ValueError("expected SemanticIDCatalog and SemanticIDTrace") - if trace.catalog_hash != catalog.stable_hash: - raise ValueError("trace catalog_hash differs from catalog") - if resolvers is None: - resolvers = {} - if not isinstance(resolvers, Mapping) or any( - not isinstance(name, str) or not name or any(c in name for c in "[]/") - for name in resolvers): - raise ValueError("resolvers must map nonempty plain names to resolvers") - from .sid_resolvers import SemanticIDResolver - if any(not isinstance(r, SemanticIDResolver) for r in resolvers.values()): - raise ValueError("resolvers must contain SemanticIDResolver instances") - views = ("SID", "CCE") + tuple(f"Item[{name}]" for name in resolvers) - rows = {view: {metric: np.zeros((len(trace.records), len(cutoffs))) - for metric in ("recall", "ndcg")} for view in views} - invalid = duplicate = total = 0 - sizes = [] - for index, record in enumerate(trace.records): - try: - address = catalog.address_sid_for_item(record.target_item_index) - tokenizer = catalog.tokenizer_sid_for_item(record.target_item_index) - except KeyError as error: - raise ValueError("trace target item is outside catalog") from error - if address != record.target_address_sid or tokenizer != record.target_tokenizer_sid: - raise ValueError("trace target codes differ from catalog") - prediction = record.prediction - validate_prediction(catalog, prediction) - addresses = [tuple(int(v) for v in row) for row in prediction.address_sids] - rank = next((i for i, value in enumerate(addresses) if value == address), None) - rows["SID"]["recall"][index], rows["SID"]["ndcg"][index] = _rank_rows(rank, cutoffs) - before, seen, reached = 0, set(), False - for value, valid in zip(addresses, prediction.catalog_validity): - if value == address: - reached = True - break - if not valid or value in seen: - before += 1 - else: - seen.add(value) - before += len(catalog.address_to_items[value]) - size = len(catalog.address_to_items[address]) - sizes.append(size) - if reached: - for col, cutoff in enumerate(cutoffs): - included = min(size, max(0, cutoff - before)) - rows["CCE"]["recall"][index, col] = included / size - rows["CCE"]["ndcg"][index, col] = ( - np.sum(1 / np.log2(before + np.arange(included) + 2)) / size) - for name, resolver in resolvers.items(): - slots = resolve_semantic_ids(catalog, prediction, resolver, - user_index=record.user_index, - query_index=record.query_index) - item_rank = next((i for i, item in enumerate(slots) - if item == record.target_item_index), None) - rows[f"Item[{name}]"]["recall"][index], rows[f"Item[{name}]"]["ndcg"][index] = _rank_rows(item_rank, cutoffs) - total += len(addresses) - invalid += int(np.count_nonzero(~prediction.catalog_validity)) - duplicate += len(addresses) - len(set(addresses)) - aggregate = {} - for view, metrics in rows.items(): - aggregate[view] = {} - for metric, values in metrics.items(): - values.setflags(write=False) - for col, cutoff in enumerate(cutoffs): - aggregate[view][f"{metric.upper() if metric == 'ndcg' else 'Recall'}@{cutoff}"] = ( - float(values[:, col].mean()) if len(values) else 0.) - return {"aggregate": aggregate, "per_query": rows, - "diagnostics": {"invalid_rate": invalid / total if total else 0., - "duplicate_rate": duplicate / total if total else 0., - "target_collision_group_size": np.asarray(sizes, dtype="int64")}, - "protocol": {"catalog_hash": catalog.stable_hash, "cutoffs": cutoffs, - "invalid_and_duplicate_slots": "preserved", - "order": "decoder_order", "CCE": "uniform_expanded_bucket_expectation", - "resolvers": {name: resolver.metadata for name, resolver in resolvers.items()}}} diff --git a/cornac/eval_methods/sid_resolvers.py b/cornac/eval_methods/sid_resolvers.py deleted file mode 100644 index 77226fe4f..000000000 --- a/cornac/eval_methods/sid_resolvers.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -"""Target-free selection of one item within each generated complete SID.""" - -import hashlib -from abc import ABC, abstractmethod - -import numpy as np - -from ..models.semantic_id import SemanticIDCatalog, SemanticIDPrediction - - -def _indices(values, name, upper=None): - values = np.asarray(values) - if (values.ndim != 1 or (values.size and - not np.issubdtype(values.dtype, np.integer)) or - np.any(values < 0) or (upper is not None and np.any(values >= upper))): - raise ValueError(f"{name} must contain covered nonnegative integer indices") - return values.astype("int64", copy=False) - - -def _unit_rows(features): - features = np.array(features, dtype="float64", copy=True) - if (features.ndim != 2 or not all(features.shape) or - not np.all(np.isfinite(features))): - raise ValueError("features must be a nonempty finite item matrix") - scales = np.max(np.abs(features), axis=1, keepdims=True) - np.divide(features, scales, out=features, where=scales != 0) - norms = np.linalg.norm(features, axis=1, keepdims=True) - np.divide(features, norms, out=features, where=norms != 0) - return np.frombuffer(features.tobytes(), dtype=features.dtype).reshape(features.shape) - - -class SemanticIDResolver(ABC): - """Score candidate members without targets; highest score wins. - - Implement ``score(query_index, user_index, candidate_items)``. The shared - operation checks scores and breaks ties by smallest global item index. - Scores never reorder generated addresses or admit outside candidates. - """ - - @abstractmethod - def score(self, query_index, user_index, candidate_items): - """Return one finite score per candidate, in candidate order.""" - raise NotImplementedError - - @property - def metadata(self): - return {"resolver": type(self).__name__} - - -class TrainingPopularity(SemanticIDResolver): - """Count explicitly supplied training events, retaining repeated events.""" - - def __init__(self, training_items, num_items): - if not isinstance(num_items, (int, np.integer)) or num_items <= 0: - raise ValueError("num_items must be a positive integer") - items = _indices(training_items, "training_items", num_items) - counts = np.bincount(items, minlength=num_items) - self.counts = np.frombuffer(counts.tobytes(), dtype=counts.dtype) - - def score(self, query_index, user_index, candidate_items): - return self.counts[list(candidate_items)] - - @property - def metadata(self): - return {"resolver": "training_popularity", "source": "training_events", - "counts_sha256": hashlib.sha256(self.counts.tobytes()).hexdigest()} - - -class HistoryCosine(SemanticIDResolver): - """Cosine to normalized sums of normalized training-event item features. - - Rows are global Cornac item indices. Supply training events only; repeats - contribute repeatedly. A zero user profile ties all members. ``feature_source`` - labels the explicit scoring feature space, which need not be the model's - tokenizer feature space. No evaluation history or target enters the profile. - """ - - def __init__(self, features, training_users, training_items, num_users, - feature_source="provided"): - self.features = _unit_rows(features) - if not isinstance(num_users, (int, np.integer)) or num_users <= 0: - raise ValueError("num_users must be a positive integer") - users = _indices(training_users, "training_users", num_users) - items = _indices(training_items, "training_items", len(self.features)) - if users.shape != items.shape: - raise ValueError("training user/item events must align") - if not isinstance(feature_source, str) or not feature_source: - raise ValueError("feature_source must be a nonempty string") - profiles = np.zeros((num_users, self.features.shape[1]), dtype="float64") - np.add.at(profiles, users, self.features[items]) - self.profiles = _unit_rows(profiles) - self.feature_source = feature_source - - def score(self, query_index, user_index, candidate_items): - if user_index is None or not 0 <= user_index < len(self.profiles): - raise ValueError("user_index is outside the training profile space") - return self.features[list(candidate_items)] @ self.profiles[user_index] - - @property - def metadata(self): - return {"resolver": "training_history_cosine", "source": "training_events", - "feature_source": self.feature_source, - "feature_join": "global_item_index", "zero_profile": "item_index_tie", - "normalized_features_sha256": hashlib.sha256(self.features.tobytes()).hexdigest()} - - -class CustomScorer(SemanticIDResolver): - """Adapt ``callback(query_index, user_index, candidate_items)`` to a resolver.""" - - def __init__(self, callback): - if not callable(callback): - raise ValueError("callback must be callable") - self.callback = callback - - def score(self, query_index, user_index, candidate_items): - return self.callback(query_index, user_index, candidate_items) - - -def validate_prediction(catalog, prediction): - """Check raw prediction validity against the admitted frozen catalog.""" - if not isinstance(catalog, SemanticIDCatalog) or not isinstance(prediction, SemanticIDPrediction): - raise ValueError("expected SemanticIDCatalog and SemanticIDPrediction") - if prediction.address_sids.shape[1] != catalog.address_sids.shape[1]: - raise ValueError("prediction address width differs from catalog") - expected = [tuple(row) in catalog.address_to_items for row in prediction.address_sids] - if not np.array_equal(prediction.catalog_validity, expected): - raise ValueError("prediction catalog_validity differs from catalog") - - -def resolve_semantic_ids(catalog, prediction, resolver, *, user_index=None, - query_index=None): - """Return item/None rank slots from raw predictions for inference or replay. - - Invalid and repeated addresses consume empty slots. Singleton buckets bypass - scoring. This operation returns no fillers, changes no input, and exposes no - target to the resolver. Choose a complete-address catalog matching the model. - """ - validate_prediction(catalog, prediction) - if not isinstance(resolver, SemanticIDResolver): - raise ValueError("resolver must be a SemanticIDResolver") - slots, seen = [], set() - for row, valid in zip(prediction.address_sids, prediction.catalog_validity): - address = tuple(int(value) for value in row) - if not valid or address in seen: - slots.append(None) - continue - seen.add(address) - items = catalog.address_to_items[address] - if len(items) == 1: - slots.append(items[0]) - continue - scores = np.asarray(resolver.score(query_index, user_index, items), dtype="float64") - if scores.shape != (len(items),) or not np.all(np.isfinite(scores)): - raise ValueError("resolver must return one finite score per candidate") - slots.append(min(zip(items, scores), key=lambda pair: (-pair[1], pair[0]))[0]) - return tuple(slots) diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index cfe8b7ba6..13d071358 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -95,10 +95,3 @@ from .vbpr import VBPR from .vmf import VMF from .wmf import WMF - -from .semantic_id import ( - SemanticIDCatalog, - SemanticIDPrediction, - SemanticIDTrace, - SemanticIDTraceRecord, -) diff --git a/cornac/models/diffgrm/recom_diffgrm.py b/cornac/models/diffgrm/recom_diffgrm.py index 2b0cf50ec..2172bc336 100644 --- a/cornac/models/diffgrm/recom_diffgrm.py +++ b/cornac/models/diffgrm/recom_diffgrm.py @@ -21,7 +21,7 @@ from tqdm.auto import trange from ..recommender import SemanticIDRecommender -from ..semantic_id import ( +from ...utils.semantic_id import ( SemanticIDCatalog, SemanticIDPrediction, ) diff --git a/cornac/models/letter/recom_letter.py b/cornac/models/letter/recom_letter.py index fe9772216..7869865d0 100644 --- a/cornac/models/letter/recom_letter.py +++ b/cornac/models/letter/recom_letter.py @@ -21,7 +21,7 @@ from tqdm.auto import trange from ..tiger.recom_tiger import TIGER -from ..semantic_id import SemanticIDCatalog, prediction_from_catalog_scores +from ...utils.semantic_id import SemanticIDCatalog, prediction_from_catalog_scores def _generator_optimizer_groups(model, weight_decay): diff --git a/cornac/models/recommender.py b/cornac/models/recommender.py index 73136c09d..60e253eba 100644 --- a/cornac/models/recommender.py +++ b/cornac/models/recommender.py @@ -770,7 +770,7 @@ class SemanticIDRecommender(NextItemRecommender): """Optional interface for recommenders that generate Semantic IDs.""" def semantic_id_catalog(self): - """Return the fitted :class:`~cornac.models.semantic_id.SemanticIDCatalog`.""" + """Return the fitted :class:`~cornac.utils.semantic_id.SemanticIDCatalog`.""" raise NotImplementedError def predict_semantic_ids(self, history_items, **context): diff --git a/cornac/models/rpg/recom_rpg.py b/cornac/models/rpg/recom_rpg.py index 6e20dbb0a..66e3353ef 100644 --- a/cornac/models/rpg/recom_rpg.py +++ b/cornac/models/rpg/recom_rpg.py @@ -19,7 +19,7 @@ from tqdm.auto import trange from ..recommender import SemanticIDRecommender -from ..semantic_id import ( +from ...utils.semantic_id import ( SemanticIDCatalog, SemanticIDPrediction, prediction_from_catalog_scores, diff --git a/cornac/models/tiger/recom_tiger.py b/cornac/models/tiger/recom_tiger.py index ea5f3c2f8..c9bffabd8 100644 --- a/cornac/models/tiger/recom_tiger.py +++ b/cornac/models/tiger/recom_tiger.py @@ -20,7 +20,7 @@ from tqdm.auto import trange from ..recommender import SemanticIDRecommender -from ..semantic_id import ( +from ...utils.semantic_id import ( SemanticIDCatalog, SemanticIDPrediction, prediction_from_catalog_scores, diff --git a/cornac/models/semantic_id.py b/cornac/utils/semantic_id.py similarity index 67% rename from cornac/models/semantic_id.py rename to cornac/utils/semantic_id.py index f5cac0cfc..a61cee424 100644 --- a/cornac/models/semantic_id.py +++ b/cornac/utils/semantic_id.py @@ -13,10 +13,18 @@ # limitations under the License. # ============================================================================ -"""Model-neutral contracts for Semantic-ID recommendation.""" +"""Model-neutral Semantic ID catalogs, predictions and item resolvers. + +Resolvers select one item from each predicted address bucket for inference or +recommendation evaluation. For example, when items 0 and 1 share an address, +TrainingPopularity([1, 1, 0], num_items=2) selects item 1 from that bucket. +Resolution preserves address order; invalid and repeated addresses leave empty +rank slots. Construct resolver state from training data only. +""" import hashlib import json +from abc import ABC, abstractmethod from collections.abc import Mapping from dataclasses import dataclass, field from types import MappingProxyType @@ -24,6 +32,8 @@ import numpy as np +# Catalogs, predictions and replay traces + def _readonly_array(values, dtype, ndim, name): original = np.asarray(values) if dtype == "int64" and original.size and not np.issubdtype(original.dtype, np.integer): @@ -382,3 +392,153 @@ def __post_init__(self): raise ValueError("catalog_hash must be a non-empty string") object.__setattr__(self, "records", records) object.__setattr__(self, "metadata", _freeze_native(self.metadata)) + + +# Item resolution + +def _indices(values, name, upper=None): + values = np.asarray(values) + if (values.ndim != 1 or (values.size and + not np.issubdtype(values.dtype, np.integer)) or + np.any(values < 0) or (upper is not None and np.any(values >= upper))): + raise ValueError(f"{name} must contain covered nonnegative integer indices") + return values.astype("int64", copy=False) + + +def _unit_rows(features): + features = np.array(features, dtype="float64", copy=True) + if (features.ndim != 2 or not all(features.shape) or + not np.all(np.isfinite(features))): + raise ValueError("features must be a nonempty finite item matrix") + scales = np.max(np.abs(features), axis=1, keepdims=True) + np.divide(features, scales, out=features, where=scales != 0) + norms = np.linalg.norm(features, axis=1, keepdims=True) + np.divide(features, norms, out=features, where=norms != 0) + return np.frombuffer(features.tobytes(), dtype=features.dtype).reshape(features.shape) + + +class SemanticIDResolver(ABC): + """Score candidate members without targets; highest score wins. + + Implement ``score(query_index, user_index, candidate_items)``. The shared + operation checks scores and breaks ties by smallest global item index. + Scores never reorder generated addresses or admit outside candidates. + """ + + @abstractmethod + def score(self, query_index, user_index, candidate_items): + """Return one finite score per candidate, in candidate order.""" + raise NotImplementedError + + @property + def metadata(self): + return {"resolver": type(self).__name__} + + +class TrainingPopularity(SemanticIDResolver): + """Count explicitly supplied training events, retaining repeated events.""" + + def __init__(self, training_items, num_items): + if not isinstance(num_items, (int, np.integer)) or num_items <= 0: + raise ValueError("num_items must be a positive integer") + items = _indices(training_items, "training_items", num_items) + counts = np.bincount(items, minlength=num_items) + self.counts = np.frombuffer(counts.tobytes(), dtype=counts.dtype) + + def score(self, query_index, user_index, candidate_items): + return self.counts[list(candidate_items)] + + @property + def metadata(self): + return {"resolver": "training_popularity", "source": "training_events", + "counts_sha256": hashlib.sha256(self.counts.tobytes()).hexdigest()} + + +class HistoryCosine(SemanticIDResolver): + """Cosine to normalized sums of normalized training-event item features. + + Rows are global Cornac item indices. Supply training events only; repeats + contribute repeatedly. A zero user profile ties all members. ``feature_source`` + labels the explicit scoring feature space, which need not be the model's + tokenizer feature space. No evaluation history or target enters the profile. + """ + + def __init__(self, features, training_users, training_items, num_users, + feature_source="provided"): + self.features = _unit_rows(features) + if not isinstance(num_users, (int, np.integer)) or num_users <= 0: + raise ValueError("num_users must be a positive integer") + users = _indices(training_users, "training_users", num_users) + items = _indices(training_items, "training_items", len(self.features)) + if users.shape != items.shape: + raise ValueError("training user/item events must align") + if not isinstance(feature_source, str) or not feature_source: + raise ValueError("feature_source must be a nonempty string") + profiles = np.zeros((num_users, self.features.shape[1]), dtype="float64") + np.add.at(profiles, users, self.features[items]) + self.profiles = _unit_rows(profiles) + self.feature_source = feature_source + + def score(self, query_index, user_index, candidate_items): + if user_index is None or not 0 <= user_index < len(self.profiles): + raise ValueError("user_index is outside the training profile space") + return self.features[list(candidate_items)] @ self.profiles[user_index] + + @property + def metadata(self): + return {"resolver": "training_history_cosine", "source": "training_events", + "feature_source": self.feature_source, + "feature_join": "global_item_index", "zero_profile": "item_index_tie", + "normalized_features_sha256": hashlib.sha256(self.features.tobytes()).hexdigest()} + + +class CustomScorer(SemanticIDResolver): + """Adapt ``callback(query_index, user_index, candidate_items)`` to a resolver.""" + + def __init__(self, callback): + if not callable(callback): + raise ValueError("callback must be callable") + self.callback = callback + + def score(self, query_index, user_index, candidate_items): + return self.callback(query_index, user_index, candidate_items) + + +def validate_prediction(catalog, prediction): + """Check raw prediction validity against the admitted frozen catalog.""" + if not isinstance(catalog, SemanticIDCatalog) or not isinstance(prediction, SemanticIDPrediction): + raise ValueError("expected SemanticIDCatalog and SemanticIDPrediction") + if prediction.address_sids.shape[1] != catalog.address_sids.shape[1]: + raise ValueError("prediction address width differs from catalog") + expected = [tuple(row) in catalog.address_to_items for row in prediction.address_sids] + if not np.array_equal(prediction.catalog_validity, expected): + raise ValueError("prediction catalog_validity differs from catalog") + + +def resolve_semantic_ids(catalog, prediction, resolver, *, user_index=None, + query_index=None): + """Return item/None rank slots from raw predictions for inference or replay. + + Invalid and repeated addresses consume empty slots. Singleton buckets bypass + scoring. This operation returns no fillers, changes no input, and exposes no + target to the resolver. Choose a complete-address catalog matching the model. + """ + validate_prediction(catalog, prediction) + if not isinstance(resolver, SemanticIDResolver): + raise ValueError("resolver must be a SemanticIDResolver") + slots, seen = [], set() + for row, valid in zip(prediction.address_sids, prediction.catalog_validity): + address = tuple(int(value) for value in row) + if not valid or address in seen: + slots.append(None) + continue + seen.add(address) + items = catalog.address_to_items[address] + if len(items) == 1: + slots.append(items[0]) + continue + scores = np.asarray(resolver.score(query_index, user_index, items), dtype="float64") + if scores.shape != (len(items),) or not np.all(np.isfinite(scores)): + raise ValueError("resolver must return one finite score per candidate") + slots.append(min(zip(items, scores), key=lambda pair: (-pair[1], pair[0]))[0]) + return tuple(slots) diff --git a/docs/source/api_ref/eval_methods.rst b/docs/source/api_ref/eval_methods.rst index 57afec443..b36d308aa 100644 --- a/docs/source/api_ref/eval_methods.rst +++ b/docs/source/api_ref/eval_methods.rst @@ -33,8 +33,14 @@ Timestamp Split .. automodule:: cornac.eval_methods.timestamp_split :members: -Semantic ID Evaluation ----------------------- +Recommendation Evaluation with Semantic IDs +------------------------------------------ + +``SemanticIDEvaluation`` subclasses ``NextItemEvaluation`` and evaluates next-item +recommendation performance, rather than the intrinsic quality of Semantic IDs. +Use it when a generated ID can map to several items: SID matching alone cannot +identify which member was recommended. ``NextItemEvaluation`` instead evaluates +the item rankings returned by ``model.rank()``. ``SemanticIDEvaluation`` evaluates TIGER, LETTER, RPG and DiffGRM through the same raw Semantic ID predictions. It reports SID accuracy, analytical @@ -50,14 +56,44 @@ duplicate addresses preserve empty rank slots. A resolver selects one item per bucket while preserving address order. Complete IDs include any disambiguating suffix; injective catalogs give identical views under the same slot rules. +CCE follows Qian Zhang, Lech Szymanski, Haibo Zhang and Jeremiah D. Deng, +`Faithful Evaluation of Semantic-ID Tokenizers for Generative Recommendation +`_ (CIKM 2026). The explicit resolvers below +are selectable item-ranking policies, distinct from CCE's expectation. + +This evaluator requires ``SemanticIDRecommender`` and supports SID models only. +For ordinary next-item models such as SASRec, use ``NextItemEvaluation`` in a +separate experiment with identical train/validation/test splits and cutoffs. +Mixed SID and ordinary models in one ``Experiment`` are not supported. + Popularity uses training events only. Supplying ``item_features`` adds history cosine: normalize item vectors, sum each user's training-event vectors (including repeats), normalize that profile and score bucket members by cosine. Features must align with global Cornac item indices. Ties, including zero profiles, choose the smallest item index. Pass fitted resolvers through ``resolvers`` or implement ``SemanticIDResolver.score(query_index, user_index, candidate_items)``; custom -state must also use training-only information. ``resolve_semantic_ids`` exposes -the same operation for inference. +state must also use training-only information. Import these utilities from +``cornac.utils.semantic_id``; ``resolve_semantic_ids`` exposes the same operation +for inference. For example:: + + import numpy as np + from cornac.utils.semantic_id import ( + SemanticIDCatalog, SemanticIDPrediction, TrainingPopularity, + resolve_semantic_ids, + ) + + codes = np.array([[0], [0], [1]]) + catalog = SemanticIDCatalog(np.arange(3), codes, codes, (2,), (2,)) + prediction = SemanticIDPrediction( + np.array([[0], [1]]), np.array([1., .5]), np.array([True, True])) + resolver = TrainingPopularity([1, 1, 0, 2], num_items=3) + resolve_semantic_ids(catalog, prediction, resolver) # (1, 2) + +Items 0 and 1 share the first ID. Popularity chooses item 1 from that bucket, +then item 2 from the next, preserving generated ID order. If item 1 is the +target, SID Recall@1 is 1, CCE Recall@1 is 0.5 and popularity-resolved Recall@1 +is 1. CCE Recall@2 is 1 because both members of the first bucket fit. Other +resolvers can choose differently while using the same predictions. For an existing Semantic ID model and item-feature modality:: @@ -71,7 +107,8 @@ See ``examples/semantic_id_evaluation_example.py`` for a standalone CPU example. Adapter authors subclass ``cornac.models.recommender.SemanticIDRecommender`` and implement ``semantic_id_catalog()`` and ``predict_semantic_ids(history_items, -**context)``. Export complete addresses separately from tokenizer prefixes and +**context)``. Use the shared types in ``cornac.utils.semantic_id``. Export +complete addresses separately from tokenizer prefixes and preserve raw decoder order, invalid slots and duplicates before item conversion. The evaluator requests ``n_candidates=max(k)``; honor it where supported and keep shorter output without fillers. ``prediction_from_catalog_scores`` supports exact @@ -79,11 +116,12 @@ scoring by maximum member score per address, with lexicographic address ties. The contract covers fixed-width integer addresses, a frozen catalog and one next-item target per query. -.. automodule:: cornac.eval_methods.semantic_id_evaluation - :members: +The evaluation module combines trace collection, offline trace evaluation via +``evaluate_semantic_id_trace`` and the ``SemanticIDEvaluation`` experiment +integration. Utilities below provide catalogs, predictions and item resolution. -.. automodule:: cornac.eval_methods.sid_evaluation +.. automodule:: cornac.eval_methods.semantic_id_evaluation :members: -.. automodule:: cornac.eval_methods.sid_resolvers +.. automodule:: cornac.utils.semantic_id :members: diff --git a/docs/source/api_ref/models.rst b/docs/source/api_ref/models.rst index e7593e53f..14b6e18a3 100644 --- a/docs/source/api_ref/models.rst +++ b/docs/source/api_ref/models.rst @@ -338,6 +338,3 @@ Semantic ID Model Contracts -------------------------- .. autoclass:: cornac.models.recommender.SemanticIDRecommender :members: - -.. automodule:: cornac.models.semantic_id - :members: diff --git a/examples/semantic_id_evaluation_example.py b/examples/semantic_id_evaluation_example.py index f47638f14..f8ba88470 100644 --- a/examples/semantic_id_evaluation_example.py +++ b/examples/semantic_id_evaluation_example.py @@ -13,13 +13,18 @@ # limitations under the License. # ============================================================================ -"""Backend-free, synthetic demonstration of shared Semantic ID evaluation.""" +"""Backend-free next-item evaluation from raw Semantic ID predictions. + +Use NextItemEvaluation for ordinary item-ranking models. This SID-only example +compares SID matching, CCE's expected item credit and explicit item selection. +""" import numpy as np import cornac from cornac.eval_methods import SemanticIDEvaluation from cornac.metrics import Recall, NDCG -from cornac.models import SemanticIDCatalog, SemanticIDPrediction, SemanticIDRecommender +from cornac.models import SemanticIDRecommender +from cornac.utils.semantic_id import SemanticIDCatalog, SemanticIDPrediction class FrozenSIDExample(SemanticIDRecommender): diff --git a/tests/cornac/eval_methods/test_semantic_id_evaluation.py b/tests/cornac/eval_methods/test_semantic_id_evaluation.py index c9706cafa..5ac437a7c 100644 --- a/tests/cornac/eval_methods/test_semantic_id_evaluation.py +++ b/tests/cornac/eval_methods/test_semantic_id_evaluation.py @@ -22,12 +22,20 @@ import pytest from cornac import Experiment -from cornac.eval_methods import (SemanticIDEvaluation, evaluate_semantic_id_trace, - TrainingPopularity, HistoryCosine, CustomScorer, resolve_semantic_ids) +from cornac.eval_methods import SemanticIDEvaluation, evaluate_semantic_id_trace from cornac.metrics import Recall, NDCG, HitRatio -from cornac.models import (SemanticIDCatalog, SemanticIDPrediction, - SemanticIDRecommender, SemanticIDTrace, SemanticIDTraceRecord) -from cornac.models.semantic_id import prediction_from_catalog_scores +from cornac.models import SemanticIDRecommender +from cornac.utils.semantic_id import ( + TrainingPopularity, + HistoryCosine, + CustomScorer, + resolve_semantic_ids, + SemanticIDCatalog, + SemanticIDPrediction, + SemanticIDTrace, + SemanticIDTraceRecord, + prediction_from_catalog_scores, +) def catalog(codes=((0,), (0,), (1,), (1,), (2,))): @@ -249,7 +257,7 @@ def test_unsupported_metric_fails_before_fit(): def test_lazy_imports(): - subprocess.run([sys.executable, "-c", "import sys; from cornac.eval_methods import SemanticIDEvaluation; from cornac.models import SemanticIDRecommender; assert 'torch' not in sys.modules; assert 'transformers' not in sys.modules"], check=True) + subprocess.run([sys.executable, "-c", "import sys; from cornac.eval_methods import SemanticIDEvaluation; from cornac.models import SemanticIDRecommender; from cornac.utils.semantic_id import SemanticIDCatalog, TrainingPopularity, HistoryCosine, resolve_semantic_ids; assert 'torch' not in sys.modules; assert 'transformers' not in sys.modules"], check=True) def test_trace_pickle_keeps_identity_and_readonly_predictions(): @@ -328,3 +336,12 @@ def test_wrong_target_codes_rejected_and_same_user_queries_are_distinct(): SemanticIDTrace(cat.stable_hash, records), (1,), {"custom": resolver}) assert seen == [(0, 0), (1, 0)] assert result["per_query"]["Item[custom]"]["recall"].tolist() == [[1.], [0.]] + + +def test_non_semantic_id_model_fails_before_fit(): + from cornac.models.recommender import NextItemRecommender + + train, test = data() + evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT") + with pytest.raises(ValueError, match="SemanticIDRecommender"): + evaluation.evaluate(NextItemRecommender("native"), [Recall(2)]) diff --git a/tests/cornac/models/test_semantic_id_adapters.py b/tests/cornac/models/test_semantic_id_adapters.py index 8272fd71b..52c65ac30 100644 --- a/tests/cornac/models/test_semantic_id_adapters.py +++ b/tests/cornac/models/test_semantic_id_adapters.py @@ -22,7 +22,7 @@ pytest.importorskip("transformers") from cornac.models import TIGER -from cornac.models.semantic_id import prediction_from_catalog_scores +from cornac.utils.semantic_id import prediction_from_catalog_scores def tiny_tiger(model_class=TIGER): From 2a6e26e46913c7148c7222295385af863432d51b Mon Sep 17 00:00:00 2001 From: hieuddo Date: Thu, 22 Oct 2026 23:20:53 +0800 Subject: [PATCH 3/5] Simplify Semantic ID evaluation utilities --- cornac/eval_methods/semantic_id_evaluation.py | 30 +++++++++------ cornac/models/diffgrm/diffgrm.py | 37 ++++++++++--------- cornac/models/tiger/recom_tiger.py | 1 - cornac/utils/semantic_id.py | 26 +------------ .../test_semantic_id_evaluation.py | 21 +++++++++++ 5 files changed, 61 insertions(+), 54 deletions(-) diff --git a/cornac/eval_methods/semantic_id_evaluation.py b/cornac/eval_methods/semantic_id_evaluation.py index 2a704576f..393988acb 100644 --- a/cornac/eval_methods/semantic_id_evaluation.py +++ b/cornac/eval_methods/semantic_id_evaluation.py @@ -158,7 +158,8 @@ def collect_semantic_id_trace(model, dataset, *, mode="last", n_candidates=20, if mode not in ("last", "next"): raise ValueError("mode must be last or next") _cutoffs((n_candidates,)) - catalog = model.semantic_id_catalog() if catalog is None else catalog + native_catalog = model.semantic_id_catalog() + catalog = native_catalog if catalog is None else catalog records = [] for [session], [mapped], [items] in dataset.si_iter(batch_size=1, shuffle=False): if len(items) < 2: @@ -174,13 +175,11 @@ def collect_semantic_id_trace(model, dataset, *, mode="last", n_candidates=20, history_items=list(items[:position]), user_index=user, query_index=query, session_index=int(session), evaluation_position=position, n_candidates=n_candidates) + validate_prediction(native_catalog, prediction) # Candidate filtering changes admission, never rank positions. validity = np.asarray([tuple(row) in catalog.address_to_items for row in prediction.address_sids]) if not np.array_equal(validity, prediction.catalog_validity): - # Verify original model validity before applying an explicit - # restricted universe; otherwise a malformed adapter is hidden. - validate_prediction(model.semantic_id_catalog(), prediction) prediction = SemanticIDPrediction(prediction.address_sids, prediction.scores, validity, prediction.decoder_diagnostics) try: @@ -232,9 +231,7 @@ def _evaluate_split(self, model, dataset, metrics, resolvers, user_based): catalog = SemanticIDCatalog(native.item_indices[mask], native.tokenizer_sids[mask], native.address_sids[mask], native.tokenizer_codebook_cardinalities, - native.address_codebook_cardinalities, - metadata={"source_catalog_hash": native.stable_hash, - "admission": "training_items"}) + native.address_codebook_cardinalities) cutoffs = tuple(dict.fromkeys(int(metric.k) for metric in metrics)) trace = collect_semantic_id_trace(model, dataset, mode=self.mode, n_candidates=max(cutoffs), catalog=catalog, @@ -243,16 +240,23 @@ def _evaluate_split(self, model, dataset, metrics, resolvers, user_based): replay = evaluate_semantic_id_trace(catalog, trace, cutoffs, resolvers) averages, per_user = OrderedDict(), OrderedDict() users = np.asarray([r.user_index for r in trace.records]) + user_queries = {int(user): np.flatnonzero(users == user) + for user in np.unique(users)} + cutoff_columns = {cutoff: column for column, cutoff in enumerate(cutoffs)} for view, rows in replay["per_query"].items(): for metric in metrics: field = "recall" if isinstance(metric, Recall) else "ndcg" - values = rows[field][:, cutoffs.index(metric.k)] + values = rows[field][:, cutoff_columns[metric.k]] name = f"{view}/{metric.name}" - grouped = {int(user): values[users == user].tolist() for user in np.unique(users)} + grouped = {user: values[indices].tolist() + for user, indices in user_queries.items()} per_user[name] = grouped - averages[name] = (float(np.mean([np.mean(v) for v in grouped.values()])) - if user_based and grouped else - float(values.mean()) if len(values) else 0.) + if not len(values): + averages[name] = 0. + elif user_based: + averages[name] = float(np.mean([np.mean(v) for v in grouped.values()])) + else: + averages[name] = float(values.mean()) result = Result(model.name, averages, per_user) result.semantic_id_trace = trace result.semantic_id_evaluation = replay @@ -287,6 +291,8 @@ def evaluate(self, model, metrics, user_based=False, show_validation=True): result.metric_avg_results["Test (s)"] = time.time() - started validation = None if show_validation and self.val_set is not None: + started = time.time() model.transform(self.val_set) validation = self._evaluate_split(model, self.val_set, metrics, resolvers, user_based) + validation.metric_avg_results["Time (s)"] = time.time() - started return result, validation diff --git a/cornac/models/diffgrm/diffgrm.py b/cornac/models/diffgrm/diffgrm.py index 5a3cd493b..9ade9b784 100644 --- a/cornac/models/diffgrm/diffgrm.py +++ b/cornac/models/diffgrm/diffgrm.py @@ -480,23 +480,25 @@ def cpd_decode_batch( catalog_set = { tuple(row) for row in np.asarray(catalog_codes).tolist() } - batch_codes, batch_scores, batch_diagnostics, batch_raw = [], [], [], [] + batch_codes, batch_scores, batch_diagnostics = [], [], [] + if return_raw: + batch_raw = [] for row in range(batch_size): complete = {} - raw_codes = [] - raw_scores = [] - raw_validity = [] + if return_raw: + raw_codes = [] + raw_scores = [] + raw_validity = [] for codes, score in zip( beams[row].cpu().tolist(), beam_scores[row].cpu().tolist() ): key = tuple(codes) is_complete = -1 not in key and np.isfinite(score) - is_valid = is_complete and ( - catalog_set is None or key in catalog_set - ) - raw_codes.append(key) - raw_scores.append(float(score)) - raw_validity.append(bool(is_valid)) + if return_raw: + raw_codes.append(key) + raw_scores.append(float(score)) + raw_validity.append(bool(is_complete and ( + catalog_set is None or key in catalog_set))) if not is_complete: continue complete[key] = max(complete.get(key, -float("inf")), score) @@ -517,13 +519,14 @@ def cpd_decode_batch( "duplicate_paths": len(beams[row]) - len(complete), } ) - batch_raw.append( - { - "address_sids": raw_codes, - "scores": raw_scores, - "catalog_validity": raw_validity, - } - ) + if return_raw: + batch_raw.append( + { + "address_sids": raw_codes, + "scores": raw_scores, + "catalog_validity": raw_validity, + } + ) if return_diagnostics and return_raw: return batch_codes, batch_scores, batch_diagnostics, batch_raw diff --git a/cornac/models/tiger/recom_tiger.py b/cornac/models/tiger/recom_tiger.py index c9bffabd8..f04be4f6c 100644 --- a/cornac/models/tiger/recom_tiger.py +++ b/cornac/models/tiger/recom_tiger.py @@ -612,7 +612,6 @@ def semantic_id_catalog(self): self.semantic_id_catalog_ = SemanticIDCatalog( np.arange(len(self.sid_table)), self.sid_table[:, :-1], self.sid_table, tuple(self.level_sizes[:-1]), tuple(self.level_sizes), - mapping_policy="disambiguation_suffix", ) return self.semantic_id_catalog_ diff --git a/cornac/utils/semantic_id.py b/cornac/utils/semantic_id.py index a61cee424..5d1eef931 100644 --- a/cornac/utils/semantic_id.py +++ b/cornac/utils/semantic_id.py @@ -108,8 +108,6 @@ class SemanticIDCatalog: address_sids: np.ndarray tokenizer_codebook_cardinalities: tuple address_codebook_cardinalities: tuple - mapping_policy: str = "preserve_all" - metadata: Mapping = field(default_factory=dict) _address_to_items: Mapping = field(init=False, repr=False, compare=False) _item_to_row: Mapping = field(init=False, repr=False, compare=False) _stable_hash: str = field(init=False, repr=False, compare=False) @@ -149,10 +147,6 @@ def __post_init__(self): ) _validate_sid_range(tokenizer_sids, tokenizer_cardinalities, "tokenizer_sids") _validate_sid_range(address_sids, address_cardinalities, "address_sids") - if not isinstance(self.mapping_policy, str) or not self.mapping_policy: - raise ValueError("mapping_policy must be a non-empty string") - metadata = _freeze_native(self.metadata) - address_to_items = {} item_to_row = {} for row, (item_index, address) in enumerate(zip(item_indices, address_sids)): @@ -175,7 +169,6 @@ def __post_init__(self): object.__setattr__( self, "address_codebook_cardinalities", address_cardinalities ) - object.__setattr__(self, "metadata", metadata) object.__setattr__(self, "_address_to_items", address_to_items) object.__setattr__(self, "_item_to_row", MappingProxyType(item_to_row)) object.__setattr__(self, "_stable_hash", self._compute_hash()) @@ -195,8 +188,6 @@ def __reduce__(self): self.address_sids, self.tokenizer_codebook_cardinalities, self.address_codebook_cardinalities, - self.mapping_policy, - _thaw_native(self.metadata), ), ) @@ -207,7 +198,7 @@ def address_to_items(self): @property def stable_hash(self): - """Stable SHA-256 identity of the arrays and catalog policy.""" + """Stable SHA-256 identity of the arrays and codebook cardinalities.""" return self._stable_hash def tokenizer_sid_for_item(self, item_index): @@ -232,8 +223,6 @@ def _compute_hash(self): payload = { "tokenizer_codebook_cardinalities": (self.tokenizer_codebook_cardinalities), "address_codebook_cardinalities": self.address_codebook_cardinalities, - "mapping_policy": self.mapping_policy, - "metadata": _thaw_native(self.metadata), } digest.update( json.dumps( @@ -303,20 +292,11 @@ def prediction_from_catalog_scores(catalog, item_scores, decoder_diagnostics=Non score = float(scores[item_index]) address_scores[address] = max(address_scores.get(address, -np.inf), score) ranked = sorted(address_scores.items(), key=lambda pair: (-pair[1], pair[0])) - diagnostics = dict(decoder_diagnostics or {}) - diagnostics.update( - { - "scored_catalog_items": len(catalog.item_indices), - "unique_complete_sids": len(ranked), - "duplicate_catalog_addresses": len(catalog.item_indices) - len(ranked), - "invalid_sids": 0, - } - ) return SemanticIDPrediction( address_sids=np.asarray([address for address, _ in ranked], dtype="int64"), scores=np.asarray([score for _, score in ranked], dtype="float64"), catalog_validity=np.ones(len(ranked), dtype="bool"), - decoder_diagnostics=diagnostics, + decoder_diagnostics=decoder_diagnostics or {}, ) @@ -351,8 +331,6 @@ def __post_init__(self): not isinstance(value, (int, np.integer)) or value < 0): raise ValueError(f"{name} must be a nonnegative integer") object.__setattr__(self, name, int(value)) - if self.query_index < 0 or self.target_item_index < 0: - raise ValueError("query_index and target_item_index must be non-negative") if not 0 <= self.history_start_position <= self.history_end_position: raise ValueError("history positions must define a non-negative interval") if self.history_end_position != self.evaluation_position: diff --git a/tests/cornac/eval_methods/test_semantic_id_evaluation.py b/tests/cornac/eval_methods/test_semantic_id_evaluation.py index 5ac437a7c..3ef78d221 100644 --- a/tests/cornac/eval_methods/test_semantic_id_evaluation.py +++ b/tests/cornac/eval_methods/test_semantic_id_evaluation.py @@ -232,6 +232,7 @@ def test_evaluation_uses_training_only_and_separates_validation(): assert validation.semantic_id_evaluation["protocol"]["resolvers"]["popularity"] == expected assert len(model.calls) == 4 assert len(validation.semantic_id_trace.records) == 1 + assert validation.metric_avg_results["Time (s)"] >= 0 @pytest.mark.parametrize("exclude", [False, True]) @@ -321,6 +322,26 @@ def predict_semantic_ids(self, history_items, **context): values.setflags(write=True) +def test_restricted_catalog_rejects_incorrect_native_validity(): + train, test = data() + + class MalformedSID(FakeSID): + def fit(self, train_set, val_set=None): + super().fit(train_set, val_set) + self.catalog_ = catalog(((0,), (1,), (2,), (3,), (4,))) + return self + + def predict_semantic_ids(self, history_items, **context): + # Address 4 belongs to the native catalog, but not the admitted one. + # Its incorrect flag must fail before admission can hide the error. + return SemanticIDPrediction([[4], [0]], [1., 0.], [False, True]) + + evaluation = SemanticIDEvaluation.from_splits( + train, test, fmt="USIT", exclude_unknowns=True) + with pytest.raises(ValueError, match="catalog_validity"): + evaluation.evaluate(MalformedSID(), [Recall(2)], show_validation=False) + + def test_wrong_target_codes_rejected_and_same_user_queries_are_distinct(): from dataclasses import replace cat = catalog() From 5b7f46da637980eb79095d5d86a4bd5ab4d05ea1 Mon Sep 17 00:00:00 2001 From: hieuddo Date: Mon, 26 Oct 2026 12:34:06 +0800 Subject: [PATCH 4/5] Integrate opt-in semantic ID metrics into next-item evaluation --- cornac/eval_methods/__init__.py | 9 - cornac/eval_methods/next_item_evaluation.py | 140 +++- cornac/eval_methods/semantic_id_evaluation.py | 298 --------- cornac/experiment/result.py | 5 +- cornac/models/__init__.py | 2 +- cornac/models/diffgrm/README.md | 15 + cornac/models/diffgrm/recom_diffgrm.py | 56 +- cornac/models/letter/README.md | 15 + cornac/models/letter/recom_letter.py | 29 +- cornac/models/recommender.py | 49 +- cornac/models/rpg/README.md | 15 + cornac/models/rpg/recom_rpg.py | 56 +- cornac/models/tiger/README.md | 18 + cornac/models/tiger/recom_tiger.py | 59 +- cornac/utils/semantic_id.py | 522 --------------- docs/source/api_ref/eval_methods.rst | 159 +++-- docs/source/api_ref/models.rst | 2 +- examples/README.md | 2 +- examples/diffgrm_example.py | 1 + examples/letter_example.py | 1 + examples/rpg_example.py | 1 + examples/semantic_id_evaluation_example.py | 52 +- examples/tiger_example.py | 1 + .../test_semantic_id_evaluation.py | 368 ----------- .../eval_methods/test_semantic_id_metrics.py | 378 +++++++++++ .../models/test_semantic_id_adapters.py | 607 +++++++++--------- 26 files changed, 1064 insertions(+), 1796 deletions(-) delete mode 100644 cornac/eval_methods/semantic_id_evaluation.py delete mode 100644 cornac/utils/semantic_id.py delete mode 100644 tests/cornac/eval_methods/test_semantic_id_evaluation.py create mode 100644 tests/cornac/eval_methods/test_semantic_id_metrics.py diff --git a/cornac/eval_methods/__init__.py b/cornac/eval_methods/__init__.py index ee15c41b3..74979c804 100644 --- a/cornac/eval_methods/__init__.py +++ b/cornac/eval_methods/__init__.py @@ -25,12 +25,6 @@ from .next_item_evaluation import NextItemEvaluation from .propensity_stratified_evaluation import PropensityStratifiedEvaluation -from .semantic_id_evaluation import ( - SemanticIDEvaluation, - collect_semantic_id_trace, - evaluate_semantic_id_trace, -) - __all__ = [ "BaseMethod", "RatioSplit", @@ -40,7 +34,4 @@ "NextBasketEvaluation", "NextItemEvaluation", "PropensityStratifiedEvaluation", - "SemanticIDEvaluation", - "collect_semantic_id_trace", - "evaluate_semantic_id_trace", ] diff --git a/cornac/eval_methods/next_item_evaluation.py b/cornac/eval_methods/next_item_evaluation.py index aceb1a26c..4b83f57bd 100644 --- a/cornac/eval_methods/next_item_evaluation.py +++ b/cornac/eval_methods/next_item_evaluation.py @@ -22,7 +22,8 @@ from ..data import SequentialDataset from ..experiment.result import Result -from ..models import NextItemRecommender +from ..models import NextItemRecommender, SemanticIDMixin +from ..metrics import Recall, NDCG from ..utils import validate_format from . import BaseMethod @@ -31,6 +32,51 @@ "next", ]) + +def _semantic_metrics(model, metrics, enabled): + if not enabled or not isinstance(model, SemanticIDMixin): + return [] + return [mt for mt in metrics if type(mt) in (Recall, NDCG) + and isinstance(mt.k, (int, np.integer)) and mt.k > 0] + + +def _semantic_metric_scores(predictions, target_sid, group_sizes, metrics): + """SID matching and uniform expected item credit after compaction.""" + seen = set() + sid_rank = None + expanded_before = 0 + target_before = 0 + for row in predictions: + sid = tuple(row) + if sid not in group_sizes or sid in seen: + continue + if sid == target_sid: + sid_rank = len(seen) + target_before = expanded_before + seen.add(sid) + expanded_before += group_sizes[sid] + + values = [] + size = group_sizes[target_sid] + for view in ("SID", "CCE"): + for metric in metrics: + if sid_rank is None: + value = 0.0 + elif view == "SID": + value = float(sid_rank < metric.k) + if type(metric) is NDCG: + value /= np.log2(sid_rank + 2) + else: + included = min(size, max(0, metric.k - target_before)) + if type(metric) is Recall: + value = included / size + else: + positions = target_before + np.arange(included) + 2 + value = float(np.sum(1.0 / np.log2(positions)) / size) + values.append(value) + return values, len(seen) + + def ranking_eval( model, metrics, @@ -40,6 +86,7 @@ def ranking_eval( exclude_unknowns=True, mode="last", verbose=False, + semantic_id_metrics=False, ): """Evaluate model on provided ranking metrics. @@ -64,6 +111,12 @@ def ranking_eval( verbose: bool, optional, default: False Output evaluation progress. + semantic_id_metrics: bool, optional, default: False + Set True to add SID and CCE views for finite-K Recall/NDCG on + SemanticIDMixin models. + Returned lists contain ordinary metrics first, then supported SID metrics, + then supported CCE metrics, in the requested metric order within each view. + Returns ------- res: (List, List) @@ -76,9 +129,24 @@ def ranking_eval( if len(metrics) == 0: return [], [] + semantic_metrics = _semantic_metrics(model, metrics, semantic_id_metrics) + if isinstance(model, SemanticIDMixin) and not semantic_id_metrics: + warnings.warn("Semantic ID metrics are disabled; reporting ordinary item metrics only.") + result_count = len(metrics) + 2 * len(semantic_metrics) avg_results = [] - session_results = [defaultdict(list) for _ in enumerate(metrics)] - user_results = [defaultdict(list) for _ in enumerate(metrics)] + session_results = [defaultdict(list) for _ in range(result_count)] + user_results = [defaultdict(list) for _ in range(result_count)] + short_queries = 0 + if semantic_metrics: + catalog = np.asarray(model.semantic_id_catalog()) + if (catalog.ndim != 2 or catalog.shape[0] != model.total_items or + catalog.shape[1] == 0 or not np.issubdtype(catalog.dtype, np.integer)): + raise ValueError("semantic_id_catalog must be an integer array (total_items, SID_length)") + candidate_count = train_set.num_items if exclude_unknowns else test_set.num_items + group_sizes = defaultdict(int) + for row in catalog[:candidate_count]: + group_sizes[tuple(row)] += 1 + max_cutoff = max(mt.k for mt in semantic_metrics) user_sessions = defaultdict(list) session_ids = [] @@ -117,7 +185,8 @@ def ranking_eval( u_gt_neg_items = np.nonzero(u_gt_neg_mask)[0] item_indices = np.nonzero(u_gt_pos_mask + u_gt_neg_mask)[0] - item_rank, item_scores = model.rank( + rank = model.rank_with_semantic_ids if semantic_metrics else model.rank + predictions = rank( user_idx, item_indices, history_items=session_items[:test_pos], @@ -127,21 +196,43 @@ def ranking_eval( extra_data=test_set.extra_data, ) - for i, mt in enumerate(metrics): - mt_score = mt.compute( + item_rank, item_scores = predictions[:2] + scores = [] + for mt in metrics: + scores.append(mt.compute( gt_pos=u_gt_pos_items, gt_neg=u_gt_neg_items, pd_rank=item_rank, pd_scores=item_scores, item_indices=item_indices, - ) + )) + if semantic_metrics: + semantic_ids = np.asarray(predictions[2]) + if (semantic_ids.ndim != 2 or semantic_ids.shape[1] != catalog.shape[1] or + not np.issubdtype(semantic_ids.dtype, np.integer)): + raise ValueError("predicted Semantic IDs must be an integer array (n_predictions, SID_length)") + if len(u_gt_pos_items): + extra_scores, distinct_count = _semantic_metric_scores( + semantic_ids, tuple(catalog[test_pos_items]), group_sizes, semantic_metrics) + else: + extra_scores = [0.0] * (2 * len(semantic_metrics)) + distinct_count = len({tuple(row) for row in semantic_ids if tuple(row) in group_sizes}) + scores.extend(extra_scores) + short_queries += distinct_count < max_cutoff + for i, mt_score in enumerate(scores): if user_based: user_results[i][user_idx].append(mt_score) else: session_results[i][sid].append(mt_score) + if short_queries: + warnings.warn( + f"{short_queries} evaluation queries returned fewer than {max_cutoff} " + "valid distinct Semantic IDs; SID/CCE metrics use the available predictions " + "without increasing the decoding budget.") + # avg results of ranking metrics - for i, mt in enumerate(metrics): + for i in range(result_count): if user_based: user_ids = list(user_sessions.keys()) user_avg_results = [np.mean(user_results[i][user_idx]) for user_idx in user_ids] @@ -190,8 +281,26 @@ class NextItemEvaluation(BaseMethod): verbose: bool, optional, default: False Output running log. + semantic_id_metrics: bool, optional, default: False + Set True for models implementing :class:`cornac.models.SemanticIDMixin` + to report ``SID-Recall@K``, ``SID-NDCG@K``, ``CCE-Recall@K`` and + ``CCE-NDCG@K`` for requested finite-K Recall/NDCG metrics. Ordinary item + metrics retain their existing names and model resolution policy. Other + metrics and non-SID models still run their ordinary item evaluation. + False reports ordinary item metrics only and warns for SID models. + Notes ----- + **Semantic IDs.** Full IDs include disambiguating suffixes. Invalid and + repeated predictions are removed before computing compact SID ranks. CCE + expands each surviving ID to its admitted item bucket and averages item + Recall/NDCG under uniform order within each bucket. Thus CCE is expected + item performance, while ordinary metrics evaluate the model's actual item + ranking. Candidate filtering also restricts bucket sizes. Predictions are + obtained once per query for all views; configured decoding budgets stay + unchanged, with one warning per split when distinct valid IDs fall below + the largest supported cutoff. No ungenerated candidates are added. + **Item content.** Modalities (e.g., ``item_feature=FeatureModality(features=..., ids=...)``) can be passed as keyword arguments to the constructor or any of the class-method builders @@ -223,8 +332,10 @@ def __init__( mode="last", exclude_unknowns=True, verbose=False, + semantic_id_metrics=False, **kwargs, ): + self.semantic_id_metrics = semantic_id_metrics super().__init__( data=data, data_size=0 if data is None else len(data), @@ -314,6 +425,7 @@ def eval( user_based=False, verbose=False, mode="last", + semantic_id_metrics=False, **kwargs, ): metric_avg_results = OrderedDict() @@ -328,11 +440,15 @@ def eval( exclude_unknowns=exclude_unknowns, mode=mode, verbose=verbose, + semantic_id_metrics=semantic_id_metrics, ) - for i, mt in enumerate(ranking_metrics): - metric_avg_results[mt.name] = avg_results[i] - metric_user_results[mt.name] = user_results[i] + names = [mt.name for mt in ranking_metrics] + semantic_metrics = _semantic_metrics(model, ranking_metrics, semantic_id_metrics) + names.extend(f"{view}-{mt.name}" for view in ("SID", "CCE") for mt in semantic_metrics) + for i, name in enumerate(names): + metric_avg_results[name] = avg_results[i] + metric_user_results[name] = user_results[i] return Result(model.name, metric_avg_results, metric_user_results) @@ -403,6 +519,7 @@ def evaluate(self, model, metrics, user_based, show_validation=True): user_based=user_based, mode=self.mode, verbose=self.verbose, + semantic_id_metrics=self.semantic_id_metrics, ) test_time = time.time() - start test_result.metric_avg_results["Train (s)"] = train_time @@ -422,6 +539,7 @@ def evaluate(self, model, metrics, user_based, show_validation=True): user_based=user_based, mode=self.mode, verbose=self.verbose, + semantic_id_metrics=self.semantic_id_metrics, ) val_time = time.time() - start val_result.metric_avg_results["Time (s)"] = val_time diff --git a/cornac/eval_methods/semantic_id_evaluation.py b/cornac/eval_methods/semantic_id_evaluation.py deleted file mode 100644 index 393988acb..000000000 --- a/cornac/eval_methods/semantic_id_evaluation.py +++ /dev/null @@ -1,298 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -"""Next-item recommendation evaluation from raw Semantic ID predictions.""" - -import time -from collections import OrderedDict -from collections.abc import Mapping - -import numpy as np - -from ..experiment.result import Result -from ..metrics import Recall, NDCG -from ..models.recommender import SemanticIDRecommender -from ..utils.semantic_id import ( - SemanticIDCatalog, - SemanticIDPrediction, - SemanticIDTrace, - SemanticIDTraceRecord, - SemanticIDResolver, - HistoryCosine, - TrainingPopularity, - resolve_semantic_ids, - validate_prediction, -) -from .next_item_evaluation import NextItemEvaluation - - -def _cutoffs(values): - values = tuple(values) - if (not values or any(isinstance(k, (bool, np.bool_)) or - not isinstance(k, (int, np.integer)) or k <= 0 for k in values) or - len(set(values)) != len(values)): - raise ValueError("cutoffs must be distinct positive finite integers") - return tuple(int(k) for k in values) - - -def _rank_rows(rank, cutoffs): - hit = np.asarray([float(rank is not None and rank < k) for k in cutoffs]) - ndcg = hit / np.log2(rank + 2) if rank is not None else hit.copy() - return hit, ndcg - - -def evaluate_semantic_id_trace(catalog, trace, cutoffs=(5, 10, 20), resolvers=None): - """Evaluate every view on the same ordered raw predictions. - - CCE is the analytical expectation under uniform permutations *within full - expanded buckets*. Earlier buckets consume their full widths. Invalid and - duplicate addresses consume one empty rank slot in every view. A member - resolver chooses one item per bucket without changing address order. - - ``resolvers`` maps display names to SemanticIDResolver instances. Callbacks - receive query/user identity and bucket members only. Results contain scalar - aggregates, per-query Recall/NDCG arrays and protocol/diagnostic sidecars. - This function averages over queries; SemanticIDEvaluation also supports the - framework's per-user aggregation. CCE follows Zhang et al., - "Faithful Evaluation of Semantic-ID Tokenizers for Generative Recommendation" - (CIKM 2026): https://arxiv.org/abs/2605.25330v2. - """ - cutoffs = _cutoffs(cutoffs) - if not isinstance(catalog, SemanticIDCatalog) or not isinstance(trace, SemanticIDTrace): - raise ValueError("expected SemanticIDCatalog and SemanticIDTrace") - if trace.catalog_hash != catalog.stable_hash: - raise ValueError("trace catalog_hash differs from catalog") - if resolvers is None: - resolvers = {} - if not isinstance(resolvers, Mapping) or any( - not isinstance(name, str) or not name or any(c in name for c in "[]/") - for name in resolvers): - raise ValueError("resolvers must map nonempty plain names to resolvers") - if any(not isinstance(r, SemanticIDResolver) for r in resolvers.values()): - raise ValueError("resolvers must contain SemanticIDResolver instances") - views = ("SID", "CCE") + tuple(f"Item[{name}]" for name in resolvers) - rows = {view: {metric: np.zeros((len(trace.records), len(cutoffs))) - for metric in ("recall", "ndcg")} for view in views} - invalid = duplicate = total = 0 - sizes = [] - for index, record in enumerate(trace.records): - try: - address = catalog.address_sid_for_item(record.target_item_index) - tokenizer = catalog.tokenizer_sid_for_item(record.target_item_index) - except KeyError as error: - raise ValueError("trace target item is outside catalog") from error - if address != record.target_address_sid or tokenizer != record.target_tokenizer_sid: - raise ValueError("trace target codes differ from catalog") - prediction = record.prediction - validate_prediction(catalog, prediction) - addresses = [tuple(int(v) for v in row) for row in prediction.address_sids] - rank = next((i for i, value in enumerate(addresses) if value == address), None) - rows["SID"]["recall"][index], rows["SID"]["ndcg"][index] = _rank_rows(rank, cutoffs) - before, seen, reached = 0, set(), False - for value, valid in zip(addresses, prediction.catalog_validity): - if value == address: - reached = True - break - if not valid or value in seen: - before += 1 - else: - seen.add(value) - before += len(catalog.address_to_items[value]) - size = len(catalog.address_to_items[address]) - sizes.append(size) - if reached: - for col, cutoff in enumerate(cutoffs): - included = min(size, max(0, cutoff - before)) - rows["CCE"]["recall"][index, col] = included / size - rows["CCE"]["ndcg"][index, col] = ( - np.sum(1 / np.log2(before + np.arange(included) + 2)) / size) - for name, resolver in resolvers.items(): - slots = resolve_semantic_ids(catalog, prediction, resolver, - user_index=record.user_index, - query_index=record.query_index) - item_rank = next((i for i, item in enumerate(slots) - if item == record.target_item_index), None) - rows[f"Item[{name}]"]["recall"][index], rows[f"Item[{name}]"]["ndcg"][index] = _rank_rows(item_rank, cutoffs) - total += len(addresses) - invalid += int(np.count_nonzero(~prediction.catalog_validity)) - duplicate += len(addresses) - len(set(addresses)) - aggregate = {} - for view, metrics in rows.items(): - aggregate[view] = {} - for metric, values in metrics.items(): - values.setflags(write=False) - for col, cutoff in enumerate(cutoffs): - aggregate[view][f"{metric.upper() if metric == 'ndcg' else 'Recall'}@{cutoff}"] = ( - float(values[:, col].mean()) if len(values) else 0.) - return {"aggregate": aggregate, "per_query": rows, - "diagnostics": {"invalid_rate": invalid / total if total else 0., - "duplicate_rate": duplicate / total if total else 0., - "target_collision_group_size": np.asarray(sizes, dtype="int64")}, - "protocol": {"catalog_hash": catalog.stable_hash, "cutoffs": cutoffs, - "invalid_and_duplicate_slots": "preserved", - "order": "decoder_order", "CCE": "uniform_expanded_bucket_expectation", - "resolvers": {name: resolver.metadata for name, resolver in resolvers.items()}}} - - -def collect_semantic_id_trace(model, dataset, *, mode="last", n_candidates=20, - catalog=None, exclude_unknowns=False, - num_train_items=None): - """Collect raw predictions once per query, with no target in model context. - - ``n_candidates`` requests a minimum budget; legitimately shorter decoder - output stays shorter. ``catalog`` may restrict the model's complete catalog - to the evaluator's admitted universe. Excluded addresses retain empty slots. - """ - if mode not in ("last", "next"): - raise ValueError("mode must be last or next") - _cutoffs((n_candidates,)) - native_catalog = model.semantic_id_catalog() - catalog = native_catalog if catalog is None else catalog - records = [] - for [session], [mapped], [items] in dataset.si_iter(batch_size=1, shuffle=False): - if len(items) < 2: - continue - user = int(dataset.uir_tuple[0][mapped[0]]) - start = 1 if mode == "next" else len(items) - 1 - for position in range(start, len(items)): - target = int(items[position]) - if exclude_unknowns and target >= num_train_items: - continue - query = len(records) - prediction = model.predict_semantic_ids( - history_items=list(items[:position]), user_index=user, - query_index=query, session_index=int(session), - evaluation_position=position, n_candidates=n_candidates) - validate_prediction(native_catalog, prediction) - # Candidate filtering changes admission, never rank positions. - validity = np.asarray([tuple(row) in catalog.address_to_items - for row in prediction.address_sids]) - if not np.array_equal(validity, prediction.catalog_validity): - prediction = SemanticIDPrediction(prediction.address_sids, - prediction.scores, validity, prediction.decoder_diagnostics) - try: - address = catalog.address_sid_for_item(target) - tokenizer = catalog.tokenizer_sid_for_item(target) - except KeyError as error: - raise ValueError("evaluation target is absent from the admitted model catalog") from error - records.append(SemanticIDTraceRecord(query, user, int(session), position, - 0, position, target, tokenizer, address, prediction)) - return SemanticIDTrace(catalog.stable_hash, tuple(records), - {"mode": mode, "requested_candidates": n_candidates}) - - -class SemanticIDEvaluation(NextItemEvaluation): - """Evaluate next-item recommendation through SID, CCE and item views. - - Use this instead of NextItemEvaluation when raw Semantic ID predictions - can refer to multiple items. NextItemEvaluation scores item rankings from - ``model.rank()``; this evaluator preserves raw decoder slots and measures - recommendation performance under CCE or explicit item resolution. It does - not evaluate the intrinsic quality of the IDs. - - Models must implement SemanticIDRecommender. Evaluate ordinary next-item - models separately with NextItemEvaluation on identical data splits and - cutoffs; a single mixed-model Experiment is not supported. - - - Accepts NextItemEvaluation dataset/splitting arguments. ``resolvers`` is an - optional mapping of names to fitted SemanticIDResolver instances; None uses - training popularity. ``item_features`` explicitly enables training-history - cosine and must use global Cornac item rows. ``feature_source`` describes - their provenance. Only Recall/NDCG with positive finite k are supported. - Existing model ranking/training and checkpoint selection remain unchanged. - """ - - def __init__(self, *args, resolvers=None, item_features=None, - feature_source="provided", **kwargs): - self.resolvers = resolvers - self.item_features = item_features - self.feature_source = feature_source - super().__init__(*args, **kwargs) - - def _evaluate_split(self, model, dataset, metrics, resolvers, user_based): - native = model.semantic_id_catalog() - catalog = native - if self.exclude_unknowns: - mask = native.item_indices < self.train_set.num_items - if not np.all(mask): - catalog = SemanticIDCatalog(native.item_indices[mask], - native.tokenizer_sids[mask], native.address_sids[mask], - native.tokenizer_codebook_cardinalities, - native.address_codebook_cardinalities) - cutoffs = tuple(dict.fromkeys(int(metric.k) for metric in metrics)) - trace = collect_semantic_id_trace(model, dataset, mode=self.mode, - n_candidates=max(cutoffs), catalog=catalog, - exclude_unknowns=self.exclude_unknowns, - num_train_items=self.train_set.num_items) - replay = evaluate_semantic_id_trace(catalog, trace, cutoffs, resolvers) - averages, per_user = OrderedDict(), OrderedDict() - users = np.asarray([r.user_index for r in trace.records]) - user_queries = {int(user): np.flatnonzero(users == user) - for user in np.unique(users)} - cutoff_columns = {cutoff: column for column, cutoff in enumerate(cutoffs)} - for view, rows in replay["per_query"].items(): - for metric in metrics: - field = "recall" if isinstance(metric, Recall) else "ndcg" - values = rows[field][:, cutoff_columns[metric.k]] - name = f"{view}/{metric.name}" - grouped = {user: values[indices].tolist() - for user, indices in user_queries.items()} - per_user[name] = grouped - if not len(values): - averages[name] = 0. - elif user_based: - averages[name] = float(np.mean([np.mean(v) for v in grouped.values()])) - else: - averages[name] = float(values.mean()) - result = Result(model.name, averages, per_user) - result.semantic_id_trace = trace - result.semantic_id_evaluation = replay - return result - - def evaluate(self, model, metrics, user_based=False, show_validation=True): - if not isinstance(model, SemanticIDRecommender): - raise ValueError("model must implement SemanticIDRecommender") - if not metrics or any(type(metric) not in (Recall, NDCG) for metric in metrics): - raise ValueError("SemanticIDEvaluation supports Recall and NDCG only") - _cutoffs(tuple(dict.fromkeys(metric.k for metric in metrics))) - if self.train_set is None or self.test_set is None: - raise ValueError("train_set and test_set are required") - self._reset() - started = time.time() - model.fit(self.train_set, self.val_set) - train_time = time.time() - started - users, items = self.train_set.uir_tuple[:2] - resolvers = (dict(self.resolvers) if self.resolvers is not None else - {"popularity": TrainingPopularity(items, self.total_items)}) - if self.item_features is not None: - if "history_cosine" in resolvers: - raise ValueError("history_cosine resolver name is already configured") - if len(self.item_features) < self.total_items: - raise ValueError("item_features must cover all global Cornac item indices") - resolvers["history_cosine"] = HistoryCosine(self.item_features, users, - items, self.total_users, feature_source=self.feature_source) - started = time.time() - model.transform(self.test_set) - result = self._evaluate_split(model, self.test_set, metrics, resolvers, user_based) - result.metric_avg_results["Train (s)"] = train_time - result.metric_avg_results["Test (s)"] = time.time() - started - validation = None - if show_validation and self.val_set is not None: - started = time.time() - model.transform(self.val_set) - validation = self._evaluate_split(model, self.val_set, metrics, resolvers, user_based) - validation.metric_avg_results["Time (s)"] = time.time() - started - return result, validation diff --git a/cornac/experiment/result.py b/cornac/experiment/result.py index 0c7f4e827..c63d5adb9 100644 --- a/cornac/experiment/result.py +++ b/cornac/experiment/result.py @@ -187,10 +187,11 @@ class ExperimentResult(list): """ def __str__(self): - headers = list(self[0].metric_avg_results.keys()) + headers = list(dict.fromkeys(m for r in self for m in r.metric_avg_results)) data, index = [], [] for r in self: - data.append([NUM_FMT.format(r.metric_avg_results[m]) for m in headers]) + data.append([NUM_FMT.format(r.metric_avg_results[m]) if m in r.metric_avg_results + else "N/A" for m in headers]) index.append(r.model_name) return _table_format(data, headers, index, h_bars=[1]) diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index 13d071358..994a344db 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -16,7 +16,7 @@ from .recommender import Recommender from .recommender import NextBasketRecommender from .recommender import NextItemRecommender -from .recommender import SemanticIDRecommender +from .recommender import SemanticIDMixin from .amr import AMR from .ann import AnnoyANN diff --git a/cornac/models/diffgrm/README.md b/cornac/models/diffgrm/README.md index 9b8c31992..049d722fa 100644 --- a/cornac/models/diffgrm/README.md +++ b/cornac/models/diffgrm/README.md @@ -31,6 +31,7 @@ from cornac.models.diffgrm import DiffGRM, DIFFGRM_SPORTS_CONFIG item_feature = FeatureModality(features=item_embeddings, ids=item_ids) eval_method = NextItemEvaluation.from_splits( + semantic_id_metrics=True, train_data=train_data, val_data=val_data, test_data=test_data, @@ -209,3 +210,17 @@ The exported paper configurations use four 256-way digits, one encoder layer, fo - Zhao Liu, Yichen Zhu, Yiqing Yang, Guoping Tang, Rui Huang, Qiang Luo, Xiao Lv, Ruiming Tang, Kun Gai, and Guorui Zhou. [DiffGRM: Diffusion-based Generative Recommendation Model](https://arxiv.org/abs/2510.21805). WWW 2026. - [Released DiffGRM repository](https://github.com/liuzhao09/DiffGRM), behavior audited at commit `ad7b971c7e525e9fea6fb8e362a5c49dccb2473c`. + + +### Semantic ID evaluation + +Set `NextItemEvaluation(..., semantic_id_metrics=True)` to report ordinary +item metrics plus `SID-Recall@K`, `SID-NDCG@K`, `CCE-Recall@K` and `CCE-NDCG@K` for +requested finite-K Recall/NDCG metrics. All views share one inference; CCE averages +item credit over uniform order within each admitted collision bucket. Invalid +and repeated predicted IDs are compacted. Other requested metrics keep ordinary +item evaluation. Configured beam sizes remain unchanged; short candidate lists +warn. Set `semantic_id_metrics=False` for ordinary item metrics only. + +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst) +and [mixed-model example](../../../examples/semantic_id_evaluation_example.py). diff --git a/cornac/models/diffgrm/recom_diffgrm.py b/cornac/models/diffgrm/recom_diffgrm.py index 2172bc336..e9b3db618 100644 --- a/cornac/models/diffgrm/recom_diffgrm.py +++ b/cornac/models/diffgrm/recom_diffgrm.py @@ -20,11 +20,7 @@ import numpy as np from tqdm.auto import trange -from ..recommender import SemanticIDRecommender -from ...utils.semantic_id import ( - SemanticIDCatalog, - SemanticIDPrediction, -) +from ..recommender import NextItemRecommender, SemanticIDMixin from ...utils import get_rng @@ -86,7 +82,7 @@ def _diffgrm_session_iter( ) -class DiffGRM(SemanticIDRecommender): +class DiffGRM(NextItemRecommender, SemanticIDMixin): """DiffGRM: Diffusion-based Generative Recommendation Model. DiffGRM combines parallel OPQ semantic IDs (PSE), on-policy @@ -622,7 +618,6 @@ def _fit_model(self, torch, val_set): self.model.load_state_dict(best_state) def fit(self, train_set, val_set=None): - self.__dict__.pop("semantic_id_catalog_", None) super().fit(train_set, val_set) if not self.trainable: return self @@ -693,13 +688,16 @@ def _ensure_device(self, torch): if next(self.model.parameters()).device != requested: self.model.to(requested) - def _score_history(self, history_items, n_candidates=None, return_raw=False): + def _score_history(self, history_items, return_semantic_ids=False): import torch from .diffgrm import cpd_decode if len(history_items) == 0: - return np.ones(self.total_items, dtype="float") + scores = np.ones(self.total_items, dtype="float") + if return_semantic_ids: + return scores, np.empty((0, self.n_digit), dtype="int64") + return scores self._ensure_device(torch) history = list(history_items)[-self.max_len :] input_iids = np.full((1, self.max_len), self.pad_idx, dtype="int64") @@ -718,7 +716,7 @@ def _score_history(self, history_items, n_candidates=None, return_raw=False): self.model, memory, padding_mask, - beam_size=max(self.beam_size, n_candidates or 0), + beam_size=self.beam_size, catalog_codes=self.sid_table, valid_code_set=self.sid_to_items_, constrained=self.scoring == "catalog", @@ -727,20 +725,16 @@ def _score_history(self, history_items, n_candidates=None, return_raw=False): else None, greedy_final=self.scoring == "released", return_diagnostics=True, - return_raw=return_raw, + return_raw=return_semantic_ids, ) codes, path_scores, diagnostics = decoded[:3] self.last_decode_time_ = time.perf_counter() - decode_start self.last_decode_diagnostics_ = diagnostics - if return_raw: - raw = decoded[3] - return SemanticIDPrediction( - np.asarray(raw["address_sids"], dtype="int64").reshape(-1, self.n_digit), - raw["scores"], raw["catalog_validity"], - dict(diagnostics, scoring=self.scoring, - beam_size=max(self.beam_size, n_candidates or 0)), - ) - return self._decoded_item_scores(codes, path_scores) + scores = self._decoded_item_scores(codes, path_scores) + if return_semantic_ids: + semantic_ids = np.asarray(decoded[3]["address_sids"], dtype="int64").reshape(-1, self.n_digit) + return scores, semantic_ids + return scores def _decoded_item_scores(self, codes, path_scores): scores = np.full(self.total_items, -1e10, dtype="float") @@ -757,24 +751,10 @@ def score(self, user_idx, history_items, **kwargs): return self._score_history(history_items) def semantic_id_catalog(self): - """Export all complete IDs independently of native collision policy.""" + """Return complete IDs independently of native collision policy.""" if not hasattr(self, "sid_table"): raise ValueError("DiffGRM must be fitted before exporting its SID catalog") - cardinalities = (self.codebook_size,) * self.n_digit - if not hasattr(self, "semantic_id_catalog_"): - self.semantic_id_catalog_ = SemanticIDCatalog( - np.arange(len(self.sid_table)), self.sid_table, self.sid_table, - cardinalities, cardinalities, - ) - return self.semantic_id_catalog_ + return self.sid_table - def predict_semantic_ids(self, history_items, **context): - """Return final CPD paths before validity filtering or deduplication.""" - if len(history_items) == 0: - return SemanticIDPrediction( - np.empty((0, self.semantic_id_catalog().address_sids.shape[1]), dtype="int64"), - [], [], {"empty_history": True}, - ) - return self._score_history( - history_items, context.get("n_candidates"), return_raw=True - ) + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + return self._score_history(history_items, return_semantic_ids=True) diff --git a/cornac/models/letter/README.md b/cornac/models/letter/README.md index ea72b59da..3e817b8e5 100644 --- a/cornac/models/letter/README.md +++ b/cornac/models/letter/README.md @@ -27,6 +27,7 @@ from cornac.models import LETTER from cornac.models.letter import LETTER_BEAUTY_CONFIG eval_method = NextItemEvaluation.from_splits( + semantic_id_metrics=True, train_data=train, val_data=val, test_data=test, @@ -94,3 +95,17 @@ The authors' Beauty index contains 12,101 items but only 12,088 unique Semantic ### Reproduction limitation The released repository provides the final Beauty Semantic-ID table, but not the trained tokenizer checkpoint, content embeddings, or SASRec checkpoint used to produce it. Exact reproduction of the paper's learned Semantic IDs therefore requires those missing artifacts; the end-to-end result above is a runnable substitute rather than an exact reconstruction of the paper's tokenizer inputs. + + +### Semantic ID evaluation + +Set `NextItemEvaluation(..., semantic_id_metrics=True)` to report ordinary +item metrics plus `SID-Recall@K`, `SID-NDCG@K`, `CCE-Recall@K` and `CCE-NDCG@K` for +requested finite-K Recall/NDCG metrics. All views share one inference; CCE averages +item credit over uniform order within each admitted collision bucket. Invalid +and repeated predicted IDs are compacted. Other requested metrics keep ordinary +item evaluation. Configured beam sizes remain unchanged; short candidate lists +warn. Set `semantic_id_metrics=False` for ordinary item metrics only. + +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst) +and [mixed-model example](../../../examples/semantic_id_evaluation_example.py). diff --git a/cornac/models/letter/recom_letter.py b/cornac/models/letter/recom_letter.py index 7869865d0..d8dacfe10 100644 --- a/cornac/models/letter/recom_letter.py +++ b/cornac/models/letter/recom_letter.py @@ -21,7 +21,6 @@ from tqdm.auto import trange from ..tiger.recom_tiger import TIGER -from ...utils.semantic_id import SemanticIDCatalog, prediction_from_catalog_scores def _generator_optimizer_groups(model, weight_decay): @@ -326,7 +325,6 @@ def _tokenize(self, torch, feats_t): def _build_semantic_ids(self, codes): """Build the official fixed-length IDs without TIGER's dedup level.""" - self.__dict__.pop("semantic_id_catalog_", None) self.sid_table = np.asarray(codes, dtype="int64") self.level_sizes = [self.rqvae_codebook_size] * self.rqvae_num_levels children = [defaultdict(set) for _ in self.level_sizes] @@ -523,7 +521,7 @@ def lr_lambda(step): if best_state is not None: self.model.load_state_dict(best_state) - def _predict_history(self, history_items, n_candidates=None, return_raw=False): + def _predict_history(self, history_items, return_semantic_ids=False): import torch self._ensure_device(torch) @@ -534,13 +532,12 @@ def _predict_history(self, history_items, n_candidates=None, return_raw=False): with torch.no_grad(): if self.scoring == "beam": beams, log_probs = self.model.generate_beam( - enc_tokens, enc_mask, max(self.n_beams, n_candidates or 0), self.prefix_children + enc_tokens, enc_mask, self.n_beams, self.prefix_children ) - if return_raw: - return self._beam_prediction(beams, log_probs, n_candidates) scores = np.full(self.total_items, -1e10, dtype="float") for sid, log_prob in zip(beams, log_probs): - scores[self.sid_to_item[sid]] = log_prob + if sid in self.sid_to_item: + scores[self.sid_to_item[sid]] = log_prob else: sid_table = torch.as_tensor( self.sid_table, dtype=torch.long, device=self.device_ @@ -548,20 +545,14 @@ def _predict_history(self, history_items, n_candidates=None, return_raw=False): scores = self.model.score_all_items( enc_tokens, enc_mask, sid_table, self.scoring_batch_size ).astype("float") - if return_raw: - return prediction_from_catalog_scores( - self.semantic_id_catalog(), scores, - {"scoring": "exact", "address_score_aggregation": "max_member"}, - ) + if return_semantic_ids: + semantic_ids = (np.asarray(beams, dtype="int64").reshape(-1, self.sid_table.shape[1]) + if self.scoring == "beam" else self._semantic_ids_from_scores(scores)) + return scores, semantic_ids return scores def semantic_id_catalog(self): - """Export LETTER's suffix-free IDs, preserving collision buckets.""" + """Return suffix-free IDs aligned to item indices.""" if not hasattr(self, "sid_table"): raise ValueError("LETTER must be fitted before exporting its SID catalog") - if not hasattr(self, "semantic_id_catalog_"): - self.semantic_id_catalog_ = SemanticIDCatalog( - np.arange(len(self.sid_table)), self.sid_table, self.sid_table, - tuple(self.level_sizes), tuple(self.level_sizes), - ) - return self.semantic_id_catalog_ + return self.sid_table diff --git a/cornac/models/recommender.py b/cornac/models/recommender.py index 60e253eba..68f8e7d5a 100644 --- a/cornac/models/recommender.py +++ b/cornac/models/recommender.py @@ -502,6 +502,10 @@ def rank(self, user_idx, item_indices=None, k=-1, **kwargs): except ScoreException: known_item_scores = np.ones(self.total_items) * self.default_score() + return self._rank_from_scores(known_item_scores, item_indices, k) + + def _rank_from_scores(self, known_item_scores, item_indices=None, k=-1): + """Apply the standard item ranking to already computed scores.""" # check if the returned scores also cover unknown items # if not, all unknown items will be given the MIN score if len(known_item_scores) == self.total_items: @@ -766,19 +770,44 @@ def score(self, user_idx, history_items, **kwargs): raise NotImplementedError("The algorithm is not able to make score prediction!") -class SemanticIDRecommender(NextItemRecommender): - """Optional interface for recommenders that generate Semantic IDs.""" +class SemanticIDMixin: + """Optional Semantic ID capability for next-item recommenders. + + Complete IDs include any item-disambiguating suffix. The combined ranking + exposes ordinary item predictions and generated IDs from the same inference; + it never reconstructs beam candidates from item-score filler values. + """ def semantic_id_catalog(self): - """Return the fitted :class:`~cornac.utils.semantic_id.SemanticIDCatalog`.""" + """Return an integer array ``(total_items, SID_length)`` aligned to items.""" + raise NotImplementedError + + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + """Return full item scores and ranked complete IDs from one inference.""" raise NotImplementedError - def predict_semantic_ids(self, history_items, **context): - """Return raw predictions before resolution, preserving decoder order. + def rank_with_semantic_ids(self, user_idx, item_indices=None, k=-1, **kwargs): + """Return item ranks, aligned item scores and raw ranked Semantic IDs. - ``context`` may contain ``user_index``, ``query_index``, session/history - positions, and ``n_candidates`` (a requested minimum output budget). - Adapters should honor the budget where supported and may return fewer - candidates when decoding terminates. Never fill from ungenerated items. + Item ranking has the same contract as :meth:`Recommender.rank`. + The third array has integer shape ``(n_predictions, SID_length)`` and + preserves the full generated list, including invalid/repeated rows, + independently of item-ranking ``k``. It is not retained on the model. + History and other context are passed in ``kwargs`` as with ``rank``. """ - raise NotImplementedError + try: + scores, semantic_ids = self._score_with_semantic_ids(user_idx, **kwargs) + except ScoreException: + scores = np.ones(self.total_items) * self.default_score() + semantic_ids = np.empty((0, self.semantic_id_catalog().shape[1]), dtype="int64") + item_rank, item_scores = self._rank_from_scores(scores, item_indices, k) + return item_rank, item_scores, semantic_ids + + def _semantic_ids_from_scores(self, scores): + """Rank exhaustive SID scores by maximum member, then lexicographic ID.""" + catalog = self.semantic_id_catalog() + group_scores = {} + for sid, score in zip(map(tuple, catalog), scores): + group_scores[sid] = max(group_scores.get(sid, -np.inf), score) + ranked = sorted(group_scores, key=lambda sid: (-group_scores[sid], sid)) + return np.asarray(ranked, dtype="int64").reshape(-1, catalog.shape[1]) diff --git a/cornac/models/rpg/README.md b/cornac/models/rpg/README.md index 4cd84808b..a7a29669a 100644 --- a/cornac/models/rpg/README.md +++ b/cornac/models/rpg/README.md @@ -35,6 +35,7 @@ from cornac.models import RPG from cornac.models.rpg import RPG_BEAUTY_CONFIG eval_method = NextItemEvaluation.from_splits( + semantic_id_metrics=True, train_data=train, val_data=val, test_data=test, @@ -140,3 +141,17 @@ within 3.1% and 8.3% of the paper's R@5, respectively; Toys is 3.5% higher. - Graph decoding is approximate and stochastic because its initial beam is sampled. Set `seed` for repeatability, or use exact scoring when deterministic full-catalog ranks are required. + + +### Semantic ID evaluation + +Set `NextItemEvaluation(..., semantic_id_metrics=True)` to report ordinary +item metrics plus `SID-Recall@K`, `SID-NDCG@K`, `CCE-Recall@K` and `CCE-NDCG@K` for +requested finite-K Recall/NDCG metrics. All views share one inference; CCE averages +item credit over uniform order within each admitted collision bucket. Invalid +and repeated predicted IDs are compacted. Other requested metrics keep ordinary +item evaluation. Configured beam sizes remain unchanged; short candidate lists +warn. Set `semantic_id_metrics=False` for ordinary item metrics only. + +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst) +and [mixed-model example](../../../examples/semantic_id_evaluation_example.py). diff --git a/cornac/models/rpg/recom_rpg.py b/cornac/models/rpg/recom_rpg.py index 66e3353ef..acb2edcff 100644 --- a/cornac/models/rpg/recom_rpg.py +++ b/cornac/models/rpg/recom_rpg.py @@ -18,12 +18,7 @@ import numpy as np from tqdm.auto import trange -from ..recommender import SemanticIDRecommender -from ...utils.semantic_id import ( - SemanticIDCatalog, - SemanticIDPrediction, - prediction_from_catalog_scores, -) +from ..recommender import NextItemRecommender, SemanticIDMixin from ...utils import get_rng @@ -103,7 +98,7 @@ def _rpg_session_iter( ) -class RPG(SemanticIDRecommender): +class RPG(NextItemRecommender, SemanticIDMixin): """RPG: Generating Long Semantic IDs in Parallel for Recommendation. RPG is the architectural counterpoint to TIGER. Instead of short *ordered* @@ -607,7 +602,6 @@ def _fit_backbone(self, torch, val_set): self.model.load_state_dict(best_state) def fit(self, train_set, val_set=None): - self.__dict__.pop("semantic_id_catalog_", None) super().fit(train_set, val_set) if not self.trainable: return self @@ -652,13 +646,16 @@ def _ensure_device(self, torch): if next(self.model.parameters()).device.type != torch.device(self.device_).type: self.model.to(self.device_) - def _predict_history(self, history_items, n_candidates=None, return_raw=False): + def _predict_history(self, history_items, return_semantic_ids=False): import torch from .rpg import graph_decode if len(history_items) == 0: - return np.ones(self.total_items, dtype="float") + scores = np.ones(self.total_items, dtype="float") + if return_semantic_ids: + return scores, np.empty((0, self.sid_table.shape[1]), dtype="int64") + return scores self._ensure_device(torch) hist = list(history_items)[-self.max_len :] hist = [self.pad_idx] * (self.max_len - len(hist)) + hist @@ -670,11 +667,8 @@ def _predict_history(self, history_items, n_candidates=None, return_raw=False): if self.scoring == "exact": scores = self._exact_item_scores(logits)[0].cpu().numpy().astype("float") - if return_raw: - return prediction_from_catalog_scores( - self.semantic_id_catalog(), scores, - {"scoring": "exact", "address_score_aggregation": "max_member"}, - ) + if return_semantic_ids: + return scores, self._semantic_ids_from_scores(scores) return scores # graph-guided beam decoding over the final propagated beam @@ -683,42 +677,24 @@ def _predict_history(self, history_items, n_candidates=None, return_raw=False): flat, self.adjacency, self.item_tokens, - max(self.n_beams, n_candidates or 0), + self.n_beams, self.propagation_steps, self.graph_rng, ) - if return_raw: - return SemanticIDPrediction( - self.sid_table[candidates], candidate_scores, - np.ones(len(candidates), dtype=bool), - {"scoring": "graph", "beam_size": max(self.n_beams, n_candidates or 0)}, - ) scores = np.full(self.total_items, -1e10, dtype="float") scores[candidates] = candidate_scores + if return_semantic_ids: + return scores, self.sid_table[candidates] return scores def semantic_id_catalog(self): - """Export un-offset OPQ codes with complete collision buckets.""" + """Return complete, un-offset OPQ codes aligned to item indices.""" if not hasattr(self, "sid_table"): raise ValueError("RPG must be fitted before exporting its SID catalog") - cardinalities = (self.codebook_size,) * self.n_codebook - if not hasattr(self, "semantic_id_catalog_"): - self.semantic_id_catalog_ = SemanticIDCatalog( - np.arange(len(self.sid_table)), self.sid_table, self.sid_table, - cardinalities, cardinalities, - ) - return self.semantic_id_catalog_ + return self.sid_table - def predict_semantic_ids(self, history_items, **context): - """Retain final graph candidate order, using the native mutable RNG.""" - if len(history_items) == 0: - return SemanticIDPrediction( - np.empty((0, self.semantic_id_catalog().address_sids.shape[1]), dtype="int64"), - [], [], {"empty_history": True}, - ) - return self._predict_history( - history_items, context.get("n_candidates"), return_raw=True - ) + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + return self._predict_history(history_items, return_semantic_ids=True) def score(self, user_idx, history_items, **kwargs): return self._predict_history(history_items) diff --git a/cornac/models/tiger/README.md b/cornac/models/tiger/README.md index d697cc987..53b05cc19 100644 --- a/cornac/models/tiger/README.md +++ b/cornac/models/tiger/README.md @@ -13,6 +13,7 @@ from cornac.models import TIGER from cornac.models.tiger import GRID_CONFIG, PAISCHER_CONFIG eval_method = NextItemEvaluation.from_splits( + semantic_id_metrics=True, train_data=train, test_data=test, val_data=val, mode="last", item_feature=FeatureModality(features=item_embeddings, ids=item_ids), ) @@ -79,3 +80,20 @@ SASRec tuned over loss {`ce`, `bce`, `bpr`, `bpr-max`} × emb {64, 128} × lr {5 - `TIGER` likely uses `Sentence-T5-XXL` embeddings, but improvements over ST5-base are minimal (sometimes even decrease). OpenAI's `text-embedding-3-large` (provided by [`RPG`](https://github.com/facebookresearch/RPG_KDD2025)) is also available, but it doesn't guarantee performance improvement. - `beauty` and `sports` are somewhat reproducible, but `toys` is ***not***, likely due to ambiguity in the item text used (possibly `descriptions` only). - Well-tuned `SASRec` and `BERT4Rec` (within cornac) still outperform `TIGER`: tuned SASRec on beauty wins every metric except a marginal R@20 edge for `TIGER` + openai embedding, at a fraction of the cost (pure ID-based recommendation). + + +### Semantic ID evaluation + +Set `NextItemEvaluation(..., semantic_id_metrics=True)` to report ordinary +item metrics plus `SID-Recall@K`, `SID-NDCG@K`, `CCE-Recall@K` and `CCE-NDCG@K` for +requested finite-K Recall/NDCG metrics. All views share one inference; CCE averages +item credit over uniform order within each admitted collision bucket. Invalid +and repeated predicted IDs are compacted. Other requested metrics keep ordinary +item evaluation. Configured beam sizes remain unchanged; short candidate lists +warn. Set `semantic_id_metrics=False` for ordinary item metrics only. + +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst) +and [mixed-model example](../../../examples/semantic_id_evaluation_example.py). + +TIGER uses complete IDs including the disambiguating suffix, so its SID and CCE +metrics coincide even when tokenizer prefixes collide. diff --git a/cornac/models/tiger/recom_tiger.py b/cornac/models/tiger/recom_tiger.py index f04be4f6c..805676e9d 100644 --- a/cornac/models/tiger/recom_tiger.py +++ b/cornac/models/tiger/recom_tiger.py @@ -19,12 +19,7 @@ import numpy as np from tqdm.auto import trange -from ..recommender import SemanticIDRecommender -from ...utils.semantic_id import ( - SemanticIDCatalog, - SemanticIDPrediction, - prediction_from_catalog_scores, -) +from ..recommender import NextItemRecommender, SemanticIDMixin from ...utils import get_rng from ..seq_utils import session_seq_iter @@ -34,7 +29,7 @@ SUPPORTED_LR_SCHEDULES = ("constant", "cosine") -class TIGER(SemanticIDRecommender): +class TIGER(NextItemRecommender, SemanticIDMixin): """TIGER: Recommender Systems with Generative Retrieval. Two-stage generative retrieval with semantic IDs: (1) an RQ-VAE quantizes @@ -366,7 +361,6 @@ def _tokenize(self, torch, feats_t): ).astype("int64") def _build_semantic_ids(self, codes): - self.__dict__.pop("semantic_id_catalog_", None) # extra level disambiguating items that share the same code tuple counters = defaultdict(int) dedup = np.zeros(len(codes), dtype="int64") @@ -537,7 +531,6 @@ def _validate(self, val_sessions, metric): return float(np.mean(results)) if results else 0.0 def fit(self, train_set, val_set=None): - self.__dict__.pop("semantic_id_catalog_", None) super().fit(train_set, val_set) if not self.trainable: return self @@ -571,7 +564,7 @@ def _ensure_device(self, torch): if next(self.model.parameters()).device.type != torch.device(self.device_).type: self.model.to(self.device_) - def _predict_history(self, history_items, n_candidates=None, return_raw=False): + def _predict_history(self, history_items, return_semantic_ids=False): import torch self._ensure_device(torch) @@ -586,53 +579,33 @@ def _predict_history(self, history_items, n_candidates=None, return_raw=False): self.model.eval() with torch.no_grad(): if self.scoring == "beam": - beams, logps = self.model.generate_beam(enc_tokens_t, enc_mask_t, max(self.n_beams, n_candidates or 0), self.prefix_children) - if return_raw: - return self._beam_prediction(beams, logps, n_candidates) + beams, logps = self.model.generate_beam(enc_tokens_t, enc_mask_t, self.n_beams, self.prefix_children) scores = np.full(self.total_items, -1e10, dtype="float") for sid, lp in zip(beams, logps): - scores[self.sid_to_item[sid]] = lp + if sid in self.sid_to_item: + scores[self.sid_to_item[sid]] = lp else: # "exact" sid_table_t = torch.tensor(self.sid_table, dtype=torch.long, device=self.device_) scores = self.model.score_all_items( enc_tokens_t, enc_mask_t, sid_table_t, self.scoring_batch_size ).astype("float") - if return_raw: - return prediction_from_catalog_scores( - self.semantic_id_catalog(), scores, - {"scoring": "exact", "address_score_aggregation": "max_member"}, - ) + if return_semantic_ids: + semantic_ids = (np.asarray(beams, dtype="int64").reshape(-1, self.sid_table.shape[1]) + if self.scoring == "beam" else self._semantic_ids_from_scores(scores)) + return scores, semantic_ids return scores def semantic_id_catalog(self): - """Export tokenizer prefixes and complete, suffix-bearing addresses.""" + """Return complete, suffix-bearing IDs aligned to item indices.""" if not hasattr(self, "sid_table"): raise ValueError("TIGER must be fitted before exporting its SID catalog") - if not hasattr(self, "semantic_id_catalog_"): - self.semantic_id_catalog_ = SemanticIDCatalog( - np.arange(len(self.sid_table)), self.sid_table[:, :-1], self.sid_table, - tuple(self.level_sizes[:-1]), tuple(self.level_sizes), - ) - return self.semantic_id_catalog_ - - def _beam_prediction(self, beams, scores, n_candidates): - catalog = self.semantic_id_catalog() - addresses = np.asarray(beams, dtype="int64").reshape( - -1, catalog.address_sids.shape[1] - ) - return SemanticIDPrediction( - addresses, scores, - [tuple(address) in catalog.address_to_items for address in addresses], - {"scoring": "beam", "beam_size": max(self.n_beams, n_candidates or 0)}, - ) + return self.sid_table - def predict_semantic_ids(self, history_items, **context): - """Decode once before item conversion, retaining invalid/duplicate slots.""" + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): if len(history_items) == 0: - return self._beam_prediction([], [], context.get("n_candidates")) - return self._predict_history( - history_items, context.get("n_candidates"), return_raw=True - ) + return np.ones(self.total_items, dtype="float"), np.empty( + (0, self.semantic_id_catalog().shape[1]), dtype="int64") + return self._predict_history(history_items, return_semantic_ids=True) def score(self, user_idx, history_items, **kwargs): if len(history_items) == 0: diff --git a/cornac/utils/semantic_id.py b/cornac/utils/semantic_id.py deleted file mode 100644 index 5d1eef931..000000000 --- a/cornac/utils/semantic_id.py +++ /dev/null @@ -1,522 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -"""Model-neutral Semantic ID catalogs, predictions and item resolvers. - -Resolvers select one item from each predicted address bucket for inference or -recommendation evaluation. For example, when items 0 and 1 share an address, -TrainingPopularity([1, 1, 0], num_items=2) selects item 1 from that bucket. -Resolution preserves address order; invalid and repeated addresses leave empty -rank slots. Construct resolver state from training data only. -""" - -import hashlib -import json -from abc import ABC, abstractmethod -from collections.abc import Mapping -from dataclasses import dataclass, field -from types import MappingProxyType - -import numpy as np - - -# Catalogs, predictions and replay traces - -def _readonly_array(values, dtype, ndim, name): - original = np.asarray(values) - if dtype == "int64" and original.size and not np.issubdtype(original.dtype, np.integer): - raise ValueError(f"{name} must contain integers") - array = np.array(values, dtype=dtype, order="C", copy=True) - if array.ndim != ndim: - raise ValueError(f"{name} must be {ndim}-dimensional") - return np.frombuffer(array.tobytes(), dtype=array.dtype).reshape(array.shape) - - -def _freeze_native(value, name="metadata"): - if isinstance(value, np.generic): - return value.item() - if isinstance(value, Mapping): - frozen = {} - for key, item in value.items(): - if not isinstance(key, str): - raise ValueError(f"{name} keys must be strings") - frozen[key] = _freeze_native(item, name) - return MappingProxyType(frozen) - if isinstance(value, (list, tuple)): - return tuple(_freeze_native(item, name) for item in value) - if value is None or isinstance(value, (str, int, float, bool)): - return value - raise ValueError(f"{name} must contain only Python-native values") - - -def _thaw_native(value): - if isinstance(value, Mapping): - return {key: _thaw_native(item) for key, item in value.items()} - if isinstance(value, tuple): - return [_thaw_native(item) for item in value] - return value - - -def _validate_cardinalities(cardinalities, width, name): - if any(isinstance(value, (bool, np.bool_)) or not isinstance(value, (int, np.integer)) - for value in cardinalities): - raise ValueError(f"{name} must contain integers") - values = tuple(int(value) for value in cardinalities) - if len(values) != width: - raise ValueError(f"{name} must contain one value per SID position") - if any(value <= 0 for value in values): - raise ValueError(f"{name} values must be positive") - return values - - -def _validate_sid_range(sids, cardinalities, name): - if sids.size == 0: - return - if sids.min() < 0: - raise ValueError(f"{name} values must be non-negative") - for position, cardinality in enumerate(cardinalities): - if sids[:, position].max() >= cardinality: - raise ValueError( - f"{name} position {position} contains a value outside " - f"[0, {cardinality})" - ) - - -@dataclass(frozen=True, eq=False) -class SemanticIDCatalog: - """Immutable item-to-SID catalog with an explicit one-to-many reverse map. - - ``tokenizer_sids`` stores the representation before collision suffixing or - reassignment. ``address_sids`` stores the complete codes generated by the model. - The two arrays may have different widths. - """ - - item_indices: np.ndarray - tokenizer_sids: np.ndarray - address_sids: np.ndarray - tokenizer_codebook_cardinalities: tuple - address_codebook_cardinalities: tuple - _address_to_items: Mapping = field(init=False, repr=False, compare=False) - _item_to_row: Mapping = field(init=False, repr=False, compare=False) - _stable_hash: str = field(init=False, repr=False, compare=False) - - def __post_init__(self): - item_indices = _readonly_array( - self.item_indices, dtype="int64", ndim=1, name="item_indices" - ) - tokenizer_sids = _readonly_array( - self.tokenizer_sids, dtype="int64", ndim=2, name="tokenizer_sids" - ) - address_sids = _readonly_array( - self.address_sids, dtype="int64", ndim=2, name="address_sids" - ) - if tokenizer_sids.shape[0] != len(item_indices): - raise ValueError("tokenizer_sids must contain one row per item") - if address_sids.shape[0] != len(item_indices): - raise ValueError("address_sids must contain one row per item") - if len(item_indices) == 0: - raise ValueError("a Semantic-ID catalog must contain at least one item") - if tokenizer_sids.shape[1] == 0 or address_sids.shape[1] == 0: - raise ValueError("SID arrays must contain at least one position") - if len(np.unique(item_indices)) != len(item_indices): - raise ValueError("item_indices must not contain duplicates") - if item_indices.size and item_indices.min() < 0: - raise ValueError("item_indices must be non-negative") - - tokenizer_cardinalities = _validate_cardinalities( - self.tokenizer_codebook_cardinalities, - tokenizer_sids.shape[1], - "tokenizer_codebook_cardinalities", - ) - address_cardinalities = _validate_cardinalities( - self.address_codebook_cardinalities, - address_sids.shape[1], - "address_codebook_cardinalities", - ) - _validate_sid_range(tokenizer_sids, tokenizer_cardinalities, "tokenizer_sids") - _validate_sid_range(address_sids, address_cardinalities, "address_sids") - address_to_items = {} - item_to_row = {} - for row, (item_index, address) in enumerate(zip(item_indices, address_sids)): - key = tuple(int(value) for value in address) - address_to_items.setdefault(key, []).append(int(item_index)) - item_to_row[int(item_index)] = row - address_to_items = MappingProxyType( - { - address: tuple(sorted(items)) - for address, items in address_to_items.items() - } - ) - - object.__setattr__(self, "item_indices", item_indices) - object.__setattr__(self, "tokenizer_sids", tokenizer_sids) - object.__setattr__(self, "address_sids", address_sids) - object.__setattr__( - self, "tokenizer_codebook_cardinalities", tokenizer_cardinalities - ) - object.__setattr__( - self, "address_codebook_cardinalities", address_cardinalities - ) - object.__setattr__(self, "_address_to_items", address_to_items) - object.__setattr__(self, "_item_to_row", MappingProxyType(item_to_row)) - object.__setattr__(self, "_stable_hash", self._compute_hash()) - - def __deepcopy__(self, memo): - """Reuse this immutable value when a fitted model is copied for saving.""" - memo[id(self)] = self - return self - - def __reduce__(self): - """Rebuild proxy-backed state through the validated public constructor.""" - return ( - type(self), - ( - self.item_indices, - self.tokenizer_sids, - self.address_sids, - self.tokenizer_codebook_cardinalities, - self.address_codebook_cardinalities, - ), - ) - - @property - def address_to_items(self): - """Mapping from generated address SID to sorted global item indices.""" - return self._address_to_items - - @property - def stable_hash(self): - """Stable SHA-256 identity of the arrays and codebook cardinalities.""" - return self._stable_hash - - def tokenizer_sid_for_item(self, item_index): - row = self._item_to_row[int(item_index)] - return tuple(int(value) for value in self.tokenizer_sids[row]) - - def address_sid_for_item(self, item_index): - row = self._item_to_row[int(item_index)] - return tuple(int(value) for value in self.address_sids[row]) - - def _compute_hash(self): - digest = hashlib.sha256(b"cornac-semantic-id-catalog-v1\0") - for name, array in ( - ("item_indices", self.item_indices), - ("tokenizer_sids", self.tokenizer_sids), - ("address_sids", self.address_sids), - ): - canonical = np.ascontiguousarray(array, dtype="= len(scores): - raise ValueError("item_scores does not cover every catalog item") - address_scores = {} - for item_index, address_row in zip(catalog.item_indices, catalog.address_sids): - address = tuple(int(value) for value in address_row) - score = float(scores[item_index]) - address_scores[address] = max(address_scores.get(address, -np.inf), score) - ranked = sorted(address_scores.items(), key=lambda pair: (-pair[1], pair[0])) - return SemanticIDPrediction( - address_sids=np.asarray([address for address, _ in ranked], dtype="int64"), - scores=np.asarray([score for _, score in ranked], dtype="float64"), - catalog_validity=np.ones(len(ranked), dtype="bool"), - decoder_diagnostics=decoder_diagnostics or {}, - ) - - -@dataclass(frozen=True, eq=False) -class SemanticIDTraceRecord: - """One evaluation query paired with its immutable raw prediction.""" - - query_index: int - user_index: int - session_index: int - evaluation_position: int - history_start_position: int - history_end_position: int - target_item_index: int - target_tokenizer_sid: tuple - target_address_sid: tuple - prediction: SemanticIDPrediction - - def __post_init__(self): - integer_fields = ( - "query_index", - "user_index", - "session_index", - "evaluation_position", - "history_start_position", - "history_end_position", - "target_item_index", - ) - for name in integer_fields: - value = getattr(self, name) - if (isinstance(value, (bool, np.bool_)) or - not isinstance(value, (int, np.integer)) or value < 0): - raise ValueError(f"{name} must be a nonnegative integer") - object.__setattr__(self, name, int(value)) - if not 0 <= self.history_start_position <= self.history_end_position: - raise ValueError("history positions must define a non-negative interval") - if self.history_end_position != self.evaluation_position: - raise ValueError( - "history_end_position must equal the target evaluation_position" - ) - tokenizer_sid = tuple(int(value) for value in self.target_tokenizer_sid) - address_sid = tuple(int(value) for value in self.target_address_sid) - if not tokenizer_sid or not address_sid: - raise ValueError("target SIDs must not be empty") - if not isinstance(self.prediction, SemanticIDPrediction): - raise ValueError("prediction must be a SemanticIDPrediction") - object.__setattr__(self, "target_tokenizer_sid", tokenizer_sid) - object.__setattr__(self, "target_address_sid", address_sid) - - -@dataclass(frozen=True, eq=False) -class SemanticIDTrace: - """Immutable raw predictions tied to one frozen catalog.""" - - catalog_hash: str - records: tuple - metadata: Mapping = field(default_factory=dict) - - def __reduce__(self): - return (type(self), (self.catalog_hash, self.records, - _thaw_native(self.metadata))) - - def __post_init__(self): - records = tuple(self.records) - if any(not isinstance(record, SemanticIDTraceRecord) for record in records): - raise ValueError("records must contain SemanticIDTraceRecord instances") - query_indices = [record.query_index for record in records] - if len(set(query_indices)) != len(query_indices): - raise ValueError("query_index must be unique within a trace") - if not isinstance(self.catalog_hash, str) or not self.catalog_hash: - raise ValueError("catalog_hash must be a non-empty string") - object.__setattr__(self, "records", records) - object.__setattr__(self, "metadata", _freeze_native(self.metadata)) - - -# Item resolution - -def _indices(values, name, upper=None): - values = np.asarray(values) - if (values.ndim != 1 or (values.size and - not np.issubdtype(values.dtype, np.integer)) or - np.any(values < 0) or (upper is not None and np.any(values >= upper))): - raise ValueError(f"{name} must contain covered nonnegative integer indices") - return values.astype("int64", copy=False) - - -def _unit_rows(features): - features = np.array(features, dtype="float64", copy=True) - if (features.ndim != 2 or not all(features.shape) or - not np.all(np.isfinite(features))): - raise ValueError("features must be a nonempty finite item matrix") - scales = np.max(np.abs(features), axis=1, keepdims=True) - np.divide(features, scales, out=features, where=scales != 0) - norms = np.linalg.norm(features, axis=1, keepdims=True) - np.divide(features, norms, out=features, where=norms != 0) - return np.frombuffer(features.tobytes(), dtype=features.dtype).reshape(features.shape) - - -class SemanticIDResolver(ABC): - """Score candidate members without targets; highest score wins. - - Implement ``score(query_index, user_index, candidate_items)``. The shared - operation checks scores and breaks ties by smallest global item index. - Scores never reorder generated addresses or admit outside candidates. - """ - - @abstractmethod - def score(self, query_index, user_index, candidate_items): - """Return one finite score per candidate, in candidate order.""" - raise NotImplementedError - - @property - def metadata(self): - return {"resolver": type(self).__name__} - - -class TrainingPopularity(SemanticIDResolver): - """Count explicitly supplied training events, retaining repeated events.""" - - def __init__(self, training_items, num_items): - if not isinstance(num_items, (int, np.integer)) or num_items <= 0: - raise ValueError("num_items must be a positive integer") - items = _indices(training_items, "training_items", num_items) - counts = np.bincount(items, minlength=num_items) - self.counts = np.frombuffer(counts.tobytes(), dtype=counts.dtype) - - def score(self, query_index, user_index, candidate_items): - return self.counts[list(candidate_items)] - - @property - def metadata(self): - return {"resolver": "training_popularity", "source": "training_events", - "counts_sha256": hashlib.sha256(self.counts.tobytes()).hexdigest()} - - -class HistoryCosine(SemanticIDResolver): - """Cosine to normalized sums of normalized training-event item features. - - Rows are global Cornac item indices. Supply training events only; repeats - contribute repeatedly. A zero user profile ties all members. ``feature_source`` - labels the explicit scoring feature space, which need not be the model's - tokenizer feature space. No evaluation history or target enters the profile. - """ - - def __init__(self, features, training_users, training_items, num_users, - feature_source="provided"): - self.features = _unit_rows(features) - if not isinstance(num_users, (int, np.integer)) or num_users <= 0: - raise ValueError("num_users must be a positive integer") - users = _indices(training_users, "training_users", num_users) - items = _indices(training_items, "training_items", len(self.features)) - if users.shape != items.shape: - raise ValueError("training user/item events must align") - if not isinstance(feature_source, str) or not feature_source: - raise ValueError("feature_source must be a nonempty string") - profiles = np.zeros((num_users, self.features.shape[1]), dtype="float64") - np.add.at(profiles, users, self.features[items]) - self.profiles = _unit_rows(profiles) - self.feature_source = feature_source - - def score(self, query_index, user_index, candidate_items): - if user_index is None or not 0 <= user_index < len(self.profiles): - raise ValueError("user_index is outside the training profile space") - return self.features[list(candidate_items)] @ self.profiles[user_index] - - @property - def metadata(self): - return {"resolver": "training_history_cosine", "source": "training_events", - "feature_source": self.feature_source, - "feature_join": "global_item_index", "zero_profile": "item_index_tie", - "normalized_features_sha256": hashlib.sha256(self.features.tobytes()).hexdigest()} - - -class CustomScorer(SemanticIDResolver): - """Adapt ``callback(query_index, user_index, candidate_items)`` to a resolver.""" - - def __init__(self, callback): - if not callable(callback): - raise ValueError("callback must be callable") - self.callback = callback - - def score(self, query_index, user_index, candidate_items): - return self.callback(query_index, user_index, candidate_items) - - -def validate_prediction(catalog, prediction): - """Check raw prediction validity against the admitted frozen catalog.""" - if not isinstance(catalog, SemanticIDCatalog) or not isinstance(prediction, SemanticIDPrediction): - raise ValueError("expected SemanticIDCatalog and SemanticIDPrediction") - if prediction.address_sids.shape[1] != catalog.address_sids.shape[1]: - raise ValueError("prediction address width differs from catalog") - expected = [tuple(row) in catalog.address_to_items for row in prediction.address_sids] - if not np.array_equal(prediction.catalog_validity, expected): - raise ValueError("prediction catalog_validity differs from catalog") - - -def resolve_semantic_ids(catalog, prediction, resolver, *, user_index=None, - query_index=None): - """Return item/None rank slots from raw predictions for inference or replay. - - Invalid and repeated addresses consume empty slots. Singleton buckets bypass - scoring. This operation returns no fillers, changes no input, and exposes no - target to the resolver. Choose a complete-address catalog matching the model. - """ - validate_prediction(catalog, prediction) - if not isinstance(resolver, SemanticIDResolver): - raise ValueError("resolver must be a SemanticIDResolver") - slots, seen = [], set() - for row, valid in zip(prediction.address_sids, prediction.catalog_validity): - address = tuple(int(value) for value in row) - if not valid or address in seen: - slots.append(None) - continue - seen.add(address) - items = catalog.address_to_items[address] - if len(items) == 1: - slots.append(items[0]) - continue - scores = np.asarray(resolver.score(query_index, user_index, items), dtype="float64") - if scores.shape != (len(items),) or not np.all(np.isfinite(scores)): - raise ValueError("resolver must return one finite score per candidate") - slots.append(min(zip(items, scores), key=lambda pair: (-pair[1], pair[0]))[0]) - return tuple(slots) diff --git a/docs/source/api_ref/eval_methods.rst b/docs/source/api_ref/eval_methods.rst index b36d308aa..82c22d4f9 100644 --- a/docs/source/api_ref/eval_methods.rst +++ b/docs/source/api_ref/eval_methods.rst @@ -33,95 +33,82 @@ Timestamp Split .. automodule:: cornac.eval_methods.timestamp_split :members: +Next Item Evaluation +-------------------- +.. automodule:: cornac.eval_methods.next_item_evaluation + :members: + Recommendation Evaluation with Semantic IDs ------------------------------------------ -``SemanticIDEvaluation`` subclasses ``NextItemEvaluation`` and evaluates next-item -recommendation performance, rather than the intrinsic quality of Semantic IDs. -Use it when a generated ID can map to several items: SID matching alone cannot -identify which member was recommended. ``NextItemEvaluation`` instead evaluates -the item rankings returned by ``model.rank()``. - -``SemanticIDEvaluation`` evaluates TIGER, LETTER, RPG and DiffGRM through the same -raw Semantic ID predictions. It reports SID accuracy, analytical -Collision-Corrected Evaluation (CCE), and exact-item accuracy under explicit -resolvers. Supported metrics are ``Recall(k)`` and ``NDCG(k)`` with positive -integer cutoffs; result names include ``SID/Recall@10``, ``CCE/NDCG@10`` and -``Item[popularity]/Recall@10``. ``user_based=True`` averages queries per user -before averaging users. - -CCE expands each complete ID into its full item bucket and averages over uniform -member permutations. Earlier buckets consume their full widths. Invalid and -duplicate addresses preserve empty rank slots. A resolver selects one item per -bucket while preserving address order. Complete IDs include any disambiguating -suffix; injective catalogs give identical views under the same slot rules. +Use ``NextItemEvaluation`` for both ordinary next-item models (such as SASRec) +and Semantic ID models (TIGER, LETTER, RPG and DiffGRM), including a mixed +``Experiment``. Its optional ``semantic_id_metrics`` switch defaults to False. +Set it to True in Semantic ID experiment setup to enable the extra views. +Ordinary item metric names and model-specific item resolution are preserved. +With the switch enabled, for each requested finite-K ``Recall`` or ``NDCG``, +SID models additionally report ``SID-Recall@K``, ``SID-NDCG@K``, ``CCE-Recall@K`` and ``CCE-NDCG@K``. +Other metrics, including AUC, MAP, MRR and unbounded cutoffs, still run their +ordinary item evaluation without additional views. Non-SID models have no SID +or CCE values; mixed result tables show ``N/A`` for those entries. + +SID metrics match the target's complete generated ID, including any +item-disambiguating suffix. Invalid IDs, IDs absent from the admitted candidate +catalog, and repeated IDs after their first occurrence are removed before +ranking. Candidate filtering respects ``exclude_unknowns`` and also restricts +each collision bucket's size. No ungenerated fallback candidates are introduced. + +Collision-Corrected Evaluation (CCE) expands surviving IDs in their original +order into their full admitted item buckets. Earlier buckets consume their full +widths. Items within a bucket have uniformly random order. CCE Recall is the +fraction of target-bucket positions inside K, and CCE NDCG averages the log +discount over those positions, divided by the full target-bucket size. + +For example, suppose items a and b share the first predicted ID, c has the +second ID, and b is the target. A model that ranks a first has ordinary +Recall@1 of 0, SID-Recall@1 of 1 and CCE-Recall@1 of 0.5. CCE-Recall@2 is 1 +because both positions of the target bucket fit. This is expected item credit, +not the accuracy of a concrete item resolver. A catalog whose complete IDs are +all unique yields identical SID and CCE metrics. TIGER's suffix-bearing IDs +have this property even when their tokenizer prefixes collide. CCE follows Qian Zhang, Lech Szymanski, Haibo Zhang and Jeremiah D. Deng, `Faithful Evaluation of Semantic-ID Tokenizers for Generative Recommendation -`_ (CIKM 2026). The explicit resolvers below -are selectable item-ranking policies, distinct from CCE's expectation. - -This evaluator requires ``SemanticIDRecommender`` and supports SID models only. -For ordinary next-item models such as SASRec, use ``NextItemEvaluation`` in a -separate experiment with identical train/validation/test splits and cutoffs. -Mixed SID and ordinary models in one ``Experiment`` are not supported. - -Popularity uses training events only. Supplying ``item_features`` adds history -cosine: normalize item vectors, sum each user's training-event vectors (including -repeats), normalize that profile and score bucket members by cosine. Features -must align with global Cornac item indices. Ties, including zero profiles, choose -the smallest item index. Pass fitted resolvers through ``resolvers`` or implement -``SemanticIDResolver.score(query_index, user_index, candidate_items)``; custom -state must also use training-only information. Import these utilities from -``cornac.utils.semantic_id``; ``resolve_semantic_ids`` exposes the same operation -for inference. For example:: - - import numpy as np - from cornac.utils.semantic_id import ( - SemanticIDCatalog, SemanticIDPrediction, TrainingPopularity, - resolve_semantic_ids, - ) - - codes = np.array([[0], [0], [1]]) - catalog = SemanticIDCatalog(np.arange(3), codes, codes, (2,), (2,)) - prediction = SemanticIDPrediction( - np.array([[0], [1]]), np.array([1., .5]), np.array([True, True])) - resolver = TrainingPopularity([1, 1, 0, 2], num_items=3) - resolve_semantic_ids(catalog, prediction, resolver) # (1, 2) - -Items 0 and 1 share the first ID. Popularity chooses item 1 from that bucket, -then item 2 from the next, preserving generated ID order. If item 1 is the -target, SID Recall@1 is 1, CCE Recall@1 is 0.5 and popularity-resolved Recall@1 -is 1. CCE Recall@2 is 1 because both members of the first bucket fit. Other -resolvers can choose differently while using the same predictions. - -For an existing Semantic ID model and item-feature modality:: - - evaluation = SemanticIDEvaluation.from_splits( - train, test, fmt="USIT", item_feature=features) - Experiment(evaluation, [TIGER(device="cpu")], [Recall(10), NDCG(10)]).run() - -See ``examples/semantic_id_evaluation_example.py`` for a standalone CPU example. -``Result.semantic_id_trace`` holds the predictions, and -``Result.semantic_id_evaluation`` holds per-query arrays and protocol metadata. - -Adapter authors subclass ``cornac.models.recommender.SemanticIDRecommender`` and -implement ``semantic_id_catalog()`` and ``predict_semantic_ids(history_items, -**context)``. Use the shared types in ``cornac.utils.semantic_id``. Export -complete addresses separately from tokenizer prefixes and -preserve raw decoder order, invalid slots and duplicates before item conversion. -The evaluator requests ``n_candidates=max(k)``; honor it where supported and keep -shorter output without fillers. ``prediction_from_catalog_scores`` supports exact -scoring by maximum member score per address, with lexicographic address ties. -The contract covers fixed-width integer addresses, a frozen catalog and one -next-item target per query. - -The evaluation module combines trace collection, offline trace evaluation via -``evaluate_semantic_id_trace`` and the ``SemanticIDEvaluation`` experiment -integration. Utilities below provide catalogs, predictions and item resolution. - -.. automodule:: cornac.eval_methods.semantic_id_evaluation - :members: - -.. automodule:: cornac.utils.semantic_id - :members: +`_ (CIKM 2026). Ordinary item metrics remain +the direct measure of the model's concrete recommendations; CCE evaluates the +uniform within-bucket assumption. + +All three views share one model inference per query. Configured decoding budgets +are preserved. Short outputs are evaluated as returned, and one warning per +split reports the number of queries with fewer valid distinct IDs than the +largest requested SID/CCE cutoff. Increase the model's beam setting explicitly +if a larger candidate budget is desired. With ``semantic_id_metrics=False``, +only ordinary item metrics are reported; SID models issue a warning. +``mode='last'``/``mode='next'`` and per-query/per-user aggregation apply to every +view in the same way. + +For example:: + + evaluation = NextItemEvaluation.from_splits( + train, test, fmt="USIT", item_feature=features, + semantic_id_metrics=True) + Experiment(evaluation, [TIGER(device="cpu"), SASRec()], + [Recall(10), NDCG(10)]).run() + +See ``examples/semantic_id_evaluation_example.py`` for a backend-free mixed-model +example with a fixed SID generator. + +Adapter authors implement ``SemanticIDMixin`` alongside ``NextItemRecommender``. +``semantic_id_catalog()`` returns a two-dimensional integer array with one row +per global item, storing complete generated IDs. ``rank_with_semantic_ids`` has +the same inputs as ``rank`` and returns ordinary item ranks, item scores aligned +to the candidate indices, and a two-dimensional integer array of raw ranked SID +rows. Its full SID output is independent of item-ranking ``k``. The model keeps +its decoding and concrete SID-to-item policy. The mixin provides shared ranking +from ``_score_with_semantic_ids(user_idx, history_items, **kwargs)``, which returns +full item scores and SID rows from the same inference. Empty SID output has +shape ``(0, SID_length)``. No persistent prediction cache is required. + +For exhaustive scoring, the built-in adapters order SID groups by their maximum +member score, with lexicographic ID ties. Beam adapters expose only generated +rows, preserving decoder order before evaluator filtering and compaction. diff --git a/docs/source/api_ref/models.rst b/docs/source/api_ref/models.rst index 14b6e18a3..fddac1a1a 100644 --- a/docs/source/api_ref/models.rst +++ b/docs/source/api_ref/models.rst @@ -336,5 +336,5 @@ Weighted Matrix Factorization (WMF) Semantic ID Model Contracts -------------------------- -.. autoclass:: cornac.models.recommender.SemanticIDRecommender +.. autoclass:: cornac.models.recommender.SemanticIDMixin :members: diff --git a/examples/README.md b/examples/README.md index da2f09dbd..604e9680f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -122,7 +122,7 @@ ## Next-Item Algorithms -[semantic_id_evaluation_example.py](semantic_id_evaluation_example.py) - Synthetic example of shared SID, CCE and explicit item-resolver evaluation through Experiment. +[semantic_id_evaluation_example.py](semantic_id_evaluation_example.py) - Backend-free mixed-model NextItemEvaluation example with ordinary item, SID and CCE metrics. [spop_yoochoose.py](spop_yoochoose.py) - Next-item recommendation based on item popularity. diff --git a/examples/diffgrm_example.py b/examples/diffgrm_example.py index e01f14a14..f66c3c377 100644 --- a/examples/diffgrm_example.py +++ b/examples/diffgrm_example.py @@ -62,6 +62,7 @@ next_item_eval = NextItemEvaluation.leave_last_out( data=data, exclude_unknowns=True, + semantic_id_metrics=True, verbose=True, item_feature=FeatureModality(features=features, ids=item_ids), ) diff --git a/examples/letter_example.py b/examples/letter_example.py index 9bdb28489..45527333d 100644 --- a/examples/letter_example.py +++ b/examples/letter_example.py @@ -69,6 +69,7 @@ val_data=val_data, test_data=test_data, exclude_unknowns=True, + semantic_id_metrics=True, verbose=True, fmt="USIT", item_feature=FeatureModality(features=content, ids=item_ids), diff --git a/examples/rpg_example.py b/examples/rpg_example.py index 0e494031b..a05699ec5 100644 --- a/examples/rpg_example.py +++ b/examples/rpg_example.py @@ -59,6 +59,7 @@ val_data=val_data, test_data=test_data, exclude_unknowns=True, + semantic_id_metrics=True, verbose=True, fmt="USIT", item_feature=FeatureModality(features=features, ids=item_ids), diff --git a/examples/semantic_id_evaluation_example.py b/examples/semantic_id_evaluation_example.py index f8ba88470..321699add 100644 --- a/examples/semantic_id_evaluation_example.py +++ b/examples/semantic_id_evaluation_example.py @@ -13,49 +13,49 @@ # limitations under the License. # ============================================================================ -"""Backend-free next-item evaluation from raw Semantic ID predictions. - -Use NextItemEvaluation for ordinary item-ranking models. This SID-only example -compares SID matching, CCE's expected item credit and explicit item selection. -""" +"""Backend-free ordinary item, SID and CCE evaluation in one Experiment.""" import numpy as np import cornac -from cornac.eval_methods import SemanticIDEvaluation +from cornac.eval_methods import NextItemEvaluation from cornac.metrics import Recall, NDCG -from cornac.models import SemanticIDRecommender -from cornac.utils.semantic_id import SemanticIDCatalog, SemanticIDPrediction +from cornac.models import NextItemRecommender, SemanticIDMixin, SPop -class FrozenSIDExample(SemanticIDRecommender): - """A fixed generator for demonstrating the interface, not a trained model.""" +class FrozenSIDExample(NextItemRecommender, SemanticIDMixin): + """A fixed generator demonstrating collisions, not a trained model.""" def __init__(self): super().__init__("FrozenSIDExample") def fit(self, train_set, val_set=None): super().fit(train_set, val_set) - codes = np.arange(self.total_items).reshape(-1, 1) // 2 - cardinalities = (int(codes.max()) + 1,) - self.catalog = SemanticIDCatalog(np.arange(self.total_items), codes, - codes, cardinalities, cardinalities) + self.codes = np.arange(self.total_items).reshape(-1, 1) // 2 return self def semantic_id_catalog(self): - return self.catalog + return self.codes + + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + # a and b share SID 0; this concrete item policy ranks a first. + scores = -np.arange(self.total_items, dtype="float64") + predicted = np.unique(self.codes, axis=0) + return scores, predicted - def predict_semantic_ids(self, history_items, **context): - addresses = np.asarray(sorted(self.catalog.address_to_items), dtype="int64") - return SemanticIDPrediction(addresses, np.zeros(len(addresses)), - np.ones(len(addresses), dtype=bool)) + def score(self, user_idx, history_items, **kwargs): + return self._score_with_semantic_ids(user_idx, history_items, **kwargs)[0] if __name__ == "__main__": - train = [("u", "train", "a", 0), ("u", "train", "b", 1), - ("u", "train", "c", 2)] + train = [("u", "train", "a", 0), ("u", "train", "b", 1), ("u", "train", "c", 2)] test = [("u", "test", "a", 3), ("u", "test", "b", 4)] - evaluation = SemanticIDEvaluation.from_splits( - train, test, fmt="USIT", item_features=np.eye(3), - feature_source="synthetic identity vectors") - cornac.Experiment(evaluation, [FrozenSIDExample()], [Recall(2), NDCG(2)], - show_validation=False).run() + evaluation = NextItemEvaluation.from_splits( + train, test, fmt="USIT", semantic_id_metrics=True + ) + cornac.Experiment( + evaluation, + [FrozenSIDExample(), SPop()], + [Recall(1), NDCG(1)], + show_validation=False, + save_dir="/tmp/cornac-semantic-id-example", + ).run() diff --git a/examples/tiger_example.py b/examples/tiger_example.py index b6c407913..6b46253ec 100644 --- a/examples/tiger_example.py +++ b/examples/tiger_example.py @@ -57,6 +57,7 @@ next_item_eval = NextItemEvaluation.leave_last_out( data=data, exclude_unknowns=True, + semantic_id_metrics=True, verbose=True, item_feature=FeatureModality(features=features, ids=item_ids), ) diff --git a/tests/cornac/eval_methods/test_semantic_id_evaluation.py b/tests/cornac/eval_methods/test_semantic_id_evaluation.py deleted file mode 100644 index 3ef78d221..000000000 --- a/tests/cornac/eval_methods/test_semantic_id_evaluation.py +++ /dev/null @@ -1,368 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -import itertools -import pickle -import subprocess -import sys - -import numpy as np -import pytest - -from cornac import Experiment -from cornac.eval_methods import SemanticIDEvaluation, evaluate_semantic_id_trace -from cornac.metrics import Recall, NDCG, HitRatio -from cornac.models import SemanticIDRecommender -from cornac.utils.semantic_id import ( - TrainingPopularity, - HistoryCosine, - CustomScorer, - resolve_semantic_ids, - SemanticIDCatalog, - SemanticIDPrediction, - SemanticIDTrace, - SemanticIDTraceRecord, - prediction_from_catalog_scores, -) - - -def catalog(codes=((0,), (0,), (1,), (1,), (2,))): - codes = np.asarray(codes) - cardinalities = tuple(int(codes[:, i].max()) + 1 for i in range(codes.shape[1])) - return SemanticIDCatalog(np.arange(len(codes)), codes, codes, cardinalities, cardinalities) - - -def prediction(cat, addresses): - values = np.asarray(addresses, dtype="int64").reshape(-1, cat.address_sids.shape[1]) - return SemanticIDPrediction(values, np.arange(len(values), dtype=float), - [tuple(row) in cat.address_to_items for row in values]) - - -def trace(cat, target, addresses, user=0, query=0): - return SemanticIDTrace(cat.stable_hash, (SemanticIDTraceRecord(query, user, - 0, 1, 0, 1, target, cat.tokenizer_sid_for_item(target), - cat.address_sid_for_item(target), prediction(cat, addresses)),)) - - -def test_catalog_freezes_copies_hash_and_pickle(): - codes = np.array([[0], [0], [1]]) - cat = catalog(codes) - codes[0, 0] = 9 - assert cat.address_to_items[(0,)] == (0, 1) - with pytest.raises(ValueError): - cat.address_sids.setflags(write=True) - loaded = pickle.loads(pickle.dumps(cat)) - assert loaded.stable_hash == cat.stable_hash - assert loaded.address_to_items == cat.address_to_items - with pytest.raises(TypeError): - cat.address_to_items[(0,)] = (1,) - with pytest.raises(ValueError, match="integers"): - catalog(((0.5,), (1.,))) - - -def test_resolution_preserves_slots_members_ties_and_target_free_context(): - cat = catalog() - pred = prediction(cat, ((8,), (1,), (1,), (0,), (2,))) - calls = [] - def score(query, user, candidates): - calls.append((query, user, candidates)) - return np.zeros(len(candidates)) - slots = resolve_semantic_ids(cat, pred, CustomScorer(score), user_index=2, query_index=7) - assert slots == (None, 2, None, 0, 4) - assert calls == [(7, 2, (2, 3)), (7, 2, (0, 1))] - assert pred.address_sids.tolist() == [[8], [1], [1], [0], [2]] - for bad in (lambda *args: [1], lambda *args: [np.nan, 0]): - with pytest.raises(ValueError, match="finite score"): - resolve_semantic_ids(cat, pred, CustomScorer(bad)) - - -def test_training_popularity_and_cosine_multiplicity_and_zero_profile(): - cat = catalog() - pop = TrainingPopularity([1, 1, 0, 3], 5) - assert resolve_semantic_ids(cat, prediction(cat, [[0], [1]]), pop) == (1, 3) - features = np.array([[1, 0], [0, 1], [1, 0], [0, 1], [0, 0]], dtype=float) - scorer = HistoryCosine(features, [0, 0, 0], [0, 1, 1], 2, "fixed-test-features") - assert resolve_semantic_ids(cat, prediction(cat, [[0], [1]]), scorer, user_index=0) == (1, 3) - assert resolve_semantic_ids(cat, prediction(cat, [[0], [1]]), scorer, user_index=1) == (0, 2) - features[:] = 0 - assert scorer.features[0, 0] == 1 - assert scorer.metadata["feature_source"] == "fixed-test-features" - with pytest.raises(ValueError, match="align"): - HistoryCosine(np.ones((5, 2)), [0], [0, 1], 2) - with pytest.raises(ValueError): - TrainingPopularity([5], 5) - with pytest.raises(ValueError): - HistoryCosine([[np.nan]], [0], [0], 1) - - -@pytest.mark.parametrize("addresses", [((0,), (1,), (2,)), - ((9,), (0,), (0,), (1,), (2,)), ((1,), (0,)), (), ((2,),)]) -def test_cce_matches_exhaustive_expansion_oracle(addresses): - cat = catalog() - buckets, seen = [], set() - for address in addresses: - if address not in cat.address_to_items or address in seen: - buckets.append(((None,),)) - else: - buckets.append(tuple(itertools.permutations(cat.address_to_items[address]))) - seen.add(address) - lists = [sum(parts, ()) for parts in itertools.product(*buckets)] - cutoffs = tuple(range(1, 9)) - for target in cat.item_indices: - result = evaluate_semantic_id_trace(cat, trace(cat, int(target), addresses), cutoffs) - for column, cutoff in enumerate(cutoffs): - hits, ndcgs = [], [] - for items in lists: - rank = items.index(target) if target in items else None - hit = float(rank is not None and rank < cutoff) - hits.append(hit) - ndcgs.append(hit / np.log2(rank + 2) if rank is not None else 0.) - assert result["per_query"]["CCE"]["recall"][0, column] == pytest.approx(np.mean(hits)) - assert result["per_query"]["CCE"]["ndcg"][0, column] == pytest.approx(np.mean(ndcgs)) - - -def test_injective_complete_ids_despite_prefix_collisions(): - cat = SemanticIDCatalog([0, 1, 2], [[0], [0], [1]], [[0, 0], [0, 1], [1, 0]], (2,), (2, 2)) - result = evaluate_semantic_id_trace(cat, trace(cat, 1, ((9, 9), (0, 0), (0, 0), (0, 1))), - (1, 3, 4), {"popularity": TrainingPopularity([0, 0], 3)}) - for metric in ("recall", "ndcg"): - np.testing.assert_array_equal(result["per_query"]["SID"][metric], result["per_query"]["CCE"][metric]) - np.testing.assert_array_equal(result["per_query"]["SID"][metric], result["per_query"]["Item[popularity]"][metric]) - - -def test_replay_rejects_mismatches_and_bad_cutoffs(): - cat = catalog() - saved = trace(cat, 1, ((0,),)) - for cutoffs in ((0,), (1.2,), (True,), (), (1, 1), (np.inf,)): - with pytest.raises(ValueError): - evaluate_semantic_id_trace(cat, saved, cutoffs) - other = catalog(((0,), (1,), (1,), (1,), (2,))) - with pytest.raises(ValueError, match="catalog_hash"): - evaluate_semantic_id_trace(other, saved) - pred = SemanticIDPrediction([[0]], [1.], [False]) - with pytest.raises(ValueError, match="validity"): - resolve_semantic_ids(cat, pred, TrainingPopularity([], 5)) - - -def test_catalog_score_conversion_declares_max_member_aggregation(): - cat = catalog() - pred = prediction_from_catalog_scores(cat, [1, 3, 2, 0, 2]) - assert pred.address_sids.tolist() == [[0], [1], [2]] - assert pred.scores.tolist() == [3, 2, 2] - - -class FakeSID(SemanticIDRecommender): - def __init__(self): - super().__init__("FakeSID") - self.calls = [] - - def fit(self, train_set, val_set=None): - super().fit(train_set, val_set) - self.catalog_ = catalog(tuple((i // 2,) for i in range(self.total_items))) - return self - - def semantic_id_catalog(self): - return self.catalog_ - - def predict_semantic_ids(self, history_items, **context): - assert set(context) == {"user_index", "query_index", "session_index", - "evaluation_position", "n_candidates"} - self.calls.append((tuple(history_items), context)) - return prediction(self.catalog_, ((9,), (0,), (0,), (1,))) - - def score(self, *args, **kwargs): - raise AssertionError("raw evaluation must never call score/rank") - - -def data(): - train = [("u0", "t0", "a", 0), ("u0", "t0", "b", 1), - ("u1", "t1", "c", 0), ("u1", "t1", "d", 1)] - test = [("u0", "s0", "a", 2), ("u0", "s0", "b", 3), ("u0", "s0", "a", 4), - ("u0", "s1", "b", 2), ("u0", "s1", "c", 3), - ("u1", "s2", "c", 2), ("u1", "s2", "d", 3)] - return train, test - - -@pytest.mark.parametrize("mode, count", [("last", 3), ("next", 4)]) -@pytest.mark.parametrize("user_based", [False, True]) -def test_experiment_collects_once_per_query_with_modes_and_user_means(mode, count, user_based, tmp_path): - train, test = data() - evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", mode=mode, - item_features=np.eye(4), feature_source="identity-features") - model = FakeSID() - experiment = Experiment(evaluation, [model], [Recall(4), NDCG(4)], - user_based=user_based, show_validation=False, verbose=False, - save_dir=str(tmp_path)) - experiment.run() - assert len(model.calls) == count - assert all(context["n_candidates"] == 4 for _, context in model.calls) - result = experiment.result[0] - assert "Item[history_cosine]/Recall@4" in result.metric_avg_results - replay = result.semantic_id_evaluation - for view, rows in replay["per_query"].items(): - for field, metric in [("recall", "Recall"), ("ndcg", "NDCG")]: - values = rows[field][:, 0] - users = [record.user_index for record in result.semantic_id_trace.records] - expected = np.mean([np.mean(values[np.array(users) == user]) for user in set(users)]) if user_based else values.mean() - assert result.metric_avg_results[f"{view}/{metric}@4"] == pytest.approx(expected) - loaded = FakeSID.load(next(tmp_path.rglob("*.pkl"))) - assert loaded.semantic_id_catalog().stable_hash == model.semantic_id_catalog().stable_hash - - -def test_evaluation_uses_training_only_and_separates_validation(): - train, test = data() - val = [("u0", "v0", "c", 5), ("u0", "v0", "b", 6)] - evaluation = SemanticIDEvaluation.from_splits(train, test, val, fmt="USIT") - model = FakeSID() - result, validation = evaluation.evaluate(model, [Recall(4)], show_validation=True) - expected = TrainingPopularity([0, 1, 2, 3], 4).metadata - assert result.semantic_id_evaluation["protocol"]["resolvers"]["popularity"] == expected - assert validation.semantic_id_evaluation["protocol"]["resolvers"]["popularity"] == expected - assert len(model.calls) == 4 - assert len(validation.semantic_id_trace.records) == 1 - assert validation.metric_avg_results["Time (s)"] >= 0 - - -@pytest.mark.parametrize("exclude", [False, True]) -def test_unknown_item_handling_and_short_predictions(exclude): - train, test = data() - test += [("u1", "s3", "a", 2), ("u1", "s3", "new", 3)] - evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", exclude_unknowns=exclude) - model = FakeSID() - result, _ = evaluation.evaluate(model, [Recall(8)], show_validation=False) - assert len(result.semantic_id_trace.records) == (3 if exclude else 4) - assert all(len(record.prediction.address_sids) == 4 for record in result.semantic_id_trace.records) - assert result.semantic_id_evaluation["diagnostics"]["invalid_rate"] > 0 - - -def test_unsupported_metric_fails_before_fit(): - train, test = data() - evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT") - for metrics in ([HitRatio(3)], [Recall(-1)], [Recall(1.5)]): - model = FakeSID() - with pytest.raises(ValueError): - evaluation.evaluate(model, metrics) - assert not hasattr(model, "catalog_") - - -def test_lazy_imports(): - subprocess.run([sys.executable, "-c", "import sys; from cornac.eval_methods import SemanticIDEvaluation; from cornac.models import SemanticIDRecommender; from cornac.utils.semantic_id import SemanticIDCatalog, TrainingPopularity, HistoryCosine, resolve_semantic_ids; assert 'torch' not in sys.modules; assert 'transformers' not in sys.modules"], check=True) - - -def test_trace_pickle_keeps_identity_and_readonly_predictions(): - cat = catalog() - saved = trace(cat, 1, ((0,), (9,))) - restored = pickle.loads(pickle.dumps(saved)) - assert restored.catalog_hash == saved.catalog_hash - with pytest.raises(ValueError): - restored.records[0].prediction.address_sids.setflags(write=True) - result = evaluate_semantic_id_trace(cat, restored, (1, 2)) - assert result["aggregate"]["CCE"]["Recall@1"] == .5 - - -def test_custom_resolver_in_experiment_and_empty_output(): - train, test = data() - custom = CustomScorer(lambda query, user, items: np.arange(len(items))) - evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", - resolvers={"custom": custom}) - result, _ = evaluation.evaluate(FakeSID(), [Recall(4)], show_validation=False) - assert "Item[custom]/Recall@4" in result.metric_avg_results - assert "Item[popularity]/Recall@4" not in result.metric_avg_results - class EmptySID(FakeSID): - def predict_semantic_ids(self, history_items, **context): - return prediction(self.catalog_, ()) - result, _ = evaluation.evaluate(EmptySID(), [Recall(4)], show_validation=False) - assert all(value == 0 for key, value in result.metric_avg_results.items() - if key.endswith("@4")) - - -def test_feature_coverage_rejected_and_trace_query_identity_unique(): - train, test = data() - evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", - item_features=np.eye(2)) - with pytest.raises(ValueError, match="cover"): - evaluation.evaluate(FakeSID(), [Recall(4)]) - cat = catalog() - saved = trace(cat, 1, ((0,),)) - with pytest.raises(ValueError, match="unique"): - SemanticIDTrace(cat.stable_hash, saved.records * 2) - - -def test_restricted_catalog_keeps_excluded_address_as_empty_slot(): - train, test = data() - class ExtraCatalogSID(FakeSID): - def fit(self, train_set, val_set=None): - super().fit(train_set, val_set) - self.catalog_ = catalog(((0,), (1,), (2,), (3,), (4,))) - return self - - def predict_semantic_ids(self, history_items, **context): - return prediction(self.catalog_, ((4,), (0,))) - evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT", exclude_unknowns=True) - result, _ = evaluation.evaluate(ExtraCatalogSID(), [Recall(1), Recall(2)], show_validation=False) - records = result.semantic_id_trace.records - assert records[0].prediction.address_sids.tolist() == [[4], [0]] - assert records[0].prediction.catalog_validity.tolist() == [False, True] - assert result.semantic_id_evaluation["per_query"]["CCE"]["recall"][0].tolist() == [0., 1.] - for values in (records[0].prediction.address_sids, records[0].prediction.scores, - records[0].prediction.catalog_validity): - with pytest.raises(ValueError): - values.setflags(write=True) - - -def test_restricted_catalog_rejects_incorrect_native_validity(): - train, test = data() - - class MalformedSID(FakeSID): - def fit(self, train_set, val_set=None): - super().fit(train_set, val_set) - self.catalog_ = catalog(((0,), (1,), (2,), (3,), (4,))) - return self - - def predict_semantic_ids(self, history_items, **context): - # Address 4 belongs to the native catalog, but not the admitted one. - # Its incorrect flag must fail before admission can hide the error. - return SemanticIDPrediction([[4], [0]], [1., 0.], [False, True]) - - evaluation = SemanticIDEvaluation.from_splits( - train, test, fmt="USIT", exclude_unknowns=True) - with pytest.raises(ValueError, match="catalog_validity"): - evaluation.evaluate(MalformedSID(), [Recall(2)], show_validation=False) - - -def test_wrong_target_codes_rejected_and_same_user_queries_are_distinct(): - from dataclasses import replace - cat = catalog() - saved = trace(cat, 1, ((0,),)) - record = replace(saved.records[0], target_address_sid=(1,)) - with pytest.raises(ValueError, match="target codes"): - evaluate_semantic_id_trace(cat, SemanticIDTrace(cat.stable_hash, (record,))) - seen = [] - resolver = CustomScorer(lambda query, user, candidates: - seen.append((query, user)) or ([0, 1] if query == 0 else [1, 0])) - records = saved.records + (replace(saved.records[0], query_index=1),) - result = evaluate_semantic_id_trace(cat, - SemanticIDTrace(cat.stable_hash, records), (1,), {"custom": resolver}) - assert seen == [(0, 0), (1, 0)] - assert result["per_query"]["Item[custom]"]["recall"].tolist() == [[1.], [0.]] - - -def test_non_semantic_id_model_fails_before_fit(): - from cornac.models.recommender import NextItemRecommender - - train, test = data() - evaluation = SemanticIDEvaluation.from_splits(train, test, fmt="USIT") - with pytest.raises(ValueError, match="SemanticIDRecommender"): - evaluation.evaluate(NextItemRecommender("native"), [Recall(2)]) diff --git a/tests/cornac/eval_methods/test_semantic_id_metrics.py b/tests/cornac/eval_methods/test_semantic_id_metrics.py new file mode 100644 index 000000000..f8574b4f5 --- /dev/null +++ b/tests/cornac/eval_methods/test_semantic_id_metrics.py @@ -0,0 +1,378 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Three-view next-item evaluation with backend-free fixed predictions.""" + +import itertools +import os +import subprocess +import sys +import warnings + +import numpy as np +import pytest + +from cornac.eval_methods import NextItemEvaluation +from cornac.eval_methods.next_item_evaluation import ( + _semantic_metric_scores, + ranking_eval, +) +from cornac.exception import ScoreException +from cornac.experiment.result import ExperimentResult +from cornac.metrics import Recall, NDCG, AUC, MAP, MRR +from cornac.models import NextItemRecommender, SemanticIDMixin, SPop + + +class FixedSID(NextItemRecommender, SemanticIDMixin): + def __init__(self, codes=(0, 0, 1), predicted=(0, 1), scores=(3.0, 2.0, 1.0)): + super().__init__("FixedSID") + self.codes = np.asarray(codes, dtype="int64").reshape(-1, 1) + self.predicted = np.asarray(predicted, dtype="int64").reshape(-1, 1) + self.scores = np.asarray(scores) + self.calls = [] + + def semantic_id_catalog(self): + return self.codes + + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + self.calls.append((list(history_items), kwargs)) + return self.scores.copy(), self.predicted.copy() + + def score(self, user_idx, history_items, **kwargs): + return self._score_with_semantic_ids(user_idx, history_items, **kwargs)[0] + + +def evaluation(**kwargs): + train = [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)] + test = [("u", "te", "a", 3), ("u", "te", "b", 4)] + kwargs.setdefault("semantic_id_metrics", True) + return NextItemEvaluation.from_splits(train, test, fmt="USIT", **kwargs) + + +@pytest.mark.parametrize("entrypoint", ["evaluate", "eval", "ranking_eval"]) +@pytest.mark.parametrize("flag", ["omitted", False, True]) +@pytest.mark.parametrize("model_class", [FixedSID, SPop]) +def test_semantic_views_require_explicit_opt_in(entrypoint, flag, model_class): + data = evaluation() + options = {} if flag == "omitted" else {"semantic_id_metrics": flag} + ev = NextItemEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", + **options, + ) + model = model_class() + enabled = flag is True and model_class is FixedSID + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + if entrypoint == "evaluate": + result, _ = ev.evaluate(model, [Recall(1)], False) + values = list(result.metric_avg_results.values())[:-2] + names = result.metric_avg_results + else: + model.fit(data.train_set) + if entrypoint == "eval": + result = NextItemEvaluation.eval( + model, data.train_set, data.test_set, True, [Recall(1)], **options + ) + values = list(result.metric_avg_results.values()) + names = result.metric_avg_results + else: + values, _ = ranking_eval( + model, [Recall(1)], data.train_set, data.test_set, **options + ) + names = None + assert len(values) == (3 if enabled else 1) + if enabled: + assert values == [0.0, 1.0, 0.5] + if names is not None: + assert ("SID-Recall@1" in names) is enabled + assert ("CCE-Recall@1" in names) is enabled + assert len(caught) == int(model_class is FixedSID and not enabled) + if caught: + assert "disabled" in str(caught[0].message) + + +@pytest.mark.parametrize("predicted", [(0, 1, 2), (1, 0), (9, 0, 0, 2, 1), ()]) +def test_cce_matches_all_bucket_permutations(predicted): + groups = {0: (0, 1), 1: (2,), 2: (3, 4, 5)} + compact = list(dict.fromkeys(sid for sid in predicted if sid in groups)) + rankings = [ + sum((list(bucket) for bucket in permutation), []) + for permutation in itertools.product( + *(list(itertools.permutations(groups[sid])) for sid in compact) + ) + ] + for target in range(6): + target_sid = next(sid for sid, members in groups.items() if target in members) + for cutoff in range(1, 8): + values, distinct = _semantic_metric_scores( + np.asarray(predicted).reshape(-1, 1), + (target_sid,), + {(sid,): len(members) for sid, members in groups.items()}, + [Recall(cutoff), NDCG(cutoff)], + ) + oracle_hits, oracle_ndcg = [], [] + for ranking in rankings: + rank = ranking.index(target) if target in ranking else None + hit = float(rank is not None and rank < cutoff) + oracle_hits.append(hit) + oracle_ndcg.append(hit / np.log2(rank + 2) if rank is not None else 0.0) + assert values[2:] == pytest.approx( + [np.mean(oracle_hits), np.mean(oracle_ndcg)] + ) + assert distinct == len(compact) + + +def test_three_views_keep_ordinary_resolution_and_share_inference(): + model = FixedSID() + result, _ = evaluation().evaluate(model, [Recall(1), NDCG(1)], False) + values = result.metric_avg_results + assert values["Recall@1"] == 0.0 + assert values["SID-Recall@1"] == 1.0 + assert values["CCE-Recall@1"] == 0.5 + assert values["CCE-NDCG@1"] == 0.5 + assert len(model.calls) == 1 + assert model.calls[0][0] == [0] + assert { + "history_mapped_ids", + "sessions", + "session_indices", + "extra_data", + } <= model.calls[0][1].keys() + assert not hasattr(result, "semantic_id_trace") + + +def test_missing_candidates_cannot_get_credit_from_score_fillers(): + model = FixedSID(predicted=(1,), scores=(-1e10, -1e10, 3.0)) + with pytest.warns(UserWarning, match="1 evaluation queries returned fewer than 2"): + result, _ = evaluation().evaluate(model, [Recall(2)], False) + assert result.metric_avg_results["SID-Recall@2"] == 0.0 + assert result.metric_avg_results["CCE-Recall@2"] == 0.0 + + +def test_invalid_and_duplicate_predictions_compact_before_cutoff(): + model = FixedSID(predicted=(9, 1, 1, 0, 0)) + result, _ = evaluation().evaluate(model, [Recall(2)], False) + assert result.metric_avg_results["SID-Recall@2"] == 1.0 + assert result.metric_avg_results["CCE-Recall@2"] == 0.5 + + +@pytest.mark.parametrize("exclude", [False, True]) +def test_candidate_filtering_changes_bucket_sizes_and_removes_unknown_only_ids(exclude): + ev = NextItemEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1)], + [("u", "te", "x", 2), ("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", + exclude_unknowns=False, + semantic_id_metrics=True, + ) + # Keep the full catalog available, then restrict the evaluated candidates. + ev.exclude_unknowns = exclude + model = FixedSID(codes=(0, 0, 0), predicted=(0,), scores=(3.0, 2.0, 1.0)) + result, _ = ev.evaluate(model, [Recall(1)], False) + assert result.metric_avg_results["CCE-Recall@1"] == pytest.approx( + 1 / (2 if exclude else 3) + ) + # An unknown-only bucket consumes no slot when excluded. + model = FixedSID(codes=(0, 0, 1), predicted=(1, 0)) + result, _ = ev.evaluate(model, [Recall(1)], False) + assert result.metric_avg_results["SID-Recall@1"] == float(exclude) + + +@pytest.mark.parametrize("mode,count", [("last", 1), ("next", 2)]) +@pytest.mark.parametrize("user_based", [False, True]) +def test_modes_user_aggregation_and_validation_share_query_counts( + mode, count, user_based +): + train = [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)] + test = [("u", "te", "a", 3), ("u", "te", "b", 4), ("u", "te", "c", 5)] + val = [("u", "va", "a", 6), ("u", "va", "b", 7)] + ev = NextItemEvaluation.from_splits( + train, test, val, fmt="USIT", mode=mode, semantic_id_metrics=True + ) + model = FixedSID() + test_result, val_result = ev.evaluate(model, [Recall(1)], user_based) + assert len(model.calls) == count + 1 + assert "CCE-Recall@1" in test_result.metric_avg_results + assert val_result.metric_avg_results["CCE-Recall@1"] == 0.5 + assert model.calls[-1][0] == [0] + + +def test_user_weighting_matches_ordinary_and_extra_views(): + ev = NextItemEvaluation.from_splits( + [("u", "tr1", "a", 0), ("u", "tr1", "b", 1), ("v", "tr2", "c", 2)], + [ + ("u", "te1", "a", 3), + ("u", "te1", "b", 4), + ("u", "te2", "a", 5), + ("u", "te2", "b", 6), + ("v", "te3", "a", 7), + ("v", "te3", "c", 8), + ], + fmt="USIT", + semantic_id_metrics=True, + ) + query, _ = ev.evaluate(FixedSID(), [Recall(1)], False) + user, _ = ev.evaluate(FixedSID(), [Recall(1)], True) + assert query.metric_avg_results["SID-Recall@1"] == pytest.approx(2 / 3) + assert user.metric_avg_results["SID-Recall@1"] == 0.5 + assert query.metric_avg_results["CCE-Recall@1"] == pytest.approx(1 / 3) + assert user.metric_avg_results["CCE-Recall@1"] == 0.25 + + +def test_only_finite_recall_ndcg_add_extra_views_without_rejecting_other_metrics(): + model = FixedSID() + metrics = [Recall(1), NDCG(1), Recall(), NDCG(), AUC(), MAP(), MRR()] + result, _ = evaluation().evaluate(model, metrics, False) + assert {mt.name for mt in metrics} <= result.metric_avg_results.keys() + assert { + name for name in result.metric_avg_results if name.startswith(("SID-", "CCE-")) + } == {"SID-Recall@1", "SID-NDCG@1", "CCE-Recall@1", "CCE-NDCG@1"} + assert len(model.calls) == 1 + with warnings.catch_warnings(record=True) as caught: + evaluation().evaluate(FixedSID(predicted=()), [MRR()], False) + assert not caught + + +def test_cutoff_lists_and_flag_off_preserve_legacy_item_metrics(): + ev = evaluation(semantic_id_metrics=False) + model = FixedSID() + with pytest.warns(UserWarning, match="disabled"): + result, _ = ev.evaluate(model, [Recall([1, 2])], False) + assert not any( + name.startswith(("SID-", "CCE-")) for name in result.metric_avg_results + ) + enabled, _ = evaluation().evaluate(FixedSID(), [Recall([1, 2])], False) + for name in ("Recall@1", "Recall@2"): + assert enabled.metric_avg_results[name] == result.metric_avg_results[name] + ordinary, _ = evaluation().evaluate(SPop(), [Recall(1)], False) + assert not any( + name.startswith(("SID-", "CCE-")) for name in ordinary.metric_avg_results + ) + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_mixed_result_table_union_and_unavailable_values(reverse): + sid_result, _ = evaluation().evaluate(FixedSID(), [Recall(1)], False) + ordinary, _ = evaluation().evaluate(SPop(), [Recall(1)], False) + results = [sid_result, ordinary] + table = str(ExperimentResult(results[::-1] if reverse else results)) + assert "SID-Recall@1" in table and "CCE-Recall@1" in table + assert table.count("N/A") == 2 + assert "nan" not in table + assert "SID-Recall@1" not in ordinary.metric_avg_results + + +@pytest.mark.parametrize("k", [-1, 1, 2]) +def test_combined_ranking_preserves_full_sid_output_and_legacy_scores(k): + ev = evaluation() + model = FixedSID(predicted=(9, 0, 0, 1)) + model.fit(ev.train_set) + ordinary = model.rank(0, [2, 1, 0], k=k, history_items=[0]) + combined = model.rank_with_semantic_ids(0, [2, 1, 0], k=k, history_items=[0]) + for expected, actual in zip(ordinary, combined): + np.testing.assert_array_equal(expected, actual) + np.testing.assert_array_equal(combined[2], model.predicted) + + +def test_rank_fallback_and_unknown_score_padding(): + ev = evaluation() + model = FixedSID(scores=(2.0, 1.0)) + model.fit(ev.train_set) + model.num_items = 2 + np.testing.assert_array_equal( + model.rank_with_semantic_ids(0, [2, 0], history_items=[0])[1], [1.0, 2.0] + ) + + def fail(*args, **kwargs): + raise ScoreException("no prediction") + + model._score_with_semantic_ids = fail + ordinary = model.rank(0, history_items=[0]) + combined = model.rank_with_semantic_ids(0, history_items=[0]) + np.testing.assert_array_equal(combined[0], ordinary[0]) + np.testing.assert_array_equal(combined[1], ordinary[1]) + assert combined[2].shape == (0, 1) + + +def test_lazy_import_without_computing_backends(): + script = """ +import sys +class BlockBackends: + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in {'torch', 'transformers', 'faiss'}: + raise ImportError('backend unavailable') +sys.meta_path.insert(0, BlockBackends()) +from cornac.models import TIGER, LETTER, RPG, DiffGRM, SemanticIDMixin +from cornac.eval_methods import NextItemEvaluation +assert 'torch' not in sys.modules +""" + subprocess.run([sys.executable, "-c", script], check=True, env=os.environ.copy()) + + +def test_score_fallback_has_no_semantic_credit(): + class Unscorable(FixedSID): + def _score_with_semantic_ids(self, *args, **kwargs): + raise ScoreException("no prediction") + + with pytest.warns(UserWarning, match="1 evaluation queries"): + result, _ = evaluation().evaluate(Unscorable(), [Recall(3)], False) + assert result.metric_avg_results["Recall@3"] == 1.0 + assert result.metric_avg_results["SID-Recall@3"] == 0.0 + assert result.metric_avg_results["CCE-Recall@3"] == 0.0 + + +def test_unique_complete_ids_make_all_three_views_equal(): + model = FixedSID(codes=(0, 1, 2), predicted=(0, 1, 2)) + result, _ = evaluation().evaluate(model, [Recall(2), NDCG(2)], False) + for name in ("Recall@2", "NDCG@2"): + assert ( + result.metric_avg_results[name] == result.metric_avg_results["SID-" + name] + ) + assert ( + result.metric_avg_results[name] == result.metric_avg_results["CCE-" + name] + ) + + +def test_empty_output_warns_once_per_split_without_budget_changes(): + train = [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)] + test = [ + ("u", "te1", "a", 3), + ("u", "te1", "b", 4), + ("u", "te2", "a", 5), + ("u", "te2", "c", 6), + ] + ev = NextItemEvaluation.from_splits( + train, test, fmt="USIT", semantic_id_metrics=True + ) + model = FixedSID(predicted=()) + with pytest.warns( + UserWarning, match="2 evaluation queries returned fewer than 2" + ) as caught: + result, _ = ev.evaluate(model, [Recall(2), NDCG(2), Recall()], False) + assert len(caught) == 1 + assert all( + result.metric_avg_results[name] == 0.0 + for name in ("SID-Recall@2", "CCE-Recall@2", "SID-NDCG@2", "CCE-NDCG@2") + ) + assert len(model.calls) == 2 + + +def test_exhaustive_sid_order_uses_max_member_and_lexicographic_ties(): + model = FixedSID(codes=(2, 2, 1), scores=(1.0, 3.0, 3.0)) + np.testing.assert_array_equal( + model._semantic_ids_from_scores(model.scores), [[1], [2]] + ) diff --git a/tests/cornac/models/test_semantic_id_adapters.py b/tests/cornac/models/test_semantic_id_adapters.py index 52c65ac30..709a0f6f1 100644 --- a/tests/cornac/models/test_semantic_id_adapters.py +++ b/tests/cornac/models/test_semantic_id_adapters.py @@ -13,25 +13,33 @@ # limitations under the License. # ============================================================================ -"""Raw SID adapters retain native scoring and decoder slot semantics.""" +"""Semantic ID adapters share decoding without changing native item scoring.""" import numpy as np import pytest torch = pytest.importorskip("torch") pytest.importorskip("transformers") - from cornac.models import TIGER -from cornac.utils.semantic_id import prediction_from_catalog_scores def tiny_tiger(model_class=TIGER): torch.manual_seed(7) model = model_class( - rqvae_num_levels=2, rqvae_codebook_size=2, n_epochs=0, - d_model=8, d_ff=16, num_heads=2, d_kv=4, - num_enc_layers=1, num_dec_layers=1, max_len=2, - dropout=0, n_beams=2, device="cpu", seed=7, + rqvae_num_levels=2, + rqvae_codebook_size=2, + n_epochs=0, + d_model=8, + d_ff=16, + num_heads=2, + d_kv=4, + num_enc_layers=1, + num_dec_layers=1, + max_len=2, + dropout=0, + n_beams=2, + device="cpu", + seed=7, ) model.num_items = 3 model.device_ = "cpu" @@ -42,261 +50,101 @@ def tiny_tiger(model_class=TIGER): def native_tiger_prediction(model, history, budget): """Direct decoder invocation with master's padding and encoder layout.""" - hist = list(history)[-model.max_len:] + hist = list(history)[-model.max_len :] hist = [model.pad_idx] * (model.max_len - len(hist)) + hist tokens = torch.tensor(model.enc_token_table[hist].reshape(1, -1)) mask = (tokens != 0).float() model.model.eval() with torch.no_grad(): if model.scoring == "beam": - return model.model.generate_beam(tokens, mask, budget, model.prefix_children) + return model.model.generate_beam( + tokens, mask, budget, model.prefix_children + ) return model.model.score_all_items( tokens, mask, torch.tensor(model.sid_table), model.scoring_batch_size ) -def test_tiger_catalog_and_raw_slots(monkeypatch): - model = tiny_tiger() - cat = model.semantic_id_catalog() - assert cat.tokenizer_sid_for_item(0) == cat.tokenizer_sid_for_item(1) - assert cat.address_sid_for_item(0) != cat.address_sid_for_item(1) - a = cat.address_sid_for_item(0) - outputs = [a, (9, 9, 9), a] - seen = [] - - def decode(tokens, mask, budget, children): - seen.append(budget) - return outputs, [-1., -2., -3.] - - monkeypatch.setattr(model.model, "generate_beam", decode) - pred = model.predict_semantic_ids(np.array([0]), n_candidates=4) - assert seen == [4] - assert model.n_beams == 2 - assert pred.address_sids.tolist() == [list(row) for row in outputs] - assert pred.catalog_validity.tolist() == [True, False, True] - assert len(model.predict_semantic_ids([]).scores) == 0 - np.testing.assert_equal(model.score(0, []), np.ones(3)) - - -@pytest.mark.parametrize("scoring", ["beam", "exact"]) -def test_tiger_native_parity_and_persistence(scoring, tmp_path): - model = tiny_tiger() - model.scoring = scoring - native = native_tiger_prediction(model, [0], model.n_beams) - if scoring == "beam": - addresses, logps = native - expected = np.full(3, -1e10) - for sid, score in zip(addresses, logps): - expected[model.sid_to_item[sid]] = score - raw = model.predict_semantic_ids([0]) - np.testing.assert_array_equal(raw.address_sids, addresses) - np.testing.assert_allclose(raw.scores, logps) - enlarged = model.predict_semantic_ids([0], n_candidates=3) - assert len(enlarged.scores) == 3 - else: - expected = native - oracle = prediction_from_catalog_scores(model.semantic_id_catalog(), expected) - raw = model.predict_semantic_ids([0]) - np.testing.assert_array_equal(raw.address_sids, oracle.address_sids) - np.testing.assert_allclose(raw.scores, oracle.scores) - np.testing.assert_allclose(model.score(0, [0]), expected) - restored = model.load(model.save(str(tmp_path))) - assert restored.semantic_id_catalog().stable_hash == model.semantic_id_catalog().stable_hash - np.testing.assert_allclose(restored.score(0, [0]), expected) - np.testing.assert_array_equal(restored.predict_semantic_ids([0]).address_sids, raw.address_sids) - - -def test_tiger_cpu_fit_and_experiment(tmp_path): - from cornac.data import FeatureModality - from cornac.eval_methods import SemanticIDEvaluation - from cornac.metrics import Recall - - evaluation = SemanticIDEvaluation.from_splits( - [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], - [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT", - item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"]), - ) - model = TIGER( - rqvae_num_levels=2, rqvae_codebook_size=2, rqvae_n_epochs=1, - rqvae_hidden_dims=(4,), rqvae_latent_dim=2, n_epochs=1, - d_model=8, d_ff=16, num_heads=2, d_kv=4, - num_enc_layers=1, num_dec_layers=1, max_len=2, - dropout=0, n_beams=2, device="cpu", seed=7, - ) - result, _ = evaluation.evaluate(model, [Recall(2)], user_based=False) - assert {"SID/Recall@2", "CCE/Recall@2", "Item[popularity]/Recall@2"} <= set(result.metric_avg_results) - restored = model.load(model.save(str(tmp_path))) - np.testing.assert_array_equal(model.predict_semantic_ids([0]).address_sids, - restored.predict_semantic_ids([0]).address_sids) - - def tiny_letter(): from cornac.data import FeatureModality - from cornac.eval_methods import SemanticIDEvaluation + from cornac.eval_methods import NextItemEvaluation from cornac.models import LETTER - evaluation = SemanticIDEvaluation.from_splits( + evaluation = NextItemEvaluation.from_splits( [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], - [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT", + [("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"]), ) model = LETTER( precomputed_semantic_ids=np.array([[0, 0], [0, 0], [1, 1]]), - rqvae_num_levels=2, rqvae_codebook_size=2, n_epochs=1, - d_model=8, d_ff=16, num_heads=2, d_kv=4, letter_base_vocab_size=8, - num_enc_layers=1, num_dec_layers=1, max_len=2, - dropout=0, n_beams=2, device="cpu", seed=7, + rqvae_num_levels=2, + rqvae_codebook_size=2, + n_epochs=1, + d_model=8, + d_ff=16, + num_heads=2, + d_kv=4, + letter_base_vocab_size=8, + num_enc_layers=1, + num_dec_layers=1, + max_len=2, + dropout=0, + n_beams=2, + device="cpu", + seed=7, ) model.fit(evaluation.train_set) return model, evaluation -@pytest.mark.parametrize("scoring", ["beam", "exact"]) -def test_letter_native_parity_collisions_and_cpu_fit(scoring, tmp_path): - from cornac.metrics import Recall - - model, evaluation = tiny_letter() - model.scoring = scoring - cat = model.semantic_id_catalog() - assert cat.address_to_items[(0, 0)] == (0, 1) - assert cat.address_sids.shape == cat.tokenizer_sids.shape == (3, 2) - inputs, mask = model._encoder_batch(torch, [[0]]) - model.model.eval() - with torch.no_grad(): - if scoring == "beam": - beams, scores = model.model.generate_beam(inputs, mask, 2, model.prefix_children) - expected = np.full(3, -1e10) - for sid, score in zip(beams, scores): - expected[model.sid_to_item[sid]] = score - raw = model.predict_semantic_ids([0], n_candidates=2) - np.testing.assert_array_equal(raw.address_sids, beams) - np.testing.assert_allclose(raw.scores, scores) - else: - expected = model.model.score_all_items(inputs, mask, torch.tensor(model.sid_table), model.scoring_batch_size) - raw = model.predict_semantic_ids([0]) - oracle = prediction_from_catalog_scores(cat, expected) - np.testing.assert_array_equal(raw.address_sids, oracle.address_sids) - np.testing.assert_allclose(raw.scores, oracle.scores) - np.testing.assert_allclose(model.score(0, [0]), expected) - restored = model.load(model.save(str(tmp_path))) - np.testing.assert_array_equal(restored.predict_semantic_ids([0]).address_sids, raw.address_sids) - assert restored.semantic_id_catalog().stable_hash == cat.stable_hash - # Avoid retraining here: the fit smoke above is sufficient; evaluate frozen model. - model.trainable = False - result, _ = evaluation.evaluate(model, [Recall(2)], user_based=False) - assert "CCE/Recall@2" in result.metric_avg_results - - -def test_letter_raw_slots_and_budget(monkeypatch): - model, _ = tiny_letter() - outputs = [(0, 0), (8, 8), (0, 0)] - budgets = [] - - def decode(tokens, mask, budget, children): - budgets.append(budget) - return outputs, [-1., -2., -3.] - - monkeypatch.setattr(model.model, "generate_beam", decode) - raw = model.predict_semantic_ids([0], n_candidates=5) - assert budgets == [5] - assert model.n_beams == 2 - assert raw.address_sids.tolist() == [list(sid) for sid in outputs] - assert raw.catalog_validity.tolist() == [True, False, True] - - def tiny_rpg(monkeypatch): from cornac.data import FeatureModality - from cornac.eval_methods import SemanticIDEvaluation + from cornac.eval_methods import NextItemEvaluation from cornac.models import RPG - evaluation = SemanticIDEvaluation.from_splits( + evaluation = NextItemEvaluation.from_splits( [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], - [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT", + [("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"]), ) model = RPG( - n_codebook=2, codebook_size=2, n_epochs=1, d_model=8, - n_layer=1, n_head=2, n_inner=16, max_len=2, - resid_dropout=0, embd_dropout=0, attn_dropout=0, - n_beams=2, graph_edges=2, propagation_steps=1, device="cpu", seed=7, + n_codebook=2, + codebook_size=2, + n_epochs=1, + d_model=8, + n_layer=1, + n_head=2, + n_inner=16, + max_len=2, + resid_dropout=0, + embd_dropout=0, + attn_dropout=0, + n_beams=2, + graph_edges=2, + propagation_steps=1, + device="cpu", + seed=7, ) # faiss OPQ uses 8-bit PQ and needs >=256 training items; freeze only # tokenization to keep this real training/graph-building CPU smoke tiny. with monkeypatch.context() as patch: - patch.setattr(RPG, "_opq_tokenize", lambda self, feats, train_mask: np.array([[0, 0], [0, 0], [1, 1]])) + patch.setattr( + RPG, + "_opq_tokenize", + lambda self, feats, train_mask: np.array([[0, 0], [0, 0], [1, 1]]), + ) model.fit(evaluation.train_set) return model, evaluation -@pytest.mark.parametrize("scoring", ["graph", "exact"]) -def test_rpg_native_parity_rng_and_persistence(monkeypatch, scoring, tmp_path): - from cornac.models.rpg.rpg import graph_decode - - model, _ = tiny_rpg(monkeypatch) - model.scoring = scoring - state = model.graph_rng.get_state() - model.model.eval() - with torch.no_grad(): - tokens = torch.tensor([[model.pad_idx, 0]]) - logits = model.model.next_item_logits(tokens, (tokens != model.pad_idx).float())[0] - if scoring == "graph": - candidates, scores = graph_decode(logits.reshape(-1).numpy(), model.adjacency, - model.item_tokens, 2, model.propagation_steps, model.graph_rng) - after_native = model.graph_rng.get_state() - expected = np.full(3, -1e10) - expected[candidates] = scores - model.graph_rng.set_state(state) - raw = model.predict_semantic_ids([0]) - np.testing.assert_array_equal(raw.address_sids, model.sid_table[candidates]) - np.testing.assert_allclose(raw.scores, scores) - assert all(np.array_equal(a, b) for a, b in zip(after_native, model.graph_rng.get_state())) - assert not all(np.array_equal(a, b) for a, b in zip(state, after_native)) - else: - expected = model._exact_item_scores(logits)[0].numpy().astype(float) - raw = model.predict_semantic_ids([0]) - oracle = prediction_from_catalog_scores(model.semantic_id_catalog(), expected) - np.testing.assert_array_equal(raw.address_sids, oracle.address_sids) - np.testing.assert_allclose(raw.scores, oracle.scores) - assert len(raw.scores) == 2 # maximum member score collapses the collision - model.graph_rng.set_state(state) - np.testing.assert_allclose(model.score(0, [0]), expected) - restored = model.load(model.save(str(tmp_path))) - # Save copied the RNG state, so compare the same next stochastic call. - np.testing.assert_array_equal(restored.predict_semantic_ids([0]).address_sids, - model.predict_semantic_ids([0]).address_sids) - assert model.semantic_id_catalog().address_to_items[(0, 0)] == (0, 1) - assert len(model.predict_semantic_ids([]).scores) == 0 - np.testing.assert_array_equal(model.score(0, []), np.ones(3)) - - -def test_rpg_graph_order_collisions_and_budget(monkeypatch): - import cornac.models.rpg.rpg as backend - model, evaluation = tiny_rpg(monkeypatch) - seen = [] - - def decode(flat, adjacency, tokens, budget, steps, rng): - assert rng is model.graph_rng - seen.append(budget) - return np.array([2, 0, 1]), np.array([-1., -2., -3.]) - - with monkeypatch.context() as patch: - patch.setattr(backend, "graph_decode", decode) - raw = model.predict_semantic_ids([0], n_candidates=5) - assert seen == [5] - assert model.n_beams == 2 - assert raw.address_sids.tolist() == [[1, 1], [0, 0], [0, 0]] - from cornac.metrics import Recall - model.trainable = False - result, _ = evaluation.evaluate(model, [Recall(2)], user_based=False) - assert "CCE/Recall@2" in result.metric_avg_results - - class ConstantCPD: n_digit = 2 codebook_size = 2 def decode_logits(self, memory, padding, partial): - return torch.tensor([[[4., 0.], [3., 0.]]]).expand(len(partial), -1, -1) + return torch.tensor([[[4.0, 0.0], [3.0, 0.0]]]).expand(len(partial), -1, -1) @pytest.mark.parametrize("diagnostics", [False, True]) @@ -305,22 +153,33 @@ def test_cpd_raw_single_batch_default_parity(diagnostics, constrained): from cornac.models.diffgrm.diffgrm import cpd_decode, cpd_decode_batch memory, padding = torch.zeros(2, 1, 4), torch.zeros(2, 1, dtype=torch.bool) - kwargs = dict(beam_size=4, catalog_codes=np.array([[0, 0], [1, 1]]), - constrained=constrained, return_diagnostics=diagnostics) + kwargs = dict( + beam_size=4, + catalog_codes=np.array([[0, 0], [1, 1]]), + constrained=constrained, + return_diagnostics=diagnostics, + ) default = cpd_decode_batch(ConstantCPD(), memory, padding, **kwargs) extra = cpd_decode_batch(ConstantCPD(), memory, padding, return_raw=True, **kwargs) assert extra[:-1] == default raw = extra[-1] assert len(raw) == 2 for entry in raw: - assert len(entry["address_sids"]) == len(entry["scores"]) == len(entry["catalog_validity"]) == 4 + assert ( + len(entry["address_sids"]) + == len(entry["scores"]) + == len(entry["catalog_validity"]) + == 4 + ) # Parallel assignment orders produce repeated complete paths. assert len(set(entry["address_sids"])) < len(entry["address_sids"]) assert entry["catalog_validity"] == [ sid in {(0, 0), (1, 1)} and np.isfinite(score) for sid, score in zip(entry["address_sids"], entry["scores"]) ] - single = cpd_decode(ConstantCPD(), memory[:1], padding[:1], return_raw=True, **kwargs) + single = cpd_decode( + ConstantCPD(), memory[:1], padding[:1], return_raw=True, **kwargs + ) assert single == tuple(part[0] for part in extra) legacy_single = cpd_decode(ConstantCPD(), memory[:1], padding[:1], **kwargs) assert legacy_single == tuple(part[0] for part in default) @@ -330,147 +189,253 @@ def test_cpd_raw_single_batch_default_parity(diagnostics, constrained): def test_cpd_raw_marks_nonfinite_and_invalid_slots(): from cornac.models.diffgrm.diffgrm import cpd_decode + memory, padding = torch.zeros(1, 1, 4), torch.zeros(1, 1, dtype=torch.bool) - codes, scores, raw = cpd_decode(ConstantCPD(), memory, padding, - beam_size=16, catalog_codes=np.array([[1, 1]]), return_raw=True) + codes, scores, raw = cpd_decode( + ConstantCPD(), + memory, + padding, + beam_size=16, + catalog_codes=np.array([[1, 1]]), + return_raw=True, + ) assert len(raw["address_sids"]) > len(codes) assert any(not valid for valid in raw["catalog_validity"]) - for sid, score, valid in zip(raw["address_sids"], raw["scores"], raw["catalog_validity"]): + for sid, score, valid in zip( + raw["address_sids"], raw["scores"], raw["catalog_validity"] + ): assert valid == (sid == (1, 1) and -1 not in sid and np.isfinite(score)) with pytest.raises(ValueError, match="permutation"): cpd_decode(ConstantCPD(), memory, padding, beam_size=2, digit_order=[0, 0]) def tiny_diffgrm(): - from cornac.eval_methods import SemanticIDEvaluation + from cornac.eval_methods import NextItemEvaluation from cornac.models import DiffGRM - evaluation = SemanticIDEvaluation.from_splits( + + evaluation = NextItemEvaluation.from_splits( [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], - [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT") + [("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", + ) model = DiffGRM( - item_sids=np.array([[0, 0], [0, 0], [1, 1]]), n_digit=2, - codebook_size=2, d_model=8, encoder_n_layer=1, decoder_n_layer=1, - n_head=2, n_inner=16, dropout=0, max_len=2, min_history=1, - n_epochs=1, n_views=1, beam_size=4, lr_schedule="constant", - device="cpu", seed=7, + item_sids=np.array([[0, 0], [0, 0], [1, 1]]), + n_digit=2, + codebook_size=2, + d_model=8, + encoder_n_layer=1, + decoder_n_layer=1, + n_head=2, + n_inner=16, + dropout=0, + max_len=2, + min_history=1, + n_epochs=1, + n_views=1, + beam_size=4, + lr_schedule="constant", + device="cpu", + seed=7, ) model.fit(evaluation.train_set) return model, evaluation +@pytest.mark.parametrize("scoring", ["beam", "exact"]) +def test_tiger_native_item_parity_and_full_ids(scoring, tmp_path): + model = tiny_tiger() + model.scoring = scoring + native = native_tiger_prediction(model, [0], model.n_beams) + if scoring == "beam": + sids, scores = native + expected = np.full(3, -1e10) + for sid, score in zip(sids, scores): + expected[model.sid_to_item[sid]] = score + else: + expected = native + sids = model._semantic_ids_from_scores(expected) + combined = model.rank_with_semantic_ids(0, np.arange(3), history_items=[0]) + np.testing.assert_allclose(combined[1], expected) + np.testing.assert_array_equal(combined[2], sids) + np.testing.assert_allclose(model.score(0, [0]), expected) + assert not np.array_equal( + model.semantic_id_catalog()[0], model.semantic_id_catalog()[1] + ) + restored = model.load(model.save(str(tmp_path))) + np.testing.assert_array_equal( + restored.rank_with_semantic_ids(0, history_items=[0])[2], combined[2] + ) + assert model.rank_with_semantic_ids(0, history_items=[])[2].shape == (0, 3) + + +@pytest.mark.parametrize("scoring", ["beam", "exact"]) +def test_letter_native_item_parity_and_collision_catalog(scoring): + model, evaluation = tiny_letter() + model.scoring = scoring + inputs, mask = model._encoder_batch(torch, [[0]]) + model.model.eval() + with torch.no_grad(): + if scoring == "beam": + sids, scores = model.model.generate_beam( + inputs, mask, model.n_beams, model.prefix_children + ) + expected = np.full(3, -1e10) + for sid, score in zip(sids, scores): + expected[model.sid_to_item[sid]] = score + else: + expected = model.model.score_all_items( + inputs, mask, torch.tensor(model.sid_table), model.scoring_batch_size + ) + sids = model._semantic_ids_from_scores(expected) + combined = model.rank_with_semantic_ids(0, np.arange(3), history_items=[0]) + np.testing.assert_allclose(combined[1], expected) + np.testing.assert_array_equal(combined[2], sids) + np.testing.assert_allclose(model.score(0, [0]), expected) + np.testing.assert_array_equal( + model.semantic_id_catalog()[0], model.semantic_id_catalog()[1] + ) + assert model.rank_with_semantic_ids(0, history_items=[])[2].shape == (0, 2) + + +@pytest.mark.parametrize("kind", ["tiger", "letter"]) +def test_beam_single_inference_keeps_raw_rows_and_configured_budget(kind, monkeypatch): + model = tiny_tiger() if kind == "tiger" else tiny_letter()[0] + sid = tuple(model.sid_table[0]) + invalid = (9,) * len(sid) + output = [sid, invalid, sid] + calls = [] + + def decode(tokens, mask, budget, children): + calls.append(budget) + return output, [-1.0, -2.0, -3.0] + + monkeypatch.setattr(model.model, "generate_beam", decode) + combined = model.rank_with_semantic_ids(0, [0, 1, 2], k=1, history_items=[0]) + assert calls == [model.n_beams] + np.testing.assert_array_equal(combined[2], output) + + +@pytest.mark.parametrize("scoring", ["graph", "exact"]) +def test_rpg_native_parity_and_single_rng_advance(monkeypatch, scoring, tmp_path): + from cornac.models.rpg.rpg import graph_decode + + model, evaluation = tiny_rpg(monkeypatch) + model.scoring = scoring + state = model.graph_rng.get_state() + model.model.eval() + with torch.no_grad(): + inputs = torch.tensor([[model.pad_idx, 0]]) + logits = model.model.next_item_logits( + inputs, (inputs != model.pad_idx).float() + )[0] + if scoring == "graph": + candidates, scores = graph_decode( + logits.reshape(-1).numpy(), + model.adjacency, + model.item_tokens, + model.n_beams, + model.propagation_steps, + model.graph_rng, + ) + after_native = model.graph_rng.get_state() + expected = np.full(3, -1e10) + expected[candidates] = scores + expected_sids = model.sid_table[candidates] + model.graph_rng.set_state(state) + else: + expected = model._exact_item_scores(logits)[0].numpy().astype(float) + expected_sids = model._semantic_ids_from_scores(expected) + combined = model.rank_with_semantic_ids(0, np.arange(3), history_items=[0]) + np.testing.assert_allclose(combined[1], expected) + np.testing.assert_array_equal(combined[2], expected_sids) + if scoring == "graph": + assert all( + np.array_equal(a, b) + for a, b in zip(after_native, model.graph_rng.get_state()) + ) + model.graph_rng.set_state(state) + ordinary = model.rank(0, np.arange(3), history_items=[0]) + np.testing.assert_array_equal(combined[0], ordinary[0]) + np.testing.assert_allclose(combined[1], ordinary[1]) + restored = model.load(model.save(str(tmp_path))) + np.testing.assert_array_equal( + restored.rank_with_semantic_ids(0, history_items=[0])[2], + model.rank_with_semantic_ids(0, history_items=[0])[2], + ) + assert model.rank_with_semantic_ids(0, history_items=[])[2].shape == (0, 2) + + @pytest.mark.parametrize("scoring", ["released", "paper", "catalog", "fixed"]) @pytest.mark.parametrize("policy", ["all", "first", "last"]) -def test_diffgrm_native_parity_raw_and_persistence(scoring, policy, tmp_path): +def test_diffgrm_native_parity_and_raw_sids(scoring, policy): from cornac.models.diffgrm.diffgrm import cpd_decode - model, _ = tiny_diffgrm() + model, evaluation = tiny_diffgrm() model.scoring = scoring model.collision_policy = policy model.model.eval() - # Original master's left-aligned history input and decode options. with torch.no_grad(): inputs = torch.tensor([[0, model.pad_idx]]) - memory, padding = model.model.encode_history(inputs, torch.tensor([[1., 0.]])) - codes, scores, diagnostics, raw = cpd_decode(model.model, memory, padding, - beam_size=model.beam_size, catalog_codes=model.sid_table, - valid_code_set=model.sid_to_items_, constrained=scoring == "catalog", + memory, padding = model.model.encode_history(inputs, torch.tensor([[1.0, 0.0]])) + codes, scores, diagnostics, raw = cpd_decode( + model.model, + memory, + padding, + beam_size=model.beam_size, + catalog_codes=model.sid_table, + valid_code_set=model.sid_to_items_, + constrained=scoring == "catalog", digit_order=model.fixed_decode_order_ if scoring == "fixed" else None, - greedy_final=scoring == "released", return_diagnostics=True, return_raw=True) + greedy_final=scoring == "released", + return_diagnostics=True, + return_raw=True, + ) expected = np.full(3, -1e10) for sid, score in zip(codes, scores): members = model.sid_to_items_[sid] - members = members[:1] if policy == "first" else members[-1:] if policy == "last" else members + members = ( + members[:1] + if policy == "first" + else members[-1:] if policy == "last" else members + ) expected[members] = score + combined = model.rank_with_semantic_ids(0, np.arange(3), history_items=[0]) + np.testing.assert_allclose(combined[1], expected) + np.testing.assert_array_equal(combined[2], raw["address_sids"]) np.testing.assert_allclose(model.score(0, [0]), expected) - pred = model.predict_semantic_ids(np.array([0])) - np.testing.assert_array_equal(pred.address_sids, raw["address_sids"]) - np.testing.assert_allclose(pred.scores, raw["scores"]) - assert pred.catalog_validity.tolist() == raw["catalog_validity"] - assert model.semantic_id_catalog().address_to_items[(0, 0)] == (0, 1) - restored = model.load(model.save(str(tmp_path))) - np.testing.assert_allclose(restored.score(0, [0]), expected) - np.testing.assert_array_equal(restored.predict_semantic_ids([0]).address_sids, pred.address_sids) - assert len(model.predict_semantic_ids([]).scores) == 0 + assert model.rank_with_semantic_ids(0, history_items=[])[2].shape == (0, 2) -def test_diffgrm_requested_budget_and_evaluation(monkeypatch): +def test_diffgrm_single_decode_resolution_independence_and_budget(monkeypatch): import cornac.models.diffgrm.diffgrm as backend from cornac.metrics import Recall model, evaluation = tiny_diffgrm() + model.trainable = False original = backend.cpd_decode - seen = [] + calls = [] def decode(*args, **kwargs): - seen.append((kwargs["beam_size"], kwargs["return_raw"])) + calls.append(kwargs["beam_size"]) return original(*args, **kwargs) - with monkeypatch.context() as patch: - patch.setattr(backend, "cpd_decode", decode) - raw = model.predict_semantic_ids([0], n_candidates=8) - assert seen == [(8, True)] - assert model.beam_size == 4 - assert len(raw.scores) == 8 - model.trainable = False - result, _ = evaluation.evaluate(model, [Recall(2)], user_based=False) - assert "CCE/Recall@2" in result.metric_avg_results - - -@pytest.mark.parametrize("kind", ["tiger", "letter", "rpg", "diffgrm"]) -def test_fitted_catalog_cached_refit_and_old_pickle(kind, monkeypatch, tmp_path): - if kind == "tiger": - # Full fitted dataset is needed below; reuse the known tiny fixture. - from cornac.data import FeatureModality - from cornac.eval_methods import SemanticIDEvaluation - evaluation = SemanticIDEvaluation.from_splits( - [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], - [("u", "te", "a", 3), ("u", "te", "b", 4)], fmt="USIT", - item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"])) - model = tiny_tiger() - model.rqvae_n_epochs = 1 - model.rqvae_hidden_dims = (4,) - model.rqvae_latent_dim = 2 - model.fit(evaluation.train_set) - elif kind == "letter": - model, evaluation = tiny_letter() - elif kind == "rpg": - model, evaluation = tiny_rpg(monkeypatch) - else: - model, evaluation = tiny_diffgrm() - first = model.semantic_id_catalog() - assert model.semantic_id_catalog() is first - model.predict_semantic_ids([0]) - assert model.semantic_id_catalog() is first - model.trainable = False - with pytest.warns(UserWarning, match="already fitted"): - model.fit(evaluation.train_set) - second = model.semantic_id_catalog() - assert second is not first - assert second.stable_hash == first.stable_hash - new_codes = np.array([[1, 0], [0, 1], [1, 0]]) - model.trainable = True - with monkeypatch.context() as patch: - if kind in ("tiger", "letter"): - patch.setattr(type(model), "_tokenize", lambda self, torch, feats: new_codes.copy()) - elif kind == "rpg": - patch.setattr(type(model), "_opq_tokenize", lambda self, feats, train_mask: new_codes.copy()) - else: - model.item_sids = new_codes - with pytest.warns(UserWarning, match="already fitted"): - model.fit(evaluation.train_set) - third = model.semantic_id_catalog() - assert third is model.semantic_id_catalog() - assert third.stable_hash != second.stable_hash - np.testing.assert_array_equal(third.tokenizer_sids, new_codes) - if kind in ("tiger", "letter"): - model._build_semantic_ids(np.array([[0, 1], [0, 0], [1, 1]])) - rebuilt = model.semantic_id_catalog() - assert rebuilt is not third - assert rebuilt.stable_hash != third.stable_hash - expected_hash = model.semantic_id_catalog().stable_hash - # Simulate an older model pickle that has SID state but no cached contract. - del model.semantic_id_catalog_ - restored = model.load(model.save(str(tmp_path))) - old_state_catalog = restored.semantic_id_catalog() - assert old_state_catalog.stable_hash == expected_hash - assert restored.semantic_id_catalog() is old_state_catalog + monkeypatch.setattr(backend, "cpd_decode", decode) + rows = [] + for policy in ("all", "first", "last"): + model.collision_policy = policy + result = evaluation.eval( + model, + evaluation.train_set, + evaluation.test_set, + True, + [Recall(1)], + semantic_id_metrics=True, + ) + rows.append( + [ + result.metric_avg_results[name] + for name in ("SID-Recall@1", "CCE-Recall@1") + ] + ) + np.testing.assert_allclose(rows, [rows[0]] * 3) + assert calls == [model.beam_size] * 3 From b22e49899ada8c01a09c2388d5f83531a9fdbe6f Mon Sep 17 00:00:00 2001 From: hieuddo Date: Thu, 29 Oct 2026 10:10:06 +0800 Subject: [PATCH 5/5] Remove redundant synthetic semantic ID example --- cornac/models/diffgrm/README.md | 3 +- cornac/models/letter/README.md | 3 +- cornac/models/rpg/README.md | 3 +- cornac/models/tiger/README.md | 3 +- docs/source/api_ref/eval_methods.rst | 3 -- examples/README.md | 2 - examples/semantic_id_evaluation_example.py | 61 ---------------------- 7 files changed, 4 insertions(+), 74 deletions(-) delete mode 100644 examples/semantic_id_evaluation_example.py diff --git a/cornac/models/diffgrm/README.md b/cornac/models/diffgrm/README.md index 049d722fa..f5af4ebc8 100644 --- a/cornac/models/diffgrm/README.md +++ b/cornac/models/diffgrm/README.md @@ -222,5 +222,4 @@ and repeated predicted IDs are compacted. Other requested metrics keep ordinary item evaluation. Configured beam sizes remain unchanged; short candidate lists warn. Set `semantic_id_metrics=False` for ordinary item metrics only. -See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst) -and [mixed-model example](../../../examples/semantic_id_evaluation_example.py). +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst). diff --git a/cornac/models/letter/README.md b/cornac/models/letter/README.md index 3e817b8e5..c812e44af 100644 --- a/cornac/models/letter/README.md +++ b/cornac/models/letter/README.md @@ -107,5 +107,4 @@ and repeated predicted IDs are compacted. Other requested metrics keep ordinary item evaluation. Configured beam sizes remain unchanged; short candidate lists warn. Set `semantic_id_metrics=False` for ordinary item metrics only. -See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst) -and [mixed-model example](../../../examples/semantic_id_evaluation_example.py). +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst). diff --git a/cornac/models/rpg/README.md b/cornac/models/rpg/README.md index a7a29669a..11f4d60af 100644 --- a/cornac/models/rpg/README.md +++ b/cornac/models/rpg/README.md @@ -153,5 +153,4 @@ and repeated predicted IDs are compacted. Other requested metrics keep ordinary item evaluation. Configured beam sizes remain unchanged; short candidate lists warn. Set `semantic_id_metrics=False` for ordinary item metrics only. -See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst) -and [mixed-model example](../../../examples/semantic_id_evaluation_example.py). +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst). diff --git a/cornac/models/tiger/README.md b/cornac/models/tiger/README.md index 53b05cc19..370df8123 100644 --- a/cornac/models/tiger/README.md +++ b/cornac/models/tiger/README.md @@ -92,8 +92,7 @@ and repeated predicted IDs are compacted. Other requested metrics keep ordinary item evaluation. Configured beam sizes remain unchanged; short candidate lists warn. Set `semantic_id_metrics=False` for ordinary item metrics only. -See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst) -and [mixed-model example](../../../examples/semantic_id_evaluation_example.py). +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst). TIGER uses complete IDs including the disambiguating suffix, so its SID and CCE metrics coincide even when tokenizer prefixes collide. diff --git a/docs/source/api_ref/eval_methods.rst b/docs/source/api_ref/eval_methods.rst index 82c22d4f9..28cd5017d 100644 --- a/docs/source/api_ref/eval_methods.rst +++ b/docs/source/api_ref/eval_methods.rst @@ -95,9 +95,6 @@ For example:: Experiment(evaluation, [TIGER(device="cpu"), SASRec()], [Recall(10), NDCG(10)]).run() -See ``examples/semantic_id_evaluation_example.py`` for a backend-free mixed-model -example with a fixed SID generator. - Adapter authors implement ``SemanticIDMixin`` alongside ``NextItemRecommender``. ``semantic_id_catalog()`` returns a two-dimensional integer array with one row per global item, storing complete generated IDs. ``rank_with_semantic_ids`` has diff --git a/examples/README.md b/examples/README.md index 604e9680f..44a1be0bb 100644 --- a/examples/README.md +++ b/examples/README.md @@ -122,8 +122,6 @@ ## Next-Item Algorithms -[semantic_id_evaluation_example.py](semantic_id_evaluation_example.py) - Backend-free mixed-model NextItemEvaluation example with ordinary item, SID and CCE metrics. - [spop_yoochoose.py](spop_yoochoose.py) - Next-item recommendation based on item popularity. [gru4rec_yoochoose.py](gru4rec_yoochoose.py) - Example of Session-based Recommendations with Recurrent Neural Networks (GRU4Rec). diff --git a/examples/semantic_id_evaluation_example.py b/examples/semantic_id_evaluation_example.py deleted file mode 100644 index 321699add..000000000 --- a/examples/semantic_id_evaluation_example.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -"""Backend-free ordinary item, SID and CCE evaluation in one Experiment.""" -import numpy as np - -import cornac -from cornac.eval_methods import NextItemEvaluation -from cornac.metrics import Recall, NDCG -from cornac.models import NextItemRecommender, SemanticIDMixin, SPop - - -class FrozenSIDExample(NextItemRecommender, SemanticIDMixin): - """A fixed generator demonstrating collisions, not a trained model.""" - - def __init__(self): - super().__init__("FrozenSIDExample") - - def fit(self, train_set, val_set=None): - super().fit(train_set, val_set) - self.codes = np.arange(self.total_items).reshape(-1, 1) // 2 - return self - - def semantic_id_catalog(self): - return self.codes - - def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): - # a and b share SID 0; this concrete item policy ranks a first. - scores = -np.arange(self.total_items, dtype="float64") - predicted = np.unique(self.codes, axis=0) - return scores, predicted - - def score(self, user_idx, history_items, **kwargs): - return self._score_with_semantic_ids(user_idx, history_items, **kwargs)[0] - - -if __name__ == "__main__": - train = [("u", "train", "a", 0), ("u", "train", "b", 1), ("u", "train", "c", 2)] - test = [("u", "test", "a", 3), ("u", "test", "b", 4)] - evaluation = NextItemEvaluation.from_splits( - train, test, fmt="USIT", semantic_id_metrics=True - ) - cornac.Experiment( - evaluation, - [FrozenSIDExample(), SPop()], - [Recall(1), NDCG(1)], - show_validation=False, - save_dir="/tmp/cornac-semantic-id-example", - ).run()