From 237c0e1aabeca32b59a631b6cfed138672f85b9e Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:21:27 +0800 Subject: [PATCH 1/5] feat(domino): add lambda curriculum and held-out backbone metrics Add the SpecForge-compatible linear lambda_base schedule, held-out corrected/backbone per-position observations, and a formal Domino recipe. Record the schedule in the training topology for exact resume and expose num_proc for constrained tokenizer hosts. --- aspec/config/train_config.py | 13 +++ aspec/models/domino.py | 10 ++- aspec/planning.py | 31 +++++++ aspec/training/checkpoint.py | 15 ++++ aspec/training/domino_trainer.py | 99 +++++++++++++++++++++- configs/recipes/domino_qwen3_8b.yaml | 93 +++++++++++++++++++++ docs/support.md | 2 +- docs/user-guide/algorithms.md | 9 +- docs/user-guide/data.md | 3 + docs/user-guide/domino.md | 63 ++++++++------ docs/user-guide/recipes.md | 1 + tests/cpu/test_domino.py | 24 +++++- tests/cpu/test_domino_integration.py | 119 ++++++++++++++++++++++++++- tests/cpu/test_release_recipes.py | 2 +- tools/recipe_catalog.json | 11 +++ 15 files changed, 456 insertions(+), 39 deletions(-) create mode 100644 configs/recipes/domino_qwen3_8b.yaml diff --git a/aspec/config/train_config.py b/aspec/config/train_config.py index 3fbb132..4fb5a92 100644 --- a/aspec/config/train_config.py +++ b/aspec/config/train_config.py @@ -240,6 +240,13 @@ class TrainingConfig: # Domino mixes corrected and backbone-only CE as # (1 - lambda_base) * corrected + lambda_base * backbone. domino_lambda_base: float = 0.3 + # Optional linear curriculum for lambda_base, mirroring the reference + # SpecForge schedule: start * max(0, 1 - step / (total_steps * ratio)). + # Leave both unset to keep the constant domino_lambda_base value. + domino_lambda_base_start: Optional[float] = None + domino_lambda_base_decay_ratio: Optional[float] = None + # Report backbone-only per-position held-out metrics alongside corrected. + domino_eval_backbone: bool = False # Pack multiple complete documents into one fixed row. The controller owns # grouping; the trainer preserves document boundaries for attention/RoPE. dflash_packing: bool = False @@ -300,6 +307,9 @@ class Config: mooncake: dict[str, Any] = field(default_factory=dict) training: TrainingConfig = field(default_factory=TrainingConfig) cache_dir: str = "./cache" + # Tokenizer worker processes for dataset preparation. Lower this on hosts + # where fork-based multiprocessing is constrained. + num_proc: int = 64 cache_key: Optional[str] = None model_download_dir: Optional[str] = None output_dir: str = "" @@ -367,6 +377,9 @@ def _validate_vllm_config(config: DictConfig) -> None: def _validate_current_scope(config: DictConfig) -> None: """Reject backend selections outside ASpec's current executable path.""" + num_proc = config.get("num_proc", 64) + if isinstance(num_proc, bool) or not isinstance(num_proc, int) or num_proc < 1: + raise ValueError("num_proc must be an integer >= 1") engine_type = config.inference.inference_engine_type target_backend = config.model.target_model_backend if engine_type not in {"sgl", "offline"} or target_backend != "sglang": diff --git a/aspec/models/domino.py b/aspec/models/domino.py index 5c810e8..fabbf84 100644 --- a/aspec/models/domino.py +++ b/aspec/models/domino.py @@ -48,6 +48,9 @@ def __init__(self, *args, lambda_base=0.0, **kwargs): # anchor slot. Both match the two documented Domino layouts. self.label_start = 1 if self.draft_model.shift_label else 0 self.prediction_start = 1 - self.label_start + # Held-out evaluation can report the backbone-only branch without + # changing the mixed objective used for training. + self.report_backbone = False self.set_lambda_base(lambda_base) def _prepare_draft_prefix( @@ -143,8 +146,11 @@ def _training_loss( "backbone_ce": base_per_token.detach(), "corrected_ce": corrected_per_token.detach(), } - # Report the corrected objective's per-position value; accuracy uses the - # corrected logits as well. + # Report the corrected objective's per-position value by default; + # accuracy uses the corrected logits as well. Held-out backbone-only + # evaluation flips the reporting branch without changing the mixed loss. + if self.report_backbone: + return loss_per_token, base_per_token, base_logits return loss_per_token, corrected_per_token, corrected_logits def _extra_training_loss( diff --git a/aspec/planning.py b/aspec/planning.py index 4b0d3d8..b83445d 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -427,6 +427,30 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]: if (isinstance(lambda_base, bool) or not isinstance(lambda_base, (int, float)) or not math.isfinite(lambda_base) or not 0.0 <= lambda_base <= 1.0): raise PlanCompileError("domino_lambda_base must be a finite number in [0, 1]") + lambda_start = _select(config, "training.domino_lambda_base_start", None) + lambda_ratio = _select(config, "training.domino_lambda_base_decay_ratio", None) + if (lambda_start is None) != (lambda_ratio is None): + raise PlanCompileError( + "domino_lambda_base_start and domino_lambda_base_decay_ratio " + "must be set together" + ) + if lambda_start is not None: + if (isinstance(lambda_start, bool) + or not isinstance(lambda_start, (int, float)) + or not math.isfinite(lambda_start) + or not 0.0 <= lambda_start <= 1.0): + raise PlanCompileError( + "domino_lambda_base_start must be a finite number in [0, 1]" + ) + if (isinstance(lambda_ratio, bool) + or not isinstance(lambda_ratio, (int, float)) + or not math.isfinite(lambda_ratio) + or lambda_ratio <= 0.0): + raise PlanCompileError( + "domino_lambda_base_decay_ratio must be a finite number > 0" + ) + if not isinstance(_select(config, "training.domino_eval_backbone", False), bool): + raise PlanCompileError("domino_eval_backbone must be boolean") target_layer_ids = raw.get("target_layer_ids") if not isinstance(target_layer_ids, list) or not target_layer_ids: @@ -1442,6 +1466,13 @@ def _workload( "ce_loss_alpha": 1.0, "l1_loss_alpha": 0.0, "lambda_base": _select(config, "training.domino_lambda_base", 0.3), + "lambda_base_start": _select( + config, "training.domino_lambda_base_start", None + ), + "lambda_base_decay_ratio": _select( + config, "training.domino_lambda_base_decay_ratio", None + ), + "eval_backbone": _select(config, "training.domino_eval_backbone", False), "shift_label": draft["domino_shift_label"], "pure_draft_prefix_len": draft["domino_pure_draft_prefix_len"], "correction_head": { diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 3738fab..28120ef 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -108,6 +108,12 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: topology["training_objective"]["lambda_base"] = getattr( args, "domino_lambda_base", 0.3 ) + topology["training_objective"]["lambda_base_start"] = getattr( + args, "domino_lambda_base_start", None + ) + topology["training_objective"]["lambda_base_decay_ratio"] = getattr( + args, "domino_lambda_base_decay_ratio", None + ) if "training_objective" in topology: fields = ("max_seq_length", "micro_batch_size", "draft_accumulation_steps", "max_grad_norm", "seed") @@ -228,6 +234,15 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec f"missing {missing}. Use continual_training=true for a " "weights-only new stage." ) + if ( + "lambda_base_start" not in objective + or "lambda_base_decay_ratio" not in objective + ) and getattr(args, "domino_lambda_base_start", None) is not None: + raise RuntimeError( + "Exact resume cannot add a lambda_base curriculum to a legacy " + "constant-lambda checkpoint; use continual_training=true for a " + "new phase." + ) if current_policy.get("dsa") is not None: # Chunk workspace is an execution bound, not the DSA objective. Preserve # every recorded semantic field while allowing a different memory budget. diff --git a/aspec/training/domino_trainer.py b/aspec/training/domino_trainer.py index d55e38b..3b42522 100644 --- a/aspec/training/domino_trainer.py +++ b/aspec/training/domino_trainer.py @@ -19,6 +19,50 @@ class DominoTrainer(DFlashTrainer): def __init__(self, args: Namespace): super().__init__(args) self.lambda_base = getattr(args, "domino_lambda_base", 0.3) + self.lambda_base_start = getattr(args, "domino_lambda_base_start", None) + self.lambda_base_decay_ratio = getattr( + args, "domino_lambda_base_decay_ratio", None + ) + self.eval_backbone = bool(getattr(args, "domino_eval_backbone", False)) + self.lambda_total_steps = int( + getattr(args, "lr_total_steps", None) + or getattr(args, "num_train_steps", 1) + ) + if (self.lambda_base_start is None) != (self.lambda_base_decay_ratio is None): + raise ValueError( + "domino_lambda_base_start and domino_lambda_base_decay_ratio " + "must be set together" + ) + if self.lambda_base_start is not None: + if not 0.0 <= float(self.lambda_base_start) <= 1.0: + raise ValueError("domino_lambda_base_start must be in [0, 1]") + if float(self.lambda_base_decay_ratio) <= 0.0: + raise ValueError("domino_lambda_base_decay_ratio must be > 0") + + def _schedule_total_steps(self) -> int: + args = getattr(self, "args", None) + value = getattr(args, "lr_total_steps", None) if args is not None else None + if not value: + value = getattr(args, "num_train_steps", None) if args is not None else None + if not value: + value = getattr(self, "lambda_total_steps", 1) + return max(1, int(value)) + + def _lambda_base_for_step(self, step: int) -> float: + """Linear backbone-weight schedule matching SpecForge's Domino trainer.""" + start = getattr(self, "lambda_base_start", None) + if start is None: + return float(getattr(self, "lambda_base", 0.3)) + decay_steps = max( + 1, + int( + self._schedule_total_steps() + * float(getattr(self, "lambda_base_decay_ratio", 1.0)) + ), + ) + progress = min(max(int(step), 0) / decay_steps, 1.0) + value = float(start) * (1.0 - progress) + return max(0.0, min(1.0, value)) def _build_draft_model(self, config): if config.block_size != self.block_size: @@ -44,7 +88,7 @@ def _build_training_wrapper(self, draft_model): loss_decay_gamma=self.loss_decay_gamma, ce_loss_alpha=self.ce_loss_alpha, l1_loss_alpha=self.l1_loss_alpha, - lambda_base=self.lambda_base, + lambda_base=self._lambda_base_for_step(0), deterministic_anchor_sampling=getattr( self.args, "deterministic_anchor_sampling", False ), @@ -67,11 +111,58 @@ def _aggregate_metrics(self, all_step_metrics, step, *, grad_norm=None): mixed = metrics.get("train/mixed_ce") if mixed is not None: metrics["train/avg_loss"] = mixed + metrics["train/lambda_base"] = self._lambda_base_for_step( + max(self.global_step - 1, 0) + ) return metrics + def _train_step(self, *args, **kwargs): + # The schedule is a function of the optimizer step, so every micro-batch + # in one step uses the same lambda. Exact resume restores global_step. + self.dflash.set_lambda_base(self._lambda_base_for_step(self.global_step)) + return super()._train_step(*args, **kwargs) + + def eval_from_cache(self): + # Evaluate with the lambda used by the most recently completed step, + # not the value the next step would use. + self.dflash.set_lambda_base( + self._lambda_base_for_step(max(self.global_step - 1, 0)) + ) + corrected = super().eval_from_cache() + if not self.eval_backbone or not corrected: + return corrected + # Reuse the cached Target features for a backbone-only reporting pass. + # The mixed loss and the correction head are unchanged by this flag. + self.dflash.report_backbone = True + try: + backbone = super().eval_from_cache() + finally: + self.dflash.report_backbone = False + return self._merge_backbone_eval_metrics(corrected, backbone) + + @staticmethod + def _merge_backbone_eval_metrics(corrected: dict, backbone: dict) -> dict: + merged = dict(corrected) + for key, value in backbone.items(): + if not key.startswith("eval/"): + continue + suffix = key[len("eval/"):] + if ( + suffix in {"avg_loss", "avg_acc"} + or "ploss_" in suffix + or suffix.startswith("acc_") + or "simulated_acc_len" in suffix + ): + merged[f"eval/backbone_{suffix}"] = value + return merged + def _aggregate_eval_metrics(self, all_step_metrics): metrics = super()._aggregate_eval_metrics(all_step_metrics) - mixed = metrics.get("eval/mixed_ce") - if mixed is not None: - metrics["eval/avg_loss"] = mixed + if not getattr(self.dflash, "report_backbone", False): + mixed = metrics.get("eval/mixed_ce") + if mixed is not None: + metrics["eval/avg_loss"] = mixed + metrics["eval/lambda_base"] = self._lambda_base_for_step( + max(self.global_step - 1, 0) + ) return metrics diff --git a/configs/recipes/domino_qwen3_8b.yaml b/configs/recipes/domino_qwen3_8b.yaml new file mode 100644 index 0000000..bf23a61 --- /dev/null +++ b/configs/recipes/domino_qwen3_8b.yaml @@ -0,0 +1,93 @@ +# Formal single-node Domino recipe: training, lambda_base curriculum, held-out +# backbone/corrected observations and exact recovery. Hyperparameters are a +# starting point, not a convergence or acceptance claim. +model: + target_model_path: ${oc.env:ASPEC_TARGET_MODEL,Qwen/Qwen3-8B} + native_context_length: 32768 + trust_remote_code: false + draft_model_config: aspec/config/domino_qwen3_8b_draft_config.json +dataset: + eval_interval: 100 + eval_micro_batch_size: 1 + train_data_path: ${oc.env:ASPEC_TRAIN_DATA} + eval_data_path: ${oc.env:ASPEC_EVAL_DATA} + chat_template: qwen3-instruct + prompt_key: conversations + min_loss_tokens: 32 + shuffle_dataset: true + dataset_seed: 42 +training: + attention_backend: flex_attention + draft_context_parallel_size: 1 + micro_batch_size: 1 + draft_accumulation_steps: 2 + learning_rate: 6e-4 + min_lr: 6e-5 + weight_decay: 0.0 + max_grad_norm: 1.0 + max_seq_length: 4096 + num_epochs: 3 + warmup_ratio: 0.04 + seed: 42 + continual_training: false + deterministic_anchor_sampling: false + training_num_gpus_per_node: 2 + training_num_nodes: 1 + placement_strategy: inference_first + fsdp_strategy: REPLICATE + fsdp_reduce_dtype: bfloat16 + compile_model: false + max_concurrent_batches: 1 + streaming_dispatch_with_training: true + pipeline_dispatch_with_training: true + prefetch_depth: 1 + save_interval: 100 + save_per_epoch: false + save_final_checkpoint: true + max_checkpoints: 2 + dflash_block_size: 16 + dflash_packing: false + dflash_num_target_layers: 5 + dflash_num_anchors: 128 + dflash_loss_objective: decay + dflash_loss_decay_gamma: 7.0 + dflash_ce_loss_alpha: 1.0 + dflash_l1_loss_alpha: 0.0 + domino_lambda_base: 0.3 + domino_lambda_base_start: 1.0 + domino_lambda_base_decay_ratio: 1.0 + domino_eval_backbone: true +inference: + inference_engine_type: sgl + store_last_hidden_states: false + target_prefill_memory_optimized: false + inference_num_gpus: 1 + inference_num_gpus_per_engine: 1 + inference_num_gpus_per_node: 3 + inference_fetch_batch: 8 + inference_buffer_threshold: 2 + max_sample_pool_size: 12 + inference_batch_size: 2 + sglang: + tp_size: 1 + mem_fraction_static: 0.5 +mooncake: + master_server_address: null + metadata_server: null + data_plane: store + protocol: tcp + enable_gpu_direct: false + allow_host_fallback: false + global_segment_size: 32GB + local_buffer_size: 8GB + host_buffer_size: 1GB + async_put_pool_size: 1 + enable_hard_pin: true + max_seq_len: 4096 + hidden_dim: 4096 +logging: + report_to: none +debug: + enable_perf_metrics: true +output_dir: outputs/domino-qwen3-8b +cache_dir: cache/domino-qwen3-8b diff --git a/docs/support.md b/docs/support.md index 49f4391..c5f82d0 100644 --- a/docs/support.md +++ b/docs/support.md @@ -12,7 +12,7 @@ and tests do not imply CPU training support. | DSpark / DFlash | Training, exact recovery, evaluation and export; Dense and selected MoE/attention structures | Objectives, feature bindings and assets remain algorithm-specific | | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | -| Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; `lambda_base` curriculum, export and consumer acceptance pending; see [scope](user-guide/domino.md) | +| Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; native export and consumer acceptance pending; see [scope](user-guide/domino.md) | | Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery; SDPA CP1, MBS1, non-packing, REPLICATE | No packing/CP, warmstart, export or consumer acceptance yet; see [scope](user-guide/medusa.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index b05d98c..e9b2b4c 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -68,10 +68,11 @@ on the retained DFlash block backbone and mixes corrected with backbone-only cross-entropy through `lambda_base`. Both documented label layouts are explicit: the unshifted DFlash convention and the official shifted convention (`shift_label=true`, `pure_draft_prefix_len=1`). Training now selects the family -through `aspec/config/domino_qwen3_8b_draft_config.json`; start from -`configs/examples/domino_qwen3_8b_4k_tcp.yaml` (FlexAttention, CP1, non-packing, Store/TCP). -Scope is dense full attention with padded inputs only. The `lambda_base` curriculum, -export and consumer validation remain pending. +through `aspec/config/domino_qwen3_8b_draft_config.json`; start from the +`domino` recipe (FlexAttention, CP1, non-packing, Store/TCP). The trainer supports a +linear `lambda_base` curriculum and held-out corrected/backbone per-position +observations. Scope is dense full attention with padded inputs only. Native export, +Target bindings and consumer/acceptance validation remain pending. ## EAGLE4: two stages diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 26981a9..f93843c 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -18,6 +18,9 @@ short unusable rows are excluded and counted. DSpark block 7 needs at least 14 s Use disjoint train/eval data and preserve identity across resume. Evidence distinguishes raw, processed, admitted, skipped and DP-truncated counts. Epoch-scoped work count is not unique-document count. +Dataset tokenization uses the top-level `num_proc` worker count (default 64). Set +`num_proc: 1` on hosts where fork-based multiprocessing is constrained or hangs. + ## Target-generated data The Qwen3-8B quality recipe generates answers from the actual Target, saving exact non-thinking prompt diff --git a/docs/user-guide/domino.md b/docs/user-guide/domino.md index 5811560..27ae9ab 100644 --- a/docs/user-guide/domino.md +++ b/docs/user-guide/domino.md @@ -4,19 +4,24 @@ This implements [ASpec #38](https://github.com/inclusionAI/ASpec/issues/38) thro `aspec.models.draft.domino.DominoDraftModel` and `aspec.models.domino.DominoModel`: a block-parallel DFlash backbone with a GRU-conditioned causal correction head. The ASpec training CLI now selects this -algorithm through `aspec/config/domino_qwen3_8b_draft_config.json`; the -[example workflow](../../configs/examples/domino_qwen3_8b_4k_tcp.yaml) runs one +algorithm through `aspec/config/domino_qwen3_8b_draft_config.json`; the maintained +[recipe](../../configs/recipes/domino_qwen3_8b.yaml) and the +[example workflow](../../configs/examples/domino_qwen3_8b_4k_tcp.yaml) both run one Target GPU plus two Draft GPUs at 4K (three GPUs total). ## Training selection - Draft config: `aspec/config/domino_qwen3_8b_draft_config.json` (Qwen3-8B, `block_size=16`, `shift_label=true`, `pure_draft_prefix_len=1`). -- Example: `configs/examples/domino_qwen3_8b_4k_tcp.yaml` (FlexAttention, CP1, MBS1, - non-packing, REPLICATE, Store/TCP transport). -- The trainer owns a fixed per-run `training.domino_lambda_base`; the curriculum - schedule is not implemented yet. Exact recovery records `lambda_base`, the label - layout and the correction-head dimensions. +- Recipe: `configs/recipes/domino_qwen3_8b.yaml` (FlexAttention, CP1, non-packing, + REPLICATE, Store/TCP transport, held-out eval and the linear `lambda_base` + curriculum). The example remains a shorter smoke-test workflow. +- `training.domino_lambda_base` is the constant fallback. Set both + `training.domino_lambda_base_start` and `training.domino_lambda_base_decay_ratio` + to apply the reference linear schedule. Exact recovery records the constant, + schedule, label layout and correction-head dimensions. +- Set `dataset.eval_data_path` and `training.domino_eval_backbone=true` to report + held-out corrected and backbone-only observations per prediction position. - The supported path fails closed outside dense full attention, padded inputs, `attention_backend=flex_attention`, `draft_context_parallel_size=1` and `dflash_packing=false`. The CUDA FlexAttention mask and the CPU dense fallback @@ -34,7 +39,10 @@ CUDA_VISIBLE_DEVICES=0 aspec target \ model.target_model_path=/path/to/Qwen3-8B ``` -Compile the example, then train with two Draft GPUs (GPUs 1 and 2): +For a maintained recipe with held-out data and the curriculum, set +`ASPEC_TRAIN_DATA` and `ASPEC_EVAL_DATA` and use +`configs/recipes/domino_qwen3_8b.yaml`. Compile the example, then train with two +Draft GPUs (GPUs 1 and 2): ```bash aspec --config configs/examples/domino_qwen3_8b_4k_tcp.yaml --dry-run \ @@ -63,18 +71,18 @@ CUDA_VISIBLE_DEVICES=1,2 RAY_ADDRESS=local \ output_dir=outputs/domino_qwen3_8b_4k_tcp_resumed ``` -The example is a short functional smoke-test recipe, not evidence that GPU execution -has been validated. Only CPU and Plan checks have been reported; the GPU run remains -pending. It uses the bundled conversations and a short step budget, not convergence, -throughput or decoding acceptance. Supply licensed data and a full learning-rate -horizon for a real run. +The example is a short functional smoke test, not convergence, throughput or +decoding-acceptance evidence. A GPU run with one Target plus two Draft GPUs has been +reported for training and exact resume; it does not establish model quality. Supply +licensed data, a full learning-rate horizon and the recipe for a real run. ## Validation status Plan compilation, registration, trainer construction, checkpoint topology and the -CPU numerical contracts are covered by `tests/cpu`. The public CI lane is CPU-only; -GPU end-to-end training and exact resume still need a hardware run before claiming -throughput or quality. +CPU numerical contracts are covered by `tests/cpu`. A GPU run with one Target and two +Draft GPUs verified 8-step training and step-4 exact resume: losses, model tensors, +optimizer state and RNG state matched the uninterrupted run. The public CI lane is +CPU-only; no throughput, quality or decoding-acceptance claim is made here. ## Reference and selected scope @@ -135,8 +143,10 @@ loss = (1 - lambda_base) * CE(corrected logits, labels) + lambda_base * CE(backbone logits, labels) ``` -`lambda_base` is an explicit per-step value in `[0, 1]`; use -`model.set_lambda_base(value)` from a trainer-owned curriculum. `lambda_base=1` +With both curriculum fields set, the trainer sets the value at optimizer step `s` to +`start * max(0, 1 - s / max(1, int(lr_total_steps * decay_ratio)))`, matching the +reference SpecForge Domino schedule. Otherwise it stays at `domino_lambda_base`. +`lambda_base=1` removes the correction contribution and keeps the selected label layout (the correction head receives no useful gradient). It equals the plain DFlash objective only for the unshifted layout (`shift_label=false`) with otherwise matching @@ -145,7 +155,14 @@ block masks, label bounds, `loss_mask` and DFlash-style exponential depth decay reused; `ce_loss_alpha=1` and `l1_loss_alpha=0` are enforced. Reporting accuracy and per-position loss use the corrected logits. The trainer reports `mixed_ce` as `train/avg_loss`/`eval/avg_loss` (the objective actually used for backward) and exposes `mixed_ce`, `backbone_ce` and -`corrected_ce` as separate token-pooled component curves. +`corrected_ce` as separate token-pooled component curves. With +`training.domino_eval_backbone=true`, each held-out evaluation also runs a +backbone-only reporting pass over the same cached Target features: +`eval/ploss_i`/`eval/acc_i` remain corrected, while +`eval/backbone_ploss_i`/`eval/backbone_acc_i` (plus +`eval/backbone_avg_loss`, `eval/backbone_avg_acc` and +`eval/backbone_simulated_acc_len`) describe the backbone-only branch. The mixed +objective and the trained correction head are unchanged by this reporting mode. ## Minimal numerical use @@ -189,7 +206,7 @@ handling, empty supervision, and frozen Target/embedding behavior. ## Remaining work -The `lambda_base` curriculum schedule, a maintained recipe catalog entry, native -export with strict reloading, Target bindings, proposal consumers and held-out -backbone-vs-corrected observations remain pending. CPU numerical and Plan checks do -not establish Qwen3-8B training quality, GPU throughput or decoding acceptance. +Native export with strict reloading, Target bindings, proposal consumers and +acceptance observations remain pending. The next planned increment is the aligned +SpecForge-vs-ASpec comparison. CPU numerical and Plan checks do not establish +Qwen3-8B training quality, GPU throughput or decoding acceptance. diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index 68cb315..6f065f1 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -17,6 +17,7 @@ connections on your hardware before training. | `quickstart` | Qwen3-8B / DSpark | 1 node, 2 GPUs: 1T + 1D; 1K | Try training, recovery and export with bundled data | | `dspark` | Qwen3-8B / Dense DSpark | 1 node, 8 GPUs: 4T + 4D; 4K | Train on your data with packing and NVLink | | `dflash` | Qwen3-8B / DFlash | 1 node, 3 GPUs: 1T + 2D; 4K | Train a block-parallel DFlash Draft | +| `domino` | Qwen3-8B / Domino | 1 node, 3 GPUs: 1T + 2D; 4K | Train a DFlash backbone with a GRU causal correction head | | `eagle3` | Qwen3-8B / EAGLE3 | 1 node, 3 GPUs: 1T + 2D; 4K | Train a recurrent EAGLE3 Draft | | `eagle4` | Qwen3-8B / EAGLE4 | 1 node, 3 GPUs: 1T + 2D; 4K; sequential stages | Train a Draft, then candidate retrieval projections | | `dmtp` | Qwen3.5-9B / MTP-conditioned DSpark | 1 node, 8 GPUs: 4T + 4D; 4K, CP1 | Train a Draft conditioned on the frozen native predictor; native assets | diff --git a/tests/cpu/test_domino.py b/tests/cpu/test_domino.py index 14a819e..e67d7a7 100644 --- a/tests/cpu/test_domino.py +++ b/tests/cpu/test_domino.py @@ -79,7 +79,7 @@ def run_and_capture(model, inputs): return result, captured[0] -def reference_objective(model, draft_hidden, inputs): +def reference_objective(model, draft_hidden, inputs, *, report="corrected"): """Dense Domino equations with explicit label/anchor/mask alignment.""" draft_model = model.draft_model anchors = inputs["injected_anchors"] @@ -150,8 +150,9 @@ def reference_objective(model, draft_hidden, inputs): binary = weights.reshape(-1) count = binary.reshape(-1, anchors.shape[1], block_size).sum(dim=(0, 1)) + reported_ce = base_ce if report == "backbone" else corrected_ce per_position = ( - corrected_ce.reshape(-1, anchors.shape[1], block_size) * weights + reported_ce.reshape(-1, anchors.shape[1], block_size) * weights ).sum(dim=(0, 1)) / count.clamp(min=1.0) denominator = flat_weights.sum().clamp_min(1e-6) components = { @@ -161,7 +162,7 @@ def reference_objective(model, draft_hidden, inputs): } return ( loss, - corrected_ce.reshape(-1, anchors.shape[1], block_size), + reported_ce.reshape(-1, anchors.shape[1], block_size), per_position, components, ) @@ -284,6 +285,23 @@ def test_lambda_one_recovers_the_backbone_only_objective(): assert torch.count_nonzero(model.draft_model.context_proj.weight.grad) > 0 +def test_backbone_reporting_switches_per_position_metrics_only(): + model = make_model(lambda_base=0.3, loss_decay_gamma=0.0) + inputs = make_inputs() + corrected, captured = run_and_capture(model, inputs) + model.report_backbone = True + backbone, _ = run_and_capture(model, inputs) + expected_loss, _, expected_per_position, _ = reference_objective( + model, captured, inputs, report="backbone" + ) + torch.testing.assert_close(backbone[0], corrected[0]) + torch.testing.assert_close(backbone[0], expected_loss, rtol=2e-5, atol=2e-6) + torch.testing.assert_close( + backbone[2], expected_per_position, rtol=2e-5, atol=2e-6 + ) + assert not torch.equal(corrected[2], backbone[2]) + + def test_two_row_layout_matches_reference_and_backward(): actual = make_model(shift_label=True, lambda_base=0.4) reference = copy.deepcopy(actual) diff --git a/tests/cpu/test_domino_integration.py b/tests/cpu/test_domino_integration.py index b00baac..ec66909 100644 --- a/tests/cpu/test_domino_integration.py +++ b/tests/cpu/test_domino_integration.py @@ -181,6 +181,88 @@ def test_trainer_aggregates_real_mixed_component(monkeypatch): assert eval_output["eval/avg_loss"] == pytest.approx(11 / 4) +def test_plan_compiles_with_domino_lambda_curriculum(tmp_path): + config, args = _config( + tmp_path, + "training.domino_lambda_base_start=1.0", + "training.domino_lambda_base_decay_ratio=1.0", + "training.domino_eval_backbone=true", + ) + plan = compile_execution_plan( + config, config_path=str(_EXAMPLE), cli_args=args + ).to_dict() + objective = plan["workload"]["objective"] + assert objective["lambda_base_start"] == 1.0 + assert objective["lambda_base_decay_ratio"] == 1.0 + assert objective["eval_backbone"] is True + + +@pytest.mark.parametrize( + "overrides", + [ + ("training.domino_lambda_base_start=1.0",), + ("training.domino_lambda_base_decay_ratio=1.0",), + ( + "training.domino_lambda_base_start=2.0", + "training.domino_lambda_base_decay_ratio=1.0", + ), + ( + "training.domino_lambda_base_start=1.0", + "training.domino_lambda_base_decay_ratio=0.0", + ), + ], +) +def test_plan_rejects_invalid_domino_lambda_curriculum(tmp_path, overrides): + config, args = _config(tmp_path, *overrides) + with pytest.raises(PlanCompileError): + compile_execution_plan(config, config_path=str(_EXAMPLE), cli_args=args) + + +def test_lambda_curriculum_matches_reference_linear_decay(): + trainer = object.__new__(DominoTrainer) + trainer.lambda_base = 0.3 + trainer.lambda_base_start = 1.0 + trainer.lambda_base_decay_ratio = 1.0 + trainer.lambda_total_steps = 8 + assert trainer._lambda_base_for_step(0) == pytest.approx(1.0) + assert trainer._lambda_base_for_step(2) == pytest.approx(0.75) + assert trainer._lambda_base_for_step(4) == pytest.approx(0.5) + assert trainer._lambda_base_for_step(8) == pytest.approx(0.0) + assert trainer._lambda_base_for_step(99) == pytest.approx(0.0) + + trainer.lambda_base_start = None + trainer.lambda_base_decay_ratio = None + assert trainer._lambda_base_for_step(4) == pytest.approx(0.3) + + +def test_backbone_eval_metrics_merge_with_prefix(): + corrected = { + "eval/ploss_0": 1.0, + "eval/acc_0": 0.5, + "eval/avg_loss": 2.0, + "eval/avg_acc": 0.25, + "eval/mixed_ce": 2.0, + "eval/backbone_ce": 3.0, + } + backbone = { + "eval/ploss_0": 9.0, + "eval/acc_0": 0.1, + "eval/avg_loss": 8.0, + "eval/avg_acc": 0.2, + "eval/simulated_acc_len": 0.3, + "eval/mixed_ce": 2.0, + } + merged = DominoTrainer._merge_backbone_eval_metrics(corrected, backbone) + assert merged["eval/ploss_0"] == 1.0 + assert merged["eval/backbone_ploss_0"] == 9.0 + assert merged["eval/backbone_acc_0"] == 0.1 + assert merged["eval/backbone_avg_loss"] == 8.0 + assert merged["eval/backbone_avg_acc"] == 0.2 + assert merged["eval/backbone_simulated_acc_len"] == 0.3 + assert merged["eval/mixed_ce"] == 2.0 + assert "eval/backbone_mixed_ce" not in merged + + def test_plan_rejects_explicit_cudnn_gqa_mode(tmp_path): raw = json.loads(_DRAFT_JSON.read_text(encoding="utf-8")) raw["cudnn_gqa_mode"] = "native" @@ -208,7 +290,9 @@ def _trainer_args(): dflash_block_size=5, dflash_num_anchors=2, dflash_num_target_layers=1, dflash_loss_objective="decay", dflash_dpace_alpha=0.5, dflash_loss_decay_gamma=7.0, dflash_ce_loss_alpha=1.0, dflash_l1_loss_alpha=0.0, - domino_lambda_base=0.3, anchor_sampling_scope="row", + domino_lambda_base=0.3, domino_lambda_base_start=None, + domino_lambda_base_decay_ratio=None, domino_eval_backbone=False, + anchor_sampling_scope="row", lr_total_steps=8, num_train_steps=8, deterministic_anchor_sampling=False, max_seq_length=128, micro_batch_size=1, draft_accumulation_steps=1, max_grad_norm=1.0, seed=42, draft_context_parallel_size=1, training_num_nodes=1, training_num_gpus_per_node=2, @@ -291,3 +375,36 @@ def test_checkpoint_topology_pins_domino_objective(): changed["training_topology"]["training_objective"]["lambda_base"] = 0.5 with pytest.raises(RuntimeError): _validate_exact_resume_topology(args, changed) + + +def test_exact_resume_rejects_adding_curriculum_to_legacy_checkpoint(): + args = _trainer_args() + legacy = _logical_training_topology(args, 2) + legacy["training_objective"].pop("lambda_base_start", None) + legacy["training_objective"].pop("lambda_base_decay_ratio", None) + metadata = {"world_size": 2, "training_topology": json.loads(json.dumps(legacy))} + _validate_exact_resume_topology(args, metadata) + + args.domino_lambda_base_start = 1.0 + args.domino_lambda_base_decay_ratio = 1.0 + with pytest.raises(RuntimeError, match="lambda_base curriculum"): + _validate_exact_resume_topology(args, metadata) + + +def test_checkpoint_topology_pins_domino_lambda_curriculum(): + args = _trainer_args() + args.domino_lambda_base_start = 1.0 + args.domino_lambda_base_decay_ratio = 1.0 + current = _logical_training_topology(args, 2) + objective = current["training_objective"] + assert objective["lambda_base_start"] == 1.0 + assert objective["lambda_base_decay_ratio"] == 1.0 + + metadata = {"world_size": 2, "training_topology": json.loads(json.dumps(current))} + _validate_exact_resume_topology(args, metadata) + changed = json.loads(json.dumps(metadata)) + changed["training_topology"]["training_objective"][ + "lambda_base_decay_ratio" + ] = 0.5 + with pytest.raises(RuntimeError): + _validate_exact_resume_topology(args, changed) diff --git a/tests/cpu/test_release_recipes.py b/tests/cpu/test_release_recipes.py index 59674f6..4deb160 100644 --- a/tests/cpu/test_release_recipes.py +++ b/tests/cpu/test_release_recipes.py @@ -83,7 +83,7 @@ def test_qwen8b_recipes_plan_without_target_override(name, monkeypatch): def test_catalog_exactly_covers_formal_files(): entries = recipes.catalog() - assert len(entries) == 13 + assert len(entries) == 14 declared = {p for entry in entries.values() for p in entry["configs"]} assert declared == { str(p.relative_to(recipes.ROOT)) for p in (recipes.ROOT / "configs/recipes").glob("*.yaml") diff --git a/tools/recipe_catalog.json b/tools/recipe_catalog.json index 6c0873f..c449cc0 100644 --- a/tools/recipe_catalog.json +++ b/tools/recipe_catalog.json @@ -32,6 +32,17 @@ "draft_gpus": 2, "native_only_export": false }, + "domino": { + "configs": [ + "configs/recipes/domino_qwen3_8b.yaml" + ], + "model": "Qwen3-8B", + "nodes": 1, + "gpus": 3, + "target_gpus": 1, + "draft_gpus": 2, + "native_only_export": true + }, "eagle3": { "configs": [ "configs/recipes/qwen3_8b_eagle3.yaml" From 6c958761e9ad981068bc2df04d295f5ce2b69847 Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:49:37 +0800 Subject: [PATCH 2/5] fix(domino): pin curriculum horizon for exact resume --- aspec/training/checkpoint.py | 21 +++++++++++++++++++++ tests/cpu/test_domino_integration.py | 13 +++++++++++++ 2 files changed, 34 insertions(+) diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 28120ef..3b9fab2 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -42,6 +42,15 @@ RNG_STATE_SCHEMA_VERSION = "aspec.rng-state/v1" +def _domino_lambda_horizon(args: Any) -> int | None: + value = getattr(args, "lr_total_steps", None) or getattr( + args, "num_train_steps", None + ) + if not value: + return None + return int(value) + + def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: """Return resume-relevant logical topology without physical host identity.""" @@ -114,6 +123,14 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: topology["training_objective"]["lambda_base_decay_ratio"] = getattr( args, "domino_lambda_base_decay_ratio", None ) + if getattr(args, "domino_lambda_base_start", None) is not None: + horizon = _domino_lambda_horizon(args) + if horizon is None: + raise RuntimeError( + "Domino lambda_base curriculum requires an effective " + "lr_total_steps or num_train_steps horizon" + ) + topology["training_objective"]["lambda_base_total_steps"] = horizon if "training_objective" in topology: fields = ("max_seq_length", "micro_batch_size", "draft_accumulation_steps", "max_grad_norm", "seed") @@ -224,6 +241,10 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec key for key in ("lambda_base", "block_size", "loss", "loss_decay_gamma") if key not in objective ] + if objective.get("lambda_base_start") is not None and ( + "lambda_base_total_steps" not in objective + ): + missing.append("lambda_base_total_steps") missing += [ key for key in ("shift_label", "pure_draft_prefix_len") if key not in semantics diff --git a/tests/cpu/test_domino_integration.py b/tests/cpu/test_domino_integration.py index ec66909..38b6936 100644 --- a/tests/cpu/test_domino_integration.py +++ b/tests/cpu/test_domino_integration.py @@ -399,6 +399,7 @@ def test_checkpoint_topology_pins_domino_lambda_curriculum(): objective = current["training_objective"] assert objective["lambda_base_start"] == 1.0 assert objective["lambda_base_decay_ratio"] == 1.0 + assert objective["lambda_base_total_steps"] == 8 metadata = {"world_size": 2, "training_topology": json.loads(json.dumps(current))} _validate_exact_resume_topology(args, metadata) @@ -408,3 +409,15 @@ def test_checkpoint_topology_pins_domino_lambda_curriculum(): ] = 0.5 with pytest.raises(RuntimeError): _validate_exact_resume_topology(args, changed) + + args.lr_total_steps = 16 + with pytest.raises(RuntimeError, match="lambda_base_total_steps"): + _validate_exact_resume_topology(args, metadata) + + missing_horizon = json.loads(json.dumps(metadata)) + del missing_horizon["training_topology"]["training_objective"][ + "lambda_base_total_steps" + ] + args.lr_total_steps = 8 + with pytest.raises(RuntimeError, match="lambda_base_total_steps"): + _validate_exact_resume_topology(args, missing_horizon) From 9e48af3611fd9f1fdd7f89d2ec52268db5aea802 Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:49:37 +0800 Subject: [PATCH 3/5] docs(domino): keep curriculum config as example until export validation CodeRabbit correctly noted that catalog promotion needs a validated native export/strict reload. Keep the held-out/curriculum configuration under configs/examples until the next PR adds that check. --- .../domino_qwen3_8b_eval.yaml} | 0 docs/user-guide/algorithms.md | 10 +++---- docs/user-guide/domino.md | 28 ++++++++++--------- docs/user-guide/recipes.md | 1 - tests/cpu/test_release_recipes.py | 2 +- tools/recipe_catalog.json | 11 -------- 6 files changed, 21 insertions(+), 31 deletions(-) rename configs/{recipes/domino_qwen3_8b.yaml => examples/domino_qwen3_8b_eval.yaml} (100%) diff --git a/configs/recipes/domino_qwen3_8b.yaml b/configs/examples/domino_qwen3_8b_eval.yaml similarity index 100% rename from configs/recipes/domino_qwen3_8b.yaml rename to configs/examples/domino_qwen3_8b_eval.yaml diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index e9b2b4c..9416f91 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -68,11 +68,11 @@ on the retained DFlash block backbone and mixes corrected with backbone-only cross-entropy through `lambda_base`. Both documented label layouts are explicit: the unshifted DFlash convention and the official shifted convention (`shift_label=true`, `pure_draft_prefix_len=1`). Training now selects the family -through `aspec/config/domino_qwen3_8b_draft_config.json`; start from the -`domino` recipe (FlexAttention, CP1, non-packing, Store/TCP). The trainer supports a -linear `lambda_base` curriculum and held-out corrected/backbone per-position -observations. Scope is dense full attention with padded inputs only. Native export, -Target bindings and consumer/acceptance validation remain pending. +through `aspec/config/domino_qwen3_8b_draft_config.json`; start from the Domino +examples (FlexAttention, CP1, non-packing, Store/TCP). The trainer supports a linear +`lambda_base` curriculum and held-out corrected/backbone per-position observations. +Scope is dense full attention with padded inputs only. Native export, recipe catalog +promotion, Target bindings and consumer/acceptance validation remain pending. ## EAGLE4: two stages diff --git a/docs/user-guide/domino.md b/docs/user-guide/domino.md index 27ae9ab..761eba6 100644 --- a/docs/user-guide/domino.md +++ b/docs/user-guide/domino.md @@ -4,22 +4,24 @@ This implements [ASpec #38](https://github.com/inclusionAI/ASpec/issues/38) thro `aspec.models.draft.domino.DominoDraftModel` and `aspec.models.domino.DominoModel`: a block-parallel DFlash backbone with a GRU-conditioned causal correction head. The ASpec training CLI now selects this -algorithm through `aspec/config/domino_qwen3_8b_draft_config.json`; the maintained -[recipe](../../configs/recipes/domino_qwen3_8b.yaml) and the -[example workflow](../../configs/examples/domino_qwen3_8b_4k_tcp.yaml) both run one -Target GPU plus two Draft GPUs at 4K (three GPUs total). +algorithm through `aspec/config/domino_qwen3_8b_draft_config.json`. The +[supplementary held-out example](../../configs/examples/domino_qwen3_8b_eval.yaml) +and the [smoke workflow](../../configs/examples/domino_qwen3_8b_4k_tcp.yaml) both run +one Target GPU plus two Draft GPUs at 4K (three GPUs total). ## Training selection - Draft config: `aspec/config/domino_qwen3_8b_draft_config.json` (Qwen3-8B, `block_size=16`, `shift_label=true`, `pure_draft_prefix_len=1`). -- Recipe: `configs/recipes/domino_qwen3_8b.yaml` (FlexAttention, CP1, non-packing, - REPLICATE, Store/TCP transport, held-out eval and the linear `lambda_base` - curriculum). The example remains a shorter smoke-test workflow. +- Held-out example: `configs/examples/domino_qwen3_8b_eval.yaml` (FlexAttention, + CP1, non-packing, REPLICATE, Store/TCP transport, held-out eval and the linear + `lambda_base` curriculum). Catalog promotion waits for native export/strict + reload validation; `domino_qwen3_8b_4k_tcp.yaml` remains a shorter smoke test. - `training.domino_lambda_base` is the constant fallback. Set both `training.domino_lambda_base_start` and `training.domino_lambda_base_decay_ratio` to apply the reference linear schedule. Exact recovery records the constant, - schedule, label layout and correction-head dimensions. + schedule, effective curriculum horizon, label layout and correction-head + dimensions. - Set `dataset.eval_data_path` and `training.domino_eval_backbone=true` to report held-out corrected and backbone-only observations per prediction position. - The supported path fails closed outside dense full attention, padded inputs, @@ -39,10 +41,9 @@ CUDA_VISIBLE_DEVICES=0 aspec target \ model.target_model_path=/path/to/Qwen3-8B ``` -For a maintained recipe with held-out data and the curriculum, set -`ASPEC_TRAIN_DATA` and `ASPEC_EVAL_DATA` and use -`configs/recipes/domino_qwen3_8b.yaml`. Compile the example, then train with two -Draft GPUs (GPUs 1 and 2): +For held-out data and the curriculum, set `ASPEC_TRAIN_DATA` and `ASPEC_EVAL_DATA` +and use `configs/examples/domino_qwen3_8b_eval.yaml`. Compile the smoke example, +then train with two Draft GPUs (GPUs 1 and 2): ```bash aspec --config configs/examples/domino_qwen3_8b_4k_tcp.yaml --dry-run \ @@ -207,6 +208,7 @@ handling, empty supervision, and frozen Target/embedding behavior. ## Remaining work Native export with strict reloading, Target bindings, proposal consumers and -acceptance observations remain pending. The next planned increment is the aligned +acceptance observations remain pending. Recipe catalog promotion also waits for the +native export/strict reload check. The next planned increment is the aligned SpecForge-vs-ASpec comparison. CPU numerical and Plan checks do not establish Qwen3-8B training quality, GPU throughput or decoding acceptance. diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index 6f065f1..68cb315 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -17,7 +17,6 @@ connections on your hardware before training. | `quickstart` | Qwen3-8B / DSpark | 1 node, 2 GPUs: 1T + 1D; 1K | Try training, recovery and export with bundled data | | `dspark` | Qwen3-8B / Dense DSpark | 1 node, 8 GPUs: 4T + 4D; 4K | Train on your data with packing and NVLink | | `dflash` | Qwen3-8B / DFlash | 1 node, 3 GPUs: 1T + 2D; 4K | Train a block-parallel DFlash Draft | -| `domino` | Qwen3-8B / Domino | 1 node, 3 GPUs: 1T + 2D; 4K | Train a DFlash backbone with a GRU causal correction head | | `eagle3` | Qwen3-8B / EAGLE3 | 1 node, 3 GPUs: 1T + 2D; 4K | Train a recurrent EAGLE3 Draft | | `eagle4` | Qwen3-8B / EAGLE4 | 1 node, 3 GPUs: 1T + 2D; 4K; sequential stages | Train a Draft, then candidate retrieval projections | | `dmtp` | Qwen3.5-9B / MTP-conditioned DSpark | 1 node, 8 GPUs: 4T + 4D; 4K, CP1 | Train a Draft conditioned on the frozen native predictor; native assets | diff --git a/tests/cpu/test_release_recipes.py b/tests/cpu/test_release_recipes.py index 4deb160..59674f6 100644 --- a/tests/cpu/test_release_recipes.py +++ b/tests/cpu/test_release_recipes.py @@ -83,7 +83,7 @@ def test_qwen8b_recipes_plan_without_target_override(name, monkeypatch): def test_catalog_exactly_covers_formal_files(): entries = recipes.catalog() - assert len(entries) == 14 + assert len(entries) == 13 declared = {p for entry in entries.values() for p in entry["configs"]} assert declared == { str(p.relative_to(recipes.ROOT)) for p in (recipes.ROOT / "configs/recipes").glob("*.yaml") diff --git a/tools/recipe_catalog.json b/tools/recipe_catalog.json index c449cc0..6c0873f 100644 --- a/tools/recipe_catalog.json +++ b/tools/recipe_catalog.json @@ -32,17 +32,6 @@ "draft_gpus": 2, "native_only_export": false }, - "domino": { - "configs": [ - "configs/recipes/domino_qwen3_8b.yaml" - ], - "model": "Qwen3-8B", - "nodes": 1, - "gpus": 3, - "target_gpus": 1, - "draft_gpus": 2, - "native_only_export": true - }, "eagle3": { "configs": [ "configs/recipes/qwen3_8b_eagle3.yaml" From e9ea40f0418f6dc9f36821a91db1d4645bf30878 Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:04:48 +0800 Subject: [PATCH 4/5] fix(domino): align lambda counter and eval anchors with SpecForge Advance lambda_base per microbatch using the 1-based reference counter, and compute corrected/backbone per-position metrics in the same forward with the same sampled anchors instead of a second eval pass. Add accumulation and anchor-slot regressions. --- aspec/models/dflash.py | 25 ++++++ aspec/models/domino.py | 46 +++++++++-- aspec/training/dflash_trainer.py | 4 + aspec/training/domino_trainer.py | 115 ++++++++++++++++----------- docs/user-guide/domino.md | 19 +++-- tests/cpu/test_domino.py | 25 +++--- tests/cpu/test_domino_integration.py | 90 +++++++++++++-------- 7 files changed, 218 insertions(+), 106 deletions(-) diff --git a/aspec/models/dflash.py b/aspec/models/dflash.py index 7c4bf17..0cc0302 100644 --- a/aspec/models/dflash.py +++ b/aspec/models/dflash.py @@ -479,6 +479,8 @@ def _compute_logits( return self.draft_model.lm_head(draft_hidden) return F.linear(draft_hidden, lm_head_weight) + collect_extra_position_metrics = False + def _extra_training_loss( self, draft_hidden: torch.Tensor, @@ -488,6 +490,18 @@ def _extra_training_loss( ) -> tuple[torch.Tensor, dict]: return logits.new_zeros(()), {} + def _extra_position_metrics( + self, + *, + bsz: int, + n_blocks: int, + target_ids: torch.Tensor, + binary_weights: torch.Tensor, + count_per_position: torch.Tensor, + ) -> dict: + """Algorithm-specific additional per-position metrics for evaluation.""" + return {} + def _training_loss( self, *, @@ -1368,6 +1382,17 @@ def forward( dim=(0, 1) ) / count_per_pos + if self.collect_extra_position_metrics: + self._last_extra_position_metrics = self._extra_position_metrics( + bsz=bsz, + n_blocks=n_blocks, + target_ids=target_ids, + binary_weights=binary_weights, + count_per_position=count_per_position, + ) + else: + self._last_extra_position_metrics = {} + return ( loss, accuracy, diff --git a/aspec/models/domino.py b/aspec/models/domino.py index fabbf84..66d2566 100644 --- a/aspec/models/domino.py +++ b/aspec/models/domino.py @@ -48,9 +48,10 @@ def __init__(self, *args, lambda_base=0.0, **kwargs): # anchor slot. Both match the two documented Domino layouts. self.label_start = 1 if self.draft_model.shift_label else 0 self.prediction_start = 1 - self.label_start - # Held-out evaluation can report the backbone-only branch without - # changing the mixed objective used for training. - self.report_backbone = False + # Held-out evaluation can collect backbone-only per-position metrics + # from the same forward and anchors as the corrected branch. + self.collect_extra_position_metrics = False + self._last_backbone_report = None self.set_lambda_base(lambda_base) def _prepare_draft_prefix( @@ -146,13 +147,42 @@ def _training_loss( "backbone_ce": base_per_token.detach(), "corrected_ce": corrected_per_token.detach(), } - # Report the corrected objective's per-position value by default; - # accuracy uses the corrected logits as well. Held-out backbone-only - # evaluation flips the reporting branch without changing the mixed loss. - if self.report_backbone: - return loss_per_token, base_per_token, base_logits + if self.collect_extra_position_metrics: + # Keep the same-forward backbone values for evaluation; the mixed + # loss and corrected reporting branch are unchanged. + self._last_backbone_report = (base_per_token.detach(), base_logits.detach()) + # Report the corrected objective's per-position value; accuracy uses the + # corrected logits as well. return loss_per_token, corrected_per_token, corrected_logits + def _extra_position_metrics( + self, + *, + bsz, + n_blocks, + target_ids, + binary_weights, + count_per_position, + ): + if not self.collect_extra_position_metrics or self._last_backbone_report is None: + return {} + base_per_token, base_logits = self._last_backbone_report + block_size = self.block_size + base_loss_pp = base_per_token.reshape(bsz, n_blocks, block_size) + vocab_size = base_logits.size(-1) + base_pred = base_logits.reshape(-1, vocab_size).argmax(-1) + correct = (base_pred.view(bsz, n_blocks, block_size) == target_ids) & ( + binary_weights > 0.5 + ) + safe_count = count_per_position.clamp(min=1.0) + loss_pp = (base_loss_pp * binary_weights).sum(dim=(0, 1)) / safe_count + acc_pp = correct.float().sum(dim=(0, 1)) / safe_count + return { + "backbone_loss_pp": loss_pp.detach(), + "backbone_acc_pp": acc_pp.detach(), + "backbone_count_pp": count_per_position.detach(), + } + def _extra_training_loss( self, draft_hidden: torch.Tensor, diff --git a/aspec/training/dflash_trainer.py b/aspec/training/dflash_trainer.py index fbd5a28..447c680 100644 --- a/aspec/training/dflash_trainer.py +++ b/aspec/training/dflash_trainer.py @@ -562,11 +562,15 @@ def eval_forward(self, batch: dict) -> dict: loss_components, _, ) = self._forward(batch) + extra_position_metrics = getattr( + self.dflash, "_last_extra_position_metrics", {} + ) or {} return { "loss_pp": loss_per_position.detach(), "acc_pp": acc_per_position.detach(), "count_pp": count_per_position.detach(), **loss_components, + **extra_position_metrics, } def _reduce_position_metrics( diff --git a/aspec/training/domino_trainer.py b/aspec/training/domino_trainer.py index 3b42522..28471c5 100644 --- a/aspec/training/domino_trainer.py +++ b/aspec/training/domino_trainer.py @@ -48,8 +48,17 @@ def _schedule_total_steps(self) -> int: value = getattr(self, "lambda_total_steps", 1) return max(1, int(value)) - def _lambda_base_for_step(self, step: int) -> float: - """Linear backbone-weight schedule matching SpecForge's Domino trainer.""" + def _accumulation_steps(self) -> int: + args = getattr(self, "args", None) + value = getattr(args, "draft_accumulation_steps", 1) if args is not None else 1 + return max(1, int(value)) + + def _lambda_base_for_micro_step(self, micro_step: int) -> float: + """Linear schedule using SpecForge's 1-based microbatch counter. + + ``total_steps`` is measured in optimizer updates; with accumulation > 1 + the counter advances per microbatch, matching the reference trainer. + """ start = getattr(self, "lambda_base_start", None) if start is None: return float(getattr(self, "lambda_base", 0.3)) @@ -60,7 +69,7 @@ def _lambda_base_for_step(self, step: int) -> float: * float(getattr(self, "lambda_base_decay_ratio", 1.0)) ), ) - progress = min(max(int(step), 0) / decay_steps, 1.0) + progress = min(max(int(micro_step), 1) / decay_steps, 1.0) value = float(start) * (1.0 - progress) return max(0.0, min(1.0, value)) @@ -88,7 +97,7 @@ def _build_training_wrapper(self, draft_model): loss_decay_gamma=self.loss_decay_gamma, ce_loss_alpha=self.ce_loss_alpha, l1_loss_alpha=self.l1_loss_alpha, - lambda_base=self._lambda_base_for_step(0), + lambda_base=self._lambda_base_for_micro_step(1), deterministic_anchor_sampling=getattr( self.args, "deterministic_anchor_sampling", False ), @@ -111,58 +120,68 @@ def _aggregate_metrics(self, all_step_metrics, step, *, grad_norm=None): mixed = metrics.get("train/mixed_ce") if mixed is not None: metrics["train/avg_loss"] = mixed - metrics["train/lambda_base"] = self._lambda_base_for_step( - max(self.global_step - 1, 0) - ) + last_micro = max(1, int(self.global_step) * self._accumulation_steps()) + metrics["train/lambda_base"] = self._lambda_base_for_micro_step(last_micro) return metrics - def _train_step(self, *args, **kwargs): - # The schedule is a function of the optimizer step, so every micro-batch - # in one step uses the same lambda. Exact resume restores global_step. - self.dflash.set_lambda_base(self._lambda_base_for_step(self.global_step)) - return super()._train_step(*args, **kwargs) + def _train_step(self, batch, accumulation_steps, step, batch_idx, num_batches): + # Match SpecForge: advance the schedule counter before every microbatch. + micro_step = ( + int(self.global_step) * int(accumulation_steps) + int(batch_idx) + 1 + ) + self.dflash.set_lambda_base(self._lambda_base_for_micro_step(micro_step)) + return super()._train_step( + batch=batch, + accumulation_steps=accumulation_steps, + step=step, + batch_idx=batch_idx, + num_batches=num_batches, + ) def eval_from_cache(self): - # Evaluate with the lambda used by the most recently completed step, - # not the value the next step would use. - self.dflash.set_lambda_base( - self._lambda_base_for_step(max(self.global_step - 1, 0)) - ) - corrected = super().eval_from_cache() - if not self.eval_backbone or not corrected: - return corrected - # Reuse the cached Target features for a backbone-only reporting pass. - # The mixed loss and the correction head are unchanged by this flag. - self.dflash.report_backbone = True + # Evaluate at the value used by the most recently completed microbatch. + last_micro = max(1, int(self.global_step) * self._accumulation_steps()) + self.dflash.set_lambda_base(self._lambda_base_for_micro_step(last_micro)) + previous = getattr(self.dflash, "collect_extra_position_metrics", False) + self.dflash.collect_extra_position_metrics = self.eval_backbone try: - backbone = super().eval_from_cache() + return super().eval_from_cache() finally: - self.dflash.report_backbone = False - return self._merge_backbone_eval_metrics(corrected, backbone) - - @staticmethod - def _merge_backbone_eval_metrics(corrected: dict, backbone: dict) -> dict: - merged = dict(corrected) - for key, value in backbone.items(): - if not key.startswith("eval/"): - continue - suffix = key[len("eval/"):] - if ( - suffix in {"avg_loss", "avg_acc"} - or "ploss_" in suffix - or suffix.startswith("acc_") - or "simulated_acc_len" in suffix - ): - merged[f"eval/backbone_{suffix}"] = value - return merged + self.dflash.collect_extra_position_metrics = previous def _aggregate_eval_metrics(self, all_step_metrics): metrics = super()._aggregate_eval_metrics(all_step_metrics) - if not getattr(self.dflash, "report_backbone", False): - mixed = metrics.get("eval/mixed_ce") - if mixed is not None: - metrics["eval/avg_loss"] = mixed - metrics["eval/lambda_base"] = self._lambda_base_for_step( - max(self.global_step - 1, 0) + mixed = metrics.get("eval/mixed_ce") + if mixed is not None: + metrics["eval/avg_loss"] = mixed + last_micro = max(1, int(self.global_step) * self._accumulation_steps()) + metrics["eval/lambda_base"] = self._lambda_base_for_micro_step(last_micro) + if not all_step_metrics or "backbone_loss_pp" not in all_step_metrics[0]: + return metrics + + backbone_loss, backbone_acc, backbone_count = self._reduce_position_metrics( + all_step_metrics, + loss_key="backbone_loss_pp", + acc_key="backbone_acc_pp", + count_key="backbone_count_pp", + ) + offset = self._anchor_slot_offset + pred_loss = backbone_loss[offset:] + pred_acc = backbone_acc[offset:] + pred_count = backbone_count[offset:] + cumulative = 1.0 + simulated_acc_len = 0.0 + for i in range(pred_acc.shape[0]): + cumulative *= pred_acc[i].item() + simulated_acc_len += cumulative + avg_loss, avg_acc = self._compute_scalar_metrics( + pred_loss, pred_acc, pred_count ) + prefix = self._prediction_metric_prefix() + metrics["eval/backbone_avg_loss"] = avg_loss + metrics["eval/backbone_avg_acc"] = avg_acc + metrics[f"eval/backbone_{prefix}simulated_acc_len"] = simulated_acc_len + for i in range(pred_loss.shape[0]): + metrics[f"eval/backbone_{prefix}ploss_{i}"] = pred_loss[i].item() + metrics[f"eval/backbone_{prefix}acc_{i}"] = pred_acc[i].item() return metrics diff --git a/docs/user-guide/domino.md b/docs/user-guide/domino.md index 761eba6..9089ccd 100644 --- a/docs/user-guide/domino.md +++ b/docs/user-guide/domino.md @@ -144,9 +144,13 @@ loss = (1 - lambda_base) * CE(corrected logits, labels) + lambda_base * CE(backbone logits, labels) ``` -With both curriculum fields set, the trainer sets the value at optimizer step `s` to -`start * max(0, 1 - s / max(1, int(lr_total_steps * decay_ratio)))`, matching the -reference SpecForge Domino schedule. Otherwise it stays at `domino_lambda_base`. +With both curriculum fields set, the trainer evaluates the value at the 1-based +microbatch counter `m`: +`start * max(0, 1 - m / max(1, int(lr_total_steps * decay_ratio)))`. +`lr_total_steps` is measured in optimizer updates; because the counter advances per +microbatch, accumulation greater than one reaches zero before the last update, +matching the reference SpecForge Domino trainer. Otherwise the value stays at +`domino_lambda_base`. `lambda_base=1` removes the correction contribution and keeps the selected label layout (the correction head receives no useful gradient). It equals the plain DFlash objective @@ -157,13 +161,14 @@ reused; `ce_loss_alpha=1` and `l1_loss_alpha=0` are enforced. Reporting accuracy corrected logits. The trainer reports `mixed_ce` as `train/avg_loss`/`eval/avg_loss` (the objective actually used for backward) and exposes `mixed_ce`, `backbone_ce` and `corrected_ce` as separate token-pooled component curves. With -`training.domino_eval_backbone=true`, each held-out evaluation also runs a -backbone-only reporting pass over the same cached Target features: +`training.domino_eval_backbone=true`, the same evaluation forward and the same +sampled anchors produce both branches: `eval/ploss_i`/`eval/acc_i` remain corrected, while `eval/backbone_ploss_i`/`eval/backbone_acc_i` (plus `eval/backbone_avg_loss`, `eval/backbone_avg_acc` and -`eval/backbone_simulated_acc_len`) describe the backbone-only branch. The mixed -objective and the trained correction head are unchanged by this reporting mode. +`eval/backbone_simulated_acc_len`) describe the backbone-only branch. There is no +second evaluation pass, and the mixed objective and trained correction head are +unchanged by this reporting mode. ## Minimal numerical use diff --git a/tests/cpu/test_domino.py b/tests/cpu/test_domino.py index e67d7a7..79a2874 100644 --- a/tests/cpu/test_domino.py +++ b/tests/cpu/test_domino.py @@ -285,21 +285,24 @@ def test_lambda_one_recovers_the_backbone_only_objective(): assert torch.count_nonzero(model.draft_model.context_proj.weight.grad) > 0 -def test_backbone_reporting_switches_per_position_metrics_only(): +def test_same_forward_backbone_metrics_share_the_anchor_slot(): model = make_model(lambda_base=0.3, loss_decay_gamma=0.0) - inputs = make_inputs() - corrected, captured = run_and_capture(model, inputs) - model.report_backbone = True - backbone, _ = run_and_capture(model, inputs) - expected_loss, _, expected_per_position, _ = reference_objective( - model, captured, inputs, report="backbone" + inputs = make_inputs(length=20) + # Exercise anchor sampling rather than injected anchors. + inputs.pop("injected_anchors") + inputs.pop("injected_keep_mask") + model.collect_extra_position_metrics = True + result, _ = run_and_capture(model, inputs) + extra = model._last_extra_position_metrics + assert extra["backbone_loss_pp"].shape == result[2].shape + # The shifted layout keeps backbone logits at position 0, so both branches + # must score the same anchor with the same sampled anchors and weights. + torch.testing.assert_close( + result[2][0], extra["backbone_loss_pp"][0], rtol=1e-6, atol=1e-6 ) - torch.testing.assert_close(backbone[0], corrected[0]) - torch.testing.assert_close(backbone[0], expected_loss, rtol=2e-5, atol=2e-6) torch.testing.assert_close( - backbone[2], expected_per_position, rtol=2e-5, atol=2e-6 + result[3][0], extra["backbone_acc_pp"][0], rtol=1e-6, atol=1e-6 ) - assert not torch.equal(corrected[2], backbone[2]) def test_two_row_layout_matches_reference_and_backward(): diff --git a/tests/cpu/test_domino_integration.py b/tests/cpu/test_domino_integration.py index 38b6936..1467213 100644 --- a/tests/cpu/test_domino_integration.py +++ b/tests/cpu/test_domino_integration.py @@ -25,6 +25,7 @@ _logical_training_topology, _validate_exact_resume_topology, ) +from aspec.training.dflash_trainer import DFlashTrainer from aspec.training.domino_trainer import DominoTrainer from aspec.training.trainer_actor import _trainer_class_for_config @@ -218,49 +219,74 @@ def test_plan_rejects_invalid_domino_lambda_curriculum(tmp_path, overrides): compile_execution_plan(config, config_path=str(_EXAMPLE), cli_args=args) -def test_lambda_curriculum_matches_reference_linear_decay(): +def test_lambda_curriculum_matches_reference_microbatch_counter(): trainer = object.__new__(DominoTrainer) trainer.lambda_base = 0.3 trainer.lambda_base_start = 1.0 trainer.lambda_base_decay_ratio = 1.0 trainer.lambda_total_steps = 8 - assert trainer._lambda_base_for_step(0) == pytest.approx(1.0) - assert trainer._lambda_base_for_step(2) == pytest.approx(0.75) - assert trainer._lambda_base_for_step(4) == pytest.approx(0.5) - assert trainer._lambda_base_for_step(8) == pytest.approx(0.0) - assert trainer._lambda_base_for_step(99) == pytest.approx(0.0) + assert trainer._lambda_base_for_micro_step(1) == pytest.approx(0.875) + assert trainer._lambda_base_for_micro_step(2) == pytest.approx(0.75) + assert trainer._lambda_base_for_micro_step(4) == pytest.approx(0.5) + assert trainer._lambda_base_for_micro_step(8) == pytest.approx(0.0) + assert trainer._lambda_base_for_micro_step(99) == pytest.approx(0.0) trainer.lambda_base_start = None trainer.lambda_base_decay_ratio = None - assert trainer._lambda_base_for_step(4) == pytest.approx(0.3) + assert trainer._lambda_base_for_micro_step(4) == pytest.approx(0.3) -def test_backbone_eval_metrics_merge_with_prefix(): - corrected = { - "eval/ploss_0": 1.0, - "eval/acc_0": 0.5, - "eval/avg_loss": 2.0, - "eval/avg_acc": 0.25, - "eval/mixed_ce": 2.0, - "eval/backbone_ce": 3.0, - } - backbone = { - "eval/ploss_0": 9.0, - "eval/acc_0": 0.1, - "eval/avg_loss": 8.0, - "eval/avg_acc": 0.2, - "eval/simulated_acc_len": 0.3, - "eval/mixed_ce": 2.0, +def test_training_loop_advances_lambda_per_microbatch(monkeypatch): + trainer = object.__new__(DominoTrainer) + trainer.global_step = 0 + trainer.lambda_base = 0.3 + trainer.lambda_base_start = 1.0 + trainer.lambda_base_decay_ratio = 1.0 + trainer.lambda_total_steps = 8 + trainer.args = SimpleNamespace(draft_accumulation_steps=2) + captured = [] + trainer.dflash = SimpleNamespace(set_lambda_base=captured.append) + monkeypatch.setattr(DFlashTrainer, "_train_step", lambda self, **kwargs: {}) + + trainer._train_step( + batch=None, accumulation_steps=2, step=1, batch_idx=0, num_batches=2 + ) + trainer._train_step( + batch=None, accumulation_steps=2, step=1, batch_idx=1, num_batches=2 + ) + assert captured == pytest.approx([0.875, 0.75]) + + +def test_backbone_eval_metrics_aggregate_per_position(monkeypatch): + import aspec.training.dflash_trainer as trainer_module + + monkeypatch.setattr(trainer_module.dist, "all_reduce", lambda *args, **kwargs: None) + monkeypatch.setattr(trainer_module.dist, "get_rank", lambda: 1) + trainer = object.__new__(DominoTrainer) + trainer._anchor_slot_offset = 0 + trainer.global_step = 4 + trainer.optimizer = SimpleNamespace(get_learning_rate=lambda: 0.0) + trainer.dflash = SimpleNamespace( + mtp_conditioning=None, collect_extra_position_metrics=False + ) + trainer.draft_model = SimpleNamespace(config=SimpleNamespace()) + trainer.loss_decay_gamma = 0.0 + trainer.args = SimpleNamespace(draft_accumulation_steps=2) + step = { + "loss_pp": torch.tensor([1.0]), + "acc_pp": torch.tensor([1.0]), + "count_pp": torch.tensor([1.0]), + "backbone_loss_pp": torch.tensor([2.0]), + "backbone_acc_pp": torch.tensor([0.5]), + "backbone_count_pp": torch.tensor([1.0]), + "mixed_ce": (torch.tensor(2.0), torch.tensor(1.0)), } - merged = DominoTrainer._merge_backbone_eval_metrics(corrected, backbone) - assert merged["eval/ploss_0"] == 1.0 - assert merged["eval/backbone_ploss_0"] == 9.0 - assert merged["eval/backbone_acc_0"] == 0.1 - assert merged["eval/backbone_avg_loss"] == 8.0 - assert merged["eval/backbone_avg_acc"] == 0.2 - assert merged["eval/backbone_simulated_acc_len"] == 0.3 - assert merged["eval/mixed_ce"] == 2.0 - assert "eval/backbone_mixed_ce" not in merged + metrics = trainer._aggregate_eval_metrics([step]) + assert metrics["eval/backbone_ploss_0"] == pytest.approx(2.0) + assert metrics["eval/backbone_acc_0"] == pytest.approx(0.5) + assert metrics["eval/backbone_avg_loss"] == pytest.approx(2.0) + assert metrics["eval/backbone_avg_acc"] == pytest.approx(0.5) + assert metrics["eval/backbone_simulated_acc_len"] == pytest.approx(0.5) def test_plan_rejects_explicit_cudnn_gqa_mode(tmp_path): From a90a42bac81e5f392a926965e087691dbb937b4e Mon Sep 17 00:00:00 2001 From: Hathway <60729800+hereisaway@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:35:45 +0800 Subject: [PATCH 5/5] fix(domino): release backbone eval logits and expand metric coverage Clear the retained backbone report after per-position metrics, and cover every backbone position, masked counts and independent accuracy against the reference. --- aspec/models/domino.py | 38 +++++++++++++++++------------ tests/cpu/test_domino.py | 52 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 17 deletions(-) diff --git a/aspec/models/domino.py b/aspec/models/domino.py index 66d2566..7a984b4 100644 --- a/aspec/models/domino.py +++ b/aspec/models/domino.py @@ -151,6 +151,9 @@ def _training_loss( # Keep the same-forward backbone values for evaluation; the mixed # loss and corrected reporting branch are unchanged. self._last_backbone_report = (base_per_token.detach(), base_logits.detach()) + else: + # Never retain the last evaluation logits across training. + self._last_backbone_report = None # Report the corrected objective's per-position value; accuracy uses the # corrected logits as well. return loss_per_token, corrected_per_token, corrected_logits @@ -167,21 +170,26 @@ def _extra_position_metrics( if not self.collect_extra_position_metrics or self._last_backbone_report is None: return {} base_per_token, base_logits = self._last_backbone_report - block_size = self.block_size - base_loss_pp = base_per_token.reshape(bsz, n_blocks, block_size) - vocab_size = base_logits.size(-1) - base_pred = base_logits.reshape(-1, vocab_size).argmax(-1) - correct = (base_pred.view(bsz, n_blocks, block_size) == target_ids) & ( - binary_weights > 0.5 - ) - safe_count = count_per_position.clamp(min=1.0) - loss_pp = (base_loss_pp * binary_weights).sum(dim=(0, 1)) / safe_count - acc_pp = correct.float().sum(dim=(0, 1)) / safe_count - return { - "backbone_loss_pp": loss_pp.detach(), - "backbone_acc_pp": acc_pp.detach(), - "backbone_count_pp": count_per_position.detach(), - } + try: + block_size = self.block_size + base_loss_pp = base_per_token.reshape(bsz, n_blocks, block_size) + vocab_size = base_logits.size(-1) + base_pred = base_logits.reshape(-1, vocab_size).argmax(-1) + correct = (base_pred.view(bsz, n_blocks, block_size) == target_ids) & ( + binary_weights > 0.5 + ) + safe_count = count_per_position.clamp(min=1.0) + loss_pp = (base_loss_pp * binary_weights).sum(dim=(0, 1)) / safe_count + acc_pp = correct.float().sum(dim=(0, 1)) / safe_count + return { + "backbone_loss_pp": loss_pp.detach(), + "backbone_acc_pp": acc_pp.detach(), + "backbone_count_pp": count_per_position.detach(), + } + finally: + # The full logits share storage with the forward graph; keep only + # the reduced position tensors and release the report immediately. + self._last_backbone_report = None def _extra_training_loss( self, diff --git a/tests/cpu/test_domino.py b/tests/cpu/test_domino.py index 79a2874..46ec3fb 100644 --- a/tests/cpu/test_domino.py +++ b/tests/cpu/test_domino.py @@ -79,7 +79,7 @@ def run_and_capture(model, inputs): return result, captured[0] -def reference_objective(model, draft_hidden, inputs, *, report="corrected"): +def reference_objective(model, draft_hidden, inputs, *, report="corrected", details=False): """Dense Domino equations with explicit label/anchor/mask alignment.""" draft_model = model.draft_model anchors = inputs["injected_anchors"] @@ -160,12 +160,27 @@ def reference_objective(model, draft_hidden, inputs, *, report="corrected"): "backbone_ce": ((base_ce * flat_weights).sum(), denominator), "corrected_ce": ((corrected_ce * flat_weights).sum(), denominator), } - return ( + result = ( loss, reported_ce.reshape(-1, anchors.shape[1], block_size), per_position, components, ) + if not details: + return result + base_pred = base_logits.reshape(-1, 32).argmax(-1) + base_correct = (base_pred == target_ids.reshape(-1)) & (binary > 0.5) + details_value = { + "backbone_per_position": ( + base_ce.reshape(-1, anchors.shape[1], block_size) * weights + ).sum(dim=(0, 1)) / count.clamp(min=1.0), + "backbone_acc_per_position": ( + base_correct.reshape(-1, anchors.shape[1], block_size).float().sum(dim=(0, 1)) + / count.clamp(min=1.0) + ), + "count_per_position": count, + } + return (*result, details_value) def _assert_parameter_gradients(actual, expected): @@ -305,6 +320,39 @@ def test_same_forward_backbone_metrics_share_the_anchor_slot(): ) +def test_extra_position_metrics_match_reference_with_masked_position(): + model = make_model(lambda_base=0.3, loss_decay_gamma=0.0) + inputs = make_inputs() + inputs["loss_mask"][0, 9] = 0 + model.collect_extra_position_metrics = True + result, captured = run_and_capture(model, inputs) + extra = model._last_extra_position_metrics + expected = reference_objective( + model, captured, inputs, report="backbone", details=True + )[4] + torch.testing.assert_close( + extra["backbone_loss_pp"], + expected["backbone_per_position"], + rtol=2e-5, + atol=2e-6, + ) + torch.testing.assert_close( + extra["backbone_acc_pp"], + expected["backbone_acc_per_position"], + rtol=1e-6, + atol=1e-6, + ) + torch.testing.assert_close( + extra["backbone_count_pp"], expected["count_per_position"], rtol=0, atol=0 + ) + assert not torch.equal(result[2], expected["backbone_per_position"]) + assert (expected["count_per_position"] < expected["count_per_position"].max()).any() + torch.testing.assert_close( + result[2][0], extra["backbone_loss_pp"][0], rtol=1e-6, atol=1e-6 + ) + assert model._last_backbone_report is None + + def test_two_row_layout_matches_reference_and_backward(): actual = make_model(shift_label=True, lambda_base=0.4) reference = copy.deepcopy(actual)