diff --git a/aspec/commands/convert_to_hf.py b/aspec/commands/convert_to_hf.py index d71b58f..22f0011 100644 --- a/aspec/commands/convert_to_hf.py +++ b/aspec/commands/convert_to_hf.py @@ -261,6 +261,15 @@ def _checkpoint_training_contract(model_dir: Path, raw_config: dict) -> dict: effective = AutoDraftModelConfig.from_dict(raw_config) if draft_model_semantics(effective) != draft_model_semantics(saved_semantics): raise ValueError("Export model semantics conflict with the checkpoint; use its saved draft_config.json") + saved_block_attention = (topology.get("draft_attention_policy") or {}).get("draft_block_attention") + if algorithm in {"dspark", "dflash"} and saved_block_attention is not None: + from aspec.config.draft_attention import ( + draft_block_attention_matches, + resolve_draft_attention, + ) + + if not draft_block_attention_matches(saved_block_attention, resolve_draft_attention(raw_config)): + raise ValueError("Export Draft block attention conflicts with the checkpoint; use its saved draft_config.json") from aspec.config.draft_structure import checkpoint_ffn_policy, resolve_draft_structure saved_ffn = checkpoint_ffn_policy(model_dir.parent, topology if metadata_path.is_file() else {}) diff --git a/aspec/config/draft_algorithm.py b/aspec/config/draft_algorithm.py index 1605840..70ca2e4 100644 --- a/aspec/config/draft_algorithm.py +++ b/aspec/config/draft_algorithm.py @@ -15,6 +15,17 @@ """Algorithm identity for the currently registered Draft architectures.""" from collections.abc import Mapping +from enum import StrEnum + + +class DSparkLossRecipe(StrEnum): + DEEPSPEC = "deepspec" + SPECULATORS_REDHAT_V1 = "speculators_redhat_v1" + + @property + def distribution_metric(self) -> str: + return "tv_loss" if self is self.SPECULATORS_REDHAT_V1 else "l1_loss" + DRAFT_ALGORITHMS = { "NativeMTPDraftModel": "mtp", diff --git a/aspec/config/draft_attention.py b/aspec/config/draft_attention.py index 52f3b04..21064f1 100644 --- a/aspec/config/draft_attention.py +++ b/aspec/config/draft_attention.py @@ -143,6 +143,9 @@ def resolve_draft_attention(raw: Mapping) -> dict: ): raise ValueError("Draft sliding_window must be a positive integer or null") schedule = raw.get("layer_types") + sliding_non_causal = raw.get("sliding_window_non_causal", True) + if not isinstance(sliding_non_causal, bool): + raise ValueError("sliding_window_non_causal must be boolean") ratios = raw.get("compress_ratios") if ratios is not None: if not isinstance(ratios, (list, tuple)) or len(ratios) != layers: @@ -421,11 +424,21 @@ def dsa_positive_integer(name: str, default: int) -> int: else kind == "sliding_attention") else "bidirectional" for kind in schedule ] if raw.get("architectures") == ["DFlash2DraftModel"] - else "bidirectional" + else ["causal" if kind == "sliding_attention" else "bidirectional" for kind in schedule] + if not sliding_non_causal else "bidirectional" ), } +def draft_block_attention_matches(saved: str | list[str], current: Mapping) -> bool: + """Compare scalar and per-layer serializations over the same layer schedule.""" + policy = current["draft_block_attention"] + layers = len(current["layer_types"]) + return ([saved] * layers if isinstance(saved, str) else saved) == ( + [policy] * layers if isinstance(policy, str) else policy + ) + + def draft_context_shard_alignment(attention: Mapping) -> int: """Derive the minimum context-shard alignment from active sparse structures.""" diff --git a/aspec/config/dspark_glm5_3_flash_draft_config.json b/aspec/config/dspark_glm5_3_flash_draft_config.json new file mode 100644 index 0000000..8bccc42 --- /dev/null +++ b/aspec/config/dspark_glm5_3_flash_draft_config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "DSparkDraftModel" + ], + "model_type": "qwen3_dspark", + "hidden_size": 4096, + "intermediate_size": 12288, + "head_dim": 64, + "num_hidden_layers": 5, + "num_attention_heads": 64, + "num_key_value_heads": 64, + "vocab_size": 154880, + "rms_norm_eps": 1e-05, + "max_position_embeddings": 1048576, + "rope_theta": 10000, + "num_target_layers": 6, + "target_hidden_size": 4096, + "target_num_hidden_layers": 45, + "target_layer_ids": [ + 19, + 27, + 31, + 35, + 39, + 43 + ], + "mask_token_id": 154856, + "block_size": 8, + "markov_rank": 256, + "markov_head_type": "vanilla", + "enable_confidence_head": true, + "confidence_head_with_markov": true, + "tie_word_embeddings": false, + "sliding_window": 2048, + "sliding_window_non_causal": false +} diff --git a/aspec/config/train_config.py b/aspec/config/train_config.py index 3fbb132..60e52bd 100644 --- a/aspec/config/train_config.py +++ b/aspec/config/train_config.py @@ -251,6 +251,7 @@ class TrainingConfig: # DSpark-specific parameters (used by DSpark trainer only) # Precision of the frozen Draft LM head during training (SM90 FP8, CP1). dspark_lm_head_precision: str = "bf16" + dspark_loss_recipe: str = "deepspec" dspark_num_anchors: int = 512 dspark_num_target_layers: int = 5 dspark_loss_decay_gamma: float = 4.0 diff --git a/aspec/config/utils.py b/aspec/config/utils.py index e5a596b..2e27186 100644 --- a/aspec/config/utils.py +++ b/aspec/config/utils.py @@ -25,11 +25,22 @@ from typing import Optional import torch -from transformers import AutoConfig, AutoTokenizer +from transformers import AutoConfig, AutoTokenizer, PretrainedConfig logger = logging.getLogger(__name__) +def load_target_config(model_path, **kwargs): + """Read GLM text geometry even in Draft environments without SGLang's registry.""" + try: + return AutoConfig.from_pretrained(model_path, **kwargs) + except ValueError: + raw, _ = PretrainedConfig.get_config_dict(model_path, **kwargs) + if raw.get("model_type") != "glm5_next": + raise + return PretrainedConfig.from_dict(raw["text_config"]) + + def _copy_config_value(value): if hasattr(value, "to_dict"): return value.to_dict() @@ -114,7 +125,7 @@ def generate_draft_model_config( Returns: dict: Generated draft model config dictionary """ - target_config = AutoConfig.from_pretrained( + target_config = load_target_config( target_model_path, cache_dir=cache_dir, trust_remote_code=trust_remote_code, diff --git a/aspec/data/renderers/__init__.py b/aspec/data/renderers/__init__.py index f0542df..87f3e1a 100644 --- a/aspec/data/renderers/__init__.py +++ b/aspec/data/renderers/__init__.py @@ -19,14 +19,17 @@ # SOFTWARE. from aspec.data.renderers.base import DEFAULT_CACHE_VERSION, ConversationRenderer +from aspec.data.renderers.glm5 import Glm5FlashRenderer from aspec.data.renderers.kimi_k3 import K3Renderer from aspec.data.renderers.registry import RENDERER_REGISTRY, RendererRegistry +RENDERER_REGISTRY.register("glm5_3_flash", Glm5FlashRenderer) RENDERER_REGISTRY.register("kimi-k3", K3Renderer) __all__ = [ "DEFAULT_CACHE_VERSION", "ConversationRenderer", + "Glm5FlashRenderer", "K3Renderer", "RENDERER_REGISTRY", "RendererRegistry", diff --git a/aspec/data/renderers/glm5.py b/aspec/data/renderers/glm5.py new file mode 100644 index 0000000..27fe708 --- /dev/null +++ b/aspec/data/renderers/glm5.py @@ -0,0 +1,72 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Text-only conversations using the GLM-5.3-Flash checkpoint template.""" + + +class Glm5FlashRenderer: + CACHE_VERSION = "v2" + roles = ("system", "user", "assistant") + + def __init__(self, tokenizer): + self.tokenizer = tokenizer + self.role_ids = {} + for role in self.roles: + marker = f"<|{role}|>" + ids = tokenizer.encode(marker, add_special_tokens=False) + if len(ids) != 1 or tokenizer.convert_ids_to_tokens(ids[0]) != marker: + raise ValueError(f"GLM tokenizer lacks its {marker} role token") + self.role_ids[ids[0]] = role + + def get_assistant_token_ids(self): + ids = {role: token for token, role in self.role_ids.items()} + return [ids["assistant"]], [ids["user"]], 0 + + def render( + self, messages, tools=None, *, max_seq_length, last_turn_only=False, generation_config=None + ): + if tools: + raise ValueError("GLM text training requires conversations without tools") + for msg in messages: + if ( + msg.get("role") not in self.roles + or not isinstance(msg.get("content"), str) + or msg.get("tool_calls") + ): + raise ValueError("Expected GLM system/user/assistant text messages") + for alias in ("thinking", "thinking_content", "reasoning"): + if alias in msg: + raise ValueError( + f"GLM messages must store reasoning in reasoning_content, not {alias!r}" + ) + for field in ("content", "reasoning_content"): + value = msg.get(field) + if value is None: + continue + if not isinstance(value, str): + raise ValueError(f"GLM {field} must be text") + if any(f"<|{role}|>" in value for role in self.roles): + raise ValueError("A GLM message contains a reserved role token") + + kwargs = { + k: v + for k, v in (generation_config or {}).items() + if k in ("reasoning_effort", "clear_thinking") + } + ids = self.tokenizer.apply_chat_template( + messages, tokenize=True, add_generation_prompt=False, return_dict=False, **kwargs + ) + mask = [0] * len(ids) + last_assistant = 0 + supervise = False + for i, token in enumerate(ids): + role = self.role_ids.get(token) + if role is not None: + supervise = role == "assistant" + if supervise: + last_assistant = i + else: + mask[i] = int(supervise) + if last_turn_only: + mask[:last_assistant] = [0] * last_assistant + return ids[:max_seq_length], mask[:max_seq_length] diff --git a/aspec/draft_artifact.py b/aspec/draft_artifact.py index 10e5f17..88958b5 100644 --- a/aspec/draft_artifact.py +++ b/aspec/draft_artifact.py @@ -74,7 +74,7 @@ def validate_native_asset(path, *, expected_config=None, weights_only=False): raise ValueError("Native artifact MTP conditioning semantics mismatch") if expected_config is not None and draft_model_semantics(config) != draft_model_semantics(expected_config): raise ValueError("Native artifact semantics disagree with the requested Draft") - from aspec.config.draft_attention import resolve_draft_attention + from aspec.config.draft_attention import draft_block_attention_matches, resolve_draft_attention attention = resolve_draft_attention(config.to_dict()) recorded_dsa = ((manifest.get("draft_structure") or {}).get("attention") or {}).get("dsa") @@ -84,6 +84,9 @@ def validate_native_asset(path, *, expected_config=None, weights_only=False): expected_attention = resolve_draft_attention( expected_config if isinstance(expected_config, dict) else expected_config.to_dict() ) + if (manifest["algorithm"] in {"dspark", "dflash"} + and not draft_block_attention_matches(attention["draft_block_attention"], expected_attention)): + raise ValueError("Native artifact Draft block attention disagrees with the requested Draft") if attention.get("dsa") is not None or expected_attention.get("dsa") is not None: def dsa_semantics(policy, *, phase_transition): dsa = policy.get("dsa") diff --git a/aspec/inference/engine/sgl_engine.py b/aspec/inference/engine/sgl_engine.py index e564d87..b2d711b 100644 --- a/aspec/inference/engine/sgl_engine.py +++ b/aspec/inference/engine/sgl_engine.py @@ -984,9 +984,9 @@ def _get_hidden_size_from_engine(self) -> int: Returns: Hidden size dimension. """ - from transformers import AutoConfig + from aspec.config.utils import load_target_config - config = AutoConfig.from_pretrained( + config = load_target_config( self.args.target_model_path, trust_remote_code=getattr(self.args, "trust_remote_code", False), ) diff --git a/aspec/kernels/cutedsl/speculators_loss.py b/aspec/kernels/cutedsl/speculators_loss.py new file mode 100644 index 0000000..50a7ed6 --- /dev/null +++ b/aspec/kernels/cutedsl/speculators_loss.py @@ -0,0 +1,482 @@ +# Copyright 2026 Ant Group Co., Ltd. +# Copyright (c) 2025, Wentao Guo, Ted Zadouri, Tri Dao (QuACK tile layout). +# SPDX-License-Identifier: Apache-2.0 +"""Hopper/Blackwell joint hard CE and overlap TV. Reduction layout adapted from QuACK 0.6.5 +softmax; objective and BF16 gradient addition reimplement Speculators b9c51bc. +""" + +import math +from functools import lru_cache + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import torch +from cutlass import BFloat16, Float32, Int32, Int64, const_expr +from quack import copy_utils +from quack.compile_utils import make_fake_tensor +from quack.epilogue.frontend import F2, gemm_epilogue +from quack.reduce import row_reduce +from quack.reduction_base import ReductionBase + + +@cute.jit +def _lane_inputs(d, t, indices, tile, row, part, copy, vocab: cutlass.Constexpr): + teacher_row = row + if const_expr(indices is not None): + teacher_row = Int32(indices[row]) + ids = cute.make_identity_tensor(d.shape) + dx = copy.partition_S(cute.local_tile(d, tile, (row, part))) + tx = copy.partition_S(cute.local_tile(t, tile, (teacher_row, part))) + ix = copy.partition_S(cute.local_tile(ids, tile, (row, part))) + rd, rt = [cute.make_rmem_tensor_like(a) for a in (dx, tx)] + rd.fill(-BFloat16.inf) + rt.fill(-BFloat16.inf) + pred = copy_utils.predicate_k(ix, limit=vocab) + copy_utils.copy(dx, rd, pred=pred) + copy_utils.copy(tx, rt, pred=pred) + columns = cute.make_rmem_tensor(rd.shape, Int32) + for i in cutlass.range_constexpr(cute.size(columns)): + columns[i] = ix[i][1] + return rd.load().to(Float32), rt.load().to(Float32), columns.load(), pred + + +class _Grad: + def __init__(self, vocab): + self.vocab = vocab + + @cute.jit + def __call__( + self, + d: cute.Tensor, + t: cute.Tensor, + indices, + stats: cute.Tensor, + labels: cute.Tensor, + gc: cute.Tensor, + gt: cute.Tensor, + out: cute.Tensor, + stream: cuda.CUstream, + ): + vec = math.gcd(self.vocab, 8) + copy = copy_utils.tiled_copy_2d(BFloat16, 128, 128, vec) + tile = (1, 128 * vec) + self.kernel(d, t, indices, stats, labels, gc, gt, out, copy, tile).launch( + grid=(d.shape[0], cute.ceil_div(self.vocab, tile[1]), 1), + block=(128, 1, 1), + stream=stream, + ) + + @cute.kernel + def kernel( + self, + d: cute.Tensor, + t: cute.Tensor, + indices, + stats: cute.Tensor, + labels: cute.Tensor, + gc: cute.Tensor, + gt: cute.Tensor, + out: cute.Tensor, + copy: cute.TiledCopy, + tile: cute.Shape, + ): + row, part, _ = cute.arch.block_idx() + thr = copy.get_slice(cute.arch.thread_idx()[0]) + x, y, col, pred = _lane_inputs(d, t, indices, tile, row, part, thr, self.vocab) + q = cute.math.exp(x - stats[row, 0], fastmath=True) + p = cute.math.exp(y - stats[row, 1], fastmath=True) + dc = gc[row] * (q - (col == Int32(labels[row])).to(Float32)) + dt = gt[row] * (q * (stats[row, 2] - (q <= p).to(Float32))) + # Upstream writes two BF16 gradients before autograd adds them. + dest = thr.partition_D(cute.local_tile(out, tile, (row, part))) + reg = cute.make_rmem_tensor_like(dest) + reg.store((dc.to(BFloat16).to(Float32) + dt.to(BFloat16).to(Float32)).to(BFloat16)) + copy_utils.copy(reg, dest, pred=pred) + + +_TILE = 2048 + + +class _Overlap(ReductionBase): + """Stream independent vocabulary tiles, then combine FP32 row partials.""" + + def __init__(self, vocab, merge): + super().__init__(BFloat16, vocab, stage=4) + self.merge_partials = merge + self.cluster_n = 1 + self.width = _TILE + self.tiles = (vocab + self.width - 1) // self.width + + @cute.jit + def __call__( + self, + d: cute.Tensor, + t: cute.Tensor, + idx, + parts: cute.Tensor, + norm: cute.Tensor, + stats: cute.Tensor, + labels: cute.Tensor, + ce: cute.Tensor, + tv: cute.Tensor, + correct: cute.Tensor, + stream: cuda.CUstream, + ): + if const_expr(not self.merge_partials): + copy = copy_utils.tiled_copy_2d(BFloat16, 128, 128, math.gcd(self.N, 8)) + self.scan(d, t, idx, parts, norm, copy).launch( + grid=(d.shape[0] * self.tiles, 1, 1), block=(128, 1, 1), stream=stream + ) + else: + self.merge(d, parts, norm, stats, labels, ce, tv, correct).launch( + grid=(d.shape[0], 1, 1), block=(128, 1, 1), stream=stream + ) + + @cute.kernel + def scan( + self, + d: cute.Tensor, + t: cute.Tensor, + idx, + parts: cute.Tensor, + norm: cute.Tensor, + copy: cute.TiledCopy, + ): + tid = cute.arch.thread_idx()[0] + bid = cute.arch.block_idx()[0] + row, part = bid // self.tiles, bid % self.tiles + x, y, c, _ = _lane_inputs( + d, t, idx, (1, self.width), row, part, copy.get_slice(tid), self.N + ) + smem = cutlass.utils.SmemAllocator() + buf, _ = self._allocate_reduction_buffer_and_mbar(smem, copy.layout_tv_tiled) + add, minimum = cute.ReductionOp.ADD, cute.ReductionOp.MIN + q = cute.math.exp(x - norm[row, 0], fastmath=True) + p = cute.math.exp(y - norm[row, 1], fastmath=True) + take = q <= p + ov, mass = row_reduce( + (cute.where(take, q, p), cute.where(take, q, 0.0)), + add, + 128, + (buf[None, None, 0], buf[None, None, 1]), + ) + i, j = row_reduce( + ( + cute.where(x == norm[row, 2], c.to(Float32), Float32(self.N)), + cute.where(y == norm[row, 3], c.to(Float32), Float32(self.N)), + ), + minimum, + 128, + (buf[None, None, 2], buf[None, None, 3]), + init_val=Float32(self.N), + ) + if tid == 0: + parts[row, 0, part] = ov + parts[row, 1, part] = mass + parts[row, 2, part] = i + parts[row, 3, part] = j + + @cute.kernel + def merge( + self, + d: cute.Tensor, + parts: cute.Tensor, + norm: cute.Tensor, + stats: cute.Tensor, + labels: cute.Tensor, + ce: cute.Tensor, + tv: cute.Tensor, + correct: cute.Tensor, + ): + row = cute.arch.block_idx()[0] + tid = cute.arch.thread_idx()[0] + smem = cutlass.utils.SmemAllocator() + layout = cute.make_layout((1, (4, 1), 4)) + buf = smem.allocate_tensor(Float32, layout) + a, b, c, e = Float32(0), Float32(0), Float32(0), Float32(0) + c, e = Float32(self.N), Float32(self.N) + if tid < self.tiles: + a, b, c, e = ( + parts[row, 0, tid], + parts[row, 1, tid], + parts[row, 2, tid], + parts[row, 3, tid], + ) + ov, mass = row_reduce( + (a, b), cute.ReductionOp.ADD, 128, (buf[None, None, 0], buf[None, None, 1]) + ) + i, j = row_reduce( + (c, e), + cute.ReductionOp.MIN, + 128, + (buf[None, None, 2], buf[None, None, 3]), + init_val=Float32(self.N), + ) + if tid == 0: + label = min(Int32(j), self.N - 1) + labels[row] = Int64(label) + correct[row] = i == j + ce[row] = norm[row, 0] - Float32(d[row, label]) + tv[row] = 1.0 - ov + stats[row, 0] = norm[row, 0] + stats[row, 1] = norm[row, 1] + stats[row, 2] = mass + + +@lru_cache(maxsize=32) +def _overlap_compile(vocab, indexed): + rows = cute.sym_int() + x = make_fake_tensor(BFloat16, (rows, vocab), math.gcd(vocab, 8)) + y = make_fake_tensor(BFloat16, (cute.sym_int(), vocab), math.gcd(vocab, 8)) if indexed else x + ids = make_fake_tensor(Int64, (rows,)) + p = make_fake_tensor(Float32, (rows, 4, (vocab + _TILE - 1) // _TILE)) + n = make_fake_tensor(Float32, (rows, 4)) + f = make_fake_tensor(Float32, (rows,)) + s = make_fake_tensor(Float32, (rows, 3)) + correct = make_fake_tensor(cutlass.Boolean, (rows,)) + return [ + cute.compile( + _Overlap(vocab, merge), + x, + y, + ids if indexed else None, + p, + n, + s, + ids, + f, + f, + correct, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + for merge in (False, True) + ] + + +class _PinnedNorm(ReductionBase): + """Partition the upstream logical CTA across a thread-block cluster. Each logical + lane keeps the same FP32 summation order, including the online tail tile.""" + + def __init__(self, vocab): + super().__init__(BFloat16, vocab, stage=4) + block = min(131072, 1 << (vocab - 1).bit_length()) + warps = 32 if block >= 32768 else 16 if block >= 8192 else 8 if block >= 2048 else 4 + self.cluster_n = max(1, warps // 4) + self.block = block + self.vec = min(math.gcd(8, vocab), max(1, block // (32 * warps))) + self.passes = max(1, block // (128 * self.cluster_n * self.vec)) + self.groups = (vocab + block - 1) // block + + def _threads_per_row(self): + return 128 + + @cute.jit + def __call__( + self, d: cute.Tensor, t: cute.Tensor, idx, norm: cute.Tensor, stream: cuda.CUstream + ): + cp = copy_utils.tiled_copy_2d(BFloat16, 128, 128, self.vec) + cluster = (1, self.cluster_n, 1) if const_expr(self.cluster_n > 1) else None + self.kernel(d, None, norm, cp, 0).launch( + grid=(d.shape[0], self.cluster_n, 1), block=(128, 1, 1), cluster=cluster, stream=stream + ) + self.kernel(t, idx, norm, cp, 1).launch( + grid=(d.shape[0], self.cluster_n, 1), block=(128, 1, 1), cluster=cluster, stream=stream + ) + + @cute.kernel + def kernel( + self, d: cute.Tensor, idx, norm: cute.Tensor, cp: cute.TiledCopy, side: cutlass.Constexpr + ): + tid = cute.arch.thread_idx()[0] + row, part, _ = cute.arch.block_idx() + smem = cutlass.utils.SmemAllocator() + buf, mb = self._allocate_reduction_buffer_and_mbar(smem, cp.layout_tv_tiled) + self._initialize_cluster(tid, mb, 4) + yrow = row + if const_expr(idx is not None): + yrow = Int32(idx[row]) + thr = cp.get_slice(tid) + gmax = -Float32.inf + denom = Float32(0) + for group in cutlass.range_constexpr(self.groups): + tile = (1, 128 * self.vec) + # Omit padded tail columns to preserve the reference FP32 summation order. + passes = min( + self.passes, + (self.N - group * self.block + 128 * self.cluster_n * self.vec - 1) + // (128 * self.cluster_n * self.vec), + ) + cache = cute.make_rmem_tensor((self.vec, passes), Float32) + for k in cutlass.range_constexpr(passes): + shard = (group * self.passes + k) * self.cluster_n + part + dx = thr.partition_S(cute.local_tile(d, tile, (yrow, shard))) + ix = thr.partition_S( + cute.local_tile(cute.make_identity_tensor(d.shape), tile, (row, shard)) + ) + pred = copy_utils.predicate_k(ix, limit=self.N) + reg = cute.make_rmem_tensor_like(dx) + reg.fill(-BFloat16.inf) + copy_utils.copy(dx, reg, pred=pred) + for j in cutlass.range_constexpr(self.vec): + cache[j, k] = Float32(reg[j]) + m = row_reduce( + cache.load(), + cute.ReductionOp.MAX, + 128, + buf[None, None, 2 * group], + mb + 2 * group if const_expr(self.cluster_n > 1) else None, + init_val=-Float32.inf, + hook_fn=cute.arch.cluster_wait + if const_expr(self.cluster_n > 1 and group == 0) + else None, + ) + newmax = cute.arch.fmax(gmax, m) + ex = cute.make_rmem_tensor_like(cache) + ex.store(cute.math.exp(cache.load() - newmax, fastmath=True)) + total = Float32(0) + for k in cutlass.range_constexpr(passes): + for j in cutlass.range_constexpr(self.vec): + total = total + ex[j, k] + z = row_reduce( + total, + cute.ReductionOp.ADD, + 128, + buf[None, None, 2 * group + 1], + mb + 2 * group + 1 if const_expr(self.cluster_n > 1) else None, + ) + if const_expr(group == 0): + denom = z + else: + denom = cute.math.fma(denom, cute.math.exp(gmax - newmax, fastmath=True), z) + gmax = newmax + if (tid == 0) & (part == 0): + norm[row, side] = gmax + cute.math.log(denom) + norm[row, side + 2] = gmax + + +@lru_cache(maxsize=32) +def _norm_compile(vocab, indexed): + rows = cute.sym_int() + x = make_fake_tensor(BFloat16, (rows, vocab), math.gcd(vocab, 8)) + y = make_fake_tensor(BFloat16, (cute.sym_int(), vocab), math.gcd(vocab, 8)) if indexed else x + ids = make_fake_tensor(Int64, (rows,)) if indexed else None + n = make_fake_tensor(Float32, (rows, 4)) + return cute.compile( + _PinnedNorm(vocab), + x, + y, + ids, + n, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + + +@lru_cache(maxsize=32) +def _grad_compile(vocab, indexed): + rows = cute.sym_int() + x = make_fake_tensor(BFloat16, (rows, vocab), math.gcd(vocab, 8)) + y = make_fake_tensor(BFloat16, (cute.sym_int(), vocab), math.gcd(vocab, 8)) if indexed else x + ids = make_fake_tensor(Int64, (rows,)) + f = make_fake_tensor(Float32, (rows,)) + stats = make_fake_tensor(Float32, (rows, 3)) + return cute.compile( + _Grad(vocab), + x, + y, + ids if indexed else None, + stats, + ids, + f, + f, + x, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + + +class _TokenLoss(torch.autograd.Function): + @staticmethod + def forward(ctx, logits, teacher, indices): + vocab = logits.shape[-1] + x = logits.view(-1, vocab) + y = teacher.view(-1, vocab) + rows = x.shape[0] + ce = x.new_empty(rows, dtype=torch.float32) + tv = torch.empty_like(ce) + ids = x.new_empty(rows, dtype=torch.int64) + stats = x.new_empty((rows, 3), dtype=torch.float32) + correct = x.new_empty(rows, dtype=torch.bool) + partial = stats.new_empty((rows, 4, (vocab + _TILE - 1) // _TILE)) + norm = stats.new_empty((rows, 4)) + _norm_compile(vocab, indices is not None)(x, y, indices, norm) + for kernel in _overlap_compile(vocab, indices is not None): + kernel(x, y, indices, partial, norm, stats, ids, ce, tv, correct) + ctx.save_for_backward(logits, teacher, indices, stats, ids) + ctx.set_materialize_grads(False) + shape = logits.shape[:-1] + labels = ids.view(shape) + correct = correct.view(shape) + ctx.mark_non_differentiable(labels, correct) + return ce.view(shape), tv.view(shape), labels, correct + + @staticmethod + def backward(ctx, grad_ce, grad_tv, _labels, _correct): + logits, teacher, indices, stats, ids = ctx.saved_tensors + x = logits.view(-1, logits.shape[-1]) + dx = torch.empty_like(x) + + def upstream(g): + return stats.new_zeros(x.shape[0]) if g is None else g.reshape(-1).contiguous() + + _grad_compile(x.shape[-1], indices is not None)( + x, + teacher.view(-1, x.shape[-1]), + indices, + stats, + ids, + upstream(grad_ce), + upstream(grad_tv), + dx, + ) + return dx.view_as(logits), None, None + + +def speculators_cutedsl_loss(logits, teacher, indices=None): + return _TokenLoss.apply(logits, teacher, indices) + + +@gemm_epilogue() +def _markov_residual(acc, c): + # Match BF16 linear -> BF16 add. A standard FP32 GEMM residual epilogue + # drops the intermediate rounding and can change the TV subgradient. + if const_expr(isinstance(acc, F2)): + rounded = F2(Float32(BFloat16(acc.lo)), Float32(BFloat16(acc.hi))) + else: + rounded = Float32(BFloat16(acc)) + return {"D": rounded + c} + + +class _MarkovProjection(torch.autograd.Function): + @staticmethod + def forward(ctx, base, embedding, weight): + a = embedding.reshape(-1, embedding.shape[-1]) + out = _markov_residual(a, weight.T, base.view(-1, base.shape[-1]), tuned=False)["D"] + ctx.save_for_backward(a, weight) + ctx.embedding_shape = embedding.shape + return out.view_as(base) + + @staticmethod + def backward(ctx, grad): + a, weight = ctx.saved_tensors + g = grad.reshape(-1, grad.shape[-1]) + return ( + grad, + (g @ weight).view(ctx.embedding_shape) if ctx.needs_input_grad[1] else None, + g.T @ a if ctx.needs_input_grad[2] else None, + ) + + +markov_residual_projection = _MarkovProjection.apply diff --git a/aspec/models/dflash.py b/aspec/models/dflash.py index 7c4bf17..4a7f2e0 100644 --- a/aspec/models/dflash.py +++ b/aspec/models/dflash.py @@ -468,6 +468,8 @@ def _block_mask_options_for_layer(self, layer_id: int) -> dict: if config is not None and getattr(config, "layer_types", None): if config.layer_types[layer_id] == "sliding_attention": options["context_window"] = config.sliding_window + if not getattr(config, "sliding_window_non_causal", True): + options["is_causal"] = True return options def _compute_logits( diff --git a/aspec/models/dspark.py b/aspec/models/dspark.py index c96c166..91d0a8b 100644 --- a/aspec/models/dspark.py +++ b/aspec/models/dspark.py @@ -18,34 +18,37 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -"""DSpark training model: DFlash training wrapper + Markov / L1 / confidence losses. +"""DSpark training with selectable DeepSpec and Red Hat objectives. -Reuses :class:`DFlashModel`'s anchor sampling, block-causal FlexAttention mask, -and MASK-token noise construction verbatim, then layers on the DSpark training -objective: +The shared DFlash backbone supplies anchor sampling, block attention and MASK-token +inputs. A Markov head biases Draft logits using the observed previous token, and a +confidence head learns the detached Draft/Target probability overlap. - - Markov-biased draft logits (teacher-forced previous token). - - Cross-entropy against the ground-truth next tokens (hard labels). - - L1 distribution distillation: ``|softmax(draft) - softmax(target)|`` where the - target distribution is the frozen LM head applied to the *target's* final - hidden state at the aligned position (requires ``last_hidden_states``). - - Confidence head BCE against the empirical per-token accept rate. +DeepSpec uses next-token CE and full L1 distillation with contiguous supervision. +The optional Red Hat recipe uses teacher-argmax CE, total variation and independent +slot supervision with per-row normalization. Both return additive loss terms so the +trainer can normalize across ranks and gradient-accumulation steps. -Combined: ``ce_alpha*ce + l1_alpha*l1 + confidence_alpha*confidence``. - -Loss formulation adapted from DeepSeek's DeepSpec -(deepspec/modeling/dspark/loss.py, MIT). Additive loss terms are returned to -the trainer so normalization covers every rank and gradient-accumulation step. +The default loss is adapted from DeepSeek's DeepSpec (deepspec/modeling/dspark/loss.py, +MIT). The Red Hat objective reimplements vllm-project/speculators b9c51bc DSpark +metrics, fused CE/TV and row reductions within ASpec's chunked heads. """ from typing import List, Optional, Tuple import torch +import torch.distributed as dist import torch.nn.functional as F +from aspec.config.draft_algorithm import DSparkLossRecipe from aspec.models.dflash import DFlashModel from aspec.models.ops.acceptance import acceptance_components, acceptance_statistics -from aspec.models.ops.loss import compiled_l1_from_logits, use_sm90_tiled_l1 +from aspec.models.ops.loss import ( + compiled_l1_from_logits, + speculators_token_losses, + use_cutedsl_speculators_loss, + use_sm90_tiled_l1, +) from aspec.models.ops.output_blocks import ( empty_prediction_terms, output_block_counts, @@ -69,7 +72,7 @@ def _cached_projection_chunks(counts, row_budget, block_size, *, copies, limit=N class DSparkModel(DFlashModel): - """DSpark training wrapper (DFlash backbone + Markov/L1/confidence heads).""" + """DFlash backbone with Markov logits and selectable DSpark distillation losses.""" prediction_start = 0 @@ -89,6 +92,7 @@ def __init__( anchor_sampling_scope: str = "row", l1_target_sampling: tuple[float, float, int] = (1.0, 1.0, -1), l1_draft_sampling: tuple[float, float, int] = (1.0, 1.0, -1), + loss_recipe: str = "deepspec", ): # Reuse DFlash anchor/mask/noise machinery. The "decay" objective drives # the per-within-block position weighting shared by all DSpark terms. @@ -111,6 +115,9 @@ def __init__( self.confidence_head_alpha = float(confidence_head_alpha) self.l1_sampling = (l1_target_sampling, l1_draft_sampling) self.l1_sampling_enabled = self.l1_sampling != ((1.0, 1.0, -1), (1.0, 1.0, -1)) + self.loss_recipe = DSparkLossRecipe(loss_recipe) + if self.loss_recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 and self.l1_sampling_enabled: + raise ValueError("speculators_redhat_v1 requires unfiltered distributions at temperature one") self.l1_loss_backend = None self.l1_kernel_invocations = {} self._use_l1_kernel_optimizations = True @@ -157,8 +164,10 @@ def uses_target_distribution(self) -> bool: return False # The confidence head regresses onto an accept rate derived from the # verifier distribution, so the L1 weight alone does not cover it. - return super().uses_target_distribution or ( - self.draft_model.confidence_head is not None and self.confidence_head_alpha > 0 + return ( + self.loss_recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 + or super().uses_target_distribution + or (self.draft_model.confidence_head is not None and self.confidence_head_alpha > 0) ) def _decay_weights(self, device: torch.device) -> torch.Tensor: @@ -172,6 +181,15 @@ def _decay_weights(self, device: torch.device) -> torch.Tensor: return torch.exp(-(k - self.prediction_start).clamp_min(0).float() / self.loss_decay_gamma) return torch.ones_like(k, dtype=torch.float32) + def _speculators_row_weights(self, mask, decay_weights): + """Normalize within a logical row before pooling rows across DP/microsteps.""" + valid = mask.sum((1, 2), keepdim=True) + if self.draft_context_parallel_size > 1: + dist.all_reduce(valid, group=self.direct_cp_group) + # CP shards contribute disjoint numerators for the same logical rows. + row_count = mask.new_tensor(mask.shape[0] / self.draft_context_parallel_size) + return decay_weights / (valid + 1e-5), decay_weights / (valid + 1e-8), row_count + def forward( self, input_ids: torch.Tensor, @@ -256,14 +274,19 @@ def forward( torch.zeros_like(safe_label_indices), ) - # eval mask = contiguous supervised prefix per block (DeepSpec - # build_eval_mask): block kept, label in-bounds, target token supervised, - # then cumprod so a gap truncates the rest of the block. + warmup_kind = self._indexer_warmup_kind() + speculators = self.loss_recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 and warmup_kind is None + teacher_indices = (safe_label_indices - 1).clamp_min(0) + mask_indices = teacher_indices if speculators else safe_label_indices target_loss_mask = torch.gather( - loss_mask.unsqueeze(1).expand(-1, n_blocks, -1), 2, safe_label_indices + loss_mask.unsqueeze(1).expand(-1, n_blocks, -1), 2, mask_indices ) eval_bool = block_keep_mask.unsqueeze(-1) & valid_label_mask & (target_loss_mask > 0.5) - eval_bool = eval_bool.to(torch.int32).cumprod(dim=-1).bool() + if speculators and attention_mask is not None: + # Padded BSHD rows keep the same label domain as an unpadded row. + eval_bool &= attention_mask.gather(1, safe_label_indices.flatten(1)).view_as(eval_bool).bool() + if not speculators: + eval_bool = eval_bool.to(torch.int32).cumprod(dim=-1).bool() eval_mask = eval_bool.float() # [B, nb, bs] eval_mask = eval_mask * ( torch.arange(self.block_size, device=device) >= self.prediction_start @@ -272,7 +295,6 @@ def forward( decay_weight_mask = eval_mask * self._decay_weights(device) local_den = decay_weight_mask.sum() - warmup_kind = self._indexer_warmup_kind() if warmup_kind is not None: if warmup_kind == "csa": numerator, loss_components = self._csa_training_loss( @@ -308,6 +330,12 @@ def forward( (numerator, local_den.detach()), ) + confidence_weights = None + if speculators: + decay_weight_mask, confidence_weights, local_den = self._speculators_row_weights( + eval_mask, decay_weight_mask + ) + hidden_4d = draft_hidden.view(bsz, n_blocks, self.block_size, -1) target_ids = torch.gather( input_ids.unsqueeze(1).expand(-1, n_blocks, -1), 2, safe_label_indices @@ -318,12 +346,12 @@ def forward( if self.uses_target_distribution: if last_hidden_states is None: raise ValueError( - "DSpark L1/confidence losses require target last_hidden_states; set " + "DSpark teacher-dependent losses require target last_hidden_states; set " "inference.store_last_hidden_states=true in the run config." ) # target distribution for the token at label_indices = target LM head # applied to the target hidden one position earlier (anchor+j). - tgt_idx = (safe_label_indices - 1).clamp(min=0) # [B, nb, bs] + tgt_idx = teacher_indices # [B, nb, bs] aligned_hidden = self._align_target_hidden_states( last_hidden_states, tgt_idx, @@ -470,14 +498,25 @@ def forward( # Keep only small FP32 L1 statistics across checkpoint replay; full # vocabulary logits still follow the projection budget above. loss_cache = {} if checkpoint_heads and self._reuse_l1_statistics else None - values = (*values, cached_teacher, cached_draft, cached_positions, loss_cache) + conf_weights = confidence_weights[:, start:stop] if confidence_weights is not None else None + if conf_weights is not None and selection is not None: + conf_weights = select_blocks(conf_weights, selection) + values = (*values, cached_teacher, cached_draft, cached_positions, loss_cache, conf_weights) if checkpoint_heads: terms = checkpoint(self._prediction_terms, *values, use_reentrant=False, preserve_rng_state=False) + elif (speculators and n_blocks <= chunk_size and bsz == 1 and cached_teacher is None + and self._reuse_head_projections and self._reuse_context_teacher_projection + and self.draft_context_parallel_size == 1 and not lm_head_weight.requires_grad + and not last_hidden_states.requires_grad and last_hidden_states.shape[1] == seq_len): + positions = tgt_idx[:, start:stop] + if selection is not None: + positions = select_blocks(positions, selection) + terms = self._prediction_terms(*values, teacher_context=(last_hidden_states, positions)) else: terms = self._prediction_terms(*values) head_terms.append(terms) - ce_num, l1_num, conf_num, ce_position_sum, correct_position_sum, acceptance_totals = ( + ce_num, distance_num, conf_num, ce_position_sum, correct_position_sum, acceptance_totals = ( sum(parts[1:], parts[0]) for parts in zip(*head_terms, strict=True) ) @@ -485,7 +524,7 @@ def forward( # before compensating for FSDP's mean gradient reduction. loss_numerator = ( self.ce_loss_alpha * ce_num - + self.l1_loss_alpha * l1_num + + self.l1_loss_alpha * distance_num + self.confidence_head_alpha * conf_num ) moe_numerator, moe_components = self._moe_training_loss( @@ -513,7 +552,7 @@ def forward( component_denominator = local_den.detach() loss_components = { "ce_loss": (ce_num.detach(), component_denominator), - "l1_loss": (l1_num.detach(), component_denominator), + self.loss_recipe.distribution_metric: (distance_num.detach(), component_denominator), "confidence_loss": (conf_num.detach(), component_denominator), **moe_components, **csa_components, @@ -561,26 +600,60 @@ def _project_draft(self, hidden, head_weight): def _prediction_terms(self, hidden, target_ids, prev_ids, head_weight, aligned_target_hidden, eval_mask, weights, cached_target_logits=None, - cached_draft_logits=None, cached_target_positions=None, loss_cache=None): + cached_draft_logits=None, cached_target_positions=None, loss_cache=None, + confidence_weights=None, *, teacher_context=None): """Additive DSpark head terms for a bounded block slice.""" # Explicit token-major GEMM: a block slice can be non-contiguous even # though the complete hidden stream is contiguous. Do not broadcast a # vocabulary projection over many tiny [block_size, hidden] matrices. logits = (cached_draft_logits if cached_draft_logits is not None else self._project_draft(hidden, head_weight)) - if self.draft_model.markov_head is not None: - logits = self.draft_model.markov_head.apply_block_logits(logits, token_ids=prev_ids) vocab = logits.shape[-1] - ce = F.cross_entropy(logits.reshape(-1, vocab), target_ids.reshape(-1), - reduction="none").view_as(weights) - ce_num = (ce * weights).sum() - l1_num = logits.new_zeros((), dtype=torch.float32) + speculators = self.loss_recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 + cute_loss = (speculators and self._use_l1_kernel_optimizations + and use_cutedsl_speculators_loss(logits, cached_target_logits, + indexed=cached_target_positions is not None)) + markov = self.draft_model.markov_head + prev_emb = None + fused_correct = None + if markov is not None: + if (cute_loss and logits.numel() // vocab >= 256 and vocab % 8 == 0 + and markov.markov_rank % 8 == 0 and markov.markov_w2.weight.is_contiguous() + and markov.markov_w2.weight.dtype == logits.dtype): + from aspec.kernels.cutedsl.speculators_loss import markov_residual_projection + + prev_emb = markov.get_prev_embeddings(prev_ids) + logits = markov_residual_projection(logits, prev_emb, markov.markov_w2.weight) + calls = self.l1_kernel_invocations + calls["cutedsl_markov_projection"] = calls.get("cutedsl_markov_projection", 0) + 1 + else: + logits = markov.apply_block_logits(logits, token_ids=prev_ids) + if not speculators: + ce = F.cross_entropy(logits.reshape(-1, vocab), target_ids.reshape(-1), + reduction="none").view_as(weights) + distance_num = logits.new_zeros((), dtype=torch.float32) accept_rate = None if aligned_target_hidden is not None: + # A long prompt can contain a short supervised region. Many anchors + # then share teacher positions even when projecting the full context + # would cost more than projecting all query rows. + # Small teacher projections do not amortize sorting and its host + # synchronization on SM100. Keep its regular CuTeDSL head path. + if (cute_loss and teacher_context is not None + and torch.cuda.get_device_capability(logits.device) == (9, 0)): + context, positions = teacher_context + unique, inverse = torch.unique(positions.reshape(-1), return_inverse=True) + # Small reductions cost more in the indexed path on Hopper. + if inverse.numel() - unique.numel() >= 64: + cached_target_logits = F.linear(context[0].index_select(0, unique), head_weight)[None] + cached_target_positions = inverse.view_as(positions) + self.loss_head_cached_context_tokens_observed = unique.numel() + self.loss_head_cached_teacher_blocks_observed = hidden.shape[1] indexed_teacher = ( - not self.l1_sampling_enabled - and self._use_l1_kernel_optimizations and cached_target_positions is not None - and use_sm90_tiled_l1(logits, cached_target_logits, True) + cached_target_positions is not None and (cute_loss or ( + not speculators and not self.l1_sampling_enabled and self._use_l1_kernel_optimizations + and use_sm90_tiled_l1(logits, cached_target_logits, True) + )) ) target_indices = None if indexed_teacher: @@ -599,36 +672,54 @@ def _prediction_terms(self, hidden, target_ids, prev_ids, head_weight, target_logits = cached_target_logits if cached_target_logits is not None else F.linear( aligned_target_hidden.reshape(-1, aligned_target_hidden.shape[-1]), head_weight ).view_as(logits) - if not self.l1_sampling_enabled and ( + if cute_loss and not target_logits.requires_grad and target_logits.dtype == logits.dtype: + from aspec.kernels.cutedsl.speculators_loss import speculators_cutedsl_loss + + ce, distance, target_ids, fused_correct = speculators_cutedsl_loss( + logits, target_logits, target_indices) + self.l1_loss_backend = "cutedsl_ce_tv" + kernel = ("indexed_" if indexed_teacher else "") + self.l1_loss_backend + calls = self.l1_kernel_invocations + calls[kernel] = calls.get(kernel, 0) + 1 + elif speculators: + ce, distance, target_ids = speculators_token_losses(logits, target_logits) + self.l1_loss_backend = "torch_compiled_tv" + elif not self.l1_sampling_enabled and ( indexed_teacher or use_sm90_tiled_l1(logits, target_logits) ): from aspec.models.ops.tiled_l1 import tiled_l1_from_logits - l1 = tiled_l1_from_logits(logits, target_logits, target_indices=target_indices, - _replay_cache=loss_cache) + distance = tiled_l1_from_logits(logits, target_logits, target_indices=target_indices, + _replay_cache=loss_cache) self.l1_loss_backend = "tiled_fp32" kernel = ("indexed_" if indexed_teacher else "") + self.l1_loss_backend self.l1_kernel_invocations[kernel] = self.l1_kernel_invocations.get(kernel, 0) + 1 else: - l1 = (compiled_l1_from_logits(logits, target_logits, *self.l1_sampling) - if self.l1_sampling_enabled else compiled_l1_from_logits(logits, target_logits)) + distance = (compiled_l1_from_logits(logits, target_logits, *self.l1_sampling) + if self.l1_sampling_enabled else compiled_l1_from_logits(logits, target_logits)) self.l1_loss_backend = "torch_compiled" if self.l1_loss_alpha > 0: - l1_num = (l1 * weights).sum() - accept_rate = (1.0 - 0.5 * l1).clamp(0.0, 1.0) + distance_num = (distance * weights).sum() + accept_rate = 1.0 - distance if speculators else (1.0 - 0.5 * distance).clamp(0.0, 1.0) + ce_num = (ce * weights).sum() conf_num = logits.new_zeros((), dtype=torch.float32) if self.draft_model.confidence_head is not None and self.confidence_head_alpha > 0: if self.draft_model.confidence_head_with_markov: - prev_emb = self.draft_model.markov_head.get_prev_embeddings(prev_ids).to(hidden.dtype) + # Keep the two embedding backward reductions independent. Sharing + # this lookup changes BF16 scatter/add rounding for repeated IDs. + prev_emb = markov.get_prev_embeddings(prev_ids).to(hidden.dtype) conf_features = torch.cat([hidden, prev_emb], dim=-1) else: conf_features = hidden - prediction = self.draft_model.confidence_head(conf_features).float() + prediction = self.draft_model.confidence_head(conf_features) + confidence_target = accept_rate.detach() + if speculators: + confidence_target = confidence_target.to(prediction.dtype) conf_num = (F.binary_cross_entropy_with_logits( - prediction, accept_rate.detach(), reduction="none" - ) * weights).sum() + prediction.float(), confidence_target.float(), reduction="none" + ) * (weights if confidence_weights is None else confidence_weights)).sum() with torch.no_grad(): - correct = (logits.argmax(-1) == target_ids) & (eval_mask > 0.5) + correct = ((logits.argmax(-1) == target_ids) if fused_correct is None else fused_correct) & (eval_mask > 0.5) position_loss = (ce * eval_mask).sum((0, 1)) position_correct = correct.float().sum((0, 1)) acceptance_totals = ( @@ -639,4 +730,4 @@ def _prediction_terms(self, hidden, target_ids, prev_ids, head_weight, (2, self.block_size + 1), dtype=torch.float32, ) ) - return ce_num, l1_num, conf_num, position_loss, position_correct, acceptance_totals + return ce_num, distance_num, conf_num, position_loss, position_correct, acceptance_totals diff --git a/aspec/models/ops/cudnn_flex_attention.py b/aspec/models/ops/cudnn_flex_attention.py index 968bac8..5eb7c12 100644 --- a/aspec/models/ops/cudnn_flex_attention.py +++ b/aspec/models/ops/cudnn_flex_attention.py @@ -382,7 +382,10 @@ def cudnn_flex_attention( raise ValueError("materialized MLA currently requires Hq=Hkv") effective_mode = resolve_cudnn_gqa_mode(gqa_mode, physical_gqa=physical_gqa) + deterministic = torch.are_deterministic_algorithms_enabled() if effective_mode == "internal": + if deterministic: + raise NotImplementedError("Deterministic backward is unavailable for internal cuDNN GQA") from aspec.models.ops.cudnn_gqa_backward import internal_gqa_attention return internal_gqa_attention( @@ -423,6 +426,7 @@ def cudnn_flex_attention( value, mask_plan=plan, softmax_scale=scale, + deterministic=deterministic, return_lse=return_lse, ) if return_lse: diff --git a/aspec/models/ops/loss.py b/aspec/models/ops/loss.py index 64fe4e8..ad2d4be 100644 --- a/aspec/models/ops/loss.py +++ b/aspec/models/ops/loss.py @@ -18,6 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from functools import lru_cache +from importlib.util import find_spec from typing import Optional import torch @@ -46,6 +48,31 @@ def use_sm90_tiled_l1(logits: torch.Tensor, target: torch.Tensor, indexed: bool ) +@lru_cache(maxsize=1) +def _have_cutedsl_loss(): + return all(find_spec(pkg) is not None for pkg in ("quack", "cutlass")) + + +def use_cutedsl_speculators_loss(logits, teacher=None, indexed=False): + """At most two reference normalization blocks and one CTA of overlap partials.""" + return ( + not torch.compiler.is_compiling() and logits.is_cuda + and logits.ndim > 0 and logits.numel() > 0 and logits.dtype == torch.bfloat16 + and logits.is_contiguous() and logits.data_ptr() % 16 == 0 + and 1 <= logits.shape[-1] <= 196608 + and (teacher is None or ( + teacher.device == logits.device and teacher.dtype == logits.dtype + and teacher.is_contiguous() and teacher.data_ptr() % 16 == 0 + and not teacher.requires_grad + and (teacher.shape == logits.shape if not indexed else + teacher.ndim == 3 and teacher.shape[0] == logits.shape[0] + and teacher.shape[-1] == logits.shape[-1]) + )) + and torch.cuda.get_device_capability(logits.device) in {(9, 0), (10, 0)} + and _have_cutedsl_loss() + ) + + def _forward_kl_from_logits(logits: torch.Tensor, target_p: torch.Tensor) -> torch.Tensor: logits_f32 = logits.float() return torch.logsumexp(logits_f32, dim=-1) - (target_p * logits_f32).sum(-1) @@ -91,6 +118,19 @@ def compiled_l1_from_logits(logits: torch.Tensor, target_logits: torch.Tensor, return (draft_probs - target_probs).abs().sum(dim=-1) +@torch.compile(dynamic=None) +def speculators_token_losses(logits: torch.Tensor, teacher_logits: torch.Tensor): + """Reimplement Speculators b9c51bc fused CE/TV with FP32 reductions.""" + labels = teacher_logits.argmax(-1) + draft = logits.float() + ce = draft.logsumexp(-1) - draft.gather(-1, labels.unsqueeze(-1)).squeeze(-1) + q = draft.softmax(-1) + p = teacher_logits.detach().float().softmax(-1) + # The published fused TV backward assigns equality to the Draft branch. + overlap = torch.where(q <= p, q, p).sum(-1) + return ce, 1.0 - overlap, labels + + def _project( hidden: torch.Tensor, lm_head_weight: torch.Tensor, diff --git a/aspec/models/target/target_utils.py b/aspec/models/target/target_utils.py index f0a2cfb..77c2b9a 100644 --- a/aspec/models/target/target_utils.py +++ b/aspec/models/target/target_utils.py @@ -29,7 +29,8 @@ import torch.nn as nn from huggingface_hub import snapshot_download from safetensors import safe_open -from transformers import AutoConfig + +from aspec.config.utils import load_target_config logger = logging.getLogger(__name__) @@ -85,7 +86,7 @@ def from_pretrained( dtype: torch.dtype = torch.bfloat16, trust_remote_code: bool = False, ) -> "TargetLMHead": - config = AutoConfig.from_pretrained( + config = load_target_config( model_path, cache_dir=cache_dir, trust_remote_code=trust_remote_code ) instance = cls(config, trust_remote_code=trust_remote_code) @@ -324,7 +325,7 @@ def load_synced_target_lm_head( trust_remote_code=trust_remote_code, ) else: - config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code) + config = load_target_config(model_path, trust_remote_code=trust_remote_code) head = TargetLMHead(config, trust_remote_code=trust_remote_code) if load_norm: head._init_norm_structure() diff --git a/aspec/planning.py b/aspec/planning.py index 4b0d3d8..c71b51f 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -34,7 +34,7 @@ from omegaconf import DictConfig, ListConfig, OmegaConf -from aspec.config.draft_algorithm import DRAFT_ALGORITHMS +from aspec.config.draft_algorithm import DRAFT_ALGORITHMS, DSparkLossRecipe from aspec.config.draft_attention import ( draft_thd_contract, resolve_draft_attention, @@ -181,6 +181,7 @@ def to_json(self) -> str: def to_runtime_args(self) -> argparse.Namespace: values = copy.deepcopy(dict(self._runtime_values)) values["aspec_plan_id"] = self.plan_id + values["aspec_target_family"] = self.workload.models.get("target", {}).get("family") values["aspec_feature_contract"] = copy.deepcopy(dict(self.workload.features)) values["aspec_locality_domains"] = copy.deepcopy( list(self.policy.resources.get("locality_domains", ())) @@ -444,6 +445,12 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]: except ValueError as exc: raise PlanCompileError(str(exc)) from exc backend = _select(config, "training.attention_backend") + try: + recipe = DSparkLossRecipe(_select(config, "training.dspark_loss_recipe", "deepspec")) + except ValueError as exc: + raise PlanCompileError(f"training.dspark_loss_recipe must be one of {list(map(str, DSparkLossRecipe))}") from exc + if recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 and family != "dspark": + raise PlanCompileError("training.dspark_loss_recipe=speculators_redhat_v1 requires a DSpark Draft") l1_sampling_enabled = False for side in ("target", "draft"): prefix = f"training.dspark_l1_{side}_" @@ -460,6 +467,8 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]: raise PlanCompileError(f"{prefix}top_k must be -1 or a positive integer") l1_sampling_enabled |= (temperature, top_p, top_k) != (1.0, 1.0, -1) if l1_sampling_enabled: + if recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1: + raise PlanCompileError("speculators_redhat_v1 requires unfiltered distributions at temperature one") indexer_warmup = any((attention.get(kind) or {}).get("training_stage") == "indexer_warmup" for kind in ("csa", "qsa", "dsa")) if family != "dspark" or _select(config, "training.dspark_l1_loss_alpha") <= 0 or indexer_warmup: @@ -785,6 +794,12 @@ def _resolve_target_family(target_path: str) -> str: local_config = _read_local_target_config(target_path) markers: list[str] = [target_path] if local_config is not None: + if ( + local_config.get("model_type") == "glm5_next" + and local_config.get("architectures") == ["Glm5NextForConditionalGeneration"] + and (local_config.get("text_config") or {}).get("num_hidden_layers") == 45 + ): + return "glm5_3_flash" markers.append(str(local_config.get("model_type", ""))) architectures = local_config.get("architectures", []) if isinstance(architectures, list): @@ -795,6 +810,8 @@ def _resolve_target_family(target_path: str) -> str: and "BailingMoeV3ForCausalLM" in architectures ): return "ling3" + elif target_path.lower().rstrip("/") in {"zai-org/glm-5.3-flash", "zhipuai/glm-5.3-flash"}: + return "glm5_3_flash" elif target_path.rstrip("/").split("/")[-1].lower() in { "ling-3.0-tiny", "ling-3.0-flash", "ling-3.1-flash", }: @@ -802,7 +819,7 @@ def _resolve_target_family(target_path: str) -> str: if any("qwen3" in marker.lower().replace("_", "") for marker in markers): return "qwen3" raise PlanCompileError( - "The supported target scope is Qwen3 and Ling-3.x, but the target model family " + "The supported target scope includes Qwen3, Ling-3.x and GLM-5.3-Flash, but the target model family " f"could not be established from {target_path!r}" ) @@ -1452,6 +1469,7 @@ def _workload( elif family == "dspark": objective = { "family": family, + "loss_recipe": _select(config, "training.dspark_loss_recipe", "deepspec"), "block_size": _select(config, "training.dflash_block_size"), "num_anchors": _select(config, "training.dspark_num_anchors"), "num_target_layers": _select(config, "training.dspark_num_target_layers"), @@ -2018,6 +2036,9 @@ def compile_execution_plan( source["resolved_config_sha256"] = _resolved_config_sha256(config) draft = _load_draft_descriptor(config) target_family = _resolve_target_family(target_path) + if target_family == "glm5_3_flash" and _select(config, "inference.target_prefill_memory_optimized", False): + raise PlanCompileError("GLM-5.3-Flash uses buffered native DFlash feature capture") + runtime = _runtime_values(config, draft, source_context_parallel_size=source_context_parallel_size) try: context_compiler = compile_offline_context_contract if offline else compile_sglang_context_contract diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 3738fab..17cbab8 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -25,6 +25,7 @@ import os import time from pathlib import Path +from types import SimpleNamespace from typing import Any import torch @@ -87,6 +88,7 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: "ce_loss_alpha", "l1_loss_alpha") if prefix == "dspark": topology["training_objective"]["lm_head_precision"] = getattr(args, "dspark_lm_head_precision", "bf16") + topology["training_objective"]["loss_recipe"] = getattr(args, "dspark_loss_recipe", "deepspec") fields += ("confidence_head_alpha",) for side in ("target", "draft"): for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)): @@ -163,6 +165,8 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec saved_objective = (saved_topology or {}).get("training_objective") or {} if saved_objective.get("lm_head_precision", "bf16") != getattr(args, "dspark_lm_head_precision", "bf16"): raise RuntimeError("Exact resume cannot change dspark_lm_head_precision; use weights-only initialization") + if saved_objective.get("loss_recipe", "deepspec") != current_topology["training_objective"]["loss_recipe"]: + raise RuntimeError("Exact resume cannot change dspark_loss_recipe; use weights-only initialization") for side in ("target", "draft"): for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)): key = f"l1_{side}_{name}" @@ -265,11 +269,19 @@ def matches_saved(value, current): saved_topology = dict(saved_topology) saved_topology["draft_model_semantics"] = draft_model_semantics(saved_topology["draft_model_semantics"]) + saved_policy = saved_topology.get("draft_attention_policy") or {} + if (current_topology.get("draft_algorithm") in {"dspark", "dflash"} + and "draft_block_attention" in saved_policy and "draft_block_attention" in current_policy): + from aspec.config.draft_attention import draft_block_attention_matches + + if draft_block_attention_matches(saved_policy["draft_block_attention"], current_policy): + saved_topology = {**saved_topology, "draft_attention_policy": { + **saved_policy, "draft_block_attention": current_policy["draft_block_attention"], + }} # Early native MTP checkpoints used the generic Draft policy label even # though their only decoder path was official causal, teacher-forced MTP. # Normalize that known serialization error at this boundary; never accept # a genuinely different attention policy or rewrite the stored checkpoint. - saved_policy = saved_topology.get("draft_attention_policy") or {} saved_semantics = saved_topology.get("draft_model_semantics") or {} objective = saved_topology.get("training_objective") or {} if (saved_topology.get("draft_algorithm") == current_topology.get("draft_algorithm") == "mtp" @@ -825,8 +837,68 @@ def resolve_initial_draft_checkpoint(path: str) -> Path: return checkpoint_path +def _speculators_dspark_rope_policy(config): + from aspec.config.utils import resolve_dflash_rope_parameters + + params = resolve_dflash_rope_parameters(config) + kind = params["rope_type"] + policy = (kind, params["rope_theta"], 1.0 if kind == "default" else params["factor"]) + if kind != "yarn": + return policy + # Use the runtime's YaRN implementation to normalize optional defaults + # and compare every parameter that affects frequencies or attention scale. + from transformers import PretrainedConfig + from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS + + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + resolved = PretrainedConfig( + rope_parameters=params, head_dim=dim, hidden_size=dim, num_attention_heads=1, + max_position_embeddings=config.max_position_embeddings, + ) + frequencies, attention_scale = ROPE_INIT_FUNCTIONS[kind](resolved, device=torch.device("cpu")) + return (*policy, tuple(frequencies.tolist()), float(attention_scale)) + + +def _bind_speculators_dspark(published, config, tensors, target_bindings): + """Validate the published Draft's layout and replace its frozen head with the Target binding.""" + # Speculators captures before a layer; ASpec numbers completed layer outputs. + policies = { + "aux_hidden_state_layer_ids": [layer + 1 for layer in config.target_layer_ids], + "mask_token_id": config.mask_token_id, + "block_size": config.block_size, + "sample_from_anchor": True, + "sliding_window_non_causal": getattr(config, "sliding_window_non_causal", True), + } + # DSpark overrides DFlash's anchor-sampling default in the pinned schema. + published = {"sliding_window_non_causal": False, "sample_from_anchor": True, **published} + if any(published.get(k) != expected for k, expected in policies.items()): + raise ValueError("Speculators DSpark capture, mask, block or attention policy mismatch") + geometry = published["transformer_layer_config"] + fields = ("sliding_window", "layer_types", "rms_norm_eps", "num_attention_heads", + "num_key_value_heads", "head_dim") + if any(geometry.get(field) != getattr(config, field) for field in fields): + raise ValueError("Speculators DSpark layer geometry differs from the configured Draft") + published_config = SimpleNamespace(**{ + "hidden_size": config.hidden_size, + "max_position_embeddings": config.max_position_embeddings, + **geometry, + }) + if _speculators_dspark_rope_policy(published_config) != _speculators_dspark_rope_policy(config): + raise ValueError("Speculators DSpark RoPE policy differs from the configured Draft") + + from aspec.models.draft.base import load_tensor_from_pretrained + + model_path, embedding_key, lm_head_key = target_bindings + for name, key in (("embed_tokens.weight", embedding_key), ("lm_head.weight", lm_head_key)): + if name not in tensors or not torch.equal(tensors[name], load_tensor_from_pretrained(model_path, key)): + raise ValueError(f"Published DSpark {name} differs from the frozen Target binding") + del tensors["lm_head.weight"] + + def load_initial_draft_weights( - draft_model: torch.nn.Module, path: str, *, allow_target_embedding: bool = False + draft_model: torch.nn.Module, path: str, *, allow_target_embedding: bool = False, + target_bindings: tuple[str, str, str] | None = None, + require_speculators_config: bool = False, ) -> Path: """Load a published draft checkpoint into a freshly constructed draft model. @@ -836,12 +908,26 @@ def load_initial_draft_weights( would otherwise leave part of the model randomly initialised for a whole run. """ checkpoint_path = resolve_initial_draft_checkpoint(path) + manifest_path = checkpoint_path.parent / "export_manifest.json" + native = manifest_path.is_file() and json.loads(manifest_path.read_text()).get("storage_format") == "aspec_native" + published = None + if target_bindings and not native: + config_path = checkpoint_path.parent / "config.json" + if config_path.is_file(): + candidate = json.loads(config_path.read_text()) + if candidate.get("speculators_model_type") == "dspark": + published = candidate + if require_speculators_config and published is None: + raise ValueError( + "GLM DSpark warm-start requires the original Speculators DSpark config.json " + "for capture policy and frozen Target binding validation" + ) with safe_open(checkpoint_path, framework="pt", device="cpu") as f: tensors = {key: f.get_tensor(key) for key in f.keys()} - manifest_path = checkpoint_path.parent / "export_manifest.json" - native = manifest_path.is_file() and json.loads(manifest_path.read_text()).get("storage_format") == "aspec_native" conditioning = getattr(draft_model.config, "mtp_conditioning", None) + if published is not None: + _bind_speculators_dspark(published, draft_model.config, tensors, target_bindings) transition = False if conditioning: saved_config_path = checkpoint_path.parent / "config.json" diff --git a/aspec/training/dflash_trainer.py b/aspec/training/dflash_trainer.py index fbd5a28..6289d5a 100644 --- a/aspec/training/dflash_trainer.py +++ b/aspec/training/dflash_trainer.py @@ -150,8 +150,18 @@ def init_model( draft_model.initialize_mtp_conditioning(target_model_path) initial_path = getattr(self.args, "initial_draft_model_path", None) if initial_path: + target_bindings = ( + target_model_path, + getattr(self.args, "embedding_key", "model.embed_tokens.weight"), + getattr(self.args, "lm_head_key", "lm_head.weight"), + ) + is_dspark = config.model_type == "qwen3_dspark" checkpoint.load_initial_draft_weights( - draft_model, initial_path, allow_target_embedding=True + draft_model, initial_path, allow_target_embedding=True, + target_bindings=target_bindings if is_dspark else None, + require_speculators_config=( + is_dspark and getattr(self.args, "aspec_target_family", None) == "glm5_3_flash" + ), ) draft_model.load_embedding( target_model_path, @@ -205,7 +215,7 @@ def init_model( self._anchor_slot_offset = self.dflash.prediction_start if hasattr(self, "confidence_head_alpha"): self._extra_loss_component_keys = [ - "ce_loss", "l1_loss", "confidence_loss", + "ce_loss", self.dflash.loss_recipe.distribution_metric, "confidence_loss", *(f"suffix_accept_rate@{index}" for index in range( self.block_size - self.dflash.prediction_start)), "suffix_tau_probabilistic", @@ -366,6 +376,8 @@ def get_feature_contract_status(self) -> dict: } ) status["execution_metadata"] = execution + if hasattr(self.dflash, "loss_recipe"): + execution["dspark_loss_recipe"] = [self.dflash.loss_recipe.value] status["draft_attention"] = {**observed, "forward_observed": True} if self._indexer_warmup_contract() is not None: main_frozen = not any( diff --git a/aspec/training/dspark_trainer.py b/aspec/training/dspark_trainer.py index f53586e..42d6055 100644 --- a/aspec/training/dspark_trainer.py +++ b/aspec/training/dspark_trainer.py @@ -27,6 +27,7 @@ from argparse import Namespace +from aspec.config.draft_algorithm import DSparkLossRecipe from aspec.models.draft.dspark import ( DSparkConfig, DSparkDraftModel, @@ -58,7 +59,8 @@ def __init__(self, args: Namespace): # Reuse the shared component reducer: all acceptance counts join the # existing batched collective and host transfer for train and eval. self._extra_loss_component_keys = [ - *self._extra_loss_component_keys, + "ce_loss", DSparkLossRecipe(getattr(args, "dspark_loss_recipe", "deepspec")).distribution_metric, + "confidence_loss", *(f"accept_rate@{index}" for index in range(self.block_size)), "tau_probabilistic", ] @@ -90,6 +92,7 @@ def _build_training_wrapper(self, draft_model): return DSparkModel( draft_model=draft_model, block_size=self.block_size, + loss_recipe=getattr(self.args, "dspark_loss_recipe", "deepspec"), num_anchors=self.num_anchors, loss_decay_gamma=self.loss_decay_gamma, ce_loss_alpha=self.ce_loss_alpha, diff --git a/configs/recipes/glm5_3_flash_dspark.yaml b/configs/recipes/glm5_3_flash_dspark.yaml new file mode 100644 index 0000000..5f403f4 --- /dev/null +++ b/configs/recipes/glm5_3_flash_dspark.yaml @@ -0,0 +1,103 @@ +# GLM-5.3-Flash FP8 Target, TP4, text-only DSpark training. +model: + target_model_path: ${oc.env:ASPEC_TARGET_MODEL,zai-org/GLM-5.3-Flash} + native_context_length: 1048576 + trust_remote_code: false + draft_model_config: aspec/config/dspark_glm5_3_flash_draft_config.json + embedding_key: model.language_model.embed_tokens.weight + lm_head_key: lm_head.weight + norm_key: model.language_model.norm.weight + +dataset: + eval_interval: 100 + eval_micro_batch_size: 1 + train_data_path: ${oc.env:ASPEC_TRAIN_DATA} + eval_data_path: ${oc.env:ASPEC_EVAL_DATA} + renderer: glm5_3_flash + prompt_key: conversations + min_loss_tokens: 32 + shuffle_dataset: true + +training: + attention_backend: cudnn_flex_attention + draft_context_parallel_size: 1 + micro_batch_size: 1 + draft_accumulation_steps: 16 + learning_rate: 6e-4 + min_lr: 6e-5 + weight_decay: 0.0 + max_concurrent_batches: 1 + streaming_dispatch_with_training: true + pipeline_dispatch_with_training: true + max_grad_norm: 1.0 + max_seq_length: 8192 + num_epochs: 3 + seed: 42 + training_num_gpus_per_node: 1 + training_num_nodes: 1 + placement_strategy: inference_first + fsdp_strategy: REPLICATE + fsdp_reduce_dtype: bfloat16 + prefetch_depth: 1 + save_interval: 100 + save_per_epoch: false + save_final_checkpoint: true + max_checkpoints: 2 + warmup_ratio: 0.04 + dflash_block_size: 8 + dflash_packing: false + dspark_loss_recipe: deepspec + dspark_num_anchors: 512 + dspark_num_target_layers: 6 + dspark_loss_decay_gamma: 4.0 + dspark_ce_loss_alpha: 0.1 + dspark_l1_loss_alpha: 0.9 + dspark_confidence_head_alpha: 1.0 + +inference: + inference_engine_type: sgl + store_last_hidden_states: true + target_prefill_memory_optimized: false + inference_num_gpus: 4 + inference_num_gpus_per_engine: 4 + inference_num_gpus_per_node: 5 + inference_fetch_batch: 8 + inference_buffer_threshold: 2 + max_sample_pool_size: 8 + inference_batch_size: 2 + sglang: + # Cold sharded checkpoint loading can exceed the generic five-minute budget. + init_timeout: 1200 + tp_size: 4 + extra_args: + watchdog_timeout: 1800 + dsa_prefill_backend: tilelang + dsa_decode_backend: tilelang + kv_cache_dtype: bf16 + moe_runner_backend: deep_gemm + mem_fraction_static: 0.9 + +mooncake: + data_plane: transfer_engine + protocol: nvlink_intra + require_nvlink: true + enable_gpu_direct: true + allow_host_fallback: false + enable_hard_pin: true + max_seq_len: 8192 + hidden_dim: 4096 + global_segment_size: 16GB + local_buffer_size: 4GB + host_buffer_size: 1GB + gpu_ring_slots: 8 + source_gpu_ring_slots: 8 + destination_gpu_ring_slots: 1 + kv_lease_ttl_s: 600 + get_retry_max_wait_seconds: 600 + +logging: + report_to: none +debug: + enable_perf_metrics: true +output_dir: outputs/glm5-3-flash-dspark +cache_dir: cache/glm5-3-flash-dspark diff --git a/docs/contributing/validation.md b/docs/contributing/validation.md index 72dbc60..86fe834 100644 --- a/docs/contributing/validation.md +++ b/docs/contributing/validation.md @@ -31,6 +31,22 @@ CUDA_VISIBLE_DEVICES='' ASPEC_RUN_RAY_INTEGRATION=1 \ This verifies driver retention until reader-process fencing, including Ray's automatic exit cleanup. It does not certify unreturning CUDA/RDMA operations, forced driver loss or host partitions. +## Native cuDNN repeatability + +The native cuDNN adapter forwards `torch.use_deterministic_algorithms(True)` to the Frontend's +deterministic backward selector. A fixed RNG seed alone does not select that path. The internal +joint-GQA path rejects deterministic mode rather than silently running a nondeterministic kernel. + +On an allocated GPU in the pinned Draft environment, check repeated MHA64 outputs and Q/K/V +gradients at 8K and 256K with: + +```bash +CUBLAS_WORKSPACE_CONFIG=:4096:8 ASPEC_RUN_CUDNN_FLEX_INTEGRATION=1 \ + bash tools/run_in_env.sh python -m pytest -q tests/test_cudnn_deterministic_gpu.py +``` + +These checks establish repeatability of the native operator's outputs and gradients. + ## Bounded GPU training, recovery and native assets Activate the installed venv, use the documented environment wrapper, and select only GPUs allocated diff --git a/docs/framework/REFERENCES.md b/docs/framework/REFERENCES.md index fa68237..cb6d3ce 100644 --- a/docs/framework/REFERENCES.md +++ b/docs/framework/REFERENCES.md @@ -25,6 +25,9 @@ the retained source identities; applicable license and notice texts are distribu | [Speculators](https://github.com/vllm-project/speculators) `3702320434bbdaa853f0846a66e6b6b8f1d97aec` | Adapted P-EAGLE model/loss and Qwen3 first-layer fusion; deterministic dense layout and ASpec input/observation boundary | | [Domino](https://github.com/jianuo-huang/Domino) `44e1e6912ba9d301899739a54bfda2ce8826980a` and [arXiv:2605.29707](https://arxiv.org/abs/2605.29707) | Public method and label-layout reference for the GRU correction head; independently implemented on the retained DFlash block backbone | | [SpecForge Domino](https://github.com/sgl-project/SpecForge/pull/571) merge `7de39e32fc845caf59bca9841595145d24e1796f` | Adapted Domino objective, predecessor alignment and label-shift handling (MIT); ASpec supplies configuration, fail-closed scope and DFlash integration | +| [Speculators DSpark](https://github.com/vllm-project/speculators/tree/b9c51bcfc134923eae2b6a6f8246ae3500d5c42f/src/speculators/models/dspark) | Reference for the optional Red Hat loss, row reduction, confidence target and causal sliding-block attention | +| [QuACK](https://github.com/Dao-AILab/quack) 0.6.5 ([Apache-2.0 license](../../licenses/QuACK-LICENSE)) | Hopper/Blackwell CE/TV adapts the softmax tile layout and uses the cluster reduction and GEMM epilogue APIs; ASpec implements the upstream loss and BF16 rounding. | +| [Red Hat GLM-5.3-Flash DSpark](https://huggingface.co/RedHatAI/GLM-5.3-Flash-speculator.dspark/tree/738c283a7e860d7dafce5785161466bc556e5cae) | Five-layer Draft geometry, capture layers and frozen Target bindings for published-weight initialization | | [Ling-3.0-flash](https://huggingface.co/inclusionAI/Ling-3.0-flash) `fcc60a52b34c69c6271d0ac8a8cb0815020b6a43` | Adapted MoE routing and independently retained reference class bodies; Apache-2.0 source headers preserved | | [EAGLE](https://github.com/SafeAILab/EAGLE), [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) and Transformers through TorchSpec | Model/data helpers in `aspec/models/draft/base.py`, shared checkpoint reads in `aspec/models/checkpoint.py`, and `aspec/data/preprocessing.py`; original source headers retain the transitive attribution | diff --git a/docs/support.md b/docs/support.md index 49f4391..02e1e98 100644 --- a/docs/support.md +++ b/docs/support.md @@ -10,6 +10,7 @@ and tests do not imply CPU training support. | Path | Implemented behavior | Boundary | | --- | --- | --- | | DSpark / DFlash | Training, exact recovery, evaluation and export; Dense and selected MoE/attention structures | Objectives, feature bindings and assets remain algorithm-specific | +| GLM-5.3-Flash / DSpark | Official FP8 TP4 Target, five-layer BF16 Draft, DeepSpec and Red Hat loss recipes, checkpoint recovery and native export | Text-only, CP1, one Draft GPU. [Training workflow and scope](user-guide/recipes.md#glm-53-flash-dspark) | | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | | Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; `lambda_base` curriculum, export and consumer acceptance pending; see [scope](user-guide/domino.md) | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index b05d98c..c0e5d21 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -20,7 +20,9 @@ RAY_ADDRESS=local aspec --config configs/recipes/qwen3_8b_eagle3.yaml \ model.target_model_path=/path/to/Qwen3-8B ``` -DSpark retains CE/L1/Markov/confidence semantics; DFlash does not use those auxiliary targets. +DSpark defaults to DeepSpec CE/L1/confidence training with a Markov head. Set +`training.dspark_loss_recipe=speculators_redhat_v1` for teacher-argmax CE and TV. +See [DSpark loss recipes](training.md#dspark-loss-recipes) for the masks and reductions. EAGLE3 uses recurrent TTT and its selected distillation objective. Its pruned vocabulary preserves the checkpoint's `d2t/t2d` mapping across recovery/export. Teacher argmax outside the selected vocabulary is filtered; omitted probability mass is not reassigned to the reduced head. @@ -55,6 +57,9 @@ aspec --config configs/recipes/qwen3_8b_eagle3.yaml --dry-run \ TTT shifts, teacher halos and supervision remain document-local. Unsupported factorized/ambiguous heads fail export rather than producing an unfaithful serving asset. +For text-only DSpark training against the official GLM-5.3-Flash FP8 Target, see the +[GLM training workflow](recipes.md#glm-53-flash-dspark). + ## P-EAGLE core prerequisite The [P-EAGLE model/loss API](peagle.md) provides a dense unsampled Qwen3 parallel diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index 68cb315..0f02371 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -24,6 +24,7 @@ connections on your hardware before training. | `ling-tiny` | Ling-3.0-tiny / Dense DSpark | 1 node, 2 GPUs: 1T + 1D; 4K | Train a small Ling Draft | | `ling-flash` | Ling-3.0-flash / Dense DSpark | 1 node, 3 GPUs: 2T (TP2) + 1D; 4K | Train against a larger Ling Target | | `ling-flash31` | Ling-3.1-flash / MLA + LingMoE DSpark | 1 node, 8 GPUs: 4T (TP4/EP4) + 4D; 4K | Train a structured Draft against the FP8 Ling Target | +| `glm5-flash` | GLM-5.3-Flash FP8 / DSpark | 1 node, 4T (TP4) + 1D, 8K | Train a text Draft from scratch or continue from Red Hat weights. [GLM training guide](#glm-53-flash-dspark) | | `shared-target` | Qwen3-8B / two independent DSpark Drafts | 1 node, 8 GPUs: 4T + 2D × 2; 4K | Compare independent Drafts sharing Target features | | `qwen27b-long` | Qwen3.8-27B / DSpark | 1 node, 8 GPUs: 3 × TP2 Target + 2D (CP2); 256K-class | Distribute long-context training with CP | | `multinode` | Qwen3-8B / DSpark | 2 nodes × 8 GPUs; each 4T + 4D; 4K | Keep features node-local and synchronize gradients across nodes | @@ -130,6 +131,67 @@ Set the same train/eval environment variables, then run this YAML directly with See [decode reproducibility](../support.md#decode-reproducibility) before interpreting acceptance or inference timing. +## GLM-5.3-Flash DSpark + +The recipe uses the official FP8 Target on four GPUs and trains a BF16 Draft with native cuDNN +MHA64 attention (`training.attention_backend=cudnn_flex_attention`) on a fifth GPU. +Attention selection is independent of the DSpark loss recipe. +Keep the attention backend fixed when reproducing or resuming a run: changing it can alter +BF16 rounding and the training trajectory. +Use the checkpoint's chat template with text-only `system`, `user` and `assistant` messages +in disjoint conversation JSONL files. Native assistant messages must use `reasoning_content` +for reasoning text; reasoning alias fields are rejected. +[Official Target](https://modelscope.cn/models/ZhipuAI/GLM-5.3-Flash) · [Draft configuration](../../aspec/config/dspark_glm5_3_flash_draft_config.json) + +Activate the patched Target environment and start the service. DeepGEMM compiles kernels for +the shapes used by the workload instead of precompiling its full shape range: + +```bash +SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 CUDA_VISIBLE_DEVICES=0,1,2,3 aspec target +``` + +In the Draft environment, set the inputs and choose one initialization. The first command trains +with the default DeepSpec loss. The second loads the published BF16 Draft, verifies its frozen +embedding and output head against the Target, and starts a new optimizer with Red Hat's loss. + +```bash +export ASPEC_TARGET_MODEL=/models/GLM-5.3-Flash +export ASPEC_TRAIN_DATA=/data/glm-train.jsonl +export ASPEC_EVAL_DATA=/data/glm-eval.jsonl +export CUDA_VISIBLE_DEVICES=4 + +aspec recipes train glm5-flash --output outputs/glm-scratch +aspec recipes train glm5-flash --output outputs/glm-warm \ + --set model.initial_draft_model_path=/models/GLM-5.3-Flash-speculator.dspark \ + --set training.dspark_loss_recipe=speculators_redhat_v1 \ + --set training.learning_rate=1e-6 --set training.min_lr=1e-7 +``` + +The published Draft directory must contain its original `config.json` and `model.safetensors`. +Its six pre-layer captures (20, 28, 32, 36, 40, 44) correspond to ASpec's zero-based layer +outputs (19, 27, 31, 35, 39, 43). The five Draft layers use a 2,048-token anchor-relative +context window and causal attention within each eight-slot block. +[Red Hat checkpoint and configuration](https://huggingface.co/RedHatAI/GLM-5.3-Flash-speculator.dspark/tree/738c283a7e860d7dafce5785161466bc556e5cae) · [Loss definitions](training.md#dspark-loss-recipes) + +To resume an interrupted ASpec run, use the same loss recipe, dataset and LR horizon. Pass +`--set training.load_path=/absolute/path/to/previous-run/checkpoints` with a fresh output path. +The checkpoint restores the optimizer, scheduler and committed work. To export one completed +iteration, run the following from the Draft environment: + +```bash +aspec export --format aspec --input-dir /path/to/checkpoints/iter_0000005 \ + --output-dir /path/to/glm-draft-asset --target-model-path "$ASPEC_TARGET_MODEL" \ + --embedding-key model.language_model.embed_tokens.weight +python -c 'import sys; from aspec.draft_artifact import load_native_draft; load_native_draft(sys.argv[1])' /path/to/glm-draft-asset +``` + +The maintained GLM path uses text-only 8K inputs, CP1 and one Draft GPU. Warm-start +initialization loads model weights and starts a new optimizer; it does not restore the published +Draft's original optimizer state. Native exports preserve the Draft tensors and frozen Target +bindings. Full-corpus convergence, reproduction of the published Draft's quality, integrated +speculative-serving throughput, multimodal inputs and training the Target's native MTP layer +remain outside the validated scope. + ## Additional structures SWA/CSA/QSA/DSA require their documented structure and stage contracts. MLA materializes K/V. diff --git a/docs/user-guide/training.md b/docs/user-guide/training.md index 66eacbf..6305d4f 100644 --- a/docs/user-guide/training.md +++ b/docs/user-guide/training.md @@ -180,9 +180,42 @@ Run `aspec --config offline.yaml --dry-run`, then `aspec --config offline.yaml`. Dataset metadata binds the Target configuration, tokenizer and weights by SHA256, plus the feature layers, dtype, layout and normalization convention. Replay validates sample checksums and tensor shapes. Set `training.max_seq_length` to cover all recorded sequences; changing preprocessing requires regenerating the dataset. The dataset identity participates in exact-resume validation. -## DSpark L1 sampling distributions - -DSpark L1 supports independent Target and Draft sampling parameters. Both sides default to `temperature=1.0`, `top_p=1.0` and `top_k=-1` (disabled). Configure either side or both: +## DSpark loss recipes + +Set `training.dspark_loss_recipe` in the run YAML or pass it as a CLI override. +Omitting it selects `deepspec`. + +| Recipe | CE label at anchor `a`, slot `k` | Distribution term | Supervision and reduction | +| --- | --- | --- | --- | +| `deepspec` | Data token at `a+k+1` | Full L1 | Use the contiguous supervised prefix starting at `a+1`. Divide the decayed sum by total valid decay weight across the optimizer window. | +| `speculators_redhat_v1` | Teacher argmax at `a+k` | Total variation (TV) | Read the mask at each `a+k` independently. Divide each row's decayed sum by its valid slot count, then average the rows. An empty row contributes zero. | + +Both use the observed token at `a+k` for the Markov head. The confidence head learns +the detached probability overlap. The existing `dspark_l1_loss_alpha` weights the +selected distribution term. Training and evaluation report it as `l1_loss` or `tv_loss`. +The default coefficients are 0.1 for CE, 0.9 for the distribution term and 1 for confidence. +In a packed batch the Red Hat reduction counts each physical packed row once. + +The Red Hat objective reimplements the GLM-5.3-Flash draft's pinned Speculators loss +on ASpec's sampled anchors, with document-local supervision. Anchor selection, Draft +architecture, Target feature capture and optimizer are configured separately. +Exact resume requires the saved recipe. Checkpoints predating this field use `deepspec`. +Use weights-only initialization to change recipes. [Upstream loss and reductions](https://github.com/vllm-project/speculators/tree/b9c51bcfc134923eae2b6a6f8246ae3500d5c42f/src/speculators/losses) + +On Hopper (SM90) and Blackwell (SM100), the Draft training environment runs contiguous BF16 Red Hat logits through +CuTeDSL. The normalizer follows the pinned fused reference's FP32 reduction order: rounding can +otherwise change which branch receives the TV gradient at equal probabilities. CE, TV and greedy +agreement share a tiled vocabulary scan. A cached teacher can be read by position without expanding +its vocabulary tensor. Hopper also projects only unique teacher positions in sparsely supervised +head chunks; Blackwell skips this extra deduplication to avoid the sorting and synchronization cost. +On eligible head chunks the Markov GEMM epilogue rounds its projection to BF16 before adding +the base logits. +Other devices, unsupported layouts or missing CuTeDSL/QuACK +use the portable PyTorch loss. This does not change the DeepSpec recipe or the Target precision. + +### DSpark L1 sampling distributions + +The DeepSpec recipe supports independent Target and Draft sampling parameters. Both sides default to `temperature=1.0`, `top_p=1.0` and `top_k=-1` (disabled). ```yaml training: diff --git a/licenses/QuACK-LICENSE b/licenses/QuACK-LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/licenses/QuACK-LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tests/cpu/test_cudnn_gqa_mode.py b/tests/cpu/test_cudnn_gqa_mode.py index 9d51afc..6686651 100644 --- a/tests/cpu/test_cudnn_gqa_mode.py +++ b/tests/cpu/test_cudnn_gqa_mode.py @@ -65,6 +65,46 @@ def test_pinned_frontend_fails_closed(monkeypatch): joint._validate_version.cache_clear() +@pytest.mark.parametrize("enabled", [False, True]) +def test_native_cudnn_propagates_deterministic_algorithms(monkeypatch, enabled): + import sys + from types import SimpleNamespace + + import torch + + from aspec.models.ops.cudnn_flex_attention import CudnnFlexMask, cudnn_flex_attention + + query = SimpleNamespace(is_cuda=True, dtype=torch.bfloat16, ndim=4, shape=(1, 8, 64, 64)) + mask = CudnnFlexMask(torch.zeros(1, 3, 8, dtype=torch.int32)) + plan, seen = object(), [] + monkeypatch.setattr(CudnnFlexMask, "plan_for", lambda *args, **kwargs: plan) + monkeypatch.setattr(torch, "are_deterministic_algorithms_enabled", lambda: enabled) + + def execute(q, k, v, **kwargs): + seen.append(kwargs) + return q + + monkeypatch.setitem(sys.modules, "cudnn.flex_attention", SimpleNamespace(flex_attn_func=execute)) + assert cudnn_flex_attention(query, query, query, mask=mask, scale=0.125) is query + assert seen == [{"mask_plan": plan, "softmax_scale": 0.125, + "deterministic": enabled, "return_lse": False}] + + +def test_internal_cudnn_determinism_fails_closed(monkeypatch): + from types import SimpleNamespace + + import torch + + from aspec.models.ops.cudnn_flex_attention import CudnnFlexMask, cudnn_flex_attention + + query = SimpleNamespace(is_cuda=True, dtype=torch.bfloat16, ndim=4, shape=(1, 8, 4, 128)) + key = SimpleNamespace(is_cuda=True, dtype=torch.bfloat16, ndim=4, shape=(1, 8, 1, 128)) + mask = CudnnFlexMask(torch.zeros(1, 3, 8, dtype=torch.int32)) + monkeypatch.setattr(torch, "are_deterministic_algorithms_enabled", lambda: True) + with pytest.raises(NotImplementedError, match="Deterministic backward"): + cudnn_flex_attention(query, key, key, mask=mask, scale=0.125, gqa_mode="internal") + + @pytest.mark.parametrize("mode", ["native", "external", "internal"]) @pytest.mark.parametrize("backend", ["cudnn_flex_attention", "math"]) def test_direct_cp_propagates_explicit_route_and_observation(monkeypatch, mode, backend): diff --git a/tests/cpu/test_dspark_loss_recipes.py b/tests/cpu/test_dspark_loss_recipes.py new file mode 100644 index 0000000..c7c0ec3 --- /dev/null +++ b/tests/cpu/test_dspark_loss_recipes.py @@ -0,0 +1,191 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""DSpark loss contracts for the Red Hat Speculators b9c51bc training objective.""" + +import copy +from pathlib import Path + +import pytest +import torch +import torch.nn.functional as F + +from aspec.config import load_config +from aspec.config.train_config import config_to_flat_args +from aspec.models.dspark import DSparkModel +from aspec.models.ops.loss import speculators_token_losses +from aspec.planning import PlanCompileError, compile_execution_plan +from aspec.training.checkpoint import _logical_training_topology, _validate_exact_resume_topology +from aspec.training.dspark_trainer import DSparkTrainer +from tests.test_dspark_loss_alignment import TinyDraft, objective_fixture + +RECIPE = "speculators_redhat_v1" +CONFIG = Path(__file__).resolve().parents[2] / "configs/examples/sglang_qwen3_8b_dspark.yaml" + + +def reference_row(draft, batch, row, anchors, keep): + """Independent dense reference, one upstream-style forward per packed row.""" + positions = anchors[:, None] + torch.arange(4) + prev = draft.markov_head.markov_w1(batch["input_ids"][row, positions]) + hidden = draft.hidden.view(-1, 4, 8) + head = batch["lm_head_weight"] + logits = F.linear(hidden, head) + draft.markov_head.markov_w2(prev) + teacher = F.linear(batch["last_hidden_states"][row, positions], head) + mask = batch["loss_mask"][row, positions] * keep[:, None] + decay = torch.exp(-torch.arange(4) / 4) + labels = teacher.argmax(-1) + ce = F.cross_entropy(logits.flatten(0, 1), labels.flatten(), reduction="none").view_as(mask) + overlap = torch.minimum(logits.softmax(-1), teacher.softmax(-1)).sum(-1) + confidence = draft.confidence_head(torch.cat([hidden, prev], dim=-1)) + bce = F.binary_cross_entropy_with_logits(confidence, overlap.detach(), reduction="none") + return { + key: (value * mask * decay).sum() / (mask.sum() + eps) + for key, value, eps in ( + ("ce_loss", ce, 1e-5), + ("tv_loss", 1 - overlap, 1e-5), + ("confidence_loss", bce, 1e-8), + ) + }, mask + + +@pytest.mark.parametrize("empty_second_row", [False, True]) +def test_speculators_rows_components_and_gradients(monkeypatch, empty_second_row): + torch.manual_seed(91) + draft = TinyDraft() + reference = copy.deepcopy(draft) + # Two blocks per head chunk exercises compaction and checkpoint replay. + model = DSparkModel(draft, block_size=4, num_anchors=2, loss_recipe=RECIPE) + batch, anchors, keep = objective_fixture() + anchors = anchors.masked_fill(~keep, 0).repeat(2, 1) + keep = keep.repeat(2, 1) + keep[1, 1:] = False + if empty_second_row: + keep[1].zero_() + for key in ("input_ids", "loss_mask", "ctx_doc_ids", "last_hidden_states", "base_position_ids"): + batch[key] = torch.cat([batch[key], batch[key].clone()]) + batch["loss_mask"][1, 4] = 0 + monkeypatch.setattr( + model, + "_draft_backbone", + lambda *a, **kw: ( + draft.hidden.expand(2, -1, -1), + anchors, + keep, + anchors.shape[1], + ), + ) + # Run the tensor operations eagerly on the CPU CI lane. + monkeypatch.setattr( + "aspec.models.dspark.speculators_token_losses", speculators_token_losses.__wrapped__ + ) + actual = model(**batch) + rows = [reference_row(reference, batch, i, anchors[i], keep[i]) for i in range(2)] + expected_terms = {key: (rows[0][0][key] + rows[1][0][key]) / 2 for key in rows[0][0]} + expected = ( + 0.1 * expected_terms["ce_loss"] + + 0.9 * expected_terms["tv_loss"] + + expected_terms["confidence_loss"] + ) + torch.testing.assert_close(actual[0], expected) + assert actual[6][1].item() == 2 + assert "l1_loss" not in actual[5] + for key, value in expected_terms.items(): + numerator, denominator = actual[5][key] + torch.testing.assert_close(numerator / denominator, value) + torch.testing.assert_close(actual[4], sum(mask.sum(0) for _, mask in rows)) + assert rows[0][1][1].tolist() == [1, 1, 0, 1] # a later slot survives a supervision gap + actual[0].backward() + expected.backward() + for param, ref in zip(draft.parameters(), reference.parameters(), strict=True): + torch.testing.assert_close(param.grad, ref.grad, atol=1e-6, rtol=1e-5) + + +def test_speculators_fused_tv_subgradient_at_partial_tie(): + # A permutation keeps the softmax normalizer equal, with one tied probability. + draft = torch.tensor([[0.0, 1.0, 2.0]], requires_grad=True) + teacher = torch.tensor([[1.0, 0.0, 2.0]], requires_grad=True) + ce, tv, labels = speculators_token_losses.__wrapped__(draft, teacher) + q, p = draft.detach().softmax(-1), teacher.detach().softmax(-1) + indicator = (q <= p).float() + tv.sum().backward() + torch.testing.assert_close(draft.grad, -q * (indicator - (q * indicator).sum(-1, keepdim=True))) + assert teacher.grad is None + torch.testing.assert_close(ce, F.cross_entropy(draft, labels, reduction="none")) + + +def test_cp_shards_contribute_one_row_across_accumulation_steps(monkeypatch): + model = DSparkModel(TinyDraft(), block_size=4, loss_recipe=RECIPE) + model.draft_context_parallel_size = 2 + model.direct_cp_group = object() + coefficient = torch.tensor(1.0, requires_grad=True) + numerator = denominator = 0 + expected_rows = [] + for valid in ([1, 0, 0, 0, 1, 1, 1, 1], [0, 0, 0, 0, 1, 0, 0, 0]): + mask = torch.tensor(valid).float().view(1, 2, 4) + decay = torch.exp(-torch.arange(4) / 4) + values = torch.arange(1.0, 9.0).view_as(mask) + expected_rows.append((values * mask * decay).sum() / (mask.sum() + 1e-5)) + for rank in range(2): + local = mask[:, rank : rank + 1] + + def all_reduce(count, group): + assert group is model.direct_cp_group + count.add_(mask[:, 1 - rank].sum()) + + monkeypatch.setattr("aspec.models.dspark.dist.all_reduce", all_reduce) + weights, _, rows = model._speculators_row_weights(local, local * decay) + numerator += (coefficient * values[:, rank : rank + 1] * weights).sum() + denominator += rows + assert denominator == 2 + (numerator / denominator).backward() + torch.testing.assert_close(coefficient.grad, torch.stack(expected_rows).mean()) + + +def test_ce_only_speculators_requires_teacher(): + model = DSparkModel(TinyDraft(), l1_loss_alpha=0, confidence_head_alpha=0, loss_recipe=RECIPE) + assert model.uses_target_hidden_states + + +@pytest.mark.parametrize("precision", ["bf16", "fp8"]) +def test_recipe_plan_wrapper_and_exact_resume(monkeypatch, precision): + overrides = ["training.attention_backend=flex_attention", + f"training.dspark_lm_head_precision={precision}"] + base = load_config(str(CONFIG), cli_args=overrides) + config = load_config( + str(CONFIG), cli_args=[*overrides, f"training.dspark_loss_recipe={RECIPE}"] + ) + default = compile_execution_plan(base) + plan = compile_execution_plan(config) + assert default.workload.objective["loss_recipe"] == "deepspec" + assert plan.workload.objective["loss_recipe"] == RECIPE + assert plan.workload.objective["lm_head_precision"] == precision + assert plan.plan_id != default.plan_id + args = config_to_flat_args(config) + args.draft_algorithm, args.world_size = "dspark", 1 + monkeypatch.setattr( + "aspec.training.trainer.Trainer.__init__", + lambda trainer, args: setattr(trainer, "args", args), + ) + trainer = DSparkTrainer(args) + wrapper = trainer._build_training_wrapper(TinyDraft()) + assert wrapper.loss_recipe == RECIPE + assert "tv_loss" in trainer._extra_loss_component_keys + metadata = {"world_size": 1, "training_topology": _logical_training_topology(args, 1)} + _validate_exact_resume_topology(args, metadata) + legacy = copy.deepcopy(metadata) + del legacy["training_topology"]["training_objective"]["loss_recipe"] + with pytest.raises(RuntimeError, match="dspark_loss_recipe"): + _validate_exact_resume_topology(args, legacy) + args.dspark_loss_recipe = "deepspec" + _validate_exact_resume_topology(args, legacy) + with pytest.raises(RuntimeError, match="dspark_loss_recipe"): + _validate_exact_resume_topology(args, metadata) + + +@pytest.mark.parametrize( + "override", ["training.dspark_loss_recipe=typo", "training.dspark_l1_target_top_k=8"] +) +def test_invalid_recipe_config(override): + config = load_config(str(CONFIG), cli_args=[f"training.dspark_loss_recipe={RECIPE}", override]) + with pytest.raises(PlanCompileError, match="dspark_loss_recipe|unfiltered"): + compile_execution_plan(config) diff --git a/tests/cpu/test_glm5_flash_draft.py b/tests/cpu/test_glm5_flash_draft.py new file mode 100644 index 0000000..cd12c5e --- /dev/null +++ b/tests/cpu/test_glm5_flash_draft.py @@ -0,0 +1,328 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +import json +from copy import deepcopy +from pathlib import Path + +import pytest +import torch +from torch import nn + +from aspec.config.utils import load_target_config +from aspec.data.renderers.glm5 import Glm5FlashRenderer +from aspec.models.dflash import _create_dflash_mask_mod +from aspec.models.draft.dspark import DSparkConfig +from aspec.models.dspark import DSparkModel +from aspec.planning import _resolve_target_family +from aspec.training.checkpoint import load_initial_draft_weights + +ROOT = Path(__file__).resolve().parents[2] +CONFIG = ROOT / "aspec/config/dspark_glm5_3_flash_draft_config.json" + + +def test_glm_config_and_capture_coordinates(tmp_path): + raw = { + "model_type": "glm5_next", + "architectures": ["Glm5NextForConditionalGeneration"], + "text_config": { + "model_type": "glm5_next_text", + "num_hidden_layers": 45, + "hidden_size": 4096, + "vocab_size": 154880, + }, + } + (tmp_path / "config.json").write_text(json.dumps(raw)) + assert _resolve_target_family(str(tmp_path)) == "glm5_3_flash" + outer = load_target_config(str(tmp_path)) + assert getattr(outer, "text_config", outer).vocab_size == 154880 + config = DSparkConfig(**json.loads(CONFIG.read_text())) + assert [k + 1 for k in config.target_layer_ids] == [20, 28, 32, 36, 40, 44] + draft = nn.Module() + draft.config = config + wrapper = DSparkModel(draft, block_size=8) + options = wrapper._block_mask_options_for_layer(0) + assert options == {"context_window": 2048, "is_causal": True} + mask = _create_dflash_mask_mod( + torch.tensor([[2050]]), torch.tensor([[True]]), 2100, 8, **options + ) + ids = torch.tensor([1, 2, 2049, 2050, 2100, 2102, 2103]) + assert mask(0, 0, torch.tensor(2), ids).tolist() == [ + False, + True, + True, + False, + True, + True, + False, + ] + + +@pytest.mark.parametrize("architecture,algorithm", [ + ("DSparkDraftModel", "dspark"), ("DFlashDraftModel", "dflash"), +]) +@pytest.mark.parametrize("layer_types", [ + ["full_attention"] * 5, + ["sliding_attention"] * 5, + ["sliding_attention", "full_attention", "sliding_attention", "full_attention", "full_attention"], +]) +@pytest.mark.parametrize("non_causal", [False, True]) +def test_draft_assets_and_exact_resume_compare_effective_block_attention( + tmp_path, architecture, algorithm, layer_types, non_causal, +): + from types import SimpleNamespace + + from safetensors.torch import save_file + + from aspec.commands.convert_to_hf import _checkpoint_training_contract + from aspec.config.draft_attention import resolve_draft_attention + from aspec.draft_artifact import _sha256, validate_native_asset + from aspec.training.checkpoint import ( + _logical_training_topology, + _validate_exact_resume_topology, + ) + + configured = json.loads(CONFIG.read_text()) + configured.update(architectures=[architecture], layer_types=layer_types, + sliding_window_non_causal=non_causal) + if algorithm == "dflash": + configured["model_type"] = "dflash" + default = dict(configured) + if non_causal: + default["sliding_window_non_causal"] = False + else: + del default["sliding_window_non_causal"] + saved_attention = resolve_draft_attention(configured) + + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(configured)) + weights_path = tmp_path / "model.safetensors" + save_file({"embed_tokens.weight": torch.zeros(1, 1)}, weights_path) + (tmp_path / "export_manifest.json").write_text(json.dumps({ + "storage_format": "aspec_native", + "algorithm": algorithm, + "config_sha256": _sha256(config_path), + "model_sha256": _sha256(weights_path), + "draft_structure": {"attention": saved_attention}, + })) + model_dir = tmp_path / "model" + model_dir.mkdir() + args = SimpleNamespace( + draft_algorithm=algorithm, draft_model_config_obj=configured, + dflash_block_size=configured["block_size"], world_size=1, + ) + metadata = {"world_size": 1, "training_topology": _logical_training_topology(args, 1)} + (tmp_path / "meta.json").write_text(json.dumps(metadata)) + validate_native_asset(tmp_path, expected_config=configured, weights_only=True) + _checkpoint_training_contract(model_dir, configured) + _validate_exact_resume_topology(args, metadata) + + args.draft_model_config_obj = default + if all(kind == "full_attention" for kind in layer_types): + validate_native_asset(tmp_path, expected_config=default, weights_only=True) + _checkpoint_training_contract(model_dir, default) + _validate_exact_resume_topology(args, metadata) + else: + with pytest.raises(ValueError, match="Native artifact Draft block attention"): + validate_native_asset(tmp_path, expected_config=default, weights_only=True) + with pytest.raises(ValueError, match="Export Draft block attention"): + _checkpoint_training_contract(model_dir, default) + with pytest.raises(RuntimeError, match="draft_attention_policy"): + _validate_exact_resume_topology(args, metadata) + + +class Tokenizer: + roles = {"<|system|>": 7, "<|user|>": 8, "<|assistant|>": 9} + + def encode(self, text, **kwargs): + return [self.roles[text]] + + def convert_ids_to_tokens(self, token): + return next(k for k, value in self.roles.items() if value == token) + + def apply_chat_template(self, messages, **kwargs): + assert kwargs == dict(tokenize=True, add_generation_prompt=False, return_dict=False) + return [7, 1, 8, 2, 9, 3, 4, 8, 5, 9, 6] + + +def test_native_role_masks(): + renderer = Glm5FlashRenderer(Tokenizer()) + messages = [{"role": "user", "content": "prompt"}] + _, mask = renderer.render(messages, max_seq_length=20) + assert mask == [0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1] + _, mask = renderer.render(messages, max_seq_length=7, last_turn_only=True) + assert mask == [0] * 7 + with pytest.raises(ValueError, match="reserved role"): + renderer.render([{"role": "user", "content": "<|assistant|>"}], max_seq_length=20) + + +@pytest.mark.parametrize("field", ["thinking", "thinking_content", "reasoning"]) +@pytest.mark.parametrize("with_canonical", [False, True]) +def test_native_reasoning_aliases_are_rejected(field, with_canonical): + renderer = Glm5FlashRenderer(Tokenizer()) + message = {"role": "assistant", "content": "answer", field: "reasoning"} + if with_canonical: + message["reasoning_content"] = "canonical reasoning" + with pytest.raises(ValueError, match="reasoning_content"): + renderer.render([message], max_seq_length=20) + + +@pytest.mark.parametrize("field", ["thinking", "thinking_content", "reasoning"]) +def test_sharegpt_reasoning_remains_canonical_before_rendering(field): + from aspec.data.preprocessing import _normalize_conversation + + class ReasoningTokenizer(Tokenizer): + def apply_chat_template(self, messages, **kwargs): + assert messages == [ + {"role": "assistant", "content": "answer", "reasoning_content": "reasoning"} + ] + return [9, 10, 11] + + messages = _normalize_conversation([{"from": "gpt", "value": "answer", field: "reasoning"}]) + renderer = Glm5FlashRenderer(ReasoningTokenizer()) + ids, mask = renderer.render(messages, max_seq_length=20) + assert ids == [9, 10, 11] + assert mask == [0, 1, 1] + + +@pytest.fixture +def published_dspark(tmp_path): + from safetensors.torch import save_file + + config = DSparkConfig(**json.loads(CONFIG.read_text())) + draft = nn.Module() + draft.config = config + draft.embed_tokens = nn.Embedding(3, 2) + published = { + "speculators_model_type": "dspark", + "aux_hidden_state_layer_ids": [20, 28, 32, 36, 40, 44], + "mask_token_id": 154856, + "block_size": 8, + "sample_from_anchor": True, + "sliding_window_non_causal": False, + "transformer_layer_config": { + "sliding_window": 2048, + "layer_types": ["sliding_attention"] * 5, + "rope_parameters": {"rope_theta": 10000}, + "rms_norm_eps": 1e-5, + "num_attention_heads": 64, + "num_key_value_heads": 64, + "head_dim": 64, + }, + } + (tmp_path / "config.json").write_text(json.dumps(published)) + target = tmp_path / "target" + target.mkdir() + embedding = torch.randn(3, 2) + head = torch.randn(3, 2) + weights = {"embed_tokens.weight": embedding, "lm_head.weight": head} + save_file(weights, tmp_path / "model.safetensors") + save_file(weights, target / "model.safetensors") + bindings = (str(target), "embed_tokens.weight", "lm_head.weight") + return draft, published, embedding, head, target, bindings + + +def test_speculators_published_head_binding(tmp_path, published_dspark): + from safetensors.torch import save_file + + draft, published, embedding, head, target, bindings = published_dspark + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + torch.testing.assert_close(draft.embed_tokens.weight, embedding) + save_file({"embed_tokens.weight": embedding, "lm_head.weight": -head}, + target / "model.safetensors") + with pytest.raises(ValueError, match="lm_head.weight differs"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + published["aux_hidden_state_layer_ids"][-1] -= 1 + (tmp_path / "config.json").write_text(json.dumps(published)) + with pytest.raises(ValueError, match="capture"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + + +def test_published_sliding_policy_uses_speculators_defaults(tmp_path, published_dspark): + draft, published, embedding, _, _, bindings = published_dspark + config_path = tmp_path / "config.json" + del published["sliding_window_non_causal"] + config_path.write_text(json.dumps(published)) + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + torch.testing.assert_close(draft.embed_tokens.weight, embedding) + + config_path.write_text(json.dumps({**published, "sliding_window_non_causal": True})) + with pytest.raises(ValueError, match="attention policy"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + del published["sample_from_anchor"] + config_path.write_text(json.dumps(published)) + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + config_path.write_text(json.dumps({**published, "sample_from_anchor": False})) + with pytest.raises(ValueError, match="capture"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + + +@pytest.mark.parametrize("config", [None, {"speculators_model_type": "dflash"}]) +@pytest.mark.parametrize("bare_file", [False, True]) +def test_bound_glm_warmstart_requires_speculators_config( + tmp_path, published_dspark, config, bare_file, monkeypatch, +): + from safetensors.torch import save_file + + draft, _, embedding, _, _, bindings = published_dspark + config_path = tmp_path / "config.json" + if config is None: + config_path.unlink() + else: + config_path.write_text(json.dumps(config)) + + weights_path = tmp_path / "model.safetensors" + weights_path.write_bytes(b"unreadable checkpoint") + with monkeypatch.context() as context: + context.setattr("aspec.training.checkpoint.safe_open", + lambda *a, **kw: pytest.fail("Tensors opened before published config validation")) + with pytest.raises(ValueError, match=r"requires the original Speculators DSpark config.json"): + load_initial_draft_weights(draft, str(weights_path if bare_file else tmp_path), + target_bindings=bindings, require_speculators_config=True) + + save_file({"embed_tokens.weight": embedding}, weights_path) + load_initial_draft_weights(draft, str(weights_path), target_bindings=bindings) + torch.testing.assert_close(draft.embed_tokens.weight, embedding) + + +@pytest.mark.parametrize("published_rope,configured_rope", [ + ({}, {"rope_type": "linear", "factor": 2.0}), + ({"rope_type": "linear", "factor": 2.0}, {}), + ({"rope_type": "linear", "factor": 2.0}, {"rope_type": "linear", "factor": 4.0}), + ({"rope_theta": 20000}, {}), + ({"rope_type": "yarn", "factor": 2.0, "beta_fast": 64}, + {"rope_type": "yarn", "factor": 2.0, "beta_fast": 32}), +]) +def test_published_dspark_rejects_different_rope(tmp_path, published_dspark, + published_rope, configured_rope): + draft, published, _, _, _, bindings = published_dspark + geometry = published["transformer_layer_config"] + geometry["rope_parameters"] = {"rope_theta": 10000, **published_rope} + geometry["max_position_embeddings"] = draft.config.max_position_embeddings + draft.config.rope_parameters = {"rope_theta": 10000, **configured_rope} + (tmp_path / "config.json").write_text(json.dumps(published)) + with pytest.raises(ValueError, match="RoPE|geometry"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + + +@pytest.mark.parametrize("published_updates,configured_updates", [ + ({"rope_parameters": None}, {}), + ({"rope_theta": 10000, "rope_parameters": None}, {}), + ({"rope_parameters": {"rope_type": "default", "factor": 1.0, "rope_theta": 10000, + "partial_rotary_factor": 1.0}}, {}), + ({"rope_theta": 10000, "rope_parameters": None, + "rope_scaling": {"type": "linear", "factor": 2.0}}, + {"rope_parameters": {"rope_type": "linear", "factor": 2.0, "rope_theta": 10000}}), + ({"rope_parameters": {"rope_type": "yarn", "factor": 2.0, "rope_theta": 10000, + "beta_fast": 32, "beta_slow": 1}}, + {"rope_parameters": {"type": "yarn", "factor": 2.0, "rope_theta": 10000}}), +]) +def test_published_dspark_accepts_equivalent_rope(tmp_path, published_dspark, + published_updates, configured_updates): + draft, published, embedding, _, _, bindings = published_dspark + published["transformer_layer_config"].update(deepcopy(published_updates)) + published["transformer_layer_config"]["max_position_embeddings"] = draft.config.max_position_embeddings + for key, value in configured_updates.items(): + setattr(draft.config, key, deepcopy(value)) + (tmp_path / "config.json").write_text(json.dumps(published)) + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + torch.testing.assert_close(draft.embed_tokens.weight, embedding) diff --git a/tests/cpu/test_release_recipes.py b/tests/cpu/test_release_recipes.py index 59674f6..0d415f3 100644 --- a/tests/cpu/test_release_recipes.py +++ b/tests/cpu/test_release_recipes.py @@ -30,6 +30,8 @@ def test_catalog_plans(name, monkeypatch): "ASPEC_TARGET_MODEL", "Qwen/Qwen3.8-27B" if name == "qwen27b-long" else recipes.catalog()[name]["model"], ) + if name == "glm5-flash": + monkeypatch.setenv("ASPEC_TARGET_MODEL", "zai-org/GLM-5.3-Flash") if name.startswith("ling"): monkeypatch.setenv( "ASPEC_TARGET_MODEL", "inclusionAI/" + recipes.catalog()[name]["model"] @@ -49,7 +51,10 @@ def test_catalog_plans(name, monkeypatch): compile_multi_draft_plan(config) else: - compile_execution_plan(config) + plan = compile_execution_plan(config) + if name == "glm5-flash": + assert plan.to_runtime_args().aspec_target_family == "glm5_3_flash" + assert plan.to_runtime_args().attention_backend == "cudnn_flex_attention" if name != "quickstart": assert config.training.num_train_steps is None assert config.training.lr_total_steps is None @@ -83,7 +88,7 @@ def test_qwen8b_recipes_plan_without_target_override(name, monkeypatch): def test_catalog_exactly_covers_formal_files(): entries = recipes.catalog() - assert len(entries) == 13 + assert len(entries) == 14 declared = {p for entry in entries.values() for p in entry["configs"]} assert declared == { str(p.relative_to(recipes.ROOT)) for p in (recipes.ROOT / "configs/recipes").glob("*.yaml") diff --git a/tests/cpu/test_target_lm_head.py b/tests/cpu/test_target_lm_head.py index 0a11b2f..b838567 100644 --- a/tests/cpu/test_target_lm_head.py +++ b/tests/cpu/test_target_lm_head.py @@ -190,8 +190,8 @@ def _patch_dist(monkeypatch, rank: int, peer_failed: bool = False) -> _Recording def _patch_follower_config(monkeypatch) -> None: monkeypatch.setattr( target_utils, - "AutoConfig", - SimpleNamespace(from_pretrained=lambda *args, **kwargs: _multimodal_config()), + "load_target_config", + lambda *args, **kwargs: _multimodal_config(), ) diff --git a/tests/test_cudnn_deterministic_gpu.py b/tests/test_cudnn_deterministic_gpu.py new file mode 100644 index 0000000..b0e9850 --- /dev/null +++ b/tests/test_cudnn_deterministic_gpu.py @@ -0,0 +1,49 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Native cuDNN MHA64 forward and backward repeatability.""" + +import os + +import pytest +import torch + +pytestmark = pytest.mark.skipif( + os.getenv("ASPEC_RUN_CUDNN_FLEX_INTEGRATION") != "1" or not torch.cuda.is_available(), + reason="requires the native CUDA cuDNN integration environment", +) + + +@pytest.mark.parametrize("context_length", [8192, 262144]) +def test_native_mha64_deterministic_forward_and_backward(context_length): + from aspec.models.ops.cudnn_flex_attention import build_dspark_mla_mask, cudnn_flex_attention + + generator = torch.Generator(device="cuda").manual_seed(93) + queries, heads, dim = 16, 4, 64 + anchors = torch.tensor([[7, context_length - 9]], device="cuda") + mask = build_dspark_mla_mask( + anchor_positions=anchors, block_keep_mask=torch.ones_like(anchors, dtype=torch.bool), + context_length=context_length, block_size=8, is_causal=True, context_window=2048, + ) + inputs = [torch.randn(1, length, heads, dim, device="cuda", dtype=torch.bfloat16, + generator=generator) for length in (queries, context_length + queries, + context_length + queries)] + gradient = torch.randn(inputs[0].shape, device="cuda", dtype=torch.bfloat16, + generator=generator) + enabled = torch.are_deterministic_algorithms_enabled() + warn_only = torch.is_deterministic_algorithms_warn_only_enabled() + try: + torch.use_deterministic_algorithms(True) + with torch.backends.cudnn.flags(deterministic=True, benchmark=False): + repeats = [] + for _ in range(2): + q, k, v = [value.detach().clone().requires_grad_() for value in inputs] + output = cudnn_flex_attention(q, k, v, mask=mask, scale=dim ** -0.5, + gqa_mode="native") + grads = torch.autograd.grad(output, (q, k, v), gradient) + assert torch.isfinite(output).all() + assert all(torch.isfinite(value).all() for value in grads) + repeats.append((output.detach(), *grads)) + for first, second in zip(*repeats, strict=True): + assert torch.equal(first, second) + finally: + torch.use_deterministic_algorithms(enabled, warn_only=warn_only) diff --git a/tests/test_direct_cp_gpu.py b/tests/test_direct_cp_gpu.py index 967c507..0bbcf97 100644 --- a/tests/test_direct_cp_gpu.py +++ b/tests/test_direct_cp_gpu.py @@ -14,7 +14,7 @@ # ruff: noqa: I001 -"""Opt-in two-GPU correctness check for Direct CP collectives and FA4.""" +"""Opt-in CUDA correctness checks for Direct CP attention and collectives.""" import os @@ -31,6 +31,70 @@ from aspec.models.ops.direct_cp import DirectCPMetadata, direct_cp_attention + +@pytest.mark.parametrize("context_length", [2103, 8192]) +def test_glm_swa_fa4_matches_dense_mask_forward_and_backward(context_length): + """GLM's MHA64 window and causal Draft slots survive sparse FA4 execution.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA is unavailable") + from torch.nn.attention import SDPBackend, sdpa_kernel + + from aspec.models.dflash import _create_dflash_mask_mod + + torch.manual_seed(73) + anchors = torch.cat(( + torch.arange(2040, 2072, device="cuda"), + torch.arange(context_length - 100, context_length - 68, device="cuda"), + torch.tensor([0, 1, 2047, 2048, 2050, context_length - 3, + context_length - 8, 100], device="cuda"), + ))[None] + keep = torch.ones_like(anchors, dtype=torch.bool) + keep[0, -1] = False + block_size = 8 + queries = anchors.numel() * block_size + shapes = [(1, 64, queries, 64), (1, 64, context_length, 64), + (1, 64, context_length, 64), (1, 64, queries, 64), (1, 64, queries, 64)] + actual_inputs = [torch.randn(*shape, device="cuda", dtype=torch.bfloat16, + requires_grad=True) for shape in shapes] + reference_inputs = [value.detach().float().requires_grad_() for value in actual_inputs] + metadata = DirectCPMetadata( + anchor_positions=anchors, block_keep_mask=keep, + context_position_ids=torch.arange(context_length, device="cuda")[None], + block_size=block_size, cp_rank=0, cp_size=1, group=None, + sequence_length=context_length, backend="fa4", context_window=2048, + draft_is_causal=True, + ) + actual, _ = direct_cp_attention(*actual_inputs, metadata) + mask_mod = _create_dflash_mask_mod(anchors, keep, context_length, block_size, + context_window=2048, is_causal=True) + dense_mask = mask_mod( + 0, 0, torch.arange(queries, device="cuda")[:, None], + torch.arange(context_length + queries, device="cuda")[None, :], + ) + token_keep = (keep[:, :, None] + & (anchors[:, :, None] + torch.arange(block_size, device="cuda") + < context_length)).flatten() + key_keep = torch.cat((torch.ones(context_length, device="cuda", dtype=torch.bool), + token_keep)) + dense_mask = (dense_mask & token_keep[:, None] & key_keep[None, :])[None, None] + query, context_key, context_value, draft_key, draft_value = reference_inputs + with sdpa_kernel(SDPBackend.MATH): + reference = torch.nn.functional.scaled_dot_product_attention( + query, torch.cat((context_key, draft_key), dim=2), + torch.cat((context_value, draft_value), dim=2), attn_mask=dense_mask, + ) + upstream = torch.randn_like(actual) + (actual.float() * upstream.float()).sum().backward() + (reference * upstream.float()).sum().backward() + pairs = [(actual, reference), *((value.grad, expected.grad) for value, expected + in zip(actual_inputs, reference_inputs, strict=True))] + for value, expected in pairs: + assert torch.isfinite(value).all() + relative = (value.float() - expected).norm() / expected.norm().clamp_min(1e-12) + assert relative < 0.01, float(relative) + assert torch.count_nonzero(actual[:, :, -block_size:]) == 0 + + def test_cp2_fa4_matches_unsharded_math_forward_and_backward(): if int(os.getenv("WORLD_SIZE", "1")) != 2: pytest.skip("launch with torchrun --standalone --nproc-per-node=2") diff --git a/tests/test_dspark_speculators_cutedsl.py b/tests/test_dspark_speculators_cutedsl.py new file mode 100644 index 0000000..8cd9e66 --- /dev/null +++ b/tests/test_dspark_speculators_cutedsl.py @@ -0,0 +1,230 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Red Hat CE/TV and the Markov epilogue, including BF16 rounding.""" + +import os + +import pytest +import torch + +pytestmark = pytest.mark.skipif( + os.getenv("ASPEC_RUN_CUTEDSL_INTEGRATION") != "1" + or not torch.cuda.is_available() + or torch.cuda.get_device_capability() not in {(9, 0), (10, 0)}, + reason="requires a Hopper/Blackwell CuTeDSL integration environment", +) + + +@pytest.mark.parametrize("vocab", [257, 154880]) +@pytest.mark.parametrize("indexed", [False, True]) +@pytest.mark.parametrize("teacher_kind", ["tie", "independent"]) +def test_speculators_cute_components_and_gradient(vocab, indexed, teacher_kind): + from aspec.kernels.cutedsl.speculators_loss import speculators_cutedsl_loss + + gen = torch.Generator(device="cuda").manual_seed(6) + rows = 32 + x = torch.randn(1, rows, vocab, generator=gen, device="cuda", dtype=torch.bfloat16) + # The identical row and a permutation exercise the <= branch independently + # of random probabilities. Duplicate maxima straddle vocabulary tiles. + x[0, :, [2, vocab - 1]] = 4 + if teacher_kind == "tie": + t = x.clone() + t[:, 1:, :2] = x[:, 1:, [1, 0]] + else: + t = torch.randn(x.shape, generator=gen, device=x.device, dtype=x.dtype) * 1.5 + 4 + idx = None + if indexed: + idx = torch.arange(rows, device=x.device).roll(1) + idx[-1] = idx[0] + x = x.index_select(1, idx) + teacher = t if idx is None else t.index_select(1, idx) + x.requires_grad_() + ce, tv, label, correct = speculators_cutedsl_loss(x, t, idx) + q = x.detach().double().softmax(-1) + p = teacher.double().softmax(-1) + if teacher_kind == "independent": + student_norm = x.detach().double().logsumexp(-1) + teacher_norm = teacher.double().logsumexp(-1) + assert (teacher_norm - student_norm).abs().min() > 0.5 + wrong_teacher = (teacher.double() - student_norm[..., None]).exp() + expected_tv = 1 - torch.minimum(q, p).sum(-1) + wrong_tv = 1 - torch.minimum(q, wrong_teacher).sum(-1) + assert (expected_tv - wrong_tv).abs().min() > 1e-3 + w = torch.linspace(-0.5, 0.5, rows, device=x.device).view(1, -1) + w[:, 5] = 0 + target = teacher.argmax(-1) + torch.testing.assert_close(label, target) + torch.testing.assert_close(correct, target == x.argmax(-1)) + ref_ce = x.double().logsumexp(-1) - x.double().gather(-1, target[..., None]).squeeze(-1) + torch.testing.assert_close(ce.double(), ref_ce, rtol=0, atol=2e-6) + torch.testing.assert_close(tv.double(), 1 - torch.minimum(q, p).sum(-1), rtol=0, atol=2e-6) + # Raw logits resolve exact ties only when the partition sums are identical. + indicator = ((x.detach() <= teacher) if teacher_kind == "tie" else (q <= p)).double() + mass = (q * indicator).sum(-1, keepdim=True) + gt = (w[..., None] * q * (mass - indicator)).bfloat16() + gc = q.clone() + gc.scatter_add_(-1, target[..., None], -torch.ones_like(target[..., None], dtype=q.dtype)) + gc = (gc * (-w[..., None])).bfloat16() + for loss, coeff, grad in ((ce, -w, gc), (tv, w, gt)): + (g,) = torch.autograd.grad(loss, x, coeff, retain_graph=True) + torch.testing.assert_close(g, grad, rtol=0.009, atol=2e-7) + (g,) = torch.autograd.grad((ce, tv), x, (-w, w)) + torch.testing.assert_close(g, gc + gt, rtol=0.012, atol=2e-7) + + +def test_markov_epilogue_rounds_before_add_and_both_inputs_receive_gradient(): + from aspec.kernels.cutedsl.speculators_loss import markov_residual_projection + + gen = torch.Generator(device="cuda").manual_seed(42) + + def sample(shape): + return torch.randn( + shape, generator=gen, dtype=torch.bfloat16, device="cuda" + ).requires_grad_() + + base, emb, weight = sample((1, 32, 8, 4096)), sample((1, 32, 8, 256)), sample((4096, 256)) + ref = base + torch.nn.functional.linear(emb, weight) + out = markov_residual_projection(base, emb, weight) + torch.testing.assert_close(out, ref, rtol=0, atol=0) + g = torch.randn(ref.shape, generator=gen, dtype=ref.dtype, device=ref.device) + xs = (base, emb, weight) + for actual, expected in zip( + torch.autograd.grad(out, xs, g), torch.autograd.grad(ref, xs, g), strict=True + ): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("context_length", [256, 262144]) +def test_markov_confidence_preserves_separate_embedding_gradient_reductions(context_length): + import copy + + from aspec.kernels.cutedsl.speculators_loss import speculators_cutedsl_loss + from aspec.models.draft.dspark import AcceptRatePredictor, VanillaMarkov + from aspec.models.dspark import DSparkModel + + torch.manual_seed(113) + draft = torch.nn.Module() + draft.markov_head = VanillaMarkov(vocab_size=1024, markov_rank=256) + draft.confidence_head = AcceptRatePredictor(32 + 256) + draft.confidence_head_with_markov = True + draft.cuda().bfloat16() + reference = copy.deepcopy(draft) + hidden = torch.randn(1, 32, 8, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + expected_hidden = hidden.detach().clone().requires_grad_() + teacher = torch.randn_like(hidden) + head = torch.randn(1024, 32, device="cuda", dtype=torch.bfloat16) + previous = torch.randint(4, (1, 32, 8), device="cuda") + weights = torch.rand(1, 32, 8, device="cuda") / 256 + context = teacher.new_zeros(1, context_length, 32) + context[:, :256] = teacher.view(1, 256, 32) + model = DSparkModel(draft, loss_recipe="speculators_redhat_v1") + actual = model._prediction_terms( + hidden, + previous, + previous, + head, + teacher, + torch.ones_like(weights), + weights, + teacher_context=(context, torch.arange(256, device="cuda").view(1, 32, 8)), + ) + (0.1 * actual[0] + 0.9 * actual[1] + actual[2]).backward() + + logits = torch.nn.functional.linear(expected_hidden, head) + logits = reference.markov_head.apply_block_logits(logits, token_ids=previous) + ce, tv, _, _ = speculators_cutedsl_loss(logits, torch.nn.functional.linear(teacher, head)) + embedding = reference.markov_head.get_prev_embeddings(previous) + prediction = reference.confidence_head(torch.cat([expected_hidden, embedding], dim=-1)) + target = (1 - tv).detach().to(prediction.dtype) + bce = torch.nn.functional.binary_cross_entropy_with_logits( + prediction.float(), target.float(), reduction="none" + ) + expected = tuple((value * weights).sum() for value in (ce, tv, bce)) + (0.1 * expected[0] + 0.9 * expected[1] + expected[2]).backward() + assert model.l1_kernel_invocations["cutedsl_markov_projection"] == 1 + for value, wanted in zip(actual[:3], expected, strict=True): + torch.testing.assert_close(value, wanted, rtol=0, atol=0) + torch.testing.assert_close(hidden.grad, expected_hidden.grad, rtol=0, atol=0) + for value, wanted in zip(draft.parameters(), reference.parameters(), strict=True): + torch.testing.assert_close(value.grad, wanted.grad, rtol=0, atol=0) + + +def test_pinned_speculators_fused_gpu_oracle(): + """Optional release gate: supply the unmodified b9c51bc src/speculators/losses/fused.py.""" + import hashlib + import importlib.util + import sys + from pathlib import Path + + from aspec.kernels.cutedsl.speculators_loss import speculators_cutedsl_loss + + reference = os.environ.get("ASPEC_SPECULATORS_FUSED_REFERENCE") + if not reference: + pytest.skip("ASPEC_SPECULATORS_FUSED_REFERENCE not supplied") + path = Path(reference).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == ( + "0465967844a5850facdb3f5c05c22a3f94569a180121c51e867939fa878b6a77" + ) + spec = importlib.util.spec_from_file_location("_aspec_pinned_speculators_fused", path) + lib = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = lib + spec.loader.exec_module(lib) + for vocab in (257, 154880): + gen = torch.Generator(device="cuda").manual_seed(79) + d = torch.randn(1, 32, vocab, generator=gen, device="cuda", dtype=torch.bfloat16) + t = d.roll(vocab // 2 + 1, dims=-1) + d.requires_grad_() + ce, tv, label, correct = speculators_cutedsl_loss(d, t) + torch.testing.assert_close(label, t.argmax(-1)) + torch.testing.assert_close(correct, d.argmax(-1) == label) + ref_ce, ref_tv = lib.fused_ce_loss(d, t), lib.fused_tv_loss(d, t) + torch.testing.assert_close(ce, ref_ce, rtol=0, atol=1e-6) + torch.testing.assert_close(tv, ref_tv, rtol=0, atol=3e-7) + # Compare TV alone: CE at the teacher label can drown out the wrong subgradient. + (actual,) = torch.autograd.grad(tv.sum(), d) + (expected,) = torch.autograd.grad(ref_tv.sum(), d) + torch.testing.assert_close(actual, expected, rtol=0.008, atol=1e-9) + + +@pytest.mark.parametrize("stride, last_offset, unique_rows", [(1, 0, 25), (8, -1, 0), (8, 0, 0)]) +def test_teacher_projection_policy_preserves_values_and_gradients(stride, last_offset, unique_rows): + from aspec.models.dspark import DSparkModel + + gen = torch.Generator(device="cuda").manual_seed(402) + teacher = torch.randn(1, 256, 32, generator=gen, device="cuda", dtype=torch.bfloat16) + head = torch.randn(1024, 32, generator=gen, device="cuda", dtype=torch.bfloat16) + ids = torch.randint(1024, (1, 256), generator=gen, device="cuda") + active = 4 + stride * torch.arange(18, device="cuda") + active[-1] += last_offset + anchors = torch.cat([active[:7], active.new_zeros(1), active[7:]])[None] + keep = anchors != 0 + positions = anchors[:, :, None] + torch.arange(8, device="cuda") + # Equal draft and teacher distributions exercise the TV subgradient at a tie. + hidden = teacher[0, positions].reshape(1, -1, 32).requires_grad_() + draft = torch.nn.Module() + draft.markov_head = draft.confidence_head = None + model = DSparkModel( + draft, + block_size=8, + num_anchors=19, + confidence_head_alpha=0, + loss_recipe="speculators_redhat_v1", + ) + model._draft_backbone = lambda *args, **kwargs: (hidden, anchors, keep, 19) + + def evaluate(): + outputs = model(ids, [], torch.ones_like(ids), head, last_hidden_states=teacher) + (grad,) = torch.autograd.grad(outputs[0], hidden) + return (*outputs[:5], grad) + + model._reuse_head_projections = False + reference = evaluate() + model._reuse_head_projections = True + actual = evaluate() + if torch.cuda.get_device_capability() == (10, 0): + unique_rows = 0 + assert model.loss_head_cached_context_tokens_observed == unique_rows + assert model.loss_head_cached_teacher_blocks_observed == (18 if unique_rows else 0) + assert model.l1_kernel_invocations.get("indexed_cutedsl_ce_tv", 0) == int(unique_rows > 0) + for observed, expected in zip(actual, reference, strict=True): + torch.testing.assert_close(observed, expected, rtol=0, atol=0) diff --git a/tests/test_fp8_lm_head.py b/tests/test_fp8_lm_head.py index ba4151a..731d4fb 100644 --- a/tests/test_fp8_lm_head.py +++ b/tests/test_fp8_lm_head.py @@ -171,7 +171,8 @@ def test_nan_propagates_through_forward_and_input_gradient(): assert hidden.grad[:2].isfinite().all() -def test_native_dspark_losses_keep_teacher_and_validation_bf16(monkeypatch): +@pytest.mark.parametrize("loss_recipe", ["deepspec", "speculators_redhat_v1"]) +def test_native_dspark_losses_keep_teacher_and_validation_bf16(monkeypatch, loss_recipe): from types import SimpleNamespace from aspec.models.draft.dspark import AcceptRatePredictor, VanillaMarkov @@ -184,7 +185,7 @@ def test_native_dspark_losses_keep_teacher_and_validation_bf16(monkeypatch): draft.markov_head = VanillaMarkov(vocab_size=256, markov_rank=16) draft.confidence_head = AcceptRatePredictor(144) draft.confidence_head_with_markov = True - model = DSparkModel(draft).cuda().to(torch.bfloat16) + model = DSparkModel(draft, loss_recipe=loss_recipe).cuda().to(torch.bfloat16) hidden = torch.randn(1, 3, 7, 128, device="cuda", dtype=torch.bfloat16, requires_grad=True) teacher = torch.randn_like(hidden, requires_grad=False) weight = torch.randn(256, 128, device="cuda", dtype=torch.bfloat16) * 0.02 diff --git a/tools/recipe_catalog.json b/tools/recipe_catalog.json index 6c0873f..e496572 100644 --- a/tools/recipe_catalog.json +++ b/tools/recipe_catalog.json @@ -146,5 +146,16 @@ "target_gpus": 4, "draft_gpus": 4, "native_only_export": true + }, + "glm5-flash": { + "configs": [ + "configs/recipes/glm5_3_flash_dspark.yaml" + ], + "model": "GLM-5.3-Flash", + "nodes": 1, + "gpus": 5, + "target_gpus": 4, + "draft_gpus": 1, + "native_only_export": true } }