From 99509928f237d2e011d2f4b4181f74cbf07d0a27 Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:13:00 +0800 Subject: [PATCH] perf(dory): reuse rotary embeddings within each stage traversal --- tests/test_dory.py | 90 +++++++++++++++++++++++++++++++++++++++++ tests/test_dory_gpu.py | 33 +++++++++++++++ vllm_rlt/models/dory.py | 29 ++++++++----- 3 files changed, 142 insertions(+), 10 deletions(-) diff --git a/tests/test_dory.py b/tests/test_dory.py index 45260837..a61afe20 100644 --- a/tests/test_dory.py +++ b/tests/test_dory.py @@ -3,6 +3,7 @@ import json import math +from collections import Counter import pytest import torch @@ -14,6 +15,7 @@ from vllm_rlt.core.kv_cache_manager import KVCacheManager from vllm_rlt.engine.llm_engine import LLMEngine from vllm_rlt.kernels.paged_attention import torch_paged_attention +from vllm_rlt.layers import RotaryEmbedding from vllm_rlt.models import AutoModelForCausalLM, DoryConfig, DoryForCausalLM ARTIFACT_PAIRS = 36 @@ -385,6 +387,94 @@ def test_batched_positions_match_reference_for_per_loop_kv(model): assert cache.num_used_blocks == 0 +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("kv_mode", ["per_loop", "last_loop"]) +def test_rotary_embeddings_are_reused_only_within_each_traversal(monkeypatch, dtype, kv_mode): + # Every stage contains repeated sliding and full attention layers. + layer_types = ("sliding_attention", "mlp", "full_attention", "mlp") * 6 + config = tiny_dory_config( + num_hidden_layers=24, + n_input_layers=8, + n_recurrent_layers=8, + n_output_layers=8, + layer_types=layer_types, + recurrent_kv_cache_mode=kv_mode, + ) + model = DoryForCausalLM(config).to(dtype=dtype).eval() + cache = make_cache(model) + assert cache.allocate("request", 2) + calls = Counter() + original = RotaryEmbedding.forward + + def record(module, hidden, positions): + calls[module.rope_theta] += 1 + return original(module, hidden, positions) + + monkeypatch.setattr(RotaryEmbedding, "forward", record) + with torch.inference_mode(): + for position in range(2): + batch = cache._prepare_batch(["request"], [0], [position]) + calls.clear() + hidden = model.prelude_prepared(torch.tensor([4 + position]), batch, cache) + assert calls == {1e4: 1, 1e7: 1} + for depth in range(config.total_ut_steps): + core = cache._prepare_batch(["request"], [depth], [position]) + calls.clear() + hidden, _ = model.recurrent_prepared(hidden, core, cache) + assert calls == {1e4: 1, 1e7: 1} + calls.clear() + model.coda_prepared(hidden, batch, cache) + assert calls == {1e4: 1, 1e7: 1} + cache.free("request") + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("kv_mode", ["per_loop", "last_loop"]) +def test_reused_rotary_matches_per_layer_recomputation(monkeypatch, dtype, kv_mode): + config = tiny_dory_config( + num_hidden_layers=24, + n_input_layers=8, + n_recurrent_layers=8, + n_output_layers=8, + layer_types=("sliding_attention", "mlp", "full_attention", "mlp") * 6, + recurrent_kv_cache_mode=kv_mode, + ) + model = DoryForCausalLM(config).to(dtype=dtype) + tokens = torch.tensor([4, 7, 9, 3, 12, 5]) + with torch.inference_mode(): + actual, cache = run_one_token_per_step(model, tokens) + + def per_layer(backbone, hidden, batch, cache, layers): + for index in layers: + layer = backbone.layers[index] + embeddings = ( + layer.mixer.rotary_emb(hidden, batch.position_ids) + if layer.is_attention + else None + ) + hidden = layer(hidden, embeddings, batch, cache) + return hidden + + monkeypatch.setattr(type(model.backbone), "forward", per_layer) + expected, reference_cache = run_one_token_per_step(model, tokens) + torch.testing.assert_close(actual["logits"], expected["logits"], atol=0, rtol=0) + for actual_loop, expected_loop in zip(actual["loops"], expected["loops"]): + torch.testing.assert_close(actual_loop, expected_loop, atol=0, rtol=0) + for layer in model.config.attention_layer_ids: + kv_layer = model.config.kv_layer_index(layer) + depths = ( + range(model.config.total_ut_steps) if kv_layer in model.recurrent_kv_layers else (0,) + ) + for depth in depths: + for actual_kv, expected_kv in zip( + cache.read(kv_layer, "request", depth), + reference_cache.read(kv_layer, "request", depth), + ): + torch.testing.assert_close(actual_kv, expected_kv, atol=0, rtol=0) + cache.free("request") + reference_cache.free("request") + + def test_recurrent_kv_is_laid_out_per_loop_plane(model): tokens = torch.tensor([4, 7, 9, 3, 12, 5]) _, cache = run_one_token_per_step(model, tokens) diff --git a/tests/test_dory_gpu.py b/tests/test_dory_gpu.py index 3a04240f..3771fa8a 100644 --- a/tests/test_dory_gpu.py +++ b/tests/test_dory_gpu.py @@ -245,6 +245,39 @@ def test_dory_execution_modes_match_cpu_oracle(mode, max_num_seqs): assert runner.prelude_graphs is None +@pytest.mark.parametrize("mode", EXECUTION_MODES) +def test_reused_rotary_embeddings_follow_positions_in_every_execution_mode(mode): + config = tiny_dory_config( + num_hidden_layers=24, + n_input_layers=8, + n_recurrent_layers=8, + n_output_layers=8, + layer_types=("sliding_attention", "mlp", "full_attention", "mlp") * 6, + ) + torch.manual_seed(42) + reference = DoryForCausalLM(config).eval() + model = DoryForCausalLM(config).to(device=DEVICE, dtype=torch.float32).eval() + model.load_state_dict(reference.state_dict()) + engine = make_engine(model, mode) + params = SamplingParams(max_tokens=6, min_loops=2, max_loops=2, ignore_eos=True) + for request_id, prompt in PROMPTS.items(): + engine.add_request(request_id, prompt, params) + try: + finished = drain(engine) + for request_id, prompt in PROMPTS.items(): + assert finished[request_id].token_ids == greedy_reference(reference, prompt, 6) + assert engine.cache_manager.num_used_blocks == 0 + if "graph" in mode: + for graphs in ( + engine.model_runner.prelude_graphs, + engine.model_runner.graphs, + engine.model_runner.coda_graphs, + ): + assert graphs.replays > 0 + finally: + engine.close() + + @pytest.mark.parametrize("mode", EXECUTION_MODES) @pytest.mark.parametrize("after", [0, 1, 3, 6, 9]) def test_dory_cancellation_releases_kv_in_every_execution_mode(mode, after): diff --git a/vllm_rlt/models/dory.py b/vllm_rlt/models/dory.py index 8e6a388e..a5f039c8 100644 --- a/vllm_rlt/models/dory.py +++ b/vllm_rlt/models/dory.py @@ -381,7 +381,7 @@ def __init__(self, config: DoryConfig, layer_idx: int) -> None: def forward( self, hidden: torch.Tensor, - position_ids: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], batch: "KVMetadata | DeviceKVMetadata", cache: "KVCacheManager", ) -> torch.Tensor: @@ -390,7 +390,7 @@ def forward( k = self.k_proj(hidden).view(count, self.num_kv_heads, self.head_dim) v = self.v_proj(hidden).view(count, self.num_kv_heads, self.head_dim) gate = self.g_proj(hidden) - cos, sin = self.rotary_emb(hidden, position_ids) + cos, sin = position_embeddings q, k = apply_rotary_pos_emb(q, k, cos, sin) # nGPT Q/K normalization happens after RoPE: queries carry the square of # the learned scale, keys only the unit-length normalization. @@ -448,12 +448,12 @@ def __init__(self, config: DoryConfig, layer_idx: int) -> None: def forward( self, hidden: torch.Tensor, - position_ids: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None, batch: "KVMetadata | DeviceKVMetadata", cache: "KVCacheManager", ) -> torch.Tensor: if self.is_attention: - update = self.mixer(hidden, position_ids, batch, cache) + update = self.mixer(hidden, position_embeddings, batch, cache) return slerp_alpha(hidden, update, self.attn_alpha, self.alpha_scale) return slerp_alpha(hidden, self.mixer(hidden), self.mlp_alpha, self.alpha_scale) @@ -475,6 +475,18 @@ def __init__(self, config: DoryConfig) -> None: self.recurrent_layers = tuple(range(*recurrent_slice)) self.output_layers = tuple(range(*output_slice)) + def forward(self, hidden, batch, cache, layers): + # Positions are shared within a traversal, but change between graph replays. + position_embeddings = {} + for index in layers: + layer = self.layers[index] + if layer.is_attention and layer.layer_type not in position_embeddings: + position_embeddings[layer.layer_type] = layer.mixer.rotary_emb( + hidden, batch.position_ids + ) + hidden = layer(hidden, position_embeddings.get(layer.layer_type), batch, cache) + return hidden + class DoryForCausalLM(nn.Module): """Inference-only Dory with eager KV boundaries and a graphable recurrent core.""" @@ -547,8 +559,7 @@ def prelude_prepared(self, tokens, batch, cache, *, generators=None) -> torch.Te if tokens.ndim != 1: raise ValueError("prelude expects packed token_ids with shape [N]") hidden = self.backbone.embeddings(tokens) - for index in self.backbone.input_layers: - hidden = self.backbone.layers[index](hidden, batch.position_ids, batch, cache) + hidden = self.backbone(hidden, batch, cache, self.backbone.input_layers) # The retained injection input is half of the carried state, and loop zero # starts from the same tensor, matching the reference ``state = input_h``. return torch.cat((hidden, hidden), dim=-1) @@ -572,8 +583,7 @@ def recurrent_prepared(self, hidden, batch, cache, *, compute_gate: bool = True) raise ValueError("recurrent expects hidden with shape [N, hidden_size]") state, injection = hidden.split(self.config.d_model, dim=-1) current = justnorm(state + injection) - for index in self.backbone.recurrent_layers: - current = self.backbone.layers[index](current, batch.position_ids, batch, cache) + current = self.backbone(current, batch, cache, self.backbone.recurrent_layers) state = justnorm(current) # Dory has no exit gate; a constant gate keeps the scheduler on the # configured depth instead of inventing an exit policy. @@ -587,8 +597,7 @@ def recurrent_prepared(self, hidden, batch, cache, *, compute_gate: bool = True) def coda_prepared(self, hidden, batch, cache, *, compute_logits: bool = True): """Run the attention-bearing output stage and the scaled LM head.""" state, _ = hidden.split(self.config.d_model, dim=-1) - for index in self.backbone.output_layers: - state = self.backbone.layers[index](state, batch.position_ids, batch, cache) + state = self.backbone(state, batch, cache, self.backbone.output_layers) return self.logits(state) if compute_logits else None def logits(self, state: torch.Tensor) -> torch.Tensor: