diff --git a/.dockerignore b/.dockerignore index c042738..6a33b46 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,6 +16,7 @@ dist .env.* **/ray-auth-token remote_exec.sh +*.override.md **/*.safetensors **/*.pt container-cache diff --git a/aspec/_environment.py b/aspec/_environment.py index 696710f..8c1a5ad 100644 --- a/aspec/_environment.py +++ b/aspec/_environment.py @@ -92,7 +92,15 @@ def command_environment(): raise ValueError("ASPEC_CUDA_COMPAT_DIR must contain libcuda.so.1.") paths.append(str(path)) purelib = Path(sysconfig.get_paths()["purelib"]) + cudnn = purelib / "nvidia/cudnn" + if not env.get("CUDNN_HOME") and (cudnn / "lib/libcudnn.so.9").is_file(): + # TE searches CUDA_HOME and the host toolkit before LD_LIBRARY_PATH. + # Bind its explicit search to this interpreter's cuDNN wheel. + env["CUDNN_HOME"] = str(cudnn) + env["ASPEC_AUTO_CUDNN_HOME"] = str(cudnn) toolkit = purelib / "nvidia/cu13" + if (toolkit / "lib/libcudart.so.13").is_file(): + env.setdefault("CUDNN_FRONTEND_CUDART_LIB_NAME", "libcudart.so.13") if not env.get("CUDA_HOME") and not env.get("CUDA_PATH") and (toolkit / "bin/nvcc").is_file(): view = _toolkit_view(toolkit, env) env["CUDA_HOME"] = str(view) @@ -101,7 +109,8 @@ def command_environment(): previous = env.get("LIBRARY_PATH", "") env["LIBRARY_PATH"] = str(view / "lib64") + (os.pathsep + previous if previous else "") paths.append(str(view / "lib64")) - paths.extend(str(p) for p in (purelib / "nvidia/cuda_runtime/lib", purelib / "torch/lib") + paths.extend(str(p) for p in (purelib / "nvidia/cuda_runtime/lib", purelib / "torch/lib", + purelib / "nvidia/cudnn/lib", purelib / "nvidia/nccl/lib") if p.is_dir()) if (toolkit / "lib").is_dir(): paths.append(str(toolkit / "lib")) diff --git a/aspec/_target_worker.py b/aspec/_target_worker.py index 5655ad4..b4068dc 100644 --- a/aspec/_target_worker.py +++ b/aspec/_target_worker.py @@ -13,6 +13,9 @@ def main(): automatic_toolkit = os.environ.pop("ASPEC_AUTO_CUDA_HOME", None) if automatic_toolkit and os.environ.get("CUDA_HOME") == automatic_toolkit: os.environ.pop("CUDA_HOME", None) + automatic_cudnn = os.environ.pop("ASPEC_AUTO_CUDNN_HOME", None) + if automatic_cudnn and os.environ.get("CUDNN_HOME") == automatic_cudnn: + os.environ.pop("CUDNN_HOME", None) os.environ["PATH"] = str(Path(sys.executable).parent) + os.pathsep + os.environ.get("PATH", "") if cache := os.environ.get("ASPEC_CACHE_HOME"): if Path(cache).name != "target": diff --git a/aspec/cli.py b/aspec/cli.py index fec66ca..49b923a 100644 --- a/aspec/cli.py +++ b/aspec/cli.py @@ -48,7 +48,10 @@ def main(): else: module = "aspec.train_entry" help_requested = arguments in (["--help"], ["-h"]) - if module == "aspec.inference.service" and not help_requested and not is_target_environment(): + megatron_target = ("--backend=megatron" in arguments or any( + a == "--backend" and b == "megatron" for a, b in zip(arguments, arguments[1:]) + )) + if module == "aspec.inference.service" and not help_requested and not is_target_environment() and not megatron_target: raise SystemExit("Start aspec target in the Target environment (pip install ling-aspec-target)") try: execute([sys.executable, "-m", module, *arguments]) diff --git a/aspec/commands/_checkpoint_audit.py b/aspec/commands/_checkpoint_audit.py index 8b616ae..10a3c75 100644 --- a/aspec/commands/_checkpoint_audit.py +++ b/aspec/commands/_checkpoint_audit.py @@ -21,19 +21,19 @@ import math -def cpu_values(value): +def cpu_values(value, *, full_tensors=False): import torch if isinstance(value, torch.Tensor): if hasattr(value, "to_local"): - value = value.to_local() + value = value.full_tensor() if full_tensors else value.to_local() return value.detach().cpu().clone() if isinstance(value, dict): - return {key: cpu_values(item) for key, item in value.items()} + return {key: cpu_values(item, full_tensors=full_tensors) for key, item in value.items()} if isinstance(value, list): - return [cpu_values(item) for item in value] + return [cpu_values(item, full_tensors=full_tensors) for item in value] if isinstance(value, tuple): - return tuple(cpu_values(item) for item in value) + return tuple(cpu_values(item, full_tensors=full_tensors) for item in value) return value @@ -80,8 +80,12 @@ def compare_checkpoint(trainer, directory): candidate_model = ModelState(trainer.model).state_dict() candidate_optimizer = OptimizerState(trainer.model, trainer.optimizer).state_dict() - original_model = cpu_values(candidate_model) - original_optimizer = cpu_values(candidate_optimizer) + # Core compute parameters are ordinary local tensors, while its DCP state + # describes logical TP tensors. Read full independent CPU references rather + # than treating a local shard as the checkpoint's complete destination. + logical_tp = getattr(trainer, "execution_backend", "fsdp") == "megatron" + original_model = cpu_values(candidate_model, full_tensors=logical_tp) + original_optimizer = cpu_values(candidate_optimizer, full_tensors=logical_tp) dcp.load({"model_state": original_model}, checkpoint_id=str(directory / "model")) dcp.load({"optim_state": original_optimizer}, checkpoint_id=str(directory / "optimizer")) statistics = { @@ -91,6 +95,8 @@ def compare_checkpoint(trainer, directory): worst_parameters = {key: {"relative_l2": 0, "name": None} for key in statistics} def compare(key, name, reference, actual): + if logical_tp and hasattr(actual, "full_tensor"): + actual = actual.full_tensor() reference = reference.reshape(-1) actual = actual.reshape(-1) if reference.shape != actual.shape: @@ -127,16 +133,25 @@ def compare(key, name, reference, actual): } +def backend_rng_identity(state): + return {"backend": state["backend"], + "cuda_tracker": tensor_hash(sorted(state["cuda_tracker"].items()))} + + def audit_state(trainer): import torch - return { + result = { "global_step": trainer.global_step, "state": state_hashes(trainer), "scheduler": cpu_values(trainer.lr_scheduler.state_dict()), "torch_rng_sha256": hashlib.sha256(torch.random.get_rng_state().numpy().tobytes()).hexdigest(), "cuda_rng_sha256": hashlib.sha256(torch.cuda.get_rng_state().cpu().numpy().tobytes()).hexdigest(), } + snapshot = getattr(trainer, "_checkpoint_backend_rng", None) + if snapshot is not None: + result["backend_rng"] = backend_rng_identity(snapshot()) + return result def add_statistics(total, reference, actual): diff --git a/aspec/commands/_recipe_smoke.py b/aspec/commands/_recipe_smoke.py index 57890da..ca07910 100644 --- a/aspec/commands/_recipe_smoke.py +++ b/aspec/commands/_recipe_smoke.py @@ -111,6 +111,11 @@ def init(self, args, role, mooncake_config=None, with_ref=False): } if any(initial[key] != value for key, value in hashes.items()): raise AssertionError("Restored RNG differs from saved checkpoint") + if "backend_rng" in initial: + from aspec.commands._checkpoint_audit import backend_rng_identity + + if backend_rng_identity(rng["backend_rng"]) != initial["backend_rng"]: + raise AssertionError("Restored model-parallel RNG differs from saved checkpoint") (self._audit_root / f"recipe-resume-diff{self._audit_suffix}.json").write_text( json.dumps({"state": compared, "rng_exact": True}, sort_keys=True) ) diff --git a/aspec/commands/doctor.py b/aspec/commands/doctor.py index 410eeaa..a2b2032 100644 --- a/aspec/commands/doctor.py +++ b/aspec/commands/doctor.py @@ -53,8 +53,10 @@ def check_dependency_metadata(distributions=None): return result -def expected_versions(target=False): - source = asset_path("requirements/sglang-v0.5.21.txt" if target else "requirements/draft.txt") +def expected_versions(target=False, *, backend=None): + manifest = ("megatron-v0.20.0.txt" if backend == "megatron" else + "sglang-v0.5.21.txt" if target else "draft.txt") + source = asset_path("requirements/" + manifest) requirements = [Requirement(line) for line in source.read_text().splitlines() if line.strip() and not line.startswith("#")] return {requirement.name: next((spec.version for spec in requirement.specifier @@ -62,7 +64,68 @@ def expected_versions(target=False): for requirement in requirements} -def inspect_environment(*, gpu=False, target=False): +def _inspect_megatron_environment(*, gpu, target): + result = {"python": sys.executable, "prefix": sys.prefix, "platform": platform.platform(), + "gpu_requested": gpu, "target_requested": target, "backend": "megatron", + "role": "target" if target else "draft", "patch": "not_required", + "checks": [], "errors": [], "dependencies": check_dependency_metadata()} + result["errors"].extend(result["dependencies"]["conflicts"]) + if error := venv_error(): + result["errors"].append(error) + if sys.version_info[:2] < (3, 12) or sys.platform != "linux": + result["errors"].append("Megatron requires Linux with Python >=3.12") + for name, expected in expected_versions(backend="megatron").items(): + try: + actual = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + actual = None + ok = actual is not None and (expected is None or actual.split("+", 1)[0] == expected) + result["checks"].append({"package": name, "expected": expected, "installed": actual, "ok": ok}) + if not ok: + result["errors"].append(f"{name}: expected {expected}, found {actual}") + for name in ("ling-aspec", "ling-aspec-target", "sglang"): + try: + importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + pass + else: + result["errors"].append(f"{name} belongs in a separate backend environment") + if gpu: + try: + import torch + import transformer_engine.pytorch as te + from mooncake.engine import TransferEngine + + from aspec.models.megatron import require_megatron + + require_megatron() + layer = te.Linear(16, 16, params_dtype=torch.bfloat16, device="cuda") + output = layer(torch.ones(2, 16, dtype=torch.bfloat16, device="cuda")) + if not torch.isfinite(output).all().item(): + raise RuntimeError("Transformer Engine BF16 linear produced non-finite values") + result["cuda"] = {"torch": torch.__version__, "runtime": torch.version.cuda, + "device_capability": list(torch.cuda.get_device_capability()), + "transformer_engine": te.__file__, + "mooncake_native_import": TransferEngine is not None} + except Exception as error: + result["errors"].append(f"Megatron GPU/native-library check failed: {error}") + result["ok"] = not result["errors"] + return result + + +def inspect_environment(*, gpu=False, target=False, backend=None): + if backend is None: + marker = Path(sys.prefix) / ".aspec-backend" + if marker.is_file() and marker.read_text().strip() == "megatron": + backend = "megatron" + else: + try: + importlib.metadata.version("ling-aspec-megatron") + backend = "megatron" + except importlib.metadata.PackageNotFoundError: + pass + if backend == "megatron": + return _inspect_megatron_environment(gpu=gpu, target=target) result = {"python": sys.executable, "prefix": sys.prefix, "platform": platform.platform(), "gpu_requested": gpu, "target_requested": target, "checks": [], "errors": []} result["dependencies"] = check_dependency_metadata() @@ -137,9 +200,10 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--gpu", action="store_true", help="Also allocate a small CUDA tensor and import Mooncake") parser.add_argument("--target", action="store_true", help="Also require SGLang and its training patch") + parser.add_argument("--backend", choices=("sglang", "megatron"), default=None) parser.add_argument("--json", action="store_true") args = parser.parse_args() - result = inspect_environment(gpu=args.gpu, target=args.target) + result = inspect_environment(gpu=args.gpu, target=args.target, backend=args.backend) if args.json: print(json.dumps(result, indent=2)) else: diff --git a/aspec/config/inference_config.py b/aspec/config/inference_config.py index 1f8a7b0..324bc68 100644 --- a/aspec/config/inference_config.py +++ b/aspec/config/inference_config.py @@ -34,6 +34,15 @@ from aspec.config.mooncake_config import MooncakeConfig +@dataclass +class MegatronConfig: + """Node-local, frozen Qwen3 prefill through upstream Megatron Core.""" + + tp_size: int = 1 + init_timeout: int = 600 + implementation: str = "transformer_engine" + + @dataclass class SGLangConfig: """Essential SGLang engine configuration. @@ -195,6 +204,7 @@ class InferenceConfig: # performance default; the Plan must explicitly admit or omit it. target_prefill_memory_optimized: bool = False offline: OfflineTrainingConfig = field(default_factory=OfflineTrainingConfig) + megatron: MegatronConfig = field(default_factory=MegatronConfig) sglang: SGLangConfig = field(default_factory=SGLangConfig) vllm: VllmConfig = field(default_factory=VllmConfig) trtllm: TrtllmConfig = field(default_factory=TrtllmConfig) diff --git a/aspec/config/megatron.py b/aspec/config/megatron.py new file mode 100644 index 0000000..a197b6f --- /dev/null +++ b/aspec/config/megatron.py @@ -0,0 +1,96 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Dependency-light capability boundary for the upstream Megatron integration.""" + +MEGATRON_VERSION = "0.20.0" +MEGATRON_REVISION = "685fb8bd970185e9ac3283fe32a5d6667bf4b268" + + +def resolve_megatron_gradient_precision(args): + """Resolve Core main-gradient/DP precision without importing the backend.""" + dtype = getattr(args, "megatron_grad_dtype", "fp32") + if not isinstance(dtype, str) or dtype not in {"fp32", "bf16"}: + raise ValueError("training.megatron_grad_dtype must be fp32 or bf16") + if getattr(args, "train_backend", "fsdp") != "megatron" and dtype != "fp32": + raise ValueError("training.megatron_grad_dtype applies only to train_backend=megatron") + resolved = "float32" if dtype == "fp32" else "bfloat16" + return {"gradient_accumulation_dtype": resolved, "gradient_reduce_dtype": resolved, + "tp_partial_gradient_reduce_dtype": "float32", "optimizer_state_dtype": "float32", + "core_grad_reduce_in_fp32": dtype == "fp32"} + + +def validate_megatron_execution(args, draft): + """Reject combinations outside the runnable dense DSpark CP1/PP1 lane.""" + training = getattr(args, "train_backend", "fsdp") + target = getattr(args, "target_model_backend", "sglang") + resolve_megatron_gradient_precision(args) + tp = getattr(args, "draft_tensor_parallel_size", 1) + if type(tp) is not int or tp < 1: + raise ValueError("training.draft_tensor_parallel_size must be a positive integer") + if training != "megatron" and tp != 1: + raise ValueError("Draft TP requires training.train_backend=megatron") + if training != "megatron" and target != "megatron": + return + if getattr(args, "cluster_mode", "single_node") != "single_node": + raise ValueError("Megatron currently requires single-node execution") + if getattr(args, "aspec_shared_source", False) or getattr(args, "aspec_shared_reader", None): + raise ValueError("Megatron shared Target execution is not supported yet") + if draft["family"] != "dspark": + raise ValueError("Megatron currently supports the DSpark objective") + if draft.get("mtp_conditioning"): + raise ValueError("Megatron currently requires an unconditioned DSpark Draft") + if getattr(args, "draft_context_parallel_size", 1) != 1: + raise ValueError("Megatron currently requires Draft CP1") + if target == "megatron": + if getattr(args, "sglang_extra_args", None) or getattr(args, "sglang_allow_context_length_override", False): + raise ValueError("Megatron Target does not apply SGLang backend overrides") + if getattr(args, "megatron_implementation", "transformer_engine") not in {"local", "transformer_engine"}: + raise ValueError("Unknown Megatron model implementation") + if not getattr(args, "target_prefill_memory_optimized", False): + raise ValueError("Megatron Target requires target_prefill_memory_optimized=true") + if getattr(args, "sglang_nnodes", 1) != 1 or getattr(args, "sglang_pp_size", 1) != 1: + raise ValueError("Megatron Target currently requires node-local TP and PP1") + if getattr(args, "train_with_decode", False): + raise ValueError("Megatron Target supports prefill-only execution") + target_tp = getattr(args, "megatron_tp_size", 1) + if type(target_tp) is not int or target_tp < 1: + raise ValueError("inference.megatron.tp_size must be a positive integer") + if target_tp not in {1, 2}: + raise ValueError("Megatron Target currently validates TP1/TP2") + if target_tp != getattr(args, "inference_num_gpus_per_engine", 1): + raise ValueError("inference.megatron.tp_size must equal inference_num_gpus_per_engine") + slots = (getattr(args, "mooncake_source_gpu_ring_slots", None) + or getattr(args, "mooncake_gpu_ring_slots", 2)) + if getattr(args, "inference_batch_size", 1) > slots: + raise ValueError("Megatron Target source ring must retain one complete inference batch") + if training == "megatron": + if getattr(args, "distributed_backend", "nccl") != "nccl": + raise ValueError("Megatron Draft requires the NCCL distributed backend") + if getattr(args, "micro_batch_size", 1) != 1: + raise ValueError("Megatron Draft currently requires micro_batch_size=1") + prefetch = max(int(getattr(args, "prefetch_depth", 0)), 0) + destination_slots = (getattr(args, "mooncake_destination_gpu_ring_slots", None) + or getattr(args, "mooncake_gpu_ring_slots", 2)) + if prefetch and destination_slots < prefetch + 2: + raise ValueError("Megatron source prefetch requires destination_gpu_ring_slots >= prefetch_depth + 2") + if tp not in {1, 2}: + raise ValueError("Megatron Draft currently validates TP1/TP2") + if getattr(args, "inference_engine_type", "sgl") == "offline": + raise ValueError("Offline replay with Megatron Draft is not validated") + if getattr(args, "fsdp_strategy", "REPLICATE") != "REPLICATE": + raise ValueError("Megatron Draft currently uses replicated DDP rather than FSDP sharding") + if getattr(args, "fsdp_cpu_offload", False): + raise ValueError("Megatron Draft CPU offload is not supported") + if getattr(args, "world_size", 1) % tp: + raise ValueError("Draft world size must be divisible by draft_tensor_parallel_size") + if getattr(args, "dflash_packing", False): + raise ValueError("Megatron Draft packed execution is not supported yet") + if getattr(args, "compile_model", False): + raise ValueError("Megatron Draft currently compiles individual optimized kernels only") + if draft.get("structure", {}).get("ffn_type", "dense") != "dense": + raise ValueError("Megatron Draft currently requires a dense FFN") + attention = draft.get("attention", {}) + if (attention.get("head_layout", {}).get("type", "gqa") not in {"gqa", "mha"} + or any(window is not None for window in attention.get("context_windows", [])) + or any(attention.get(kind) is not None for kind in ("csa", "qsa", "dsa"))): + raise ValueError("Megatron Draft currently requires dense full GQA/MHA attention") diff --git a/aspec/config/train_config.py b/aspec/config/train_config.py index 3fbb132..2c5870b 100644 --- a/aspec/config/train_config.py +++ b/aspec/config/train_config.py @@ -34,6 +34,7 @@ resolve_draft_execution_capability, ) from aspec.config.inference_config import InferenceConfig, _validate_inference_batch_config +from aspec.config.megatron import resolve_megatron_gradient_precision from aspec.config.mooncake_config import resolve_rdma_routes from aspec.inference.concurrency import direct_output_sample_capacity from aspec.utils.logging import logger @@ -140,6 +141,7 @@ class TrainingConfig: # Attention backend and context parallelism are independent execution # dimensions. CP>1 currently requires the FA4 backend. draft_context_parallel_size: int = 1 + draft_tensor_parallel_size: int = 1 colocate: bool = False continual_training: bool = False # A weights-only phase transition normally starts a fresh data stream. Set @@ -214,6 +216,9 @@ class TrainingConfig: # Anchor budget per physical row (legacy) or original source sample. anchor_sampling_scope: str = "row" train_backend: str = "fsdp" + # Core main-gradient accumulation and DP reduction. Optimizer state and + # the dedicated TP-partial norm-gradient reduction remain FP32. + megatron_grad_dtype: str = "fp32" train_env_vars: str = "{}" train_with_decode: bool = False training_num_gpus_per_node: int = 1 @@ -367,12 +372,14 @@ def _validate_vllm_config(config: DictConfig) -> None: def _validate_current_scope(config: DictConfig) -> None: """Reject backend selections outside ASpec's current executable path.""" + resolve_megatron_gradient_precision(config.training) engine_type = config.inference.inference_engine_type target_backend = config.model.target_model_backend - if engine_type not in {"sgl", "offline"} or target_backend != "sglang": + supported = {"sgl": "sglang", "offline": "sglang", "megatron": "megatron"} + if supported.get(engine_type) != target_backend: raise NotImplementedError( - "ASpec currently supports inference.inference_engine_type=sgl or offline with " - "model.target_model_backend=sglang; " + "ASpec currently supports sgl/offline with model.target_model_backend=sglang, " + "or megatron with model.target_model_backend=megatron; " f"got engine_type={engine_type!r}, target_backend={target_backend!r}" ) @@ -854,19 +861,20 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral if not config.inference.target_prefill_memory_optimized: return - if config.inference.inference_engine_type != "sgl": + if config.inference.inference_engine_type not in {"sgl", "megatron"}: raise NotImplementedError( - "target_prefill_memory_optimized currently supports SGLang only" + "target_prefill_memory_optimized requires a live Target backend" ) if config.training.train_with_decode: raise NotImplementedError( "target_prefill_memory_optimized requires prefill-only Target execution" ) - if config.inference.inference_num_gpus_per_engine != 1: + if config.inference.inference_engine_type == "sgl" and config.inference.inference_num_gpus_per_engine != 1: raise NotImplementedError( "target_prefill_memory_optimized currently requires one GPU per Target engine" ) - if config.inference.sglang.tp_size != 1 or config.inference.sglang.pp_size != 1: + if (config.inference.inference_engine_type == "sgl" + and (config.inference.sglang.tp_size != 1 or config.inference.sglang.pp_size != 1)): raise NotImplementedError( "target_prefill_memory_optimized currently requires SGLang TP1/PP1" ) @@ -935,6 +943,10 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral logical_dp_size = world_size // cp_size if is_direct_cp_backend( config.training.attention_backend ) else world_size + tp_size = config.training.draft_tensor_parallel_size if config.training.train_backend == "megatron" else 1 + if type(tp_size) is not int or tp_size < 1: + raise ValueError("training.draft_tensor_parallel_size must be a positive integer") + logical_dp_size //= tp_size dispatch_batch_size = config.training.micro_batch_size * logical_dp_size inference_gpus = config.inference.inference_num_gpus if not isinstance(inference_gpus, int) or inference_gpus < 1: @@ -942,15 +954,16 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral "target_prefill_memory_optimized requires a positive inference_num_gpus" ) if node_local_features: - engine_count = config.inference.inference_num_gpus_per_node + engine_count = (config.inference.inference_num_gpus_per_node + // config.inference.inference_num_gpus_per_engine) dispatch_batch_size = ( config.training.micro_batch_size * config.training.training_num_gpus_per_node - // cp_size + // cp_size // tp_size ) capacity_scope = "per node" else: - engine_count = inference_gpus + engine_count = inference_gpus // config.inference.inference_num_gpus_per_engine capacity_scope = "across the run" resident_sample_capacity = direct_output_sample_capacity( engine_count=engine_count, @@ -1050,6 +1063,7 @@ def load_config( "mooncake": "mooncake_", "offline": "offline_", "sglang": "sglang_", + "megatron": "megatron_", "vllm": "vllm_", "trtllm": "trtllm_", "tokenspeed": "tokenspeed_", diff --git a/aspec/controller/eval.py b/aspec/controller/eval.py index 667e141..e9452d8 100644 --- a/aspec/controller/eval.py +++ b/aspec/controller/eval.py @@ -72,15 +72,23 @@ def eval_cache_identity(args, dataset_identity, count): identity = OmegaConf.to_container(OmegaConf.create(identity), resolve=True) from aspec.runtime_identity import SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity - versions = {} - for name in ("sglang", "torch", "transformers", "flash-attn-4", "flashinfer-python", "sglang-kernel"): - try: - versions[name] = importlib.metadata.version(name) - except importlib.metadata.PackageNotFoundError: - versions[name] = None + if getattr(args, "target_model_backend", None) == "megatron": + identity["megatron_tp_size"] = args.megatron_tp_size + identity["megatron_implementation"] = args.megatron_implementation + producer_code = args.megatron_target_cache_identity + versions = producer_code["dependencies"] + else: + versions = {} + for name in ("sglang", "torch", "transformers", "flash-attn-4", "flashinfer-python", "sglang-kernel"): + try: + versions[name] = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + versions[name] = None + producer_code = module_runtime_identity( + "sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS)["effective_tree_sha256"] return {"schema": "aspec.eval-cache/v2", "dataset": dataset_identity, "samples": count, "target_snapshot": snapshot, "inputs": identity, - "producer_code": module_runtime_identity("sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS)["effective_tree_sha256"], + "producer_code": producer_code, "consumer_cache_code": {name: module_runtime_identity(name)["effective_tree_sha256"] for name in ("aspec.training.data_fetcher", "aspec.utils.processing", "aspec.data.utils")}, diff --git a/aspec/controller/inference_manager.py b/aspec/controller/inference_manager.py index 9018e41..9efe859 100644 --- a/aspec/controller/inference_manager.py +++ b/aspec/controller/inference_manager.py @@ -1216,7 +1216,7 @@ async def leased_engine(): multimodal_inputs=kwargs["multimodal_inputs"], ) if ( - getattr(self.args, "inference_engine_type", "sgl") == "sgl" + getattr(self.args, "inference_engine_type", "sgl") in {"sgl", "megatron"} and not self._train_with_decode ): generate_kwargs["request_ids"] = request_ids diff --git a/aspec/controller/loop.py b/aspec/controller/loop.py index a038e21..8408a59 100644 --- a/aspec/controller/loop.py +++ b/aspec/controller/loop.py @@ -436,7 +436,7 @@ def _safe_training_cleanup( feature_contract_error = None read_owner_fence = closed_reader_fence if train_group is None else None source_teardown_safe = True - external_target = (getattr(args, "inference_engine_type", None) == "sgl" + external_target = (getattr(args, "inference_engine_type", None) in {"sgl", "megatron"} and bool(getattr(args, "target_service", None))) def record(component: str, state: str, error: Exception | None = None) -> None: diff --git a/aspec/evidence.py b/aspec/evidence.py index ccfc32e..0d829d7 100644 --- a/aspec/evidence.py +++ b/aspec/evidence.py @@ -34,6 +34,7 @@ from aspec.planning import ExecutionPlan from aspec.runtime_control import is_termination_exception from aspec.runtime_identity import ( + MEGATRON_RUNTIME_RELATIVE_PATHS, SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity, ) @@ -43,6 +44,9 @@ "ling-aspec", "ling-aspec-runtime", "ling-aspec-target", + "ling-aspec-megatron", + "megatron-core", + "transformer-engine", "torch", "ray", "sglang", @@ -208,7 +212,8 @@ def _module_source_identity(module: str) -> dict[str, Any]: runtime_identity = module_runtime_identity( module, relative_paths=( - SGLANG_RUNTIME_RELATIVE_PATHS if module == "sglang" else None + SGLANG_RUNTIME_RELATIVE_PATHS if module == "sglang" else + MEGATRON_RUNTIME_RELATIVE_PATHS if module == "megatron.core" else None ), ) identity: dict[str, Any] = { @@ -306,7 +311,9 @@ def __init__( "aspec": _source_identity(), "packages": _package_versions(), "package_scope": "draft_driver", - "backends": {"sglang": _module_source_identity("sglang")}, + "backends": ({"megatron": _module_source_identity("megatron.core")} + if plan.policy.backends["target_features"] == "megatron" else + {"sglang": _module_source_identity("sglang")}), "host": { "hostname": platform.node(), "system": platform.system(), @@ -433,13 +440,16 @@ def record_actor_initialization( target_runtimes = [dict(status["runtime"]) for status in (inference_engine_statuses or ()) if isinstance(status.get("runtime"), Mapping)] if target_runtimes: + target_backend = self._plan.policy.backends["target_features"] backend_identities = [item.get("backend_identity") for item in target_runtimes] if (len(target_runtimes) != inference_engine_count + or any(item.get("backend", "sglang") != target_backend for item in target_runtimes) or any(not isinstance(item, Mapping) or not item.get("effective_tree_sha256") or item.get("missing_files") for item in backend_identities) or len({item["effective_tree_sha256"] for item in backend_identities}) != 1): - raise RuntimeError("Target workers reported incomplete or different SGLang identities") - self._data["source"]["backends"]["sglang"] = dict(backend_identities[0]) + name = "SGLang" if target_backend == "sglang" else "Megatron" + raise RuntimeError(f"Target workers reported incomplete or different {name} identities") + self._data["source"]["backends"][target_backend] = dict(backend_identities[0]) self._data["source"]["target_runtimes"] = target_runtimes prior_runtime = self._data["observations"]["runtime"] @@ -500,12 +510,12 @@ def record_actor_initialization( capacity = status.get("token_capacity") if not isinstance(capacity, Mapping): raise RuntimeError( - f"SGLang engine {status.get('rank')} did not report token capacity" + f"Target engine {status.get('rank')} did not report token capacity" ) observed_context = capacity.get("context") if observed_context != planned_context: raise RuntimeError( - "Observed SGLang context contract does not match the Plan: " + "Observed Target context contract does not match the Plan: " f"rank={status.get('rank')}, planned={planned_context}, " f"observed={observed_context}" ) @@ -513,7 +523,7 @@ def record_actor_initialization( "backend_context_length" ]: raise RuntimeError( - "Observed SGLang context length does not match the Plan: " + "Observed Target context length does not match the Plan: " f"rank={status.get('rank')}, " f"planned={backend_tokens['backend_context_length']}, " f"observed={capacity.get('context_length')}" @@ -570,7 +580,7 @@ def record_actor_initialization( { "predicate": ("all_planned_offline_replay_initializers_completed" if self._plan.policy.backends["target_features"] == "offline_replay" - else "all_planned_sglang_engine_initializers_completed"), + else f"all_planned_{self._plan.policy.backends['target_features']}_engine_initializers_completed"), "observed_engine_count": inference_engine_count, }, ) @@ -599,6 +609,18 @@ def _validate_actor_placement( training = [dict(item) for item in (training_actor_statuses or ())] inference = [dict(item) for item in (inference_engine_statuses or ())] + if self._plan.policy.backends["training"] == "megatron": + expected_tp = self._plan.policy.parallelism["training"]["tensor_parallel"] + expected_dp = self._plan.policy.parallelism["training"]["data_parallel"] + if (not training or any(item.get("backend") != "megatron" + or item.get("tensor_parallel_size") != expected_tp + or item.get("data_parallel_size") != expected_dp + or not item.get("initialized") for item in training)): + raise RuntimeError("Megatron trainer/backend TP was not observed as planned") + identities = [item.get("runtime", {}).get("backend_identity", {}) for item in training] + if (any(not item.get("effective_tree_sha256") or item.get("missing_files") for item in identities) + or len({item.get("effective_tree_sha256") for item in identities}) != 1): + raise RuntimeError("Megatron trainers reported incomplete or different Core identities") if training and len(training) != self._plan.policy.resources["training"]["world_size"]: raise RuntimeError("trainer placement status count does not match the Plan") if inference and len(inference) != self._plan.policy.resources["inference"]["engines"]: @@ -833,7 +855,7 @@ def record_training(self, summary: Mapping[str, Any]) -> None: transfer_observation = { "predicate": ("optimizer_step_consumed_validated_offline_features_via_mooncake" if self._plan.policy.backends["target_features"] == "offline_replay" - else "optimizer_step_consumed_validated_sglang_features_via_mooncake"), + else f"optimizer_step_consumed_validated_{self._plan.policy.backends['target_features']}_features_via_mooncake"), "optimizer_steps_observed": training["optimizer_steps_observed"], "feature_contract_id": planned_contract_id, "producer_count": feature_contract["producer_count"], diff --git a/aspec/feature_contract.py b/aspec/feature_contract.py index ae40798..5a370d1 100644 --- a/aspec/feature_contract.py +++ b/aspec/feature_contract.py @@ -67,6 +67,7 @@ def build_target_feature_contract( target_num_hidden_layers: int, aux_hidden_state_layers: list[int], store_last_hidden_states: bool, + producer_backend: str = "sglang", mtp: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Build the payload contract without starting a backend. @@ -76,6 +77,8 @@ def build_target_feature_contract( differences still produce distinct identities. """ + if producer_backend not in {"sglang", "megatron"}: + raise FeatureContractError(f"Unsupported Target producer backend: {producer_backend!r}") if not isinstance(target_hidden_size, int) or target_hidden_size <= 0: raise FeatureContractError( f"draft target_hidden_size must be a positive integer, got {target_hidden_size!r}" @@ -168,7 +171,7 @@ def build_target_feature_contract( contract: dict[str, Any] = { "schema_version": FEATURE_CONTRACT_SCHEMA_VERSION, - "producer": {"backend": "sglang", "mode": "prefill_only"}, + "producer": {"backend": producer_backend, "mode": "prefill_only"}, "consumer": { "target_distribution": target_distribution, }, @@ -197,7 +200,10 @@ def validate_feature_contract(contract: Mapping[str, Any]) -> None: raise FeatureContractError( f"Target feature contract identity mismatch: {observed_id!r} != {expected_id!r}" ) - if contract.get("producer") != {"backend": "sglang", "mode": "prefill_only"}: + if contract.get("producer") not in ( + {"backend": "sglang", "mode": "prefill_only"}, + {"backend": "megatron", "mode": "prefill_only"}, + ): raise FeatureContractError( f"Unsupported target feature producer contract: {contract.get('producer')!r}" ) diff --git a/aspec/inference/context.py b/aspec/inference/context.py index b6c034d..59bf8d4 100644 --- a/aspec/inference/context.py +++ b/aspec/inference/context.py @@ -269,6 +269,13 @@ def configure_sglang_context_override(contract: Mapping[str, Any]) -> bool: return enabled +def compile_megatron_context_contract(**kwargs): + """Frozen prefill consumes exactly the requested sequence, without decode reserve.""" + contract = compile_offline_context_contract(**kwargs) + contract["backend_context_length"] = contract["effective_training_max_sequence_length"] + return contract + + def compile_offline_context_contract( *, max_sequence_length, context_parallel_size, model_config, native_context_length, allow_context_length_override, context_shard_alignment=1, diff --git a/aspec/inference/engine/megatron_engine.py b/aspec/inference/engine/megatron_engine.py new file mode 100644 index 0000000..8366afb --- /dev/null +++ b/aspec/inference/engine/megatron_engine.py @@ -0,0 +1,294 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Frozen upstream Megatron Target workers behind ASpec's Target service. + +Each engine owns a role-local TP world. Only the publishing rank owns source +storage; TP activations are replicated because sequence parallelism is disabled. +Control pipes carry tokens and descriptors, never feature tensors. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import os +import sys +import threading +import traceback +import uuid +from datetime import timedelta + +import torch + +from aspec.feature_contract import FeatureContractRuntime +from aspec.inference.target_prefill import TargetPrefillDirectSink, TargetPrefillRequest +from aspec.ray.ray_actor import RayActor +from aspec.utils.misc import get_current_node_ip + + +def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, engine_rank): + import torch.distributed as dist + + from aspec.models.megatron import initialize_parallelism, load_qwen3_target + + store = None + try: + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(gpu) for gpu in gpu_ids) + torch.cuda.set_device(local_rank) + dist.init_process_group( + "nccl", init_method=f"tcp://127.0.0.1:{port}", + rank=local_rank, world_size=len(gpu_ids), + timeout=timedelta(seconds=getattr(args, "megatron_init_timeout", 600)), + ) + groups = initialize_parallelism(len(gpu_ids)) + model, config = load_qwen3_target( + args.target_model_path, groups, + implementation=getattr(args, "megatron_implementation", "transformer_engine"), + ) + captures = list(args.aux_hidden_states_layers) + runtime = FeatureContractRuntime( + args.aspec_feature_contract, role="producer", participant=f"megatron_engine_{engine_rank}", + ) + runtime.validate_producer_binding( + hidden_size=config.hidden_size, num_hidden_layers=config.num_hidden_layers, + aux_hidden_state_layers=captures, store_last_hidden_states=args.store_last_hidden_states, + ) + if local_rank == 0: + from aspec.transfer.mooncake.eagle_store import EagleMooncakeStore + + source_config = dataclasses.replace(mooncake_config, global_segment_size=0, async_put_pool_size=0) + store = EagleMooncakeStore(source_config) + store.setup(device=local_rank, p2p_role="source") + status = { + "pid": os.getpid(), "hidden_size": config.hidden_size, + "python": sys.executable, "prefix": sys.prefix, + "aspec": __import__("aspec").__file__, + "megatron": __import__("megatron.core", fromlist=["__file__"]).__file__, + "torch": torch.__version__, "cuda": torch.version.cuda, + "tensor_parallel_size": len(gpu_ids), "local_rank": local_rank, + "device_capability": list(torch.cuda.get_device_capability()), + "model_implementation": getattr(args, "megatron_implementation", "transformer_engine"), + "model_class": type(model).__name__, + "hybrid_layer_pattern": model.hybrid_layer_pattern, + "target_frozen": not any(p.requires_grad for p in model.parameters()), + } + connection.send({"ok": True, "value": status}) + while True: + method, payload = connection.recv() + if method == "shutdown": + if store is not None: + store.close() + dist.destroy_process_group() + connection.send({"ok": True, "value": None}) + return + if method == "status": + features = runtime.status() + if store is not None: + features["transport"] = store.get_p2p_status() + connection.send({"ok": True, "value": features}) + continue + if method != "generate": + raise ValueError(f"Unknown Megatron Target operation: {method}") + outputs = [] + for sample in payload: + ids = torch.tensor(sample["tokens"], dtype=torch.long, device="cuda") + length = ids.numel() + if not 0 < length <= args.max_seq_length: + raise ValueError("Target input length is outside the planned context") + request = TargetPrefillRequest(sample["request_id"], sample["data_id"], length) + sink = None + try: + if local_rank == 0: + sink = TargetPrefillDirectSink( + store=store, requests=[request], capture_layers=captures, + hidden_size=config.hidden_size, store_last_hidden_states=args.store_last_hidden_states, + source_engine=f"megatron_{engine_rank}", context_parallel_size=1, + ) + sink.begin(input_ids=ids, request_ids=[request.request_id], + sequence_lengths=[length], prefix_lengths=[0]) + final = model.prefill(ids.unsqueeze(0), captures, sink.capture if sink is not None else None) + if local_rank == 0: + sink.finish(final) + key = sink.commit_request(request.request_id) + shapes = {"hidden_states": (length, len(captures) * config.hidden_size), + "input_ids": (length,)} + dtypes = {"hidden_states": torch.bfloat16, "input_ids": torch.int64} + if args.store_last_hidden_states: + shapes["last_hidden_states"] = (length, config.hidden_size) + dtypes["last_hidden_states"] = torch.bfloat16 + metadata = { + "request_id": request.request_id, "engine_rank": engine_rank, + "aspec_feature_contract_id": runtime.contract_id, + "source_node_ip": get_current_node_ip(), + "feature_affinity": getattr(args, "mooncake_feature_affinity", "any"), + "target_feature_output": "direct_to_transport", + "transport_data_plane": "transfer_engine", + "transport_protocol": mooncake_config.protocol, + "feature_layout": "full", + "draft_context_parallel_size": 1, "draft_feature_layout": "full", + "draft_attention_backend": args.attention_backend, + } + from aspec.config.draft_execution import is_direct_cp_backend + + if is_direct_cp_backend(args.attention_backend): + metadata.update(direct_cp_sharded=True, direct_cp_size=1) + runtime.validate_metadata(tensor_shapes=shapes, tensor_dtypes=dtypes, metadata=metadata) + sink.validate_publishable() + sink.publish() + outputs.append({"mooncake_key": key, "tensor_shapes": shapes, + "tensor_dtypes": dtypes, "metadata": metadata}) + except BaseException: + if sink is not None: + sink.abort() + raise + connection.send({"ok": True, "value": outputs if local_rank == 0 else None}) + except BaseException as error: + try: + connection.send({"ok": False, "error": repr(error), "traceback": traceback.format_exc()}) + finally: + # Published storage remains owned until the service fences readers + # and observes this process exiting, including native-call failures. + connection.close() + if store is not None: + # Keep all published CUDA registrations alive on failure. The + # Target service terminates this world only after its read fence. + threading.Event().wait() + + +class MegatronEngine(RayActor): + """Ray control facade; model and source storage live in owned child workers.""" + + def __init__(self, args, rank, base_gpu_id=None, num_gpus_per_engine=1, + node_rank=0, engine_group=0, gpu_ids=None): + self.args, self.rank = args, rank + self.base_gpu_id = base_gpu_id + self.gpu_ids = list(gpu_ids or range(base_gpu_id, base_gpu_id + num_gpus_per_engine)) + self.num_gpus_per_engine = num_gpus_per_engine + self._connections, self._processes = [], [] + self._status = [] + self._lock = threading.Lock() + self._mooncake_config = None + self._features = None + self._failed = False + + def _receive(self): + try: + return self._receive_workers() + except BaseException: + self._failed = True + raise + + def _receive_workers(self): + results = [] + for connection, process in zip(self._connections, self._processes, strict=True): + if not connection.poll(getattr(self.args, "megatron_init_timeout", 600)): + raise TimeoutError(f"Megatron Target worker {process.pid} did not complete") + result = connection.recv() + if not result["ok"]: + raise RuntimeError(f"Megatron Target worker failed: {result['error']}\n{result['traceback']}") + results.append(result["value"]) + return results + + def _call(self, method, payload=None): + with self._lock: + if not self.health_check(): + raise RuntimeError("Megatron Target worker world is unavailable") + try: + for connection in self._connections: + connection.send((method, payload)) + return self._receive()[0] + except BaseException: + self._failed = True + raise + + def init(self, mooncake_config=None, pre_allocated_port=None): + if self._processes: + raise RuntimeError("Megatron Target engine cannot be initialized twice") + if mooncake_config is None or mooncake_config.data_plane != "transfer_engine": + raise ValueError("Megatron Target requires the Mooncake transfer_engine data plane") + if pre_allocated_port is None: + raise ValueError("Megatron Target requires a reserved initialization port") + self._mooncake_config = mooncake_config + context = torch.multiprocessing.get_context("spawn") + for rank in range(self.num_gpus_per_engine): + parent, child = context.Pipe() + process = context.Process(target=_target_worker, args=( + child, rank, self.gpu_ids, self.args, mooncake_config, pre_allocated_port, self.rank, + )) + process.start() + child.close() + self._connections.append(parent) + self._processes.append(process) + self._status = self._receive() + + async def generate(self, data_id, input_ids_ref=None, packed_loss_mask_list=None, + formatted_prompts=None, request_ids=None, return_last_hidden_states=False, + return_logits=True, multimodal_inputs=None): + if multimodal_inputs: + raise ValueError("Megatron Target currently supports text-only Qwen3") + if (input_ids_ref is None) == (formatted_prompts is None): + raise ValueError("Provide exactly one of input_ids_ref or formatted_prompts") + if formatted_prompts is not None: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(self.args.target_model_path, trust_remote_code=False) + tokens = tokenizer(formatted_prompts, add_special_tokens=False)["input_ids"] + else: + tokens = [value.reshape(-1).tolist() for value in input_ids_ref] + ids = ([f"{data_id}_{index}" for index in range(len(tokens))] + if isinstance(data_id, str) else list(data_id)) + if len(ids) != len(tokens): + raise ValueError("Target data identities and token sequences differ in length") + if request_ids is None: + request_ids = [uuid.uuid4().hex for _ in tokens] + elif isinstance(request_ids, str): + request_ids = [f"{request_ids}_{index}" for index in range(len(tokens))] + if len(request_ids) != len(tokens): + raise ValueError("Target request identities and token sequences differ in length") + payload = [dict(tokens=value, data_id=data, request_id=request) + for value, data, request in zip(tokens, ids, request_ids, strict=True)] + return await asyncio.to_thread(self._call, "generate", payload) + + def get_status(self): + return { + "rank": self.rank, "pid": os.getpid(), "node_ip": self.get_node_ip(), + "initialized": bool(self._status) and self.health_check(), + "base_gpu_id": self.base_gpu_id, "engine_gpu_ids": self.gpu_ids, + "num_gpus_per_engine": self.num_gpus_per_engine, + "hidden_size": self._status[0]["hidden_size"] if self._status else None, + "target_prefill_memory_optimized": True, "workers": self._status, + "token_capacity": {"context": self.args.aspec_context_contract, + "context_length": self.args.aspec_context_contract["backend_context_length"], + "profiled_max_total_tokens": None, + "required_max_total_tokens": None, + "max_prefill_tokens": self.args.max_seq_length, + "requested_max_running_requests": 1, + "allocation_policy": "sequential_full_prefill_without_kv_pool"}, + } + + def health_check(self, timeout=5.0): + return (not self._failed and bool(self._processes) + and all(process.is_alive() for process in self._processes)) + + def get_mooncake_config(self): + return self._mooncake_config + + def get_feature_contract_status(self): + if self._processes and self.health_check(): + self._features = self._call("status") + return self._features or {"role": "producer", "state": "pending_observation"} + + async def prepare_feature_contract_status(self): + return await asyncio.to_thread(self.get_feature_contract_status) + + async def shutdown(self): + if self._processes: + await asyncio.to_thread(self._call, "shutdown") + for process in self._processes: + process.join(timeout=30) + if process.is_alive(): + raise RuntimeError("Megatron Target worker did not exit after source drain") + for connection in self._connections: + connection.close() + self._processes.clear() + self._connections.clear() diff --git a/aspec/inference/engine/target_actor.py b/aspec/inference/engine/target_actor.py index 308a294..2da486f 100644 --- a/aspec/inference/engine/target_actor.py +++ b/aspec/inference/engine/target_actor.py @@ -18,9 +18,16 @@ def __init__(self, *args, **kwargs): self.runtime_identity = code_identity() self.gpu_ids = kwargs.pop("gpu_ids") - from aspec.inference.engine.sgl_engine import SglEngine + arguments = kwargs.get("args", args[0] if args else None) + self.backend = getattr(arguments, "target_model_backend", "sglang") + if self.backend == "megatron": + from aspec.inference.engine.megatron_engine import MegatronEngine - self.engine = SglEngine(*args, **kwargs) + self.engine = MegatronEngine(*args, gpu_ids=self.gpu_ids, **kwargs) + else: + from aspec.inference.engine.sgl_engine import SglEngine + + self.engine = SglEngine(*args, **kwargs) def init(self, **kwargs): return self.engine.init(**kwargs) @@ -37,13 +44,19 @@ def find_free_port(self, *args, **kwargs): def get_status(self): import psutil import ray - import sglang import torch import aspec - from aspec.runtime_identity import SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity - - backend = module_runtime_identity("sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS) + from aspec.runtime_identity import ( + MEGATRON_RUNTIME_RELATIVE_PATHS, + SGLANG_RUNTIME_RELATIVE_PATHS, + module_runtime_identity, + ) + + if self.backend == "megatron": + backend = module_runtime_identity("megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS) + else: + backend = module_runtime_identity("sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS) backend_identity = { "path": backend["path"], "git_revision": None, "git_dirty": None, "effective_tree_sha256": backend["effective_tree_sha256"], @@ -52,18 +65,24 @@ def get_status(self): result = self.engine.get_status() result["engine_gpu_ids"] = self.gpu_ids result["created"] = psutil.Process().create_time() + from aspec.inference.target_runtime import dependency_versions + result["runtime"] = { **self.runtime_identity, "python": sys.executable, "prefix": sys.prefix, "aspec": str(Path(aspec.__file__).resolve()), "ray": str(Path(ray.__file__).resolve()), - "sglang": str(Path(sglang.__file__).resolve()), "backend_identity": backend_identity, - "sglang_version": importlib.metadata.version("sglang"), + "backend": self.backend, + "dependencies": dependency_versions("sglang", "megatron-core", "transformer-engine", + "flashinfer-python", "sglang-kernel"), "torch": torch.__version__, "cuda": torch.version.cuda, - "cutlass": importlib.metadata.version("nvidia-cutlass-dsl"), - "quack": importlib.metadata.version("quack-kernels"), + "cutlass": dependency_versions("nvidia-cutlass-dsl")["nvidia-cutlass-dsl"], + "quack": dependency_versions("quack-kernels")["quack-kernels"], } + if self.backend == "sglang": + result["runtime"]["sglang"] = backend["path"] + result["runtime"]["sglang_version"] = importlib.metadata.version("sglang") return result def get_mooncake_config(self): diff --git a/aspec/inference/factory.py b/aspec/inference/factory.py index 45fdccc..9305f81 100644 --- a/aspec/inference/factory.py +++ b/aspec/inference/factory.py @@ -37,10 +37,9 @@ def _require_supported_engine(engine_type: str) -> None: - if engine_type not in {"sgl", "offline"}: + if engine_type not in {"sgl", "offline", "megatron"}: raise NotImplementedError( - "ASpec currently supports the SGLang or offline replay path " - f"(inference_engine_type='sgl' or 'offline'); got {engine_type!r}" + f"Supported inference engines are sgl, megatron and offline; got {engine_type!r}" ) @@ -59,7 +58,7 @@ def create_inference_engines(args, inference_pg, mooncake_config, engine_group: _wait_for_init(init_refs, "OfflineReplay", timeout=300) return engines - if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed"): + if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed", "megatron"): raise ValueError(f"Unknown inference_engine_type: {engine_type}") logger.info(f"Using {engine_type} engine for inference") @@ -94,14 +93,14 @@ def prepare_inference_engines(args, inference_pg, mooncake_config, engine_group: if engine_type == "offline": return _prepare_offline_replay_engines(args, mooncake_config, engine_group) - if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed"): + if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed", "megatron"): raise ValueError(f"Unknown inference_engine_type: {engine_type}") logger.info(f"Preparing {engine_type} inference engines...") if engine_type == "hf": engines, init_refs = _prepare_hf_engines(args, inference_pg, mooncake_config, engine_group) - elif engine_type == "sgl": + elif engine_type in {"sgl", "megatron"}: engines, init_refs = _prepare_sgl_engines(args, inference_pg, mooncake_config, engine_group) elif engine_type == "trtllm": engines, init_refs = _prepare_trtllm_engines( @@ -159,7 +158,7 @@ def init_engines(args, pg, engine_type: str, mooncake_config=None, engine_group: _require_supported_engine(engine_type) if engine_type == "hf": return _init_hf_engines(args, pg, mooncake_config, engine_group) - elif engine_type == "sgl": + elif engine_type in {"sgl", "megatron"}: return _init_sgl_engines(args, pg, mooncake_config, engine_group) elif engine_type == "vllm": return _init_vllm_engines(args, pg, mooncake_config, engine_group) diff --git a/aspec/inference/managed.py b/aspec/inference/managed.py index 47a89ca..f2c12e3 100644 --- a/aspec/inference/managed.py +++ b/aspec/inference/managed.py @@ -53,7 +53,8 @@ def managed_target(args, *, draft_gpu_count=None): config = root / "target-service.json" config.write_text(json.dumps({"model": {"target_model_path": args.target_model_path}})) command = [settings["python_executable"], "-m", "aspec._target_worker", "-m", - "aspec.inference.service", "--config", str(config), "--address", address] + "aspec.inference.service", "--config", str(config), "--address", address, + "--backend", getattr(args, "target_model_backend", "sglang")] log_path = root / "target-service.log" with log_path.open("w") as log: process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, @@ -61,7 +62,10 @@ def managed_target(args, *, draft_gpu_count=None): ready = False prior_devices = os.environ.get("CUDA_VISIBLE_DEVICES") try: - deadline = time.monotonic() + float(getattr(args, "sglang_init_timeout", 300)) + timeout_field = ("megatron_init_timeout" + if getattr(args, "target_model_backend", "sglang") == "megatron" + else "sglang_init_timeout") + deadline = time.monotonic() + float(getattr(args, timeout_field, 300)) while time.monotonic() < deadline: if process.poll() is not None: raise RuntimeError(f"Target exited before readiness; see {log_path}") diff --git a/aspec/inference/service.py b/aspec/inference/service.py index af37079..5b6b696 100644 --- a/aspec/inference/service.py +++ b/aspec/inference/service.py @@ -93,6 +93,8 @@ def create(self, value): if set(arguments) - TARGET_ARGUMENTS: raise ValueError("Unknown Target configuration fields") args = Namespace(**arguments) + if getattr(args, "target_model_backend", "sglang") != self.runtime.get("backend", "sglang"): + raise ValueError("Requested Target backend differs from the service environment") validate_target_tp(getattr(args, "sglang_nnodes", 1), value["node_rank"]) if self.model and args.target_model_path != self.model: raise ValueError("Target model differs from the service configuration") @@ -437,6 +439,7 @@ def main(): logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser(description="Run Target in this environment; Draft connects by address.") parser.add_argument("--config", type=Path) + parser.add_argument("--backend", choices=("sglang", "megatron"), default="sglang") parser.add_argument("--address", help="Listening http://host:port; defaults to the config or localhost:30000") args = parser.parse_args() config = OmegaConf.load(args.config) if args.config else OmegaConf.create({}) @@ -446,7 +449,7 @@ def main(): devices = visible_gpu_ids() if not devices: parser.error("Select allocated Target GPUs with CUDA_VISIBLE_DEVICES") - runtime = inventory() # Check this environment and patch before starting processes. + runtime = inventory(args.backend) service = TargetService(model=model, devices=devices, runtime=runtime) parsed = urlsplit(address) server = http_server(service, parsed.hostname, parsed.port) diff --git a/aspec/inference/service_protocol.py b/aspec/inference/service_protocol.py index 9de6d54..c66f6e0 100644 --- a/aspec/inference/service_protocol.py +++ b/aspec/inference/service_protocol.py @@ -23,7 +23,8 @@ # to this interface. Optimizers, datasets, logging credentials and driver # objects remain in Draft. Unknown Draft settings do not change this protocol. TARGET_ARGUMENTS = frozenset(""" -target_model_path trust_remote_code aux_hidden_states_layers store_last_hidden_states native_context_length +target_model_path target_model_backend trust_remote_code aux_hidden_states_layers store_last_hidden_states native_context_length +megatron_tp_size megatron_init_timeout megatron_implementation draft_algorithm mooncake_hidden_dim sglang_mem_fraction_static sglang_pp_size sglang_nnodes sglang_tp_size sglang_extra_args sglang_allow_context_length_override sglang_enable_multimodal diff --git a/aspec/inference/target_runtime.py b/aspec/inference/target_runtime.py index 3527c93..35851de 100644 --- a/aspec/inference/target_runtime.py +++ b/aspec/inference/target_runtime.py @@ -35,8 +35,19 @@ def dependency_versions(*names): return result -def inventory(): +def inventory(backend="sglang"): """Read Target identity in its own interpreter, without starting GPU services.""" + if backend == "megatron": + from aspec.models.megatron import require_megatron + from aspec.runtime_identity import MEGATRON_RUNTIME_RELATIVE_PATHS, module_runtime_identity + + require_megatron() + return {**code_identity(), "backend": backend, "python": sys.executable, "prefix": sys.prefix, + "megatron": module_runtime_identity("megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS), + "dependencies": dependency_versions("torch", "ray", "megatron-core", "transformer-engine", + "flashinfer-python", "sglang-kernel")} + if backend != "sglang": + raise ValueError(f"Unsupported Target backend: {backend}") from aspec.commands.doctor import inspect_environment from aspec.runtime_identity import SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity diff --git a/aspec/models/megatron.py b/aspec/models/megatron.py new file mode 100644 index 0000000..8b3327f --- /dev/null +++ b/aspec/models/megatron.py @@ -0,0 +1,198 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Public Megatron Core construction and strict Qwen3 weight mapping. + +Uses Megatron Core 0.20's public model/spec/process-group interfaces. Qwen3 +mapping follows the grouped QKV and gate/up layouts documented by NVIDIA's +Megatron Bridge Qwen3 adapter. +""" + +from __future__ import annotations + +import importlib.metadata +import json +from pathlib import Path + +import torch +import torch.distributed as dist +from safetensors import safe_open + +from aspec.config.megatron import MEGATRON_VERSION + + +def require_megatron(): + from packaging.version import Version + + try: + version = importlib.metadata.version("megatron-core") + except importlib.metadata.PackageNotFoundError as exc: + raise RuntimeError("Install the pinned ASpec Megatron environment") from exc + if Version(version).base_version != MEGATRON_VERSION: + raise RuntimeError(f"Megatron Core {MEGATRON_VERSION} is required; found {version}") + return version + + +def initialize_parallelism(tp_size: int, *, seed: int = 42): + """The caller's default process group contains only one execution role.""" + require_megatron() + from megatron.core import parallel_state + from megatron.core.process_groups_config import ProcessGroupCollection + + if not dist.is_initialized(): + raise RuntimeError("Initialize the role-local process group before Megatron") + if parallel_state.model_parallel_is_initialized(): + raise RuntimeError("A Megatron model-parallel world is already initialized") + parallel_state.initialize_model_parallel( + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + create_gloo_process_groups=False, + ) + from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + + model_parallel_cuda_manual_seed(seed) + return ProcessGroupCollection.use_mpu_process_groups() + + +def transformer_config(config, tp_size: int): + from megatron.core.transformer.transformer_config import TransformerConfig + + hidden = int(config.hidden_size) + heads = int(config.num_attention_heads) + groups = int(config.num_key_value_heads) + intermediate = int(config.intermediate_size) + if any(value % tp_size for value in (heads, groups, intermediate, config.vocab_size)): + raise ValueError("Qwen3 heads, KV groups, FFN and vocabulary must divide Target TP") + if config.model_type != "qwen3" or getattr(config, "num_experts", None): + raise ValueError("Megatron Target currently requires dense Qwen3") + if getattr(config, "attention_bias", False): + raise ValueError("Megatron Qwen3 prefill requires bias-free attention projections") + if getattr(config, "quantization_config", None) or getattr(config, "hidden_act", "silu") != "silu": + raise ValueError("Megatron Qwen3 requires unquantized dense SwiGLU weights") + rope = getattr(config, "rope_parameters", None) or {} + scaling = getattr(config, "rope_scaling", None) or rope + if scaling.get("rope_type", scaling.get("type", "default")) != "default": + raise ValueError("Megatron Qwen3 currently requires the original default RoPE policy") + return TransformerConfig( + num_layers=2 * int(config.num_hidden_layers), + hidden_size=hidden, + ffn_hidden_size=intermediate, + num_attention_heads=heads, + num_query_groups=groups, + kv_channels=int(getattr(config, "head_dim", None) or hidden // heads), + normalization="RMSNorm", + layernorm_epsilon=float(config.rms_norm_eps), + qk_layernorm=True, + gated_linear_unit=True, + activation_func=torch.nn.functional.silu, + add_bias_linear=False, + add_qkv_bias=False, + attention_dropout=0.0, + hidden_dropout=0.0, + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + params_dtype=torch.bfloat16, + bf16=True, + perform_initialization=False, + use_cpu_initialization=False, + gradient_accumulation_fusion=False, + ) + + +class SafetensorReader: + """Read one source parameter at a time, with an explicit model identity.""" + + def __init__(self, path): + from huggingface_hub import snapshot_download + + self.root = Path(path) + if not self.root.is_dir(): + self.root = Path(snapshot_download(path, allow_patterns=["*.json", "*.safetensors"])) + index = self.root / "model.safetensors.index.json" + if index.is_file(): + self.files = json.loads(index.read_text())["weight_map"] + else: + with safe_open(self.root / "model.safetensors", framework="pt", device="cpu") as source: + self.files = {key: "model.safetensors" for key in source.keys()} + self.read = set() + + def tensor(self, name): + if name not in self.files: + raise ValueError(f"Target checkpoint is missing required parameter {name}") + self.read.add(name) + with safe_open(self.root / self.files[name], framework="pt", device="cpu") as source: + return source.get_tensor(name) + + +def _shard(value, axis, rank, size): + if value.shape[axis] % size: + raise ValueError(f"Tensor dimension {axis} cannot be partitioned over TP{size}") + return value.chunk(size, dim=axis)[rank].contiguous() + + +def qwen3_parameter(reader, name, config, rank, tp_size): + """Map public MCore parameter names, preserving native Qwen3 values.""" + if name == "embedding.word_embeddings.weight": + return _shard(reader.tensor("model.embed_tokens.weight"), 0, rank, tp_size) + if name == "final_norm.weight": + return reader.tensor("model.norm.weight") + pieces = name.split(".") + if pieces[:2] != ["decoder", "layers"] or len(pieces) < 5: + raise ValueError(f"Unsupported Megatron Qwen3 parameter: {name}") + prefix = f"model.layers.{int(pieces[2]) // 2}." + suffix = ".".join(pieces[3:]) + aliases = { + "input_layernorm.weight": "input_layernorm.weight", + "pre_mlp_layernorm.weight": "post_attention_layernorm.weight", + "self_attention.q_layernorm.weight": "self_attn.q_norm.weight", + "self_attention.k_layernorm.weight": "self_attn.k_norm.weight", + } + if suffix in aliases: + return reader.tensor(prefix + aliases[suffix]) + if suffix == "self_attention.linear_qkv.weight": + groups = int(config.num_key_value_heads) + heads_per_group = int(config.num_attention_heads) // groups + width = int(getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads) + query = reader.tensor(prefix + "self_attn.q_proj.weight").view(groups, heads_per_group, width, -1) + key = reader.tensor(prefix + "self_attn.k_proj.weight").view(groups, 1, width, -1) + value = reader.tensor(prefix + "self_attn.v_proj.weight").view(groups, 1, width, -1) + return _shard(torch.cat((query, key, value), dim=1), 0, rank, tp_size).flatten(0, 2) + if suffix == "self_attention.linear_proj.weight": + return _shard(reader.tensor(prefix + "self_attn.o_proj.weight"), 1, rank, tp_size) + if suffix == "mlp.linear_fc1.weight": + return torch.cat([ + _shard(reader.tensor(prefix + "mlp." + part + ".weight"), 0, rank, tp_size) + for part in ("gate_proj", "up_proj") + ], dim=0) + if suffix == "mlp.linear_fc2.weight": + return _shard(reader.tensor(prefix + "mlp.down_proj.weight"), 1, rank, tp_size) + raise ValueError(f"Unsupported Megatron Qwen3 parameter: {name}") + + +def load_qwen3_target(path, groups, *, implementation="transformer_engine"): + """Load the frozen feature-only HybridModel, including its final norm.""" + from transformers import AutoConfig + + from aspec.models.megatron_target import build_target + + reader = SafetensorReader(path) + config = AutoConfig.from_pretrained(reader.root, trust_remote_code=False) + tp_size = dist.get_world_size(groups.tp) + rank = dist.get_rank(groups.tp) + core_config = transformer_config(config, tp_size) + model = build_target(config, core_config, groups, implementation) + model.to(dtype=torch.bfloat16) + with torch.no_grad(): + for name, parameter in model.named_parameters(): + value = qwen3_parameter(reader, name, config, rank, tp_size) + if tuple(value.shape) != tuple(parameter.shape): + raise ValueError(f"Qwen3 mapping shape mismatch for {name}: {value.shape} != {parameter.shape}") + parameter.copy_(value) + unused = set(reader.files) - reader.read + # The feature-only Target never constructs the vocabulary projection. + unused.discard("lm_head.weight") + if unused: + raise ValueError(f"Unmapped Target checkpoint parameters: {sorted(unused)}") + model.eval().requires_grad_(False) + return model, config diff --git a/aspec/models/megatron_draft.py b/aspec/models/megatron_draft.py new file mode 100644 index 0000000..970cccc --- /dev/null +++ b/aspec/models/megatron_draft.py @@ -0,0 +1,121 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Megatron TP projections around the retained optimized DSpark computation. + +The vocabulary objective and its bounded/tiled caches remain replicated in this +CP1 implementation. Only attention heads and dense FFN channels are partitioned. +This avoids gathering vocabulary logits or changing global probability semantics. +""" + +import torch +from torch.distributed.tensor import DTensor, Shard + +from aspec.models.megatron import require_megatron + + +def draft_transformer_config(config, tp_size): + require_megatron() + from megatron.core.transformer.transformer_config import TransformerConfig + + for name in ("num_attention_heads", "num_key_value_heads", "intermediate_size"): + if getattr(config, name) % tp_size: + raise ValueError(f"Draft {name} must be divisible by TP{tp_size}") + return TransformerConfig( + num_layers=config.num_hidden_layers, hidden_size=config.hidden_size, + ffn_hidden_size=config.intermediate_size, + num_attention_heads=config.num_attention_heads, + num_query_groups=config.num_key_value_heads, + kv_channels=int(getattr(config, "head_dim", None) + or config.hidden_size // config.num_attention_heads), normalization="RMSNorm", + layernorm_epsilon=config.rms_norm_eps, gated_linear_unit=True, + activation_func=torch.nn.functional.silu, add_bias_linear=False, + params_dtype=torch.bfloat16, bf16=True, + tensor_model_parallel_size=tp_size, context_parallel_size=1, + pipeline_model_parallel_size=1, sequence_parallel=False, + perform_initialization=False, use_cpu_initialization=False, + gradient_accumulation_fusion=False, attention_dropout=0.0, hidden_dropout=0.0, + ) + + +def checkpoint_tensor(tensor, parameter, tp_mesh): + """Expose the logical tensor to DCP without changing compute parameters.""" + axis = getattr(parameter, "_aspec_tp_axis", None) + if isinstance(tensor, DTensor) or axis is None or tensor.ndim == 0: + return tensor + return DTensor.from_local(tensor, device_mesh=tp_mesh, placements=(Shard(axis),), + run_check=False) + + +def partition_dspark(draft, core_config, groups, tp_mesh): + from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear + + class Column(ColumnParallelLinear): + def forward(self, value): + output, bias = super().forward(value) + if bias is not None: + raise RuntimeError("DSpark TP projections must be bias-free") + return output + + class Row(RowParallelLinear): + def forward(self, value): + output, bias = super().forward(value) + if bias is not None: + raise RuntimeError("DSpark TP projections must be bias-free") + return output + + rank, size = groups.tp.rank(), groups.tp.size() + + def replace(owner, name, axis): + original = getattr(owner, name) + if original.bias is not None: + raise ValueError("Megatron DSpark requires bias-free projections") + options = {"gather_output": False} if axis == 0 else {"input_is_parallel": True} + layer_type = Column if axis == 0 else Row + replacement = layer_type( + original.in_features, original.out_features, config=core_config, + init_method=core_config.init_method, bias=False, skip_bias_add=False, + tp_group=groups.tp, **options, + ) + with torch.no_grad(): + replacement.weight.copy_(original.weight.chunk(size, dim=axis)[rank]) + replacement.weight.requires_grad_(original.weight.requires_grad) + replacement.weight._aspec_tp_axis = axis + setattr(owner, name, replacement) + + for layer in draft.layers: + attention = layer.self_attn + for name in ("q_proj", "k_proj", "v_proj"): + replace(attention, name, 0) + replace(attention, "o_proj", 1) + attention.num_heads //= size + attention.num_kv_heads //= size + for name in ("gate_proj", "up_proj"): + replace(layer.mlp, name, 0) + replace(layer.mlp, "down_proj", 1) + draft.cuda() + for layer in draft.layers: + for norm in (layer.self_attn.q_norm, layer.self_attn.k_norm): + norm.weight._aspec_tp_partial_gradient = True + draft._aspec_tp_mesh = tp_mesh + draft._aspec_tp_group = groups.tp + return draft + + +def model_checkpoint_state(model, tp_mesh): + parameters = dict(model.named_parameters()) + return {name: checkpoint_tensor(value, parameters[name], tp_mesh) + if name in parameters else value + for name, value in model.state_dict().items() if isinstance(value, torch.Tensor)} + + +def load_model_checkpoint_state(model, state): + current = {name: value for name, value in model.state_dict().items() + if isinstance(value, torch.Tensor)} + if set(current) != set(state): + raise ValueError("Megatron checkpoint does not cover the complete DSpark state") + with torch.no_grad(): + for name, value in state.items(): + local = value.to_local() if isinstance(value, DTensor) else value + if current[name].shape != local.shape: + raise ValueError(f"Megatron checkpoint shape mismatch: {name}") + current[name].copy_(local) diff --git a/aspec/models/megatron_target.py b/aspec/models/megatron_target.py new file mode 100644 index 0000000..84b3803 --- /dev/null +++ b/aspec/models/megatron_target.py @@ -0,0 +1,156 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Frozen single-sequence Qwen3 prefill on the upstream Core HybridStack.""" + +from contextvars import ContextVar +from dataclasses import dataclass +from functools import partial + +import torch +from flashinfer.norm import fused_add_rmsnorm, rmsnorm +from flashinfer.rope import apply_rope_with_cos_sin_cache_inplace +from megatron.core.models.backends import get_backend +from megatron.core.models.hybrid.hybrid_block import HybridStack, HybridStackSubmodules +from megatron.core.models.hybrid.hybrid_model import HybridModel +from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.mlp import MLP, MLPSubmodules +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_layer import TransformerLayer, TransformerLayerSubmodules +from sgl_kernel import silu_and_mul +from sgl_kernel.flash_attn import flash_attn_varlen_func + + +class NormWeight(torch.nn.Module): + """Checkpoint gamma for a normalization executed by the prefill kernels.""" + + def __init__(self, *, config, hidden_size, eps): + super().__init__() + self.weight = torch.nn.Parameter(torch.empty(hidden_size, dtype=config.params_dtype)) + self.eps = eps + + +@dataclass +class _Prefill: + positions: torch.Tensor + bounds: torch.Tensor + layers: frozenset + capture: object + residual: torch.Tensor | None = None + rope_cache: torch.Tensor | None = None + + def normalize(self, branch, norm): + if self.residual is None: + self.residual = branch + return rmsnorm(branch, norm.weight, norm.eps) + fused_add_rmsnorm(branch, self.residual, norm.weight, norm.eps) + return branch + + +# Core passes a single activation between HybridStack positions. The fused norm +# also needs the pending residual. Scope that second stream to one model call. +_prefill = ContextVar("qwen3_target_prefill") + + +def _normalize_heads(value, norm): + # Warp reduction order for Qwen3 Q/K gamma, with FP32 lane accumulators. + x = value.float() + lanes = x.unflatten(-1, (32, -1)) + sums = lanes[..., 0].square() + for component in lanes.unbind(-1)[1:]: + sums = torch.addcmul(sums, component, component) + for stride in (16, 8, 4, 2, 1): + sums = sums[..., :stride] + sums[..., stride:] + inv_rms = torch.rsqrt(sums / x.shape[-1] + norm.eps) + return (x * inv_rms * norm.weight.float()).to(value.dtype) + + +class Qwen3AttentionLayer(TransformerLayer): + def forward(self, hidden_states, rotary_pos_emb, **kwargs): + state = _prefill.get() + attention = self.self_attention + width = self.config.kv_channels + length = hidden_states.size(0) + if state.rope_cache is None: + angles = rotary_pos_emb[:length, 0, 0, :width // 2].contiguous() + state.rope_cache = torch.cat((angles.cos(), angles.sin()), dim=-1) + normalized = state.normalize(hidden_states.squeeze(1), self.input_layernorm) + packed, _ = attention.linear_qkv(normalized.unsqueeze(1)) + queries = self.config.num_attention_heads // self.config.num_query_groups + packed = packed.reshape(length, -1, queries + 2, width) + q = _normalize_heads(packed[:, :, :queries].reshape(length, -1, width), attention.q_layernorm) + k = _normalize_heads(packed[:, :, queries], attention.k_layernorm) + v = packed[:, :, queries + 1].contiguous() + apply_rope_with_cos_sin_cache_inplace( + state.positions, q.flatten(1), k.flatten(1), width, state.rope_cache, True, + ) + context = flash_attn_varlen_func(q, k, v, state.bounds, state.bounds, length, length, causal=True) + branch, _ = attention.linear_proj(context.reshape(length, 1, -1)) + return branch, None + + +class Qwen3MLPLayer(TransformerLayer): + def forward(self, hidden_states, **kwargs): + state = _prefill.get() + normalized = state.normalize(hidden_states.squeeze(1), self.pre_mlp_layernorm) + gate_up, _ = self.mlp.linear_fc1(normalized.unsqueeze(1)) + branch, _ = self.mlp.linear_fc2(silu_and_mul(gate_up.squeeze(1)).unsqueeze(1)) + logical_layer = (self.layer_number - 1) // 2 + if state.capture is not None and logical_layer in state.layers: + state.capture(logical_layer, branch.squeeze(1), state.residual) + return branch, None + + +class Qwen3Target(HybridModel): + """ASpec feature endpoint. Core owns embeddings, layer order and TP projections.""" + + @torch.inference_mode() + def prefill(self, ids, layers, capture=None): + if ids.ndim != 2 or ids.size(0) != 1 or not 0 < ids.size(1) <= self.max_sequence_length: + raise ValueError("Qwen3 Target expects one nonempty sequence within its context") + if any(layer < 0 or layer >= self.config.num_layers // 2 for layer in layers): + raise ValueError("Qwen3 Target capture is outside the logical decoder") + positions = torch.arange(ids.size(1), device=ids.device) + bounds = torch.tensor([0, ids.size(1)], device=ids.device, dtype=torch.int32) + state = _Prefill(positions, bounds, frozenset(layers), capture) + token = _prefill.set(state) + try: + branch = self(input_ids=ids, position_ids=positions.unsqueeze(0), attention_mask=None) + return state.normalize(branch.squeeze(1), self.final_norm) + finally: + _prefill.reset(token) + + +def build_target(config, core_config, groups, implementation): + if core_config.kv_channels % 32: + raise ValueError("Qwen3 prefill requires heads divisible by a warp") + backend = get_backend(implementation) + attention = ModuleSpec( + module=SelfAttention, params={"attn_mask_type": AttnMaskType.causal}, + submodules=SelfAttentionSubmodules( + linear_qkv=backend.column_parallel_linear(), linear_proj=backend.row_parallel_linear(), + core_attention=backend.core_attention(), q_layernorm=NormWeight, k_layernorm=NormWeight, + ), + ) + mlp = partial(MLP.as_mlp_submodule, submodules=MLPSubmodules( + linear_fc1=backend.column_parallel_linear(), linear_fc2=backend.row_parallel_linear(), + )) + stack = ModuleSpec(module=HybridStack, submodules=HybridStackSubmodules( + attention_layer=ModuleSpec(module=Qwen3AttentionLayer, submodules=TransformerLayerSubmodules( + input_layernorm=NormWeight, self_attention=attention)), + mlp_layer=ModuleSpec(module=Qwen3MLPLayer, submodules=TransformerLayerSubmodules( + pre_mlp_layernorm=NormWeight, mlp=mlp)), + )) + rope = getattr(config, "rope_parameters", None) or {} + with torch.device("cuda"): + model = Qwen3Target( + config=core_config, hybrid_stack_spec=stack, + hybrid_layer_pattern="*-" * config.num_hidden_layers, + vocab_size=config.vocab_size, max_sequence_length=config.max_position_embeddings, + post_process=False, position_embedding_type="rope", + rotary_base=rope.get("rope_theta", getattr(config, "rope_theta", 1000000.0)), + pg_collection=groups, + ) + model.final_norm = NormWeight(config=core_config, hidden_size=config.hidden_size, + eps=core_config.layernorm_epsilon) + return model diff --git a/aspec/planning.py b/aspec/planning.py index 4b0d3d8..62c794f 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -47,12 +47,14 @@ resolve_draft_execution_capability, ) from aspec.config.draft_structure import resolve_draft_structure +from aspec.config.megatron import resolve_megatron_gradient_precision, validate_megatron_execution from aspec.config.mooncake_config import MooncakeConfig, resolve_rdma_routes from aspec.config.train_config import config_to_flat_args from aspec.feature_contract import FeatureContractError, build_target_feature_contract from aspec.inference.concurrency import direct_output_sample_capacity from aspec.inference.context import ( ContextContractError, + compile_megatron_context_contract, compile_offline_context_contract, compile_sglang_context_contract, ) @@ -817,7 +819,16 @@ def _resolve_batch_size(args: argparse.Namespace) -> None: ) args.attention_backend = execution.attention_backend args.draft_context_parallel_size = execution.context_parallel_size - if execution.direct_cp_backend: + if getattr(args, "train_backend", "fsdp") == "megatron": + tp_size = getattr(args, "draft_tensor_parallel_size", 1) + if type(tp_size) is not int or tp_size < 1 or world_size % tp_size: + raise PlanCompileError("Draft world size must be divisible by positive draft_tensor_parallel_size") + dp_size = world_size // tp_size + args.dp_size = dp_size + args.sp_size = 1 + args.per_dp_rank_batch_size = args.micro_batch_size + sp_size = 1 + elif execution.direct_cp_backend: cp_size = execution.context_parallel_size if world_size % cp_size != 0: raise PlanCompileError( @@ -912,10 +923,14 @@ def _validate_usp_args(args: argparse.Namespace) -> None: def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: - if getattr(args, "train_backend", None) != "fsdp": + if getattr(args, "train_backend", None) not in {"fsdp", "megatron"}: raise PlanCompileError( - f"The current training backend is fsdp; got {getattr(args, 'train_backend', None)!r}" + f"Training backend must be fsdp or megatron; got {getattr(args, 'train_backend', None)!r}" ) + try: + validate_megatron_execution(args, draft) + except ValueError as exc: + raise PlanCompileError(str(exc)) from exc cluster_mode = getattr(args, "cluster_mode", "single_node") if cluster_mode not in {"single_node", "static_multi_node"}: raise PlanCompileError(f"Unsupported cluster.mode {cluster_mode!r}") @@ -1018,7 +1033,10 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: raise PlanCompileError("static_multi_node requires cluster.shared_root") expected_tp = getattr(args, "sglang_nnodes", 1) * gpus_per_engine - if not offline and getattr(args, "sglang_tp_size", None) != expected_tp: + target_tp = (getattr(args, "megatron_tp_size", 1) + if getattr(args, "target_model_backend", "sglang") == "megatron" + else getattr(args, "sglang_tp_size", None)) + if not offline and target_tp != expected_tp: raise PlanCompileError( "inference.sglang.tp_size must equal sglang.nnodes * " f"inference_num_gpus_per_engine ({expected_tp})" @@ -1171,15 +1189,16 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: getattr(args, "target_prefill_memory_optimized", False) ) if direct_target_output: - if getattr(args, "inference_engine_type", "sgl") != "sgl": + if getattr(args, "inference_engine_type", "sgl") not in {"sgl", "megatron"}: raise PlanCompileError( - "target_prefill_memory_optimized currently requires SGLang" + "target_prefill_memory_optimized requires a live Target backend" ) if getattr(args, "train_with_decode", False): raise PlanCompileError( "target_prefill_memory_optimized requires prefill-only Target execution" ) - if gpus_per_engine != 1 or getattr(args, "sglang_tp_size", 1) != 1: + if (getattr(args, "inference_engine_type", "sgl") == "sgl" + and (gpus_per_engine != 1 or getattr(args, "sglang_tp_size", 1) != 1)): raise PlanCompileError( "target_prefill_memory_optimized currently requires Target TP1" ) @@ -1241,6 +1260,7 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: dispatch_batch_size = ( int(getattr(args, "training_num_gpus_per_node", 0)) // int(getattr(args, "draft_context_parallel_size", 1)) + // int(getattr(args, "draft_tensor_parallel_size", 1)) * args.per_dp_rank_batch_size ) capacity_scope = "per node" @@ -1534,6 +1554,7 @@ def _workload( target_num_hidden_layers=draft["target_num_hidden_layers"], aux_hidden_state_layers=list(draft["target_layer_ids"]), store_last_hidden_states=store_last, + producer_backend=_select(config, "model.target_model_backend", "sglang"), **({"mtp": {"native_config": draft["mtp_conditioning"]["native_config"], "embedding_key": _select(config, "model.embedding_key")}} if draft.get("mtp_conditioning") and draft["mtp_conditioning"]["mode"] == "hidden" @@ -1660,6 +1681,22 @@ def _policy( _select(config, "inference.inference_batch_size"), ), ) + megatron_input_memory = None + if runtime.get("train_backend") == "megatron": + # Receive/collate copies on the consumer thread. Leader source prefetch + # borrows registered destination slots, counted separately as rings. + resident_batches = 2 + feature_width = int(_select(config, "mooncake.hidden_dim", 4096)) + feature_bytes_per_token = 2 * feature_width * ( + len(runtime["aux_hidden_states_layers"]) + int(runtime.get("store_last_hidden_states", False))) + # Token IDs plus masks/positions alongside the captured feature ABI. + bytes_per_batch = (feature_bytes_per_token + 40) * int(runtime["max_seq_length"]) * int(runtime["micro_batch_size"]) + megatron_input_memory = { + "storage": "consumer_cuda_payloads_replicated_over_tp", + "resident_batch_bound_per_rank": resident_batches, + "payload_bytes_bound_per_rank": resident_batches * bytes_per_batch, + "excludes": "registered_rings_model_activations_and_projection_caches", + } cluster_nodes: list[dict[str, Any]] = [] locality_domains: list[dict[str, Any]] = [] ready_watermark_by_domain: dict[str, int] = {} @@ -1776,7 +1813,8 @@ def _policy( return ExecutionPolicy( backends={ - "target_features": "offline_replay" if runtime["inference_engine_type"] == "offline" else "sglang", + "target_features": ("offline_replay" if runtime["inference_engine_type"] == "offline" + else _select(config, "model.target_model_backend")), "training": _select(config, "training.train_backend"), "transport": "mooncake", }, @@ -1805,7 +1843,7 @@ def _policy( "world_size": runtime["world_size"], }, "inference": { - "gpu_owner": "target_service" if runtime["inference_engine_type"] == "sgl" else None, + "gpu_owner": "target_service" if runtime["inference_engine_type"] != "offline" else None, "nodes": ( len(_select(config, "inference.inference_node_ips") or ()) if cluster_mode == "static_multi_node" @@ -1826,15 +1864,22 @@ def _policy( parallelism={ "training": { "data_parallel": runtime["dp_size"], + "tensor_parallel": _select(config, "training.draft_tensor_parallel_size", 1), "sequence_parallel": runtime["sp_size"], "context_parallel": _select( config, "training.draft_context_parallel_size", 1 ), "attention_backend": _select(config, "training.attention_backend"), - "fsdp_strategy": _select(config, "training.fsdp_strategy"), + "fsdp_strategy": (None if runtime["train_backend"] == "megatron" + else _select(config, "training.fsdp_strategy")), + **({**resolve_megatron_gradient_precision(argparse.Namespace(**runtime)), + "vocabulary": "replicated"} + if runtime["train_backend"] == "megatron" else {}), }, "inference": { - "tensor_parallel": _select(config, "inference.sglang.tp_size"), + "tensor_parallel": (_select(config, "inference.megatron.tp_size") + if runtime["inference_engine_type"] == "megatron" + else _select(config, "inference.sglang.tp_size")), "pipeline_parallel": _select(config, "inference.sglang.pp_size"), "source_context_parallel_size": runtime["source_context_parallel_size"], }, @@ -1878,15 +1923,18 @@ def _policy( }, capacity={ "device_memory": { + **({"megatron_input_staging": megatron_input_memory} if megatron_input_memory else {}), **({"mtp_conditioning": conditioning_memory} if conditioning_memory else {}), **({"target_mtp_features": target_mtp_memory} if target_mtp_memory else {}), - "sglang_static_fraction": (None if runtime["inference_engine_type"] == "offline" + "sglang_static_fraction": (None if runtime["inference_engine_type"] != "sgl" else _select(config, "inference.sglang.mem_fraction_static")) }, "backend_tokens": { **copy.deepcopy(dict(context_contract)), "profiled_max_total_tokens": None, - "state": "not_applicable_offline" if runtime["inference_engine_type"] == "offline" else "pending_backend_probe", + "state": ("not_applicable_offline" if runtime["inference_engine_type"] == "offline" + else "no_persistent_kv_pool" if runtime["inference_engine_type"] == "megatron" + else "pending_backend_probe"), }, "transfer_storage": { "data_plane": data_plane, @@ -2018,9 +2066,18 @@ def compile_execution_plan( source["resolved_config_sha256"] = _resolved_config_sha256(config) draft = _load_draft_descriptor(config) target_family = _resolve_target_family(target_path) + if (_select(config, "training.train_backend") == "megatron" + or _select(config, "model.target_model_backend") == "megatron"): + target_raw = _read_local_target_config(target_path) + if target_family != "qwen3" or target_raw and target_raw.get("model_type") != "qwen3": + raise PlanCompileError("Megatron integration currently requires dense Qwen3 Target weights") runtime = _runtime_values(config, draft, source_context_parallel_size=source_context_parallel_size) try: - context_compiler = compile_offline_context_contract if offline else compile_sglang_context_contract + context_compiler = ( + compile_offline_context_contract if offline else + compile_megatron_context_contract if runtime["inference_engine_type"] == "megatron" + else compile_sglang_context_contract + ) context_contract = context_compiler( max_sequence_length=_select(config, "training.max_seq_length"), context_parallel_size=int( @@ -2108,7 +2165,8 @@ def compile_execution_plan( name="target_feature_engine", requirement="required", requested=_select(config, "inference.inference_engine_type"), - resolved="offline_replay" if offline else "sglang", + resolved=("offline_replay" if offline else + _select(config, "model.target_model_backend", "sglang")), ), CapabilityDecision( name="draft_trainer", @@ -2120,7 +2178,7 @@ def compile_execution_plan( name="training_backend", requirement="required", requested=_select(config, "training.train_backend"), - resolved="fsdp", + resolved=_select(config, "training.train_backend", "fsdp"), ), CapabilityDecision( name="static_multi_node_execution", diff --git a/aspec/ray/placement_group.py b/aspec/ray/placement_group.py index c945e3f..07624e0 100644 --- a/aspec/ray/placement_group.py +++ b/aspec/ray/placement_group.py @@ -559,7 +559,7 @@ def create_placement_groups(args, roles: set[str] | None = None): _ensure_ray_initialized() - if getattr(args, "inference_engine_type", None) == "sgl": + if getattr(args, "inference_engine_type", None) in {"sgl", "megatron"}: managed_local = (getattr(args, "target_service", None) or {}).get("mode") == "managed-local" if managed_local: live_nodes = [node for node in ray.nodes() if node.get("Alive")] diff --git a/aspec/ray/train_group.py b/aspec/ray/train_group.py index 1699f43..2faca83 100644 --- a/aspec/ray/train_group.py +++ b/aspec/ray/train_group.py @@ -298,6 +298,11 @@ def set_eval_queues( def _expand_queues_for_usp(self, queues): actor_count = len(self._actor_handlers) + if getattr(self.args, "train_backend", "fsdp") == "megatron": + tp = self.args.draft_tensor_parallel_size + if actor_count % tp or len(queues) != actor_count // tp: + raise ValueError("Megatron needs one input queue per data-parallel replica") + return [queues[rank // tp] for rank in range(actor_count)] if len(queues) == actor_count: return queues diff --git a/aspec/runtime_identity.py b/aspec/runtime_identity.py index 6c8461a..d9014c4 100644 --- a/aspec/runtime_identity.py +++ b/aspec/runtime_identity.py @@ -24,6 +24,18 @@ # Patched files and their native v0.5.21 request/capture/runner dependencies. # A bounded set catches divergent checkouts without hashing the entire repo. +MEGATRON_RUNTIME_RELATIVE_PATHS = ( + "package_info.py", + "models/hybrid/hybrid_model.py", + "models/hybrid/hybrid_block.py", + "models/hybrid/hybrid_layer_allocation.py", + "tensor_parallel/layers.py", + "tensor_parallel/mappings.py", + "distributed/distributed_data_parallel.py", + "transformer/spec_utils.py", + "transformer/transformer_block.py", +) + SGLANG_RUNTIME_RELATIVE_PATHS = ( "kernels/ops/attention/triton_gdn_fused_proj.py", "kernels/ops/attention/fla/chunk_delta_h.py", diff --git a/aspec/train_entry.py b/aspec/train_entry.py index 0a3602d..1c62922 100644 --- a/aspec/train_entry.py +++ b/aspec/train_entry.py @@ -302,9 +302,9 @@ def _validate_and_configure_dflash(args, draft_model_config) -> None: algo = "DSpark" if is_dspark else "DFlash" engine_type = getattr(args, "inference_engine_type", "sgl") - if engine_type not in {"sgl", "offline"}: + if engine_type not in {"sgl", "megatron", "offline"}: raise NotImplementedError( - f"{algo} currently supports inference_engine_type='sgl' or 'offline', got '{engine_type}'." + f"{algo} requires inference_engine_type=sgl, megatron or offline; got {engine_type!r}" ) if getattr(args, "defer_tokenization", False): raise NotImplementedError(f"{algo} does not support defer_tokenization=True.") @@ -458,6 +458,15 @@ def train_async_no_generation(args, evidence: RunEvidenceRecorder | None = None) inference_engine_statuses = ray.get( [engine.get_status.remote() for engine in inference_engines] ) + if args.inference_engine_type == "megatron": + # The Draft interpreter may not have any Target dependencies installed. + runtime = inference_engine_statuses[0]["runtime"] + args.megatron_target_cache_identity = { + "adapter": runtime["code_sha256"], + "core": runtime["backend_identity"]["effective_tree_sha256"], + "dependencies": runtime["dependencies"], + "torch": runtime["torch"], "cuda": runtime["cuda"], + } if evidence is not None: evidence.record_actor_initialization( training_actor_count=n_train, @@ -489,7 +498,7 @@ def train_async_no_generation(args, evidence: RunEvidenceRecorder | None = None) timer.log_summary() except BaseException: - if args.inference_engine_type == "sgl": + if args.inference_engine_type in {"sgl", "megatron"}: rollback_unused_targets(inference_engines) raise diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 3738fab..1e0eb04 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -51,6 +51,8 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: "training_num_gpus_per_node", "attention_backend", "draft_context_parallel_size", + "draft_tensor_parallel_size", + "train_backend", "sp_ulysses_size", "sp_ring_size", "fsdp_strategy", @@ -59,6 +61,8 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: value = getattr(args, field, None) if value is not None: topology[field] = value + if getattr(args, "train_backend", "fsdp") == "megatron": + topology["megatron_grad_dtype"] = getattr(args, "megatron_grad_dtype", "fp32") if getattr(args, "draft_algorithm", None) == "mtp": topology["training_objective"] = { "steps": args.mtp_steps, "step_weight_beta": args.mtp_step_weight_beta, @@ -159,6 +163,15 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec if saved_topology is not None and not isinstance(saved_topology, dict): raise RuntimeError("Exact resume checkpoint has invalid training_topology metadata") current_topology = _logical_training_topology(args, current_world_size) + if ((saved_topology or {}).get("train_backend", "fsdp") + != current_topology.get("train_backend", "fsdp") + or (saved_topology or {}).get("draft_tensor_parallel_size", 1) + != current_topology.get("draft_tensor_parallel_size", 1)): + raise RuntimeError("Exact resume cannot change the training backend or Draft TP; use weights-only initialization") + if (current_topology.get("train_backend") == "megatron" + and (saved_topology or {}).get("megatron_grad_dtype", "fp32") + != current_topology["megatron_grad_dtype"]): + raise RuntimeError("Exact resume cannot change Megatron gradient precision; use weights-only initialization") if getattr(args, "draft_algorithm", None) == "dspark": saved_objective = (saved_topology or {}).get("training_objective") or {} if saved_objective.get("lm_head_precision", "bf16") != getattr(args, "dspark_lm_head_precision", "bf16"): @@ -321,7 +334,8 @@ def _rng_state_path(checkpoint_dir: Path, rank: int) -> Path: return checkpoint_dir / "rng" / f"rank-{int(rank):05d}.pt" -def _save_rank_rng_state(checkpoint_dir: Path, *, rank: int, world_size: int) -> None: +def _save_rank_rng_state(checkpoint_dir: Path, *, rank: int, world_size: int, + backend_rng_state=None) -> None: path = _rng_state_path(checkpoint_dir, rank) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") state = { @@ -330,6 +344,7 @@ def _save_rank_rng_state(checkpoint_dir: Path, *, rank: int, world_size: int) -> "world_size": int(world_size), "torch": torch.get_rng_state(), "cuda": torch.cuda.get_rng_state() if torch.cuda.is_available() else None, + **({"backend_rng": backend_rng_state} if backend_rng_state is not None else {}), } try: torch.save(state, temporary) @@ -436,10 +451,15 @@ def __init__(self, model): self.model = model def state_dict(self): + if callable(getattr(self.model, "_aspec_checkpoint_state", None)): + return {"model": self.model._aspec_checkpoint_state()} model_state_dict, _ = get_state_dict(self.model, optimizers=[]) return {"model": model_state_dict} def load_state_dict(self, state_dict): + if callable(getattr(self.model, "_aspec_load_checkpoint_state", None)): + self.model._aspec_load_checkpoint_state(state_dict["model"]) + return set_state_dict( self.model, optimizers=[], model_state_dict=state_dict["model"], optim_state_dict=None ) @@ -484,7 +504,8 @@ def _optimizer_load_template(self): master = self.optimizer.fp32_params[index] if len(parts) != 2 or field not in {"step", "exp_avg", "exp_avg_sq"}: raise ValueError(f"Unsupported BF16 Adam checkpoint state: {key}") - expected_shape = () if field == "step" else tuple(master.shape) + logical_master = self._checkpoint_tensor(index, master) + expected_shape = () if field == "step" else tuple(logical_master.shape) dtype = value.properties.dtype valid_dtype = ( dtype in {torch.float32, torch.float64} @@ -501,11 +522,24 @@ def _optimizer_load_template(self): self._load_template = {**template, "state": states} return self._load_template + def _checkpoint_tensor(self, index, tensor): + wrap = getattr(self.optimizer, "checkpoint_tensor", None) + return wrap(index, tensor) if wrap is not None else tensor + + def _checkpoint_optimizer_state(self, state): + if not hasattr(self.optimizer, "checkpoint_tensor"): + return state + return {**state, "state": { + index: {key: self._checkpoint_tensor(int(index), value) + if isinstance(value, torch.Tensor) else value for key, value in values.items()} + for index, values in state["state"].items() + }} + def state_dict(self): if self._is_bf16_optimizer: names = _fp32_param_names(self.optimizer) return { - "optim": ( + "optim": self._checkpoint_optimizer_state( self._optimizer_load_template() if self._load_path is not None else self.optimizer.state_dict() @@ -513,8 +547,8 @@ def state_dict(self): # Keyed by name: which parameters are trainable can differ between the run # that saved and the run that loads. "fp32_params": { - name: param.data - for name, param in zip(names, self.optimizer.fp32_params, strict=True) + name: self._checkpoint_tensor(index, param.data) + for index, (name, param) in enumerate(zip(names, self.optimizer.fp32_params, strict=True)) }, } _, optimizer_state_dict = get_state_dict(self.model, optimizers=[self.optimizer]) @@ -522,7 +556,16 @@ def state_dict(self): def load_state_dict(self, state_dict): if self._is_bf16_optimizer: - self.optimizer.load_state_dict(state_dict["optim"]) + from torch.distributed.tensor import DTensor + + optim = state_dict["optim"] + if hasattr(self.optimizer, "checkpoint_tensor"): + optim = {**optim, "state": { + index: {key: value.to_local() if isinstance(value, DTensor) else value + for key, value in values.items()} + for index, values in optim["state"].items() + }} + self.optimizer.load_state_dict(optim) saved = state_dict.get("fp32_params") if saved: names = _fp32_param_names(self.optimizer) @@ -537,7 +580,10 @@ def load_state_dict(self, state_dict): ) with torch.no_grad(): for name, master in zip(names, self.optimizer.fp32_params, strict=True): - master.data.copy_(saved[name]) + value = saved[name] + if isinstance(value, DTensor) and not isinstance(master, DTensor): + value = value.to_local() + master.copy_(value) logger.info("Restored fp32 master params from checkpoint") return set_state_dict( @@ -1075,6 +1121,9 @@ def finalize_load(actor: Any, checkpoint_payload: dict[str, Any] | None) -> None rng_state, legacy=isinstance(rng_state, dict) and "schema_version" not in rng_state, ) + restore_backend_rng = getattr(actor, "_restore_backend_rng", None) + if restore_backend_rng is not None: + restore_backend_rng(rng_state.get("backend_rng")) torch.set_rng_state(rng_state["torch"]) if torch.cuda.is_available() and rng_state.get("cuda") is not None: cuda_state = rng_state["cuda"] @@ -1144,6 +1193,8 @@ def save(actor: Any, step: int) -> None: checkpoint_dir, rank=dist.get_rank(), world_size=dist.get_world_size(), + backend_rng_state=(actor._checkpoint_backend_rng() + if hasattr(actor, "_checkpoint_backend_rng") else None), ) if dist.get_rank() == 0: diff --git a/aspec/training/data_fetcher.py b/aspec/training/data_fetcher.py index d807116..2f535bf 100644 --- a/aspec/training/data_fetcher.py +++ b/aspec/training/data_fetcher.py @@ -139,8 +139,18 @@ def __init__( packing_enabled: bool = False, feature_affinity: str = "any", consumer_node_ip: str | None = None, + tensor_parallel_group=None, + tensor_parallel_control_group=None, ): self.ray_queue = ray_queue + if (tensor_parallel_group is None) != (tensor_parallel_control_group is None): + raise ValueError("TP input requires paired GPU-data and CPU-control groups") + self.tensor_parallel_group = tensor_parallel_group + self.tensor_parallel_control_group = tensor_parallel_control_group + self._tensor_parallel_device = (torch.cuda.current_device() + if tensor_parallel_group is not None else None) + self._tp_source_prefetch = None + self._tensor_parallel_samples = self._tensor_parallel_payload_bytes = 0 self.mooncake_store = mooncake_store self.device = device self.prefetch_factor = prefetch_factor @@ -186,11 +196,21 @@ def __init__( ): raise ValueError("packing requires non-USP training and positive max_seq_length") self._init_sp_context() + if tensor_parallel_group is not None and prefetch_factor > 0 and dist.get_rank(tensor_parallel_group) == 0: + from aspec.training.tp_source_prefetch import TPSourcePrefetch + + self._tp_source_prefetch = TPSourcePrefetch( + self._read_tp_source, lambda item: _release_p2p_receive_leases(item[1]), depth=prefetch_factor) def request_stop(self) -> None: """Cancel future queue reads without interrupting a native transfer.""" self._stop_requested.set() + def close(self): + self.request_stop() + if self._tp_source_prefetch is not None: + self._tp_source_prefetch.close() + def _next_queue_item(self): # Poll only the queue wait, not Mooncake itself: close must be able to # join an idle prefetch worker before its registered buffers are freed. @@ -540,8 +560,14 @@ def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: logger.debug(f"__iter__: waiting for item from ray_queue (yield_count={yield_count})") try: - item = self._next_queue_item() + if self.tensor_parallel_group is not None: + item, data = self._tensor_parallel_item() + else: + item = self._next_queue_item() + data = None except Exception as e: + if self.tensor_parallel_group is not None: + raise logger.warning(f"__iter__: Exception waiting for data: {e}, timeout={self.timeout}") break @@ -550,7 +576,8 @@ def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: break logger.debug(f"__iter__: got item, mooncake_key={item.mooncake_key}") - data = self._load_from_mooncake(item) + if self.tensor_parallel_group is None: + data = self._load_from_mooncake(item) neutralized_count = self._resolve_and_neutralize_loss_mask( data, item.mooncake_key, neutralized_count @@ -584,6 +611,86 @@ def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: logger.debug(f"__iter__: yielding batch {yield_count}, keys={list(data.keys())}") yield data + def _tensor_parallel_item(self): + with torch.cuda.device(self._tensor_parallel_device): + return self._tensor_parallel_item_on_device() + + def _read_tp_source(self): + # No collation, GPU clones or collectives run on this producer thread. + # P2P GET writes preallocated destination slots; leases bound prefetch. + with torch.cuda.device(self._tensor_parallel_device): + item = self._next_queue_item() + return item, self._load_from_mooncake(item) if item is not None else {} + + def _tensor_parallel_item_on_device(self): + """Read a source lease once, then broadcast its full CP1 payload over TP. + + Source release and work-ledger acknowledgement belong to the TP leader. + Source GET may be prefetched by the leader. Every TP broadcast executes + on the consumer thread between microbatches, in model collective order. + Its destination lease remains attached through the consumer microbatch. + """ + group = self.tensor_parallel_group + leader = dist.get_global_rank(group, 0) + is_leader = dist.get_rank(group) == 0 + control, data, leases = [None], None, [] + if is_leader: + try: + if self._tp_source_prefetch is not None: + item, data = self._tp_source_prefetch.next() + else: + item, data = self._read_tp_source() + leases = _take_p2p_receive_leases(data) + control[0] = { + "item": item, + "values": {k: v for k, v in data.items() if not isinstance(v, torch.Tensor)}, + "tensors": [(k, tuple(v.shape), v.dtype, v.device.type) + for k, v in data.items() if isinstance(v, torch.Tensor)], + } + except BaseException as error: + control[0] = {"error": f"{type(error).__name__}: {error}"} + try: + # Waiting for the next queue item must never launch a waiting GPU + # collective on followers: CUDA allocation/synchronization in the + # current model step could otherwise wait for undispatched input. + dist.broadcast_object_list(control, src=leader, group=self.tensor_parallel_control_group) + message = control[0] + if "error" in message: + raise RuntimeError("TP input leader failed: " + message["error"]) + item = message["item"] + if item is None: + return None, None + if not is_leader: + data = dict(message["values"]) + self._feature_contract_runtime.validate_metadata( + tensor_shapes=item.tensor_shapes, tensor_dtypes=item.tensor_dtypes, + metadata=item.metadata, + ) + for key, shape, dtype, original_device in message["tensors"]: + buffer = (data[key].to(device="cuda").contiguous() if is_leader + else torch.empty(shape, device="cuda", dtype=dtype)) + dist.broadcast(buffer, src=leader, group=group) + if not is_leader: + data[key] = buffer.cpu() if original_device == "cpu" else buffer + if not is_leader: + names = self._feature_contract_runtime.contract["tensors"] + self._validate_received_feature_tensors( + {name: data[name] for name in names}, expected_shapes=item.tensor_shapes, + expected_dtypes={name: data[name].dtype for name in names}, + ) + if leases: + data["_mooncake_p2p_receive_leases"] = leases + leases = [] + self._tensor_parallel_samples += 1 + self._tensor_parallel_payload_bytes += sum( + data[key].numel() * data[key].element_size() + for key, _shape, _dtype, _device in message["tensors"] + ) + return item, data + finally: + for lease in leases: + lease.release() + def _direct_cp_get_sharded_item( self, neutralized_count: int ) -> tuple[Dict[str, torch.Tensor] | None, int]: @@ -957,6 +1064,14 @@ def get_feature_contract_status(self) -> dict[str, Any]: status["data_admission"] = { "neutralized_rank_samples": self._neutralized_rank_samples, } + if getattr(self, "tensor_parallel_group", None) is not None: + status["tensor_parallel_input"] = { + "rank": dist.get_rank(self.tensor_parallel_group), + "size": dist.get_world_size(self.tensor_parallel_group), + "broadcast_samples": self._tensor_parallel_samples, + "broadcast_payload_bytes": self._tensor_parallel_payload_bytes, + "source_read_owner": dist.get_rank(self.tensor_parallel_group) == 0, + } get_transport_status = getattr(self.mooncake_store, "get_p2p_status", None) transport = ( get_transport_status() if callable(get_transport_status) else None @@ -1011,6 +1126,8 @@ def create_mooncake_dataloader( packing_enabled: bool = False, feature_affinity: str = "any", consumer_node_ip: str | None = None, + tensor_parallel_group=None, + tensor_parallel_control_group=None, ) -> DataLoader: """Create a DataLoader that fetches from mooncake via queue. @@ -1028,7 +1145,7 @@ def create_mooncake_dataloader( collator: Collator for padding and batching samples. device: Target device for tensors. batch_size: Number of samples per batch (= per_dp_rank_batch_size). - prefetch_factor: Unused, kept for API compatibility. + prefetch_factor: Source GET queue depth for TP input; unused by other paths. timeout: Timeout in seconds for waiting on queue. None means wait forever. assistant_header_ids: Token IDs for assistant header (for loss mask skip check). end_token_ids: Token IDs for end of turn (for loss mask skip check). @@ -1061,6 +1178,8 @@ def create_mooncake_dataloader( packing_enabled=packing_enabled, feature_affinity=feature_affinity, consumer_node_ip=consumer_node_ip, + tensor_parallel_group=tensor_parallel_group, + tensor_parallel_control_group=tensor_parallel_control_group, ) def dataloader_collator(samples): @@ -1128,6 +1247,8 @@ def __init__( packing_enabled: bool = False, feature_affinity: str = "any", consumer_node_ip: str | None = None, + tensor_parallel_group=None, + tensor_parallel_control_group=None, ): self.batch_size = batch_size self._dataloader = create_mooncake_dataloader( @@ -1154,6 +1275,8 @@ def __init__( packing_enabled=packing_enabled, feature_affinity=feature_affinity, consumer_node_ip=consumer_node_ip, + tensor_parallel_group=tensor_parallel_group, + tensor_parallel_control_group=tensor_parallel_control_group, ) def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: @@ -1165,6 +1288,9 @@ def get_feature_contract_status(self) -> dict[str, Any]: def request_stop(self) -> None: self._dataloader.dataset.request_stop() + def close(self): + self._dataloader.dataset.close() + class PrefetchedDataFetcher: """Wraps MooncakeDataFetcher with async pre-fetching. diff --git a/aspec/training/dflash_trainer.py b/aspec/training/dflash_trainer.py index fbd5a28..344318c 100644 --- a/aspec/training/dflash_trainer.py +++ b/aspec/training/dflash_trainer.py @@ -94,6 +94,22 @@ def __init__(self, args: Namespace): self.verifier_norm: Optional[torch.nn.Module] = None self._mtp_binding_validated = False + _optimizer_class = BF16Optimizer + + def _loads_initial_weights_here(self): + return dist.get_rank() == 0 + + def _wrap_training_model(self, model, draft_model): + full_state = model.state_dict() if dist.get_rank() == 0 else {} + model = apply_fsdp2( + model, mesh=self.dp_mesh, cpu_offload=self.fsdp_cpu_offload, + args=self.args, modules_to_shard=list(draft_model.layers), + ) + return fsdp2_load_full_state_dict( + model, full_state, self.dp_mesh, + cpu_offload=True if self.fsdp_cpu_offload else None, + ) + def init_model( self, draft_model_config, @@ -145,7 +161,7 @@ def init_model( draft_model = self._build_draft_model(config) conditioning = getattr(config, "mtp_conditioning", None) - if dist.get_rank() == 0: + if self._loads_initial_weights_here(): if conditioning: draft_model.initialize_mtp_conditioning(target_model_path) initial_path = getattr(self.args, "initial_draft_model_path", None) @@ -175,22 +191,7 @@ def init_model( dflash_model = self._build_training_wrapper(draft_model) - full_state = dflash_model.state_dict() if dist.get_rank() == 0 else {} - - dflash_model = apply_fsdp2( - dflash_model, - mesh=self.dp_mesh, - cpu_offload=self.fsdp_cpu_offload, - args=self.args, - modules_to_shard=list(draft_model.layers), - ) - - dflash_model = fsdp2_load_full_state_dict( - dflash_model, - full_state, - self.dp_mesh, - cpu_offload=True if self.fsdp_cpu_offload else None, - ) + dflash_model = self._wrap_training_model(dflash_model, draft_model) if getattr(self.args, "compile_model", False): logger.info("Compiling DFlash model with torch.compile (inductor backend)") @@ -215,7 +216,7 @@ def init_model( decay_style = getattr(self.args, "lr_decay_style", "cosine") warmup_ratio = getattr(self.args, "warmup_ratio", 0.1) - self.optimizer = BF16Optimizer( + self.optimizer = self._optimizer_class( self.draft_model, lr=self.args.learning_rate, weight_decay=getattr(self.args, "weight_decay", 0.0), @@ -274,7 +275,7 @@ def init_model( self.prof.on_init_end() - logger.info(f"[Rank {self.dp_rank}] DFlash model initialized with FSDP2") + logger.info("[DP rank %s] Draft initialized with %s", self.dp_rank, self.execution_backend) return 0 diff --git a/aspec/training/megatron_optimizer.py b/aspec/training/megatron_optimizer.py new file mode 100644 index 0000000..1117b0f --- /dev/null +++ b/aspec/training/megatron_optimizer.py @@ -0,0 +1,47 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Retain ASpec's BF16 master-AdamW semantics over Megatron TP/DDP buffers.""" + +import torch +import torch.distributed as dist + +from aspec.models.megatron_draft import checkpoint_tensor +from aspec.training.optimizer import BF16Optimizer + + +class MegatronBF16Optimizer(BF16Optimizer): + def _model_gradient(self, parameter): + return parameter.main_grad + + def checkpoint_tensor(self, index, value): + return checkpoint_tensor(value, self.model_params[index], self.model._aspec_tp_mesh) + + def _clip_gradients(self): + group = self.model._aspec_tp_group + # Replicated vocabulary/confidence/norm parameters contribute once; + # every distinct attention/FFN shard contributes to the global norm. + gradients = [master.grad for parameter, master in zip( + self.model_params, self.fp32_params, strict=True) + if master.grad is not None + and (getattr(parameter, "_aspec_tp_axis", None) is not None or group.rank() == 0)] + norm = torch.nn.utils.get_total_norm(gradients).square() + dist.all_reduce(norm, op=dist.ReduceOp.SUM, group=group) + norm.sqrt_() + coefficient = (self.max_grad_norm / (norm + 1e-6)).clamp(max=1.0) + if self._fuse_clip_scaling: + self.optimizer.grad_scale = coefficient.reciprocal() + else: + from aspec.models.ops.gradient_scaling import scale_gradients_ + + self.optimizer.grad_scale = None + scale_gradients_([p.grad for p in self.fp32_params if p.grad is not None], coefficient) + return norm + + def step(self, closure=None): + norm = super().step(closure) + self.model._aspec_ddp.zero_grad_buffer() + return norm + + def zero_grad(self, set_to_none=True): + super().zero_grad(set_to_none=set_to_none) + self.model._aspec_ddp.zero_grad_buffer() diff --git a/aspec/training/megatron_trainer.py b/aspec/training/megatron_trainer.py new file mode 100644 index 0000000..209253b --- /dev/null +++ b/aspec/training/megatron_trainer.py @@ -0,0 +1,206 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""DSpark's retained training loop with upstream Megatron TP and DDP. + +Inspired by the public Megatron Core 0.20 custom-model DDP interface. DSpark +algorithm, batching, optimizer-step normalization and recovery remain ASpec-owned. +""" + +import weakref +from pathlib import Path + +import torch +import torch.distributed as dist +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.tensor import DTensor + +from aspec.config.megatron import resolve_megatron_gradient_precision +from aspec.models.megatron import initialize_parallelism +from aspec.models.megatron_draft import ( + draft_transformer_config, + load_model_checkpoint_state, + model_checkpoint_state, + partition_dspark, +) +from aspec.training.dspark_trainer import DSparkTrainer +from aspec.training.megatron_optimizer import MegatronBF16Optimizer + + +class MegatronDSparkTrainer(DSparkTrainer): + execution_backend = "megatron" + _optimizer_class = MegatronBF16Optimizer + + def _setup_device_mesh(self): + self.groups = initialize_parallelism(self.args.draft_tensor_parallel_size, + seed=getattr(self.args, "seed", 42)) + self.tp_size, self.tp_rank = self.groups.tp.size(), self.groups.tp.rank() + self.dp_size, self.dp_rank = self.groups.dp.size(), self.groups.dp.rank() + self.dp_group = self.groups.dp + self.cp_size, self.cp_rank, self.cp_group = 1, 0, None + self.cache_rank = dist.get_rank() + self.tp_mesh = DeviceMesh.from_group(self.groups.tp, "cuda") + self.dp_mesh = DeviceMesh.from_group(self.groups.dp, "cuda") + self.mesh = self.grad_sync_mesh = self.dp_mesh + # Only source GET is prefetched. Input collectives run on the model's + # consumer thread, in a consistent order across overlapping TP/DP groups. + for replica in range(self.dp_size): + ranks = list(range(replica * self.tp_size, (replica + 1) * self.tp_size)) + group = dist.new_group(ranks, backend="nccl") + control_group = dist.new_group(ranks, backend="gloo") + eval_group = dist.new_group(ranks, backend="nccl") + eval_control_group = dist.new_group(ranks, backend="gloo") + if replica == self.dp_rank: + self.input_tp_group = group + self.input_tp_control_group = control_group + self.eval_input_tp_group = eval_group + self.eval_input_tp_control_group = eval_control_group + + def _get_init_weight_context_manager(self): + return lambda: torch.device("cpu") + + def _loads_initial_weights_here(self): + return True + + def _build_training_wrapper(self, draft_model): + wrapper = super()._build_training_wrapper(draft_model) + sample = wrapper._sample_anchor_positions + wrapper._aspec_anchor_tp_broadcasts = 0 + + def shared_anchors(*args, **kwargs): + anchors, keep = sample(*args, **kwargs) + if self.tp_size > 1: + leader = dist.get_global_rank(self.groups.tp, 0) + dist.broadcast(anchors, src=leader, group=self.groups.tp) + dist.broadcast(keep, src=leader, group=self.groups.tp) + wrapper._aspec_anchor_tp_broadcasts += 1 + return anchors, keep + + wrapper._sample_anchor_positions = shared_anchors + return wrapper + + def _wrap_training_model(self, model, draft_model): + from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig + from megatron.core.tensor_parallel import ( + set_defaults_if_not_set_tensor_model_parallel_attributes, + ) + + config = draft_transformer_config(draft_model.config, self.tp_size) + partition_dspark(draft_model, config, self.groups, self.tp_mesh) + for parameter in model.parameters(): + set_defaults_if_not_set_tensor_model_parallel_attributes(parameter) + precision = resolve_megatron_gradient_precision(self.args) + ddp = DistributedDataParallel( + config, DistributedDataParallelConfig( + grad_reduce_in_fp32=precision["core_grad_reduce_in_fp32"], overlap_grad_reduce=True, + average_in_collective=True, use_distributed_optimizer=False), + model, pg_collection=self.groups, + ) + ddp.broadcast_params() + self._tp_partial_gradients = [p.main_grad for p in model.parameters() + if getattr(p, "_aspec_tp_partial_gradient", False)] + self._tp_partial_buffer = torch.empty( + sum(g.numel() for g in self._tp_partial_gradients), device="cuda", dtype=torch.float32, + ) + self._tp_partial_views = list(self._tp_partial_buffer.split( + [g.numel() for g in self._tp_partial_gradients])) + self._gradient_precision = { + "configured": getattr(self.args, "megatron_grad_dtype", "fp32"), + **precision, + "observed_main_grad_dtypes": sorted({str(p.main_grad.dtype) for p in model.parameters() + if p.requires_grad}), + "observed_core_grad_reduce_in_fp32": ddp.ddp_config.grad_reduce_in_fp32, + "observed_tp_partial_buffer_dtype": str(self._tp_partial_buffer.dtype), + } + # A weak reference avoids making a second module parent / state-dict cycle. + object.__setattr__(draft_model, "_aspec_ddp", weakref.proxy(ddp)) + ddp._aspec_checkpoint_state = lambda: model_checkpoint_state(model, self.tp_mesh) + ddp._aspec_load_checkpoint_state = lambda state: load_model_checkpoint_state(model, state) + return ddp + + def _normalize_accumulated_gradients(self, local_denominator, accumulation_steps): + self.model.finish_grad_sync() + if self.tp_size > 1: + flattened = [g.view(-1) for g in self._tp_partial_gradients] + torch._foreach_copy_(self._tp_partial_views, flattened) + dist.all_reduce(self._tp_partial_buffer, group=self.groups.tp) + torch._foreach_copy_(flattened, self._tp_partial_views) + denominator = local_denominator.detach().clone() + dist.all_reduce(denominator, group=self.dp_group) + scale = torch.where( + denominator > 0, + self.dp_size * accumulation_steps / denominator.clamp_min(1e-6), + torch.zeros_like(denominator), + ) + from aspec.models.ops.gradient_scaling import scale_gradients_ + + scale_gradients_([p.main_grad for p in self.optimizer.model_params], scale) + return denominator + + def _checkpoint_backend_rng(self): + from megatron.core.tensor_parallel.random import get_cuda_rng_tracker + + return {"backend": "megatron", "cuda_tracker": get_cuda_rng_tracker().get_states()} + + def _restore_backend_rng(self, state): + from megatron.core.tensor_parallel.random import get_cuda_rng_tracker + + if not isinstance(state, dict) or state.get("backend") != "megatron": + raise RuntimeError("Exact Megatron resume requires the model-parallel RNG tracker") + saved = state.get("cuda_tracker") + if not isinstance(saved, dict) or set(saved) != set(get_cuda_rng_tracker().get_states()): + raise RuntimeError("Megatron RNG tracker streams differ from the checkpoint") + if any(not isinstance(value, torch.Tensor) or value.dtype != torch.uint8 + or value.device.type != "cpu" or value.ndim != 1 for value in saved.values()): + raise RuntimeError("Invalid Megatron RNG tracker state") + get_cuda_rng_tracker().set_states(saved) + + def pop_committed_work(self, optimizer_step): + local = super().pop_committed_work(optimizer_step) + replicas = [None] * self.tp_size + dist.all_gather_object(replicas, local, group=self.groups.tp) + if any(items != local for items in replicas): + raise RuntimeError("TP ranks completed different ASpec work identities") + return local if self.tp_rank == 0 else [] + + def _kernel_observation(self): + wrapper = getattr(self, "dflash", None) + if getattr(wrapper, "l1_loss_backend", None) is None: + return None + return { + "l1_backend": wrapper.l1_loss_backend, + "l1_kernel_invocations": dict(wrapper.l1_kernel_invocations), + "anchor_tp_broadcasts": wrapper._aspec_anchor_tp_broadcasts, + "qk_preparation_backends": [layer.self_attn.qk_prepare_backend + for layer in self.draft_model.layers], + "gradient_checkpointing_enabled": self.draft_model.gradient_checkpointing, + "optimizer_fused_clip_scaling": self.optimizer._fuse_clip_scaling, + "gradient_precision": dict(self._gradient_precision), + "last_batch": {name.removesuffix("_observed"): value + for name, value in vars(wrapper).items() + if name.endswith("_observed") and type(value) in (int, float)}, + "attention_layers": list(self.draft_model.last_attention_observations), + } + + def get_feature_contract_status(self): + status = super().get_feature_contract_status() + if observation := self._kernel_observation(): + status["dspark_execution"] = observation + return status + + def save_draft_model_for_serving(self, output_dir): + error = None + try: + if self.dp_rank == 0: + state = model_checkpoint_state(self.draft_model, self.tp_mesh) + state = {name: (value.full_tensor() if isinstance(value, DTensor) else value).cpu() + for name, value in state.items()} + if self.tp_rank == 0: + root = Path(output_dir) + root.mkdir(parents=True, exist_ok=True) + self.draft_model.config.save_pretrained(root) + torch.save(state, root / "pytorch_model.bin") + except Exception as caught: + error = caught + message = self._synchronize_serving_error(error) + if message is not None: + raise RuntimeError("Megatron Draft export failed: " + message) diff --git a/aspec/training/optimizer.py b/aspec/training/optimizer.py index bb5a06e..929a414 100644 --- a/aspec/training/optimizer.py +++ b/aspec/training/optimizer.py @@ -85,6 +85,20 @@ def __init__( wsd_decay_style=wsd_decay_style, ) + def _model_gradient(self, parameter): + return parameter.grad + + def _clip_gradients(self): + if self._fuse_clip_scaling: + grad_norm = torch.nn.utils.get_total_norm( + [p.grad for p in self.fp32_params if p.grad is not None] + ) + coefficient = (self.max_grad_norm / (grad_norm + 1e-6)).clamp(max=1.0) + self.optimizer.grad_scale = coefficient.reciprocal() + return grad_norm + self.optimizer.grad_scale = None + return torch.nn.utils.clip_grad_norm_(self.fp32_params, self.max_grad_norm) + def step(self, closure=None): """Perform optimizer step with gradient clipping. @@ -98,27 +112,17 @@ def step(self, closure=None): grad_destinations = [] grad_sources = [] for p, mp, g in zip(self.model_params, self.fp32_params, self.fp32_grads): - if p.grad is not None: + gradient = self._model_gradient(p) + if gradient is not None: grad_destinations.append(g) - grad_sources.append(p.grad) + grad_sources.append(gradient) mp.grad = g else: mp.grad = None if grad_destinations: _copy_tensors_(grad_destinations, grad_sources) - if self._fuse_clip_scaling: - grad_norm = torch.nn.utils.get_total_norm( - [p.grad for p in self.fp32_params if p.grad is not None] - ) - # Fused AdamW already supports device-side gradient unscaling. - # Apply the same clipping coefficient there, avoiding a complete - # read/write pass over the FP32 gradient buffers on Hopper. - coefficient = (self.max_grad_norm / (grad_norm + 1e-6)).clamp(max=1.0) - self.optimizer.grad_scale = coefficient.reciprocal() - else: - self.optimizer.grad_scale = None - grad_norm = torch.nn.utils.clip_grad_norm_(self.fp32_params, self.max_grad_norm) + grad_norm = self._clip_gradients() # Fused AdamW consumes this device scalar through the same path used by # GradScaler. A nonzero value skips parameter, moment, weight-decay, and diff --git a/aspec/training/tp_source_prefetch.py b/aspec/training/tp_source_prefetch.py new file mode 100644 index 0000000..460aa55 --- /dev/null +++ b/aspec/training/tp_source_prefetch.py @@ -0,0 +1,63 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Bounded source GET prefetch; GPU collectives stay on the consumer thread.""" + +import queue +import threading + + +class TPSourcePrefetch: + def __init__(self, read, release, *, depth): + self.read, self.release = read, release + self.queue = queue.Queue(maxsize=depth) + self.stop = threading.Event() + self.thread = None + self.error = None + self.exhausted = False + + def _run(self): + try: + while not self.stop.is_set(): + item = self.read() + while not self.stop.is_set(): + try: + self.queue.put(item, timeout=0.1) + break + except queue.Full: + continue + else: + self.release(item) + return + if item[0] is None: + return + except BaseException as error: + self.error = error + + def next(self): + if self.exhausted: + return None, {} + if self.thread is None: + self.thread = threading.Thread(target=self._run, daemon=True, name="tp-source-get") + self.thread.start() + while not self.stop.is_set(): + if self.error is not None: + raise self.error + try: + item = self.queue.get(timeout=0.1) + self.exhausted = item[0] is None + return item + except queue.Empty: + continue + raise StopIteration + + def close(self): + self.stop.set() + if self.thread is not None: + self.thread.join(timeout=10) + if self.thread.is_alive(): + raise RuntimeError("TP source GET did not stop; retain backing storage until the read fence") + while True: + try: + self.release(self.queue.get_nowait()) + except queue.Empty: + return diff --git a/aspec/training/trainer.py b/aspec/training/trainer.py index adf4900..f4eb9d9 100644 --- a/aspec/training/trainer.py +++ b/aspec/training/trainer.py @@ -74,12 +74,15 @@ def _consumer_participant( *, dp_rank: int, sequence_rank: int | None = None, + tensor_rank: int = 0, ) -> str: """Build a stable, unique identity for one feature/storage consumer.""" participant = f"draft_{role}_dp_rank_{dp_rank}" if sequence_rank is not None: participant = f"{participant}_sequence_rank_{sequence_rank}" + if tensor_rank: + participant = f"{participant}_tensor_rank_{tensor_rank}" return participant @@ -140,6 +143,8 @@ class Trainer(abc.ABC): logic via ``init_model``, ``_train_step``, and ``_aggregate_metrics``. """ + execution_backend = "fsdp" + def __init__(self, args: Namespace): self.args = args @@ -328,6 +333,7 @@ def set_train_queue( ) prefetch_depth = getattr(self.args, "prefetch_depth", 0) + tp_input = getattr(self, "input_tp_group", None) is not None gpu_device = torch.cuda.current_device() # When prefetching, stage data on CPU to avoid GPU contention between @@ -340,6 +346,7 @@ def set_train_queue( collator=collator, device=fetch_device, batch_size=per_dp_rank_batch_size, + prefetch_factor=prefetch_depth if tp_input else 2, assistant_header_ids=self.assistant_header_ids, end_token_ids=self.end_token_ids, dynamic_loss_mask=self.dynamic_loss_mask, @@ -355,15 +362,18 @@ def set_train_queue( "trainer", dp_rank=self.dp_rank, sequence_rank=sequence_rank, + tensor_rank=getattr(self, "tp_rank", 0), ), lifecycle_observer=lifecycle_observer, packing_enabled=packing_enabled, feature_affinity=getattr( self.args, "mooncake_feature_affinity", "any" ), + tensor_parallel_group=getattr(self, "input_tp_group", None), + tensor_parallel_control_group=getattr(self, "input_tp_control_group", None), ) - if prefetch_depth > 0: + if prefetch_depth > 0 and not tp_input: self.data_fetcher = PrefetchedDataFetcher( inner_fetcher, prefetch_depth=prefetch_depth, @@ -372,12 +382,13 @@ def set_train_queue( logger.info( f"[Rank {self.dp_rank}] Prefetched data fetcher initialized " f"(batch_size={per_dp_rank_batch_size}, prefetch_depth={prefetch_depth}, " - f"staging=CPU, target={gpu_device})" + f"staging={'GPU TP' if getattr(self, 'input_tp_group', None) is not None else 'CPU'}, target={gpu_device})" ) else: self.data_fetcher = inner_fetcher logger.info( f"[Rank {self.dp_rank}] Data fetcher initialized with batch_size={per_dp_rank_batch_size}" + + (f", source_prefetch_depth={prefetch_depth}, TP broadcasts on consumer thread" if tp_input else "") ) # ------------------------------------------------------------------ @@ -412,6 +423,7 @@ def set_eval_queue( collator=collator, device=torch.cuda.current_device(), batch_size=per_dp_rank_batch_size, + prefetch_factor=0, assistant_header_ids=self.assistant_header_ids, end_token_ids=self.end_token_ids, dynamic_loss_mask=self.dynamic_loss_mask, @@ -427,8 +439,11 @@ def set_eval_queue( "eval", dp_rank=self.dp_rank, sequence_rank=sequence_rank, + tensor_rank=getattr(self, "tp_rank", 0), ), lifecycle_observer=lifecycle_observer, + tensor_parallel_group=getattr(self, "eval_input_tp_group", None), + tensor_parallel_control_group=getattr(self, "eval_input_tp_control_group", None), ) self._eval_collator = collator self._eval_cache: list[dict] = [] diff --git a/aspec/training/trainer_actor.py b/aspec/training/trainer_actor.py index 3077962..2a6339d 100644 --- a/aspec/training/trainer_actor.py +++ b/aspec/training/trainer_actor.py @@ -127,7 +127,14 @@ def init(self, args: Namespace, role: str, mooncake_config=None, with_ref: bool if draft_model_config is None and getattr(args, "draft_model_config", None): draft_model_config = AutoDraftModelConfig.from_file(args.draft_model_config) - trainer_class = _trainer_class_for_config(draft_model_config) + if getattr(args, "train_backend", "fsdp") == "megatron": + from aspec.training.megatron_trainer import MegatronDSparkTrainer + + if not isinstance(draft_model_config, DSparkConfig): + raise ValueError("Megatron Draft currently requires DSpark") + trainer_class = MegatronDSparkTrainer + else: + trainer_class = _trainer_class_for_config(draft_model_config) self._trainer = trainer_class(args) target_model_path = getattr(args, "target_model_path", None) @@ -194,7 +201,8 @@ def get_status(self) -> dict: import aspec from aspec.inference.target_runtime import code_identity, dependency_versions - return { + trainer = getattr(self, "_trainer", None) + result = { "rank": self._rank, "world_size": self._world_size, "node_ip": self.get_node_ip(), @@ -204,11 +212,26 @@ def get_status(self) -> dict: "assigned_gpu_ids": [int(float(value)) for value in ray.get_gpu_ids()], "local_gpu_id": int(os.environ["LOCAL_RANK"]), "initialized": hasattr(self, "_trainer"), + "backend": getattr(trainer, "execution_backend", None), + "tensor_parallel_size": getattr(trainer, "tp_size", 1), + "tensor_parallel_rank": getattr(trainer, "tp_rank", 0), + "data_parallel_rank": getattr(trainer, "dp_rank", None), + "data_parallel_size": getattr(trainer, "dp_size", None), "runtime": {**code_identity(), "python": sys.executable, "prefix": sys.prefix, "aspec": aspec.__file__, "ray": ray.__file__, "torch": torch.__version__, "cuda": torch.version.cuda, - "dependencies": dependency_versions("nvidia-cutlass-dsl", "quack-kernels")}, + "dependencies": dependency_versions("nvidia-cutlass-dsl", "quack-kernels", + "megatron-core", "transformer-engine")}, } + if getattr(trainer, "execution_backend", None) == "megatron": + from aspec.runtime_identity import ( + MEGATRON_RUNTIME_RELATIVE_PATHS, + module_runtime_identity, + ) + + result["runtime"]["backend_identity"] = module_runtime_identity( + "megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS) + return result def save_model(self, step: int, force_sync: bool = False) -> None: self._trainer.save_model(step, force_sync) diff --git a/aspec/utils/env.py b/aspec/utils/env.py index 9e00c0c..1d37846 100644 --- a/aspec/utils/env.py +++ b/aspec/utils/env.py @@ -21,6 +21,10 @@ # compatibility directory. Ray runtime_env workers do not reliably retain # the driver's process loader path, so forward the resolved value exactly. "LD_LIBRARY_PATH", + "CUDA_HOME", + "CUDNN_HOME", + "CUDNN_FRONTEND_CUDART_LIB_NAME", + "CUDA_DEVICE_MAX_CONNECTIONS", # Source snapshots and explicitly selected backend adapters must resolve to # the same code in the driver and every Ray actor. "PYTHONPATH", @@ -81,6 +85,9 @@ def get_aspec_env_vars(args=None) -> dict[str, str]: **{k: "1" for k in _RAY_NOSET_VISIBLE_DEVICES_KEYS}, } env.update({k: os.environ[k] for k in _ASPEC_ENV_KEYS if k in os.environ}) + if (getattr(args, "train_backend", None) == "megatron" + or getattr(args, "target_model_backend", None) == "megatron"): + env.setdefault("CUDA_DEVICE_MAX_CONNECTIONS", "1") if getattr(args, "cluster_mode", "single_node") == "static_multi_node": # Ray is started from each node's own venv/environment. A driver's # library/cache paths must not overwrite another node's local layout. diff --git a/aspec/utils/train_dump.py b/aspec/utils/train_dump.py index 5a19912..c8c42f1 100644 --- a/aspec/utils/train_dump.py +++ b/aspec/utils/train_dump.py @@ -43,8 +43,9 @@ def extract_gradients(model: nn.Module) -> Dict[str, torch.Tensor]: """ gradients = {} for name, param in model.named_parameters(): - if param.grad is not None: - gradients[name] = param.grad.detach().clone().cpu() + gradient = param.grad if param.grad is not None else getattr(param, "main_grad", None) + if gradient is not None: + gradients[name] = gradient.detach().clone().cpu() return gradients diff --git a/docs/README.md b/docs/README.md index 18ba362..1abb6fe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,7 @@ The [project roadmap](../ROADMAP.md) lists implemented features and planned exte | Task | Guide | | --- | --- | | Configure resources, batches and Plan | [Configuration](user-guide/configuration.md) | +| Select Megatron versions and isolated environments | [Megatron integration](user-guide/megatron.md) | | Choose DSpark, DFlash, EAGLE or native MTP | [Algorithms](user-guide/algorithms.md) | | Choose Dense/MoE/MLA, SWA/CSA/QSA or packing/CP | [Structures](user-guide/structures.md) | | Use Ling Targets | [Ling recipes](user-guide/ling.md) | diff --git a/docs/framework/REFERENCES.md b/docs/framework/REFERENCES.md index fa68237..838df17 100644 --- a/docs/framework/REFERENCES.md +++ b/docs/framework/REFERENCES.md @@ -12,6 +12,8 @@ the retained source identities; applicable license and notice texts are distribu | [AngelSpec](https://github.com/Tencent/AngelSpec) `d3412bed231025ea85d35dbf5e0af44f1ac62a5b` | Thread-default reference retained in `aspec/utils/env.py` | | Retained [AngelSpec](https://github.com/Tencent/AngelSpec) snapshot `0966eb139d1c4054767e4a37bfc8a33db5989a72` | Concurrency, Direct-CP, Target-prefill, model-configuration and tensor-transfer regression cases; original source/license ancestry remains applicable | | [Megatron-LM](https://github.com/NVIDIA/Megatron-LM) and [FlashAttention](https://github.com/Dao-AILab/flash-attention) | Declared Direct CP, sparse metadata, output compaction and attention bridge derivations | +| Upstream [Megatron Core 0.20.0](https://github.com/NVIDIA/Megatron-LM/tree/685fb8bd970185e9ac3283fe32a5d6667bf4b268) | External, unmodified runtime through public model/spec, TP and DDP interfaces; independently authored ASpec Qwen3/DSpark adapters reuse the retained Target norm and training graph | +| [Megatron Bridge Qwen3 mapping](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/f46e89eafb935fb0c75209afdb4502322250d0c7/src/megatron/bridge/models/qwen/qwen3_bridge.py) | Public grouped-QKV/gated-MLP mapping reference; inspired ASpec's independent strict HF-to-Core loader, with explicit BF16 norm order and per-feature comparisons | | Official [SGLang](https://github.com/sgl-project/sglang) v0.5.21 `e00930c5489053f26d86b179cee0d087f846acbb` | Current Target runtime plus `aspec/compat/patches/sglang-v0.5.21.patch` | | [cuDNN Frontend](https://github.com/NVIDIA/cudnn-frontend) `cb8cec8a9884f77fa514767237502bef7b7ad9f0` | Source of the adapted ASpec-owned DSpark Q/K CuTeDSL specialization and selected Frost primitives | | cuDNN Frontend **1.31.0 wheel**, with the FlashAttention/CuTe source chain retained | Adapted SM90/SM100 joint GQA backward, pre/postprocess, layouts and host dispatch; [source paths, hashes and licenses](../../aspec/kernels/cutedsl/cudnn_gqa/README.md) | diff --git a/docs/releasing.md b/docs/releasing.md index 08f8116..551de73 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,7 +1,10 @@ # Releasing -ASpec publishes the Draft package `ling-aspec`, Target package `ling-aspec-target`, and -shared code package `ling-aspec-runtime` at the same version. Python imports and the CLI use `aspec`. +ASpec builds the Draft package `ling-aspec`, SGLang Target package `ling-aspec-target`, +Megatron environment package `ling-aspec-megatron`, and shared code package +`ling-aspec-runtime` at the same version. Python imports and the CLI use `aspec`. +The [Megatron guide](user-guide/megatron.md) describes its native build prerequisites +and independent environment profile. ## Versioning @@ -39,8 +42,8 @@ maintenance tools. `setup.py` collects resources from their canonical source loc In the release repository, configure a GitHub environment named `pypi`: - Add the environment secret `PYPI_API_TOKEN` with an account-wide PyPI token whose account can - publish all three distributions. This workflow uploads them together; a token scoped to one - project cannot authorize the other two. + publish all four distributions. This workflow uploads them together; a token scoped to one + project cannot authorize the others. - Allow deployments from `v*` tags. - Add required reviewers if releases need approval before upload. diff --git a/docs/support.md b/docs/support.md index 49f4391..3f299e4 100644 --- a/docs/support.md +++ b/docs/support.md @@ -29,6 +29,10 @@ passing Plan validation establishes legality, not performance, convergence or do ## Execution and assets +- Experimental upstream Megatron Core Target/Draft adapters cover Qwen3-8B dense DSpark, + single-node CP1/PP1, Target TP1/TP2 and Draft TP1/TP2 with DP. The roles select backends + independently and use separate environments. See [scope and validation](user-guide/megatron.md) + for current evidence and numerical/replay boundaries; no Core fork is required. - Single-node online training supports required NVLink and explicitly selected Store/transport paths. With packing enabled, DSpark/DFlash full and sliding attention preserve document semantics with FA4 or native cuDNN flattened-token execution at CP1 and above. Selected-entry CSA/QSA/DSA diff --git a/docs/user-guide/installation.md b/docs/user-guide/installation.md index 85b1d1a..c83ad4d 100644 --- a/docs/user-guide/installation.md +++ b/docs/user-guide/installation.md @@ -5,6 +5,8 @@ GPUs and a CUDA 13-compatible driver. The Target package supplies its matching C the host still needs the C++20 compiler and native libraries listed below. See [native prerequisites](installation-advanced.md#requirements) for the compiler, glibc and RDMA library requirements, or use the [container](containers.md). +The experimental [Megatron profile](megatron.md) uses Python >=3.12, a separate environment +package and staged Transformer Engine build; follow that guide when choosing Megatron for either role. ## Install from PyPI diff --git a/docs/user-guide/megatron.md b/docs/user-guide/megatron.md new file mode 100644 index 0000000..7a57ba8 --- /dev/null +++ b/docs/user-guide/megatron.md @@ -0,0 +1,260 @@ +# Megatron Target and Draft + +ASpec supports the public Megatron Core 0.20.0 API +(`core_v0.20.0`, revision `685fb8bd970185e9ac3283fe32a5d6667bf4b268`). +Select the Target and Draft independently. The validated configuration uses +Qwen3-8B prefill and dense full-attention DSpark, BF16, microbatch size one, +single-node CP1/PP1, and TP1 or TP2. The Megatron Draft supports DP replicas. +Shared Target, decode, MTP conditioning and Megatron Draft packing are outside +this configuration. + +## Dependency profiles + +| Distribution | Environment | +| --- | --- | +| `ling-aspec-runtime` | Shared ASpec code, CLI and service protocol | +| `ling-aspec` | Existing ASpec Draft dependencies | +| `ling-aspec-target` | SGLang Target dependencies | +| `ling-aspec-megatron` | Megatron dependencies for either role | + +Install each role in its own isolated venv. All ASpec distributions in a run use +the same release version. Install Megatron with the common runtime directly; +combining its profile with either default environment package creates unsupported +dependency mixtures. Backend imports remain lazy. + +Megatron requires Python >=3.12. Its profile pins Torch 2.13.0, CUDA toolkit 13.0.3, +Transformer Engine 2.20.2, FA4 b34, CUTLASS DSL 4.7.1 and Quack 0.6.5. The feature-only Target uses FlashInfer 0.6.18 +and the standalone sglang-kernel 0.4.7 package on SM90. The ordinary +Draft and SGLang profiles retain their independent versions. + +Transformer Engine's PyTorch extension must be built against the execution +environment's Torch ABI. An unconstrained build environment can select a different +Torch. Source setup installs the build prerequisites first, then builds with +`--no-build-isolation`, selecting the venv's CUDA 13 toolkit and NCCL/cuDNN headers +explicitly. Including all NVIDIA wheel headers can put the transport's CUDA 12 +runtime headers before CUDA 13; the setup helper avoids this mixture. + +From a source checkout: + +```bash +bash setup.sh --megatron --python python3.12 --venv /absolute/path/to/draft-venv +bash setup.sh --megatron --target --python python3.12 --venv /absolute/path/to/target-venv +``` + +For a built release directory, create a Python 3.12 venv, activate it, and install: + +```bash +pip install -r /path/to/release/runtime/megatron-build.txt +aspec_nvidia=$(python -c 'import pathlib,sysconfig; print(pathlib.Path(sysconfig.get_paths()["purelib"]) / "nvidia")') +CUDA_HOME="$aspec_nvidia/cu13" \ + NVTE_CUDA_INCLUDE_PATH="$aspec_nvidia/cu13/include" \ + CPLUS_INCLUDE_PATH="$aspec_nvidia/cudnn/include:$aspec_nvidia/nccl/include" \ + NVTE_BUILD_USE_NVIDIA_WHEELS=0 MAX_JOBS=4 pip install --no-build-isolation \ + --find-links /path/to/release ling-aspec-megatron +aspec doctor --backend megatron --gpu +``` + +The host needs a compatible CUDA driver and the native prerequisites in the +[installation guide](installation-advanced.md). + +Each process prepares native-library paths from its own interpreter. The optional +local supervisor removes automatically selected Draft library paths before starting +Target. A separate Target service requires a service address; managed-local mode +also requires an explicit Target interpreter. Neither role imports the other's +Python dependency graph or transfers Python model objects across the boundary. + +## Target and Draft adapters + +The frozen Target uses Core's `HybridModel` and `HybridStack`. Each logical +Qwen3 block contributes an attention position (`*`) and an MLP position (`-`). +For Qwen3-8B the stack executes 72 positions. The adapter maps the grouped QKV +and gate/up weights into the Core TP projections and loads the final norm. +The feature-only model omits the vocabulary head. + +ASpec supplies two Target-specific layer specs for the single-sequence prefill. +They preserve the BF16 residual stream across fused add/RMSNorm boundaries, +the Q/K normalization reduction order, RoPE, causal FA3 and fused SwiGLU. +At an auxiliary MLP boundary the TP leader adds the pending branch and residual +directly into the transport slot. The final feature uses the checkpoint's RMSNorm. +Each request goes through the same Core model forward. Each engine processes one +sequence at a time. + +Megatron Draft partitions attention heads and dense FFN channels using public +Core TP projections, and uses Core DDP gradient buffers with ASpec's FP32-master +AdamW update and schedule. The retained DSpark graph still owns attention/QK +kernels, block compaction, bounded projection caches, tiled/indexed L1, statistics, +selective recomputation and deferred metrics. Vocabulary, Markov and confidence +computations remain replicated across TP. This preserves the existing global +softmax/L1 semantics; vocabulary sharding is not a claimed capability. + +Megatron Draft gradient precision is selected through the normal training +configuration: + +```yaml +training: + train_backend: megatron + megatron_grad_dtype: bf16 # fp32 (default) or bf16 +``` + +`fp32` preserves the existing default. `bf16` passes +`grad_reduce_in_fp32=False` to Core's ordinary DDP configuration, selecting BF16 +main-gradient accumulation and DP reduction for the BF16 model parameters. The dedicated TP-partial Q/K norm-gradient +reduction, optimizer masters and Adam moments remain FP32 under both settings. +This option applies only to Megatron Draft; the existing Torch/REPLICATE BF16 +gradient path is unchanged. + +The Plan records accumulation, DP-reduction and TP-partial-reduction precision +separately. Runtime execution observations include actual main-gradient dtypes +and the Core DDP flag. Exact resume preserves this setting; checkpoints without +the field are interpreted as FP32. Changing gradient precision requires +a weights-only new stage rather than an exact continuation. + +One TP leader prefetches source GETs into bounded, preallocated destination rings. +TP broadcasts and collation run on the consumer thread between microbatches, in +the same order as model collectives. Availability and tensor metadata use a CPU +Gloo group; no background reader launches NCCL work. Training and evaluation own +separate input groups. With source prefetch enabled, destination slots must cover +`prefetch_depth + 2` samples, including the active consumer and an in-progress GET. +The destination lease remains attached until the consumer finishes. Source release +occurs once per DP replica. Anchors/keep masks are shared from the TP leader; +commit collection checks matching work identity and reports it once per replica. + +Gradients are normalized by the complete optimizer window's supervision weight, +pooled across DP. TP-partial Q/K norm gradients are reduced once, and clipping +counts each replicated parameter once. Checkpoints describe logical TP shards and +restore model, FP32 masters, moments, schedule and both ASpec and Core RNG streams. +Exact resume requires the same Draft backend, TP topology and world size. Cross-backend weights-only +initialization is a separate stage. Native export gathers the logical Draft weights +for the existing strict loader. + +## Run Qwen3-8B with DSpark + +Start with SGLang Target and Megatron Draft in separate environments. Activate +the Megatron Draft venv and select two allocated GPUs: + +```bash +source /absolute/path/to/megatron-draft-venv/bin/activate +CUDA_VISIBLE_DEVICES=0,1 aspec quickstart qwen3-8b \ + --target /absolute/path/to/Qwen3-8B --output outputs/megatron-draft-001 \ + --managed-target /absolute/path/to/sglang-target-venv/bin/python \ + --set training.train_backend=megatron \ + --set training.megatron_grad_dtype=bf16 \ + --set mooncake.destination_gpu_ring_slots=3 +``` + +This exercises training, fresh-process recovery, native export and strict loading. +The two-GPU Quickstart budget covers TP1 for both roles. Megatron Draft TP2 uses +`training.draft_tensor_parallel_size=2` and two Draft GPUs per DP replica. +Megatron Target is selected independently with `model.target_model_backend=megatron` +and `inference.inference_engine_type=megatron`; its TP2 settings are +`inference.megatron.tp_size=2` and `inference.inference_num_gpus_per_engine=2`. + +## Verification + +Run the fixed-feature comparison in each Draft's own environment using the same +initial weights, SGLang feature cache, anchors, schedule and gradient precision. +Prepare the initial weights with `megatron_torch_alignment.py --backend fsdp +--phase prepare`. For Torch TP1 use `--backend fsdp`. For Torch TP2 the +verification tool uses a native PyTorch DTensor adapter around the existing +DSpark trainer and optimizer. This adapter is available only in the validation +tool. The public Torch training entry point supports TP1. + +A Torch TP2/DP1 reference run is: + +```bash +CUDA_VISIBLE_DEVICES=0,1 python -m aspec._environment \ + /absolute/path/to/torch-venv/bin/torchrun --standalone --nproc_per_node=2 \ + /absolute/path/to/tools/validation/megatron_torch_alignment.py \ + --attention-backward-deterministic \ + --backend torch-tp --phase checkpoint-control --tp 2 \ + --steps 100 --split-step 50 \ + --tensor-steps 1,2,5,10,20,30,40,50,60,70,80,90,100 \ + --state-steps 1,10,25,50,75,100 \ + --source /absolute/path/to/ASpec \ + --model /absolute/path/to/Qwen3-8B \ + --teacher /absolute/path/to/aspec-eval-cache.pt \ + --initial /absolute/path/to/shared-initial/model.safetensors \ + --output /tmp/torch-tp2-dp1-reference +``` + +For the Megatron run, use its venv and checkout, replace `--backend torch-tp` +with `--backend megatron --megatron-grad-dtype bf16`, and add +`--reference /tmp/torch-tp2-dp1-reference` after the reference completes. Use four +workers and TP2 for DP2 in both roles; add update 11 to both tensor milestone lists. +The deterministic option verifies the cuDNN-FE attention backward policy and +writes its call counts on every rank. + +`checkpoint-control` saves after update 50 and continues to 100. In fresh +processes, use `--phase resume --resume .../checkpoints/iter_0000051` to restore +the saved state. Compare against the uninterrupted run with `--reference` and +`--no-save-reference-tensors`. The tool records loss components, gradient norm +and learning rate each step, plus logical gradients and optimizer state at the +requested milestones. + +For Target comparison, run `tools/validation/megatron_target_aspec_alignment.py` +with `torchrun` in the Megatron environment: + +```bash +python -m aspec._environment /absolute/path/to/megatron-venv/bin/torchrun \ + --standalone --nproc_per_node=2 \ + /absolute/path/to/tools/validation/megatron_target_aspec_alignment.py \ + --model /absolute/path/to/Qwen3-8B \ + --reference-cache /absolute/path/to/aspec-eval-cache.pt \ + --length 4096 --output /tmp/target-tp2.json +``` + +This compares requested residual captures and final post-norm features with the +existing ASpec/SGLang producer, using the same checkpoint and token IDs. The +cache's producer and value-stage identity are checked. Choose an SGLang cache produced +at the same TP size as the Target for the matched comparison. + +To exercise online training and recovery with pretokenized Parquet inputs: + +```bash +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 python -m aspec._environment \ + /absolute/path/to/megatron-draft-venv/bin/python \ + -m tools.validation.megatron_pipeline_acceptance \ + --model /absolute/path/to/Qwen3-8B \ + --train-data /absolute/path/to/train.parquet \ + --eval-data /absolute/path/to/eval.parquet \ + --target-python /absolute/path/to/megatron-target-venv/bin/python \ + --target-tp 2 --draft-tp 2 --dp 2 --accumulation 2 \ + --sequence-length 4096 --anchors 512 --steps 4 \ + --output /tmp/megatron-online-check +``` + +This uses the ordinary launcher and recipe audits, restarts the driver/workers +from a native checkpoint, checks state/RNG/work restoration, continues training, +and exports an asset for strict loading. Use compiler caches on a filesystem +supporting the toolchain's concurrent cache operations. Functional pipeline +completion and Target feature equivalence remain separate gates. + +## Validated configurations + +The deterministic online comparison on SM90 used 200 Open PerfectBlend prompts +with Qwen3-8B responses, two samples per optimizer update, a 4096-token context +limit and 512 anchors. DP1 accumulated two microbatches and DP2 accumulated one +per replica. Each pair used the same Target TP size, Draft topology, initial +weights and BF16 gradients. + +| Draft topology | SGLang Target + Torch Draft mean objective | Megatron Target + Megatron Draft mean objective | Max absolute per-step rel diff | +| --- | ---: | ---: | ---: | +| TP1/DP1 | 2.83849453 | 2.83849453 | 0% | +| TP2/DP1 | 2.83903998 | 2.83903998 | 0% | +| TP2/DP2 | 2.85058107 | 2.85058107 | 0% | + +The objective is 0.1 × CE + 0.9 × L1 + confidence loss, averaged over 100 +optimizer updates. Final Draft weights, FP32 masters and Adam moments matched +at every topology. For TP1, set `inference.sglang.extra_args.prefill_max_requests=1` +in the reference profile to match Megatron's single-request prefill. Batching +requests changes the token-row count of the first MLP down projection and can +change BF16 rounding. + +The independent fixed-feature checks at TP2/DP1 and TP2/DP2 verified that +restoring after update 50 reproduced the remaining 50 updates and final state +on each backend. Online mixed-backend checks covered SGLang Target with +Megatron Draft, Megatron Target with Torch Draft, feature transport, export +and strict native loading. The 4K fixture exercised CuTeDSL Q/K preparation, +tiled FP32 L1 and fused optimizer clipping. Multi-chunk projection caching, +optional backbone checkpointing, throughput, peak memory and held-out quality +require separate measurements. diff --git a/requirements/megatron-build.txt b/requirements/megatron-build.txt new file mode 100644 index 0000000..1099fb4 --- /dev/null +++ b/requirements/megatron-build.txt @@ -0,0 +1,10 @@ +# Install before building native Megatron/TE extensions, in the same venv. +torch==2.13.0 +cuda-toolkit[cccl,nvcc]==13.0.3.0 +numpy<2.4 +setuptools>=80 +wheel +packaging>=24.2 +pybind11 +cmake +ninja diff --git a/requirements/megatron-v0.20.0.txt b/requirements/megatron-v0.20.0.txt new file mode 100644 index 0000000..9c980b2 --- /dev/null +++ b/requirements/megatron-v0.20.0.txt @@ -0,0 +1,21 @@ +# Independent Megatron environment for either Target or Draft (Python >=3.12). +# Upstream core_v0.20.0: 685fb8bd970185e9ac3283fe32a5d6667bf4b268. +megatron-core==0.20.0 +transformer-engine[pytorch,core_cu13]==2.20.2 +torch==2.13.0 +torchvision==0.28.0 +transformers==5.12.1 +cuda-toolkit[cccl,nvcc]==13.0.3.0 +apache-tvm-ffi==0.1.11 +# Native prefill kernels for the feature-only HybridModel. +flashinfer-python==0.6.18 +sglang-kernel==0.4.7 +cuda-tile==1.6.0rc5 +flash-attn-4==4.0.0b34 +nvidia-cutlass-dsl[cu13]==4.7.1 +quack-kernels[cu13]==0.6.5 +nvidia-cudnn-frontend==1.31.0 +einops==0.8.2 +cmake +ninja +numba diff --git a/requirements/sglang-v0.5.21.txt b/requirements/sglang-v0.5.21.txt index eb37893..7a43663 100644 --- a/requirements/sglang-v0.5.21.txt +++ b/requirements/sglang-v0.5.21.txt @@ -1,5 +1,7 @@ # Target-only environment: retain SGLang's declared dependency versions. sglang==0.5.21 +# Already required by SGLang; explicit pin admits this prerelease in uv too. +cuda-tile==1.6.0rc5 flash-attn-4==4.0.0b34 cuda-toolkit[nvcc]==13.0.3.0 # Mooncake's native wheel needs libcudart.so.12 alongside Torch's CUDA 13. diff --git a/setup.py b/setup.py index 62324ac..9c61c64 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,9 @@ def run(self): ROOT / "examples/data/sample_conversations.jsonl", ROOT / "tools/recipe_catalog.json", ROOT / "requirements/sglang-v0.5.21.txt", - ROOT / "requirements/draft.txt"] + ROOT / "requirements/draft.txt", + ROOT / "requirements/megatron-v0.20.0.txt", + ROOT / "requirements/megatron-build.txt"] for source in assets: destination = package / "resources" / source.relative_to(ROOT) destination.parent.mkdir(parents=True, exist_ok=True) diff --git a/setup.sh b/setup.sh index d65f0a1..45c66a8 100755 --- a/setup.sh +++ b/setup.sh @@ -5,6 +5,7 @@ venv="$repo_root/.venv" base_python="" options=() role=draft +backend=default explicit_venv=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -13,10 +14,11 @@ while [[ $# -gt 0 ]]; do if [[ $1 == --venv ]]; then venv=$2; explicit_venv=1; else base_python=$2; fi shift 2 ;; --target) role=target; options+=("$1"); shift ;; + --megatron) backend=megatron; options+=("$1"); shift ;; --dev|--gpu) options+=("$1"); shift ;; --help) - echo 'Usage: bash setup.sh [--venv PATH] [--python EXECUTABLE] [--target] [--dev] [--gpu]' - echo 'Creates an isolated Linux Python >=3.11 venv and installs pinned dependencies.' + echo 'Usage: bash setup.sh [--venv PATH] [--python EXECUTABLE] [--target] [--megatron] [--dev] [--gpu]' + echo 'Creates an isolated Linux Python >=3.11 venv (>=3.12 for Megatron) and installs pinned dependencies.' echo 'Defaults to .venv. Existing venvs require this checkout owner.' echo 'Does not install host packages/drivers, download models, start training or clean clusters.' exit 0 ;; @@ -24,24 +26,28 @@ while [[ $# -gt 0 ]]; do esac done if [[ $role == target && $explicit_venv == 0 ]]; then venv="$repo_root/.venv-target"; fi +if [[ $backend == megatron && $explicit_venv == 0 ]]; then venv="$repo_root/.venv-megatron-$role"; fi [[ $(uname -s) == Linux && $(uname -m) == x86_64 ]] || { echo 'Use Linux x86-64; see docs/user-guide/installation.md.' >&2; exit 2; } [[ -z ${PYTHONPATH:-} && -z ${PYTHONHOME:-} ]] || { echo 'Unset PYTHONPATH and PYTHONHOME before setup.' >&2; exit 2; } +minimum_python=11 +if [[ $backend == megatron ]]; then minimum_python=12; fi if [[ -z $base_python ]]; then for candidate in python3.12 python3.11 python3; do - if command -v "$candidate" >/dev/null && "$candidate" -c 'import sys; sys.exit(sys.version_info[:2] < (3,11))'; then + if command -v "$candidate" >/dev/null && "$candidate" -c 'import sys; sys.exit(sys.version_info[:2] < (3, int(sys.argv[1])))' "$minimum_python"; then base_python=$candidate; break fi done fi -[[ -n $base_python ]] || { echo 'Install Python >=3.11 or pass --python EXECUTABLE.' >&2; exit 2; } -"$base_python" - <<'CHECK' +[[ -n $base_python ]] || { echo "Install Python >=3.$minimum_python or pass --python EXECUTABLE." >&2; exit 2; } +"$base_python" - "$minimum_python" <<'CHECK' import sys -if sys.version_info[:2] < (3, 11): - raise SystemExit("Python >=3.11 is required") +minimum = (3, int(sys.argv[1])) +if sys.version_info[:2] < minimum: + raise SystemExit(f"Python >=3.{minimum[1]} is required") CHECK venv=$("$base_python" -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$venv") if [[ -e $venv || -L $venv ]]; then diff --git a/tests/cpu/test_alignment_rel_diff.py b/tests/cpu/test_alignment_rel_diff.py new file mode 100644 index 0000000..5647f7c --- /dev/null +++ b/tests/cpu/test_alignment_rel_diff.py @@ -0,0 +1,59 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Guard numerical reports against hidden normalization/near-zero errors.""" + +import math + +import pytest +import torch + +from tools.validation.megatron_torch_alignment import compare_vectors, scalar_diff, selected_steps + + +def test_scalar_rel_diff_preserves_sign_and_exposes_zero_floor(): + assert scalar_diff(9, 10)["signed_rel_diff"] == -.1 + assert scalar_diff(9, 10)["abs_rel_diff"] == .1 + row = scalar_diff(1e-14, 0) + assert row["denominator_floored"] + assert row["absolute_difference"] == 1e-14 + assert row["signed_rel_diff"] == .01 + with pytest.raises(ValueError, match="Nonfinite"): + scalar_diff(float("nan"), 1) + + +def test_tensor_rel_diff_uses_logical_elements_and_reports_each_parameter(): + expected = {"large": torch.tensor([1., 2., 3.]), "small": torch.tensor([4.])} + actual = {"large": torch.tensor([2., 2., 3.]), "small": torch.tensor([2.])} + row = compare_vectors(actual, expected, device="cpu") + assert row["relative_l2"] == pytest.approx(math.sqrt(5 / 30)) + assert row["relative_l1"] == pytest.approx(3 / 10) + assert row["relative_max"] == .5 + assert row["mean_absolute"] == .75 + assert row["pointwise_rel_diff_mean"] == .375 + assert row["pointwise_rel_diff_quantiles"]["p50"] == .25 + assert row["pointwise_exceed_counts"]["0.1"] == 2 + assert row["different_elements"] == 2 + assert row["numel"] == 4 + assert {r["parameter"] for r in row["per_parameter"]} == set(expected) + + +def test_tensor_rel_diff_zero_reference_is_explicit_and_does_not_drop_elements(): + row = compare_vectors({"w": torch.tensor([0., 1e-12])}, + {"w": torch.zeros(2)}, device="cpu") + assert row["relative_l2"] is None + assert row["zero_reference"] + assert row["denominator_floored_count"] == 2 + assert row["pointwise_rel_diff_mean"] == pytest.approx(.5) + equal = compare_vectors({"w": torch.zeros(2)}, {"w": torch.zeros(2)}, device="cpu") + assert equal["relative_l2"] == 0 + assert equal["bitwise_equal"] + with pytest.raises(ValueError, match="Nonfinite"): + compare_vectors({"w": torch.tensor([float("inf")])}, + {"w": torch.ones(1)}, device="cpu") + + +def test_declared_observation_steps_do_not_extend_the_trajectory(): + assert selected_steps("1,2,10,100", 100) == {1, 2, 10, 100} + assert selected_steps("", 100) == set() + with pytest.raises(ValueError, match="within"): + selected_steps("101", 100) diff --git a/tests/cpu/test_megatron.py b/tests/cpu/test_megatron.py new file mode 100644 index 0000000..1879783 --- /dev/null +++ b/tests/cpu/test_megatron.py @@ -0,0 +1,267 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import subprocess +import sys +import threading +from argparse import Namespace +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +import torch.nn.functional as F + +from aspec.config.megatron import validate_megatron_execution +from aspec.config.train_config import load_config +from aspec.feature_contract import build_target_feature_contract, validate_feature_contract +from aspec.models.megatron import qwen3_parameter +from aspec.planning import compile_execution_plan + + +def test_baseline_import_does_not_require_megatron(): + result = subprocess.run([sys.executable, "-c", """ +import sys +class RejectMegatron: + def find_spec(self, name, *args): + if name == 'megatron' or name.startswith('megatron.'): + raise AssertionError('Baseline imported an optional backend') +sys.meta_path.insert(0, RejectMegatron()) +import aspec.inference.factory +import aspec.training.trainer_actor +"""], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def settings(**overrides): + return Namespace(**{"train_backend": "megatron", "target_model_backend": "megatron", + "draft_tensor_parallel_size": 2, "world_size": 4, + "megatron_tp_size": 2, "inference_num_gpus_per_engine": 2, + "target_prefill_memory_optimized": True, **overrides}) + + +@pytest.mark.parametrize("target,training", [("megatron", "fsdp"), ("sglang", "megatron"), + ("megatron", "megatron")]) +def test_independent_target_draft_pairings(target, training): + validate_megatron_execution(settings(target_model_backend=target, train_backend=training, + draft_tensor_parallel_size=1 if training == "fsdp" else 2), + {"family": "dspark"}) + + +@pytest.mark.parametrize("dtype,resolved", [("fp32", "float32"), ("bf16", "bfloat16")]) +def test_gradient_precision_configuration_reaches_plan_and_runtime(dtype, resolved): + recipe = Path(__file__).resolve().parents[2] / "configs/recipes/quickstart_qwen3_8b.yaml" + config = load_config(str(recipe), cli_args=[ + "training.train_backend=megatron", "training.draft_tensor_parallel_size=2", + "training.training_num_gpus_per_node=2", "training.prefetch_depth=0", + f"training.megatron_grad_dtype={dtype}", + ]) + plan = compile_execution_plan(config) + assert plan.to_runtime_args().megatron_grad_dtype == dtype + policy = plan.policy.parallelism["training"] + assert policy["gradient_accumulation_dtype"] == policy["gradient_reduce_dtype"] == resolved + assert policy["core_grad_reduce_in_fp32"] == (dtype == "fp32") + assert policy["tp_partial_gradient_reduce_dtype"] == policy["optimizer_state_dtype"] == "float32" + + +@pytest.mark.parametrize("backend,dtype", [("megatron", "fp16"), ("megatron", "float32"), + ("megatron", None), ("fsdp", "bf16")]) +def test_gradient_precision_rejects_invalid_or_wrong_backend_configuration(backend, dtype): + with pytest.raises(ValueError, match="megatron_grad_dtype"): + validate_megatron_execution(settings(train_backend=backend, draft_tensor_parallel_size=1, + megatron_grad_dtype=dtype), {"family": "dspark"}) + + +@pytest.mark.parametrize("saved,current,allowed", [ + (None, "fp32", True), (None, "bf16", False), + ("fp32", "fp32", True), ("bf16", "bf16", True), + ("fp32", "bf16", False), ("bf16", "fp32", False), +]) +def test_exact_resume_preserves_megatron_gradient_precision(saved, current, allowed): + from aspec.training.checkpoint import ( + _logical_training_topology, + _validate_exact_resume_topology, + ) + + args = Namespace(train_backend="megatron", draft_tensor_parallel_size=1, + world_size=1, megatron_grad_dtype=current) + topology = _logical_training_topology(args, 1) + if saved is None: + topology.pop("megatron_grad_dtype") + else: + topology["megatron_grad_dtype"] = saved + metadata = {"world_size": 1, "training_topology": topology} + if allowed: + _validate_exact_resume_topology(args, metadata) + else: + with pytest.raises(RuntimeError, match="gradient precision"): + _validate_exact_resume_topology(args, metadata) + + +@pytest.mark.parametrize("overrides,message", [ + ({"world_size": 3}, "divisible"), + ({"draft_context_parallel_size": 2}, "CP1"), + ({"dflash_packing": True}, "packed"), + ({"fsdp_cpu_offload": True}, "offload"), + ({"distributed_backend": "gloo"}, "NCCL"), + ({"micro_batch_size": 2}, "micro_batch_size"), + ({"prefetch_depth": 1, "mooncake_destination_gpu_ring_slots": 1}, "destination_gpu_ring_slots"), + ({"sglang_pp_size": 2}, "PP1"), + ({"target_prefill_memory_optimized": False}, "memory_optimized"), + ({"inference_num_gpus_per_engine": 1}, "must equal"), + ({"train_backend": "fsdp", "draft_tensor_parallel_size": 2}, "Draft TP"), +]) +def test_megatron_fails_closed_before_process_launch(overrides, message): + with pytest.raises(ValueError, match=message): + validate_megatron_execution(settings(**overrides), {"family": "dspark"}) + + +def test_producer_backend_is_part_of_the_feature_identity(): + kwargs = dict(drafter_family="dspark", target_hidden_size=16, target_num_hidden_layers=4, + aux_hidden_state_layers=[0, 2], store_last_hidden_states=True) + sglang = build_target_feature_contract(**kwargs) + megatron = build_target_feature_contract(**kwargs, producer_backend="megatron") + validate_feature_contract(megatron) + assert sglang["contract_id"] != megatron["contract_id"] + + +def test_tp_source_capacity_counts_dp_replicas_and_target_engines(): + recipe = Path(__file__).resolve().parents[2] / "configs/recipes/quickstart_qwen3_8b.yaml" + overrides = [ + "model.target_model_backend=megatron", "inference.inference_engine_type=megatron", + "training.train_backend=megatron", "training.draft_tensor_parallel_size=2", + "training.training_num_gpus_per_node=4", "training.draft_accumulation_steps=1", + "inference.megatron.tp_size=2", "inference.inference_num_gpus=2", + "inference.inference_num_gpus_per_engine=2", "inference.inference_batch_size=1", + "mooncake.source_gpu_ring_slots=2", + "mooncake.destination_gpu_ring_slots=3", + ] + config = load_config(str(recipe), cli_args=overrides) + plan = compile_execution_plan(config) + assert plan.to_runtime_args().dp_size == 2 + assert plan.policy.pipeline["global_batch_size"] == 2 + # Two Target GPUs still represent one engine with one source ring. + with pytest.raises(ValueError, match="cannot retain one complete Draft dispatch"): + load_config(base_config=config, cli_args=["mooncake.source_gpu_ring_slots=1"]) + + +def test_tp_receive_slot_remains_owned_until_consumer_releases_it(monkeypatch): + from aspec.training.data_fetcher import MooncakeDataset, _release_p2p_receive_leases + + dataset = object.__new__(MooncakeDataset) + dataset.tensor_parallel_group = object() + dataset.tensor_parallel_control_group = object() + dataset._tp_source_prefetch = None + dataset._tensor_parallel_samples = dataset._tensor_parallel_payload_bytes = 0 + dataset._next_queue_item = lambda: "sample" + receive_buffer = torch.zeros(3) + ring = {"busy": False, "reads": 0, "releases": 0} + + def release(): + assert ring["busy"] + ring["busy"] = False + ring["releases"] += 1 + + def load(_item): + assert not ring["busy"], "next prefetch cannot wait on the active consumer" + ring["busy"] = True + ring["reads"] += 1 + receive_buffer.fill_(ring["reads"]) + return {"features": receive_buffer, + "_mooncake_p2p_receive_lease": SimpleNamespace(release=release)} + + dataset._load_from_mooncake = load + dataset._read_tp_source = lambda: ("sample", load("sample")) + monkeypatch.setattr(torch.distributed, "get_global_rank", lambda *_: 0) + monkeypatch.setattr(torch.distributed, "get_rank", lambda *_: 0) + def broadcast_control(*_, group, **__): + assert group is dataset.tensor_parallel_control_group + + monkeypatch.setattr(torch.distributed, "broadcast_object_list", broadcast_control) + monkeypatch.setattr(torch.distributed, "broadcast", lambda *_, **__: None) + original_to = torch.Tensor.to + + def to(tensor, *args, **kwargs): + return tensor if kwargs.get("device") == "cuda" else original_to(tensor, *args, **kwargs) + + monkeypatch.setattr(torch.Tensor, "to", to) + _, first = dataset._tensor_parallel_item_on_device() + assert first["features"].tolist() == [1, 1, 1] + assert ring["busy"] and ring["releases"] == 0 + _release_p2p_receive_leases(first) + _, second = dataset._tensor_parallel_item_on_device() + assert second["features"].tolist() == [2, 2, 2] + _release_p2p_receive_leases(second) + assert ring == {"busy": False, "reads": 2, "releases": 2} + + +def test_grouped_qkv_tp_mapping_preserves_independent_hf_projections(): + generator = torch.Generator().manual_seed(1729) + tensors = {f"model.layers.0.self_attn.{name}_proj.weight": torch.randn(rows, 16, generator=generator) + for name, rows in (("q", 16), ("k", 8), ("v", 8))} + reader = SimpleNamespace(tensor=lambda key: tensors[key]) + config = SimpleNamespace(num_attention_heads=4, num_key_value_heads=2, hidden_size=16, head_dim=4) + inputs = torch.randn(3, 16, generator=generator) + queries, keys, values = [], [], [] + for rank in range(2): + mapped = qwen3_parameter(reader, "decoder.layers.0.self_attention.linear_qkv.weight", + config, rank, 2) + projected = F.linear(inputs, mapped).view(3, 1, 4, 4) + queries.append(projected[:, :, :2].flatten(1)) + keys.append(projected[:, :, 2].flatten(1)) + values.append(projected[:, :, 3].flatten(1)) + for name, shards in (("q", queries), ("k", keys), ("v", values)): + torch.testing.assert_close(torch.cat(shards, dim=-1), + F.linear(inputs, tensors[f"model.layers.0.self_attn.{name}_proj.weight"])) + + +def test_source_prefetch_only_reads_in_background_and_drains_unused_leases(): + from aspec.training.tp_source_prefetch import TPSourcePrefetch + + main_thread = threading.get_ident() + calls, released = [], [] + second_read = threading.Event() + values = iter((("one", {"id": 1}), ("two", {"id": 2}), (None, {}))) + + def read(): + calls.append(threading.get_ident()) + item = next(values) + if item[0] == "two": + second_read.set() + return item + + prefetch = TPSourcePrefetch(read, lambda item: released.append(item[1]), depth=1) + assert prefetch.next() == ("one", {"id": 1}) + assert second_read.wait(timeout=2) + prefetch.close() + assert calls and all(thread != main_thread for thread in calls) + assert all(item.get("id") in (None, 2) for item in released) + assert {"id": 2} in released + + +def test_hybrid_physical_layers_map_to_qwen3_logical_block(): + tensors = {name: torch.rand(8) for name in ( + "model.layers.9.input_layernorm.weight", "model.layers.9.post_attention_layernorm.weight", + "model.norm.weight", + )} + reader = SimpleNamespace(tensor=tensors.__getitem__) + for core_name, source_name in ( + ("decoder.layers.18.input_layernorm.weight", "model.layers.9.input_layernorm.weight"), + ("decoder.layers.19.pre_mlp_layernorm.weight", "model.layers.9.post_attention_layernorm.weight"), + ("final_norm.weight", "model.norm.weight"), + ): + assert qwen3_parameter(reader, core_name, None, 0, 1) is tensors[source_name] + + +def test_megatron_eval_cache_binds_target_worker_and_topology(tmp_path): + from aspec.controller.eval import eval_cache_identity + + args = SimpleNamespace( + target_model_path=str(tmp_path), target_model_backend="megatron", + megatron_tp_size=1, megatron_implementation="transformer_engine", + megatron_target_cache_identity={"adapter": "candidate", "core": "core", "dependencies": {}}, + ) + first = eval_cache_identity(args, "sha256:dataset", 8) + assert first["producer_code"] == args.megatron_target_cache_identity + args.megatron_tp_size = 2 + assert first != eval_cache_identity(args, "sha256:dataset", 8) diff --git a/tests/cpu/test_release_audit.py b/tests/cpu/test_release_audit.py index 0d79272..30ad450 100644 --- a/tests/cpu/test_release_audit.py +++ b/tests/cpu/test_release_audit.py @@ -61,7 +61,7 @@ def _artifacts( link.type = tarfile.SYMTYPE link.linkname = "target" output.addfile(link) - for role in ("ling-aspec", "ling-aspec-target"): + for role in ("ling-aspec", "ling-aspec-target", "ling-aspec-megatron"): normalized = role.replace("-", "_") metadata = (f"Name: {role}\nVersion: {wheel_version}\n" f"Requires-Dist: ling-aspec-runtime=={wheel_version}\n") @@ -81,7 +81,7 @@ def test_release_audit_accepts_complete_artifact_pair(tmp_path): result = audit(tmp_path) assert result[wheel.name] == len(WHEEL_REQUIRED) + 1 assert result[source.name] == len(SDIST_REQUIRED) - assert len(result) == 6 + assert len(result) == 8 @pytest.mark.parametrize("member", ["aspec/compat/sglang_patch.py", @@ -119,7 +119,7 @@ def test_release_audit_rejects_non_source_members(tmp_path, member): def test_release_audit_requires_one_wheel_and_one_sdist(tmp_path): - with pytest.raises(ValueError, match="exactly three wheels and three sdists"): + with pytest.raises(ValueError, match="exactly four wheels and four sdists"): audit(tmp_path) diff --git a/tests/cpu/test_target_service.py b/tests/cpu/test_target_service.py index ddcb8d5..d790327 100644 --- a/tests/cpu/test_target_service.py +++ b/tests/cpu/test_target_service.py @@ -57,7 +57,7 @@ def register(sig, handler): monkeypatch.setattr(module, "TargetService", lambda **_: instance) monkeypatch.setattr(module, "http_server", lambda *_: server) monkeypatch.setattr(module, "visible_gpu_ids", lambda: [0]) - monkeypatch.setattr(module, "inventory", lambda: {"python": sys.executable}) + monkeypatch.setattr(module, "inventory", lambda backend="sglang": {"python": sys.executable}) monkeypatch.setattr(module, "ray", MagicMock()) monkeypatch.setattr(module.tempfile, "mkdtemp", lambda **_: str(tmp_path)) monkeypatch.setattr(module.time, "sleep", lambda seconds: events.append(("wait", seconds))) diff --git a/tests/cpu/test_torch_tp_validation.py b/tests/cpu/test_torch_tp_validation.py new file mode 100644 index 0000000..dc26184 --- /dev/null +++ b/tests/cpu/test_torch_tp_validation.py @@ -0,0 +1,61 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Check native DTensor TP gradients against independent split-linear autograd.""" + +from datetime import timedelta + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from torch.distributed.device_mesh import init_device_mesh + +from tools.validation.torch_tp_dspark import TorchTPLinear + + +def _projection_worker(rank, rendezvous): + torch.set_num_threads(1) + dist.init_process_group("gloo", init_method=rendezvous, rank=rank, world_size=2, + timeout=timedelta(seconds=60)) + try: + mesh = init_device_mesh("cpu", (2,)) + for dtype in (torch.float32, torch.bfloat16): + for axis in (0, 1): + torch.manual_seed(114) + full = torch.nn.Linear(8, 12, bias=False, dtype=dtype) + layer = TorchTPLinear(full, mesh, axis) + reference_weight = full.weight.detach().clone().requires_grad_(True) + reference_input = torch.randn(2, 3, 8, dtype=dtype, requires_grad=True) + gradient = torch.randn(2, 3, 12, dtype=dtype) + pieces = reference_weight.chunk(2, dim=axis) + if axis == 0: + expected = torch.cat([torch.nn.functional.linear(reference_input, w) + for w in pieces], dim=-1) + local = reference_input.detach().clone().requires_grad_(True) + local_gradient = gradient.chunk(2, dim=-1)[rank] + else: + partials = [torch.nn.functional.linear(x.contiguous(), w.contiguous()) + for x, w in zip(reference_input.chunk(2, dim=-1), pieces, strict=True)] + expected = partials[0] + partials[1] + local = reference_input.detach().chunk(2, dim=-1)[rank].clone().requires_grad_(True) + local_gradient = gradient + actual = layer(local) + expected_output = expected.chunk(2, dim=-1)[rank] if axis == 0 else expected + torch.testing.assert_close(actual, expected_output, rtol=0, atol=0) + # Exercise native BF16 leaf accumulation across two microbatches. + for _ in range(2): + (layer(local).float() * local_gradient.float()).sum().backward() + (expected.float() * gradient.float()).sum().backward(retain_graph=True) + torch.testing.assert_close(layer.weight.grad, + reference_weight.grad.chunk(2, dim=axis)[rank], rtol=0, atol=0) + expected_input_gradient = (reference_input.grad if axis == 0 + else reference_input.grad.chunk(2, dim=-1)[rank]) + torch.testing.assert_close(local.grad, expected_input_gradient, rtol=0, atol=0) + assert layer.weight.grad.dtype == dtype + finally: + dist.destroy_process_group() + + +@pytest.mark.skipif(not dist.is_gloo_available(), reason="Requires native Gloo collectives") +def test_dtensor_column_and_row_shards_preserve_forward_and_accumulated_backward(tmp_path): + mp.spawn(_projection_worker, args=((tmp_path / "rendezvous").as_uri(),), nprocs=2, join=True) diff --git a/tests/test_optimizer_state_keys.py b/tests/test_optimizer_state_keys.py index 80c192c..d08e05f 100644 --- a/tests/test_optimizer_state_keys.py +++ b/tests/test_optimizer_state_keys.py @@ -146,6 +146,30 @@ def load_state_dict(self, value): self.optimizer.load_state_dict(value) +def test_restore_preserves_dtensor_master_weights(tmp_path): + import torch.distributed as dist + from torch.distributed.device_mesh import init_device_mesh + from torch.distributed.tensor import Shard, distribute_tensor + + dist.init_process_group("gloo", init_method=(tmp_path / "rendezvous").as_uri(), + rank=0, world_size=1) + try: + model = nn.Linear(4, 2, bias=False) + mesh = init_device_mesh("cpu", (1,)) + model.weight = nn.Parameter(distribute_tensor(model.weight.detach(), mesh, [Shard(0)])) + optimizer = _StubBF16Optimizer(model) + state = OptimizerState(model, optimizer) + saved = state.state_dict() + expected = saved["fp32_params"]["weight"].clone() + saved["fp32_params"]["weight"] = expected + optimizer.fp32_params[0].zero_() + + state.load_state_dict(saved) + torch.testing.assert_close(optimizer.fp32_params[0].to_local(), expected.to_local(), rtol=0, atol=0) + finally: + dist.destroy_process_group() + + def test_dcp_exact_resume_restores_lazy_adam_moments_without_an_extra_update(tmp_path): original = _RealAdamBF16Surface(nn.Linear(3, 2)) original.fp32_params[0].grad = torch.ones_like(original.fp32_params[0]) diff --git a/tools/audit_release.py b/tools/audit_release.py index 663a75d..a59fcd4 100644 --- a/tools/audit_release.py +++ b/tools/audit_release.py @@ -75,6 +75,8 @@ "aspec/compat/patches/sglang-v0.5.21.patch", "requirements/sglang-v0.5.21.txt", "requirements/draft.txt", + "requirements/megatron-v0.20.0.txt", + "requirements/megatron-build.txt", "tools/audit_release.py", "tools/recipe_catalog.json", "tools/build_release.py", @@ -107,9 +109,19 @@ "aspec/commands/convert_pretokenized_jsonl.py", "aspec/commands/_recipe_smoke.py", "aspec/commands/_checkpoint_audit.py", + "aspec/config/megatron.py", + "aspec/models/megatron.py", + "aspec/models/megatron_target.py", + "aspec/models/megatron_draft.py", + "aspec/training/megatron_trainer.py", + "aspec/training/megatron_optimizer.py", + "aspec/training/tp_source_prefetch.py", + "aspec/inference/engine/megatron_engine.py", "aspec/commands/_recipe_capacity.py", "aspec/resources/requirements/sglang-v0.5.21.txt", "aspec/resources/requirements/draft.txt", + "aspec/resources/requirements/megatron-v0.20.0.txt", + "aspec/resources/requirements/megatron-build.txt", "aspec/resources/tools/recipe_catalog.json", "aspec/resources/examples/data/sample_conversations.jsonl", } @@ -146,7 +158,7 @@ def _identity(raw: bytes, artifact: Path) -> tuple[str, str]: version = metadata.get("Version") if not name or not version: raise ValueError(f"{artifact.name}: package metadata lacks Name or Version") - if name not in {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime"}: + if name not in {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime", "ling-aspec-megatron"}: raise ValueError(f"{artifact.name}: unexpected distribution name {name!r}") if name != "ling-aspec-runtime": requirements = {value.replace(" ", "") for value in metadata.get_all("Requires-Dist", [])} @@ -198,7 +210,7 @@ def _sdist_members(path: Path) -> tuple[set[str], tuple[str, str]]: missing = SDIST_REQUIRED - names if root.startswith("ling_aspec_runtime-") else {"PKG-INFO", "LICENSE", "README.md", "pyproject.toml"} - names if missing: raise ValueError(f"{path.name}: missing source members {sorted(missing)}") - if not root.startswith(("ling_aspec-", "ling_aspec_target-", "ling_aspec_runtime-")): + if not root.startswith(("ling_aspec-", "ling_aspec_target-", "ling_aspec_runtime-", "ling_aspec_megatron-")): raise ValueError(f"{path.name}: unexpected source root {root!r}") with tarfile.open(path, mode="r:gz") as source: metadata_member = source.getmember(f"{root}/PKG-INFO") @@ -212,19 +224,19 @@ def _sdist_members(path: Path) -> tuple[set[str], tuple[str, str]]: def audit(directory: Path) -> dict[str, int]: wheels = sorted(directory.glob("*.whl")) sdists = sorted(directory.glob("*.tar.gz")) - if len(wheels) != 3 or len(sdists) != 3: - raise ValueError(f"{directory}: expected exactly three wheels and three sdists") + if len(wheels) != 4 or len(sdists) != 4: + raise ValueError(f"{directory}: expected exactly four wheels and four sdists") result, identities = {}, {} for artifact in [*wheels, *sdists]: members, identity = (_wheel_members(artifact) if artifact.suffix == ".whl" else _sdist_members(artifact)) result[artifact.name] = len(members) identities.setdefault(identity, []).append(artifact.suffix == ".whl") - expected = {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime"} + expected = {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime", "ling-aspec-megatron"} if ({name for name, version in identities} != expected or len({version for name, version in identities}) != 1 or any(sorted(kinds) != [False, True] for kinds in identities.values())): - raise ValueError("wheel identity does not match the three version-aligned source identities") + raise ValueError("wheel identity does not match the four version-aligned source identities") return result diff --git a/tools/build_release.py b/tools/build_release.py index ee00aef..befd036 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -33,10 +33,13 @@ def build_release(directory): subprocess.run([sys.executable, "-m", "build", "--outdir", str(directory)], cwd=root, check=True) project = tomllib.loads((root / "pyproject.toml").read_text())["project"] - for role, manifest in (("draft", "draft.txt"), ("target", "sglang-v0.5.21.txt")): + for role, manifest, name, python_min in ( + ("draft", "draft.txt", "ling-aspec", "3.11"), + ("target", "sglang-v0.5.21.txt", "ling-aspec-target", "3.11"), + ("megatron", "megatron-v0.20.0.txt", "ling-aspec-megatron", "3.12"), + ): with tempfile.TemporaryDirectory(prefix="aspec-role-") as staging: stage = Path(staging) - name = "ling-aspec" if role == "draft" else "ling-aspec-target" dependencies = [f"ling-aspec-runtime=={project['version']}"] dependencies += [line.strip() for line in (root / "requirements" / manifest).read_text().splitlines() if line.strip() and not line.lstrip().startswith("#")] @@ -45,7 +48,7 @@ def build_release(directory): f"name = {json.dumps(name)}\nversion = {json.dumps(project['version'])}\n" f"description = {json.dumps('ASpec ' + role + ' environment')}\n" "readme = \"README.md\"\nlicense = \"Apache-2.0\"\n" - "license-files = [\"LICENSE\"]\nrequires-python = \">=3.11\"\n" + f"license-files = [\"LICENSE\"]\nrequires-python = \">={python_min}\"\n" f"dependencies = {json.dumps(dependencies)}\n\n" "[project.urls]\nHomepage = \"https://github.com/inclusionAI/ASpec\"\n" "\n[tool.setuptools]\npackages = []\n") @@ -59,7 +62,7 @@ def build_release(directory): print(f"PASS {artifact}: {count} reviewed members", flush=True) runtime = directory / "runtime" runtime.mkdir() - for name in ("sglang-v0.5.21.txt", "draft.txt"): + for name in ("sglang-v0.5.21.txt", "draft.txt", "megatron-v0.20.0.txt", "megatron-build.txt"): shutil.copyfile(root / "requirements" / name, runtime / name) files = sorted(path for path in directory.rglob("*") if path.is_file()) (directory / "SHA256SUMS").write_text("".join( diff --git a/tools/install.sh b/tools/install.sh index 0a088ab..d030c4e 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -2,14 +2,16 @@ set -euo pipefail dev=0 role=draft +backend=default gpu=() for argument in "$@"; do case "$argument" in --dev) dev=1 ;; --target) role=target ;; + --megatron) backend=megatron ;; --gpu) gpu=(--gpu) ;; --help) - echo 'Usage: bash tools/install.sh [--target] [--dev] [--gpu]' + echo 'Usage: bash tools/install.sh [--target] [--megatron] [--dev] [--gpu]' echo 'For an already activated isolated venv. New users: bash setup.sh' exit 0 ;; *) printf 'Unknown argument: %s\n' "$argument" >&2; exit 2 ;; @@ -48,9 +50,27 @@ if [[ $role == target ]]; then fi extras="." (( dev == 0 )) || extras=".[dev]" -python -m pip install -r "$manifest" -e "$extras" +if [[ $backend == megatron ]]; then + python -c 'import sys; sys.exit("Megatron requires Python >=3.12" if sys.version_info[:2] < (3,12) else 0)' + manifest=requirements/megatron-v0.20.0.txt + check+=(--backend megatron) + # Build TE against the actual execution Torch, including its native headers. + # An unconstrained isolated build can select a different Torch ABI. + python -m pip install -r requirements/megatron-build.txt + aspec_nvidia=$(python -c 'import pathlib,sysconfig; print(pathlib.Path(sysconfig.get_paths()["purelib"]) / "nvidia")') + # The transport also needs CUDA 12's runtime. Including every NVIDIA wheel + # would put its headers ahead of CUDA 13 and break TE's C++ compilation. + CUDA_HOME="$aspec_nvidia/cu13" \ + NVTE_CUDA_INCLUDE_PATH="$aspec_nvidia/cu13/include" \ + CPLUS_INCLUDE_PATH="$aspec_nvidia/cudnn/include:$aspec_nvidia/nccl/include:${CPLUS_INCLUDE_PATH:-}" \ + NVTE_BUILD_USE_NVIDIA_WHEELS=0 MAX_JOBS="${MAX_JOBS:-4}" \ + python -m pip install --no-build-isolation -r "$manifest" -e "$extras" + printf 'megatron\n' > "$VIRTUAL_ENV/.aspec-backend" +else + python -m pip install -r "$manifest" -e "$extras" +fi printf '%s\n' "$role" > "$VIRTUAL_ENV/.aspec-role" -if [[ $role == target ]]; then +if [[ $role == target && $backend != megatron ]]; then aspec patch-sglang aspec patch-sglang --check fi diff --git a/tools/validation/megatron_pipeline_acceptance.py b/tools/validation/megatron_pipeline_acceptance.py new file mode 100644 index 0000000..dd07ffa --- /dev/null +++ b/tools/validation/megatron_pipeline_acceptance.py @@ -0,0 +1,125 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Bounded mixed-backend train/resume/export through the normal ASpec launcher. + +Reuses ASpec's existing recipe audits. Unlike the bundled two-GPU Quickstart, +this validation entry keeps explicit TP, DP and gradient-accumulation settings. +Input capacity, future bitwise replay, quality and throughput are separate gates. +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +from omegaconf import OmegaConf + +from aspec.commands.quickstart import build_profile +from aspec.commands.recipes import audit_run, digest, execute, export_asset + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--train-data", required=True, type=Path) + parser.add_argument("--eval-data", type=Path) + parser.add_argument("--target-python", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--target-backend", choices=("sglang", "megatron"), default="megatron") + parser.add_argument("--draft-backend", choices=("fsdp", "megatron"), default="megatron") + parser.add_argument("--target-tp", type=int, choices=(1, 2), default=1) + parser.add_argument("--draft-tp", type=int, choices=(1, 2), default=1) + parser.add_argument("--dp", type=int, default=1) + parser.add_argument("--accumulation", type=int, default=2) + parser.add_argument("--sequence-length", type=int, default=4096) + parser.add_argument("--anchors", type=int, default=512) + parser.add_argument("--steps", type=int, default=4) + parser.add_argument("--gradient-checkpointing", action="store_true") + parser.add_argument("--attention-backend", choices=("fa4", "cudnn_flex_attention"), + default="cudnn_flex_attention") + parser.add_argument("--target-address", default="http://127.0.0.1:30000") + args = parser.parse_args() + if min(args.dp, args.accumulation, args.sequence_length, args.anchors) < 1: + parser.error("DP, accumulation, sequence length and anchors must be positive") + if args.steps < 2 or args.steps % 2: + parser.error("Use an even step count >=2") + if args.target_backend == "sglang" and args.target_tp != 1: + parser.error("The retained SGLang Target path requires TP1") + if args.draft_backend == "fsdp" and args.draft_tp != 1: + parser.error("Draft TP requires the Megatron backend") + if not args.train_data.is_file() or not args.target_python.is_absolute() or not args.target_python.is_file(): + parser.error("Provide existing data and an absolute Target interpreter path") + if args.eval_data is not None and not args.eval_data.is_file(): + parser.error("Evaluation data must exist") + world = args.draft_tp * args.dp + budget = world + args.target_tp + devices = os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",") + if len(devices) != budget or any(not item.strip() for item in devices) or len(set(devices)) != budget: + parser.error(f"Select exactly {budget} distinct Draft+Target GPUs") + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=False) + config = build_profile("qwen3-8b", args.model) + config = OmegaConf.merge(config, OmegaConf.create({ + "dataset": {"train_data_path": str(args.train_data.resolve()), "chat_template": None, + "eval_data_path": str(args.eval_data.resolve()) if args.eval_data else None, + "eval_interval": args.steps // 2}, + "model": {"target_model_backend": args.target_backend}, + "training": { + "train_backend": args.draft_backend, "draft_tensor_parallel_size": args.draft_tp, + "training_num_gpus_per_node": world, "draft_accumulation_steps": args.accumulation, + "attention_backend": args.attention_backend, "dflash_packing": False, + "gradient_checkpointing": args.gradient_checkpointing, + "max_seq_length": args.sequence_length, "dspark_num_anchors": args.anchors, + "num_epochs": 100, "lr_total_steps": args.steps, + "save_interval": args.steps // 2, "save_final_checkpoint": True, + }, + "inference": { + "inference_engine_type": "megatron" if args.target_backend == "megatron" else "sgl", + "inference_num_gpus": args.target_tp, "inference_num_gpus_per_engine": args.target_tp, + "inference_num_gpus_per_node": budget, + "megatron": {"tp_size": args.target_tp}, + "target_service": {"mode": "managed-local", "address": args.target_address, + "python_executable": str(args.target_python)}, + }, + "mooncake": {"max_seq_len": args.sequence_length, + "destination_gpu_ring_slots": 3 if args.draft_backend == "megatron" else 1, + "source_gpu_ring_slots": max(12, 2 * args.dp)}, + })) + profile = root / "profile.yaml" + OmegaConf.save(config, profile) + result = {"status": "running", "scope": "functional train/resume/native export", + "source_input": {"path": str(args.train_data.resolve()), "sha256": digest(args.train_data)}, + "world_size": world, "dp": args.dp, "draft_tp": args.draft_tp, + "target_tp": args.target_tp, "accumulation": args.accumulation} + result["gradient_checkpointing"] = args.gradient_checkpointing + if args.eval_data: + result["eval_input"] = {"path": str(args.eval_data.resolve()), "sha256": digest(args.eval_data)} + try: + doctor = [sys.executable, "-m", "aspec.commands.doctor", "--gpu"] + if args.draft_backend == "megatron": + doctor += ["--backend", "megatron"] + execute(doctor, root / "doctor.log") + execute([sys.executable, "-m", "aspec.train_entry", "--config", profile, "--dry-run"], + root / "plan.log") + for label, limit in (("first", args.steps // 2), ("resumed", args.steps)): + run = root / label + overrides = [f"training.num_train_steps={limit}", f"cache_dir={root / 'cache'}"] + if label == "resumed": + overrides.append(f"training.load_path={root / 'first/checkpoints'}") + execute([sys.executable, "-m", "aspec.commands._recipe_smoke", "--config", profile, + "--output", run, "--gpus", budget, *overrides], root / f"{label}.log") + audit_run(run, limit, label == "resumed", require_eval=bool(args.eval_data), expected_ranks=world) + export_asset(root / "resumed/checkpoints" / f"iter_{args.steps + 1:07d}", root / "asset", + log_prefix=root / "asset") + result["status"] = "passed" + except BaseException as error: + result.update(status="failed", error=str(error)) + raise + finally: + (root / "acceptance.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/validation/megatron_target_aspec_alignment.py b/tools/validation/megatron_target_aspec_alignment.py new file mode 100644 index 0000000..153db02 --- /dev/null +++ b/tools/validation/megatron_target_aspec_alignment.py @@ -0,0 +1,96 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Compare frozen Qwen3 Target features with a same-TP ASpec/SGLang cache. + +Run with torchrun in the Megatron environment. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path + +import torch +import torch.distributed as dist + +from aspec.models.megatron import initialize_parallelism, load_qwen3_target + + +def sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(8 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--reference-cache", type=Path, required=True) + parser.add_argument("--sample", type=int, default=0) + parser.add_argument("--length", type=int, default=4096) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl") + cached = torch.load(args.reference_cache, map_location="cpu", weights_only=False, mmap=True) + identity = cached["identity"] + contract = identity["inputs"]["aspec_feature_contract"] + if contract["producer"]["backend"] != "sglang": + raise ValueError("Reference cache must come from the existing ASpec SGLang producer") + if contract["tensors"]["last_hidden_states"]["value_stage"] != "post_norm": + raise ValueError("Expected the existing post-norm Target feature ABI") + sample = cached["samples"][args.sample] + if sample["input_ids"].shape[0] != 1 or sample["input_ids"].shape[1] < args.length: + raise ValueError("Select one complete reference sequence") + layers = contract["aux_hidden_state_layers"] + groups = initialize_parallelism(dist.get_world_size()) + model, config = load_qwen3_target(args.model, groups) + captured = {} + + def capture(layer, branch, residual): + captured[str(layer)] = branch + residual + + versions = [p._version for p in model.parameters()] + ids = sample["input_ids"][:, :args.length].cuda() + captured["final"] = model.prefill(ids, layers, capture) + frozen = (not any(p.requires_grad for p in model.parameters()) + and versions == [p._version for p in model.parameters()]) + verdict = [None] + if dist.get_rank() == 0: + expected = {str(layer): sample["hidden_states"][0, :args.length, + index * config.hidden_size:(index + 1) * config.hidden_size].cuda() + for index, layer in enumerate(layers)} + expected["final"] = sample["last_hidden_states"][0, :args.length].cuda() + measurements = {} + for name, target in expected.items(): + value, target = captured[name].double(), target.double() + measurements[name] = { + "mean": value.mean().item(), "reference_mean": target.mean().item(), + "equal": torch.equal(value, target), + "relative_l2": ((value - target).norm() / target.norm().clamp_min(1e-12)).item(), + "max_absolute": (value - target).abs().max().item(), + "cosine": torch.nn.functional.cosine_similarity(value.flatten(), target.flatten(), dim=0).item()} + passed = frozen and all(row["relative_l2"] <= 0.02 and row["cosine"] >= 0.999 + for row in measurements.values()) + result = {"reference_tp": identity["inputs"].get("sglang_tp_size"), "model_class": type(model).__name__, + "passed": passed, "reference": "existing_aspec_sglang_feature_cache", + "reference_sha256": sha(args.reference_cache), "reference_identity": identity, + "sample": args.sample, "length": args.length, "tp": dist.get_world_size(), + "target_frozen": frozen, "features": measurements, + "tolerances": {"relative_l2_max": 0.02, "cosine_min": 0.999}} + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as stream: + json.dump(result, stream, indent=2) + print(json.dumps({key: value for key, value in result.items() if key != "reference_identity"}), flush=True) + verdict[0] = passed + dist.broadcast_object_list(verdict, src=0) + dist.destroy_process_group() + if not verdict[0]: + raise SystemExit("Megatron features did not align with the existing ASpec Target") + + +if __name__ == "__main__": + main() diff --git a/tools/validation/megatron_torch_alignment.py b/tools/validation/megatron_torch_alignment.py new file mode 100644 index 0000000..7f8aa0a --- /dev/null +++ b/tools/validation/megatron_torch_alignment.py @@ -0,0 +1,662 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Compare native ASpec Torch and Megatron DSpark updates on fixed Target features. + +Run each backend in its own environment. Measure the objective at every update +and compare logical gradients, parameters and optimizer state at selected steps. +A fresh process can restore a checkpoint and compare with a continuous run. +""" + +import argparse +import copy +import hashlib +import importlib.util +import inspect +import json +import math +import os +import time +from pathlib import Path +from unittest.mock import patch + +import torch +import torch.distributed as dist +from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict +from torch.distributed.tensor import DTensor + +import aspec + + +def file_sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(8 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def tensor_record(value): + value = value.detach().cpu().contiguous() + return {"shape": list(value.shape), "dtype": str(value.dtype), + "sha256": hashlib.sha256(value.reshape(-1).view(torch.uint8).numpy().tobytes()).hexdigest()} + + +def write_json(path, value): + path = Path(path) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n") + temporary.replace(path) + + +def local_state(trainer): + def record(value): + if isinstance(value, DTensor): + value = value.to_local() + if isinstance(value, torch.Tensor): + return tensor_record(value) + if isinstance(value, dict): + return {str(k): record(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [record(v) for v in value] + return value + + return {"model": record(trainer.model.state_dict()), + "masters": record(trainer.optimizer.fp32_params), + "optimizer": record(trainer.optimizer.state_dict()), + "scheduler": record(trainer.lr_scheduler.state_dict()), + "torch_rng": record(torch.get_rng_state()), + "cuda_rng": record(torch.cuda.get_rng_state()), + "backend_rng": (record(trainer._checkpoint_backend_rng()) + if hasattr(trainer, "_checkpoint_backend_rng") else None), + "frozen_target_head": tensor_record(trainer.target_lm_head_weight), + "global_step": trainer.global_step} + + +def logical_cpu(trainer, value, parameter): + if value is None: + return None + if isinstance(value, DTensor): + value = value.full_tensor() + elif getattr(parameter, "_aspec_tp_axis", None) is not None: + if hasattr(trainer, "logical_tensor"): + value = trainer.logical_tensor(value, parameter).full_tensor() + else: + from aspec.models.megatron_draft import checkpoint_tensor + + value = checkpoint_tensor(value, parameter, trainer.tp_mesh).full_tensor() + return value.detach().cpu().clone() + + +def named_trainable(trainer): + result = [(name, parameter) for name, parameter in trainer.draft_model.named_parameters() + if parameter.requires_grad] + assert all(p is expected for (_, p), expected in zip( + result, trainer.optimizer.model_params, strict=True)) + return result + + +def logical_optimizer_state(trainer): + result = {key: {} for key in ("model", "masters", "exp_avg", "exp_avg_sq")} + for index, (name, parameter) in enumerate(named_trainable(trainer)): + master = trainer.optimizer.fp32_params[index] + result["model"][name] = logical_cpu(trainer, parameter, parameter) + result["masters"][name] = logical_cpu(trainer, master, parameter) + state = trainer.optimizer.optimizer.state.get(master, {}) + for key in ("exp_avg", "exp_avg_sq"): + if key in state: + result[key][name] = logical_cpu(trainer, state[key], parameter) + return result + + +def scalar_diff(actual, reference, epsilon=1e-12): + """Reference-relative signed difference, with an explicit denominator floor.""" + actual, reference = float(actual), float(reference) + if not math.isfinite(actual) or not math.isfinite(reference): + raise ValueError("Nonfinite scalar in numerical comparison") + delta = actual - reference + return {"actual": actual, "reference": reference, "signed_difference": delta, + "absolute_difference": abs(delta), + "signed_rel_diff": delta / max(abs(reference), epsilon), + "abs_rel_diff": abs(delta) / max(abs(reference), epsilon), + "denominator_floored": abs(reference) < epsilon, "epsilon": epsilon} + + +def compare_vectors(actual, expected, *, device=None): + """Full-vector reductions and explicit, near-zero-aware pointwise rel diffs. + + Norms/means/maxima/threshold counts use every element and FP64 reductions. + Only quantiles are sampled: systematic coordinates in the logical vector, + independent of values, with separate per-parameter samples. No training state + or RNG is changed. The denominator floor is max(1e-12, 1e-6 * reference RMS) + per parameter; floored coordinates are counted rather than silently removed. + """ + if set(actual) != set(expected): + raise ValueError("Compared trainable parameter sets differ") + device = device or ("cuda" if torch.cuda.is_available() else "cpu") + count = sum(v.numel() for v in expected.values() if v is not None) + stride = max(1, math.ceil(count / 262144)) + totals = torch.zeros(14, dtype=torch.float64, device=device) + global_samples, per_parameter = [], [] + exceed_counts = torch.zeros(4, dtype=torch.int64, device=device) + thresholds = (.01, .03, .1, 1.0) + exact, offset = True, 0 + + def ratio(numerator, denominator): + return numerator / denominator if denominator else (0.0 if numerator == 0 else None) + + def summarize(values): + n, e2, b2, a2, e1, b1, es, dot, floors, point_sum, maximum, reference_max, point_max, mismatches = values + return {"numel": int(n), "relative_l2": ratio(math.sqrt(e2), math.sqrt(b2)), + "relative_l1": ratio(e1, b1), "relative_max": ratio(maximum, reference_max), + "rmse": math.sqrt(e2 / max(n, 1)), "mean_absolute": e1 / max(n, 1), + "max_absolute": maximum, "signed_mean_difference": es / max(n, 1), + "reference_rms": math.sqrt(b2 / max(n, 1)), + "cosine": ratio(dot, math.sqrt(a2 * b2)), + "pointwise_rel_diff_mean": point_sum / max(n, 1), + "pointwise_rel_diff_max": point_max, "denominator_floored_count": int(floors), + "different_elements": int(mismatches), "zero_reference": b2 == 0} + + def quantiles(values): + if not values or not sum(v.numel() for v in values): + return {"sample_size": 0, "p50": None, "p95": None, "p99": None} + values = torch.cat(values).float() + return {"sample_size": values.numel(), + **{label: val for label, val in zip(("p50", "p95", "p99"), + torch.quantile(values, torch.tensor([.5, .95, .99], device=device)).tolist(), strict=True)}} + + for name, value in actual.items(): + reference = expected[name] + if value is None or reference is None: + if value is not None or reference is not None: + raise ValueError(f"Missing gradient differs: {name}") + continue + if value.shape != reference.shape: + raise ValueError(f"Compared tensor shapes differ: {name}") + a, b = value.to(device).float().flatten(), reference.to(device).float().flatten() + if not (torch.isfinite(a).all() & torch.isfinite(b).all()).item(): + raise ValueError(f"Nonfinite tensor in numerical comparison: {name}") + ad, bd = a.double(), b.double() + dd = delta = ad - bd + e2, b2 = dd.square().sum(), bd.square().sum() + floor = max(1e-12, 1e-6 * (b2 / max(b.numel(), 1)).sqrt().item()) + pointwise = delta.abs() / bd.abs().clamp_min(floor) + exceeds = torch.stack([(pointwise > threshold).sum() for threshold in thresholds]) + exceed_counts += exceeds + row = torch.stack((b.new_tensor(b.numel()), e2, b2, ad.square().sum(), dd.abs().sum(), + bd.abs().sum(), dd.sum(), (ad * bd).sum(), (b.abs() < floor).sum(), + pointwise.double().sum(), delta.abs().max(), b.abs().max(), + pointwise.max(), (a != b).sum())).double() + totals[:10] += row[:10] + totals[10:13] = torch.maximum(totals[10:13], row[10:13]) + totals[13] += row[13] + values = row.tolist() + exact = exact and value.dtype == reference.dtype and values[13] == 0 + row_summary = summarize(values) + per_parameter.append({"parameter": name, "squared_error": values[1], + "denominator_floor": floor, **row_summary, + "pointwise_exceed_counts": dict(zip(map(str, thresholds), exceeds.tolist(), strict=True)), + "pointwise_rel_diff_quantiles": quantiles([ + pointwise[::max(1, math.ceil(pointwise.numel() / 4096))]])}) + global_samples.append(pointwise[(-offset) % stride::stride].clone()) + offset += b.numel() + del a, b, delta, ad, bd, dd, pointwise + return {**summarize(totals.tolist()), "bitwise_equal": exact, + "pointwise_exceed_counts": dict(zip(map(str, thresholds), exceed_counts.tolist(), strict=True)), + "pointwise_rel_diff_quantiles": quantiles(global_samples), + "quantile_method": "systematic logical coordinates, global <=262144, per-parameter <=4096", + "denominator_floor_policy": "max(1e-12, 1e-6 * per-parameter reference RMS)", + "per_parameter": per_parameter, + "largest_errors": sorted(per_parameter, key=lambda row: row["squared_error"], reverse=True)[:5]} + + +def selected_steps(value, maximum): + if value == "all": + return set(range(1, maximum + 1)) + result = {int(step) for step in value.split(",") if step} + if any(step < 1 or step > maximum for step in result): + raise ValueError("Observation steps must be within the training trajectory") + return result + + +def wait_reference(path, timeout): + deadline = time.monotonic() + timeout + while not path.is_file(): + if time.monotonic() >= deadline: + raise TimeoutError(f"Reference observation unavailable: {path}") + time.sleep(.5) + + +def source_identity(trainer, expected): + from aspec.training import checkpoint + + root = Path(aspec.__file__).resolve().parent + if root != expected.resolve() / "aspec": + raise RuntimeError(f"Wrong ASpec checkout: {root}, expected {expected}") + files = {} + for method in (type(trainer), trainer._train_core_from_queue, trainer._train_step, + trainer._normalize_accumulated_gradients, type(trainer.optimizer), + type(trainer.optimizer).step, checkpoint.save, checkpoint.load): + path = Path(inspect.getfile(method)).resolve() + if path == Path(__file__).with_name("torch_tp_dspark.py").resolve(): + files["validation/torch_tp_dspark.py"] = file_sha(path) + else: + files[str(path.relative_to(root))] = file_sha(path) + return {"aspec": str(root), "python": os.sys.executable, "prefix": os.sys.prefix, + "torch": torch.__version__, "trainer": type(trainer).__name__, "files": files, + "harness_sha256": file_sha(__file__)} + + +def save_native_checkpoint(trainer, runtime, committed, fixture_sha, output, steps): + from aspec.training import checkpoint + + rank = dist.get_rank() + if rank == 0: + checkpoint.prepare_checkpoint_save(runtime.checkpoint_dir, trainer.global_step) + dist.barrier() + trainer.save_model(trainer.global_step, force_sync=True) + if rank == 0: + checkpoint.save_checkpoint_work_state(runtime.checkpoint_dir, trainer.global_step, { + "committed_work_ids": committed, "dataset_fingerprint": "sha256:" + fixture_sha, + "dataset_size": steps * 2 * trainer.dp_size}) + dist.barrier() + write_json(output / f"checkpoint-state-rank{rank}.json", local_state(trainer)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=("fsdp", "megatron", "torch-tp"), required=True) + parser.add_argument("--phase", choices=("prepare", "uninterrupted", "checkpoint-control", "split", "resume"), required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--teacher", type=Path, required=True) + parser.add_argument("--initial", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--reference", type=Path, action="append", default=[]) + parser.add_argument("--resume", type=Path) + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--megatron-grad-dtype", choices=("fp32", "bf16")) + parser.add_argument("--steps", type=int, default=4) + parser.add_argument("--stop-after", type=int, help="Stop at this update without shortening the LR horizon") + parser.add_argument("--split-step", type=int, default=2) + parser.add_argument("--length", type=int, default=4096) + parser.add_argument("--anchors", type=int, default=512) + parser.add_argument("--tensor-steps", default="all", help="Full gradient/update observations: all or comma-separated steps") + parser.add_argument("--state-steps", default="", help="Model/master/moment observations: comma-separated steps") + parser.add_argument("--save-reference-tensors", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--reference-wait-seconds", type=float, default=0) + parser.add_argument("--attention-backend", choices=("fa4", "cudnn_flex_attention"), default="cudnn_flex_attention") + parser.add_argument("--attention-backward-deterministic", action="store_true", + help="Require native deterministic cuDNN-FE attention backward on every rank") + args = parser.parse_args() + if args.backend == "fsdp" and args.tp != 1: + parser.error("The Torch baseline uses its existing TP1 execution") + if args.backend != "megatron" and args.megatron_grad_dtype is not None: + parser.error("Megatron gradient precision applies only to the Megatron backend") + if not args.attention_backward_deterministic: + run(args) + return + if args.attention_backend != "cudnn_flex_attention" or args.phase == "prepare": + parser.error("The deterministic backward check requires a cuDNN training run") + + from cudnn.flex_attention import autograd, flex_attn_func + + backward = autograd._flex_attention_backward + calls = {"forward": 0, "backward": 0} + + def deterministic_forward(*inputs, **options): + options["deterministic"] = True + calls["forward"] += 1 + return flex_attn_func(*inputs, **options) + + def checked_backward(*inputs, **options): + if not options.get("deterministic"): + raise RuntimeError("cuDNN attention backward did not receive deterministic=True") + calls["backward"] += 1 + return backward(*inputs, **options) + + with (patch("cudnn.flex_attention.flex_attn_func", deterministic_forward), + patch.object(autograd, "_flex_attention_backward", checked_backward)): + run(args, attention_calls=calls) + + +def run(args, *, attention_calls=None): + from aspec.config.train_config import load_config + from aspec.models.draft.auto import AutoDraftModelConfig + from aspec.planning import compile_execution_plan + from aspec.training import checkpoint + from aspec.training.dspark_trainer import DSparkTrainer + from aspec.utils.distributed import init_direct_cp_groups, init_gloo_group + + if args.backend == "torch-tp": + class RejectMegatronImports: + def find_spec(self, fullname, path=None, target=None): + if fullname == "megatron" or fullname.startswith("megatron."): + raise RuntimeError("The independent Torch TP validation cannot import Megatron Core") + + os.sys.meta_path.insert(0, RejectMegatronImports()) + tensor_steps = selected_steps(args.tensor_steps, args.steps) + state_steps = selected_steps(args.state_steps, args.steps) + torch.set_num_threads(4) + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl", device_id=torch.device("cuda", torch.cuda.current_device())) + init_gloo_group() + init_direct_cp_groups(1) + rank, world = dist.get_rank(), dist.get_world_size() + output = args.output.resolve() + if rank == 0: + output.mkdir(parents=True, exist_ok=False) + dist.barrier() + os.environ.setdefault("ASPEC_TRAIN_DATA", str(args.source / "examples/data/sample_conversations.jsonl")) + os.environ.setdefault("ASPEC_EVAL_DATA", str(args.source / "examples/data/sample_conversations.jsonl")) + base_backend = "fsdp" if args.backend == "torch-tp" else args.backend + overrides = [f"model.target_model_path={args.model}", + f"model.draft_model_config={args.source / 'aspec/config/dspark_draft_config.json'}", + f"training.train_backend={base_backend}", "training.dflash_packing=false", + f"training.training_num_gpus_per_node={world}", "training.draft_accumulation_steps=2", + "training.deterministic_anchor_sampling=true", f"training.max_seq_length={args.length}", + f"training.dspark_num_anchors={args.anchors}", f"training.num_train_steps={args.steps}", + f"training.lr_total_steps={args.steps}", f"training.attention_backend={args.attention_backend}", + "inference.target_prefill_memory_optimized=true", + "mooncake.source_gpu_ring_slots=12", "mooncake.destination_gpu_ring_slots=3", + f"output_dir={output}"] + if args.initial: + overrides.append(f"model.initial_draft_model_path={args.initial}") + if args.resume: + overrides.append(f"training.load_path={args.resume}") + if args.backend == "megatron": + overrides.append(f"training.draft_tensor_parallel_size={args.tp}") + if args.megatron_grad_dtype is not None: + overrides.append(f"training.megatron_grad_dtype={args.megatron_grad_dtype}") + config = load_config(str(args.source / "configs/recipes/qwen3_8b_dspark.yaml"), cli_args=overrides) + plan = compile_execution_plan(config) + runtime = plan.to_runtime_args() + runtime.rank, runtime.world_size = rank, world + if args.backend == "megatron": + from aspec.training.megatron_trainer import MegatronDSparkTrainer + + trainer = MegatronDSparkTrainer(runtime) + elif args.backend == "torch-tp": + # The maintained public Plan still rejects Torch Draft TP. This + # explicitly recorded validation extension tests its native DTensor + # arithmetic without claiming production Plan/backend support. + runtime.draft_tensor_parallel_size = args.tp + path = Path(__file__).with_name("torch_tp_dspark.py") + spec = importlib.util.spec_from_file_location("aspec_torch_tp_validation", path) + module = importlib.util.module_from_spec(spec) + os.sys.modules[spec.name] = module + spec.loader.exec_module(module) + trainer = module.TorchTPDSparkTrainer(runtime) + write_json(output / f"torch-tp-validation-rank{rank}.json", { + "scope": "validation-only extension of existing Torch DSpark trainer; not a production backend", + "base_plan_backend": "fsdp", "base_plan_draft_tp": 1, + "observed_draft_tp": trainer.tp_size, "observed_dp": trainer.dp_size, + "tp_operators": "native PyTorch DTensor linear/redistribute/autograd", + "adapter_sha256": file_sha(path), + }) + else: + trainer = DSparkTrainer(runtime) + trainer.init_model(AutoDraftModelConfig.from_file(runtime.draft_model_config), args.model) + identity = source_identity(trainer, args.source) + if args.backend == "megatron": + write_json(output / f"gradient-precision-rank{rank}.json", trainer._gradient_precision) + if args.backend != "megatron" and any(name == "megatron" or name.startswith("megatron.") for name in os.sys.modules): + raise RuntimeError("The baseline imported Megatron") + write_json(output / f"origin-rank{rank}.json", identity) + if args.phase == "prepare": + if args.backend != "fsdp" or world != 1: + raise ValueError("Prepare the common initialization with the actual TP1/DP1 Torch trainer") + from safetensors.torch import save_file + + state = get_model_state_dict(trainer.draft_model, options=StateDictOptions(full_state_dict=True, cpu_offload=True)) + save_file({name: value.contiguous() for name, value in state.items()}, str(output / "model.safetensors")) + trainer.draft_model.config.save_pretrained(output) + write_json(output / "prepared.json", {"source": identity, "initial_sha256": file_sha(output / "model.safetensors")}) + dist.destroy_process_group() + return + + initial_state = local_state(trainer) + write_json(output / f"initial-state-rank{rank}.json", initial_state) + if args.phase == "resume": + if args.resume is None: + raise ValueError("Resume requires a native checkpoint") + expected = json.loads((args.resume.parent.parent / f"checkpoint-state-rank{rank}.json").read_text()) + restored = initial_state == expected + write_json(output / f"restore-rank{rank}.json", {"exact": restored}) + if not restored: + differences = [key for key in expected if expected[key] != initial_state[key]] + raise RuntimeError(f"Native checkpoint state did not restore exactly on rank {rank}: {differences}") + del initial_state + saved_work = checkpoint.load_checkpoint_work_state(runtime) if args.resume else None + committed = list(saved_work["committed_work_ids"]) if saved_work else [] + samples = torch.load(args.teacher, map_location="cpu", weights_only=False, mmap=True)["samples"] + fixture_sha = file_sha(args.teacher) + manifest = {"teacher_sha256": fixture_sha, "initial_sha256": file_sha(args.initial) if args.initial else None, + "steps": args.steps, "length": args.length, "anchors": args.anchors, + "accumulation": 2, "attention_backend": args.attention_backend, + "dp_size": trainer.dp_size, "seed": runtime.seed, + "learning_rate": runtime.learning_rate, "warmup_ratio": runtime.warmup_ratio, + "objective_weights": [trainer.ce_loss_alpha, trainer.l1_loss_alpha, trainer.confidence_head_alpha]} + if rank == 0: + write_json(output / "run-manifest.json", manifest) + write_json(output / "runtime-config.json", json.loads(json.dumps(vars(runtime), default=str))) + for reference in args.reference: + wait_reference(reference / "run-manifest.json", args.reference_wait_seconds) + if manifest != json.loads((reference / "run-manifest.json").read_text()): + raise RuntimeError("Compared trajectories differ in fixed inputs or training contract") + trainable = named_trainable(trainer) + publish = trainer.dp_rank == 0 + writer = publish and getattr(trainer, "tp_rank", 0) == 0 + if args.phase != "resume" and publish: + logical = logical_optimizer_state(trainer) + if writer: + initial = {key: {name: tensor_record(value) for name, value in tensors.items()} + for key, tensors in logical.items()} + write_json(output / "initial-logical.json", initial) + for reference in args.reference: + wait_reference(reference / "initial-logical.json", args.reference_wait_seconds) + expected = json.loads((reference / "initial-logical.json").read_text()) + if initial != expected: + raise RuntimeError("Compared trainers did not start from identical logical weights/masters") + del logical + observation = {"terms": [], "anchors": [], "gradients": {}, "before": {}} + original_forward = trainer._forward + + def observed_forward(*values, **kwargs): + result = original_forward(*values, **kwargs) + observation["terms"].append(torch.stack([part.detach() for part in result[-1]])) + return result + + trainer._forward = observed_forward + original_sample = trainer.dflash._sample_anchor_positions + + def observed_sample(*values, **kwargs): + result = original_sample(*values, **kwargs) + observation["anchors"].append(tuple(value.detach().clone() for value in result)) + return result + + trainer.dflash._sample_anchor_positions = observed_sample + original_step = trainer.optimizer.step + + def observed_step(*values, **kwargs): + if publish and trainer.global_step + 1 in tensor_steps: + for index, (name, parameter) in enumerate(trainable): + gradient = (parameter.main_grad if hasattr(parameter, "main_grad") else parameter.grad) + observation["gradients"][name] = logical_cpu(trainer, gradient, parameter) + observation["before"][name] = logical_cpu(trainer, trainer.optimizer.fp32_params[index], parameter) + norm = original_step(*values, **kwargs) + observation["norm"] = norm.detach().clone() + return norm + + trainer.optimizer.step = observed_step + trainer._work_lifecycle_enabled = True + stop = args.split_step if args.phase == "split" else args.steps + if args.stop_after is not None: + if not trainer.global_step < args.stop_after <= stop: + raise ValueError("stop-after must follow the initial step and stay within the declared horizon") + stop = args.stop_after + while trainer.global_step < stop: + step = trainer.global_step + 1 + observation.update(terms=[], anchors=[], gradients={}, before={}) + batches = [] + for micro in range(2): + index = ((step - 1) * 2 * trainer.dp_size + micro * trainer.dp_size + trainer.dp_rank) % len(samples) + batch = {key: value[:, :args.length] if isinstance(value, torch.Tensor) else copy.deepcopy(value) + for key, value in samples[index].items()} + batch["_aspec_work"] = [{"execution_id": f"window-{step}/dp-{trainer.dp_rank}/micro-{micro}"}] + batches.append(batch) + trainer.data_fetcher = batches + metrics = trainer._train_core_from_queue(step=step - 1, num_batches=2) + if hasattr(metrics, "finalize"): + metrics = metrics.finalize() + local_work = trainer.pop_committed_work(step) + participants = [None] * world + dist.all_gather_object(participants, local_work) + new_work = [item for replica in participants for item in replica] + if len(new_work) != 2 * trainer.dp_size or set(new_work).intersection(committed): + raise RuntimeError("Committed work was duplicated or omitted") + committed.extend(new_work) + totals = torch.stack(observation["terms"]).sum(0) + dist.all_reduce(totals, group=trainer.dp_group) + anchors = [{"positions": tensor_record(a), "keep": tensor_record(k)} for a, k in observation["anchors"]] + write_json(output / f"anchors-step{step}-rank{rank}.json", anchors) + participant_anchors = [None] * world + dist.all_gather_object(participant_anchors, {"replica": trainer.dp_rank, "anchors": anchors}) + by_replica = {} + for participant in participant_anchors: + replica = str(participant["replica"]) + if replica in by_replica and by_replica[replica] != participant["anchors"]: + raise RuntimeError("TP participants used different anchors") + by_replica[replica] = participant["anchors"] + updates = {} + if publish and step in tensor_steps: + for index, (name, parameter) in enumerate(trainable): + current = logical_cpu(trainer, trainer.optimizer.fp32_params[index], parameter) + if writer: + updates[name] = current - observation["before"][name] + if writer: + vectors = {"gradients": observation["gradients"], "updates": updates} + record = {"step": step, "loss": (totals[0] / totals[1]).item(), + "numerator": totals[0].item(), "denominator": totals[1].item(), + "gradient_norm": observation["norm"].item(), + "learning_rate_after_update": trainer.optimizer.get_learning_rate(), + "gradient_dtypes": sorted({str(v.dtype) for v in vectors["gradients"].values() if v is not None}), + "anchors": anchors, "anchors_by_replica": by_replica, + "metrics": metrics, "comparisons": {}, + "full_tensor_observation": step in tensor_steps, + "fixture_sample_indices_by_replica": { + str(replica): [((step - 1) * 2 * trainer.dp_size + micro * trainer.dp_size + replica) + % len(samples) for micro in range(2)] + for replica in range(trainer.dp_size)}} + if args.save_reference_tensors and step in tensor_steps and args.phase in {"uninterrupted", "checkpoint-control"}: + torch.save(vectors, output / f"vectors-step{step}.pt") + for reference in args.reference: + wait_reference(reference / f"step{step}.json", args.reference_wait_seconds) + expected_record = json.loads((reference / f"step{step}.json").read_text()) + measurements = {} + if step in tensor_steps: + expected = torch.load(reference / f"vectors-step{step}.pt", weights_only=True, mmap=True) + measurements = {key: compare_vectors(value, expected[key]) for key, value in vectors.items()} + del expected + scalar_values = {"objective": record["loss"], "gradient_norm": record["gradient_norm"], + "learning_rate": record["learning_rate_after_update"], + **{key: value for key, value in metrics.items() if key.startswith("train/") + and isinstance(value, (int, float))}} + expected_scalars = {"objective": expected_record["loss"], + "gradient_norm": expected_record["gradient_norm"], + "learning_rate": expected_record["learning_rate_after_update"], + **expected_record["metrics"]} + measurements["scalars"] = {key: scalar_diff(value, expected_scalars[key]) + for key, value in scalar_values.items()} + measurements.update(loss_relative_error=abs(record["loss"] / expected_record["loss"] - 1), + denominator_exact=record["denominator"] == expected_record["denominator"], + anchors_exact=by_replica == expected_record["anchors_by_replica"], + learning_rate_exact=record["learning_rate_after_update"] == expected_record["learning_rate_after_update"]) + measurements["tolerance_passed"] = (measurements["loss_relative_error"] <= 0.01 + and measurements["denominator_exact"] and measurements["anchors_exact"] + and measurements["learning_rate_exact"] + and (step not in tensor_steps or all( + measurements[key]["relative_l2"] is not None + and measurements[key]["relative_l2"] <= 0.03 for key in vectors))) + measurements["tolerance_scope"] = "loss/gradient/update" if step in tensor_steps else "loss only; full tensors not observed" + record["comparisons"][str(reference)] = measurements + if publish and step in state_steps: + logical = logical_optimizer_state(trainer) + if writer: + if args.save_reference_tensors: + torch.save(logical, output / f"logical-state-step{step}.pt") + state_comparisons = {} + for reference in args.reference: + wait_reference(reference / f"step{step}.json", args.reference_wait_seconds) + expected = torch.load(reference / f"logical-state-step{step}.pt", weights_only=True, mmap=True) + state_comparisons[str(reference)] = {key: compare_vectors(value, expected[key]) + for key, value in logical.items()} + del expected + write_json(output / f"logical-state-step{step}.json", state_comparisons) + del logical + if writer: + write_json(output / f"step{step}.json", record) + print(json.dumps({"step": step, "backend": args.backend, "loss": record["loss"], + "full_tensors": step in tensor_steps, + "scalar_rel_diffs": {ref: {key: values["abs_rel_diff"] for key, values in c["scalars"].items() + if key in {"objective", "train/ce_loss", "train/l1_loss", "train/confidence_loss"}} + for ref, c in record["comparisons"].items()}}), flush=True) + observation.update(gradients={}, before={}, anchors=[], terms=[]) + if writer: + del vectors + del updates, batches + dist.barrier() + if args.phase == "checkpoint-control" and step == args.split_step: + save_native_checkpoint(trainer, runtime, committed, fixture_sha, output, args.steps) + + if args.phase == "split": + save_native_checkpoint(trainer, runtime, committed, fixture_sha, output, args.steps) + write_json(output / f"final-state-rank{rank}.json", local_state(trainer)) + if trainer.global_step == args.steps and publish and args.steps not in state_steps: + logical = logical_optimizer_state(trainer) + if writer: + if args.save_reference_tensors and args.phase in {"uninterrupted", "checkpoint-control"}: + torch.save(logical, output / "final-logical.pt") + comparisons = {} + for reference in args.reference: + wait_reference(reference / "completed.json", args.reference_wait_seconds) + expected = torch.load(reference / "final-logical.pt", weights_only=True, mmap=True) + comparisons[str(reference)] = {key: compare_vectors(values, expected[key]) + for key, values in logical.items()} + write_json(output / "final-logical-comparisons.json", comparisons) + del logical + elif trainer.global_step == args.steps and writer: + write_json(output / "final-logical-comparisons.json", + json.loads((output / f"logical-state-step{args.steps}.json").read_text())) + if attention_calls is not None: + if not attention_calls["backward"]: + raise RuntimeError("No native deterministic cuDNN attention backward calls were observed") + write_json(output / f"attention-backward-rank{rank}.json", attention_calls) + if rank == 0: + if args.backend == "torch-tp": + write_json(output / "torch-tp-execution.json", { + "optimizer_fused_clip_scaling": trainer.optimizer._fuse_clip_scaling, + "qk_preparation_backends": [layer.self_attn.qk_prepare_backend + for layer in trainer.draft_model.layers], + "l1_backend": trainer.dflash.l1_loss_backend, + "l1_kernel_invocations": dict(trainer.dflash.l1_kernel_invocations), + "megatron_core_imported": any(name == "megatron" or name.startswith("megatron.") + for name in os.sys.modules), + }) + write_json(output / "completed.json", {"phase": args.phase, "steps": trainer.global_step, + "backend": args.backend, "world_size": world, "tp": args.tp, + "committed_work_ids": committed, "teacher_sha256": fixture_sha, + "runtime": identity, "attention_backend": args.attention_backend}) + write_json(output / "observation-policy.json", { + "tensor_steps": sorted(tensor_steps), "state_steps": sorted(state_steps), + "save_reference_tensors": args.save_reference_tensors, + "scalar_rel_diff": "(actual-reference) / max(abs(reference), 1e-12)", + "tensor_pointwise_rel_diff": "abs(actual-reference) / max(abs(reference), max(1e-12, 1e-6 * per-parameter reference RMS))", + "trajectory": "independent native updates from identical initial weights; subsequent differences include accumulated drift", + "fixture_samples": len(samples), "fixture_repeats": True, + "initial_sha256": file_sha(args.initial) if args.initial else None}) + dist.barrier() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/tools/validation/torch_tp_dspark.py b/tools/validation/torch_tp_dspark.py new file mode 100644 index 0000000..8d46ef2 --- /dev/null +++ b/tools/validation/torch_tp_dspark.py @@ -0,0 +1,187 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Validation-only native PyTorch TP for the retained ASpec DSpark trainer. + +Independently implements the public DTensor linear placements; no Megatron +operators or optimizer are imported. Plain local Parameters preserve ASpec's +fused master-AdamW/clipping path. DTensor wraps operands inside each projection +so its native autograd owns TP collectives. This is not a production backend: +the public Torch configuration continues to reject Draft TP greater than one. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.distributed.device_mesh import init_device_mesh +from torch.distributed.tensor import DTensor, Replicate, Shard + +from aspec.training.dspark_trainer import DSparkTrainer +from aspec.training.fsdp import apply_fsdp2 +from aspec.training.optimizer import BF16Optimizer + + +class TorchTPLinear(torch.nn.Module): + def __init__(self, original, mesh, axis): + super().__init__() + if original.bias is not None: + raise ValueError("The Torch TP validation path requires bias-free projections") + self.mesh, self.axis = mesh, axis + self.weight = torch.nn.Parameter( + original.weight.chunk(mesh.size(), dim=axis)[mesh.get_local_rank()].clone(), + requires_grad=original.weight.requires_grad, + ) + self.weight._aspec_tp_axis = axis + + def forward(self, value): + placement = Replicate() if self.axis == 0 else Shard(value.ndim - 1) + inputs = DTensor.from_local(value, self.mesh, (placement,), run_check=False) + weight = DTensor.from_local(self.weight, self.mesh, (Shard(self.axis),), run_check=False) + result = F.linear(inputs, weight) + if self.axis == 1: + result = result.redistribute(placements=(Replicate(),)) + return result.to_local() + + +class TorchTPOptimizer(BF16Optimizer): + def checkpoint_tensor(self, index, value): + return self.model._validation_logical_tensor(value, self.model_params[index]) + + def _clip_gradients(self): + group = self.model._validation_tp_group + gradients = [master.grad for parameter, master in zip( + self.model_params, self.fp32_params, strict=True) + if master.grad is not None + and (hasattr(parameter, "_aspec_tp_axis") or group.rank() == 0)] + # The global vector contains each replicated parameter once, and every + # distinct shard. Retain the native Torch norm and fused clip-scaling. + norm_squared = torch.nn.utils.get_total_norm(gradients).square() + dist.all_reduce(norm_squared, group=group) + norm = norm_squared.sqrt_() + coefficient = (self.max_grad_norm / (norm + 1e-6)).clamp(max=1.0) + if self._fuse_clip_scaling: + self.optimizer.grad_scale = coefficient.reciprocal() + else: + from aspec.models.ops.gradient_scaling import scale_gradients_ + + self.optimizer.grad_scale = None + scale_gradients_([p.grad for p in self.fp32_params if p.grad is not None], coefficient) + return norm + + +class TorchTPDSparkTrainer(DSparkTrainer): + execution_backend = "torch-tp-validation" + _optimizer_class = TorchTPOptimizer + + def _setup_device_mesh(self): + size = self.args.draft_tensor_parallel_size + if size != 2 or dist.get_world_size() % size: + raise ValueError("The Torch TP validation adapter requires TP2 and complete DP replicas") + mesh = init_device_mesh("cuda", (dist.get_world_size() // size, size), + mesh_dim_names=("dp", "tp")) + self.tp_mesh, self.dp_mesh = mesh["tp"], mesh["dp"] + self.tp_group, self.dp_group = self.tp_mesh.get_group(), self.dp_mesh.get_group() + self.tp_size, self.tp_rank = size, self.tp_mesh.get_local_rank() + self.dp_size, self.dp_rank = self.dp_mesh.size(), self.dp_mesh.get_local_rank() + self.mesh = self.grad_sync_mesh = self.dp_mesh + self.cp_size, self.cp_rank, self.cp_group = 1, 0, None + self.cache_rank = dist.get_rank() + + def _get_init_weight_context_manager(self): + return lambda: torch.device("cpu") + + def _loads_initial_weights_here(self): + return True + + def _build_training_wrapper(self, draft_model): + wrapper = super()._build_training_wrapper(draft_model) + original = wrapper._sample_anchor_positions + + def sample(*args, **kwargs): + anchors, keep = original(*args, **kwargs) + leader = dist.get_global_rank(self.tp_group, 0) + dist.broadcast(anchors, src=leader, group=self.tp_group) + dist.broadcast(keep, src=leader, group=self.tp_group) + return anchors, keep + + wrapper._sample_anchor_positions = sample + return wrapper + + def logical_tensor(self, value, parameter): + axis = getattr(parameter, "_aspec_tp_axis", None) + if axis is None or value.ndim == 0: + return value + return DTensor.from_local(value, self.tp_mesh, (Shard(axis),), run_check=False) + + def _wrap_training_model(self, model, draft_model): + if (self.args.fsdp_strategy != "REPLICATE" or self.args.draft_context_parallel_size != 1 + or self.args.dflash_packing or self.args.compile_model): + raise ValueError("Torch TP validation requires native REPLICATE, padded CP1 execution") + config = draft_model.config + if any(getattr(config, key) % self.tp_size for key in ( + "num_attention_heads", "num_key_value_heads", "intermediate_size", + )): + raise ValueError("Attention heads and FFN width must divide the Torch TP size") + draft_model.cuda() + for layer in draft_model.layers: + for owner, names, axis in ( + (layer.self_attn, ("q_proj", "k_proj", "v_proj"), 0), + (layer.self_attn, ("o_proj",), 1), + (layer.mlp, ("gate_proj", "up_proj"), 0), + (layer.mlp, ("down_proj",), 1), + ): + for name in names: + setattr(owner, name, TorchTPLinear(getattr(owner, name), self.tp_mesh, axis)) + layer.self_attn.num_heads //= self.tp_size + layer.self_attn.num_kv_heads //= self.tp_size + self._partial_norm_parameters = [norm.weight for layer in draft_model.layers + for norm in (layer.self_attn.q_norm, layer.self_attn.k_norm)] + # Head-shared norm weights receive contributions from local heads only. + # Sum their partial derivatives in FP32, retaining BF16 leaf gradients. + self._partial_buffer = torch.empty(sum(p.numel() for p in self._partial_norm_parameters), + device="cuda", dtype=torch.float32) + self._partial_views = list(self._partial_buffer.split( + [p.numel() for p in self._partial_norm_parameters])) + model = apply_fsdp2(model, mesh=self.dp_mesh, args=self.args) + object.__setattr__(draft_model, "_validation_tp_group", self.tp_group) + object.__setattr__(draft_model, "_validation_logical_tensor", self.logical_tensor) + + def state(): + parameters = dict(model.named_parameters()) + return {name: self.logical_tensor(value, parameters[name]) if name in parameters else value + for name, value in model.state_dict().items() if isinstance(value, torch.Tensor)} + + def load(values): + current = model.state_dict() + if set(current) != set(values): + raise ValueError("The Torch TP checkpoint must cover the full model") + with torch.no_grad(): + for name, value in values.items(): + local = value.to_local() if isinstance(value, DTensor) else value + if current[name].shape != local.shape: + raise ValueError(f"Torch TP checkpoint shape mismatch: {name}") + current[name].copy_(local) + + model._aspec_checkpoint_state, model._aspec_load_checkpoint_state = state, load + return model + + def _normalize_accumulated_gradients(self, local_denominator, accumulation_steps): + flat = [p.grad.view(-1) for p in self._partial_norm_parameters] + torch._foreach_copy_(self._partial_views, flat) + dist.all_reduce(self._partial_buffer, group=self.tp_group) + torch._foreach_copy_(flat, self._partial_views) + return super()._normalize_accumulated_gradients(local_denominator, accumulation_steps) + + def pop_committed_work(self, optimizer_step): + local = super().pop_committed_work(optimizer_step) + participants = [None] * self.tp_size + dist.all_gather_object(participants, local, group=self.tp_group) + if any(work != local for work in participants): + raise RuntimeError("Torch TP participants committed different work") + return local if self.tp_rank == 0 else [] + + def _checkpoint_backend_rng(self): + return {"backend": "torch-tp-validation", "tp_size": self.tp_size} + + def _restore_backend_rng(self, state): + if state != self._checkpoint_backend_rng(): + raise RuntimeError("Torch TP validation resume cannot change backend or TP size")