diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 6dea683618..9a7543fce0 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -3273,3 +3273,319 @@ def run(x, module=model): forward=case in ("bias", "unfused"), backward=case != "linear" and bool(targets), ) + + +def _ops_activation_reference( + name, + x, + *, + beta1=4.0, + beta2=25.0, + limit=7.0, + alpha=1.702, + glu_linear_offset=1.0, + interleave_size=None, +): + gate = None + if name in ("GLU", "GEGLU", "QGEGLU", "ReGLU", "SReGLU", "SwiGLU", "SiTUGLU", "ClampedSwiGLU"): + if interleave_size is None: + x, gate = x.chunk(2, dim=-1) + else: + x, gate = x.reshape(*x.shape[:-1], -1, 2, interleave_size).unbind(-2) + x, gate = x.flatten(-2), gate.flatten(-2) + if name in ("GELU", "GEGLU"): + x = torch.nn.functional.gelu(x, approximate="tanh") + elif name in ("QGELU", "QGEGLU"): + x = x * torch.sigmoid(1.702 * x) + elif name in ("ReLU", "ReGLU"): + x = x.relu() + elif name in ("SReLU", "SReGLU"): + x = x.relu().square() + elif name in ("SiLU", "SwiGLU"): + x = torch.nn.functional.silu(x) + elif name == "SiTUGLU": + x = beta1 * torch.tanh(x / beta1) * x.sigmoid() + gate = beta2 * torch.tanh(gate / beta2) + elif name == "ClampedSwiGLU": + x = x.clamp(max=limit) + x = x * torch.sigmoid(alpha * x) + gate = gate.clamp(min=-limit, max=limit) + glu_linear_offset + elif name == "GLU": + x = x.sigmoid() + else: + raise ValueError(name) + return x if gate is None else x * gate + + +def _check_te_ops_activation_compile( + activation, + *, + dtype=torch.float32, + layout="contiguous", + cache_input=False, + autocast=False, + training=True, + interleave_size=None, + dynamic=False, + activation_kwargs=None, +): + if (layout == "fp8" or cache_input) and not fp8_available: + pytest.skip(reason_for_no_fp8) + torch._dynamo.reset() + activation_kwargs = activation_kwargs or {} + kwargs = {"cache_quantized_input": cache_input, **activation_kwargs} + if interleave_size is not None: + kwargs["glu_interleave_size"] = interleave_size + model = te.ops.Sequential(getattr(te.ops, activation)(**kwargs)) + + def run(x): + with torch.autocast("cuda", dtype=torch.bfloat16, enabled=autocast): + return model(x) + + compiled, graphs = _compile_with_graphs(run) + eager_reference = autocast or dynamic + tols = {} if eager_reference else dtype_tols(dtype) + for step, rows in enumerate((16, 16, 32, 48) if dynamic else (32,)): + shape = (rows, 64) if dynamic else (2, 16, 64) + x = torch.randn(shape, device="cuda", dtype=dtype) + if layout == "fp8" or (cache_input and not eager_reference): + quantizer = Float8CurrentScalingQuantizer(tex.DType.kFloat8E4M3, device="cuda") + x = quantizer(x) + if layout != "fp8": + x = x.dequantize() + if layout == "strided": + x = x.transpose(0, 1) + x = x.detach().requires_grad_(training) + if dynamic: + torch._dynamo.mark_dynamic(x, 0) + ref_x = ( + x.detach().clone() + if eager_reference + else (x.dequantize() if layout == "fp8" else x).detach().double() + ).requires_grad_(training) + with torch.set_grad_enabled(training): + if layout == "strided": + with pytest.raises( + Exception, match="requires contiguous inputs under torch.compile" + ): + compiled(x) + actual = run(x) + else: + actual = compiled(x) + expected = ( + run(ref_x) + if eager_reference + else _ops_activation_reference( + activation, ref_x, interleave_size=interleave_size, **activation_kwargs + ) + ) + torch.testing.assert_close(actual.to(expected.dtype), expected, **tols) + if training: + dy = torch.randn_like(actual) + got = torch.autograd.grad(actual, x, dy)[0] + want = torch.autograd.grad(expected, ref_x, dy.to(expected.dtype))[0] + torch.testing.assert_close(got.to(want.dtype), want, **tols) + if autocast: + assert actual.dtype == torch.bfloat16 + torch.testing.assert_close(x, ref_x) + if dynamic: + if step == 1: + graph_count = len(graphs) + elif step > 1: + assert len(graphs) == graph_count + if layout != "strided": + _assert_custom_ops(graphs, activation.lower(), backward=training) + + +@pytest.mark.parametrize( + "activation", + [ + "GELU", + "GEGLU", + "GLU", + "QGELU", + "QGEGLU", + "ReLU", + "ReGLU", + "SReLU", + "SReGLU", + "SiLU", + "SwiGLU", + "SiTUGLU", + "ClampedSwiGLU", + ], +) +def test_te_ops_activation_compile(activation): + _check_te_ops_activation_compile(activation, dtype=torch.bfloat16) + + +@pytest.mark.parametrize("activation", ["GELU", "GEGLU", "SwiGLU"]) +@pytest.mark.parametrize( + "dtype,layout,cache_input", + [ + (torch.float32, "contiguous", False), + (torch.float16, "contiguous", False), + (torch.bfloat16, "strided", False), + (torch.bfloat16, "fp8", False), + (torch.bfloat16, "contiguous", True), + (torch.bfloat16, "fp8", True), + ], +) +def test_te_ops_activation_variants_compile(activation, dtype, layout, cache_input): + _check_te_ops_activation_compile( + activation, dtype=dtype, layout=layout, cache_input=cache_input + ) + + +@pytest.mark.parametrize("activation", ["GELU", "SwiGLU", "SiTUGLU", "ClampedSwiGLU"]) +@pytest.mark.parametrize("cache_input", [False, True]) +@pytest.mark.parametrize("training", [False, True]) +def test_te_ops_activation_autocast_compile(activation, cache_input, training): + _check_te_ops_activation_compile( + activation, cache_input=cache_input, autocast=True, training=training + ) + + +@pytest.mark.parametrize("activation", ["SwiGLU", "SiTUGLU", "ClampedSwiGLU"]) +@pytest.mark.parametrize("cache_input", [False, True]) +@pytest.mark.parametrize("interleave_size", [None, 4]) +def test_te_ops_swiglu_dynamic_compile(activation, cache_input, interleave_size): + _check_te_ops_activation_compile( + activation, cache_input=cache_input, interleave_size=interleave_size, dynamic=True + ) + + +@pytest.mark.parametrize("beta1,beta2", [(0.75, 1.5), (3.0, 12.0)]) +def test_te_ops_situglu_parameters_compile(beta1, beta2): + _check_te_ops_activation_compile( + "SiTUGLU", + cache_input=True, + interleave_size=4, + activation_kwargs={"beta1": beta1, "beta2": beta2}, + ) + + +@pytest.mark.parametrize("limit,alpha,glu_linear_offset", [(0.75, 1.3, 0.0), (1.25, 2.0, 0.5)]) +def test_te_ops_clamped_swiglu_parameters_compile(limit, alpha, glu_linear_offset): + _check_te_ops_activation_compile( + "ClampedSwiGLU", + cache_input=True, + interleave_size=4, + activation_kwargs={"limit": limit, "alpha": alpha, "glu_linear_offset": glu_linear_offset}, + ) + + +@pytest.mark.parametrize("quantization", [None, "fp8", "nvfp4", "nvfp4_no_rht"]) +@pytest.mark.parametrize( + "activation,training,mode", + [ + ("GELU", True, "default"), + ("ReLU", True, "default"), + ("GEGLU", True, "default"), + ("SwiGLU", True, "default"), + ("SiTUGLU", True, "default"), + ("ClampedSwiGLU", True, "default"), + ("GELU", False, "default"), + ("SwiGLU", False, "default"), + ("GELU", True, "reduce-overhead"), + ("SwiGLU", True, "reduce-overhead"), + ], +) +def test_te_ops_activation_mlp_compile(activation, quantization, training, mode, monkeypatch): + nvfp4 = quantization in ("nvfp4", "nvfp4_no_rht") + if nvfp4 and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) + if quantization == "fp8" and not fp8_available: + pytest.skip(reason_for_no_fp8) + dtype = torch.float16 if quantization is None else torch.bfloat16 + tols = {"rtol": 0, "atol": 0} if nvfp4 or not training else {} + torch._dynamo.reset() + counters.clear() + monkeypatch.setattr(torch.backends.cuda.matmul, "allow_tf32", False) + in_features, hidden = (128, 128) if nvfp4 else (32, 64) + gated = activation in ("GEGLU", "SwiGLU", "SiTUGLU", "ClampedSwiGLU") + cache_input = training and activation == "GELU" and quantization == "fp8" + rows = 64 if activation == "GELU" and quantization == "nvfp4" else 32 + model = ( + te.ops.Sequential( + te.ops.Linear(in_features, hidden * (2 if gated else 1), device="cuda", dtype=dtype), + getattr(te.ops, activation)(cache_quantized_input=cache_input), + te.ops.Linear(hidden, in_features, device="cuda", dtype=dtype), + ) + .train(training) + .requires_grad_(training) + ) + if training: + with torch.no_grad(): + for param in model.parameters(): + param.copy_(torch.randint(0, 3, param.shape, device=param.device) / 32) + eager_model = copy.deepcopy(model) + if nvfp4: + quant_recipe = recipe.NVFP4BlockScaling( + disable_rht=quantization == "nvfp4_no_rht", disable_stochastic_rounding=True + ) + else: + quant_recipe = recipe.Float8CurrentScaling(backward_override=None) if quantization else None + + def run(x, module=model): + if quant_recipe is None: + return module(x) + with te.autocast(recipe=quant_recipe): + return module(x) + + if mode == "default": + compiled, graphs = _compile_with_graphs(run) + else: + compiled, graphs = torch.compile(run, fullgraph=True, mode=mode), [] + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + eager_optimizer = torch.optim.SGD(eager_model.parameters(), lr=0.01) + for step in range(4 if training else 1): + torch.compiler.cudagraph_mark_step_begin() + shape = (2, rows // 2, in_features) if training else (rows, in_features) + x = torch.randn(shape, device="cuda", dtype=dtype, requires_grad=training) + if training and step == 0: + # Keep the independent reference reductions well-conditioned. + with torch.no_grad(): + x.copy_(torch.randint(0, 9, x.shape, device=x.device) / 8) + eager_x = x.detach().clone().requires_grad_(training) + with torch.set_grad_enabled(training): + actual = compiled(x) + expected = run(eager_x, eager_model) + torch.testing.assert_close(actual, expected, **tols) + if not training: + continue + dy = torch.randn_like(actual) + if step == 0: + dy.copy_(torch.randint(0, 9, dy.shape, device=dy.device) / 8) + actual.backward(dy) + expected.backward(dy) + torch.testing.assert_close(x.grad, eager_x.grad, **tols) + for param, reference in zip(model.parameters(), eager_model.parameters()): + torch.testing.assert_close(param.grad, reference.grad, **tols) + if quantization is None and step == 0: + ref_x = x.detach().clone().requires_grad_() + params = [p.detach().clone().requires_grad_() for p in model.parameters()] + ref = torch.nn.functional.linear(ref_x.float(), *[p.float() for p in params[:2]]) + ref = _ops_activation_reference(activation, ref.to(dtype).float()).to(dtype) + ref = torch.nn.functional.linear(ref.float(), *[p.float() for p in params[2:]]) + ref = ref.to(dtype) + grads = torch.autograd.grad(ref, (ref_x, *params), dy) + torch.testing.assert_close(actual, ref, **dtype_tols(dtype)) + for got, want in zip((x.grad, *(p.grad for p in model.parameters())), grads): + torch.testing.assert_close(got, want, **dtype_tols(dtype)) + optimizer.step() + eager_optimizer.step() + optimizer.zero_grad(set_to_none=True) + eager_optimizer.zero_grad(set_to_none=True) + if step == 1: + graph_count = len(graphs) + elif step > 1: + assert len(graphs) == graph_count + del actual, expected, x, eager_x + if mode == "default": + fused = training and quantization is not None and activation in ("GELU", "ReLU") + _assert_custom_ops(graphs, activation.lower(), backward=training and not fused) + _assert_custom_ops(graphs, "backwardactivationbias", forward=False, backward=fused) + else: + assert not counters["inductor"]["cudagraph_skips"], counters["inductor"] + assert counters["inductor"]["cudagraph_recorded_non_static_inputs"] > 0 diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index f3621f9e12..f15e28700d 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -7,24 +7,55 @@ from __future__ import annotations from typing import Optional +import torch + from .tensor_spec import TensorSpec from ..quantized_tensor import Quantizer +def fake_activation( + input_: TensorSpec, + quantizer: Optional[Quantizer] = None, + *, + gated: bool = False, + dtype: Optional[torch.dtype] = None, +) -> TensorSpec: + """Describe an activation output, including optional quantization.""" + shape = (*input_.shape[:-1], input_.shape[-1] // 2 if gated else input_.shape[-1]) + dtype = dtype or input_.dtype + return TensorSpec(shape=shape, dtype=dtype, device=input_.device, quantizer=quantizer) + + +def fake_activation_backward( + input_: TensorSpec, + quantizer: Optional[Quantizer] = None, + *, + dtype: Optional[torch.dtype] = None, +) -> TensorSpec: + """Describe the gradient of the original, pre-activation input.""" + return TensorSpec( + shape=input_.shape, + dtype=dtype or input_.dtype, + device=input_.device, + quantizer=quantizer, + ) + + def fake_bgrad( grad_output: TensorSpec, quantizer: Optional[Quantizer] = None, + *, + dtype: Optional[torch.dtype] = None, ) -> tuple[TensorSpec, TensorSpec]: """Describe bias reduction and optional gradient quantization, in native order.""" - db = TensorSpec( - shape=(grad_output.shape[-1],), dtype=grad_output.dtype, device=grad_output.device - ) + dtype = dtype or grad_output.dtype + db = TensorSpec(shape=(grad_output.shape[-1],), dtype=dtype, device=grad_output.device) dx = ( grad_output - if quantizer is None + if quantizer is None and dtype == grad_output.dtype else TensorSpec( shape=grad_output.shape, - dtype=grad_output.dtype, + dtype=dtype, device=grad_output.device, quantizer=quantizer, ) diff --git a/transformer_engine/pytorch/ops/basic/activation.py b/transformer_engine/pytorch/ops/basic/activation.py index 21a376ca3d..09511ef076 100644 --- a/transformer_engine/pytorch/ops/basic/activation.py +++ b/transformer_engine/pytorch/ops/basic/activation.py @@ -8,17 +8,20 @@ import abc import math from collections.abc import Iterable, Sequence +from dataclasses import dataclass from typing import Any, Optional import torch import transformer_engine_torch as tex from ...constants import DType +from ...dynamo import TensorSpec, TensorOrQuantized +from ...dynamo.fake import fake_activation, fake_activation_backward from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload from ...tensor.float8_tensor import Float8CurrentScalingQuantizer, Quantizer from ...utils import _compile_safe_warn, clear_tensor_data from ..op import BasicOperation, OperationContext -from .._common import maybe_autocast_dtype, maybe_dequantize +from .._common import is_quantized_tensor, maybe_autocast_dtype, maybe_dequantize __all__ = [ "GELU", @@ -36,6 +39,28 @@ ] +@dataclass(slots=True) +class ActivationFwdArgs: + """Activation inputs and quantization configuration.""" + + input_: TensorOrQuantized + dtype: torch.dtype + output_quantizer: Optional[Quantizer] + input_quantizer: Optional[Quantizer] + grad_input_quantizer: Optional[Quantizer] + requires_grad: bool + + +@dataclass(slots=True) +class ActivationBwdArgs: + """Activation gradient and saved input.""" + + grad_output: TensorOrQuantized + input_: TensorOrQuantized + dtype: torch.dtype + grad_input_quantizer: Optional[Quantizer] + + class _ActivationOperation(BasicOperation, metaclass=abc.ABCMeta): r"""Apply activation function @@ -69,79 +94,132 @@ def __init__(self, *, cache_quantized_input: bool = False): super().__init__() self.cache_quantized_input: bool = cache_quantized_input + @staticmethod @abc.abstractmethod - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: """Forward implementation Implementation from transformer_engine.pytorch.cpp_extensions. """ + @staticmethod @abc.abstractmethod - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: """Backward implementation Implementation from transformer_engine_torch. """ - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: + fwd_args_type = ActivationFwdArgs + bwd_args_type = ActivationBwdArgs + _output_halves_last_dim: bool = False - # Compute dtype - dtype: torch.dtype = maybe_autocast_dtype(default_dtype=input_.dtype) + def compile_unsupported_reason(self, mode: str) -> Optional[str]: + if is_cpu_offload_enabled(): + return "CPU offloading" + return super().compile_unsupported_reason(mode) + + def pack_forward_args( + self, + basic_op_ctxs, + input_, + *, + prev_op_grad_output_quantizer, + next_op_input_quantizer, + basic_op_kwargs, + **unused, # pylint: disable=unused-argument + ) -> ActivationFwdArgs: + if basic_op_kwargs[0]: + raise ValueError("Activation forward does not expect keyword arguments") + if ( + torch.compiler.is_compiling() + and not is_quantized_tensor(input_) + and not input_.is_contiguous() + ): + raise RuntimeError("Activation forward requires contiguous inputs under torch.compile") + dtype = maybe_autocast_dtype(default_dtype=input_.dtype) if dtype not in (torch.float32, torch.float16, torch.bfloat16): raise RuntimeError(f"Unsupported dtype ({dtype})") - - # Check input tensor - x = maybe_dequantize(input_.contiguous(), dtype) - - # Launch kernel - y = self._activation_forward_impl(x, next_op_input_quantizer) - - # Quantize input to FP8 before caching if needed + input_quantizer = None if self.cache_quantized_input: - input_quantizer = Float8CurrentScalingQuantizer(DType.kFloat8E4M3, x.device) + input_quantizer = Float8CurrentScalingQuantizer(DType.kFloat8E4M3, input_.device) input_quantizer.set_usage(rowwise=True, columnwise=False) - x = input_quantizer(x) - - # Save state for backward pass - if ctx.requires_grad: - if is_cpu_offload_enabled(): - mark_activation_offload(x) - ctx.save_for_backward(x) - ctx.dtype = dtype - ctx.prev_op_grad_output_quantizer = prev_op_grad_output_quantizer - - return y - - def op_backward( - self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, tuple[()]]: - - # Saved tensors from forward pass - (x,) = ctx.saved_tensors - - # Check input tensor - x = maybe_dequantize(x.contiguous(), ctx.dtype) - - # Check grad output tensor - dy = maybe_dequantize(grad_output.contiguous(), x.dtype) + input_quantizer.internal = True + return self.fwd_args_type( + input_, + dtype, + next_op_input_quantizer, + input_quantizer, + prev_op_grad_output_quantizer, + basic_op_ctxs[0].requires_grad, + ) - # Launch kernel - dx = self._activation_backward_impl(dy, x, ctx.prev_op_grad_output_quantizer) + @classmethod + def forward_compute( + cls, args: ActivationFwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument + x = maybe_dequantize(args.input_, args.dtype).contiguous() + y = cls._activation_forward_impl(x, args.output_quantizer) + if args.input_quantizer is not None: + x = args.input_quantizer(x) + return y, [()], (x,) if args.requires_grad and x is not args.input_ else () + + @classmethod + def forward_compute_fake(cls, args: ActivationFwdArgs): + x = args.input_ + y = fake_activation( + x, + args.output_quantizer, + dtype=args.dtype, + gated=cls._output_halves_last_dim, + ) + saved = () + if args.requires_grad and ( + args.input_quantizer is not None or is_quantized_tensor(x) or x.dtype != args.dtype + ): + saved = ( + TensorSpec( + shape=x.shape, dtype=args.dtype, device=x.device, quantizer=args.input_quantizer + ), + ) + return y, [()], saved - # Clear input tensor if possible - clear_tensor_data(x) + def forward_setup_context(self, basic_op_ctxs, args, aux) -> None: + ctx = basic_op_ctxs[0] + x = aux[0] if aux else args.input_ + if is_cpu_offload_enabled(): + mark_activation_offload(x) + ctx.save_for_backward(x) + ctx.dtype = args.dtype + ctx.prev_op_grad_output_quantizer = args.grad_input_quantizer + + def pack_backward_args( + self, basic_op_ctxs, grad_output, **unused # pylint: disable=unused-argument + ) -> ActivationBwdArgs: + ctx = basic_op_ctxs[0] + return self.bwd_args_type( + grad_output, ctx.saved_tensors[0], ctx.dtype, ctx.prev_op_grad_output_quantizer + ) - return dx, () + @classmethod + def backward_compute(cls, args: ActivationBwdArgs, *, in_custom_op: bool = False): + x = maybe_dequantize(args.input_, args.dtype).contiguous() + dy = maybe_dequantize(args.grad_output, args.dtype).contiguous() + dx = cls._activation_backward_impl(dy, x, args.grad_input_quantizer) + if not in_custom_op: + clear_tensor_data(x) + return dx, [()], [()] + + @classmethod + def backward_compute_fake(cls, args: ActivationBwdArgs): + dx = fake_activation_backward( + args.input_, + args.grad_input_quantizer, + dtype=args.dtype, + ) + return dx, [()], [()] class GELU(_ActivationOperation): @@ -157,10 +235,12 @@ class GELU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.gelu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dgelu(*args, **kwargs) @@ -189,10 +269,14 @@ class GLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.glu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dglu(*args, **kwargs) @@ -224,10 +308,14 @@ class GEGLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.geglu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dgeglu(*args, **kwargs) @@ -243,10 +331,12 @@ class QGELU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.qgelu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dqgelu(*args, **kwargs) @@ -276,10 +366,14 @@ class QGEGLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.qgeglu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dqgeglu(*args, **kwargs) @@ -292,10 +386,12 @@ class ReLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.relu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.drelu(*args, **kwargs) @@ -321,10 +417,14 @@ class ReGLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.reglu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dreglu(*args, **kwargs) @@ -339,10 +439,12 @@ class SReLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.srelu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dsrelu(*args, **kwargs) @@ -592,10 +694,14 @@ class SReGLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.sreglu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dsreglu(*args, **kwargs) @@ -608,8 +714,10 @@ class SiLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.silu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dsilu(*args, **kwargs) diff --git a/transformer_engine/pytorch/ops/basic/bias.py b/transformer_engine/pytorch/ops/basic/bias.py index c5a9dda4f6..7d012c8eed 100644 --- a/transformer_engine/pytorch/ops/basic/bias.py +++ b/transformer_engine/pytorch/ops/basic/bias.py @@ -15,6 +15,7 @@ from ...dynamo import TensorOrQuantized, TensorSpec from ...dynamo.fake import fake_bgrad from ...quantization import FP8GlobalStateManager +from .._common import is_quantized_tensor from ..op import BasicOperation from ...utils import canonicalize_device, canonicalize_dtype from ...tensor import Quantizer @@ -143,6 +144,13 @@ def pack_forward_args( ) -> BiasFwdArgs: if basic_op_kwargs[0]: raise ValueError("Bias forward does not expect keyword arguments") + if torch.compiler.is_compiling() and ( + (not is_quantized_tensor(input_) and not input_.is_contiguous()) + or not self.bias.is_contiguous() + ): + raise RuntimeError( + "Bias forward requires contiguous inputs and bias under torch.compile" + ) quantizer = prev_op_grad_output_quantizer if FP8GlobalStateManager.is_fp8_enabled(): if FP8GlobalStateManager.get_fp8_recipe().backward_override is not None: diff --git a/transformer_engine/pytorch/ops/basic/swiglu.py b/transformer_engine/pytorch/ops/basic/swiglu.py index 71ff010ae5..1162b2d31f 100644 --- a/transformer_engine/pytorch/ops/basic/swiglu.py +++ b/transformer_engine/pytorch/ops/basic/swiglu.py @@ -7,17 +7,18 @@ from __future__ import annotations from collections.abc import Iterable, Sequence import math +from dataclasses import dataclass from typing import Any, Optional import torch import transformer_engine_torch as tex -from ...constants import DType from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload -from ...tensor import Float8CurrentScalingQuantizer, Quantizer +from ...tensor import Quantizer from ...utils import clear_tensor_data from ..op import BasicOperation, OperationContext from .._common import maybe_autocast_dtype, maybe_dequantize +from .activation import _ActivationOperation, ActivationFwdArgs, ActivationBwdArgs __all__ = [ "SwiGLU", @@ -29,7 +30,62 @@ ] -class SwiGLU(BasicOperation): +@dataclass(slots=True) +class SwiGLUFwdArgs(ActivationFwdArgs): + """Activation configuration with the GLU input layout.""" + + glu_interleave_size: Optional[int] = None + + +@dataclass(slots=True) +class SwiGLUBwdArgs(ActivationBwdArgs): + """Activation gradient with the GLU input layout.""" + + glu_interleave_size: Optional[int] = None + + +@dataclass(slots=True) +class SiTUGLUFwdArgs(SwiGLUFwdArgs): + """SwiGLU inputs with gate and up-branch soft caps.""" + + beta1: float = 4.0 + beta2: float = 25.0 + + +@dataclass(slots=True) +class SiTUGLUBwdArgs(SwiGLUBwdArgs): + """SwiGLU gradients with gate and up-branch soft caps.""" + + beta1: float = 4.0 + beta2: float = 25.0 + + +@dataclass(slots=True) +class ClampedSwiGLUFwdArgs(SwiGLUFwdArgs): + """SwiGLU inputs with clipping and gate parameters.""" + + limit: float = 7.0 + alpha: float = 1.702 + glu_linear_offset: float = 1.0 + + +@dataclass(slots=True) +class ClampedSwiGLUBwdArgs(SwiGLUBwdArgs): + """SwiGLU gradients with clipping and gate parameters.""" + + limit: float = 7.0 + alpha: float = 1.702 + glu_linear_offset: float = 1.0 + + +def _interleave_glu(tensor: torch.Tensor, size: int, *, reverse: bool = False) -> torch.Tensor: + shape = tensor.shape + blocks = shape[-1] // (2 * size) + middle = (blocks, 2) if reverse else (2, blocks) + return tensor.reshape(-1, *middle, size).transpose(1, 2).contiguous().view(shape) + + +class SwiGLU(_ActivationOperation): r"""Swish gated linear unit The input tensor is split into chunks :math:``a`` and :math:``b`` @@ -88,125 +144,62 @@ def __init__( self.cache_quantized_input: bool = cache_quantized_input self.glu_interleave_size: Optional[int] = glu_interleave_size - def _tex_swiglu_forward( - self, - input_: torch.Tensor, - quantizer: Optional[Quantizer], - ) -> torch.Tensor: - """Call the Transformer Engine SwiGLU forward kernel.""" + fwd_args_type = SwiGLUFwdArgs + bwd_args_type = SwiGLUBwdArgs + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(input_, quantizer, **_kwargs) -> torch.Tensor: return tex.swiglu(input_, quantizer) - def _tex_swiglu_backward( - self, - grad_output: torch.Tensor, - input_: torch.Tensor, - quantizer: Optional[Quantizer], - ) -> torch.Tensor: - """Call the Transformer Engine SwiGLU backward kernel.""" + @staticmethod + def _activation_backward_impl(grad_output, input_, quantizer, **_kwargs) -> torch.Tensor: return tex.dswiglu(grad_output, input_, quantizer) - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: - - # Compute dtype - dtype: torch.dtype = maybe_autocast_dtype(default_dtype=input_.dtype) - if dtype not in (torch.float32, torch.float16, torch.bfloat16): - raise RuntimeError(f"Unsupported dtype ({dtype})") - - # Check input tensor - input_ = maybe_dequantize(input_.contiguous(), dtype) - - # Remove interleaving if needed - swiglu_in = input_ - if self.glu_interleave_size is not None: - shape = swiglu_in.size() - swiglu_in = swiglu_in.reshape( - -1, - shape[-1] // (2 * self.glu_interleave_size), - 2, - self.glu_interleave_size, - ) - swiglu_in = swiglu_in.transpose(1, 2).contiguous() - swiglu_in = swiglu_in.view(shape) + def pack_forward_args(self, *args, **kwargs) -> SwiGLUFwdArgs: + args = super().pack_forward_args(*args, **kwargs) + args.glu_interleave_size = self.glu_interleave_size + return args - # Launch kernel - out = self._tex_swiglu_forward(swiglu_in, next_op_input_quantizer) - - # Quantize input to FP8 before caching if needed - if self.cache_quantized_input: - input_quantizer = Float8CurrentScalingQuantizer( - DType.kFloat8E4M3, - input_.device, - ) - input_quantizer.set_usage(rowwise=True, columnwise=False) - input_ = input_quantizer(input_) - - # Save state for backward pass - if ctx.requires_grad: - if is_cpu_offload_enabled(): - mark_activation_offload(input_) - ctx.save_for_backward(input_) - ctx.dtype = dtype - ctx.prev_op_grad_output_quantizer = prev_op_grad_output_quantizer - - return out - - def op_backward( - self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, tuple[()]]: - - # Saved tensors from forward pass - (input_,) = ctx.saved_tensors - - # Make sure tensors have correct dtypes - x = maybe_dequantize(input_.contiguous(), ctx.dtype) - dy = maybe_dequantize(grad_output.contiguous(), ctx.dtype) - - # Remove interleaving if needed + @classmethod + def forward_compute( + cls, args: SwiGLUFwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument + x = maybe_dequantize(args.input_, args.dtype).contiguous() swiglu_in = x - if self.glu_interleave_size is not None: - shape = swiglu_in.size() - swiglu_in = swiglu_in.reshape( - -1, - shape[-1] // (2 * self.glu_interleave_size), - 2, - self.glu_interleave_size, - ) - swiglu_in = swiglu_in.transpose(1, 2).contiguous() - swiglu_in = swiglu_in.view(shape) - - # Quantizer for grad input - quantizer = ctx.prev_op_grad_output_quantizer - if self.glu_interleave_size is not None: + if args.glu_interleave_size is not None: + swiglu_in = _interleave_glu(x, args.glu_interleave_size, reverse=True) + out = cls._activation_forward_impl(swiglu_in, args.output_quantizer, args=args) + if args.input_quantizer is not None: + x = args.input_quantizer(x) + return out, [()], (x,) if args.requires_grad and x is not args.input_ else () + + def pack_backward_args(self, *args, **kwargs) -> SwiGLUBwdArgs: + args = super().pack_backward_args(*args, **kwargs) + args.glu_interleave_size = self.glu_interleave_size + return args + + @classmethod + def backward_compute(cls, args: SwiGLUBwdArgs, *, in_custom_op: bool = False): + x = maybe_dequantize(args.input_, args.dtype).contiguous() + dy = maybe_dequantize(args.grad_output, args.dtype).contiguous() + quantizer = args.grad_input_quantizer + if args.glu_interleave_size is not None: + x = _interleave_glu(x, args.glu_interleave_size, reverse=True) quantizer = None + dx = cls._activation_backward_impl(dy, x, quantizer, args=args) + if args.glu_interleave_size is not None: + dx = _interleave_glu(dx, args.glu_interleave_size) + if not in_custom_op: + clear_tensor_data(args.input_) + return dx, [()], [()] - # Launch kernel - grad_swiglu_in = self._tex_swiglu_backward(dy, swiglu_in, quantizer) - - # Apply interleaving if needed - dx = grad_swiglu_in - if self.glu_interleave_size is not None: - shape = dx.size() - dx = dx.reshape( - -1, - 2, - shape[-1] // (2 * self.glu_interleave_size), - self.glu_interleave_size, - ) - dx = dx.transpose(1, 2).contiguous() - dx = dx.view(shape) - - # Clear input tensor if possible - clear_tensor_data(input_) - - return dx, () + @classmethod + def backward_compute_fake(cls, args: SwiGLUBwdArgs): + dx, params, extras = super().backward_compute_fake(args) + if args.glu_interleave_size is not None: + dx.quantizer = None + return dx, params, extras class SiTUGLU(SwiGLU): @@ -234,6 +227,9 @@ class SiTUGLU(SwiGLU): Block-interleaved GLU layout, as in :class:`SwiGLU`. """ + fwd_args_type = SiTUGLUFwdArgs + bwd_args_type = SiTUGLUBwdArgs + def __init__( self, *, @@ -253,23 +249,39 @@ def __init__( if not math.isfinite(self.beta2) or self.beta2 <= 0.0: raise ValueError(f"beta2 must be finite and positive, got {self.beta2}") - def _tex_swiglu_forward( - self, + @staticmethod + def _activation_forward_impl( input_: torch.Tensor, quantizer: Optional[Quantizer], + *, + args: SiTUGLUFwdArgs, ) -> torch.Tensor: - return tex.situglu(input_, quantizer, self.beta1, self.beta2) + return tex.situglu(input_, quantizer, args.beta1, args.beta2) - def _tex_swiglu_backward( - self, + @staticmethod + def _activation_backward_impl( grad_output: torch.Tensor, input_: torch.Tensor, quantizer: Optional[Quantizer], + *, + args: SiTUGLUBwdArgs, ) -> torch.Tensor: - return tex.dsituglu(grad_output, input_, quantizer, self.beta1, self.beta2) + return tex.dsituglu(grad_output, input_, quantizer, args.beta1, args.beta2) + + def pack_forward_args(self, *args, **kwargs) -> SiTUGLUFwdArgs: + args = super().pack_forward_args(*args, **kwargs) + args.beta1 = self.beta1 + args.beta2 = self.beta2 + return args + def pack_backward_args(self, *args, **kwargs) -> SiTUGLUBwdArgs: + args = super().pack_backward_args(*args, **kwargs) + args.beta1 = self.beta1 + args.beta2 = self.beta2 + return args -class ClampedSwiGLU(BasicOperation): + +class ClampedSwiGLU(SwiGLU): r"""GPT-OSS Implementation based on `GPT-OSS `__. @@ -302,6 +314,9 @@ class ClampedSwiGLU(BasicOperation): """ + fwd_args_type = ClampedSwiGLUFwdArgs + bwd_args_type = ClampedSwiGLUBwdArgs + def __init__( self, *, @@ -311,142 +326,48 @@ def __init__( cache_quantized_input: bool = False, glu_interleave_size: Optional[int] = None, ): - super().__init__() + super().__init__( + cache_quantized_input=cache_quantized_input, + glu_interleave_size=glu_interleave_size, + ) self.limit: float = limit self.alpha: float = alpha self.glu_linear_offset: float = glu_linear_offset - self.cache_quantized_input: bool = cache_quantized_input - self.glu_interleave_size: Optional[int] = glu_interleave_size - def _tex_clamped_swiglu_forward( - self, - swiglu_in: torch.Tensor, - next_op_input_quantizer: Optional[Quantizer], + @staticmethod + def _activation_forward_impl( + input_: torch.Tensor, + quantizer: Optional[Quantizer], + *, + args: ClampedSwiGLUFwdArgs, ) -> torch.Tensor: - """Call :func:`tex.clamped_swiglu` with this op's ``limit`` / ``alpha`` / ``glu_linear_offset``.""" - return tex.clamped_swiglu( - swiglu_in, - next_op_input_quantizer, - self.limit, - self.alpha, - self.glu_linear_offset, - ) + return tex.clamped_swiglu(input_, quantizer, args.limit, args.alpha, args.glu_linear_offset) - def _tex_clamped_dswiglu( - self, - dy: torch.Tensor, - swiglu_in: torch.Tensor, + @staticmethod + def _activation_backward_impl( + grad_output: torch.Tensor, + input_: torch.Tensor, quantizer: Optional[Quantizer], + *, + args: ClampedSwiGLUBwdArgs, ) -> torch.Tensor: - """Call :func:`tex.clamped_dswiglu` with this op's ``limit`` / ``alpha`` / ``glu_linear_offset``.""" return tex.clamped_dswiglu( - dy, - swiglu_in, - quantizer, - self.limit, - self.alpha, - self.glu_linear_offset, + grad_output, input_, quantizer, args.limit, args.alpha, args.glu_linear_offset ) - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: - - # Compute dtype - dtype: torch.dtype = maybe_autocast_dtype(default_dtype=input_.dtype) - if dtype not in (torch.float32, torch.float16, torch.bfloat16): - raise RuntimeError(f"Unsupported dtype ({dtype})") - - # Check input tensor - x = maybe_dequantize(input_.contiguous(), dtype) - - # Remove interleaving if needed - swiglu_in = x - if self.glu_interleave_size is not None: - shape = swiglu_in.size() - swiglu_in = swiglu_in.reshape( - -1, - shape[-1] // (2 * self.glu_interleave_size), - 2, - self.glu_interleave_size, - ) - swiglu_in = swiglu_in.transpose(1, 2).contiguous() - swiglu_in = swiglu_in.view(shape) - - # Launch kernel - out = self._tex_clamped_swiglu_forward(swiglu_in, next_op_input_quantizer) - - # Quantize input to FP8 before caching if needed - if self.cache_quantized_input: - input_quantizer = Float8CurrentScalingQuantizer(DType.kFloat8E4M3, x.device) - input_quantizer.set_usage(rowwise=True, columnwise=False) - x = input_quantizer(x) - - # Save state for backward pass - if ctx.requires_grad: - if is_cpu_offload_enabled(): - mark_activation_offload(x) - ctx.save_for_backward(x) - ctx.dtype = dtype - ctx.prev_op_grad_output_quantizer = prev_op_grad_output_quantizer - - return out - - def op_backward( - self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, tuple[()]]: - - # Saved tensors from forward pass - (input_,) = ctx.saved_tensors - - # Make sure tensors have correct dtypes - x = maybe_dequantize(input_.contiguous(), ctx.dtype) - dy = maybe_dequantize(grad_output.contiguous(), ctx.dtype) - - # Remove interleaving if needed - swiglu_in = x - if self.glu_interleave_size is not None: - shape = swiglu_in.size() - swiglu_in = swiglu_in.reshape( - -1, - shape[-1] // (2 * self.glu_interleave_size), - 2, - self.glu_interleave_size, - ) - swiglu_in = swiglu_in.transpose(1, 2).contiguous() - swiglu_in = swiglu_in.view(shape) - - # Quantizer for grad input - quantizer = ctx.prev_op_grad_output_quantizer - if self.glu_interleave_size is not None: - quantizer = None - - # Launch kernel - grad_swiglu_in = self._tex_clamped_dswiglu(dy, swiglu_in, quantizer) - - # Apply interleaving if needed - dx = grad_swiglu_in - if self.glu_interleave_size is not None: - shape = dx.size() - dx = dx.reshape( - -1, - 2, - shape[-1] // (2 * self.glu_interleave_size), - self.glu_interleave_size, - ) - dx = dx.transpose(1, 2).contiguous() - dx = dx.view(shape) - - # Clear input tensor if possible - clear_tensor_data(input_) - - return dx, () + def pack_forward_args(self, *args, **kwargs) -> ClampedSwiGLUFwdArgs: + args = super().pack_forward_args(*args, **kwargs) + args.limit = self.limit + args.alpha = self.alpha + args.glu_linear_offset = self.glu_linear_offset + return args + + def pack_backward_args(self, *args, **kwargs) -> ClampedSwiGLUBwdArgs: + args = super().pack_backward_args(*args, **kwargs) + args.limit = self.limit + args.alpha = self.alpha + args.glu_linear_offset = self.glu_linear_offset + return args class _ScaledGLU(BasicOperation): diff --git a/transformer_engine/pytorch/ops/fused/backward_activation_bias.py b/transformer_engine/pytorch/ops/fused/backward_activation_bias.py index 3950316a3c..dedb6c45e1 100644 --- a/transformer_engine/pytorch/ops/fused/backward_activation_bias.py +++ b/transformer_engine/pytorch/ops/fused/backward_activation_bias.py @@ -6,29 +6,36 @@ from __future__ import annotations from typing import Optional - -import torch +from dataclasses import dataclass import transformer_engine_torch as tex from transformer_engine.pytorch.quantization import Recipe from transformer_engine.pytorch.ops.basic import Bias from transformer_engine.pytorch.ops.basic.activation import ( _ActivationOperation, + ActivationBwdArgs, GELU, ReLU, ) from transformer_engine.pytorch.ops.op import ( FusedOperation, FusibleOperation, - OperationContext, ) from ...utils import clear_tensor_data +from ...dynamo.fake import fake_bgrad from .._common import maybe_dequantize _fused_activations = {GELU: tex.dbias_dgelu, ReLU: tex.dbias_drelu} _fusible_activations = tuple(_fused_activations.keys()) +@dataclass(slots=True) +class BackwardActivationBiasArgs(ActivationBwdArgs): + """Activation gradient with a fused bias reduction.""" + + activation: str + + class BackwardActivationBias(FusedOperation): """Fused backward dact + dbias + quantize @@ -38,47 +45,44 @@ class BackwardActivationBias(FusedOperation): def __init__(self, *, bias: Bias, activation: _ActivationOperation): super().__init__((bias, activation)) - self._fused_function = _fused_activations[type(activation)] - - def fuser_backward( - self, - basic_op_ctxs: list[OperationContext], - grad_output: torch.Tensor, - *, - basic_op_grad_extra_outputs: list[tuple[torch.Tensor, ...]], - ) -> tuple[ - torch.Tensor, - list[tuple[Optional[torch.Tensor], ...]], - list[tuple[()]], - ]: - # Get basic operation contexts - bias_op_ctx = basic_op_ctxs[0] - activation_op_ctx = basic_op_ctxs[1] + bwd_args_type = BackwardActivationBiasArgs - # Saved tensors from forward pass - (act_input,) = activation_op_ctx.saved_tensors + def compile_unsupported_reason(self, mode: str) -> Optional[str]: + reason = super().compile_unsupported_reason(mode) + return reason or self.basic_ops[1].compile_unsupported_reason(mode) - # Check activation input tensor - act_input = maybe_dequantize(act_input.contiguous(), activation_op_ctx.dtype) - - # Check grad output tensor - dy = maybe_dequantize(grad_output.contiguous(), act_input.dtype) - - # Get previous op quantizer - quantizer = bias_op_ctx.grad_input_quantizer + def pack_backward_args( + self, basic_op_ctxs, grad_output, **unused # pylint: disable=unused-argument + ) -> BackwardActivationBiasArgs: + bias_ctx, act_ctx = basic_op_ctxs + quantizer = bias_ctx.grad_input_quantizer if quantizer is None: raise RuntimeError( "BackwardActivationBias requires previous op's grad output quantizer, " "but Bias context has no quantizer" ) + return BackwardActivationBiasArgs( + grad_output, + act_ctx.saved_tensors[0], + act_ctx.dtype, + quantizer, + type(self.basic_ops[1]).__name__.lower(), + ) + + @classmethod + def backward_compute(cls, args: BackwardActivationBiasArgs, *, in_custom_op: bool = False): + x = maybe_dequantize(args.input_, args.dtype).contiguous() + dy = maybe_dequantize(args.grad_output, args.dtype).contiguous() + compute = {"gelu": tex.dbias_dgelu, "relu": tex.dbias_drelu}[args.activation] + db, dx = compute(dy, x, args.grad_input_quantizer) + if not in_custom_op: + clear_tensor_data(x) + return dx, [(db,), ()], [(), ()] - # Launch kernel - db, dx = self._fused_function(dy, act_input, quantizer) - - # Clear activation input tensor - clear_tensor_data(act_input) - + @classmethod + def backward_compute_fake(cls, args: BackwardActivationBiasArgs): + db, dx = fake_bgrad(args.grad_output, args.grad_input_quantizer, dtype=args.dtype) return dx, [(db,), ()], [(), ()] @staticmethod