From 494ee99f6259e8843b3f2361116cee57b4aae52f Mon Sep 17 00:00:00 2001 From: Prerana Date: Sun, 1 Nov 2026 12:51:08 -0500 Subject: [PATCH 1/2] fix: warn when TorchScript tracer finds no bmm/matmul due to SDPA When attention uses SDPA, it is traced as a single fused op, so qmodel_prep (TorchScript path) finds no bmm/matmul to attach QBmm to, and BMM quantization (nbits_bmm1/nbits_bmm2) is silently skipped. Log a warning in this case, as the Dynamo path already does, and point to attn_implementation="eager". Add tests with a small BERT using sdpa (warns) and eager (no warning). Refs #208 (adds the interim warning suggested there; does not add SDPA support) Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Prerana --- fms_mo/utils/torchscript_utils.py | 9 ++++ tests/models/test_qmodelprep.py | 71 +++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/fms_mo/utils/torchscript_utils.py b/fms_mo/utils/torchscript_utils.py index 87c50763..a58fa80d 100644 --- a/fms_mo/utils/torchscript_utils.py +++ b/fms_mo/utils/torchscript_utils.py @@ -1606,6 +1606,15 @@ def model_analyzer_ts( logger.info( f"Context manager is set to intercept {quant_config['which2patch_contextmanager']}" ) + # if attention uses SDPA, it is traced as a single fused op, so no bmm/matmul is found + if num_qbmm_attached == 0 and reconstructed_graph.DFS( + ["aten::scaled_dot_product_attention"] + ): + logger.warning( + "No bmm and matmul are found. Likely SDPA is enabled, so BMM quantization " + "(nbits_bmm1/nbits_bmm2) will be skipped. Load the model with " + 'attn_implementation="eager" to quantize the attention matmuls.' + ) # 1.2.6 sync QKV's clipval (or any linears that share parent) # here we identify the groups that share parent (at this point, they are still nn.Linear) diff --git a/tests/models/test_qmodelprep.py b/tests/models/test_qmodelprep.py index 476aa404..407c7793 100644 --- a/tests/models/test_qmodelprep.py +++ b/tests/models/test_qmodelprep.py @@ -16,6 +16,9 @@ Test qmodel_prep functionality on Toy Models, Resnet50, Vision Transformer, and BERT """ +# Standard +import logging + # Third Party import pytest import torch @@ -391,3 +394,71 @@ def test_bert_dynamo_wi_qbmm( assert all( m.num_module_called == 2 for m in other_qmodules ), "Modules other than QBmm were not called properly." + + +def _tiny_bert_with_input(attn_implementation: str): + """ + Create a small randomly initialized BERT (no download) and its input + + Args: + attn_implementation (str): "eager" or "sdpa" + + Returns: + tuple: BertModel and its input dict + """ + config = transformers.BertConfig.from_dict( + { + "hidden_size": 64, + "num_hidden_layers": 2, + "num_attention_heads": 2, + "intermediate_size": 128, + "vocab_size": 100, + } + ) + model = transformers.BertModel._from_config( + config, attn_implementation=attn_implementation + ).eval() + inputs = { + "input_ids": torch.randint(0, 100, (1, 8)), + "attention_mask": torch.ones(1, 8, dtype=torch.long), + } + return model, inputs + + +def test_torchscript_sdpa_warns_no_bmm(config_int8: dict, caplog): + """ + With SDPA attention, TorchScript tracer finds no bmm/matmul to attach QBmm to, qmodel_prep + should warn the user instead of silently skipping BMM quantization. + + Args: + config_int8 (dict): Recipe Config w/ int8 settings + caplog (pytest.LogCaptureFixture): captures log records + """ + model, inputs = _tiny_bert_with_input("sdpa") + config_int8["nbits_bmm1"] = 8 + config_int8["nbits_bmm2"] = 8 + + with caplog.at_level(logging.WARNING): + qmodel_prep(model, inputs, config_int8, use_dynamo=False) + + assert "Likely SDPA is enabled" in caplog.text + assert not any("QBmm" in n for n, _ in model.named_modules()) + + +def test_torchscript_eager_no_sdpa_warning(config_int8: dict, caplog): + """ + With eager attention, bmm/matmul are found and QBmm are attached, no SDPA warning expected. + + Args: + config_int8 (dict): Recipe Config w/ int8 settings + caplog (pytest.LogCaptureFixture): captures log records + """ + model, inputs = _tiny_bert_with_input("eager") + config_int8["nbits_bmm1"] = 8 + config_int8["nbits_bmm2"] = 8 + + with caplog.at_level(logging.WARNING): + qmodel_prep(model, inputs, config_int8, use_dynamo=False) + + assert "Likely SDPA is enabled" not in caplog.text + assert any("QBmm" in n for n, _ in model.named_modules()) From 6919fe28db1b9c5d9aa7dc671abc0785cac94785 Mon Sep 17 00:00:00 2001 From: Prerana Date: Mon, 9 Nov 2026 10:51:56 -0500 Subject: [PATCH 2/2] docs: note that SDPA warning text is asserted by unit tests Addresses review feedback on #214. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Prerana --- fms_mo/utils/torchscript_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fms_mo/utils/torchscript_utils.py b/fms_mo/utils/torchscript_utils.py index a58fa80d..ee50d9a6 100644 --- a/fms_mo/utils/torchscript_utils.py +++ b/fms_mo/utils/torchscript_utils.py @@ -1607,6 +1607,10 @@ def model_analyzer_ts( f"Context manager is set to intercept {quant_config['which2patch_contextmanager']}" ) # if attention uses SDPA, it is traced as a single fused op, so no bmm/matmul is found + # NOTE: the substring "Likely SDPA is enabled" is asserted by these tests in + # tests/models/test_qmodelprep.py, keep them in sync if the message changes: + # - test_torchscript_sdpa_warns_no_bmm + # - test_torchscript_eager_no_sdpa_warning if num_qbmm_attached == 0 and reconstructed_graph.DFS( ["aten::scaled_dot_product_attention"] ):