diff --git a/fms_mo/utils/torchscript_utils.py b/fms_mo/utils/torchscript_utils.py index 87c5076..ee50d9a 100644 --- a/fms_mo/utils/torchscript_utils.py +++ b/fms_mo/utils/torchscript_utils.py @@ -1606,6 +1606,19 @@ def model_analyzer_ts( logger.info( f"Context manager is set to intercept {quant_config['which2patch_contextmanager']}" ) + # if attention uses SDPA, it is traced as a single fused op, so no bmm/matmul is found + # NOTE: the substring "Likely SDPA is enabled" is asserted by these tests in + # tests/models/test_qmodelprep.py, keep them in sync if the message changes: + # - test_torchscript_sdpa_warns_no_bmm + # - test_torchscript_eager_no_sdpa_warning + if num_qbmm_attached == 0 and reconstructed_graph.DFS( + ["aten::scaled_dot_product_attention"] + ): + logger.warning( + "No bmm and matmul are found. Likely SDPA is enabled, so BMM quantization " + "(nbits_bmm1/nbits_bmm2) will be skipped. Load the model with " + 'attn_implementation="eager" to quantize the attention matmuls.' + ) # 1.2.6 sync QKV's clipval (or any linears that share parent) # here we identify the groups that share parent (at this point, they are still nn.Linear) diff --git a/tests/models/test_qmodelprep.py b/tests/models/test_qmodelprep.py index 476aa40..407c779 100644 --- a/tests/models/test_qmodelprep.py +++ b/tests/models/test_qmodelprep.py @@ -16,6 +16,9 @@ Test qmodel_prep functionality on Toy Models, Resnet50, Vision Transformer, and BERT """ +# Standard +import logging + # Third Party import pytest import torch @@ -391,3 +394,71 @@ def test_bert_dynamo_wi_qbmm( assert all( m.num_module_called == 2 for m in other_qmodules ), "Modules other than QBmm were not called properly." + + +def _tiny_bert_with_input(attn_implementation: str): + """ + Create a small randomly initialized BERT (no download) and its input + + Args: + attn_implementation (str): "eager" or "sdpa" + + Returns: + tuple: BertModel and its input dict + """ + config = transformers.BertConfig.from_dict( + { + "hidden_size": 64, + "num_hidden_layers": 2, + "num_attention_heads": 2, + "intermediate_size": 128, + "vocab_size": 100, + } + ) + model = transformers.BertModel._from_config( + config, attn_implementation=attn_implementation + ).eval() + inputs = { + "input_ids": torch.randint(0, 100, (1, 8)), + "attention_mask": torch.ones(1, 8, dtype=torch.long), + } + return model, inputs + + +def test_torchscript_sdpa_warns_no_bmm(config_int8: dict, caplog): + """ + With SDPA attention, TorchScript tracer finds no bmm/matmul to attach QBmm to, qmodel_prep + should warn the user instead of silently skipping BMM quantization. + + Args: + config_int8 (dict): Recipe Config w/ int8 settings + caplog (pytest.LogCaptureFixture): captures log records + """ + model, inputs = _tiny_bert_with_input("sdpa") + config_int8["nbits_bmm1"] = 8 + config_int8["nbits_bmm2"] = 8 + + with caplog.at_level(logging.WARNING): + qmodel_prep(model, inputs, config_int8, use_dynamo=False) + + assert "Likely SDPA is enabled" in caplog.text + assert not any("QBmm" in n for n, _ in model.named_modules()) + + +def test_torchscript_eager_no_sdpa_warning(config_int8: dict, caplog): + """ + With eager attention, bmm/matmul are found and QBmm are attached, no SDPA warning expected. + + Args: + config_int8 (dict): Recipe Config w/ int8 settings + caplog (pytest.LogCaptureFixture): captures log records + """ + model, inputs = _tiny_bert_with_input("eager") + config_int8["nbits_bmm1"] = 8 + config_int8["nbits_bmm2"] = 8 + + with caplog.at_level(logging.WARNING): + qmodel_prep(model, inputs, config_int8, use_dynamo=False) + + assert "Likely SDPA is enabled" not in caplog.text + assert any("QBmm" in n for n, _ in model.named_modules())