Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions fms_mo/utils/torchscript_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1606,6 +1606,19 @@ def model_analyzer_ts(
logger.info(
f"Context manager is set to intercept {quant_config['which2patch_contextmanager']}"
)
# if attention uses SDPA, it is traced as a single fused op, so no bmm/matmul is found
# NOTE: the substring "Likely SDPA is enabled" is asserted by these tests in
# tests/models/test_qmodelprep.py, keep them in sync if the message changes:
# - test_torchscript_sdpa_warns_no_bmm
# - test_torchscript_eager_no_sdpa_warning
if num_qbmm_attached == 0 and reconstructed_graph.DFS(
["aten::scaled_dot_product_attention"]
):
logger.warning(
"No bmm and matmul are found. Likely SDPA is enabled, so BMM quantization "
"(nbits_bmm1/nbits_bmm2) will be skipped. Load the model with "
'attn_implementation="eager" to quantize the attention matmuls.'
)

# 1.2.6 sync QKV's clipval (or any linears that share parent)
# here we identify the groups that share parent (at this point, they are still nn.Linear)
Expand Down
71 changes: 71 additions & 0 deletions tests/models/test_qmodelprep.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
Test qmodel_prep functionality on Toy Models, Resnet50, Vision Transformer, and BERT
"""

# Standard
import logging

# Third Party
import pytest
import torch
Expand Down Expand Up @@ -391,3 +394,71 @@ def test_bert_dynamo_wi_qbmm(
assert all(
m.num_module_called == 2 for m in other_qmodules
), "Modules other than QBmm were not called properly."


def _tiny_bert_with_input(attn_implementation: str):
"""
Create a small randomly initialized BERT (no download) and its input
Args:
attn_implementation (str): "eager" or "sdpa"
Returns:
tuple: BertModel and its input dict
"""
config = transformers.BertConfig.from_dict(
{
"hidden_size": 64,
"num_hidden_layers": 2,
"num_attention_heads": 2,
"intermediate_size": 128,
"vocab_size": 100,
}
)
model = transformers.BertModel._from_config(
config, attn_implementation=attn_implementation
).eval()
inputs = {
"input_ids": torch.randint(0, 100, (1, 8)),
"attention_mask": torch.ones(1, 8, dtype=torch.long),
}
return model, inputs


def test_torchscript_sdpa_warns_no_bmm(config_int8: dict, caplog):
"""
With SDPA attention, TorchScript tracer finds no bmm/matmul to attach QBmm to, qmodel_prep
should warn the user instead of silently skipping BMM quantization.
Args:
config_int8 (dict): Recipe Config w/ int8 settings
caplog (pytest.LogCaptureFixture): captures log records
"""
model, inputs = _tiny_bert_with_input("sdpa")
config_int8["nbits_bmm1"] = 8
config_int8["nbits_bmm2"] = 8

with caplog.at_level(logging.WARNING):
qmodel_prep(model, inputs, config_int8, use_dynamo=False)

assert "Likely SDPA is enabled" in caplog.text
assert not any("QBmm" in n for n, _ in model.named_modules())


def test_torchscript_eager_no_sdpa_warning(config_int8: dict, caplog):
"""
With eager attention, bmm/matmul are found and QBmm are attached, no SDPA warning expected.
Args:
config_int8 (dict): Recipe Config w/ int8 settings
caplog (pytest.LogCaptureFixture): captures log records
"""
model, inputs = _tiny_bert_with_input("eager")
config_int8["nbits_bmm1"] = 8
config_int8["nbits_bmm2"] = 8

with caplog.at_level(logging.WARNING):
qmodel_prep(model, inputs, config_int8, use_dynamo=False)

assert "Likely SDPA is enabled" not in caplog.text
assert any("QBmm" in n for n, _ in model.named_modules())
Loading