diff --git a/backends/qualcomm/__init__.py b/backends/qualcomm/__init__.py index a37caed3e11..1156ed40dd5 100644 --- a/backends/qualcomm/__init__.py +++ b/backends/qualcomm/__init__.py @@ -1,131 +1,11 @@ -import os -import platform -import threading - -# The Qualcomm SDK setup below is deferred rather than run here, so that importing this -# package has no side effects. It used to run at import time, which meant that merely -# importing the package downloaded the SDK and libc++ over the network, re-executed the -# interpreter under a staged loader, rewrote QNN_SDK_ROOT and LD_LIBRARY_PATH, and raised -# when a machine had no network. It also disabled PyTorch's MKLDNN backend for the whole -# process on any AMD host, which affects every other model in that interpreter. +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved # -# Nothing about loading this package needs any of that. The native adaptor does not link -# the SDK; it resolves QNN symbols with dlopen when a model is actually compiled, and says -# so plainly when they are missing. So setup happens on the first call that needs it. - -_sdk_ready = False -# Guards the flag above. Python's import lock does not, because the setup is now called from -# several modules rather than from one package __init__, and two threads importing two of them -# concurrently would otherwise both run an installer that downloads and rewrites the -# environment. -_sdk_lock = threading.Lock() - - -def setup_qnn_sdk() -> None: - """Make the Qualcomm SDK usable in this process, once. - - Safe to call repeatedly and from more than one thread: the work happens on the first call and - later calls return immediately. Called by the code paths that need the SDK, so a caller does - not have to. - """ - if _sdk_ready: - return - with _sdk_lock: - # Another thread may have finished while this one waited. - if _sdk_ready: - return - _setup_qnn_sdk_locked() - - -def _setup_qnn_sdk_locked() -> None: - global _sdk_ready - - # The wheel build imports this package to collect its files, and has no use for an SDK. - if os.getenv("EXECUTORCH_BUILDING_WHEEL", "0").lower() in ("1", "true", "yes"): - _sdk_ready = True - return - - # A preinstalled SDK is used as it is. - qnn_sdk_root = os.getenv("QNN_SDK_ROOT", None) - if qnn_sdk_root: - print(f"[QNN] Using QNN SDK at {qnn_sdk_root} (from QNN_SDK_ROOT)", flush=True) - _sdk_ready = True - return - - # Downloading a prebuilt SDK is only possible for the platform it is published for. - # Decided here rather than by asking the downloader, so that a build which does not - # package the downloader still gets this far and returns. - if not _is_linux_x86(): - _sdk_ready = True - return - - if not _install_qnn_sdk(): - from .scripts.download_qnn_sdk import QNN_ZIP_URL - - raise RuntimeError( - "Failed to set up QNN SDK.\n\n" - "To resolve, try one of:\n" - " 1. Download the SDK manually from:\n" - f" {QNN_ZIP_URL}\n" - " Or go to step 2 if QNN SDK already exists.\n" - " 2. Set QNN_SDK_ROOT to an existing SDK installation:\n" - " export QNN_SDK_ROOT=/path/to/qualcomm/sdk\n" - " export LD_LIBRARY_PATH=" - "$QNN_SDK_ROOT/lib/x86_64-linux-clang/:$LD_LIBRARY_PATH" - ) - - _sdk_ready = True - - -def _is_linux_x86() -> bool: - """True when a prebuilt Qualcomm SDK is published for this platform.""" - return platform.system().lower() == "linux" and platform.machine().lower() in ( - "x86_64", - "amd64", - "i386", - "i686", - ) - - -def _install_qnn_sdk() -> bool: - # Imported here rather than at module scope because the downloader lives in a sibling - # directory that some builds do not package, and it imports the network stack. Importing - # this package must not require either. - try: - from .scripts.download_qnn_sdk import install_qnn_sdk - except ModuleNotFoundError as error: - # Only when the downloader itself is absent. A dependency missing from inside it is a - # different problem and is left to speak for itself. - if not (error.name or "").startswith(f"{__name__}.scripts"): - raise - raise RuntimeError( - "This build cannot download a QNN SDK. Set QNN_SDK_ROOT to an existing " - "installation:\n" - " export QNN_SDK_ROOT=/path/to/qualcomm/sdk\n" - " export LD_LIBRARY_PATH=" - "$QNN_SDK_ROOT/lib/x86_64-linux-clang/:$LD_LIBRARY_PATH" - ) from error - - return install_qnn_sdk() - - -def disable_mkldnn_on_amd() -> None: - """Turn off PyTorch's MKLDNN backend on an AMD host. - - MKLDNN crashes on some AMD hosts, which is why this exists. The original comment described - it as producing wrong results; what was measured is a core dump, from a plain convolution - with nothing from this backend involved. - - This changes a global PyTorch setting, so it is applied by the QNN compile paths rather than - at import, where it would also change how unrelated models run in the same interpreter. - """ - import torch - - try: - import cpuinfo - except ImportError: - raise ImportError("Please install the cpuinfo with pip install py-cpuinfo.") - - vendor = cpuinfo.get_cpu_info().get("vendor_id_raw", "") or "" - if "amd" in vendor.lower(): - torch.backends.mkldnn.enabled = False +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Deliberately empty. Importing this package must have no side effects, and it must not be the +# home of anything a submodule needs, because build systems that assemble a package from a file +# list can leave this file out and synthesize an empty one in its place. The Qualcomm SDK setup +# that used to live here is in utils/qnn_sdk_setup.py, called by the code paths that start a +# backend. diff --git a/backends/qualcomm/debugger/utils.py b/backends/qualcomm/debugger/utils.py index 97a0de9c310..f1c4c535847 100644 --- a/backends/qualcomm/debugger/utils.py +++ b/backends/qualcomm/debugger/utils.py @@ -10,6 +10,7 @@ import pandas as pd import torch from executorch.backends.qualcomm.serialization.qc_schema import QcomChipset +from executorch.backends.qualcomm.utils.qnn_sdk_setup import setup_qnn_sdk from executorch.backends.qualcomm.utils.utils import dump_context_from_pte from graphviz import Digraph @@ -199,10 +200,20 @@ def __init__( build_folder, workspace="/data/local/tmp/qnn_executorch_test", ): + # Makes the SDK usable first, because this tool runs a binary from inside it. Setup used to + # happen while this module was imported, which set the variable read below as a side effect. + setup_qnn_sdk() + self.qnn_sdk = os.environ.get("QNN_SDK_ROOT", None) self.ndk = os.environ.get("ANDROID_NDK_ROOT", None) - assert self.qnn_sdk, "QNN_SDK_ROOT was not found in environment variable" - assert self.ndk, "ANDROID_NDK_ROOT was not found in environment variable" + # Raised rather than asserted, because an assert is stripped under `python -O` and both + # values are used to build real paths a few lines below. + if not self.qnn_sdk: + raise EnvironmentError("QNN_SDK_ROOT was not found in environment variable") + if not self.ndk: + raise EnvironmentError( + "ANDROID_NDK_ROOT was not found in environment variable" + ) self.tmp_dir = tmp_dir self.workspace = workspace diff --git a/backends/qualcomm/export_utils.py b/backends/qualcomm/export_utils.py index 4a06519945e..cd0b15aec61 100644 --- a/backends/qualcomm/export_utils.py +++ b/backends/qualcomm/export_utils.py @@ -45,6 +45,10 @@ HEXAGON_SDK_ROOT, HEXAGON_TOOLS_ROOT, ) +from executorch.backends.qualcomm.utils.qnn_sdk_setup import ( + disable_mkldnn_on_amd, + setup_qnn_sdk, +) from executorch.backends.qualcomm.utils.utils import ( generate_gpu_compiler_spec, generate_htp_compiler_spec, @@ -129,9 +133,14 @@ def __post_init__(self): assert not ( self.compile_only and self.pre_gen_pte ), "Cannot set both compile_only and pre_gen_pte as true" - assert ( - "QNN_SDK_ROOT" in os.environ - ), "Environment variable QNN_SDK_ROOT must be set." + # Every example script builds this config before doing anything else, and the check + # below needs the SDK path. It used to be set as a side effect of importing the backend. + setup_qnn_sdk() + # A usable path, not merely the key being present, because everything below builds real + # paths from it. Raised rather than asserted: an assert is stripped under `python -O`, + # which would turn this into a confusing failure much later. + if not os.environ.get("QNN_SDK_ROOT"): + raise EnvironmentError("Environment variable QNN_SDK_ROOT must be set.") if (not self.compile_only and not self.enable_x86_64) and self.device is None: raise RuntimeError( "device serial is required if not compile only or run on x86 emulator. Please specify a device serial." @@ -146,9 +155,15 @@ def __post_init__(self): elif get_soc_to_lpai_hw_ver_map()[ self.soc_model ] == LpaiHardwareVersion.V6 and is_qnn_sdk_version_less_than("2.39"): + # Read once, because building this message by querying again raises when there is + # no SDK, which replaces the useful error below with a confusing one. + try: + current = get_sdk_build_id() + except Exception: + current = "unknown, no usable SDK found" raise RuntimeError( f"Target soc_model({self.soc_model}) with LPAI backend v6 requires QNN SDK version >= 2.39. \n" - f"Current QNN SDK version: {get_sdk_build_id()}" + f"Current QNN SDK version: {current}" ) if self.seed: torch.manual_seed(self.seed) @@ -539,6 +554,13 @@ def build_executorch_binary( ) return + # Both applied here rather than deeper in the lowering. The quantized path below runs the + # model once per calibration sample, which is a real eager run and so needs the AMD guard + # first, and the SDK setup can re-execute the interpreter, which must not happen after a + # model has been traced and calibrated. + setup_qnn_sdk() + disable_mkldnn_on_amd() + sample_input = dataset[0] if ( qnn_config.backend == QnnExecuTorchBackendType.kGpuBackend diff --git a/backends/qualcomm/quantizer/backend_opinfo_adapter.py b/backends/qualcomm/quantizer/backend_opinfo_adapter.py index 3595e52f582..ae7ab009004 100644 --- a/backends/qualcomm/quantizer/backend_opinfo_adapter.py +++ b/backends/qualcomm/quantizer/backend_opinfo_adapter.py @@ -45,6 +45,39 @@ def add_qnn_python_path(): backend_opinfo = None _HAS_BACKEND_OPINFO = False +_WARNED_ABOUT_FALLBACK = False + + +def _load_backend_opinfo() -> bool: + """Looks for the SDK's op info again, once the SDK has been set up. + + The attempt above runs while this module is imported, which can be before anything has made + the SDK usable. Deciding only there would cache the no-op fallback for the rest of the + process, silently dropping every quantization constraint check. + """ + global backend_opinfo, _HAS_BACKEND_OPINFO + if _HAS_BACKEND_OPINFO: + return True + + from executorch.backends.qualcomm.utils.qnn_sdk_setup import setup_qnn_sdk + + try: + # Inside the try, because a failure here means the same thing as an SDK without this API: + # the op info is unavailable. Letting it escape would make building a quantizer raise on + # any host with no usable SDK, where it used to fall back. + setup_qnn_sdk() + add_qnn_python_path() + from qti.aisw.converters.common import backend_opinfo as loaded + except Exception: + # Logged rather than dropped, because the fallback below is silent about which of the + # several possible causes applied: no SDK, an SDK without this API, or a real error. + logging.debug("The QNN op info could not be loaded", exc_info=True) + return False + + backend_opinfo = loaded + _HAS_BACKEND_OPINFO = True + return True + class _NoOpBackendOpInfo: def __init__(self, *args, **kwargs): @@ -57,21 +90,18 @@ def get_op_info(self, op_name: str): return [] -class _NoOpNamespace: - HTP = 1 - LPAI = 3 - BackendOpInfo = _NoOpBackendOpInfo - - -if not _HAS_BACKEND_OPINFO: +def _warn_once_about_the_fallback() -> None: + global _WARNED_ABOUT_FALLBACK + if _WARNED_ABOUT_FALLBACK: + return + _WARNED_ABOUT_FALLBACK = True logging.warning( "The backend_opinfo module couldn't be imported, so the abstract implementation will be used instead. This might be because $QNN_SDK_ROOT/lib/python isn't included in your PYTHONPATH, or the `BackendOpInfo` API isn't available in your QNN SDK version. Note that the `BackendOpInfo` API is supported starting from QNN SDK 2.41 and above." ) - backend_opinfo = _NoOpNamespace() @lru_cache() -def get_backend_opinfo(backend: str, soc_model: QcomChipset): +def _get_backend_opinfo_cached(backend: str, soc_model: QcomChipset): backend_type = getattr(backend_opinfo, backend.upper()) # For qnn 2.41, it only supports HTP backend # It will support LPAI backend as soon as possible. @@ -80,12 +110,28 @@ def get_backend_opinfo(backend: str, soc_model: QcomChipset): try: return backend_opinfo.BackendOpInfo(backend_type, soc_model) except Exception: - print( - f"The 'BackendOpInfo' APIs may not be available for this backend {backend}." + logging.warning( + "The 'BackendOpInfo' APIs may not be available for this backend %s.", + backend, ) return _NoOpBackendOpInfo() +def get_backend_opinfo(backend: str, soc_model: QcomChipset): + # The SDK is looked for again here rather than only while this module was imported, because + # that ran before anything had made the SDK usable. + # + # Deliberately not cached itself, so a later successful setup can still recover the real + # constraint checks. The inner function is cached, and it does return the do-nothing checker + # for a backend the SDK has no op info for, which is a fixed fact about that backend rather + # than something a later setup changes. + if not _load_backend_opinfo(): + _warn_once_about_the_fallback() + return _NoOpBackendOpInfo() + + return _get_backend_opinfo_cached(backend, soc_model) + + # Helper functions for normalizing OpInfo objects (moved from backend_opinfo_adapter) def _normalize_datatype_info(datatype_info: Any) -> PortDatatypeConstraints: """ diff --git a/backends/qualcomm/quantizer/quantizer.py b/backends/qualcomm/quantizer/quantizer.py index d7ac219d927..ffe3e1d43a5 100644 --- a/backends/qualcomm/quantizer/quantizer.py +++ b/backends/qualcomm/quantizer/quantizer.py @@ -40,6 +40,7 @@ QnnExecuTorchBackendType, ) from executorch.backends.qualcomm.utils.constants import QCOM_QUANT_ANNOTATION_KEY +from executorch.backends.qualcomm.utils.qnn_sdk_setup import disable_mkldnn_on_amd from torch._ops import OpOverload from torch.fx import GraphModule @@ -390,6 +391,11 @@ def __init__( self.supported_ops: Set[OpOverload] = set(self._rules_map.keys()) self.quant_ops: Set[OpOverload] = self.supported_ops.copy() + # Applied when the quantizer is built, because that is upstream of calibration, and + # calibration runs the model eagerly. The AMD crash this prevents needs a real + # convolution, so a guard applied at lowering time comes after the risk has passed. + disable_mkldnn_on_amd() + # Load backend_opinfo of current backend and soc_model self.backend_opinfo = get_backend_opinfo(str(backend), soc_model) diff --git a/backends/qualcomm/recipes/qnn_recipe_provider.py b/backends/qualcomm/recipes/qnn_recipe_provider.py index c1b42fd4f73..d5a745ecf94 100644 --- a/backends/qualcomm/recipes/qnn_recipe_provider.py +++ b/backends/qualcomm/recipes/qnn_recipe_provider.py @@ -21,6 +21,10 @@ QcomChipset, QnnExecuTorchBackendType, ) +from executorch.backends.qualcomm.utils.qnn_sdk_setup import ( + disable_mkldnn_on_amd, + setup_qnn_sdk, +) from executorch.backends.qualcomm.utils.utils import ( generate_htp_compiler_spec, generate_qnn_executorch_compiler_spec, @@ -53,6 +57,12 @@ def create_recipe( self._validate_recipe_kwargs(recipe_type, kwargs) + # Done while the recipe is built, which is before the export pipeline traces anything. The + # installer can re-execute the interpreter on an old glibc, and doing that partway through + # would discard a model that has already been traced. + setup_qnn_sdk() + disable_mkldnn_on_amd() + if recipe_type == QNNRecipeType.FP16: return self._build_fp16_recipe(recipe_type, kwargs) diff --git a/backends/qualcomm/tests/test_import_side_effects.py b/backends/qualcomm/tests/test_import_side_effects.py index f5b08fddb9c..488fe96b90e 100644 --- a/backends/qualcomm/tests/test_import_side_effects.py +++ b/backends/qualcomm/tests/test_import_side_effects.py @@ -4,174 +4,671 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""Tests that importing the Qualcomm backend package has no side effects. - -The package's __init__ used to do real work at import time. Merely importing it fetched the -SDK and libc++ over the network, could re-execute the interpreter under a staged loader, -rewrote QNN_SDK_ROOT and LD_LIBRARY_PATH, raised on a machine with no network, and disabled -PyTorch's MKLDNN backend for the whole process on any AMD host. None of that is needed to -load the package: the native adaptor resolves QNN symbols with dlopen when a model is -compiled, so setup belongs on the paths that compile. - -These are unit tests because the behaviour under test is what happens during import, which -is observable without a Qualcomm SDK or a Qualcomm device. +"""Tests that loading the Qualcomm backend has no side effects. + +Setup used to run while the package's ``__init__`` was imported. Merely importing it fetched the +SDK and libc++ over the network, could re-execute the interpreter under a staged loader, rewrote +QNN_SDK_ROOT and LD_LIBRARY_PATH, raised on a machine with no network, and disabled PyTorch's +MKLDNN backend for the whole process on any AMD host. None of that is needed to load the +package: the native adaptor resolves QNN symbols with dlopen when a backend is started, so setup +belongs on the calls that start one. + +Two separate properties are checked. First, that no module runs the setup while being imported, +which is asserted against the module source because that is what an import executes. Second, +that the setup itself behaves, which is asserted by calling it. + +The source checks ask the module's loader for its source rather than opening a file, so they +also hold where the package was assembled into an archive, and a module with no source of its +own reads as empty instead of raising. """ import ast -import importlib +import builtins +import os +import platform import sys import types -from pathlib import Path import executorch.backends.qualcomm as qnn +import executorch.backends.qualcomm.builders.node_visitor as node_visitor +import executorch.backends.qualcomm.debugger.utils as debugger_utils +import executorch.backends.qualcomm.quantizer.validators as validators +import executorch.backends.qualcomm.utils.check_qnn_version as check_qnn_version +import executorch.backends.qualcomm.utils.qnn_manager_lifecycle as qnn_manager_lifecycle +import executorch.backends.qualcomm.utils.qnn_sdk_setup as qnn_sdk_setup +import executorch.backends.qualcomm.utils.utils as qnn_utils import pytest import torch +from executorch.backends.qualcomm.serialization.qc_schema import ( + QnnExecuTorchBackendType, +) + +try: + from executorch.backends.qualcomm.recipes.qnn_recipe_types import QNNRecipeType +except Exception: # the recipe package pulls in optional dependencies + QNNRecipeType = None + +# Every module that reaches the SDK. Each one is imported by ordinary use of the backend, so a +# setup call in any of them is a setup call at import time. +CONSUMER_MODULES = [ + node_visitor, + qnn_utils, + check_qnn_version, + qnn_manager_lifecycle, + validators, + debugger_utils, +] + +SETUP_NAMES = {"setup_qnn_sdk", "disable_mkldnn_on_amd", "install_qnn_sdk"} + + +def module_source(module): + """Returns `module`'s own source text, or an empty string when it has none. + + Read through the loader rather than from the file, so this also works where the package was + assembled into an archive and no file exists to open. A loader is allowed to report missing + source either by returning None or by raising ImportError, and a build system that + synthesizes an empty file leaves a module with no source of its own, so both have to read as + empty here rather than failing the check. + """ + loader = getattr(module, "__loader__", None) + get_source = getattr(loader, "get_source", None) + if get_source is None: + return "" + try: + return get_source(module.__name__) or "" + except ImportError: + return "" + + +def calls_made_while_importing(module): + """Returns the names called by `module`'s own top level statements. + + Walks into every top level statement rather than only bare expressions, because a call + guarded by an `if` still runs during the import. That guarded shape is what the original + defect looked like, so a check that skipped it would not have caught it. + + A function or class body is skipped, since an import does not enter it, but the parts of the + definition around it are not: a decorator, a base class and a default argument are all + evaluated at import time. A version check inside a `skipIf` decorator is exactly how setup + crept back onto the import path once already. + """ + tree = ast.parse(module_source(module)) + called = set() + + def record(nodes): + for root in nodes: + for node in ast.walk(root): + if isinstance(node, ast.Call): + target = node.func + if isinstance(target, ast.Name): + called.add(target.id) + elif isinstance(target, ast.Attribute): + called.add(target.attr) + + for statement in tree.body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + # The body is not entered during an import, but the parts of the definition that + # surround it are evaluated: decorators, base classes, and default arguments. + record(statement.decorator_list) + if isinstance(statement, ast.ClassDef): + record(statement.bases) + record(kw.value for kw in statement.keywords) + else: + args = statement.args + record(d for d in args.defaults if d is not None) + record(d for d in args.kw_defaults if d is not None) + continue + record([statement]) + return called @pytest.fixture def fake_cpuinfo(monkeypatch): """Replaces cpuinfo so a vendor can be chosen without an AMD host.""" - def install(vendor): + def install(vendor, key="vendor_id_raw"): module = types.ModuleType("cpuinfo") - module.get_cpu_info = lambda: {"vendor_id_raw": vendor} + module.get_cpu_info = lambda: {key: vendor} monkeypatch.setitem(sys.modules, "cpuinfo", module) return install -def test_importing_the_package_leaves_mkldnn_alone(monkeypatch, fake_cpuinfo): - """Import must not change a global PyTorch setting, even on an AMD host. +@pytest.fixture +def ready_to_set_up(monkeypatch): + """Puts the setup back in its pre-run state, on a host that could download an SDK.""" + monkeypatch.setattr(qnn_sdk_setup, "_sdk_ready", False) + monkeypatch.setattr(qnn_sdk_setup, "_is_linux_x86", lambda: True) + monkeypatch.delenv("QNN_SDK_ROOT", raising=False) + monkeypatch.delenv("EXECUTORCH_BUILDING_WHEEL", raising=False) + + +@pytest.fixture +def recorded_install(monkeypatch): + """Replaces the installer with one that records its calls instead of downloading.""" + calls = [] + + def install(): + calls.append(1) + return True + + monkeypatch.setattr(qnn_sdk_setup, "_install_qnn_sdk", install) + return calls + + +@pytest.mark.parametrize( + "module", CONSUMER_MODULES, ids=lambda module: module.__name__.rsplit(".", 1)[-1] +) +def test_importing_a_module_does_not_run_the_setup(module): + """No module may set up the SDK while it is being imported. + + Asserted against the source rather than by importing with a stub in place: by the time a test + runs, the import has already happened, and re-importing rebinds the real functions and + discards any stub. What matters is that no top level statement calls them. + """ + assert not calls_made_while_importing(module) & SETUP_NAMES + - The setting affects every model in the interpreter, not just a Qualcomm one, so an import - that flips it changes how unrelated code runs. +def test_the_package_root_stays_inert(): + """The package's __init__ must hold nothing a submodule needs. - A reload is used deliberately: it re-executes every module level statement, which is exactly - what an import does, and the stubbed cpuinfo is read inside those statements rather than - bound at import, so unlike a stubbed installer it survives. + A build system that assembles a package from a file list can leave this file out and put an + empty one in its place, which silently removes anything defined here. Keeping it free of top + level statements means the real file and a synthesized empty one behave the same. + """ + assert not ast.parse(module_source(qnn)).body + + +@pytest.mark.parametrize( + "source", + [ + "setup_qnn_sdk()\n", + "if True:\n setup_qnn_sdk()\n", + "@skip_if(setup_qnn_sdk())\ndef f():\n pass\n", + "class C(base(setup_qnn_sdk())):\n pass\n", + "def f(x=setup_qnn_sdk()):\n pass\n", + ], + ids=["plain", "guarded", "decorator", "class-base", "default-arg"], +) +def test_the_check_sees_every_shape_that_runs_at_import(source): + """Guards the two checks above, which pass trivially if the walk finds nothing. + + All five shapes run while a module is imported. The last three are easy to miss because they + sit on a function or class definition, whose body an import does not enter, and a version + check inside a decorator is how setup crept back onto the import path once already. + """ + module = types.ModuleType("shaped_setup_call") + module.__loader__ = types.SimpleNamespace(get_source=lambda name: source) + + assert calls_made_while_importing(module) & SETUP_NAMES + + +def test_the_check_treats_a_sourceless_module_as_empty(): + """A synthesized empty file has no source, and that has to read as empty, not raise.""" + module = types.ModuleType("synthesized_package") + module.__loader__ = types.SimpleNamespace(get_source=lambda name: None) + + assert module_source(module) == "" + assert not calls_made_while_importing(module) + + +def test_the_check_treats_a_refusing_loader_as_empty(): + """A loader may report missing source by raising instead of returning None.""" + + def refuse(name): + raise ImportError(f"no source available for {name}") + + module = types.ModuleType("bytecode_only_package") + module.__loader__ = types.SimpleNamespace(get_source=refuse) + + assert module_source(module) == "" + assert not calls_made_while_importing(module) + + +class _StopAfterTrace(Exception): + """Ends an entry point at the trace, so a test can check what ran before it.""" + + +@pytest.mark.parametrize( + "entry_point", ["to_edge_transform_and_lower_to_qnn", "capture_program"] +) +def test_setup_and_the_amd_guard_precede_the_trace(monkeypatch, entry_point): + """Both have to happen at the entry point, not deeper in the lowering. + + The SDK setup can download one, and on an old glibc the installer re-executes the + interpreter, so running it after a trace would throw the traced model away. The AMD guard + prevents a crash that needs a real convolution, which calibration and a plain forward pass + run even though a trace does not. + + Driven by recording the real call order rather than by comparing source positions, because a + call inside a function that is never called would satisfy a source order check. + """ + order = [] + monkeypatch.setattr( + qnn_utils, "disable_mkldnn_on_amd", lambda: order.append("guard") + ) + monkeypatch.setattr(qnn_utils, "setup_qnn_sdk", lambda: order.append("setup")) + + def stop_here(*args, **kwargs): + order.append("trace") + raise _StopAfterTrace("stopped at the trace") + + monkeypatch.setattr(torch.export, "export", stop_here) + + # The entry point is expected to fail on the stub inputs, one way or another. What matters is + # that both have already run by then. Setup can re-execute the interpreter, which would throw + # away a traced model, and the guard has to precede any eager run. + with pytest.raises((_StopAfterTrace, ValueError, TypeError, AttributeError)): + getattr(qnn_utils, entry_point)(torch.nn.Identity(), (torch.ones(1),), []) + + assert order, f"{entry_point} applied neither the setup nor the AMD guard" + assert "setup" in order, f"{entry_point} does not set up the SDK before tracing" + assert "guard" in order, f"{entry_point} does not apply the AMD guard" + if "trace" in order: + assert order.index("setup") < order.index("trace") + assert order.index("guard") < order.index("trace") + + +def test_the_op_info_is_looked_for_again_after_setup(monkeypatch): + """The import-time attempt can run before anything has made the SDK usable. + + Drives the real retry, rather than stubbing it, so a version that only trusts the + import-time result fails here. + """ + from executorch.backends.qualcomm.quantizer import backend_opinfo_adapter as adapter + + # As if the import-time attempt had failed, which is the case this exists for. + monkeypatch.setattr(adapter, "_HAS_BACKEND_OPINFO", False) + monkeypatch.setattr(adapter, "backend_opinfo", None) + + staged = types.ModuleType("qti.aisw.converters.common") + staged.backend_opinfo = types.SimpleNamespace( + HTP=1, BackendOpInfo=lambda *a: "real" + ) + + def stage_the_sdk(): + # What setup does for real: makes the SDK importable. + monkeypatch.setitem(sys.modules, "qti", types.ModuleType("qti")) + monkeypatch.setitem(sys.modules, "qti.aisw", types.ModuleType("qti.aisw")) + monkeypatch.setitem( + sys.modules, "qti.aisw.converters", types.ModuleType("qti.aisw.converters") + ) + monkeypatch.setitem(sys.modules, "qti.aisw.converters.common", staged) + + # Patched at its source, because the retry imports it inside the function. + monkeypatch.setattr(qnn_sdk_setup, "setup_qnn_sdk", lambda: None) + monkeypatch.setattr(adapter, "add_qnn_python_path", stage_the_sdk) + + assert adapter._load_backend_opinfo() is True + assert adapter._HAS_BACKEND_OPINFO + + +def test_the_op_info_fallback_is_not_cached_forever(monkeypatch): + """A do-nothing checker must not be pinned once the SDK becomes usable. + + The real getter is cached, so caching the fallback too would keep returning it for those + arguments for the rest of the process, silently dropping every constraint check. + """ + from executorch.backends.qualcomm.quantizer import backend_opinfo_adapter as adapter + + ready = [] + monkeypatch.setattr(adapter, "_load_backend_opinfo", lambda: bool(ready)) + monkeypatch.setattr(adapter, "_warn_once_about_the_fallback", lambda: None) + monkeypatch.setattr( + adapter, "_get_backend_opinfo_cached", lambda backend, soc: "real" + ) + + before = adapter.get_backend_opinfo("HTP", 1) + ready.append(True) + after = adapter.get_backend_opinfo("HTP", 1) + + assert isinstance(before, adapter._NoOpBackendOpInfo) + assert after == "real" + + +@pytest.mark.parametrize( + "helper,unset_answer", + [ + ("is_qnn_sdk_version_less_than", True), + ("is_qnn_sdk_version_greater_than", False), + ], +) +def test_a_broken_sdk_is_not_reported_as_an_old_one(monkeypatch, helper, unset_answer): + """An unreadable SDK must not look like an old SDK. + + With no SDK path there is no version to compare, so a fallback answer is right. A library + that is present but cannot be read is a different problem, and swallowing it too made the + two indistinguishable, so a real failure was silently reported as a version gate. + """ + + def raise_it(error): + def raiser(): + raise error + + return raiser + + no_path = check_qnn_version.QnnSdkRootNotSet("QNN_SDK_ROOT must be set.") + monkeypatch.setattr(check_qnn_version, "get_sdk_build_id", raise_it(no_path)) + + assert getattr(check_qnn_version, helper)("2.48") is unset_answer + + unreadable = OSError("cannot open libQnnHtp.so") + monkeypatch.setattr(check_qnn_version, "get_sdk_build_id", raise_it(unreadable)) + + with pytest.raises(OSError): + getattr(check_qnn_version, helper)("2.48") + + +def test_reusing_a_cached_manager_still_applies_the_guard( + monkeypatch, fake_cpuinfo, recorded_install +): + """A second lowering must not run with the setting back on. + + The manager is built once and reused, so applying the guard only where it is built would let + every later lowering run in exactly the configuration the guard exists to prevent. """ fake_cpuinfo("AuthenticAMD") + monkeypatch.setattr(qnn_sdk_setup, "_vendor_is_amd", None) + # Marked ready, because this test is about the guard and nothing here may reach the real + # installer: it downloads about a gigabyte and rewrites the environment for the whole process. + monkeypatch.setattr(qnn_sdk_setup, "_sdk_ready", True) + registry = qnn_manager_lifecycle.QnnManagerRegistry() + backend_type = QnnExecuTorchBackendType.kHtpBackend + registry._registry[backend_type] = object() + monkeypatch.setattr(torch.backends.mkldnn, "enabled", True) + registry.get_or_create_qnn_manager(backend_type, b"") - importlib.reload(qnn) + assert not torch.backends.mkldnn.enabled + # Nothing may reach the real installer from a test: it downloads about a gigabyte and + # rewrites the environment for the rest of the process. + assert not recorded_install - assert torch.backends.mkldnn.enabled +@pytest.mark.parametrize( + "module_name,attribute,call", + [ + ("qnn_utils", "update_spill_fill_size", lambda f: f([])), + ( + "export_utils", + "QnnConfig", + lambda f: f(soc_model="SM8650", build_folder="/tmp/bf"), + ), + ( + "target_recipes", + "get_android_recipe", + lambda f: f("android-arm64-snapdragon-fp16"), + ), + ("quantizer", "QnnQuantizer", lambda f: f()), + ("qnn_utils", "from_context_binary", lambda f: f("nope.bin", "g")), + ("qnn_utils", "skip_annotation", lambda f: f(None, None, [], (), None)), + ( + "recipe_provider", + "QNNRecipeProvider", + lambda f: f().create_recipe( + QNNRecipeType.FP16 if QNNRecipeType else "qnn_fp16", + soc_model="SM8650", + ), + ), + ("cli", "execute", lambda f: f(None)), + ], +) +def test_every_entry_point_reaches_the_setup(monkeypatch, module_name, attribute, call): + """Each of these needs a usable SDK, and none of them is a lowering entry point. -def test_importing_the_package_does_not_set_up_the_sdk(): - """Import must not run the installer, which downloads and rewrites the environment. - - Asserted against the module source rather than by importing with a stub in place. A reload - re-executes the module, which rebinds the real installer and discards any stub, so a stubbed - reload can only ever observe an empty call list and would pass even if the call came back. - What actually matters is that no module level statement calls it. + Without a test per call site, deleting one is invisible: the whole premise is that setup no + longer happens at import, so a missing call is a broken path rather than a slower one. """ - module = ast.parse(Path(qnn.__file__).read_text()) - called_at_module_level = { - node.func.id - for statement in module.body - if isinstance(statement, ast.Expr) - for node in ast.walk(statement) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + # Imported here rather than at module scope, so one missing optional dependency cannot stop + # the whole file from being collected. + paths = { + "qnn_utils": "executorch.backends.qualcomm.utils.utils", + "export_utils": "executorch.backends.qualcomm.export_utils", + "target_recipes": "executorch.export.target_recipes", + "quantizer": "executorch.backends.qualcomm.quantizer.quantizer", + "recipe_provider": "executorch.backends.qualcomm.recipes.qnn_recipe_provider", + "cli": "executorch.examples.qualcomm.util_scripts.cli", + "qaihub_export": "executorch.examples.qualcomm.qaihub_scripts.utils.export", } + module = pytest.importorskip(paths[module_name]) + reached = [] + # Patched at the source module too, because some call sites import the helpers inside the + # function, where rebinding the caller's attribute would miss them. + for name in ("setup_qnn_sdk", "disable_mkldnn_on_amd"): + recorder = (lambda n: lambda *a, **k: reached.append(n))(name) + monkeypatch.setattr(qnn_sdk_setup, name, recorder) + if hasattr(module, name): + monkeypatch.setattr(module, name, recorder) + # Linux x86 only on some paths, so the platform gate must not short-circuit the probe. Patched + # where each module looks it up, since they import the platform module separately. + monkeypatch.setattr(platform, "system", lambda: "Linux") + monkeypatch.setattr(platform, "machine", lambda: "x86_64") + export_utils = pytest.importorskip("executorch.export.utils") + monkeypatch.setattr( + export_utils, "is_supported_platform_for_qnn_lowering", lambda: True + ) + if hasattr(module, "is_supported_platform_for_qnn_lowering"): + monkeypatch.setattr( + module, "is_supported_platform_for_qnn_lowering", lambda: True + ) + + # Every one of these fails on the stub input. What matters is what ran before it did. + try: + call(getattr(module, attribute)) + except Exception as error: # noqa: BLE001 + # A path that never got past its own availability guard proves nothing either way, and + # that depends on what is installed rather than on this change. + if not reached and "not available" in str(error): + pytest.skip(f"{module_name} is not usable in this environment: {error}") + + assert reached, f"{module_name}.{attribute} reaches neither helper" + + +def test_a_failed_install_does_not_leave_a_broken_sdk_path( + monkeypatch, ready_to_set_up +): + """A failed install must not look like a usable SDK to the next caller. + + The installer writes QNN_SDK_ROOT before it tries to load the library, so a failure after + that point leaves the variable pointing at a tree that does not work. Left in place, the next + call takes the preinstalled branch and reports success on a broken SDK. + """ + attempts = [] + + def poisoning_install(): + os.environ["QNN_SDK_ROOT"] = "/staged/but/broken" + attempts.append(1) + return False + + monkeypatch.setattr(qnn_sdk_setup, "_install_qnn_sdk", poisoning_install) + + with pytest.raises(RuntimeError): + qnn_sdk_setup.setup_qnn_sdk() + + assert os.environ.get("QNN_SDK_ROOT") is None - assert "install_qnn_sdk" not in called_at_module_level - assert "setup_qnn_sdk" not in called_at_module_level - # A guard on the two above, which would pass just as happily if the parse found nothing. - assert any( - isinstance(statement, ast.FunctionDef) and statement.name == "setup_qnn_sdk" - for statement in module.body + # The docstring promises the next caller tries again, which only holds if the path was cleared. + with pytest.raises(RuntimeError): + qnn_sdk_setup.setup_qnn_sdk() + + assert len(attempts) == 2 + + +def test_building_a_quantizer_applies_the_amd_guard(monkeypatch, fake_cpuinfo): + """Everything that calibrates builds a quantizer first, so the guard belongs there. + + The standalone quantize command in the example CLI is the case that made this matter: it runs + calibration itself, and calibration is a real eager run of the model, which is what the guard + is for. It reaches this constructor through make_quantizer. + """ + fake_cpuinfo("AuthenticAMD") + monkeypatch.setattr(qnn_sdk_setup, "_vendor_is_amd", None) + monkeypatch.setattr(qnn_sdk_setup, "_sdk_ready", True) + monkeypatch.setattr(torch.backends.mkldnn, "enabled", True) + quantizer_module = pytest.importorskip( + "executorch.backends.qualcomm.quantizer.quantizer" ) + quantizer_module.QnnQuantizer() + + assert not torch.backends.mkldnn.enabled -def test_the_package_imports_without_the_downloader(tmp_path, monkeypatch): - """The package must load in a build that does not ship the downloader directory. - Some builds assemble a package from an explicit file list and leave the sibling `scripts` - directory out. A module level import of it then fails, and with it every module in the - backend, which is what this guards. +def test_a_missing_cpuinfo_does_not_stop_a_lowering(monkeypatch): + """The AMD guard runs on every lowering, but it only matters on an AMD host. + + Raising when py-cpuinfo is absent would fail a lowering on an Intel or ARM machine over a + dependency that machine has no use for. """ - package = tmp_path / "qnnpkg" - package.mkdir() - (package / "__init__.py").write_text(Path(qnn.__file__).read_text()) - monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setattr(qnn_sdk_setup, "_vendor_is_amd", None) + monkeypatch.setitem(sys.modules, "cpuinfo", None) + monkeypatch.setattr(torch.backends.mkldnn, "enabled", True) - module = importlib.import_module("qnnpkg") + qnn_sdk_setup.disable_mkldnn_on_amd() - assert module.setup_qnn_sdk is not None + assert torch.backends.mkldnn.enabled -def test_platform_check_needs_no_downloader(tmp_path, monkeypatch): - """Setup must return on a platform with no prebuilt SDK, downloader present or not. +def test_setup_is_idempotent(ready_to_set_up, recorded_install): + """Several call sites each ask for it, so only the first may do the work.""" + qnn_sdk_setup.setup_qnn_sdk() + qnn_sdk_setup.setup_qnn_sdk() + + assert len(recorded_install) == 1 + + +def test_setup_honours_a_preinstalled_sdk( + monkeypatch, ready_to_set_up, recorded_install +): + monkeypatch.setenv("QNN_SDK_ROOT", "/opt/qcom/sdk") + + qnn_sdk_setup.setup_qnn_sdk() - The platform is decided locally for this reason. Asking the downloader would import it, so - a host that needs no download at all would fail on a build that does not ship it. + assert not recorded_install + + +def test_setup_leaves_an_empty_sdk_root_alone( + monkeypatch, ready_to_set_up, recorded_install +): + """Setting the variable at all means something else manages the SDK. + + An empty value used to read as "not set" and fall through to the installer, which downloads + an SDK and rewrites the environment underneath whatever set it. """ - package = tmp_path / "qnnpkg2" - package.mkdir() - (package / "__init__.py").write_text(Path(qnn.__file__).read_text()) - monkeypatch.syspath_prepend(str(tmp_path)) - monkeypatch.delenv("QNN_SDK_ROOT", raising=False) - monkeypatch.delenv("EXECUTORCH_BUILDING_WHEEL", raising=False) + monkeypatch.setenv("QNN_SDK_ROOT", "") + + qnn_sdk_setup.setup_qnn_sdk() + + assert not recorded_install + + +def test_setup_skips_a_wheel_build(monkeypatch, ready_to_set_up, recorded_install): + monkeypatch.setenv("EXECUTORCH_BUILDING_WHEEL", "1") + + qnn_sdk_setup.setup_qnn_sdk() - module = importlib.import_module("qnnpkg2") - monkeypatch.setattr(module, "_is_linux_x86", lambda: False) + assert not recorded_install - module.setup_qnn_sdk() +def test_setup_skips_a_platform_with_no_published_sdk( + monkeypatch, ready_to_set_up, recorded_install +): + monkeypatch.setattr(qnn_sdk_setup, "_is_linux_x86", lambda: False) -def test_a_missing_downloader_names_the_way_out(tmp_path, monkeypatch): - """On a platform that would download, an absent downloader has to say what to do instead. + qnn_sdk_setup.setup_qnn_sdk() + + assert not recorded_install + + +@pytest.mark.parametrize( + "system,machine,expected", + [ + ("Linux", "x86_64", True), + ("Linux", "AMD64", True), + ("Linux", "i686", True), + ("Linux", "aarch64", False), + ("Darwin", "arm64", False), + ("Darwin", "x86_64", False), + ("Windows", "AMD64", False), + ], +) +def test_the_platform_check_reads_the_real_platform( + monkeypatch, system, machine, expected +): + """Drives the real function, so its machine list is actually covered. + + Patching the check itself away would leave its body unexecuted, and the case it exists to + prevent, reaching for the downloader just to name the platform, would ship unnoticed. + """ + monkeypatch.setattr(platform, "system", lambda: system) + monkeypatch.setattr(platform, "machine", lambda: machine) + # Absent downloader, so answering at all proves the check does not consult it. + monkeypatch.setitem( + sys.modules, "executorch.backends.qualcomm.scripts.download_qnn_sdk", None + ) + + assert qnn_sdk_setup._is_linux_x86() is expected + + +def test_a_missing_downloader_names_the_way_out(monkeypatch, ready_to_set_up): + """An absent downloader has to say what to do, on a platform that would have downloaded. A bare ModuleNotFoundError names an internal packaging detail and leaves the reader nothing to act on. """ - package = tmp_path / "qnnpkg3" - package.mkdir() - (package / "__init__.py").write_text(Path(qnn.__file__).read_text()) - monkeypatch.syspath_prepend(str(tmp_path)) - monkeypatch.delenv("QNN_SDK_ROOT", raising=False) - monkeypatch.delenv("EXECUTORCH_BUILDING_WHEEL", raising=False) - module = importlib.import_module("qnnpkg3") - monkeypatch.setattr(module, "_is_linux_x86", lambda: True) + def no_downloader(name, *args, **kwargs): + if name.startswith("executorch.backends.qualcomm.scripts"): + raise ModuleNotFoundError(f"No module named {name!r}", name=name) + return original_import(name, *args, **kwargs) - with pytest.raises(RuntimeError, match="QNN_SDK_ROOT"): - module.setup_qnn_sdk() + original_import = builtins.__import__ + monkeypatch.setattr(builtins, "__import__", no_downloader) + # Matched on what the message promises, not just the variable name. Both error paths + # mention QNN_SDK_ROOT, so that alone cannot tell them apart or spot an empty message. + with pytest.raises(RuntimeError, match="cannot download"): + qnn_sdk_setup.setup_qnn_sdk() -def test_setup_is_idempotent(monkeypatch): - """The compile paths each call it, so only the first call may do the work.""" - calls = [] - monkeypatch.setattr(qnn, "_install_qnn_sdk", lambda: calls.append(1) or True) - monkeypatch.setattr(qnn, "_is_linux_x86", lambda: True) - monkeypatch.setattr(qnn, "_sdk_ready", False) - monkeypatch.delenv("QNN_SDK_ROOT", raising=False) - monkeypatch.delenv("EXECUTORCH_BUILDING_WHEEL", raising=False) - - qnn.setup_qnn_sdk() - qnn.setup_qnn_sdk() + with pytest.raises(RuntimeError, match=r"export QNN_SDK_ROOT="): + qnn_sdk_setup.setup_qnn_sdk() - assert len(calls) == 1 +def test_a_dependency_missing_inside_the_downloader_speaks_for_itself( + monkeypatch, ready_to_set_up +): + """Only an absent downloader is reworded. Its own missing dependency must not be hidden. -def test_setup_honours_a_preinstalled_sdk(monkeypatch): - calls = [] - monkeypatch.setattr(qnn, "_install_qnn_sdk", lambda: calls.append(1) or True) - monkeypatch.setattr(qnn, "_sdk_ready", False) - monkeypatch.setenv("QNN_SDK_ROOT", "/opt/qcom/sdk") + Reporting a missing `requests` as "set QNN_SDK_ROOT" would send the reader after the wrong + problem. + """ - qnn.setup_qnn_sdk() + def no_requests(name, *args, **kwargs): + if name == "requests": + raise ModuleNotFoundError("No module named 'requests'", name="requests") + return original_import(name, *args, **kwargs) + + original_import = builtins.__import__ + # Dropped from the module cache so the import below really runs and really fails on + # `requests`. A cached downloader would sail past the block and call the real installer, + # because the fixture has already cleared QNN_SDK_ROOT and forced the platform check true. + monkeypatch.delitem( + sys.modules, + "executorch.backends.qualcomm.scripts.download_qnn_sdk", + raising=False, + ) + monkeypatch.setattr(builtins, "__import__", no_requests) - assert not calls + with pytest.raises(ModuleNotFoundError, match="requests"): + qnn_sdk_setup.setup_qnn_sdk() -def test_setup_runs_once_under_concurrent_callers(monkeypatch): - """Several modules call it, so two threads can reach it at the same time. +def test_setup_runs_once_under_concurrent_callers(monkeypatch, ready_to_set_up): + """Several call sites reach it, so two threads can arrive at the same time. Without a lock they both see the flag unset and both run the installer, which downloads an - SDK and rewrites the environment. The import lock does not cover this, because the calls - come from different modules rather than from one package __init__. + SDK and rewrites the environment. """ import threading import time @@ -185,13 +682,9 @@ def slow_install(): calls.append(1) return True - monkeypatch.setattr(qnn, "_install_qnn_sdk", slow_install) - monkeypatch.setattr(qnn, "_is_linux_x86", lambda: True) - monkeypatch.setattr(qnn, "_sdk_ready", False) - monkeypatch.delenv("QNN_SDK_ROOT", raising=False) - monkeypatch.delenv("EXECUTORCH_BUILDING_WHEEL", raising=False) + monkeypatch.setattr(qnn_sdk_setup, "_install_qnn_sdk", slow_install) - threads = [threading.Thread(target=qnn.setup_qnn_sdk) for _ in range(8)] + threads = [threading.Thread(target=qnn_sdk_setup.setup_qnn_sdk) for _ in range(8)] for thread in threads: thread.start() for thread in threads: @@ -203,32 +696,79 @@ def slow_install(): assert len(calls) == 1 -def test_setup_reports_a_failed_install(monkeypatch): +def test_setup_reports_a_failed_install(monkeypatch, ready_to_set_up): """A failure has to name the two ways out, since it cannot be resolved automatically.""" - monkeypatch.setattr(qnn, "_install_qnn_sdk", lambda: False) - monkeypatch.setattr(qnn, "_is_linux_x86", lambda: True) - monkeypatch.setattr(qnn, "_sdk_ready", False) - monkeypatch.delenv("QNN_SDK_ROOT", raising=False) - monkeypatch.delenv("EXECUTORCH_BUILDING_WHEEL", raising=False) + monkeypatch.setattr(qnn_sdk_setup, "_install_qnn_sdk", lambda: False) - with pytest.raises(RuntimeError, match="QNN_SDK_ROOT"): - qnn.setup_qnn_sdk() + # Both ways out are asserted, so a message trimmed down to the variable name fails here. + with pytest.raises(RuntimeError, match="Download the SDK manually"): + qnn_sdk_setup.setup_qnn_sdk() + with pytest.raises(RuntimeError, match=r"export QNN_SDK_ROOT="): + qnn_sdk_setup.setup_qnn_sdk() + +@pytest.mark.parametrize("key", ["vendor_id_raw", "vendor_id"]) @pytest.mark.parametrize( "vendor,expected", [ - # The vendor string an AMD host reports. MKLDNN produces wrong results there. + # The vendor string an AMD host reports. MKLDNN crashes there. ("AuthenticAMD", False), ("GenuineIntel", True), # A host that reports nothing, such as Apple silicon. ("", True), ], ) -def test_mkldnn_is_disabled_only_on_amd(monkeypatch, fake_cpuinfo, vendor, expected): - fake_cpuinfo(vendor) +def test_mkldnn_is_disabled_only_on_amd( + monkeypatch, fake_cpuinfo, key, vendor, expected +): + """Releases of py-cpuinfo disagree on the key, so both spellings have to be read.""" + fake_cpuinfo(vendor, key=key) monkeypatch.setattr(torch.backends.mkldnn, "enabled", True) + monkeypatch.setattr(qnn_sdk_setup, "_vendor_is_amd", None) + + qnn_sdk_setup.disable_mkldnn_on_amd() + + assert torch.backends.mkldnn.enabled == expected + + +def test_the_vendor_is_read_at_most_once(monkeypatch): + """Reading it costs a subprocess, and the call sites reach it more than once per lowering. + + py-cpuinfo caches nothing, so without a cache here every lowering paid for the read several + times over, which was enough to push a delegate test job into its timeout. + """ + reads = [] + module = types.ModuleType("cpuinfo") + module.get_cpu_info = lambda: reads.append(1) or {"vendor_id_raw": "GenuineIntel"} + monkeypatch.setitem(sys.modules, "cpuinfo", module) + monkeypatch.setattr(qnn_sdk_setup, "_vendor_is_amd", None) + + for _ in range(5): + qnn_sdk_setup.disable_mkldnn_on_amd() + + assert len(reads) == 1 + + +def test_the_guard_holds_when_a_caller_re_enables_mkldnn(monkeypatch, fake_cpuinfo): + """A later lowering must not run with the setting back on. + + Caching the decision rather than the vendor would leave the second lowering in exactly the + configuration the guard exists to prevent, because the guard would return early. + """ + reads = [] + module = types.ModuleType("cpuinfo") + module.get_cpu_info = lambda: reads.append(1) or {"vendor_id_raw": "AuthenticAMD"} + monkeypatch.setitem(sys.modules, "cpuinfo", module) + monkeypatch.setattr(torch.backends.mkldnn, "enabled", True) + monkeypatch.setattr(qnn_sdk_setup, "_vendor_is_amd", None) + + qnn_sdk_setup.disable_mkldnn_on_amd() + assert not torch.backends.mkldnn.enabled - qnn.disable_mkldnn_on_amd() + torch.backends.mkldnn.enabled = True + qnn_sdk_setup.disable_mkldnn_on_amd() - assert torch.backends.mkldnn.enabled is expected + assert not torch.backends.mkldnn.enabled + # Re-applied without paying for the vendor read again. + assert len(reads) == 1 diff --git a/backends/qualcomm/utils/check_qnn_version.py b/backends/qualcomm/utils/check_qnn_version.py index d38629a78b1..700858b45d8 100644 --- a/backends/qualcomm/utils/check_qnn_version.py +++ b/backends/qualcomm/utils/check_qnn_version.py @@ -37,10 +37,19 @@ def _get_sdk_build_id(qnn_sdk_root: str): return PyQnnManagerAdaptor.GetQnnSdkBuildId(htp_library_path) +class QnnSdkRootNotSet(EnvironmentError): + """No SDK path is configured, so there is no version to read. + + Its own type because the version helpers below fall back when they see it, while any other + failure means the SDK is there but unreadable, which they must not hide. EnvironmentError is + an alias for OSError, so catching that would swallow both. + """ + + def get_sdk_build_id(): qnn_sdk_root = os.environ.get("QNN_SDK_ROOT") if not qnn_sdk_root: - raise EnvironmentError( + raise QnnSdkRootNotSet( "QNN_SDK_ROOT must be set to query the QNN SDK build id." ) return _get_sdk_build_id(qnn_sdk_root) @@ -49,7 +58,10 @@ def get_sdk_build_id(): def is_qnn_sdk_version_less_than(target_version): try: current_version = get_sdk_build_id() - except Exception: + except QnnSdkRootNotSet: + # No SDK path set, so there is no version to compare. Treated as older than any target, + # which is what the callers want when they gate a newer feature. Any other failure means + # the SDK is there but unreadable, and that must not be reported as an old version. return True match = re.search(r"v(\d+)\.(\d+)", current_version) @@ -68,7 +80,9 @@ def is_qnn_sdk_version_less_than(target_version): def is_qnn_sdk_version_greater_than(target_version): try: current_version = get_sdk_build_id() - except Exception: + except QnnSdkRootNotSet: + # No SDK path set, so there is no version to compare. Treated as not newer than any + # target, the conservative answer. Any other failure is left to propagate. return False match = re.search(r"v(\d+)\.(\d+)", current_version) diff --git a/backends/qualcomm/utils/qnn_manager_lifecycle.py b/backends/qualcomm/utils/qnn_manager_lifecycle.py index daee778d297..c5f42043d94 100644 --- a/backends/qualcomm/utils/qnn_manager_lifecycle.py +++ b/backends/qualcomm/utils/qnn_manager_lifecycle.py @@ -11,6 +11,10 @@ from executorch.backends.qualcomm.serialization.qc_schema_serialize import ( flatbuffer_to_option, ) +from executorch.backends.qualcomm.utils.qnn_sdk_setup import ( + disable_mkldnn_on_amd, + setup_qnn_sdk, +) from executorch.exir.backend.compile_spec_schema import CompileSpec # Thread-local storage for QnnManager instances @@ -25,6 +29,12 @@ def __init__(self): def get_or_create_qnn_manager( self, backend_type: QnnExecuTorchBackendType, option: bytes ) -> PyQnnManager.QnnManager: + # Outside the branch below, so reusing a cached manager still re-applies them. Both are + # cheap on a repeat call, and the AMD guard has to hold for every lowering, not only the + # one that happened to build the manager. + setup_qnn_sdk() + disable_mkldnn_on_amd() + if backend_type not in self._registry: qnn_manager = PyQnnManager.QnnManager(option) err = qnn_manager.InitBackend() @@ -90,4 +100,8 @@ def get_current_qnn_manager( return QnnManagerRegistry().get_or_create_qnn_manager( backend_type, generate_qnn_executorch_option(compile_specs) ) + + # Re-applied even though the manager already exists, because a caller may have turned the + # setting back on since it was built, and this is a lowering about to run. + disable_mkldnn_on_amd() return active_registry._registry[backend_type] diff --git a/backends/qualcomm/utils/qnn_sdk_setup.py b/backends/qualcomm/utils/qnn_sdk_setup.py new file mode 100644 index 00000000000..307457aafc1 --- /dev/null +++ b/backends/qualcomm/utils/qnn_sdk_setup.py @@ -0,0 +1,183 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Making the Qualcomm SDK usable, on the paths that need it. + +Deliberately not in the package's ``__init__``, where it used to run: loading the package needs +no SDK, because the native adaptor links no QNN library and resolves those symbols with +``dlopen`` when a backend starts. So setup belongs on the calls that start one. +""" + +import logging +import os +import platform +import threading + +_sdk_ready = False +# Guards the flag above. Two threads starting a backend at the same time would otherwise both +# run an installer that downloads and rewrites the environment. +_sdk_lock = threading.Lock() + +_vendor_is_amd = None +_vendor_lock = threading.Lock() + + +def setup_qnn_sdk() -> None: + """Make the Qualcomm SDK usable in this process, once. + + Safe to call repeatedly and from more than one thread. After a success, later calls return + immediately. After a failure nothing is remembered, so the next caller tries again, which is + deliberate: a download can fail for a reason that goes away, such as a dropped network. + """ + if _sdk_ready: + return + with _sdk_lock: + # Another thread may have finished while this one waited. + if _sdk_ready: + return + _setup_qnn_sdk_locked() + + +def _setup_qnn_sdk_locked() -> None: + global _sdk_ready + + # The wheel build imports this package to collect its files, and has no use for an SDK. + if os.getenv("EXECUTORCH_BUILDING_WHEEL", "0").lower() in ("1", "true", "yes"): + _sdk_ready = True + return + + # An empty value counts as set, because a caller that exports the variable at all has taken + # charge of the SDK, often supplying it through LD_LIBRARY_PATH instead. Treating that as + # unset downloads a second copy and rewrites both variables underneath them. + # + # Anywhere that goes on to build a real path from the value rejects an empty one. Read the + # two rules together as: empty means "not mine to fetch", not "a usable SDK lives here". + qnn_sdk_root = os.getenv("QNN_SDK_ROOT") + if qnn_sdk_root is not None: + # Reported differently when empty, because naming it as a path would read as though a + # location had been found. + if qnn_sdk_root: + logging.info("[QNN] Using QNN SDK at %s (from QNN_SDK_ROOT)", qnn_sdk_root) + else: + logging.info( + "[QNN] QNN_SDK_ROOT is set but empty, so the SDK is left to the caller" + ) + _sdk_ready = True + return + + # Decided here rather than by asking the downloader, so a host with no published SDK returns + # without needing a module some builds do not package. + if not _is_linux_x86(): + _sdk_ready = True + return + + if not _install_qnn_sdk(): + from executorch.backends.qualcomm.scripts.download_qnn_sdk import QNN_ZIP_URL + + # Cleared because the installer writes QNN_SDK_ROOT before it tries to load the library, + # so a failure here leaves it pointing at a tree that does not work. Left in place, the + # next call would take the preinstalled branch above and report success on a broken SDK, + # and a later version query aborts the interpreter rather than raising. + os.environ.pop("QNN_SDK_ROOT", None) + + raise RuntimeError( + "Failed to set up QNN SDK.\n\n" + "To resolve, try one of:\n" + " 1. Download the SDK manually from:\n" + f" {QNN_ZIP_URL}\n" + " Or go to step 2 if QNN SDK already exists.\n" + " 2. Set QNN_SDK_ROOT to an existing SDK installation:\n" + " export QNN_SDK_ROOT=/path/to/qualcomm/sdk\n" + " export LD_LIBRARY_PATH=" + "$QNN_SDK_ROOT/lib/x86_64-linux-clang/:$LD_LIBRARY_PATH" + ) + + _sdk_ready = True + + +def _is_linux_x86() -> bool: + """True when a prebuilt Qualcomm SDK is published for this platform.""" + return platform.system().lower() == "linux" and platform.machine().lower() in ( + "x86_64", + "amd64", + "i386", + "i686", + ) + + +def _install_qnn_sdk() -> bool: + # Imported here rather than at module scope because the downloader lives in a directory some + # builds do not package, and it imports the network stack. + try: + from executorch.backends.qualcomm.scripts.download_qnn_sdk import ( + install_qnn_sdk, + ) + except ModuleNotFoundError as error: + # Only when the downloader itself is absent. A dependency missing from inside it is a + # different problem and is left to speak for itself. + if not (error.name or "").startswith("executorch.backends.qualcomm.scripts"): + raise + raise RuntimeError( + "This build cannot download a QNN SDK. Set QNN_SDK_ROOT to an existing " + "installation:\n" + " export QNN_SDK_ROOT=/path/to/qualcomm/sdk\n" + " export LD_LIBRARY_PATH=" + "$QNN_SDK_ROOT/lib/x86_64-linux-clang/:$LD_LIBRARY_PATH" + ) from error + + return install_qnn_sdk() + + +def disable_mkldnn_on_amd() -> None: + """Turn off PyTorch's MKLDNN backend on an AMD host. + + MKLDNN crashes on some AMD hosts, which is why this exists. The original comment described + it as producing wrong results; what was measured is a core dump, from a plain convolution + with nothing from this backend involved. + + This changes a global PyTorch setting, so it is applied by the calls that start a backend + rather than at import, where it would also change how unrelated models run in the same + interpreter. + + The setting is re-applied on every call, since a caller may have turned it back on in + between, but the vendor behind the decision is read only once. Reading it spawns a + subprocess, and the call sites reach here more than once per lowering. + """ + if not _host_is_amd(): + return + + import torch + + torch.backends.mkldnn.enabled = False + + +def _host_is_amd() -> bool: + global _vendor_is_amd + if _vendor_is_amd is not None: + return _vendor_is_amd + with _vendor_lock: + if _vendor_is_amd is None: + _vendor_is_amd = _read_cpu_vendor().lower().find("amd") != -1 + return _vendor_is_amd + + +def _read_cpu_vendor() -> str: + try: + import cpuinfo + except ImportError: + # Warned about rather than raised, because this runs on every lowering and the guard it + # feeds only matters on an AMD host. Aborting here would fail a lowering on an Intel or + # ARM machine over a dependency that machine has no use for. + logging.warning( + "[QNN] py-cpuinfo is not installed, so the CPU vendor cannot be read and the AMD " + "MKLDNN workaround is skipped. Install it with: pip install py-cpuinfo" + ) + return "" + + # Releases of py-cpuinfo disagree on the key: older ones report "vendor_id", newer ones + # "vendor_id_raw". Reading only one of them finds nothing on the other, silently. + info = cpuinfo.get_cpu_info() + return info.get("vendor_id_raw") or info.get("vendor_id") or "" diff --git a/backends/qualcomm/utils/utils.py b/backends/qualcomm/utils/utils.py index 32da32288fd..fb88c5a5d45 100644 --- a/backends/qualcomm/utils/utils.py +++ b/backends/qualcomm/utils/utils.py @@ -61,6 +61,10 @@ QCOM_QUANTIZED_IO, ) from executorch.backends.qualcomm.utils.qnn_manager_lifecycle import QnnManagerContext +from executorch.backends.qualcomm.utils.qnn_sdk_setup import ( + disable_mkldnn_on_amd, + setup_qnn_sdk, +) from executorch.exir import ExirExportedProgram, to_edge from executorch.exir.backend.compile_spec_schema import CompileSpec from executorch.exir.lowered_backend_module import LoweredBackendModule @@ -239,6 +243,10 @@ def dump_context_from_pte(pte_path) -> List[str]: def update_spill_fill_size( exported_program: ExportedProgram | List[LoweredBackendModule], ): + # Reads a context binary through a QnnManager, so the SDK has to be usable first. + setup_qnn_sdk() + disable_mkldnn_on_amd() + # check if user specifies to use multi_contexts # this is a generic approach in case there exists multiple backends def get_program_info(program): @@ -384,6 +392,17 @@ def to_edge_transform_and_lower_to_qnn( EdgeProgramManager: The manager for the edge program after transformation and lowering. """ + # Both applied at the entry point rather than deeper in the lowering. + # + # The SDK setup can download one, and on an old glibc the installer re-executes the + # interpreter to pick up a staged loader. Doing that partway through would throw away a model + # that has already been traced, so it happens before any of that work. + # + # The AMD guard needs to precede any eager run of the model. The crash it prevents needs a + # real convolution to execute, which a trace does not do (it works on fake tensors) but + # calibration and a plain forward pass do. + setup_qnn_sdk() + disable_mkldnn_on_amd() def ensure_graph_specific_dict(value, graph_names): """ @@ -517,6 +536,10 @@ def capture_program( DeprecationWarning, stacklevel=1, ) + # See to_edge_transform_and_lower_to_qnn for why both happen at the entry point. + setup_qnn_sdk() + disable_mkldnn_on_amd() + ep = torch.export.export(module, inputs, dynamic_shapes=dynamic_shapes, strict=True) pass_manager = get_qnn_pass_manager_cls(QnnExecuTorchBackendType.kHtpBackend)() ep = pass_manager.transform_for_export_pipeline(ep) @@ -726,6 +749,12 @@ def skip_annotation( from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + # Both before the tracing and calibration below. Calibration runs the model eagerly, which is + # what the AMD guard is for, and the installer can re-execute the interpreter, which must not + # happen after a model has been traced. + setup_qnn_sdk() + disable_mkldnn_on_amd() + def prepare_subgm(subgm, subgm_name): # prepare current submodule for quantization annotation subgm_prepared = prepare_pt2e(subgm, quantizer) @@ -810,6 +839,10 @@ def from_context_binary( # noqa: C901 soc_model: QcomChipset = QcomChipset.SM8650, custom_info: Dict = None, ): + # Reads a context binary through a QnnManager, so the SDK has to be usable first. + setup_qnn_sdk() + disable_mkldnn_on_amd() + from pathlib import Path def implement_op(custom_op, op_name, outputs): @@ -1257,12 +1290,22 @@ def generate_qnn_executorch_compiler_spec( # noqa: C901 "Please choose the following SOC: " f"{list(get_soc_to_lpai_hw_ver_map().keys())}" ) - elif get_soc_to_lpai_hw_ver_map()[ + # Before the version check below, because setup may install a newer SDK than the one this + # process can currently see. Asking first would read the version of whatever happened to be + # there and could reject a target the installed SDK actually supports. + setup_qnn_sdk() + if get_soc_to_lpai_hw_ver_map()[ soc_model.name ] == LpaiHardwareVersion.V6 and is_qnn_sdk_version_less_than("2.39"): + # Read once, because building this message by querying again raises when there is no + # SDK, which replaces the useful error below with a confusing one. + try: + current = get_sdk_build_id() + except Exception: + current = "unknown, no usable SDK found" raise ValueError( f"Target soc_model({soc_model.name}) with LPAI backend v6 requires QNN SDK version >= 2.39. \n" - f"Current QNN SDK version: {get_sdk_build_id()}" + f"Current QNN SDK version: {current}" ) qnn_executorch_options.shared_buffer = shared_buffer diff --git a/examples/qualcomm/oss_scripts/llama/wrappers/base_component.py b/examples/qualcomm/oss_scripts/llama/wrappers/base_component.py index 1fc08163912..93f9b87b4e2 100644 --- a/examples/qualcomm/oss_scripts/llama/wrappers/base_component.py +++ b/examples/qualcomm/oss_scripts/llama/wrappers/base_component.py @@ -23,6 +23,7 @@ get_sdk_build_id, is_qnn_sdk_version_less_than, ) +from executorch.backends.qualcomm.utils.qnn_sdk_setup import setup_qnn_sdk from executorch.examples.qualcomm.oss_scripts.llama import LLMModelConfig from executorch.examples.qualcomm.oss_scripts.llama.decoder_constants import ( AUDIO_ENCODER, @@ -118,9 +119,18 @@ def process_model_args( model_args.kv_io_bit_width = quant_recipe.get_kv_io_bit_width() if config.masked_softmax: + # Before the version check, because setup may install a newer SDK than this process can + # currently see, and asking first could disable the feature on an SDK that supports it. + setup_qnn_sdk() if is_qnn_sdk_version_less_than("2.35"): + # Read once, because building this message by querying again raises when there is no + # SDK, so the warning that disables the feature would fail instead of disabling it. + try: + current = get_sdk_build_id() + except Exception: + current = "unknown, no usable SDK found" logging.warning( - f"Masked softmax is supported after QNN SDK 2.35. Given sdk version {get_sdk_build_id()}" + f"Masked softmax is supported after QNN SDK 2.35. Given sdk version {current}" " is lower the target version. Disabling the feature." ) model_args.enable_masked_softmax = False diff --git a/examples/qualcomm/qaihub_scripts/utils/export.py b/examples/qualcomm/qaihub_scripts/utils/export.py index a144e74a82c..645f6266ddf 100644 --- a/examples/qualcomm/qaihub_scripts/utils/export.py +++ b/examples/qualcomm/qaihub_scripts/utils/export.py @@ -168,7 +168,6 @@ def to_context_binary( logger.info(f"Generating context binary for {model_lib}") # leverage SimpleADB for model library conversion lib_name = Path(model_lib).stem - sdk_root = os.getenv("QNN_SDK_ROOT") qnn_config = QnnConfig( soc_model=soc_model, build_folder=build_folder, @@ -176,6 +175,9 @@ def to_context_binary( host=host, target=target, ) + # Read after the config is built, because building it is what sets up the SDK and so may be + # what puts this variable in the environment. + sdk_root = os.getenv("QNN_SDK_ROOT") adb = SimpleADB( qnn_config=qnn_config, pte_path=model_lib, diff --git a/examples/qualcomm/util_scripts/cli.py b/examples/qualcomm/util_scripts/cli.py index 914145c9095..fc31af40da1 100644 --- a/examples/qualcomm/util_scripts/cli.py +++ b/examples/qualcomm/util_scripts/cli.py @@ -38,6 +38,7 @@ QnnExecuTorchLpaiTargetEnv, ) from executorch.backends.qualcomm.utils.constants import QCOM_PASS_ACTIVATE_KEY +from executorch.backends.qualcomm.utils.qnn_sdk_setup import setup_qnn_sdk from executorch.backends.qualcomm.utils.utils import ( draw_graph, dump_context_from_pte, @@ -298,6 +299,10 @@ def compile(args): def execute(args): logger = get_logger() + # The SDK has to be usable before the graph metadata is read below, because that opens a + # native QNN manager. It used to be set up as a side effect of importing the backend. + setup_qnn_sdk() + pte_name = Path(args.artifact).stem # get input order diff --git a/export/target_recipes.py b/export/target_recipes.py index eac35c08bf7..29472f0d288 100644 --- a/export/target_recipes.py +++ b/export/target_recipes.py @@ -7,7 +7,7 @@ """ Target-specific recipe functions for simplified multi-backend deployment. -This module provides platform-specific functions that abstract away backend +This module provides platform-specific functions that abstract away backend selection and combine multiple backends optimally for target hardware. """ @@ -146,21 +146,16 @@ def get_android_recipe( ) try: - # Qualcomm QNN backend runs QNN sdk download on first use - # with a pip install, so wrap it in a try/except # pyre-ignore from executorch.backends.qualcomm.recipes import QNNRecipeType - - # (1) if this is called from a pip install, the QNN SDK will be available - # (2) if this is called from a source build, check if qnn is available otherwise, had to run build.sh - if os.getenv("QNN_SDK_ROOT", None) is None: - raise ValueError( - "QNN SDK not found, cannot use QNN recipes. First run `./backends/qualcomm/scripts/build.sh`, if building from source" - ) - except Exception as e: + from executorch.backends.qualcomm.utils import qnn_sdk_setup + except ImportError as e: + # Only an import failure means the backend is not installed. Catching everything here + # would rewrite a real bug inside those modules as a missing backend and hide its + # traceback. raise ValueError( "QNN backend is not available. Please ensure the Qualcomm backend " - "is properly installed and configured, " + "is properly installed and configured." ) from e android_configs: Dict[str, List[RecipeType]] = { @@ -168,6 +163,7 @@ def get_android_recipe( "android-arm64-snapdragon-fp16": [QNNRecipeType.FP16, XNNPackRecipeType.FP32], } + # Rejected before the SDK is fetched, so a mistyped target does not cost a full download. if target_config not in android_configs: supported = list(android_configs.keys()) raise ValueError( @@ -175,6 +171,29 @@ def get_android_recipe( f"Supported: {supported}" ) + # Outside the import block, so a setup failure says what actually went wrong rather than + # being rewritten as "the backend is not available". + # + # The SDK is fetched here for a pip install that has not set one up by hand. It used to + # happen while the backend was imported, which meant every import fetched an SDK whether + # or not one was wanted. + qnn_sdk_setup.setup_qnn_sdk() + + # Checked for a usable path, not merely for the variable being present. Setup treats an empty + # value as the caller taking charge of the SDK, but a recipe needs a real path, and every other + # reader of this variable rejects an empty one. + qnn_sdk_root = os.getenv("QNN_SDK_ROOT") + if qnn_sdk_root is not None and not qnn_sdk_root: + raise ValueError( + "QNN_SDK_ROOT is set but empty, so no SDK was fetched. Set it to the path of an SDK, " + "or unset it to have one downloaded." + ) + if not qnn_sdk_root: + raise ValueError( + "QNN SDK not found, cannot use QNN recipes. First run " + "`./backends/qualcomm/scripts/build.sh`, if building from source" + ) + kwargs = kwargs or {} if target_config == "android-arm64-snapdragon-fp16": diff --git a/setup.py b/setup.py index 89470b2644f..c80131657f5 100644 --- a/setup.py +++ b/setup.py @@ -657,11 +657,10 @@ def _base_dependencies() -> List[str]: "packaging", "pandas>=2.2.2; python_version >= '3.10'", "parameterized", - # backends/qualcomm/__init__.py cannot be imported from a clean install - # without both of these. It reads the CPU vendor to disable an mkldnn path on - # AMD, and the module it imports first does a module-scope `import requests`, - # so declaring only the cpuinfo half leaves the import failing on the line - # before. + # The Qualcomm backend needs both, on different paths. Any lowering reads the CPU vendor + # to disable an mkldnn path that crashes on AMD, whether or not an SDK is already set up. + # Fetching an SDK additionally needs requests, which its downloader imports at module + # scope. Neither is needed merely to import the backend. "py-cpuinfo", "requests", "pytorch-tokenizers",