diff --git a/scripts/generate_wrappers.py b/scripts/generate_wrappers.py
index 16c0bf64b..4b98dfb9b 100644
--- a/scripts/generate_wrappers.py
+++ b/scripts/generate_wrappers.py
@@ -831,8 +831,10 @@ def _generate_call(op_name, call, method=True, supports_triton_config=False):
                     "    if (config_dict.has_value()) {\n"
                     "      triton_config_ptr = "
                     "triton::jit::ConfigFromPyDict(*config_dict);\n"
-                    "      triton_config_ptr->set_implementation_index(\n"
-                    "          config.implementation_index());\n"
+                    "      if (!config.needs_implementation_resolution()) {\n"
+                    "        triton_config_ptr->set_implementation_index(\n"
+                    "            config.implementation_index());\n"
+                    "      }\n"
                     "    }\n"
                 )
                 extra_pybind = ', py::arg("config") = py::none()'
@@ -872,7 +874,7 @@ def _generate_call(op_name, call, method=True, supports_triton_config=False):
                 f"    Config config;\n"
                 f"    if (implementation_index.has_value()) {{\n"
                 f"      config.set_implementation_index(*implementation_index);\n"
-                f"    }} else {{\n"
+                f"    }} else if (!TuningManager::Instance().IsEnabled()) {{\n"
                 f"      config.set_implementation_index(\n"
                 f"          {default_impl_index});\n"
                 f"    }}\n"
@@ -959,7 +961,8 @@ def _overload_order_key(node):
 #include "generated/bindings/generated_dispatch.h"
 #include "handle.h"
 #include "host_range_profiler.h"
-#include "pybind11_utils.h"{triton_config_include}
+#include "pybind11_utils.h"
+#include "tuning.h"{triton_config_include}
 
 namespace py = pybind11;
 
@@ -2068,13 +2071,15 @@ def _generate_ops_module_source(bind_func_names, op_includes=(), monolithic=Fals
             for bind_func_name in bind_func_names
         )
 
-    module_calls = "BindHostRangeProfileControls(m);"
+    module_calls = """TuningManager::Instance().InitializeFromEnvironment();
+BindHostRangeProfileControls(m);"""
     if bind_func_calls:
         module_calls = f"{module_calls}\n{bind_func_calls}"
 
     return f"""#include <pybind11/pybind11.h>
 
 #include "host_range_profiler.h"
+#include "tuning.h"
 
 {pre_namespace}
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 91ee03e57..4b245bcb3 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -51,6 +51,19 @@ endfunction()
 
 include(GNUInstallDirs)
 
+find_package(nlohmann_json 3.12.0 CONFIG QUIET)
+if(NOT TARGET nlohmann_json::nlohmann_json)
+    if(POLICY CMP0135)
+        cmake_policy(SET CMP0135 NEW)
+    endif()
+    include(FetchContent)
+    FetchContent_Declare(nlohmann_json
+        URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz
+        URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa
+    )
+    FetchContent_MakeAvailable(nlohmann_json)
+endif()
+
 file(GLOB BASE_SRCS CONFIGURE_DEPENDS "*.cc")
 list(FILTER BASE_SRCS EXCLUDE REGEX ".*tensor\\.cc$")
 target_sources(infiniops PRIVATE ${BASE_SRCS})
@@ -59,6 +72,7 @@ target_link_libraries(infiniops PUBLIC
     $<BUILD_INTERFACE:infinirt>
     $<INSTALL_INTERFACE:InfiniRT::infinirt>
 )
+target_link_libraries(infiniops PRIVATE nlohmann_json::nlohmann_json)
 
 set(INFINI_RT_INCLUDE_FLAGS "")
 foreach(_include_dir IN LISTS INFINI_RT_INCLUDE_DIRS)
diff --git a/src/config.h b/src/config.h
index e156497bd..ac371e13e 100644
--- a/src/config.h
+++ b/src/config.h
@@ -3,6 +3,7 @@
 
 #include <cstddef>
 #include <memory>
+#include <optional>
 
 #include "cloneable.h"
 
@@ -16,14 +17,20 @@ class Config {
     return std::make_unique<Config>(*this);
   }
 
-  std::size_t implementation_index() const { return implementation_index_; }
+  std::size_t implementation_index() const {
+    return implementation_index_.value_or(0);
+  }
 
   void set_implementation_index(std::size_t implementation_index) {
     implementation_index_ = implementation_index;
   }
 
+  bool needs_implementation_resolution() const {
+    return !implementation_index_.has_value();
+  }
+
  private:
-  std::size_t implementation_index_{0};
+  std::optional<std::size_t> implementation_index_;
 };
 
 }  // namespace infini::ops
diff --git a/src/operator.h b/src/operator.h
index 9ab9ce71a..172420484 100644
--- a/src/operator.h
+++ b/src/operator.h
@@ -1,10 +1,14 @@
 #ifndef INFINI_OPS_OPERATOR_H_
 #define INFINI_OPS_OPERATOR_H_
 
+#include <algorithm>
 #include <atomic>
 #include <cassert>
+#include <chrono>
 #include <cstdio>
 #include <cstdlib>
+#include <iostream>
+#include <limits>
 #include <memory>
 #include <optional>
 #include <string_view>
@@ -18,7 +22,9 @@
 #include "dispatcher.h"
 #include "handle.h"
 #include "host_range_profiler.h"
+#include "runtime.h"
 #include "tensor.h"
+#include "tuning.h"
 
 namespace infini::ops::detail {
 
@@ -79,6 +85,34 @@ bool ListContains(ValueType value, List<values...>) {
   return ((value == static_cast<ValueType>(values)) || ...);
 }
 
+inline void SyncDevice(Device::Type dev_type) {
+  if (!ListContains(dev_type, ActiveDevices<void>{})) {
+    return;
+  }
+  DispatchFunc<ActiveDevices<void>>(
+      dev_type,
+      [](auto device_tag) {
+        constexpr Device::Type kDev = decltype(device_tag)::value;
+        infini::rt::runtime::Runtime<kDev>::DeviceSynchronize();
+      },
+      "SyncDevice");
+}
+
+inline Device::Type FirstDeviceType() { return Device::Type::kCount; }
+
+template <typename First, typename... Rest>
+Device::Type FirstDeviceType(const First& first, const Rest&... rest) {
+  if constexpr (std::is_same_v<std::decay_t<First>, Tensor>) {
+    return first.device().type();
+  } else if constexpr (std::is_same_v<std::decay_t<First>,
+                                      std::vector<Tensor>>) {
+    return first.empty() ? FirstDeviceType(rest...)
+                         : first.front().device().type();
+  } else {
+    return FirstDeviceType(rest...);
+  }
+}
+
 template <typename TensorLike, typename = void>
 class IsTensorLike : public std::false_type {};
 
@@ -199,6 +233,21 @@ struct CacheKeyBuilder {
   }
 };
 
+namespace detail {
+
+template <typename Key, typename... Args>
+std::size_t ResolveImplementationIndex(const Config& config,
+                                       Device::Type dev_type,
+                                       const Args&... args);
+
+template <typename Key, typename... Args>
+std::size_t ResolveImplementationIndexOnline(const Handle& handle,
+                                             const Config& config,
+                                             Device::Type dev_type,
+                                             const Args&... args);
+
+}  // namespace detail
+
 template <typename Key, Device::Type kDev>
 struct ActiveImplementations;
 
@@ -246,13 +295,24 @@ class Operator : public OperatorBase {
   template <typename... Args>
   static std::unique_ptr<Operator> Make(const Config& config,
                                         const Tensor tensor, Args&&... args) {
-    return MakeWithDevice(config, tensor.device().type(), tensor,
+    const auto dev_type = tensor.device().type();
+    if (!TuningManager::Instance().IsEnabled() ||
+        !config.needs_implementation_resolution()) {
+      return MakeWithDevice(config, dev_type, tensor,
+                            std::forward<Args>(args)...);
+    }
+
+    auto resolved_config = config.Clone();
+    resolved_config->set_implementation_index(
+        detail::ResolveImplementationIndex<Key>(config, dev_type, tensor,
+                                                args...));
+    return MakeWithDevice(*resolved_config, dev_type, tensor,
                           std::forward<Args>(args)...);
   }
 
   template <typename... Args>
   static std::unique_ptr<Operator> Make(const Tensor tensor, Args&&... args) {
-    return Make(DefaultConfig(tensor.device().type()), tensor,
+    return Make(ImplicitConfig(tensor.device().type()), tensor,
                 std::as_const(args)...);
   }
 
@@ -262,7 +322,18 @@ class Operator : public OperatorBase {
                                         Args&&... args) {
     assert(!tensors.empty() && "operator tensor list input cannot be empty");
 
-    return MakeWithDevice(config, tensors.front().device().type(), tensors,
+    const auto dev_type = tensors.front().device().type();
+    if (!TuningManager::Instance().IsEnabled() ||
+        !config.needs_implementation_resolution()) {
+      return MakeWithDevice(config, dev_type, tensors,
+                            std::forward<Args>(args)...);
+    }
+
+    auto resolved_config = config.Clone();
+    resolved_config->set_implementation_index(
+        detail::ResolveImplementationIndex<Key>(config, dev_type, tensors,
+                                                args...));
+    return MakeWithDevice(*resolved_config, dev_type, tensors,
                           std::forward<Args>(args)...);
   }
 
@@ -271,7 +342,7 @@ class Operator : public OperatorBase {
                                         Args&&... args) {
     assert(!tensors.empty() && "operator tensor list input cannot be empty");
 
-    return Make(DefaultConfig(tensors.front().device().type()), tensors,
+    return Make(ImplicitConfig(tensors.front().device().type()), tensors,
                 std::as_const(args)...);
   }
 
@@ -293,12 +364,28 @@ class Operator : public OperatorBase {
       generation = cache_generation;
     }
 
+    std::unique_ptr<Config> resolved_config;
+    const Config* effective_config = &config;
+    if (TuningManager::Instance().IsEnabled() &&
+        config.needs_implementation_resolution()) {
+      const auto dev_type = detail::FirstDeviceType(args...);
+      assert(dev_type != Device::Type::kCount &&
+             "operator call requires at least one tensor argument");
+
+      const auto resolved_implementation_index =
+          detail::ResolveImplementationIndexOnline<Key>(handle, config,
+                                                        dev_type, args...);
+      resolved_config = config.Clone();
+      resolved_config->set_implementation_index(resolved_implementation_index);
+      effective_config = resolved_config.get();
+    }
+
 #if defined(INFINI_OPS_ENABLE_HOST_RANGE_PROFILING)
     auto key = [&]() {
       HostRangeScope host_range_cache_key{HostRangeLayer::kCacheKey};
-      return CacheKeyBuilder<Key>{}(config, args...);
+      return CacheKeyBuilder<Key>{}(*effective_config, args...);
     }();
-    detail::TraceOperatorCall<Key>(key, config);
+    detail::TraceOperatorCall<Key>(key, *effective_config);
 
     auto it = [&]() {
       HostRangeScope host_range_cache_lookup{HostRangeLayer::kCacheLookup};
@@ -308,17 +395,18 @@ class Operator : public OperatorBase {
     if (it == cache.end()) {
       HostRangeScope host_range_cache_construct{
           HostRangeLayer::kCacheConstruct};
-      auto new_op = Make(config, args...);
+      auto new_op = Make(*effective_config, args...);
       it = cache.emplace(std::move(key), std::move(new_op)).first;
     }
 #else
-    auto key = CacheKeyBuilder<Key>{}(config, args...);
-    detail::TraceOperatorCall<Key>(key, config);
+    auto key = CacheKeyBuilder<Key>{}(*effective_config, args...);
+    detail::TraceOperatorCall<Key>(key, *effective_config);
 
     auto it{cache.find(key)};
 
     if (it == cache.end()) {
-      it = cache.emplace(std::move(key), Make(config, args...)).first;
+      it =
+          cache.emplace(std::move(key), Make(*effective_config, args...)).first;
     }
 #endif
 
@@ -331,7 +419,7 @@ class Operator : public OperatorBase {
 
   template <typename... Args>
   static void Call(const Tensor tensor, const Args&... args) {
-    return Call({}, DefaultConfig(tensor.device().type()), tensor, args...);
+    return Call({}, ImplicitConfig(tensor.device().type()), tensor, args...);
   }
 
   template <
@@ -414,6 +502,11 @@ class Operator : public OperatorBase {
     return config;
   }
 
+  static Config ImplicitConfig(Device::Type dev_type) {
+    if (TuningManager::Instance().IsEnabled()) return Config{};
+    return DefaultConfig(dev_type);
+  }
+
   template <typename TensorLike, typename... Args>
   static auto CallReturning(const TensorLike& tensor, const Args&... args) {
     auto out = Key::MakeReturnValue(tensor, args...);
@@ -505,6 +598,126 @@ struct ActiveImplementations {
       Key, kDev, std::make_index_sequence<kMaxImplementations>>::type;
 };
 
+namespace detail {
+
+template <typename Key, typename... Args>
+std::size_t ResolveImplementationIndex(const Config& config,
+                                       Device::Type dev_type,
+                                       const Args&... args) {
+  if (!config.needs_implementation_resolution()) {
+    return config.implementation_index();
+  }
+
+  auto indices = Operator<Key>::active_implementation_indices(dev_type);
+  if (indices.empty()) return config.implementation_index();
+
+  auto signature = TuningSignature::Build(args...);
+  constexpr auto op_name = OperatorName<Key>();
+  auto tuned_index =
+      TuningManager::Instance().Lookup(op_name, dev_type, signature);
+  auto chosen = indices.front();
+
+  if (tuned_index.has_value()) {
+    bool is_valid = std::find(indices.begin(), indices.end(), *tuned_index) !=
+                    indices.end();
+    if (is_valid) {
+      chosen = *tuned_index;
+    } else {
+      std::cerr << "[Tuning] Warning: tuned implementation " << *tuned_index
+                << " for " << op_name << " on "
+                << Device::StringFromType(dev_type)
+                << " is not available (compiled indices:";
+      for (auto idx : indices) std::cerr << " " << idx;
+      std::cerr << "), falling back to " << chosen << std::endl;
+    }
+  }
+
+  return chosen;
+}
+
+template <typename Key, typename... Args>
+double BenchmarkImplementation(const Handle& handle, Device::Type dev_type,
+                               std::size_t impl_index, const Args&... args) {
+  Config fixed;
+  fixed.set_implementation_index(impl_index);
+
+  auto op = Operator<Key>::Make(fixed, args...);
+
+  const auto& tuning = TuningManager::Instance();
+  const int warmup = tuning.warmup_count();
+  const int repeat = tuning.repeat_count();
+
+  for (int i = 0; i < warmup; ++i) {
+    (*op)(handle, args...);
+  }
+  SyncDevice(dev_type);
+
+  double best = std::numeric_limits<double>::infinity();
+  for (int i = 0; i < repeat; ++i) {
+    auto start = std::chrono::steady_clock::now();
+    (*op)(handle, args...);
+    SyncDevice(dev_type);
+    auto end = std::chrono::steady_clock::now();
+    double elapsed = std::chrono::duration<double>(end - start).count();
+    best = std::min(best, elapsed);
+  }
+  return best;
+}
+
+template <typename Key, typename... Args>
+std::size_t ResolveImplementationIndexOnline(const Handle& handle,
+                                             const Config& config,
+                                             Device::Type dev_type,
+                                             const Args&... args) {
+  if (!config.needs_implementation_resolution()) {
+    return config.implementation_index();
+  }
+
+  auto& tuning = TuningManager::Instance();
+  if (!tuning.IsEnabled()) {
+    return ResolveImplementationIndex<Key>(config, dev_type, args...);
+  }
+
+  auto indices = Operator<Key>::active_implementation_indices(dev_type);
+  if (indices.empty()) return config.implementation_index();
+
+  auto signature = TuningSignature::Build(args...);
+  constexpr auto op_name = OperatorName<Key>();
+  auto tuned = tuning.Lookup(op_name, dev_type, signature);
+  std::size_t chosen;
+
+  if (tuned.has_value() &&
+      std::find(indices.begin(), indices.end(), *tuned) != indices.end()) {
+    chosen = *tuned;
+  } else if (indices.size() == 1) {
+    chosen = indices.front();
+    tuning.Record(op_name, dev_type, signature, chosen);
+    std::cout << "[Tuning] " << op_name << " on "
+              << Device::StringFromType(dev_type)
+              << ": single impl, chose index " << chosen << std::endl;
+  } else {
+    chosen = indices.front();
+    double best_time = std::numeric_limits<double>::infinity();
+    for (auto idx : indices) {
+      double time =
+          BenchmarkImplementation<Key>(handle, dev_type, idx, args...);
+      if (time < best_time) {
+        best_time = time;
+        chosen = idx;
+      }
+    }
+    tuning.Record(op_name, dev_type, signature, chosen);
+    std::cout << "[Tuning] " << op_name << " on "
+              << Device::StringFromType(dev_type) << ": benchmarked "
+              << indices.size() << " impls, chose index " << chosen << " ("
+              << best_time * 1e6 << " us)" << std::endl;
+  }
+
+  return chosen;
+}
+
+}  // namespace detail
+
 }  // namespace infini::ops
 
 #endif
diff --git a/src/tuning.cc b/src/tuning.cc
new file mode 100644
index 000000000..4f24d4329
--- /dev/null
+++ b/src/tuning.cc
@@ -0,0 +1,235 @@
+#include "tuning.h"
+
+#include <algorithm>
+#include <cctype>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <nlohmann/json.hpp>
+#include <string_view>
+#include <utility>
+
+namespace infini::ops {
+
+namespace {
+
+using Json = nlohmann::json;
+
+constexpr int kTuningCacheVersion = 1;
+constexpr char kDefaultTuningPath[] = "tuning.json";
+
+bool EqualsIgnoreCase(std::string_view value, std::string_view expected) {
+  return value.size() == expected.size() &&
+         std::equal(value.begin(), value.end(), expected.begin(),
+                    [](char left, char right) {
+                      return std::toupper(static_cast<unsigned char>(left)) ==
+                             std::toupper(static_cast<unsigned char>(right));
+                    });
+}
+
+bool EnvFlag(const char* name) {
+  const char* value = std::getenv(name);
+  if (!value || !*value) return false;
+
+  const std::string_view flag{value};
+  return flag == "1" || EqualsIgnoreCase(flag, "ON") ||
+         EqualsIgnoreCase(flag, "TRUE");
+}
+
+int EnvInt(const char* name, int fallback) {
+  const char* value = std::getenv(name);
+  if (!value || !*value) return fallback;
+
+  int parsed = std::atoi(value);
+  return parsed > 0 ? parsed : fallback;
+}
+
+const Json* FindMember(const Json& object, const char* name) {
+  if (!object.is_object()) return nullptr;
+
+  auto iterator = object.find(name);
+  return iterator == object.end() ? nullptr : &*iterator;
+}
+
+bool IsInteger(const Json& value) {
+  return value.is_number_integer() || value.is_number_unsigned();
+}
+
+template <auto... device_types>
+std::optional<Device::Type> DeviceTypeFromString(std::string_view name,
+                                                 List<device_types...>) {
+  const Device::Type types[]{device_types...};
+  for (auto type : types) {
+    if (name == Device::StringFromType(type)) return type;
+  }
+  return std::nullopt;
+}
+
+Json SignatureToJson(const TuningSignature& signature) {
+  Json tensors = Json::array();
+  for (const auto& tensor : signature.tensors) {
+    tensors.push_back(
+        {{"shape", tensor.shape}, {"dtype", static_cast<int>(tensor.dtype)}});
+  }
+
+  return {{"tensors", std::move(tensors)}, {"scalars", signature.scalars}};
+}
+
+std::optional<TuningSignature> SignatureFromJson(const Json& value) {
+  const Json* tensors = FindMember(value, "tensors");
+  const Json* scalars = FindMember(value, "scalars");
+  if (!tensors || !tensors->is_array() || !scalars || !scalars->is_array()) {
+    return std::nullopt;
+  }
+
+  TuningSignature parsed;
+  for (const auto& tensor : *tensors) {
+    const Json* shape = FindMember(tensor, "shape");
+    const Json* dtype = FindMember(tensor, "dtype");
+    if (!shape || !shape->is_array() || !dtype || !IsInteger(*dtype)) {
+      return std::nullopt;
+    }
+
+    TuningSignature::TensorSig tensor_signature;
+    for (const auto& dimension : *shape) {
+      if (!IsInteger(dimension)) return std::nullopt;
+
+      tensor_signature.shape.push_back(dimension.get<int64_t>());
+    }
+    tensor_signature.dtype = static_cast<DataType>(dtype->get<int64_t>());
+    parsed.tensors.push_back(std::move(tensor_signature));
+  }
+
+  for (const auto& scalar : *scalars) {
+    if (!scalar.is_number()) return std::nullopt;
+
+    parsed.scalars.push_back(scalar.get<double>());
+  }
+
+  return parsed;
+}
+
+}  // namespace
+
+TuningManager& TuningManager::Instance() {
+  static TuningManager instance;
+  return instance;
+}
+
+void TuningManager::InitializeFromEnvironment() {
+  if (!EnvFlag("INFINI_OPS_ENABLE_AUTO_TUNING")) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    enabled_ = false;
+    cache_.clear();
+    json_path_.clear();
+    return;
+  }
+
+  warmup_count_ = EnvInt("INFINI_OPS_TUNING_WARMUP", kDefaultWarmupCount);
+  repeat_count_ = EnvInt("INFINI_OPS_TUNING_REPEAT", kDefaultRepeatCount);
+
+  const char* path = std::getenv("INFINI_OPS_TUNING_PATH");
+  LoadTuningCache(path && *path ? path : kDefaultTuningPath);
+}
+
+void TuningManager::LoadTuningCache(const std::string& json_path) {
+  std::lock_guard<std::mutex> lock(mutex_);
+
+  cache_.clear();
+  json_path_ = json_path;
+  enabled_ = true;
+
+  std::ifstream file(json_path);
+  if (!file.is_open()) return;
+
+  Json root = Json::parse(file, nullptr, false);
+  const Json* version = FindMember(root, "version");
+  const Json* entries = FindMember(root, "entries");
+  if (root.is_discarded() || !version || !IsInteger(*version) || !entries ||
+      !entries->is_array()) {
+    std::cerr << "[TuningManager] Warning: failed to parse " << json_path
+              << ", starting with an empty cache" << std::endl;
+    cache_.clear();
+    return;
+  }
+
+  if (version->get<int>() != kTuningCacheVersion) {
+    std::cerr << "[TuningManager] Warning: tuning.json version "
+              << version->get<int>() << " not supported (expected "
+              << kTuningCacheVersion << ")" << std::endl;
+    return;
+  }
+
+  for (const auto& entry : *entries) {
+    const Json* operator_name = FindMember(entry, "operator");
+    const Json* device_name = FindMember(entry, "device");
+    const Json* signature_json = FindMember(entry, "signature");
+    const Json* best_implementation = FindMember(entry, "best_implementation");
+    if (!operator_name || !operator_name->is_string() || !device_name ||
+        !device_name->is_string() || !signature_json || !best_implementation ||
+        !IsInteger(*best_implementation)) {
+      continue;
+    }
+
+    auto device = DeviceTypeFromString(
+        device_name->get_ref<const std::string&>(), AllDeviceTypes{});
+    auto signature = SignatureFromJson(*signature_json);
+    if (!device.has_value() || !signature.has_value()) {
+      continue;
+    }
+
+    CacheKey key{operator_name->get_ref<const std::string&>(), *device,
+                 std::move(*signature)};
+    cache_[std::move(key)] = best_implementation->get<std::size_t>();
+  }
+
+  std::cout << "[TuningManager] Loaded " << cache_.size()
+            << " tuning entries from " << json_path << std::endl;
+}
+
+std::optional<std::size_t> TuningManager::Lookup(
+    std::string_view operator_name, Device::Type device,
+    const TuningSignature& signature) const {
+  if (!enabled_) return std::nullopt;
+
+  std::lock_guard<std::mutex> lock(mutex_);
+  CacheKey key{std::string{operator_name}, device, signature};
+  auto iterator = cache_.find(key);
+  if (iterator == cache_.end()) return std::nullopt;
+
+  return iterator->second;
+}
+
+void TuningManager::Record(std::string_view operator_name, Device::Type device,
+                           const TuningSignature& signature,
+                           std::size_t best_index) {
+  if (!enabled_) return;
+
+  std::lock_guard<std::mutex> lock(mutex_);
+  CacheKey key{std::string{operator_name}, device, signature};
+  cache_[key] = best_index;
+  FlushToDiskLocked();
+}
+
+void TuningManager::FlushToDiskLocked() const {
+  std::ofstream out(json_path_, std::ios::trunc);
+  if (!out.is_open()) {
+    std::cerr << "[TuningManager] Warning: cannot write tuning cache to "
+              << json_path_ << std::endl;
+    return;
+  }
+
+  Json entries = Json::array();
+  for (const auto& [key, best_implementation] : cache_) {
+    entries.push_back(
+        {{"operator", key.operator_name},
+         {"device", std::string{Device::StringFromType(key.device)}},
+         {"signature", SignatureToJson(key.signature)},
+         {"best_implementation", best_implementation}});
+  }
+
+  Json root{{"version", kTuningCacheVersion}, {"entries", std::move(entries)}};
+  out << root.dump(2) << '\n';
+}
+
+}  // namespace infini::ops
diff --git a/src/tuning.h b/src/tuning.h
new file mode 100644
index 000000000..27f451eba
--- /dev/null
+++ b/src/tuning.h
@@ -0,0 +1,176 @@
+#ifndef INFINI_OPS_TUNING_H_
+#define INFINI_OPS_TUNING_H_
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <unordered_map>
+#include <vector>
+
+#include "data_type.h"
+#include "device.h"
+#include "tensor.h"
+
+namespace infini::ops {
+
+namespace detail {
+
+template <typename T>
+void CombineTuningHash(std::size_t& hash, const T& value) {
+  hash ^= std::hash<T>{}(value) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
+}
+
+}  // namespace detail
+
+struct TuningSignature {
+  struct TensorSig {
+    std::vector<int64_t> shape;
+    DataType dtype;
+
+    bool operator==(const TensorSig& other) const {
+      return shape == other.shape && dtype == other.dtype;
+    }
+  };
+
+  std::vector<TensorSig> tensors;
+  std::vector<double> scalars;
+
+  template <typename... Args>
+  static TuningSignature Build(const Args&... args) {
+    TuningSignature sig;
+    (sig.Absorb(args), ...);
+    return sig;
+  }
+
+  bool operator==(const TuningSignature& other) const {
+    return tensors == other.tensors && scalars == other.scalars;
+  }
+
+  std::size_t Hash() const {
+    std::size_t hash = 0;
+    for (const auto& t : tensors) {
+      for (auto dimension : t.shape) {
+        detail::CombineTuningHash(hash, dimension);
+      }
+      detail::CombineTuningHash(hash, static_cast<int>(t.dtype));
+    }
+    for (auto s : scalars) {
+      detail::CombineTuningHash(hash, s);
+    }
+    return hash;
+  }
+
+ private:
+  void Absorb(const Tensor& t) {
+    std::vector<int64_t> shape_vec;
+    for (std::size_t i = 0; i < t.shape().size(); ++i) {
+      shape_vec.push_back(static_cast<int64_t>(t.shape()[i]));
+    }
+    tensors.push_back({shape_vec, t.dtype()});
+  }
+
+  void Absorb(const std::optional<Tensor>& t) {
+    if (t.has_value()) {
+      Absorb(*t);
+    }
+  }
+
+  void Absorb(const std::vector<Tensor>& ts) {
+    for (const auto& t : ts) {
+      Absorb(t);
+    }
+  }
+
+  template <typename T>
+  void Absorb(const T& v) {
+    if constexpr (std::is_arithmetic_v<T>) {
+      scalars.push_back(static_cast<double>(v));
+    } else if constexpr (std::is_enum_v<T>) {
+      scalars.push_back(static_cast<double>(static_cast<int64_t>(v)));
+    }
+  }
+
+  template <typename T>
+  void Absorb(const std::optional<T>& v) {
+    if (v.has_value()) {
+      Absorb(*v);
+    }
+  }
+};
+
+class TuningManager {
+ public:
+  static TuningManager& Instance();
+
+  void InitializeFromEnvironment();
+
+  void LoadTuningCache(const std::string& json_path);
+
+  std::optional<std::size_t> Lookup(std::string_view operator_name,
+                                    Device::Type device,
+                                    const TuningSignature& signature) const;
+
+  void Record(std::string_view operator_name, Device::Type device,
+              const TuningSignature& signature, std::size_t best_index);
+
+  bool IsEnabled() const { return enabled_; }
+
+  int warmup_count() const { return warmup_count_; }
+
+  int repeat_count() const { return repeat_count_; }
+
+ private:
+  TuningManager() = default;
+
+  TuningManager(const TuningManager&) = delete;
+
+  TuningManager& operator=(const TuningManager&) = delete;
+
+  static constexpr int kDefaultWarmupCount = 1;
+
+  static constexpr int kDefaultRepeatCount = 5;
+
+  struct CacheKey {
+    std::string operator_name;
+    Device::Type device;
+    TuningSignature signature;
+
+    bool operator==(const CacheKey& other) const {
+      return operator_name == other.operator_name && device == other.device &&
+             signature == other.signature;
+    }
+  };
+
+  struct CacheKeyHash {
+    std::size_t operator()(const CacheKey& key) const {
+      std::size_t hash = 0;
+      detail::CombineTuningHash(hash, key.operator_name);
+      detail::CombineTuningHash(hash, static_cast<int>(key.device));
+      detail::CombineTuningHash(hash, key.signature.Hash());
+      return hash;
+    }
+  };
+
+  void FlushToDiskLocked() const;
+
+  std::unordered_map<CacheKey, std::size_t, CacheKeyHash> cache_;
+
+  bool enabled_{false};
+
+  std::string json_path_;
+
+  int warmup_count_{kDefaultWarmupCount};
+
+  int repeat_count_{kDefaultRepeatCount};
+
+  mutable std::mutex mutex_;
+};
+
+}  // namespace infini::ops
+
+#endif  // INFINI_OPS_TUNING_H_
diff --git a/tests/test_cpp_api.py b/tests/test_cpp_api.py
index 6d2c79cde..4cf69dcf9 100644
--- a/tests/test_cpp_api.py
+++ b/tests/test_cpp_api.py
@@ -8,7 +8,7 @@
 
 
 def test_cpp_operator_call_instantiation_smoke(tmp_path):
-    install_prefix = _install_prefix()
+    install_prefixes = _install_prefixes()
     source_dir = tmp_path / "source"
     build_dir = tmp_path / "build"
     source_dir.mkdir()
@@ -23,7 +23,7 @@ def test_cpp_operator_call_instantiation_smoke(tmp_path):
             str(source_dir),
             "-B",
             str(build_dir),
-            f"-DCMAKE_PREFIX_PATH={install_prefix}",
+            f"-DCMAKE_PREFIX_PATH={';'.join(map(str, install_prefixes))}",
             f"-DCMAKE_CXX_COMPILER={_compiler('CXX', 'c++')}",
         ]
     )
@@ -32,28 +32,7 @@ def test_cpp_operator_call_instantiation_smoke(tmp_path):
 
 
 def test_cpp_operator_call_trace_is_json(tmp_path):
-    install_prefix = _install_prefix()
-    include_dir = install_prefix / "include"
-    library_dir = _library_dir(install_prefix)
-    source = tmp_path / "add_trace.cc"
-    binary = tmp_path / "add_trace"
-    source.write_text(_ADD_SMOKE_SOURCE)
-
-    _run(
-        [
-            _compiler("CXX", "c++"),
-            "-std=c++17",
-            "-Werror",
-            f"-I{include_dir}",
-            str(source),
-            f"-L{library_dir}",
-            "-linfiniops",
-            "-linfinirt",
-            f"-Wl,-rpath,{library_dir}",
-            "-o",
-            str(binary),
-        ]
-    )
+    binary = _compile_cpp(tmp_path, "add_trace", _ADD_SMOKE_SOURCE)
     env = os.environ.copy()
     env["INFINI_OPS_TRACE_CALLS"] = "1"
     result = _run([str(binary)], env=env)
@@ -70,83 +49,102 @@ def test_cpp_operator_call_trace_is_json(tmp_path):
 
 
 def test_cpp_returning_call_smoke(tmp_path):
-    install_prefix = _install_prefix()
-    include_dir = install_prefix / "include"
-    library_dir = _library_dir(install_prefix)
-    source = tmp_path / "add_return_smoke.cc"
-    binary = tmp_path / "add_return_smoke"
-    source.write_text(_ADD_RETURN_SMOKE_SOURCE)
+    binary = _compile_cpp(tmp_path, "add_return_smoke", _ADD_RETURN_SMOKE_SOURCE)
+    _run([str(binary)])
 
-    _run(
-        [
-            _compiler("CXX", "c++"),
-            "-std=c++17",
-            "-Werror",
-            f"-I{include_dir}",
-            str(source),
-            f"-L{library_dir}",
-            "-linfiniops",
-            "-linfinirt",
-            f"-Wl,-rpath,{library_dir}",
-            "-o",
-            str(binary),
-        ]
+
+def test_tuning_cache_round_trip(tmp_path):
+    binary = _compile_cpp(tmp_path, "tuning_cache", _TUNING_CACHE_SOURCE)
+    cache_path = tmp_path / "tuning.json"
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "INFINI_OPS_ENABLE_AUTO_TUNING": "1",
+            "INFINI_OPS_TUNING_PATH": str(cache_path),
+            "INFINI_OPS_TUNING_WARMUP": "2",
+            "INFINI_OPS_TUNING_REPEAT": "3",
+        }
     )
-    _run([str(binary)])
 
+    _run([str(binary), "initialize", str(cache_path)], env=environment)
 
-def test_cpp_configless_calls_use_first_active_implementation(tmp_path):
-    install_prefix = _install_prefix()
-    include_dir = install_prefix / "include"
-    library_dir = _library_dir(install_prefix)
-    source = tmp_path / "configless_active_implementation.cc"
-    binary = tmp_path / "configless_active_implementation"
-    source.write_text(_CONFIGLESS_ACTIVE_IMPLEMENTATION_SOURCE)
+    cache = json.loads(cache_path.read_text())
+    assert cache["version"] == 1
+    entry = cache["entries"][0]
+    assert entry["operator"] == "Add"
+    assert entry["best_implementation"] == 7
+
+    _run([str(binary), "lookup", str(cache_path)])
+
+    cache_path.write_text("{")
+    _run([str(binary), "miss", str(cache_path)])
+
+
+def test_tuning_disabled_by_default(tmp_path):
+    binary = _compile_cpp(tmp_path, "tuning_disabled", _TUNING_CACHE_SOURCE)
+
+    for index, setting in enumerate((None, "0", "OFF", "false")):
+        cache_path = tmp_path / f"disabled-{index}.json"
+        environment = os.environ.copy()
+        environment["INFINI_OPS_TUNING_PATH"] = str(cache_path)
+        if setting is None:
+            environment.pop("INFINI_OPS_ENABLE_AUTO_TUNING", None)
+        else:
+            environment["INFINI_OPS_ENABLE_AUTO_TUNING"] = setting
+
+        _run([str(binary), "disabled", str(cache_path)], env=environment)
+        assert not cache_path.exists()
+
+
+def _compile_cpp(tmp_path, name, source_text):
+    install_prefixes = _install_prefixes()
+    include_dirs = [prefix / "include" for prefix in install_prefixes]
+    library_dirs = list(
+        dict.fromkeys(
+            (
+                _library_dir(install_prefixes, "libinfiniops.so"),
+                _library_dir(install_prefixes, "libinfinirt.so"),
+            )
+        )
+    )
+    source = tmp_path / f"{name}.cc"
+    binary = tmp_path / name
+    source.write_text(source_text)
 
     _run(
         [
             _compiler("CXX", "c++"),
             "-std=c++17",
             "-Werror",
-            f"-I{include_dir}",
+            "-Wno-error=deprecated-declarations",
+            *(f"-I{include_dir}" for include_dir in include_dirs),
             str(source),
-            f"-L{library_dir}",
+            *(f"-L{library_dir}" for library_dir in library_dirs),
             "-linfiniops",
             "-linfinirt",
-            f"-Wl,-rpath,{library_dir}",
+            *(f"-Wl,-rpath,{library_dir}" for library_dir in library_dirs),
             "-o",
             str(binary),
         ]
     )
-    _run([str(binary)])
 
+    return binary
 
-def test_cpp_polymorphic_context_smoke(tmp_path):
-    install_prefix = _install_prefix()
-    include_dir = install_prefix / "include"
-    library_dir = _library_dir(install_prefix)
-    source = tmp_path / "polymorphic_context.cc"
-    binary = tmp_path / "polymorphic_context"
-    source.write_text(_POLYMORPHIC_CONTEXT_SOURCE)
 
-    _run(
-        [
-            _compiler("CXX", "c++"),
-            "-std=c++17",
-            "-Werror",
-            f"-I{include_dir}",
-            str(source),
-            f"-L{library_dir}",
-            "-linfiniops",
-            "-linfinirt",
-            f"-Wl,-rpath,{library_dir}",
-            "-o",
-            str(binary),
-        ]
+def test_cpp_configless_calls_use_first_active_implementation(tmp_path):
+    binary = _compile_cpp(
+        tmp_path,
+        "configless_active_implementation",
+        _CONFIGLESS_ACTIVE_IMPLEMENTATION_SOURCE,
     )
     _run([str(binary)])
 
 
+def test_cpp_polymorphic_context_smoke(tmp_path):
+    binary = _compile_cpp(tmp_path, "polymorphic_context", _POLYMORPHIC_CONTEXT_SOURCE)
+    _run([str(binary)])
+
+
 @pytest.mark.parametrize(
     "header",
     (
@@ -156,8 +154,7 @@ def test_cpp_polymorphic_context_smoke(tmp_path):
     ),
 )
 def test_cpp_base_headers_compile_with_metadata_views(tmp_path, header):
-    install_prefix = _install_prefix()
-    include_dir = install_prefix / "include"
+    include_dirs = [prefix / "include" for prefix in _install_prefixes()]
     source = tmp_path / f"{Path(header).stem}_metadata_view.cc"
     source.write_text(f"#include <{header}>\n\nint main() {{ return 0; }}\n")
 
@@ -168,7 +165,7 @@ def test_cpp_base_headers_compile_with_metadata_views(tmp_path, header):
             "-Werror",
             "-UNDEBUG",
             "-fsyntax-only",
-            f"-I{include_dir}",
+            *(f"-I{include_dir}" for include_dir in include_dirs),
             str(source),
         ]
     )
@@ -183,13 +180,22 @@ def _install_prefix():
     pytest.skip("`INFINI_OPS_INSTALL_PREFIX` is not set.")
 
 
-def _library_dir(prefix):
-    for name in ("lib", "lib64"):
-        library_dir = prefix / name
-        if (library_dir / "libinfiniops.so").exists():
-            return library_dir
+def _install_prefixes():
+    infiniops_prefix = _install_prefix()
+    prefixes = [infiniops_prefix]
+    if infinirt_root := os.environ.get("INFINI_RT_ROOT"):
+        prefixes.append(Path(infinirt_root))
+
+    return list(dict.fromkeys(prefixes))
+
 
-    pytest.skip(f"`libinfiniops.so` was not found under `{prefix}`.")
+def _library_dir(prefixes, library_name):
+    for prefix in prefixes:
+        for library_dir in (prefix, prefix / "lib", prefix / "lib64"):
+            if (library_dir / library_name).exists():
+                return library_dir
+
+    pytest.skip(f"`{library_name}` was not found under any configured prefix.")
 
 
 def _compiler(env_name, default):
@@ -209,7 +215,7 @@ def _run(command, **kwargs):
     except FileNotFoundError as error:
         pytest.skip(f"`{command[0]}` is not available: {error}")
     except subprocess.CalledProcessError as error:
-        output = "\n".join((error.stdout, error.stderr)).strip()
+        output = f"{error.stdout}\n{error.stderr}".strip()
         raise AssertionError(output) from error
 
 
@@ -228,6 +234,59 @@ def _run(command, **kwargs):
 )
 
 
+_TUNING_CACHE_SOURCE = textwrap.dedent(
+    r"""
+    #include <tuning.h>
+
+    #include <string>
+
+    int main(int argc, char** argv) {
+      if (argc != 3) {
+        return 2;
+      }
+
+      infini::ops::TuningSignature signature;
+      signature.tensors.push_back(
+          {{2, 3}, infini::ops::DataType::kFloat32});
+      signature.scalars.push_back(1.5);
+
+      auto& manager = infini::ops::TuningManager::Instance();
+      const std::string mode{argv[1]};
+
+      if (mode == "initialize") {
+        manager.InitializeFromEnvironment();
+        if (!manager.IsEnabled() || manager.warmup_count() != 2 ||
+            manager.repeat_count() != 3) {
+          return 1;
+        }
+        manager.Record("Add", infini::ops::Device::Type::kCpu, signature, 7);
+        return 0;
+      }
+
+      if (mode == "disabled") {
+        manager.InitializeFromEnvironment();
+        if (manager.IsEnabled()) {
+          return 1;
+        }
+        manager.Record("Add", infini::ops::Device::Type::kCpu, signature, 7);
+        return 0;
+      }
+
+      manager.LoadTuningCache(argv[2]);
+      auto implementation = manager.Lookup(
+          "Add", infini::ops::Device::Type::kCpu, signature);
+      if (mode == "lookup") {
+        return implementation == std::optional<std::size_t>{7} ? 0 : 1;
+      }
+      if (mode == "miss") {
+        return implementation.has_value() ? 1 : 0;
+      }
+      return 2;
+    }
+    """
+).lstrip()
+
+
 _ADD_SMOKE_SOURCE = textwrap.dedent(
     r"""
     #include <infini/ops.h>
diff --git a/tests/test_generate_wrappers.py b/tests/test_generate_wrappers.py
index 0f34483a0..4829e7f1d 100644
--- a/tests/test_generate_wrappers.py
+++ b/tests/test_generate_wrappers.py
@@ -369,31 +369,30 @@ class Mul {
 
     text = module._generate_pybind11(operator)
 
+    assert '#include "tuning.h"' in text
     assert "std::size_t DefaultImplementationIndexForMul" in text
     assert (
         "config.set_implementation_index("
         "DefaultImplementationIndexForMul(DeviceFromPybind11Handle(input).type()))"
     ) in text
     assert "std::optional<std::size_t> implementation_index" in text
-    assert "if (implementation_index.has_value())" in text
-    assert "config.set_implementation_index(*implementation_index)" in text
-    assert "auto converted_first_tensor{TensorFromPybind11Handle(input)};" in text
     assert (
-        "DefaultImplementationIndexForMul(converted_first_tensor.device().type()))"
+        "if (implementation_index.has_value()) {\n"
+        "      config.set_implementation_index(*implementation_index);\n"
+        "    } else if (!TuningManager::Instance().IsEnabled()) {\n"
+        "      config.set_implementation_index(\n"
+        "          DefaultImplementationIndexForMul("
+        "converted_first_tensor.device().type()));\n"
+        "    }"
     ) in text
+    assert "auto converted_first_tensor{TensorFromPybind11Handle(input)};" in text
     assert "std::move(converted_first_tensor)" not in text
     assert text.count("DeviceFromPybind11Handle(input)") == 1
-    assert (
-        "config.set_implementation_index("
-        "DefaultImplementationIndexForMul(DeviceFromPybind11Handle(input).type()))"
-    ) in text
     assert "implementation_index.value_or(" not in text
     assert 'py::arg("implementation_index") = py::none()' in text
 
 
-def test_pybind_default_implementation_reuses_first_vector_tensor(
-    monkeypatch, tmp_path
-):
+def test_pybind_reuses_first_vector_tensor_conversion(monkeypatch, tmp_path):
     module = _load_generator_module()
     base_header = tmp_path / "cat.h"
     base_header.write_text(
@@ -424,6 +423,7 @@ class Cat {
     assert (
         "auto converted_first_tensor{VectorTensorFromPybind11Handle(inputs)};" in text
     )
+    assert "generated_dispatch::CallCat(handle, config, converted_first_tensor," in text
     assert "converted_first_tensor.at(0).device().type()" in text
     assert "std::move(converted_first_tensor)" not in text
     assert "DeviceFromPybind11Handle(inputs.at(0))" not in text
@@ -574,6 +574,21 @@ def test_generated_ops_module_exposes_host_range_profile_controls_once():
         assert target in binding
 
 
+@pytest.mark.parametrize("monolithic", [False, True])
+def test_generated_ops_module_initializes_tuning(monolithic):
+    module = _load_generator_module()
+
+    text = module._generate_ops_module_source(
+        ["BindAdd"],
+        op_includes=['#include "base/add.h"'],
+        monolithic=monolithic,
+    )
+
+    assert text.count('#include "tuning.h"') == 1
+    assert text.count("TuningManager::Instance().InitializeFromEnvironment();") == 1
+    assert text.count("BindHostRangeProfileControls(m);") == 1
+
+
 def test_iluvatar_custom_compilers_receive_host_range_profile_definition():
     cmake = (pathlib.Path(__file__).parents[1] / "src" / "CMakeLists.txt").read_text(
         encoding="utf-8"
@@ -994,10 +1009,12 @@ def test_triton_binding_uses_backend_metadata_and_shared_config_parser(
 
     binding = module._generate_pybind11(operator)
 
+    assert '#include "tuning.h"' in binding
     assert '#include "triton/jit/pybind11_config.h"' in binding
     assert "std::unique_ptr<Config> triton_config_ptr;" in binding
     assert "triton::jit::ConfigFromPyDict(*config_dict)" in binding
     assert "triton_config_ptr->set_implementation_index(" in binding
+    assert "if (!config.needs_implementation_resolution())" in binding
     assert "config.implementation_index()" in binding
     assert "if (triton_config_ptr)" in binding
     assert "generated_dispatch::MakeAdd(*triton_config_ptr," in binding