From 26270c0085c912251a26c4c01b8f2fa89ef8d06e Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Sat, 14 Nov 2026 12:32:39 -0500 Subject: [PATCH] Backport llama.cpp#31437: vm_allocate host buffers on every Apple platform iOS host buffers came from posix_memalign, whose freed pages libmalloc keeps in the app footprint (llamadart#2004). The iOS artifact check now rejects a _posix_memalign import. --- CMakeLists.txt | 32 ++++++++++++++----- README.md | 7 +++-- patches/README.md | 1 + patches/ggml_apple_vm_allocate.patch | 23 ++++++++++++++ patches/ggml_metal_apple_vm_allocate.patch | 36 ++++++++++++++++++++++ tests/test_tools.py | 23 ++++++++++++++ tools/validate_artifacts.py | 11 +++++++ 7 files changed, 124 insertions(+), 9 deletions(-) create mode 100644 patches/ggml_apple_vm_allocate.patch create mode 100644 patches/ggml_metal_apple_vm_allocate.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index b038e6c..91cbde5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,17 +20,31 @@ add_subdirectory("${SD_NATIVE_UPSTREAM_DIR}" sd) # Fixes the pinned upstream lacks (patches/README.md). Each patch is applied to # build-tree copies of the upstream sources it changes, and the library # compiles the copies, so the submodule stays pristine. Configuring fails when -# a patch no longer applies or changes nothing. +# a patch no longer applies or changes nothing. A patch's sources belong to +# one target: stable-diffusion unless TARGET names another, such as ggml-base. find_package(Git REQUIRED) function(sd_native_carry_patch patch) + cmake_parse_arguments(PARSE_ARGV 1 carry "" "TARGET" "") + if(NOT carry_TARGET) + set(carry_TARGET stable-diffusion) + endif() + set(carry_paths ${carry_UNPARSED_ARGUMENTS}) get_filename_component(name "${patch}" NAME_WE) set(patched_root "${CMAKE_BINARY_DIR}/patched") set(patched_dir "${patched_root}/${name}") file(REMOVE_RECURSE "${patched_dir}") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/patches/${patch}") - get_target_property(sources stable-diffusion SOURCES) - foreach(path IN LISTS ARGN) + get_target_property(target_dir ${carry_TARGET} SOURCE_DIR) + get_target_property(target_sources ${carry_TARGET} SOURCES) + set(sources) + foreach(source IN LISTS target_sources) + if(NOT source MATCHES "^\\$<") + get_filename_component(source "${source}" ABSOLUTE BASE_DIR "${target_dir}") + endif() + list(APPEND sources "${source}") + endforeach() + foreach(path IN LISTS carry_paths) set(upstream_file "${SD_NATIVE_UPSTREAM_DIR}/${path}") string(MD5 source_key "${upstream_file}") get_property(current_source GLOBAL PROPERTY "SD_NATIVE_PATCHED_${source_key}") @@ -38,7 +52,7 @@ function(sd_native_carry_patch patch) set(current_source "${upstream_file}") endif() if(NOT "${current_source}" IN_LIST sources) - message(FATAL_ERROR "patches/${patch}: ${path} is not a stable-diffusion source") + message(FATAL_ERROR "patches/${patch}: ${path} is not a ${carry_TARGET} source") endif() file(READ "${current_source}" text) # A Windows checkout can convert upstream's line endings; patches use LF. @@ -60,7 +74,7 @@ function(sd_native_carry_patch patch) message(FATAL_ERROR "patches/${patch} does not apply to the pinned upstream:\n${output}") endif() - foreach(path IN LISTS ARGN) + foreach(path IN LISTS carry_paths) set(upstream_file "${SD_NATIVE_UPSTREAM_DIR}/${path}") string(MD5 source_key "${upstream_file}") get_property(current_source GLOBAL PROPERTY "SD_NATIVE_PATCHED_${source_key}") @@ -79,10 +93,10 @@ function(sd_native_carry_patch patch) set_property(GLOBAL PROPERTY "SD_NATIVE_PATCHED_${source_key}" "${patched_file}") # Quoted includes in the copy must still find the upstream files next to it. get_filename_component(upstream_file_dir "${upstream_file}" DIRECTORY) - set_source_files_properties("${patched_file}" TARGET_DIRECTORY stable-diffusion + set_source_files_properties("${patched_file}" TARGET_DIRECTORY ${carry_TARGET} PROPERTIES INCLUDE_DIRECTORIES "${upstream_file_dir}") endforeach() - set_property(TARGET stable-diffusion PROPERTY SOURCES ${sources}) + set_property(TARGET ${carry_TARGET} PROPERTY SOURCES ${sources}) message(STATUS "Carrying patches/${patch}") endfunction() @@ -91,6 +105,10 @@ sd_native_carry_patch(torch_zip_entry_lookup.patch src/model_io/torch_zip_io.cpp sd_native_carry_patch(fp8_metal_f16_load.patch src/model_manager.cpp src/core/backend_fit.cpp) sd_native_carry_patch(lora_immediate_fallback.patch src/pipeline/diffusion_engine.cpp) sd_native_carry_patch(intrinsic_tiny_decoder_policy.patch src/stable-diffusion.cpp src/pipeline/diffusion_engine.cpp) +sd_native_carry_patch(ggml_apple_vm_allocate.patch TARGET ggml-base ggml/src/ggml.c) +if(TARGET ggml-metal) + sd_native_carry_patch(ggml_metal_apple_vm_allocate.patch TARGET ggml-metal ggml/src/ggml-metal/ggml-metal-device.m) +endif() # Wrapper exports are compiled into the upstream library target, so they share # its SD_API export macro and reach upstream without patching it. diff --git a/README.md b/README.md index f219754..bcc6644 100644 --- a/README.md +++ b/README.md @@ -657,14 +657,17 @@ the macOS probe and builds the iOS device and simulator slices. Upstream 1017 has its [ABI audit, wrapper review, FP8 evidence and outstanding gates](docs/upstream_1017_qualification.md); the carry inventory is in the [upstream 982 record](docs/upstream_982_qualification.md). -It carries four fixes the pin lacks: unbatched CFG in Metal builds (below), +It carries five fixes the pin lacks: unbatched CFG in Metal builds (below), the [`pickle_line_span.patch`](patches/README.md) for legacy checkpoints above 2 GiB, the [`fp8_metal_f16_load.patch`](patches/README.md), which loads FP8 weights as F16 on Metal instead of aborting, so they take twice their file size there, and the [`lora_immediate_fallback.patch`](patches/README.md), which applies a LoRA at runtime instead of crashing where the `immediately` mode cannot merge it: into FP8 weights, and on Metal, which cannot add into the F16 -convolution weights every model has. +convolution weights every model has, and the +[`ggml_apple_vm_allocate.patch`](patches/README.md), which backports +[llama.cpp#31437](https://github.com/ggml-org/llama.cpp/pull/31437) so iOS +host buffers go back to the system when freed. Upstream's `generate_image` now returns an owned result with per-image generation metadata; `sd_dart_generate_image` keeps returning an image array that `free_sd_images` frees and drops that metadata. Since upstream 982 a null diff --git a/patches/README.md b/patches/README.md index 0a6fab3..4b3ae7d 100644 --- a/patches/README.md +++ b/patches/README.md @@ -24,6 +24,7 @@ reports. | `lora_immediate_fallback.patch` | `src/pipeline/diffusion_engine.cpp` | The `immediately` LoRA apply mode, which `auto` selects for unquantized weights, crashes the CPU merging into FP8 weights (ggml-cpu can neither multiply nor write FP8) and aborts Metal merging into F16 weights (ggml-metal adds only equal types), which every model has: convolution weights load as F16. FP8 weights, and every model on a backend that cannot run that add outside host memory, now take the LoRA `at_runtime`, with a warning when `immediately` was requested ([#31](https://github.com/leehack/stable-diffusion-native/issues/31)). | `tests/test_fp8_runtime.py` | | `intrinsic_tiny_decoder_policy.patch` | `src/stable-diffusion.cpp`, `src/pipeline/diffusion_engine.cpp` | Owner-only `sd_dart_new_sd_ctx_intrinsic_tiny_v1` context contract for [llamadart #1047](https://github.com/leehack/llamadart/issues/1047), not an upstream defect: automatic model refresh is valid upstream behavior. The public C API cannot preserve a decoder invariant, so minimal context creation/free and candidate/execution insertion points reject a full-VAE replacement before runner probes or decode. Real unchanged/refreshed SDXS and unrestricted reload controls exercise the boundary. Retire when upstream exposes an equivalent persistent decoder policy. | `tests/test_decoder_policy_runtime.py` | +| `ggml_apple_vm_allocate.patch`, `ggml_metal_apple_vm_allocate.patch` | `ggml/src/ggml.c`, `ggml/src/ggml-metal/ggml-metal-device.m` | On iOS, ggml's CPU and Metal host buffers come from `posix_memalign`/`free`, and libmalloc keeps the freed pages in the app's memory footprint, so an unloaded model still counts toward the jetsam limit ([llamadart #2004](https://github.com/leehack/llamadart/issues/2004)). Every Apple platform now uses `vm_allocate`/`vm_deallocate`, as macOS already did. A backport of llama.cpp `ea24b952c` ([ggml-org/llama.cpp#31437](https://github.com/ggml-org/llama.cpp/pull/31437)); retire when the pinned ggml includes it. Two patches because the files belong to the `ggml-base` and `ggml-metal` targets. | `tools/validate_artifacts.py` (an iOS library must not import `_posix_memalign`), `tests/test_tools.py` | Patches touching the same source compose in declaration order. Each next patch applies to the preceding build-tree copy; the submodule stays pristine. diff --git a/patches/ggml_apple_vm_allocate.patch b/patches/ggml_apple_vm_allocate.patch new file mode 100644 index 0000000..d0230a3 --- /dev/null +++ b/patches/ggml_apple_vm_allocate.patch @@ -0,0 +1,23 @@ +diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c +index da9ccad..d379020 100644 +--- a/ggml/src/ggml.c ++++ b/ggml/src/ggml.c +@@ -357,7 +357,8 @@ void * ggml_aligned_malloc(size_t size) { + void * aligned_memory = NULL; + #ifdef GGML_USE_CPU_HBM + int result = hbw_posix_memalign(&aligned_memory, alignment, size); +- #elif TARGET_OS_OSX && !defined(GGML_TSAN_ENABLED) ++ #elif defined(__APPLE__) && !defined(GGML_TSAN_ENABLED) ++ // libmalloc can keep freed pages in the process footprint, vm_deallocate returns them at once + GGML_UNUSED(alignment); + kern_return_t alloc_status = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t *) &aligned_memory, size, VM_FLAGS_ANYWHERE); + int result = EFAULT; +@@ -404,7 +405,7 @@ void ggml_aligned_free(void * ptr, size_t size) { + if (ptr != NULL) { + hbw_free(ptr); + } +-#elif TARGET_OS_OSX && !defined(GGML_TSAN_ENABLED) ++#elif defined(__APPLE__) && !defined(GGML_TSAN_ENABLED) + if (ptr != NULL) { + vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ptr, size); + } diff --git a/patches/ggml_metal_apple_vm_allocate.patch b/patches/ggml_metal_apple_vm_allocate.patch new file mode 100644 index 0000000..41e2839 --- /dev/null +++ b/patches/ggml_metal_apple_vm_allocate.patch @@ -0,0 +1,36 @@ +diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m +index 4451990..a59f37f 100644 +--- a/ggml/src/ggml-metal/ggml-metal-device.m ++++ b/ggml/src/ggml-metal/ggml-metal-device.m +@@ -2093,19 +2093,11 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) { + static void * ggml_metal_host_malloc(size_t n) { + void * data = NULL; + +-#if TARGET_OS_OSX + kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE); + if (err != KERN_SUCCESS) { + GGML_LOG_ERROR("%s: error: vm_allocate failed\n", __func__); + return NULL; + } +-#else +- const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n); +- if (result != 0) { +- GGML_LOG_ERROR("%s: error: posix_memalign failed\n", __func__); +- return NULL; +- } +-#endif + + return data; + } +@@ -2287,11 +2279,7 @@ void ggml_metal_buffer_free(ggml_metal_buffer_t buf) { + } + + if (buf->is_shared && buf->owned) { +-#if TARGET_OS_OSX + vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)buf->all_data, buf->all_size); +-#else +- free(buf->all_data); +-#endif + } + + free(buf); diff --git a/tests/test_tools.py b/tests/test_tools.py index 0a7044c..9a8b778 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -15,6 +15,7 @@ APPLE_STATIC_DESTRUCTOR_IMPORT, ELF_STATIC_DESTRUCTOR_IMPORT, ELF_STREAM_INIT_IMPORT, + IOS_HOST_BUFFER_IMPORT, elf_imports_symbol, elf_undefined_symbols, imports_symbol, @@ -100,6 +101,21 @@ def test_wrapper_header_adds_only_the_documented_exports(self): } """ +HOST_BUFFER_WITH_POSIX_MEMALIGN = """ +#include +void *host_buffer(unsigned long n) { + void *p = 0; + return posix_memalign(&p, 16384, n) == 0 ? p : 0; +} +""" +HOST_BUFFER_WITH_VM_ALLOCATE = """ +#include +void *host_buffer(unsigned long n) { + vm_address_t p = 0; + return vm_allocate(mach_task_self(), &p, n, VM_FLAGS_ANYWHERE) == KERN_SUCCESS ? (void *) p : 0; +} +""" + @unittest.skipUnless(sys.platform == "darwin", "reads Mach-O imports with nm") class StaticDestructorImportTest(unittest.TestCase): @@ -125,6 +141,13 @@ def test_reads_the_named_architecture_of_a_universal_file(self): for arch in ("arm64", "x86_64"): self.assertTrue(imports_symbol(system, APPLE_STATIC_DESTRUCTOR_IMPORT, arch)) + def test_tells_posix_memalign_host_buffers_from_vm_allocate_ones(self): + with tempfile.TemporaryDirectory() as tmp: + old = self.build(tmp, "old", HOST_BUFFER_WITH_POSIX_MEMALIGN) + new = self.build(tmp, "new", HOST_BUFFER_WITH_VM_ALLOCATE) + self.assertTrue(imports_symbol(old, IOS_HOST_BUFFER_IMPORT)) + self.assertFalse(imports_symbol(new, IOS_HOST_BUFFER_IMPORT)) + # `readelf --dyn-syms -W` of GNU binutils and of LLVM: the first appends the # index of a symbol's version, the second does not. diff --git a/tools/validate_artifacts.py b/tools/validate_artifacts.py index 3acac94..0631a32 100644 --- a/tools/validate_artifacts.py +++ b/tools/validate_artifacts.py @@ -43,6 +43,12 @@ # are still inside it (leehack/llamadart#949). Android libraries are left as # they were and import it. ELF_STATIC_DESTRUCTOR_IMPORT = "__cxa_atexit" +# ggml's host buffers on iOS come from `posix_memalign`, and libmalloc keeps +# their freed pages in the app's memory footprint for as long as it likes, so a +# model that is unloaded still counts toward the jetsam limit +# (leehack/llamadart#2004). `patches/ggml_apple_vm_allocate.patch` makes every +# Apple build use `vm_allocate`, as macOS already did. +IOS_HOST_BUFFER_IMPORT = "_posix_memalign" # `std::ios_base::Init::Init()`. Before GCC 13 every source that includes # holds an object of that class, whose destructor flushes the # standard streams when the last one goes. A Linux library drops that @@ -184,6 +190,11 @@ def validate(target: Target) -> list[str]: f"imports {APPLE_STATIC_DESTRUCTOR_IMPORT}: static destructors bypass exit teardown" ) + if target.os == "ios" and imports_symbol(library, IOS_HOST_BUFFER_IMPORT): + problems.append( + f"imports {IOS_HOST_BUFFER_IMPORT}: host buffers outlive their free in the app footprint" + ) + if target.os == "linux": imports = elf_undefined_symbols(output(["readelf", "--dyn-syms", "-W", str(library)])) if ELF_STATIC_DESTRUCTOR_IMPORT in imports: