Carry llama.cpp#31437: vm_allocate host buffers on iOS (llamadart#2004)

CMakeLists.txt

@@ -20,25 +20,39 @@ add_subdirectory("${SD_NATIVE_UPSTREAM_DIR}" sd)
 # Fixes the pinned upstream lacks (patches/README.md). Each patch is applied to
 # build-tree copies of the upstream sources it changes, and the library
 # compiles the copies, so the submodule stays pristine. Configuring fails when
-# a patch no longer applies or changes nothing.
+# a patch no longer applies or changes nothing. A patch's sources belong to
+# one target: stable-diffusion unless TARGET names another, such as ggml-base.
 find_package(Git REQUIRED)
 function(sd_native_carry_patch patch)
+    cmake_parse_arguments(PARSE_ARGV 1 carry "" "TARGET" "")
+    if(NOT carry_TARGET)
+        set(carry_TARGET stable-diffusion)
+    endif()
+    set(carry_paths ${carry_UNPARSED_ARGUMENTS})
     get_filename_component(name "${patch}" NAME_WE)
     set(patched_root "${CMAKE_BINARY_DIR}/patched")
     set(patched_dir "${patched_root}/${name}")
     file(REMOVE_RECURSE "${patched_dir}")
     set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
         "${CMAKE_CURRENT_SOURCE_DIR}/patches/${patch}")
-    get_target_property(sources stable-diffusion SOURCES)
-    foreach(path IN LISTS ARGN)
+    get_target_property(target_dir ${carry_TARGET} SOURCE_DIR)
+    get_target_property(target_sources ${carry_TARGET} SOURCES)
+    set(sources)
+    foreach(source IN LISTS target_sources)
+        if(NOT source MATCHES "^\\$<")
+            get_filename_component(source "${source}" ABSOLUTE BASE_DIR "${target_dir}")
+        endif()
+        list(APPEND sources "${source}")
+    endforeach()
+    foreach(path IN LISTS carry_paths)
         set(upstream_file "${SD_NATIVE_UPSTREAM_DIR}/${path}")
         string(MD5 source_key "${upstream_file}")
         get_property(current_source GLOBAL PROPERTY "SD_NATIVE_PATCHED_${source_key}")
         if(NOT current_source)
             set(current_source "${upstream_file}")
         endif()
         if(NOT "${current_source}" IN_LIST sources)
-            message(FATAL_ERROR "patches/${patch}: ${path} is not a stable-diffusion source")
+            message(FATAL_ERROR "patches/${patch}: ${path} is not a ${carry_TARGET} source")
         endif()
         file(READ "${current_source}" text)
         # A Windows checkout can convert upstream's line endings; patches use LF.
@@ -60,7 +74,7 @@ function(sd_native_carry_patch patch)
         message(FATAL_ERROR "patches/${patch} does not apply to the pinned upstream:\n${output}")
     endif()
 
-    foreach(path IN LISTS ARGN)
+    foreach(path IN LISTS carry_paths)
         set(upstream_file "${SD_NATIVE_UPSTREAM_DIR}/${path}")
         string(MD5 source_key "${upstream_file}")
         get_property(current_source GLOBAL PROPERTY "SD_NATIVE_PATCHED_${source_key}")
@@ -79,10 +93,10 @@ function(sd_native_carry_patch patch)
         set_property(GLOBAL PROPERTY "SD_NATIVE_PATCHED_${source_key}" "${patched_file}")
         # Quoted includes in the copy must still find the upstream files next to it.
         get_filename_component(upstream_file_dir "${upstream_file}" DIRECTORY)
-        set_source_files_properties("${patched_file}" TARGET_DIRECTORY stable-diffusion
+        set_source_files_properties("${patched_file}" TARGET_DIRECTORY ${carry_TARGET}
             PROPERTIES INCLUDE_DIRECTORIES "${upstream_file_dir}")
     endforeach()
-    set_property(TARGET stable-diffusion PROPERTY SOURCES ${sources})
+    set_property(TARGET ${carry_TARGET} PROPERTY SOURCES ${sources})
     message(STATUS "Carrying patches/${patch}")
 endfunction()
 
@@ -91,6 +105,10 @@ sd_native_carry_patch(torch_zip_entry_lookup.patch src/model_io/torch_zip_io.cpp
 sd_native_carry_patch(fp8_metal_f16_load.patch src/model_manager.cpp src/core/backend_fit.cpp)
 sd_native_carry_patch(lora_immediate_fallback.patch src/pipeline/diffusion_engine.cpp)
 sd_native_carry_patch(intrinsic_tiny_decoder_policy.patch src/stable-diffusion.cpp src/pipeline/diffusion_engine.cpp)
+sd_native_carry_patch(ggml_apple_vm_allocate.patch TARGET ggml-base ggml/src/ggml.c)
+if(TARGET ggml-metal)
+    sd_native_carry_patch(ggml_metal_apple_vm_allocate.patch TARGET ggml-metal ggml/src/ggml-metal/ggml-metal-device.m)
+endif()
 
 # Wrapper exports are compiled into the upstream library target, so they share
 # its SD_API export macro and reach upstream without patching it.

README.md

@@ -657,14 +657,17 @@ the macOS probe and builds the iOS device and simulator slices.
 Upstream 1017 has its
 [ABI audit, wrapper review, FP8 evidence and outstanding gates](docs/upstream_1017_qualification.md);
 the carry inventory is in the [upstream 982 record](docs/upstream_982_qualification.md).
-It carries four fixes the pin lacks: unbatched CFG in Metal builds (below),
+It carries five fixes the pin lacks: unbatched CFG in Metal builds (below),
 the [`pickle_line_span.patch`](patches/README.md) for legacy checkpoints above
 2 GiB, the [`fp8_metal_f16_load.patch`](patches/README.md), which loads FP8
 weights as F16 on Metal instead of aborting, so they take twice their file size
 there, and the [`lora_immediate_fallback.patch`](patches/README.md), which
 applies a LoRA at runtime instead of crashing where the `immediately` mode
 cannot merge it: into FP8 weights, and on Metal, which cannot add into the F16
-convolution weights every model has.
+convolution weights every model has, and the
+[`ggml_apple_vm_allocate.patch`](patches/README.md), which backports
+[llama.cpp#31437](https://github.com/ggml-org/llama.cpp/pull/31437) so iOS
+host buffers go back to the system when freed.
 Upstream's `generate_image` now returns an owned result with per-image
 generation metadata; `sd_dart_generate_image` keeps returning an image array
 that `free_sd_images` frees and drops that metadata. Since upstream 982 a null

patches/README.md

@@ -24,6 +24,7 @@ reports.
 | `lora_immediate_fallback.patch` | `src/pipeline/diffusion_engine.cpp` | The `immediately` LoRA apply mode, which `auto` selects for unquantized weights, crashes the CPU merging into FP8 weights (ggml-cpu can neither multiply nor write FP8) and aborts Metal merging into F16 weights (ggml-metal adds only equal types), which every model has: convolution weights load as F16. FP8 weights, and every model on a backend that cannot run that add outside host memory, now take the LoRA `at_runtime`, with a warning when `immediately` was requested ([#31](https://github.com/leehack/stable-diffusion-native/issues/31)). | `tests/test_fp8_runtime.py` |
 
 | `intrinsic_tiny_decoder_policy.patch` | `src/stable-diffusion.cpp`, `src/pipeline/diffusion_engine.cpp` | Owner-only `sd_dart_new_sd_ctx_intrinsic_tiny_v1` context contract for [llamadart #1047](https://github.com/leehack/llamadart/issues/1047), not an upstream defect: automatic model refresh is valid upstream behavior. The public C API cannot preserve a decoder invariant, so minimal context creation/free and candidate/execution insertion points reject a full-VAE replacement before runner probes or decode. Real unchanged/refreshed SDXS and unrestricted reload controls exercise the boundary. Retire when upstream exposes an equivalent persistent decoder policy. | `tests/test_decoder_policy_runtime.py` |
+| `ggml_apple_vm_allocate.patch`, `ggml_metal_apple_vm_allocate.patch` | `ggml/src/ggml.c`, `ggml/src/ggml-metal/ggml-metal-device.m` | On iOS, ggml's CPU and Metal host buffers come from `posix_memalign`/`free`, and libmalloc keeps the freed pages in the app's memory footprint, so an unloaded model still counts toward the jetsam limit ([llamadart #2004](https://github.com/leehack/llamadart/issues/2004)). Every Apple platform now uses `vm_allocate`/`vm_deallocate`, as macOS already did. A backport of llama.cpp `ea24b952c` ([ggml-org/llama.cpp#31437](https://github.com/ggml-org/llama.cpp/pull/31437)); retire when the pinned ggml includes it. Two patches because the files belong to the `ggml-base` and `ggml-metal` targets. | `tools/validate_artifacts.py` (an iOS library must not import `_posix_memalign`), `tests/test_tools.py` |
 
 Patches touching the same source compose in declaration order. Each next
 patch applies to the preceding build-tree copy; the submodule stays pristine.

patches/ggml_apple_vm_allocate.patch

@@ -0,0 +1,23 @@
+diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
+index da9ccad..d379020 100644
+--- a/ggml/src/ggml.c
++++ b/ggml/src/ggml.c
+@@ -357,7 +357,8 @@ void * ggml_aligned_malloc(size_t size) {
+     void * aligned_memory = NULL;
+   #ifdef GGML_USE_CPU_HBM
+     int result = hbw_posix_memalign(&aligned_memory, alignment, size);
+-  #elif TARGET_OS_OSX && !defined(GGML_TSAN_ENABLED)
++  #elif defined(__APPLE__) && !defined(GGML_TSAN_ENABLED)
++    // libmalloc can keep freed pages in the process footprint, vm_deallocate returns them at once
+     GGML_UNUSED(alignment);
+     kern_return_t alloc_status = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t *) &aligned_memory, size, VM_FLAGS_ANYWHERE);
+     int result = EFAULT;
+@@ -404,7 +405,7 @@ void ggml_aligned_free(void * ptr, size_t size) {
+     if (ptr != NULL) {
+         hbw_free(ptr);
+     }
+-#elif TARGET_OS_OSX && !defined(GGML_TSAN_ENABLED)
++#elif defined(__APPLE__) && !defined(GGML_TSAN_ENABLED)
+     if (ptr != NULL) {
+         vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ptr, size);
+     }

patches/ggml_metal_apple_vm_allocate.patch

@@ -0,0 +1,36 @@
+diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
+index 4451990..a59f37f 100644
+--- a/ggml/src/ggml-metal/ggml-metal-device.m
++++ b/ggml/src/ggml-metal/ggml-metal-device.m
+@@ -2093,19 +2093,11 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) {
+ static void * ggml_metal_host_malloc(size_t n) {
+     void * data = NULL;
+
+-#if TARGET_OS_OSX
+     kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE);
+     if (err != KERN_SUCCESS) {
+         GGML_LOG_ERROR("%s: error: vm_allocate failed\n", __func__);
+         return NULL;
+     }
+-#else
+-    const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);
+-    if (result != 0) {
+-        GGML_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);
+-        return NULL;
+-    }
+-#endif
+
+     return data;
+ }
+@@ -2287,11 +2279,7 @@ void ggml_metal_buffer_free(ggml_metal_buffer_t buf) {
+     }
+
+     if (buf->is_shared && buf->owned) {
+-#if TARGET_OS_OSX
+         vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)buf->all_data, buf->all_size);
+-#else
+-        free(buf->all_data);
+-#endif
+     }
+
+     free(buf);

tests/test_tools.py

@@ -15,6 +15,7 @@
     APPLE_STATIC_DESTRUCTOR_IMPORT,
     ELF_STATIC_DESTRUCTOR_IMPORT,
     ELF_STREAM_INIT_IMPORT,
+    IOS_HOST_BUFFER_IMPORT,
     elf_imports_symbol,
     elf_undefined_symbols,
     imports_symbol,
@@ -100,6 +101,21 @@ def test_wrapper_header_adds_only_the_documented_exports(self):
 }
 """
 
+HOST_BUFFER_WITH_POSIX_MEMALIGN = """
+#include <stdlib.h>
+void *host_buffer(unsigned long n) {
+    void *p = 0;
+    return posix_memalign(&p, 16384, n) == 0 ? p : 0;
+}
+"""
+HOST_BUFFER_WITH_VM_ALLOCATE = """
+#include <mach/mach.h>
+void *host_buffer(unsigned long n) {
+    vm_address_t p = 0;
+    return vm_allocate(mach_task_self(), &p, n, VM_FLAGS_ANYWHERE) == KERN_SUCCESS ? (void *) p : 0;
+}
+"""
+
 
 @unittest.skipUnless(sys.platform == "darwin", "reads Mach-O imports with nm")
 class StaticDestructorImportTest(unittest.TestCase):
@@ -125,6 +141,13 @@ def test_reads_the_named_architecture_of_a_universal_file(self):
             for arch in ("arm64", "x86_64"):
                 self.assertTrue(imports_symbol(system, APPLE_STATIC_DESTRUCTOR_IMPORT, arch))
 
+    def test_tells_posix_memalign_host_buffers_from_vm_allocate_ones(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            old = self.build(tmp, "old", HOST_BUFFER_WITH_POSIX_MEMALIGN)
+            new = self.build(tmp, "new", HOST_BUFFER_WITH_VM_ALLOCATE)
+            self.assertTrue(imports_symbol(old, IOS_HOST_BUFFER_IMPORT))
+            self.assertFalse(imports_symbol(new, IOS_HOST_BUFFER_IMPORT))
+
 
 # `readelf --dyn-syms -W` of GNU binutils and of LLVM: the first appends the
 # index of a symbol's version, the second does not.

tools/validate_artifacts.py

@@ -43,6 +43,12 @@
 # are still inside it (leehack/llamadart#949). Android libraries are left as
 # they were and import it.
 ELF_STATIC_DESTRUCTOR_IMPORT = "__cxa_atexit"
+# ggml's host buffers on iOS come from `posix_memalign`, and libmalloc keeps
+# their freed pages in the app's memory footprint for as long as it likes, so a
+# model that is unloaded still counts toward the jetsam limit
+# (leehack/llamadart#2004). `patches/ggml_apple_vm_allocate.patch` makes every
+# Apple build use `vm_allocate`, as macOS already did.
+IOS_HOST_BUFFER_IMPORT = "_posix_memalign"
 # `std::ios_base::Init::Init()`. Before GCC 13 every source that includes
 # <iostream> holds an object of that class, whose destructor flushes the
 # standard streams when the last one goes. A Linux library drops that
@@ -184,6 +190,11 @@ def validate(target: Target) -> list[str]:
             f"imports {APPLE_STATIC_DESTRUCTOR_IMPORT}: static destructors bypass exit teardown"
         )
 
+    if target.os == "ios" and imports_symbol(library, IOS_HOST_BUFFER_IMPORT):
+        problems.append(
+            f"imports {IOS_HOST_BUFFER_IMPORT}: host buffers outlive their free in the app footprint"
+        )
+
     if target.os == "linux":
         imports = elf_undefined_symbols(output(["readelf", "--dyn-syms", "-W", str(library)]))
         if ELF_STATIC_DESTRUCTOR_IMPORT in imports: