From bbef88594909c8bc1e94d0da1f07496540a1df72 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 15 Nov 2026 17:09:15 +0800 Subject: [PATCH 1/3] refactor(api): adopt opaque options, tensors, and results --- CMakeLists.txt | 5 + docs/README.md | 2 + docs/development/c_api.md | 63 ++ docs/guides/backend.md | 9 +- docs/guides/c_api.md | 507 ++++++++++++ docs/guides/caching.md | 10 +- docs/guides/generation_metadata.md | 49 +- docs/guides/image_preprocessing.md | 24 +- docs/guides/prompt_images.md | 22 +- docs/guides/sage_attention.md | 6 +- docs/guides/tokenizers.md | 10 +- docs/models/minimax_h3.md | 4 +- docs/troubleshooting.md | 2 +- examples/cli/CMakeLists.txt | 1 - examples/cli/README.md | 7 + examples/cli/main.cpp | 264 +++---- examples/common/common.cpp | 1045 +++++++++++-------------- examples/common/common.h | 185 +++-- examples/common/media_io.cpp | 30 +- examples/common/media_io.h | 32 +- examples/common/options.h | 185 +++++ examples/common/resource_owners.hpp | 486 ++++++++++-- examples/server/README.md | 10 + examples/server/api.md | 12 +- examples/server/async_jobs.cpp | 45 +- examples/server/main.cpp | 12 +- examples/server/model_catalog.cpp | 87 +- examples/server/routes_models.cpp | 4 +- examples/server/routes_openai.cpp | 24 +- examples/server/routes_sdapi.cpp | 185 +++-- examples/server/routes_sdcpp.cpp | 117 ++- examples/server/runtime.cpp | 111 +-- examples/server/runtime.h | 22 +- include/stable-diffusion.h | 949 ++++++++-------------- scripts/format-code.ps1 | 3 + scripts/format-code.sh | 1 + src/api/api.cpp | 245 ++++++ src/api/api.h | 34 + src/api/options.cpp | 897 +++++++++++++++++++++ src/api/options.h | 25 + src/api/result.cpp | 241 ++++++ src/api/result.h | 57 ++ src/api/tensor.cpp | 191 +++++ src/api/tensor.h | 93 +++ src/conditioning/conditioner.hpp | 8 +- src/config.cpp | 133 ++++ src/config.h | 295 +++++++ src/core/backend_fit.cpp | 3 +- src/core/backend_fit.h | 3 +- src/core/cuda/int8.cu | 6 +- src/core/ggml_tensor_utils.cpp | 2 +- src/core/ggml_tensor_utils.h | 6 +- src/core/media.h | 33 + src/core/option_types.cpp | 265 +++++++ src/core/option_types.h | 165 ++++ src/core/util.cpp | 7 +- src/core/util.h | 8 +- src/detailer.cpp | 208 ++--- src/detailer.h | 6 +- src/extensions/generation_extension.h | 7 +- src/generation_info.cpp | 87 +- src/generation_info.h | 11 +- src/model/diffusion/control.hpp | 2 + src/model/integrations/sd.cpp | 1 + src/model/vae/minimax_h3_vae.hpp | 10 +- src/model/vae/vae.hpp | 18 +- src/model/vae/vae_tiling.hpp | 4 +- src/model_definition.h | 2 + src/pipeline/diffusion_engine.cpp | 58 +- src/pipeline/diffusion_engine.h | 57 +- src/pipeline/generation.h | 20 +- src/pipeline/image.cpp | 34 +- src/pipeline/model_builders.cpp | 1 + src/pipeline/model_builders.h | 3 +- src/pipeline/relay_vsr.cpp | 5 +- src/pipeline/request.cpp | 54 +- src/pipeline/request.h | 95 ++- src/pipeline/seedvr2.cpp | 3 +- src/pipeline/video.cpp | 63 +- src/pipeline/video_upscale.cpp | 96 ++- src/pipeline/video_upscale.h | 8 +- src/runtime/denoiser.hpp | 2 + src/runtime/image_preprocess.cpp | 16 +- src/runtime/image_preprocess.h | 25 +- src/runtime/preprocessing.hpp | 6 +- src/runtime/sample-cache.cpp | 15 +- src/runtime/sample-cache.h | 4 +- src/stable-diffusion.cpp | 998 +++-------------------- src/upscaler.cpp | 145 ++-- src/upscaler.h | 4 +- 90 files changed, 5952 insertions(+), 3363 deletions(-) create mode 100644 docs/development/c_api.md create mode 100644 docs/guides/c_api.md create mode 100644 examples/common/options.h create mode 100644 src/api/api.cpp create mode 100644 src/api/api.h create mode 100644 src/api/options.cpp create mode 100644 src/api/options.h create mode 100644 src/api/result.cpp create mode 100644 src/api/result.h create mode 100644 src/api/tensor.cpp create mode 100644 src/api/tensor.h create mode 100644 src/config.cpp create mode 100644 src/config.h create mode 100644 src/core/media.h create mode 100644 src/core/option_types.cpp create mode 100644 src/core/option_types.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 98ec75f4ee..59107d4f40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -240,6 +240,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS "src/*.h" "src/*.cpp" "src/*.hpp" + "src/api/*.h" + "src/api/*.cpp" "src/conditioning/*.h" "src/conditioning/*.cpp" "src/conditioning/*.hpp" @@ -431,6 +433,9 @@ target_include_directories(${SD_LIB} PUBLIC include) target_include_directories(${SD_LIB} PRIVATE src src/core) target_include_directories(${SD_LIB} PUBLIC thirdparty) target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17) +set_target_properties(${SD_LIB} PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) if (SD_BUILD_EXAMPLES) diff --git a/docs/README.md b/docs/README.md index 4e998a5e34..5490146b20 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,6 +15,7 @@ For the project overview and supported models, see the [project README](../READM ## Generation and inputs +- [C API and migration](guides/c_api.md) - [Sampling methods and schedulers](guides/sampling.md) - [Image editing](guides/edit.md) - [Images in prompts](guides/prompt_images.md) @@ -38,6 +39,7 @@ For the project overview and supported models, see the [project README](../READM ## Development +- [C API internals](development/c_api.md) - [Model configuration and integration conventions](development/model_config.md) - [ggml operator extensions](development/ggml_extensions.md) - [Backend performance probes](development/backend_probe.md) diff --git a/docs/development/c_api.md b/docs/development/c_api.md new file mode 100644 index 0000000000..58587a6fac --- /dev/null +++ b/docs/development/c_api.md @@ -0,0 +1,63 @@ +# C API Internals + +The public declarations live in [stable-diffusion.h](../../include/stable-diffusion.h). +See the [C API guide](../guides/c_api.md) for keys, value types and ownership rules. + +## Internal modules + +| Module | Responsibility | +| --- | --- | +| `src/api/options.h/.cpp` | Opaque options, key registration, setter validation and conversion to typed configuration. | +| `src/config.h/.cpp` | Typed configuration and the library defaults for each configuration group. | +| `src/api/tensor.h/.cpp` | Opaque tensor handles backed by `sd::Tensor`, reference counting and conversion to private media views. | +| `src/core/media.h` | Private raw media views used by inference and codecs. | +| `src/api/result.h/.cpp` | Internal result storage, opaque result accessors, metadata attachment and media ownership. | +| `src/api/api.h/.cpp` | Context, generation, callback and query entry points, plus thread-local diagnostics and exception-to-status handling. | +| `src/generation_info.h/.cpp` | Construction of generation metadata JSON. | +| `src/pipeline/generation.h` | Internal generation entry declarations. | +| `src/stable-diffusion.cpp` | Internal typed image/video generation and model-specific validation. | +| `src/core/option_types.h/.cpp` | Internal mode enums, name lookup and parsing. | + +Options are validated and converted once at the API boundary. The runtime reads +typed configuration fields. Internal callers such as ADetailer pass typed +configuration directly to the internal generation entry; they do not rebuild +options or call public setters. + +The option map alone records whether a key was supplied. Zero, false, empty +strings and explicit automatic choices remain entries until `unset` removes +them. Each conversion starts with a fresh `OptConfig`, then overlays supplied +values and validates combinations. Reusing the output configuration cannot +retain a previous request's overrides. ADetailer applies its operation-specific +strength default before the overlay. + +Fixed defaults live in the configuration constructors. Internal automatic +policies (such as model-selected sampler/scheduler and automatic eta/flow shift) +are resolved by the pipeline after refreshing the model sources. Their internal +sentinel values describe an automatic policy, not the presence of an input key. +Do not infer presence by comparing a typed value against its default. + +CLI/server inference inputs use `std::optional`; their option builders emit +only engaged fields. Recipe defaults fill missing startup inputs; request +fields then override the configured recipe/startup values. Explicit empty +numeric arrays replace inherited arrays, and JSON `null` clears an optional +input. Frontend-only settings such as output formats and listen ports keep +local defaults. Frontend checks cover request syntax, file inputs and server +resource policy. Parameter ranges are checked by the public setters used by the +option builders; combinations and model applicability are checked when executing. +Examples include only public library headers and do not query library defaults. +Capability defaults describe configured overrides; null means that the library +selects the value, and result metadata reports effective values. Post-generation +ADetailer clears inherited dimensions and strength before applying pass overrides, +so its operation defaults also come from the library. + +Result storage owns tensors and metadata. Tensor handles contain a dtype-specific +`sd::Tensor` and an atomic reference count; options retain handles rather than +copying tensor payloads. Raw pipeline output buffers are converted to tensors at +the boundary and released there. Internal producers with an `sd::Tensor` can +move it directly into a handle. Video results keep per-frame tensors and assemble +a contiguous video tensor only when requested. Cached assembly is synchronized. +Preview tensors are independent snapshots and may be retained by callbacks. +Metadata construction is separate from its attachment to a result. + +These headers are private implementation details and are not installed as part +of the public C API. diff --git a/docs/guides/backend.md b/docs/guides/backend.md index cc6545da92..f7edddf592 100644 --- a/docs/guides/backend.md +++ b/docs/guides/backend.md @@ -273,8 +273,8 @@ flags are deprecated and emit a warning, but remain supported when `--conv-direct` is omitted. Use `--conv-direct auto,diffusion=on,controlnet=on` and `--conv-direct auto,vae=on`, respectively. An explicit `--conv-direct` takes precedence over both legacy flags regardless of order. -Library callers can set the appended `sd_ctx_params_t::conv_direct_mode` field -to the same string; null or empty selects `auto` while honoring legacy true +Library callers can set the `conv_direct_mode` context-option key with +`sd_opt_set_string()` to the same string; unset or empty selects `auto` while honoring legacy true flags. The context copies the string. ## Flash attention (`--flash-attn on|off|auto`) @@ -339,9 +339,10 @@ row-split configurations, which can fall back to layer split during initializati Use `--flash-attn on` and `--flash-attn off,diffusion=on`, respectively. Both aliases write the same mode string as `--flash-attn`; the last occurrence of any of these options wins. Library callers -can set `sd_ctx_params_t::flash_attn_mode` to the same string, including +can set the `flash_attn_mode` context-option key with `sd_opt_set_string()` +to the same string, including `"on"`, `"off"`, `"auto"`, or `"auto,te=off,vae=on"`. -A null or empty string uses automatic selection unless a +An unset or empty string uses automatic selection unless a legacy `flash_attn` or `diffusion_flash_attn` boolean is set. The context copies the string; no new enum is required. diff --git a/docs/guides/c_api.md b/docs/guides/c_api.md new file mode 100644 index 0000000000..055056d57e --- /dev/null +++ b/docs/guides/c_api.md @@ -0,0 +1,507 @@ +# C API + +Use purpose-specific opaque options for model loading, generation and +postprocessing, and +opaque `sd_result_t` objects to read generated outputs. The library allocates and +frees these objects; applications must not allocate them by size or access fields. + +## Loading and generation + +This C example loads a checkpoint and generates an image. Check every setter: +an unsuccessful setter preserves the previous value, including its unset state. + +```c +#include +#include "stable-diffusion.h" + +int generate(const char* model_path) { + int rc = 1; + sd_ctx_t* ctx = NULL; + sd_result_t* result = NULL; + sd_ctx_opt_t* ctx_opt = sd_ctx_opt_create(); + sd_img_gen_opt_t* image_opt = NULL; + if (!ctx_opt) goto error; + if (sd_opt_set_string(ctx_opt, "model_path", model_path) != SD_STATUS_OK || + sd_opt_set_int(ctx_opt, "n_threads", 8) != SD_STATUS_OK || + sd_ctx_create(ctx_opt, &ctx) != SD_STATUS_OK) + goto error; + sd_opt_free(ctx_opt); + ctx_opt = NULL; + + image_opt = sd_img_gen_opt_create(); + if (!image_opt) goto error; + if (sd_opt_set_string(image_opt, "prompt", "a small house") != SD_STATUS_OK || + sd_opt_set_int(image_opt, "width", 512) != SD_STATUS_OK || + sd_opt_set_int(image_opt, "height", 512) != SD_STATUS_OK || + sd_opt_set_int(image_opt, "seed", 42) != SD_STATUS_OK || + sd_opt_set_int(image_opt, "sample.steps", 20) != SD_STATUS_OK || + sd_opt_set_string(image_opt, "sample.method", "euler") != SD_STATUS_OK || + sd_generate_image(ctx, image_opt, &result) != SD_STATUS_OK) + goto error; + + for (size_t i = 0; i < sd_result_get_image_count(result); ++i) { + const sd_tensor_t* image = sd_result_get_image(result, i); + const char* info = sd_result_get_generation_info(result, i); + /* Read pixels with sd_tensor_data(image), and consume the UTF-8 JSON info. */ + (void) image; + (void) info; + } + rc = 0; + goto cleanup; +error: + fprintf(stderr, "%s\n", sd_get_last_error()); +cleanup: + sd_result_free(result); + sd_ctx_free(ctx); + sd_opt_free(ctx_opt); + sd_opt_free(image_opt); + return rc; +} +``` + +Use `sd_generate_video()` with options from `sd_vid_gen_opt_create()` to generate video. Its result +exposes frames through the same image accessors, optional audio through +`sd_result_get_audio()`, and effective frame rate through `sd_result_get_fps()`. +`sd_result_get_kind()` distinguishes image batches from videos. Video generation +metadata is sequence-wide and is returned for every valid frame index. + +All option names are aliases of the same opaque `sd_opt_t` type and share +all `sd_opt_set_*()`, `sd_opt_unset()` and `sd_opt_free()` functions. The constructor +permanently binds the object's purpose. Setters reject keys for another purpose +immediately, and operation entry points reject an option object of the wrong +purpose even if empty. For example, `prompt` cannot be set on context options, +and image options cannot be passed to `sd_ctx_create()`. These checks occur at +runtime; the C typedef aliases provide descriptive names rather than distinct +compile-time types. + +## Operations + +Each operation has its own constructor and uses the shared setters and +`sd_opt_free()`. Creation and execution return `sd_status_t`; output pointers +are required and set to `NULL` on failure. All outputs use `sd_result_t`. + +| Option constructor | Operation | +| --- | --- | +| `sd_ctx_opt_create()` | `sd_ctx_create()` | +| `sd_img_gen_opt_create()` | `sd_generate_image()` | +| `sd_vid_gen_opt_create()` | `sd_generate_video()` | +| `sd_upscaler_opt_create()` | `sd_upscaler_create()` | +| `sd_upscale_opt_create()` | `sd_upscale()` | +| `sd_vid_upscaler_opt_create()` | `sd_vid_upscaler_create()` | +| `sd_vid_upscale_opt_create()` | `sd_upscale_video()` | +| `sd_adetailer_opt_create()` | `sd_adetailer_create()` | +| `sd_adetail_opt_create()` | `sd_adetail()` | + +Image upscaling loads `model_path` with optional `n_threads`, `backend`, +`params_backend`, `conv_direct` and `tile_size` (default 128). Execution takes +an RGB `image`. Its scale is fixed by the loaded model and can be queried with +`sd_upscaler_get_scale()`; metadata records that actual scale. The former +image-upscale factor argument was unused and has been removed. + +```c +sd_upscaler_opt_t* load = sd_upscaler_opt_create(); +sd_upscaler_t* upscaler = NULL; +sd_upscale_opt_t* opt = NULL; +sd_result_t* result = NULL; +sd_status_t status = load ? sd_opt_set_string(load, "model_path", model_path) + : SD_STATUS_OUT_OF_MEMORY; +if (status == SD_STATUS_OK) status = sd_upscaler_create(load, &upscaler); +if (status == SD_STATUS_OK) { + opt = sd_upscale_opt_create(); + status = sd_opt_set_tensor(opt, "image", image); + if (status == SD_STATUS_OK) status = sd_upscale(upscaler, opt, &result); +} +/* Consume result on success. */ +sd_result_free(result); +sd_opt_free(opt); +sd_upscaler_free(upscaler); +sd_opt_free(load); +``` + +Video-upscaler loading requires `diffusion_model_path`, `vae_path` and +`upscale_factor`; backend/thread/attention settings are optional. The scale must +match both checkpoints (RelayVSR uses 2 or 4). Execution takes RGB `frames` with +equal dimensions. Defaults are FPS 24, seed 42, RelayVSR `window_size` 16, and +SeedVR2 `temporal_chunk_size` 5. Free the model with `sd_vid_upscaler_free()`. + +ADetailer loading takes the detector `model_path` and optional backend/thread +settings. Call `sd_adetail(adetailer, diffusion_ctx, opt, &result)` with an +`sd_adetail_opt_t`. Set `image` for the source image, and reuse the applicable +image-generation keys for inpainting: `prompt`, `negative_prompt`, dimensions, +strength, seed, sampling, LoRAs, cache, VAE tiling and circular axes. Inpainting +defaults to 512×512 and strength 0.4. `adetail.prompt` and +`adetail.negative_prompt` supply region prompt overrides; `adetail.args` accepts +the existing detector/mask/inpaint argument string documented by the CLI +`--extra-ad-args` option. Reference/control images, hires and batching are not +ADetailer options. Free the detector with `sd_adetailer_free()`. + +## String-valued modes + +Sampler, scheduler, prediction, RNG, cache, control, hires and VAE option enums +are internal. Set their documented string values through `sd_opt_set_string()`. +Leave `sample.method` and `sample.scheduler` unset, or set them to `"auto"`, +to select defaults for the loaded model and sampler. + +The bundled CLI and server retain these modes as strings and pass them directly +to setters. Enum parsers, enum name helpers, parameter configuration layouts and +name tables are internal and are not exported by the shared library. + +Preview registration also uses strings: + +```c +sd_status_t status = sd_set_preview_callback(callback, "proj", 5, + true, false, user_data); +/* Disable registration when the context is idle. */ +sd_set_preview_callback(NULL, "none", 1, false, false, NULL); +``` + +Preview modes are `none`, `proj`, `tae` and `vae`. A null or unknown mode returns +`SD_STATUS_INVALID_ARGUMENT` and preserves the previous registration. See the +public header for interval, noisy/denoised and callback lifetime semantics. + +## Tensor data + +All media payloads use opaque `sd_tensor_t` handles, backed by the library's +`sd::Tensor`. The declarations are in `stable-diffusion.h`; no additional +public header is required. Dimensions are ordered from most contiguous to least +contiguous, matching `sd::Tensor`: + +| Media | Element type | Shape | +| --- | --- | --- | +| Image | `SD_TENSOR_U8` | `[channels, width, height]` | +| Audio samples | `SD_TENSOR_F32` | `[channels, samples]` | +| Video frames | `SD_TENSOR_U8` | `[channels, width, height, frames]` | + +Channels are interleaved. Audio duration is `samples / sample_rate`; sample count +and channel count come from the tensor dimensions. `sd_audio_t` adds `sample_rate` +to a sample tensor. `sd_video_t` adds FPS and an optional audio descriptor to a +video tensor. Image dimensions and channel count are entirely in the tensor. + +`sd_tensor_create(type, rank, shape, data, data_size)` copies the supplied bytes. +The byte count must exactly match the element type and dimensions. General +tensors support U8, I32, I64, F32 and F64, rank up to 8, rank-zero scalars and +zero-sized dimensions. Media keys validate their required type, rank and positive +dimensions. + +```c +int64_t shape[] = {3, width, height}; +sd_tensor_t* image = sd_tensor_create(SD_TENSOR_U8, 3, shape, + pixels, (size_t)width * height * 3); +if (image) { + sd_status_t status = sd_opt_set_tensor(opt, "init_image", image); + sd_tensor_free(image); /* opt retains the tensor after a successful setter. */ + (void)status; +} +``` + +Use `sd_opt_set_tensors()` for image lists such as `ref_images`; individual images +may have different dimensions. Use one 4D tensor for `control_frames` and video +upscaler `frames`. Use `sd_opt_set_audios()` and `sd_opt_set_videos()` for lists +whose items also carry sample-rate/FPS metadata. These setters retain all nested +tensors. Freeing the caller's tensor handles or descriptor arrays does not +invalidate configured options. + +`sd_result_get_image()` returns a U8 image/frame tensor. `sd_result_get_audio()` +returns sample-rate metadata and a F32 sample tensor. `sd_result_get_video()` +returns FPS, optional audio and a contiguous 4D tensor, assembled and cached on +first access. Reading individual video frames avoids that assembly and copy. + +Tensor data is read-only. `sd_tensor_retain()` extends the lifetime of any tensor, +including result and preview tensors, independently of its original owner; +release each owned reference with `sd_tensor_free()`. Retain/free accept NULL and +preserve the last diagnostic. Preview callbacks receive an array of borrowed +image tensors. Prompt-image callbacks return a borrowed image tensor that stays +valid until the next callback. `sd_preprocess_canny()` returns a new image tensor +and leaves its input unchanged. + +## Values, validation and ownership + +- Unset keys retain existing model-loading and generation defaults. + `sd_opt_unset()` restores this state and succeeds for an applicable known absent key. + Zero, false, empty strings and empty arrays remain explicit values, subject to + that key's validation. `NULL` strings are invalid; use `unset` for defaults. +- Keys and enum values are case-sensitive. Unknown keys return + `SD_STATUS_UNKNOWN_KEY`; incorrect setter types return `SD_STATUS_TYPE_MISMATCH`. + Numeric overflow/range errors and invalid enum names are rejected by setters. + No automatic conversion occurs between integer, floating-point and boolean values. +- Setters and unset reject keys for another purpose with `SD_STATUS_UNSUPPORTED`, + even when set to a default-looking value. Loading/generation checks the option + object purpose and parameter combinations. Model-dependent features are checked at generation: + for example UCache requires UNet and EasyCache/CacheDIT require DiT. Known + high-noise sampling settings and `moe_boundary` are ignored without a high-noise + stage; `vace_strength` is ignored by non-VACE models. Actual control-frame inputs + still require VACE. Existing size/frame alignment and model-dependent automatic + defaults are retained. +- Scalar strings, numeric arrays, LoRA paths and embedding names/paths are copied. + Tensor arrays and audio/video metadata are copied, and tensor references are + retained. Empty arrays accept a `NULL` pointer and count 0. +- Successful context creation owns the configuration it retains. Free or reuse + the loading params immediately afterwards. Generation params may be freed or + changed after the generation call returns. Do not mutate/free an option object + while another operation is using it. +- Results own their media and metadata independently of the context and params. + Read-only views remain valid until `sd_result_free()`. Do not free or modify + buffers through a view. Image results have no audio and return FPS 0. Invalid + indices return `NULL` with a diagnostic. All new free functions accept `NULL`. +- Output pointers are required and set to `NULL` on failure. Partial batch + cancellation can return successful completed outputs; full cancellation returns + `SD_STATUS_CANCELLED` without a result. Other failures distinguish invalid + arguments, unsupported features, memory exhaustion, loading and generation. +- `sd_get_last_error()` returns a thread-local diagnostic. Status/accessor calls + clear it on success; free functions preserve it. Read it before another API call. + +Option and tensor objects have no public field-layout contract. Future settings +normally add a key while retaining existing key names, types and meanings. To +keep an output after releasing the result, retain its tensor first; release the +retained handle with `sd_tensor_free()` when finished. Result metadata and media +metadata descriptors remain borrowed from the result and should be copied when +needed independently. + +## Keys + +Types below correspond to `sd_opt_set_()`. `context`, `image` and `video` +refer to model loading and image/video generation. Other operation labels match +the corresponding option constructor, such as `vid_upscale` for +`sd_vid_upscale_opt_create()`. +Array lengths are supplied to setters; there are no separately writable count keys. + +Top-level keys retain their existing parameter meanings. Nested names map as follows: + +| Key prefix | Configuration | +| --- | --- | +| `sample.*` | Base sampling settings; `method`, `steps`, `sigmas`, `args` select the sampler, step count, custom schedule and extra arguments. | +| `high_noise_sample.*` | High-noise sampling settings for two-stage video models. | +| `sample.guidance.*` | Text/image CFG, distilled guidance and SLG settings. | +| `cache.*` | [Inference cache](caching.md) settings. | +| `hires.*` | High-resolution second pass and upscaling settings. | +| `vae_tiling.*` | Spatial/temporal VAE tiling settings. | +| `photomaker.*` | PhotoMaker identity images/embedding and style strength. | +| `pulid.*` | PuLID identity embedding path and weight. | +| `image_preprocess.rules` | [Image preprocessing](image_preprocessing.md) rule string. | + +The library supplies inference defaults. CLI/server inputs that are omitted do +not override them. Model recipes may supply presets; explicit startup options +and request fields take priority over recipe values. A supplied value equal to +a default is still explicit and undergoes model applicability checks. + +Generation defaults include 512×512, a random seed (`-1`), 20 sampling steps, +model-selected sampler/scheduler, text CFG 7, distilled guidance 3.5, strength +0.75, and disabled caching/hires/tiling. Image batches default to 1; videos default +to 6 requested frames at 16 FPS, subject to model alignment. Omitted eta, flow shift +and image CFG retain automatic defaults; do not pass infinity to request defaults. +Use `unset`. High-noise steps default to `-1` (automatic allocation). The high-noise pass +shares the base scheduler, sigmas, flow shift and timestep shifting; set those +through `sample.*`. Separate high-noise keys for these settings are not supported. + +Sampler/scheduler values follow the [sampling guide](sampling.md); `"auto"` +requests automatic selection. RNG values are `std_default`, `cuda`, `cpu`, `auto`. +Prediction modes are `auto`, `eps`, `v`, `edm_v`, `sd3_flow`, `flux_flow`, +`sefi_flow`, `minit2i_flow`, `sensenova_u1_flow`, `flux3_flow`, `h3_av_flow`, +`ming_image_flow`. Weight types use the names returned by `sd_type_name()`, +plus `auto`. VAE formats are `auto`, `flux`, `sd3`, +`flux2`, `wan`. LoRA apply modes are `auto`, `immediately`, `at_runtime`. +Cache modes are `disabled`, `easycache`, `ucache`, `dbcache`, `taylorseer`, +`cache_dit`, `spectrum`. Control types are `none`, `openpose`, `depth`, `scribble`, `canny`, `normal`, +`segment`, `tile`, `repaint`, with the existing aliases such as `hed` and `lineart`. Hires upscalers are `none`, `latent`, +`latent_nearest`, `latent_nearest_exact`, `latent_antialiased`, `latent_bicubic`, +`latent_bicubic_antialiased`, `lanczos`, `nearest`, `model`. + +Integers must fit the underlying field (32 bits except `seed`, which is signed +64-bit). Dimensions, batch size, video frames, FPS and base steps are positive. +Floating-point settings must be finite and fit float32; strength, denoising +strength, percent/range settings and MOE boundary lie in [0, 1]. Custom sigma +arrays are empty or strictly decreasing nonnegative finite values ending in zero, +with at least two entries. SLG layer indices are nonnegative integers. + +| Key | Setter type | Operation | +| --- | --- | --- | +| `adetail.args` | `string` | adetail | +| `adetail.negative_prompt` | `string` | adetail | +| `adetail.prompt` | `string` | adetail | +| `attn_scale` | `float` | context | +| `audio_encoder_path` | `string` | context | +| `audio_vae_path` | `string` | context | +| `auto_fit` | `bool` | context | +| `backend` | `string` | context, upscaler, vid_upscaler, adetailer | +| `batch_count` | `int` | image | +| `batched_cfg` | `bool` | context | +| `cache.bn_compute_blocks` | `int` | image, video, adetail | +| `cache.end_percent` | `float` | image, video, adetail | +| `cache.error_decay_rate` | `float` | image, video, adetail | +| `cache.fn_compute_blocks` | `int` | image, video, adetail | +| `cache.max_cached_steps` | `int` | image, video, adetail | +| `cache.max_continuous_cached_steps` | `int` | image, video, adetail | +| `cache.max_warmup_steps` | `int` | image, video, adetail | +| `cache.mode` | `string` | image, video, adetail | +| `cache.reset_error_on_compute` | `bool` | image, video, adetail | +| `cache.residual_diff_threshold` | `float` | image, video, adetail | +| `cache.reuse_threshold` | `float` | image, video, adetail | +| `cache.scm_mask` | `string` | image, video, adetail | +| `cache.scm_policy_dynamic` | `bool` | image, video, adetail | +| `cache.spectrum_flex_window` | `float` | image, video, adetail | +| `cache.spectrum_lam` | `float` | image, video, adetail | +| `cache.spectrum_m` | `int` | image, video, adetail | +| `cache.spectrum_stop_percent` | `float` | image, video, adetail | +| `cache.spectrum_w` | `float` | image, video, adetail | +| `cache.spectrum_warmup_steps` | `int` | image, video, adetail | +| `cache.spectrum_window_size` | `int` | image, video, adetail | +| `cache.start_percent` | `float` | image, video, adetail | +| `cache.taylorseer_n_derivatives` | `int` | image, video, adetail | +| `cache.taylorseer_skip_interval` | `int` | image, video, adetail | +| `cache.use_relative_threshold` | `bool` | image, video, adetail | +| `circular_x` | `bool` | image, video, adetail | +| `circular_y` | `bool` | image, video, adetail | +| `clip_g_path` | `string` | context | +| `clip_l_path` | `string` | context | +| `clip_skip` | `int` | image, video, adetail | +| `clip_vision_path` | `string` | context | +| `conditioning_cache_size` | `int` | context | +| `control_frames` | `tensor` | video | +| `control_image` | `tensor` | image | +| `control_net_path` | `string` | context | +| `control_strength` | `float` | image | +| `control_type` | `string` | image | +| `conv_direct` | `bool` | upscaler | +| `conv_direct_mode` | `string` | context, vid_upscaler | +| `diffusion_conv_direct` | `bool` | context | +| `diffusion_flash_attn` | `bool` | context | +| `diffusion_model_path` | `string` | context, vid_upscaler | +| `disable_prefetch` | `bool` | context | +| `disable_segmented_compute` | `bool` | context | +| `eager_load` | `bool` | context | +| `embeddings` | `embeddings` | context | +| `embeddings_connectors_path` | `string` | context | +| `enable_mmap` | `bool` | context | +| `end_image` | `tensor` | video | +| `flash_attn` | `bool` | context | +| `flash_attn_mode` | `string` | context, vid_upscaler | +| `force_sdxl_vae_conv_scale` | `bool` | context | +| `fps` | `int` | video, vid_upscale | +| `frames` | `tensor` | vid_upscale | +| `height` | `int` | image, video, adetail | +| `high_noise_diffusion_model_path` | `string` | context | +| `high_noise_sample.args` | `string` | video | +| `high_noise_sample.eta` | `float` | video | +| `high_noise_sample.guidance.distilled_guidance` | `float` | video | +| `high_noise_sample.guidance.img_cfg` | `float` | video | +| `high_noise_sample.guidance.slg.layer_end` | `float` | video | +| `high_noise_sample.guidance.slg.layer_start` | `float` | video | +| `high_noise_sample.guidance.slg.layers` | `ints` | video | +| `high_noise_sample.guidance.slg.scale` | `float` | video | +| `high_noise_sample.guidance.txt_cfg` | `float` | video | +| `high_noise_sample.method` | `string` | video | +| `high_noise_sample.steps` | `int` | video | +| `hires.denoising_strength` | `float` | image, video | +| `hires.enabled` | `bool` | image, video | +| `hires.model_path` | `string` | image, video | +| `hires.scale` | `float` | image, video | +| `hires.sigmas` | `floats` | image, video | +| `hires.steps` | `int` | image, video | +| `hires.target_height` | `int` | image, video | +| `hires.target_width` | `int` | image, video | +| `hires.upscale_tile_size` | `int` | image, video | +| `hires.upscaler` | `string` | image, video | +| `image` | `tensor` | upscale, adetail | +| `image_preprocess.rules` | `string` | image, video | +| `init_image` | `tensor` | image, video | +| `ip_adapter_image` | `tensor` | image | +| `ip_adapter_path` | `string` | context | +| `ip_adapter_strength` | `float` | image | +| `linear_scale` | `float` | context | +| `llm_path` | `string` | context | +| `llm_vision_path` | `string` | context | +| `lora_apply_mode` | `string` | context | +| `loras` | `loras` | image, video, adetail | +| `mask_image` | `tensor` | image | +| `max_vram` | `string` | context | +| `model_args` | `string` | context | +| `model_path` | `string` | context, upscaler, adetailer | +| `moe_boundary` | `float` | video | +| `motion_module_path` | `string` | context | +| `n_threads` | `int` | context, upscaler, vid_upscaler, adetailer | +| `negative_prompt` | `string` | image, video, adetail | +| `params_backend` | `string` | context, upscaler, vid_upscaler, adetailer | +| `photo_maker_path` | `string` | context | +| `photomaker.id_embed_path` | `string` | image | +| `photomaker.id_images` | `tensors` | image | +| `photomaker.style_strength` | `float` | image | +| `prediction` | `string` | context | +| `prompt` | `string` | image, video, adetail | +| `pulid.id_embedding_path` | `string` | image | +| `pulid.id_weight` | `float` | image | +| `pulid_weights_path` | `string` | context | +| `qwen_image_layers` | `int` | image | +| `ref_audios` | `audios` | video | +| `ref_image_args` | `string` | image, video | +| `ref_images` | `tensors` | image, video | +| `ref_videos` | `videos` | video | +| `rng_type` | `string` | context | +| `rpc_servers` | `string` | context | +| `sage_attn` | `bool` | context | +| `sample.args` | `string` | image, video, adetail | +| `sample.eta` | `float` | image, video, adetail | +| `sample.flow_shift` | `float` | image, video, adetail | +| `sample.guidance.distilled_guidance` | `float` | image, video, adetail | +| `sample.guidance.img_cfg` | `float` | image, video, adetail | +| `sample.guidance.slg.layer_end` | `float` | image, video, adetail | +| `sample.guidance.slg.layer_start` | `float` | image, video, adetail | +| `sample.guidance.slg.layers` | `ints` | image, video, adetail | +| `sample.guidance.slg.scale` | `float` | image, video, adetail | +| `sample.guidance.txt_cfg` | `float` | image, video, adetail | +| `sample.method` | `string` | image, video, adetail | +| `sample.scheduler` | `string` | image, video, adetail | +| `sample.shifted_timestep` | `int` | image, video, adetail | +| `sample.sigmas` | `floats` | image, video, adetail | +| `sample.steps` | `int` | image, video, adetail | +| `sampler_rng_type` | `string` | context | +| `seed` | `int` | image, video, vid_upscale, adetail | +| `split_mode` | `string` | context | +| `strength` | `float` | image, video, adetail | +| `t5xxl_path` | `string` | context | +| `tae_preview_only` | `bool` | context | +| `taesd_path` | `string` | context | +| `temporal_chunk_size` | `int` | vid_upscale | +| `tensor_type_rules` | `string` | context | +| `tile_size` | `int` | upscaler | +| `tokenizer` | `string` | context | +| `uncond_diffusion_model_path` | `string` | context | +| `upscale_factor` | `int` | vid_upscaler | +| `vace_strength` | `float` | video | +| `vae_conv_direct` | `bool` | context | +| `vae_format` | `string` | context | +| `vae_path` | `string` | context, vid_upscaler | +| `vae_tiling.enabled` | `bool` | image, video, adetail | +| `vae_tiling.extra_tiling_args` | `string` | image, video, adetail | +| `vae_tiling.rel_size_h` | `float` | image, video, adetail | +| `vae_tiling.rel_size_w` | `float` | image, video, adetail | +| `vae_tiling.target_overlap` | `float` | image, video, adetail | +| `vae_tiling.temporal_tiling` | `bool` | image, video, adetail | +| `vae_tiling.tile_size_h` | `int` | image, video, adetail | +| `vae_tiling.tile_size_w` | `int` | image, video, adetail | +| `video_frames` | `int` | video | +| `width` | `int` | image, video, adetail | +| `window_size` | `int` | vid_upscale | +| `wtype` | `string` | context | + +## Migration + +This is a breaking C API change. The former public generation parameter layouts, +their initialization/formatting helpers, `new_sd_ctx()`, `free_sd_ctx()`, +`generate_image()`, `generate_video()` and public result containers/destructors +have been removed. Applications and bindings must migrate and rebuild; no +compatibility layer is provided. + +Create context/image/video options with the corresponding constructor, populate +them through the shared typed setters, and use `sd_ctx_create()`, +`sd_generate_image()` and `sd_generate_video()`. Read or retain outputs through +`sd_result_get_*()` and `sd_tensor_retain()` and release the result with +`sd_result_free()`. ADetailer and image/video upscalers use purpose-specific opt constructors and +return the same opaque result type. Their former public parameter structures, +long positional signatures and bool-returning entry points have been removed. +Device selection and cancellation controls retain their interfaces. Preview +registration now uses a mode string and returns a status. + +Native generation JSON retains its existing schema and names, including +`sample_params.sample_method`; translate these to the corresponding option keys +(`sample.method`, etc.) when replaying metadata. See +[generation results and metadata](generation_metadata.md). diff --git a/docs/guides/caching.md b/docs/guides/caching.md index f69cfbeca8..66dfc27a00 100644 --- a/docs/guides/caching.md +++ b/docs/guides/caching.md @@ -5,11 +5,11 @@ Caching methods accelerate diffusion inference by reusing intermediate computati ### Conditioning Cache Conditioning results are cached per model context using an LRU cache. The default -capacity is **0 (disabled) for `sd-cli`** and **4 entries for `sd-server` and the C -API**. Set `--conditioning-cache-size N` to change the limit; `0` disables caching. -For example, `sd-cli -m model.safetensors -p "a cat" --conditioning-cache-size 4` -enables the cache in the CLI. The C API option is -`sd_ctx_params_t::conditioning_cache_size`, initialized by `sd_ctx_params_init()`. +capacity is **4 entries for `sd-cli`, `sd-server` and the C API**. +Set `--conditioning-cache-size N` to change the limit; `0` disables caching. +For example, `sd-cli -m model.safetensors -p "a cat" --conditioning-cache-size 8` +increases the CLI cache capacity to 8 entries. The C API option is +`conditioning_cache_size`, set with `sd_opt_set_int()` on context options. This cache is independent of the diffusion-step `--cache-mode` options below. ### Cache Modes diff --git a/docs/guides/generation_metadata.md b/docs/guides/generation_metadata.md index 1f7fc3338c..f24bda25fb 100644 --- a/docs/guides/generation_metadata.md +++ b/docs/guides/generation_metadata.md @@ -1,30 +1,31 @@ # Generation results and metadata -`generate_image()`, `generate_video()`, `adetail_image()` and `upscale()` return -owned results containing the media and its generation record. This is a breaking -C API change: applications and bindings must update their calls and rebuild. +`sd_generate_image()`, `sd_generate_video()`, `sd_adetail()`, `sd_upscale()` and +`sd_upscale_video()` return opaque `sd_result_t` objects containing generated media +and generation metadata. See the [C API guide](c_api.md) for option creation and +migration from the removed parameter/result structures. ```c -sd_img_gen_params_t params; -sd_img_gen_params_init(¶ms); -params.prompt = "a small house"; - -sd_image_generation_result_t* result = NULL; -if (generate_image(ctx, ¶ms, &result)) { - for (int i = 0; i < result->count; ++i) { - const sd_image_result_t* item = &result->items[i]; - /* Consume item->image and item->generation_info together. */ +sd_img_gen_opt_t* opt = sd_img_gen_opt_create(); +sd_result_t* result = NULL; +if (opt && sd_opt_set_string(opt, "prompt", "a small house") == SD_STATUS_OK && + sd_generate_image(ctx, opt, &result) == SD_STATUS_OK) { + for (size_t i = 0; i < sd_result_get_image_count(result); ++i) { + const sd_tensor_t* image = sd_result_get_image(result, i); + const char* info = sd_result_get_generation_info(result, i); + /* Consume image and info together. */ } - free_sd_image_generation_result(result); } +sd_result_free(result); +sd_opt_free(opt); ``` -`result_out` is required and is set to `NULL` on failure. A successful result -owns its pixels and UTF-8 JSON strings. The video result additionally owns its -audio and contains the effective `fps` and `frame_count`. Results remain valid -after freeing the generation context. Use the matching library result destructor; -do not separately free fields unless transferring ownership and clearing them. -Both result destructors accept `NULL`. +`result_out` is required and set to `NULL` on failure. Successful results own +their pixels, audio and UTF-8 JSON independently of the generation context. +Video results expose effective FPS and frame count through accessors; each valid +frame index returns the shared video generation record. Result views are borrowed +until `sd_result_free()`. Use `sd_tensor_retain()` to keep media independently, +and release the retained handle with `sd_tensor_free()`. Free functions accept `NULL`. ## Reproduction parameters @@ -33,7 +34,7 @@ same loaded context and the same input images, masks, reference media and other assets. It is created after `GenerationRequest` resolves the request, before preprocessing or sampling. A failed call returns no result or metadata. -Fields follow the C generation parameter structs: `prompt`, `negative_prompt`, +The JSON schema retains its established generation field names: `prompt`, `negative_prompt`, `width`, `height`, `seed`, `sample_params`, `hires`, `cache`, `vae_tiling_params`, `image_preprocess`, and the image/video-specific options. `kind` distinguishes image and video parameters. There is no inferred `task`. @@ -77,10 +78,10 @@ and shares them across a batch. With a stochastic VAE encoder, replaying a later image alone using its sampling seed can change those input latents. Per-image seeds therefore do not guarantee independent reproduction for these paths. -To replay a record, initialize the matching C parameter struct, populate its -fields from the JSON, restore the same input buffers, and call generation with -the same context. Keep custom-sigma/SLG arrays and string storage alive for the -call. Paths supplied as generation parameters (LoRAs, hires models and identity +To replay a record, create image/video options, translate JSON fields to the documented C API +keys, restore the same input buffers, and call generation with the same context. +Setters copy strings and numeric arrays and retain media tensors; keep options alive +for the call. Paths supplied as generation parameters (LoRAs, hires models and identity embeddings) are retained. `models` identifies successfully loaded context model files by component (`model`, `diffusion_model`, `clip_l`, `vae`, etc.), using filenames with extensions and omitting directories. Optional components disabled diff --git a/docs/guides/image_preprocessing.md b/docs/guides/image_preprocessing.md index c097e5a901..dc0aca8bc3 100644 --- a/docs/guides/image_preprocessing.md +++ b/docs/guides/image_preprocessing.md @@ -156,23 +156,23 @@ user rules for replay with the same input images; image encodings and channel co ## C API -Set `image_preprocess` on the existing image/video generation parameters. -Generation returns media and actual parameters together; see [generation metadata](generation_metadata.md): +Set the `image_preprocess.rules` key on image/video options. Generation returns +media and actual parameters together; see [generation metadata](generation_metadata.md): ```c -sd_img_gen_params_t params; -sd_img_gen_params_init(¶ms); +sd_img_gen_opt_t* opt = sd_img_gen_opt_create(); +sd_result_t* result = NULL; /* Set prompt, original-resolution input images, and generation options. */ -params.image_preprocess.rules = "target=init,mode=crop-resize,filter=lanczos;" - "target=mask,filter=nearest-exact"; -sd_image_generation_result_t* result = NULL; -bool ok = generate_image(ctx, ¶ms, &result); -/* Consume result->items and their generation_info on success. */ -free_sd_image_generation_result(result); +sd_status_t status = sd_opt_set_string(opt, "image_preprocess.rules", + "target=init,mode=crop-resize,filter=lanczos;target=mask,filter=nearest-exact"); +if (status == SD_STATUS_OK) + status = sd_generate_image(ctx, opt, &result); +/* Read outputs through sd_result_get_*() on success. */ +sd_result_free(result); +sd_opt_free(opt); ``` -Both generation parameter initializers set `image_preprocess.rules` to `NULL`, -selecting input presets. Rule strings are borrowed for the synchronous call. +Leaving the key unset selects input presets. Rule strings are copied by setters. The library owns temporary transformed pixels; caller images and arrays are not modified. Add `canny=true` to the desired target's rule in `image_preprocess.rules` to enable Canny. diff --git a/docs/guides/prompt_images.md b/docs/guides/prompt_images.md index 726a797fee..e6c301f0c4 100644 --- a/docs/guides/prompt_images.md +++ b/docs/guides/prompt_images.md @@ -47,24 +47,30 @@ smaller source image if this limit is exceeded. ## C API Register a context-local image resolver with `sd_set_prompt_image_callback()`, -then call the existing `generate_image()` or `generate_video()` function. The +then call `sd_generate_image()` or `sd_generate_video()`. The library passes each marker's `source` string unchanged to the callback, allowing paths, application resource IDs, or in-memory images. The library does not open files or fetch URLs itself. ```c -bool load_prompt_image(const char* source, void* user_data, sd_image_t* image_out); +bool load_prompt_image(const char* source, void* user_data, const sd_tensor_t** image_out); -params.prompt = "Draw the subject from in a watercolor landscape."; -sd_image_generation_result_t* result = NULL; +sd_img_gen_opt_t* opt = sd_img_gen_opt_create(); +sd_status_t status = sd_opt_set_string(opt, "prompt", + "Draw the subject from in a watercolor landscape."); +sd_result_t* result = NULL; sd_set_prompt_image_callback(ctx, load_prompt_image, application_images); -bool ok = generate_image(ctx, ¶ms, &result); +if (status == SD_STATUS_OK) + status = sd_generate_image(ctx, opt, &result); sd_set_prompt_image_callback(ctx, NULL, NULL); +sd_result_free(result); +sd_opt_free(opt); ``` -The loader returns `true` with an image containing interleaved 8-bit pixels and -one to four channels. The caller owns the pixels. The library copies them before -calling the loader again, so a loader may reuse its buffer. Repeated sources +The loader returns `true` with a borrowed U8 tensor of shape +`[channels,width,height]`, containing interleaved pixels and one to four channels. +The caller owns the tensor and keeps it valid until the next callback. The +library copies its pixels before calling the loader again. Repeated sources within one prompt are loaded once and appear at every specified position. Resolved image pixels participate in conditioning-cache comparisons. diff --git a/docs/guides/sage_attention.md b/docs/guides/sage_attention.md index 2c5d9602d3..2f3a0d09ca 100644 --- a/docs/guides/sage_attention.md +++ b/docs/guides/sage_attention.md @@ -61,7 +61,7 @@ speed, and report model loading and first-step initialization separately. Quantization, smoothing, and format conversion costs are included in generation time, so short sequences may not benefit. -Library callers set `sd_ctx_params_t.sage_attn = true` before `new_sd_ctx()`, -like `diffusion_flash_attn`. Context creation fails if the requested feature is -unavailable. Initialize the parameter structure with `sd_ctx_params_init()`. +Library callers set `sage_attn` with `sd_opt_set_bool(ctx_opt, "sage_attn", true)` +before `sd_ctx_create()`. Context creation fails if the requested feature is +unavailable. Create options with `sd_ctx_opt_create()`. Rebuild library callers against the updated public header. diff --git a/docs/guides/tokenizers.md b/docs/guides/tokenizers.md index 272df9d438..fb93550a9d 100644 --- a/docs/guides/tokenizers.md +++ b/docs/guides/tokenizers.md @@ -52,15 +52,17 @@ sd-cli --diffusion-model sd3.gguf --clip_l clip_l.safetensors \ -p "a cat" ``` -The C API accepts the same string in `sd_ctx_params_t::tokenizer`. A null or +The C API accepts the same string through the `tokenizer` context-option key. An unset or empty value keeps an embedded tokenizer where available; PiD and Lens require a nonempty main tokenizer path. The CLI passes the string through; `TokenizerConfig` parses and validates it when text encoders are initialized. ```c -sd_ctx_params_t params; -sd_ctx_params_init(¶ms); -params.tokenizer = "clip-l=tokenizer_clip.json,clip-g=tokenizer_clip.json"; +sd_ctx_opt_t* opt = sd_ctx_opt_create(); +sd_status_t status = sd_opt_set_string(opt, "tokenizer", + "clip-l=tokenizer_clip.json,clip-g=tokenizer_clip.json"); +/* Set model paths, then call sd_ctx_create(opt, &ctx). */ +sd_opt_free(opt); ``` Rebuild applications against the updated public header when using the updated diff --git a/docs/models/minimax_h3.md b/docs/models/minimax_h3.md index dc4c52eb5c..862b04d3e9 100644 --- a/docs/models/minimax_h3.md +++ b/docs/models/minimax_h3.md @@ -254,7 +254,7 @@ Reference images keep their aspect ratio and are only downscaled when they exceed the configured limit; neither mode scales a reference up. The limit is selected by `ref_image_size`. The default, `match`, uses the pixel area of the requested generation canvas. Pass `--ref-image-args "ref_image_size=max"` (or -`ref_image_size` in `sd_vid_gen_params_t.ref_image_args`) to use a 2048 px short +`ref_image_size` in the `ref_image_args` video-option string) to use a 2048 px short edge instead, which preserves more identity detail at a higher sampling cost: reference latents are attended at every sampling step. @@ -266,7 +266,7 @@ narrower than 32 pixels. the 32-pixel grid and reports a warning. The C API exposes the same inputs through `ref_images`, `ref_videos`, and -`ref_audios` in `sd_vid_gen_params_t`. Each `sd_ref_video_t` supplies its own +the `ref_audios` video-option key. Each `sd_video_t` supplies its own frame rate and optional soundtrack; non-24-fps inputs are resampled internally. #### Sol-H3-VAE decoding diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 27731e1cce..c667cd4b89 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -45,7 +45,7 @@ the intended output scale: The two values can be set independently and apply across model components. The default `0` preserves each model's built-in settings; `1` explicitly disables the corresponding scaling. Overrides must be finite positive values. C API users can -set `linear_scale` and `attn_scale` in `sd_ctx_params_t`. +set the `linear_scale` and `attn_scale` context-option keys with `sd_opt_set_float()`. If the problem persists after trying the relevant steps above, [submit a bug report](https://github.com/leejet/stable-diffusion.cpp/issues/new?template=bug_report.yml). diff --git a/examples/cli/CMakeLists.txt b/examples/cli/CMakeLists.txt index 90164cfc02..36b9243e35 100644 --- a/examples/cli/CMakeLists.txt +++ b/examples/cli/CMakeLists.txt @@ -14,7 +14,6 @@ if(APPLE) endif() target_include_directories(${TARGET} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/.." - "${PROJECT_SOURCE_DIR}/src" ) install(TARGETS ${TARGET} RUNTIME) target_link_libraries(${TARGET} PRIVATE stable-diffusion zip ${CMAKE_THREAD_LIBS_INIT}) diff --git a/examples/cli/README.md b/examples/cli/README.md index 78dd25adb8..0316f7bd02 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -17,6 +17,13 @@ From the build directory, show the full list of command-line arguments: See [sampling methods and schedulers](../../docs/guides/sampling.md) for supported names, defaults, and custom sigma schedules. +Omitted inference options use the [library defaults](../../docs/guides/c_api.md#keys). +In particular, an omitted seed requests a random seed; use `--seed 42` for a +repeatable run. Video generation defaults to 6 requested frames before model +alignment. SLG layers remain unset unless `--skip-layers` is supplied, and the +conditioning cache uses the library default. Zero and false are explicit inputs; +invalid values are reported rather than treated as omitted. + Logging defaults to `info`. Use `--log-level ` to select `debug`, `verbose`, `info`, `warn`, or `error` (from most to least detailed). Each level includes messages at that level and all less detailed levels. `-v` and `--verbose` are diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 44462eb69b..a29fd70b60 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -35,28 +35,28 @@ std::regex format_specifier_regex("(?:[^%]|^)(?:%%)*(%\\d{0,3}d)"); struct SDCliParams { std::string input_frames; - int video_upscale_factor = 0; - int vsr_window_size = 16; - int vsr_temporal_chunk_size = 5; - SDMode mode = IMG_GEN; - std::string output_path = "output.png"; - int output_begin_idx = -1; - int compression_quality = 90; + int video_upscale_factor = 0; + std::optional vsr_window_size; + std::optional vsr_temporal_chunk_size; + SDMode mode = IMG_GEN; + std::string output_path = "output.png"; + int output_begin_idx = -1; + int compression_quality = 90; std::string image_path; std::string metadata_format = "text"; sd_log_level_t log_level = SD_LOG_INFO; bool convert_name = false; - preview_t preview_method = PREVIEW_NONE; - int preview_interval = 1; - std::string preview_path = "preview.png"; - int preview_fps = 16; - bool preview_noisy = false; - bool color = false; - bool metadata_raw = false; - bool metadata_brief = false; - bool metadata_all = false; + std::string preview_method = "none"; + int preview_interval = 1; + std::string preview_path = "preview.png"; + int preview_fps = 16; + bool preview_noisy = false; + bool color = false; + bool metadata_raw = false; + bool metadata_brief = false; + bool metadata_all = false; std::string imatrix_out; std::vector imatrix_in; @@ -167,18 +167,7 @@ struct SDCliParams { if (++index >= argc) { return -1; } - const char* preview = argv[index]; - int preview_found = -1; - for (int m = 0; m < PREVIEW_COUNT; m++) { - if (!strcmp(preview, previews_str[m])) { - preview_found = m; - } - } - if (preview_found == -1) { - LOG_ERROR("error: preview method %s", preview); - return -1; - } - preview_method = (preview_t)preview_found; + preview_method = argv[index]; return 1; }; @@ -203,7 +192,7 @@ struct SDCliParams { on_mode_arg}, {"", "--preview", - std::string("preview method. must be one of the following [") + previews_str[0] + ", " + previews_str[1] + ", " + previews_str[2] + ", " + previews_str[3] + "] (default is " + previews_str[PREVIEW_NONE] + ")", + std::string("preview method. must be one of the following [") + previews_str[0] + ", " + previews_str[1] + ", " + previews_str[2] + ", " + previews_str[3] + "] (default is " + previews_str[0] + ")", on_preview_arg}, {"-h", "--help", @@ -272,7 +261,7 @@ struct SDCliParams { << " log_level: " << log_level_name(log_level) << ",\n" << " color: " << (color ? "true" : "false") << ",\n" << " convert_name: " << (convert_name ? "true" : "false") << ",\n" - << " preview_method: " << previews_str[preview_method] << ",\n" + << " preview_method: " << preview_method << ",\n" << " preview_interval: " << preview_interval << ",\n" << " preview_path: \"" << preview_path << "\",\n" << " preview_fps: " << preview_fps << ",\n" @@ -368,10 +357,10 @@ bool load_images_from_dir(const std::string dir, return false; } - images.emplace_back(sd_image_t{(uint32_t)width, - (uint32_t)height, - (uint32_t)loaded_channel, - image_buffer}); + images.emplace_back(SDImage{(uint32_t)width, + (uint32_t)height, + (uint32_t)loaded_channel, + image_buffer}); if (max_image_num > 0 && static_cast(images.size()) >= max_image_num) { break; @@ -402,7 +391,11 @@ std::string format_frame_idx(std::string pattern, int frame_idx) { int continuous_preview_counter = 0; -void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, void* data) { +void step_callback(int step, int frame_count, const sd_tensor_t* const* tensors, bool is_noisy, void* data) { + std::vector views; + for (int i = 0; i < frame_count; ++i) + views.push_back(tensor_image_view(tensors[i])); + SDImage* image = views.data(); (void)step; (void)is_noisy; SDCliParams* cli_params = (SDCliParams*)data; @@ -424,7 +417,7 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, LOG_ERROR("save preview image to '%s' failed", path.u8string().c_str()); } } else { - int fps = cli_params->preview_method == PREVIEW_PROJ ? cli_params->preview_fps / 4 : cli_params->preview_fps; + int fps = cli_params->preview_method == "proj" ? cli_params->preview_fps / 4 : cli_params->preview_fps; if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, fps, cli_params->compression_quality) != 0) { LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str()); } @@ -456,7 +449,7 @@ bool save_results(const SDCliParams& cli_params, const SDGenerationParams& gen_params, SDImageVec& results, int num_results, - const sd_audio_t* generated_audio, + const SDAudio* generated_audio, const std::string& video_info) { if (results.empty() || num_results <= 0) { return false; @@ -497,7 +490,7 @@ bool save_results(const SDCliParams& cli_params, } auto write_image = [&](const fs::path& path, int idx) { - const sd_image_t& img = results[idx]; + const SDImage& img = results[idx]; if (!img.data) return false; @@ -556,7 +549,7 @@ bool save_results(const SDCliParams& cli_params, std::string final_ext_lower = ext.u8string(); std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower); const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm"); - if (create_video_from_sd_images(video_path.u8string().c_str(), results.data(), num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) { + if (create_video_from_sd_images(video_path.u8string().c_str(), results.data(), num_results, gen_params.fps.value(), cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) { LOG_INFO("save result video to '%s'", video_path.u8string().c_str()); if (generated_audio != nullptr && !mux_audio) { fs::path wav_path = video_path; @@ -591,10 +584,8 @@ bool save_results(const SDCliParams& cli_params, } static bool apply_adetailer(sd_ctx_t* sd_ctx, - const sd_ctx_params_t& sd_ctx_params, const SDContextParams& ctx_params, - const SDGenerationParams& gen_params, - const sd_img_gen_params_t& img_gen_params, + SDGenerationParams& gen_params, SDMode mode, SDImageVec& results, int num_results) { @@ -602,17 +593,12 @@ static bool apply_adetailer(sd_ctx_t* sd_ctx, return true; } - sd_adetailer_params_t ad_params{}; - ad_params.prompt = gen_params.ad_prompt.empty() ? nullptr : gen_params.ad_prompt.c_str(); - ad_params.negative_prompt = gen_params.ad_negative_prompt.empty() ? nullptr : gen_params.ad_negative_prompt.c_str(); - ad_params.extra_ad_args = gen_params.extra_ad_args.c_str(); - - ADetailerCtxPtr ad_ctx(new_adetailer_ctx(gen_params.ad_model_path.c_str(), - ctx_params.n_threads, - sd_ctx_params.backend, - sd_ctx_params.params_backend)); + ADetailerCtxPtr ad_ctx(create_sd_adetailer(gen_params.ad_model_path.c_str(), + ctx_params.n_threads, + ctx_params.effective_backend.c_str(), + ctx_params.effective_params_backend.c_str())); if (ad_ctx == nullptr) { - LOG_ERROR("new_adetailer_ctx failed"); + LOG_ERROR("sd_adetailer_create failed"); return false; } @@ -620,30 +606,29 @@ static bool apply_adetailer(sd_ctx_t* sd_ctx, if (results[i].data == nullptr) { continue; } - sd_img_gen_params_t ad_generation_params = img_gen_params; - ad_generation_params.seed = generation_seed(results.generation_info(i), img_gen_params.seed); + SDOptBuilder ad_options(gen_params.to_image_opt(true).release()); + ad_options.set_int("seed", generation_seed(results.generation_info(i), gen_params.seed.value_or(-1))); if (mode == IMG_GEN) { - ad_generation_params.width = 512; - ad_generation_params.height = 512; - ad_generation_params.strength = 0.4f; + ad_options.unset("width"); + ad_options.unset("height"); + ad_options.unset("strength"); } - sd_image_generation_result_t* detailed = nullptr; - if (!adetail_image(ad_ctx.get(), - sd_ctx, - results[i], - &ad_params, - &ad_generation_params, - &detailed) || - detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) { - free_sd_image_generation_result(detailed); + sd_result_t* detailed = nullptr; + if (!adetail_sd_image(ad_ctx.get(), + sd_ctx, + results[i], + ad_options.finish(), + &detailed) || + detailed == nullptr || sd_result_get_image_count(detailed) <= 0 || sd_tensor_data(sd_result_get_image(detailed, 0)) == nullptr) { + sd_result_free(detailed); LOG_ERROR("ADetailer failed for image %d", i + 1); return false; } - results.append_generation_info(i, detailed->items[0].generation_info, ctx_params.lora_model_dir); - free(results[i].data); - results[i] = detailed->items[0].image; - detailed->items[0].image = {}; - free_sd_image_generation_result(detailed); + results.append_generation_info(i, sd_result_get_generation_info(detailed, 0), ctx_params.lora_model_dir); + sd_image_free(&results[i]); + results[i] = take_result_image(detailed, 0); + + sd_result_free(detailed); } return true; } @@ -656,7 +641,6 @@ int main(int argc, const char* argv[]) { SDCliParams cli_params; SDContextParams ctx_params; - ctx_params.conditioning_cache_size = 0; SDGenerationParams gen_params; sd_set_log_callback(sd_log_cb, (void*)&cli_params); @@ -685,10 +669,6 @@ int main(int argc, const char* argv[]) { return 1; } ctx_params.prepare_backend_assignments(); - if (ctx_params.diffusion_model_path.empty() || ctx_params.vae_path.empty() || gen_params.fps <= 0) { - LOG_ERROR("vid_upscale requires --diffusion-model, --vae and positive --fps"); - return 1; - } std::vector owned_frames; if (!gen_params.init_image_path.empty()) { owned_frames.emplace_back(); @@ -702,44 +682,45 @@ int main(int argc, const char* argv[]) { } if (owned_frames.size() > size_t(INT_MAX)) return 1; - std::vector frames; + std::vector frames; for (const auto& frame : owned_frames) frames.push_back(frame.get()); - sd_video_upscaler_ctx_params_t params; - sd_video_upscaler_ctx_params_init(¶ms); + SDVideoUpscalerParams params; + params.diffusion_model_path = ctx_params.diffusion_model_path.c_str(); params.vae_path = ctx_params.vae_path.c_str(); params.upscale_factor = cli_params.video_upscale_factor; params.n_threads = ctx_params.n_threads; params.backend = ctx_params.effective_backend.c_str(); params.params_backend = ctx_params.effective_params_backend.c_str(); - params.flash_attn_mode = ctx_params.flash_attn_mode.c_str(); - std::string conv_direct_mode = ctx_params.conv_direct_mode; + params.flash_attn_mode = ctx_params.flash_attn_mode ? ctx_params.flash_attn_mode->c_str() : nullptr; + std::string conv_direct_mode = ctx_params.conv_direct_mode.value_or(""); if (conv_direct_mode.empty()) { conv_direct_mode = "auto"; - if (ctx_params.diffusion_conv_direct) + if (ctx_params.diffusion_conv_direct.value_or(false)) conv_direct_mode += ",diffusion=on"; - if (ctx_params.vae_conv_direct) + if (ctx_params.vae_conv_direct.value_or(false)) conv_direct_mode += ",vae=on"; } params.conv_direct_mode = conv_direct_mode.c_str(); - std::unique_ptr upscaler( - new_video_upscaler_ctx(¶ms), - free_video_upscaler_ctx); + std::unique_ptr upscaler( + create_sd_vid_upscaler(params), + sd_vid_upscaler_free); if (!upscaler) return 1; - sd_video_generation_result_t* output = nullptr; - sd_video_upscale_params_t upscale_params; - sd_video_upscale_params_init(&upscale_params); + sd_result_t* output = nullptr; + SDVideoUpscaleParams upscale_params; + upscale_params.frames = frames.data(); upscale_params.frame_count = int(frames.size()); upscale_params.fps = gen_params.fps; upscale_params.window_size = cli_params.vsr_window_size; upscale_params.temporal_chunk_size = cli_params.vsr_temporal_chunk_size; upscale_params.seed = gen_params.seed; - if (!upscale_video(upscaler.get(), &upscale_params, &output)) + if (!upscale_sd_video(upscaler.get(), upscale_params, &output)) return 1; - std::string video_info = output->generation_info; + std::string video_info = sd_result_get_generation_info(output, 0); + gen_params.fps = sd_result_get_fps(output); SDImageVec results; results.adopt_video(output); return save_results(cli_params, ctx_params, gen_params, results, int(results.size()), nullptr, video_info) ? 0 : 1; @@ -750,7 +731,7 @@ int main(int argc, const char* argv[]) { return 1; } - if (gen_params.video_frames > 4) { + if (cli_params.mode == VID_GEN || gen_params.video_frames > 4) { size_t last_dot_pos = cli_params.preview_path.find_last_of("."); std::string base_path = cli_params.preview_path; std::string file_ext = ""; @@ -763,14 +744,18 @@ int main(int argc, const char* argv[]) { cli_params.preview_path = base_path + ".avi"; } } - cli_params.preview_fps = gen_params.fps; - - sd_set_preview_callback(step_callback, - cli_params.preview_method, - cli_params.preview_interval, - !cli_params.preview_noisy, - cli_params.preview_noisy, - (void*)&cli_params); + if (gen_params.fps) + cli_params.preview_fps = *gen_params.fps; + + if (sd_set_preview_callback(step_callback, + cli_params.preview_method.c_str(), + cli_params.preview_interval, + !cli_params.preview_noisy, + cli_params.preview_noisy, + (void*)&cli_params) != SD_STATUS_OK) { + LOG_ERROR("%s", sd_get_last_error()); + return 1; + } LOG_VERBOSE("version: %s", version_string().c_str()); LOG_VERBOSE("%s", sd_get_system_info()); @@ -813,7 +798,7 @@ int main(int argc, const char* argv[]) { ctx_params.wtype, ctx_params.tensor_type_rules.c_str(), cli_params.convert_name, - ctx_params.n_threads, + ctx_params.n_threads.value_or(0), loras.empty() ? nullptr : loras.data(), (int)loras.size()); if (!success) { @@ -945,7 +930,7 @@ int main(int argc, const char* argv[]) { gen_params.control_frames.clear(); if (!load_images_from_dir(gen_params.control_video_path, gen_params.control_frames, - gen_params.video_frames)) { + gen_params.video_frames.value_or(0))) { return 1; } } @@ -958,60 +943,55 @@ int main(int argc, const char* argv[]) { } } - sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(); + SDOptPtr sd_ctx_params = ctx_params.to_context_opt(); SDImageVec results; - int num_results = 0; - sd_audio_t* generated_audio = nullptr; + int num_results = 0; + SDAudio* generated_audio = nullptr; std::string video_info; if (cli_params.mode == UPSCALE) { num_results = 1; results.push_back(gen_params.init_image.release()); } else { - SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params)); + SDCtxPtr sd_ctx(create_sd_context(sd_ctx_params)); if (sd_ctx == nullptr) { - LOG_INFO("new_sd_ctx_t failed"); + LOG_INFO("sd_ctx_create failed"); return 1; } SDImageOwner prompt_image; sd_set_prompt_image_callback( - sd_ctx.get(), [](const char* source, void* data, sd_image_t* image_out) -> bool { + sd_ctx.get(), [](const char* source, void* data, const sd_tensor_t** image_out) -> bool { auto& image = *static_cast(data); - sd_image_t loaded{}; + SDImage loaded{}; if (!load_sd_image_from_file(&loaded, source, 0, 0, 3)) { return false; } image.reset(loaded); - *image_out = loaded; + *image_out = image.tensor(); return true; }, &prompt_image); - sd_img_gen_params_t img_gen_params{}; const bool use_img_gen_params = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER; - if (use_img_gen_params) { - img_gen_params = gen_params.to_sd_img_gen_params_t(); - } - if (cli_params.mode == IMG_GEN) { - sd_image_generation_result_t* generated = nullptr; - generate_image(sd_ctx.get(), &img_gen_params, &generated); + sd_result_t* generated = nullptr; + generate_sd_image(sd_ctx.get(), gen_params.to_image_opt(), &generated); results.adopt(generated, ctx_params.lora_model_dir); num_results = results.count(); } else if (cli_params.mode == ADETAILER) { num_results = 1; results.push_back(gen_params.init_image.release()); } else if (cli_params.mode == VID_GEN) { - sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t(); - sd_video_generation_result_t* generated = nullptr; - if (generate_video(sd_ctx.get(), &vid_gen_params, &generated)) { - gen_params.fps = cli_params.preview_fps = generated->fps; - generated_audio = generated->audio; - generated->audio = nullptr; - num_results = generated->frame_count; + SDOptPtr vid_gen_params = gen_params.to_video_opt(); + sd_result_t* generated = nullptr; + if (generate_sd_video(sd_ctx.get(), vid_gen_params, &generated)) { + gen_params.fps = cli_params.preview_fps = sd_result_get_fps(generated); + generated_audio = retain_result_audio(generated); + + num_results = sd_result_get_image_count(generated); results.adopt_video(generated, ctx_params.lora_model_dir); video_info = results.generation_info(0); } @@ -1024,10 +1004,8 @@ int main(int argc, const char* argv[]) { if (use_img_gen_params && !apply_adetailer(sd_ctx.get(), - sd_ctx_params, ctx_params, gen_params, - img_gen_params, cli_params.mode, results, num_results)) { @@ -1035,17 +1013,16 @@ int main(int argc, const char* argv[]) { } } - int upscale_factor = 4; // unused for RealESRGAN_x4plus_anime_6B.pth if (ctx_params.esrgan_path.size() > 0 && gen_params.upscale_repeats > 0) { - UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(ctx_params.esrgan_path.c_str(), - ctx_params.diffusion_conv_direct, - ctx_params.n_threads, - gen_params.upscale_tile_size, - sd_ctx_params.backend, - sd_ctx_params.params_backend)); + UpscalerCtxPtr upscaler_ctx(create_sd_upscaler(ctx_params.esrgan_path.c_str(), + ctx_params.diffusion_conv_direct, + ctx_params.n_threads, + gen_params.upscale_tile_size, + ctx_params.effective_backend.c_str(), + ctx_params.effective_params_backend.c_str())); if (upscaler_ctx == nullptr) { - LOG_ERROR("new_upscaler_ctx failed"); + LOG_ERROR("sd_upscaler_create failed"); } else { for (int i = 0; i < num_results; i++) { if (results[i].data == nullptr) { @@ -1054,20 +1031,17 @@ int main(int argc, const char* argv[]) { SDImageOwner current_image(results[i]); results[i] = {0, 0, 0, nullptr}; for (int u = 0; u < gen_params.upscale_repeats; ++u) { - sd_image_generation_result_t* upscaled = nullptr; - bool upscale_ok = upscale(upscaler_ctx.get(), - current_image.get(), - upscale_factor, - &upscaled); - if (!upscale_ok || !upscaled || upscaled->count <= 0 || upscaled->items[0].image.data == nullptr) { - free_sd_image_generation_result(upscaled); + sd_result_t* upscaled = nullptr; + bool upscale_ok = upscale_sd_image(upscaler_ctx.get(), current_image.get(), &upscaled); + if (!upscale_ok || !upscaled || sd_result_get_image_count(upscaled) <= 0 || sd_tensor_data(sd_result_get_image(upscaled, 0)) == nullptr) { + sd_result_free(upscaled); LOG_ERROR("upscale failed"); break; } - results.append_generation_info(i, upscaled->items[0].generation_info); - sd_image_t upscaled_image = upscaled->items[0].image; - upscaled->items[0].image = {}; - free_sd_image_generation_result(upscaled); + results.append_generation_info(i, sd_result_get_generation_info(upscaled, 0)); + SDImage upscaled_image = take_result_image(upscaled, 0); + + sd_result_free(upscaled); current_image.reset(upscaled_image); } results[i] = current_image.release(); // Set the final upscaled image as the result diff --git a/examples/common/common.cpp b/examples/common/common.cpp index d92c8193bf..fbf977d6a0 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -38,25 +38,6 @@ const char* const modes_str[] = { "vid_upscale", }; -static sd_vae_format_t str_to_vae_format(const std::string& value) { - if (value == "auto") { - return SD_VAE_FORMAT_AUTO; - } - if (value == "flux") { - return SD_VAE_FORMAT_FLUX; - } - if (value == "sd3") { - return SD_VAE_FORMAT_SD3; - } - if (value == "flux2") { - return SD_VAE_FORMAT_FLUX2; - } - if (value == "wan") { - return SD_VAE_FORMAT_WAN; - } - return SD_VAE_FORMAT_COUNT; -} - static void prepend_backend_assignment(std::string& spec, const char* assignment) { spec = spec.empty() ? assignment : std::string(assignment) + "," + spec; } @@ -286,13 +267,13 @@ bool parse_options(int argc, const char** argv, const std::vector& o invalid_arg = true; return; } - if (option.concat && !option.target->empty()) { + if (option.concat && !option.target.get().empty()) { if (option.concat > 0 && option.concat <= 0xff) { - *option.target += static_cast(option.concat); + option.target.set(option.target.get() + static_cast(option.concat)); } - *option.target += argv_to_utf8(i, argv); + option.target.set(option.target.get() + argv_to_utf8(i, argv)); } else { - *option.target = argv_to_utf8(i, argv); + option.target.set(argv_to_utf8(i, argv)); } found_arg = true; })) @@ -304,7 +285,7 @@ bool parse_options(int argc, const char** argv, const std::vector& o return; } try { - *option.target = std::stoi(argv[i]); + option.target.set(std::stoi(argv[i])); } catch (const std::invalid_argument&) { invalid_arg = true; } @@ -318,7 +299,7 @@ bool parse_options(int argc, const char** argv, const std::vector& o return; } try { - *option.target = std::stof(argv[i]); + option.target.set(std::stof(argv[i])); } catch (const std::invalid_argument&) { invalid_arg = true; } @@ -327,8 +308,8 @@ bool parse_options(int argc, const char** argv, const std::vector& o break; if (match_and_apply(options.bool_options, [&](auto& option) { - *option.target = option.keep_true ? true : false; - found_arg = true; + option.target.set(option.keep_true); + found_arg = true; })) break; @@ -360,7 +341,7 @@ bool parse_options(int argc, const char** argv, const std::vector& o return true; } -static int parse_scale_override(int argc, const char** argv, int index, float& scale) { +static int parse_scale_override(int argc, const char** argv, int index, std::optional& scale) { if (++index >= argc) { return -1; } @@ -379,7 +360,7 @@ static int parse_scale_override(int argc, const char** argv, int index, float& s return 1; } -static int parse_on_off_arg(int argc, const char** argv, int index, const char* option, bool& value) { +static int parse_on_off_arg(int argc, const char** argv, int index, const char* option, std::optional& value) { if (++index >= argc) { LOG_ERROR("%s requires 'on' or 'off'", option); return -1; @@ -587,12 +568,11 @@ ArgOptions SDContextParams::get_options() { options.int_options = { {"-t", "--threads", - "number of threads to use during computation (default: -1). " - "If threads <= 0, then threads will be set to the number of CPU physical cores", + "positive number of computation threads; unset uses the library CPU core selection", &n_threads}, {"", "--conditioning-cache-size", - "maximum number of conditioning results cached per model context (default: " + std::to_string(conditioning_cache_size) + ", 0 disables caching)", + "maximum cached conditioning results; unset uses the library default, 0 disables caching", &conditioning_cache_size}, }; @@ -667,12 +647,7 @@ ArgOptions SDContextParams::get_options() { return -1; } const char* arg = argv[index]; - rng_type = str_to_rng_type(arg); - if (rng_type == RNG_TYPE_COUNT) { - LOG_ERROR("error: invalid rng type %s", - arg); - return -1; - } + rng_type = arg; return 1; }; @@ -681,12 +656,7 @@ ArgOptions SDContextParams::get_options() { return -1; } const char* arg = argv[index]; - sampler_rng_type = str_to_rng_type(arg); - if (sampler_rng_type == RNG_TYPE_COUNT) { - LOG_ERROR("error: invalid sampler rng type %s", - arg); - return -1; - } + sampler_rng_type = arg; return 1; }; @@ -695,12 +665,7 @@ ArgOptions SDContextParams::get_options() { return -1; } const char* arg = argv[index]; - prediction = str_to_prediction(arg); - if (prediction == PREDICTION_COUNT) { - LOG_ERROR("error: invalid prediction type %s", - arg); - return -1; - } + prediction = arg; return 1; }; @@ -709,12 +674,7 @@ ArgOptions SDContextParams::get_options() { return -1; } const char* arg = argv[index]; - lora_apply_mode = str_to_lora_apply_mode(arg); - if (lora_apply_mode == LORA_APPLY_MODE_COUNT) { - LOG_ERROR("error: invalid lora apply model %s", - arg); - return -1; - } + lora_apply_mode = arg; return 1; }; @@ -835,7 +795,7 @@ ArgOptions SDContextParams::get_options() { } bool SDContextParams::print_devices() const { - if (!sd_add_rpc_servers(rpc_servers.c_str())) { + if (!sd_add_rpc_servers(rpc_servers ? rpc_servers->c_str() : nullptr)) { return false; } size_t device_list_size = sd_list_devices(nullptr, 0); @@ -878,20 +838,12 @@ void SDContextParams::build_embedding_map() { } bool SDContextParams::resolve(SDMode mode) { - if (n_threads <= 0) { - n_threads = sd_get_num_physical_cores(); - } - build_embedding_map(); return true; } bool SDContextParams::validate(SDMode mode) { - if (conditioning_cache_size < 0) { - LOG_ERROR("error: conditioning-cache-size must be non-negative"); - return false; - } if (mode == CONVERT) { const bool has_convert_input = model_path.length() != 0 || clip_l_path.length() != 0 || @@ -915,8 +867,8 @@ bool SDContextParams::validate(SDMode mode) { } } - if (str_to_vae_format(vae_format) == SD_VAE_FORMAT_COUNT) { - LOG_ERROR("error: vae_format must be 'auto', 'flux', 'sd3', 'flux2', or 'wan'"); + if (!to_context_opt()) { + LOG_ERROR("%s", sd_get_last_error()); return false; } @@ -934,8 +886,8 @@ bool SDContextParams::resolve_and_validate(SDMode mode) { } void SDContextParams::prepare_backend_assignments() { - effective_backend = backend; - effective_params_backend = params_backend; + effective_backend = backend.value_or(""); + effective_params_backend = params_backend.value_or(""); if (offload_params_to_cpu) { prepend_backend_assignment(effective_params_backend, "*=cpu"); @@ -995,39 +947,39 @@ std::string SDContextParams::to_string() const { << " lora_model_dir: \"" << lora_model_dir << "\",\n" << " hires_upscalers_dir: \"" << hires_upscalers_dir << "\",\n" << " photo_maker_path: \"" << photo_maker_path << "\",\n" - << " rng_type: " << sd_rng_type_name(rng_type) << ",\n" - << " sampler_rng_type: " << sd_rng_type_name(sampler_rng_type) << ",\n" + << " rng_type: " << rng_type << ",\n" + << " sampler_rng_type: " << sampler_rng_type << ",\n" << " offload_params_to_cpu: " << (offload_params_to_cpu ? "true" : "false") << ",\n" << " max_vram: \"" << max_vram << "\",\n" - << " disable_prefetch: " << (disable_prefetch ? "true" : "false") << ",\n" - << " disable_segmented_compute: " << (disable_segmented_compute ? "true" : "false") << ",\n" - << " batched_cfg: " << (batched_cfg ? "true" : "false") << ",\n" - << " eager_load: " << (eager_load ? "true" : "false") << ",\n" + << " disable_prefetch: " << disable_prefetch << ",\n" + << " disable_segmented_compute: " << disable_segmented_compute << ",\n" + << " batched_cfg: " << batched_cfg << ",\n" + << " eager_load: " << eager_load << ",\n" << " backend: \"" << backend << "\",\n" << " params_backend: \"" << params_backend << "\",\n" << " split_mode: \"" << split_mode << "\",\n" << " model_args: \"" << model_args << "\",\n" - << " auto_fit: " << (auto_fit ? "true" : "false") << ",\n" - << " enable_mmap: " << (enable_mmap ? "true" : "false") << ",\n" + << " auto_fit: " << auto_fit << ",\n" + << " enable_mmap: " << enable_mmap << ",\n" << " control_net_cpu: " << (control_net_cpu ? "true" : "false") << ",\n" << " clip_on_cpu: " << (clip_on_cpu ? "true" : "false") << ",\n" << " vae_on_cpu: " << (vae_on_cpu ? "true" : "false") << ",\n" << " flash_attn_mode: " << flash_attn_mode << ",\n" - << " conv_direct_mode: " << (conv_direct_mode.empty() ? "auto (legacy flags honored)" : conv_direct_mode) << ",\n" - << " sage_attn: " << (sage_attn ? "true" : "false") << ",\n" + << " conv_direct_mode: " << conv_direct_mode << ",\n" + << " sage_attn: " << sage_attn << ",\n" << " linear_scale: " << linear_scale << ",\n" << " attn_scale: " << attn_scale << ",\n" - << " diffusion_conv_direct: " << (diffusion_conv_direct ? "true" : "false") << ",\n" - << " vae_conv_direct: " << (vae_conv_direct ? "true" : "false") << ",\n" - << " taesd_preview_only: " << (taesd_preview_only ? "true" : "false") << ",\n" - << " prediction: " << sd_prediction_name(prediction) << ",\n" - << " lora_apply_mode: " << sd_lora_apply_mode_name(lora_apply_mode) << ",\n" - << " force_sdxl_vae_conv_scale: " << (force_sdxl_vae_conv_scale ? "true" : "false") << "\n" + << " diffusion_conv_direct: " << diffusion_conv_direct << ",\n" + << " vae_conv_direct: " << vae_conv_direct << ",\n" + << " taesd_preview_only: " << taesd_preview_only << ",\n" + << " prediction: " << prediction << ",\n" + << " lora_apply_mode: " << lora_apply_mode << ",\n" + << " force_sdxl_vae_conv_scale: " << force_sdxl_vae_conv_scale << "\n" << "}"; return oss.str(); } -sd_ctx_params_t SDContextParams::to_sd_ctx_params_t() { +SDOptPtr SDContextParams::to_context_opt() { prepare_backend_assignments(); embedding_vec.clear(); embedding_vec.reserve(embedding_map.size()); @@ -1038,67 +990,66 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t() { embedding_vec.emplace_back(item); } - sd_ctx_params_t sd_ctx_params; - sd_ctx_params_init(&sd_ctx_params); - sd_ctx_params.model_path = model_path.c_str(); - sd_ctx_params.clip_l_path = clip_l_path.c_str(); - sd_ctx_params.clip_g_path = clip_g_path.c_str(); - sd_ctx_params.clip_vision_path = clip_vision_path.c_str(); - sd_ctx_params.t5xxl_path = t5xxl_path.c_str(); - sd_ctx_params.llm_path = llm_path.c_str(); - sd_ctx_params.llm_vision_path = llm_vision_path.c_str(); - sd_ctx_params.tokenizer = tokenizer.c_str(); - sd_ctx_params.diffusion_model_path = diffusion_model_path.c_str(); - sd_ctx_params.high_noise_diffusion_model_path = high_noise_diffusion_model_path.c_str(); - sd_ctx_params.uncond_diffusion_model_path = uncond_diffusion_model_path.c_str(); - sd_ctx_params.embeddings_connectors_path = embeddings_connectors_path.c_str(); - sd_ctx_params.vae_path = vae_path.c_str(); - sd_ctx_params.audio_vae_path = audio_vae_path.c_str(); - sd_ctx_params.audio_encoder_path = audio_encoder_path.c_str(); - sd_ctx_params.taesd_path = taesd_path.c_str(); - sd_ctx_params.control_net_path = control_net_path.c_str(); - sd_ctx_params.ip_adapter_path = ip_adapter_path.c_str(); - sd_ctx_params.motion_module_path = motion_module_path.c_str(); - sd_ctx_params.embeddings = embedding_vec.data(); - sd_ctx_params.embedding_count = static_cast(embedding_vec.size()); - sd_ctx_params.photo_maker_path = photo_maker_path.c_str(); - sd_ctx_params.pulid_weights_path = pulid_weights_path.c_str(); - sd_ctx_params.tensor_type_rules = tensor_type_rules.c_str(); - sd_ctx_params.n_threads = n_threads; - sd_ctx_params.conditioning_cache_size = conditioning_cache_size; - sd_ctx_params.wtype = wtype; - sd_ctx_params.rng_type = rng_type; - sd_ctx_params.sampler_rng_type = sampler_rng_type; - sd_ctx_params.prediction = prediction; - sd_ctx_params.lora_apply_mode = lora_apply_mode; - sd_ctx_params.enable_mmap = enable_mmap; - sd_ctx_params.flash_attn_mode = flash_attn_mode.c_str(); - sd_ctx_params.conv_direct_mode = conv_direct_mode.c_str(); - sd_ctx_params.sage_attn = sage_attn; - sd_ctx_params.linear_scale = linear_scale; - sd_ctx_params.attn_scale = attn_scale; - sd_ctx_params.tae_preview_only = taesd_preview_only; - sd_ctx_params.diffusion_conv_direct = diffusion_conv_direct; - sd_ctx_params.vae_conv_direct = vae_conv_direct; - sd_ctx_params.force_sdxl_vae_conv_scale = force_sdxl_vae_conv_scale; - sd_ctx_params.vae_format = str_to_vae_format(vae_format); - sd_ctx_params.max_vram = max_vram.c_str(); - sd_ctx_params.disable_prefetch = disable_prefetch; - sd_ctx_params.disable_segmented_compute = disable_segmented_compute; - sd_ctx_params.batched_cfg = batched_cfg; - sd_ctx_params.eager_load = eager_load; - sd_ctx_params.backend = effective_backend.c_str(); - sd_ctx_params.params_backend = effective_params_backend.c_str(); - sd_ctx_params.split_mode = split_mode.c_str(); - sd_ctx_params.auto_fit = auto_fit; - sd_ctx_params.rpc_servers = rpc_servers.c_str(); - sd_ctx_params.model_args = model_args.empty() ? nullptr : model_args.c_str(); - return sd_ctx_params; + SDOptBuilder opt(sd_ctx_opt_create()); + opt.set_string("model_path", model_path); + opt.set_string("clip_l_path", clip_l_path); + opt.set_string("clip_g_path", clip_g_path); + opt.set_string("clip_vision_path", clip_vision_path); + opt.set_string("t5xxl_path", t5xxl_path); + opt.set_string("llm_path", llm_path); + opt.set_string("llm_vision_path", llm_vision_path); + opt.set_string("tokenizer", tokenizer); + opt.set_string("diffusion_model_path", diffusion_model_path); + opt.set_string("high_noise_diffusion_model_path", high_noise_diffusion_model_path); + opt.set_string("uncond_diffusion_model_path", uncond_diffusion_model_path); + opt.set_string("embeddings_connectors_path", embeddings_connectors_path); + opt.set_string("vae_path", vae_path); + opt.set_string("audio_vae_path", audio_vae_path); + opt.set_string("audio_encoder_path", audio_encoder_path); + opt.set_string("taesd_path", taesd_path); + opt.set_string("control_net_path", control_net_path); + opt.set_string("ip_adapter_path", ip_adapter_path); + opt.set_string("motion_module_path", motion_module_path); + opt.set_string("photo_maker_path", photo_maker_path); + opt.set_string("pulid_weights_path", pulid_weights_path); + opt.set_string("tensor_type_rules", tensor_type_rules); + opt.set_int("n_threads", n_threads); + opt.set_int("conditioning_cache_size", conditioning_cache_size); + if (wtype != SD_TYPE_COUNT) + opt.set_string("wtype", sd_type_name(wtype)); + opt.set_string("rng_type", rng_type); + opt.set_string("sampler_rng_type", sampler_rng_type); + opt.set_string("prediction", prediction); + opt.set_string("lora_apply_mode", lora_apply_mode); + opt.set_bool("enable_mmap", enable_mmap); + opt.set_string("flash_attn_mode", flash_attn_mode); + opt.set_string("conv_direct_mode", conv_direct_mode); + opt.set_bool("sage_attn", sage_attn); + opt.set_float("linear_scale", linear_scale); + opt.set_float("attn_scale", attn_scale); + opt.set_bool("tae_preview_only", taesd_preview_only); + opt.set_bool("diffusion_conv_direct", diffusion_conv_direct); + opt.set_bool("vae_conv_direct", vae_conv_direct); + opt.set_bool("force_sdxl_vae_conv_scale", force_sdxl_vae_conv_scale); + opt.set_string("vae_format", vae_format); + opt.set_string("max_vram", max_vram); + opt.set_bool("disable_prefetch", disable_prefetch); + opt.set_bool("disable_segmented_compute", disable_segmented_compute); + opt.set_bool("batched_cfg", batched_cfg); + opt.set_bool("eager_load", eager_load); + if (backend || !effective_backend.empty()) + opt.set_string("backend", effective_backend); + if (params_backend || !effective_params_backend.empty()) + opt.set_string("params_backend", effective_params_backend); + opt.set_string("split_mode", split_mode); + opt.set_bool("auto_fit", auto_fit); + opt.set_string("rpc_servers", rpc_servers); + opt.set_string("model_args", model_args); + opt.set_embeddings("embeddings", embedding_vec.data(), embedding_vec.size()); + return opt.finish(); } SDGenerationParams::SDGenerationParams() { - sd_sample_params_init(&sample_params); - sd_sample_params_init(&high_noise_sample_params); } ArgOptions SDGenerationParams::get_options() { @@ -1247,7 +1198,7 @@ ArgOptions SDGenerationParams::get_options() { &qwen_image_layers}, {"", "--video-frames", - "video frames (default: 1)", + "video frames; unset uses the library default", &video_frames}, {"", "--fps", @@ -1440,12 +1391,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } const char* arg = argv[index]; - sample_params.sample_method = str_to_sample_method(arg); - if (sample_params.sample_method == SAMPLE_METHOD_COUNT) { - LOG_ERROR("error: invalid sample method %s", - arg); - return -1; - } + sample_params.sample_method = arg; return 1; }; @@ -1454,12 +1400,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } const char* arg = argv[index]; - high_noise_sample_params.sample_method = str_to_sample_method(arg); - if (high_noise_sample_params.sample_method == SAMPLE_METHOD_COUNT) { - LOG_ERROR("error: invalid high noise sample method %s", - arg); - return -1; - } + high_noise_sample_params.sample_method = arg; return 1; }; @@ -1468,12 +1409,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } const char* arg = argv[index]; - sample_params.scheduler = str_to_scheduler(arg); - if (sample_params.scheduler == SCHEDULER_COUNT) { - LOG_ERROR("error: invalid scheduler %s", - arg); - return -1; - } + sample_params.scheduler = arg; return 1; }; @@ -1531,7 +1467,7 @@ ArgOptions SDGenerationParams::get_options() { return 1; }; - auto parse_sigmas_arg = [&](const char* value, std::vector* target, const char* option_name) { + auto parse_sigmas_arg = [&](const char* value, std::optional>* target, const char* option_name) { if (target == nullptr || value == nullptr) { return -1; } @@ -1556,7 +1492,7 @@ ArgOptions SDGenerationParams::get_options() { sigmas_str.pop_back(); } - size_t before = target->size(); + std::vector parsed; std::stringstream ss(sigmas_str); std::string item; while (std::getline(ss, item, ',')) { @@ -1564,7 +1500,7 @@ ArgOptions SDGenerationParams::get_options() { item.erase(item.find_last_not_of(" \t\n\r\f\v") + 1); if (!item.empty()) { try { - target->push_back(std::stof(item)); + parsed.push_back(std::stof(item)); } catch (const std::invalid_argument&) { LOG_ERROR("error: invalid float value '%s' in %s", item.c_str(), option_name); return -1; @@ -1575,10 +1511,11 @@ ArgOptions SDGenerationParams::get_options() { } } - if (target->size() == before && !sigmas_str.empty()) { + if (parsed.empty() && !sigmas_str.empty()) { LOG_ERROR("error: could not parse any sigma values from '%s'", value); return -1; } + *target = std::move(parsed); return 1; }; @@ -1763,20 +1700,23 @@ ArgOptions SDGenerationParams::get_options() { return 1; }; - std::string sample_methods = sample_method_to_str[0]; - for (int i = 1; i < SAMPLE_METHOD_COUNT; i++) { - sample_methods += ", " + std::string(sample_method_to_str[i]); + std::string sample_methods; + for (const char* name : example_options::sample_methods) { + if (!sample_methods.empty()) + sample_methods += ", "; + sample_methods += name; } - - std::string schedulers = scheduler_to_str[0]; - for (int i = 1; i < SCHEDULER_COUNT; i++) { - schedulers += ", " + std::string(scheduler_to_str[i]); + std::string schedulers; + for (const char* name : example_options::schedulers) { + if (!schedulers.empty()) + schedulers += ", "; + schedulers += name; } options.manual_options = { {"-s", "--seed", - "RNG seed (default: 42, use random seed for < 0)", + "RNG seed; unset or a negative value requests a random seed", on_seed_arg}, {"", "--sampling-method", @@ -1803,11 +1743,11 @@ ArgOptions SDGenerationParams::get_options() { on_hires_sigmas_arg}, {"", "--skip-layers", - "layers to skip for SLG steps (default: [7,8,9])", + "layers to skip for SLG steps; unset uses the library default", on_skip_layers_arg}, {"", "--high-noise-skip-layers", - "(high noise) layers to skip for SLG steps (default: [7,8,9])", + "(high noise) layers to skip for SLG steps; unset uses the library default", on_high_noise_skip_layers_arg}, {"-r", "--ref-image", @@ -2219,10 +2159,44 @@ bool SDGenerationParams::parse_prompt_images_json(const std::string& json_str) { return true; } +template +static void load_json_input(const json& object, const char* key, Field& out) { + if (!object.contains(key)) + return; + const auto& value = object.at(key); + if (value.is_null()) { + out = Field{}; + return; + } + using T = typename InputType::type; + bool valid = false; + if constexpr (std::is_same_v) + valid = value.is_string(); + else if constexpr (std::is_same_v) + valid = value.is_boolean(); + else if constexpr (std::is_integral_v) { + valid = value.is_number_integer(); + if (valid) { + if (value.is_number_unsigned()) + valid = value.get() <= static_cast(std::numeric_limits::max()); + else { + const auto integer = value.get(); + valid = integer >= std::numeric_limits::min() && integer <= std::numeric_limits::max(); + } + } + } else if constexpr (std::is_floating_point_v) + valid = value.is_number(); + else + valid = value.is_array(); + if (!valid) + throw std::invalid_argument(std::string(key) + ": invalid value type or range"); + out = value.get(); +} + /** Populates generation params from a JSON payload (server APIs); returns false on any malformed field. */ bool SDGenerationParams::from_json_str( const std::string& json_str, - const std::function& lora_path_resolver) { + const std::function& lora_path_resolver) try { json j; try { j = json::parse(json_str); @@ -2236,33 +2210,7 @@ bool SDGenerationParams::from_json_str( if (j.contains("prompt_images") && !parse_prompt_images_json(j["prompt_images"].dump())) return false; - auto load_if_exists = [&](const char* key, auto& out) { - if (j.contains(key)) { - using T = std::decay_t; - if constexpr (std::is_same_v) { - if (j[key].is_string()) - out = j[key]; - } else if constexpr (std::is_same_v || std::is_same_v) { - if (j[key].is_number_integer()) - out = j[key]; - } else if constexpr (std::is_same_v) { - if (j[key].is_number()) - out = j[key]; - } else if constexpr (std::is_same_v) { - if (j[key].is_boolean()) - out = j[key]; - } else if constexpr (std::is_same_v>) { - if (j[key].is_array()) - out = j[key].get>(); - } else if constexpr (std::is_same_v>) { - if (j[key].is_array()) - out = j[key].get>(); - } else if constexpr (std::is_same_v>) { - if (j[key].is_array()) - out = j[key].get>(); - } - } - }; + auto load_if_exists = [&](const char* key, auto& out) { load_json_input(j, key, out); }; load_if_exists("prompt", prompt); load_if_exists("negative_prompt", negative_prompt); @@ -2297,98 +2245,42 @@ bool SDGenerationParams::from_json_str( if (j.contains("hires") && j["hires"].is_object()) { const json& hires_json = j["hires"]; - if (hires_json.contains("enabled") && hires_json["enabled"].is_boolean()) { - hires_enabled = hires_json["enabled"]; - } - if (hires_json.contains("upscaler") && hires_json["upscaler"].is_string()) { - hires_upscaler = hires_json["upscaler"]; - } - if (hires_json.contains("scale") && hires_json["scale"].is_number()) { - hires_scale = hires_json["scale"]; - } - if (hires_json.contains("target_width") && hires_json["target_width"].is_number_integer()) { - hires_width = hires_json["target_width"]; - } - if (hires_json.contains("target_height") && hires_json["target_height"].is_number_integer()) { - hires_height = hires_json["target_height"]; - } - if (hires_json.contains("steps") && hires_json["steps"].is_number_integer()) { - hires_steps = hires_json["steps"]; - } - if (hires_json.contains("denoising_strength") && hires_json["denoising_strength"].is_number()) { - hires_denoising_strength = hires_json["denoising_strength"]; - } - if (hires_json.contains("custom_sigmas") && hires_json["custom_sigmas"].is_array()) { - hires_custom_sigmas = hires_json["custom_sigmas"].get>(); - } - if (hires_json.contains("upscale_tile_size") && hires_json["upscale_tile_size"].is_number_integer()) { - hires_upscale_tile_size = hires_json["upscale_tile_size"]; - } + load_json_input(hires_json, "enabled", hires_enabled); + load_json_input(hires_json, "upscaler", hires_upscaler); + load_json_input(hires_json, "scale", hires_scale); + load_json_input(hires_json, "target_width", hires_width); + load_json_input(hires_json, "target_height", hires_height); + load_json_input(hires_json, "steps", hires_steps); + load_json_input(hires_json, "denoising_strength", hires_denoising_strength); + load_json_input(hires_json, "custom_sigmas", hires_custom_sigmas); + load_json_input(hires_json, "upscale_tile_size", hires_upscale_tile_size); } auto parse_sample_params_json = [&](const json& sample_json, - sd_sample_params_t& target_params, + SDSampleParams& target_params, std::string& target_extra_sample_args, - std::vector& target_skip_layers, - std::vector* target_custom_sigmas) { - if (sample_json.contains("sample_steps") && sample_json["sample_steps"].is_number_integer()) { - target_params.sample_steps = sample_json["sample_steps"]; - } - if (sample_json.contains("eta") && sample_json["eta"].is_number()) { - target_params.eta = sample_json["eta"]; - } - if (sample_json.contains("shifted_timestep") && sample_json["shifted_timestep"].is_number_integer()) { - target_params.shifted_timestep = sample_json["shifted_timestep"]; - } - if (sample_json.contains("flow_shift") && sample_json["flow_shift"].is_number()) { - target_params.flow_shift = sample_json["flow_shift"]; - } - if (sample_json.contains("extra_sample_args") && sample_json["extra_sample_args"].is_string()) { - target_extra_sample_args = sample_json["extra_sample_args"].get(); - } - if (target_custom_sigmas != nullptr && - sample_json.contains("custom_sigmas") && - sample_json["custom_sigmas"].is_array() && - !sample_json["custom_sigmas"].empty()) { // empty [] = "not specified", keep server/CLI default - *target_custom_sigmas = sample_json["custom_sigmas"].get>(); - } - if (sample_json.contains("sample_method") && sample_json["sample_method"].is_string()) { - enum sample_method_t tmp = str_to_sample_method(sample_json["sample_method"].get().c_str()); - if (tmp != SAMPLE_METHOD_COUNT) { - target_params.sample_method = tmp; - } - } - if (sample_json.contains("scheduler") && sample_json["scheduler"].is_string()) { - enum scheduler_t tmp = str_to_scheduler(sample_json["scheduler"].get().c_str()); - if (tmp != SCHEDULER_COUNT) { - target_params.scheduler = tmp; - } - } + std::optional>& target_skip_layers, + std::optional>* target_custom_sigmas) { + load_json_input(sample_json, "sample_steps", target_params.sample_steps); + load_json_input(sample_json, "eta", target_params.eta); + load_json_input(sample_json, "shifted_timestep", target_params.shifted_timestep); + load_json_input(sample_json, "flow_shift", target_params.flow_shift); + load_json_input(sample_json, "extra_sample_args", target_extra_sample_args); + if (target_custom_sigmas) + load_json_input(sample_json, "custom_sigmas", *target_custom_sigmas); + load_json_input(sample_json, "sample_method", target_params.sample_method); + load_json_input(sample_json, "scheduler", target_params.scheduler); if (sample_json.contains("guidance") && sample_json["guidance"].is_object()) { const json& guidance_json = sample_json["guidance"]; - if (guidance_json.contains("txt_cfg") && guidance_json["txt_cfg"].is_number()) { - target_params.guidance.txt_cfg = guidance_json["txt_cfg"]; - } - if (guidance_json.contains("img_cfg") && guidance_json["img_cfg"].is_number()) { - target_params.guidance.img_cfg = guidance_json["img_cfg"]; - } - if (guidance_json.contains("distilled_guidance") && guidance_json["distilled_guidance"].is_number()) { - target_params.guidance.distilled_guidance = guidance_json["distilled_guidance"]; - } + load_json_input(guidance_json, "txt_cfg", target_params.guidance.txt_cfg); + load_json_input(guidance_json, "img_cfg", target_params.guidance.img_cfg); + load_json_input(guidance_json, "distilled_guidance", target_params.guidance.distilled_guidance); if (guidance_json.contains("slg") && guidance_json["slg"].is_object()) { const json& slg_json = guidance_json["slg"]; - if (slg_json.contains("layers") && slg_json["layers"].is_array()) { - target_skip_layers = slg_json["layers"].get>(); - } - if (slg_json.contains("layer_start") && slg_json["layer_start"].is_number()) { - target_params.guidance.slg.layer_start = slg_json["layer_start"]; - } - if (slg_json.contains("layer_end") && slg_json["layer_end"].is_number()) { - target_params.guidance.slg.layer_end = slg_json["layer_end"]; - } - if (slg_json.contains("scale") && slg_json["scale"].is_number()) { - target_params.guidance.slg.scale = slg_json["scale"]; - } + load_json_input(slg_json, "layers", target_skip_layers); + load_json_input(slg_json, "layer_start", target_params.guidance.slg.layer_start); + load_json_input(slg_json, "layer_end", target_params.guidance.slg.layer_end); + load_json_input(slg_json, "scale", target_params.guidance.slg.scale); } } }; @@ -2406,30 +2298,14 @@ bool SDGenerationParams::from_json_str( if (j.contains("vae_tiling_params") && j["vae_tiling_params"].is_object()) { const json& tiling_json = j["vae_tiling_params"]; - if (tiling_json.contains("enabled") && tiling_json["enabled"].is_boolean()) { - vae_tiling_params.enabled = tiling_json["enabled"]; - } - if (tiling_json.contains("temporal_tiling") && tiling_json["temporal_tiling"].is_boolean()) { - vae_tiling_params.temporal_tiling = tiling_json["temporal_tiling"]; - } - if (tiling_json.contains("tile_size_w") && tiling_json["tile_size_w"].is_number_integer()) { - vae_tiling_params.tile_size_w = tiling_json["tile_size_w"]; - } - if (tiling_json.contains("tile_size_h") && tiling_json["tile_size_h"].is_number_integer()) { - vae_tiling_params.tile_size_h = tiling_json["tile_size_h"]; - } - if (tiling_json.contains("target_overlap") && tiling_json["target_overlap"].is_number()) { - vae_tiling_params.target_overlap = tiling_json["target_overlap"]; - } - if (tiling_json.contains("rel_size_w") && tiling_json["rel_size_w"].is_number()) { - vae_tiling_params.rel_size_w = tiling_json["rel_size_w"]; - } - if (tiling_json.contains("rel_size_h") && tiling_json["rel_size_h"].is_number()) { - vae_tiling_params.rel_size_h = tiling_json["rel_size_h"]; - } - if (tiling_json.contains("extra_tiling_args") && tiling_json["extra_tiling_args"].is_string()) { - extra_tiling_args = tiling_json["extra_tiling_args"].get(); - } + load_json_input(tiling_json, "enabled", vae_tiling_params.enabled); + load_json_input(tiling_json, "temporal_tiling", vae_tiling_params.temporal_tiling); + load_json_input(tiling_json, "tile_size_w", vae_tiling_params.tile_size_w); + load_json_input(tiling_json, "tile_size_h", vae_tiling_params.tile_size_h); + load_json_input(tiling_json, "target_overlap", vae_tiling_params.target_overlap); + load_json_input(tiling_json, "rel_size_w", vae_tiling_params.rel_size_w); + load_json_input(tiling_json, "rel_size_h", vae_tiling_params.rel_size_h); + load_json_input(tiling_json, "extra_tiling_args", extra_tiling_args); } if (!parse_lora_json_field(j, lora_path_resolver, lora_map, high_noise_lora_map)) { @@ -2460,6 +2336,9 @@ bool SDGenerationParams::from_json_str( } return true; +} catch (const std::exception& error) { + LOG_ERROR("invalid generation parameters: %s", error.what()); + return false; } bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_dir) { @@ -2543,27 +2422,26 @@ bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d } bool SDGenerationParams::width_and_height_are_set() const { - return width > 0 && height > 0; + return width.has_value() && height.has_value(); } void SDGenerationParams::set_width_and_height_if_unset(int w, int h) { - if (!width_and_height_are_set()) { - LOG_INFO("set width x height to %d x %d", w, h); - width = w; + if (!width) + width = w; + if (!height) height = h; - } } -int SDGenerationParams::get_resolved_width() const { - return (width > 0) ? width : 512; +int SDGenerationParams::get_requested_width() const { + return width.value_or(0); } -int SDGenerationParams::get_resolved_height() const { - return (height > 0) ? height : 512; +int SDGenerationParams::get_requested_height() const { + return height.value_or(0); } bool SDGenerationParams::initialize_cache_params() { - sd_cache_params_init(&cache_params); + cache_params = SDCacheParams{}; auto parse_named_params = [&](const std::string& opt_str) -> bool { std::stringstream ss(opt_str); @@ -2629,19 +2507,19 @@ bool SDGenerationParams::initialize_cache_params() { if (!cache_mode.empty()) { if (cache_mode == "disabled") { - cache_params.mode = SD_CACHE_DISABLED; + cache_params.mode = "disabled"; } else if (cache_mode == "easycache") { - cache_params.mode = SD_CACHE_EASYCACHE; + cache_params.mode = "easycache"; } else if (cache_mode == "ucache") { - cache_params.mode = SD_CACHE_UCACHE; + cache_params.mode = "ucache"; } else if (cache_mode == "dbcache") { - cache_params.mode = SD_CACHE_DBCACHE; + cache_params.mode = "dbcache"; } else if (cache_mode == "taylorseer") { - cache_params.mode = SD_CACHE_TAYLORSEER; + cache_params.mode = "taylorseer"; } else if (cache_mode == "cache-dit") { - cache_params.mode = SD_CACHE_CACHE_DIT; + cache_params.mode = "cache_dit"; } else if (cache_mode == "spectrum") { - cache_params.mode = SD_CACHE_SPECTRUM; + cache_params.mode = "spectrum"; } else { LOG_ERROR("error: invalid cache mode '%s'", cache_mode.c_str()); return false; @@ -2652,9 +2530,9 @@ bool SDGenerationParams::initialize_cache_params() { return false; } - if (cache_params.mode == SD_CACHE_DBCACHE || - cache_params.mode == SD_CACHE_TAYLORSEER || - cache_params.mode == SD_CACHE_CACHE_DIT) { + if (cache_params.mode == "dbcache" || + cache_params.mode == "taylorseer" || + cache_params.mode == "cache_dit") { cache_params.scm_policy_dynamic = scm_policy_dynamic; } @@ -2664,47 +2542,35 @@ bool SDGenerationParams::initialize_cache_params() { bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict, bool validate_missing_loras) { vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); - if (high_noise_sample_params.sample_steps <= 0) { - high_noise_sample_params.sample_steps = -1; - } - if (!initialize_cache_params()) { return false; } - if (strict) { - batch_count = std::clamp(batch_count, 1, 8); - sample_params.sample_steps = std::clamp(sample_params.sample_steps, 1, 100); - } - - resolved_control_type = SD_CONTROL_TYPE_NONE; - if (!control_type.empty()) { - resolved_control_type = str_to_sd_control_type(control_type.c_str()); - if (resolved_control_type == SD_CONTROL_TYPE_COUNT) { - LOG_ERROR("invalid control type '%s' (openpose, depth, scribble, canny, normal, segment, tile, repaint)", control_type.c_str()); - return false; - } + if (strict && ((batch_count && (*batch_count < 1 || *batch_count > 8)) || + (sample_params.sample_steps && (*sample_params.sample_steps < 1 || *sample_params.sample_steps > 100)))) { + LOG_ERROR("server requests require batch_count in [1, 8] and sample_steps in [1, 100]"); + return false; } hires_upscaler_model_path.clear(); - if (hires_enabled) { - if (hires_upscaler.empty()) { - hires_upscaler = "Latent"; - } - resolved_hires_upscaler = str_to_sd_hires_upscaler(hires_upscaler.c_str()); - if (resolved_hires_upscaler == SD_HIRES_UPSCALER_NONE) { - hires_enabled = false; - } else if (resolved_hires_upscaler == SD_HIRES_UPSCALER_COUNT) { + resolved_hires_upscaler.clear(); + if (hires_upscaler) { + static const std::map names = { + {"None", "none"}, {"Latent", "latent"}, {"Latent (nearest)", "latent_nearest"}, {"Latent (nearest-exact)", "latent_nearest_exact"}, {"Latent (antialiased)", "latent_antialiased"}, {"Latent (bicubic)", "latent_bicubic"}, {"Latent (bicubic antialiased)", "latent_bicubic_antialiased"}, {"Lanczos", "lanczos"}, {"Nearest", "nearest"}, {"Model", "model"}}; + const auto it = names.find(*hires_upscaler); + if (it != names.end()) { + resolved_hires_upscaler = it->second; + } else if (std::any_of(names.begin(), names.end(), [&](const auto& name) { return name.second == *hires_upscaler; })) { + resolved_hires_upscaler = *hires_upscaler; + } else { static const std::vector valid_ext = {".gguf", ".safetensors", ".pt", ".pth"}; - if (!resolve_model_file_from_dir(hires_upscaler, - hires_upscalers_dir, - valid_ext, - "hires upscaler", - hires_upscaler_model_path)) { + if (hires_enabled.value_or(false) && + !resolve_model_file_from_dir(*hires_upscaler, hires_upscalers_dir, valid_ext, "hires upscaler", hires_upscaler_model_path)) return false; - } - resolved_hires_upscaler = SD_HIRES_UPSCALER_MODEL; + resolved_hires_upscaler = "model"; } + if (resolved_hires_upscaler == "none") + hires_enabled = false; } prompt_with_lora = prompt; @@ -2722,54 +2588,6 @@ bool SDGenerationParams::validate(SDMode mode) { LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode"); return false; } - if (batch_count <= 0) { - LOG_ERROR("error: batch_count must be greater than 0"); - return false; - } - - if (qwen_image_layers < 0) { - LOG_ERROR("error: qwen_image_layers must be non-negative"); - return false; - } - - if (sample_params.sample_steps <= 0) { - LOG_ERROR("error: the sample_steps must be greater than 0\n"); - return false; - } - - if (strength < 0.f || strength > 1.f) { - LOG_ERROR("error: can only work with strength in [0.0, 1.0]\n"); - return false; - } - - if (sample_params.guidance.txt_cfg < 0.f) { - LOG_ERROR("error: cfg_scale must be positive"); - return false; - } - - if (!cache_mode.empty()) { - if (cache_mode == "easycache" || cache_mode == "ucache") { - if (cache_params.reuse_threshold < 0.0f) { - LOG_ERROR("error: cache threshold must be non-negative"); - return false; - } - if (cache_params.start_percent < 0.0f || cache_params.start_percent >= 1.0f || - cache_params.end_percent <= 0.0f || cache_params.end_percent > 1.0f || - cache_params.start_percent >= cache_params.end_percent) { - LOG_ERROR("error: cache start/end percents must satisfy 0.0 <= start < end <= 1.0"); - return false; - } - } - } - - if (mode == VID_GEN && video_frames <= 0) { - return false; - } - - if (mode == VID_GEN && fps <= 0) { - return false; - } - if (ref_video_audio_paths.size() + ref_video_audios.size() > ref_video_paths.size() + ref_videos.size()) { LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video"); return false; @@ -2781,46 +2599,10 @@ bool SDGenerationParams::validate(SDMode mode) { return false; } - if (sample_params.shifted_timestep < 0 || sample_params.shifted_timestep > 1000) { - LOG_ERROR("error: shifted_timestep must be in range [0, 1000]"); - return false; - } - if (upscale_repeats < 1) { return false; } - if (upscale_tile_size < 1) { - return false; - } - - if (hires_enabled) { - if (hires_width < 0 || hires_height < 0) { - LOG_ERROR("error: hires target width and height must be >= 0"); - return false; - } - if (hires_scale <= 0.f && hires_width <= 0 && hires_height <= 0) { - LOG_ERROR("error: hires scale must be positive when target size is not set"); - return false; - } - if (hires_steps < 0) { - LOG_ERROR("error: hires steps must be >= 0"); - return false; - } - if (hires_denoising_strength <= 0.f || hires_denoising_strength > 1.f) { - LOG_ERROR("error: hires denoising strength must be in (0.0, 1.0]"); - return false; - } - if (!hires_custom_sigmas.empty() && hires_custom_sigmas.size() < 2) { - LOG_ERROR("error: hires custom sigmas must contain at least two values"); - return false; - } - if (hires_upscale_tile_size < 1) { - LOG_ERROR("error: hires upscale tile size must be positive"); - return false; - } - } - if (mode == UPSCALE || mode == ADETAILER) { if (init_image_path.length() == 0) { LOG_ERROR("error: %s mode needs an init image (--init-img)\n", @@ -2834,6 +2616,13 @@ bool SDGenerationParams::validate(SDMode mode) { return false; } + if (mode == IMG_GEN || mode == ADETAILER || mode == VID_GEN) { + auto opt = mode == VID_GEN ? to_video_opt() : to_image_opt(mode == ADETAILER); + if (!opt) { + LOG_ERROR("%s", sd_get_last_error()); + return false; + } + } return true; } @@ -2850,10 +2639,68 @@ bool SDGenerationParams::resolve_and_validate(SDMode mode, return true; } -sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { - sd_img_gen_params_t params; - sd_img_gen_params_init(¶ms); +static void set_sample_options(SDOptBuilder& opt, const std::string& prefix, const SDSampleParams& params, const std::optional>& layers, const std::optional>& sigmas, const std::string& extra_args) { + if (!extra_args.empty()) + opt.set_string((prefix + "args").c_str(), extra_args); + opt.set_float((prefix + "eta").c_str(), params.eta); + opt.set_float((prefix + "guidance.distilled_guidance").c_str(), params.guidance.distilled_guidance); + opt.set_float((prefix + "guidance.img_cfg").c_str(), params.guidance.img_cfg); + opt.set_float((prefix + "guidance.slg.layer_end").c_str(), params.guidance.slg.layer_end); + opt.set_float((prefix + "guidance.slg.layer_start").c_str(), params.guidance.slg.layer_start); + if (layers) + opt.set_ints((prefix + "guidance.slg.layers").c_str(), layers->data(), layers->size()); + opt.set_float((prefix + "guidance.slg.scale").c_str(), params.guidance.slg.scale); + opt.set_float((prefix + "guidance.txt_cfg").c_str(), params.guidance.txt_cfg); + opt.set_string((prefix + "method").c_str(), params.sample_method); + if (prefix == "sample.") { + opt.set_string("sample.scheduler", params.scheduler); + opt.set_float("sample.flow_shift", params.flow_shift); + opt.set_int("sample.shifted_timestep", params.shifted_timestep); + if (sigmas) + opt.set_floats((prefix + "sigmas").c_str(), sigmas->data(), sigmas->size()); + } + opt.set_int((prefix + "steps").c_str(), params.sample_steps); +} + +static void set_cache_options(SDOptBuilder& opt, const SDCacheParams& params) { + opt.set_int("cache.bn_compute_blocks", params.Bn_compute_blocks); + opt.set_float("cache.end_percent", params.end_percent); + opt.set_float("cache.error_decay_rate", params.error_decay_rate); + opt.set_int("cache.fn_compute_blocks", params.Fn_compute_blocks); + opt.set_int("cache.max_cached_steps", params.max_cached_steps); + opt.set_int("cache.max_continuous_cached_steps", params.max_continuous_cached_steps); + opt.set_int("cache.max_warmup_steps", params.max_warmup_steps); + opt.set_string("cache.mode", params.mode); + opt.set_bool("cache.reset_error_on_compute", params.reset_error_on_compute); + opt.set_float("cache.residual_diff_threshold", params.residual_diff_threshold); + opt.set_float("cache.reuse_threshold", params.reuse_threshold); + opt.set_string("cache.scm_mask", params.scm_mask); + opt.set_bool("cache.scm_policy_dynamic", params.scm_policy_dynamic); + opt.set_float("cache.spectrum_flex_window", params.spectrum_flex_window); + opt.set_float("cache.spectrum_lam", params.spectrum_lam); + opt.set_int("cache.spectrum_m", params.spectrum_m); + opt.set_float("cache.spectrum_stop_percent", params.spectrum_stop_percent); + opt.set_float("cache.spectrum_w", params.spectrum_w); + opt.set_int("cache.spectrum_warmup_steps", params.spectrum_warmup_steps); + opt.set_int("cache.spectrum_window_size", params.spectrum_window_size); + opt.set_float("cache.start_percent", params.start_percent); + opt.set_int("cache.taylorseer_n_derivatives", params.taylorseer_n_derivatives); + opt.set_int("cache.taylorseer_skip_interval", params.taylorseer_skip_interval); + opt.set_bool("cache.use_relative_threshold", params.use_relative_threshold); +} +static void set_tiling_options(SDOptBuilder& opt, const SDTilingParams& params) { + opt.set_bool("vae_tiling.enabled", params.enabled); + opt.set_string("vae_tiling.extra_tiling_args", params.extra_tiling_args); + opt.set_float("vae_tiling.rel_size_h", params.rel_size_h); + opt.set_float("vae_tiling.rel_size_w", params.rel_size_w); + opt.set_float("vae_tiling.target_overlap", params.target_overlap); + opt.set_bool("vae_tiling.temporal_tiling", params.temporal_tiling); + opt.set_int("vae_tiling.tile_size_h", params.tile_size_h); + opt.set_int("vae_tiling.tile_size_w", params.tile_size_w); +} + +SDOptPtr SDGenerationParams::to_image_opt(bool adetail) { lora_vec.clear(); lora_vec.reserve(lora_map.size() + high_noise_lora_map.size()); for (const auto& kv : lora_map) { @@ -2875,85 +2722,101 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { pm_id_image_views.push_back(image.get()); } - sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data(); - sample_params.guidance.slg.layer_count = skip_layers.size(); - high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data(); - high_noise_sample_params.guidance.slg.layer_count = high_noise_skip_layers.size(); - sample_params.custom_sigmas = custom_sigmas.empty() ? nullptr : custom_sigmas.data(); - sample_params.custom_sigmas_count = static_cast(custom_sigmas.size()); - sample_params.extra_sample_args = extra_sample_args.empty() ? nullptr : extra_sample_args.c_str(); - high_noise_sample_params.extra_sample_args = high_noise_extra_sample_args.empty() ? nullptr : high_noise_extra_sample_args.c_str(); - vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); - cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str(); - - sd_pm_params_t pm_params = { - pm_id_image_views.empty() ? nullptr : pm_id_image_views.data(), - static_cast(pm_id_image_views.size()), - pm_id_embed_path.empty() ? nullptr : pm_id_embed_path.c_str(), - pm_style_strength, - }; - - sd_pulid_params_t pulid_params = { - pulid_id_embedding_path.empty() ? nullptr : pulid_id_embedding_path.c_str(), - pulid_id_weight, - }; - - if (increase_ref_index) { - if (!ref_image_args.empty()) { - ref_image_args += ","; - } - ref_image_args += "ref_index_mode=increase"; - LOG_WARN("Notice: --increase-ref-index is deprecated. Use --ref-image-args \"ref_index_mode=increase\" instead."); - } - - params.loras = lora_vec.empty() ? nullptr : lora_vec.data(); - params.lora_count = static_cast(lora_vec.size()); - params.prompt = prompt.c_str(); - params.negative_prompt = negative_prompt.c_str(); - params.clip_skip = clip_skip; - params.init_image = init_image.get(); - params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); - params.ref_images_count = static_cast(ref_image_views.size()); - params.ref_image_args = ref_image_args.c_str(); - params.mask_image = mask_image.get(); - params.width = get_resolved_width(); - params.height = get_resolved_height(); - params.sample_params = sample_params; - params.strength = strength; - params.seed = seed; - params.batch_count = batch_count; - params.qwen_image_layers = qwen_image_layers; - params.control_image = control_image.get(); - params.control_strength = control_strength; - params.control_type = resolved_control_type; - params.ip_adapter_image = ip_adapter_image.get(); - params.ip_adapter_strength = ip_adapter_strength; - params.pm_params = pm_params; - params.pulid_params = pulid_params; - params.vae_tiling_params = vae_tiling_params; - params.cache = cache_params; - - params.hires.enabled = hires_enabled; - params.hires.upscaler = resolved_hires_upscaler; - params.hires.model_path = hires_upscaler_model_path.empty() ? nullptr : hires_upscaler_model_path.c_str(); - params.hires.scale = hires_scale; - params.hires.target_width = hires_width; - params.hires.target_height = hires_height; - params.hires.steps = hires_steps; - params.hires.denoising_strength = hires_denoising_strength; - params.hires.upscale_tile_size = hires_upscale_tile_size; - params.hires.custom_sigmas = hires_custom_sigmas.empty() ? nullptr : hires_custom_sigmas.data(); - params.hires.custom_sigmas_count = static_cast(hires_custom_sigmas.size()); - params.circular_x = circular || circular_x; - params.circular_y = circular || circular_y; - params.image_preprocess = {image_preprocess.c_str()}; - return params; + vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); + cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str(); + + std::string effective_ref_image_args = ref_image_args; + if (!adetail && increase_ref_index) { + if (!effective_ref_image_args.empty()) + effective_ref_image_args += ","; + effective_ref_image_args += "ref_index_mode=increase"; + } + + SDOptBuilder opt(adetail ? sd_adetail_opt_create() : sd_img_gen_opt_create()); + opt.set_string("prompt", prompt); + opt.set_string("negative_prompt", negative_prompt); + opt.set_int("clip_skip", clip_skip); + if (!adetail) + opt.set_image("init_image", init_image.get()); + if (!adetail) + opt.set_string("ref_image_args", effective_ref_image_args); + if (!adetail) + opt.set_image("mask_image", mask_image.get()); + opt.set_int("width", width); + opt.set_int("height", height); + opt.set_float("strength", strength); + opt.set_int("seed", seed); + if (!adetail) + opt.set_int("batch_count", batch_count); + if (!adetail) + opt.set_int("qwen_image_layers", qwen_image_layers); + if (!adetail) + opt.set_image("control_image", control_image.get()); + if (!adetail) + opt.set_float("control_strength", control_strength); + if (!adetail) + opt.set_string("control_type", control_type); + if (!adetail) + opt.set_image("ip_adapter_image", ip_adapter_image.get()); + if (!adetail) + opt.set_float("ip_adapter_strength", ip_adapter_strength); + if (!adetail) + opt.set_bool("hires.enabled", hires_enabled); + if (!adetail) + opt.set_string("hires.upscaler", resolved_hires_upscaler.empty() ? hires_upscaler : std::optional(resolved_hires_upscaler)); + if (!adetail) + opt.set_string("hires.model_path", hires_upscaler_model_path); + if (!adetail) + opt.set_float("hires.scale", hires_scale); + if (!adetail) + opt.set_int("hires.target_width", hires_width); + if (!adetail) + opt.set_int("hires.target_height", hires_height); + if (!adetail) + opt.set_int("hires.steps", hires_steps); + if (!adetail) + opt.set_float("hires.denoising_strength", hires_denoising_strength); + if (!adetail) + opt.set_int("hires.upscale_tile_size", hires_upscale_tile_size); + if (circular) + opt.set_bool("circular_x", true); + else + opt.set_bool("circular_x", circular_x); + if (circular) + opt.set_bool("circular_y", true); + else + opt.set_bool("circular_y", circular_y); + opt.set_loras("loras", lora_vec.data(), lora_vec.size()); + if (!adetail) + opt.set_images("ref_images", ref_image_views.data(), ref_image_views.size()); + if (!adetail) + opt.set_images("photomaker.id_images", pm_id_image_views.data(), pm_id_image_views.size()); + if (!adetail) + opt.set_string("photomaker.id_embed_path", pm_id_embed_path); + if (!adetail) + opt.set_float("photomaker.style_strength", pm_style_strength); + if (!adetail) + opt.set_string("pulid.id_embedding_path", pulid_id_embedding_path); + if (!adetail) + opt.set_float("pulid.id_weight", pulid_id_weight); + if (!adetail) + opt.set_floats("hires.sigmas", hires_custom_sigmas); + if (!adetail) + opt.set_string("image_preprocess.rules", image_preprocess); + set_sample_options(opt, "sample.", sample_params, skip_layers, custom_sigmas, extra_sample_args); + set_cache_options(opt, cache_params); + set_tiling_options(opt, vae_tiling_params); + if (adetail) { + if (!ad_prompt.empty()) + opt.set_string("adetail.prompt", ad_prompt); + if (!ad_negative_prompt.empty()) + opt.set_string("adetail.negative_prompt", ad_negative_prompt); + opt.set_string("adetail.args", extra_ad_args); + } + return opt.finish(); } -sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { - sd_vid_gen_params_t params; - sd_vid_gen_params_init(¶ms); - +SDOptPtr SDGenerationParams::to_video_opt() { lora_vec.clear(); lora_vec.reserve(lora_map.size() + high_noise_lora_map.size()); for (const auto& kv : lora_map) { @@ -2985,7 +2848,7 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { for (auto& frame : ref_videos[i]) { frame_views.push_back(frame.get()); } - sd_audio_t audio = i < ref_video_audios.size() ? ref_video_audios[i].get() : sd_audio_t{}; + SDAudio audio = i < ref_video_audios.size() ? ref_video_audios[i].get() : SDAudio{}; ref_video_views.push_back({frame_views.empty() ? nullptr : frame_views.data(), static_cast(frame_views.size()), 24, @@ -2998,65 +2861,61 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { ref_audio_views.push_back(audio.get()); } - sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data(); - sample_params.guidance.slg.layer_count = skip_layers.size(); - high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data(); - high_noise_sample_params.guidance.slg.layer_count = high_noise_skip_layers.size(); - sample_params.custom_sigmas = custom_sigmas.empty() ? nullptr : custom_sigmas.data(); - sample_params.custom_sigmas_count = static_cast(custom_sigmas.size()); - sample_params.extra_sample_args = extra_sample_args.empty() ? nullptr : extra_sample_args.c_str(); - high_noise_sample_params.extra_sample_args = high_noise_extra_sample_args.empty() ? nullptr : high_noise_extra_sample_args.c_str(); - vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); - cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str(); - - params.loras = lora_vec.empty() ? nullptr : lora_vec.data(); - params.lora_count = static_cast(lora_vec.size()); - params.prompt = prompt.c_str(); - params.negative_prompt = negative_prompt.c_str(); - params.clip_skip = clip_skip; - params.init_image = init_image.get(); - params.end_image = end_image.get(); - params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); - params.ref_images_count = static_cast(ref_image_views.size()); - params.ref_videos = ref_video_views.empty() ? nullptr : ref_video_views.data(); - params.ref_videos_count = static_cast(ref_video_views.size()); - params.ref_audios = ref_audio_views.empty() ? nullptr : ref_audio_views.data(); - params.ref_audios_count = static_cast(ref_audio_views.size()); - params.control_frames = control_frame_views.empty() ? nullptr : control_frame_views.data(); - params.control_frames_size = static_cast(control_frame_views.size()); - params.width = get_resolved_width(); - params.height = get_resolved_height(); - params.sample_params = sample_params; - params.high_noise_sample_params = high_noise_sample_params; - params.moe_boundary = moe_boundary; - params.strength = strength; - params.seed = seed; - params.video_frames = video_frames; - params.fps = fps; - params.vace_strength = vace_strength; - params.vae_tiling_params = vae_tiling_params; - params.cache = cache_params; - params.hires.enabled = hires_enabled; - params.hires.upscaler = resolved_hires_upscaler; - params.hires.model_path = hires_upscaler_model_path.empty() ? nullptr : hires_upscaler_model_path.c_str(); - params.hires.scale = hires_scale; - params.hires.target_width = hires_width; - params.hires.target_height = hires_height; - params.hires.steps = hires_steps; - params.hires.denoising_strength = hires_denoising_strength; - params.hires.upscale_tile_size = hires_upscale_tile_size; - params.hires.custom_sigmas = hires_custom_sigmas.empty() ? nullptr : hires_custom_sigmas.data(); - params.hires.custom_sigmas_count = static_cast(hires_custom_sigmas.size()); - params.circular_x = circular || circular_x; - params.circular_y = circular || circular_y; - params.image_preprocess = {image_preprocess.c_str()}; - params.ref_image_args = ref_image_args.c_str(); - return params; + vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); + cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str(); + + SDOptBuilder opt(sd_vid_gen_opt_create()); + opt.set_string("prompt", prompt); + opt.set_string("negative_prompt", negative_prompt); + opt.set_int("clip_skip", clip_skip); + opt.set_image("init_image", init_image.get()); + opt.set_image("end_image", end_image.get()); + opt.set_int("width", width); + opt.set_int("height", height); + opt.set_float("moe_boundary", moe_boundary); + opt.set_float("strength", strength); + opt.set_int("seed", seed); + opt.set_int("video_frames", video_frames); + opt.set_int("fps", fps); + opt.set_float("vace_strength", vace_strength); + opt.set_bool("hires.enabled", hires_enabled); + opt.set_string("hires.upscaler", resolved_hires_upscaler.empty() ? hires_upscaler : std::optional(resolved_hires_upscaler)); + opt.set_string("hires.model_path", hires_upscaler_model_path); + opt.set_float("hires.scale", hires_scale); + opt.set_int("hires.target_width", hires_width); + opt.set_int("hires.target_height", hires_height); + opt.set_int("hires.steps", hires_steps); + opt.set_float("hires.denoising_strength", hires_denoising_strength); + opt.set_int("hires.upscale_tile_size", hires_upscale_tile_size); + if (circular) + opt.set_bool("circular_x", true); + else + opt.set_bool("circular_x", circular_x); + if (circular) + opt.set_bool("circular_y", true); + else + opt.set_bool("circular_y", circular_y); + opt.set_string("ref_image_args", ref_image_args); + opt.set_loras("loras", lora_vec.data(), lora_vec.size()); + opt.set_images("ref_images", ref_image_views.data(), ref_image_views.size()); + opt.set_videos("ref_videos", ref_video_views.data(), ref_video_views.size()); + opt.set_audios("ref_audios", ref_audio_views.data(), ref_audio_views.size()); + opt.set_frames("control_frames", control_frame_views.data(), control_frame_views.size()); + opt.set_floats("hires.sigmas", hires_custom_sigmas); + opt.set_string("image_preprocess.rules", image_preprocess); + opt.set_string("sample.scheduler", high_noise_sample_params.scheduler); + opt.set_float("sample.flow_shift", high_noise_sample_params.flow_shift); + opt.set_int("sample.shifted_timestep", high_noise_sample_params.shifted_timestep); + set_sample_options(opt, "sample.", sample_params, skip_layers, custom_sigmas, extra_sample_args); + set_sample_options(opt, "high_noise_sample.", high_noise_sample_params, high_noise_skip_layers, {}, high_noise_extra_sample_args); + set_cache_options(opt, cache_params); + set_tiling_options(opt, vae_tiling_params); + return opt.finish(); } std::string SDGenerationParams::to_string() const { - FreeUniquePtr sample_params_str(sd_sample_params_to_str(&sample_params)); - FreeUniquePtr high_noise_sample_params_str(sd_sample_params_to_str(&high_noise_sample_params)); + const std::string sample_params_str = sample_params.sample_method.value_or("unset") + ", " + sample_params.scheduler.value_or("unset") + ", steps=" + (sample_params.sample_steps ? std::to_string(*sample_params.sample_steps) : "unset"); + const std::string high_noise_sample_params_str = high_noise_sample_params.sample_method.value_or("unset") + ", " + high_noise_sample_params.scheduler.value_or("unset") + ", steps=" + (high_noise_sample_params.sample_steps ? std::to_string(*high_noise_sample_params.sample_steps) : "unset"); std::ostringstream lora_ss; lora_ss << "{\n"; @@ -3113,15 +2972,15 @@ std::string SDGenerationParams::to_string() const { << " pm_id_images_dir: \"" << pm_id_images_dir << "\",\n" << " pm_id_embed_path: \"" << pm_id_embed_path << "\",\n" << " pm_style_strength: " << pm_style_strength << ",\n" - << " skip_layers: " << vec_to_string(skip_layers) << ",\n" - << " sample_params: " << SAFE_STR(sample_params_str.get()) << ",\n" - << " high_noise_skip_layers: " << vec_to_string(high_noise_skip_layers) << ",\n" - << " high_noise_sample_params: " << SAFE_STR(high_noise_sample_params_str.get()) << ",\n" - << " custom_sigmas: " << vec_to_string(custom_sigmas) << ",\n" + << " skip_layers: " << (skip_layers ? vec_to_string(*skip_layers) : "unset") << ",\n" + << " sample_params: " << SAFE_STR(sample_params_str.c_str()) << ",\n" + << " high_noise_skip_layers: " << (high_noise_skip_layers ? vec_to_string(*high_noise_skip_layers) : "unset") << ",\n" + << " high_noise_sample_params: " << SAFE_STR(high_noise_sample_params_str.c_str()) << ",\n" + << " custom_sigmas: " << (custom_sigmas ? vec_to_string(*custom_sigmas) : "unset") << ",\n" << " cache_mode: \"" << cache_mode << "\",\n" << " cache_option: \"" << cache_option << "\",\n" << " cache: " - << (cache_params.mode != SD_CACHE_DISABLED ? "enabled" : "disabled") + << cache_params.mode << " (threshold=" << cache_params.reuse_threshold << ", start=" << cache_params.start_percent << ", end=" << cache_params.end_percent << "),\n" @@ -3135,7 +2994,7 @@ std::string SDGenerationParams::to_string() const { << " seed: " << seed << ",\n" << " upscale_repeats: " << upscale_repeats << ",\n" << " upscale_tile_size: " << upscale_tile_size << ",\n" - << " hires: { enabled: " << (hires_enabled ? "true" : "false") + << " hires: { enabled: " << hires_enabled << ", upscaler: \"" << hires_upscaler << "\"" << ", model_path: \"" << hires_upscaler_model_path << "\"" << ", scale: " << hires_scale @@ -3143,7 +3002,7 @@ std::string SDGenerationParams::to_string() const { << ", target_height: " << hires_height << ", steps: " << hires_steps << ", denoising_strength: " << hires_denoising_strength - << ", custom_sigmas: " << vec_to_string(hires_custom_sigmas) + << ", custom_sigmas: " << (hires_custom_sigmas ? vec_to_string(*hires_custom_sigmas) : "unset") << ", upscale_tile_size: " << hires_upscale_tile_size << " },\n" << " vae_tiling_params: { " << vae_tiling_params.enabled << ", " diff --git a/examples/common/common.h b/examples/common/common.h index 43e11efe32..eb4a043a7f 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -2,6 +2,7 @@ #define __EXAMPLES_COMMON_COMMON_H__ #include "generation_metadata.h" +#include "options.h" #include #include @@ -40,21 +41,21 @@ struct StringOption { std::string long_name; std::string desc; int concat; - std::string* target; + OptionTarget target; }; struct IntOption { std::string short_name; std::string long_name; std::string desc; - int* target; + OptionTarget target; }; struct FloatOption { std::string short_name; std::string long_name; std::string desc; - float* target; + OptionTarget target; }; struct BoolOption { @@ -62,7 +63,7 @@ struct BoolOption { std::string long_name; std::string desc; bool keep_true; - bool* target; + OptionTarget target; }; struct ManualFunction { @@ -123,8 +124,8 @@ bool decode_base64_image(const std::string& encoded_input, SDImageOwner& out_image); struct SDContextParams { - int n_threads = -1; - int conditioning_cache_size = 4; + std::optional n_threads; + std::optional conditioning_cache_size; std::string model_path; std::string clip_l_path; std::string clip_g_path; @@ -138,7 +139,7 @@ struct SDContextParams { std::string uncond_diffusion_model_path; std::string embeddings_connectors_path; std::string vae_path; - std::string vae_format = "auto"; + std::optional vae_format; std::string audio_vae_path; std::string audio_encoder_path; std::string taesd_path; @@ -157,42 +158,41 @@ struct SDContextParams { std::map embedding_map; std::vector embedding_vec; - rng_type_t rng_type = CUDA_RNG; - rng_type_t sampler_rng_type = RNG_TYPE_COUNT; - bool offload_params_to_cpu = false; - std::string max_vram = "0"; - bool disable_prefetch = false; - bool disable_segmented_compute = false; - bool eager_load = false; - bool batched_cfg = true; - std::string backend; - std::string params_backend; - std::string split_mode; - std::string model_args; - bool auto_fit = true; - std::string rpc_servers; + std::optional rng_type; + std::optional sampler_rng_type; + bool offload_params_to_cpu = false; + std::optional max_vram; + std::optional disable_prefetch; + std::optional disable_segmented_compute; + std::optional eager_load; + std::optional batched_cfg; + std::optional backend; + std::optional params_backend; + std::optional split_mode; + std::optional model_args; + std::optional auto_fit; + std::optional rpc_servers; bool list_devices = false; std::string effective_backend; std::string effective_params_backend; - bool enable_mmap = false; - bool control_net_cpu = false; - bool clip_on_cpu = false; - bool vae_on_cpu = false; - std::string flash_attn_mode = "auto"; - std::string conv_direct_mode; // Empty selects auto while honoring legacy convolution flags. - bool sage_attn = false; - bool diffusion_conv_direct = false; - bool vae_conv_direct = false; - bool taesd_preview_only = false; - - prediction_t prediction = PREDICTION_COUNT; - lora_apply_mode_t lora_apply_mode = LORA_APPLY_AUTO; - - bool force_sdxl_vae_conv_scale = false; - float linear_scale = 0.f; - float attn_scale = 0.f; - - float flow_shift = INFINITY; + std::optional enable_mmap; + bool control_net_cpu = false; + bool clip_on_cpu = false; + bool vae_on_cpu = false; + std::optional flash_attn_mode; + std::optional conv_direct_mode; + std::optional sage_attn; + std::optional diffusion_conv_direct; + std::optional vae_conv_direct; + std::optional taesd_preview_only; + + std::optional prediction; + std::optional lora_apply_mode; + + std::optional force_sdxl_vae_conv_scale; + std::optional linear_scale; + std::optional attn_scale; + ArgOptions get_options(); bool print_devices() const; void build_embedding_map(); @@ -201,7 +201,7 @@ struct SDContextParams { bool validate(SDMode mode); bool resolve_and_validate(SDMode mode); std::string to_string() const; - sd_ctx_params_t to_sd_ctx_params_t(); + SDOptPtr to_context_opt(); }; struct SDGenerationParams { @@ -212,15 +212,15 @@ struct SDGenerationParams { std::string ad_prompt; std::string ad_negative_prompt; std::string extra_ad_args; - int clip_skip = -1; // <= 0 represents unspecified - int width = -1; - int height = -1; - int batch_count = 1; - int qwen_image_layers = 3; - int64_t seed = 42; - float strength = 0.75f; - float control_strength = 0.9f; - float ip_adapter_strength = 1.0f; + std::optional clip_skip; + std::optional width; + std::optional height; + std::optional batch_count; + std::optional qwen_image_layers; + std::optional seed; + std::optional strength; + std::optional control_strength; + std::optional ip_adapter_strength; bool increase_ref_index = false; bool embed_image_metadata = true; @@ -228,7 +228,7 @@ struct SDGenerationParams { std::string end_image_path; std::string mask_image_path; std::string control_image_path; - std::string control_type; + std::optional control_type; std::string ip_adapter_image_path; std::vector ref_image_paths; std::vector ref_video_paths; @@ -236,26 +236,26 @@ struct SDGenerationParams { std::vector ref_audio_paths; std::string control_video_path; - sd_sample_params_t sample_params; - sd_sample_params_t high_noise_sample_params; + SDSampleParams sample_params; + SDSampleParams high_noise_sample_params; std::string extra_sample_args; std::string high_noise_extra_sample_args; - std::vector skip_layers = {7, 8, 9}; - std::vector high_noise_skip_layers = {7, 8, 9}; + std::optional> skip_layers; + std::optional> high_noise_skip_layers; - std::vector custom_sigmas; + std::optional> custom_sigmas; std::string cache_mode; std::string cache_option; std::string scm_mask; - bool scm_policy_dynamic = true; - sd_cache_params_t cache_params{}; - - float moe_boundary = 0.875f; - int video_frames = 1; - int fps = 16; - float vace_strength = 1.f; - sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; + std::optional scm_policy_dynamic; + SDCacheParams cache_params{}; + + std::optional moe_boundary; + std::optional video_frames; + std::optional fps; + std::optional vace_strength; + SDTilingParams vae_tiling_params; std::string extra_tiling_args; std::string ref_image_args; @@ -263,28 +263,28 @@ struct SDGenerationParams { std::string pm_id_images_dir; std::string pm_id_embed_path; - float pm_style_strength = 20.f; + std::optional pm_style_strength; std::string pulid_id_embedding_path; - float pulid_id_weight = 1.0f; + std::optional pulid_id_weight; - int upscale_repeats = 1; - int upscale_tile_size = 128; + int upscale_repeats = 1; + std::optional upscale_tile_size; - bool circular = false; - bool circular_x = false; - bool circular_y = false; + bool circular = false; + std::optional circular_x; + std::optional circular_y; - bool hires_enabled = false; - std::string hires_upscaler = "Latent"; + std::optional hires_enabled; + std::optional hires_upscaler; std::string hires_upscaler_model_path; - float hires_scale = 2.f; - int hires_width = 0; - int hires_height = 0; - int hires_steps = 0; - float hires_denoising_strength = 0.7f; - int hires_upscale_tile_size = 128; - std::vector hires_custom_sigmas; + std::optional hires_scale; + std::optional hires_width; + std::optional hires_height; + std::optional hires_steps; + std::optional hires_denoising_strength; + std::optional hires_upscale_tile_size; + std::optional> hires_custom_sigmas; std::map lora_map; std::map high_noise_lora_map; @@ -292,8 +292,7 @@ struct SDGenerationParams { // Derived and normalized fields. std::string prompt_with_lora; // for metadata record only std::vector lora_vec; - sd_hires_upscaler_t resolved_hires_upscaler; - sd_control_type_t resolved_control_type = SD_CONTROL_TYPE_NONE; + std::string resolved_hires_upscaler; // Owned execution payload. SDImageOwner init_image; @@ -309,13 +308,13 @@ struct SDGenerationParams { std::vector pm_id_images; std::vector control_frames; - // Backing storage for sd_img_gen_params_t view fields. - std::vector ref_image_views; - std::vector> ref_video_frame_views; - std::vector ref_video_views; - std::vector ref_audio_views; - std::vector pm_id_image_views; - std::vector control_frame_views; + // Media views used while populating the opaque options. + std::vector ref_image_views; + std::vector> ref_video_frame_views; + std::vector ref_video_views; + std::vector ref_audio_views; + std::vector pm_id_image_views; + std::vector control_frame_views; SDGenerationParams(); SDGenerationParams(const SDGenerationParams& other) = default; @@ -331,16 +330,16 @@ struct SDGenerationParams { bool extract_and_remove_lora(const std::string& lora_model_dir); bool width_and_height_are_set() const; void set_width_and_height_if_unset(int w, int h); - int get_resolved_width() const; - int get_resolved_height() const; + int get_requested_width() const; + int get_requested_height() const; bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false, bool validate_missing_loras = false); bool validate(SDMode mode); bool resolve_and_validate(SDMode mode, const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false); - sd_img_gen_params_t to_sd_img_gen_params_t(); - sd_vid_gen_params_t to_sd_vid_gen_params_t(); + SDOptPtr to_image_opt(bool adetail = false); + SDOptPtr to_video_opt(); std::string to_string() const; }; diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp index e878c889e6..4159e46695 100644 --- a/examples/common/media_io.cpp +++ b/examples/common/media_io.cpp @@ -409,7 +409,7 @@ bool extract_vp8_frame_from_webp(const std::vector& webp_data, std::vec return false; } -bool encode_sd_image_to_vp8_frame(const sd_image_t& image, int quality, std::vector& vp8_frame) { +bool encode_sd_image_to_vp8_frame(const SDImage& image, int quality, std::vector& vp8_frame) { if (image.data == nullptr || image.width == 0 || image.height == 0) { return false; } @@ -659,7 +659,7 @@ void write_fourcc(std::vector& data, const char* fourcc) { data.insert(data.end(), fourcc, fourcc + 4); } -static std::vector audio_to_pcm16_bytes(const sd_audio_t* audio) { +static std::vector audio_to_pcm16_bytes(const SDAudio* audio) { if (audio == nullptr || audio->data == nullptr || audio->sample_count == 0 || audio->channels == 0 || audio->sample_rate == 0) { return {}; } @@ -674,7 +674,7 @@ static std::vector audio_to_pcm16_bytes(const sd_audio_t* audio) { return bytes; } -static std::pair audio_sample_range_for_video_frame(const sd_audio_t* audio, int frame_idx, int num_frames, int fps) { +static std::pair audio_sample_range_for_video_frame(const SDAudio* audio, int frame_idx, int num_frames, int fps) { if (audio == nullptr || fps <= 0 || num_frames <= 0) { return {0, 0}; } @@ -783,7 +783,7 @@ uint8_t* load_image_from_file(const char* image_path, return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, out_channel); } -bool load_sd_image_from_file(sd_image_t* image, +bool load_sd_image_from_file(SDImage* image, const char* image_path, int expected_width, int expected_height, @@ -836,7 +836,7 @@ static void append_avi_metadata(std::vector& data, const std::string& p size_t start_pos = data.size(); } -std::vector create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +std::vector create_mjpg_avi_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { if (num_images == 0) { fprintf(stderr, "Error: Image array is empty.\n"); return {}; @@ -1043,7 +1043,7 @@ std::vector create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images return avi_data; } -int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +int create_mjpg_avi_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { std::vector avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters); if (avi_data.empty()) { return -1; @@ -1056,7 +1056,7 @@ int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int } #ifdef SD_USE_WEBP -std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality) { +std::vector create_animated_webp_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality) { if (num_images == 0) { fprintf(stderr, "Error: Image array is empty.\n"); return {}; @@ -1102,7 +1102,7 @@ std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* i int timestamp_ms = 0; for (int i = 0; i < num_images; ++i) { - const sd_image_t& image = images[i]; + const SDImage& image = images[i]; if (static_cast(image.width) != width || static_cast(image.height) != height) { fprintf(stderr, "Error: Frame dimensions do not match.\n"); return {}; @@ -1160,7 +1160,7 @@ std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* i return std::vector(webp_data.data.bytes, webp_data.data.bytes + webp_data.data.size); } -int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) { +int create_animated_webp_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality) { std::vector webp_data = create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality); if (webp_data.empty()) { return -1; @@ -1174,7 +1174,7 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images #endif #ifdef SD_USE_WEBM -std::vector create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +std::vector create_webm_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { if (num_images == 0) { fprintf(stderr, "Error: Image array is empty.\n"); return {}; @@ -1261,7 +1261,7 @@ std::vector create_webm_from_sd_images_to_vector(sd_image_t* images, in uint64_t timestamp_ns = 0; for (int i = 0; i < num_images; ++i) { - const sd_image_t& image = images[i]; + const SDImage& image = images[i]; if (static_cast(image.width) != width || static_cast(image.height) != height) { fprintf(stderr, "Error: Frame dimensions do not match.\n"); return -1; @@ -1314,7 +1314,7 @@ std::vector create_webm_from_sd_images_to_vector(sd_image_t* images, in return writer.data(); } -int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +int create_webm_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { std::vector webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters); if (webm_data.empty()) { return -1; @@ -1328,11 +1328,11 @@ int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num #endif std::vector create_video_from_sd_images_to_vector(const std::string& output_format, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality, - const sd_audio_t* audio, + const SDAudio* audio, const std::string& parameters) { std::string format = output_format; std::transform(format.begin(), format.end(), format.begin(), @@ -1359,7 +1359,7 @@ std::vector create_video_from_sd_images_to_vector(const std::string& ou return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters); } -int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +int create_video_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { std::string path = filename ? filename : ""; auto pos = path.find_last_of('.'); std::string ext = pos == std::string::npos ? "" : path.substr(pos); diff --git a/examples/common/media_io.h b/examples/common/media_io.h index 7feb0ada25..d765db7fcc 100644 --- a/examples/common/media_io.h +++ b/examples/common/media_io.h @@ -5,7 +5,7 @@ #include #include -#include "stable-diffusion.h" +#include "resource_owners.hpp" enum class EncodedImageFormat { JPEG, @@ -43,7 +43,7 @@ uint8_t* load_image_from_file(const char* image_path, int expected_height = 0, int expected_channel = 3); -bool load_sd_image_from_file(sd_image_t* image, +bool load_sd_image_from_file(SDImage* image, const char* image_path, int expected_width = 0, int expected_height = 0, @@ -59,26 +59,26 @@ uint8_t* load_image_from_memory(const char* image_bytes, int expected_channel = 3); int create_mjpg_avi_from_sd_images(const char* filename, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); -std::vector create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, +std::vector create_mjpg_avi_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); #ifdef SD_USE_WEBP int create_animated_webp_from_sd_images(const char* filename, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90); -std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* images, +std::vector create_animated_webp_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality = 90); @@ -86,33 +86,33 @@ std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* i #ifdef SD_USE_WEBM int create_webm_from_sd_images(const char* filename, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); -std::vector create_webm_from_sd_images_to_vector(sd_image_t* images, +std::vector create_webm_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); #endif int create_video_from_sd_images(const char* filename, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); std::vector create_video_from_sd_images_to_vector(const std::string& output_format, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); bool write_wav_to_file(const std::string& path, diff --git a/examples/common/options.h b/examples/common/options.h new file mode 100644 index 0000000000..3b860328f8 --- /dev/null +++ b/examples/common/options.h @@ -0,0 +1,185 @@ +#ifndef SD_EXAMPLES_OPTIONS_H +#define SD_EXAMPLES_OPTIONS_H + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nlohmann { + template + struct adl_serializer> { + static void to_json(json& out, const std::optional& value) { + out = value ? json(*value) : json(nullptr); + } + static void from_json(const json& in, std::optional& value) { + value = in.is_null() ? std::nullopt : std::optional(in.get()); + } + }; +} + +template +std::ostream& operator<<(std::ostream& out, const std::optional& value) { + if (!value) + return out << "unset"; + if constexpr (std::is_same_v) + return out << (*value ? "true" : "false"); + else + return out << *value; +} + +template +class OptionTarget { + T* value_ = nullptr; + std::optional* input_ = nullptr; + +public: + OptionTarget(T* value) + : value_(value) {} + OptionTarget(std::optional* input) + : input_(input) {} + T get() const { return input_ ? input_->value_or(T{}) : *value_; } + void set(T value) const { + if (input_) + *input_ = std::move(value); + else + *value_ = std::move(value); + } +}; + +template +struct InputType { + using type = T; +}; +template +struct InputType> { + using type = T; +}; + +struct SDTilingParams { + std::optional enabled; + std::optional temporal_tiling; + std::optional tile_size_w; + std::optional tile_size_h; + std::optional target_overlap; + std::optional rel_size_w; + std::optional rel_size_h; + const char* extra_tiling_args = nullptr; +}; + +struct SDSLGParams { + std::optional layer_start; + std::optional layer_end; + std::optional scale; +}; + +struct SDGuidanceParams { + std::optional txt_cfg; + std::optional img_cfg; + std::optional distilled_guidance; + SDSLGParams slg; +}; + +struct SDSampleParams { + SDGuidanceParams guidance; + std::optional scheduler; + std::optional sample_method; + std::optional sample_steps; + std::optional eta; + std::optional shifted_timestep; + std::optional flow_shift; +}; + +struct SDCacheParams { + std::optional mode; + std::optional reuse_threshold; + std::optional start_percent; + std::optional end_percent; + std::optional error_decay_rate; + std::optional use_relative_threshold; + std::optional reset_error_on_compute; + std::optional Fn_compute_blocks; + std::optional Bn_compute_blocks; + std::optional residual_diff_threshold; + std::optional max_warmup_steps; + std::optional max_cached_steps; + std::optional max_continuous_cached_steps; + std::optional taylorseer_n_derivatives; + std::optional taylorseer_skip_interval; + const char* scm_mask = nullptr; + std::optional scm_policy_dynamic; + std::optional spectrum_w; + std::optional spectrum_m; + std::optional spectrum_lam; + std::optional spectrum_window_size; + std::optional spectrum_flex_window; + std::optional spectrum_warmup_steps; + std::optional spectrum_stop_percent; +}; + +namespace example_options { + inline constexpr const char* sample_methods[] = { + "euler", + "euler_a", + "heun", + "dpm2", + "dpm++2s_a", + "dpm++2m", + "dpm++2mv2", + "ipndm", + "ipndm_v", + "lcm", + "ddim_trailing", + "tcd", + "res_multistep", + "res_2s", + "er_sde", + "euler_cfg_pp", + "euler_a_cfg_pp", + "euler_ge", + "dpm++2m_sde", + "dpm++2m_sde_bt", + "lms", + }; + + inline constexpr const char* schedulers[] = { + "discrete", + "karras", + "exponential", + "ays", + "gits", + "sgm_uniform", + "simple", + "smoothstep", + "kl_optimal", + "lcm", + "bong_tangent", + "ltx2", + "logit_normal", + "flux2", + "flux", + "beta", + "llada_image", + }; + + inline constexpr const char* preview_modes[] = { + "none", + "proj", + "tae", + "vae", + }; + + template + bool contains(const char* const (&values)[N], const std::string& value) { + for (const char* entry : values) + if (value == entry) + return true; + return false; + } +} + +#endif diff --git a/examples/common/resource_owners.hpp b/examples/common/resource_owners.hpp index 8a8f843146..8ba69e7a9c 100644 --- a/examples/common/resource_owners.hpp +++ b/examples/common/resource_owners.hpp @@ -4,13 +4,136 @@ #include #include #include +#include #include +#include +#include #include #include +#include #include "generation_metadata.h" +#include "log.h" #include "stable-diffusion.h" +struct SDImage { + uint32_t width; + uint32_t height; + uint32_t channel; + uint8_t* data; + sd_tensor_t* storage = nullptr; +}; + +struct SDAudio { + uint32_t sample_rate; + uint32_t channels; + uint64_t sample_count; + float* data; + sd_tensor_t* storage = nullptr; +}; + +struct SDReferenceVideo { + SDImage* frames; + int frame_count; + int fps; + SDAudio audio; +}; + +struct SDTensorDeleter { + void operator()(sd_tensor_t* tensor) const { sd_tensor_free(tensor); } +}; +using SDTensorPtr = std::unique_ptr; + +static inline SDImage tensor_image_view(const sd_tensor_t* tensor) { + if (!tensor) + return {}; + return {uint32_t(sd_tensor_dim(tensor, 1)), uint32_t(sd_tensor_dim(tensor, 2)), uint32_t(sd_tensor_dim(tensor, 0)), + const_cast(static_cast(sd_tensor_data(tensor)))}; +} + +static inline void sd_image_free(SDImage* image) { + if (!image) + return; + if (image->storage) + sd_tensor_free(image->storage); + else + free(image->data); + *image = {}; +} + +static inline void free_sd_audio(SDAudio* audio) { + if (!audio) + return; + if (audio->storage) + sd_tensor_free(audio->storage); + else + free(audio->data); + free(audio); +} + +static inline SDTensorPtr image_tensor(const SDImage& image) { + if (!image.data) + return {}; + if (image.storage && sd_tensor_data(image.storage) == image.data && + sd_tensor_dim(image.storage, 0) == image.channel && sd_tensor_dim(image.storage, 1) == image.width && sd_tensor_dim(image.storage, 2) == image.height) + return SDTensorPtr(sd_tensor_retain(image.storage)); + const int64_t shape[] = {image.channel, image.width, image.height}; + return SDTensorPtr(sd_tensor_create(SD_TENSOR_U8, 3, shape, image.data, size_t(image.width) * image.height * image.channel)); +} + +static inline SDTensorPtr audio_tensor(const SDAudio& audio) { + if (!audio.data) + return {}; + if (audio.storage) + return SDTensorPtr(sd_tensor_retain(audio.storage)); + const int64_t shape[] = {audio.channels, static_cast(audio.sample_count)}; + return SDTensorPtr(sd_tensor_create(SD_TENSOR_F32, 2, shape, audio.data, size_t(audio.sample_count) * audio.channels * sizeof(float))); +} + +static inline SDTensorPtr frames_tensor(const SDImage* frames, size_t count) { + if (!frames || !count) + return {}; + const auto& first = frames[0]; + const size_t frame_bytes = size_t(first.width) * first.height * first.channel; + if (!frame_bytes || count > SIZE_MAX / frame_bytes) { + LOG_ERROR("invalid video frame storage size"); + return {}; + } + std::vector pixels(frame_bytes * count); + for (size_t i = 0; i < count; ++i) { + if (!frames[i].data || frames[i].width != first.width || frames[i].height != first.height || frames[i].channel != first.channel) { + LOG_ERROR("video frames must have matching dimensions and channels"); + return {}; + } + memcpy(pixels.data() + i * frame_bytes, frames[i].data, frame_bytes); + } + const int64_t shape[] = {first.channel, first.width, first.height, static_cast(count)}; + return SDTensorPtr(sd_tensor_create(SD_TENSOR_U8, 4, shape, pixels.data(), pixels.size())); +} + +static inline SDAudio* retain_result_audio(const sd_result_t* result) { + const auto* audio = sd_result_get_audio(result); + if (!audio) + return nullptr; + auto* view = static_cast(malloc(sizeof(SDAudio))); + if (!view) + return nullptr; + *view = {audio->sample_rate, uint32_t(sd_tensor_dim(audio->data, 0)), uint64_t(sd_tensor_dim(audio->data, 1)), + const_cast(static_cast(sd_tensor_data(audio->data))), sd_tensor_retain(audio->data)}; + return view; +} + +static inline bool preprocess_canny(SDImage& image, float high, float low, float weak, float strong, bool inverse) { + auto input = image_tensor(image); + sd_tensor_t* output = nullptr; + if (!input || sd_preprocess_canny(input.get(), high, low, weak, strong, inverse, &output) != SD_STATUS_OK) + return false; + sd_image_free(&image); + image = tensor_image_view(output); + image.storage = output; + return true; +} + struct FreeDeleter { void operator()(void* ptr) const { free(ptr); @@ -28,23 +151,23 @@ struct FileCloser { struct SDCtxDeleter { void operator()(sd_ctx_t* ctx) const { if (ctx != nullptr) { - free_sd_ctx(ctx); + sd_ctx_free(ctx); } } }; struct UpscalerCtxDeleter { - void operator()(upscaler_ctx_t* ctx) const { + void operator()(sd_upscaler_t* ctx) const { if (ctx != nullptr) { - free_upscaler_ctx(ctx); + sd_upscaler_free(ctx); } } }; struct ADetailerCtxDeleter { - void operator()(adetailer_ctx_t* ctx) const { + void operator()(sd_adetailer_t* ctx) const { if (ctx != nullptr) { - free_adetailer_ctx(ctx); + sd_adetailer_free(ctx); } } }; @@ -54,12 +177,280 @@ using FreeUniquePtr = std::unique_ptr; using FilePtr = std::unique_ptr; using SDCtxPtr = std::unique_ptr; -using UpscalerCtxPtr = std::unique_ptr; -using ADetailerCtxPtr = std::unique_ptr; +using UpscalerCtxPtr = std::unique_ptr; +using ADetailerCtxPtr = std::unique_ptr; + +struct SDOptDeleter { + void operator()(sd_opt_t* opt) const { sd_opt_free(opt); } +}; +struct SDResultDeleter { + void operator()(sd_result_t* result) const { sd_result_free(result); } +}; +using SDOptPtr = std::unique_ptr; +using SDResultPtr = std::unique_ptr; + +class SDOptBuilder { + SDOptPtr opt; + sd_status_t status; + +public: + explicit SDOptBuilder(sd_opt_t* value) + : opt(value), status(value ? SD_STATUS_OK : SD_STATUS_OUT_OF_MEMORY) {} + SDOptPtr finish() { return status == SD_STATUS_OK ? std::move(opt) : SDOptPtr{}; } + void unset(const char* key) { + if (status == SD_STATUS_OK) + status = sd_opt_unset(opt.get(), key); + } + void set_floats(const char* key, const std::optional>& value) { + if (value) + set_floats(key, value->data(), value->size()); + } + void set_bool(const char* key, const std::optional& value) { + if (value) + set_bool(key, *value); + } + template + void set_int(const char* key, const std::optional& value) { + if (value) + set_int(key, *value); + } + template + void set_float(const char* key, const std::optional& value) { + if (value) + set_float(key, *value); + } + void set_string(const char* key, const std::optional& value) { + if (value) + set_string(key, *value); + } + void set_bool(const char* key, bool value) { + if (status == SD_STATUS_OK) + status = sd_opt_set_bool(opt.get(), key, value); + } + void set_int(const char* key, int64_t value) { + if (status == SD_STATUS_OK) + status = sd_opt_set_int(opt.get(), key, value); + } + void set_float(const char* key, double value) { + if (status == SD_STATUS_OK) + status = sd_opt_set_float(opt.get(), key, value); + } + void set_string(const char* key, const std::string& value) { + if (status == SD_STATUS_OK) + status = sd_opt_set_string(opt.get(), key, value.c_str()); + } + void set_string(const char* key, const char* value) { + if (value && status == SD_STATUS_OK) + status = sd_opt_set_string(opt.get(), key, value); + } + void set_image(const char* key, const SDImage& value) { + if (!value.data || status != SD_STATUS_OK) + return; + auto tensor = image_tensor(value); + if (!tensor) { + status = SD_STATUS_INVALID_ARGUMENT; + return; + } + status = sd_opt_set_tensor(opt.get(), key, tensor.get()); + } + void set_images(const char* key, const SDImage* value, size_t count) { + if (status != SD_STATUS_OK) + return; + std::vector storage; + std::vector tensors; + storage.reserve(count); + tensors.reserve(count); + for (size_t i = 0; i < count; ++i) { + storage.push_back(image_tensor(value[i])); + if (!storage.back()) { + status = SD_STATUS_INVALID_ARGUMENT; + return; + } + tensors.push_back(storage.back().get()); + } + status = sd_opt_set_tensors(opt.get(), key, tensors.data(), count); + } + void set_frames(const char* key, const SDImage* value, size_t count) { + if (!count || status != SD_STATUS_OK) + return; + auto tensor = frames_tensor(value, count); + if (!tensor) { + status = SD_STATUS_INVALID_ARGUMENT; + return; + } + status = sd_opt_set_tensor(opt.get(), key, tensor.get()); + } + void set_floats(const char* key, const float* value, size_t count) { + if (status == SD_STATUS_OK) + status = sd_opt_set_floats(opt.get(), key, value, count); + } + void set_ints(const char* key, const int32_t* value, size_t count) { + if (status == SD_STATUS_OK) + status = sd_opt_set_ints(opt.get(), key, value, count); + } + void set_loras(const char* key, const sd_lora_t* value, size_t count) { + if (status == SD_STATUS_OK) + status = sd_opt_set_loras(opt.get(), key, value, count); + } + void set_embeddings(const char* key, const sd_embedding_t* value, size_t count) { + if (status == SD_STATUS_OK) + status = sd_opt_set_embeddings(opt.get(), key, value, count); + } + void set_audios(const char* key, const SDAudio* value, size_t count) { + if (status != SD_STATUS_OK) + return; + std::vector storage; + std::vector audios; + storage.reserve(count); + audios.reserve(count); + for (size_t i = 0; i < count; ++i) { + storage.push_back(audio_tensor(value[i])); + audios.push_back({value[i].sample_rate, storage.back().get()}); + } + status = sd_opt_set_audios(opt.get(), key, audios.data(), count); + } + void set_videos(const char* key, const SDReferenceVideo* value, size_t count) { + if (status != SD_STATUS_OK) + return; + std::vector storage; + std::vector videos; + storage.reserve(count * 2); + videos.reserve(count); + for (size_t i = 0; i < count; ++i) { + storage.push_back(frames_tensor(value[i].frames, value[i].frame_count)); + const auto* frames = storage.back().get(); + storage.push_back(audio_tensor(value[i].audio)); + videos.push_back({frames, value[i].fps, {value[i].audio.sample_rate, storage.back().get()}}); + } + status = sd_opt_set_videos(opt.get(), key, videos.data(), count); + } +}; + +static inline sd_ctx_t* create_sd_context(const SDOptPtr& opt) { + sd_ctx_t* ctx = nullptr; + if (opt) + sd_ctx_create(opt.get(), &ctx); + return ctx; +} + +static inline bool generate_sd_image(sd_ctx_t* ctx, const SDOptPtr& opt, sd_result_t** result) { + if (result) + *result = nullptr; + return opt && sd_generate_image(ctx, opt.get(), result) == SD_STATUS_OK; +} + +static inline bool generate_sd_video(sd_ctx_t* ctx, const SDOptPtr& opt, sd_result_t** result) { + if (result) + *result = nullptr; + return opt && sd_generate_video(ctx, opt.get(), result) == SD_STATUS_OK; +} + +static inline sd_upscaler_t* create_sd_upscaler(const char* path, std::optional direct, std::optional threads, std::optional tile, const char* backend, const char* params_backend) { + SDOptPtr opt(sd_upscaler_opt_create()); + if (!opt || sd_opt_set_string(opt.get(), "model_path", path) != SD_STATUS_OK || + (direct && sd_opt_set_bool(opt.get(), "conv_direct", *direct) != SD_STATUS_OK) || + (tile && sd_opt_set_int(opt.get(), "tile_size", *tile) != SD_STATUS_OK) || + (threads && sd_opt_set_int(opt.get(), "n_threads", *threads) != SD_STATUS_OK) || + (backend && sd_opt_set_string(opt.get(), "backend", backend) != SD_STATUS_OK) || + (params_backend && sd_opt_set_string(opt.get(), "params_backend", params_backend) != SD_STATUS_OK)) + return nullptr; + sd_upscaler_t* result = nullptr; + sd_upscaler_create(opt.get(), &result); + return result; +} + +static inline bool upscale_sd_image(sd_upscaler_t* ctx, SDImage image, sd_result_t** result) { + if (result) + *result = nullptr; + SDOptPtr opt(sd_upscale_opt_create()); + auto tensor = image_tensor(image); + return opt && tensor && sd_opt_set_tensor(opt.get(), "image", tensor.get()) == SD_STATUS_OK && + sd_upscale(ctx, opt.get(), result) == SD_STATUS_OK; +} + +static inline sd_adetailer_t* create_sd_adetailer(const char* path, std::optional threads, const char* backend, const char* params_backend) { + SDOptPtr opt(sd_adetailer_opt_create()); + if (!opt || sd_opt_set_string(opt.get(), "model_path", path) != SD_STATUS_OK || + (threads && sd_opt_set_int(opt.get(), "n_threads", *threads) != SD_STATUS_OK) || + (backend && sd_opt_set_string(opt.get(), "backend", backend) != SD_STATUS_OK) || + (params_backend && sd_opt_set_string(opt.get(), "params_backend", params_backend) != SD_STATUS_OK)) + return nullptr; + sd_adetailer_t* result = nullptr; + sd_adetailer_create(opt.get(), &result); + return result; +} + +static inline bool adetail_sd_image(sd_adetailer_t* detector, sd_ctx_t* ctx, SDImage image, SDOptPtr opt, sd_result_t** result) { + if (result) + *result = nullptr; + auto tensor = image_tensor(image); + return opt && tensor && sd_opt_set_tensor(opt.get(), "image", tensor.get()) == SD_STATUS_OK && + sd_adetail(detector, ctx, opt.get(), result) == SD_STATUS_OK; +} + +struct SDVideoUpscalerParams { + const char* diffusion_model_path{}; + const char* vae_path{}; + // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints. + int upscale_factor{}; + std::optional n_threads; + const char* backend{}; + const char* params_backend{}; + // auto/on/off or diffusion/vae assignments; null selects auto. + const char* flash_attn_mode{}; + const char* conv_direct_mode{}; +}; + +struct SDVideoUpscaleParams { + const SDImage* frames{}; + int frame_count{}; + std::optional fps; + std::optional seed; + // RelayVSR reference window, including both endpoints; defaults to 16. + std::optional window_size; + // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5. + std::optional temporal_chunk_size; +}; + +static inline sd_vid_upscaler_t* create_sd_vid_upscaler(const SDVideoUpscalerParams& config) { + SDOptPtr opt(sd_vid_upscaler_opt_create()); + if (!opt || sd_opt_set_string(opt.get(), "diffusion_model_path", config.diffusion_model_path) != SD_STATUS_OK || + sd_opt_set_string(opt.get(), "vae_path", config.vae_path) != SD_STATUS_OK || + sd_opt_set_int(opt.get(), "upscale_factor", config.upscale_factor) != SD_STATUS_OK || + (config.n_threads && sd_opt_set_int(opt.get(), "n_threads", *config.n_threads) != SD_STATUS_OK) || + (config.backend && sd_opt_set_string(opt.get(), "backend", config.backend) != SD_STATUS_OK) || + (config.params_backend && sd_opt_set_string(opt.get(), "params_backend", config.params_backend) != SD_STATUS_OK) || + (config.flash_attn_mode && sd_opt_set_string(opt.get(), "flash_attn_mode", config.flash_attn_mode) != SD_STATUS_OK) || + (config.conv_direct_mode && sd_opt_set_string(opt.get(), "conv_direct_mode", config.conv_direct_mode) != SD_STATUS_OK)) + return nullptr; + sd_vid_upscaler_t* result = nullptr; + sd_vid_upscaler_create(opt.get(), &result); + return result; +} + +static inline bool upscale_sd_video(sd_vid_upscaler_t* ctx, const SDVideoUpscaleParams& config, sd_result_t** result) { + if (result) + *result = nullptr; + SDOptBuilder opt(sd_vid_upscale_opt_create()); + opt.set_frames("frames", config.frames, config.frame_count); + opt.set_int("fps", config.fps); + opt.set_int("seed", config.seed); + opt.set_int("window_size", config.window_size); + opt.set_int("temporal_chunk_size", config.temporal_chunk_size); + auto params = opt.finish(); + return params && sd_upscale_video(ctx, params.get(), result) == SD_STATUS_OK; +} + +static inline SDImage take_result_image(sd_result_t* result, size_t index) { + const auto* tensor = sd_result_get_image(result, index); + auto image = tensor_image_view(tensor); + image.storage = sd_tensor_retain(tensor); + return image; +} class SDImageOwner { private: - static sd_image_t copy_image(const sd_image_t& image) { + static SDImage copy_image(const SDImage& image) { if (image.data == nullptr) { return {image.width, image.height, image.channel, nullptr}; } @@ -74,11 +465,12 @@ class SDImageOwner { return {image.width, image.height, image.channel, raw_copy}; } - sd_image_t image_ = {0, 0, 0, nullptr}; + SDImage image_ = {0, 0, 0, nullptr}; + SDTensorPtr public_tensor_; public: SDImageOwner() = default; - explicit SDImageOwner(sd_image_t image) + explicit SDImageOwner(SDImage image) : image_(image) { } @@ -109,9 +501,14 @@ class SDImageOwner { reset(); } - sd_image_t* put() { + const sd_tensor_t* tensor() { + public_tensor_ = image_tensor(image_); + return public_tensor_.get(); + } + + SDImage* put() { if (image_.data != nullptr) { - free(image_.data); + sd_image_free(&image_); image_.data = nullptr; } image_.width = 0; @@ -120,23 +517,23 @@ class SDImageOwner { return &image_; } - sd_image_t& get() { + SDImage& get() { return image_; } - const sd_image_t& get() const { + const SDImage& get() const { return image_; } - sd_image_t release() { - sd_image_t image = image_; - image_ = {0, 0, 0, nullptr}; + SDImage release() { + SDImage image = image_; + image_ = {0, 0, 0, nullptr}; return image; } - void reset(sd_image_t image = {0, 0, 0, nullptr}) { + void reset(SDImage image = {0, 0, 0, nullptr}) { if (image_.data != nullptr) { - free(image_.data); + sd_image_free(&image_); } image_ = image; } @@ -161,7 +558,7 @@ class SDAudioOwner { return samples_.empty(); } - sd_audio_t get() { + SDAudio get() { return {sample_rate_, channels_, channels_ == 0 ? 0 : static_cast(samples_.size() / channels_), @@ -175,7 +572,7 @@ class SDAudioOwner { class SDImageVec { private: - std::vector images_; + std::vector images_; std::vector generation_infos_; public: @@ -201,7 +598,7 @@ class SDImageVec { clear(); } - void push_back(sd_image_t image) { + void push_back(SDImage image) { images_.push_back(image); } @@ -223,32 +620,25 @@ class SDImageVec { generation_infos_.at(index) = merge_generation_info(generation_infos_.at(index), generation_info_with_relative_lora_paths(info, lora_model_dir)); } - void adopt(sd_image_generation_result_t* result, const std::string& lora_model_dir = {}) { + void adopt(sd_result_t* result, const std::string& lora_model_dir = {}) { clear(); + SDResultPtr owner(result); if (!result) return; - for (int i = 0; i < result->count; ++i) { - images_.push_back(result->items[i].image); - result->items[i].image = {}; - generation_infos_.push_back(generation_info_with_relative_lora_paths(result->items[i].generation_info, lora_model_dir)); + const size_t count = sd_result_get_image_count(result); + images_.reserve(count); + generation_infos_.reserve(count); + for (size_t i = 0; i < count; ++i) { + generation_infos_.push_back(generation_info_with_relative_lora_paths(sd_result_get_generation_info(result, i), lora_model_dir)); + images_.push_back(take_result_image(result, i)); } - free_sd_image_generation_result(result); } - void adopt_video(sd_video_generation_result_t* result, const std::string& lora_model_dir = {}) { - clear(); - if (!result) - return; - const auto info = generation_info_with_relative_lora_paths(result->generation_info, lora_model_dir); - for (int i = 0; i < result->frame_count; ++i) { - images_.push_back(result->frames[i]); - result->frames[i] = {}; - generation_infos_.push_back(info); - } - free_sd_video_generation_result(result); + void adopt_video(sd_result_t* result, const std::string& lora_model_dir = {}) { + adopt(result, lora_model_dir); } - void adopt(sd_image_t* images, int count) { + void adopt(SDImage* images, int count) { clear(); if (images == nullptr || count <= 0) { free(images); @@ -278,33 +668,33 @@ class SDImageVec { return !images_.empty(); } - sd_image_t* data() { + SDImage* data() { return images_.data(); } - const sd_image_t* data() const { + const SDImage* data() const { return images_.data(); } - sd_image_t& operator[](size_t index) { + SDImage& operator[](size_t index) { return images_[index]; } - const sd_image_t& operator[](size_t index) const { + const SDImage& operator[](size_t index) const { return images_[index]; } - std::vector& raw() { + std::vector& raw() { return images_; } - const std::vector& raw() const { + const std::vector& raw() const { return images_; } void clear() { - for (sd_image_t& image : images_) { - free(image.data); + for (SDImage& image : images_) { + sd_image_free(&image); image.data = nullptr; } images_.clear(); diff --git a/examples/server/README.md b/examples/server/README.md index 6b6e75052f..8673eb80c4 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -10,6 +10,16 @@ and HTTP APIs. Download or build the server before starting; see the - [API](#api): native and compatibility endpoints. - [Frontend development](#frontend-development): build or customize the Web UI. +Inference settings follow this priority: request fields, explicit server startup +options, model recipe presets, then [library defaults](../../docs/guides/c_api.md#keys). +Omitted request fields retain the configured override. In native JSON requests, +`null` clears an optional inference field so the library selects its value; +`0`, `false`, empty strings and empty numeric arrays are explicit values. +Capability `defaults` and `defaults_by_mode` contain configured overrides, with +`null` for optional settings left to the library. Effective settings are returned +in generation metadata. Output formats, queue limits and listen addresses keep +server-local defaults. + ## Web UI ### Start the server diff --git a/examples/server/api.md b/examples/server/api.md index dca1351b37..24419240a8 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -229,6 +229,9 @@ values reflect these declarations. A loaded recipe is snapshotted: refreshing recipe files does not change the active context's capabilities until reloading. Command-line preloading uses the runtime-only all-features recipe instead of model-family inference and component paths; variant-specific recipe declarations are more precise. +Known high-noise sampling settings and `moe_boundary` are ignored when no +high-noise stage is loaded, and `vace_strength` is ignored by non-VACE models. +Actual control-frame inputs still require VACE. These declarations govern the Web UI and advertised capabilities, not API request validation. Existing generation APIs continue accepting their existing @@ -1282,7 +1285,7 @@ and `ref_images[]`. RGB-only models drop it, so sending RGBA is safe for every m If omitted or null: -- single-image fields map to an empty `sd_image_t` +- single-image fields clear the corresponding image tensor - array fields map to an empty C-style array, represented as `pointer = nullptr` and `count = 0` ### Field Mapping Summary @@ -1645,7 +1648,7 @@ Frame ordering rules: If omitted or null: -- single-image fields map to an empty `sd_image_t` +- single-image fields clear the corresponding image tensor - array fields map to an empty C-style array, represented as `pointer = nullptr` and `count = 0` ### Field Mapping Summary @@ -1707,6 +1710,11 @@ Sampling fields: High-noise sampling fields: +The two video stages share the scheduler, flow shift and timestep shifting. +`high_noise_sample_params.scheduler`, `.flow_shift` and `.shifted_timestep` +are accepted as aliases for those shared settings. An explicitly supplied +corresponding `sample_params` field takes precedence. + | Field | Type | | --- | --- | | `high_noise_sample_params.scheduler` | `string` | diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp index 3513c4a8f8..bb88fb6aa5 100644 --- a/examples/server/async_jobs.cpp +++ b/examples/server/async_jobs.cpp @@ -43,14 +43,18 @@ class JobCancellationMonitor { } }; -static void preview_callback(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data) { +static void preview_callback(int step, int frame_count, const sd_tensor_t* const* tensors, bool is_noisy, void* data) { + std::vector views; + for (int i = 0; i < frame_count; ++i) + views.push_back(tensor_image_view(tensors[i])); + SDImage* frames = views.data(); (void)is_noisy; if (frame_count <= 0 || frames == nullptr || frames[0].data == nullptr) { return; } AsyncGenerationJob* job = static_cast(data); - const sd_image_t& frame = frames[0]; + const SDImage& frame = frames[0]; const sd_preview_info_t info = sd_get_preview_info(); auto image_bytes = encode_image_to_vector(EncodedImageFormat::PNG, @@ -73,23 +77,20 @@ static void preview_callback(int step, int frame_count, sd_image_t* frames, bool } static void clear_preview_callback() { - sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr); + sd_set_preview_callback(nullptr, "none", 1, false, false, nullptr); } static void set_preview_callback(AsyncGenerationJob& job) { - preview_t mode = str_to_preview(job.kind == AsyncJobKind::ImgGen - ? job.img_gen.preview_mode.c_str() - : job.vid_gen.preview_mode.c_str()); - if (mode == PREVIEW_COUNT || mode == PREVIEW_NONE) { + const auto& mode = job.kind == AsyncJobKind::ImgGen ? job.img_gen.preview_mode : job.vid_gen.preview_mode; + if (mode == "none") return; - } int interval = job.kind == AsyncJobKind::ImgGen ? job.img_gen.preview_interval : job.vid_gen.preview_interval; if (interval <= 0) { interval = 1; } - sd_set_preview_callback(preview_callback, mode, interval, true, false, &job); + sd_set_preview_callback(preview_callback, mode.c_str(), interval, true, false, &job); } static void update_job_progress(AsyncGenerationJob& job, const char* stage, const char* detail = "", int sample_pass = -1, int64_t completed = 0, int64_t total = 0, const char* unit = "") { @@ -343,8 +344,8 @@ bool execute_img_gen_job(ServerRuntime& runtime, std::vector& output_images, json& output_info, std::string& error_message) { - output_info = json::array(); - sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t(); + output_info = json::array(); + SDOptPtr params = job.img_gen.to_image_opt(); SDImageVec results; @@ -356,15 +357,15 @@ bool execute_img_gen_job(ServerRuntime& runtime, JobCancellationMonitor cancellation(runtime.sd_ctx, job); JobCallbacksGuard callbacks(runtime.sd_ctx, job); PromptImageCallbackGuard prompt_images(runtime.sd_ctx, job.img_gen.gen_params); - sd_image_generation_result_t* generated = nullptr; - generate_image(runtime.sd_ctx, ¶ms, &generated); + sd_result_t* generated = nullptr; + generate_sd_image(runtime.sd_ctx, params, &generated); results.adopt(generated, runtime.ctx_params->lora_model_dir); if (job.cancel_requested.load()) { return false; } if (!results.empty() && job.img_gen.adetailer_enabled && !runtime.default_gen_params->ad_model_path.empty()) { callbacks.begin_detailing(); - if (!apply_server_adetailer(runtime, job.img_gen, params, results)) { + if (!apply_server_adetailer(runtime, job.img_gen, results)) { error_message = "ADetailer failed"; return false; } @@ -424,12 +425,12 @@ bool execute_vid_gen_job(ServerRuntime& runtime, int& output_fps, json& output_info, std::string& error_message) { - sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t(); + SDOptPtr params = job.vid_gen.to_video_opt(); std::string str_params; SDImageVec results; - int num_results = 0; - sd_audio_t* generated_audio = nullptr; + int num_results = 0; + SDAudio* generated_audio = nullptr; { std::lock_guard lock(*runtime.sd_ctx_mutex); @@ -439,11 +440,11 @@ bool execute_vid_gen_job(ServerRuntime& runtime, JobCancellationMonitor cancellation(runtime.sd_ctx, job); JobCallbacksGuard callbacks(runtime.sd_ctx, job); PromptImageCallbackGuard prompt_images(runtime.sd_ctx, job.vid_gen.gen_params); - sd_video_generation_result_t* generated = nullptr; - if (generate_video(runtime.sd_ctx, ¶ms, &generated)) { - output_fps = generated->fps; - generated_audio = generated->audio; - generated->audio = nullptr; + sd_result_t* generated = nullptr; + if (generate_sd_video(runtime.sd_ctx, params, &generated)) { + output_fps = sd_result_get_fps(generated); + generated_audio = retain_result_audio(generated); + results.adopt_video(generated, runtime.ctx_params->lora_model_dir); output_info = json::parse(results.generation_info(0)); if (job.vid_gen.gen_params.embed_image_metadata) diff --git a/examples/server/main.cpp b/examples/server/main.cpp index 67d557a4e5..391a1ec9ad 100644 --- a/examples/server/main.cpp +++ b/examples/server/main.cpp @@ -52,8 +52,6 @@ static void parse_args(int argc, exit(ctx_params.print_devices() ? 0 : 1); } - const bool random_seed_requested = default_gen_params.seed < 0; - if (!svr_params.resolve_and_validate() || !ctx_params.resolve_and_validate(ctx_params.model_path.empty() && ctx_params.diffusion_model_path.empty() ? METADATA : IMG_GEN) || !default_gen_params.resolve_and_validate(IMG_GEN, @@ -62,10 +60,6 @@ static void parse_args(int argc, print_usage(argv[0], options_vec); exit(1); } - - if (random_seed_requested) { - default_gen_params.seed = -1; - } } void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) { @@ -93,16 +87,16 @@ int main(int argc, const char** argv) { LOG_DEBUG("%s", ctx_params.to_string().c_str()); LOG_DEBUG("%s", default_gen_params.to_string().c_str()); - sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(); + SDOptPtr sd_ctx_params = ctx_params.to_context_opt(); ServerModels models; if (!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) { const auto started = std::chrono::steady_clock::now(); - models.context.reset(new_sd_ctx(&sd_ctx_params)); + models.context.reset(create_sd_context(sd_ctx_params)); models.status["load_elapsed_seconds"] = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); } if ((!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) && !models.context) { - LOG_ERROR("new_sd_ctx_t failed"); + LOG_ERROR("sd_ctx_create failed"); return 1; } diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp index f665567096..bc264bca3e 100644 --- a/examples/server/model_catalog.cpp +++ b/examples/server/model_catalog.cpp @@ -193,19 +193,26 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para throw std::runtime_error(std::string(key) + " is out of range"); } if (std::string(key) == "width") - params.width = static_cast(value); + if (!params.width) + params.width = static_cast(value); if (std::string(key) == "height") - params.height = static_cast(value); + if (!params.height) + params.height = static_cast(value); if (std::string(key) == "steps") - params.sample_params.sample_steps = static_cast(value); + if (!params.sample_params.sample_steps) + params.sample_params.sample_steps = static_cast(value); if (field == "shifted_timestep") - params.sample_params.shifted_timestep = static_cast(value); + if (!params.sample_params.shifted_timestep) + params.sample_params.shifted_timestep = static_cast(value); if (field == "video_frames") - params.video_frames = static_cast(value); + if (!params.video_frames) + params.video_frames = static_cast(value); if (field == "fps") - params.fps = static_cast(value); + if (!params.fps) + params.fps = static_cast(value); if (field == "high_noise_steps") - params.high_noise_sample_params.sample_steps = static_cast(value); + if (!params.high_noise_sample_params.sample_steps) + params.high_noise_sample_params.sample_steps = static_cast(value); } for (const char* key : {"cfg_scale", "strength", "moe_boundary", "img_cfg_scale", "distilled_guidance", "flow_shift", "high_noise_cfg_scale", "high_noise_flow_shift"}) { if (!defaults.contains(key)) { @@ -219,21 +226,29 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para throw std::runtime_error(std::string(key) + " is out of range"); } if (std::string(key) == "strength") - params.strength = value; + if (!params.strength) + params.strength = value; if (std::string(key) == "moe_boundary") - params.moe_boundary = value; + if (!params.moe_boundary) + params.moe_boundary = value; if (std::string(key) == "cfg_scale") - params.sample_params.guidance.txt_cfg = value; + if (!params.sample_params.guidance.txt_cfg) + params.sample_params.guidance.txt_cfg = value; if (std::string(key) == "img_cfg_scale") - params.sample_params.guidance.img_cfg = value; + if (!params.sample_params.guidance.img_cfg) + params.sample_params.guidance.img_cfg = value; if (std::string(key) == "distilled_guidance") - params.sample_params.guidance.distilled_guidance = value; + if (!params.sample_params.guidance.distilled_guidance) + params.sample_params.guidance.distilled_guidance = value; if (std::string(key) == "flow_shift") - params.sample_params.flow_shift = value; + if (!params.sample_params.flow_shift) + params.sample_params.flow_shift = value; if (std::string(key) == "high_noise_cfg_scale") - params.high_noise_sample_params.guidance.txt_cfg = value; + if (!params.high_noise_sample_params.guidance.txt_cfg) + params.high_noise_sample_params.guidance.txt_cfg = value; if (std::string(key) == "high_noise_flow_shift") - params.high_noise_sample_params.flow_shift = value; + if (!params.sample_params.flow_shift) + params.sample_params.flow_shift = value; } if (defaults.contains("custom_sigmas")) { const auto& sigmas = defaults.at("custom_sigmas"); @@ -243,40 +258,28 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para if (!sigma.is_number() || !std::isfinite(sigma.get()) || sigma.get() < 0) throw std::runtime_error("custom_sigmas must contain finite nonnegative numbers"); } - params.custom_sigmas = sigmas.get>(); + if (!params.custom_sigmas) + params.custom_sigmas = sigmas.get>(); } for (const char* key : {"sampling_method", "high_noise_sampling_method"}) { if (!defaults.contains(key)) continue; auto& sample = std::string(key) == "sampling_method" ? params.sample_params : params.high_noise_sample_params; const auto name = defaults.at(key).get(); - bool found = false; - for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) { - if (name == sd_sample_method_name(static_cast(i))) { - sample.sample_method = static_cast(i); - found = true; - break; - } - } - if (!found) + if (!example_options::contains(example_options::sample_methods, name)) throw std::runtime_error(std::string("unknown ") + key); + if (!sample.sample_method) + sample.sample_method = name; } for (const char* key : {"scheduler", "high_noise_scheduler"}) { if (!defaults.contains(key)) continue; - auto& sample = std::string(key) == "scheduler" ? params.sample_params : params.high_noise_sample_params; + auto& sample = params.sample_params; const auto name = defaults.at(key).get(); - bool found = false; - for (int i = 0; i < SCHEDULER_COUNT; ++i) { - if (name == sd_scheduler_name(static_cast(i)) || - (name == "normal" && i == DISCRETE_SCHEDULER)) { - sample.scheduler = static_cast(i); - found = true; - break; - } - } - if (!found) + if (!example_options::contains(example_options::schedulers, name) && name != "normal") throw std::runtime_error(std::string("unknown ") + key); + if (!sample.scheduler) + sample.scheduler = name; } } @@ -286,16 +289,20 @@ static void apply_recipe_context(const json& settings, SDContextParams& params) const auto format = settings.at("vae_format").get(); if (format != "auto" && format != "flux" && format != "sd3" && format != "flux2" && format != "wan") throw std::runtime_error("unknown vae_format"); - params.vae_format = format; + if (!params.vae_format) + params.vae_format = format; } if (settings.contains("model_args")) { - params.model_args = settings.at("model_args").get(); + if (!params.model_args) + params.model_args = settings.at("model_args").get(); } if (settings.contains("backend")) { - params.backend = settings.at("backend").get(); + if (!params.backend) + params.backend = settings.at("backend").get(); } if (settings.contains("params_backend")) { - params.params_backend = settings.at("params_backend").get(); + if (!params.params_backend) + params.params_backend = settings.at("params_backend").get(); } } diff --git a/examples/server/routes_models.cpp b/examples/server/routes_models.cpp index f33057758c..9dbd6ed761 100644 --- a/examples/server/routes_models.cpp +++ b/examples/server/routes_models.cpp @@ -112,8 +112,8 @@ static void change_model(ServerRuntime& runtime, const httplib::Request& req, ht *runtime.ctx_params = unload ? models.base_context : std::move(context); *runtime.default_gen_params = unload ? models.base_defaults : std::move(defaults); if (!unload) { - auto params = runtime.ctx_params->to_sd_ctx_params_t(); - models.context.reset(new_sd_ctx(¶ms)); + auto params = runtime.ctx_params->to_context_opt(); + models.context.reset(create_sd_context(params)); if (!models.context) { throw std::runtime_error("Model initialization failed; check the server log for component compatibility and memory errors"); } diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp index 30ca7f3285..e67bd20159 100644 --- a/examples/server/routes_openai.cpp +++ b/examples/server/routes_openai.cpp @@ -35,8 +35,8 @@ static bool build_openai_generation_request(const httplib::Request& req, std::string size = j.value("size", ""); std::string output_format = j.value("output_format", "png"); int output_compression = j.value("output_compression", 100); - int width = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->width : 512; - int height = runtime.default_gen_params->height > 0 ? runtime.default_gen_params->height : 512; + auto width = runtime.default_gen_params->width; + auto height = runtime.default_gen_params->height; if (!size.empty()) { auto pos = size.find('x'); if (pos != std::string::npos) { @@ -120,8 +120,8 @@ static bool build_openai_edit_request(const httplib::Request& req, } std::string size = req.form.get_field("size"); - int width = -1; - int height = -1; + auto width = runtime.default_gen_params->width; + auto height = runtime.default_gen_params->height; if (!size.empty()) { auto pos = size.find('x'); if (pos != std::string::npos) { @@ -185,12 +185,12 @@ static bool build_openai_edit_request(const httplib::Request& req, mask_w, mask_h, mask_channel, 0, 0, 1); request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw}); - const sd_image_t& mask_image = request.gen_params.mask_image.get(); + const SDImage& mask_image = request.gen_params.mask_image.get(); request.gen_params.set_width_and_height_if_unset(mask_image.width, mask_image.height); } else { request.gen_params.mask_image.reset({ - (uint32_t)request.gen_params.get_resolved_width(), - (uint32_t)request.gen_params.get_resolved_height(), + (uint32_t)request.gen_params.get_requested_width(), + (uint32_t)request.gen_params.get_requested_height(), 1, nullptr, }); @@ -218,17 +218,17 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime, ImgGenJobRequest& request, SDImageVec& results, std::string& error_message) { - sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t(); - int num_results = 0; + SDOptPtr img_gen_params = request.to_image_opt(); + int num_results = 0; { std::lock_guard lock(*runtime.sd_ctx_mutex); PromptImageCallbackGuard prompt_images(runtime.sd_ctx, request.gen_params); - sd_image_generation_result_t* generated = nullptr; - generate_image(runtime.sd_ctx, &img_gen_params, &generated); + sd_result_t* generated = nullptr; + generate_sd_image(runtime.sd_ctx, img_gen_params, &generated); results.adopt(generated, runtime.ctx_params->lora_model_dir); num_results = results.count(); - if (!results.empty() && !apply_server_adetailer(runtime, request, img_gen_params, results)) { + if (!results.empty() && !apply_server_adetailer(runtime, request, results)) { error_message = "ADetailer failed"; return false; } diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp index 26ed7c9709..e9f60c7555 100644 --- a/examples/server/routes_sdapi.cpp +++ b/examples/server/routes_sdapi.cpp @@ -47,41 +47,38 @@ static std::string lower_ascii(std::string value) { return value; } -static enum sample_method_t get_sdapi_sample_method(std::string name) { - enum sample_method_t result = str_to_sample_method(name.c_str()); - if (result != SAMPLE_METHOD_COUNT) { - return result; - } - +static std::string get_sdapi_sample_method(std::string name) { + if (example_options::contains(example_options::sample_methods, name)) + return name; name = lower_ascii(name); - static const std::unordered_map hardcoded{ - {"euler a", EULER_A_SAMPLE_METHOD}, - {"k_euler_a", EULER_A_SAMPLE_METHOD}, - {"euler", EULER_SAMPLE_METHOD}, - {"k_euler", EULER_SAMPLE_METHOD}, - {"heun", HEUN_SAMPLE_METHOD}, - {"k_heun", HEUN_SAMPLE_METHOD}, - {"dpm2", DPM2_SAMPLE_METHOD}, - {"k_dpm_2", DPM2_SAMPLE_METHOD}, - {"lcm", LCM_SAMPLE_METHOD}, - {"ddim", DDIM_TRAILING_SAMPLE_METHOD}, - {"dpm++ 2m", DPMPP2M_SAMPLE_METHOD}, - {"k_dpmpp_2m", DPMPP2M_SAMPLE_METHOD}, - {"dpm++ 2m sde", DPMPP2M_SDE_SAMPLE_METHOD}, - {"k_dpmpp_2m_sde", DPMPP2M_SDE_SAMPLE_METHOD}, - {"dpm++ 2m sde gpu", DPMPP2M_SDE_BT_SAMPLE_METHOD}, - {"k_dpmpp_2m_sde_gpu", DPMPP2M_SDE_BT_SAMPLE_METHOD}, - {"res multistep", RES_MULTISTEP_SAMPLE_METHOD}, - {"k_res_multistep", RES_MULTISTEP_SAMPLE_METHOD}, - {"res 2s", RES_2S_SAMPLE_METHOD}, - {"k_res_2s", RES_2S_SAMPLE_METHOD}, - {"euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD}, - {"k_euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD}, - {"euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD}, - {"k_euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD}, + static const std::unordered_map hardcoded{ + {"euler a", "euler_a"}, + {"k_euler_a", "euler_a"}, + {"euler", "euler"}, + {"k_euler", "euler"}, + {"heun", "heun"}, + {"k_heun", "heun"}, + {"dpm2", "dpm2"}, + {"k_dpm_2", "dpm2"}, + {"lcm", "lcm"}, + {"ddim", "ddim_trailing"}, + {"dpm++ 2m", "dpm++2m"}, + {"k_dpmpp_2m", "dpm++2m"}, + {"dpm++ 2m sde", "dpm++2m_sde"}, + {"k_dpmpp_2m_sde", "dpm++2m_sde"}, + {"dpm++ 2m sde gpu", "dpm++2m_sde_bt"}, + {"k_dpmpp_2m_sde_gpu", "dpm++2m_sde_bt"}, + {"res multistep", "res_multistep"}, + {"k_res_multistep", "res_multistep"}, + {"res 2s", "res_2s"}, + {"k_res_2s", "res_2s"}, + {"euler_cfg_pp", "euler_cfg_pp"}, + {"k_euler_cfg_pp", "euler_cfg_pp"}, + {"euler_a_cfg_pp", "euler_a_cfg_pp"}, + {"k_euler_a_cfg_pp", "euler_a_cfg_pp"}, }; auto it = hardcoded.find(name); - return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT; + return it != hardcoded.end() ? it->second : std::string{}; } static bool parse_sdapi_adetailer(const json& body, @@ -191,10 +188,11 @@ static bool parse_sdapi_adetailer(const json& body, ServerADetailerPass pass; pass.prompt = text("ad_prompt", ""); pass.negative_prompt = text("ad_negative_prompt", ""); - pass.width = request.gen_params.get_resolved_width(); - pass.height = request.gen_params.get_resolved_height(); - pass.strength = static_cast(number("ad_denoising_strength", 0.4, 0, 1)); - auto append = [&](const char* key, const std::string& value) { + pass.width = request.gen_params.width; + pass.height = request.gen_params.height; + if (config.contains("ad_denoising_strength")) + pass.strength = static_cast(number("ad_denoising_strength", 0, 0, 1)); + auto append = [&](const char* key, const std::string& value) { if (!pass.extra_args.empty()) pass.extra_args += ","; pass.extra_args += std::string(key) + "=" + value; @@ -218,7 +216,7 @@ static bool parse_sdapi_adetailer(const json& body, append("mask_min_ratio", json(min_ratio).dump()); append("mask_max_ratio", json(max_ratio).dump()); // Bound pixel operations before converting them to the native mask parameters. - const int mask_limit = std::clamp(std::max(pass.width, pass.height), 32, 16384); + const int mask_limit = k_max_generation_dimension; append_integer("dilate_erode", "ad_dilate_erode", 4, -mask_limit, mask_limit); append_integer("x_offset", "ad_x_offset", 0, -mask_limit, mask_limit); append_integer("y_offset", "ad_y_offset", 0, -mask_limit, mask_limit); @@ -231,8 +229,12 @@ static bool parse_sdapi_adetailer(const json& body, append("mask_mode", merge == "None" ? "none" : merge == "Merge" ? "merge" : "merge_invert"); if (boolean("ad_use_inpaint_width_height", false)) { - pass.width = integer("ad_inpaint_width", 512, 1, k_max_generation_dimension); - pass.height = integer("ad_inpaint_height", 512, 1, k_max_generation_dimension); + pass.width.reset(); + pass.height.reset(); + if (config.contains("ad_inpaint_width")) + pass.width = integer("ad_inpaint_width", 0, 1, k_max_generation_dimension); + if (config.contains("ad_inpaint_height")) + pass.height = integer("ad_inpaint_height", 0, 1, k_max_generation_dimension); } if (boolean("ad_use_steps", false)) append_integer("steps", "ad_steps", 28, 1, k_max_adetailer_steps); @@ -251,20 +253,20 @@ static bool parse_sdapi_adetailer(const json& body, if (legacy_karras) sampler.resize(sampler.size() - suffix.size()); const auto method = get_sdapi_sample_method(sampler); - if (method == SAMPLE_METHOD_COUNT) + if (method.empty()) throw std::invalid_argument("unsupported ADetailer sampler"); - append("sample_method", sd_sample_method_name(method)); + append("sample_method", method); if (legacy_karras && text("ad_scheduler", "Use same scheduler") == "Use same scheduler") - append("scheduler", sd_scheduler_name(KARRAS_SCHEDULER)); + append("scheduler", "karras"); } const auto scheduler = text("ad_scheduler", "Use same scheduler"); if (scheduler == "Automatic") { pass.automatic_scheduler = true; } else if (scheduler != "Use same scheduler") { - const auto value = str_to_scheduler(scheduler.c_str()); - if (value == SCHEDULER_COUNT) + const auto value = scheduler; + if (!example_options::contains(example_options::schedulers, value) && value != "normal") throw std::invalid_argument("unsupported ADetailer scheduler"); - append("scheduler", sd_scheduler_name(value)); + append("scheduler", value == "normal" ? "discrete" : value); } } request.adetailer_passes.push_back(std::move(pass)); @@ -287,21 +289,9 @@ static bool build_sdapi_img_gen_request(const json& j, std::string& error_message) { std::string prompt = j.value("prompt", ""); std::string negative_prompt = j.value("negative_prompt", ""); - int width = j.value("width", 512); - int height = j.value("height", 512); - int steps = j.value("steps", runtime.default_gen_params->sample_params.sample_steps); - float cfg_scale = j.value("cfg_scale", runtime.default_gen_params->sample_params.guidance.txt_cfg); - int64_t seed = j.value("seed", -1); - int batch_size = j.value("batch_size", 1); - int clip_skip = j.value("clip_skip", -1); std::string sampler_name = j.value("sampler_name", ""); std::string scheduler_name = j.value("scheduler", ""); - if (width <= 0 || height <= 0) { - error_message = "width and height must be positive"; - return false; - } - if (prompt.empty()) { error_message = "prompt required"; return false; @@ -309,15 +299,20 @@ static bool build_sdapi_img_gen_request(const json& j, request.gen_params = *runtime.default_gen_params; - request.gen_params.prompt = prompt; - request.gen_params.negative_prompt = negative_prompt; - request.gen_params.seed = seed; - request.gen_params.sample_params.sample_steps = steps; - request.gen_params.batch_count = batch_size; - request.gen_params.sample_params.guidance.txt_cfg = cfg_scale; - const bool recipe_dimensions = !img2img && model_status(runtime)["recipe"].is_string(); - request.gen_params.width = j.value("width", recipe_dimensions ? runtime.default_gen_params->width : -1); - request.gen_params.height = j.value("height", recipe_dimensions ? runtime.default_gen_params->height : -1); + request.gen_params.prompt = prompt; + if (j.contains("negative_prompt")) + request.gen_params.negative_prompt = negative_prompt; + auto load_input = [&](const char* key, auto& value) { + if (j.contains(key)) + j.at(key).get_to(value); + }; + load_input("seed", request.gen_params.seed); + load_input("steps", request.gen_params.sample_params.sample_steps); + load_input("batch_size", request.gen_params.batch_count); + load_input("cfg_scale", request.gen_params.sample_params.guidance.txt_cfg); + load_input("width", request.gen_params.width); + load_input("height", request.gen_params.height); + load_input("clip_skip", request.gen_params.clip_skip); if (!img2img && j.value("enable_hr", false)) { request.gen_params.hires_enabled = true; @@ -337,17 +332,13 @@ static bool build_sdapi_img_gen_request(const json& j, return false; } - if (clip_skip > 0) { - request.gen_params.clip_skip = clip_skip; - } - - enum sample_method_t sample_method = get_sdapi_sample_method(sampler_name); - if (sample_method != SAMPLE_METHOD_COUNT) { + const auto sample_method = get_sdapi_sample_method(sampler_name); + if (!sample_method.empty()) { request.gen_params.sample_params.sample_method = sample_method; } - enum scheduler_t scheduler = str_to_scheduler(scheduler_name.c_str()); - if (scheduler != SCHEDULER_COUNT) { + const auto scheduler = scheduler_name; + if (example_options::contains(example_options::schedulers, scheduler) || scheduler == "normal") { request.gen_params.sample_params.scheduler = scheduler; } @@ -404,7 +395,7 @@ static bool build_sdapi_img_gen_request(const json& j, 0, 0, request.gen_params.init_image)) { - const sd_image_t& image = request.gen_params.init_image.get(); + const SDImage& image = request.gen_params.init_image.get(); request.gen_params.set_width_and_height_if_unset(image.width, image.height); } } @@ -415,10 +406,10 @@ static bool build_sdapi_img_gen_request(const json& j, 0, 0, request.gen_params.mask_image)) { - const sd_image_t& image = request.gen_params.mask_image.get(); + const SDImage& image = request.gen_params.mask_image.get(); request.gen_params.set_width_and_height_if_unset(image.width, image.height); } - sd_image_t& mask_image = request.gen_params.mask_image.get(); + SDImage& mask_image = request.gen_params.mask_image.get(); bool inpainting_mask_invert = j.value("inpainting_mask_invert", 0) != 0; if (inpainting_mask_invert && mask_image.data != nullptr) { for (uint32_t i = 0; i < mask_image.width * mask_image.height; ++i) { @@ -445,7 +436,7 @@ static bool build_sdapi_img_gen_request(const json& j, 0, 0, 0, image_owner)) { - const sd_image_t& image = image_owner.get(); + const SDImage& image = image_owner.get(); request.gen_params.set_width_and_height_if_unset(image.width, image.height); request.gen_params.ref_images.push_back(std::move(image_owner)); } @@ -499,14 +490,18 @@ struct SDAPIPreviewState { static SDAPIPreviewState g_preview; -static void preview_callback(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data) { +static void preview_callback(int step, int frame_count, const sd_tensor_t* const* tensors, bool is_noisy, void* data) { + std::vector views; + for (int i = 0; i < frame_count; ++i) + views.push_back(tensor_image_view(tensors[i])); + SDImage* frames = views.data(); (void)is_noisy; (void)data; if (frame_count <= 0 || frames == nullptr || frames[0].data == nullptr) { return; } - const sd_image_t& frame = frames[0]; + const SDImage& frame = frames[0]; const sd_preview_info_t info = sd_get_preview_info(); auto image_bytes = encode_image_to_vector(EncodedImageFormat::JPEG, @@ -530,7 +525,7 @@ static void preview_callback(int step, int frame_count, sd_image_t* frames, bool } struct SDAPIPreviewGuard { - SDAPIPreviewGuard(const std::string& id_task, const sd_img_gen_params_t& params, const std::string& job_type, bool detailing = false) { + SDAPIPreviewGuard(const std::string& id_task, const SDGenerationParams& params, const std::string& job_type, bool detailing = false) { std::lock_guard lock(g_preview.mutex); g_preview.id_task = id_task; g_preview.total = 0; @@ -539,13 +534,13 @@ struct SDAPIPreviewGuard { g_preview.step = 0; g_preview.active = true; // Each batch item has a base pass and, when enabled, a hires pass. - g_preview.job_count = static_cast(std::max(1, params.batch_count)) * (params.hires.enabled ? 2 : 1) + (detailing ? 1 : 0); + g_preview.job_count = static_cast(std::max(1, params.batch_count.value_or(1))) * (params.hires_enabled.value_or(false) ? 2 : 1) + (detailing ? 1 : 0); g_preview.job_no = 0; - sd_set_preview_callback(preview_callback, PREVIEW_PROJ, 1, true, false, nullptr); + sd_set_preview_callback(preview_callback, "proj", 1, true, false, nullptr); } void begin_detailing() { - sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr); + sd_set_preview_callback(nullptr, "none", 1, false, false, nullptr); std::lock_guard lock(g_preview.mutex); g_preview.job_type = "adetailer"; g_preview.job_no = static_cast(g_preview.job_count - 1); @@ -554,7 +549,7 @@ struct SDAPIPreviewGuard { } ~SDAPIPreviewGuard() { - sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr); + sd_set_preview_callback(nullptr, "none", 1, false, false, nullptr); std::lock_guard lock(g_preview.mutex); g_preview.active = false; g_preview.b64.clear(); @@ -594,7 +589,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { LOG_DEBUG("%s\n", request.gen_params.to_string().c_str()); - sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t(); + SDOptPtr img_gen_params = request.to_image_opt(); SDImageVec results; int num_results = 0; @@ -602,14 +597,14 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { std::lock_guard lock(*runtime->sd_ctx_mutex); PromptImageCallbackGuard prompt_images(runtime->sd_ctx, request.gen_params); const bool detailing = request.adetailer_enabled && !runtime->default_gen_params->ad_model_path.empty(); - SDAPIPreviewGuard preview_guard(id_task, img_gen_params, img2img ? "img2img" : "txt2img", detailing); - sd_image_generation_result_t* generated = nullptr; - generate_image(runtime->sd_ctx, &img_gen_params, &generated); + SDAPIPreviewGuard preview_guard(id_task, request.gen_params, img2img ? "img2img" : "txt2img", detailing); + sd_result_t* generated = nullptr; + generate_sd_image(runtime->sd_ctx, img_gen_params, &generated); results.adopt(generated, runtime->ctx_params->lora_model_dir); num_results = results.count(); if (detailing && !results.empty()) preview_guard.begin_detailing(); - if (!results.empty() && !apply_server_adetailer(*runtime, request, img_gen_params, results)) { + if (!results.empty() && !apply_server_adetailer(*runtime, request, results)) { res.status = 500; res.set_content(R"({"error":"ADetailer failed"})", "application/json"); return; @@ -748,8 +743,8 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { routes.get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) { std::vector sampler_names; sampler_names.push_back("default"); - for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) { - sampler_names.push_back(sd_sample_method_name((sample_method_t)i)); + for (const char* name : example_options::sample_methods) { + sampler_names.push_back(name); } json r = json::array(); for (auto name : sampler_names) { @@ -765,9 +760,9 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { routes.get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) { std::vector scheduler_names; scheduler_names.push_back("default"); - for (int i = 0; i < SCHEDULER_COUNT; i++) { - scheduler_names.push_back(sd_scheduler_name((scheduler_t)i)); - if (i == DISCRETE_SCHEDULER) { + for (const std::string name : example_options::schedulers) { + scheduler_names.push_back(name); + if (name == "discrete") { scheduler_names.push_back("normal"); } } diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index 1d11b89adc..a0cc075eb1 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -15,7 +15,7 @@ namespace fs = std::filesystem; static constexpr uint32_t k_max_upscale_dimension = 8192; -static bool valid_upscale_dimensions(const sd_image_t& image, int factor, int repeats) { +static bool valid_upscale_dimensions(const SDImage& image, int factor, int repeats) { if (image.width == 0 || image.height == 0 || factor < 1 || repeats < 1 || repeats > 4) { return false; } @@ -31,51 +31,20 @@ static bool valid_upscale_dimensions(const sd_image_t& image, int factor, int re return true; } -static bool parse_cache_mode(const std::string& mode_str, sd_cache_mode_t& mode_out) { - if (mode_str == "disabled") { - mode_out = SD_CACHE_DISABLED; - return true; - } - if (mode_str == "easycache") { - mode_out = SD_CACHE_EASYCACHE; - return true; - } - if (mode_str == "ucache") { - mode_out = SD_CACHE_UCACHE; - return true; - } - if (mode_str == "dbcache") { - mode_out = SD_CACHE_DBCACHE; - return true; - } - if (mode_str == "taylorseer") { - mode_out = SD_CACHE_TAYLORSEER; - return true; - } - if (mode_str == "cache-dit") { - mode_out = SD_CACHE_CACHE_DIT; - return true; - } - if (mode_str == "spectrum") { - mode_out = SD_CACHE_SPECTRUM; - return true; - } - return false; -} - -static json finite_number_or_null(float value) { - return std::isfinite(value) ? json(value) : json(nullptr); -} - -static const char* capability_scheduler_name(enum scheduler_t scheduler) { - return scheduler < SCHEDULER_COUNT ? sd_scheduler_name(scheduler) : "default"; +static bool parse_cache_mode(const std::string& value, std::string& mode) { + const std::string canonical = value == "cache-dit" ? "cache_dit" : value; + SDOptPtr opt(sd_img_gen_opt_create()); + if (!opt || sd_opt_set_string(opt.get(), "cache.mode", canonical.c_str()) != SD_STATUS_OK) + return false; + mode = canonical; + return true; } -static const char* capability_sample_method_name(enum sample_method_t sample_method) { - return sample_method < SAMPLE_METHOD_COUNT ? sd_sample_method_name(sample_method) : "default"; +static json finite_number_or_null(const std::optional& value) { + return value && std::isfinite(*value) ? json(*value) : json(nullptr); } -static json make_vae_tiling_json(const sd_tiling_params_t& params) { +static json make_vae_tiling_json(const SDTilingParams& params) { return { {"enabled", params.enabled}, {"temporal_tiling", params.temporal_tiling}, @@ -99,11 +68,11 @@ static fs::path resolve_display_model_path(const ServerRuntime& runtime) { return {}; } -static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector& skip_layers, const std::vector& custom_sigmas = {}) { +static json make_sample_params_json(const SDSampleParams& sample_params, const std::optional>& skip_layers, const std::optional>& custom_sigmas = {}) { const auto& guidance = sample_params.guidance; return { - {"scheduler", capability_scheduler_name(sample_params.scheduler)}, - {"sample_method", capability_sample_method_name(sample_params.sample_method)}, + {"scheduler", sample_params.scheduler}, + {"sample_method", sample_params.sample_method}, {"sample_steps", sample_params.sample_steps}, {"custom_sigmas", custom_sigmas}, {"eta", finite_number_or_null(sample_params.eta)}, @@ -144,8 +113,8 @@ static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const {"prompt", defaults.prompt}, {"negative_prompt", defaults.negative_prompt}, {"clip_skip", defaults.clip_skip}, - {"width", defaults.width > 0 ? defaults.width : 512}, - {"height", defaults.height > 0 ? defaults.height : 512}, + {"width", defaults.width}, + {"height", defaults.height}, {"strength", defaults.strength}, {"seed", defaults.seed}, {"batch_count", defaults.batch_count}, @@ -173,8 +142,8 @@ static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const {"prompt", defaults.prompt}, {"negative_prompt", defaults.negative_prompt}, {"clip_skip", defaults.clip_skip}, - {"width", defaults.width > 0 ? defaults.width : 512}, - {"height", defaults.height > 0 ? defaults.height : 512}, + {"width", defaults.width}, + {"height", defaults.height}, {"strength", defaults.strength}, {"seed", defaults.seed}, {"video_frames", defaults.video_frames}, @@ -234,8 +203,8 @@ static json make_vid_gen_features_json() { static json make_preview_modes_json() { json modes = json::array(); - for (int i = 0; i < PREVIEW_COUNT; ++i) { - modes.push_back(sd_preview_name((preview_t)i)); + for (const char* name : example_options::preview_modes) { + modes.push_back(name); } return modes; } @@ -258,13 +227,13 @@ static json make_capabilities_json(ServerRuntime& runtime) { json available_upscalers = json::array(); json supported_modes = json::array(); - for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) { - samplers.push_back(sd_sample_method_name((sample_method_t)i)); + for (const char* name : example_options::sample_methods) { + samplers.push_back(name); } - for (int i = 0; i < SCHEDULER_COUNT; ++i) { - schedulers.push_back(sd_scheduler_name((scheduler_t)i)); - if (i == DISCRETE_SCHEDULER) { + for (const std::string name : example_options::schedulers) { + schedulers.push_back(name); + if (name == "discrete") { schedulers.push_back("normal"); } } @@ -546,7 +515,9 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { res.set_content(json({{"error", error_message}}).dump(), "application/json"); return; } - const int tile_size = std::max(32, body.value("tile_size", runtime->default_gen_params->upscale_tile_size)); + const auto tile_size = body.value("tile_size", runtime->default_gen_params->upscale_tile_size); + if (tile_size && *tile_size < 32) + throw std::invalid_argument("tile_size must be at least 32"); const int repeats = std::clamp(body.value("repeats", 1), 1, 4); const std::string wanted = body.value("upscaler", std::string()); @@ -597,18 +568,18 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { // One GPU: an upscale must not run while a generation is using it. std::lock_guard ctx_lock(*runtime->sd_ctx_mutex); - UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(model_path.c_str(), - runtime->ctx_params->diffusion_conv_direct, - runtime->ctx_params->n_threads, - tile_size, - runtime->ctx_params->backend.c_str(), - runtime->ctx_params->params_backend.c_str())); + UpscalerCtxPtr upscaler_ctx(create_sd_upscaler(model_path.c_str(), + runtime->ctx_params->diffusion_conv_direct, + runtime->ctx_params->n_threads, + tile_size, + runtime->ctx_params->backend ? runtime->ctx_params->backend->c_str() : nullptr, + runtime->ctx_params->params_backend ? runtime->ctx_params->params_backend->c_str() : nullptr)); if (upscaler_ctx == nullptr) { res.status = 500; res.set_content(R"({"error":"the upscaler model could not be loaded"})", "application/json"); return; } - const int factor = get_upscale_factor(upscaler_ctx.get()); + const int factor = sd_upscaler_get_scale(upscaler_ctx.get()); // The model file may have changed since its metadata was cached. if (!valid_upscale_dimensions(input.get(), factor, repeats)) { res.status = 400; @@ -619,24 +590,24 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { SDImageOwner current(input.release()); std::string generation_info; for (int i = 0; i < repeats; ++i) { - sd_image_generation_result_t* generated = nullptr; - if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &generated) || - !generated || generated->count <= 0 || generated->items[0].image.data == nullptr) { - free_sd_image_generation_result(generated); + sd_result_t* generated = nullptr; + if (!upscale_sd_image(upscaler_ctx.get(), current.get(), &generated) || + !generated || sd_result_get_image_count(generated) <= 0 || sd_tensor_data(sd_result_get_image(generated, 0)) == nullptr) { + sd_result_free(generated); res.status = 500; res.set_content(R"({"error":"upscale failed"})", "application/json"); return; } - generation_info = merge_generation_info(generation_info, generated->items[0].generation_info); - sd_image_t produced = generated->items[0].image; - generated->items[0].image = {}; - free_sd_image_generation_result(generated); + generation_info = merge_generation_info(generation_info, sd_result_get_generation_info(generated, 0)); + SDImage produced = take_result_image(generated, 0); + + sd_result_free(generated); current.reset(produced); } const std::string& format = output_options.output_format; const int compression = output_options.output_compression; - const sd_image_t result = current.get(); + const SDImage result = current.get(); auto image_bytes = encode_image_to_vector(format == "jpeg" ? EncodedImageFormat::JPEG : format == "webp" ? EncodedImageFormat::WEBP : EncodedImageFormat::PNG, diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp index 499693a929..6ac3866c77 100644 --- a/examples/server/runtime.cpp +++ b/examples/server/runtime.cpp @@ -19,15 +19,17 @@ namespace fs = std::filesystem; -bool PromptImageCallbackGuard::load_image(const char* source, void* data, sd_image_t* image_out) { - const auto& params = static_cast(data)->params_; +bool PromptImageCallbackGuard::load_image(const char* source, void* data, const sd_tensor_t** image_out) { + auto& guard = *static_cast(data); + const auto& params = guard.params_; const auto it = params.prompt_images.find(source); if (it == params.prompt_images.end()) { LOG_ERROR("Unknown prompt image ID: %s", source); return false; } - *image_out = it->second.get(); - return image_out->data != nullptr; + guard.image_tensor_ = image_tensor(it->second.get()); + *image_out = guard.image_tensor_.get(); + return *image_out != nullptr; } PromptImageCallbackGuard::PromptImageCallbackGuard(sd_ctx_t* ctx, const SDGenerationParams& params) @@ -42,34 +44,35 @@ PromptImageCallbackGuard::~PromptImageCallbackGuard() { bool validate_server_img_gen_request(const ServerRuntime& runtime, const ImgGenJobRequest& request, std::string& error_message) { - auto dimensions_valid = [&](double width, double height) { - if (!std::isfinite(width) || !std::isfinite(height) || - width < 1 || height < 1 || width > k_max_generation_dimension || height > k_max_generation_dimension) { - error_message = "generation width and height must be between 1 and " + std::to_string(k_max_generation_dimension); + auto within_dimension_limit = [&](double value) { + if (!std::isfinite(value) || value > k_max_generation_dimension) { + error_message = "generation dimensions must not exceed " + std::to_string(k_max_generation_dimension); return false; } return true; }; const auto& params = request.gen_params; - const int width = params.get_resolved_width(); - const int height = params.get_resolved_height(); - if (!dimensions_valid(width, height)) { + if ((params.width && !within_dimension_limit(*params.width)) || + (params.height && !within_dimension_limit(*params.height))) return false; - } - if (params.hires_enabled) { - const double hires_width = params.hires_width > 0 ? params.hires_width : params.hires_height > 0 ? params.hires_height - : std::round(width * params.hires_scale); - const double hires_height = params.hires_height > 0 ? params.hires_height : params.hires_width > 0 ? params.hires_width - : std::round(height * params.hires_scale); - if (!dimensions_valid(hires_width, hires_height)) { + if (params.hires_enabled.value_or(false)) { + if ((params.hires_width && !within_dimension_limit(*params.hires_width)) || + (params.hires_height && !within_dimension_limit(*params.hires_height))) return false; + if (params.hires_width.value_or(0) <= 0 && params.hires_height.value_or(0) <= 0) { + // Use the documented library defaults for this resource-policy check only. + const float scale = params.hires_scale.value_or(2.f); + if (!within_dimension_limit(std::round(params.width.value_or(512) * scale)) || + !within_dimension_limit(std::round(params.height.value_or(512) * scale))) + return false; } } if (!request.adetailer_enabled || runtime.default_gen_params->ad_model_path.empty()) { return true; } auto pass_valid = [&](const ServerADetailerPass& pass) { - if (!dimensions_valid(pass.width, pass.height)) { + if ((pass.width && !within_dimension_limit(*pass.width)) || + (pass.height && !within_dimension_limit(*pass.height))) { return false; } // Match the native key normalization, but leave full option parsing to ADetailer. @@ -114,10 +117,9 @@ bool validate_server_img_gen_request(const ServerRuntime& runtime, } bool apply_server_adetailer(ServerRuntime& runtime, - const ImgGenJobRequest& request, - const sd_img_gen_params_t& params, + ImgGenJobRequest& request, SDImageVec& results) { - const auto& gen_params = request.gen_params; + auto& gen_params = request.gen_params; const auto& ad_model_path = runtime.default_gen_params->ad_model_path; if (!request.adetailer_enabled || ad_model_path.empty()) { return true; @@ -137,53 +139,54 @@ bool apply_server_adetailer(ServerRuntime& runtime, passes.push_back(std::move(pass)); } - ADetailerCtxPtr ad_ctx(new_adetailer_ctx(ad_model_path.c_str(), - runtime.ctx_params->n_threads, - runtime.ctx_params->effective_backend.c_str(), - runtime.ctx_params->effective_params_backend.c_str())); + ADetailerCtxPtr ad_ctx(create_sd_adetailer(ad_model_path.c_str(), + runtime.ctx_params->n_threads, + runtime.ctx_params->effective_backend.c_str(), + runtime.ctx_params->effective_params_backend.c_str())); if (ad_ctx == nullptr) { - LOG_ERROR("new_adetailer_ctx failed"); + LOG_ERROR("sd_adetailer_create failed"); return false; } for (const auto& pass : passes) { - sd_adetailer_params_t ad_params{}; - ad_params.prompt = pass.prompt.empty() ? nullptr : pass.prompt.c_str(); - ad_params.negative_prompt = pass.negative_prompt.empty() ? nullptr : pass.negative_prompt.c_str(); - ad_params.extra_ad_args = pass.extra_args.c_str(); for (int i = 0; i < results.count(); ++i) { if (results[i].data == nullptr) { continue; } - sd_img_gen_params_t ad_generation_params = params; - ad_generation_params.seed = generation_seed(results.generation_info(i)); - ad_generation_params.width = pass.width; - ad_generation_params.height = pass.height; - ad_generation_params.strength = pass.strength; - if (pass.clip_skip > 0) { - ad_generation_params.clip_skip = pass.clip_skip; + SDOptBuilder ad_options(gen_params.to_image_opt(true).release()); + ad_options.unset("width"); + ad_options.unset("height"); + ad_options.unset("strength"); + ad_options.set_string("adetail.prompt", pass.prompt); + ad_options.set_string("adetail.negative_prompt", pass.negative_prompt); + ad_options.set_string("adetail.args", pass.extra_args); + ad_options.set_int("seed", generation_seed(results.generation_info(i))); + ad_options.set_int("width", pass.width); + ad_options.set_int("height", pass.height); + ad_options.set_float("strength", pass.strength); + if (pass.clip_skip) { + ad_options.set_int("clip_skip", pass.clip_skip); } if (pass.automatic_scheduler) { - ad_generation_params.sample_params.scheduler = SCHEDULER_COUNT; + ad_options.set_string("sample.scheduler", "auto"); } - sd_image_generation_result_t* detailed = nullptr; - if (!adetail_image(ad_ctx.get(), - runtime.sd_ctx, - results[i], - &ad_params, - &ad_generation_params, - &detailed) || - detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) { - free_sd_image_generation_result(detailed); + sd_result_t* detailed = nullptr; + if (!adetail_sd_image(ad_ctx.get(), + runtime.sd_ctx, + results[i], + ad_options.finish(), + &detailed) || + detailed == nullptr || sd_result_get_image_count(detailed) <= 0 || sd_tensor_data(sd_result_get_image(detailed, 0)) == nullptr) { + sd_result_free(detailed); LOG_ERROR("ADetailer failed for image %d", i + 1); return false; } - results.append_generation_info(i, detailed->items[0].generation_info, runtime.ctx_params->lora_model_dir); - free(results[i].data); - results[i] = detailed->items[0].image; - detailed->items[0].image = {}; - free_sd_image_generation_result(detailed); + results.append_generation_info(i, sd_result_get_generation_info(detailed, 0), runtime.ctx_params->lora_model_dir); + sd_image_free(&results[i]); + results[i] = take_result_image(detailed, 0); + + sd_result_free(detailed); } } return true; @@ -582,7 +585,7 @@ void refresh_upscaler_cache(ServerRuntime& rt) { }); upscaler_entry.image_upscale_factor = previous != previous_cache.end() ? previous->image_upscale_factor - : get_upscaler_model_scale(upscaler_entry.fullpath.c_str()); + : sd_upscaler_probe_scale(upscaler_entry.fullpath.c_str()); if (upscaler_entry.image_upscale_factor > 0) { upscaler_entry.scale = upscaler_entry.image_upscale_factor; upscaler_entry.model_name = "ESRGAN_" + std::to_string(upscaler_entry.scale) + "x"; diff --git a/examples/server/runtime.h b/examples/server/runtime.h index b21f6f5678..abb9766f71 100644 --- a/examples/server/runtime.h +++ b/examples/server/runtime.h @@ -73,10 +73,10 @@ struct ServerADetailerPass { std::string prompt; std::string negative_prompt; std::string extra_args; - int width = 512; - int height = 512; - float strength = 0.4f; - int clip_skip = -1; + std::optional width; + std::optional height; + std::optional strength; + std::optional clip_skip; bool automatic_scheduler = false; }; @@ -89,16 +89,15 @@ struct ImgGenJobRequest { bool adetailer_enabled = true; std::vector adetailer_passes; - sd_img_gen_params_t to_sd_img_gen_params_t() { - return gen_params.to_sd_img_gen_params_t(); - } + SDOptPtr to_image_opt() { return gen_params.to_image_opt(); } }; class PromptImageCallbackGuard { sd_ctx_t* ctx_; const SDGenerationParams& params_; + SDTensorPtr image_tensor_; - static bool load_image(const char* source, void* data, sd_image_t* image_out); + static bool load_image(const char* source, void* data, const sd_tensor_t** image_out); public: PromptImageCallbackGuard(sd_ctx_t* ctx, const SDGenerationParams& params); @@ -121,9 +120,7 @@ struct VidGenJobRequest { std::string preview_mode = "none"; int preview_interval = 1; - sd_vid_gen_params_t to_sd_vid_gen_params_t() { - return gen_params.to_sd_vid_gen_params_t(); - } + SDOptPtr to_video_opt() { return gen_params.to_video_opt(); } }; std::string base64_encode(const std::vector& bytes); @@ -147,6 +144,5 @@ std::string get_lora_full_path(ServerRuntime& rt, const std::string& path); void refresh_upscaler_cache(ServerRuntime& rt); int64_t unix_timestamp_now_ms(); bool apply_server_adetailer(ServerRuntime& runtime, - const ImgGenJobRequest& request, - const sd_img_gen_params_t& params, + ImgGenJobRequest& request, SDImageVec& results); diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 3a06aea688..c5a25c6d10 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -19,487 +19,222 @@ #endif #endif -#ifdef __cplusplus -extern "C" { -#endif - #include #include #include #include -enum rng_type_t { - STD_DEFAULT_RNG, - CUDA_RNG, - CPU_RNG, - RNG_TYPE_COUNT -}; - -enum sample_method_t { - EULER_SAMPLE_METHOD, - EULER_A_SAMPLE_METHOD, - HEUN_SAMPLE_METHOD, - DPM2_SAMPLE_METHOD, - DPMPP2S_A_SAMPLE_METHOD, - DPMPP2M_SAMPLE_METHOD, - DPMPP2Mv2_SAMPLE_METHOD, - IPNDM_SAMPLE_METHOD, - IPNDM_V_SAMPLE_METHOD, - LCM_SAMPLE_METHOD, - DDIM_TRAILING_SAMPLE_METHOD, - TCD_SAMPLE_METHOD, - RES_MULTISTEP_SAMPLE_METHOD, - RES_2S_SAMPLE_METHOD, - ER_SDE_SAMPLE_METHOD, - EULER_CFG_PP_SAMPLE_METHOD, - EULER_A_CFG_PP_SAMPLE_METHOD, - EULER_GE_SAMPLE_METHOD, - DPMPP2M_SDE_SAMPLE_METHOD, - DPMPP2M_SDE_BT_SAMPLE_METHOD, - LMS_SAMPLE_METHOD, - SAMPLE_METHOD_COUNT -}; - -extern SD_API const char* sample_method_to_str[]; - -enum scheduler_t { - DISCRETE_SCHEDULER, - KARRAS_SCHEDULER, - EXPONENTIAL_SCHEDULER, - AYS_SCHEDULER, - GITS_SCHEDULER, - SGM_UNIFORM_SCHEDULER, - SIMPLE_SCHEDULER, - SMOOTHSTEP_SCHEDULER, - KL_OPTIMAL_SCHEDULER, - LCM_SCHEDULER, - BONG_TANGENT_SCHEDULER, - LTX2_SCHEDULER, - LOGIT_NORMAL_SCHEDULER, - FLUX2_SCHEDULER, - FLUX_SCHEDULER, - BETA_SCHEDULER, - LLADA_IMAGE_SCHEDULER, - SCHEDULER_COUNT -}; - -extern SD_API const char* scheduler_to_str[]; - -enum prediction_t { - EPS_PRED, - V_PRED, - EDM_V_PRED, - FLOW_PRED, - FLUX_FLOW_PRED, - SEFI_FLOW_PRED, - MINIT2I_FLOW_PRED, - SENSENOVA_U1_FLOW_PRED, - FLUX3_FLOW_PRED, - H3_AV_FLOW_PRED, - MING_IMAGE_FLOW_PRED, - PREDICTION_COUNT -}; - -// same as enum ggml_type -enum sd_type_t { - SD_TYPE_F32 = 0, - SD_TYPE_F16 = 1, - SD_TYPE_Q4_0 = 2, - SD_TYPE_Q4_1 = 3, - // SD_TYPE_Q4_2 = 4, support has been removed - // SD_TYPE_Q4_3 = 5, support has been removed - SD_TYPE_Q5_0 = 6, - SD_TYPE_Q5_1 = 7, - SD_TYPE_Q8_0 = 8, - SD_TYPE_Q8_1 = 9, - SD_TYPE_Q2_K = 10, - SD_TYPE_Q3_K = 11, - SD_TYPE_Q4_K = 12, - SD_TYPE_Q5_K = 13, - SD_TYPE_Q6_K = 14, - SD_TYPE_Q8_K = 15, - SD_TYPE_IQ2_XXS = 16, - SD_TYPE_IQ2_XS = 17, - SD_TYPE_IQ3_XXS = 18, - SD_TYPE_IQ1_S = 19, - SD_TYPE_IQ4_NL = 20, - SD_TYPE_IQ3_S = 21, - SD_TYPE_IQ2_S = 22, - SD_TYPE_IQ4_XS = 23, - SD_TYPE_I8 = 24, - SD_TYPE_I16 = 25, - SD_TYPE_I32 = 26, - SD_TYPE_I64 = 27, - SD_TYPE_F64 = 28, - SD_TYPE_IQ1_M = 29, - SD_TYPE_BF16 = 30, - // SD_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files - // SD_TYPE_Q4_0_4_8 = 32, - // SD_TYPE_Q4_0_8_8 = 33, - SD_TYPE_TQ1_0 = 34, - SD_TYPE_TQ2_0 = 35, - // SD_TYPE_IQ4_NL_4_4 = 36, - // SD_TYPE_IQ4_NL_4_8 = 37, - // SD_TYPE_IQ4_NL_8_8 = 38, - SD_TYPE_MXFP4 = 39, // MXFP4 (1 block) - SD_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) - SD_TYPE_Q1_0 = 41, - SD_TYPE_Q2_0 = 42, - SD_TYPE_F8_E4M3 = 43, - SD_TYPE_F8_E5M2 = 44, - SD_TYPE_COUNT = 45, -}; - -enum sd_log_level_t { - SD_LOG_DEBUG, - SD_LOG_VERBOSE, - SD_LOG_INFO, - SD_LOG_WARN, - SD_LOG_ERROR -}; - -enum preview_t { - PREVIEW_NONE, - PREVIEW_PROJ, - PREVIEW_TAE, - PREVIEW_VAE, - PREVIEW_COUNT -}; - -enum lora_apply_mode_t { - LORA_APPLY_AUTO, - LORA_APPLY_IMMEDIATELY, - LORA_APPLY_AT_RUNTIME, - LORA_APPLY_MODE_COUNT, -}; - -typedef struct { - bool enabled; - bool temporal_tiling; - // Spatial tile dimensions in image pixels for both encode and decode; 0 uses 256. - int tile_size_w; - int tile_size_h; - float target_overlap; - // Positive values override tile_size: <= 1 is a dimension fraction, > 1 a target tile count. - float rel_size_w; - float rel_size_h; - const char* extra_tiling_args; -} sd_tiling_params_t; - -typedef struct { - const char* name; - const char* path; -} sd_embedding_t; +#ifdef __cplusplus +extern "C" { +#endif -enum sd_vae_format_t { - SD_VAE_FORMAT_AUTO = -1, - SD_VAE_FORMAT_FLUX, - SD_VAE_FORMAT_SD3, - SD_VAE_FORMAT_FLUX2, - SD_VAE_FORMAT_WAN, - SD_VAE_FORMAT_COUNT, -}; +// Opaque handles -typedef struct { - const char* model_path; - const char* clip_l_path; - const char* clip_g_path; - const char* clip_vision_path; - const char* t5xxl_path; - const char* llm_path; - const char* llm_vision_path; - const char* diffusion_model_path; - const char* high_noise_diffusion_model_path; - const char* uncond_diffusion_model_path; - const char* embeddings_connectors_path; - const char* vae_path; - const char* audio_vae_path; - const char* audio_encoder_path; - const char* taesd_path; - const char* control_net_path; - const char* ip_adapter_path; - const char* motion_module_path; - const sd_embedding_t* embeddings; - uint32_t embedding_count; - const char* photo_maker_path; - const char* pulid_weights_path; - const char* tensor_type_rules; - int n_threads; - enum sd_type_t wtype; - enum rng_type_t rng_type; - enum rng_type_t sampler_rng_type; - enum prediction_t prediction; - enum lora_apply_mode_t lora_apply_mode; - bool enable_mmap; - bool flash_attn; - bool diffusion_flash_attn; - bool tae_preview_only; - bool diffusion_conv_direct; - bool vae_conv_direct; - bool force_sdxl_vae_conv_scale; - enum sd_vae_format_t vae_format; - const char* max_vram; // Optional per-device GiB budget for managed weights and runner buffers; 0 uses live free VRAM without an explicit budget - bool disable_prefetch; // Disable asynchronous next-segment weight prefetch - bool eager_load; // Load all params into the params backend at model-load time instead of lazily on first use - const char* backend; - const char* params_backend; - const char* split_mode; // weight distribution for multi-device modules: layer (default) or row, or per-module assignments e.g. "diffusion=row" - bool auto_fit; - const char* rpc_servers; - const char* model_args; - bool disable_segmented_compute; // Force monolithic graph execution even when automatic graph cutting would fit memory better - bool batched_cfg; // Run the conditional and unconditional CFG branches in one batched UNet forward when supported - float linear_scale; // Override linear input scaling; 0 keeps the model default - float attn_scale; // Override flash-attention K/V scaling; 0 keeps the model default - const char* tokenizer; // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens - bool sage_attn; - int conditioning_cache_size; // Maximum cached conditioning entries per context; 0 disables caching (default: 4) - const char* flash_attn_mode; // on/off/auto or module assignments (e.g. "auto,te=off"); NULL/empty honors legacy flags. - const char* conv_direct_mode; // on/off/auto or diffusion/vae/controlnet assignments; NULL/empty honors legacy flags. -} sd_ctx_params_t; +typedef struct sd_tensor_t sd_tensor_t; +typedef struct sd_ctx_t sd_ctx_t; +typedef struct sd_result sd_result_t; +typedef struct sd_opt sd_opt_t; +typedef struct sd_upscaler sd_upscaler_t; +typedef struct sd_vid_upscaler sd_vid_upscaler_t; +typedef struct sd_adetailer sd_adetailer_t; + +// Status and diagnostics + +typedef enum { + SD_STATUS_OK = 0, + SD_STATUS_INVALID_ARGUMENT = 1, + SD_STATUS_UNKNOWN_KEY = 2, + SD_STATUS_TYPE_MISMATCH = 3, + SD_STATUS_OUT_OF_RANGE = 4, + SD_STATUS_UNSUPPORTED = 5, + SD_STATUS_OUT_OF_MEMORY = 6, + SD_STATUS_LOAD_FAILED = 7, + SD_STATUS_GENERATION_FAILED = 8, + SD_STATUS_BUSY = 9, + SD_STATUS_CANCELLED = 10, +} sd_status_t; + +// Thread-local diagnostic, cleared by successful status/accessor calls. +// Valid until the next API call; free functions and tensor retain preserve it. +SD_API const char* sd_get_last_error(void); + +// Tensors and media + +typedef int32_t sd_tensor_type_t; +#define SD_TENSOR_U8 1 +#define SD_TENSOR_I32 2 +#define SD_TENSOR_I64 3 +#define SD_TENSOR_F32 4 +#define SD_TENSOR_F64 5 typedef struct { uint32_t sample_rate; - uint32_t channels; - uint64_t sample_count; - float* data; + const sd_tensor_t* data; } sd_audio_t; typedef struct { - uint32_t width; - uint32_t height; - uint32_t channel; - uint8_t* data; -} sd_image_t; - -typedef struct { - sd_image_t image; - // Owned UTF-8 JSON using the sdcpp.generation/v1 schema. - char* generation_info; -} sd_image_result_t; - -typedef struct { - sd_image_result_t* items; - int count; -} sd_image_generation_result_t; - -typedef struct { - sd_image_t* frames; - int frame_count; - sd_audio_t* audio; + const sd_tensor_t* data; int fps; - char* generation_info; -} sd_video_generation_result_t; + sd_audio_t audio; +} sd_video_t; + +// Contiguous dimension first. Images: U8 [C,W,H], audio: F32 [C,N], video: U8 [C,W,H,T]. +// create copies data; data_size must exactly match type and shape. Rank <= 8. +// Rank zero is a scalar; zero dimensions represent empty tensors. +SD_API sd_tensor_t* sd_tensor_create( + sd_tensor_type_t type, + size_t rank, + const int64_t* shape, + const void* data, + size_t data_size); + +// retain returns the same immutable handle with one additional ownership reference. +SD_API sd_tensor_t* sd_tensor_retain(const sd_tensor_t* tensor); +SD_API void sd_tensor_free(sd_tensor_t* tensor); +SD_API sd_tensor_type_t sd_tensor_type(const sd_tensor_t* tensor); +SD_API size_t sd_tensor_rank(const sd_tensor_t* tensor); +SD_API int64_t sd_tensor_dim(const sd_tensor_t* tensor, size_t axis); +SD_API size_t sd_tensor_byte_size(const sd_tensor_t* tensor); +SD_API const void* sd_tensor_data(const sd_tensor_t* tensor); + +// Options typedef struct { - // Semicolon-separated target=...,key=value rules. NULL preserves defaults. - const char* rules; -} sd_image_preprocess_params_t; + const char* name; + const char* path; +} sd_embedding_t; typedef struct { - sd_image_t* frames; - int frame_count; - int fps; - sd_audio_t audio; -} sd_ref_video_t; + bool is_high_noise; + float multiplier; + const char* path; +} sd_lora_t; -typedef struct { - int* layers; - size_t layer_count; - float layer_start; - float layer_end; - float scale; -} sd_slg_params_t; +typedef sd_opt_t sd_ctx_opt_t; +typedef sd_opt_t sd_img_gen_opt_t; +typedef sd_opt_t sd_vid_gen_opt_t; +typedef sd_opt_t sd_upscaler_opt_t; +typedef sd_opt_t sd_upscale_opt_t; +typedef sd_opt_t sd_vid_upscaler_opt_t; +typedef sd_opt_t sd_vid_upscale_opt_t; +typedef sd_opt_t sd_adetailer_opt_t; +typedef sd_opt_t sd_adetail_opt_t; + +// Parameter keys and types are documented in docs/guides/c_api.md. +// Setters replace values atomically; a failed setter preserves the previous value. +// Strings, arrays and media descriptors are copied. Tensor storage is retained; +// callers may free their tensor handles after a successful setter. +// Do not mutate/free params concurrently with an operation using them. +// The aliases share one C type; constructors bind their purpose at runtime. +SD_API sd_ctx_opt_t* sd_ctx_opt_create(void); +SD_API sd_img_gen_opt_t* sd_img_gen_opt_create(void); +SD_API sd_vid_gen_opt_t* sd_vid_gen_opt_create(void); +SD_API sd_upscaler_opt_t* sd_upscaler_opt_create(void); +SD_API sd_upscale_opt_t* sd_upscale_opt_create(void); +SD_API sd_vid_upscaler_opt_t* sd_vid_upscaler_opt_create(void); +SD_API sd_vid_upscale_opt_t* sd_vid_upscale_opt_create(void); +SD_API sd_adetailer_opt_t* sd_adetailer_opt_create(void); +SD_API sd_adetail_opt_t* sd_adetail_opt_create(void); +SD_API void sd_opt_free(sd_opt_t* params); + +SD_API sd_status_t sd_opt_set_bool(sd_opt_t* params, const char* key, bool value); +SD_API sd_status_t sd_opt_set_int(sd_opt_t* params, const char* key, int64_t value); +SD_API sd_status_t sd_opt_set_float(sd_opt_t* params, const char* key, double value); +SD_API sd_status_t sd_opt_set_string(sd_opt_t* params, const char* key, const char* value); +SD_API sd_status_t sd_opt_set_tensor(sd_opt_t* params, const char* key, const sd_tensor_t* tensor); +SD_API sd_status_t sd_opt_set_tensors( + sd_opt_t* params, + const char* key, + const sd_tensor_t* const* tensors, + size_t count); +SD_API sd_status_t sd_opt_set_floats(sd_opt_t* params, const char* key, const float* values, size_t count); +SD_API sd_status_t sd_opt_set_ints(sd_opt_t* params, const char* key, const int32_t* values, size_t count); +SD_API sd_status_t sd_opt_set_loras(sd_opt_t* params, const char* key, const sd_lora_t* values, size_t count); +SD_API sd_status_t sd_opt_set_embeddings( + sd_opt_t* params, + const char* key, + const sd_embedding_t* values, + size_t count); +SD_API sd_status_t sd_opt_set_audios(sd_opt_t* params, const char* key, const sd_audio_t* values, size_t count); +SD_API sd_status_t sd_opt_set_videos(sd_opt_t* params, const char* key, const sd_video_t* values, size_t count); + +// Setters/unset reject keys for another purpose. Unsetting an applicable known +// absent key succeeds; unknown keys fail. NULL is not a string value. +SD_API sd_status_t sd_opt_unset(sd_opt_t* params, const char* key); + +// Context and generation + +// Output pointers are required and set to NULL on failure. Context creation retains +// its own configuration; params may be freed immediately after the call returns. +SD_API sd_status_t sd_ctx_create(const sd_ctx_opt_t* params, sd_ctx_t** ctx_out); +SD_API void sd_ctx_free(sd_ctx_t* ctx); + +SD_API sd_status_t sd_generate_image(sd_ctx_t* ctx, const sd_img_gen_opt_t* params, sd_result_t** result_out); +SD_API sd_status_t sd_generate_video(sd_ctx_t* ctx, const sd_vid_gen_opt_t* params, sd_result_t** result_out); -typedef struct { - float txt_cfg; - float img_cfg; - float distilled_guidance; - sd_slg_params_t slg; -} sd_guidance_params_t; +SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx); +SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx); -typedef struct { - sd_guidance_params_t guidance; - enum scheduler_t scheduler; - enum sample_method_t sample_method; - int sample_steps; - float eta; - int shifted_timestep; - float* custom_sigmas; - int custom_sigmas_count; - float flow_shift; - const char* extra_sample_args; -} sd_sample_params_t; +// ControlNet hot-swap APIs are not safe to call while generation is in flight. +SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path); +SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx); +SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx); -typedef struct { - sd_image_t* id_images; - int id_images_count; - const char* id_embed_path; - float style_strength; -} sd_pm_params_t; // photo maker +// Requires a loaded context; returns a static string owned by the library, or "Unknown". +SD_API const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx); -typedef struct { - const char* id_embedding_path; - float id_weight; -} sd_pulid_params_t; - -enum sd_cache_mode_t { - SD_CACHE_DISABLED = 0, - SD_CACHE_EASYCACHE, - SD_CACHE_UCACHE, - SD_CACHE_DBCACHE, - SD_CACHE_TAYLORSEER, - SD_CACHE_CACHE_DIT, - SD_CACHE_SPECTRUM, +enum sd_cancel_mode_t { + // Stop the current generation as soon as possible. + SD_CANCEL_ALL, + // Finish the current image sample, then skip additional batch latents and return completed images. + SD_CANCEL_NEW_LATENTS, + // Clear a pending cancellation request. + SD_CANCEL_RESET }; -typedef struct { - enum sd_cache_mode_t mode; - float reuse_threshold; - float start_percent; - float end_percent; - float error_decay_rate; - bool use_relative_threshold; - bool reset_error_on_compute; - int Fn_compute_blocks; - int Bn_compute_blocks; - float residual_diff_threshold; - int max_warmup_steps; - int max_cached_steps; - int max_continuous_cached_steps; - int taylorseer_n_derivatives; - int taylorseer_skip_interval; - const char* scm_mask; - bool scm_policy_dynamic; - float spectrum_w; - int spectrum_m; - float spectrum_lam; - int spectrum_window_size; - float spectrum_flex_window; - int spectrum_warmup_steps; - float spectrum_stop_percent; -} sd_cache_params_t; +SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode); -typedef struct { - bool is_high_noise; - float multiplier; - const char* path; -} sd_lora_t; +// Results -enum sd_hires_upscaler_t { - SD_HIRES_UPSCALER_NONE, - SD_HIRES_UPSCALER_LATENT, - SD_HIRES_UPSCALER_LATENT_NEAREST, - SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT, - SD_HIRES_UPSCALER_LATENT_ANTIALIASED, - SD_HIRES_UPSCALER_LATENT_BICUBIC, - SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED, - SD_HIRES_UPSCALER_LANCZOS, - SD_HIRES_UPSCALER_NEAREST, - SD_HIRES_UPSCALER_MODEL, - SD_HIRES_UPSCALER_COUNT, -}; +typedef enum { + SD_RESULT_INVALID = 0, + SD_RESULT_IMAGE = 1, + SD_RESULT_VIDEO = 2, +} sd_result_kind_t; -enum sd_control_type_t { - SD_CONTROL_TYPE_NONE, - SD_CONTROL_TYPE_OPENPOSE, - SD_CONTROL_TYPE_DEPTH, - SD_CONTROL_TYPE_SCRIBBLE, // thick lines: scribble, hed, softedge, ted - SD_CONTROL_TYPE_CANNY, // thin lines: canny, lineart, anime lineart, mlsd - SD_CONTROL_TYPE_NORMAL, - SD_CONTROL_TYPE_SEGMENT, - SD_CONTROL_TYPE_TILE, - SD_CONTROL_TYPE_REPAINT, // inpaint, outpaint - SD_CONTROL_TYPE_COUNT, -}; +// Results own all outputs independently of ctx and params. Views are read-only and +// borrowed until sd_result_free(). Retain a tensor to keep its data independently. +// Media descriptors contain metadata and borrowed immutable tensor handles. +SD_API sd_result_kind_t sd_result_get_kind(const sd_result_t* result); +SD_API size_t sd_result_get_image_count(const sd_result_t* result); +SD_API const sd_tensor_t* sd_result_get_image(const sd_result_t* result, size_t index); -typedef struct { - bool enabled; - enum sd_hires_upscaler_t upscaler; - const char* model_path; - float scale; - int target_width; - int target_height; - int steps; - float denoising_strength; - int upscale_tile_size; - float* custom_sigmas; - int custom_sigmas_count; -} sd_hires_params_t; +// Materializes contiguous U8 [C,W,H,T] storage on first access. +SD_API const sd_video_t* sd_result_get_video(const sd_result_t* result); -typedef struct { - const sd_lora_t* loras; - uint32_t lora_count; - const char* prompt; - const char* negative_prompt; - int clip_skip; - sd_image_t init_image; - sd_image_t* ref_images; - int ref_images_count; - const char* ref_image_args; - sd_image_t mask_image; - int width; - int height; - sd_sample_params_t sample_params; - float strength; - int64_t seed; - int batch_count; - sd_image_t control_image; - float control_strength; - sd_image_t ip_adapter_image; - float ip_adapter_strength; - sd_pm_params_t pm_params; - sd_pulid_params_t pulid_params; - sd_tiling_params_t vae_tiling_params; - sd_cache_params_t cache; - sd_hires_params_t hires; - int qwen_image_layers; - bool circular_x; - bool circular_y; - sd_image_preprocess_params_t image_preprocess; - enum sd_control_type_t control_type; -} sd_img_gen_params_t; +// Image metadata is per image; video metadata is shared by all frame indices. +SD_API const char* sd_result_get_generation_info(const sd_result_t* result, size_t index); +SD_API const sd_audio_t* sd_result_get_audio(const sd_result_t* result); +SD_API int sd_result_get_fps(const sd_result_t* result); -typedef struct { - const sd_lora_t* loras; - uint32_t lora_count; - const char* prompt; - const char* negative_prompt; - int clip_skip; - sd_image_t init_image; - sd_image_t end_image; - sd_image_t* ref_images; - int ref_images_count; - sd_ref_video_t* ref_videos; - int ref_videos_count; - sd_audio_t* ref_audios; - int ref_audios_count; - sd_image_t* control_frames; - int control_frames_size; - int width; - int height; - sd_sample_params_t sample_params; - sd_sample_params_t high_noise_sample_params; - float moe_boundary; - float strength; - int64_t seed; - int video_frames; - int fps; - float vace_strength; - sd_tiling_params_t vae_tiling_params; - sd_cache_params_t cache; - sd_hires_params_t hires; - bool circular_x; - bool circular_y; - sd_image_preprocess_params_t image_preprocess; - const char* ref_image_args; -} sd_vid_gen_params_t; +SD_API void sd_result_free(sd_result_t* result); -typedef struct sd_ctx_t sd_ctx_t; -struct ggml_tensor; +// Callbacks and progress + +enum sd_log_level_t { + SD_LOG_DEBUG, + SD_LOG_VERBOSE, + SD_LOG_INFO, + SD_LOG_WARN, + SD_LOG_ERROR +}; typedef void (*sd_log_cb_t)(enum sd_log_level_t level, const char* text, void* data); +SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); + typedef void (*sd_progress_cb_t)(int step, int steps, float time, void* data); +SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data); + typedef struct { const char* stage; const char* detail; @@ -509,222 +244,174 @@ typedef struct { int64_t total; } sd_generation_progress_t; typedef void (*sd_generation_progress_cb_t)(const sd_generation_progress_t* progress, void* data); -typedef void (*sd_preview_cb_t)(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data); + +// Context-local generation events, independent of preview and the legacy progress callback. +// Strings and the event are valid only during the synchronous callback. Callbacks must not throw. +// total == 0 means unknown; sample_pass counts from 1 since callback registration. +// Register/clear only while the context is idle. Passing NULL clears the callback. +SD_API void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data); typedef struct { int sample_pass; int total_steps; } sd_preview_info_t; -typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data); -// Resolves markers. Pixels are borrowed and copied before the -// next callback; the caller retains ownership. Register/clear while ctx is idle. -typedef bool (*sd_prompt_image_cb_t)(const char* source, void* data, sd_image_t* image_out); -SD_API void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data); +// Frames are borrowed during the callback; retain a tensor to keep its snapshot. +typedef void (*sd_preview_cb_t)(int step, int frame_count, const sd_tensor_t* const* frames, bool is_noisy, void* data); -SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); -SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data); -// Context-local generation events, independent of preview and the legacy progress callback. -// Strings and the event are valid only during the synchronous callback. Callbacks must not throw. -// total == 0 means unknown; sample_pass counts from 1 since callback registration. -// Register/clear only while the context is idle. Passing nullptr clears the callback. -SD_API void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data); // In each sampling pass, a positive interval previews every Nth denoiser step, while a // negative interval previews only completed logical step -interval. Zero previews the final // completed step of the first sampling pass (base-resolution or high-noise). -SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data); +// Mode is "none", "proj", "tae" or "vae"; invalid input preserves registration. +SD_API sd_status_t sd_set_preview_callback( + sd_preview_cb_t cb, + const char* mode, + int interval, + bool denoised, + bool noisy, + void* data); + // Query from a preview callback. Passes are numbered from 1 since the last // sd_set_preview_callback call; total_steps is the actual count in that pass. // Both fields are zero before sampling starts. SD_API sd_preview_info_t sd_get_preview_info(); -SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data); -SD_API int32_t sd_get_num_physical_cores(); -SD_API const char* sd_get_system_info(); -SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx); -SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx); - -// ControlNet hot-swap APIs are not safe to call while generation is in flight. -SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path); -SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx); -SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx); -SD_API const char* sd_type_name(enum sd_type_t type); -SD_API enum sd_type_t str_to_sd_type(const char* str); -SD_API const char* sd_rng_type_name(enum rng_type_t rng_type); -SD_API enum rng_type_t str_to_rng_type(const char* str); -SD_API const char* sd_sample_method_name(enum sample_method_t sample_method); -SD_API enum sample_method_t str_to_sample_method(const char* str); -SD_API const char* sd_scheduler_name(enum scheduler_t scheduler); -SD_API enum scheduler_t str_to_scheduler(const char* str); -SD_API const char* sd_prediction_name(enum prediction_t prediction); -SD_API enum prediction_t str_to_prediction(const char* str); -SD_API const char* sd_preview_name(enum preview_t preview); -SD_API enum preview_t str_to_preview(const char* str); -SD_API const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode); -SD_API enum lora_apply_mode_t str_to_lora_apply_mode(const char* str); -SD_API const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler); -SD_API enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str); -SD_API const char* sd_control_type_name(enum sd_control_type_t control_type); -// Also accepts the names of the kinds a type stands for (hed, lineart, mlsd, ...); -// returns SD_CONTROL_TYPE_COUNT when the name is unknown. -SD_API enum sd_control_type_t str_to_sd_control_type(const char* str); - -SD_API void sd_cache_params_init(sd_cache_params_t* cache_params); -SD_API void sd_hires_params_init(sd_hires_params_t* hires_params); - -SD_API void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params); -SD_API char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params); - -SD_API sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params); -SD_API void free_sd_ctx(sd_ctx_t* sd_ctx); -SD_API void free_sd_audio(sd_audio_t* audio); - -SD_API void sd_sample_params_init(sd_sample_params_t* sample_params); -SD_API char* sd_sample_params_to_str(const sd_sample_params_t* sample_params); +// Resolves markers. The returned U8 [C,W,H] tensor is borrowed +// and copied before the next callback. Register/clear while ctx is idle. +typedef bool (*sd_prompt_image_cb_t)(const char* source, void* data, const sd_tensor_t** image_out); +SD_API void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data); -// Requires a loaded context; returns a static string owned by the library, or "Unknown". -SD_API const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx); +struct ggml_tensor; -SD_API enum sample_method_t sd_get_default_sample_method(const sd_ctx_t* sd_ctx); -SD_API enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_method_t sample_method); +typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data); +SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data); -SD_API void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params); -SD_API char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params); -SD_API bool generate_image(sd_ctx_t* sd_ctx, - const sd_img_gen_params_t* sd_img_gen_params, - sd_image_generation_result_t** result_out); +// Image upscaling -// Results own their pixels, audio and generation_info, independently of the context. -// Generation calls require result_out and set it to NULL on failure. -SD_API void free_sd_image_generation_result(sd_image_generation_result_t* result); -SD_API void free_sd_video_generation_result(sd_video_generation_result_t* result); +SD_API sd_status_t sd_upscaler_create(const sd_upscaler_opt_t* opt, sd_upscaler_t** out); +SD_API void sd_upscaler_free(sd_upscaler_t* upscaler); +SD_API sd_status_t sd_upscale(sd_upscaler_t* upscaler, const sd_upscale_opt_t* opt, sd_result_t** out); +SD_API int sd_upscaler_get_scale(const sd_upscaler_t* upscaler); -enum sd_cancel_mode_t { - // Stop the current generation as soon as possible. - SD_CANCEL_ALL, - // Finish the current image sample, then skip additional batch latents and return completed images. - SD_CANCEL_NEW_LATENTS, - // Clear a pending cancellation request. - SD_CANCEL_RESET -}; +// Reads model metadata only; returns 0 if it is not a recognized RGB upscaler. +SD_API int sd_upscaler_probe_scale(const char* model_path); -SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode); +// Video upscaling -SD_API void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params); -SD_API bool generate_video(sd_ctx_t* sd_ctx, - const sd_vid_gen_params_t* sd_vid_gen_params, - sd_video_generation_result_t** result_out); +SD_API sd_status_t sd_vid_upscaler_create(const sd_vid_upscaler_opt_t* opt, sd_vid_upscaler_t** out); +SD_API void sd_vid_upscaler_free(sd_vid_upscaler_t* upscaler); +SD_API sd_status_t sd_upscale_video( + sd_vid_upscaler_t* upscaler, + const sd_vid_upscale_opt_t* opt, + sd_result_t** out); -typedef struct upscaler_ctx_t upscaler_ctx_t; +// ADetailer -SD_API upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path, - bool direct, - int n_threads, - int tile_size, - const char* backend, - const char* params_backend); -SD_API void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); +SD_API sd_status_t sd_adetailer_create(const sd_adetailer_opt_t* opt, sd_adetailer_t** out); +SD_API void sd_adetailer_free(sd_adetailer_t* adetailer); +SD_API sd_status_t sd_adetail( + sd_adetailer_t* adetailer, + sd_ctx_t* ctx, + const sd_adetail_opt_t* opt, + sd_result_t** out); -SD_API bool upscale(upscaler_ctx_t* upscaler_ctx, - sd_image_t input_image, - uint32_t upscale_factor, - sd_image_generation_result_t** result_out); +// Image preprocessing -SD_API int get_upscale_factor(upscaler_ctx_t* upscaler_ctx); -// Reads model metadata only; returns 0 if the file is not a recognized RGB ESRGAN model. -SD_API int get_upscaler_model_scale(const char* model_path); +SD_API sd_status_t sd_preprocess_canny( + const sd_tensor_t* image, + float high_threshold, + float low_threshold, + float weak, + float strong, + bool inverse, + sd_tensor_t** image_out); -typedef struct video_upscaler_ctx_t video_upscaler_ctx_t; +// Weight types and model conversion -typedef struct { - const char* diffusion_model_path; - const char* vae_path; - // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints. - int upscale_factor; - int n_threads; - const char* backend; - const char* params_backend; - // auto/on/off or diffusion/vae assignments; null selects auto. - const char* flash_attn_mode; - const char* conv_direct_mode; -} sd_video_upscaler_ctx_params_t; +// same as enum ggml_type +enum sd_type_t { + SD_TYPE_F32 = 0, + SD_TYPE_F16 = 1, + SD_TYPE_Q4_0 = 2, + SD_TYPE_Q4_1 = 3, + // SD_TYPE_Q4_2 = 4, support has been removed + // SD_TYPE_Q4_3 = 5, support has been removed + SD_TYPE_Q5_0 = 6, + SD_TYPE_Q5_1 = 7, + SD_TYPE_Q8_0 = 8, + SD_TYPE_Q8_1 = 9, + SD_TYPE_Q2_K = 10, + SD_TYPE_Q3_K = 11, + SD_TYPE_Q4_K = 12, + SD_TYPE_Q5_K = 13, + SD_TYPE_Q6_K = 14, + SD_TYPE_Q8_K = 15, + SD_TYPE_IQ2_XXS = 16, + SD_TYPE_IQ2_XS = 17, + SD_TYPE_IQ3_XXS = 18, + SD_TYPE_IQ1_S = 19, + SD_TYPE_IQ4_NL = 20, + SD_TYPE_IQ3_S = 21, + SD_TYPE_IQ2_S = 22, + SD_TYPE_IQ4_XS = 23, + SD_TYPE_I8 = 24, + SD_TYPE_I16 = 25, + SD_TYPE_I32 = 26, + SD_TYPE_I64 = 27, + SD_TYPE_F64 = 28, + SD_TYPE_IQ1_M = 29, + SD_TYPE_BF16 = 30, + // SD_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files + // SD_TYPE_Q4_0_4_8 = 32, + // SD_TYPE_Q4_0_8_8 = 33, + SD_TYPE_TQ1_0 = 34, + SD_TYPE_TQ2_0 = 35, + // SD_TYPE_IQ4_NL_4_4 = 36, + // SD_TYPE_IQ4_NL_4_8 = 37, + // SD_TYPE_IQ4_NL_8_8 = 38, + SD_TYPE_MXFP4 = 39, // MXFP4 (1 block) + SD_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + SD_TYPE_Q1_0 = 41, + SD_TYPE_Q2_0 = 42, + SD_TYPE_F8_E4M3 = 43, + SD_TYPE_F8_E5M2 = 44, + SD_TYPE_COUNT = 45, +}; -typedef struct { - const sd_image_t* frames; - int frame_count; - int fps; - int64_t seed; - // RelayVSR reference window, including both endpoints; defaults to 16. - int window_size; - // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5. - int temporal_chunk_size; -} sd_video_upscale_params_t; - -SD_API void sd_video_upscaler_ctx_params_init(sd_video_upscaler_ctx_params_t* params); -SD_API void sd_video_upscale_params_init(sd_video_upscale_params_t* params); -SD_API video_upscaler_ctx_t* new_video_upscaler_ctx(const sd_video_upscaler_ctx_params_t* params); -SD_API void free_video_upscaler_ctx(video_upscaler_ctx_t* ctx); -// Frames are RGB, have equal dimensions, and remain owned by the caller. -// Free the result with free_sd_video_generation_result(). -SD_API bool upscale_video(video_upscaler_ctx_t* ctx, - const sd_video_upscale_params_t* params, - sd_video_generation_result_t** result_out); - -typedef struct adetailer_ctx_t adetailer_ctx_t; +SD_API const char* sd_type_name(enum sd_type_t type); +SD_API enum sd_type_t str_to_sd_type(const char* str); -typedef struct { - const char* prompt; - const char* negative_prompt; - const char* extra_ad_args; -} sd_adetailer_params_t; - -SD_API adetailer_ctx_t* new_adetailer_ctx(const char* detector_path, - int n_threads, - const char* backend, - const char* params_backend); -SD_API void free_adetailer_ctx(adetailer_ctx_t* adetailer_ctx); -SD_API bool adetail_image(adetailer_ctx_t* adetailer_ctx, - sd_ctx_t* sd_ctx, - sd_image_t input_image, - const sd_adetailer_params_t* adetailer_params, - const sd_img_gen_params_t* inpaint_params, - sd_image_generation_result_t** result_out); - -SD_API bool convert(const char* input_path, - const char* vae_path, - const char* output_path, - enum sd_type_t output_type, - const char* tensor_type_rules, - bool convert_name); - -SD_API bool convert_with_components(const char* model_path, - const char* clip_l_path, - const char* clip_g_path, - const char* t5xxl_path, - const char* diffusion_model_path, - const char* vae_path, - const char* output_path, - enum sd_type_t output_type, - const char* tensor_type_rules, - bool convert_name, - int n_threads, - const sd_lora_t* loras, - int lora_count); - -SD_API bool preprocess_canny(sd_image_t image, - float high_threshold, - float low_threshold, - float weak, - float strong, - bool inverse); +SD_API bool convert( + const char* input_path, + const char* vae_path, + const char* output_path, + enum sd_type_t output_type, + const char* tensor_type_rules, + bool convert_name); + +SD_API bool convert_with_components( + const char* model_path, + const char* clip_l_path, + const char* clip_g_path, + const char* t5xxl_path, + const char* diffusion_model_path, + const char* vae_path, + const char* output_path, + enum sd_type_t output_type, + const char* tensor_type_rules, + bool convert_name, + int n_threads, + const sd_lora_t* loras, + int lora_count); + +// Importance matrix SD_API bool load_imatrix(const char* imatrix_path); SD_API void save_imatrix(const char* imatrix_path); SD_API void enable_imatrix_collection(void); SD_API void disable_imatrix_collection(void); -SD_API const char* sd_commit(void); -SD_API const char* sd_version(void); +// Devices and system information // List available ggml backend devices, one `namedescription` per line. // The names are the device names accepted by the --backend / --params-backend @@ -737,9 +424,13 @@ SD_API size_t sd_list_devices(char* buffer, size_t buffer_size); // Returns false if RPC is unavailable or an endpoint has no usable devices. SD_API bool sd_add_rpc_servers(const char* servers); -// for C API, caller needs to call free_sd_images to free the memory after use -// This helps avoid CRT problems on Windows when memory is allocated in the library but freed in the caller, which may use a different CRT. -SD_API void free_sd_images(sd_image_t* result_images, int num_images); +SD_API int32_t sd_get_num_physical_cores(); +SD_API const char* sd_get_system_info(); + +// Build information + +SD_API const char* sd_commit(void); +SD_API const char* sd_version(void); #ifdef __cplusplus } diff --git a/scripts/format-code.ps1 b/scripts/format-code.ps1 index 02097e6bb9..782e09205e 100644 --- a/scripts/format-code.ps1 +++ b/scripts/format-code.ps1 @@ -2,6 +2,9 @@ $patterns = @( "src/*.cpp" "src/*.h" "src/*.hpp" + "src/api/*.cpp" + "src/api/*.h" + "src/api/*.hpp" "src/conditioning/*.cpp" "src/conditioning/*.h" "src/conditioning/*.hpp" diff --git a/scripts/format-code.sh b/scripts/format-code.sh index dd4d0fa95b..c2806f6473 100644 --- a/scripts/format-code.sh +++ b/scripts/format-code.sh @@ -6,6 +6,7 @@ repo_root="$(cd "$(dirname "$0")/.." && pwd)" cd "$repo_root" for f in src/*.cpp src/*.h src/*.hpp \ + src/api/*.cpp src/api/*.h src/api/*.hpp \ src/conditioning/*.cpp src/conditioning/*.h src/conditioning/*.hpp \ src/core/*.cpp src/core/*.h src/core/*.hpp \ src/core/cpu/*.cpp src/core/cpu/*.h \ diff --git a/src/api/api.cpp b/src/api/api.cpp new file mode 100644 index 0000000000..72cd6d4049 --- /dev/null +++ b/src/api/api.cpp @@ -0,0 +1,245 @@ +#include "api.h" +#include "options.h" +#include "result.h" +#include "tensor.h" + +#include +#include +#include + +#include "core/ggml_extend.h" +#include "core/option_types.h" +#include "core/util.h" +#include "pipeline/diffusion_engine.h" +#include "pipeline/generation.h" + +static thread_local char last_error[512] = {}; + +void sd::api::clear_error() { + last_error[0] = '\0'; +} + +sd_status_t sd::api::error(sd_status_t status, const char* key, const char* reason) { + snprintf(last_error, sizeof(last_error), "%s%s%s", key ? key : "", key ? ": " : "", reason); + return status; +} + +const char* sd_get_last_error(void) { + return last_error; +} + +const char* sd_type_name(enum sd_type_t type) { + if (type == SD_TYPE_F8_E4M3) { + return "f8_e4m3"; + } + if (type == SD_TYPE_F8_E5M2) { + return "f8_e5m2"; + } + const auto ggml_type = sd_type_to_ggml_type(type); + if (ggml_type != GGML_TYPE_COUNT) { + return ggml_type_name(ggml_type); + } + return "NONE"; +} + +enum sd_type_t str_to_sd_type(const char* str) { + if (!strcmp(str, "f8_e4m3")) { + return SD_TYPE_F8_E4M3; + } + if (!strcmp(str, "f8_e5m2")) { + return SD_TYPE_F8_E5M2; + } + for (int i = 0; i < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT); i++) { + auto trait = ggml_get_type_traits((ggml_type)i); + if (!strcmp(str, trait->type_name)) { + return (enum sd_type_t)i; + } + } + return SD_TYPE_COUNT; +} + +sd_status_t sd_set_preview_callback(sd_preview_cb_t cb, const char* mode, int interval, bool denoised, bool noisy, void* data) { + sd::api::clear_error(); + if (!mode) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "preview.mode", "mode is required"); + const auto parsed = str_to_preview(mode); + if (parsed == PREVIEW_COUNT) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "preview.mode", "unknown preview mode"); + sd_set_preview_callback_impl(cb, parsed, interval, denoised, noisy, data); + return SD_STATUS_OK; +} + +void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data) { + if (ctx == nullptr || ctx->sd == nullptr) { + return; + } + std::lock_guard lock(ctx->sd->execution_mutex); + if (!ctx->sd->executing_) { + ctx->sd->prompt_image_callback = cb; + ctx->sd->prompt_image_callback_data = data; + } +} + +void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data) { + if (ctx == nullptr || ctx->sd == nullptr) { + return; + } + std::lock_guard lock(ctx->sd->execution_mutex); + if (!ctx->sd->executing_) { + ctx->sd->generation_progress.set_callback(cb, data); + } +} + +static sd_ctx_t* create_context(const sd::ContextConfig& params) { + auto engine = std::make_unique(); + if (!engine->init(¶ms)) + return nullptr; + auto context = std::make_unique(); + context->sd = engine.release(); + return context.release(); +} + +sd_status_t sd_ctx_create(const sd_ctx_opt_t* params, sd_ctx_t** ctx_out) { + if (ctx_out) + *ctx_out = nullptr; + return sd::api::call([&] { + if (!ctx_out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "ctx_out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(params, sd::api::Context, config); + const auto& context = config.context; + if (status != SD_STATUS_OK) + return status; + *ctx_out = create_context(context); + return *ctx_out ? SD_STATUS_OK : sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "context creation failed; see log for details"); + }, + SD_STATUS_LOAD_FAILED); +} + +void sd_ctx_free(sd_ctx_t* sd_ctx) { + if (!sd_ctx) + return; + if (sd_ctx->sd != nullptr) { + delete sd_ctx->sd; + sd_ctx->sd = nullptr; + } + delete sd_ctx; +} + +SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode) { + if (sd_ctx && sd_ctx->sd) { + if (mode < SD_CANCEL_ALL || mode > SD_CANCEL_RESET) { + mode = SD_CANCEL_ALL; + } + sd_ctx->sd->set_cancel_flag(mode); + } +} + +SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + return sd_version_supports_image_generation(sd_ctx->sd->version); +} + +SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + if (sd_ctx->sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd_ctx->sd->version)) { + return true; + } + return sd_version_supports_video_generation(sd_ctx->sd->version); +} + +SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) { + return false; + } + return sd_ctx->sd->load_control_net_from_file(path); +} + +SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + return sd_ctx->sd->unload_control_net(); +} + +SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + if (sd_ctx->sd->control_net != nullptr) { + return true; + } + // z-image keeps control net weights inside the diffusion runner, registered + // under ModelComponent::ControlNet instead of a separate control_net runner. + return sd_ctx->sd->model_manager != nullptr && + sd_ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0; +} + +const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_ctx->sd->version >= VERSION_COUNT) { + return "Unknown"; + } + return sd_version_name(sd_ctx->sd->version); +} + +sd_status_t sd_generate_image(sd_ctx_t* ctx, const sd_img_gen_opt_t* params, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + return sd::api::call([&] { + if (!result_out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result_out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(params, sd::api::Image, config); + const auto& image = config.image; + if (status != SD_STATUS_OK) + return status; + sd::ImageGenerationResult* images = nullptr; + status = sd::generate_image(ctx, image, &images, params); + if (status != SD_STATUS_OK) + return status; + return sd::make_image_result(images, result_out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result"); + }, + SD_STATUS_GENERATION_FAILED); +} + +sd_status_t sd_generate_video(sd_ctx_t* ctx, const sd_vid_gen_opt_t* params, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + return sd::api::call([&] { + if (!result_out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result_out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(params, sd::api::Video, config); + const auto& video = config.video; + if (status != SD_STATUS_OK) + return status; + sd::VideoGenerationResult* frames = nullptr; + status = sd::generate_video(ctx, video, &frames, params); + if (status != SD_STATUS_OK) + return status; + return sd::make_video_result(frames, result_out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result"); + }, + SD_STATUS_GENERATION_FAILED); +} + +sd_status_t sd_preprocess_canny(const sd_tensor_t* image, float high_threshold, float low_threshold, float weak, float strong, bool inverse, sd_tensor_t** image_out) { + if (image_out) + *image_out = nullptr; + return sd::api::call([&] { + if (!image || !image_out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "image and image_out are required"); + sd::api::image_view(*image); + auto output = sd::api::allocate_tensor(image->type, image->shape); + memcpy(output.get()->mutable_data(), image->data(), image->bytes); + if (!sd::preprocess_canny(sd::api::image_view(*output.get()), high_threshold, low_threshold, weak, strong, inverse)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "Canny preprocessing failed"); + *image_out = sd_tensor_retain(output.get()); + return SD_STATUS_OK; + }); +} diff --git a/src/api/api.h b/src/api/api.h new file mode 100644 index 0000000000..545859e960 --- /dev/null +++ b/src/api/api.h @@ -0,0 +1,34 @@ +#ifndef SD_INTERNAL_API_H +#define SD_INTERNAL_API_H + +#include +#include + +#include "stable-diffusion.h" + +class StableDiffusionGGML; + +struct sd_ctx_t { + StableDiffusionGGML* sd = nullptr; +}; + +namespace sd::api { + void clear_error(); + sd_status_t error(sd_status_t status, const char* key, const char* reason); + + template + sd_status_t call(Function&& function, sd_status_t failure = SD_STATUS_INVALID_ARGUMENT) noexcept { + clear_error(); + try { + return function(); + } catch (const std::bad_alloc&) { + return error(SD_STATUS_OUT_OF_MEMORY, nullptr, "out of memory"); + } catch (const std::exception& exception) { + return error(failure, nullptr, exception.what()); + } catch (...) { + return error(failure, nullptr, "unexpected exception"); + } + } +} + +#endif // SD_INTERNAL_API_H diff --git a/src/api/options.cpp b/src/api/options.cpp new file mode 100644 index 0000000000..591cce3448 --- /dev/null +++ b/src/api/options.cpp @@ -0,0 +1,897 @@ +#include "options.h" +#include "core/option_types.h" +#include "tensor.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum class ParamType { Bool, + Int, + Float, + String, + Tensor, + Tensors, + Floats, + Ints, + Loras, + Embeddings, + Audios, + Videos }; + +struct LoraValues { + std::vector paths; + std::vector items; +}; + +struct EmbeddingValues { + std::vector strings; + std::vector items; +}; + +struct TensorValue { + sd::api::TensorRef tensor; + std::vector frames; +}; + +struct ImageValues { + std::vector tensors; + std::vector items; +}; + +struct AudioValues { + std::vector tensors; + std::vector items; +}; + +struct VideoValues { + std::vector tensors; + std::vector> frames; + std::vector items; +}; + +using ParamValue = std::variant, std::vector, LoraValues, EmbeddingValues, AudioValues, VideoValues>; + +struct sd_opt { + sd::api::Operation operation; + std::map values; +}; + +struct ParamSpec { + const char* key; + ParamType type; + int operations; + std::function apply; + int64_t int_min = std::numeric_limits::min(); + int64_t int_max = std::numeric_limits::max(); + double float_min = -std::numeric_limits::max(); + double float_max = std::numeric_limits::max(); + std::function valid; + const char* invalid_reason = "invalid parameter value"; + bool video_tensor = false; +}; + +static sd_cache_mode_t parse_cache(const char* text) { + static const char* names[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"}; + for (int i = 0; i < 7; ++i) + if (strcmp(text, names[i]) == 0) + return static_cast(i); + return static_cast(-1); +} + +static sd_vae_format_t parse_vae_format(const char* text) { + static const char* names[] = {"auto", "flux", "sd3", "flux2", "wan"}; + for (int i = 0; i < 5; ++i) + if (strcmp(text, names[i]) == 0) + return static_cast(i - 1); + return SD_VAE_FORMAT_COUNT; +} + +static sd_hires_upscaler_t parse_hires_upscaler(const char* text) { + static const char* names[] = {"none", "latent", "latent_nearest", "latent_nearest_exact", + "latent_antialiased", "latent_bicubic", "latent_bicubic_antialiased", + "lanczos", "nearest", "model"}; + for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; ++i) + if (strcmp(text, names[i]) == 0) + return static_cast(i); + return SD_HIRES_UPSCALER_COUNT; +} + +static bool valid_sigmas(const ParamValue& value) { + const auto& sigmas = std::get>(value); + if (sigmas.empty()) + return true; + if (sigmas.size() < 2 || sigmas.back() != 0.f) + return false; + for (size_t i = 0; i < sigmas.size(); ++i) + if (sigmas[i] < 0.f || (i && sigmas[i] >= sigmas[i - 1])) + return false; + return true; +} + +static bool valid_layers(const ParamValue& value) { + const auto& layers = std::get>(value); + return std::all_of(layers.begin(), layers.end(), [](int index) { return index >= 0; }); +} + +template +static ParamSpec parameter(const char* key, ParamType type, int operations, Apply apply) { + return {key, type, operations, [apply](const ParamValue& value, sd::OptConfig& config) { + apply(std::get(value), config); + }}; +} + +template +static ParamSpec integer_parameter(const char* key, int operations, Apply apply, int64_t min = INT64_MIN, int64_t max = INT64_MAX) { + auto spec = parameter(key, ParamType::Int, operations, apply); + spec.int_min = min; + spec.int_max = max; + return spec; +} + +template +static ParamSpec float_parameter(const char* key, int operations, Apply apply, double min = -std::numeric_limits::max(), double max = std::numeric_limits::max()) { + auto spec = parameter(key, ParamType::Float, operations, apply); + spec.float_min = min; + spec.float_max = max; + return spec; +} + +template +struct ArrayStorage; +template <> +struct ArrayStorage { + using Value = std::vector; + static constexpr auto type = ParamType::Floats; +}; +template <> +struct ArrayStorage { + using Value = std::vector; + static constexpr auto type = ParamType::Ints; +}; +template <> +struct ArrayStorage { + using Value = ImageValues; + static constexpr auto type = ParamType::Tensors; +}; +template <> +struct ArrayStorage { + using Value = AudioValues; + static constexpr auto type = ParamType::Audios; +}; +template <> +struct ArrayStorage { + using Value = VideoValues; + static constexpr auto type = ParamType::Videos; +}; +template <> +struct ArrayStorage { + using Value = LoraValues; + static constexpr auto type = ParamType::Loras; +}; +template <> +struct ArrayStorage { + using Value = EmbeddingValues; + static constexpr auto type = ParamType::Embeddings; +}; + +template +static const auto& array_items(const std::vector& value) { + return value; +} +static const auto& array_items(const ImageValues& value) { + return value.items; +} +static const auto& array_items(const AudioValues& value) { + return value.items; +} +static const auto& array_items(const VideoValues& value) { + return value.items; +} +static const auto& array_items(const LoraValues& value) { + return value.items; +} +static const auto& array_items(const EmbeddingValues& value) { + return value.items; +} + +template +class ParamGroup { + std::vector& specs; + int operations; + Select select; + + template + ParamSpec binding(const char* key, ParamType type, Field Object::* field, Convert convert) const { + return parameter(key, type, operations, [select = select, field, convert](const Value& value, sd::OptConfig& config) { + const auto decoded = convert(value); + std::apply([&](auto&... target) { ((target.*field = decoded), ...); }, select(config)); + }); + } + +public: + ParamGroup(std::vector& specs, int operations, Select select) + : specs(specs), operations(operations), select(select) {} + + ParamGroup for_operations(int mask) const { return {specs, mask, select}; } + + template + void boolean(const char* key, bool Object::* field) { + specs.push_back(binding(key, ParamType::Bool, field, [](bool value) { return value; })); + } + + template + void integer(const char* key, Field Object::* field, int64_t min = INT64_MIN, int64_t max = INT64_MAX) { + auto spec = binding(key, ParamType::Int, field, [](int64_t value) { return static_cast(value); }); + spec.int_min = min; + spec.int_max = max; + specs.push_back(std::move(spec)); + } + + template + void number(const char* key, float Object::* field, double min = -std::numeric_limits::max(), double max = std::numeric_limits::max()) { + auto spec = binding(key, ParamType::Float, field, [](double value) { return static_cast(value); }); + spec.float_min = min; + spec.float_max = max; + specs.push_back(std::move(spec)); + } + + template + void string(const char* key, const char* Object::* field) { + specs.push_back(binding(key, ParamType::String, field, [](const std::string& value) { return value.c_str(); })); + } + + template + void image(const char* key, sd::Image Object::* field) { + specs.push_back(binding(key, ParamType::Tensor, field, [](const TensorValue& value) { return sd::api::image_view(*value.tensor.get()); })); + } + + template + void choice(const char* key, Enum Object::* field, Enum (*parse)(const char*), Enum invalid, bool allow_auto = false) { + auto spec = binding(key, ParamType::String, field, [=](const std::string& value) { + return allow_auto && value == "auto" ? invalid : parse(value.c_str()); + }); + spec.valid = [=](const ParamValue& value) { + const auto& text = std::get(value); + return (allow_auto && text == "auto") || parse(text.c_str()) != invalid; + }; + specs.push_back(std::move(spec)); + } + + template + void frames(const char* key, Element* Object::* data, Count Object::* count) { + auto spec = parameter(key, ParamType::Tensor, operations, [select = select, data, count](const TensorValue& value, sd::OptConfig& config) { + std::apply([&](auto&... target) { + ((target.*data = const_cast(value.frames.data()), target.*count = static_cast(value.frames.size())), ...); + }, + select(config)); + }); + spec.video_tensor = true; + specs.push_back(std::move(spec)); + } + + template + void array(const char* key, Element* Object::* data, Count Object::* count, bool (*valid)(const ParamValue&) = nullptr, const char* reason = "invalid parameter value") { + using Storage = ArrayStorage>; + using Value = typename Storage::Value; + auto spec = parameter(key, Storage::type, operations, [select = select, data, count](const Value& value, sd::OptConfig& config) { + const auto& items = array_items(value); + std::apply([&](auto&... target) { + ((target.*data = const_cast(items.data()), target.*count = static_cast(items.size())), ...); + }, + select(config)); + }); + spec.valid = valid; + spec.invalid_reason = reason; + specs.push_back(std::move(spec)); + } +}; + +static void add_context_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup ctx(specs, Context, [](sd::OptConfig& config) { return std::tie(config.context); }); + + ctx.number("attn_scale", &sd::ContextConfig::attn_scale, 0, std::numeric_limits::max()); + ctx.string("audio_encoder_path", &sd::ContextConfig::audio_encoder_path); + ctx.string("audio_vae_path", &sd::ContextConfig::audio_vae_path); + ctx.boolean("auto_fit", &sd::ContextConfig::auto_fit); + ctx.boolean("batched_cfg", &sd::ContextConfig::batched_cfg); + ctx.string("clip_g_path", &sd::ContextConfig::clip_g_path); + ctx.string("clip_l_path", &sd::ContextConfig::clip_l_path); + ctx.string("clip_vision_path", &sd::ContextConfig::clip_vision_path); + ctx.integer("conditioning_cache_size", &sd::ContextConfig::conditioning_cache_size, 0, INT32_MAX); + ctx.string("control_net_path", &sd::ContextConfig::control_net_path); + ctx.boolean("diffusion_conv_direct", &sd::ContextConfig::diffusion_conv_direct); + ctx.boolean("diffusion_flash_attn", &sd::ContextConfig::diffusion_flash_attn); + ctx.boolean("disable_prefetch", &sd::ContextConfig::disable_prefetch); + ctx.boolean("disable_segmented_compute", &sd::ContextConfig::disable_segmented_compute); + ctx.boolean("eager_load", &sd::ContextConfig::eager_load); + ctx.array("embeddings", &sd::ContextConfig::embeddings, &sd::ContextConfig::embedding_count); + ctx.string("embeddings_connectors_path", &sd::ContextConfig::embeddings_connectors_path); + ctx.boolean("enable_mmap", &sd::ContextConfig::enable_mmap); + ctx.boolean("flash_attn", &sd::ContextConfig::flash_attn); + ctx.boolean("force_sdxl_vae_conv_scale", &sd::ContextConfig::force_sdxl_vae_conv_scale); + ctx.string("high_noise_diffusion_model_path", &sd::ContextConfig::high_noise_diffusion_model_path); + ctx.string("ip_adapter_path", &sd::ContextConfig::ip_adapter_path); + ctx.number("linear_scale", &sd::ContextConfig::linear_scale, 0, std::numeric_limits::max()); + ctx.string("llm_path", &sd::ContextConfig::llm_path); + ctx.string("llm_vision_path", &sd::ContextConfig::llm_vision_path); + ctx.choice("lora_apply_mode", &sd::ContextConfig::lora_apply_mode, str_to_lora_apply_mode, LORA_APPLY_MODE_COUNT); + ctx.string("max_vram", &sd::ContextConfig::max_vram); + ctx.string("model_args", &sd::ContextConfig::model_args); + ctx.for_operations(Context | Upscaler | ADetailer).string("model_path", &sd::ContextConfig::model_path); + ctx.string("motion_module_path", &sd::ContextConfig::motion_module_path); + ctx.string("photo_maker_path", &sd::ContextConfig::photo_maker_path); + ctx.choice("prediction", &sd::ContextConfig::prediction, str_to_prediction, PREDICTION_COUNT, true); + ctx.string("pulid_weights_path", &sd::ContextConfig::pulid_weights_path); + ctx.choice("rng_type", &sd::ContextConfig::rng_type, str_to_rng_type, RNG_TYPE_COUNT, true); + ctx.string("rpc_servers", &sd::ContextConfig::rpc_servers); + ctx.boolean("sage_attn", &sd::ContextConfig::sage_attn); + ctx.choice("sampler_rng_type", &sd::ContextConfig::sampler_rng_type, str_to_rng_type, RNG_TYPE_COUNT, true); + ctx.string("split_mode", &sd::ContextConfig::split_mode); + ctx.string("t5xxl_path", &sd::ContextConfig::t5xxl_path); + ctx.boolean("tae_preview_only", &sd::ContextConfig::tae_preview_only); + ctx.string("taesd_path", &sd::ContextConfig::taesd_path); + ctx.string("tensor_type_rules", &sd::ContextConfig::tensor_type_rules); + ctx.string("tokenizer", &sd::ContextConfig::tokenizer); + ctx.string("uncond_diffusion_model_path", &sd::ContextConfig::uncond_diffusion_model_path); + ctx.boolean("vae_conv_direct", &sd::ContextConfig::vae_conv_direct); + ctx.choice("vae_format", &sd::ContextConfig::vae_format, parse_vae_format, SD_VAE_FORMAT_COUNT); + ctx.choice("wtype", &sd::ContextConfig::wtype, str_to_sd_type, SD_TYPE_COUNT, true); + + specs.push_back(parameter("backend", ParamType::String, Context | Upscaler | VidUpscaler | ADetailer, [](const std::string& value, sd::OptConfig& config) { config.context.backend = value.c_str(); config.post.video_upscaler.backend = value.c_str(); })); + specs.push_back(parameter("conv_direct_mode", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.conv_direct_mode = value.c_str(); config.post.video_upscaler.conv_direct_mode = value.c_str(); })); + specs.push_back(parameter("diffusion_model_path", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.diffusion_model_path = value.c_str(); config.post.video_upscaler.diffusion_model_path = value.c_str(); })); + specs.push_back(parameter("flash_attn_mode", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.flash_attn_mode = value.c_str(); config.post.video_upscaler.flash_attn_mode = value.c_str(); })); + specs.push_back(integer_parameter( + "n_threads", Context | Upscaler | VidUpscaler | ADetailer, [](const int64_t& value, sd::OptConfig& config) { config.context.n_threads = static_cast(value); config.post.video_upscaler.n_threads = static_cast(value); }, 1, INT32_MAX)); + specs.push_back(parameter("params_backend", ParamType::String, Context | Upscaler | VidUpscaler | ADetailer, [](const std::string& value, sd::OptConfig& config) { config.context.params_backend = value.c_str(); config.post.video_upscaler.params_backend = value.c_str(); })); + specs.push_back(parameter("vae_path", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.vae_path = value.c_str(); config.post.video_upscaler.vae_path = value.c_str(); })); +} + +static void add_generation_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup img(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image); }); + ParamGroup vid(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video); }); + ParamGroup preprocess(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.image_preprocess, config.video.image_preprocess); }); + ParamGroup photomaker(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image.pm_params); }); + ParamGroup pulid(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image.pulid_params); }); + + img.integer("batch_count", &sd::ImageConfig::batch_count, 1, INT32_MAX); + img.image("control_image", &sd::ImageConfig::control_image); + img.number("control_strength", &sd::ImageConfig::control_strength, 0, std::numeric_limits::max()); + img.choice("control_type", &sd::ImageConfig::control_type, str_to_sd_control_type, SD_CONTROL_TYPE_COUNT); + img.image("ip_adapter_image", &sd::ImageConfig::ip_adapter_image); + img.number("ip_adapter_strength", &sd::ImageConfig::ip_adapter_strength, 0, std::numeric_limits::max()); + img.image("mask_image", &sd::ImageConfig::mask_image); + img.integer("qwen_image_layers", &sd::ImageConfig::qwen_image_layers, 0, INT32_MAX); + + vid.frames("control_frames", &sd::VideoConfig::control_frames, &sd::VideoConfig::control_frames_size); + vid.image("end_image", &sd::VideoConfig::end_image); + vid.number("moe_boundary", &sd::VideoConfig::moe_boundary, 0, 1); + vid.array("ref_audios", &sd::VideoConfig::ref_audios, &sd::VideoConfig::ref_audios_count); + vid.array("ref_videos", &sd::VideoConfig::ref_videos, &sd::VideoConfig::ref_videos_count); + vid.number("vace_strength", &sd::VideoConfig::vace_strength, 0, std::numeric_limits::max()); + vid.integer("video_frames", &sd::VideoConfig::video_frames, 1, INT32_MAX); + + preprocess.string("image_preprocess.rules", &sd::ImagePreprocessConfig::rules); + + photomaker.string("photomaker.id_embed_path", &sd::PhotoMakerConfig::id_embed_path); + photomaker.array("photomaker.id_images", &sd::PhotoMakerConfig::id_images, &sd::PhotoMakerConfig::id_images_count); + photomaker.number("photomaker.style_strength", &sd::PhotoMakerConfig::style_strength, 0, 100); + + pulid.string("pulid.id_embedding_path", &sd::PuLIDConfig::id_embedding_path); + pulid.number("pulid.id_weight", &sd::PuLIDConfig::id_weight, 0, std::numeric_limits::max()); + + specs.push_back(parameter("circular_x", ParamType::Bool, Image | Video | ADetail, [](const bool& value, sd::OptConfig& config) { config.image.circular_x = value; config.video.circular_x = value; })); + specs.push_back(parameter("circular_y", ParamType::Bool, Image | Video | ADetail, [](const bool& value, sd::OptConfig& config) { config.image.circular_y = value; config.video.circular_y = value; })); + specs.push_back(integer_parameter( + "clip_skip", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.clip_skip = static_cast(value); config.video.clip_skip = static_cast(value); }, INT32_MIN, INT32_MAX)); + specs.push_back(integer_parameter( + "fps", Video | VidUpscale, [](const int64_t& value, sd::OptConfig& config) { config.video.fps = static_cast(value); config.post.video_upscale.fps = static_cast(value); }, 1, INT32_MAX)); + specs.push_back(integer_parameter( + "height", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.height = static_cast(value); config.video.height = static_cast(value); }, 1, INT32_MAX)); + specs.push_back(parameter("init_image", ParamType::Tensor, Image | Video, [](const TensorValue& value, sd::OptConfig& config) { config.image.init_image = sd::api::image_view(*value.tensor.get()); config.video.init_image = config.image.init_image; })); + specs.push_back(parameter("loras", ParamType::Loras, Image | Video | ADetail, [](const LoraValues& value, sd::OptConfig& config) { config.image.loras = value.items.data(); config.image.lora_count = static_cast(value.items.size()); config.video.loras = value.items.data(); config.video.lora_count = static_cast(value.items.size()); })); + specs.push_back(parameter("negative_prompt", ParamType::String, Image | Video | ADetail, [](const std::string& value, sd::OptConfig& config) { config.image.negative_prompt = value.c_str(); config.video.negative_prompt = value.c_str(); })); + specs.push_back(parameter("prompt", ParamType::String, Image | Video | ADetail, [](const std::string& value, sd::OptConfig& config) { config.image.prompt = value.c_str(); config.video.prompt = value.c_str(); })); + specs.push_back(parameter("ref_image_args", ParamType::String, Image | Video, [](const std::string& value, sd::OptConfig& config) { config.image.ref_image_args = value.c_str(); config.video.ref_image_args = value.c_str(); })); + specs.push_back(parameter("ref_images", ParamType::Tensors, Image | Video, [](const ImageValues& value, sd::OptConfig& config) { config.image.ref_images = const_cast(value.items.data()); config.image.ref_images_count = static_cast(value.items.size()); config.video.ref_images = config.image.ref_images; config.video.ref_images_count = config.image.ref_images_count; })); + specs.push_back(integer_parameter("seed", Image | Video | VidUpscale | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.seed = static_cast(value); config.video.seed = static_cast(value); config.post.video_upscale.seed = value; })); + specs.push_back(float_parameter( + "strength", Image | Video | ADetail, [](const double& value, sd::OptConfig& config) { config.image.strength = static_cast(value); config.video.strength = static_cast(value); }, 0, 1)); + specs.push_back(integer_parameter( + "width", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.width = static_cast(value); config.video.width = static_cast(value); }, 1, INT32_MAX)); +} + +static void add_sampling_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup high_noise(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params); }); + ParamGroup high_noise_guidance(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params.guidance); }); + ParamGroup high_noise_slg(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params.guidance.slg); }); + ParamGroup sample(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params, config.video.sample_params); }); + ParamGroup guidance(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params.guidance, config.video.sample_params.guidance); }); + ParamGroup slg(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params.guidance.slg, config.video.sample_params.guidance.slg); }); + + high_noise.string("high_noise_sample.args", &sd::SampleConfig::extra_sample_args); + high_noise.number("high_noise_sample.eta", &sd::SampleConfig::eta, 0, std::numeric_limits::max()); + high_noise.choice("high_noise_sample.method", &sd::SampleConfig::sample_method, str_to_sample_method, SAMPLE_METHOD_COUNT, true); + high_noise.integer("high_noise_sample.steps", &sd::SampleConfig::sample_steps, -1, INT32_MAX); + + high_noise_guidance.number("high_noise_sample.guidance.distilled_guidance", &sd::GuidanceConfig::distilled_guidance); + high_noise_guidance.number("high_noise_sample.guidance.img_cfg", &sd::GuidanceConfig::img_cfg); + high_noise_guidance.number("high_noise_sample.guidance.txt_cfg", &sd::GuidanceConfig::txt_cfg); + + high_noise_slg.number("high_noise_sample.guidance.slg.layer_end", &sd::SLGConfig::layer_end, 0, 1); + high_noise_slg.number("high_noise_sample.guidance.slg.layer_start", &sd::SLGConfig::layer_start, 0, 1); + high_noise_slg.array("high_noise_sample.guidance.slg.layers", &sd::SLGConfig::layers, &sd::SLGConfig::layer_count, valid_layers, "layer indices must be nonnegative"); + high_noise_slg.number("high_noise_sample.guidance.slg.scale", &sd::SLGConfig::scale, 0, std::numeric_limits::max()); + + sample.string("sample.args", &sd::SampleConfig::extra_sample_args); + sample.number("sample.eta", &sd::SampleConfig::eta, 0, std::numeric_limits::max()); + sample.number("sample.flow_shift", &sd::SampleConfig::flow_shift, 0, std::numeric_limits::max()); + sample.choice("sample.method", &sd::SampleConfig::sample_method, str_to_sample_method, SAMPLE_METHOD_COUNT, true); + sample.choice("sample.scheduler", &sd::SampleConfig::scheduler, str_to_scheduler, SCHEDULER_COUNT, true); + sample.integer("sample.shifted_timestep", &sd::SampleConfig::shifted_timestep, 0, INT32_MAX); + sample.array("sample.sigmas", &sd::SampleConfig::custom_sigmas, &sd::SampleConfig::custom_sigmas_count, valid_sigmas, "sigmas require at least two strictly decreasing nonnegative entries ending in zero, or an empty array"); + sample.integer("sample.steps", &sd::SampleConfig::sample_steps, 1, INT32_MAX); + + guidance.number("sample.guidance.distilled_guidance", &sd::GuidanceConfig::distilled_guidance); + guidance.number("sample.guidance.img_cfg", &sd::GuidanceConfig::img_cfg); + guidance.number("sample.guidance.txt_cfg", &sd::GuidanceConfig::txt_cfg); + + slg.number("sample.guidance.slg.layer_end", &sd::SLGConfig::layer_end, 0, 1); + slg.number("sample.guidance.slg.layer_start", &sd::SLGConfig::layer_start, 0, 1); + slg.array("sample.guidance.slg.layers", &sd::SLGConfig::layers, &sd::SLGConfig::layer_count, valid_layers, "layer indices must be nonnegative"); + slg.number("sample.guidance.slg.scale", &sd::SLGConfig::scale, 0, std::numeric_limits::max()); +} + +static void add_cache_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup cache(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.cache, config.video.cache); }); + + cache.integer("cache.bn_compute_blocks", &sd::CacheConfig::Bn_compute_blocks, 0, INT32_MAX); + cache.number("cache.end_percent", &sd::CacheConfig::end_percent, 0, 1); + cache.number("cache.error_decay_rate", &sd::CacheConfig::error_decay_rate, 0, 1); + cache.integer("cache.fn_compute_blocks", &sd::CacheConfig::Fn_compute_blocks, 0, INT32_MAX); + cache.integer("cache.max_cached_steps", &sd::CacheConfig::max_cached_steps, INT32_MIN, INT32_MAX); + cache.integer("cache.max_continuous_cached_steps", &sd::CacheConfig::max_continuous_cached_steps, INT32_MIN, INT32_MAX); + cache.integer("cache.max_warmup_steps", &sd::CacheConfig::max_warmup_steps, 0, INT32_MAX); + cache.choice("cache.mode", &sd::CacheConfig::mode, parse_cache, static_cast(-1)); + cache.boolean("cache.reset_error_on_compute", &sd::CacheConfig::reset_error_on_compute); + cache.number("cache.residual_diff_threshold", &sd::CacheConfig::residual_diff_threshold, 0, std::numeric_limits::max()); + cache.number("cache.reuse_threshold", &sd::CacheConfig::reuse_threshold, 0, std::numeric_limits::max()); + cache.string("cache.scm_mask", &sd::CacheConfig::scm_mask); + cache.boolean("cache.scm_policy_dynamic", &sd::CacheConfig::scm_policy_dynamic); + cache.number("cache.spectrum_flex_window", &sd::CacheConfig::spectrum_flex_window, 0, std::numeric_limits::max()); + cache.number("cache.spectrum_lam", &sd::CacheConfig::spectrum_lam, 0, std::numeric_limits::max()); + cache.integer("cache.spectrum_m", &sd::CacheConfig::spectrum_m, 1, INT32_MAX); + cache.number("cache.spectrum_stop_percent", &sd::CacheConfig::spectrum_stop_percent, 0, 1); + cache.number("cache.spectrum_w", &sd::CacheConfig::spectrum_w, 0, std::numeric_limits::max()); + cache.integer("cache.spectrum_warmup_steps", &sd::CacheConfig::spectrum_warmup_steps, 0, INT32_MAX); + cache.integer("cache.spectrum_window_size", &sd::CacheConfig::spectrum_window_size, 1, INT32_MAX); + cache.number("cache.start_percent", &sd::CacheConfig::start_percent, 0, 1); + cache.integer("cache.taylorseer_n_derivatives", &sd::CacheConfig::taylorseer_n_derivatives, 0, INT32_MAX); + cache.integer("cache.taylorseer_skip_interval", &sd::CacheConfig::taylorseer_skip_interval, 1, INT32_MAX); + cache.boolean("cache.use_relative_threshold", &sd::CacheConfig::use_relative_threshold); +} + +static void add_hires_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup hires(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.hires, config.video.hires); }); + + hires.number("hires.denoising_strength", &sd::HiresConfig::denoising_strength, 0, 1); + hires.boolean("hires.enabled", &sd::HiresConfig::enabled); + hires.string("hires.model_path", &sd::HiresConfig::model_path); + hires.number("hires.scale", &sd::HiresConfig::scale, 0, std::numeric_limits::max()); + hires.array("hires.sigmas", &sd::HiresConfig::custom_sigmas, &sd::HiresConfig::custom_sigmas_count, valid_sigmas, "sigmas require at least two strictly decreasing nonnegative entries ending in zero, or an empty array"); + hires.integer("hires.steps", &sd::HiresConfig::steps, 0, INT32_MAX); + hires.integer("hires.target_height", &sd::HiresConfig::target_height, 0, INT32_MAX); + hires.integer("hires.target_width", &sd::HiresConfig::target_width, 0, INT32_MAX); + hires.integer("hires.upscale_tile_size", &sd::HiresConfig::upscale_tile_size, 1, INT32_MAX); + hires.choice("hires.upscaler", &sd::HiresConfig::upscaler, parse_hires_upscaler, SD_HIRES_UPSCALER_COUNT); +} + +static void add_tiling_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup tiling(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.vae_tiling_params, config.video.vae_tiling_params); }); + + tiling.boolean("vae_tiling.enabled", &sd::TilingConfig::enabled); + tiling.string("vae_tiling.extra_tiling_args", &sd::TilingConfig::extra_tiling_args); + tiling.number("vae_tiling.rel_size_h", &sd::TilingConfig::rel_size_h, 0, std::numeric_limits::max()); + tiling.number("vae_tiling.rel_size_w", &sd::TilingConfig::rel_size_w, 0, std::numeric_limits::max()); + tiling.number("vae_tiling.target_overlap", &sd::TilingConfig::target_overlap, 0, 0.99); + tiling.boolean("vae_tiling.temporal_tiling", &sd::TilingConfig::temporal_tiling); + tiling.integer("vae_tiling.tile_size_h", &sd::TilingConfig::tile_size_h, 0, INT32_MAX); + tiling.integer("vae_tiling.tile_size_w", &sd::TilingConfig::tile_size_w, 0, INT32_MAX); +} + +static void add_postprocess_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup input(specs, Upscale | ADetail, [](sd::OptConfig& config) { return std::tie(config.post); }); + ParamGroup upscaler(specs, Upscaler, [](sd::OptConfig& config) { return std::tie(config.post); }); + ParamGroup video_upscaler(specs, VidUpscaler, [](sd::OptConfig& config) { return std::tie(config.post.video_upscaler); }); + ParamGroup video_upscale(specs, VidUpscale, [](sd::OptConfig& config) { return std::tie(config.post.video_upscale); }); + ParamGroup adetail(specs, ADetail, [](sd::OptConfig& config) { return std::tie(config.post.adetail); }); + + input.image("image", &sd::PostprocessConfig::image); + + upscaler.boolean("conv_direct", &sd::PostprocessConfig::conv_direct); + upscaler.integer("tile_size", &sd::PostprocessConfig::tile_size, 1, INT32_MAX); + + video_upscaler.integer("upscale_factor", &sd::VideoUpscalerConfig::upscale_factor, 1, INT32_MAX); + + video_upscale.frames("frames", &sd::VideoUpscaleConfig::frames, &sd::VideoUpscaleConfig::frame_count); + video_upscale.integer("window_size", &sd::VideoUpscaleConfig::window_size, 1, INT32_MAX); + video_upscale.integer("temporal_chunk_size", &sd::VideoUpscaleConfig::temporal_chunk_size, 1, INT32_MAX); + + adetail.string("adetail.prompt", &sd::ADetailConfig::prompt); + adetail.string("adetail.negative_prompt", &sd::ADetailConfig::negative_prompt); + adetail.string("adetail.args", &sd::ADetailConfig::extra_ad_args); +} + +static std::vector make_specs() { + std::vector specs; + add_context_specs(specs); + add_generation_specs(specs); + add_sampling_specs(specs); + add_cache_specs(specs); + add_hires_specs(specs); + add_tiling_specs(specs); + add_postprocess_specs(specs); + return specs; +} + +static const ParamSpec* find_spec(const char* key) { + static const auto specs = make_specs(); + if (!key) + return nullptr; + for (const auto& spec : specs) + if (strcmp(key, spec.key) == 0) + return &spec; + return nullptr; +} + +static sd_status_t check_key(sd_opt_t* params, const char* key, ParamType type, const ParamSpec** out) { + if (!params || !key) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "params and key are required"); + *out = find_spec(key); + if (!*out) + return sd::api::error(SD_STATUS_UNKNOWN_KEY, key, "unknown parameter"); + if (!((*out)->operations & params->operation)) + return sd::api::error(SD_STATUS_UNSUPPORTED, key, "parameter does not apply to this option object"); + if ((*out)->type != type) + return sd::api::error(SD_STATUS_TYPE_MISMATCH, key, "incorrect setter type"); + return SD_STATUS_OK; +} + +static sd_status_t store_value(sd_opt_t* params, const ParamSpec& spec, ParamValue value) { + if (spec.type == ParamType::Int) { + const auto number = std::get(value); + if (number < spec.int_min || number > spec.int_max) + return sd::api::error(SD_STATUS_OUT_OF_RANGE, spec.key, "integer outside supported range"); + } + if (spec.type == ParamType::Float) { + const auto number = std::get(value); + if (!std::isfinite(number) || number < spec.float_min || number > spec.float_max) + return sd::api::error(SD_STATUS_OUT_OF_RANGE, spec.key, "float must be finite and within supported range"); + } + if (spec.valid && !spec.valid(value)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, spec.key, spec.invalid_reason); + params->values.insert_or_assign(spec.key, std::move(value)); + return SD_STATUS_OK; +} + +static sd_opt_t* create_opt(sd::api::Operation operation) { + sd_opt_t* params = nullptr; + sd::api::call([&] { params = new sd_opt_t{operation, {}}; return SD_STATUS_OK; }); + return params; +} + +sd_ctx_opt_t* sd_ctx_opt_create(void) { + return create_opt(sd::api::Context); +} +sd_img_gen_opt_t* sd_img_gen_opt_create(void) { + return create_opt(sd::api::Image); +} +sd_vid_gen_opt_t* sd_vid_gen_opt_create(void) { + return create_opt(sd::api::Video); +} +sd_upscaler_opt_t* sd_upscaler_opt_create(void) { + return create_opt(sd::api::Upscaler); +} + +sd_upscale_opt_t* sd_upscale_opt_create(void) { + return create_opt(sd::api::Upscale); +} + +sd_vid_upscaler_opt_t* sd_vid_upscaler_opt_create(void) { + return create_opt(sd::api::VidUpscaler); +} + +sd_vid_upscale_opt_t* sd_vid_upscale_opt_create(void) { + return create_opt(sd::api::VidUpscale); +} + +sd_adetailer_opt_t* sd_adetailer_opt_create(void) { + return create_opt(sd::api::ADetailer); +} + +sd_adetail_opt_t* sd_adetail_opt_create(void) { + return create_opt(sd::api::ADetail); +} + +void sd_opt_free(sd_opt_t* params) { + delete params; +} + +sd_status_t sd_opt_set_bool(sd_opt_t* params, const char* key, bool value) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::Bool, &spec); + if (status != SD_STATUS_OK) + return status; + return store_value(params, *spec, ParamValue(value)); + }); +} + +sd_status_t sd_opt_set_int(sd_opt_t* params, const char* key, int64_t value) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::Int, &spec); + if (status != SD_STATUS_OK) + return status; + return store_value(params, *spec, ParamValue(value)); + }); +} + +sd_status_t sd_opt_set_float(sd_opt_t* params, const char* key, double value) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::Float, &spec); + if (status != SD_STATUS_OK) + return status; + return store_value(params, *spec, ParamValue(value)); + }); +} + +sd_status_t sd_opt_set_string(sd_opt_t* params, const char* key, const char* value) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::String, &spec); + if (status != SD_STATUS_OK) + return status; + if (!value) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "string is NULL; use unset to restore defaults"); + return store_value(params, *spec, std::string(value)); + }); +} + +sd_status_t sd_opt_set_tensor(sd_opt_t* params, const char* key, const sd_tensor_t* tensor) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::Tensor, &spec); + if (status != SD_STATUS_OK) + return status; + if (!tensor) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "tensor is required"); + TensorValue value; + value.tensor = sd::api::TensorRef(tensor); + try { + if (spec->video_tensor) + value.frames = sd::api::video_frames(*tensor); + else + sd::api::image_view(*tensor); + } catch (const std::invalid_argument& error) { + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, error.what()); + } + return store_value(params, *spec, std::move(value)); + }); +} + +template +static sd_status_t set_array(sd_opt_t* params, const char* key, const T* values, size_t count, ParamType type, Function build) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, type, &spec); + if (status != SD_STATUS_OK) + return status; + if (count > INT32_MAX) + return sd::api::error(SD_STATUS_OUT_OF_RANGE, key, "too many array elements"); + if (!values && count) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "array is NULL with nonzero count"); + return build(*spec); + }); +} + +sd_status_t sd_opt_set_tensors(sd_opt_t* params, const char* key, const sd_tensor_t* const* values, size_t count) { + return set_array(params, key, values, count, ParamType::Tensors, [&](const ParamSpec& spec) { + ImageValues copy; + copy.tensors.reserve(count); + copy.items.reserve(count); + for (size_t i = 0; i < count; ++i) { + if (!values[i]) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "tensor is required"); + copy.items.push_back(sd::api::image_view(*values[i])); + copy.tensors.emplace_back(values[i]); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_floats(sd_opt_t* params, const char* key, const float* values, size_t count) { + return set_array(params, key, values, count, ParamType::Floats, [&](const ParamSpec& spec) { + std::vector copy; + for (size_t i = 0; i < count; ++i) { + if (!std::isfinite(values[i])) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "array values must be finite"); + copy.push_back(values[i]); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_ints(sd_opt_t* params, const char* key, const int32_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Ints, [&](const ParamSpec& spec) { + std::vector copy; + for (size_t i = 0; i < count; ++i) { + copy.push_back(values[i]); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_loras(sd_opt_t* params, const char* key, const sd_lora_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Loras, [&](const ParamSpec& spec) { + LoraValues copy; + copy.paths.reserve(count); + for (size_t i = 0; i < count; ++i) { + if (!values[i].path || !*values[i].path || !std::isfinite(values[i].multiplier)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "LoRA requires a path and finite multiplier"); + copy.paths.emplace_back(values[i].path); + copy.items.push_back({values[i].is_high_noise, values[i].multiplier, copy.paths.back().c_str()}); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_embeddings(sd_opt_t* params, const char* key, const sd_embedding_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Embeddings, [&](const ParamSpec& spec) { + EmbeddingValues copy; + copy.strings.reserve(count * 2); + for (size_t i = 0; i < count; ++i) { + if (!values[i].name || !*values[i].name || !values[i].path || !*values[i].path) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "embedding requires a name and path"); + copy.strings.emplace_back(values[i].name); + const char* name = copy.strings.back().c_str(); + copy.strings.emplace_back(values[i].path); + copy.items.push_back({name, copy.strings.back().c_str()}); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_audios(sd_opt_t* params, const char* key, const sd_audio_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Audios, [&](const ParamSpec& spec) { + AudioValues copy; + copy.tensors.reserve(count); + copy.items.reserve(count); + for (size_t i = 0; i < count; ++i) { + copy.items.push_back(sd::api::audio_view(values[i])); + copy.tensors.emplace_back(values[i].data); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_videos(sd_opt_t* params, const char* key, const sd_video_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Videos, [&](const ParamSpec& spec) { + VideoValues copy; + copy.frames.reserve(count); + copy.items.reserve(count); + for (size_t i = 0; i < count; ++i) { + const auto& video = values[i]; + if (!video.data || video.fps <= 0) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "video data and positive fps are required"); + copy.frames.push_back(sd::api::video_frames(*video.data)); + copy.tensors.emplace_back(video.data); + sd::Audio audio{}; + if (video.audio.data) { + audio = sd::api::audio_view(video.audio); + copy.tensors.emplace_back(video.audio.data); + } + copy.items.push_back({copy.frames.back().data(), static_cast(copy.frames.back().size()), video.fps, audio}); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_unset(sd_opt_t* params, const char* key) { + return sd::api::call([&] { + if (!params || !key) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "params and key are required"); + const auto* spec = find_spec(key); + if (!spec) + return sd::api::error(SD_STATUS_UNKNOWN_KEY, key, "unknown parameter"); + if (!(spec->operations & params->operation)) + return sd::api::error(SD_STATUS_UNSUPPORTED, key, "parameter does not apply to this option object"); + params->values.erase(key); + return SD_STATUS_OK; + }); +} + +bool sd::api::is_set(const sd_opt_t* params, const char* key) { + return params && params->values.count(key) != 0; +} + +bool sd::api::has_prefix(const sd_opt_t* params, const char* prefix) { + if (!params) + return false; + const auto it = params->values.lower_bound(prefix); + return it != params->values.end() && it->first.compare(0, strlen(prefix), prefix) == 0; +} + +sd_status_t sd::api::resolve(const sd_opt_t* params, Operation operation, OptConfig& config) { + if (!params) + return error(SD_STATUS_INVALID_ARGUMENT, nullptr, "params is required"); + if (params->operation != operation) + return error(SD_STATUS_UNSUPPORTED, nullptr, "option object has the wrong purpose for this operation"); + config = OptConfig{}; + if (operation == ADetail) + config.image.strength = 0.4f; + for (const auto& entry : params->values) { + const auto* spec = find_spec(entry.first.c_str()); + if (!(spec->operations & operation)) + return error(SD_STATUS_UNSUPPORTED, spec->key, "parameter does not apply to this operation"); + spec->apply(entry.second, config); + } + const auto& image = config.image; + const auto& video = config.video; + const bool sampling = operation == Image || operation == Video || operation == ADetail; + const auto& cache = operation == Video ? video.cache : image.cache; + if (sampling && cache.mode != SD_CACHE_DISABLED) { + if (cache.start_percent >= cache.end_percent) + return error(SD_STATUS_INVALID_ARGUMENT, "cache", "start_percent must precede end_percent"); + } + if (sampling) { + const auto& sample = operation == Video ? video.sample_params : image.sample_params; + if (sample.guidance.slg.layer_start > sample.guidance.slg.layer_end) + return error(SD_STATUS_INVALID_ARGUMENT, "sample.guidance.slg", "layer_start must not exceed layer_end"); + const auto& hires = operation == Video ? video.hires : image.hires; + if (hires.enabled && hires.upscaler != SD_HIRES_UPSCALER_NONE && + hires.scale <= 0.f && hires.target_width <= 0 && hires.target_height <= 0) + return error(SD_STATUS_INVALID_ARGUMENT, "hires", "enabled hires requires a positive scale or target size"); + } + if (operation == Video && video.high_noise_sample_params.guidance.slg.layer_start > video.high_noise_sample_params.guidance.slg.layer_end) + return error(SD_STATUS_INVALID_ARGUMENT, "high_noise_sample.guidance.slg", "layer_start must not exceed layer_end"); + if ((operation == Upscaler || operation == ADetailer) && (!config.context.model_path || !*config.context.model_path)) + return error(SD_STATUS_INVALID_ARGUMENT, "model_path", "model path is required"); + if (operation == VidUpscaler) { + const auto& load = config.post.video_upscaler; + if (!load.diffusion_model_path || !*load.diffusion_model_path || !load.vae_path || !*load.vae_path || load.upscale_factor <= 0) + return error(SD_STATUS_INVALID_ARGUMENT, nullptr, "diffusion_model_path, vae_path and upscale_factor are required"); + } + if ((operation == Upscale || operation == ADetail) && !config.post.image.data) + return error(SD_STATUS_INVALID_ARGUMENT, "image", "input image is required"); + if (operation == Upscale && config.post.image.channel != 3) + return error(SD_STATUS_INVALID_ARGUMENT, "image", "upscaling requires an RGB image"); + if (operation == VidUpscale) { + const auto& input = config.post.video_upscale; + if (!input.frames || input.frame_count <= 0) + return error(SD_STATUS_INVALID_ARGUMENT, "frames", "video upscaling requires nonempty frames"); + for (int i = 0; i < input.frame_count; ++i) + if (input.frames[i].channel != 3 || input.frames[i].width != input.frames[0].width || input.frames[i].height != input.frames[0].height) + return error(SD_STATUS_INVALID_ARGUMENT, "frames", "frames must be RGB and have equal dimensions"); + } + return SD_STATUS_OK; +} diff --git a/src/api/options.h b/src/api/options.h new file mode 100644 index 0000000000..6517829939 --- /dev/null +++ b/src/api/options.h @@ -0,0 +1,25 @@ +#ifndef SD_OPTIONS_H +#define SD_OPTIONS_H + +#include "api.h" +#include "config.h" + +namespace sd::api { + enum Operation { + Context = 1, + Image = 2, + Video = 4, + Upscaler = 8, + Upscale = 16, + VidUpscaler = 32, + VidUpscale = 64, + ADetailer = 128, + ADetail = 256 + }; + + sd_status_t resolve(const sd_opt_t* params, Operation operation, OptConfig& config); + bool is_set(const sd_opt_t* params, const char* key); + bool has_prefix(const sd_opt_t* params, const char* prefix); +} + +#endif // SD_OPTIONS_H diff --git a/src/api/result.cpp b/src/api/result.cpp new file mode 100644 index 0000000000..ebfbad474c --- /dev/null +++ b/src/api/result.cpp @@ -0,0 +1,241 @@ +#include "result.h" +#include "api.h" + +#include +#include +#include + +namespace sd { + using nlohmann::json; + + char* copy_generation_info(const json& info) { + const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace); + char* result = static_cast(malloc(text.size() + 1)); + if (result) + memcpy(result, text.c_str(), text.size() + 1); + return result; + } + + bool make_image_generation_result(sd::Image* images, int count, const std::vector& infos, sd::ImageGenerationResult** result_out) { + if (!result_out || !images || count <= 0 || infos.size() != static_cast(count)) { + sd::free_images(images, count); + return false; + } + auto* result = static_cast(calloc(1, sizeof(sd::ImageGenerationResult))); + if (!result) { + sd::free_images(images, count); + return false; + } + result->items = static_cast(calloc(count, sizeof(sd::ImageResult))); + if (!result->items) { + free(result); + sd::free_images(images, count); + return false; + } + result->count = count; + try { + for (int i = 0; i < count; ++i) { + result->items[i].image = images[i]; + images[i] = {}; + result->items[i].generation_info = copy_generation_info(infos[i]); + if (!result->items[i].generation_info) { + sd::free_images(images, count); + sd::free_image_generation_result(result); + return false; + } + } + } catch (...) { + sd::free_images(images, count); + sd::free_image_generation_result(result); + return false; + } + free(images); + *result_out = result; + return true; + } +} + +void sd::free_image_generation_result(sd::ImageGenerationResult* result) { + if (!result) + return; + for (int i = 0; i < result->count; ++i) { + free(result->items[i].image.data); + free(result->items[i].generation_info); + } + free(result->items); + free(result); +} + +void sd::free_video_generation_result(sd::VideoGenerationResult* result) { + if (!result) + return; + sd::free_images(result->frames, result->frame_count); + sd::free_audio(result->audio); + free(result->generation_info); + free(result); +} + +static sd::api::TensorRef consume_image(sd::Image& image) { + auto* data = image.data; + image.data = nullptr; + return sd::api::own_tensor(SD_TENSOR_U8, {image.channel, image.width, image.height}, data); +} + +bool sd::make_image_result(ImageGenerationResult* images, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + std::unique_ptr owner(images, free_image_generation_result); + if (!result_out || !images) + return false; + auto result = std::make_unique(); + result->kind = SD_RESULT_IMAGE; + result->images.reserve(images->count); + result->generation_infos.reserve(images->count); + for (int i = 0; i < images->count; ++i) { + result->images.push_back(consume_image(images->items[i].image)); + result->generation_infos.emplace_back(images->items[i].generation_info ? images->items[i].generation_info : ""); + } + *result_out = result.release(); + return true; +} + +bool sd::make_image_result(sd::Image* images, int count, const std::vector& infos, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + ImageGenerationResult* raw = nullptr; + if (!make_image_generation_result(images, count, infos, &raw)) + return false; + return make_image_result(raw, result_out); +} + +bool sd::make_video_result(VideoGenerationResult* video, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + std::unique_ptr owner(video, free_video_generation_result); + if (!result_out || !video) + return false; + auto result = std::make_unique(); + result->kind = SD_RESULT_VIDEO; + result->fps = video->fps; + result->images.reserve(video->frame_count); + for (int i = 0; i < video->frame_count; ++i) + result->images.push_back(consume_image(video->frames[i])); + result->generation_infos.emplace_back(video->generation_info ? video->generation_info : ""); + if (video->audio) { + auto& audio = *video->audio; + auto* data = audio.data; + audio.data = nullptr; + result->audio_tensor = sd::api::own_tensor(SD_TENSOR_F32, {audio.channels, static_cast(audio.sample_count)}, data); + result->audio = {audio.sample_rate, result->audio_tensor.get()}; + } + *result_out = result.release(); + return true; +} + +void sd::free_images(sd::Image* result_images, int num_images) { + if (result_images == nullptr) { + return; + } + + for (int i = 0; i < num_images; ++i) { + if (result_images[i].data != nullptr) { + free(result_images[i].data); + result_images[i].data = nullptr; + } + } + + free(result_images); +} + +void sd::free_audio(sd::Audio* audio) { + if (audio == nullptr) { + return; + } + free(audio->data); + audio->data = nullptr; + free(audio); +} + +void sd::free_image(sd::Image* image) { + if (image) { + free(image->data); + *image = {}; + } +} + +void sd_result_free(sd_result_t* result) { + delete result; +} + +sd_result_kind_t sd_result_get_kind(const sd_result_t* result) { + sd::api::clear_error(); + if (!result) { + sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result is required"); + return SD_RESULT_INVALID; + } + return result->kind; +} + +size_t sd_result_get_image_count(const sd_result_t* result) { + if (!sd_result_get_kind(result)) + return 0; + return result->images.size(); +} + +const sd_tensor_t* sd_result_get_image(const sd_result_t* result, size_t index) { + if (index >= sd_result_get_image_count(result)) { + sd::api::error(result ? SD_STATUS_OUT_OF_RANGE : SD_STATUS_INVALID_ARGUMENT, nullptr, "invalid result or image index"); + return nullptr; + } + return result->images[index].get(); +} + +const char* sd_result_get_generation_info(const sd_result_t* result, size_t index) { + if (!sd_result_get_image(result, index)) + return nullptr; + return result->generation_infos[result->kind == SD_RESULT_VIDEO ? 0 : index].c_str(); +} + +const sd_audio_t* sd_result_get_audio(const sd_result_t* result) { + if (!sd_result_get_kind(result)) + return nullptr; + return result->audio.data ? &result->audio : nullptr; +} + +int sd_result_get_fps(const sd_result_t* result) { + if (!sd_result_get_kind(result)) + return 0; + return result->fps; +} + +const sd_video_t* sd_result_get_video(const sd_result_t* result) { + const sd_video_t* output = nullptr; + sd::api::call([&] { + if (!result) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result is required"); + if (result->kind != SD_RESULT_VIDEO) + return sd::api::error(SD_STATUS_TYPE_MISMATCH, nullptr, "result is not a video"); + std::lock_guard lock(result->video_mutex); + if (!result->video_tensor) { + if (result->images.empty()) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "video has no frames"); + const auto& first = *result->images.front().get(); + auto shape = first.shape; + shape.push_back(static_cast(result->images.size())); + for (const auto& frame : result->images) + if (frame->type != first.type || frame->shape != first.shape) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "video frames must have identical shapes and types"); + auto packed = sd::api::allocate_tensor(first.type, std::move(shape)); + auto* destination = static_cast(packed.get()->mutable_data()); + for (const auto& frame : result->images) { + memcpy(destination, frame->data(), frame->bytes); + destination += frame->bytes; + } + result->video_tensor = std::move(packed); + result->video = {result->video_tensor.get(), result->fps, result->audio}; + } + output = &result->video; + return SD_STATUS_OK; + }); + return output; +} diff --git a/src/api/result.h b/src/api/result.h new file mode 100644 index 0000000000..452055b478 --- /dev/null +++ b/src/api/result.h @@ -0,0 +1,57 @@ +#ifndef SD_RESULT_H +#define SD_RESULT_H + +#include +#include +#include +#include +#include "tensor.h" + +#include "json.hpp" +#include "stable-diffusion.h" + +namespace sd { + struct ImageResult { + sd::Image image; + // Owned UTF-8 JSON using the sdcpp.generation/v1 schema. + char* generation_info; + }; + + struct ImageGenerationResult { + sd::ImageResult* items; + int count; + }; + + struct VideoGenerationResult { + sd::Image* frames; + int frame_count; + sd::Audio* audio; + int fps; + char* generation_info; + }; + + void free_image_generation_result(ImageGenerationResult* result); + void free_video_generation_result(VideoGenerationResult* result); + void free_images(sd::Image* images, int count); + + char* copy_generation_info(const nlohmann::json& info); + // Consumes images even on failure. + bool make_image_generation_result(sd::Image* images, int count, const std::vector& infos, sd::ImageGenerationResult** result_out); + bool make_image_result(ImageGenerationResult* images, sd_result_t** result_out); + bool make_image_result(sd::Image* images, int count, const std::vector& infos, sd_result_t** result_out); + bool make_video_result(VideoGenerationResult* video, sd_result_t** result_out); +} + +struct sd_result { + sd_result_kind_t kind = SD_RESULT_INVALID; + std::vector images; + std::vector generation_infos; + sd::api::TensorRef audio_tensor; + sd_audio_t audio{}; + int fps = 0; + mutable std::mutex video_mutex; + mutable sd::api::TensorRef video_tensor; + mutable sd_video_t video{}; +}; + +#endif // SD_RESULT_H diff --git a/src/api/tensor.cpp b/src/api/tensor.cpp new file mode 100644 index 0000000000..756753560f --- /dev/null +++ b/src/api/tensor.cpp @@ -0,0 +1,191 @@ +#include "tensor.h" +#include "api.h" + +#include +#include +#include +#include +#include + +static size_t element_size(sd_tensor_type_t type) { + switch (type) { + case SD_TENSOR_U8: + return sizeof(uint8_t); + case SD_TENSOR_I32: + return sizeof(int32_t); + case SD_TENSOR_I64: + return sizeof(int64_t); + case SD_TENSOR_F32: + return sizeof(float); + case SD_TENSOR_F64: + return sizeof(double); + default: + throw std::invalid_argument("invalid tensor type"); + } +} + +size_t sd::api::tensor_byte_size(sd_tensor_type_t type, const std::vector& shape) { + if (shape.size() > 8) + throw std::invalid_argument("tensor rank must not exceed 8"); + size_t count = 1; + for (int64_t dim : shape) { + if (dim < 0 || (count && uint64_t(dim) > SIZE_MAX / count)) + throw std::invalid_argument("negative or overflowing tensor dimensions"); + count *= static_cast(dim); + } + const size_t size = element_size(type); + if (count > uint64_t(INT64_MAX) || count > SIZE_MAX / size) + throw std::invalid_argument("tensor byte size overflow"); + return count * size; +} + +static sd_tensor_t::Storage allocate_storage(sd_tensor_type_t type, std::vector shape, size_t bytes) { + if (bytes && shape.empty()) + shape.push_back(1); + if (!bytes) + shape.clear(); + switch (type) { + case SD_TENSOR_U8: + return sd::Tensor(std::move(shape)); + case SD_TENSOR_I32: + return sd::Tensor(std::move(shape)); + case SD_TENSOR_I64: + return sd::Tensor(std::move(shape)); + case SD_TENSOR_F32: + return sd::Tensor(std::move(shape)); + case SD_TENSOR_F64: + return sd::Tensor(std::move(shape)); + default: + throw std::invalid_argument("invalid tensor type"); + } +} + +sd_tensor_t::sd_tensor_t(sd_tensor_type_t type, std::vector shape) + : type(type), shape(std::move(shape)), bytes(sd::api::tensor_byte_size(type, this->shape)), storage_(allocate_storage(type, this->shape, bytes)) {} + +static std::vector validated_shape(sd_tensor_type_t type, std::vector shape, const void* data, size_t size) { + if (size != sd::api::tensor_byte_size(type, shape) || (size && !data)) + throw std::invalid_argument("tensor data size does not match type and shape"); + return shape; +} + +sd_tensor_t::sd_tensor_t(sd_tensor_type_t type, std::vector shape, const void* data, size_t size) + : sd_tensor_t(type, validated_shape(type, std::move(shape), data, size)) { + if (bytes) + memcpy(mutable_data(), data, bytes); +} + +sd::api::TensorRef sd::api::allocate_tensor(sd_tensor_type_t type, std::vector shape) { + return TensorRef::adopt(new sd_tensor_t(type, std::move(shape))); +} + +sd::api::TensorRef sd::api::own_tensor(sd_tensor_type_t type, std::vector shape, void* data) { + std::unique_ptr owner(data, std::free); + const size_t bytes = tensor_byte_size(type, shape); + return TensorRef::adopt(new sd_tensor_t(type, std::move(shape), data, bytes)); +} + +sd::Image sd::api::image_view(const sd_tensor_t& tensor) { + const auto& s = tensor.shape; + if (tensor.type != SD_TENSOR_U8 || s.size() != 3 || s[0] < 1 || s[0] > 4 || + s[1] <= 0 || s[2] <= 0 || s[1] > INT32_MAX || s[2] > INT32_MAX || !tensor.data()) + throw std::invalid_argument("image requires U8 [channels,width,height], channels in [1,4], and positive dimensions"); + return {uint32_t(s[1]), uint32_t(s[2]), uint32_t(s[0]), const_cast(static_cast(tensor.data()))}; +} + +sd::Audio sd::api::audio_view(const sd_audio_t& audio) { + if (!audio.data || audio.sample_rate == 0) + throw std::invalid_argument("audio requires samples and a positive sample rate"); + const auto& t = *audio.data; + if (t.type != SD_TENSOR_F32 || t.shape.size() != 2 || t.shape[0] <= 0 || t.shape[0] > UINT32_MAX || t.shape[1] <= 0 || !t.data()) + throw std::invalid_argument("audio requires F32 [channels,samples] with positive dimensions"); + return {audio.sample_rate, uint32_t(t.shape[0]), uint64_t(t.shape[1]), const_cast(static_cast(t.data()))}; +} + +std::vector sd::api::video_frames(const sd_tensor_t& tensor) { + const auto& s = tensor.shape; + if (tensor.type != SD_TENSOR_U8 || s.size() != 4 || s[0] < 1 || s[0] > 4 || s[1] <= 0 || s[2] <= 0 || s[3] <= 0 || + s[1] > INT32_MAX || s[2] > INT32_MAX || s[3] > INT32_MAX || !tensor.data()) + throw std::invalid_argument("video requires U8 [channels,width,height,frames] with positive dimensions"); + const size_t stride = size_t(s[0]) * size_t(s[1]) * size_t(s[2]); + auto* data = const_cast(static_cast(tensor.data())); + std::vector result; + result.reserve(size_t(s[3])); + for (int64_t i = 0; i < s[3]; ++i) + result.push_back({uint32_t(s[1]), uint32_t(s[2]), uint32_t(s[0]), data + size_t(i) * stride}); + return result; +} + +void sd::api::preview_images(const std::function& callback, int step, int count, const sd::Image* images, bool noisy, void* data) { + std::vector tensors; + std::vector views; + tensors.reserve(count); + views.reserve(count); + for (int i = 0; i < count; ++i) { + const auto& image = images[i]; + const int64_t shape[] = {image.channel, image.width, image.height}; + auto* tensor = sd_tensor_create(SD_TENSOR_U8, 3, shape, image.data, size_t(image.channel) * image.width * image.height); + if (!tensor) + throw std::bad_alloc(); + tensors.push_back(TensorRef::adopt(tensor)); + views.push_back(tensors.back().get()); + } + callback(step, count, views.data(), noisy, data); +} + +sd_tensor_t* sd_tensor_create(sd_tensor_type_t type, size_t rank, const int64_t* shape, const void* data, size_t data_size) { + sd_tensor_t* result = nullptr; + sd::api::call([&] { + if (rank > 8 || (rank && !shape)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "invalid tensor shape"); + std::vector dimensions; + if (rank) + dimensions.assign(shape, shape + rank); + const size_t bytes = sd::api::tensor_byte_size(type, dimensions); + if (data_size != bytes || (bytes && !data)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "tensor data size does not match type and shape"); + result = new sd_tensor_t(type, std::move(dimensions), data, data_size); + return SD_STATUS_OK; + }); + return result; +} + +sd_tensor_t* sd_tensor_retain(const sd_tensor_t* tensor) { + if (tensor) + tensor->references.fetch_add(1, std::memory_order_relaxed); + return const_cast(tensor); +} + +void sd_tensor_free(sd_tensor_t* tensor) { + if (tensor && tensor->references.fetch_sub(1, std::memory_order_acq_rel) == 1) + delete tensor; +} + +static bool valid_tensor(const sd_tensor_t* tensor) { + sd::api::clear_error(); + if (!tensor) + sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "tensor is required"); + return tensor != nullptr; +} + +sd_tensor_type_t sd_tensor_type(const sd_tensor_t* tensor) { + return valid_tensor(tensor) ? tensor->type : 0; +} +size_t sd_tensor_rank(const sd_tensor_t* tensor) { + return valid_tensor(tensor) ? tensor->shape.size() : 0; +} +size_t sd_tensor_byte_size(const sd_tensor_t* tensor) { + return valid_tensor(tensor) ? tensor->bytes : 0; +} +const void* sd_tensor_data(const sd_tensor_t* tensor) { + return valid_tensor(tensor) ? tensor->data() : nullptr; +} +int64_t sd_tensor_dim(const sd_tensor_t* tensor, size_t axis) { + if (!valid_tensor(tensor)) + return 0; + if (axis >= tensor->shape.size()) { + sd::api::error(SD_STATUS_OUT_OF_RANGE, nullptr, "invalid tensor axis"); + return 0; + } + return tensor->shape[axis]; +} diff --git a/src/api/tensor.h b/src/api/tensor.h new file mode 100644 index 0000000000..9d5728708a --- /dev/null +++ b/src/api/tensor.h @@ -0,0 +1,93 @@ +#ifndef __SD_API_TENSOR_H__ +#define __SD_API_TENSOR_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "core/tensor.hpp" + +#include "core/media.h" +#include "stable-diffusion.h" + +namespace sd::api { + size_t tensor_byte_size(sd_tensor_type_t type, const std::vector& shape); +} + +struct sd_tensor_t { + using Storage = std::variant, sd::Tensor, sd::Tensor, sd::Tensor, sd::Tensor>; + mutable std::atomic references{1}; + sd_tensor_type_t type; + std::vector shape; + size_t bytes; + +private: + Storage storage_; + +public: + sd_tensor_t(sd_tensor_type_t type, std::vector shape); + sd_tensor_t(sd_tensor_type_t type, std::vector shape, const void* data, size_t size); + + template + explicit sd_tensor_t(sd::Tensor tensor) + : type(std::is_same_v ? SD_TENSOR_U8 : std::is_same_v ? SD_TENSOR_I32 + : std::is_same_v ? SD_TENSOR_I64 + : std::is_same_v ? SD_TENSOR_F32 + : SD_TENSOR_F64), + shape(tensor.shape()), + bytes(sd::api::tensor_byte_size(type, shape)), + storage_(std::move(tensor)) { + static_assert(std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v, "unsupported tensor type"); + if (std::get>(storage_).numel() != static_cast(bytes / sizeof(T))) + throw std::invalid_argument("tensor shape and storage size do not match"); + } + + const void* data() const { + return bytes ? std::visit([](const auto& tensor) -> const void* { return tensor.data(); }, storage_) : nullptr; + } + void* mutable_data() { + return bytes ? std::visit([](auto& tensor) -> void* { return tensor.data(); }, storage_) : nullptr; + } +}; + +namespace sd::api { + class TensorRef { + sd_tensor_t* tensor_ = nullptr; + + public: + TensorRef() = default; + explicit TensorRef(const sd_tensor_t* tensor) + : tensor_(sd_tensor_retain(tensor)) {} + static TensorRef adopt(sd_tensor_t* tensor) { + TensorRef result; + result.tensor_ = tensor; + return result; + } + TensorRef(const TensorRef& other) + : TensorRef(other.tensor_) {} + TensorRef(TensorRef&& other) noexcept + : tensor_(other.tensor_) { other.tensor_ = nullptr; } + TensorRef& operator=(TensorRef other) noexcept { + std::swap(tensor_, other.tensor_); + return *this; + } + ~TensorRef() { sd_tensor_free(tensor_); } + sd_tensor_t* get() const { return tensor_; } + const sd_tensor_t* operator->() const { return tensor_; } + explicit operator bool() const { return tensor_ != nullptr; } + }; + + size_t tensor_byte_size(sd_tensor_type_t type, const std::vector& shape); + TensorRef own_tensor(sd_tensor_type_t type, std::vector shape, void* data); + TensorRef allocate_tensor(sd_tensor_type_t type, std::vector shape); + sd::Image image_view(const sd_tensor_t& tensor); + sd::Audio audio_view(const sd_audio_t& audio); + std::vector video_frames(const sd_tensor_t& tensor); + void preview_images(const std::function& callback, int step, int count, const sd::Image* images, bool noisy, void* data); +} + +#endif // __SD_API_TENSOR_H__ diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index eeebb778e1..808375c4e6 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -1997,13 +1997,13 @@ struct LLMEmbedder : public Conditioner { : version(version) { if (!tokenizers.has(TokenizerConfig::MAIN)) { if (sd_version_is_lens(version)) { - throw std::runtime_error("Lens requires an external GPT-OSS tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); + throw std::runtime_error("Lens requires an external GPT-OSS tokenizer.json; pass --tokenizer FILE or set the tokenizer context option"); } if (sd_version_is_pid(version)) { - throw std::runtime_error("PiD requires an external Gemma 2 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); + throw std::runtime_error("PiD requires an external Gemma 2 tokenizer.json; pass --tokenizer FILE or set the tokenizer context option"); } if (version == VERSION_MOSS_IMAGE) { - throw std::runtime_error("MOSS-Image requires the official Qwen3.5 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); + throw std::runtime_error("MOSS-Image requires the official Qwen3.5 tokenizer.json; pass --tokenizer FILE or set the tokenizer context option"); } } LLM::LLMArch arch = LLM::LLMArch::QWEN2_5_VL; @@ -3643,7 +3643,7 @@ struct LLaDAImageEmbedder : public Conditioner { text_projection_prefix(text_projection_prefix), sigvq_prefix(sigvq_prefix) { if (!tokenizers.has(TokenizerConfig::MAIN)) { - throw std::runtime_error("LLaDA-Image requires an external LLaDA2 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); + throw std::runtime_error("LLaDA-Image requires an external LLaDA2 tokenizer.json; pass --tokenizer FILE or set the tokenizer context option"); } llm = std::make_shared(LLM::LLMArch::LLADA2_MOE, backend, diff --git a/src/config.cpp b/src/config.cpp new file mode 100644 index 0000000000..5cffb854eb --- /dev/null +++ b/src/config.cpp @@ -0,0 +1,133 @@ +#include "config.h" + +#include + +namespace sd { + + SampleConfig::SampleConfig() { + guidance.txt_cfg = 7.0f; + guidance.img_cfg = INFINITY; + guidance.distilled_guidance = 3.5f; + guidance.slg.layers = nullptr; + guidance.slg.layer_count = 0; + guidance.slg.layer_start = 0.01f; + guidance.slg.layer_end = 0.2f; + guidance.slg.scale = 0.f; + scheduler = SCHEDULER_COUNT; + sample_method = SAMPLE_METHOD_COUNT; + shifted_timestep = 0; + sample_steps = 20; + eta = INFINITY; + custom_sigmas = nullptr; + custom_sigmas_count = 0; + flow_shift = INFINITY; + extra_sample_args = nullptr; + } + + CacheConfig::CacheConfig() { + mode = SD_CACHE_DISABLED; + reuse_threshold = INFINITY; + start_percent = 0.15f; + end_percent = 0.95f; + error_decay_rate = 1.0f; + use_relative_threshold = true; + reset_error_on_compute = true; + Fn_compute_blocks = 8; + Bn_compute_blocks = 0; + residual_diff_threshold = 0.08f; + max_warmup_steps = 8; + max_cached_steps = -1; + max_continuous_cached_steps = -1; + taylorseer_n_derivatives = 1; + taylorseer_skip_interval = 1; + scm_mask = nullptr; + scm_policy_dynamic = true; + spectrum_w = 0.40f; + spectrum_m = 3; + spectrum_lam = 1.0f; + spectrum_window_size = 2; + spectrum_flex_window = 0.50f; + spectrum_warmup_steps = 4; + spectrum_stop_percent = 0.9f; + } + + HiresConfig::HiresConfig() { + enabled = false; + upscaler = SD_HIRES_UPSCALER_LATENT; + model_path = nullptr; + scale = 2.f; + target_width = 0; + target_height = 0; + steps = 0; + denoising_strength = 0.7f; + upscale_tile_size = 128; + custom_sigmas = nullptr; + custom_sigmas_count = 0; + } + + ContextConfig::ContextConfig() { + n_threads = sd_get_num_physical_cores(); + conditioning_cache_size = 4; + wtype = SD_TYPE_COUNT; + rng_type = CUDA_RNG; + sampler_rng_type = RNG_TYPE_COUNT; + prediction = PREDICTION_COUNT; + lora_apply_mode = LORA_APPLY_AUTO; + max_vram = nullptr; + disable_prefetch = false; + disable_segmented_compute = false; + batched_cfg = true; + eager_load = false; + enable_mmap = false; + flash_attn_mode = nullptr; + conv_direct_mode = nullptr; + diffusion_flash_attn = false; + sage_attn = false; + linear_scale = 0.f; + attn_scale = 0.f; + vae_format = SD_VAE_FORMAT_AUTO; + backend = nullptr; + params_backend = nullptr; + split_mode = nullptr; + auto_fit = true; + rpc_servers = nullptr; + model_args = nullptr; + pulid_weights_path = nullptr; + } + + ImageConfig::ImageConfig() { + clip_skip = -1; + ref_images_count = 0; + ref_image_args = ""; + width = 512; + height = 512; + strength = 0.75f; + seed = -1; + batch_count = 1; + control_strength = 0.9f; + ip_adapter_strength = 1.0f; + qwen_image_layers = 3; + circular_x = false; + circular_y = false; + pm_params = {nullptr, 0, nullptr, 20.f}; + pulid_params = {nullptr, 1.0f}; + vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; + } + + VideoConfig::VideoConfig() { + high_noise_sample_params.sample_steps = -1; + width = 512; + height = 512; + strength = 0.75f; + seed = -1; + video_frames = 6; + fps = 16; + moe_boundary = 0.875f; + vace_strength = 1.f; + vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; + circular_x = false; + circular_y = false; + ref_image_args = ""; + } + +} // namespace sd diff --git a/src/config.h b/src/config.h new file mode 100644 index 0000000000..7331ccc4b9 --- /dev/null +++ b/src/config.h @@ -0,0 +1,295 @@ +#ifndef SD_CONFIG_H +#define SD_CONFIG_H + +#include "core/media.h" + +#include "core/option_types.h" +#include "stable-diffusion.h" + +namespace sd { + struct TilingConfig { + bool enabled; + bool temporal_tiling; + // Spatial tile dimensions in image pixels for both encode and decode; 0 uses 256. + int tile_size_w; + int tile_size_h; + float target_overlap; + // Positive values override tile_size: <= 1 is a dimension fraction, > 1 a target tile count. + float rel_size_w; + float rel_size_h; + const char* extra_tiling_args; + }; + + struct ImagePreprocessConfig { + // Semicolon-separated target=...,key=value rules. NULL preserves defaults. + const char* rules; + }; + + struct SLGConfig { + int* layers; + size_t layer_count; + float layer_start; + float layer_end; + float scale; + }; + + struct GuidanceConfig { + float txt_cfg; + float img_cfg; + float distilled_guidance; + sd::SLGConfig slg; + }; + + struct SampleConfig { + sd::GuidanceConfig guidance; + enum scheduler_t scheduler; + enum sample_method_t sample_method; + int sample_steps; + float eta; + int shifted_timestep; + float* custom_sigmas; + int custom_sigmas_count; + float flow_shift; + const char* extra_sample_args; + + SampleConfig(); + }; + + struct PhotoMakerConfig { + sd::Image* id_images; + int id_images_count; + const char* id_embed_path; + float style_strength; + }; + + struct PuLIDConfig { + const char* id_embedding_path; + float id_weight; + }; + + struct CacheConfig { + enum sd_cache_mode_t mode; + float reuse_threshold; + float start_percent; + float end_percent; + float error_decay_rate; + bool use_relative_threshold; + bool reset_error_on_compute; + int Fn_compute_blocks; + int Bn_compute_blocks; + float residual_diff_threshold; + int max_warmup_steps; + int max_cached_steps; + int max_continuous_cached_steps; + int taylorseer_n_derivatives; + int taylorseer_skip_interval; + const char* scm_mask; + bool scm_policy_dynamic; + float spectrum_w; + int spectrum_m; + float spectrum_lam; + int spectrum_window_size; + float spectrum_flex_window; + int spectrum_warmup_steps; + float spectrum_stop_percent; + + CacheConfig(); + }; + + struct HiresConfig { + bool enabled; + enum sd_hires_upscaler_t upscaler; + const char* model_path; + float scale; + int target_width; + int target_height; + int steps; + float denoising_strength; + int upscale_tile_size; + float* custom_sigmas; + int custom_sigmas_count; + + HiresConfig(); + }; + + struct ContextConfig { + const char* model_path{}; + const char* clip_l_path{}; + const char* clip_g_path{}; + const char* clip_vision_path{}; + const char* t5xxl_path{}; + const char* llm_path{}; + const char* llm_vision_path{}; + const char* diffusion_model_path{}; + const char* high_noise_diffusion_model_path{}; + const char* uncond_diffusion_model_path{}; + const char* embeddings_connectors_path{}; + const char* vae_path{}; + const char* audio_vae_path{}; + const char* audio_encoder_path{}; + const char* taesd_path{}; + const char* control_net_path{}; + const char* ip_adapter_path{}; + const char* motion_module_path{}; + const sd_embedding_t* embeddings{}; + uint32_t embedding_count{}; + const char* photo_maker_path{}; + const char* pulid_weights_path{}; + const char* tensor_type_rules{}; + int n_threads{}; + enum sd_type_t wtype {}; + enum rng_type_t rng_type {}; + enum rng_type_t sampler_rng_type {}; + enum prediction_t prediction {}; + enum lora_apply_mode_t lora_apply_mode {}; + bool enable_mmap{}; + bool flash_attn{}; + bool diffusion_flash_attn{}; + bool tae_preview_only{}; + bool diffusion_conv_direct{}; + bool vae_conv_direct{}; + bool force_sdxl_vae_conv_scale{}; + enum sd_vae_format_t vae_format {}; + const char* max_vram{}; // Optional per-device GiB budget for managed weights and runner buffers; 0 uses live free VRAM without an explicit budget + bool disable_prefetch{}; // Disable asynchronous next-segment weight prefetch + bool eager_load{}; // Load all params into the params backend at model-load time instead of lazily on first use + const char* backend{}; + const char* params_backend{}; + const char* split_mode{}; // weight distribution for multi-device modules: layer (default) or row, or per-module assignments e.g. "diffusion=row" + bool auto_fit{}; + const char* rpc_servers{}; + const char* model_args{}; + bool disable_segmented_compute{}; // Force monolithic graph execution even when automatic graph cutting would fit memory better + bool batched_cfg{}; // Run the conditional and unconditional CFG branches in one batched UNet forward when supported + float linear_scale{}; // Override linear input scaling; 0 keeps the model default + float attn_scale{}; // Override flash-attention K/V scaling; 0 keeps the model default + const char* tokenizer{}; // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens + bool sage_attn{}; + int conditioning_cache_size{}; // Maximum cached conditioning entries per context; 0 disables caching (default: 4) + const char* flash_attn_mode{}; // on/off/auto or module assignments (e.g. "auto,te=off"); NULL/empty honors legacy flags. + const char* conv_direct_mode{}; // on/off/auto or diffusion/vae/controlnet assignments; NULL/empty honors legacy flags. + + ContextConfig(); + }; + + struct ImageConfig { + const sd_lora_t* loras{}; + uint32_t lora_count{}; + const char* prompt{}; + const char* negative_prompt{}; + int clip_skip{}; + sd::Image init_image{}; + sd::Image* ref_images{}; + int ref_images_count{}; + const char* ref_image_args{}; + sd::Image mask_image{}; + int width{}; + int height{}; + sd::SampleConfig sample_params{}; + float strength{}; + int64_t seed{}; + int batch_count{}; + sd::Image control_image{}; + float control_strength{}; + sd::Image ip_adapter_image{}; + float ip_adapter_strength{}; + sd::PhotoMakerConfig pm_params{}; + sd::PuLIDConfig pulid_params{}; + sd::TilingConfig vae_tiling_params{}; + sd::CacheConfig cache{}; + sd::HiresConfig hires{}; + int qwen_image_layers{}; + bool circular_x{}; + bool circular_y{}; + sd::ImagePreprocessConfig image_preprocess{}; + enum sd_control_type_t control_type {}; + + ImageConfig(); + }; + + struct VideoConfig { + const sd_lora_t* loras{}; + uint32_t lora_count{}; + const char* prompt{}; + const char* negative_prompt{}; + int clip_skip{}; + sd::Image init_image{}; + sd::Image end_image{}; + sd::Image* ref_images{}; + int ref_images_count{}; + sd::ReferenceVideo* ref_videos{}; + int ref_videos_count{}; + sd::Audio* ref_audios{}; + int ref_audios_count{}; + sd::Image* control_frames{}; + int control_frames_size{}; + int width{}; + int height{}; + sd::SampleConfig sample_params{}; + sd::SampleConfig high_noise_sample_params{}; + float moe_boundary{}; + float strength{}; + int64_t seed{}; + int video_frames{}; + int fps{}; + float vace_strength{}; + sd::TilingConfig vae_tiling_params{}; + sd::CacheConfig cache{}; + sd::HiresConfig hires{}; + bool circular_x{}; + bool circular_y{}; + sd::ImagePreprocessConfig image_preprocess{}; + const char* ref_image_args{}; + + VideoConfig(); + }; + + struct VideoUpscalerConfig { + const char* diffusion_model_path{}; + const char* vae_path{}; + // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints. + int upscale_factor{}; + int n_threads{}; + const char* backend{}; + const char* params_backend{}; + // auto/on/off or diffusion/vae assignments; null selects auto. + const char* flash_attn_mode{}; + const char* conv_direct_mode{}; + }; + + struct VideoUpscaleConfig { + const sd::Image* frames{}; + int frame_count{}; + int fps = 24; + int64_t seed = 42; + // RelayVSR reference window, including both endpoints; defaults to 16. + int window_size = 16; + // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5. + int temporal_chunk_size = 5; + }; + + struct ADetailConfig { + const char* prompt{}; + const char* negative_prompt{}; + const char* extra_ad_args{}; + }; + + struct PostprocessConfig { + sd::Image image{}; + bool conv_direct = false; + int tile_size = 128; + VideoUpscalerConfig video_upscaler; + VideoUpscaleConfig video_upscale; + ADetailConfig adetail; + }; + + struct OptConfig { + ContextConfig context; + ImageConfig image; + VideoConfig video; + PostprocessConfig post; + }; + +} // namespace sd + +#endif // SD_CONFIG_H diff --git a/src/core/backend_fit.cpp b/src/core/backend_fit.cpp index 120cbe423f..123894de21 100644 --- a/src/core/backend_fit.cpp +++ b/src/core/backend_fit.cpp @@ -1,4 +1,5 @@ #include "backend_fit.h" +#include "config.h" #include #include @@ -489,7 +490,7 @@ namespace sd::backend_fit { return true; } - bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params, + bool prepare_vae_decode_retry_tiling(sd::TilingConfig& tiling_params, bool prefer_temporal_tiling, ggml_status status, int latent_tile_size_w, diff --git a/src/core/backend_fit.h b/src/core/backend_fit.h index ad1b60c614..61393ad6b2 100644 --- a/src/core/backend_fit.h +++ b/src/core/backend_fit.h @@ -3,6 +3,7 @@ #include +#include "config.h" #include "core/ggml_graph_cut.h" #include "model_loader.h" #include "stable-diffusion.h" @@ -15,7 +16,7 @@ namespace sd::backend_fit { std::string& runtime_spec, std::string& params_spec); - bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params, + bool prepare_vae_decode_retry_tiling(sd::TilingConfig& tiling_params, bool prefer_temporal_tiling, ggml_status status, int latent_tile_size_w, diff --git a/src/core/cuda/int8.cu b/src/core/cuda/int8.cu index 1b1ee67d4a..46afe433a4 100644 --- a/src/core/cuda/int8.cu +++ b/src/core/cuda/int8.cu @@ -339,9 +339,9 @@ namespace SD_CUDA_NAMESPACE { } else { rotated = values[indices[i]]; } - int value = row_scale == 0.0f ? 0 : __float2int_rn(rotated * inv_scale); - value = max(-127, min(127, value)); - output[indices[i]] = (int8_t)value; + int value = row_scale == 0.0f ? 0 : __float2int_rn(rotated * inv_scale); + value = max(-127, min(127, value)); + output[indices[i]] = (int8_t)value; } if (group == 0 && tid == 0 && dst_stride != k) { *((float*)(dst + row * dst_stride + k)) = row_scale; diff --git a/src/core/ggml_tensor_utils.cpp b/src/core/ggml_tensor_utils.cpp index 5639c7ed98..44846c6b65 100644 --- a/src/core/ggml_tensor_utils.cpp +++ b/src/core/ggml_tensor_utils.cpp @@ -200,7 +200,7 @@ uint8_t* ggml_tensor_to_sd_image(ggml_tensor* input, int idx, bool video) { return image_data; } -void sd_image_to_ggml_tensor(sd_image_t image, +void sd_image_to_ggml_tensor(sd::Image image, ggml_tensor* tensor, bool scale) { GGML_ASSERT(image.width == tensor->ne[0]); diff --git a/src/core/ggml_tensor_utils.h b/src/core/ggml_tensor_utils.h index 65cc5f2f48..f7b101ce1b 100644 --- a/src/core/ggml_tensor_utils.h +++ b/src/core/ggml_tensor_utils.h @@ -1,6 +1,8 @@ #ifndef __SD_CORE_GGML_TENSOR_UTILS_H__ #define __SD_CORE_GGML_TENSOR_UTILS_H__ +#include "core/media.h" + #include #include #include @@ -58,7 +60,7 @@ __STATIC_INLINE__ ggml_fp16_t ggml_ext_tensor_get_f16(const ggml_tensor* tensor, return *(ggml_fp16_t*)((char*)(tensor->data) + i3 * tensor->nb[3] + i2 * tensor->nb[2] + i1 * tensor->nb[1] + i0 * tensor->nb[0]); } -__STATIC_INLINE__ float sd_image_get_f32(sd_image_t image, int64_t iw, int64_t ih, int64_t ic, bool scale = true) { +__STATIC_INLINE__ float sd_image_get_f32(sd::Image image, int64_t iw, int64_t ih, int64_t ic, bool scale = true) { float value = *(image.data + ih * image.width * image.channel + iw * image.channel + ic); if (scale) { value /= 255.f; @@ -146,7 +148,7 @@ uint8_t* ggml_tensor_to_sd_image(ggml_tensor* input, uint8_t* image_data = nullp uint8_t* ggml_tensor_to_sd_image(ggml_tensor* input, int idx, bool video = false); -void sd_image_to_ggml_tensor(sd_image_t image, +void sd_image_to_ggml_tensor(sd::Image image, ggml_tensor* tensor, bool scale = true); diff --git a/src/core/media.h b/src/core/media.h new file mode 100644 index 0000000000..3ee1d4ebe2 --- /dev/null +++ b/src/core/media.h @@ -0,0 +1,33 @@ +#ifndef __SD_CORE_MEDIA_H__ +#define __SD_CORE_MEDIA_H__ + +#include + +namespace sd { + struct Image { + uint32_t width; + uint32_t height; + uint32_t channel; + uint8_t* data; + }; + + struct Audio { + uint32_t sample_rate; + uint32_t channels; + uint64_t sample_count; + float* data; + }; + + struct ReferenceVideo { + Image* frames; + int frame_count; + int fps; + Audio audio; + }; + + bool preprocess_canny(Image image, float high_threshold, float low_threshold, float weak, float strong, bool inverse); + void free_image(Image* image); + void free_audio(Audio* audio); +} + +#endif // __SD_CORE_MEDIA_H__ diff --git a/src/core/option_types.cpp b/src/core/option_types.cpp new file mode 100644 index 0000000000..2c9a1c1bc9 --- /dev/null +++ b/src/core/option_types.cpp @@ -0,0 +1,265 @@ +#include "option_types.h" + +#include + +static constexpr const char* rng_type_to_str[] = { + "std_default", + "cuda", + "cpu", +}; + +const char* sd_rng_type_name(enum rng_type_t rng_type) { + if (rng_type < RNG_TYPE_COUNT) { + return rng_type_to_str[rng_type]; + } + return "NONE"; +} + +enum rng_type_t str_to_rng_type(const char* str) { + for (int i = 0; i < RNG_TYPE_COUNT; i++) { + if (!strcmp(str, rng_type_to_str[i])) { + return (enum rng_type_t)i; + } + } + return RNG_TYPE_COUNT; +} + +static constexpr const char* sample_method_to_str[] = { + "euler", + "euler_a", + "heun", + "dpm2", + "dpm++2s_a", + "dpm++2m", + "dpm++2mv2", + "ipndm", + "ipndm_v", + "lcm", + "ddim_trailing", + "tcd", + "res_multistep", + "res_2s", + "er_sde", + "euler_cfg_pp", + "euler_a_cfg_pp", + "euler_ge", + "dpm++2m_sde", + "dpm++2m_sde_bt", + "lms", +}; + +static_assert(SAMPLE_METHOD_COUNT == sizeof(sample_method_to_str) / sizeof(sample_method_to_str[0]), + "\nnumber of elements in sample_method_to_str[] != SAMPLE_METHOD_COUNT"); + +const char* sd_sample_method_name(enum sample_method_t sample_method) { + if (sample_method < SAMPLE_METHOD_COUNT) { + return sample_method_to_str[sample_method]; + } + return "NONE"; +} + +enum sample_method_t str_to_sample_method(const char* str) { + for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) { + if (!strcmp(str, sample_method_to_str[i])) { + return (enum sample_method_t)i; + } + } + return SAMPLE_METHOD_COUNT; +} + +static constexpr const char* scheduler_to_str[] = { + "discrete", + "karras", + "exponential", + "ays", + "gits", + "sgm_uniform", + "simple", + "smoothstep", + "kl_optimal", + "lcm", + "bong_tangent", + "ltx2", + "logit_normal", + "flux2", + "flux", + "beta", + "llada_image", +}; + +static_assert(SCHEDULER_COUNT == sizeof(scheduler_to_str) / sizeof(scheduler_to_str[0]), + "\nnumber of elements in scheduler_to_str[] != SCHEDULER_COUNT"); + +const char* sd_scheduler_name(enum scheduler_t scheduler) { + if (scheduler < SCHEDULER_COUNT) { + return scheduler_to_str[scheduler]; + } + return "NONE"; +} + +enum scheduler_t str_to_scheduler(const char* str) { + if (!strcmp(str, "normal")) { + return DISCRETE_SCHEDULER; + } + for (int i = 0; i < SCHEDULER_COUNT; i++) { + if (!strcmp(str, scheduler_to_str[i])) { + return (enum scheduler_t)i; + } + } + return SCHEDULER_COUNT; +} + +static constexpr const char* prediction_to_str[] = { + "eps", + "v", + "edm_v", + "sd3_flow", + "flux_flow", + "sefi_flow", + "minit2i_flow", + "sensenova_u1_flow", + "flux3_flow", + "h3_av_flow", + "ming_image_flow", +}; + +const char* sd_prediction_name(enum prediction_t prediction) { + if (prediction >= EPS_PRED && prediction < PREDICTION_COUNT) { + return prediction_to_str[prediction]; + } + return "NONE"; +} + +enum prediction_t str_to_prediction(const char* str) { + for (int i = 0; i < PREDICTION_COUNT; i++) { + if (!strcmp(str, prediction_to_str[i])) { + return (enum prediction_t)i; + } + } + return PREDICTION_COUNT; +} + +static constexpr const char* preview_to_str[] = { + "none", + "proj", + "tae", + "vae", +}; + +const char* sd_preview_name(enum preview_t preview) { + if (preview < PREVIEW_COUNT) { + return preview_to_str[preview]; + } + return "NONE"; +} + +enum preview_t str_to_preview(const char* str) { + for (int i = 0; i < PREVIEW_COUNT; i++) { + if (!strcmp(str, preview_to_str[i])) { + return (enum preview_t)i; + } + } + return PREVIEW_COUNT; +} + +static constexpr const char* lora_apply_mode_to_str[] = { + "auto", + "immediately", + "at_runtime", +}; + +const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode) { + if (mode < LORA_APPLY_MODE_COUNT) { + return lora_apply_mode_to_str[mode]; + } + return "NONE"; +} + +enum lora_apply_mode_t str_to_lora_apply_mode(const char* str) { + for (int i = 0; i < LORA_APPLY_MODE_COUNT; i++) { + if (!strcmp(str, lora_apply_mode_to_str[i])) { + return (enum lora_apply_mode_t)i; + } + } + return LORA_APPLY_MODE_COUNT; +} + +static constexpr const char* hires_upscaler_to_str[] = { + "None", + "Latent", + "Latent (nearest)", + "Latent (nearest-exact)", + "Latent (antialiased)", + "Latent (bicubic)", + "Latent (bicubic antialiased)", + "Lanczos", + "Nearest", + "Model", +}; + +const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler) { + if (upscaler >= SD_HIRES_UPSCALER_NONE && upscaler < SD_HIRES_UPSCALER_COUNT) { + return hires_upscaler_to_str[upscaler]; + } + return "NONE"; +} + +enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str) { + for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; i++) { + if (!strcmp(str, hires_upscaler_to_str[i])) { + return (enum sd_hires_upscaler_t)i; + } + } + return SD_HIRES_UPSCALER_COUNT; +} + +static constexpr const char* control_type_to_str[] = { + "none", + "openpose", + "depth", + "scribble", + "canny", + "normal", + "segment", + "tile", + "repaint", +}; + +static_assert(SD_CONTROL_TYPE_COUNT == sizeof(control_type_to_str) / sizeof(control_type_to_str[0]), + "\nnumber of elements in control_type_to_str[] != SD_CONTROL_TYPE_COUNT"); + +const char* sd_control_type_name(enum sd_control_type_t control_type) { + if (control_type >= SD_CONTROL_TYPE_NONE && control_type < SD_CONTROL_TYPE_COUNT) { + return control_type_to_str[control_type]; + } + return "NONE"; +} + +enum sd_control_type_t str_to_sd_control_type(const char* str) { + for (int i = 0; i < SD_CONTROL_TYPE_COUNT; i++) { + if (!strcmp(str, control_type_to_str[i])) { + return (enum sd_control_type_t)i; + } + } + static const struct { + const char* alias; + enum sd_control_type_t type; + } aliases[] = { + {"hed", SD_CONTROL_TYPE_SCRIBBLE}, + {"softedge", SD_CONTROL_TYPE_SCRIBBLE}, + {"pidi", SD_CONTROL_TYPE_SCRIBBLE}, + {"ted", SD_CONTROL_TYPE_SCRIBBLE}, + {"lineart", SD_CONTROL_TYPE_CANNY}, + {"anime_lineart", SD_CONTROL_TYPE_CANNY}, + {"mlsd", SD_CONTROL_TYPE_CANNY}, + {"seg", SD_CONTROL_TYPE_SEGMENT}, + {"inpaint", SD_CONTROL_TYPE_REPAINT}, + {"outpaint", SD_CONTROL_TYPE_REPAINT}, + }; + for (const auto& entry : aliases) { + if (!strcmp(str, entry.alias)) { + return entry.type; + } + } + return SD_CONTROL_TYPE_COUNT; +} diff --git a/src/core/option_types.h b/src/core/option_types.h new file mode 100644 index 0000000000..280cde237d --- /dev/null +++ b/src/core/option_types.h @@ -0,0 +1,165 @@ +#ifndef __SD_CORE_OPTION_TYPES_H__ +#define __SD_CORE_OPTION_TYPES_H__ + +#include "stable-diffusion.h" + +enum rng_type_t { + STD_DEFAULT_RNG, + CUDA_RNG, + CPU_RNG, + RNG_TYPE_COUNT +}; + +enum sample_method_t { + EULER_SAMPLE_METHOD, + EULER_A_SAMPLE_METHOD, + HEUN_SAMPLE_METHOD, + DPM2_SAMPLE_METHOD, + DPMPP2S_A_SAMPLE_METHOD, + DPMPP2M_SAMPLE_METHOD, + DPMPP2Mv2_SAMPLE_METHOD, + IPNDM_SAMPLE_METHOD, + IPNDM_V_SAMPLE_METHOD, + LCM_SAMPLE_METHOD, + DDIM_TRAILING_SAMPLE_METHOD, + TCD_SAMPLE_METHOD, + RES_MULTISTEP_SAMPLE_METHOD, + RES_2S_SAMPLE_METHOD, + ER_SDE_SAMPLE_METHOD, + EULER_CFG_PP_SAMPLE_METHOD, + EULER_A_CFG_PP_SAMPLE_METHOD, + EULER_GE_SAMPLE_METHOD, + DPMPP2M_SDE_SAMPLE_METHOD, + DPMPP2M_SDE_BT_SAMPLE_METHOD, + LMS_SAMPLE_METHOD, + SAMPLE_METHOD_COUNT +}; + +enum scheduler_t { + DISCRETE_SCHEDULER, + KARRAS_SCHEDULER, + EXPONENTIAL_SCHEDULER, + AYS_SCHEDULER, + GITS_SCHEDULER, + SGM_UNIFORM_SCHEDULER, + SIMPLE_SCHEDULER, + SMOOTHSTEP_SCHEDULER, + KL_OPTIMAL_SCHEDULER, + LCM_SCHEDULER, + BONG_TANGENT_SCHEDULER, + LTX2_SCHEDULER, + LOGIT_NORMAL_SCHEDULER, + FLUX2_SCHEDULER, + FLUX_SCHEDULER, + BETA_SCHEDULER, + LLADA_IMAGE_SCHEDULER, + SCHEDULER_COUNT +}; + +enum prediction_t { + EPS_PRED, + V_PRED, + EDM_V_PRED, + FLOW_PRED, + FLUX_FLOW_PRED, + SEFI_FLOW_PRED, + MINIT2I_FLOW_PRED, + SENSENOVA_U1_FLOW_PRED, + FLUX3_FLOW_PRED, + H3_AV_FLOW_PRED, + MING_IMAGE_FLOW_PRED, + PREDICTION_COUNT +}; + +enum lora_apply_mode_t { + LORA_APPLY_AUTO, + LORA_APPLY_IMMEDIATELY, + LORA_APPLY_AT_RUNTIME, + LORA_APPLY_MODE_COUNT, +}; + +enum sd_vae_format_t { + SD_VAE_FORMAT_AUTO = -1, + SD_VAE_FORMAT_FLUX, + SD_VAE_FORMAT_SD3, + SD_VAE_FORMAT_FLUX2, + SD_VAE_FORMAT_WAN, + SD_VAE_FORMAT_COUNT, +}; + +enum sd_cache_mode_t { + SD_CACHE_DISABLED = 0, + SD_CACHE_EASYCACHE, + SD_CACHE_UCACHE, + SD_CACHE_DBCACHE, + SD_CACHE_TAYLORSEER, + SD_CACHE_CACHE_DIT, + SD_CACHE_SPECTRUM, +}; + +enum sd_hires_upscaler_t { + SD_HIRES_UPSCALER_NONE, + SD_HIRES_UPSCALER_LATENT, + SD_HIRES_UPSCALER_LATENT_NEAREST, + SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT, + SD_HIRES_UPSCALER_LATENT_ANTIALIASED, + SD_HIRES_UPSCALER_LATENT_BICUBIC, + SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED, + SD_HIRES_UPSCALER_LANCZOS, + SD_HIRES_UPSCALER_NEAREST, + SD_HIRES_UPSCALER_MODEL, + SD_HIRES_UPSCALER_COUNT, +}; + +enum sd_control_type_t { + SD_CONTROL_TYPE_NONE, + SD_CONTROL_TYPE_OPENPOSE, + SD_CONTROL_TYPE_DEPTH, + SD_CONTROL_TYPE_SCRIBBLE, // thick lines: scribble, hed, softedge, ted + SD_CONTROL_TYPE_CANNY, // thin lines: canny, lineart, anime lineart, mlsd + SD_CONTROL_TYPE_NORMAL, + SD_CONTROL_TYPE_SEGMENT, + SD_CONTROL_TYPE_TILE, + SD_CONTROL_TYPE_REPAINT, // inpaint, outpaint + SD_CONTROL_TYPE_COUNT, +}; + +const char* sd_rng_type_name(enum rng_type_t rng_type); + +enum rng_type_t str_to_rng_type(const char* str); + +const char* sd_sample_method_name(enum sample_method_t sample_method); + +enum sample_method_t str_to_sample_method(const char* str); + +const char* sd_scheduler_name(enum scheduler_t scheduler); + +enum scheduler_t str_to_scheduler(const char* str); + +const char* sd_prediction_name(enum prediction_t prediction); + +enum prediction_t str_to_prediction(const char* str); + +const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode); + +enum lora_apply_mode_t str_to_lora_apply_mode(const char* str); + +const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler); + +enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str); + +const char* sd_control_type_name(enum sd_control_type_t control_type); + +enum sd_control_type_t str_to_sd_control_type(const char* str); + +enum preview_t { + PREVIEW_NONE, + PREVIEW_PROJ, + PREVIEW_TAE, + PREVIEW_VAE, + PREVIEW_COUNT +}; +const char* sd_preview_name(enum preview_t preview); +enum preview_t str_to_preview(const char* str); + +#endif // __SD_CORE_OPTION_TYPES_H__ diff --git a/src/core/util.cpp b/src/core/util.cpp index 9de1ace24f..220134f1f0 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -17,6 +17,7 @@ #include #include #include "core/ggml_tensor_utils.h" +#include "core/option_types.h" #include "runtime/preprocessing.hpp" #if defined(__APPLE__) && defined(__MACH__) @@ -721,7 +722,7 @@ void sd_set_progress_callback(sd_progress_cb_t cb, void* data) { sd_progress_cb = cb; sd_progress_cb_data = data; } -void sd_set_preview_callback(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data) { +void sd_set_preview_callback_impl(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data) { sd_preview_info = {}; sd_preview_cb = cb; sd_preview_cb_data = data; @@ -782,7 +783,7 @@ void* sd_get_progress_callback_data() { return sd_progress_cb_data; } -sd_image_t tensor_to_sd_image(const sd::Tensor& tensor, int frame_index) { +sd::Image tensor_to_sd_image(const sd::Tensor& tensor, int frame_index) { const auto& shape = tensor.shape(); GGML_ASSERT(shape.size() == 4 || shape.size() == 5); int width = static_cast(shape[0]); @@ -799,7 +800,7 @@ sd_image_t tensor_to_sd_image(const sd::Tensor& tensor, int frame_index) }; } -sd::Tensor sd_image_to_tensor(sd_image_t image, +sd::Tensor sd_image_to_tensor(sd::Image image, int target_width, int target_height, bool scale) { diff --git a/src/core/util.h b/src/core/util.h index 0929f60e01..082565a4e3 100644 --- a/src/core/util.h +++ b/src/core/util.h @@ -1,12 +1,15 @@ #ifndef __SD_CORE_UTIL_H__ #define __SD_CORE_UTIL_H__ +#include "core/media.h" + #include #include #include #include #include +#include "core/option_types.h" #include "core/tensor.hpp" #include "ggml-backend.h" #include "stable-diffusion.h" @@ -41,9 +44,9 @@ std::string utf32_to_utf8(const std::u32string& utf32_str); std::u32string unicode_value_to_utf32(int unicode_value); // std::string sd_basename(const std::string& path); -sd_image_t tensor_to_sd_image(const sd::Tensor& tensor, int frame_index = 0); +sd::Image tensor_to_sd_image(const sd::Tensor& tensor, int frame_index = 0); -sd::Tensor sd_image_to_tensor(sd_image_t image, +sd::Tensor sd_image_to_tensor(sd::Image image, int target_width = -1, int target_height = -1, bool scale = true); @@ -105,6 +108,7 @@ void* sd_get_progress_callback_data(); void sd_begin_preview_pass(int total_steps); sd_preview_cb_t sd_get_preview_callback(); void* sd_get_preview_callback_data(); +void sd_set_preview_callback_impl(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data); preview_t sd_get_preview_mode(); int sd_get_preview_interval(); bool sd_should_preview_denoised(); diff --git a/src/detailer.cpp b/src/detailer.cpp index 9c48ca4787..990d0a349c 100644 --- a/src/detailer.cpp +++ b/src/detailer.cpp @@ -1,5 +1,9 @@ #include "detailer.h" +#include "api/options.h" +#include "api/result.h" +#include "core/option_types.h" #include "generation_info.h" +#include "pipeline/generation.h" #include "pipeline/request.h" #include @@ -15,8 +19,8 @@ #include "json.hpp" #include "model_loader.h" -struct adetailer_ctx_t { - ADetailerGGML* detailer = nullptr; +struct sd_adetailer { + std::unique_ptr detailer; }; struct LetterboxInput { @@ -96,7 +100,7 @@ static bool parse_float(const std::string& value, float* result) { } } -static float image_channel(const sd_image_t& image, int x, int y, int channel) { +static float image_channel(const sd::Image& image, int x, int y, int channel) { if (image.data == nullptr || image.width == 0 || image.height == 0 || image.channel == 0) { return 0.f; } @@ -107,7 +111,7 @@ static float image_channel(const sd_image_t& image, int x, int y, int channel) { return image.data[index] / 255.f; } -static float bilinear_channel(const sd_image_t& image, float x, float y, int channel) { +static float bilinear_channel(const sd::Image& image, float x, float y, int channel) { int x0 = static_cast(std::floor(x)); int y0 = static_cast(std::floor(y)); int x1 = x0 + 1; @@ -119,7 +123,7 @@ static float bilinear_channel(const sd_image_t& image, float x, float y, int cha return a * (1.f - wy) + b * wy; } -static LetterboxInput make_letterbox_input(sd_image_t image, int input_size) { +static LetterboxInput make_letterbox_input(sd::Image image, int input_size) { LetterboxInput result; result.tensor = sd::full({input_size, input_size, 3, 1}, 114.f / 255.f); result.scale = std::min(static_cast(input_size) / image.width, @@ -180,7 +184,7 @@ static float box_iou(const ADetailerDetection& a, const ADetailerDetection& b) { static std::vector decode_detections(const sd::Tensor& raw, const YOLOv8Config& config, const LetterboxInput& letterbox, - sd_image_t image, + sd::Image image, const ADetailerParams& params) { std::vector candidates; if (raw.empty() || raw.dim() < 2) { @@ -510,9 +514,9 @@ static CropRegion expand_crop(CropRegion crop, return crop; } -static sd_image_t resize_crop_image(const sd_image_t& source, const CropRegion& crop, int width, int height) { - sd_image_t output = {static_cast(width), static_cast(height), 3, nullptr}; - output.data = static_cast(malloc(static_cast(width) * height * 3)); +static sd::Image resize_crop_image(const sd::Image& source, const CropRegion& crop, int width, int height) { + sd::Image output = {static_cast(width), static_cast(height), 3, nullptr}; + output.data = static_cast(malloc(static_cast(width) * height * 3)); if (output.data == nullptr) { return {0, 0, 0, nullptr}; } @@ -531,9 +535,9 @@ static sd_image_t resize_crop_image(const sd_image_t& source, const CropRegion& return output; } -static sd_image_t resize_crop_mask(const Mask& source, const CropRegion& crop, int width, int height) { - sd_image_t output = {static_cast(width), static_cast(height), 1, nullptr}; - output.data = static_cast(malloc(static_cast(width) * height)); +static sd::Image resize_crop_mask(const Mask& source, const CropRegion& crop, int width, int height) { + sd::Image output = {static_cast(width), static_cast(height), 1, nullptr}; + output.data = static_cast(malloc(static_cast(width) * height)); if (output.data == nullptr) { return {0, 0, 0, nullptr}; } @@ -549,11 +553,11 @@ static sd_image_t resize_crop_mask(const Mask& source, const CropRegion& crop, i return output; } -static sd_image_t copy_for_detailing(const sd_image_t& source) { +static sd::Image copy_for_detailing(const sd::Image& source) { if (source.channel == 4) { - sd_image_t output = source; - size_t bytes = static_cast(source.width) * source.height * source.channel; - output.data = static_cast(malloc(bytes)); + sd::Image output = source; + size_t bytes = static_cast(source.width) * source.height * source.channel; + output.data = static_cast(malloc(bytes)); if (output.data != nullptr) { memcpy(output.data, source.data, bytes); } @@ -563,8 +567,8 @@ static sd_image_t copy_for_detailing(const sd_image_t& source) { return resize_crop_image(source, full, source.width, source.height); } -static void composite_crop(sd_image_t* destination, - const sd_image_t& generated, +static void composite_crop(sd::Image* destination, + const sd::Image& generated, const Mask& feather_mask, const CropRegion& crop) { int crop_width = crop.x2 - crop.x1; @@ -724,7 +728,7 @@ bool ADetailerGGML::load_from_file(const std::string& detector_path) { return true; } -std::vector ADetailerGGML::predict(sd_image_t image, +std::vector ADetailerGGML::predict(sd::Image image, const ADetailerParams& params) { LetterboxInput input = make_letterbox_input(image, params.input_size); int64_t start = ggml_time_ms(); @@ -758,8 +762,8 @@ std::vector ADetailerGGML::predict(sd_image_t image, return detections; } -static bool parse_adetailer_params(const sd_adetailer_params_t& public_params, - const sd_img_gen_params_t& inpaint_params, +static bool parse_adetailer_params(const sd::ADetailConfig& public_params, + const sd::ImageConfig& inpaint_params, ADetailerParams* params) { if (params == nullptr) { return false; @@ -883,41 +887,32 @@ static bool parse_adetailer_params(const sd_adetailer_params_t& public_params, return true; } -adetailer_ctx_t* new_adetailer_ctx(const char* detector_path, - int n_threads, - const char* backend, - const char* params_backend) { - if (detector_path == nullptr || detector_path[0] == '\0') { - return nullptr; - } - auto* context = static_cast(calloc(1, sizeof(adetailer_ctx_t))); - if (context == nullptr) { - return nullptr; - } - context->detailer = new ADetailerGGML(n_threads, SAFE_STR(backend), SAFE_STR(params_backend)); - if (context->detailer == nullptr || !context->detailer->load_from_file(detector_path)) { - delete context->detailer; - free(context); - return nullptr; - } - return context; +sd_status_t sd_adetailer_create(const sd_adetailer_opt_t* opt, sd_adetailer_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::ADetailer, config); + if (status != SD_STATUS_OK) + return status; + auto result = std::make_unique(); + const auto& ctx = config.context; + result->detailer = std::make_unique(ctx.n_threads, SAFE_STR(ctx.backend), SAFE_STR(ctx.params_backend)); + if (!result->detailer->load_from_file(ctx.model_path)) + return sd::api::error(SD_STATUS_LOAD_FAILED, "model_path", "could not load ADetailer"); + *out = result.release(); + return SD_STATUS_OK; + }, + SD_STATUS_LOAD_FAILED); } -void free_adetailer_ctx(adetailer_ctx_t* context) { - if (context == nullptr) { - return; - } - delete context->detailer; - context->detailer = nullptr; - free(context); +void sd_adetailer_free(sd_adetailer_t* adetailer) { + delete adetailer; } -bool adetail_image(adetailer_ctx_t* context, - sd_ctx_t* sd_ctx, - sd_image_t input_image, - const sd_adetailer_params_t* public_params, - const sd_img_gen_params_t* inpaint_params, - sd_image_generation_result_t** result_out) try { +static bool run_adetail(sd_adetailer_t* context, sd_ctx_t* sd_ctx, sd::Image input_image, const sd::ADetailConfig* public_params, const sd::ImageConfig* inpaint_params, sd_result_t** result_out, sd_status_t& failure) { if (result_out) *result_out = nullptr; if (!result_out) @@ -929,6 +924,7 @@ bool adetail_image(adetailer_ctx_t* context, ADetailerParams params; if (!parse_adetailer_params(*public_params, *inpaint_params, ¶ms)) { + failure = SD_STATUS_INVALID_ARGUMENT; return false; } @@ -939,8 +935,9 @@ bool adetail_image(adetailer_ctx_t* context, filter_and_sort_detections(&detections, input_image.width, input_image.height, params); auto masks = make_masks(detections, input_image.width, input_image.height, params); - sd_image_t current = copy_for_detailing(input_image); + sd::Image current = copy_for_detailing(input_image); if (current.data == nullptr) { + failure = SD_STATUS_OUT_OF_MEMORY; return false; } @@ -959,38 +956,39 @@ bool adetail_image(adetailer_ctx_t* context, if (!mask_bbox(masks[i], &crop)) { continue; } - crop = params.inpaint_only_masked ? expand_crop(crop, - current.width, - current.height, - params.inpaint_padding, - params.inpaint_width, - params.inpaint_height) - : CropRegion{0, 0, static_cast(current.width), static_cast(current.height)}; - sd_image_t local_image = resize_crop_image(current, crop, params.inpaint_width, params.inpaint_height); - sd_image_t local_mask = resize_crop_mask(masks[i], crop, params.inpaint_width, params.inpaint_height); + crop = params.inpaint_only_masked ? expand_crop(crop, + current.width, + current.height, + params.inpaint_padding, + params.inpaint_width, + params.inpaint_height) + : CropRegion{0, 0, static_cast(current.width), static_cast(current.height)}; + sd::Image local_image = resize_crop_image(current, crop, params.inpaint_width, params.inpaint_height); + sd::Image local_mask = resize_crop_mask(masks[i], crop, params.inpaint_width, params.inpaint_height); if (local_image.data == nullptr || local_mask.data == nullptr) { + failure = SD_STATUS_OUT_OF_MEMORY; free(local_image.data); free(local_mask.data); return false; } - sd_img_gen_params_t generation = resolved_inpaint; - generation.prompt = prompt.c_str(); - generation.negative_prompt = negative_prompt.c_str(); - generation.init_image = local_image; - generation.mask_image = local_mask; - generation.width = params.inpaint_width; - generation.height = params.inpaint_height; - generation.strength = params.denoising_strength; - generation.seed = static_cast((static_cast(resolved_inpaint.seed) + i) & INT64_MAX); - generation.batch_count = 1; - generation.ref_images = nullptr; - generation.ref_images_count = 0; - generation.control_image = {}; - generation.pm_params = {}; - generation.pulid_params = {}; - generation.hires.enabled = false; - generation.image_preprocess = {}; + sd::ImageConfig generation = resolved_inpaint; + generation.prompt = prompt.c_str(); + generation.negative_prompt = negative_prompt.c_str(); + generation.init_image = local_image; + generation.mask_image = local_mask; + generation.width = params.inpaint_width; + generation.height = params.inpaint_height; + generation.strength = params.denoising_strength; + generation.seed = static_cast((static_cast(resolved_inpaint.seed) + i) & INT64_MAX); + generation.batch_count = 1; + generation.ref_images = nullptr; + generation.ref_images_count = 0; + generation.control_image = {}; + generation.pm_params = {}; + generation.pulid_params = {}; + generation.hires.enabled = false; + generation.image_preprocess = {}; if (params.steps > 0) { generation.sample_params.sample_steps = params.steps; generation.sample_params.custom_sigmas = nullptr; @@ -1006,32 +1004,62 @@ bool adetail_image(adetailer_ctx_t* context, generation.sample_params.scheduler = params.scheduler; } - sd_image_generation_result_t* generated = nullptr; - bool success = generate_image(sd_ctx, &generation, &generated); + sd::ImageGenerationResult* generated = nullptr; + failure = sd::generate_image(sd_ctx, generation, &generated); + std::unique_ptr generated_owner(generated, sd::free_image_generation_result); free(local_image.data); free(local_mask.data); - if (!success || generated == nullptr || generated->count <= 0 || generated->items[0].image.data == nullptr) { - free_sd_image_generation_result(generated); + if (failure != SD_STATUS_OK) + return false; + if (!generated || generated->count <= 0 || !generated->items[0].image.data) { + failure = sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "inpainting returned no image"); return false; } - std::unique_ptr generated_owner(generated, &free_sd_image_generation_result); Mask feather = gaussian_blur_mask(masks[i], params.mask_blur); composite_crop(¤t, generated->items[0].image, feather, crop); ++applied; } const std::vector infos{info}; - auto* outputs = static_cast(calloc(1, sizeof(sd_image_t))); - if (!outputs) + auto* outputs = static_cast(calloc(1, sizeof(sd::Image))); + if (!outputs) { + failure = SD_STATUS_OUT_OF_MEMORY; return false; + } outputs[0] = current; current_owner.release(); - if (!sd::make_image_generation_result(outputs, 1, infos, result_out)) + if (!sd::make_image_result(outputs, 1, infos, result_out)) { + failure = SD_STATUS_OUT_OF_MEMORY; return false; + } LOG_INFO("ADetailer applied %d mask(s)", applied); return true; -} catch (const std::exception& e) { - LOG_ERROR("ADetailer failed: %s", e.what()); - return false; +} + +sd_status_t sd_adetail(sd_adetailer_t* adetailer, sd_ctx_t* ctx, const sd_adetail_opt_t* opt, sd_result_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::ADetail, config); + if (status != SD_STATUS_OK) + return status; + if (!adetailer || !ctx) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "adetailer and diffusion context are required"); + if (!sd_ctx_supports_image_generation(ctx)) + return sd::api::error(SD_STATUS_UNSUPPORTED, nullptr, "inpainting requires an image-generation model"); + sd_status_t failure = SD_STATUS_GENERATION_FAILED; + if (!run_adetail(adetailer, ctx, config.post.image, &config.post.adetail, &config.image, out, failure)) { + if (failure == SD_STATUS_OK) + failure = SD_STATUS_GENERATION_FAILED; + if (*sd_get_last_error()) + return failure; + return sd::api::error(failure, nullptr, "ADetailer failed; see log for details"); + } + return SD_STATUS_OK; + }, + SD_STATUS_GENERATION_FAILED); } diff --git a/src/detailer.h b/src/detailer.h index 0e3ae50f5a..67ede5bdad 100644 --- a/src/detailer.h +++ b/src/detailer.h @@ -1,6 +1,10 @@ #ifndef __SD_DETAILER_H__ #define __SD_DETAILER_H__ +#include "core/media.h" + +#include "core/option_types.h" + #include #include #include @@ -70,7 +74,7 @@ struct ADetailerGGML { ~ADetailerGGML(); bool load_from_file(const std::string& detector_path); - std::vector predict(sd_image_t image, + std::vector predict(sd::Image image, const ADetailerParams& params); }; diff --git a/src/extensions/generation_extension.h b/src/extensions/generation_extension.h index e5ffd770e1..018092e2de 100644 --- a/src/extensions/generation_extension.h +++ b/src/extensions/generation_extension.h @@ -9,6 +9,7 @@ #include #include "conditioning/conditioner.hpp" +#include "config.h" #include "core/ggml_extend_backend.h" #include "model/diffusion/model.hpp" #include "model_loader.h" @@ -16,7 +17,7 @@ #include "stable-diffusion.h" struct GenerationExtensionInitContext { - const sd_ctx_params_t* params; + const sd::ContextConfig* params; SDVersion version; const String2TensorStorage& tensor_storage_map; bool photomaker_source_available; @@ -30,8 +31,8 @@ struct GenerationExtensionInitContext { struct GenerationExtensionConditionContext { Conditioner* conditioner; ConditionerParams& condition_params; - const sd_pm_params_t& pm_params; - const sd_pulid_params_t& pulid_params; + const sd::PhotoMakerConfig& pm_params; + const sd::PuLIDConfig& pulid_params; int n_threads; int total_steps; }; diff --git a/src/generation_info.cpp b/src/generation_info.cpp index 0d4829139d..f56ef98840 100644 --- a/src/generation_info.cpp +++ b/src/generation_info.cpp @@ -1,9 +1,10 @@ #include "generation_info.h" +#include "config.h" +#include "core/option_types.h" #include #include #include -#include #include "pipeline/request.h" #include "runtime/guidance.h" @@ -30,7 +31,7 @@ namespace sd { return std::strtod(buf, nullptr); } - static json sampling_info(const sd_sample_params_t& p, bool high_noise = false) { + static json sampling_info(const sd::SampleConfig& p, bool high_noise = false) { const auto& g = p.guidance; json info = {{"sample_steps", p.sample_steps}, {"guidance", {{"txt_cfg", g.txt_cfg}, {"distilled_guidance", g.distilled_guidance}}}}; if (!high_noise && p.shifted_timestep > 0) @@ -55,7 +56,7 @@ namespace sd { return info; } - static json hires_info(const sd_hires_params_t& p) { + static json hires_info(const sd::HiresConfig& p) { json info = {{"enabled", p.enabled}}; if (!p.enabled) return info; @@ -69,7 +70,7 @@ namespace sd { return info; } - static json cache_info(const sd_cache_params_t& p) { + static json cache_info(const sd::CacheConfig& p) { static const char* modes[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"}; if (p.mode < SD_CACHE_DISABLED || p.mode > SD_CACHE_SPECTRUM) return {{"mode", static_cast(p.mode)}}; @@ -144,7 +145,7 @@ namespace sd { return info; } - static json image_info(const sd_img_gen_params_t& p) { + static json image_info(const sd::ImageConfig& p) { auto info = common_info(p, "image"); if (p.init_image.data) info["strength"] = p.strength; @@ -177,7 +178,7 @@ namespace sd { return p; } - json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params) { + json generation_info(const pipeline::GenerationRequest& request, const sd::ImageConfig& params) { auto p = resolved_params(request, params); p.qwen_image_layers = request.qwen_image_layers; auto info = image_info(p); @@ -188,7 +189,7 @@ namespace sd { return info; } - json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params) { + json generation_info(const pipeline::GenerationRequest& request, const sd::VideoConfig& params) { const auto p = resolved_params(request, params); auto info = common_info(p, "video"); if (!request.models.empty()) @@ -208,14 +209,13 @@ namespace sd { return info; } - json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params) { + json adetailer_generation_info(const sd::ADetailConfig& params, const sd::ImageConfig& inpaint_params) { auto inpaint = image_info(inpaint_params); inpaint["strength"] = inpaint_params.strength; // ADetailer clears these inputs before generating each crop. for (const auto* key : {"control_strength", "control_type", "pm_params", "pulid_params", "hires", "image_preprocess"}) inpaint.erase(key); - sd_img_gen_params_t defaults; - sd_img_gen_params_init(&defaults); + sd::ImageConfig defaults; if (inpaint_params.qwen_image_layers != defaults.qwen_image_layers) inpaint["qwen_image_layers"] = inpaint_params.qwen_image_layers; json adetailer = json::object(); @@ -232,69 +232,4 @@ namespace sd { return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"upscale", {{"upscale_factor", upscale_factor}}}}; } - char* copy_generation_info(const json& info) { - const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace); - char* result = static_cast(malloc(text.size() + 1)); - if (result) - memcpy(result, text.c_str(), text.size() + 1); - return result; - } - - bool make_image_generation_result(sd_image_t* images, int count, const std::vector& infos, sd_image_generation_result_t** result_out) { - if (!result_out || !images || count <= 0 || infos.size() != static_cast(count)) { - free_sd_images(images, count); - return false; - } - auto* result = static_cast(calloc(1, sizeof(sd_image_generation_result_t))); - if (!result) { - free_sd_images(images, count); - return false; - } - result->items = static_cast(calloc(count, sizeof(sd_image_result_t))); - if (!result->items) { - free(result); - free_sd_images(images, count); - return false; - } - result->count = count; - try { - for (int i = 0; i < count; ++i) { - result->items[i].image = images[i]; - images[i] = {}; - result->items[i].generation_info = copy_generation_info(infos[i]); - if (!result->items[i].generation_info) { - free_sd_images(images, count); - free_sd_image_generation_result(result); - return false; - } - } - } catch (...) { - free_sd_images(images, count); - free_sd_image_generation_result(result); - return false; - } - free(images); - *result_out = result; - return true; - } -} - -void free_sd_image_generation_result(sd_image_generation_result_t* result) { - if (!result) - return; - for (int i = 0; i < result->count; ++i) { - free(result->items[i].image.data); - free(result->items[i].generation_info); - } - free(result->items); - free(result); -} - -void free_sd_video_generation_result(sd_video_generation_result_t* result) { - if (!result) - return; - free_sd_images(result->frames, result->frame_count); - free_sd_audio(result->audio); - free(result->generation_info); - free(result); -} +} // namespace sd diff --git a/src/generation_info.h b/src/generation_info.h index 56688ee812..d84673f7a2 100644 --- a/src/generation_info.h +++ b/src/generation_info.h @@ -3,6 +3,7 @@ #include +#include "config.h" #include "json.hpp" #include "stable-diffusion.h" @@ -11,13 +12,11 @@ namespace sd::pipeline { } namespace sd { - nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params); - nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params); - nlohmann::json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params); + nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd::ImageConfig& params); + nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd::VideoConfig& params); + nlohmann::json adetailer_generation_info(const sd::ADetailConfig& params, const sd::ImageConfig& inpaint_params); nlohmann::json upscale_generation_info(uint32_t upscale_factor); - char* copy_generation_info(const nlohmann::json& info); - // Consumes images even on failure. - bool make_image_generation_result(sd_image_t* images, int count, const std::vector& infos, sd_image_generation_result_t** result_out); + } #endif // __SD_GENERATION_INFO_H__ diff --git a/src/model/diffusion/control.hpp b/src/model/diffusion/control.hpp index 4ad85ecf8c..2fc3dee36b 100644 --- a/src/model/diffusion/control.hpp +++ b/src/model/diffusion/control.hpp @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_DIFFUSION_CONTROL_HPP__ #define __SD_MODEL_DIFFUSION_CONTROL_HPP__ +#include "core/option_types.h" + #include "model/common/block.hpp" #include "stable-diffusion.h" diff --git a/src/model/integrations/sd.cpp b/src/model/integrations/sd.cpp index a36ff8e36d..24a9acfc17 100644 --- a/src/model/integrations/sd.cpp +++ b/src/model/integrations/sd.cpp @@ -1,4 +1,5 @@ #include "conditioning/conditioner.hpp" +#include "core/option_types.h" #include "model/diffusion/unet.hpp" #include "model/vae/auto_encoder_kl.hpp" #include "model_definition.h" diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp index c43c2c7152..3010cb82b8 100644 --- a/src/model/vae/minimax_h3_vae.hpp +++ b/src/model/vae/minimax_h3_vae.hpp @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__ #define __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__ +#include "config.h" + #include #include #include @@ -719,11 +721,11 @@ namespace MiniMaxH3VAE { tensor.shape()[3]}); } - sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const override { + sd::TilingConfig resolve_tiling_params(sd::TilingConfig params) const override { return config.sol ? params : resolve_original_tiling_params(params); } - static sd_tiling_params_t resolve_original_tiling_params(sd_tiling_params_t params) { + static sd::TilingConfig resolve_original_tiling_params(sd::TilingConfig params) { if (!params.enabled) { params.target_overlap = 0.25f; } @@ -774,7 +776,7 @@ namespace MiniMaxH3VAE { sd::Tensor encode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool circular_x = false, bool circular_y = false) override { auto input = ensure_video_shape(x); @@ -860,7 +862,7 @@ namespace MiniMaxH3VAE { sd::Tensor decode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool decode_video = false, bool circular_x = false, bool circular_y = false, diff --git a/src/model/vae/vae.hpp b/src/model/vae/vae.hpp index 3c66965f69..f0a51c60a8 100644 --- a/src/model/vae/vae.hpp +++ b/src/model/vae/vae.hpp @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_VAE_VAE_HPP__ #define __SD_MODEL_VAE_VAE_HPP__ +#include "config.h" + #include #include @@ -82,7 +84,7 @@ struct VAE : public GGMLRunner { sd::Tensor compute_with_temporal_tiling(const int n_threads, const sd::Tensor& input, VAETemporalDirection direction, - const sd_tiling_params_t& tiling_params) { + const sd::TilingConfig& tiling_params) { if (!tiling_params.temporal_tiling || input.dim() != 5 || input.shape()[2] <= 1) { return _compute(n_threads, input, direction == VAETemporalDirection::DECODE); } @@ -126,7 +128,7 @@ struct VAE : public GGMLRunner { bool circular_x, bool circular_y, bool decode_graph, - const sd_tiling_params_t& tiling_params, + const sd::TilingConfig& tiling_params, const char* error_message, bool silent = false) { auto on_processing = [&](const sd::Tensor& input_tile) { @@ -201,14 +203,14 @@ struct VAE : public GGMLRunner { return supports_temporal_tiling(VAETemporalDirection::DECODE); } - virtual sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const { + virtual sd::TilingConfig resolve_tiling_params(sd::TilingConfig params) const { return params; } bool get_tile_sizes(int& tile_size_w, int& tile_size_h, float& tile_overlap, - const sd_tiling_params_t& params, + const sd::TilingConfig& params, int64_t latent_w, int64_t latent_h) { const auto tiling = resolve_tiling_params(params); @@ -247,7 +249,7 @@ struct VAE : public GGMLRunner { virtual sd::Tensor encode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool circular_x = false, bool circular_y = false) { return encode(n_threads, x, tiling_params, circular_x, circular_y, true); @@ -255,7 +257,7 @@ struct VAE : public GGMLRunner { sd::Tensor encode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool circular_x, bool circular_y, bool auto_end) { @@ -313,7 +315,7 @@ struct VAE : public GGMLRunner { virtual sd::Tensor decode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool decode_video = false, bool circular_x = false, bool circular_y = false, @@ -323,7 +325,7 @@ struct VAE : public GGMLRunner { sd::Tensor decode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool decode_video, bool circular_x, bool circular_y, diff --git a/src/model/vae/vae_tiling.hpp b/src/model/vae/vae_tiling.hpp index c88ff9f056..a343084327 100644 --- a/src/model/vae/vae_tiling.hpp +++ b/src/model/vae/vae_tiling.hpp @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_VAE_VAE_TILING_HPP__ #define __SD_MODEL_VAE_VAE_TILING_HPP__ +#include "config.h" + #include #include #include @@ -36,7 +38,7 @@ struct VAETemporalTilePlan { std::vector tiles; }; -inline VAETemporalTilingConfig resolve_vae_temporal_tiling_config(const sd_tiling_params_t& params, +inline VAETemporalTilingConfig resolve_vae_temporal_tiling_config(const sd::TilingConfig& params, int default_tile_frames, int default_overlap) { VAETemporalTilingConfig config; diff --git a/src/model_definition.h b/src/model_definition.h index 2b49994958..1a0f2df488 100644 --- a/src/model_definition.h +++ b/src/model_definition.h @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_DEFINITION_H__ #define __SD_MODEL_DEFINITION_H__ +#include "core/option_types.h" + #include #include "model.h" #include "stable-diffusion.h" diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 8c33376d5b..c2ae6b83e6 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -1,10 +1,13 @@ #include "diffusion_engine.h" +#include "api/tensor.h" +#include "core/option_types.h" #include #include #include #include #include +#include #include #include #include @@ -23,6 +26,7 @@ #include "model.h" #include "model_definition.h" +#include "config.h" #include "core/rng.hpp" #include "core/rng_mt19937.hpp" #include "core/rng_philox.hpp" @@ -890,7 +894,7 @@ bool StableDiffusionGGML::set_sage_attention_enabled(bool enabled) { return true; } -bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) { +bool StableDiffusionGGML::init(const sd::ContextConfig* sd_ctx_params) { #ifdef SD_USE_UPSTREAM_GGML LOG_WARN( "Using upstream GGML: INT8 tensorwise/convrot is disabled and FP8 weights are " @@ -1890,12 +1894,12 @@ SDCondition StableDiffusionGGML::get_learned_condition(const ConditionerParams& return {}; } if (!sd::resolve_prompt_images(input, [&](const std::string& source) -> sd::Tensor { - sd_image_t image{}; - if (!prompt_image_callback(source.c_str(), prompt_image_callback_data, &image) || - image.data == nullptr || image.width == 0 || image.height == 0 || image.channel < 1 || image.channel > 4) { + const sd_tensor_t* tensor = nullptr; + if (!prompt_image_callback(source.c_str(), prompt_image_callback_data, &tensor) || !tensor) { LOG_ERROR("Failed to resolve prompt image: %s", source.c_str()); return {}; } + auto image = sd::api::image_view(*tensor); auto pixels = sd_image_to_tensor(image); if (image.channel <= 2) { auto gray = sd::ops::slice(pixels, 2, 0, 1); @@ -1918,8 +1922,8 @@ void StableDiffusionGGML::reset_generation_extensions() { } } -void StableDiffusionGGML::prepare_generation_extensions(const sd_pm_params_t& pm_params, - const sd_pulid_params_t& pulid_params, +void StableDiffusionGGML::prepare_generation_extensions(const sd::PhotoMakerConfig& pm_params, + const sd::PuLIDConfig& pulid_params, ConditionerParams& condition_params, int total_steps) { reset_generation_extensions(); @@ -1961,7 +1965,7 @@ sd::Tensor StableDiffusionGGML::get_clip_vision_output(const sd::Tensor StableDiffusionGGML::get_audio_embedding(const sd_audio_t& audio) { +sd::Tensor StableDiffusionGGML::get_audio_embedding(const sd::Audio& audio) { if (audio_encoder == nullptr) { LOG_ERROR("audio encoder model is not loaded"); return {}; @@ -1983,7 +1987,7 @@ sd::Tensor StableDiffusionGGML::get_audio_embedding(const sd_audio_t& aud return audio_encoder->compute(n_threads, mono); } -void StableDiffusionGGML::compute_ip_adapter_tokens(const sd_image_t& image, float strength) { +void StableDiffusionGGML::compute_ip_adapter_tokens(const sd::Image& image, float strength) { ip_adapter_tokens = {}; ip_adapter_uncond_tokens = {}; ip_adapter_strength = strength; @@ -2083,7 +2087,7 @@ void StableDiffusionGGML::preview_image(int step, const sd::Tensor& latents, enum SDVersion version, preview_t preview_mode, - std::function step_callback, + std::function step_callback, void* step_callback_data, bool is_noisy) { bool is_video = preview_latent_tensor_is_video(latents); @@ -2172,17 +2176,14 @@ void StableDiffusionGGML::preview_image(int step, uint32_t img_height = static_cast(_latents.shape()[1]) * patch_sz; uint32_t img_channels = latent_alpha_proj != nullptr ? 4 : 3; - uint8_t* data = (uint8_t*)malloc(frames * img_width * img_height * img_channels * sizeof(uint8_t)); + std::unique_ptr data(static_cast(malloc(size_t(frames) * img_width * img_height * img_channels)), free); GGML_ASSERT(data != nullptr); - preview_latent_video(data, _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias); - sd_image_t* images = (sd_image_t*)malloc(frames * sizeof(sd_image_t)); - GGML_ASSERT(images != nullptr); + preview_latent_video(data.get(), _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias); + std::vector images(frames); for (uint32_t i = 0; i < frames; i++) { - images[i] = {img_width, img_height, img_channels, data + i * img_width * img_height * img_channels}; + images[i] = {img_width, img_height, img_channels, data.get() + size_t(i) * img_width * img_height * img_channels}; } - step_callback(step, frames, images, is_noisy, step_callback_data); - free(data); - free(images); + sd::api::preview_images(step_callback, step, frames, images.data(), is_noisy, step_callback_data); return; } @@ -2201,19 +2202,20 @@ void StableDiffusionGGML::preview_image(int step, return; } - is_video = preview_latent_tensor_is_video(decoded); - uint32_t frames = is_video ? static_cast(decoded.shape()[2]) : 1; - sd_image_t* images = (sd_image_t*)malloc(frames * sizeof(sd_image_t)); + is_video = preview_latent_tensor_is_video(decoded); + uint32_t frames = is_video ? static_cast(decoded.shape()[2]) : 1; + auto release_images = [frames](sd::Image* images) { + for (uint32_t i = 0; i < frames; ++i) + free(images[i].data); + free(images); + }; + std::unique_ptr images(static_cast(calloc(frames, sizeof(sd::Image))), release_images); GGML_ASSERT(images != nullptr); for (uint32_t i = 0; i < frames; ++i) { - images[i] = tensor_to_sd_image(decoded, static_cast(i)); + images.get()[i] = tensor_to_sd_image(decoded, static_cast(i)); } - step_callback(step, frames, images, is_noisy, step_callback_data); - for (uint32_t i = 0; i < frames; ++i) { - free(images[i].data); - } - free(images); + sd::api::preview_images(step_callback, step, frames, images.get(), is_noisy, step_callback_data); return; } @@ -2352,7 +2354,7 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr& control_image, float control_strength, - const sd_guidance_params_t& guidance, + const sd::GuidanceConfig& guidance, float eta, int shifted_timestep, sample_method_t method, @@ -2366,7 +2368,7 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr& video_positions) { struct RunnerEndOnExit { diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h index d2a05e07a8..ae05dc9cd3 100644 --- a/src/pipeline/diffusion_engine.h +++ b/src/pipeline/diffusion_engine.h @@ -1,6 +1,10 @@ #ifndef __SD_PIPELINE_DIFFUSION_ENGINE_H__ #define __SD_PIPELINE_DIFFUSION_ENGINE_H__ +#include "core/media.h" + +#include "core/option_types.h" + #include #include #include @@ -14,6 +18,7 @@ #include #include +#include "config.h" #include "core/ggml_extend_backend.h" #include "core/ggml_graph_cut.h" #include "core/tensor.hpp" @@ -99,8 +104,8 @@ class StableDiffusionGGML { int animatediff_num_frames = 0; std::string taesd_path; - sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr}; - bool enable_mmap = false; + sd::TilingConfig vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr}; + bool enable_mmap = false; sd::ggml_graph_cut::MaxVramAssignment max_vram_assignment; bool disable_prefetch = false; bool disable_segmented_compute = false; @@ -124,7 +129,7 @@ class StableDiffusionGGML { using RunnerGroups = std::set; struct ModelConfig { - sd_ctx_params_t params{}; + sd::ContextConfig params{}; std::list strings; std::vector embeddings; std::map models; @@ -134,22 +139,22 @@ class StableDiffusionGGML { bool photomaker_source_available = false; bool animatediff_loaded = false; - explicit ModelConfig(const sd_ctx_params_t& initial) + explicit ModelConfig(const sd::ContextConfig& initial) : params(initial) { - for (auto member : {&sd_ctx_params_t::model_path, &sd_ctx_params_t::clip_l_path, - &sd_ctx_params_t::clip_g_path, &sd_ctx_params_t::clip_vision_path, - &sd_ctx_params_t::t5xxl_path, &sd_ctx_params_t::llm_path, - &sd_ctx_params_t::llm_vision_path, &sd_ctx_params_t::diffusion_model_path, - &sd_ctx_params_t::tokenizer, &sd_ctx_params_t::flash_attn_mode, &sd_ctx_params_t::conv_direct_mode, - &sd_ctx_params_t::high_noise_diffusion_model_path, &sd_ctx_params_t::uncond_diffusion_model_path, - &sd_ctx_params_t::embeddings_connectors_path, &sd_ctx_params_t::vae_path, - &sd_ctx_params_t::audio_vae_path, &sd_ctx_params_t::taesd_path, - &sd_ctx_params_t::control_net_path, &sd_ctx_params_t::ip_adapter_path, - &sd_ctx_params_t::motion_module_path, &sd_ctx_params_t::photo_maker_path, - &sd_ctx_params_t::pulid_weights_path, &sd_ctx_params_t::tensor_type_rules, - &sd_ctx_params_t::max_vram, &sd_ctx_params_t::backend, - &sd_ctx_params_t::params_backend, &sd_ctx_params_t::split_mode, - &sd_ctx_params_t::rpc_servers, &sd_ctx_params_t::model_args}) { + for (auto member : {&sd::ContextConfig::model_path, &sd::ContextConfig::clip_l_path, + &sd::ContextConfig::clip_g_path, &sd::ContextConfig::clip_vision_path, + &sd::ContextConfig::t5xxl_path, &sd::ContextConfig::llm_path, + &sd::ContextConfig::llm_vision_path, &sd::ContextConfig::diffusion_model_path, + &sd::ContextConfig::tokenizer, &sd::ContextConfig::flash_attn_mode, &sd::ContextConfig::conv_direct_mode, + &sd::ContextConfig::high_noise_diffusion_model_path, &sd::ContextConfig::uncond_diffusion_model_path, + &sd::ContextConfig::embeddings_connectors_path, &sd::ContextConfig::vae_path, + &sd::ContextConfig::audio_vae_path, &sd::ContextConfig::audio_encoder_path, &sd::ContextConfig::taesd_path, + &sd::ContextConfig::control_net_path, &sd::ContextConfig::ip_adapter_path, + &sd::ContextConfig::motion_module_path, &sd::ContextConfig::photo_maker_path, + &sd::ContextConfig::pulid_weights_path, &sd::ContextConfig::tensor_type_rules, + &sd::ContextConfig::max_vram, &sd::ContextConfig::backend, + &sd::ContextConfig::params_backend, &sd::ContextConfig::split_mode, + &sd::ContextConfig::rpc_servers, &sd::ContextConfig::model_args}) { strings.emplace_back(SAFE_STR(initial.*member)); params.*member = strings.back().c_str(); } @@ -338,7 +343,7 @@ class StableDiffusionGGML { bool init_model_loader(ModelLoader& model_loader, ModelConfig& configuration); - bool init(const sd_ctx_params_t* sd_ctx_params); + bool init(const sd::ContextConfig* sd_ctx_params); bool set_sage_attention_enabled(bool enabled); bool uses_tae() const; @@ -394,8 +399,8 @@ class StableDiffusionGGML { void reset_generation_extensions(); - void prepare_generation_extensions(const sd_pm_params_t& pm_params, - const sd_pulid_params_t& pulid_params, + void prepare_generation_extensions(const sd::PhotoMakerConfig& pm_params, + const sd::PuLIDConfig& pulid_params, ConditionerParams& condition_params, int total_steps); @@ -404,9 +409,9 @@ class StableDiffusionGGML { int clip_skip = -1, bool zero_out_masked = false); - sd::Tensor get_audio_embedding(const sd_audio_t& audio); + sd::Tensor get_audio_embedding(const sd::Audio& audio); - void compute_ip_adapter_tokens(const sd_image_t& image, float strength); + void compute_ip_adapter_tokens(const sd::Image& image, float strength); std::vector process_timesteps(const std::vector& timesteps, const sd::Tensor& init_latent, @@ -421,7 +426,7 @@ class StableDiffusionGGML { const sd::Tensor& latents, enum SDVersion version, preview_t preview_mode, - std::function step_callback, + std::function step_callback, void* step_callback_data, bool is_noisy); @@ -466,7 +471,7 @@ class StableDiffusionGGML { const SDCondition& img_uncond, const sd::Tensor& control_image, float control_strength, - const sd_guidance_params_t& guidance, + const sd::GuidanceConfig& guidance, float eta, int shifted_timestep, sample_method_t method, @@ -480,7 +485,7 @@ class StableDiffusionGGML { float vace_strength, int audio_length, float frame_rate, - const sd_cache_params_t* cache_params, + const sd::CacheConfig* cache_params, bool preview_final_step, const sd::Tensor& video_positions = {}); diff --git a/src/pipeline/generation.h b/src/pipeline/generation.h index 0bc3913150..08511f2afb 100644 --- a/src/pipeline/generation.h +++ b/src/pipeline/generation.h @@ -1,12 +1,22 @@ #ifndef __SD_PIPELINE_GENERATION_H__ #define __SD_PIPELINE_GENERATION_H__ +#include "core/media.h" + #include "conditioning/conditioner.hpp" +#include "config.h" #include "json.hpp" #include "stable-diffusion.h" class StableDiffusionGGML; +namespace sd { + struct ImageGenerationResult; + struct VideoGenerationResult; + sd_status_t generate_image(sd_ctx_t* ctx, const ImageConfig& config, ImageGenerationResult** result_out, const sd_opt_t* source = nullptr) noexcept; + sd_status_t generate_video(sd_ctx_t* ctx, const VideoConfig& config, VideoGenerationResult** result_out, const sd_opt_t* source = nullptr) noexcept; +} + static inline bool sd_version_supports_animatediff(SDVersion version) { return version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX; } @@ -53,16 +63,16 @@ namespace sd::pipeline { // Callers hold ExecutionScope; AnimateDiff reuses the image path within the same scope. bool generate_image(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, - sd_image_t** images_out, + const sd::ImageConfig* sd_img_gen_params, + sd::Image** images_out, int* num_images_out, std::vector* infos_out = nullptr); bool generate_video(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, - sd_image_t** frames_out, + const sd::VideoConfig* sd_vid_gen_params, + sd::Image** frames_out, int* num_frames_out, - sd_audio_t** audio_out, + sd::Audio** audio_out, int* fps_out, nlohmann::json* info_out = nullptr); diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp index 5e7a349583..f2cdcea9c5 100644 --- a/src/pipeline/image.cpp +++ b/src/pipeline/image.cpp @@ -1,3 +1,5 @@ +#include "api/result.h" +#include "core/option_types.h" #include "generation.h" #include "generation_info.h" @@ -22,7 +24,7 @@ namespace sd::pipeline { }; static CircularAxesState configure_image_vae_axes(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, const GenerationRequest& request) { CircularAxesState original_axes = {sd->circular_x, sd->circular_y}; @@ -77,7 +79,7 @@ namespace sd::pipeline { public: ImageVaeAxesGuard(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, const GenerationRequest& request) : sd(sd), original_axes(configure_image_vae_axes(sd, sd_img_gen_params, request)) {} @@ -184,7 +186,7 @@ namespace sd::pipeline { } static std::optional prepare_image_generation_latents(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, GenerationRequest* request, SamplePlan* plan, const RefImageParams& ref_image_params) { @@ -541,7 +543,7 @@ namespace sd::pipeline { } static std::optional prepare_image_generation_embeds(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, GenerationRequest* request, SamplePlan* plan, ImageGenerationLatents* latents, @@ -669,10 +671,10 @@ namespace sd::pipeline { return embeds; } - static sd_image_t* decode_image_outputs(StableDiffusionGGML* sd, - const GenerationRequest& request, - const std::vector>& final_latents, - int* num_images_out) { + static sd::Image* decode_image_outputs(StableDiffusionGGML* sd, + const GenerationRequest& request, + const std::vector>& final_latents, + int* num_images_out) { if (final_latents.empty()) { LOG_ERROR("no latent images to decode"); return nullptr; @@ -760,8 +762,8 @@ namespace sd::pipeline { return nullptr; } - int image_count = static_cast(decoded_images.size()); - sd_image_t* result_images = (sd_image_t*)calloc(image_count, sizeof(sd_image_t)); + int image_count = static_cast(decoded_images.size()); + sd::Image* result_images = (sd::Image*)calloc(image_count, sizeof(sd::Image)); if (result_images == nullptr) { return nullptr; } @@ -915,8 +917,8 @@ namespace sd::pipeline { } bool generate_image(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, - sd_image_t** images_out, + const sd::ImageConfig* sd_img_gen_params, + sd::Image** images_out, int* num_images_out, std::vector* infos_out) { if (infos_out) @@ -944,7 +946,7 @@ namespace sd::pipeline { GenerationRequest request(sd, sd_img_gen_params); const auto info = sd::generation_info(request, *sd_img_gen_params); sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules); - sd_img_gen_params_t processed_params = *sd_img_gen_params; + sd::ImageConfig processed_params = *sd_img_gen_params; if (!preprocessing.prepare_inputs(processed_params, request.width, request.height)) return false; sd_img_gen_params = &processed_params; @@ -953,7 +955,6 @@ namespace sd::pipeline { sd->rng->manual_seed(request.seed); sd->sampler_rng->manual_seed(request.seed); - sd->set_flow_shift(sd_img_gen_params->sample_params.flow_shift); if (!sd->apply_loras(sd_img_gen_params->loras, sd_img_gen_params->lora_count)) return false; sd->apply_circular_axes(sd_img_gen_params->circular_x, sd_img_gen_params->circular_y); @@ -1084,7 +1085,6 @@ namespace sd::pipeline { make_hires_sigma_schedule(sd, request.hires, request.sample_params, - plan.sample_method, plan.sample_steps, sd->get_image_seq_len(request.hires.target_height, request.hires.target_width), &hires_scheduler_steps); @@ -1217,7 +1217,7 @@ namespace sd::pipeline { (*infos_out)[i]["seed"] = std::to_string(request.seed_for_batch(i / images_per_latent)); } } catch (...) { - free_sd_images(result, num_images); + sd::free_images(result, num_images); throw; } } @@ -1231,7 +1231,7 @@ namespace sd::pipeline { if (images_out != nullptr) { *images_out = result; } else { - free_sd_images(result, num_images); + sd::free_images(result, num_images); } return true; } diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp index afc0ce8d7f..389b199829 100644 --- a/src/pipeline/model_builders.cpp +++ b/src/pipeline/model_builders.cpp @@ -1,4 +1,5 @@ #include "model_builders.h" +#include "core/option_types.h" #include #include diff --git a/src/pipeline/model_builders.h b/src/pipeline/model_builders.h index bb51ccc14a..1f56e4a1d2 100644 --- a/src/pipeline/model_builders.h +++ b/src/pipeline/model_builders.h @@ -4,6 +4,7 @@ #include #include +#include "config.h" #include "core/ggml_extend_backend.h" #include "model.h" #include "stable-diffusion.h" @@ -28,7 +29,7 @@ namespace IPAdapter { namespace sd::model_builders { struct Context { - const sd_ctx_params_t& params; + const sd::ContextConfig& params; SDVersion version; const String2TensorStorage& tensor_storage_map; SDBackendManager& backends; diff --git a/src/pipeline/relay_vsr.cpp b/src/pipeline/relay_vsr.cpp index c8b0d72fdf..20d70eaecb 100644 --- a/src/pipeline/relay_vsr.cpp +++ b/src/pipeline/relay_vsr.cpp @@ -1,3 +1,4 @@ +#include "api/result.h" #include "pipeline/video_upscale.h" #include @@ -11,7 +12,7 @@ namespace sd { namespace pipeline { - static sd::Tensor prepare_vsr_frame(const sd_image_t& frame, int width, int height) { + static sd::Tensor prepare_vsr_frame(const sd::Image& frame, int width, int height) { auto result = sd_image_to_tensor(frame, -1, -1, false); result /= 127.5f; result -= 1.f; @@ -103,7 +104,7 @@ namespace sd { flash_->runner_end(); } - bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t* result) override { + bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) override { const auto* frames = params.frames; const int frame_count = params.frame_count, fps = params.fps, window_size = params.window_size; const int64_t seed = params.seed; diff --git a/src/pipeline/request.cpp b/src/pipeline/request.cpp index 739b8e43a1..bbf3ab778f 100644 --- a/src/pipeline/request.cpp +++ b/src/pipeline/request.cpp @@ -1,5 +1,7 @@ #include "request.h" #include "conditioning/conditioner.hpp" +#include "config.h" +#include "core/option_types.h" #include #include @@ -93,9 +95,9 @@ namespace sd::pipeline { return scheduler; } - float resolve_eta(StableDiffusionGGML* sd, - float eta, - enum sample_method_t sample_method) { + static float resolve_eta(StableDiffusionGGML* sd, + float eta, + enum sample_method_t sample_method) { if (eta == INFINITY) { if (sd->version == VERSION_HIDREAM_O1) { return 8.f; @@ -120,7 +122,7 @@ namespace sd::pipeline { return eta; } - GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params) { + GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd::ImageConfig* sd_img_gen_params) { prompt = SAFE_STR(sd_img_gen_params->prompt); negative_prompt = SAFE_STR(sd_img_gen_params->negative_prompt); width = sd_img_gen_params->width; @@ -136,7 +138,6 @@ namespace sd::pipeline { strength = sd_img_gen_params->strength; control_strength = sd_img_gen_params->control_strength; control_type = sd_img_gen_params->control_type; - eta = sd_img_gen_params->sample_params.eta; has_ref_images = sd_img_gen_params->ref_images_count > 0; guidance = sd_img_gen_params->sample_params.guidance; pm_params = sd_img_gen_params->pm_params; @@ -156,7 +157,7 @@ namespace sd::pipeline { models.erase("pulid_weights"); } - GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params) { + GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd::VideoConfig* sd_vid_gen_params) { prompt = SAFE_STR(sd_vid_gen_params->prompt); negative_prompt = SAFE_STR(sd_vid_gen_params->negative_prompt); width = sd_vid_gen_params->width; @@ -293,7 +294,7 @@ namespace sd::pipeline { } void GenerationRequest::resolve_guidance(StableDiffusionGGML* sd, - sd_guidance_params_t* guidance, + sd::GuidanceConfig* guidance, bool* use_uncond, bool* use_img_uncond, bool has_ref_images, @@ -388,16 +389,21 @@ namespace sd::pipeline { sample_params.sample_method = resolve_sample_method(sd, sample_params.sample_method); sample_params.scheduler = resolve_scheduler(sd, sample_params.scheduler, sample_params.sample_method); sample_params.eta = resolve_eta(sd, sample_params.eta, sample_params.sample_method); + sd->set_flow_shift(sample_params.flow_shift); + if (sd->is_flow_denoiser()) + sample_params.flow_shift = sd->active_flow_shift; if (frames > 0) { - high_noise_sample_params.guidance = high_noise_guidance; - high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method); - high_noise_sample_params.scheduler = resolve_scheduler(sd, high_noise_sample_params.scheduler, high_noise_sample_params.sample_method); - high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method); + high_noise_sample_params.guidance = high_noise_guidance; + high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method); + high_noise_sample_params.scheduler = sample_params.scheduler; + high_noise_sample_params.flow_shift = sample_params.flow_shift; + high_noise_sample_params.shifted_timestep = sample_params.shifted_timestep; + high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method); } } SamplePlan::SamplePlan(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, const GenerationRequest& request) { sample_method = request.sample_params.sample_method; extra_sample_args = request.sample_params.extra_sample_args; @@ -407,7 +413,7 @@ namespace sd::pipeline { } SamplePlan::SamplePlan(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, const GenerationRequest& request) { sample_method = request.sample_params.sample_method; extra_sample_args = request.sample_params.extra_sample_args; @@ -425,9 +431,8 @@ namespace sd::pipeline { void SamplePlan::resolve(StableDiffusionGGML* sd, const GenerationRequest* request, - const sd_sample_params_t* sample_params) { - sample_method = resolve_sample_method(sd, sample_method); - + const sd::SampleConfig* sample_params) { + scheduler = sample_params->scheduler; total_steps = sample_steps + std::max(0, high_noise_sample_steps); if (sample_params->custom_sigmas_count > 0) { @@ -446,9 +451,6 @@ namespace sd::pipeline { } } } else { - scheduler = resolve_scheduler(sd, - sample_params->scheduler, - sample_method); int sample_seq_len = sd->get_image_seq_len(request->height, request->width); if (sd_version_is_ltxav(sd->version) && request->frames > 0) { int latent_frames = ((request->frames - 1) / 8) + 1; @@ -463,8 +465,6 @@ namespace sd::pipeline { sample_params->extra_sample_args); } - eta = resolve_eta(sd, eta, sample_method); - if (high_noise_sample_steps < 0) { for (size_t i = 0; i < sigmas.size(); ++i) { if (sigmas[i] < moe_boundary) { @@ -477,17 +477,13 @@ namespace sd::pipeline { LOG_INFO("sampling using %s method", sampling_methods_str[sample_method]); if (high_noise_sample_steps > 0) { - high_noise_sample_method = resolve_sample_method(sd, - high_noise_sample_method); - high_noise_eta = resolve_eta(sd, high_noise_eta, high_noise_sample_method); LOG_INFO("sampling(high noise) using %s method", sampling_methods_str[high_noise_sample_method]); } } std::vector make_hires_sigma_schedule(StableDiffusionGGML* sd, - const sd_hires_params_t& hires, - const sd_sample_params_t& sample_params, - sample_method_t sample_method, + const sd::HiresConfig& hires, + const sd::SampleConfig& sample_params, int default_steps, int sample_seq_len, int* scheduler_steps_out) { @@ -512,9 +508,7 @@ namespace sd::pipeline { int scheduler_steps = static_cast(effective_steps / hires.denoising_strength); scheduler_steps = std::max(1, scheduler_steps); - scheduler_t scheduler = resolve_scheduler(sd, - sample_params.scheduler, - sample_method); + scheduler_t scheduler = sample_params.scheduler; std::vector sigmas = sd->denoiser->get_sigmas(scheduler_steps, sample_seq_len, scheduler, diff --git a/src/pipeline/request.h b/src/pipeline/request.h index b1ab6404cb..1e15db8349 100644 --- a/src/pipeline/request.h +++ b/src/pipeline/request.h @@ -1,10 +1,13 @@ #ifndef __SD_PIPELINE_REQUEST_H__ #define __SD_PIPELINE_REQUEST_H__ +#include "core/option_types.h" + #include #include #include +#include "config.h" #include "stable-diffusion.h" class StableDiffusionGGML; @@ -19,52 +22,47 @@ namespace sd::pipeline { int64_t resolve_seed(int64_t seed); - float resolve_eta(StableDiffusionGGML* sd, - float eta, - enum sample_method_t sample_method); - struct GenerationRequest { std::string prompt; std::string negative_prompt; std::map models; - int width = -1; - int height = -1; - int clip_skip = -1; - int vae_scale_factor = -1; - int diffusion_model_down_factor = -1; - int64_t seed = -1; - bool use_uncond = false; - bool use_img_uncond = false; - bool use_high_noise_uncond = false; - bool use_high_noise_img_uncond = false; - bool has_ref_images = false; - bool is_layered_image = false; - bool use_high_noise = false; - bool use_vace = false; - const sd_cache_params_t* cache_params = nullptr; - int batch_count = 1; - int qwen_image_layers = 3; - int shifted_timestep = 0; - float strength = 1.f; - float control_strength = 0.f; - sd_control_type_t control_type = SD_CONTROL_TYPE_NONE; - float eta = 0.f; - sd_sample_params_t sample_params = {}; - sd_sample_params_t high_noise_sample_params = {}; - sd_guidance_params_t guidance = {}; - sd_guidance_params_t high_noise_guidance = {}; - sd_pm_params_t pm_params = {}; - sd_pulid_params_t pulid_params = {}; - sd_hires_params_t hires = {}; - sd_tiling_params_t vae_tiling_params = {}; - int frames = -1; - int requested_frames = -1; - int fps = 16; - float vace_strength = 1.f; - - GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params); - - GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params); + int width = -1; + int height = -1; + int clip_skip = -1; + int vae_scale_factor = -1; + int diffusion_model_down_factor = -1; + int64_t seed = -1; + bool use_uncond = false; + bool use_img_uncond = false; + bool use_high_noise_uncond = false; + bool use_high_noise_img_uncond = false; + bool has_ref_images = false; + bool is_layered_image = false; + bool use_high_noise = false; + bool use_vace = false; + const sd::CacheConfig* cache_params = nullptr; + int batch_count = 1; + int qwen_image_layers = 3; + int shifted_timestep = 0; + float strength = 1.f; + float control_strength = 0.f; + sd_control_type_t control_type = SD_CONTROL_TYPE_NONE; + sd::SampleConfig sample_params = {}; + sd::SampleConfig high_noise_sample_params = {}; + sd::GuidanceConfig guidance = {}; + sd::GuidanceConfig high_noise_guidance = {}; + sd::PhotoMakerConfig pm_params = {}; + sd::PuLIDConfig pulid_params = {}; + sd::HiresConfig hires = {}; + sd::TilingConfig vae_tiling_params = {}; + int frames = -1; + int requested_frames = -1; + int fps = 16; + float vace_strength = 1.f; + + GenerationRequest(StableDiffusionGGML* sd, const sd::ImageConfig* sd_img_gen_params); + + GenerationRequest(StableDiffusionGGML* sd, const sd::VideoConfig* sd_vid_gen_params); int64_t seed_for_batch(int index) const; @@ -75,7 +73,7 @@ namespace sd::pipeline { void resolve_hires(); static void resolve_guidance(StableDiffusionGGML* sd, - sd_guidance_params_t* guidance, + sd::GuidanceConfig* guidance, bool* use_uncond, bool* use_img_uncond, bool has_ref_images, @@ -99,22 +97,21 @@ namespace sd::pipeline { std::vector sigmas; SamplePlan(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, const GenerationRequest& request); SamplePlan(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, const GenerationRequest& request); void resolve(StableDiffusionGGML* sd, const GenerationRequest* request, - const sd_sample_params_t* sample_params); + const sd::SampleConfig* sample_params); }; std::vector make_hires_sigma_schedule(StableDiffusionGGML* sd, - const sd_hires_params_t& hires, - const sd_sample_params_t& sample_params, - sample_method_t sample_method, + const sd::HiresConfig& hires, + const sd::SampleConfig& sample_params, int default_steps, int sample_seq_len, int* scheduler_steps_out); diff --git a/src/pipeline/seedvr2.cpp b/src/pipeline/seedvr2.cpp index 5fe2eaa444..cb07876fd9 100644 --- a/src/pipeline/seedvr2.cpp +++ b/src/pipeline/seedvr2.cpp @@ -1,3 +1,4 @@ +#include "api/result.h" #include "pipeline/video_upscale.h" #include @@ -54,7 +55,7 @@ namespace sd { vae_->runner_end(); } - bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t* result) override { + bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) override { const int chunk = params.temporal_chunk_size; if (chunk < 1 || chunk > INT_MAX - 3 || (chunk - 1) % 4) { LOG_ERROR("SeedVR2 temporal chunk size must be 1 or 4n+1 input frames"); diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index ab645407df..529bbb60ce 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -1,3 +1,4 @@ +#include "core/option_types.h" #include "generation.h" #include "generation_info.h" @@ -22,8 +23,8 @@ namespace sd::pipeline { - static sd_audio_t* waveform_to_sd_audio(const StableDiffusionGGML* sd, - const sd::Tensor& waveform) { + static sd::Audio* waveform_to_sd_audio(const StableDiffusionGGML* sd, + const sd::Tensor& waveform) { if (sd == nullptr || waveform.empty()) { return nullptr; } @@ -34,7 +35,7 @@ namespace sd::pipeline { return nullptr; } - sd_audio_t* audio = (sd_audio_t*)malloc(sizeof(sd_audio_t)); + sd::Audio* audio = (sd::Audio*)malloc(sizeof(sd::Audio)); if (audio == nullptr) { return nullptr; } @@ -374,7 +375,7 @@ namespace sd::pipeline { return sd::zeros({audio_length, 2, 32, 1}); } - static sd::Tensor prepare_minimax_h3_reference_waveform(const sd_audio_t& audio, + static sd::Tensor prepare_minimax_h3_reference_waveform(const sd::Audio& audio, int target_sample_rate = 32000) { if (audio.data == nullptr || audio.sample_count == 0 || audio.channels == 0 || audio.sample_rate == 0) { return {}; @@ -490,7 +491,7 @@ namespace sd::pipeline { } static std::optional prepare_video_generation_latents(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, GenerationRequest* request, const RefImageParams& ref_image_params) { ImageGenerationLatents latents; @@ -576,7 +577,7 @@ namespace sd::pipeline { std::vector{width, height, image.shape()[2], image.shape()[3]}); }; - auto encode_reference_audio = [&](const sd_audio_t& audio, + auto encode_reference_audio = [&](const sd::Audio& audio, int32_t* audio_index) -> bool { if (sd->audio_vae_model == nullptr) { LOG_ERROR("MiniMax-H3 Ref2VA audio requires --audio-vae with encoder weights"); @@ -1194,7 +1195,7 @@ namespace sd::pipeline { } static std::optional prepare_video_generation_embeds(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, const GenerationRequest& request, const ImageGenerationLatents& latents, const RefImageParams& ref_image_params) { @@ -1264,10 +1265,10 @@ namespace sd::pipeline { return embeds; } - static sd_image_t* decode_video_outputs(StableDiffusionGGML* sd, - const GenerationRequest& request, - const sd::Tensor& final_latent, - int* num_frames_out) { + static sd::Image* decode_video_outputs(StableDiffusionGGML* sd, + const GenerationRequest& request, + const sd::Tensor& final_latent, + int* num_frames_out) { if (final_latent.empty()) { LOG_ERROR("no latent video to decode"); return nullptr; @@ -1305,7 +1306,7 @@ namespace sd::pipeline { vid = sd::ops::slice(vid, 2, 0, request.frames); } - sd_image_t* result_images = (sd_image_t*)calloc(vid.shape()[2], sizeof(sd_image_t)); + sd::Image* result_images = (sd::Image*)calloc(vid.shape()[2], sizeof(sd::Image)); if (result_images == nullptr) { return nullptr; } @@ -1661,7 +1662,7 @@ namespace sd::pipeline { } static bool apply_ltxv_refine_image_conditioning(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, const GenerationRequest& request, const ImageGenerationLatents& latents, sd::Tensor* latent, @@ -1765,8 +1766,8 @@ namespace sd::pipeline { } static bool generate_animatediff_video(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, - sd_image_t** frames_out, + const sd::VideoConfig* sd_vid_gen_params, + sd::Image** frames_out, int* num_frames_out) { int n_frames = sd_vid_gen_params->video_frames; if (n_frames < 1) { @@ -1778,8 +1779,8 @@ namespace sd::pipeline { n_frames = 32; } - sd_img_gen_params_t img_gen_params; - sd_img_gen_params_init(&img_gen_params); + sd::ImageConfig img_gen_params; + img_gen_params.loras = sd_vid_gen_params->loras; img_gen_params.lora_count = sd_vid_gen_params->lora_count; img_gen_params.prompt = sd_vid_gen_params->prompt; @@ -1815,10 +1816,10 @@ namespace sd::pipeline { } bool generate_video(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, - sd_image_t** frames_out, + const sd::VideoConfig* sd_vid_gen_params, + sd::Image** frames_out, int* num_frames_out, - sd_audio_t** audio_out, + sd::Audio** audio_out, int* fps_out, nlohmann::json* info_out) { if (info_out) @@ -1847,7 +1848,7 @@ namespace sd::pipeline { sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params; sd->apply_circular_axes(sd_vid_gen_params->circular_x, sd_vid_gen_params->circular_y); sd::ImagePreprocessor preprocessing(sd_vid_gen_params->image_preprocess.rules); - sd_vid_gen_params_t processed_params = *sd_vid_gen_params; + sd::VideoConfig processed_params = *sd_vid_gen_params; if (!preprocessing.prepare_inputs(processed_params, request.width, request.height)) return false; sd_vid_gen_params = &processed_params; @@ -1897,7 +1898,6 @@ namespace sd::pipeline { sd->rng->manual_seed(request.seed); sd->sampler_rng->manual_seed(request.seed); - sd->set_flow_shift(sd_vid_gen_params->sample_params.flow_shift); if (!sd->apply_loras(sd_vid_gen_params->loras, sd_vid_gen_params->lora_count)) return false; sd->reset_generation_extensions(); @@ -2141,13 +2141,10 @@ namespace sd::pipeline { make_hires_sigma_schedule(sd, request.hires, request.sample_params, - hires_sample_method, plan.sample_steps, sd->get_image_seq_len(hires_request.height, hires_request.width) * T, &hires_scheduler_steps); - float hires_eta = resolve_eta(sd, - sd_vid_gen_params->sample_params.eta, - hires_sample_method); + float hires_eta = plan.eta; LOG_VERBOSE("sample(latent upscale) %dx%dx%d", W, H, T); LOG_INFO("video latent spatial upscale refine: scheduler_steps=%d, denoising_strength=%.2f, sampler=%s, sigma_sched_size=%zu%s", @@ -2197,11 +2194,11 @@ namespace sd::pipeline { int64_t latent_end = ggml_time_ms(); LOG_INFO("generating latent video completed, taking %.2fs", (latent_end - latent_start) * 1.0f / 1000); - sd_audio_t* generated_audio = nullptr; + sd::Audio* generated_audio = nullptr; if (sd->version == VERSION_WAN2_2_S2V && sd_vid_gen_params->ref_audios_count > 0) { // Return the driving track for muxing with the generated video. - const sd_audio_t& driving = sd_vid_gen_params->ref_audios[0]; - generated_audio = (sd_audio_t*)malloc(sizeof(sd_audio_t)); + const sd::Audio& driving = sd_vid_gen_params->ref_audios[0]; + generated_audio = (sd::Audio*)malloc(sizeof(sd::Audio)); if (generated_audio != nullptr) { generated_audio->sample_rate = driving.sample_rate; generated_audio->channels = driving.channels; @@ -2270,13 +2267,13 @@ namespace sd::pipeline { if (sd->get_cancel_flag() == SD_CANCEL_ALL) { LOG_ERROR("cancelling generation before video decode"); - free_sd_audio(generated_audio); + sd::free_audio(generated_audio); return false; } auto result = decode_video_outputs(sd, latent_upscale_enabled ? hires_request : request, final_latent, num_frames_out); LOG_DEBUG("decode_video_outputs returned %s", result == nullptr ? "nullptr (failed)" : "frames"); if (result == nullptr) { - free_sd_audio(generated_audio); + sd::free_audio(generated_audio); return false; } @@ -2305,14 +2302,14 @@ namespace sd::pipeline { try { *info_out = info; } catch (...) { - free_sd_audio(generated_audio); + sd::free_audio(generated_audio); throw; } } if (audio_out != nullptr) { *audio_out = generated_audio; } else { - free_sd_audio(generated_audio); + sd::free_audio(generated_audio); } return true; } diff --git a/src/pipeline/video_upscale.cpp b/src/pipeline/video_upscale.cpp index f9b631e637..98837e8e7a 100644 --- a/src/pipeline/video_upscale.cpp +++ b/src/pipeline/video_upscale.cpp @@ -1,4 +1,7 @@ #include "pipeline/video_upscale.h" +#include "api/options.h" +#include "api/result.h" +#include "generation_info.h" #include @@ -66,7 +69,7 @@ namespace sd { model_.reset(); } - bool VideoUpscaler::init(const sd_video_upscaler_ctx_params_t& params) { + bool VideoUpscaler::init(const sd::VideoUpscalerConfig& params) { if (!params.diffusion_model_path || !*params.diffusion_model_path || !params.vae_path || !*params.vae_path || params.upscale_factor <= 0) { LOG_ERROR("video upscale requires diffusion and decoder model paths and a positive upscale factor"); @@ -110,7 +113,7 @@ namespace sd { return model_->init(runtime_, weights, upscale_factor_); } - bool VideoUpscaler::upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t** result_out) { + bool VideoUpscaler::upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult** result_out) { if (!result_out) return false; *result_out = nullptr; @@ -130,11 +133,11 @@ namespace sd { } } sd::ParallelScope scope(executor_.get()); - auto result = std::unique_ptr( - static_cast(calloc(1, sizeof(sd_video_generation_result_t))), free_sd_video_generation_result); + auto result = std::unique_ptr( + static_cast(calloc(1, sizeof(sd::VideoGenerationResult))), sd::free_video_generation_result); if (!result) return false; - result->frames = static_cast(calloc(params.frame_count, sizeof(sd_image_t))); + result->frames = static_cast(calloc(params.frame_count, sizeof(sd::Image))); if (!result->frames) return false; result->frame_count = params.frame_count; @@ -148,55 +151,50 @@ namespace sd { } // namespace pipeline } // namespace sd -struct video_upscaler_ctx_t { +struct sd_vid_upscaler { sd::pipeline::VideoUpscaler upscaler; }; -void sd_video_upscaler_ctx_params_init(sd_video_upscaler_ctx_params_t* params) { - if (!params) - return; - *params = {}; - params->n_threads = sd_get_num_physical_cores(); +sd_status_t sd_vid_upscaler_create(const sd_vid_upscaler_opt_t* opt, sd_vid_upscaler_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::VidUpscaler, config); + if (status != SD_STATUS_OK) + return status; + auto result = std::make_unique(); + if (!result->upscaler.init(config.post.video_upscaler)) + return sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not load video upscaler"); + *out = result.release(); + return SD_STATUS_OK; + }, + SD_STATUS_LOAD_FAILED); } -void sd_video_upscale_params_init(sd_video_upscale_params_t* params) { - if (!params) - return; - *params = {}; - params->fps = 24; - params->seed = 42; - params->window_size = 16; - params->temporal_chunk_size = 5; +void sd_vid_upscaler_free(sd_vid_upscaler_t* upscaler) { + delete upscaler; } -video_upscaler_ctx_t* new_video_upscaler_ctx(const sd_video_upscaler_ctx_params_t* params) { - if (!params) - return nullptr; - try { - auto ctx = std::make_unique(); - if (!ctx->upscaler.init(*params)) - return nullptr; - return ctx.release(); - } catch (const std::exception& error) { - LOG_ERROR("video upscaler initialization failed: %s", error.what()); - return nullptr; - } -} - -bool upscale_video(video_upscaler_ctx_t* ctx, const sd_video_upscale_params_t* params, sd_video_generation_result_t** result_out) { - if (!result_out) - return false; - *result_out = nullptr; - if (!ctx || !params) - return false; - try { - return ctx->upscaler.upscale(*params, result_out); - } catch (const std::exception& error) { - LOG_ERROR("video upscale failed: %s", error.what()); - return false; - } -} - -void free_video_upscaler_ctx(video_upscaler_ctx_t* ctx) { - delete ctx; +sd_status_t sd_upscale_video(sd_vid_upscaler_t* upscaler, const sd_vid_upscale_opt_t* opt, sd_result_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::VidUpscale, config); + if (status != SD_STATUS_OK) + return status; + if (!upscaler) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required"); + sd::VideoGenerationResult* video = nullptr; + if (!upscaler->upscaler.upscale(config.post.video_upscale, &video)) + return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "video upscale failed"); + return sd::make_video_result(video, out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result"); + }, + SD_STATUS_GENERATION_FAILED); } diff --git a/src/pipeline/video_upscale.h b/src/pipeline/video_upscale.h index ec5815c44f..51ab7681ed 100644 --- a/src/pipeline/video_upscale.h +++ b/src/pipeline/video_upscale.h @@ -4,6 +4,8 @@ #include #include +#include "api/result.h" +#include "config.h" #include "core/ggml_extend_backend.h" #include "core/tensor.hpp" #include "model_manager.h" @@ -34,7 +36,7 @@ namespace sd { virtual ~VideoUpscaleModel() = default; virtual bool init(VideoUpscaleRuntime& runtime, const String2TensorStorage& weights, int upscale_factor) = 0; virtual void end() = 0; - virtual bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t* result) = 0; + virtual bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) = 0; }; bool is_relay_vsr_model(const String2TensorStorage& weights); @@ -50,8 +52,8 @@ namespace sd { public: ~VideoUpscaler(); - bool init(const sd_video_upscaler_ctx_params_t& params); - bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t** result_out); + bool init(const sd::VideoUpscalerConfig& params); + bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult** result_out); }; } // namespace pipeline diff --git a/src/runtime/denoiser.hpp b/src/runtime/denoiser.hpp index 701d8d9983..b947074694 100644 --- a/src/runtime/denoiser.hpp +++ b/src/runtime/denoiser.hpp @@ -1,6 +1,8 @@ #ifndef __SD_RUNTIME_DENOISER_HPP__ #define __SD_RUNTIME_DENOISER_HPP__ +#include "core/option_types.h" + #include #include #include diff --git a/src/runtime/image_preprocess.cpp b/src/runtime/image_preprocess.cpp index 445f0d1fc8..d5e17d9984 100644 --- a/src/runtime/image_preprocess.cpp +++ b/src/runtime/image_preprocess.cpp @@ -196,7 +196,7 @@ namespace sd { return options; } - bool ImagePreprocessor::validate_inputs(const sd_img_gen_params_t& params) const { + bool ImagePreprocessor::validate_inputs(const sd::ImageConfig& params) const { const std::map counts = { {ImageTarget::Init, params.init_image.data != nullptr}, {ImageTarget::Mask, params.mask_image.data != nullptr}, @@ -215,7 +215,7 @@ namespace sd { return valid_; } - bool ImagePreprocessor::validate_inputs(const sd_vid_gen_params_t& params) const { + bool ImagePreprocessor::validate_inputs(const sd::VideoConfig& params) const { const std::map counts = { {ImageTarget::Init, params.init_image.data != nullptr}, {ImageTarget::End, params.end_image.data != nullptr}, @@ -370,7 +370,7 @@ namespace sd { return apply_transform(image, options, p, label, default_filter); } - Tensor ImagePreprocessor::preprocess_input(sd_image_t image, ImageTarget target, int index, int width, int height) { + Tensor ImagePreprocessor::preprocess_input(sd::Image image, ImageTarget target, int index, int width, int height) { if (image.data == nullptr || image.width == 0 || image.height == 0 || image.width > INT_MAX || image.height > INT_MAX || image.channel < 1 || image.channel > 4) { fail(std::string(enum_name(target, image_targets)) + ": invalid input image"); return {}; @@ -419,7 +419,7 @@ namespace sd { std::free(image.data); } - bool ImagePreprocessor::prepare_image(sd_image_t& image, ImageTarget target, int index, int width, int height) { + bool ImagePreprocessor::prepare_image(sd::Image& image, ImageTarget target, int index, int width, int height) { if (image.data == nullptr) return true; auto options = resolve_options(target, index); @@ -431,13 +431,13 @@ namespace sd { if (output.data == nullptr) return fail("could not allocate input preprocessing buffer"); owned_images_.push_back(output); - if (canny && !preprocess_canny(output, 0.08f, 0.08f, 0.8f, 1.f, false)) + if (canny && !sd::preprocess_canny(output, 0.08f, 0.08f, 0.8f, 1.f, false)) return fail("Canny preprocessing failed"); image = output; return true; } - bool ImagePreprocessor::prepare_array(sd_image_t*& images, int count, ImageTarget target, std::vector& storage, int width, int height) { + bool ImagePreprocessor::prepare_array(sd::Image*& images, int count, ImageTarget target, std::vector& storage, int width, int height) { if (count < 0 || (count > 0 && images == nullptr)) return fail(std::string("invalid image array: ") + enum_name(target, image_targets)); if (count == 0) @@ -453,7 +453,7 @@ namespace sd { return true; } - bool ImagePreprocessor::prepare_inputs(sd_img_gen_params_t& params, int width, int height) { + bool ImagePreprocessor::prepare_inputs(sd::ImageConfig& params, int width, int height) { if (prepared_) return fail("inputs have already been prepared"); prepared_ = true; @@ -470,7 +470,7 @@ namespace sd { return true; } - bool ImagePreprocessor::prepare_inputs(sd_vid_gen_params_t& params, int width, int height) { + bool ImagePreprocessor::prepare_inputs(sd::VideoConfig& params, int width, int height) { if (prepared_) return fail("inputs have already been prepared"); prepared_ = true; diff --git a/src/runtime/image_preprocess.h b/src/runtime/image_preprocess.h index 7dfc458992..078befdb9c 100644 --- a/src/runtime/image_preprocess.h +++ b/src/runtime/image_preprocess.h @@ -1,11 +1,14 @@ #ifndef __SD_RUNTIME_IMAGE_PREPROCESS_H__ #define __SD_RUNTIME_IMAGE_PREPROCESS_H__ +#include "core/media.h" + #include #include #include #include +#include "config.h" #include "core/tensor.hpp" #include "stable-diffusion.h" @@ -61,30 +64,30 @@ namespace sd { ImageTransform init_transform_; bool has_init_transform_ = false; bool prepared_ = false; - std::vector owned_images_; - std::vector ref_images_; - std::vector id_images_; - std::vector control_frames_; + std::vector owned_images_; + std::vector ref_images_; + std::vector id_images_; + std::vector control_frames_; bool fail(const std::string& message) const; std::map resolve_options(ImageTarget target, int index) const; Tensor apply_transform(const Tensor& image, const std::map& options, ImageTransform plan, const std::string& label, ops::InterpolateMode default_filter) const; - bool prepare_image(sd_image_t& image, ImageTarget target, int index, int width, int height); - bool prepare_array(sd_image_t*& images, int count, ImageTarget target, std::vector& storage, int width = -1, int height = -1); + bool prepare_image(sd::Image& image, ImageTarget target, int index, int width, int height); + bool prepare_array(sd::Image*& images, int count, ImageTarget target, std::vector& storage, int width = -1, int height = -1); public: explicit ImagePreprocessor(const char* rules = nullptr); ~ImagePreprocessor(); ImagePreprocessor(const ImagePreprocessor&) = delete; ImagePreprocessor& operator=(const ImagePreprocessor&) = delete; - bool prepare_inputs(sd_img_gen_params_t& params, int width, int height); - bool prepare_inputs(sd_vid_gen_params_t& params, int width, int height); + bool prepare_inputs(sd::ImageConfig& params, int width, int height); + bool prepare_inputs(sd::VideoConfig& params, int width, int height); bool is_valid() const { return valid_; } - bool validate_inputs(const sd_img_gen_params_t& params) const; - bool validate_inputs(const sd_vid_gen_params_t& params) const; + bool validate_inputs(const sd::ImageConfig& params) const; + bool validate_inputs(const sd::VideoConfig& params) const; Tensor apply_geometry(const Tensor& image, ImageTarget target, int index, int width, int height, ImageResizeMode default_mode = ImageResizeMode::Stretch, ops::InterpolateMode default_filter = ops::InterpolateMode::Nearest, ImageTransform* plan_out = nullptr) const; - Tensor preprocess_input(sd_image_t image, ImageTarget target, int index = 0, int width = -1, int height = -1); + Tensor preprocess_input(sd::Image image, ImageTarget target, int index = 0, int width = -1, int height = -1); }; } // namespace sd diff --git a/src/runtime/preprocessing.hpp b/src/runtime/preprocessing.hpp index ab4ca29b64..da91e4666b 100644 --- a/src/runtime/preprocessing.hpp +++ b/src/runtime/preprocessing.hpp @@ -1,6 +1,8 @@ #ifndef __SD_RUNTIME_PREPROCESSING_HPP__ #define __SD_RUNTIME_PREPROCESSING_HPP__ +#include "core/media.h" + #include #include @@ -93,7 +95,7 @@ static inline void preprocessing_tensor_frame_to_sd_image(const sd::Tensor sd_image_to_preprocessing_tensor(sd_image_t image) { +static inline sd::Tensor sd_image_to_preprocessing_tensor(sd::Image image) { sd::Tensor tensor({static_cast(image.width), static_cast(image.height), static_cast(image.channel), 1}); for (uint32_t y = 0; y < image.height; ++y) { for (uint32_t x = 0; x < image.width; ++x) { @@ -293,7 +295,7 @@ static inline void threshold_hystersis(sd::Tensor* img, float high_thresh } } -bool preprocess_canny(sd_image_t img, float high_threshold, float low_threshold, float weak, float strong, bool inverse) { +bool sd::preprocess_canny(sd::Image img, float high_threshold, float low_threshold, float weak, float strong, bool inverse) { float kX[9] = { -1, 0, 1, -2, 0, 2, diff --git a/src/runtime/sample-cache.cpp b/src/runtime/sample-cache.cpp index 0b92c73358..c8ea2c12ce 100644 --- a/src/runtime/sample-cache.cpp +++ b/src/runtime/sample-cache.cpp @@ -1,8 +1,9 @@ #include "runtime/sample-cache.h" +#include "config.h" namespace sd_sample { - static float get_cache_reuse_threshold(const sd_cache_params_t& params) { + static float get_cache_reuse_threshold(const sd::CacheConfig& params) { float reuse_threshold = params.reuse_threshold; if (reuse_threshold == INFINITY) { if (params.mode == SD_CACHE_EASYCACHE) { @@ -26,7 +27,7 @@ namespace sd_sample { return mode == SampleCacheMode::CACHEDIT; } - static bool has_valid_cache_percent_range(const sd_cache_params_t& cache_params) { + static bool has_valid_cache_percent_range(const sd::CacheConfig& cache_params) { if (cache_params.mode != SD_CACHE_EASYCACHE && cache_params.mode != SD_CACHE_UCACHE) { return true; } @@ -40,7 +41,7 @@ namespace sd_sample { static void init_easycache_runtime(SampleCacheRuntime& runtime, SDVersion version, - const sd_cache_params_t& cache_params, + const sd::CacheConfig& cache_params, Denoiser* denoiser) { if (!sd_version_is_dit(version)) { LOG_WARN("EasyCache requested but not supported for this model type"); @@ -68,7 +69,7 @@ namespace sd_sample { static void init_ucache_runtime(SampleCacheRuntime& runtime, SDVersion version, - const sd_cache_params_t& cache_params, + const sd::CacheConfig& cache_params, Denoiser* denoiser, const std::vector& sigmas) { if (!sd_version_is_unet(version)) { @@ -104,7 +105,7 @@ namespace sd_sample { static void init_cachedit_runtime(SampleCacheRuntime& runtime, SDVersion version, - const sd_cache_params_t& cache_params, + const sd::CacheConfig& cache_params, const std::vector& sigmas) { if (!sd_version_is_dit(version)) { LOG_WARN("CacheDIT requested but not supported for this model type (only DiT models)"); @@ -147,7 +148,7 @@ namespace sd_sample { static void init_spectrum_runtime(SampleCacheRuntime& runtime, SDVersion version, - const sd_cache_params_t& cache_params, + const sd::CacheConfig& cache_params, const std::vector& sigmas) { if (!sd_version_is_unet(version) && !sd_version_is_dit(version)) { LOG_WARN("Spectrum requested but not supported for this model type (only UNET and DiT models)"); @@ -174,7 +175,7 @@ namespace sd_sample { } SampleCacheRuntime init_sample_cache_runtime(SDVersion version, - const sd_cache_params_t* cache_params, + const sd::CacheConfig* cache_params, Denoiser* denoiser, const std::vector& sigmas) { SampleCacheRuntime runtime; diff --git a/src/runtime/sample-cache.h b/src/runtime/sample-cache.h index 30ff66e5fe..fd22edc819 100644 --- a/src/runtime/sample-cache.h +++ b/src/runtime/sample-cache.h @@ -1,6 +1,8 @@ #ifndef __SD_RUNTIME_SAMPLE_CACHE_H__ #define __SD_RUNTIME_SAMPLE_CACHE_H__ +#include "config.h" + #include #include "core/tensor.hpp" @@ -51,7 +53,7 @@ namespace sd_sample { }; SampleCacheRuntime init_sample_cache_runtime(SDVersion version, - const sd_cache_params_t* cache_params, + const sd::CacheConfig* cache_params, Denoiser* denoiser, const std::vector& sigmas); diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index df2cfcbbbb..acc3672cf0 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -1,901 +1,135 @@ -#include "stable-diffusion.h" -#include "generation_info.h" - -#include -#include -#include -#include - -#include "core/ggml_extend.h" -#include "core/util.h" +#include "api/api.h" +#include "api/options.h" +#include "api/result.h" #include "pipeline/diffusion_engine.h" #include "pipeline/generation.h" -#include "pipeline/request.h" - -#define NONE_STR "NONE" - -static float get_cache_reuse_threshold(const sd_cache_params_t& params) { - float reuse_threshold = params.reuse_threshold; - if (reuse_threshold == INFINITY) { - if (params.mode == SD_CACHE_EASYCACHE) { - reuse_threshold = 0.2f; - } else if (params.mode == SD_CACHE_UCACHE) { - reuse_threshold = 1.0f; - } - } - return std::max(0.0f, reuse_threshold); -} - -const char* sd_type_name(enum sd_type_t type) { - if (type == SD_TYPE_F8_E4M3) { - return "f8_e4m3"; - } - if (type == SD_TYPE_F8_E5M2) { - return "f8_e5m2"; - } - const auto ggml_type = sd_type_to_ggml_type(type); - if (ggml_type != GGML_TYPE_COUNT) { - return ggml_type_name(ggml_type); - } - return NONE_STR; -} - -enum sd_type_t str_to_sd_type(const char* str) { - if (!strcmp(str, "f8_e4m3")) { - return SD_TYPE_F8_E4M3; - } - if (!strcmp(str, "f8_e5m2")) { - return SD_TYPE_F8_E5M2; - } - for (int i = 0; i < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT); i++) { - auto trait = ggml_get_type_traits((ggml_type)i); - if (!strcmp(str, trait->type_name)) { - return (enum sd_type_t)i; - } - } - return SD_TYPE_COUNT; -} - -const char* rng_type_to_str[] = { - "std_default", - "cuda", - "cpu", -}; - -const char* sd_rng_type_name(enum rng_type_t rng_type) { - if (rng_type < RNG_TYPE_COUNT) { - return rng_type_to_str[rng_type]; - } - return NONE_STR; -} - -enum rng_type_t str_to_rng_type(const char* str) { - for (int i = 0; i < RNG_TYPE_COUNT; i++) { - if (!strcmp(str, rng_type_to_str[i])) { - return (enum rng_type_t)i; - } - } - return RNG_TYPE_COUNT; -} - -const char* sample_method_to_str[] = { - "euler", - "euler_a", - "heun", - "dpm2", - "dpm++2s_a", - "dpm++2m", - "dpm++2mv2", - "ipndm", - "ipndm_v", - "lcm", - "ddim_trailing", - "tcd", - "res_multistep", - "res_2s", - "er_sde", - "euler_cfg_pp", - "euler_a_cfg_pp", - "euler_ge", - "dpm++2m_sde", - "dpm++2m_sde_bt", - "lms", -}; - -static_assert(SAMPLE_METHOD_COUNT == sizeof(sample_method_to_str) / sizeof(sample_method_to_str[0]), - "\nnumber of elements in sample_method_to_str[] != SAMPLE_METHOD_COUNT"); - -const char* sd_sample_method_name(enum sample_method_t sample_method) { - if (sample_method < SAMPLE_METHOD_COUNT) { - return sample_method_to_str[sample_method]; - } - return NONE_STR; -} - -enum sample_method_t str_to_sample_method(const char* str) { - for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) { - if (!strcmp(str, sample_method_to_str[i])) { - return (enum sample_method_t)i; - } - } - return SAMPLE_METHOD_COUNT; -} - -const char* scheduler_to_str[] = { - "discrete", - "karras", - "exponential", - "ays", - "gits", - "sgm_uniform", - "simple", - "smoothstep", - "kl_optimal", - "lcm", - "bong_tangent", - "ltx2", - "logit_normal", - "flux2", - "flux", - "beta", - "llada_image", -}; - -static_assert(SCHEDULER_COUNT == sizeof(scheduler_to_str) / sizeof(scheduler_to_str[0]), - "\nnumber of elements in scheduler_to_str[] != SCHEDULER_COUNT"); - -const char* sd_scheduler_name(enum scheduler_t scheduler) { - if (scheduler < SCHEDULER_COUNT) { - return scheduler_to_str[scheduler]; - } - return NONE_STR; -} - -enum scheduler_t str_to_scheduler(const char* str) { - if (!strcmp(str, "normal")) { - return DISCRETE_SCHEDULER; - } - for (int i = 0; i < SCHEDULER_COUNT; i++) { - if (!strcmp(str, scheduler_to_str[i])) { - return (enum scheduler_t)i; - } - } - return SCHEDULER_COUNT; -} - -const char* prediction_to_str[] = { - "eps", - "v", - "edm_v", - "sd3_flow", - "flux_flow", - "sefi_flow", - "minit2i_flow", - "sensenova_u1_flow", - "flux3_flow", - "h3_av_flow", - "ming_image_flow", -}; - -const char* sd_prediction_name(enum prediction_t prediction) { - if (prediction >= EPS_PRED && prediction < PREDICTION_COUNT) { - return prediction_to_str[prediction]; - } - return NONE_STR; -} - -enum prediction_t str_to_prediction(const char* str) { - for (int i = 0; i < PREDICTION_COUNT; i++) { - if (!strcmp(str, prediction_to_str[i])) { - return (enum prediction_t)i; - } - } - return PREDICTION_COUNT; -} - -const char* preview_to_str[] = { - "none", - "proj", - "tae", - "vae", -}; - -const char* sd_preview_name(enum preview_t preview) { - if (preview < PREVIEW_COUNT) { - return preview_to_str[preview]; - } - return NONE_STR; -} - -enum preview_t str_to_preview(const char* str) { - for (int i = 0; i < PREVIEW_COUNT; i++) { - if (!strcmp(str, preview_to_str[i])) { - return (enum preview_t)i; - } - } - return PREVIEW_COUNT; -} - -const char* lora_apply_mode_to_str[] = { - "auto", - "immediately", - "at_runtime", -}; - -const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode) { - if (mode < LORA_APPLY_MODE_COUNT) { - return lora_apply_mode_to_str[mode]; - } - return NONE_STR; -} - -enum lora_apply_mode_t str_to_lora_apply_mode(const char* str) { - for (int i = 0; i < LORA_APPLY_MODE_COUNT; i++) { - if (!strcmp(str, lora_apply_mode_to_str[i])) { - return (enum lora_apply_mode_t)i; - } - } - return LORA_APPLY_MODE_COUNT; -} - -const char* hires_upscaler_to_str[] = { - "None", - "Latent", - "Latent (nearest)", - "Latent (nearest-exact)", - "Latent (antialiased)", - "Latent (bicubic)", - "Latent (bicubic antialiased)", - "Lanczos", - "Nearest", - "Model", -}; - -const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler) { - if (upscaler >= SD_HIRES_UPSCALER_NONE && upscaler < SD_HIRES_UPSCALER_COUNT) { - return hires_upscaler_to_str[upscaler]; - } - return NONE_STR; -} - -enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str) { - for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; i++) { - if (!strcmp(str, hires_upscaler_to_str[i])) { - return (enum sd_hires_upscaler_t)i; - } - } - return SD_HIRES_UPSCALER_COUNT; -} - -const char* control_type_to_str[] = { - "none", - "openpose", - "depth", - "scribble", - "canny", - "normal", - "segment", - "tile", - "repaint", -}; - -static_assert(SD_CONTROL_TYPE_COUNT == sizeof(control_type_to_str) / sizeof(control_type_to_str[0]), - "\nnumber of elements in control_type_to_str[] != SD_CONTROL_TYPE_COUNT"); -const char* sd_control_type_name(enum sd_control_type_t control_type) { - if (control_type >= SD_CONTROL_TYPE_NONE && control_type < SD_CONTROL_TYPE_COUNT) { - return control_type_to_str[control_type]; - } - return NONE_STR; -} - -enum sd_control_type_t str_to_sd_control_type(const char* str) { - for (int i = 0; i < SD_CONTROL_TYPE_COUNT; i++) { - if (!strcmp(str, control_type_to_str[i])) { - return (enum sd_control_type_t)i; - } - } - static const struct { - const char* alias; - enum sd_control_type_t type; - } aliases[] = { - {"hed", SD_CONTROL_TYPE_SCRIBBLE}, - {"softedge", SD_CONTROL_TYPE_SCRIBBLE}, - {"pidi", SD_CONTROL_TYPE_SCRIBBLE}, - {"ted", SD_CONTROL_TYPE_SCRIBBLE}, - {"lineart", SD_CONTROL_TYPE_CANNY}, - {"anime_lineart", SD_CONTROL_TYPE_CANNY}, - {"mlsd", SD_CONTROL_TYPE_CANNY}, - {"seg", SD_CONTROL_TYPE_SEGMENT}, - {"inpaint", SD_CONTROL_TYPE_REPAINT}, - {"outpaint", SD_CONTROL_TYPE_REPAINT}, - }; - for (const auto& entry : aliases) { - if (!strcmp(str, entry.alias)) { - return entry.type; - } - } - return SD_CONTROL_TYPE_COUNT; -} - -const char* sd_vae_format_name(enum sd_vae_format_t format) { - switch (format) { - case SD_VAE_FORMAT_AUTO: - return "auto"; - case SD_VAE_FORMAT_FLUX: - return "flux"; - case SD_VAE_FORMAT_SD3: - return "sd3"; - case SD_VAE_FORMAT_FLUX2: - return "flux2"; - case SD_VAE_FORMAT_WAN: - return "wan"; - default: - return NONE_STR; - } -} - -void sd_cache_params_init(sd_cache_params_t* cache_params) { - *cache_params = {}; - cache_params->mode = SD_CACHE_DISABLED; - cache_params->reuse_threshold = INFINITY; - cache_params->start_percent = 0.15f; - cache_params->end_percent = 0.95f; - cache_params->error_decay_rate = 1.0f; - cache_params->use_relative_threshold = true; - cache_params->reset_error_on_compute = true; - cache_params->Fn_compute_blocks = 8; - cache_params->Bn_compute_blocks = 0; - cache_params->residual_diff_threshold = 0.08f; - cache_params->max_warmup_steps = 8; - cache_params->max_cached_steps = -1; - cache_params->max_continuous_cached_steps = -1; - cache_params->taylorseer_n_derivatives = 1; - cache_params->taylorseer_skip_interval = 1; - cache_params->scm_mask = nullptr; - cache_params->scm_policy_dynamic = true; - cache_params->spectrum_w = 0.40f; - cache_params->spectrum_m = 3; - cache_params->spectrum_lam = 1.0f; - cache_params->spectrum_window_size = 2; - cache_params->spectrum_flex_window = 0.50f; - cache_params->spectrum_warmup_steps = 4; - cache_params->spectrum_stop_percent = 0.9f; -} - -void sd_hires_params_init(sd_hires_params_t* hires_params) { - *hires_params = {}; - hires_params->enabled = false; - hires_params->upscaler = SD_HIRES_UPSCALER_LATENT; - hires_params->model_path = nullptr; - hires_params->scale = 2.0f; - hires_params->target_width = 0; - hires_params->target_height = 0; - hires_params->steps = 0; - hires_params->denoising_strength = 0.7f; - hires_params->upscale_tile_size = 128; - hires_params->custom_sigmas = nullptr; - hires_params->custom_sigmas_count = 0; -} - -void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) { - *sd_ctx_params = {}; - sd_ctx_params->n_threads = sd_get_num_physical_cores(); - sd_ctx_params->conditioning_cache_size = 4; - sd_ctx_params->wtype = SD_TYPE_COUNT; - sd_ctx_params->rng_type = CUDA_RNG; - sd_ctx_params->sampler_rng_type = RNG_TYPE_COUNT; - sd_ctx_params->prediction = PREDICTION_COUNT; - sd_ctx_params->lora_apply_mode = LORA_APPLY_AUTO; - sd_ctx_params->max_vram = nullptr; - sd_ctx_params->disable_prefetch = false; - sd_ctx_params->disable_segmented_compute = false; - sd_ctx_params->batched_cfg = true; - sd_ctx_params->eager_load = false; - sd_ctx_params->enable_mmap = false; - sd_ctx_params->flash_attn_mode = nullptr; - sd_ctx_params->conv_direct_mode = nullptr; - sd_ctx_params->diffusion_flash_attn = false; - sd_ctx_params->sage_attn = false; - sd_ctx_params->linear_scale = 0.f; - sd_ctx_params->attn_scale = 0.f; - sd_ctx_params->vae_format = SD_VAE_FORMAT_AUTO; - sd_ctx_params->backend = nullptr; - sd_ctx_params->params_backend = nullptr; - sd_ctx_params->split_mode = nullptr; - sd_ctx_params->auto_fit = true; - sd_ctx_params->rpc_servers = nullptr; - sd_ctx_params->model_args = nullptr; - sd_ctx_params->pulid_weights_path = nullptr; -} - -char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { - char* buf = (char*)malloc(8192); - if (!buf) - return nullptr; - buf[0] = '\0'; - - snprintf(buf + strlen(buf), 8192 - strlen(buf), - "model_path: %s\n" - "clip_l_path: %s\n" - "clip_g_path: %s\n" - "clip_vision_path: %s\n" - "t5xxl_path: %s\n" - "llm_path: %s\n" - "llm_vision_path: %s\n" - "tokenizer: %s\n" - "diffusion_model_path: %s\n" - "high_noise_diffusion_model_path: %s\n" - "uncond_diffusion_model_path: %s\n" - "embeddings_connectors_path: %s\n" - "vae_path: %s\n" - "audio_vae_path: %s\n" - "audio_encoder_path: %s\n" - "taesd_path: %s\n" - "control_net_path: %s\n" - "photo_maker_path: %s\n" - "pulid_weights_path: %s\n" - "tensor_type_rules: %s\n" - "n_threads: %d\n" - "conditioning_cache_size: %d\n" - "wtype: %s\n" - "rng_type: %s\n" - "sampler_rng_type: %s\n" - "prediction: %s\n" - "max_vram: %s\n" - "disable_prefetch: %s\n" - "disable_segmented_compute: %s\n" - "eager_load: %s\n" - "backend: %s\n" - "params_backend: %s\n" - "split_mode: %s\n" - "model_args: %s\n" - "auto_fit: %s\n" - "flash_attn: %s\n" - "flash_attn_mode: %s\n" - "conv_direct_mode: %s\n" - "diffusion_flash_attn: %s\n" - "sage_attn: %s\n" - "linear_scale: %g\n" - "attn_scale: %g\n" - "vae_format: %s\n", - SAFE_STR(sd_ctx_params->model_path), - SAFE_STR(sd_ctx_params->clip_l_path), - SAFE_STR(sd_ctx_params->clip_g_path), - SAFE_STR(sd_ctx_params->clip_vision_path), - SAFE_STR(sd_ctx_params->t5xxl_path), - SAFE_STR(sd_ctx_params->llm_path), - SAFE_STR(sd_ctx_params->llm_vision_path), - SAFE_STR(sd_ctx_params->tokenizer), - SAFE_STR(sd_ctx_params->diffusion_model_path), - SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path), - SAFE_STR(sd_ctx_params->uncond_diffusion_model_path), - SAFE_STR(sd_ctx_params->embeddings_connectors_path), - SAFE_STR(sd_ctx_params->vae_path), - SAFE_STR(sd_ctx_params->audio_vae_path), - SAFE_STR(sd_ctx_params->audio_encoder_path), - SAFE_STR(sd_ctx_params->taesd_path), - SAFE_STR(sd_ctx_params->control_net_path), - SAFE_STR(sd_ctx_params->photo_maker_path), - SAFE_STR(sd_ctx_params->pulid_weights_path), - SAFE_STR(sd_ctx_params->tensor_type_rules), - sd_ctx_params->n_threads, - sd_ctx_params->conditioning_cache_size, - sd_type_name(sd_ctx_params->wtype), - sd_rng_type_name(sd_ctx_params->rng_type), - sd_rng_type_name(sd_ctx_params->sampler_rng_type), - sd_prediction_name(sd_ctx_params->prediction), - SAFE_STR(sd_ctx_params->max_vram), - BOOL_STR(sd_ctx_params->disable_prefetch), - BOOL_STR(sd_ctx_params->disable_segmented_compute), - BOOL_STR(sd_ctx_params->eager_load), - SAFE_STR(sd_ctx_params->backend), - SAFE_STR(sd_ctx_params->params_backend), - SAFE_STR(sd_ctx_params->split_mode), - SAFE_STR(sd_ctx_params->model_args), - BOOL_STR(sd_ctx_params->auto_fit), - BOOL_STR(sd_ctx_params->flash_attn), - sd_ctx_params->flash_attn_mode && sd_ctx_params->flash_attn_mode[0] ? sd_ctx_params->flash_attn_mode : "auto (legacy flags honored)", - sd_ctx_params->conv_direct_mode && sd_ctx_params->conv_direct_mode[0] ? sd_ctx_params->conv_direct_mode : "auto (legacy flags honored)", - BOOL_STR(sd_ctx_params->diffusion_flash_attn), - BOOL_STR(sd_ctx_params->sage_attn), - sd_ctx_params->linear_scale, - sd_ctx_params->attn_scale, - sd_vae_format_name(sd_ctx_params->vae_format)); - - return buf; -} - -void sd_sample_params_init(sd_sample_params_t* sample_params) { - *sample_params = {}; - sample_params->guidance.txt_cfg = 7.0f; - sample_params->guidance.img_cfg = INFINITY; - sample_params->guidance.distilled_guidance = 3.5f; - sample_params->guidance.slg.layer_count = 0; - sample_params->guidance.slg.layer_start = 0.01f; - sample_params->guidance.slg.layer_end = 0.2f; - sample_params->guidance.slg.scale = 0.f; - sample_params->scheduler = SCHEDULER_COUNT; - sample_params->sample_method = SAMPLE_METHOD_COUNT; - sample_params->sample_steps = 20; - sample_params->eta = INFINITY; - sample_params->custom_sigmas = nullptr; - sample_params->custom_sigmas_count = 0; - sample_params->flow_shift = INFINITY; - sample_params->extra_sample_args = nullptr; -} - -char* sd_sample_params_to_str(const sd_sample_params_t* sample_params) { - char* buf = (char*)malloc(4096); - if (!buf) - return nullptr; - buf[0] = '\0'; - - snprintf(buf + strlen(buf), 4096 - strlen(buf), - "(txt_cfg: %.2f, " - "img_cfg: %.2f, " - "distilled_guidance: %.2f, " - "slg.layer_count: %zu, " - "slg.layer_start: %.2f, " - "slg.layer_end: %.2f, " - "slg.scale: %.2f, " - "scheduler: %s, " - "sample_method: %s, " - "sample_steps: %d, " - "eta: %.2f, " - "shifted_timestep: %d, " - "flow_shift: %.2f, " - "extra_sample_args: %s)", - sample_params->guidance.txt_cfg, - std::isfinite(sample_params->guidance.img_cfg) - ? sample_params->guidance.img_cfg - : sample_params->guidance.txt_cfg, - sample_params->guidance.distilled_guidance, - sample_params->guidance.slg.layer_count, - sample_params->guidance.slg.layer_start, - sample_params->guidance.slg.layer_end, - sample_params->guidance.slg.scale, - sd_scheduler_name(sample_params->scheduler), - sd_sample_method_name(sample_params->sample_method), - sample_params->sample_steps, - sample_params->eta, - sample_params->shifted_timestep, - sample_params->flow_shift, - SAFE_STR(sample_params->extra_sample_args)); - - return buf; -} - -void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params) { - *sd_img_gen_params = {}; - sd_sample_params_init(&sd_img_gen_params->sample_params); - sd_img_gen_params->clip_skip = -1; - sd_img_gen_params->ref_images_count = 0; - sd_img_gen_params->ref_image_args = ""; - sd_img_gen_params->width = 512; - sd_img_gen_params->height = 512; - sd_img_gen_params->strength = 0.75f; - sd_img_gen_params->seed = -1; - sd_img_gen_params->batch_count = 1; - sd_img_gen_params->control_strength = 0.9f; - sd_img_gen_params->ip_adapter_strength = 1.0f; - sd_img_gen_params->qwen_image_layers = 3; - sd_img_gen_params->circular_x = false; - sd_img_gen_params->circular_y = false; - sd_img_gen_params->pm_params = {nullptr, 0, nullptr, 20.f}; - sd_img_gen_params->pulid_params = {nullptr, 1.0f}; - sd_img_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; - sd_cache_params_init(&sd_img_gen_params->cache); - sd_hires_params_init(&sd_img_gen_params->hires); -} - -char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params) { - char* buf = (char*)malloc(4096); - if (!buf) - return nullptr; - buf[0] = '\0'; - - char* sample_params_str = sd_sample_params_to_str(&sd_img_gen_params->sample_params); - - snprintf(buf + strlen(buf), 4096 - strlen(buf), - "prompt: %s\n" - "negative_prompt: %s\n" - "clip_skip: %d\n" - "width: %d\n" - "height: %d\n" - "sample_params: %s\n" - "strength: %.2f\n" - "seed: %" PRId64 - "\n" - "batch_count: %d\n" - "qwen_image_layers: %d\n" - "ref_images_count: %d\n" - "ref_image_args: %s\n" - "control_strength: %.2f\n" - "control_type: %s\n" - "photo maker: {style_strength = %.2f, id_images_count = %d, id_embed_path = %s}\n" - "VAE tiling: %s (temporal=%s, extra_tiling_args=%s)\n" - "circular_x: %s\n" - "circular_y: %s\n" - "hires: {enabled=%s, upscaler=%s, model_path=%s, scale=%.2f, target=%dx%d, steps=%d, denoising_strength=%.2f}\n", - SAFE_STR(sd_img_gen_params->prompt), - SAFE_STR(sd_img_gen_params->negative_prompt), - sd_img_gen_params->clip_skip, - sd_img_gen_params->width, - sd_img_gen_params->height, - SAFE_STR(sample_params_str), - sd_img_gen_params->strength, - sd_img_gen_params->seed, - sd_img_gen_params->batch_count, - sd_img_gen_params->qwen_image_layers, - sd_img_gen_params->ref_images_count, - SAFE_STR(sd_img_gen_params->ref_image_args), - sd_img_gen_params->control_strength, - sd_control_type_name(sd_img_gen_params->control_type), - sd_img_gen_params->pm_params.style_strength, - sd_img_gen_params->pm_params.id_images_count, - SAFE_STR(sd_img_gen_params->pm_params.id_embed_path), - BOOL_STR(sd_img_gen_params->vae_tiling_params.enabled), - BOOL_STR(sd_img_gen_params->vae_tiling_params.temporal_tiling), - SAFE_STR(sd_img_gen_params->vae_tiling_params.extra_tiling_args), - BOOL_STR(sd_img_gen_params->circular_x), - BOOL_STR(sd_img_gen_params->circular_y), - BOOL_STR(sd_img_gen_params->hires.enabled), - sd_hires_upscaler_name(sd_img_gen_params->hires.upscaler), - SAFE_STR(sd_img_gen_params->hires.model_path), - sd_img_gen_params->hires.scale, - sd_img_gen_params->hires.target_width, - sd_img_gen_params->hires.target_height, - sd_img_gen_params->hires.steps, - sd_img_gen_params->hires.denoising_strength); - const char* cache_mode_str = "disabled"; - if (sd_img_gen_params->cache.mode == SD_CACHE_EASYCACHE) { - cache_mode_str = "easycache"; - } else if (sd_img_gen_params->cache.mode == SD_CACHE_UCACHE) { - cache_mode_str = "ucache"; - } - snprintf(buf + strlen(buf), 4096 - strlen(buf), - "cache: %s (threshold=%.3f, start=%.2f, end=%.2f)\n", - cache_mode_str, - get_cache_reuse_threshold(sd_img_gen_params->cache), - sd_img_gen_params->cache.start_percent, - sd_img_gen_params->cache.end_percent); - free(sample_params_str); - return buf; -} - -void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params) { - *sd_vid_gen_params = {}; - sd_sample_params_init(&sd_vid_gen_params->sample_params); - sd_sample_params_init(&sd_vid_gen_params->high_noise_sample_params); - sd_vid_gen_params->high_noise_sample_params.sample_steps = -1; - sd_vid_gen_params->width = 512; - sd_vid_gen_params->height = 512; - sd_vid_gen_params->strength = 0.75f; - sd_vid_gen_params->seed = -1; - sd_vid_gen_params->video_frames = 6; - sd_vid_gen_params->fps = 16; - sd_vid_gen_params->moe_boundary = 0.875f; - sd_vid_gen_params->vace_strength = 1.f; - sd_vid_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; - sd_vid_gen_params->hires.enabled = false; - sd_vid_gen_params->hires.upscaler = SD_HIRES_UPSCALER_LATENT; - sd_vid_gen_params->hires.scale = 2.f; - sd_vid_gen_params->hires.target_width = 0; - sd_vid_gen_params->hires.target_height = 0; - sd_vid_gen_params->hires.steps = 0; - sd_vid_gen_params->hires.denoising_strength = 0.7f; - sd_vid_gen_params->hires.upscale_tile_size = 128; - sd_vid_gen_params->hires.custom_sigmas = nullptr; - sd_vid_gen_params->hires.custom_sigmas_count = 0; - sd_vid_gen_params->circular_x = false; - sd_vid_gen_params->circular_y = false; - sd_vid_gen_params->ref_image_args = ""; - sd_cache_params_init(&sd_vid_gen_params->cache); -} - -struct sd_ctx_t { - StableDiffusionGGML* sd = nullptr; -}; - -void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data) { - if (ctx == nullptr || ctx->sd == nullptr) { - return; - } - std::lock_guard lock(ctx->sd->execution_mutex); - if (!ctx->sd->executing_) { - ctx->sd->prompt_image_callback = cb; - ctx->sd->prompt_image_callback_data = data; - } -} - -void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data) { - if (ctx == nullptr || ctx->sd == nullptr) { - return; - } - std::lock_guard lock(ctx->sd->execution_mutex); - if (!ctx->sd->executing_) { - ctx->sd->generation_progress.set_callback(cb, data); - } -} - -sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params) { - sd_ctx_t* sd_ctx = (sd_ctx_t*)malloc(sizeof(sd_ctx_t)); - if (sd_ctx == nullptr) { - return nullptr; - } - - sd_ctx->sd = new StableDiffusionGGML(); - if (sd_ctx->sd == nullptr) { - free(sd_ctx); - return nullptr; - } - - if (!sd_ctx->sd->init(sd_ctx_params)) { - delete sd_ctx->sd; - sd_ctx->sd = nullptr; - free(sd_ctx); - return nullptr; - } - return sd_ctx; -} - -void free_sd_ctx(sd_ctx_t* sd_ctx) { - if (sd_ctx->sd != nullptr) { - delete sd_ctx->sd; - sd_ctx->sd = nullptr; - } - free(sd_ctx); -} - -SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode) { - if (sd_ctx && sd_ctx->sd) { - if (mode < SD_CANCEL_ALL || mode > SD_CANCEL_RESET) { - mode = SD_CANCEL_ALL; - } - sd_ctx->sd->set_cancel_flag(mode); - } -} - -void free_sd_audio(sd_audio_t* audio) { - if (audio == nullptr) { - return; - } - free(audio->data); - audio->data = nullptr; - free(audio); -} - -SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { - return false; - } - return sd_version_supports_image_generation(sd_ctx->sd->version); -} - -SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { - return false; - } - if (sd_ctx->sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd_ctx->sd->version)) { - return true; - } - return sd_version_supports_video_generation(sd_ctx->sd->version); -} - -SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) { - return false; - } - return sd_ctx->sd->load_control_net_from_file(path); -} - -SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { - return false; - } - return sd_ctx->sd->unload_control_net(); -} - -SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { - return false; - } - if (sd_ctx->sd->control_net != nullptr) { - return true; - } - // z-image keeps control net weights inside the diffusion runner, registered - // under ModelComponent::ControlNet instead of a separate control_net runner. - return sd_ctx->sd->model_manager != nullptr && - sd_ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0; -} - -const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_ctx->sd->version >= VERSION_COUNT) { - return "Unknown"; - } - return sd_version_name(sd_ctx->sd->version); -} - -enum sample_method_t sd_get_default_sample_method(const sd_ctx_t* sd_ctx) { - return sd::pipeline::default_sample_method(sd_ctx != nullptr ? sd_ctx->sd : nullptr); -} - -enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_method_t sample_method) { - return sd::pipeline::default_scheduler(sd_ctx != nullptr ? sd_ctx->sd : nullptr, sample_method); -} - -SD_API bool generate_image(sd_ctx_t* sd_ctx, - const sd_img_gen_params_t* params, - sd_image_generation_result_t** result_out) { +#include +#include +#include +#include + +static sd_status_t validate_generation(StableDiffusionGGML* engine, const sd::CacheConfig& cache, bool video) { + if (video ? !sd_version_supports_video_generation(engine->version) && + !(engine->config_->animatediff_loaded && sd_version_supports_animatediff(engine->version)) + : !sd_version_supports_image_generation(engine->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, nullptr, "model does not support this operation"); + if (cache.mode == SD_CACHE_UCACHE && !sd_version_is_unet(engine->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "ucache requires a UNet model"); + if ((cache.mode == SD_CACHE_EASYCACHE || cache.mode == SD_CACHE_DBCACHE || + cache.mode == SD_CACHE_TAYLORSEER || cache.mode == SD_CACHE_CACHE_DIT) && + !sd_version_is_dit(engine->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "selected cache requires a DiT model"); + if (cache.mode == SD_CACHE_SPECTRUM && !sd_version_is_unet(engine->version) && !sd_version_is_dit(engine->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "spectrum requires a UNet or DiT model"); + return SD_STATUS_OK; +} + +static sd_status_t generation_failure(StableDiffusionGGML* engine) { + if (engine->get_cancel_flag() == SD_CANCEL_ALL) + return sd::api::error(SD_STATUS_CANCELLED, nullptr, "generation cancelled"); + return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "generation failed; see log for details"); +} + +static sd_status_t run_image(sd_ctx_t* ctx, const sd::ImageConfig* params, sd::ImageGenerationResult** result_out, const sd_opt_t* keys = nullptr) { if (result_out) *result_out = nullptr; - if (!result_out || !sd_ctx || !sd_ctx->sd || !params) - return false; - StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd); + if (!result_out || !ctx || !ctx->sd || !params) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context, params and result_out are required"); + StableDiffusionGGML::ExecutionScope execution(*ctx->sd); if (!execution.ready) - return false; - sd_image_t* images = nullptr; - int count = 0; + return execution.operation.acquired + ? sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not refresh model sources") + : sd::api::error(SD_STATUS_BUSY, nullptr, "context is already executing"); + auto status = validate_generation(ctx->sd, params->cache, false); + if (status != SD_STATUS_OK) + return status; + if (sd::api::is_set(keys, "qwen_image_layers") && ctx->sd->version != VERSION_QWEN_IMAGE_LAYERED) + return sd::api::error(SD_STATUS_UNSUPPORTED, "qwen_image_layers", "requires Qwen Image Layered"); + const bool native_control = sd_version_is_control(ctx->sd->version) || ctx->sd->version == VERSION_FLEX_2 || + (sd_version_is_z_image(ctx->sd->version) && ctx->sd->model_manager && + ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0); + if (params->control_image.data && !ctx->sd->control_net && !native_control) + return sd::api::error(SD_STATUS_UNSUPPORTED, "control_image", "model has no active control-image component"); + if (params->ip_adapter_image.data && !ctx->sd->ip_adapter) + return sd::api::error(SD_STATUS_UNSUPPORTED, "ip_adapter_image", "no IP-Adapter is loaded"); + if (params->pm_params.id_images_count > 0 && !ctx->sd->config_->photomaker_source_available) + return sd::api::error(SD_STATUS_UNSUPPORTED, "photomaker.id_images", "no PhotoMaker component is loaded"); + if (params->pulid_params.id_embedding_path && *params->pulid_params.id_embedding_path && + (ctx->sd->version != VERSION_FLUX || !ctx->sd->config_->params.pulid_weights_path || + !*ctx->sd->config_->params.pulid_weights_path)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "pulid.id_embedding_path", "requires a Flux model with PuLID weights"); + if (params->sample_params.guidance.img_cfg != INFINITY && params->sample_params.guidance.img_cfg != 1.f && + !sd_version_is_inpaint_or_unet_edit(ctx->sd->version) && + !(params->ref_images_count > 0 && sd_version_supports_ref_latent_img_cfg(ctx->sd->version))) + return sd::api::error(SD_STATUS_UNSUPPORTED, "sample.guidance.img_cfg", "model does not support image CFG for these inputs"); + sd::Image* images = nullptr; + int count = 0; std::vector infos; try { - if (!sd::pipeline::generate_image(sd_ctx->sd, params, &images, &count, &infos)) { - free_sd_images(images, count); - return false; + if (!sd::pipeline::generate_image(ctx->sd, params, &images, &count, &infos)) { + sd::free_images(images, count); + return generation_failure(ctx->sd); } - } catch (const std::exception& e) { - LOG_ERROR("image generation failed: %s", e.what()); - free_sd_images(images, count); - return false; + } catch (...) { + sd::free_images(images, count); + throw; } - return sd::make_image_generation_result(images, count, infos, result_out); + return sd::make_image_generation_result(images, count, infos, result_out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate image result"); } -SD_API bool generate_video(sd_ctx_t* sd_ctx, - const sd_vid_gen_params_t* params, - sd_video_generation_result_t** result_out) { +static sd_status_t run_video(sd_ctx_t* ctx, const sd::VideoConfig* params, sd::VideoGenerationResult** result_out, const sd_opt_t* keys = nullptr) { if (result_out) *result_out = nullptr; - if (!result_out || !sd_ctx || !sd_ctx->sd || !params) - return false; - StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd); + if (!result_out || !ctx || !ctx->sd || !params) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context, params and result_out are required"); + StableDiffusionGGML::ExecutionScope execution(*ctx->sd); if (!execution.ready) - return false; - auto* result = static_cast(calloc(1, sizeof(sd_video_generation_result_t))); + return execution.operation.acquired + ? sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not refresh model sources") + : sd::api::error(SD_STATUS_BUSY, nullptr, "context is already executing"); + auto status = validate_generation(ctx->sd, params->cache, true); + if (status != SD_STATUS_OK) + return status; + const bool vace = ctx->sd->diffusion_model && + (ctx->sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" || + ctx->sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B"); + if (params->control_frames_size > 0 && !vace) + return sd::api::error(SD_STATUS_UNSUPPORTED, "control_frames", "requires a VACE model"); + if (params->ref_videos_count > 0 && !sd_version_is_minimax_h3(ctx->sd->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "ref_videos", "requires a MiniMax-H3 model"); + if (params->ref_audios_count > 0 && !sd_version_is_minimax_h3(ctx->sd->version) && ctx->sd->version != VERSION_WAN2_2_S2V) + return sd::api::error(SD_STATUS_UNSUPPORTED, "ref_audios", "requires MiniMax-H3 or Wan2.2 S2V"); + if (ctx->sd->version == VERSION_WAN2_2_S2V) { + if (params->end_image.data) + return sd::api::error(SD_STATUS_UNSUPPORTED, "end_image", "Wan2.2 S2V does not support an end image"); + if (params->ref_audios_count > 1 || (params->ref_audios_count && !ctx->sd->audio_encoder)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "ref_audios", "Wan2.2 S2V requires one driving track and an audio encoder"); + } + auto result = std::unique_ptr( + static_cast(calloc(1, sizeof(sd::VideoGenerationResult))), sd::free_video_generation_result); if (!result) - return false; - try { - nlohmann::json info; - if (!sd::pipeline::generate_video(sd_ctx->sd, params, &result->frames, &result->frame_count, - &result->audio, &result->fps, &info)) { - free_sd_video_generation_result(result); - return false; - } - result->generation_info = sd::copy_generation_info(info); - if (!result->generation_info) { - free_sd_video_generation_result(result); - return false; - } - } catch (const std::exception& e) { - LOG_ERROR("video generation failed: %s", e.what()); - free_sd_video_generation_result(result); - return false; - } - *result_out = result; - return true; + return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate video result"); + nlohmann::json info; + if (!sd::pipeline::generate_video(ctx->sd, params, &result->frames, &result->frame_count, + &result->audio, &result->fps, &info)) + return generation_failure(ctx->sd); + result->generation_info = sd::copy_generation_info(info); + if (!result->generation_info) + return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate video metadata"); + *result_out = result.release(); + return SD_STATUS_OK; +} + +sd_status_t sd::generate_image(sd_ctx_t* ctx, const ImageConfig& config, ImageGenerationResult** result_out, const sd_opt_t* source) noexcept { + if (result_out) + *result_out = nullptr; + return sd::api::call([&] { return run_image(ctx, &config, result_out, source); }, SD_STATUS_GENERATION_FAILED); } -SD_API void free_sd_images(sd_image_t* result_images, int num_images) { - if (result_images == nullptr) { - return; - } - - for (int i = 0; i < num_images; ++i) { - if (result_images[i].data != nullptr) { - free(result_images[i].data); - result_images[i].data = nullptr; - } - } - - free(result_images); +sd_status_t sd::generate_video(sd_ctx_t* ctx, const VideoConfig& config, VideoGenerationResult** result_out, const sd_opt_t* source) noexcept { + if (result_out) + *result_out = nullptr; + return sd::api::call([&] { return run_video(ctx, &config, result_out, source); }, SD_STATUS_GENERATION_FAILED); } diff --git a/src/upscaler.cpp b/src/upscaler.cpp index 82aa1f464d..9a16c0adef 100644 --- a/src/upscaler.cpp +++ b/src/upscaler.cpp @@ -1,4 +1,6 @@ #include "upscaler.h" +#include "api/options.h" +#include "api/result.h" #include "core/ggml_extend_backend.h" #include "core/util.h" #include "generation_info.h" @@ -60,13 +62,12 @@ sd::Tensor UpscalerGGML::upscale_tensor(const sd::Tensor& input_te return sd::pipeline::upscale_image_tensor(*model, input_tensor, n_threads, tile_size); } -sd_image_t UpscalerGGML::upscale(sd_image_t input_image, uint32_t upscale_factor) { +sd::Image UpscalerGGML::upscale(sd::Image input_image) { sd::ParallelScope tensor_scope(&tensor_executor); - // upscale_factor, unused for RealESRGAN_x4plus_anime_6B.pth - sd_image_t upscaled_image = {0, 0, 0, nullptr}; - const int scale = get_scale(); - int output_width = (int)input_image.width * scale; - int output_height = (int)input_image.height * scale; + sd::Image upscaled_image = {0, 0, 0, nullptr}; + const int scale = get_scale(); + int output_width = (int)input_image.width * scale; + int output_height = (int)input_image.height * scale; LOG_INFO("upscaling from (%i x %i) to (%i x %i)", input_image.width, input_image.height, output_width, output_height); @@ -77,81 +78,83 @@ sd_image_t UpscalerGGML::upscale(sd_image_t input_image, uint32_t upscale_factor if (upscaled.empty()) { return upscaled_image; } - sd_image_t upscaled_data = tensor_to_sd_image(upscaled); - int64_t t3 = ggml_time_ms(); + sd::Image upscaled_data = tensor_to_sd_image(upscaled); + int64_t t3 = ggml_time_ms(); LOG_INFO("input_image_tensor upscaled, taking %.2fs", (t3 - t0) / 1000.0f); upscaled_image = upscaled_data; return upscaled_image; } -struct upscaler_ctx_t { - UpscalerGGML* upscaler = nullptr; +struct sd_upscaler { + std::unique_ptr upscaler; }; -upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path_c_str, - bool direct, - int n_threads, - int tile_size, - const char* backend, - const char* params_backend) { - upscaler_ctx_t* upscaler_ctx = (upscaler_ctx_t*)malloc(sizeof(upscaler_ctx_t)); - if (upscaler_ctx == nullptr) { - return nullptr; - } - std::string esrgan_path(esrgan_path_c_str); - - upscaler_ctx->upscaler = new UpscalerGGML(n_threads, direct, tile_size, SAFE_STR(backend), SAFE_STR(params_backend)); - if (upscaler_ctx->upscaler == nullptr) { - return nullptr; - } - - if (!upscaler_ctx->upscaler->load_from_file(esrgan_path, n_threads)) { - delete upscaler_ctx->upscaler; - upscaler_ctx->upscaler = nullptr; - free(upscaler_ctx); - return nullptr; - } - return upscaler_ctx; +sd_status_t sd_upscaler_create(const sd_upscaler_opt_t* opt, sd_upscaler_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::Upscaler, config); + if (status != SD_STATUS_OK) + return status; + auto result = std::make_unique(); + const auto& ctx = config.context; + result->upscaler = std::make_unique(ctx.n_threads, config.post.conv_direct, config.post.tile_size, + SAFE_STR(ctx.backend), SAFE_STR(ctx.params_backend)); + if (!result->upscaler->load_from_file(ctx.model_path, ctx.n_threads)) + return sd::api::error(SD_STATUS_LOAD_FAILED, "model_path", "could not load upscaler"); + *out = result.release(); + return SD_STATUS_OK; + }, + SD_STATUS_LOAD_FAILED); } -bool upscale(upscaler_ctx_t* upscaler_ctx, - sd_image_t input_image, - uint32_t upscale_factor, - sd_image_generation_result_t** result_out) try { - if (result_out) - *result_out = nullptr; - if (!result_out) - return false; - if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) { - return false; - } - - sd_image_t* result_images = (sd_image_t*)calloc(1, sizeof(sd_image_t)); - if (result_images == nullptr) { - return false; - } - - auto release_images = [](sd_image_t* images) { free_sd_images(images, 1); }; - std::unique_ptr image_owner(result_images, release_images); - const std::vector infos{sd::upscale_generation_info(upscale_factor)}; - result_images[0] = upscaler_ctx->upscaler->upscale(input_image, upscale_factor); - if (!result_images[0].data) - return false; +void sd_upscaler_free(sd_upscaler_t* upscaler) { + delete upscaler; +} - return sd::make_image_generation_result(image_owner.release(), 1, infos, result_out); -} catch (const std::exception& e) { - LOG_ERROR("upscale failed: %s", e.what()); - return false; +sd_status_t sd_upscale(sd_upscaler_t* upscaler, const sd_upscale_opt_t* opt, sd_result_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::Upscale, config); + if (status != SD_STATUS_OK) + return status; + if (!upscaler || !upscaler->upscaler) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required"); + const auto& image = config.post.image; + const auto scale = upscaler->upscaler->get_scale(); + if (scale <= 0 || image.width > uint32_t(INT32_MAX / scale) || image.height > uint32_t(INT32_MAX / scale)) + return sd::api::error(SD_STATUS_OUT_OF_RANGE, "image", "upscaled dimensions exceed the supported range"); + auto release = [](sd::Image* images) { sd::free_images(images, 1); }; + std::unique_ptr images(static_cast(calloc(1, sizeof(sd::Image))), release); + if (!images) + return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result"); + images.get()[0] = upscaler->upscaler->upscale(image); + if (!images->data) + return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "upscale failed"); + const std::vector infos{sd::upscale_generation_info(scale)}; + return sd::make_image_result(images.release(), 1, infos, out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result metadata"); + }, + SD_STATUS_GENERATION_FAILED); } -int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) { - if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) { - return 1; +int sd_upscaler_get_scale(const sd_upscaler_t* upscaler) { + sd::api::clear_error(); + if (!upscaler || !upscaler->upscaler) { + sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required"); + return 0; } - return upscaler_ctx->upscaler->get_scale(); + return upscaler->upscaler->get_scale(); } -int get_upscaler_model_scale(const char* model_path) { +int sd_upscaler_probe_scale(const char* model_path) { if (model_path == nullptr || model_path[0] == '\0') { return 0; } @@ -178,13 +181,3 @@ int get_upscaler_model_scale(const char* model_path) { return 0; } } - -void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx) { - if (!upscaler_ctx) - return; - if (upscaler_ctx->upscaler != nullptr) { - delete upscaler_ctx->upscaler; - upscaler_ctx->upscaler = nullptr; - } - free(upscaler_ctx); -} diff --git a/src/upscaler.h b/src/upscaler.h index f3d09384e6..e3cf09fd2d 100644 --- a/src/upscaler.h +++ b/src/upscaler.h @@ -1,6 +1,8 @@ #ifndef __SD_UPSCALER_H__ #define __SD_UPSCALER_H__ +#include "core/media.h" + #include "core/ggml_extend_backend.h" #include "core/tensor.hpp" #include "model_manager.h" @@ -35,7 +37,7 @@ struct UpscalerGGML { void set_max_graph_vram_bytes(size_t max_vram_bytes); int get_scale(); sd::Tensor upscale_tensor(const sd::Tensor& input_tensor); - sd_image_t upscale(sd_image_t input_image, uint32_t upscale_factor); + sd::Image upscale(sd::Image input_image); }; #endif // __SD_UPSCALER_H__ From 1d32eb5a4d962b1277a2f01e4841060372cde429 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 15 Nov 2026 17:41:48 +0800 Subject: [PATCH 2/3] fix: preserve parameter precedence and media buffer ownership --- examples/common/common.cpp | 34 +++++++++++++++++++++++++------ examples/server/api.md | 5 +++-- examples/server/model_catalog.cpp | 10 ++++----- examples/server/routes_sdcpp.cpp | 7 +++++-- src/api/result.cpp | 6 ++++-- src/detailer.cpp | 2 ++ 6 files changed, 47 insertions(+), 17 deletions(-) diff --git a/examples/common/common.cpp b/examples/common/common.cpp index fbf977d6a0..1169bcb571 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -1418,7 +1418,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } std::string layers_str = argv[index]; - if (layers_str[0] != '[' || layers_str[layers_str.size() - 1] != ']') { + if (layers_str.size() < 2 || layers_str.front() != '[' || layers_str.back() != ']') { return -1; } @@ -1431,8 +1431,12 @@ ArgOptions SDGenerationParams::get_options() { std::vector layers; for (const auto& token : tokens) { try { - layers.push_back(std::stoi(token)); - } catch (const std::invalid_argument&) { + size_t used = 0; + const int layer = std::stoi(token, &used); + if (used != token.size()) + return -1; + layers.push_back(layer); + } catch (const std::exception&) { return -1; } } @@ -1445,7 +1449,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } std::string layers_str = argv[index]; - if (layers_str[0] != '[' || layers_str[layers_str.size() - 1] != ']') { + if (layers_str.size() < 2 || layers_str.front() != '[' || layers_str.back() != ']') { return -1; } @@ -1458,8 +1462,12 @@ ArgOptions SDGenerationParams::get_options() { std::vector layers; for (const auto& token : tokens) { try { - layers.push_back(std::stoi(token)); - } catch (const std::invalid_argument&) { + size_t used = 0; + const int layer = std::stoi(token, &used); + if (used != token.size()) + return -1; + layers.push_back(layer); + } catch (const std::exception&) { return -1; } } @@ -2296,6 +2304,20 @@ bool SDGenerationParams::from_json_str( nullptr); } + auto merge_shared_sample_input = [&](const char* key, auto member) { + const auto base = j.find("sample_params"); + const auto high = j.find("high_noise_sample_params"); + if (base != j.end() && base->is_object() && base->contains(key)) { + if (base->at(key).is_null()) + (high_noise_sample_params.*member).reset(); + } else if (high != j.end() && high->is_object() && high->contains(key)) { + sample_params.*member = high_noise_sample_params.*member; + } + }; + merge_shared_sample_input("scheduler", &SDSampleParams::scheduler); + merge_shared_sample_input("flow_shift", &SDSampleParams::flow_shift); + merge_shared_sample_input("shifted_timestep", &SDSampleParams::shifted_timestep); + if (j.contains("vae_tiling_params") && j["vae_tiling_params"].is_object()) { const json& tiling_json = j["vae_tiling_params"]; load_json_input(tiling_json, "enabled", vae_tiling_params.enabled); diff --git a/examples/server/api.md b/examples/server/api.md index 24419240a8..17008c613a 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -1712,8 +1712,9 @@ High-noise sampling fields: The two video stages share the scheduler, flow shift and timestep shifting. `high_noise_sample_params.scheduler`, `.flow_shift` and `.shifted_timestep` -are accepted as aliases for those shared settings. An explicitly supplied -corresponding `sample_params` field takes precedence. +are accepted as aliases for those shared settings. Within one request, a +corresponding `sample_params` field takes precedence. Request aliases override +inherited startup and recipe values; `null` clears the inherited shared setting. | Field | Type | | --- | --- | diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp index bc264bca3e..5a538fd529 100644 --- a/examples/server/model_catalog.cpp +++ b/examples/server/model_catalog.cpp @@ -241,14 +241,14 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para if (!params.sample_params.guidance.distilled_guidance) params.sample_params.guidance.distilled_guidance = value; if (std::string(key) == "flow_shift") - if (!params.sample_params.flow_shift) + if (!params.sample_params.flow_shift && !params.high_noise_sample_params.flow_shift) params.sample_params.flow_shift = value; if (std::string(key) == "high_noise_cfg_scale") if (!params.high_noise_sample_params.guidance.txt_cfg) params.high_noise_sample_params.guidance.txt_cfg = value; if (std::string(key) == "high_noise_flow_shift") - if (!params.sample_params.flow_shift) - params.sample_params.flow_shift = value; + if (!params.sample_params.flow_shift && !params.high_noise_sample_params.flow_shift) + params.high_noise_sample_params.flow_shift = value; } if (defaults.contains("custom_sigmas")) { const auto& sigmas = defaults.at("custom_sigmas"); @@ -274,11 +274,11 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para for (const char* key : {"scheduler", "high_noise_scheduler"}) { if (!defaults.contains(key)) continue; - auto& sample = params.sample_params; + auto& sample = std::string(key) == "scheduler" ? params.sample_params : params.high_noise_sample_params; const auto name = defaults.at(key).get(); if (!example_options::contains(example_options::schedulers, name) && name != "normal") throw std::runtime_error(std::string("unknown ") + key); - if (!sample.scheduler) + if (!params.sample_params.scheduler && !params.high_noise_sample_params.scheduler) sample.scheduler = name; } } diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index a0cc075eb1..e103b93132 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -516,8 +516,11 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { return; } const auto tile_size = body.value("tile_size", runtime->default_gen_params->upscale_tile_size); - if (tile_size && *tile_size < 32) - throw std::invalid_argument("tile_size must be at least 32"); + if (tile_size && *tile_size < 32) { + res.status = 400; + res.set_content(json({{"error", "tile_size must be at least 32"}}).dump(), "application/json"); + return; + } const int repeats = std::clamp(body.value("repeats", 1), 1, 4); const std::string wanted = body.value("upscaler", std::string()); diff --git a/src/api/result.cpp b/src/api/result.cpp index ebfbad474c..34db27ca5f 100644 --- a/src/api/result.cpp +++ b/src/api/result.cpp @@ -76,9 +76,10 @@ void sd::free_video_generation_result(sd::VideoGenerationResult* result) { } static sd::api::TensorRef consume_image(sd::Image& image) { + std::vector shape{image.channel, image.width, image.height}; auto* data = image.data; image.data = nullptr; - return sd::api::own_tensor(SD_TENSOR_U8, {image.channel, image.width, image.height}, data); + return sd::api::own_tensor(SD_TENSOR_U8, std::move(shape), data); } bool sd::make_image_result(ImageGenerationResult* images, sd_result_t** result_out) { @@ -123,9 +124,10 @@ bool sd::make_video_result(VideoGenerationResult* video, sd_result_t** result_ou result->generation_infos.emplace_back(video->generation_info ? video->generation_info : ""); if (video->audio) { auto& audio = *video->audio; + std::vector shape{audio.channels, static_cast(audio.sample_count)}; auto* data = audio.data; audio.data = nullptr; - result->audio_tensor = sd::api::own_tensor(SD_TENSOR_F32, {audio.channels, static_cast(audio.sample_count)}, data); + result->audio_tensor = sd::api::own_tensor(SD_TENSOR_F32, std::move(shape), data); result->audio = {audio.sample_rate, result->audio_tensor.get()}; } *result_out = result.release(); diff --git a/src/detailer.cpp b/src/detailer.cpp index 990d0a349c..234afce2fe 100644 --- a/src/detailer.cpp +++ b/src/detailer.cpp @@ -989,6 +989,8 @@ static bool run_adetail(sd_adetailer_t* context, sd_ctx_t* sd_ctx, sd::Image inp generation.pulid_params = {}; generation.hires.enabled = false; generation.image_preprocess = {}; + + generation.sample_params.guidance.img_cfg = INFINITY; if (params.steps > 0) { generation.sample_params.sample_steps = params.steps; generation.sample_params.custom_sigmas = nullptr; From 823dbc3140dd37e4a2f9bae3cd88aadfdb9e58d1 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 15 Nov 2026 18:26:41 +0800 Subject: [PATCH 3/3] fix(server): restrict high-noise alias merging to video requests --- examples/common/common.cpp | 14 -------------- examples/server/api.md | 3 ++- examples/server/routes_sdcpp.cpp | 15 +++++++++++++++ src/api/result.cpp | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 1169bcb571..0c155c1b89 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -2304,20 +2304,6 @@ bool SDGenerationParams::from_json_str( nullptr); } - auto merge_shared_sample_input = [&](const char* key, auto member) { - const auto base = j.find("sample_params"); - const auto high = j.find("high_noise_sample_params"); - if (base != j.end() && base->is_object() && base->contains(key)) { - if (base->at(key).is_null()) - (high_noise_sample_params.*member).reset(); - } else if (high != j.end() && high->is_object() && high->contains(key)) { - sample_params.*member = high_noise_sample_params.*member; - } - }; - merge_shared_sample_input("scheduler", &SDSampleParams::scheduler); - merge_shared_sample_input("flow_shift", &SDSampleParams::flow_shift); - merge_shared_sample_input("shifted_timestep", &SDSampleParams::shifted_timestep); - if (j.contains("vae_tiling_params") && j["vae_tiling_params"].is_object()) { const json& tiling_json = j["vae_tiling_params"]; load_json_input(tiling_json, "enabled", vae_tiling_params.enabled); diff --git a/examples/server/api.md b/examples/server/api.md index 17008c613a..9f48e247f8 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -1712,7 +1712,8 @@ High-noise sampling fields: The two video stages share the scheduler, flow shift and timestep shifting. `high_noise_sample_params.scheduler`, `.flow_shift` and `.shifted_timestep` -are accepted as aliases for those shared settings. Within one request, a +are accepted as aliases for those shared settings in video requests. Image +requests ignore these high-noise fields. Within one video request, a corresponding `sample_params` field takes precedence. Request aliases override inherited startup and recipe values; `null` clears the inherited shared setting. diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index e103b93132..c6e49710b1 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -452,6 +452,21 @@ static bool parse_vid_gen_request(const json& body, return false; } + auto& params = request.gen_params; + auto merge_shared_sample_input = [&](const char* key, auto member) { + const auto base = body.find("sample_params"); + const auto high = body.find("high_noise_sample_params"); + if (base != body.end() && base->is_object() && base->contains(key)) { + if (base->at(key).is_null()) + (params.high_noise_sample_params.*member).reset(); + } else if (high != body.end() && high->is_object() && high->contains(key)) { + params.sample_params.*member = params.high_noise_sample_params.*member; + } + }; + merge_shared_sample_input("scheduler", &SDSampleParams::scheduler); + merge_shared_sample_input("flow_shift", &SDSampleParams::flow_shift); + merge_shared_sample_input("shifted_timestep", &SDSampleParams::shifted_timestep); + std::string output_format = body.value("output_format", "webm"); int output_compression = body.value("output_compression", 100); if (!assign_output_options(request, output_format, output_compression, error_message)) { diff --git a/src/api/result.cpp b/src/api/result.cpp index 34db27ca5f..388f948cee 100644 --- a/src/api/result.cpp +++ b/src/api/result.cpp @@ -123,7 +123,7 @@ bool sd::make_video_result(VideoGenerationResult* video, sd_result_t** result_ou result->images.push_back(consume_image(video->frames[i])); result->generation_infos.emplace_back(video->generation_info ? video->generation_info : ""); if (video->audio) { - auto& audio = *video->audio; + auto& audio = *video->audio; std::vector shape{audio.channels, static_cast(audio.sample_count)}; auto* data = audio.data; audio.data = nullptr;