From edccc49f102b8bd680ff4b2ab076e42d2484990a Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:18:36 -0600 Subject: [PATCH 1/7] echo_tts: add guidance_scale and cfg_mode options not wired in to actually do anything yet, just the required bits to store cfg_mode session-option and guidance_scale request-option and validate usage rebased on upstream/main: moved "text_chunk_size" check to `require_supported_request_options`. --- .../engine/community_models/echo_tts/config.h | 12 ++- .../community_models/echo_tts/session.h | 2 + model_specs/echo_tts.json | 23 ++++- src/community_models/echo_tts/session.cpp | 84 ++++++++++++++++--- 4 files changed, 104 insertions(+), 17 deletions(-) diff --git a/include/engine/community_models/echo_tts/config.h b/include/engine/community_models/echo_tts/config.h index 5f09d583e..596e93261 100644 --- a/include/engine/community_models/echo_tts/config.h +++ b/include/engine/community_models/echo_tts/config.h @@ -161,11 +161,19 @@ struct EchoPcaState { float latent_scale = 1.0F; }; +// Which CFG method to use when sampling (may be expanded in the future) +enum class EchoCfgMode { + Independent, // Reference independent 2-CFG (speaker, text) method + Joint, // Single CFG (speaker+text) method +}; + // Request-level sampler configuration, parsed from spec options. struct EchoSamplerOptions { int num_steps = 40; - float cfg_scale_text = 3.0F; - float cfg_scale_speaker = 8.0F; + EchoCfgMode cfg_mode = EchoCfgMode::Independent; + float cfg_scale_text = 3.0F; // Independent-CFG only + float cfg_scale_speaker = 8.0F; // Independent-CFG only + float cfg_scale = 8.0F; // Joint-CFG only float cfg_min_t = 0.5F; float cfg_max_t = 1.0F; // Evaluate the two unconditional lanes only every Nth step inside the CFG diff --git a/include/engine/community_models/echo_tts/session.h b/include/engine/community_models/echo_tts/session.h index 40a9ffacd..443b032b7 100644 --- a/include/engine/community_models/echo_tts/session.h +++ b/include/engine/community_models/echo_tts/session.h @@ -1,5 +1,6 @@ #pragma once +#include "config.h" #include "engine/community_models/echo_tts/config.h" #include "engine/framework/assets/resource_bundle.h" #include "engine/framework/model_spec/metadata.h" @@ -86,6 +87,7 @@ class EchoTtsSession final std::shared_ptr contract_; std::unique_ptr dit_; std::unique_ptr codec_; + EchoCfgMode cfg_mode_ = EchoCfgMode::Independent; int64_t reference_max_samples_ = 0; bool mem_saver_ = false; std::vector speaker_latent_; diff --git a/model_specs/echo_tts.json b/model_specs/echo_tts.json index b48c6ec34..bae3fcae5 100644 --- a/model_specs/echo_tts.json +++ b/model_specs/echo_tts.json @@ -38,7 +38,7 @@ { "name": "text_guidance_scale", "type": "float", - "description": "Classifier-free guidance scale on the text condition. Advertised range is recommended, not enforced by the parser.", + "description": "Classifier-free guidance scale on the text condition (independent-cfg only). Advertised range is recommended, not enforced by the parser.", "required": false, "min": 0, "default": 3 @@ -46,7 +46,15 @@ { "name": "speaker_guidance_scale", "type": "float", - "description": "Classifier-free guidance scale on the speaker condition. Advertised range is recommended, not enforced by the parser.", + "description": "Classifier-free guidance scale on the speaker condition (independent-cfg only). Advertised range is recommended, not enforced by the parser.", + "required": false, + "min": 0, + "default": 8 + }, + { + "name": "guidance_scale", + "type": "float", + "description": "Classifier-free guidance scale on the text and speaker condition (join-cfg only). Advertised range is recommended, not enforced by the parser.", "required": false, "min": 0, "default": 8 @@ -126,6 +134,17 @@ "required": false, "description": "Release Fish S1-DAC decode graph space before encoding reference speaker audio and encode graph space after. Avoids having both allocations resident simulaneously at the cost of rebuilding the graphs when a speaker cache-miss occurs.", "default": false + }, + { + "name": "cfg_mode", + "type": "enum", + "description": "Type of classifier free guidance used.", + "values": [ + "independent", + "joint" + ], + "required": false, + "default": "independent" } ], "load": [] diff --git a/src/community_models/echo_tts/session.cpp b/src/community_models/echo_tts/session.cpp index b263a7d95..d18660311 100644 --- a/src/community_models/echo_tts/session.cpp +++ b/src/community_models/echo_tts/session.cpp @@ -119,6 +119,26 @@ int64_t quantize_window(int64_t frames, int64_t max_frames) { return std::min(frames, max_frames); } +std::string trim_ascii(std::string text) { + while (!text.empty() && (text.front() == ' ' || text.front() == '\n' || + text.front() == '\r' || text.front() == '\t')) { + text.erase(text.begin()); + } + while (!text.empty() && (text.back() == ' ' || text.back() == '\n' || + text.back() == '\r' || text.back() == '\t')) { + text.pop_back(); + } + return text; +} + +std::string lower_ascii(std::string text) { + std::transform(text.begin(), text.end(), text.begin(), + [](unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + return text; +} + // Predicts how many latents this chunk needs. Deliberately generous: a window // that is too short costs a full-length retry, while one that is slightly too // long only wastes the difference. @@ -183,18 +203,37 @@ runtime::SessionOptions require_supported_session_options( runtime::SessionOptions options, const std::shared_ptr &contract) { // Older standalone GGUF packages embed a contract that predates the - // echo_tts.mem_saver option; keep them usable while still validating + // echo_tts.mem_saver + options; keep them usable while still validating // the value below. auto validation_options = options; - if (contract->session_option_keys.find("echo_tts.mem_saver") == - contract->session_option_keys.end()) { - validation_options.options.erase("echo_tts.mem_saver"); + for (auto key : {"echo_tts.mem_saver", "echo_tts.cfg_mode"}) { + if (contract->session_option_keys.find(key) == + contract->session_option_keys.end()) { + validation_options.options.erase(key); + } } runtime::validate_spec_backed_session_options( validation_options, *contract, kFamily, "Echo-TTS"); return options; } +void require_supported_request_options( + const std::unordered_map & options, + const engine::model_spec::ModelContract & contract) { + // Older standalone GGUF packages embed a contract that predates the + // 'guidance_interval' option; keep them usable while still validating + // the value below. + auto validation_options = options; + for (auto key : {"guidance_scale", "text_chunk_size"}) { + if (contract.request_option_keys.find(key) == + contract.request_option_keys.end()) { + validation_options.erase(key); + } + } + runtime::validate_spec_backed_request_options(validation_options, contract, "Echo-TTS"); +} + + } // namespace EchoTtsSession::EchoTtsSession( @@ -213,6 +252,17 @@ EchoTtsSession::EchoTtsSession( RuntimeSessionBase::options().options, {"echo_tts.mem_saver"})) { mem_saver_ = runtime::parse_bool_option(*value, "echo_tts.mem_saver"); } + if (const auto value = runtime::find_option( + RuntimeSessionBase::options().options, {"echo_tts.cfg_mode"})) { + const auto mode = lower_ascii(trim_ascii(*value)); + if (mode == "independent") { + cfg_mode_ = EchoCfgMode::Independent; + } else if (mode == "joint") { + cfg_mode_ = EchoCfgMode::Joint; + } else { + throw std::runtime_error("echo_tts.cfg_mode must be 'independent' or 'joint'"); + } + } const auto slots = runtime::parse_int_option( RuntimeSessionBase::options().options, {"echo_tts.reference_cache_slots"}); if (slots.has_value()) { @@ -244,10 +294,23 @@ EchoSamplerOptions EchoTtsSession::parse_sampler_options( EchoSamplerOptions sampler; sampler.num_steps = runtime::parse_int_option(options, {"num_inference_steps"}).value_or(sampler.num_steps); - sampler.cfg_scale_text = runtime::parse_float_option(options, {"text_guidance_scale"}) - .value_or(sampler.cfg_scale_text); - sampler.cfg_scale_speaker = runtime::parse_float_option(options, {"speaker_guidance_scale"}) + sampler.cfg_mode = cfg_mode_; // Session-only option (for now), copy from the session + if (cfg_mode_ == EchoCfgMode::Independent) { + sampler.cfg_scale_text = runtime::parse_float_option(options, {"text_guidance_scale"}) + .value_or(sampler.cfg_scale_text); + sampler.cfg_scale_speaker = runtime::parse_float_option(options, {"speaker_guidance_scale"}) .value_or(sampler.cfg_scale_speaker); + if (runtime::find_option(options, {"guidance_scale"})) { + throw std::runtime_error("Echo-TTS guidance_scale is invalid for independent-CFG sampling"); + } + } else { // EchoCfgMode::Joint + sampler.cfg_scale = runtime::parse_float_option(options, {"guidance_scale"}) + .value_or(sampler.cfg_scale); + if (runtime::find_option(options, {"text_guidance_scale"}) || + runtime::find_option(options, {"speaker_guidance_scale"})) { + throw std::runtime_error("Echo-TTS text_guidance_scale and speaker_guidance_scale are invalid for joint-CFG sampling"); + } + } if (const auto truncation = runtime::parse_float_option(options, {"truncation_factor"})) { sampler.truncation_factor = *truncation; } @@ -582,12 +645,7 @@ runtime::AudioBuffer EchoTtsSession::synthesize_chunk( runtime::TaskResult EchoTtsSession::run(const runtime::TaskRequest & request) { require_prepared("Echo-TTS run"); - auto validation_options = request.options; - // Older embedded specs omit this control; the chunk parser validates its value. - if (contract_->request_option_keys.find("text_chunk_size") == contract_->request_option_keys.end()) { - validation_options.erase("text_chunk_size"); - } - runtime::validate_spec_backed_request_options(validation_options, *contract_, "Echo-TTS"); + require_supported_request_options(request.options, *contract_); if (!request.text_input.has_value() || request.text_input->text.empty()) { throw std::runtime_error("Echo-TTS requires text input"); From 39dc21f8f793415ffd192ce0ed26ee47a2bf338f Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:48:56 -0600 Subject: [PATCH 2/7] echo_tts: use cfg_mode during sampling New denoiser graph, masking, and combining during sampling. Not changed yet: kv_cache allocation (kMaxCfgLanes), as that will take more work to route the session option to the right place. --- .../community_models/echo_tts/sampler.h | 11 +++++- src/community_models/echo_tts/dit.cpp | 22 +++++++----- src/community_models/echo_tts/sampler.cpp | 36 ++++++++++++++++--- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/include/engine/community_models/echo_tts/sampler.h b/include/engine/community_models/echo_tts/sampler.h index 1cbd75522..ae57b5bec 100644 --- a/include/engine/community_models/echo_tts/sampler.h +++ b/include/engine/community_models/echo_tts/sampler.h @@ -31,12 +31,21 @@ bool cfg_active(float t, float cfg_min_t, float cfg_max_t); // v = v_cond // + w_text * (v_cond - v_uncond_text) // + w_speaker * (v_cond - v_uncond_speaker) -std::vector combine_cfg_lanes( +std::vector combine_cfg_lanes_independent( const std::vector & lanes, int64_t lane_elements, float cfg_scale_text, float cfg_scale_speaker); +// Combines the two CFG lanes into a single velocity, following the +// joint-guidance form: +// v = v_cond +// + w_joint * (v_cond - v_undcond) +std::vector combine_cfg_lanes_joint( + const std::vector & lanes, + int64_t lane_elements, + float cfg_scale); + // Runs the sampler loop. `denoise` supplies the model evaluation and // `initial_noise` the starting latent, both injected so this can be tested // without a backend. `on_kv_rescale`, when set, is invoked at the timestep where diff --git a/src/community_models/echo_tts/dit.cpp b/src/community_models/echo_tts/dit.cpp index 891cdc85f..14908379b 100644 --- a/src/community_models/echo_tts/dit.cpp +++ b/src/community_models/echo_tts/dit.cpp @@ -281,15 +281,15 @@ class EchoDiTRuntime::Impl { if (!conditioning_ready_) { throw std::runtime_error("Echo-TTS denoise() called before prepare_conditioning()"); } - if (lanes != 1 && lanes != 3) { - throw std::runtime_error("Echo-TTS denoiser supports 1 or 3 CFG lanes"); + if (lanes != 1 && lanes != 2 && lanes != 3) { + throw std::runtime_error("Echo-TTS denoiser supports 1, 2, or 3 CFG lanes"); } const int64_t elements = sequence_length_ * config_.latent_size; if (static_cast(x.size()) != elements) { throw std::runtime_error("Echo-TTS denoiser received a mis-shaped latent"); } - auto & graph = lanes == 1 ? single_ : triple_; + auto & graph = lanes == 1 ? single_ : lanes == 2 ? joint_ : independent_; if (graph.graph == nullptr) { build_denoiser_graph(graph, lanes); } @@ -453,15 +453,17 @@ class EchoDiTRuntime::Impl { } // Denoiser mask, laid out per lane as [self | text | speaker]. Lane 0 is - // fully conditional, lane 1 drops text, lane 2 drops speaker, reproducing - // upstream's concatenated cond/uncond masks. + // fully conditional, + // joint-cfg: lane 1 drops text and speaker + // independent-cfg: lane 1 drops text, lane 2 drops speaker, reproducing + // upstream's concatenated cond/uncond masks. std::vector make_denoiser_mask(int lanes) const { const int64_t keys = sequence_length_ + text_length_ + speaker_tokens_; std::vector mask( static_cast(static_cast(lanes) * sequence_length_ * keys), 0.0F); for (int lane = 0; lane < lanes; ++lane) { - const bool text_on = lane != 1; - const bool speaker_on = lane != 2; + const bool text_on = lane != 1; // Joint: lane 0; Independent: lanes 0, 2 + const bool speaker_on = lane != (lanes - 1); // Joint: lane 0; Independent: lanes 0, 1 for (int64_t q = 0; q < sequence_length_; ++q) { float * row = mask.data() + (static_cast(lane) * sequence_length_ + q) * keys; @@ -779,7 +781,8 @@ class EchoDiTRuntime::Impl { void release_denoiser_graphs() { release_denoiser_graph(single_); - release_denoiser_graph(triple_); + release_denoiser_graph(joint_); + release_denoiser_graph(independent_); } void release_kv_cache() { @@ -831,7 +834,8 @@ class EchoDiTRuntime::Impl { core::TensorValue speaker_positions_; DenoiserGraph single_; - DenoiserGraph triple_; + DenoiserGraph joint_; + DenoiserGraph independent_; }; EchoDiTRuntime::EchoDiTRuntime( diff --git a/src/community_models/echo_tts/sampler.cpp b/src/community_models/echo_tts/sampler.cpp index 61e5749e7..b9abb80de 100644 --- a/src/community_models/echo_tts/sampler.cpp +++ b/src/community_models/echo_tts/sampler.cpp @@ -30,7 +30,7 @@ bool cfg_active(float t, float cfg_min_t, float cfg_max_t) { return t >= cfg_min_t && t <= cfg_max_t; } -std::vector combine_cfg_lanes( +std::vector combine_cfg_lanes_independent( const std::vector & lanes, int64_t lane_elements, float cfg_scale_text, @@ -53,6 +53,26 @@ std::vector combine_cfg_lanes( return out; } +std::vector combine_cfg_lanes_joint( + const std::vector & lanes, + int64_t lane_elements, + float cfg_scale) { + if (lane_elements <= 0 || + static_cast(lanes.size()) != lane_elements * 2) { + throw std::runtime_error("Echo-TTS Joint-CFG combine expects exactly 2 lanes"); + } + const float * cond = lanes.data(); + const float * uncond = lanes.data() + lane_elements; + + std::vector out(static_cast(lane_elements)); + for (int64_t i = 0; i < lane_elements; ++i) { + const float c = cond[i]; + out[static_cast(i)] = + c + cfg_scale * (c - uncond[i]); + } + return out; +} + std::vector run_euler_sampler( const EchoSamplerOptions & options, int64_t sequence_length, @@ -84,6 +104,7 @@ std::vector run_euler_sampler( // is the whole additive term, w_text * (v_cond - v_text) + w_speaker * // (v_cond - v_speaker), held in absolute units rather than as a ratio so a // reused correction cannot amplify a small v_cond. + const int n_cfg_lanes = options.cfg_mode == EchoCfgMode::Joint ? 2 : 3; std::vector cfg_delta; int steps_since_refresh = 0; @@ -98,12 +119,17 @@ std::vector run_euler_sampler( // applied before one has been measured. const bool refresh = cfg_delta.empty() || steps_since_refresh >= cfg_interval - 1; if (refresh) { - auto lanes = denoise(x_t, t, 3); - if (static_cast(lanes.size()) != elements * 3) { + auto lanes = denoise(x_t, t, n_cfg_lanes); + if (static_cast(lanes.size()) != elements * n_cfg_lanes) { throw std::runtime_error("Echo-TTS denoiser returned mis-shaped CFG lanes"); } - v_pred = combine_cfg_lanes( - lanes, elements, options.cfg_scale_text, options.cfg_scale_speaker); + if (n_cfg_lanes == 2) { + v_pred = combine_cfg_lanes_joint( + lanes, elements, options.cfg_scale); + } else { + v_pred = combine_cfg_lanes_independent( + lanes, elements, options.cfg_scale_text, options.cfg_scale_speaker); + } if (cfg_interval > 1) { cfg_delta.resize(static_cast(elements)); for (int64_t i = 0; i < elements; ++i) { From 4d5b10f0b43d310aeacc32e68f475119f15ee489 Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:54:14 -0600 Subject: [PATCH 3/7] echo_tts: adjust kv cache max lanes allocation depending on cfg_mode --- include/engine/community_models/echo_tts/dit.h | 2 +- src/community_models/echo_tts/dit.cpp | 12 ++++++------ src/community_models/echo_tts/session.cpp | 15 ++++++++++++++- tests/echo_tts/echo_tts_dit_parity.cpp | 2 +- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/include/engine/community_models/echo_tts/dit.h b/include/engine/community_models/echo_tts/dit.h index 35a976599..a41918121 100644 --- a/include/engine/community_models/echo_tts/dit.h +++ b/include/engine/community_models/echo_tts/dit.h @@ -47,7 +47,7 @@ class EchoDiTRuntime { // Runs the text and speaker encoders and populates the cached key/value // projections. Must be called before sample(). - void prepare_conditioning(const EchoConditioning & conditioning); + void prepare_conditioning(const EchoConditioning & conditioning, int max_kv_lanes); // Runs the dual-CFG Euler sampler and returns the final latent, shaped // (sequence_length, latent_size) row-major. diff --git a/src/community_models/echo_tts/dit.cpp b/src/community_models/echo_tts/dit.cpp index 14908379b..fc6a672c6 100644 --- a/src/community_models/echo_tts/dit.cpp +++ b/src/community_models/echo_tts/dit.cpp @@ -214,7 +214,7 @@ class EchoDiTRuntime::Impl { const EchoTtsConfig & config() const noexcept { return config_; } bool conditioning_ready() const noexcept { return conditioning_ready_; } - void prepare_conditioning(const EchoConditioning & conditioning) { + void prepare_conditioning(const EchoConditioning & conditioning, int max_kv_lanes) { validate_conditioning(conditioning); // Any change in conditioning length invalidates every cached graph, @@ -235,7 +235,7 @@ class EchoDiTRuntime::Impl { conditioning.speaker_mask[static_cast(i * config_.speaker_patch_size)]; } - allocate_kv_cache(); + allocate_kv_cache(max_kv_lanes); build_conditioning_graph(); core::write_tensor_i32(text_ids_, conditioning.text_input_ids); @@ -492,7 +492,7 @@ class EchoDiTRuntime::Impl { // The cache lives in its own context and backend buffer so that both the // conditioning graph (which writes it) and the denoiser graphs (which read // it) can reference the same tensors as leaves. - void allocate_kv_cache() { + void allocate_kv_cache(int max_kv_lanes) { const int64_t heads = config_.num_heads; const int64_t head_dim = config_.head_dim(); const size_t tensor_count = static_cast(config_.num_layers) * 4; @@ -508,7 +508,7 @@ class EchoDiTRuntime::Impl { // batch and short-circuits to a passthrough. The single-lane graph // reads lane 0, which is a contiguous prefix because the lane axis is // outermost in ggml's layout. - kv_lanes_ = echo_kv_expand_disabled() ? 1 : kMaxCfgLanes; + kv_lanes_ = echo_kv_expand_disabled() ? 1 : max_kv_lanes; auto make = [&](int64_t tokens) { return core::make_tensor( ctx, GGML_TYPE_F32, @@ -850,8 +850,8 @@ EchoDiTRuntime::~EchoDiTRuntime() = default; const EchoTtsConfig & EchoDiTRuntime::config() const noexcept { return impl_->config(); } -void EchoDiTRuntime::prepare_conditioning(const EchoConditioning & conditioning) { - impl_->prepare_conditioning(conditioning); +void EchoDiTRuntime::prepare_conditioning(const EchoConditioning & conditioning, int max_kv_lanes) { + impl_->prepare_conditioning(conditioning, max_kv_lanes); } std::vector EchoDiTRuntime::denoise_once(const std::vector & x, float t, int lanes) { diff --git a/src/community_models/echo_tts/session.cpp b/src/community_models/echo_tts/session.cpp index d18660311..68a816ac4 100644 --- a/src/community_models/echo_tts/session.cpp +++ b/src/community_models/echo_tts/session.cpp @@ -119,6 +119,19 @@ int64_t quantize_window(int64_t frames, int64_t max_frames) { return std::min(frames, max_frames); } +// For KV-cache allocation mostly, determine number of kv-lanes to allocate +// (may belong in a different source file?) +int cfg_mode_to_kv_lanes(EchoCfgMode cfg_mode) { + switch (cfg_mode) { + case EchoCfgMode::Joint: + return 2; + case EchoCfgMode::Independent: + return 3; + default: + throw std::runtime_error("Echo-TTS unrecognized CFG mode."); + } +} + std::string trim_ascii(std::string text) { while (!text.empty() && (text.front() == ' ' || text.front() == '\n' || text.front() == '\r' || text.front() == '\t')) { @@ -547,7 +560,7 @@ runtime::AudioBuffer EchoTtsSession::synthesize_chunk( conditioning.speaker_mask.assign(static_cast(speaker_frames_), 1.0F); conditioning.speaker_frames = speaker_frames_; - dit_->prepare_conditioning(conditioning); + dit_->prepare_conditioning(conditioning, cfg_mode_to_kv_lanes(cfg_mode_)); // Adaptive window: OFF by default, enable with AUDIOCPP_ECHO_TTS_ADAPTIVE_WINDOW=1. // diff --git a/tests/echo_tts/echo_tts_dit_parity.cpp b/tests/echo_tts/echo_tts_dit_parity.cpp index dbaa72357..452ef54a2 100644 --- a/tests/echo_tts/echo_tts_dit_parity.cpp +++ b/tests/echo_tts/echo_tts_dit_parity.cpp @@ -318,7 +318,7 @@ int main(int argc, char ** argv) try { std::cout << "text_length=" << conditioning.text_length << " speaker_frames=" << conditioning.speaker_frames << "\n"; - dit.prepare_conditioning(conditioning); + dit.prepare_conditioning(conditioning, 3); // 3 kv-lanes = independent-cfg bool ok = true; From bbaa9b658d7fdc74c2bdc396eb420a62918fee59 Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:14:49 -0600 Subject: [PATCH 4/7] echo_tts: remove kMaxCfgLanes as it's no longer needed --- src/community_models/echo_tts/dit.cpp | 9 +++++---- src/community_models/echo_tts/dit_blocks.inc | 4 ---- src/community_models/echo_tts/session.cpp | 4 ++++ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/community_models/echo_tts/dit.cpp b/src/community_models/echo_tts/dit.cpp index fc6a672c6..919f51015 100644 --- a/src/community_models/echo_tts/dit.cpp +++ b/src/community_models/echo_tts/dit.cpp @@ -503,8 +503,9 @@ class EchoDiTRuntime::Impl { throw std::runtime_error("Echo-TTS KV cache context initialization failed"); } core::ModuleBuildContext ctx{kv_ctx_.get(), "echo_tts.kv_cache", backend_type_}; - // The cache is allocated at the widest lane count any graph will ask - // for, so joint_attention's expand() finds dims[0] already equal to the + // The cache is allocated at the widest lane count any graph will ask for in the + // session-configured CFG mode, + // so joint_attention's expand() finds dims[0] already equal to the // batch and short-circuits to a passthrough. The single-lane graph // reads lane 0, which is a contiguous prefix because the lane axis is // outermost in ggml's layout. @@ -653,12 +654,12 @@ class EchoDiTRuntime::Impl { // Reconciles the cache's lane count with the graph's. // - // With pre-expansion on, the cache is allocated at kMaxCfgLanes and the + // With pre-expansion on, the cache is allocated at 2 or 3 lanes and the // single-lane graph reads a prefix of it; the lane axis is outermost, so // that is a view with no copy. // // With pre-expansion off (AUDIOCPP_ECHO_TTS_NO_KV_EXPAND=1) the cache is - // batch-1 and the three-lane graph is *wider* than it. Narrowing is not + // batch-1 and the two- or three-lane graphs are *wider* than it. Narrowing is not // possible and not wanted: returning the cache unchanged leaves // joint_attention's expand() to broadcast it per step, which is exactly the // pre-patch behaviour the flag exists to restore. diff --git a/src/community_models/echo_tts/dit_blocks.inc b/src/community_models/echo_tts/dit_blocks.inc index a4c2350ca..a4cc17c3d 100644 --- a/src/community_models/echo_tts/dit_blocks.inc +++ b/src/community_models/echo_tts/dit_blocks.inc @@ -34,10 +34,6 @@ constexpr size_t kWeightContextBytes = 6144ull * 1024ull * 1024ull; constexpr size_t kGraphArenaBytes = 512ull * 1024ull * 1024ull; constexpr float kRopeTheta = 10000.0F; -// cond, text-uncond, speaker-uncond. The KV cache is sized to this so the -// three-lane graph never has to broadcast it at sampling time. -constexpr int64_t kMaxCfgLanes = 3; - // Additive mask value for disallowed keys. -INFINITY would be exact but // produces NaN when an entire row is masked, which happens for the // speaker-uncond CFG lane; a large finite penalty is the standard ggml diff --git a/src/community_models/echo_tts/session.cpp b/src/community_models/echo_tts/session.cpp index 68a816ac4..d9eef91d3 100644 --- a/src/community_models/echo_tts/session.cpp +++ b/src/community_models/echo_tts/session.cpp @@ -560,6 +560,10 @@ runtime::AudioBuffer EchoTtsSession::synthesize_chunk( conditioning.speaker_mask.assign(static_cast(speaker_frames_), 1.0F); conditioning.speaker_frames = speaker_frames_; + // Note: with cfg_mode_ as a Session-option, it's safe to set the KV cache + // allocation based on the current cfg_mode_. If we ever allow cfg_mode_ as a + // request-option, this logic will need to be adjusted to the maximum needed + // by any allowed cfg mode. dit_->prepare_conditioning(conditioning, cfg_mode_to_kv_lanes(cfg_mode_)); // Adaptive window: OFF by default, enable with AUDIOCPP_ECHO_TTS_ADAPTIVE_WINDOW=1. From f80bf8f95de7ceb4e2af888f0f17f8ab2ce9658c Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:59:15 -0600 Subject: [PATCH 5/7] Use the new InvalidRequestError where appropriate. --- src/community_models/echo_tts/session.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/community_models/echo_tts/session.cpp b/src/community_models/echo_tts/session.cpp index d9eef91d3..5bfac2c4b 100644 --- a/src/community_models/echo_tts/session.cpp +++ b/src/community_models/echo_tts/session.cpp @@ -7,6 +7,7 @@ #include "engine/framework/audio/conversion.h" #include "engine/framework/audio/resampling.h" #include "engine/framework/audio/waveform_ops.h" +#include "engine/framework/runtime/errors.h" #include "engine/framework/runtime/options.h" #include "engine/framework/runtime/spec_backed_model.h" #include "engine/framework/text/chunking.h" @@ -314,14 +315,14 @@ EchoSamplerOptions EchoTtsSession::parse_sampler_options( sampler.cfg_scale_speaker = runtime::parse_float_option(options, {"speaker_guidance_scale"}) .value_or(sampler.cfg_scale_speaker); if (runtime::find_option(options, {"guidance_scale"})) { - throw std::runtime_error("Echo-TTS guidance_scale is invalid for independent-CFG sampling"); + throw runtime::InvalidRequestError("Echo-TTS guidance_scale is invalid for independent-CFG sampling"); } } else { // EchoCfgMode::Joint sampler.cfg_scale = runtime::parse_float_option(options, {"guidance_scale"}) .value_or(sampler.cfg_scale); if (runtime::find_option(options, {"text_guidance_scale"}) || runtime::find_option(options, {"speaker_guidance_scale"})) { - throw std::runtime_error("Echo-TTS text_guidance_scale and speaker_guidance_scale are invalid for joint-CFG sampling"); + throw runtime::InvalidRequestError("Echo-TTS text_guidance_scale and speaker_guidance_scale are invalid for joint-CFG sampling"); } } if (const auto truncation = runtime::parse_float_option(options, {"truncation_factor"})) { @@ -329,7 +330,7 @@ EchoSamplerOptions EchoTtsSession::parse_sampler_options( } if (const auto kv_scale = runtime::parse_float_option(options, {"speaker_kv_scale"})) { if (*kv_scale == 0.0F) { - throw std::runtime_error("speaker_kv_scale must be nonzero"); + throw runtime::InvalidRequestError("speaker_kv_scale must be nonzero"); } // 1.0 is the documented "disabled" value, not a scale to apply. if (*kv_scale != 1.0F) { @@ -346,7 +347,7 @@ EchoSamplerOptions EchoTtsSession::parse_sampler_options( sampler.sequence_length = assets_->config.max_sequence_length; if (const auto window_sec = runtime::parse_float_option(options, {"max_duration_sec"})) { if (!(*window_sec > 0.0F)) { - throw std::runtime_error("Echo-TTS max_duration_sec must be positive"); + throw runtime::InvalidRequestError("Echo-TTS max_duration_sec must be positive"); } // The window is quantised to whole latent frames. Round *down* so the // option keeps its name: the result is never longer than what was asked @@ -362,12 +363,12 @@ EchoSamplerOptions EchoTtsSession::parse_sampler_options( } if (const auto interval = runtime::parse_int_option(options, {"guidance_interval"})) { if (*interval < 1) { - throw std::runtime_error("Echo-TTS guidance_interval must be at least 1"); + throw runtime::InvalidRequestError("Echo-TTS guidance_interval must be at least 1"); } sampler.cfg_interval = *interval; } if (sampler.num_steps <= 0) { - throw std::runtime_error("Echo-TTS num_inference_steps must be positive"); + throw runtime::InvalidRequestError("Echo-TTS num_inference_steps must be positive"); } return sampler; } @@ -431,7 +432,7 @@ int64_t EchoTtsSession::resolve_reference_max_samples( return kDefaultReferenceMaxSamples; } if (!(*seconds > 0.0F)) { - throw std::runtime_error("Echo-TTS reference_duration_sec must be positive"); + throw runtime::InvalidRequestError("Echo-TTS reference_duration_sec must be positive"); } const auto requested = static_cast( static_cast(*seconds) * static_cast(kSampleRate)); @@ -665,7 +666,7 @@ runtime::TaskResult EchoTtsSession::run(const runtime::TaskRequest & request) { require_supported_request_options(request.options, *contract_); if (!request.text_input.has_value() || request.text_input->text.empty()) { - throw std::runtime_error("Echo-TTS requires text input"); + throw runtime::InvalidRequestError("Echo-TTS requires text input"); } if (!request.voice.has_value() || !request.voice->speaker.has_value() || !request.voice->speaker->audio.has_value()) { From 6ee9951366e12b60ad2618c87e88241f80a60f2d Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:11:06 -0600 Subject: [PATCH 6/7] echo-tts: documentation on mem_saver and cfg_mode --- docs/community_models/echo_tts_performance.md | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/community_models/echo_tts_performance.md b/docs/community_models/echo_tts_performance.md index 6d9aab1d8..25c96450f 100644 --- a/docs/community_models/echo_tts_performance.md +++ b/docs/community_models/echo_tts_performance.md @@ -118,6 +118,45 @@ Verified against a mock denoiser at 0.011% max deviation for interval 2 and 0.020% for interval 3 -- but that mock had deliberately smooth uncond offsets, so treat those as a lower bound, not a measurement on real audio. +### 5. `echo_tts.mem_saver` (opt-in, default false) +Mem-saver frees VRAM by free-ing the audio encoder allocations after reference +speaker encoding. For cached reference speakers, this has no performance hit, +for an uncached reference speaker this has a small performance hit. Peak VRAM +often occurs during speaker encoding, but depends on the length of prompts you +send (long prompts make the audio decode buffers larger than the encode buffers). +Actual savings will depend on model quantization and backend architecture. For +Q8_0 model weights, the following were measured on a GTX 1080 Ti and a RTX 4070: +- GTX 1080 Ti (false/true): 8122 MB / 7416 MB (8.7% savings) +- RTX 4070 (false/true): 6084 MB / 5356 MB (12.0% savings) + +### 6. `echo_tts.cfg_mode` (opt-in, default independent) +Echo-TTS allows two classifier free guidance modes: the reference Independent +mode with a guidance scale for the reference speaker and a guidance scale for the +text, and a Joint mode with a single combined guidance scale. Joint-CFG allows +for a slightly smaller VRAM allocation, and fewer forwards passes. While actual +results will depend on the backend architecture, the following numbers are +provided as examples measured on a GTX 1080 Ti and an RTX 4080: +- GTX 1080 Ti (independent/joint): 16.9s / 11.9s (0.7x, or ~30% faster) +- RTX 4070 (independent/joint): 3.5s / 2.7s (0.77x, or ~23% faster) + +The setting also has an effect on audio quality. Measured by @5uck1ess on an RTX +5090, 4 English prompts x 3 seeds run through Whisper-v3 WER and WavLM SIM: + +| Mode & Scale | WER | UTMOS | SIM | +|---|---|---|---| +| independent, 3.0/8.0 | 0.069 | 4.20 | 0.831 | +| joint, 4.0 | 0.081 | 4.32 | 0.804 | +| joint, 6.0 | 0.060 | 4.31 | 0.821 | +| joint, 8.0 | 0.076 | 4.29 | 0.829 | +| joint, 10.0 | 0.069 | 4.26 | 0.830 | + +Joint-CFG at scale 8.0 is at parity, below ~6.0 scale similarity drops. By ear: +near parity, independent is slightly better on a few, clone is almost identical. +@lojack5 also notes: Joint-CFG the most noticable (by ear) difference is at the +first ~0.2 seconds of the generated audio, where the speaker doesn't as closely +match the reference as Independent-CFG does. After the beginning of the +generation, the speaker quickly matches the reference audio. + --- ## Suggested config From 72c54b561a3523789e630961fd154a1c06bbc34f Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:30:33 -0600 Subject: [PATCH 7/7] echo-tts: fix masking logic for single-lane (CFG inactive) sampling --- src/community_models/echo_tts/dit.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/community_models/echo_tts/dit.cpp b/src/community_models/echo_tts/dit.cpp index 919f51015..670ed2228 100644 --- a/src/community_models/echo_tts/dit.cpp +++ b/src/community_models/echo_tts/dit.cpp @@ -462,8 +462,9 @@ class EchoDiTRuntime::Impl { std::vector mask( static_cast(static_cast(lanes) * sequence_length_ * keys), 0.0F); for (int lane = 0; lane < lanes; ++lane) { - const bool text_on = lane != 1; // Joint: lane 0; Independent: lanes 0, 2 - const bool speaker_on = lane != (lanes - 1); // Joint: lane 0; Independent: lanes 0, 1 + // Note: logic needs to be correct for both lanes == 1 and lanes == 2/3 + const bool text_on = lane != 1; + const bool speaker_on = lane == 0 || lane != (lanes - 1); for (int64_t q = 0; q < sequence_length_; ++q) { float * row = mask.data() + (static_cast(lane) * sequence_length_ + q) * keys;