From 08692294cdbc07fb2a59a9fa9bde22e5cde242e6 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:39:21 +0000 Subject: [PATCH] audio: ffmpeg_dec: add libavfilter graph support (afftdn, alimiter) Add on-DSP libavfilter audio filter graph support (ffmpeg-filter.c) to the ffmpeg_dec module: - Build an abuffer(src) -> -> abuffersink(sink) graph with deferred DP-thread initialization so heavy filter setup (such as afftdn FFT table generation) runs off the IPC/IDC worker stack. - Support FFT-based noise reduction (CONFIG_FFMPEG_FILTER_AFFTDN, afftdn) and lookahead brickwall peak limiting (CONFIG_FFMPEG_FILTER_ALIMITER, alimiter), selectable via Kconfig default, IPC4 snd_codec profile, or runtime module configuration. - Extend ffmpeg_dec-convert.c with shared PCM-to-AVFrame and AVFrame-to-PCM conversions using Q-format macros, and split HiFi3/HiFi4 VFPU SIMD conversion intrinsics into ffmpeg-convert-hifi.c. - Provide C90 strtod/strtof in the libc shims so libavfilter option parsing works in both built-in (=y) and LLEXT (=m) builds. Signed-off-by: Liam Girdwood --- src/audio/ffmpeg_dec/CMakeLists.txt | 14 +- src/audio/ffmpeg_dec/Kconfig | 58 +- src/audio/ffmpeg_dec/ffmpeg-convert-hifi.c | 158 +++ src/audio/ffmpeg_dec/ffmpeg-filter.c | 945 ++++++++++++++++++ src/audio/ffmpeg_dec/ffmpeg.cmake | 34 +- .../ffmpeg_dec/ffmpeg_dec-builtin-libc.c | 87 +- src/audio/ffmpeg_dec/ffmpeg_dec-convert.c | 366 ++++--- src/audio/ffmpeg_dec/ffmpeg_dec-convert.h | 25 +- src/audio/ffmpeg_dec/ffmpeg_dec-shims.c | 227 ++++- src/audio/ffmpeg_dec/ffmpeg_dec.c | 45 + src/audio/ffmpeg_dec/ffmpeg_dec.h | 20 + src/audio/ffmpeg_dec/llext/CMakeLists.txt | 22 +- src/kernel/audio/base_fw.c | 2 +- 13 files changed, 1830 insertions(+), 173 deletions(-) create mode 100644 src/audio/ffmpeg_dec/ffmpeg-convert-hifi.c create mode 100644 src/audio/ffmpeg_dec/ffmpeg-filter.c diff --git a/src/audio/ffmpeg_dec/CMakeLists.txt b/src/audio/ffmpeg_dec/CMakeLists.txt index 5952bc266d71..c7863b8b0c61 100644 --- a/src/audio/ffmpeg_dec/CMakeLists.txt +++ b/src/audio/ffmpeg_dec/CMakeLists.txt @@ -24,6 +24,7 @@ else() else() add_local_sources(sof ffmpeg_dec-ffmpeg.c) add_local_sources(sof ffmpeg_dec-convert.c) + add_local_sources(sof ffmpeg-convert-hifi.c) if(CONFIG_FFMPEG_ENC) add_local_sources(sof ffmpeg-encode.c) endif() @@ -33,6 +34,9 @@ else() if(CONFIG_FFMPEG_ENC_AAC) add_local_sources(sof ffmpeg-enc-aac.c) endif() + if(CONFIG_FFMPEG_FILTER) + add_local_sources(sof ffmpeg-filter.c) + endif() # Scalar soft-float for the SOF glue that touches float. core-isa.h reports # XCHAL_HAVE_FP=1 so LLVM would lower the FLT/FLTP -> Q1.31 sample conversion # in ffmpeg_dec_emit_frame() to hardware FP0 ops (lsi/mul.s/trunc.s), which @@ -47,7 +51,9 @@ else() set(_ffmpeg_dec_sources ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg_dec.c ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg_dec-ffmpeg.c - ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg_dec-convert.c) + ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg_dec-convert.c + ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg-convert-hifi.c + ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg_dec-builtin-libc.c) if(CONFIG_FFMPEG_ENC) list(APPEND _ffmpeg_dec_sources ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg-encode.c) endif() @@ -57,6 +63,9 @@ else() if(CONFIG_FFMPEG_ENC_AAC) list(APPEND _ffmpeg_dec_sources ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg-enc-aac.c) endif() + if(CONFIG_FFMPEG_FILTER) + list(APPEND _ffmpeg_dec_sources ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg-filter.c) + endif() if(CMAKE_C_COMPILER_ID MATCHES "Clang") set_source_files_properties( ${_ffmpeg_dec_sources} @@ -103,6 +112,9 @@ else() # stubs in ffmpeg_dec-builtin-libc.c). FLAC is integer-only so it never # calls libm at runtime, but the references still resolve to real kernels # now, and lossy decoders (AAC/Opus/Vorbis) need them to decode correctly. + if(CONFIG_FFMPEG_BUILD_AVFILTER) + target_link_libraries(SOF INTERFACE avfilter) + endif() target_link_libraries(SOF INTERFACE avcodec swresample avutil) # Encode mode: libshine MP3 encoder archive (built by ffmpeg.cmake # section 3b when FFMPEG_ENC_MP3 is set). Linked AFTER avcodec, diff --git a/src/audio/ffmpeg_dec/Kconfig b/src/audio/ffmpeg_dec/Kconfig index 55bab4de5573..23d56c197f7a 100644 --- a/src/audio/ffmpeg_dec/Kconfig +++ b/src/audio/ffmpeg_dec/Kconfig @@ -77,11 +77,65 @@ config FFMPEG_ENC bool default y if FFMPEG_ENC_MP3 || FFMPEG_ENC_AAC -# Selected automatically when a decoder that needs IEEE float math is +comment "FFmpeg audio filters to build (libavfilter)" + depends on IPC_MAJOR_4 + +config FFMPEG_FILTER_AFFTDN + bool "FFT noise reduction (afftdn)" + depends on IPC_MAJOR_4 + help + FFT-based audio denoiser (libavfilter af_afftdn). Builds libavfilter + with abuffer, abuffersink, aformat and afftdn, plus the avfilter-graph + runtime path (ffmpeg-filter.c). + +config FFMPEG_FILTER_ALIMITER + bool "Lookahead peak limiter (alimiter)" + depends on IPC_MAJOR_4 + help + Lookahead brickwall peak limiter (libavfilter af_alimiter). Prevents + digital clipping and provides speaker protection. Builds libavfilter + with abuffer, abuffersink, aformat and alimiter, plus the avfilter-graph + runtime path (ffmpeg-filter.c). + +# Turns on libavfilter in the FFmpeg cross-build when any filter is selected. +config FFMPEG_BUILD_AVFILTER + bool + default y if FFMPEG_FILTER_AFFTDN || FFMPEG_FILTER_ALIMITER + +# Gates compilation of the runtime avfilter-graph path (ffmpeg-filter.c). +config FFMPEG_FILTER + bool + default y if FFMPEG_BUILD_AVFILTER + +choice + prompt "Default FFmpeg audio filter" + depends on FFMPEG_FILTER + default FFMPEG_FILTER_DEFAULT_AFFTDN if FFMPEG_FILTER_AFFTDN + default FFMPEG_FILTER_DEFAULT_ALIMITER if FFMPEG_FILTER_ALIMITER + +config FFMPEG_FILTER_DEFAULT_AFFTDN + bool "FFT noise reduction (afftdn)" + depends on FFMPEG_FILTER_AFFTDN + help + Select afftdn (FFT noise reduction) as the default libavfilter audio + filter when no explicit filter profile or runtime configuration is + supplied. + +config FFMPEG_FILTER_DEFAULT_ALIMITER + bool "Lookahead peak limiter (alimiter)" + depends on FFMPEG_FILTER_ALIMITER + help + Select alimiter (lookahead brickwall peak limiter) as the default + libavfilter audio filter when no explicit filter profile or runtime + configuration is supplied. + +endchoice + +# Selected automatically when a decoder or filter that needs IEEE float math is # built. Gates compilation of the fast single-precision libm (fastmathf.c). config FFMPEG_DEC_FLOAT_MATH bool - default y if FFMPEG_DEC_AAC || FFMPEG_DEC_OPUS || FFMPEG_DEC_MP3 + default y if FFMPEG_DEC_AAC || FFMPEG_DEC_OPUS || FFMPEG_DEC_MP3 || FFMPEG_BUILD_AVFILTER config FFMPEG_DEC_COLD_SPLIT bool "Place cold FFmpeg code (av_cold init/setup) in DRAM [EXPERIMENTAL]" diff --git a/src/audio/ffmpeg_dec/ffmpeg-convert-hifi.c b/src/audio/ffmpeg_dec/ffmpeg-convert-hifi.c new file mode 100644 index 000000000000..ba80cedefa21 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg-convert-hifi.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Xtensa HiFi3 / HiFi4 / HiFi5 VFPU SIMD routines for FFmpeg PCM format +// conversion and interleaving. +// +// Each helper processes the SIMD-aligned prefix of @nb_samples and returns the +// number of samples per channel converted (0 when the target core or compiler +// does not provide the required TIE intrinsics). The caller in +// ffmpeg_dec-convert.c finishes any remaining tail samples in scalar C. + +#include +#include "ffmpeg_dec-convert.h" + +#if defined(__XTENSA__) && defined(__has_include) +#if __has_include() +#include +#endif + +#if defined(__has_builtin) && __has_builtin(__builtin_xtensa_trunc_sx2) && \ + __has_builtin(__builtin_xtensa_float_sx2) && \ + defined(XCHAL_HAVE_HIFI4_VFPU) && XCHAL_HAVE_HIFI4_VFPU && \ + __has_include() +#include +#define FFMPEG_DEC_CONV_VFPU 1 +#endif + +#if defined(XCHAL_HAVE_HIFI3) && XCHAL_HAVE_HIFI3 && __has_include() +#include +#define FFMPEG_DEC_CONV_HIFI3 1 +#endif +#endif + +int ffmpeg_hifi_fltp_to_s32(const float *const *src, int32_t *dst, + int channels, int nb_samples) +{ +#if defined(FFMPEG_DEC_CONV_VFPU) + int i = 0; + + if (channels == 1) { + const float *c0 = src[0]; + + for (; i + 2 <= nb_samples; i += 2) { + int32_t tmp[2] __attribute__((aligned(8))); + ae_int32x2 f = AE_L32X2_I((const ae_int32x2 *)&c0[i], 0); + ae_int32x2 q = XT_TRUNC_SX2((ae_xtfloatx2)f, 31); + + AE_S32X2_I(q, (ae_int32x2 *)tmp, 0); + dst[i + 0] = tmp[0]; + dst[i + 1] = tmp[1]; + } + return i; + } + + if (channels == 2) { + const float *c0 = src[0]; + const float *c1 = src[1]; + + for (; i + 2 <= nb_samples; i += 2) { + ae_int32x2 f0 = AE_L32X2_I((const ae_int32x2 *)&c0[i], 0); + ae_int32x2 f1 = AE_L32X2_I((const ae_int32x2 *)&c1[i], 0); + ae_int32x2 q0 = XT_TRUNC_SX2((ae_xtfloatx2)f0, 31); + ae_int32x2 q1 = XT_TRUNC_SX2((ae_xtfloatx2)f1, 31); + int32_t tmp0[2] __attribute__((aligned(8))); + int32_t tmp1[2] __attribute__((aligned(8))); + + AE_S32X2_I(q0, tmp0, 0); + AE_S32X2_I(q1, tmp1, 0); + + dst[2 * i + 0] = tmp0[0]; + dst[2 * i + 1] = tmp1[0]; + dst[2 * i + 2] = tmp0[1]; + dst[2 * i + 3] = tmp1[1]; + } + return i; + } + + for (int c = 0; c < channels; c++) { + const float *pl = src[c]; + int32_t *dc = dst + c; + + for (i = 0; i + 2 <= nb_samples; i += 2) { + int32_t tmp[2] __attribute__((aligned(8))); + ae_int32x2 f = AE_L32X2_I((const ae_int32x2 *)&pl[i], 0); + ae_int32x2 q = XT_TRUNC_SX2((ae_xtfloatx2)f, 31); + + AE_S32X2_I(q, (ae_int32x2 *)tmp, 0); + dc[i * channels] = tmp[0]; + dc[(i + 1) * channels] = tmp[1]; + } + } + return nb_samples & ~1; +#else + (void)src; + (void)dst; + (void)channels; + (void)nb_samples; + return 0; +#endif +} + +int ffmpeg_hifi_s32_to_fltp(const int32_t *src, float *const *dst, + int channels, int nb_samples) +{ +#if defined(FFMPEG_DEC_CONV_VFPU) + int i = 0; + + for (int c = 0; c < channels; c++) { + float *pl = dst[c]; + const int32_t *sc = src + c; + + for (i = 0; i + 2 <= nb_samples; i += 2) { + int32_t tmp[2] __attribute__((aligned(8))); + ae_int32x2 p; + ae_xtfloatx2 r; + + tmp[0] = sc[i * channels]; + tmp[1] = sc[(i + 1) * channels]; + p = AE_L32X2_I((const ae_int32x2 *)tmp, 0); + r = XT_FLOAT_SX2(p, 31); + AE_S32X2_I((ae_int32x2)r, (ae_int32x2 *)&pl[i], 0); + } + } + return nb_samples & ~1; +#else + (void)src; + (void)dst; + (void)channels; + (void)nb_samples; + return 0; +#endif +} + +int ffmpeg_hifi_s32p_to_s32_stereo(const int32_t *c0, const int32_t *c1, + int32_t *dst, int nb_samples) +{ +#if defined(FFMPEG_DEC_CONV_HIFI3) + int i = 0; + + for (; i + 2 <= nb_samples; i += 2) { + ae_int32x2 v0 = AE_L32X2_I((const ae_int32x2 *)&c0[i], 0); + ae_int32x2 v1 = AE_L32X2_I((const ae_int32x2 *)&c1[i], 0); + ae_int32x2 pair0 = AE_SEL32_LL(v0, v1); + ae_int32x2 pair1 = AE_SEL32_HH(v0, v1); + + AE_S32X2_I(pair0, (ae_int32x2 *)&dst[2 * i + 0], 0); + AE_S32X2_I(pair1, (ae_int32x2 *)&dst[2 * i + 2], 0); + } + return i; +#else + (void)c0; + (void)c1; + (void)dst; + (void)nb_samples; + return 0; +#endif +} diff --git a/src/audio/ffmpeg_dec/ffmpeg-filter.c b/src/audio/ffmpeg_dec/ffmpeg-filter.c new file mode 100644 index 000000000000..87469ac928b0 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg-filter.c @@ -0,0 +1,945 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// libavfilter graph backend for the ffmpeg_dec module: runs an FFmpeg audio +// filter (e.g. afftdn FFT noise reduction, alimiter lookahead peak limiter) +// over PCM. Unlike the decoder/encoder paths (send_packet/receive_frame or +// send_frame/receive_packet), filters use the avfilter graph API: +// +// abuffer(src) -> -> abuffersink(sink) +// +// PCM AVFrames are pushed into the source and pulled filtered from the sink. +// Built when CONFIG_FFMPEG_FILTER is enabled (selected by any +// CONFIG_FFMPEG_FILTER_* option), which pulls libavfilter into the module. + +#include +#include +#include +#include +#include +#include +#include +#include +#include "ffmpeg_dec.h" +#include "ffmpeg_dec-convert.h" + +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +#define FFMPEG_AF_MAX_NAME 32 +#define FFMPEG_AF_MAX_ARGS 128 +#define FFMPEG_AF_LAYOUT_STR_SIZE 64 + +enum ffmpeg_af_filter_id { + FFMPEG_AF_FILTER_DEFAULT = 0, + FFMPEG_AF_FILTER_AFFTDN = 1, + FFMPEG_AF_FILTER_ALIMITER = 2, +}; + +struct ffmpeg_af_filter_desc { + enum ffmpeg_af_filter_id id; + const char *name; + const char *default_args; + enum AVSampleFormat sample_fmt; +}; + +static const struct ffmpeg_af_filter_desc ffmpeg_af_filters[] = { +#if defined(CONFIG_FFMPEG_FILTER_AFFTDN) + { + .id = FFMPEG_AF_FILTER_AFFTDN, + .name = "afftdn", + .default_args = "nr=28:nf=-20", + .sample_fmt = AV_SAMPLE_FMT_FLTP, + }, +#endif +#if defined(CONFIG_FFMPEG_FILTER_ALIMITER) + { + .id = FFMPEG_AF_FILTER_ALIMITER, + .name = "alimiter", + .default_args = "limit=0.95:attack=5:release=50:level=true", + .sample_fmt = AV_SAMPLE_FMT_DBL, + }, +#endif +}; + +static const struct ffmpeg_af_filter_desc *ffmpeg_af_default_filter(void) +{ +#if defined(CONFIG_FFMPEG_FILTER_DEFAULT_ALIMITER) + for (size_t i = 0; i < ARRAY_SIZE(ffmpeg_af_filters); i++) { + if (ffmpeg_af_filters[i].id == FFMPEG_AF_FILTER_ALIMITER) + return &ffmpeg_af_filters[i]; + } +#endif + if (ARRAY_SIZE(ffmpeg_af_filters) > 0) + return &ffmpeg_af_filters[0]; + return NULL; +} + +static const struct ffmpeg_af_filter_desc *ffmpeg_af_find_by_id(uint32_t id) +{ + for (size_t i = 0; i < ARRAY_SIZE(ffmpeg_af_filters); i++) { + if (ffmpeg_af_filters[i].id == id) + return &ffmpeg_af_filters[i]; + } + return NULL; +} + +static const struct ffmpeg_af_filter_desc *ffmpeg_af_find_by_name(const char *name) +{ + for (size_t i = 0; i < ARRAY_SIZE(ffmpeg_af_filters); i++) { + if (strcmp(ffmpeg_af_filters[i].name, name) == 0) + return &ffmpeg_af_filters[i]; + } + return NULL; +} + +/** + * struct ffmpeg_af_graph - one audio filter graph instance. + * @graph: The avfilter graph. + * @src: abuffer source context (frames pushed here). + * @sink: abuffersink context (filtered frames pulled here). + * @in_frame: Persistent pre-allocated input AVFrame. + * @out_frame: Persistent pre-allocated output AVFrame. + * @desc: Selected filter descriptor. + * @filter_name: Filter name string (e.g. "afftdn", "alimiter"). + * @filter_args: Filter argument string passed to avfilter_graph_create_filter(). + * @frame_size: Number of samples per filter advance block (rate / 80). + * @channels: Channel count. + * @rate: Sample rate (Hz). + * @in_frame_fmt: SOF input valid sample format (enum sof_ipc_frame). + * @out_frame_fmt: SOF output valid sample format (enum sof_ipc_frame). + * @in_frame_bytes: Bytes per multichannel input audio frame. + * @out_frame_bytes: Bytes per multichannel output audio frame. + * @pcm_in: Linear staging buffer for input interleaved PCM. + * @pcm_in_count: Current number of audio frames in @pcm_in. + * @pcm_in_cap: Maximum capacity of @pcm_in in audio frames. + * @pcm_out: Linear staging FIFO for filtered interleaved PCM. + * @pcm_out_count: Current number of unread audio frames in @pcm_out. + * @pcm_out_rd: Read offset in audio frames for draining @pcm_out. + * @pcm_out_cap: Maximum capacity of @pcm_out in audio frames. + * @eof_sent: True once NULL EOF frame has been submitted to @src. + * @eof_flushed: True once @sink has returned AVERROR_EOF. + */ +struct ffmpeg_af_graph { + AVFilterGraph *graph; + AVFilterContext *src; + AVFilterContext *sink; + AVFrame *in_frame; + AVFrame *out_frame; + const struct ffmpeg_af_filter_desc *desc; + char filter_name[FFMPEG_AF_MAX_NAME]; + char filter_args[FFMPEG_AF_MAX_ARGS]; + int frame_size; + int channels; + int rate; + int in_frame_fmt; + int out_frame_fmt; + int in_frame_bytes; + int out_frame_bytes; + + uint8_t *pcm_in; + size_t pcm_in_count; + size_t pcm_in_cap; + + uint8_t *pcm_out; + size_t pcm_out_count; + size_t pcm_out_rd; + size_t pcm_out_cap; + + bool eof_sent; + bool eof_flushed; +}; + +static void ffmpeg_af_close(struct processing_module *mod, struct ffmpeg_af_graph *g) +{ + if (!g) + return; + + if (g->in_frame) + av_frame_free(&g->in_frame); + if (g->out_frame) + av_frame_free(&g->out_frame); + if (g->pcm_in) { + mod_free(mod, g->pcm_in); + g->pcm_in = NULL; + } + if (g->pcm_out) { + mod_free(mod, g->pcm_out); + g->pcm_out = NULL; + } + if (g->graph) + avfilter_graph_free(&g->graph); + g->src = NULL; + g->sink = NULL; + g->pcm_in_count = 0; + g->pcm_out_count = 0; + g->pcm_out_rd = 0; + g->eof_sent = false; + g->eof_flushed = false; +} + +static void ffmpeg_af_quiet_log(void *avcl, int level, const char *fmt, va_list vl) +{ + (void)avcl; + (void)level; + (void)fmt; + (void)vl; +} + +/** + * ffmpeg_af_open() - Build a src -> -> sink graph for the given PCM + * format. + * @mod: Pointer to module data. + * @g: Graph to initialise. + * @filter: Filter name, e.g. "afftdn" or "alimiter". + * @args: Filter argument string, or NULL for filter defaults. + * @fmt: Filter sample format (e.g. AV_SAMPLE_FMT_FLTP or AV_SAMPLE_FMT_DBL). + * + * Return: Zero on success, negative errno otherwise. + */ +static int ffmpeg_af_open(struct processing_module *mod, + struct ffmpeg_af_graph *g, const char *filter, + const char *args, enum AVSampleFormat fmt) +{ + struct comp_dev *dev = mod->dev; + const AVFilter *abuffer = avfilter_get_by_name("abuffer"); + const AVFilter *abuffersink = avfilter_get_by_name("abuffersink"); + const AVFilter *filt = avfilter_get_by_name(filter); + AVFilterContext *filt_ctx = NULL; + AVChannelLayout ch_layout = { 0 }; + char layout_str[FFMPEG_AF_LAYOUT_STR_SIZE]; + AVRational tb; + int ret; + + /* Phase 1: Look up avfilter descriptors and validate stream parameters. */ + if (!abuffer || !abuffersink || !filt) { + comp_err(dev, "avfilter '%s' or buffer endpoint not found", filter); + return -ENODEV; + } + + if (!g->rate) { + comp_warn(dev, "rate unset, defaulting to 48000 Hz"); + g->rate = 48000; + } + if (!g->channels) { + comp_warn(dev, "channels unset, defaulting to 2"); + g->channels = 2; + } + if (g->in_frame_bytes <= 0 || g->in_frame_bytes > 32) { + comp_warn(dev, "in_frame_bytes %d invalid, defaulting to %zu", + g->in_frame_bytes, g->channels * sizeof(int16_t)); + g->in_frame_bytes = g->channels * sizeof(int16_t); + } + if (g->out_frame_bytes <= 0 || g->out_frame_bytes > 32) { + comp_warn(dev, "out_frame_bytes %d invalid, defaulting to %zu", + g->out_frame_bytes, g->channels * sizeof(int16_t)); + g->out_frame_bytes = g->channels * sizeof(int16_t); + } + g->frame_size = g->rate / 80; + if (g->frame_size <= 0) { + comp_warn(dev, "frame_size %d invalid, defaulting to 600", g->frame_size); + g->frame_size = 600; + } + + av_log_set_callback(ffmpeg_af_quiet_log); + av_log_set_level(AV_LOG_QUIET); + + /* Phase 2: Allocate the filter graph and configure the abuffer source node. */ + ffmpeg_af_close(mod, g); + g->graph = avfilter_graph_alloc(); + if (!g->graph) + return -ENOMEM; + + g->src = avfilter_graph_alloc_filter(g->graph, abuffer, "in"); + if (!g->src) { + ret = -ENOMEM; + goto err; + } + + tb.num = 1; + tb.den = g->rate; + av_opt_set_q(g->src, "time_base", tb, AV_OPT_SEARCH_CHILDREN); + av_opt_set_int(g->src, "sample_rate", g->rate, AV_OPT_SEARCH_CHILDREN); + av_opt_set_sample_fmt(g->src, "sample_fmt", fmt, AV_OPT_SEARCH_CHILDREN); + av_channel_layout_default(&ch_layout, g->channels); + av_channel_layout_describe(&ch_layout, layout_str, FFMPEG_AF_LAYOUT_STR_SIZE); + av_opt_set(g->src, "channel_layout", layout_str, AV_OPT_SEARCH_CHILDREN); + + ret = avfilter_init_dict(g->src, NULL); + if (ret < 0) { + comp_err(dev, "abuffer init failed %d", ret); + av_channel_layout_uninit(&ch_layout); + ret = -EIO; + goto err; + } + + /* Phase 3: Instantiate the processing filter node and abuffersink output node. */ + ret = avfilter_graph_create_filter(&filt_ctx, filt, "filter", + args, NULL, g->graph); + if (ret < 0) { + comp_err(dev, "create filter '%s' failed %d", filter, ret); + av_channel_layout_uninit(&ch_layout); + ret = -EIO; + goto err; + } + + ret = avfilter_graph_create_filter(&g->sink, abuffersink, "out", NULL, NULL, g->graph); + if (ret < 0) { + comp_err(dev, "create abuffersink failed %d", ret); + av_channel_layout_uninit(&ch_layout); + ret = -EIO; + goto err; + } + av_opt_set(g->sink, "ch_layouts", layout_str, AV_OPT_SEARCH_CHILDREN); + av_channel_layout_uninit(&ch_layout); + + /* Phase 4: Link abuffer -> filter -> abuffersink and negotiate graph formats. */ + ret = avfilter_link(g->src, 0, filt_ctx, 0); + if (!ret) + ret = avfilter_link(filt_ctx, 0, g->sink, 0); + if (ret < 0) { + comp_err(dev, "avfilter_link failed %d", ret); + ret = -EIO; + goto err; + } + + ret = avfilter_graph_config(g->graph, NULL); + if (ret < 0) { + comp_err(dev, "avfilter_graph_config failed %d", ret); + ret = -EIO; + goto err; + } + + /* Phase 5: Allocate persistent AVFrames and linear PCM staging buffers. */ + g->in_frame = av_frame_alloc(); + g->out_frame = av_frame_alloc(); + if (!g->in_frame || !g->out_frame) { + ret = -ENOMEM; + goto err; + } + + g->in_frame->format = fmt; + g->in_frame->nb_samples = g->frame_size; + g->in_frame->sample_rate = g->rate; + av_channel_layout_default(&g->in_frame->ch_layout, g->channels); + ret = av_frame_get_buffer(g->in_frame, 0); + if (ret < 0) { + comp_err(dev, "av_frame_get_buffer failed %d", ret); + ret = -ENOMEM; + goto err; + } + + /* Allocate persistent staging buffers for 4 blocks to absorb filter latency. */ + g->pcm_in_cap = (size_t)g->frame_size * 4; + g->pcm_out_cap = (size_t)g->frame_size * 4; + g->pcm_in = mod_alloc(mod, g->pcm_in_cap * (size_t)g->in_frame_bytes); + g->pcm_out = mod_alloc(mod, g->pcm_out_cap * (size_t)g->out_frame_bytes); + if (!g->pcm_in || !g->pcm_out) { + ret = -ENOMEM; + goto err; + } + + g->pcm_in_count = 0; + g->pcm_out_count = 0; + g->pcm_out_rd = 0; + g->eof_sent = false; + g->eof_flushed = false; + + comp_info(dev, "ffmpeg_af: '%s' open OK, rate %d ch %d frame_size %d", + filter, g->rate, g->channels, g->frame_size); + return 0; + +err: + ffmpeg_af_close(mod, g); + return ret; +} + +static void ffmpeg_af_store_out_frame(struct ffmpeg_af_graph *g, + const AVFrame *frame) +{ + size_t avail_bytes; + uint8_t *dst; + int ret; + + if (g->pcm_out_rd > 0) { + if (g->pcm_out_count > 0) { + memmove(g->pcm_out, + g->pcm_out + g->pcm_out_rd * (size_t)g->out_frame_bytes, + g->pcm_out_count * (size_t)g->out_frame_bytes); + } + g->pcm_out_rd = 0; + } + + if (g->pcm_out_count >= g->pcm_out_cap) + return; + + dst = g->pcm_out + g->pcm_out_count * (size_t)g->out_frame_bytes; + avail_bytes = (g->pcm_out_cap - g->pcm_out_count) * (size_t)g->out_frame_bytes; + ret = ffmpeg_dec_convert_frame(frame, dst, avail_bytes, + g->out_frame_fmt, g->channels); + if (ret > 0) + g->pcm_out_count += (size_t)ret / (size_t)g->out_frame_bytes; +} + +static int ffmpeg_af_pull_output(struct processing_module *mod, + struct ffmpeg_af_graph *g) +{ + struct comp_dev *dev = mod->dev; + int ret; + + while ((int)(g->pcm_out_cap - g->pcm_out_count) >= g->frame_size) { + ret = av_buffersink_get_samples(g->sink, g->out_frame, g->frame_size); + if (ret == AVERROR(EAGAIN)) + break; + if (ret == AVERROR_EOF) { + g->eof_flushed = true; + break; + } + if (ret < 0) { + comp_err(dev, "av_buffersink_get_samples failed %d", ret); + return -EIO; + } + + ffmpeg_af_store_out_frame(g, g->out_frame); + av_frame_unref(g->out_frame); + } + + return 0; +} + +static int ffmpeg_af_submit_input(struct processing_module *mod, + struct ffmpeg_af_graph *g, int nb_samples) +{ + struct comp_dev *dev = mod->dev; + int ret; + + ret = av_frame_make_writable(g->in_frame); + if (ret < 0) { + comp_err(dev, "av_frame_make_writable failed %d", ret); + return -EIO; + } + + g->in_frame->nb_samples = nb_samples; + ret = ffmpeg_dec_pcm_to_frame(g->pcm_in, g->in_frame, + g->in_frame_fmt, g->channels, nb_samples); + if (ret < 0) { + g->in_frame->nb_samples = g->frame_size; + comp_err(dev, "ffmpeg_dec_pcm_to_frame failed %d", ret); + return ret; + } + + if (g->pcm_in_count > (size_t)nb_samples) { + memmove(g->pcm_in, + g->pcm_in + (size_t)nb_samples * (size_t)g->in_frame_bytes, + (g->pcm_in_count - (size_t)nb_samples) * + (size_t)g->in_frame_bytes); + } + g->pcm_in_count -= (size_t)nb_samples; + + ret = av_buffersrc_add_frame_flags(g->src, g->in_frame, + AV_BUFFERSRC_FLAG_KEEP_REF); + g->in_frame->nb_samples = g->frame_size; + if (ret < 0) { + comp_err(dev, "av_buffersrc_add_frame_flags failed %d", ret); + return -EIO; + } + + return ffmpeg_af_pull_output(mod, g); +} + +static void ffmpeg_af_drain_sink(struct ffmpeg_af_graph *g, struct sof_sink *sink) +{ + size_t free_bytes, req_bytes; + int free_frames, to_drain; + void *dst, *buf_start; + size_t buf_size; + int ret; + + if (g->pcm_out_count == 0) + return; + + free_bytes = sink_get_free_size(sink); + free_frames = (int)(free_bytes / (size_t)g->out_frame_bytes); + to_drain = MIN((int)g->pcm_out_count, free_frames); + if (to_drain <= 0) + return; + + req_bytes = (size_t)to_drain * (size_t)g->out_frame_bytes; + ret = sink_get_buffer(sink, req_bytes, &dst, &buf_start, &buf_size); + if (ret == 0) { + ffmpeg_dec_copy_to_circular(dst, buf_start, buf_size, + g->pcm_out + g->pcm_out_rd * (size_t)g->out_frame_bytes, + req_bytes); + sink_commit_buffer(sink, req_bytes); + g->pcm_out_rd += (size_t)to_drain; + g->pcm_out_count -= (size_t)to_drain; + if (g->pcm_out_count == 0) + g->pcm_out_rd = 0; + } +} + +int ffmpeg_af_mod_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct ffmpeg_dec_comp_data *cd; + struct ffmpeg_af_graph *g; + const struct ffmpeg_af_filter_desc *desc; + + ffmpeg_dec_libc_bind(mod); + comp_info(dev, "entry"); + + if (dev->ipc_config.proc_domain != COMP_PROCESSING_DOMAIN_DP) { + comp_err(dev, "ffmpeg_af requires DP processing domain"); + return -EINVAL; + } + + desc = ffmpeg_af_default_filter(); + if (!desc) { + comp_err(dev, "no FFmpeg audio filters enabled in build"); + return -ENOTSUP; + } + + if (mod->priv.cfg.ext_data && + mod->priv.cfg.ext_data->module_data && + mod->priv.cfg.ext_data->module_data_size >= sizeof(struct snd_codec)) { + const struct snd_codec *codec_params = mod->priv.cfg.ext_data->module_data; + + if (codec_params->profile) { + const struct ffmpeg_af_filter_desc *req_desc = + ffmpeg_af_find_by_id(codec_params->profile); + + if (!req_desc) { + comp_err(dev, "unsupported or disabled filter profile %u", + codec_params->profile); + return -ENOTSUP; + } + desc = req_desc; + } + } + + cd = mod_zalloc(mod, sizeof(*cd)); + if (!cd) + return -ENOMEM; + cd->is_filter = true; + md->private = cd; + + md->mpd.in_buff_size = 9600; + md->mpd.out_buff_size = 9600; + + g = mod_zalloc(mod, sizeof(*g)); + if (!g) { + mod_free(mod, cd); + md->private = NULL; + return -ENOMEM; + } + + g->desc = desc; + snprintf(g->filter_name, sizeof(g->filter_name), "%s", desc->name); + snprintf(g->filter_args, sizeof(g->filter_args), "%s", desc->default_args); + cd->backend_data = g; + + cd->eos_msg = ffmpeg_dec_eos_notification_init(mod); + if (!cd->eos_msg) { + comp_err(dev, "failed to allocate EOS notification"); + mod_free(mod, g); + mod_free(mod, cd); + md->private = NULL; + return -ENOMEM; + } + + return 0; +} + +int ffmpeg_af_mod_set_config(struct processing_module *mod, + const uint8_t *data, size_t size) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + struct ffmpeg_af_graph *g; + char buf[FFMPEG_AF_MAX_NAME + FFMPEG_AF_MAX_ARGS]; + const struct ffmpeg_af_filter_desc *desc; + char *colon; + size_t len; + + if (!cd || !cd->backend_data || !data || !size) + return 0; + + g = cd->backend_data; + if (dev->state == COMP_STATE_ACTIVE || cd->configured || g->graph) { + comp_err(dev, "cannot reconfigure filter while active (state %d)", + dev->state); + return -EBUSY; + } + + len = MIN(size, sizeof(buf) - 1); + memcpy_s(buf, sizeof(buf) - 1, data, len); + buf[len] = '\0'; + + colon = strchr(buf, ':'); + if (colon) + *colon = '\0'; + + desc = ffmpeg_af_find_by_name(buf); + if (!desc) { + comp_err(dev, "unknown or disabled filter '%s'", buf); + return -ENOTSUP; + } + + g->desc = desc; + snprintf(g->filter_name, sizeof(g->filter_name), "%s", desc->name); + if (colon && colon[1] != '\0') + snprintf(g->filter_args, sizeof(g->filter_args), "%s", colon + 1); + else + snprintf(g->filter_args, sizeof(g->filter_args), "%s", desc->default_args); + + comp_info(dev, "configured filter '%s' args '%s'", g->filter_name, g->filter_args); + return 0; +} + +static int ffmpeg_af_validate_pcm_fmt(struct comp_dev *dev, int container_fmt, + int valid_fmt, int channels, + int frame_bytes, const char *dir) +{ + int expected_bps; + + switch (container_fmt) { + case SOF_IPC_FRAME_S16_LE: + expected_bps = 2; + break; + case SOF_IPC_FRAME_S24_4LE: + case SOF_IPC_FRAME_S32_LE: + expected_bps = 4; + break; + default: + comp_err(dev, "unsupported %s container format %d", dir, container_fmt); + return -EINVAL; + } + + switch (valid_fmt) { + case SOF_IPC_FRAME_S16_LE: + if (expected_bps != 2) { + comp_err(dev, "unsupported %s S16 valid format in %d-byte container", + dir, expected_bps); + return -EINVAL; + } + break; + case SOF_IPC_FRAME_S24_4LE: + case SOF_IPC_FRAME_S32_LE: + if (expected_bps != 4) { + comp_err(dev, "unsupported %s valid format %d in %d-byte container", + dir, valid_fmt, expected_bps); + return -EINVAL; + } + break; + default: + comp_err(dev, "unsupported %s valid sample format %d", dir, valid_fmt); + return -EINVAL; + } + + if (channels <= 0 || frame_bytes != channels * expected_bps) { + comp_err(dev, "invalid %s frame_bytes %d for %d ch (%d B/sample)", + dir, frame_bytes, channels, expected_bps); + return -EINVAL; + } + + return 0; +} + +int ffmpeg_af_mod_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + struct ffmpeg_af_graph *g; + struct sof_source *source; + struct sof_sink *sink; + int in_container_fmt, out_container_fmt; + int ret; + + if (!cd || !cd->backend_data) + return -EINVAL; + if (num_of_sources < 1 || !sources[0] || num_of_sinks < 1 || !sinks[0]) { + comp_err(dev, "missing source or sink"); + return -EINVAL; + } + + ffmpeg_dec_libc_bind(mod); + g = cd->backend_data; + source = sources[0]; + sink = sinks[0]; + + g->rate = sink_get_rate(sink); + g->channels = sink_get_channels(sink); + out_container_fmt = sink_get_frm_fmt(sink); + g->out_frame_fmt = sink_get_valid_fmt(sink); + g->out_frame_bytes = sink_get_frame_bytes(sink); + + in_container_fmt = source_get_frm_fmt(source); + g->in_frame_fmt = source_get_valid_fmt(source); + g->in_frame_bytes = source_get_frame_bytes(source); + + if (source_get_rate(source) && g->rate && + source_get_rate(source) != (uint32_t)g->rate) { + comp_err(dev, "source rate %u != sink rate %d", + source_get_rate(source), g->rate); + return -EINVAL; + } + if (source_get_channels(source) && g->channels && + source_get_channels(source) != (uint32_t)g->channels) { + comp_err(dev, "source channels %u != sink channels %d", + source_get_channels(source), g->channels); + return -EINVAL; + } + + ret = ffmpeg_af_validate_pcm_fmt(dev, in_container_fmt, g->in_frame_fmt, + g->channels, g->in_frame_bytes, "input"); + if (ret < 0) + return ret; + + ret = ffmpeg_af_validate_pcm_fmt(dev, out_container_fmt, g->out_frame_fmt, + g->channels, g->out_frame_bytes, "output"); + if (ret < 0) + return ret; + + cd->out_rate = g->rate; + cd->out_channels = g->channels; + cd->out_frame_fmt = g->out_frame_fmt; + cd->out_frame_bytes = g->out_frame_bytes; + + comp_info(dev, "ffmpeg_af: %s, rate %d ch %d in_fmt %d (%d B) out_fmt %d (%d B)", + g->filter_name, g->rate, g->channels, + g->in_frame_fmt, g->in_frame_bytes, + g->out_frame_fmt, g->out_frame_bytes); + + /* + * NB: Lazy open deferred to DP thread. + * prepare() is dispatched across cores via IDC and runs on the IDC + * worker thread with a small stack. Filter open (avfilter_graph_config / + * afftdn table building) requires a deep stack and takes tens of ms. + * Defer ffmpeg_af_open() to the first process() call on the module's + * own DP thread. + */ + cd->configured = false; + cd->eos_sent = false; + g->eof_sent = false; + g->eof_flushed = false; + + return 0; +} + +int ffmpeg_af_mod_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + struct sof_source *source; + struct sof_sink *sink; + struct ffmpeg_af_graph *g; + size_t avail_bytes, req_bytes; + int avail_frames, space_frames, to_read; + const void *sp, *sstart; + size_t sbytes; + int ret; + enum sof_audio_buffer_state src_state; + bool host_eos, src_exhausted; + + ffmpeg_dec_libc_bind(mod); + if (!cd || !cd->backend_data) + return -EINVAL; + + g = cd->backend_data; + + /* Lazy open: runs once on the module's DP thread with deep stack. */ + if (!cd->configured) { + enum AVSampleFormat fmt = g->desc ? g->desc->sample_fmt : AV_SAMPLE_FMT_FLTP; + + ret = ffmpeg_af_open(mod, g, g->filter_name, g->filter_args, fmt); + if (ret < 0) { + comp_err(dev, "ffmpeg_af_open failed %d", ret); + return ret; + } + cd->configured = true; + return 0; /* proceed to process on next cycle */ + } + + if (num_of_sources < 1 || num_of_sinks < 1) + return -EINVAL; + + source = sources[0]; + sink = sinks[0]; + src_state = source_get_state(source); + host_eos = src_state == AUDIOBUF_STATE_END_OF_STREAM || + src_state == AUDIOBUF_STATE_END_OF_STREAM_FLUSH; + + /* 1. Drain previously staged filtered samples into the sink if space is available. */ + ffmpeg_af_drain_sink(g, sink); + + /* 2. Read new input samples from source into pcm_in staging buffer. */ + avail_bytes = source_get_data_available(source); + space_frames = (int)(g->pcm_in_cap - g->pcm_in_count); + avail_frames = (int)(avail_bytes / (size_t)g->in_frame_bytes); + to_read = MIN(avail_frames, space_frames); + if (to_read > 0) { + req_bytes = (size_t)to_read * (size_t)g->in_frame_bytes; + ret = source_get_data(source, req_bytes, &sp, &sstart, &sbytes); + if (ret == 0) { + ffmpeg_dec_copy_from_circular(g->pcm_in + + g->pcm_in_count * (size_t)g->in_frame_bytes, + sp, sstart, sbytes, req_bytes); + source_release_data(source, req_bytes); + g->pcm_in_count += (size_t)to_read; + } + } + + src_exhausted = host_eos && + source_get_data_available(source) < (size_t)g->in_frame_bytes; + + /* 3. Process at most one filter frame per cycle to yield to scheduler and IDC. */ + if ((int)g->pcm_in_count >= g->frame_size && + (int)(g->pcm_out_cap - g->pcm_out_count) >= g->frame_size) { + ret = ffmpeg_af_submit_input(mod, g, g->frame_size); + if (ret < 0) + return ret; + } else if (src_exhausted && g->pcm_in_count > 0 && + (g->pcm_out_cap - g->pcm_out_count) >= g->pcm_in_count) { + /* Flush remaining partial input block (< frame_size) at EOS. */ + ret = ffmpeg_af_submit_input(mod, g, (int)g->pcm_in_count); + if (ret < 0) + return ret; + } + + /* 4. At EOS after all input is submitted, send NULL EOF frame and drain graph. */ + if (src_exhausted && g->pcm_in_count == 0 && !g->eof_flushed && + (int)(g->pcm_out_cap - g->pcm_out_count) >= g->frame_size) { + if (!g->eof_sent) { + ret = av_buffersrc_add_frame_flags(g->src, NULL, 0); + if (ret < 0) { + comp_err(dev, "av_buffersrc EOF flush failed %d", ret); + return -EIO; + } + g->eof_sent = true; + } + ret = ffmpeg_af_pull_output(mod, g); + if (ret < 0) + return ret; + } + + /* 5. Drain any freshly produced samples to the sink. */ + ffmpeg_af_drain_sink(g, sink); + + /* 6. Signal EOS once source, filter graph, and output staging are fully drained. */ + if (src_exhausted && g->pcm_in_count == 0 && g->eof_flushed && + g->pcm_out_count == 0 && !cd->eos_sent) + ffmpeg_dec_signal_eos(mod, sink); + + return 0; +} + +bool ffmpeg_af_is_ready_to_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_af_graph *g; + + if (num_of_sources < 1 || num_of_sinks < 1) + return false; + + if (!cd) + return false; + + /* Run lazy open on DP thread if not configured yet. */ + if (!cd->configured) + return true; + + g = cd->backend_data; + if (!g) + return false; + + if (cd->eos_sent && g->pcm_out_count == 0) + return false; + + if (g->pcm_out_count > 0 && + sink_get_free_size(sinks[0]) >= (size_t)g->out_frame_bytes) + return true; + + if (source_get_data_available(sources[0]) >= (size_t)g->in_frame_bytes && + g->pcm_in_count < g->pcm_in_cap) + return true; + + if ((int)g->pcm_in_count >= g->frame_size && + (int)(g->pcm_out_cap - g->pcm_out_count) >= g->frame_size) + return true; + + if (!cd->eos_sent) { + enum sof_audio_buffer_state src_state = source_get_state(sources[0]); + bool host_eos = src_state == AUDIOBUF_STATE_END_OF_STREAM || + src_state == AUDIOBUF_STATE_END_OF_STREAM_FLUSH; + + if (host_eos && + source_get_data_available(sources[0]) < (size_t)g->in_frame_bytes) { + if (g->pcm_in_count > 0 && + (g->pcm_out_cap - g->pcm_out_count) >= g->pcm_in_count) + return true; + if (g->pcm_in_count == 0 && !g->eof_flushed && + (int)(g->pcm_out_cap - g->pcm_out_count) >= g->frame_size) + return true; + if (g->pcm_in_count == 0 && g->eof_flushed && + g->pcm_out_count == 0) + return true; + } + } + + return false; +} + +int ffmpeg_af_mod_reset(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_af_graph *g; + + ffmpeg_dec_libc_bind(mod); + if (!cd || !cd->backend_data) + return 0; + + g = cd->backend_data; + ffmpeg_af_close(mod, g); + cd->configured = false; + cd->eos_sent = false; + return 0; +} + +int ffmpeg_af_mod_free(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + ffmpeg_dec_libc_bind(mod); + if (!cd) + return 0; + + if (cd->eos_msg) { + mod_ipc_msg_free(mod, cd->eos_msg); + cd->eos_msg = NULL; + } + + if (cd->backend_data) { + ffmpeg_af_close(mod, cd->backend_data); + mod_free(mod, cd->backend_data); + cd->backend_data = NULL; + } + mod_free(mod, cd); + md->private = NULL; + return 0; +} diff --git a/src/audio/ffmpeg_dec/ffmpeg.cmake b/src/audio/ffmpeg_dec/ffmpeg.cmake index 9d77ca6181ae..90312c2ff566 100644 --- a/src/audio/ffmpeg_dec/ffmpeg.cmake +++ b/src/audio/ffmpeg_dec/ffmpeg.cmake @@ -42,9 +42,9 @@ if(CONFIG_FFMPEG_DEC_MP3) list(APPEND _ff_decoders mp3) list(APPEND _ff_parsers mpegaudio) endif() -# A decoder is required unless this is an encoder-only build. -if(NOT _ff_decoders AND NOT CONFIG_FFMPEG_ENC_MP3 AND NOT CONFIG_FFMPEG_ENC_AAC) - message(FATAL_ERROR "ffmpeg_dec: no decoder or encoder selected (Kconfig FFMPEG_DEC_*/FFMPEG_ENC_*)") +# At least one decoder, encoder, or filter must be selected. +if(NOT _ff_decoders AND NOT CONFIG_FFMPEG_ENC_MP3 AND NOT CONFIG_FFMPEG_ENC_AAC AND NOT CONFIG_FFMPEG_BUILD_AVFILTER) + message(FATAL_ERROR "ffmpeg_dec: no decoder, encoder, or filter selected (Kconfig FFMPEG_DEC_*/FFMPEG_ENC_*/FFMPEG_FILTER_*)") endif() set(_ff_dec_cfg "") if(_ff_decoders) @@ -53,6 +53,24 @@ if(_ff_decoders) set(_ff_dec_cfg --enable-decoder=${_ff_dec_csv} --enable-parser=${_ff_par_csv}) endif() +# --- 2b. Kconfig -> libavfilter audio filters (off unless a filter is chosen) --- +set(_ff_avfilter_cfg --disable-avfilter) +if(CONFIG_FFMPEG_BUILD_AVFILTER) + set(_ff_filters "") + if(CONFIG_FFMPEG_FILTER_AFFTDN) + list(APPEND _ff_filters afftdn) + endif() + if(CONFIG_FFMPEG_FILTER_ALIMITER) + list(APPEND _ff_filters alimiter) + endif() + list(JOIN _ff_filters "," _ff_filt_csv) + # abuffer/abuffersink feed/drain a filter graph; aformat negotiates format. + set(_ff_avfilter_cfg + --enable-avfilter + --enable-filter=abuffer,abuffersink,aformat,${_ff_filt_csv}) + message(STATUS "ffmpeg_dec: enabling avfilter, filters [${_ff_filt_csv}]") +endif() + # --- 3. Derive the cross toolchain + per-compiler cross flags --- # GCC 14 (Zephyr SDK) promotes these to errors; FFmpeg 7.x trips them. set(_ff_extra_cflags "-fPIC -mlongcalls -mtext-section-literals -Wno-error=incompatible-pointer-types -Wno-error=implicit-function-declaration") @@ -346,6 +364,11 @@ if(CONFIG_FFMPEG_DEC_COLD_SPLIT) ${FFMPEG_INSTALL_DIR}/lib/libavutil.a COMMAND ${_ff_objcopy} --rename-section .text=.cold ${FFMPEG_INSTALL_DIR}/lib/libswresample.a) + if(CONFIG_FFMPEG_BUILD_AVFILTER) + list(APPEND _ff_cold_rename + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libavfilter.a) + endif() endif() # --- 3b. MP3 encoder via libshine (fixed-point; FFmpeg has no native one) --- @@ -514,6 +537,9 @@ set(_ff_byproducts ${FFMPEG_INSTALL_DIR}/lib/libavcodec.a ${FFMPEG_INSTALL_DIR}/lib/libavutil.a ${FFMPEG_INSTALL_DIR}/lib/libswresample.a) +if(CONFIG_FFMPEG_BUILD_AVFILTER) + list(APPEND _ff_byproducts ${FFMPEG_INSTALL_DIR}/lib/libavfilter.a) +endif() ExternalProject_Add(ffmpeg_ext DEPENDS ${_ff_shine_dep} ${_ff_vo_aacenc_dep} @@ -532,7 +558,7 @@ ExternalProject_Add(ffmpeg_ext # C intrinsics) and is required to build ff_float_dsp_init_xtensa. --disable-all --disable-everything --disable-autodetect --disable-programs --disable-doc --disable-network - --disable-avformat --disable-avdevice --disable-avfilter + --disable-avformat --disable-avdevice ${_ff_avfilter_cfg} --disable-swscale --disable-pthreads --disable-w32threads --disable-os2threads --disable-runtime-cpudetect --disable-debug diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-builtin-libc.c b/src/audio/ffmpeg_dec/ffmpeg_dec-builtin-libc.c index 9a4a5d79a057..a1c1aabffc7a 100644 --- a/src/audio/ffmpeg_dec/ffmpeg_dec-builtin-libc.c +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-builtin-libc.c @@ -19,7 +19,12 @@ // getenv, clock, gmtime, mktime, strftime): NEVER used on a SOF DSP nor on // the FLAC decode path -> no-op stubs. av_log()'s stderr path is bypassed by // the Zephyr LOG wrapper installed in ffmpeg_dec-ffmpeg.c. -// * strtod: not on the FLAC decode path -> return-0 stub. +// * strtod / strtof: Zephyr's CONFIG_MINIMAL_LIBC (lib/libc/minimal/source/ +// stdlib/) only implements atoi/strtol/strtoul and omits strtod/strtof, and +// the toolchain's prebuilt newlib libc.a cannot be linked into the base +// firmware image (compiled without -mlongcalls and conflicts with minimal +// libc). libavfilter's av_opt_set_from_string() -> av_strtod() calls strtod +// to parse filter argument strings (e.g. "limit=0.95:attack=5:release=50"). // * __errno: the double libm libav* references (acos/sin/exp/log/pow/scalbn/ // __isinfd/...) is NOT stubbed here -- it is the real newlib math for this // core, compiled from vendored newlib source with -mlongcalls into @@ -69,14 +74,21 @@ __attribute__((weak)) void *gmtime(const void *timep) { (void)timep; return NULL __attribute__((weak)) long mktime(void *tm) { (void)tm; return -1; } __attribute__((weak)) size_t strftime(char *s, size_t max, const char *fmt, const void *tm) { (void)fmt; (void)tm; if (max) s[0] = '\0'; return 0; } +/* + * Zephyr's CONFIG_MINIMAL_LIBC omits strtod/strtof (only strtol/strtoul are + * provided in zephyr/lib/libc/minimal/source/stdlib/), and the SDK's prebuilt + * libc.a cannot be linked into a -nostdlib / --no-relax firmware image. + * libavfilter's option parser (av_strtod) requires strtod to parse filter + * parameter strings such as "nr=28:nf=-20" and "limit=0.95:attack=5". + */ __attribute__((weak)) double strtod(const char *nptr, char **endptr) { const char *s = nptr; double val = 0.0; double sign = 1.0; - double frac = 0.0; - double div = 1.0; int has_digits = 0; + int frac_exp = 0; + int exp_val = 0; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; @@ -94,7 +106,7 @@ __attribute__((weak)) double strtod(const char *nptr, char **endptr) s += 3; if (endptr) *endptr = (char *)s; - return sign * (1.0 / 0.0); + return sign * __builtin_inf(); } if ((s[0] == 'n' || s[0] == 'N') && (s[1] == 'a' || s[1] == 'A') && @@ -102,12 +114,15 @@ __attribute__((weak)) double strtod(const char *nptr, char **endptr) s += 3; if (endptr) *endptr = (char *)s; - return 0.0 / 0.0; + return __builtin_nan(""); } while (*s >= '0' && *s <= '9') { has_digits = 1; - val = val * 10.0 + (double)(*s - '0'); + if (val < 1e18) + val = val * 10.0 + (double)(*s - '0'); + else + frac_exp++; s++; } @@ -115,20 +130,24 @@ __attribute__((weak)) double strtod(const char *nptr, char **endptr) s++; while (*s >= '0' && *s <= '9') { has_digits = 1; - div *= 10.0; - frac = frac * 10.0 + (double)(*s - '0'); + if (val < 1e18) { + val = val * 10.0 + (double)(*s - '0'); + frac_exp--; + } s++; } - val += frac / div; } - if (has_digits && (*s == 'e' || *s == 'E')) { + if (!has_digits) { + if (endptr) + *endptr = (char *)nptr; + return 0.0; + } + + if (*s == 'e' || *s == 'E') { const char *es = s + 1; int exp_sign = 1; - int exp_val = 0; int exp_digits = 0; - double p = 1.0; - int i; if (*es == '-') { exp_sign = -1; @@ -138,24 +157,44 @@ __attribute__((weak)) double strtod(const char *nptr, char **endptr) } while (*es >= '0' && *es <= '9') { exp_digits = 1; - exp_val = exp_val * 10 + (*es - '0'); + if (exp_val < 10000) + exp_val = exp_val * 10 + (*es - '0'); es++; } if (exp_digits) { - for (i = 0; i < exp_val; i++) - p *= 10.0; - if (exp_sign < 0) - val /= p; - else - val *= p; + exp_val *= exp_sign; s = es; + } else { + exp_val = 0; } } - if (!has_digits) { - if (endptr) - *endptr = (char *)nptr; - return 0.0; + exp_val += frac_exp; + if (val != 0.0) { + while (exp_val >= 8) { + val *= 1e8; + if (val == __builtin_inf()) + break; + exp_val -= 8; + } + while (exp_val > 0) { + val *= 10.0; + if (val == __builtin_inf()) + break; + exp_val--; + } + while (exp_val <= -8) { + val *= 1e-8; + if (val == 0.0) + break; + exp_val += 8; + } + while (exp_val < 0) { + val *= 0.1; + if (val == 0.0) + break; + exp_val++; + } } if (endptr) diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-convert.c b/src/audio/ffmpeg_dec/ffmpeg_dec-convert.c index a499efb62a91..6b69cb5deae9 100644 --- a/src/audio/ffmpeg_dec/ffmpeg_dec-convert.c +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-convert.c @@ -5,6 +5,7 @@ // Fast-path PCM format conversion and interleaving for ffmpeg_dec. #include +#include #include #include #include @@ -12,24 +13,6 @@ #include #include "ffmpeg_dec-convert.h" -#if defined(__XTENSA__) && defined(__has_include) -#if __has_include() -#include -#endif - -#if defined(__has_builtin) && __has_builtin(__builtin_xtensa_trunc_sx2) && \ - defined(XCHAL_HAVE_HIFI4_VFPU) && XCHAL_HAVE_HIFI4_VFPU && \ - __has_include() -#include -#define FFMPEG_DEC_CONV_VFPU 1 -#endif - -#if defined(XCHAL_HAVE_HIFI3) && XCHAL_HAVE_HIFI3 && __has_include() -#include -#define FFMPEG_DEC_CONV_HIFI3 1 -#endif -#endif - /* * Branch-free soft-float-free float -> Q1.31 for f in [-1.0, 1.0). * @@ -62,67 +45,75 @@ static inline int32_t f32_to_q31_scalar(float f) static inline int16_t f32_to_s16_scalar(float f) { int32_t q = f32_to_q31_scalar(f); - return (int16_t)(q >> 16); + + return (int16_t)Q_SHIFT(q, 31, 15); +} + +static inline int32_t f64_to_q31_scalar(double d) +{ + if (d >= 1.0) + return INT32_MAX; + if (d <= -1.0) + return INT32_MIN; + return sat_int32((int64_t)(d * (double)((int64_t)1 << 31))); } /* ------------------------------------------------------------------------- - * Fast Path: FLTP (Planar Float) -> S32_LE + * Fast Path: FLTP (Planar Float) -> S32_LE / S24_4LE * ------------------------------------------------------------------------- */ -static void conv_fltp_to_s32_stereo(const AVFrame *frame, int32_t *dst, int nb_samples) +static void conv_fltp_to_s32(const AVFrame *frame, int32_t *dst, + int channels, int nb_samples) { - const float *c0 = (const float *)frame->data[0]; - const float *c1 = (const float *)frame->data[1]; - int i = 0; + const float *const *src = (const float *const *)frame->data; + int i = ffmpeg_hifi_fltp_to_s32(src, dst, channels, nb_samples); + + if (channels == 1) { + const float *c0 = src[0]; -#if defined(FFMPEG_DEC_CONV_VFPU) - for (; i + 2 <= nb_samples; i += 2) { - ae_int32x2 f0 = AE_L32X2_I((const ae_int32x2 *)&c0[i], 0); - ae_int32x2 f1 = AE_L32X2_I((const ae_int32x2 *)&c1[i], 0); - ae_int32x2 q0 = XT_TRUNC_SX2((ae_xtfloatx2)f0, 31); - ae_int32x2 q1 = XT_TRUNC_SX2((ae_xtfloatx2)f1, 31); - int32_t tmp0[2] __attribute__((aligned(8))); - int32_t tmp1[2] __attribute__((aligned(8))); - - AE_S32X2_I(q0, tmp0, 0); - AE_S32X2_I(q1, tmp1, 0); - - dst[2 * i + 0] = tmp0[0]; - dst[2 * i + 1] = tmp1[0]; - dst[2 * i + 2] = tmp0[1]; - dst[2 * i + 3] = tmp1[1]; + for (; i < nb_samples; i++) + dst[i] = f32_to_q31_scalar(c0[i]); + return; } -#else - for (; i + 4 <= nb_samples; i += 4) { - dst[2 * i + 0] = f32_to_q31_scalar(c0[i + 0]); - dst[2 * i + 1] = f32_to_q31_scalar(c1[i + 0]); - dst[2 * i + 2] = f32_to_q31_scalar(c0[i + 1]); - dst[2 * i + 3] = f32_to_q31_scalar(c1[i + 1]); - dst[2 * i + 4] = f32_to_q31_scalar(c0[i + 2]); - dst[2 * i + 5] = f32_to_q31_scalar(c1[i + 2]); - dst[2 * i + 6] = f32_to_q31_scalar(c0[i + 3]); - dst[2 * i + 7] = f32_to_q31_scalar(c1[i + 3]); + + if (channels == 2) { + const float *c0 = src[0]; + const float *c1 = src[1]; + + for (; i + 4 <= nb_samples; i += 4) { + dst[2 * i + 0] = f32_to_q31_scalar(c0[i + 0]); + dst[2 * i + 1] = f32_to_q31_scalar(c1[i + 0]); + dst[2 * i + 2] = f32_to_q31_scalar(c0[i + 1]); + dst[2 * i + 3] = f32_to_q31_scalar(c1[i + 1]); + dst[2 * i + 4] = f32_to_q31_scalar(c0[i + 2]); + dst[2 * i + 5] = f32_to_q31_scalar(c1[i + 2]); + dst[2 * i + 6] = f32_to_q31_scalar(c0[i + 3]); + dst[2 * i + 7] = f32_to_q31_scalar(c1[i + 3]); + } + for (; i < nb_samples; i++) { + dst[2 * i + 0] = f32_to_q31_scalar(c0[i]); + dst[2 * i + 1] = f32_to_q31_scalar(c1[i]); + } + return; } -#endif + for (; i < nb_samples; i++) { - dst[2 * i + 0] = f32_to_q31_scalar(c0[i]); - dst[2 * i + 1] = f32_to_q31_scalar(c1[i]); + for (int c = 0; c < channels; c++) + dst[i * channels + c] = f32_to_q31_scalar(src[c][i]); } } -static void conv_fltp_to_s32_mono(const AVFrame *frame, int32_t *dst, int nb_samples) +static void conv_fltp_to_s24(const AVFrame *frame, int32_t *dst, + int channels, int nb_samples) { - const float *c0 = (const float *)frame->data[0]; - int i = 0; + const float *const *src = (const float *const *)frame->data; + + for (int i = 0; i < nb_samples; i++) { + for (int c = 0; c < channels; c++) { + int32_t q = f32_to_q31_scalar(src[c][i]); -#if defined(FFMPEG_DEC_CONV_VFPU) - for (; i + 2 <= nb_samples; i += 2) { - ae_int32x2 f = AE_L32X2_I((const ae_int32x2 *)&c0[i], 0); - ae_int32x2 q = XT_TRUNC_SX2((ae_xtfloatx2)f, 31); - AE_S32X2_I(q, (ae_int32x2 *)&dst[i], 0); + dst[i * channels + c] = Q_SHIFT(q, 31, 23); + } } -#endif - for (; i < nb_samples; i++) - dst[i] = f32_to_q31_scalar(c0[i]); } /* ------------------------------------------------------------------------- @@ -153,9 +144,8 @@ static void conv_fltp_to_s16_stereo(const AVFrame *frame, int16_t *dst, int nb_s static void conv_fltp_to_s16_mono(const AVFrame *frame, int16_t *dst, int nb_samples) { const float *c0 = (const float *)frame->data[0]; - int i = 0; - for (; i < nb_samples; i++) + for (int i = 0; i < nb_samples; i++) dst[i] = f32_to_s16_scalar(c0[i]); } @@ -193,28 +183,19 @@ static void conv_s16p_to_s32_stereo(const AVFrame *frame, int32_t *dst, int nb_s const int16_t *c1 = (const int16_t *)frame->data[1]; int i = 0; -#if defined(FFMPEG_DEC_CONV_HIFI3) - for (; i + 2 <= nb_samples; i += 2) { - dst[2 * i + 0] = (int32_t)c0[i + 0] << 16; - dst[2 * i + 1] = (int32_t)c1[i + 0] << 16; - dst[2 * i + 2] = (int32_t)c0[i + 1] << 16; - dst[2 * i + 3] = (int32_t)c1[i + 1] << 16; - } -#else for (; i + 4 <= nb_samples; i += 4) { - dst[2 * i + 0] = (int32_t)c0[i + 0] << 16; - dst[2 * i + 1] = (int32_t)c1[i + 0] << 16; - dst[2 * i + 2] = (int32_t)c0[i + 1] << 16; - dst[2 * i + 3] = (int32_t)c1[i + 1] << 16; - dst[2 * i + 4] = (int32_t)c0[i + 2] << 16; - dst[2 * i + 5] = (int32_t)c1[i + 2] << 16; - dst[2 * i + 6] = (int32_t)c0[i + 3] << 16; - dst[2 * i + 7] = (int32_t)c1[i + 3] << 16; + dst[2 * i + 0] = Q_SHIFT_LEFT((int32_t)c0[i + 0], 15, 31); + dst[2 * i + 1] = Q_SHIFT_LEFT((int32_t)c1[i + 0], 15, 31); + dst[2 * i + 2] = Q_SHIFT_LEFT((int32_t)c0[i + 1], 15, 31); + dst[2 * i + 3] = Q_SHIFT_LEFT((int32_t)c1[i + 1], 15, 31); + dst[2 * i + 4] = Q_SHIFT_LEFT((int32_t)c0[i + 2], 15, 31); + dst[2 * i + 5] = Q_SHIFT_LEFT((int32_t)c1[i + 2], 15, 31); + dst[2 * i + 6] = Q_SHIFT_LEFT((int32_t)c0[i + 3], 15, 31); + dst[2 * i + 7] = Q_SHIFT_LEFT((int32_t)c1[i + 3], 15, 31); } -#endif for (; i < nb_samples; i++) { - dst[2 * i + 0] = (int32_t)c0[i] << 16; - dst[2 * i + 1] = (int32_t)c1[i] << 16; + dst[2 * i + 0] = Q_SHIFT_LEFT((int32_t)c0[i], 15, 31); + dst[2 * i + 1] = Q_SHIFT_LEFT((int32_t)c1[i], 15, 31); } } @@ -226,13 +207,13 @@ static void conv_s16_to_s32_interleaved(const int16_t *src, int32_t *dst, int to int i = 0; for (; i + 4 <= total_samples; i += 4) { - dst[i + 0] = (int32_t)src[i + 0] << 16; - dst[i + 1] = (int32_t)src[i + 1] << 16; - dst[i + 2] = (int32_t)src[i + 2] << 16; - dst[i + 3] = (int32_t)src[i + 3] << 16; + dst[i + 0] = Q_SHIFT_LEFT((int32_t)src[i + 0], 15, 31); + dst[i + 1] = Q_SHIFT_LEFT((int32_t)src[i + 1], 15, 31); + dst[i + 2] = Q_SHIFT_LEFT((int32_t)src[i + 2], 15, 31); + dst[i + 3] = Q_SHIFT_LEFT((int32_t)src[i + 3], 15, 31); } for (; i < total_samples; i++) - dst[i] = (int32_t)src[i] << 16; + dst[i] = Q_SHIFT_LEFT((int32_t)src[i], 15, 31); } /* ------------------------------------------------------------------------- @@ -242,19 +223,8 @@ static void conv_s32p_to_s32_stereo(const AVFrame *frame, int32_t *dst, int nb_s { const int32_t *c0 = (const int32_t *)frame->data[0]; const int32_t *c1 = (const int32_t *)frame->data[1]; - int i = 0; - -#if defined(FFMPEG_DEC_CONV_HIFI3) - for (; i + 2 <= nb_samples; i += 2) { - ae_int32x2 v0 = AE_L32X2_I((const ae_int32x2 *)&c0[i], 0); - ae_int32x2 v1 = AE_L32X2_I((const ae_int32x2 *)&c1[i], 0); - ae_int32x2 pair0 = AE_SEL32_LL(v0, v1); - ae_int32x2 pair1 = AE_SEL32_HH(v0, v1); + int i = ffmpeg_hifi_s32p_to_s32_stereo(c0, c1, dst, nb_samples); - AE_S32X2_I(pair0, (ae_int32x2 *)&dst[2 * i + 0], 0); - AE_S32X2_I(pair1, (ae_int32x2 *)&dst[2 * i + 2], 0); - } -#else for (; i + 4 <= nb_samples; i += 4) { dst[2 * i + 0] = c0[i + 0]; dst[2 * i + 1] = c1[i + 0]; @@ -265,7 +235,6 @@ static void conv_s32p_to_s32_stereo(const AVFrame *frame, int32_t *dst, int nb_s dst[2 * i + 6] = c0[i + 3]; dst[2 * i + 7] = c1[i + 3]; } -#endif for (; i < nb_samples; i++) { dst[2 * i + 0] = c0[i]; dst[2 * i + 1] = c1[i]; @@ -273,23 +242,26 @@ static void conv_s32p_to_s32_stereo(const AVFrame *frame, int32_t *dst, int nb_s } /* ------------------------------------------------------------------------- - * Fallback: Generic Sample-by-Sample Conversion + * Fallback: Generic Sample-by-Sample Conversion (AVFrame -> SOF PCM) * ------------------------------------------------------------------------- */ static inline int32_t src_sample_to_q31(const uint8_t *p, int fmt) { switch (fmt) { case AV_SAMPLE_FMT_U8: case AV_SAMPLE_FMT_U8P: - return ((int32_t)*p - 0x80) << 24; + return Q_SHIFT_LEFT((int32_t)*p - 0x80, 7, 31); case AV_SAMPLE_FMT_S16: case AV_SAMPLE_FMT_S16P: - return (int32_t)*(const int16_t *)p << 16; + return Q_SHIFT_LEFT((int32_t)*(const int16_t *)p, 15, 31); case AV_SAMPLE_FMT_S32: case AV_SAMPLE_FMT_S32P: return *(const int32_t *)p; case AV_SAMPLE_FMT_FLT: case AV_SAMPLE_FMT_FLTP: return f32_to_q31_scalar(*(const float *)p); + case AV_SAMPLE_FMT_DBL: + case AV_SAMPLE_FMT_DBLP: + return f64_to_q31_scalar(*(const double *)p); default: return 0; } @@ -299,13 +271,13 @@ static inline void q31_to_sink_sample(uint8_t *p, int32_t v, int fmt) { switch (fmt) { case SOF_IPC_FRAME_S16_LE: - *(int16_t *)p = (int16_t)(v >> 16); + *(int16_t *)p = (int16_t)Q_SHIFT(v, 31, 15); break; case SOF_IPC_FRAME_S24_4LE: - *(int32_t *)p = v >> 8; + *(int32_t *)p = Q_SHIFT(v, 31, 23); break; case SOF_IPC_FRAME_FLOAT: - *(float *)p = (float)v / 2147483648.0f; + *(float *)p = Q_CONVERT_QTOF(v, 31); break; case SOF_IPC_FRAME_S32_LE: default: @@ -335,7 +307,146 @@ static void conv_generic(const AVFrame *frame, uint8_t *out, int sink_fmt, int c } /* ------------------------------------------------------------------------- - * Top-Level Dispatcher + * Input Conversion: SOF Interleaved PCM -> AVFrame (FLTP / DBL) + * ------------------------------------------------------------------------- */ +static void conv_s16_to_fltp(const int16_t *src, float *const *dst, + int channels, int nb_samples) +{ + int i = 0; + + if (channels == 2) { + float *dst0 = dst[0]; + float *dst1 = dst[1]; + + for (; i + 4 <= nb_samples; i += 4) { + dst0[i + 0] = Q_CONVERT_QTOF(src[(i + 0) * 2 + 0], 15); + dst1[i + 0] = Q_CONVERT_QTOF(src[(i + 0) * 2 + 1], 15); + dst0[i + 1] = Q_CONVERT_QTOF(src[(i + 1) * 2 + 0], 15); + dst1[i + 1] = Q_CONVERT_QTOF(src[(i + 1) * 2 + 1], 15); + dst0[i + 2] = Q_CONVERT_QTOF(src[(i + 2) * 2 + 0], 15); + dst1[i + 2] = Q_CONVERT_QTOF(src[(i + 2) * 2 + 1], 15); + dst0[i + 3] = Q_CONVERT_QTOF(src[(i + 3) * 2 + 0], 15); + dst1[i + 3] = Q_CONVERT_QTOF(src[(i + 3) * 2 + 1], 15); + } + for (; i < nb_samples; i++) { + dst0[i] = Q_CONVERT_QTOF(src[i * 2 + 0], 15); + dst1[i] = Q_CONVERT_QTOF(src[i * 2 + 1], 15); + } + return; + } + + for (; i < nb_samples; i++) { + for (int c = 0; c < channels; c++) + dst[c][i] = Q_CONVERT_QTOF(src[i * channels + c], 15); + } +} + +static void conv_s24_to_fltp(const int32_t *src, float *const *dst, + int channels, int nb_samples) +{ + for (int i = 0; i < nb_samples; i++) { + for (int c = 0; c < channels; c++) { + int32_t s = sign_extend_s24(src[i * channels + c]); + + dst[c][i] = Q_CONVERT_QTOF(s, 23); + } + } +} + +static void conv_s32_to_fltp(const int32_t *src, float *const *dst, + int channels, int nb_samples) +{ + int i = ffmpeg_hifi_s32_to_fltp(src, dst, channels, nb_samples); + + if (channels == 2) { + float *dst0 = dst[0]; + float *dst1 = dst[1]; + + for (; i + 4 <= nb_samples; i += 4) { + dst0[i + 0] = Q_CONVERT_QTOF(src[(i + 0) * 2 + 0], 31); + dst1[i + 0] = Q_CONVERT_QTOF(src[(i + 0) * 2 + 1], 31); + dst0[i + 1] = Q_CONVERT_QTOF(src[(i + 1) * 2 + 0], 31); + dst1[i + 1] = Q_CONVERT_QTOF(src[(i + 1) * 2 + 1], 31); + dst0[i + 2] = Q_CONVERT_QTOF(src[(i + 2) * 2 + 0], 31); + dst1[i + 2] = Q_CONVERT_QTOF(src[(i + 2) * 2 + 1], 31); + dst0[i + 3] = Q_CONVERT_QTOF(src[(i + 3) * 2 + 0], 31); + dst1[i + 3] = Q_CONVERT_QTOF(src[(i + 3) * 2 + 1], 31); + } + for (; i < nb_samples; i++) { + dst0[i] = Q_CONVERT_QTOF(src[i * 2 + 0], 31); + dst1[i] = Q_CONVERT_QTOF(src[i * 2 + 1], 31); + } + return; + } + + for (; i < nb_samples; i++) { + for (int c = 0; c < channels; c++) + dst[c][i] = Q_CONVERT_QTOF(src[i * channels + c], 31); + } +} + +int ffmpeg_dec_pcm_to_frame(const uint8_t *in, AVFrame *frame, + int src_fmt, int channels, int nb_samples) +{ + if (!in || !frame || channels <= 0 || nb_samples <= 0) + return -EINVAL; + if (frame->ch_layout.nb_channels != channels) + return -EINVAL; + + if (frame->format == AV_SAMPLE_FMT_DBL) { + double *dst = (double *)frame->data[0]; + int total = nb_samples * channels; + + switch (src_fmt) { + case SOF_IPC_FRAME_S16_LE: { + const int16_t *src16 = (const int16_t *)in; + + for (int i = 0; i < total; i++) + dst[i] = Q_CONVERT_QTOD(src16[i], 15); + return 0; + } + case SOF_IPC_FRAME_S24_4LE: { + const int32_t *src24 = (const int32_t *)in; + + for (int i = 0; i < total; i++) + dst[i] = Q_CONVERT_QTOD(sign_extend_s24(src24[i]), 23); + return 0; + } + case SOF_IPC_FRAME_S32_LE: { + const int32_t *src32 = (const int32_t *)in; + + for (int i = 0; i < total; i++) + dst[i] = Q_CONVERT_QTOD(src32[i], 31); + return 0; + } + default: + return -EINVAL; + } + } + + if (frame->format == AV_SAMPLE_FMT_FLTP) { + float *const *dst = (float *const *)frame->data; + + switch (src_fmt) { + case SOF_IPC_FRAME_S16_LE: + conv_s16_to_fltp((const int16_t *)in, dst, channels, nb_samples); + return 0; + case SOF_IPC_FRAME_S24_4LE: + conv_s24_to_fltp((const int32_t *)in, dst, channels, nb_samples); + return 0; + case SOF_IPC_FRAME_S32_LE: + conv_s32_to_fltp((const int32_t *)in, dst, channels, nb_samples); + return 0; + default: + return -EINVAL; + } + } + + return -EINVAL; +} + +/* ------------------------------------------------------------------------- + * Top-Level Output Dispatcher: AVFrame -> SOF Interleaved PCM * ------------------------------------------------------------------------- */ int ffmpeg_dec_convert_frame(const AVFrame *frame, uint8_t *out, size_t out_size, int sink_fmt, int channels) @@ -368,18 +479,25 @@ int ffmpeg_dec_convert_frame(const AVFrame *frame, uint8_t *out, size_t out_size } } - /* 2. Stereo Fast Paths (the dominant audio decoding configuration) */ + /* 2. Planar Float (FLTP) -> S32_LE / S24_4LE for any channel count */ + if (frame->format == AV_SAMPLE_FMT_FLTP) { + if (sink_fmt == SOF_IPC_FRAME_S32_LE) { + conv_fltp_to_s32(frame, (int32_t *)out, channels, frame->nb_samples); + return (int)need; + } + if (sink_fmt == SOF_IPC_FRAME_S24_4LE) { + conv_fltp_to_s24(frame, (int32_t *)out, channels, frame->nb_samples); + return (int)need; + } + } + + /* 3. Stereo Fast Paths (the dominant audio decoding configuration) */ if (channels == 2) { - /* AAC / Opus (FLTP) */ - if (frame->format == AV_SAMPLE_FMT_FLTP) { - if (sink_fmt == SOF_IPC_FRAME_S32_LE) { - conv_fltp_to_s32_stereo(frame, (int32_t *)out, frame->nb_samples); - return (int)need; - } - if (sink_fmt == SOF_IPC_FRAME_S16_LE) { - conv_fltp_to_s16_stereo(frame, (int16_t *)out, frame->nb_samples); - return (int)need; - } + /* AAC / Opus / afftdn (FLTP) -> S16_LE */ + if (frame->format == AV_SAMPLE_FMT_FLTP && + sink_fmt == SOF_IPC_FRAME_S16_LE) { + conv_fltp_to_s16_stereo(frame, (int16_t *)out, frame->nb_samples); + return (int)need; } /* MP3 / FLAC (S16P) */ @@ -409,11 +527,11 @@ int ffmpeg_dec_convert_frame(const AVFrame *frame, uint8_t *out, size_t out_size } } - /* 3. Mono Fast Paths */ + /* 4. Mono Fast Paths */ if (channels == 1) { if (frame->format == AV_SAMPLE_FMT_FLTP || frame->format == AV_SAMPLE_FMT_FLT) { if (sink_fmt == SOF_IPC_FRAME_S32_LE) { - conv_fltp_to_s32_mono(frame, (int32_t *)out, frame->nb_samples); + conv_fltp_to_s32(frame, (int32_t *)out, 1, frame->nb_samples); return (int)need; } if (sink_fmt == SOF_IPC_FRAME_S16_LE) { @@ -430,7 +548,7 @@ int ffmpeg_dec_convert_frame(const AVFrame *frame, uint8_t *out, size_t out_size } } - /* 4. General Fallback */ + /* 5. General Fallback */ conv_generic(frame, out, sink_fmt, channels); return (int)need; } diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-convert.h b/src/audio/ffmpeg_dec/ffmpeg_dec-convert.h index 9ba02adc8911..8571f8475272 100644 --- a/src/audio/ffmpeg_dec/ffmpeg_dec-convert.h +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-convert.h @@ -13,10 +13,10 @@ /** * ffmpeg_dec_convert_frame - Convert and interleave an AVFrame into a SOF sink buffer. - * @frame: Decoded libavcodec AVFrame. + * @frame: Decoded or filtered libav* AVFrame. * @out: Destination buffer. * @out_size: Capacity of destination buffer in bytes. - * @sink_fmt: SOF IPC frame format (SOF_IPC_FRAME_*). + * @sink_fmt: SOF valid sample format (SOF_IPC_FRAME_*). * @channels: Output channel count. * * Dispatches to specialized HiFi/SIMD and unrolled scalar conversion routines @@ -28,4 +28,25 @@ int ffmpeg_dec_convert_frame(const AVFrame *frame, uint8_t *out, size_t out_size, int sink_fmt, int channels); +/** + * ffmpeg_dec_pcm_to_frame - Convert interleaved SOF PCM into an AVFrame. + * @in: Source interleaved PCM buffer. + * @frame: Destination pre-allocated AVFrame (AV_SAMPLE_FMT_FLTP or AV_SAMPLE_FMT_DBL). + * @src_fmt: SOF valid sample format (SOF_IPC_FRAME_S16_LE / S24_4LE / S32_LE). + * @channels: Input channel count. + * @nb_samples: Number of audio frames (samples per channel) to convert. + * + * Returns 0 on success or negative errno on invalid parameters. + */ +int ffmpeg_dec_pcm_to_frame(const uint8_t *in, AVFrame *frame, + int src_fmt, int channels, int nb_samples); + +/* HiFi SIMD helpers implemented in ffmpeg-convert-hifi.c */ +int ffmpeg_hifi_fltp_to_s32(const float *const *src, int32_t *dst, + int channels, int nb_samples); +int ffmpeg_hifi_s32_to_fltp(const int32_t *src, float *const *dst, + int channels, int nb_samples); +int ffmpeg_hifi_s32p_to_s32_stereo(const int32_t *c0, const int32_t *c1, + int32_t *dst, int nb_samples); + #endif /* __SOF_AUDIO_FFMPEG_DEC_CONVERT_H__ */ diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c b/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c index c57c621cc14d..11770a2f031c 100644 --- a/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c @@ -16,7 +16,10 @@ // a compact real implementation. // * str/mem helpers not exported by the core (memchr, strchr, ...): real. // * bsearch, __bswap*: real. -// * strto* : not on the FLAC decode path -> return-0 stubs. +// * strto* : Zephyr's CONFIG_MINIMAL_LIBC omits strtod/strtof and an +// isolated LLEXT module cannot link the toolchain's non-PIC libc.a. +// libavfilter's av_opt_set_from_string() calls strtod/strtol to parse +// floating-point, integer, and boolean filter argument strings. // * libm (pow/sin/sqrt/...): FLAC is lossless integer and never calls these; // SOF's fixed-point math is range-limited (sqrt 0-2, exp +/-8, 16-bit sin) so // it cannot serve as a double libm. These are RESOLVE-ONLY STUBS that let the @@ -179,18 +182,224 @@ uint64_t __bswapdi2(uint64_t x) __bswapsi2((uint32_t)(x >> 32)); } -/* ============================ strto* (unused stubs) ======================= */ +/* ============================ strto* ===================================== */ +/* + * Zephyr's CONFIG_MINIMAL_LIBC omits strtod/strtof (only strtol/strtoul exist + * in zephyr/lib/libc/minimal/source/stdlib/), and an LLEXT module cannot link + * the toolchain's non-PIC newlib libc.a or import strtod from the base image. + * libavfilter's option parser (av_strtod) requires strtod to parse filter + * parameter strings such as "nr=28:nf=-20" and "limit=0.95:attack=5". + */ double strtod(const char *nptr, char **endptr) -{ if (endptr) *endptr = (char *)nptr; return 0; } -long strtol(const char *nptr, char **endptr, int base) -{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } +{ + const char *s = nptr; + double val = 0.0; + double sign = 1.0; + int has_digits = 0; + int frac_exp = 0; + int exp_val = 0; + + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') + s++; + + if (*s == '-') { + sign = -1.0; + s++; + } else if (*s == '+') { + s++; + } + + if ((s[0] == 'i' || s[0] == 'I') && + (s[1] == 'n' || s[1] == 'N') && + (s[2] == 'f' || s[2] == 'F')) { + s += 3; + if (endptr) + *endptr = (char *)s; + return sign * __builtin_inf(); + } + if ((s[0] == 'n' || s[0] == 'N') && + (s[1] == 'a' || s[1] == 'A') && + (s[2] == 'n' || s[2] == 'N')) { + s += 3; + if (endptr) + *endptr = (char *)s; + return __builtin_nan(""); + } + + while (*s >= '0' && *s <= '9') { + has_digits = 1; + if (val < 1e18) + val = val * 10.0 + (double)(*s - '0'); + else + frac_exp++; + s++; + } + + if (*s == '.') { + s++; + while (*s >= '0' && *s <= '9') { + has_digits = 1; + if (val < 1e18) { + val = val * 10.0 + (double)(*s - '0'); + frac_exp--; + } + s++; + } + } + + if (!has_digits) { + if (endptr) + *endptr = (char *)nptr; + return 0.0; + } + + if (*s == 'e' || *s == 'E') { + const char *es = s + 1; + int exp_sign = 1; + int exp_digits = 0; + + if (*es == '-') { + exp_sign = -1; + es++; + } else if (*es == '+') { + es++; + } + while (*es >= '0' && *es <= '9') { + exp_digits = 1; + if (exp_val < 10000) + exp_val = exp_val * 10 + (*es - '0'); + es++; + } + if (exp_digits) { + exp_val *= exp_sign; + s = es; + } else { + exp_val = 0; + } + } + + exp_val += frac_exp; + if (val != 0.0) { + while (exp_val >= 8) { + val *= 1e8; + if (val == __builtin_inf()) + break; + exp_val -= 8; + } + while (exp_val > 0) { + val *= 10.0; + if (val == __builtin_inf()) + break; + exp_val--; + } + while (exp_val <= -8) { + val *= 1e-8; + if (val == 0.0) + break; + exp_val += 8; + } + while (exp_val < 0) { + val *= 0.1; + if (val == 0.0) + break; + exp_val++; + } + } + + if (endptr) + *endptr = (char *)s; + + return sign * val; +} + +float strtof(const char *nptr, char **endptr) +{ + return (float)strtod(nptr, endptr); +} + +static unsigned long long ffmpeg_dec_strtoull(const char *nptr, char **endptr, + int base, int *is_neg) +{ + const char *s = nptr; + unsigned long long acc = 0; + int neg = 0, any = 0, d; + + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') + s++; + + if (*s == '-') { + neg = 1; + s++; + } else if (*s == '+') { + s++; + } + + if (base < 0 || base == 1 || base > 36) { + if (endptr) + *endptr = (char *)nptr; + if (is_neg) + *is_neg = 0; + return 0; + } + + if ((base == 0 || base == 16) && s[0] == '0' && (s[1] == 'x' || s[1] == 'X') && + ((s[2] >= '0' && s[2] <= '9') || + (s[2] >= 'a' && s[2] <= 'f') || + (s[2] >= 'A' && s[2] <= 'F'))) { + s += 2; + base = 16; + } + if (base == 0) + base = (*s == '0') ? 8 : 10; + + for (;; s++) { + if (*s >= '0' && *s <= '9') + d = *s - '0'; + else if (*s >= 'a' && *s <= 'z') + d = *s - 'a' + 10; + else if (*s >= 'A' && *s <= 'Z') + d = *s - 'A' + 10; + else + break; + if (d >= base) + break; + any = 1; + acc = acc * (unsigned long long)base + (unsigned long long)d; + } + + if (endptr) + *endptr = (char *)(any ? s : nptr); + if (is_neg) + *is_neg = any ? neg : 0; + return any ? acc : 0; +} + +unsigned long long strtoull(const char *nptr, char **endptr, int base) +{ + int neg = 0; + unsigned long long uv = ffmpeg_dec_strtoull(nptr, endptr, base, &neg); + + return neg ? -uv : uv; +} + long long strtoll(const char *nptr, char **endptr, int base) -{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } +{ + int neg = 0; + unsigned long long uv = ffmpeg_dec_strtoull(nptr, endptr, base, &neg); + + return neg ? -(long long)uv : (long long)uv; +} + +long strtol(const char *nptr, char **endptr, int base) +{ + return (long)strtoll(nptr, endptr, base); +} + unsigned long strtoul(const char *nptr, char **endptr, int base) -{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } -unsigned long long strtoull(const char *nptr, char **endptr, int base) -{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } +{ + return (unsigned long)strtoull(nptr, endptr, base); +} /* ============================ errno / libm =============================== */ /* diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec.c b/src/audio/ffmpeg_dec/ffmpeg_dec.c index 445909ad0f35..3411e058ddf5 100644 --- a/src/audio/ffmpeg_dec/ffmpeg_dec.c +++ b/src/audio/ffmpeg_dec/ffmpeg_dec.c @@ -290,6 +290,7 @@ __cold static int ffmpeg_dec_init(struct processing_module *mod) if (mod->priv.cfg.ext_data && mod->priv.cfg.ext_data->module_data && mod->priv.cfg.ext_data->module_data_size >= sizeof(struct snd_codec)) { + const struct snd_codec *codec_params = mod->priv.cfg.ext_data->module_data; const uint8_t *init_bytes = mod->priv.cfg.ext_data->module_data; uint32_t direction = SOF_IPC_STREAM_PLAYBACK; @@ -311,6 +312,18 @@ __cold static int ffmpeg_dec_init(struct processing_module *mod) comp_err(dev, "invalid compress stream direction %u", direction); return -EINVAL; } + if (codec_params->id == SND_AUDIOCODEC_PCM) { +#if defined(CONFIG_FFMPEG_FILTER) + return ffmpeg_af_mod_init(mod); +#else + comp_err(dev, "PCM filter requested but no filters enabled"); + return -ENOTSUP; +#endif + } +#if defined(CONFIG_FFMPEG_FILTER) + } else { + return ffmpeg_af_mod_init(mod); +#endif } #endif @@ -428,6 +441,11 @@ static int ffmpeg_dec_prepare(struct processing_module *mod, return ffmpeg_enc_mod_prepare(mod, sources, num_of_sources, sinks, num_of_sinks); #endif +#if defined(CONFIG_FFMPEG_FILTER) + if (cd && cd->is_filter) + return ffmpeg_af_mod_prepare(mod, sources, num_of_sources, + sinks, num_of_sinks); +#endif /* * The PCM format the decoder must produce is that of the sink. @@ -542,6 +560,11 @@ static bool ffmpeg_dec_is_ready_to_process(struct processing_module *mod, return ffmpeg_enc_is_ready_to_process(mod, sources, num_of_sources, sinks, num_of_sinks); #endif +#if defined(CONFIG_FFMPEG_FILTER) + if (cd && cd->is_filter) + return ffmpeg_af_is_ready_to_process(mod, sources, num_of_sources, + sinks, num_of_sinks); +#endif if (num_of_sources < 1 || num_of_sinks < 1) return false; @@ -588,6 +611,11 @@ static int ffmpeg_dec_process(struct processing_module *mod, return ffmpeg_enc_mod_process(mod, sources, num_of_sources, sinks, num_of_sinks); #endif +#if defined(CONFIG_FFMPEG_FILTER) + if (cd && cd->is_filter) + return ffmpeg_af_mod_process(mod, sources, num_of_sources, + sinks, num_of_sinks); +#endif if (num_of_sources < 1 || num_of_sinks < 1) return -EINVAL; @@ -811,6 +839,15 @@ ffmpeg_dec_set_config(struct processing_module *mod, uint32_t config_id, return 0; } +#if defined(CONFIG_FFMPEG_FILTER) + { + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + if (cd && cd->is_filter) + return ffmpeg_af_mod_set_config(mod, config->data, config->size); + } +#endif + return ffmpeg_dec_store_extradata(mod, config->data, config->size); } @@ -832,6 +869,10 @@ static int ffmpeg_dec_reset(struct processing_module *mod) if (cd && cd->is_encoder) return ffmpeg_enc_mod_reset(mod); #endif +#if defined(CONFIG_FFMPEG_FILTER) + if (cd && cd->is_filter) + return ffmpeg_af_mod_reset(mod); +#endif /* Discard any PCM staged mid-stream; the pipeline is restarting. */ cd->pcm_avail = 0; @@ -869,6 +910,10 @@ __cold static int ffmpeg_dec_free(struct processing_module *mod) if (cd && cd->is_encoder) return ffmpeg_enc_mod_free(mod); #endif +#if defined(CONFIG_FFMPEG_FILTER) + if (cd && cd->is_filter) + return ffmpeg_af_mod_free(mod); +#endif if (cd->backend->free) cd->backend->free(mod); diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec.h b/src/audio/ffmpeg_dec/ffmpeg_dec.h index a01e7236c893..c05dfdf207c5 100644 --- a/src/audio/ffmpeg_dec/ffmpeg_dec.h +++ b/src/audio/ffmpeg_dec/ffmpeg_dec.h @@ -89,6 +89,25 @@ bool ffmpeg_enc_is_ready_to_process(struct processing_module *mod, int ffmpeg_enc_mod_reset(struct processing_module *mod); int ffmpeg_enc_mod_free(struct processing_module *mod); +/* Module-interface entry points for filter mode (CONFIG_FFMPEG_FILTER): + * PCM in -> libavfilter graph -> PCM out (ffmpeg-filter.c), + * selected at runtime for SND_AUDIOCODEC_PCM or PCM pipelines. + */ +int ffmpeg_af_mod_init(struct processing_module *mod); +int ffmpeg_af_mod_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks); +int ffmpeg_af_mod_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks); +bool ffmpeg_af_is_ready_to_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks); +int ffmpeg_af_mod_set_config(struct processing_module *mod, + const uint8_t *data, size_t size); +int ffmpeg_af_mod_reset(struct processing_module *mod); +int ffmpeg_af_mod_free(struct processing_module *mod); + /** * struct ffmpeg_enc_codec_ops - Per-codec encoder operations. * @id: SOF internal codec enum (enum ffmpeg_dec_codec). @@ -256,6 +275,7 @@ struct ffmpeg_dec_comp_data { int out_frame_fmt; uint32_t out_frame_bytes; bool is_encoder; + bool is_filter; bool configured; uint8_t *in_buf; size_t in_buf_size; diff --git a/src/audio/ffmpeg_dec/llext/CMakeLists.txt b/src/audio/ffmpeg_dec/llext/CMakeLists.txt index 4fbeae15cc4f..09fbd971c48a 100644 --- a/src/audio/ffmpeg_dec/llext/CMakeLists.txt +++ b/src/audio/ffmpeg_dec/llext/CMakeLists.txt @@ -24,6 +24,7 @@ else() ../ffmpeg_dec.c ../ffmpeg_dec-ffmpeg.c ../ffmpeg_dec-convert.c + ../ffmpeg-convert-hifi.c ../ffmpeg_dec-shims.c ../ffmpeg_dec-alloc.c) if(CONFIG_FFMPEG_DEC_FLAC) @@ -46,12 +47,10 @@ else() # libavfilter into the link; it must precede the libs it depends on. set(ffmpeg_dec_libs avcodec swresample avutil) set(ffmpeg_dec_libpaths "${FFMPEG_INSTALL_DIR}/lib") - # Real libm (newlib double math) so lossy codecs (MP3/AAC/Opus) decode correctly - get_filename_component(_ff_libm_dir "${FFMPEG_TARGET_LIBM}" DIRECTORY) - list(APPEND ffmpeg_dec_libpaths "${_ff_libm_dir}") - list(APPEND ffmpeg_dec_libs newlib_m) + if(CONFIG_FFMPEG_FILTER) + list(APPEND ffmpeg_dec_srcs ../ffmpeg-filter.c) + endif() if(CONFIG_FFMPEG_BUILD_AVFILTER) - list(APPEND ffmpeg_dec_srcs ../ffmpeg_dec-filter.c) set(ffmpeg_dec_libs avfilter ${ffmpeg_dec_libs}) endif() @@ -71,11 +70,22 @@ else() list(APPEND ffmpeg_dec_libpaths "${VO_AACENC_INSTALL}/lib") endif() + # Real libm (newlib double math) LAST so libav*/filter references resolve against it + get_filename_component(_ff_libm_dir "${FFMPEG_TARGET_LIBM}" DIRECTORY) + list(APPEND ffmpeg_dec_libpaths "${_ff_libm_dir}") + list(APPEND ffmpeg_dec_libs newlib_m) + # Cold split: module code must also use -mlongcalls to reach FFmpeg code # that now lives in the far (DRAM) .cold region. set(ffmpeg_dec_cflags "") if(CONFIG_FFMPEG_DEC_COLD_SPLIT) - set(ffmpeg_dec_cflags -mlongcalls) + list(APPEND ffmpeg_dec_cflags -mlongcalls) + endif() + # Disable scalar hardware FP0 on Clang targets (CONFIG_FPU=n) so float + # conversions and strtod/strtof emit soft-float libcalls instead of + # faulting FP0 instructions. + if(CMAKE_C_COMPILER_ID MATCHES "Clang") + list(APPEND ffmpeg_dec_cflags "SHELL:-Xclang -target-feature -Xclang -fp") endif() set(ffmpeg_dec_includes "${FFMPEG_INSTALL_DIR}/include") diff --git a/src/kernel/audio/base_fw.c b/src/kernel/audio/base_fw.c index 8addf4cab3e9..c1c8d69413d7 100644 --- a/src/kernel/audio/base_fw.c +++ b/src/kernel/audio/base_fw.c @@ -110,7 +110,7 @@ static void get_codec_info(struct sof_tlv **tuple) codec_info.items[codec_info.count++] = SET_CODEC_INFO_ITEM(SND_AUDIOCODEC_BESPOKE, SOF_IPC_STREAM_CAPTURE); #endif -#ifdef CONFIG_SOF_COMPRESS_CODEC_PCM_DEC +#if defined(CONFIG_SOF_COMPRESS_CODEC_PCM_DEC) || defined(CONFIG_FFMPEG_FILTER) codec_info.items[codec_info.count++] = SET_CODEC_INFO_ITEM(SND_AUDIOCODEC_PCM, SOF_IPC_STREAM_PLAYBACK); #endif