From e74f0675f29fcd61ed94860e62d79a1bffa82b67 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:13:33 +0000 Subject: [PATCH 1/9] schedule: split the kernel part of zephyr_domain.c into src/kernel/schedule zephyr_domain.c mixes the LL scheduler thread function and the parts of the timer domain that run in the LL scheduler threads with the thread and timer setup, which needs kernel mode. Move the kernel part to src/kernel/schedule/zephyr_domain-kernel.c: - the LL scheduler thread stacks and the timer interrupt handler; - domain register and unregister of builds without user-space LL, which create and abort the scheduler threads and start and stop the timer; - the user-space LL thread init and free, which also register the LL threads as SOF system call callers, the kernel-only per-core thread table and the thread lookups used for access grants; - the domain ops and zephyr_domain_init(). zephyr_domain.c keeps the thread function, the user-space LL register and unregister bookkeeping, the cross-core stream block and unblock and ll_sch_is_current(). The domain data structures and the functions the kernel part refers to are declared in the new scheduler-private header sof/schedule/zephyr_ll.h. zephyr_domain.c no longer includes the timer, userspace helper, platform and memory map headers. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/zephyr_ll.h | 61 +++ src/kernel/schedule/CMakeLists.txt | 4 + src/kernel/schedule/zephyr_domain-kernel.c | 460 ++++++++++++++++++++ src/schedule/zephyr_domain.c | 471 +-------------------- 4 files changed, 537 insertions(+), 459 deletions(-) create mode 100644 src/include/sof/schedule/zephyr_ll.h create mode 100644 src/kernel/schedule/zephyr_domain-kernel.c diff --git a/src/include/sof/schedule/zephyr_ll.h b/src/include/sof/schedule/zephyr_ll.h new file mode 100644 index 000000000000..126b07ee1879 --- /dev/null +++ b/src/include/sof/schedule/zephyr_ll.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2019-2026 Intel Corporation. + */ + +/* + * Private data of the Zephyr LL scheduler and its timer domain, shared by + * the parts in src/schedule, which also run in the LL scheduler threads, and + * the kernel parts in src/kernel/schedule. Not for use outside the scheduler. + */ + +#ifndef __SOF_SCHEDULE_ZEPHYR_LL_H__ +#define __SOF_SCHEDULE_ZEPHYR_LL_H__ + +#include +#include +#include +#include + +struct zephyr_domain_thread { + struct k_thread *ll_thread; + struct k_sem *sem; +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_thread ll_thread_obj; + struct k_sem sem_obj; +#endif + void (*handler)(void *arg); + void *arg; +}; + +struct zephyr_domain { + struct k_timer *timer; +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_timer timer_obj; +#endif + struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; + struct ll_schedule_domain *ll_domain; +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL + atomic_t block; + struct k_mutex block_mutex; + struct k_condvar block_condvar; +#endif +}; + +/* LL scheduler thread function, p1 is the domain and p2 the core */ +void zephyr_domain_thread_fn(void *p1, void *p2, void *p3); + +#ifdef CONFIG_SOF_USERSPACE_LL +int zephyr_domain_register_user(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg); +int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks); +#endif + +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL +void zephyr_domain_block(struct ll_schedule_domain *domain); +void zephyr_domain_unblock(struct ll_schedule_domain *domain); +#endif + +#endif /* __SOF_SCHEDULE_ZEPHYR_LL_H__ */ diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 277378449165..6b0d7011f409 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -2,6 +2,10 @@ is_zephyr(zephyr) if(zephyr) + zephyr_library_sources( + zephyr_domain-kernel.c + ) + zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL zephyr_ll_user.c ) diff --git a/src/kernel/schedule/zephyr_domain-kernel.c b/src/kernel/schedule/zephyr_domain-kernel.c new file mode 100644 index 000000000000..6acbb5e00904 --- /dev/null +++ b/src/kernel/schedule/zephyr_domain-kernel.c @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019-2021 Intel Corporation. All rights reserved. +// +// Author: Tomasz Lauda + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); + +/* Kernel part of the Zephyr LL scheduler timer domain: the scheduler threads, the + * timer and its interrupt handler. The thread function and the parts that run in the + * LL scheduler threads are in src/schedule/zephyr_domain.c + */ + +/* + * Currently the Zephyr clock rate is part it's Kconfig known at build time. + * SOF on Intel CAVS platforms currently only aligns with Zephyr when both + * use the CAVS 19.2 MHz SSP clock. TODO - needs runtime alignment. + */ +#if CONFIG_XTENSA && CONFIG_CAVS && !CONFIG_INTEL_ADSP_TIMER +#error "Zephyr uses 19.2MHz clock derived from SSP which must be enabled." +#endif + +#define ZEPHYR_LL_STACK_SIZE 8192 + +#if CONFIG_SOF_USERSPACE_LL +K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); +#else +K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); +#endif + +/* Timer callback: runs in timer IRQ context */ +static void zephyr_domain_timer_fn(struct k_timer *timer) +{ + struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer); + int core; + + /* + * A race is possible when the Zephyr LL scheduling domain is being + * unregistered while a timer IRQ is processed on a different core. Then + * the timer is removed by the former but then re-added by the latter, + * but this time with no user data and no handler set. This leads to the + * timer continuing to trigger and then leading to a lock up when it is + * registered again next time. + */ + if (!zephyr_domain) { + k_timer_stop(timer); + return; + } + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + if (dt->handler) + k_sem_give(dt->sem); + } +} + +/* The normal kernel-space implementation for register/unregister */ +#ifndef CONFIG_SOF_USERSPACE_LL + +static int zephyr_domain_register(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + char thread_name[] = "ll_thread0"; + k_tid_t thread; + k_spinlock_key_t key; + + tr_dbg(&ll_tr, "entry"); + + /* domain work only needs registered once on each core */ + if (dt->handler) + return 0; + + dt->handler = handler; + dt->arg = arg; + + /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ + k_sem_init(dt->sem, 0, 10); + + thread_name[sizeof(thread_name) - 2] = '0' + core; + + /* not allocated dynamically when LL in kernel space */ + dt->ll_thread = &dt->ll_thread_obj; + + thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, + zephyr_domain_thread_fn, zephyr_domain, INT_TO_POINTER(core), + NULL, CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER); + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, core); +#endif + k_thread_name_set(thread, thread_name); + + k_thread_start(thread); + + key = k_spin_lock(&domain->lock); + + if (!k_timer_user_data_get(zephyr_domain->timer)) { + k_timeout_t start = {0}; + + k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); + k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); + + k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); + + /* Enable the watchdog */ + watchdog_enable(core); + } + + k_spin_unlock(&domain->lock, key); + + tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", + domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); + + return 0; +} + +static int zephyr_domain_unregister(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + k_spinlock_key_t key; + + tr_dbg(&ll_tr, "entry"); + + /* tasks still registered on this core */ + if (num_tasks) + return 0; + + key = k_spin_lock(&domain->lock); + + if (!atomic_read(&domain->total_num_tasks)) { + /* Disable the watchdog */ + watchdog_disable(core); + + k_timer_stop(zephyr_domain->timer); + k_timer_user_data_set(zephyr_domain->timer, NULL); + } + + zephyr_domain->domain_thread[core].handler = NULL; + + k_spin_unlock(&domain->lock, key); + + tr_info(&ll_tr, "domain->type %d domain->clk %d", + domain->type, domain->clk); + + /* + * If running in the context of the domain thread, k_thread_abort() will + * not return + */ + k_thread_abort(zephyr_domain->domain_thread[core].ll_thread); + + tr_dbg(&ll_tr, "exit"); + + return 0; +} + +#else /* CONFIG_SOF_USERSPACE_LL */ + +/* + * Kernel-owned per-core LL thread table. Populated from the privileged + * domain-thread init path and consulted by z_impl_zephyr_ll_task_sem_alloc() + * so that privileged code never has to traverse the user-accessible + * scheduler/domain objects to find the LL thread of a given core. + */ +static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT]; + +/* + * Privileged thread initialization for userspace LL scheduling. + * Creates the scheduling thread, sets up timer, grants access to kernel + * objects. Must be called from kernel context before any user-space + * domain_register() calls. + */ +static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, + struct task *task) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + struct zephyr_domain_thread *dt; + char thread_name[] = "ll_thread0"; + k_tid_t thread; + int core = task->core; + + tr_dbg(&ll_tr, "entry"); + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + dt = zephyr_domain->domain_thread + core; + + /* thread only needs to be created once per core */ + if (dt->ll_thread) + return 0; + + dt->handler = NULL; + dt->arg = NULL; + + dt->sem = k_object_alloc(K_OBJ_SEM); + if (!dt->sem) { + tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core); + return -ENOMEM; + } + + /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ + k_sem_init(dt->sem, 0, 10); + + thread_name[sizeof(thread_name) - 2] = '0' + core; + + /* + * Allocate thread structure dynamically. Keep the handle in a local: + * the domain is user-writable, so dt->ll_thread must not be read back + * for the kernel-only tables below. + */ + thread = k_object_alloc(K_OBJ_THREAD); + if (!thread) { + tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); + k_object_free(dt->sem); + dt->sem = NULL; + return -ENOMEM; + } + + k_thread_create(thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, + zephyr_domain_thread_fn, zephyr_domain, + INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, + K_USER, K_FOREVER); + + /* record in the kernel-only table for syscall-context lookups */ + ll_thread_tid[core] = thread; + sof_sysuser_thread_set(SOF_SYSUSER_LL, core, thread); + dt->ll_thread = thread; + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, core); +#endif + k_thread_name_set(thread, thread_name); + + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + user_grant_uaol_access_all(thread); + tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); + + k_thread_start(thread); + + k_mutex_lock(domain->lock, K_FOREVER); + if (!k_timer_user_data_get(zephyr_domain->timer)) { + k_timeout_t start = {0}; + + k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); + k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); + + k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); + + /* Enable the watchdog */ + watchdog_enable(core); + } + + k_mutex_unlock(domain->lock); + + tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", + domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); + + return 0; +} + +/* + * Free resources acquired by zephyr_domain_thread_init(). + * Stops the timer, aborts the scheduling thread and frees the thread object. + * Must be called from kernel context. + */ +static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, + uint32_t num_tasks) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + tr_dbg(&ll_tr, "thread_free entry, core %d, num_tasks %u", core, num_tasks); + + /* Still tasks on other cores, only clean up this core's thread */ + k_mutex_lock(domain->lock, K_FOREVER); + + if (!num_tasks && !atomic_read(&domain->total_num_tasks)) { + /* Last task globally: stop the timer and watchdog */ + watchdog_disable(core); + + k_timer_stop(zephyr_domain->timer); + k_timer_user_data_set(zephyr_domain->timer, NULL); + } + + dt->handler = NULL; + dt->arg = NULL; + + k_mutex_unlock(domain->lock); + + if (dt->ll_thread) { + sof_sysuser_thread_set(SOF_SYSUSER_LL, core, NULL); + ll_thread_tid[core] = NULL; + k_thread_abort(dt->ll_thread); + k_object_free(dt->ll_thread); + dt->ll_thread = NULL; + } + + if (dt->sem) { + k_object_free(dt->sem); + dt->sem = NULL; + } + + tr_info(&ll_tr, "thread_free done, core %d", core); +} + +struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + tr_dbg(&ll_tr, "entry"); + + return dt->ll_thread; +} + +/* + * Return the LL scheduling thread for an explicitly given core. + * + * Reads a kernel-only table keyed by core, so it is safe to call from a + * privileged syscall context without dereferencing any user-accessible + * scheduler or domain object, and without relying on cpu_get_id(). + */ +struct k_thread *zephyr_domain_thread_tid_for_core(int core) +{ + if (core < 0 || core >= CONFIG_CORE_COUNT) + return NULL; + + return ll_thread_tid[core]; +} + +struct k_thread *zephyr_ll_domain_thread(void) +{ + struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); + + if (!ll_domain) + return NULL; + + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); + + return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; +} + +#endif /* CONFIG_SOF_USERSPACE_LL */ + +APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { +#ifdef CONFIG_SOF_USERSPACE_LL + .domain_register = zephyr_domain_register_user, + .domain_unregister = zephyr_domain_unregister_user, + .domain_thread_init = zephyr_domain_thread_init, + .domain_thread_free = zephyr_domain_thread_free, +#else + .domain_register = zephyr_domain_register, + .domain_unregister = zephyr_domain_unregister, +#endif +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL + .domain_block = zephyr_domain_block, + .domain_unblock = zephyr_domain_unblock, +#endif +}; + +/* Core 0 only */ +struct ll_schedule_domain *zephyr_domain_init(int clk) +{ + struct ll_schedule_domain *domain; + struct zephyr_domain *zephyr_domain; + int core; + + domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, false, + &zephyr_domain_ops); + if (!domain) { + tr_err(&ll_tr, "domain init failed"); + return NULL; + } + +#if CONFIG_SOF_USERSPACE_LL + zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*zephyr_domain), sizeof(void *)); + if (zephyr_domain) + memset(zephyr_domain, 0, sizeof(*zephyr_domain)); +#else + zephyr_domain = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*zephyr_domain)); +#endif + if (!zephyr_domain) { + tr_err(&ll_tr, "domain allocation failed"); + k_panic(); + return NULL; + } + + zephyr_domain->ll_domain = domain; + +#if CONFIG_SOF_USERSPACE_LL + /* Allocate timer dynamically for userspace access */ + zephyr_domain->timer = k_object_alloc(K_OBJ_TIMER); + if (!zephyr_domain->timer) { + tr_err(&ll_tr, "timer allocation failed"); + k_panic(); + return NULL; + } +#else + /* not allocated dynamically when LL in kernel space */ + zephyr_domain->timer = &zephyr_domain->timer_obj; +#endif + +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL + atomic_set(&zephyr_domain->block, 0); + k_mutex_init(&zephyr_domain->block_mutex); + k_condvar_init(&zephyr_domain->block_condvar); +#endif + + ll_sch_domain_set_pdata(domain, zephyr_domain); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { +#ifndef CONFIG_SOF_USERSPACE_LL + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + /* not allocated dynamically when LL in kernel space */ + dt->sem = &dt->sem_obj; +#endif + } + + return domain; +} diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 58d1f4b7c0fe..11e14dbc187c 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -4,19 +4,15 @@ // // Author: Tomasz Lauda -#include #include #include -#include #include -#include #include #include -#include #include #include +#include #include -#include #include #include #include @@ -29,47 +25,9 @@ LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); -/* - * Currently the Zephyr clock rate is part it's Kconfig known at build time. - * SOF on Intel CAVS platforms currently only aligns with Zephyr when both - * use the CAVS 19.2 MHz SSP clock. TODO - needs runtime alignment. +/* The thread and timer setup, which runs in kernel mode, is in + * src/kernel/schedule/zephyr_domain-kernel.c */ -#if CONFIG_XTENSA && CONFIG_CAVS && !CONFIG_INTEL_ADSP_TIMER -#error "Zephyr uses 19.2MHz clock derived from SSP which must be enabled." -#endif - -#define ZEPHYR_LL_STACK_SIZE 8192 - -#if CONFIG_SOF_USERSPACE_LL -K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); -#else -K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); -#endif - -struct zephyr_domain_thread { - struct k_thread *ll_thread; - struct k_sem *sem; -#ifndef CONFIG_SOF_USERSPACE_LL - struct k_thread ll_thread_obj; - struct k_sem sem_obj; -#endif - void (*handler)(void *arg); - void *arg; -}; - -struct zephyr_domain { - struct k_timer *timer; -#ifndef CONFIG_SOF_USERSPACE_LL - struct k_timer timer_obj; -#endif - struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; - struct ll_schedule_domain *ll_domain; -#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL - atomic_t block; - struct k_mutex block_mutex; - struct k_condvar block_condvar; -#endif -}; /* perf measurement windows size 2^x */ #define CYCLES_WINDOW_SIZE CONFIG_SCHEDULE_LL_STATS_LOG_WINDOW_SIZE @@ -87,7 +45,7 @@ static inline void stats_report(unsigned int runs, int core, unsigned int cycles } #endif /* CONFIG_SCHEDULE_LL_STATS_LOG */ -static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) +void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) { struct zephyr_domain *zephyr_domain = p1; int core = POINTER_TO_INT(p2); @@ -156,253 +114,14 @@ static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) } } -/* Timer callback: runs in timer IRQ context */ -static void zephyr_domain_timer_fn(struct k_timer *timer) -{ - struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer); - int core; - - /* - * A race is possible when the Zephyr LL scheduling domain is being - * unregistered while a timer IRQ is processed on a different core. Then - * the timer is removed by the former but then re-added by the latter, - * but this time with no user data and no handler set. This leads to the - * timer continuing to trigger and then leading to a lock up when it is - * registered again next time. - */ - if (!zephyr_domain) { - k_timer_stop(timer); - return; - } - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - if (dt->handler) - k_sem_give(dt->sem); - } -} - -/* The normal kernel-space implementation for register/unregister */ -#ifndef CONFIG_SOF_USERSPACE_LL - -static int zephyr_domain_register(struct ll_schedule_domain *domain, - struct task *task, - void (*handler)(void *arg), void *arg) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - char thread_name[] = "ll_thread0"; - k_tid_t thread; - k_spinlock_key_t key; - - tr_dbg(&ll_tr, "entry"); - - /* domain work only needs registered once on each core */ - if (dt->handler) - return 0; - - dt->handler = handler; - dt->arg = arg; - - /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ - k_sem_init(dt->sem, 0, 10); - - thread_name[sizeof(thread_name) - 2] = '0' + core; - - /* not allocated dynamically when LL in kernel space */ - dt->ll_thread = &dt->ll_thread_obj; - - thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, - zephyr_domain_thread_fn, zephyr_domain, INT_TO_POINTER(core), - NULL, CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER); - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, core); -#endif - k_thread_name_set(thread, thread_name); - - k_thread_start(thread); - - key = k_spin_lock(&domain->lock); - - if (!k_timer_user_data_get(zephyr_domain->timer)) { - k_timeout_t start = {0}; - - k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); - k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); - - k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); - - /* Enable the watchdog */ - watchdog_enable(core); - } - - k_spin_unlock(&domain->lock, key); - - tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", - domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); - - return 0; -} - -static int zephyr_domain_unregister(struct ll_schedule_domain *domain, - struct task *task, uint32_t num_tasks) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - k_spinlock_key_t key; - - tr_dbg(&ll_tr, "entry"); - - /* tasks still registered on this core */ - if (num_tasks) - return 0; - - key = k_spin_lock(&domain->lock); - - if (!atomic_read(&domain->total_num_tasks)) { - /* Disable the watchdog */ - watchdog_disable(core); - - k_timer_stop(zephyr_domain->timer); - k_timer_user_data_set(zephyr_domain->timer, NULL); - } - - zephyr_domain->domain_thread[core].handler = NULL; - - k_spin_unlock(&domain->lock, key); - - tr_info(&ll_tr, "domain->type %d domain->clk %d", - domain->type, domain->clk); - - /* - * If running in the context of the domain thread, k_thread_abort() will - * not return - */ - k_thread_abort(zephyr_domain->domain_thread[core].ll_thread); - - tr_dbg(&ll_tr, "exit"); - - return 0; -} - -#else /* CONFIG_SOF_USERSPACE_LL */ - -/* - * Kernel-owned per-core LL thread table. Populated from the privileged - * domain-thread init path and consulted by z_impl_zephyr_ll_task_sem_alloc() - * so that privileged code never has to traverse the user-accessible - * scheduler/domain objects to find the LL thread of a given core. - */ -static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT]; - -/* - * Privileged thread initialization for userspace LL scheduling. - * Creates the scheduling thread, sets up timer, grants access to kernel - * objects. Must be called from kernel context before any user-space - * domain_register() calls. - */ -static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, - struct task *task) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - struct zephyr_domain_thread *dt; - char thread_name[] = "ll_thread0"; - k_tid_t thread; - int core = task->core; - - tr_dbg(&ll_tr, "entry"); - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - dt = zephyr_domain->domain_thread + core; - - /* thread only needs to be created once per core */ - if (dt->ll_thread) - return 0; - - dt->handler = NULL; - dt->arg = NULL; - - dt->sem = k_object_alloc(K_OBJ_SEM); - if (!dt->sem) { - tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core); - return -ENOMEM; - } - - /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ - k_sem_init(dt->sem, 0, 10); - - thread_name[sizeof(thread_name) - 2] = '0' + core; - - /* - * Allocate thread structure dynamically. Keep the handle in a local: - * the domain is user-writable, so dt->ll_thread must not be read back - * for the kernel-only tables below. - */ - thread = k_object_alloc(K_OBJ_THREAD); - if (!thread) { - tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); - k_object_free(dt->sem); - dt->sem = NULL; - return -ENOMEM; - } - - k_thread_create(thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, - zephyr_domain_thread_fn, zephyr_domain, - INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, - K_USER, K_FOREVER); - - /* record in the kernel-only table for syscall-context lookups */ - ll_thread_tid[core] = thread; - sof_sysuser_thread_set(SOF_SYSUSER_LL, core, thread); - dt->ll_thread = thread; - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, core); -#endif - k_thread_name_set(thread, thread_name); - - k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); - user_grant_dai_access_all(thread); - user_grant_dma_access_all(thread); - user_grant_uaol_access_all(thread); - tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); - - k_thread_start(thread); - - k_mutex_lock(domain->lock, K_FOREVER); - if (!k_timer_user_data_get(zephyr_domain->timer)) { - k_timeout_t start = {0}; - - k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); - k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); - - k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); - - /* Enable the watchdog */ - watchdog_enable(core); - } - - k_mutex_unlock(domain->lock); - - tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", - domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); - - return 0; -} - +#ifdef CONFIG_SOF_USERSPACE_LL /* * User-space register: bookkeeping only. The privileged thread setup has * already been done by domain_thread_init() called from kernel context. */ -static int zephyr_domain_register_user(struct ll_schedule_domain *domain, - struct task *task, - void (*handler)(void *arg), void *arg) +int zephyr_domain_register_user(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); struct zephyr_domain_thread *dt; @@ -433,8 +152,8 @@ static int zephyr_domain_register_user(struct ll_schedule_domain *domain, return 0; } -static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, - struct task *task, uint32_t num_tasks) +int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); int core = task->core; @@ -461,95 +180,10 @@ static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, return 0; } - -/* - * Free resources acquired by zephyr_domain_thread_init(). - * Stops the timer, aborts the scheduling thread and frees the thread object. - * Must be called from kernel context. - */ -static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, - uint32_t num_tasks) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - tr_dbg(&ll_tr, "thread_free entry, core %d, num_tasks %u", core, num_tasks); - - /* Still tasks on other cores, only clean up this core's thread */ - k_mutex_lock(domain->lock, K_FOREVER); - - if (!num_tasks && !atomic_read(&domain->total_num_tasks)) { - /* Last task globally: stop the timer and watchdog */ - watchdog_disable(core); - - k_timer_stop(zephyr_domain->timer); - k_timer_user_data_set(zephyr_domain->timer, NULL); - } - - dt->handler = NULL; - dt->arg = NULL; - - k_mutex_unlock(domain->lock); - - if (dt->ll_thread) { - sof_sysuser_thread_set(SOF_SYSUSER_LL, core, NULL); - ll_thread_tid[core] = NULL; - k_thread_abort(dt->ll_thread); - k_object_free(dt->ll_thread); - dt->ll_thread = NULL; - } - - if (dt->sem) { - k_object_free(dt->sem); - dt->sem = NULL; - } - - tr_info(&ll_tr, "thread_free done, core %d", core); -} - -struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - tr_dbg(&ll_tr, "entry"); - - return dt->ll_thread; -} - -/* - * Return the LL scheduling thread for an explicitly given core. - * - * Reads a kernel-only table keyed by core, so it is safe to call from a - * privileged syscall context without dereferencing any user-accessible - * scheduler or domain object, and without relying on cpu_get_id(). - */ -struct k_thread *zephyr_domain_thread_tid_for_core(int core) -{ - if (core < 0 || core >= CONFIG_CORE_COUNT) - return NULL; - - return ll_thread_tid[core]; -} - -struct k_thread *zephyr_ll_domain_thread(void) -{ - struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); - - if (!ll_domain) - return NULL; - - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); - - return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; -} - #endif /* CONFIG_SOF_USERSPACE_LL */ #if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL -static void zephyr_domain_block(struct ll_schedule_domain *domain) +void zephyr_domain_block(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -560,7 +194,7 @@ static void zephyr_domain_block(struct ll_schedule_domain *domain) k_mutex_unlock(&zephyr_domain->block_mutex); } -static void zephyr_domain_unblock(struct ll_schedule_domain *domain) +void zephyr_domain_unblock(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -573,87 +207,6 @@ static void zephyr_domain_unblock(struct ll_schedule_domain *domain) } #endif /* CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL */ -APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { -#ifdef CONFIG_SOF_USERSPACE_LL - .domain_register = zephyr_domain_register_user, - .domain_unregister = zephyr_domain_unregister_user, - .domain_thread_init = zephyr_domain_thread_init, - .domain_thread_free = zephyr_domain_thread_free, -#else - .domain_register = zephyr_domain_register, - .domain_unregister = zephyr_domain_unregister, -#endif -#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL - .domain_block = zephyr_domain_block, - .domain_unblock = zephyr_domain_unblock, -#endif -}; - -/* Core 0 only */ -struct ll_schedule_domain *zephyr_domain_init(int clk) -{ - struct ll_schedule_domain *domain; - struct zephyr_domain *zephyr_domain; - int core; - - domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, false, - &zephyr_domain_ops); - if (!domain) { - tr_err(&ll_tr, "domain init failed"); - return NULL; - } - -#if CONFIG_SOF_USERSPACE_LL - zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*zephyr_domain), sizeof(void *)); - if (zephyr_domain) - memset(zephyr_domain, 0, sizeof(*zephyr_domain)); -#else - zephyr_domain = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*zephyr_domain)); -#endif - if (!zephyr_domain) { - tr_err(&ll_tr, "domain allocation failed"); - k_panic(); - return NULL; - } - - zephyr_domain->ll_domain = domain; - -#if CONFIG_SOF_USERSPACE_LL - /* Allocate timer dynamically for userspace access */ - zephyr_domain->timer = k_object_alloc(K_OBJ_TIMER); - if (!zephyr_domain->timer) { - tr_err(&ll_tr, "timer allocation failed"); - k_panic(); - return NULL; - } -#else - /* not allocated dynamically when LL in kernel space */ - zephyr_domain->timer = &zephyr_domain->timer_obj; -#endif - -#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL - atomic_set(&zephyr_domain->block, 0); - k_mutex_init(&zephyr_domain->block_mutex); - k_condvar_init(&zephyr_domain->block_condvar); -#endif - - ll_sch_domain_set_pdata(domain, zephyr_domain); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { -#ifndef CONFIG_SOF_USERSPACE_LL - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - /* not allocated dynamically when LL in kernel space */ - dt->sem = &dt->sem_obj; -#endif - } - - return domain; -} - /* Check if currently running in the LL scheduler thread context */ bool ll_sch_is_current(void) { From 1c5bed39ac94320b167c7bbc24f86e1a8fd0738f Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:14:23 +0000 Subject: [PATCH 2/9] schedule: split the kernel part of zephyr_ll.c into src/kernel/schedule zephyr_ll.c is the LL scheduler, which runs in the LL scheduler threads and, with user-space LL, in user mode. Its setup and a few other parts need kernel mode. Move them to src/kernel/schedule/zephyr_ll-kernel.c: - zephyr_ll_scheduler_init(), zephyr_ll_scheduler_free() and the scheduler ops, which allocate and free the per-core scheduler data and lock and register the scheduler at boot; - the kernel-only copy of the per-core lock table, which the access grants use; - the kernel half of the zephyr_ll_task_sem_alloc() and zephyr_ll_task_sem_free() system calls, with their caller checks, which allocate the per-task semaphores as kernel objects and grant the LL thread access to them; - zephyr_ll_init_context(), which sets up the LL thread of a core and grants it access to the scheduler lock and the task semaphore; - user_ll_grant_access(), which grants a thread access to the scheduler lock of a core. The scheduler and per-task data structures, the per-core lock table and the scheduler ops are declared in sof/schedule/zephyr_ll.h, and zephyr_ll_assert_core(), which both parts use, becomes a static inline there. The kernel part is built under the same condition as zephyr_ll.c. The code is unchanged, but the compiler inlines differently in the smaller zephyr_ll.c: in builds with user-space support, zephyr_ll_task_done() now inlines the system call wrapper of a log message instead of calling an out-of-line copy of it, and the register allocation of zephyr_ll_task_free() and zephyr_ll_task_init() differs. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/zephyr_ll.h | 48 ++++ src/kernel/schedule/CMakeLists.txt | 7 + src/kernel/schedule/zephyr_ll-kernel.c | 313 +++++++++++++++++++++++ src/schedule/zephyr_ll.c | 332 +------------------------ 4 files changed, 381 insertions(+), 319 deletions(-) create mode 100644 src/kernel/schedule/zephyr_ll-kernel.c diff --git a/src/include/sof/schedule/zephyr_ll.h b/src/include/sof/schedule/zephyr_ll.h index 126b07ee1879..14a7b75c1821 100644 --- a/src/include/sof/schedule/zephyr_ll.h +++ b/src/include/sof/schedule/zephyr_ll.h @@ -12,11 +12,59 @@ #ifndef __SOF_SCHEDULE_ZEPHYR_LL_H__ #define __SOF_SCHEDULE_ZEPHYR_LL_H__ +#include +#include #include #include #include +#include #include +/* per-scheduler data */ +struct zephyr_ll { + struct list_item tasks; /* list of ll tasks */ + unsigned int n_tasks; /* task counter */ + struct ll_schedule_domain *ll_domain; /* scheduling domain */ + unsigned int core; /* core ID of this instance */ +#if CONFIG_SOF_USERSPACE_LL + struct k_mutex *lock; /* mutex for userspace */ +#endif + struct k_heap *heap; +}; + +/* per-task scheduler data */ +struct zephyr_ll_pdata { + bool run; + bool freeing; + struct k_sem *sem_p; +#if !CONFIG_DYNAMIC_OBJECTS + struct k_sem sem; +#endif +}; + +static inline void zephyr_ll_assert_core(const struct zephyr_ll *sch) +{ + assert(CONFIG_CORE_COUNT == 1 || IS_ENABLED(CONFIG_SOF_USERSPACE_LL) || + sch->core == cpu_get_id()); +} + +/* LL scheduler ops, see zephyr_ll.c */ +int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, + uint64_t period); +int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *before); +int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *after); +int zephyr_ll_task_free(void *data, struct task *task); +int zephyr_ll_task_cancel(void *data, struct task *task); + +#if CONFIG_SOF_USERSPACE_LL +/* per-core LL scheduler locks, see zephyr_ll.c */ +extern struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; + +struct k_thread *zephyr_ll_init_context(void *data, struct task *task); +#endif + struct zephyr_domain_thread { struct k_thread *ll_thread; struct k_sem *sem; diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 6b0d7011f409..5fb06ef0adaa 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -6,6 +6,13 @@ if(zephyr) zephyr_domain-kernel.c ) + # The kernel part of zephyr_ll.c, built with it, see src/schedule/CMakeLists.txt + if(NOT CONFIG_SOC_ACP_6_0 OR CONFIG_DMA_DOMAIN) + zephyr_library_sources( + zephyr_ll-kernel.c + ) + endif() + zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL zephyr_ll_user.c ) diff --git a/src/kernel/schedule/zephyr_ll-kernel.c b/src/kernel/schedule/zephyr_ll-kernel.c new file mode 100644 index 000000000000..c0a8398b3dc8 --- /dev/null +++ b/src/kernel/schedule/zephyr_ll-kernel.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Guennadi Liakhovetski + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); + +/* Kernel part of the Zephyr LL scheduler: the scheduler initialization, the task + * semaphore syscalls and the access grants to the scheduler objects. The rest is in + * src/schedule/zephyr_ll.c + */ + +#if CONFIG_SOF_USERSPACE_LL +/* + * Kernel-only copy of zephyr_ll_locks[]. That array is writable by user + * threads, so kernel code granting access to the locks must use this copy, + * otherwise a user thread could make it grant access to any kernel object. + */ +static struct k_mutex *zephyr_ll_klocks[CONFIG_CORE_COUNT]; +#endif + +#if CONFIG_DYNAMIC_OBJECTS +static struct list_item zephyr_ll_task_sem_list = LIST_INIT(zephyr_ll_task_sem_list); + +struct zephyr_ll_task_sem { + struct task *task; + struct k_sem *sem; + struct list_item list; +}; + +/* + * The syscall validators pass in pdata that they read once from user memory + * and validated, so these helpers must not read task->priv_data again. + */ +static int zephyr_ll_task_sem_alloc_pdata(struct task *task, struct zephyr_ll_pdata *pdata) +{ + struct zephyr_ll_task_sem *ts = rmalloc(SOF_MEM_FLAG_COHERENT, sizeof(*ts)); + + if (!ts) + return -ENOMEM; + + ts->sem = k_object_alloc(K_OBJ_SEM); + if (!ts->sem) { + rfree(ts); + return -ENOMEM; + } + + k_sem_init(ts->sem, 0, 1); + +#if CONFIG_SOF_USERSPACE_LL + /* + * The per-task semaphore is signalled from zephyr_ll_task_done(), + * which runs in the (unprivileged) LL scheduler thread when a task is + * freed while it is still running. k_object_alloc() only grants access + * to the calling thread (the IPC handler that creates the task), so the + * LL thread must be granted access explicitly, otherwise its + * k_sem_give() traps with a userspace permission fault. + * + * Resolve the LL thread from kernel-only per-core state keyed by the + * task's target core; never traverse the user-accessible scheduler or + * domain objects from privileged context. + */ + struct k_thread *ll_tid = zephyr_domain_thread_tid_for_core(task->core); + + if (ll_tid) + k_thread_access_grant(ll_tid, ts->sem); +#endif /* CONFIG_SOF_USERSPACE_LL */ + + ts->task = task; + pdata->sem_p = ts->sem; + /* List is protected by IPC serialization */ + list_item_append(&ts->list, &zephyr_ll_task_sem_list); + + return 0; +} + +int z_impl_zephyr_ll_task_sem_alloc(struct task *task) +{ + return zephyr_ll_task_sem_alloc_pdata(task, task->priv_data); +} + +static int zephyr_ll_task_sem_free_pdata(struct task *task, struct zephyr_ll_pdata *pdata) +{ + struct list_item *list; + struct zephyr_ll_task_sem *ts; + bool found = false; + + /* List is protected by IPC serialization */ + list_for_item(list, &zephyr_ll_task_sem_list) { + ts = container_of(list, struct zephyr_ll_task_sem, list); + if (ts->task == task) { + found = true; + break; + } + } + + if (!found) + return -ENOENT; + + if (pdata->sem_p != ts->sem) + return -EINVAL; + + list_item_del(list); + k_object_free(ts->sem); + rfree(ts); + + return 0; +} + +int z_impl_zephyr_ll_task_sem_free(struct task *task) +{ + return zephyr_ll_task_sem_free_pdata(task, task->priv_data); +} + +#ifdef CONFIG_USERSPACE +#include +#include +/* + * Read task->priv_data exactly once. The task lives in user memory and a + * thread on another core can change the pointer after it has been checked. + * + * @return validated pdata pointer, or NULL if the task has none + */ +static struct zephyr_ll_pdata *zephyr_ll_task_pdata_vrfy(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + K_OOPS(k_usermode_from_copy(&pdata, &task->priv_data, sizeof(pdata))); + if (pdata) + K_OOPS(K_SYSCALL_MEMORY_WRITE(pdata, sizeof(*pdata))); + + return pdata; +} + +static inline int z_vrfy_zephyr_ll_task_sem_alloc(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); + + if (!task) + return -EINVAL; + pdata = zephyr_ll_task_pdata_vrfy(task); + if (!pdata) + return -EINVAL; + + return zephyr_ll_task_sem_alloc_pdata(task, pdata); +} +#include + +static inline int z_vrfy_zephyr_ll_task_sem_free(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); + + if (!task) + return -EINVAL; + pdata = zephyr_ll_task_pdata_vrfy(task); + if (!pdata) + return -EINVAL; + + return zephyr_ll_task_sem_free_pdata(task, pdata); +} +#include +#endif /* CONFIG_USERSPACE */ +#endif /* CONFIG_DYNAMIC_OBJECTS */ + +#if CONFIG_SOF_USERSPACE_LL +struct k_thread *zephyr_ll_init_context(void *data, struct task *task) +{ + struct zephyr_ll *sch = data; + struct zephyr_ll_pdata *pdata = task->priv_data; + int ret; + + /* + * Use domain_thread_init() for privileged setup (thread creation, + * timer, access grants). domain_register() is now bookkeeping only + * and will be called later from user context when scheduling tasks. + */ + ret = domain_thread_init(sch->ll_domain, task); + if (ret < 0) { + tr_err(&ll_tr, "cannot init_context %d", ret); + return NULL; + } + + assert(!k_is_user_context()); + k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock, pdata->sem_p); + + tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + + return zephyr_domain_thread_tid(sch->ll_domain); +} + +void user_ll_grant_access(struct k_thread *thread, int core) +{ + assert(core < CONFIG_CORE_COUNT && zephyr_ll_klocks[core] != NULL); + + k_thread_access_grant(thread, zephyr_ll_klocks[core]); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY +static void zephyr_ll_scheduler_free(void *data, uint32_t flags) +{ + struct zephyr_ll *sch = data; + + zephyr_ll_assert_core(sch); + +#if CONFIG_SOF_USERSPACE_LL + zephyr_ll_locks[sch->core] = NULL; + zephyr_ll_klocks[sch->core] = NULL; + k_object_free(sch->lock); +#endif + + if (sch->n_tasks) + tr_err(&ll_tr, "%u tasks are still active!", + sch->n_tasks); + +#if CONFIG_SOF_USERSPACE_LL + domain_thread_free(sch->ll_domain, sch->n_tasks); +#endif + sof_heap_free(sch->heap, sch); +} +#endif /* CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY */ + +static const struct scheduler_ops zephyr_ll_ops = { + .schedule_task = zephyr_ll_task_schedule, + .schedule_task_before = zephyr_ll_task_schedule_before, + .schedule_task_after = zephyr_ll_task_schedule_after, + .schedule_task_free = zephyr_ll_task_free, + .schedule_task_cancel = zephyr_ll_task_cancel, +#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY + .scheduler_free = zephyr_ll_scheduler_free, +#endif +#if CONFIG_SOF_USERSPACE_LL + .scheduler_init_context = zephyr_ll_init_context, +#endif +}; + +/* TODO: low-power mode clock support */ +/* Runs on each core during initialisation with the same domain argument */ +__cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) +{ + struct zephyr_ll *sch; + int core = cpu_get_id(); + struct k_heap *heap = sof_sys_heap_get(); + int flags = SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT; + + assert_can_be_cold(); + +#if CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; +#endif + tr_dbg(&ll_tr, "init on core %d", core); + + /* initialize per-core scheduler private data */ + sch = sof_heap_alloc(heap, flags, sizeof(*sch), 0); + if (!sch) { + tr_err(&ll_tr, "allocation failed"); + return -ENOMEM; + } + + memset(sch, 0, sizeof(*sch)); + + list_init(&sch->tasks); + sch->ll_domain = domain; + sch->core = core; + sch->n_tasks = 0; + sch->heap = heap; + +#if CONFIG_SOF_USERSPACE_LL + /* Allocate mutex dynamically for userspace access */ + sch->lock = k_object_alloc(K_OBJ_MUTEX); + if (!sch->lock) { + tr_err(&ll_tr, "mutex allocation failed"); + sof_heap_free(sch->heap, sch); + return -ENOMEM; + } + assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); + zephyr_ll_locks[core] = sch->lock; + zephyr_ll_klocks[core] = sch->lock; + k_mutex_init(sch->lock); + + tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); +#endif + + scheduler_init(domain->type, &zephyr_ll_ops, sch); + + return 0; +} diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index e04a05edd742..7daa415b8fd1 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -26,27 +27,9 @@ SOF_DEFINE_REG_UUID(zll_sched); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(ll_tr, SOF_UUID(zll_sched_uuid), LOG_LEVEL_INFO); -/* per-scheduler data */ -struct zephyr_ll { - struct list_item tasks; /* list of ll tasks */ - unsigned int n_tasks; /* task counter */ - struct ll_schedule_domain *ll_domain; /* scheduling domain */ - unsigned int core; /* core ID of this instance */ -#if CONFIG_SOF_USERSPACE_LL - struct k_mutex *lock; /* mutex for userspace */ -#endif - struct k_heap *heap; -}; - -/* per-task scheduler data */ -struct zephyr_ll_pdata { - bool run; - bool freeing; - struct k_sem *sem_p; -#if !CONFIG_DYNAMIC_OBJECTS - struct k_sem sem; -#endif -}; +/* The scheduler syscalls and access grants, which run in kernel mode, are in + * src/kernel/schedule/zephyr_ll-kernel.c + */ #if CONFIG_SOF_USERSPACE_LL /* @@ -54,14 +37,7 @@ struct zephyr_ll_pdata { * can read the pointer for syscalls. The actual lock object resides * in kernel memory, there are just handles! */ -static APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; - -/* - * Kernel-only copy of zephyr_ll_locks[]. That array is writable by user - * threads, so kernel code granting access to the locks must use this copy, - * otherwise a user thread could make it grant access to any kernel object. - */ -static struct k_mutex *zephyr_ll_klocks[CONFIG_CORE_COUNT]; +APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; /* * Shadow ownership tracking for the per-core LL mutex, used by @@ -118,12 +94,6 @@ static void zephyr_ll_unlock(struct zephyr_ll *sch, uint32_t *flags) #endif } -static void zephyr_ll_assert_core(const struct zephyr_ll *sch) -{ - assert(CONFIG_CORE_COUNT == 1 || IS_ENABLED(CONFIG_SOF_USERSPACE_LL) || - sch->core == cpu_get_id()); -} - /* Locking: caller should hold the domain lock */ static void zephyr_ll_task_done(struct zephyr_ll *sch, struct task *task) @@ -433,183 +403,35 @@ static int zephyr_ll_task_schedule_common(struct zephyr_ll *sch, struct task *ta return 0; } -static int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, - uint64_t period) +int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, + uint64_t period) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, NULL, false); } -static int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, - uint64_t period, struct task *before) +int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *before) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, before, true); } -static int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, - uint64_t period, struct task *after) +int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *after) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, after, false); } -#if CONFIG_DYNAMIC_OBJECTS -static struct list_item zephyr_ll_task_sem_list = LIST_INIT(zephyr_ll_task_sem_list); - -struct zephyr_ll_task_sem { - struct task *task; - struct k_sem *sem; - struct list_item list; -}; - -/* - * The syscall validators pass in pdata that they read once from user memory - * and validated, so these helpers must not read task->priv_data again. - */ -static int zephyr_ll_task_sem_alloc_pdata(struct task *task, struct zephyr_ll_pdata *pdata) -{ - struct zephyr_ll_task_sem *ts = rmalloc(SOF_MEM_FLAG_COHERENT, sizeof(*ts)); - - if (!ts) - return -ENOMEM; - - ts->sem = k_object_alloc(K_OBJ_SEM); - if (!ts->sem) { - rfree(ts); - return -ENOMEM; - } - - k_sem_init(ts->sem, 0, 1); - -#if CONFIG_SOF_USERSPACE_LL - /* - * The per-task semaphore is signalled from zephyr_ll_task_done(), - * which runs in the (unprivileged) LL scheduler thread when a task is - * freed while it is still running. k_object_alloc() only grants access - * to the calling thread (the IPC handler that creates the task), so the - * LL thread must be granted access explicitly, otherwise its - * k_sem_give() traps with a userspace permission fault. - * - * Resolve the LL thread from kernel-only per-core state keyed by the - * task's target core; never traverse the user-accessible scheduler or - * domain objects from privileged context. - */ - struct k_thread *ll_tid = zephyr_domain_thread_tid_for_core(task->core); - - if (ll_tid) - k_thread_access_grant(ll_tid, ts->sem); -#endif /* CONFIG_SOF_USERSPACE_LL */ - - ts->task = task; - pdata->sem_p = ts->sem; - /* List is protected by IPC serialization */ - list_item_append(&ts->list, &zephyr_ll_task_sem_list); - - return 0; -} - -int z_impl_zephyr_ll_task_sem_alloc(struct task *task) -{ - return zephyr_ll_task_sem_alloc_pdata(task, task->priv_data); -} - -static int zephyr_ll_task_sem_free_pdata(struct task *task, struct zephyr_ll_pdata *pdata) -{ - struct list_item *list; - struct zephyr_ll_task_sem *ts; - bool found = false; - - /* List is protected by IPC serialization */ - list_for_item(list, &zephyr_ll_task_sem_list) { - ts = container_of(list, struct zephyr_ll_task_sem, list); - if (ts->task == task) { - found = true; - break; - } - } - - if (!found) - return -ENOENT; - - if (pdata->sem_p != ts->sem) - return -EINVAL; - - list_item_del(list); - k_object_free(ts->sem); - rfree(ts); - - return 0; -} - -int z_impl_zephyr_ll_task_sem_free(struct task *task) -{ - return zephyr_ll_task_sem_free_pdata(task, task->priv_data); -} - -#ifdef CONFIG_USERSPACE -#include -#include -/* - * Read task->priv_data exactly once. The task lives in user memory and a - * thread on another core can change the pointer after it has been checked. - * - * @return validated pdata pointer, or NULL if the task has none - */ -static struct zephyr_ll_pdata *zephyr_ll_task_pdata_vrfy(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); - K_OOPS(k_usermode_from_copy(&pdata, &task->priv_data, sizeof(pdata))); - if (pdata) - K_OOPS(K_SYSCALL_MEMORY_WRITE(pdata, sizeof(*pdata))); - - return pdata; -} - -static inline int z_vrfy_zephyr_ll_task_sem_alloc(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); - - if (!task) - return -EINVAL; - pdata = zephyr_ll_task_pdata_vrfy(task); - if (!pdata) - return -EINVAL; - - return zephyr_ll_task_sem_alloc_pdata(task, pdata); -} -#include - -static inline int z_vrfy_zephyr_ll_task_sem_free(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); - - if (!task) - return -EINVAL; - pdata = zephyr_ll_task_pdata_vrfy(task); - if (!pdata) - return -EINVAL; - - return zephyr_ll_task_sem_free_pdata(task, pdata); -} -#include -#endif /* CONFIG_USERSPACE */ -#endif /* CONFIG_DYNAMIC_OBJECTS */ - /* * This is synchronous - after this returns the object can be destroyed! * Assertion: under Zephyr this is always called from a thread context! */ -static int zephyr_ll_task_free(void *data, struct task *task) +int zephyr_ll_task_free(void *data, struct task *task) { struct zephyr_ll *sch = data; uint32_t flags; @@ -677,7 +499,7 @@ static int zephyr_ll_task_free(void *data, struct task *task) } /* This is asynchronous */ -static int zephyr_ll_task_cancel(void *data, struct task *task) +int zephyr_ll_task_cancel(void *data, struct task *task) { struct zephyr_ll *sch = data; uint32_t flags; @@ -706,74 +528,6 @@ static int zephyr_ll_task_cancel(void *data, struct task *task) return 0; } -#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY -static void zephyr_ll_scheduler_free(void *data, uint32_t flags) -{ - struct zephyr_ll *sch = data; - - zephyr_ll_assert_core(sch); - -#if CONFIG_SOF_USERSPACE_LL - zephyr_ll_locks[sch->core] = NULL; - zephyr_ll_klocks[sch->core] = NULL; - k_object_free(sch->lock); -#endif - - if (sch->n_tasks) - tr_err(&ll_tr, "%u tasks are still active!", - sch->n_tasks); - -#if CONFIG_SOF_USERSPACE_LL - domain_thread_free(sch->ll_domain, sch->n_tasks); -#endif - sof_heap_free(sch->heap, sch); -} -#endif /* CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY */ - -#if CONFIG_SOF_USERSPACE_LL -struct k_thread *zephyr_ll_init_context(void *data, struct task *task) -{ - struct zephyr_ll *sch = data; - struct zephyr_ll_pdata *pdata = task->priv_data; - int ret; - - /* - * Use domain_thread_init() for privileged setup (thread creation, - * timer, access grants). domain_register() is now bookkeeping only - * and will be called later from user context when scheduling tasks. - */ - ret = domain_thread_init(sch->ll_domain, task); - if (ret < 0) { - tr_err(&ll_tr, "cannot init_context %d", ret); - return NULL; - } - - assert(!k_is_user_context()); - k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock, pdata->sem_p); - - tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, - zephyr_domain_thread_tid(sch->ll_domain)); - tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, - zephyr_domain_thread_tid(sch->ll_domain)); - - return zephyr_domain_thread_tid(sch->ll_domain); -} -#endif /* CONFIG_SOF_USERSPACE_LL */ - -static const struct scheduler_ops zephyr_ll_ops = { - .schedule_task = zephyr_ll_task_schedule, - .schedule_task_before = zephyr_ll_task_schedule_before, - .schedule_task_after = zephyr_ll_task_schedule_after, - .schedule_task_free = zephyr_ll_task_free, - .schedule_task_cancel = zephyr_ll_task_cancel, -#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY - .scheduler_free = zephyr_ll_scheduler_free, -#endif -#if CONFIG_SOF_USERSPACE_LL - .scheduler_init_context = zephyr_ll_init_context, -#endif -}; - #if CONFIG_SOF_USERSPACE_LL struct task *zephyr_ll_task_alloc(void) { @@ -787,13 +541,6 @@ struct task *zephyr_ll_task_alloc(void) return task; } -void user_ll_grant_access(struct k_thread *thread, int core) -{ - assert(core < CONFIG_CORE_COUNT && zephyr_ll_klocks[core] != NULL); - - k_thread_access_grant(thread, zephyr_ll_klocks[core]); -} - /** * Lock the LL scheduler to prevent it from processing tasks. * @@ -901,59 +648,6 @@ int zephyr_ll_task_init(struct task *task, } EXPORT_SYMBOL(zephyr_ll_task_init); -/* TODO: low-power mode clock support */ -/* Runs on each core during initialisation with the same domain argument */ -__cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) -{ - struct zephyr_ll *sch; - int core = cpu_get_id(); - struct k_heap *heap = sof_sys_heap_get(); - int flags = SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT; - - assert_can_be_cold(); - -#if CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); - flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; -#endif - tr_dbg(&ll_tr, "init on core %d", core); - - /* initialize per-core scheduler private data */ - sch = sof_heap_alloc(heap, flags, sizeof(*sch), 0); - if (!sch) { - tr_err(&ll_tr, "allocation failed"); - return -ENOMEM; - } - - memset(sch, 0, sizeof(*sch)); - - list_init(&sch->tasks); - sch->ll_domain = domain; - sch->core = core; - sch->n_tasks = 0; - sch->heap = heap; - -#if CONFIG_SOF_USERSPACE_LL - /* Allocate mutex dynamically for userspace access */ - sch->lock = k_object_alloc(K_OBJ_MUTEX); - if (!sch->lock) { - tr_err(&ll_tr, "mutex allocation failed"); - sof_heap_free(sch->heap, sch); - return -ENOMEM; - } - assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); - zephyr_ll_locks[core] = sch->lock; - zephyr_ll_klocks[core] = sch->lock; - k_mutex_init(sch->lock); - - tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); -#endif - - scheduler_init(domain->type, &zephyr_ll_ops, sch); - - return 0; -} - void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { From 683681f70ba2aee0718855c983c97ab7d4d28043 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:26:02 +0000 Subject: [PATCH 3/9] schedule: move the DP scheduler private header to src/include zephyr_dp_schedule.h holds the DP scheduler data structures and helpers shared by the DP scheduler sources. The kernel parts of those sources move to src/kernel/schedule in the next patches and need it too, so move it to sof/schedule/zephyr_dp_schedule.h. Add the include guard that it did not have. The generated code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/{ => include/sof}/schedule/zephyr_dp_schedule.h | 10 +++++++++- src/schedule/zephyr_dp_schedule.c | 2 +- src/schedule/zephyr_dp_schedule_application.c | 2 +- src/schedule/zephyr_dp_schedule_thread.c | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) rename src/{ => include/sof}/schedule/zephyr_dp_schedule.h (92%) diff --git a/src/schedule/zephyr_dp_schedule.h b/src/include/sof/schedule/zephyr_dp_schedule.h similarity index 92% rename from src/schedule/zephyr_dp_schedule.h rename to src/include/sof/schedule/zephyr_dp_schedule.h index 07d8ee4a4754..f84e1901d5ae 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/include/sof/schedule/zephyr_dp_schedule.h @@ -6,11 +6,17 @@ */ /** - * @file src/schedule/zephyr_dp_schedule.h + * @file include/sof/schedule/zephyr_dp_schedule.h * @brief Internal Data Processing (DP) scheduler structures, memory partitions, and thread helpers. * @ingroup scheduling_api + * + * Shared by the DP scheduler sources in src/schedule and their kernel parts in + * src/kernel/schedule. Not for use outside the scheduler. */ +#ifndef __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ +#define __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ + #include #include #include @@ -102,3 +108,5 @@ void scheduler_dp_unlock(unsigned int key); * @param[in] core Target DSP core ID. */ void scheduler_dp_grant(k_tid_t thread_id, uint16_t core); + +#endif /* __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ */ diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 9ff0eb1928a5..21a019dc9b20 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -25,7 +25,7 @@ #include #include -#include "zephyr_dp_schedule.h" +#include #include diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 523a1e0886f2..54885466e9eb 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -30,7 +30,7 @@ #include #include -#include "zephyr_dp_schedule.h" +#include LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index 8e33ee1ab2ce..46cbadfeade3 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -25,7 +25,7 @@ #include #include -#include "zephyr_dp_schedule.h" +#include LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; From 8de1d779454db90c52c9f9dc344485aeefb9e305 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:28:55 +0000 Subject: [PATCH 4/9] schedule: split the kernel part of zephyr_dp_schedule.c into src/kernel/schedule zephyr_dp_schedule.c holds the DP scheduler code common to its thread and application variants. Part of it needs kernel mode. Move it to src/kernel/schedule/zephyr_dp_schedule-kernel.c: - the per-core DP scheduler locks, which are kernel objects, and scheduler_dp_grant(), which grants threads access to them; - the scheduler_dp_ll_tick() system call, with the description of the DP scheduling it drives, and the scheduler_dp_check_deadline() system call; - scheduler_dp_init(), which allocates the per-core scheduler data, registers the scheduler and its ops at boot and sets up the LL tick task, together with the scheduler UUID, trace context and log module. zephyr_dp_schedule.c keeps the lock and unlock helpers, the task schedule, stop, cancel and free ops and the task info query. The locks and the ops are declared in sof/schedule/zephyr_dp_schedule.h. The code is unchanged, but z_impl_scheduler_dp_ll_tick() now calls the lock and unlock helpers instead of inlining them, and in the application variant scheduler_dp_task_free() calls scheduler_dp_task_stop() instead of inlining it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/zephyr_dp_schedule.h | 14 + src/kernel/schedule/CMakeLists.txt | 4 + .../schedule/zephyr_dp_schedule-kernel.c | 299 ++++++++++++++++++ src/schedule/zephyr_dp_schedule.c | 274 +--------------- 4 files changed, 326 insertions(+), 265 deletions(-) create mode 100644 src/kernel/schedule/zephyr_dp_schedule-kernel.c diff --git a/src/include/sof/schedule/zephyr_dp_schedule.h b/src/include/sof/schedule/zephyr_dp_schedule.h index f84e1901d5ae..1056efbfec4c 100644 --- a/src/include/sof/schedule/zephyr_dp_schedule.h +++ b/src/include/sof/schedule/zephyr_dp_schedule.h @@ -72,6 +72,20 @@ struct task_dp_pdata { #endif }; +/** @brief Per-core DP scheduler locks, see scheduler_dp_lock(). */ +extern struct k_sem dp_lock[CONFIG_MP_MAX_NUM_CPUS]; + +/** + * @brief DP scheduler ops, see struct scheduler_ops. + */ +int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, + uint64_t period); +int scheduler_dp_task_stop(void *data, struct task *task); +int scheduler_dp_task_free(void *data, struct task *task); +#if CONFIG_SOF_USERSPACE_APPLICATION +int scheduler_dp_task_cancel(void *data, struct task *task); +#endif + /** * @brief Recalculate deadlines and evaluate readiness for all active DP tasks on an LL tick. * @param[in,out] dp_sch Pointer to per-core DP scheduler state container. diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 5fb06ef0adaa..5c878b19f3c3 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -16,4 +16,8 @@ if(zephyr) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL zephyr_ll_user.c ) + + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule-kernel.c + ) endif() diff --git a/src/kernel/schedule/zephyr_dp_schedule-kernel.c b/src/kernel/schedule/zephyr_dp_schedule-kernel.c new file mode 100644 index 000000000000..7f0d7c285d62 --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule-kernel.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2023 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); +SOF_DEFINE_REG_UUID(dp_sched); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); + +/* Kernel part of the DP scheduler: the scheduler locks, their access grants, the LL + * tick system call and the scheduler initialization. The rest is in + * src/schedule/zephyr_dp_schedule.c + */ + +#define DP_LOCK_INIT(i, _) Z_SEM_INITIALIZER(dp_lock[i], 1, 1) +#define DP_LOCK_INIT_LIST LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,)) + +/* User threads don't need access to this array. Access is performed from + * the kernel space via a syscall. Array must be placed in special section + * to be qualified as initialized by the gen_kobject_list.py script. + */ +STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST }; + +void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) +{ + if (thread_id) + k_thread_access_grant(thread_id, &dp_lock[core]); +} + +/* dummy LL task - to start LL on secondary cores */ +static enum task_state scheduler_dp_ll_tick_dummy(void *data) +{ + return SOF_TASK_STATE_RESCHEDULE; +} + +/* + * function called after every LL tick + * + * This function checks if the queued DP tasks are ready to processing (meaning + * the module run by the task has enough data at all sources and enough free space + * on all sinks) + * + * if the task becomes ready, a deadline is set allowing Zephyr to schedule threads + * in right order + * + * TODO: currently there's a limitation - DP module must be surrounded by LL modules. + * it simplifies algorithm - there's no need to browse through DP chains calculating + * deadlines for each module in function of all modules execution status. + * Now is simple - modules deadline is its start + tick time. + * + * example: + * Lets assume we do have a pipeline: + * + * LL1 -> DP1 -> LL2 -> DP2 -> LL3 -> DP3 -> LL4 + * + * all LLs starts in 1ms tick + * + * for simplification lets assume + * - all LLs are on primary core, all DPs on secondary (100% CPU is for DP) + * - context switching requires 0 cycles + * + * DP1 - starts every 1ms, needs 0.5ms to finish processing + * DP2 - starts every 2ms, needs 0.6ms to finish processing + * DP3 - starts every 10ms, needs 0.3ms to finish processing + * + * TICK0 + * only LL1 is ready to run + * LL1 processing (producing data chunk for DP1) + * + * TICK1 + * LL1 is ready to run + * DP1 is ready tu run (has data from LL1) set deadline to TICK2 + * LL1 processing (producing second data chunk for DP1) + * DP1 processing for 0.5ms (consuming first data chunk, producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK2 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK3 + * LL2 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK3 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK4 + * LL2 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK4 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK5 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK6 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of second data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * !!!!!! Note here - DP1 must do before DP2 as it MUST finish in this tick. DP2 can wait + * >>>>>>> this is what we call EDF - EARIEST DEADLINE FIRST <<<<<< + * + * TICK5 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK6 + * LL2 is ready to run + * DP2 is in progress, deadline is set to TICK6 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of second data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.1ms (producing TWO data chunks for LL3) + * CPU is idle for 0.4ms (60% used) + * + * TICK6 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK7 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK8 + * LL3 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of second data chunk for DP2) + * LL3 processing (producing 10% of first data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * + * + * (........ 9 more cycles - LL3 procuces 100% of data for DP3......) + * + * + * TICK15 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK16 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK17 + * LL3 is ready to run + * DP3 is ready to run set deadline to TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of data chunk for DP2) + * LL3 processing (producing 10% of second data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used - + * !!! note that DP3 is ready but has no chance to get CPU in this cycle + * + * TICK16 + * LL1 is ready to run set deadline to TICK17 + * DP1 is ready tu run + * LL2 is ready to run + * DP2 is in progress, deadline is set to TICK17 + * LL3 is ready to run + * DP3 is in progress, deadline is set to TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * LL3 processing (producing 10% of second data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.1ms (producing data) + * DP3 processing for 0.2ms (producing 10 data chunks for LL4) + * 90% CPU used + * + * TICK17 + * LL1 is ready to run + * DP1 is ready tu run + * LL2 is ready to run + * DP2 is ready to run + * LL3 is ready to run + * LL4 is ready to run + * !! NOTE that DP3 is not ready - it will be ready again in TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * LL3 processing (producing next 10% of second data chunk for DP3) + * LL4 processing (consuming 10% of data prepared by DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * + * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 + * needed 1.2ms for processing - but the example would be too complicated) + */ + +void z_impl_scheduler_dp_ll_tick(void) +{ + unsigned int lock_key; +#if CONFIG_SOF_USERSPACE_LL + struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); +#else + struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); +#endif + + if (!dp_sch) + return; + + /* remember current timestamp as "NOW" */ + dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); + + lock_key = scheduler_dp_lock(cpu_get_id()); + scheduler_dp_recalculate(dp_sch); + scheduler_dp_unlock(lock_key); +} + +#ifdef CONFIG_USERSPACE +#include +#include +void z_vrfy_scheduler_dp_ll_tick(void) +{ + SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_LL); + + z_impl_scheduler_dp_ll_tick(); +} +#include +#endif + +void z_impl_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) +{ + if (deadline_clock_ticks && + (int32_t)(k_cycle_get_32() - deadline_clock_ticks) > 0) + telemetry_notify_dp_deadline_miss(core); +} + +#ifdef CONFIG_USERSPACE +void z_vrfy_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) +{ + K_OOPS(K_SYSCALL_VERIFY(core < CONFIG_MP_MAX_NUM_CPUS)); + z_impl_scheduler_dp_check_deadline(core, deadline_clock_ticks); +} +#include +#endif + +APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { + .schedule_task = scheduler_dp_task_shedule, +#if CONFIG_SOF_USERSPACE_APPLICATION + .schedule_task_cancel = scheduler_dp_task_cancel, +#else + .schedule_task_cancel = scheduler_dp_task_stop, +#endif + .schedule_task_free = scheduler_dp_task_free, +}; + +/* Runs on each core */ +__cold int scheduler_dp_init(void) +{ + assert_can_be_cold(); + + struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); + if (!dp_sch) + return -ENOMEM; + + memset(dp_sch, 0, sizeof(*dp_sch)); + list_init(&dp_sch->tasks); + + scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); + + /* init src of DP tick */ + return schedule_task_init_ll(&dp_sch->ll_tick_src, + SOF_UUID(dp_sched_uuid), + SOF_SCHEDULE_LL_TIMER, + 0, scheduler_dp_ll_tick_dummy, dp_sch, + cpu_get_id(), 0); +} diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 21a019dc9b20..ac50500abd7f 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -32,21 +31,12 @@ #include #include -LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); -SOF_DEFINE_REG_UUID(dp_sched); +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); - -#define DP_LOCK_INIT(i, _) Z_SEM_INITIALIZER(dp_lock[i], 1, 1) -#define DP_LOCK_INIT_LIST LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,)) - -/* User threads don't need access to this array. Access is performed from - * the kernel space via a syscall. Array must be placed in special section - * to be qualified as initialized by the gen_kobject_list.py script. +/* The scheduler locks, the LL tick system call and the scheduler initialization, which + * run in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule-kernel.c */ -static -STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST }; /* Each per-core instance of DP scheduler has separate structures; hence, locks are per-core. * @@ -63,221 +53,8 @@ void scheduler_dp_unlock(unsigned int key) k_sem_give(&dp_lock[key]); } -void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) -{ - if (thread_id) - k_thread_access_grant(thread_id, &dp_lock[core]); -} - -/* dummy LL task - to start LL on secondary cores */ -static enum task_state scheduler_dp_ll_tick_dummy(void *data) -{ - return SOF_TASK_STATE_RESCHEDULE; -} - -/* - * function called after every LL tick - * - * This function checks if the queued DP tasks are ready to processing (meaning - * the module run by the task has enough data at all sources and enough free space - * on all sinks) - * - * if the task becomes ready, a deadline is set allowing Zephyr to schedule threads - * in right order - * - * TODO: currently there's a limitation - DP module must be surrounded by LL modules. - * it simplifies algorithm - there's no need to browse through DP chains calculating - * deadlines for each module in function of all modules execution status. - * Now is simple - modules deadline is its start + tick time. - * - * example: - * Lets assume we do have a pipeline: - * - * LL1 -> DP1 -> LL2 -> DP2 -> LL3 -> DP3 -> LL4 - * - * all LLs starts in 1ms tick - * - * for simplification lets assume - * - all LLs are on primary core, all DPs on secondary (100% CPU is for DP) - * - context switching requires 0 cycles - * - * DP1 - starts every 1ms, needs 0.5ms to finish processing - * DP2 - starts every 2ms, needs 0.6ms to finish processing - * DP3 - starts every 10ms, needs 0.3ms to finish processing - * - * TICK0 - * only LL1 is ready to run - * LL1 processing (producing data chunk for DP1) - * - * TICK1 - * LL1 is ready to run - * DP1 is ready tu run (has data from LL1) set deadline to TICK2 - * LL1 processing (producing second data chunk for DP1) - * DP1 processing for 0.5ms (consuming first data chunk, producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK2 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK3 - * LL2 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK3 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK4 - * LL2 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK4 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK5 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK6 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of second data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * !!!!!! Note here - DP1 must do before DP2 as it MUST finish in this tick. DP2 can wait - * >>>>>>> this is what we call EDF - EARIEST DEADLINE FIRST <<<<<< - * - * TICK5 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK6 - * LL2 is ready to run - * DP2 is in progress, deadline is set to TICK6 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of second data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.1ms (producing TWO data chunks for LL3) - * CPU is idle for 0.4ms (60% used) - * - * TICK6 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK7 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK8 - * LL3 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of second data chunk for DP2) - * LL3 processing (producing 10% of first data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * - * - * (........ 9 more cycles - LL3 procuces 100% of data for DP3......) - * - * - * TICK15 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK16 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK17 - * LL3 is ready to run - * DP3 is ready to run set deadline to TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of data chunk for DP2) - * LL3 processing (producing 10% of second data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - - * !!! note that DP3 is ready but has no chance to get CPU in this cycle - * - * TICK16 - * LL1 is ready to run set deadline to TICK17 - * DP1 is ready tu run - * LL2 is ready to run - * DP2 is in progress, deadline is set to TICK17 - * LL3 is ready to run - * DP3 is in progress, deadline is set to TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * LL3 processing (producing 10% of second data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.1ms (producing data) - * DP3 processing for 0.2ms (producing 10 data chunks for LL4) - * 90% CPU used - * - * TICK17 - * LL1 is ready to run - * DP1 is ready tu run - * LL2 is ready to run - * DP2 is ready to run - * LL3 is ready to run - * LL4 is ready to run - * !! NOTE that DP3 is not ready - it will be ready again in TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * LL3 processing (producing next 10% of second data chunk for DP3) - * LL4 processing (consuming 10% of data prepared by DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * - * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 - * needed 1.2ms for processing - but the example would be too complicated) - */ - -void z_impl_scheduler_dp_ll_tick(void) -{ - unsigned int lock_key; -#if CONFIG_SOF_USERSPACE_LL - struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); -#else - struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); -#endif - - if (!dp_sch) - return; - - /* remember current timestamp as "NOW" */ - dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); - - lock_key = scheduler_dp_lock(cpu_get_id()); - scheduler_dp_recalculate(dp_sch); - scheduler_dp_unlock(lock_key); -} - -#ifdef CONFIG_USERSPACE -#include -#include -void z_vrfy_scheduler_dp_ll_tick(void) -{ - SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_LL); - - z_impl_scheduler_dp_ll_tick(); -} -#include -#endif - -void z_impl_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) -{ - if (deadline_clock_ticks && - (int32_t)(k_cycle_get_32() - deadline_clock_ticks) > 0) - telemetry_notify_dp_deadline_miss(core); -} - -#ifdef CONFIG_USERSPACE -void z_vrfy_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) -{ - K_OOPS(K_SYSCALL_VERIFY(core < CONFIG_MP_MAX_NUM_CPUS)); - z_impl_scheduler_dp_check_deadline(core, deadline_clock_ticks); -} -#include -#endif - #if CONFIG_SOF_USERSPACE_APPLICATION -static int scheduler_dp_task_cancel(void *data, struct task *task) +int scheduler_dp_task_cancel(void *data, struct task *task) { /* Should never be called */ k_panic(); @@ -285,7 +62,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) } #endif -static int scheduler_dp_task_stop(void *data, struct task *task) +int scheduler_dp_task_stop(void *data, struct task *task) { unsigned int lock_key; struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; @@ -313,7 +90,7 @@ static int scheduler_dp_task_stop(void *data, struct task *task) return 0; } -static int scheduler_dp_task_free(void *data, struct task *task) +int scheduler_dp_task_free(void *data, struct task *task) { struct task_dp_pdata *pdata = task->priv_data; int ret; @@ -337,8 +114,8 @@ static int scheduler_dp_task_free(void *data, struct task *task) return ret; } -static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, - uint64_t period) +int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, + uint64_t period) { struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; struct task_dp_pdata *pdata = task->priv_data; @@ -374,39 +151,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return 0; } -APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { - .schedule_task = scheduler_dp_task_shedule, -#if CONFIG_SOF_USERSPACE_APPLICATION - .schedule_task_cancel = scheduler_dp_task_cancel, -#else - .schedule_task_cancel = scheduler_dp_task_stop, -#endif - .schedule_task_free = scheduler_dp_task_free, -}; - -/* Runs on each core */ -__cold int scheduler_dp_init(void) -{ - assert_can_be_cold(); - - struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); - if (!dp_sch) - return -ENOMEM; - - memset(dp_sch, 0, sizeof(*dp_sch)); - list_init(&dp_sch->tasks); - - scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); - - /* init src of DP tick */ - return schedule_task_init_ll(&dp_sch->ll_tick_src, - SOF_UUID(dp_sched_uuid), - SOF_SCHEDULE_LL_TIMER, - 0, scheduler_dp_ll_tick_dummy, dp_sch, - cpu_get_id(), 0); -} - void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { unsigned int lock_key; From 421a25cdf7b4068373b8d7b20fa66d43ff84814b Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:29:27 +0000 Subject: [PATCH 5/9] schedule: split the kernel part of zephyr_dp_schedule_thread.c into src/kernel/schedule In the thread variant of the DP scheduler, scheduler_dp_task_init() allocates the DP thread, its stack and event, creates and pins the thread and, for user-space modules, grants it access to its objects and adds it to the module memory domain. Move it, and the kernel half of the scheduler_dp_internal_free() system call, which frees those objects, to src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c. zephyr_dp_schedule_thread.c keeps the DP thread function and the task readiness and deadline calculation. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/kernel/schedule/CMakeLists.txt | 5 + .../zephyr_dp_schedule_thread-kernel.c | 208 ++++++++++++++++++ src/schedule/zephyr_dp_schedule_thread.c | 177 +-------------- 3 files changed, 217 insertions(+), 173 deletions(-) create mode 100644 src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 5c878b19f3c3..4afc98e320bf 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -20,4 +20,9 @@ if(zephyr) zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER zephyr_dp_schedule-kernel.c ) + if(NOT CONFIG_SOF_USERSPACE_APPLICATION) + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule_thread-kernel.c + ) + endif() endif() diff --git a/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c b/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c new file mode 100644 index 000000000000..05c85ad98b1e --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + * Adrian Warecki + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; + +/* Kernel part of the thread variant of the DP scheduler: the DP thread creation and + * the scheduler_dp_internal_free() system call. The DP thread function is in + * src/schedule/zephyr_dp_schedule_thread.c + */ + +int scheduler_dp_task_init(struct task **task, + const struct sof_uuid_entry *uid, + const struct task_ops *ops, + struct processing_module *mod, + uint16_t core, + size_t stack_size, + uint32_t options) +{ + void __sparse_cache *p_stack = NULL; + struct k_heap *const user_heap = mod->dev->drv->user_heap; + + /* memory allocation helper structure */ + struct { + struct task task; /* keep first, used for freeing below */ + struct task_dp_pdata pdata; + } *task_memory; + + int ret; + + /* must be called on the same core the task will be binded to */ + assert(cpu_get_id() == core); + + /* + * allocate memory + * to avoid multiple malloc operations allocate all required memory as a single structure + * and return pointer to task_memory->task + * As the structure contains zephyr kernel specific data, it must be located in + * shared, non cached memory + */ + task_memory = sof_heap_alloc(user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*task_memory), 0); + if (!task_memory) { + tr_err(&dp_tr, "memory alloc failed"); + return -ENOMEM; + } + + memset(task_memory, 0, sizeof(*task_memory)); + /* allocate stack - must be aligned and cached so a separate alloc */ + p_stack = user_stack_allocate(stack_size, options); + if (!p_stack) { + tr_err(&dp_tr, "stack alloc failed"); + ret = -ENOMEM; + goto err; + } + + /* internal SOF task init */ + ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, + mod, core, options); + if (ret < 0) { + tr_err(&dp_tr, "schedule_task_init failed"); + goto err; + } + + struct task_dp_pdata *pdata = &task_memory->pdata; + + /* Point to event_struct event for kernel threads synchronization */ + /* It will be overwritten for K_USER threads to dynamic ones. */ + pdata->event = &pdata->event_struct; + pdata->thread = &pdata->thread_struct; + +#ifdef CONFIG_USERSPACE + if (options & K_USER) { + pdata->event = k_object_alloc(K_OBJ_EVENT); + if (!pdata->event) { + tr_err(&dp_tr, "Event object allocation failed"); + ret = -ENOMEM; + goto err; + } + + pdata->thread = k_object_alloc(K_OBJ_THREAD); + if (!pdata->thread) { + tr_err(&dp_tr, "Thread object allocation failed"); + ret = -ENOMEM; + goto err; + } + } +#endif /* CONFIG_USERSPACE */ + + /* initialize other task structures */ + task_memory->task.ops.complete = ops->complete; + task_memory->task.ops.get_deadline = ops->get_deadline; + task_memory->task.state = SOF_TASK_STATE_INIT; + task_memory->task.core = core; + task_memory->task.priv_data = pdata; + k_event_init(pdata->event); + + /* success, fill the structures */ + pdata->p_stack = p_stack; + pdata->mod = mod; + + /* create a zephyr thread for the task */ + pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)p_stack, + stack_size, dp_thread_fn, &task_memory->task, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, task_memory->task.flags, + K_FOREVER); + +#ifdef CONFIG_SCHED_CPU_MASK + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(pdata->thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } +#endif + +#ifdef CONFIG_USERSPACE + if (options & K_USER) { + k_thread_access_grant(pdata->thread_id, pdata->event); + scheduler_dp_grant(pdata->thread_id, core); + + ret = k_mem_domain_add_thread(pdata->mod->user_ctx->comp_dom, pdata->thread_id); + if (ret < 0) { + tr_err(&dp_tr, "k_mem_domain_add_thread() failed %d", ret); + goto e_thread; + } + +#if IS_ENABLED(CONFIG_SOF_USERSPACE_MOD_IPC_BY_DP_THREAD) + pdata->ipc_work_item = userspace_proxy_register_ipc_handler(mod, pdata->event); + assert(pdata->ipc_work_item); +#endif + } +#endif /* CONFIG_USERSPACE */ + + /* start the thread, it should immediately stop at an event */ + k_thread_start(pdata->thread_id); + + /* success, fill output parameter */ + *task = &task_memory->task; + return 0; + +e_thread: + k_thread_abort(pdata->thread_id); +err: + /* cleanup - free all allocated resources */ + if (user_stack_free((__sparse_force void *)p_stack)) + tr_err(&dp_tr, "user_stack_free failed!"); + + /* k_object_free looks for a pointer in the list, any invalid value can be passed */ + k_object_free(task_memory->pdata.event); + k_object_free(task_memory->pdata.thread); + sof_heap_free(user_heap, task_memory); + return ret; +} + +void z_impl_scheduler_dp_internal_free(struct task *task) +{ + struct task_dp_pdata *pdata = task->priv_data; + +#ifdef CONFIG_USERSPACE + if (pdata->event != &pdata->event_struct) + k_object_free(pdata->event); + if (pdata->thread != &pdata->thread_struct) + k_object_free(pdata->thread); +#endif + + /* task is the first member in task_memory above */ + sof_heap_free(pdata->mod->dev->drv->user_heap, task); +} + +#ifdef CONFIG_USERSPACE +#include + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + /* + * With the thread DP scheduler variant scheduler_dp_internal_free() is + * never called from the userspace context + */ + K_OOPS(true); +} +#include +#endif diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index 46cbadfeade3..db2311c3d90a 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -30,6 +30,10 @@ LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; +/* The DP thread creation and the scheduler_dp_internal_free() system call, which run + * in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c + */ + /* Go through all DP tasks and recalculate their readiness and deadlines * NOT REENTRANT, should be called with scheduler_dp_lock() */ @@ -214,176 +218,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3) if (task->state == SOF_TASK_STATE_COMPLETED) task_complete(task); } - -int scheduler_dp_task_init(struct task **task, - const struct sof_uuid_entry *uid, - const struct task_ops *ops, - struct processing_module *mod, - uint16_t core, - size_t stack_size, - uint32_t options) -{ - void __sparse_cache *p_stack = NULL; - struct k_heap *const user_heap = mod->dev->drv->user_heap; - - /* memory allocation helper structure */ - struct { - struct task task; /* keep first, used for freeing below */ - struct task_dp_pdata pdata; - } *task_memory; - - int ret; - - /* must be called on the same core the task will be binded to */ - assert(cpu_get_id() == core); - - /* - * allocate memory - * to avoid multiple malloc operations allocate all required memory as a single structure - * and return pointer to task_memory->task - * As the structure contains zephyr kernel specific data, it must be located in - * shared, non cached memory - */ - task_memory = sof_heap_alloc(user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory), 0); - if (!task_memory) { - tr_err(&dp_tr, "memory alloc failed"); - return -ENOMEM; - } - - memset(task_memory, 0, sizeof(*task_memory)); - /* allocate stack - must be aligned and cached so a separate alloc */ - p_stack = user_stack_allocate(stack_size, options); - if (!p_stack) { - tr_err(&dp_tr, "stack alloc failed"); - ret = -ENOMEM; - goto err; - } - - /* internal SOF task init */ - ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, - mod, core, options); - if (ret < 0) { - tr_err(&dp_tr, "schedule_task_init failed"); - goto err; - } - - struct task_dp_pdata *pdata = &task_memory->pdata; - - /* Point to event_struct event for kernel threads synchronization */ - /* It will be overwritten for K_USER threads to dynamic ones. */ - pdata->event = &pdata->event_struct; - pdata->thread = &pdata->thread_struct; - -#ifdef CONFIG_USERSPACE - if (options & K_USER) { - pdata->event = k_object_alloc(K_OBJ_EVENT); - if (!pdata->event) { - tr_err(&dp_tr, "Event object allocation failed"); - ret = -ENOMEM; - goto err; - } - - pdata->thread = k_object_alloc(K_OBJ_THREAD); - if (!pdata->thread) { - tr_err(&dp_tr, "Thread object allocation failed"); - ret = -ENOMEM; - goto err; - } - } -#endif /* CONFIG_USERSPACE */ - - /* initialize other task structures */ - task_memory->task.ops.complete = ops->complete; - task_memory->task.ops.get_deadline = ops->get_deadline; - task_memory->task.state = SOF_TASK_STATE_INIT; - task_memory->task.core = core; - task_memory->task.priv_data = pdata; - k_event_init(pdata->event); - - /* success, fill the structures */ - pdata->p_stack = p_stack; - pdata->mod = mod; - - /* create a zephyr thread for the task */ - pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)p_stack, - stack_size, dp_thread_fn, &task_memory->task, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, task_memory->task.flags, - K_FOREVER); - -#ifdef CONFIG_SCHED_CPU_MASK - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(pdata->thread_id, core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto e_thread; - } -#endif - -#ifdef CONFIG_USERSPACE - if (options & K_USER) { - k_thread_access_grant(pdata->thread_id, pdata->event); - scheduler_dp_grant(pdata->thread_id, core); - - ret = k_mem_domain_add_thread(pdata->mod->user_ctx->comp_dom, pdata->thread_id); - if (ret < 0) { - tr_err(&dp_tr, "k_mem_domain_add_thread() failed %d", ret); - goto e_thread; - } - -#if IS_ENABLED(CONFIG_SOF_USERSPACE_MOD_IPC_BY_DP_THREAD) - pdata->ipc_work_item = userspace_proxy_register_ipc_handler(mod, pdata->event); - assert(pdata->ipc_work_item); -#endif - } -#endif /* CONFIG_USERSPACE */ - - /* start the thread, it should immediately stop at an event */ - k_thread_start(pdata->thread_id); - - /* success, fill output parameter */ - *task = &task_memory->task; - return 0; - -e_thread: - k_thread_abort(pdata->thread_id); -err: - /* cleanup - free all allocated resources */ - if (user_stack_free((__sparse_force void *)p_stack)) - tr_err(&dp_tr, "user_stack_free failed!"); - - /* k_object_free looks for a pointer in the list, any invalid value can be passed */ - k_object_free(task_memory->pdata.event); - k_object_free(task_memory->pdata.thread); - sof_heap_free(user_heap, task_memory); - return ret; -} - -void z_impl_scheduler_dp_internal_free(struct task *task) -{ - struct task_dp_pdata *pdata = task->priv_data; - -#ifdef CONFIG_USERSPACE - if (pdata->event != &pdata->event_struct) - k_object_free(pdata->event); - if (pdata->thread != &pdata->thread_struct) - k_object_free(pdata->thread); -#endif - - /* task is the first member in task_memory above */ - sof_heap_free(pdata->mod->dev->drv->user_heap, task); -} - -#ifdef CONFIG_USERSPACE -#include - -void z_vrfy_scheduler_dp_internal_free(struct task *task) -{ - /* - * With the thread DP scheduler variant scheduler_dp_internal_free() is - * never called from the userspace context - */ - K_OOPS(true); -} -#include -#endif From 85408823e4d45d5a55d7acb0afba2fe07176b43b Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:30:22 +0000 Subject: [PATCH 6/9] schedule: split the kernel part of zephyr_dp_schedule_application.c into src/kernel/schedule In the application variant of the DP scheduler, scheduler_dp_task_init() allocates the DP thread and its kernel objects, sets up the memory domain of the module, grants the DP, IPC and LL threads access, and records the kernel objects of the task in a kernel-only list. Move it to src/kernel/schedule/zephyr_dp_schedule_application-kernel.c, with: - the kernel object record list and its helpers; - the memory domain pool and teardown; - the kernel half of the scheduler_dp_internal_free() system call and its verification of the caller; - the per-core DP thread synchronization semaphores, which are kernel objects. zephyr_dp_schedule_application.c keeps the IPC packing and unpacking, scheduler_dp_thread_ipc(), the task readiness and deadline calculation and the DP thread function. The task memory layout and the packed IPC structure, which both parts use, and the semaphores are declared in sof/schedule/zephyr_dp_schedule.h. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/zephyr_dp_schedule.h | 57 ++ src/kernel/schedule/CMakeLists.txt | 6 +- .../zephyr_dp_schedule_application-kernel.c | 536 +++++++++++++++++ src/schedule/zephyr_dp_schedule_application.c | 545 +----------------- 4 files changed, 601 insertions(+), 543 deletions(-) create mode 100644 src/kernel/schedule/zephyr_dp_schedule_application-kernel.c diff --git a/src/include/sof/schedule/zephyr_dp_schedule.h b/src/include/sof/schedule/zephyr_dp_schedule.h index 1056efbfec4c..582193a6a816 100644 --- a/src/include/sof/schedule/zephyr_dp_schedule.h +++ b/src/include/sof/schedule/zephyr_dp_schedule.h @@ -18,10 +18,14 @@ #define __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ #include +#include #include #include #include +#include +#include + #include #include @@ -72,6 +76,59 @@ struct task_dp_pdata { #endif }; +#if CONFIG_SOF_USERSPACE_APPLICATION +/** @brief Per-core DP thread start and IPC completion semaphores. */ +extern struct k_sem dp_sync[CONFIG_CORE_COUNT]; + +struct ipc4_flat { + unsigned int cmd; + int ret; + union { + struct { + struct ipc4_module_bind_unbind bu; + enum bind_type type; + } bind; + struct { + unsigned int trigger_cmd; + enum ipc4_pipeline_state state; + int n_sources; + int n_sinks; + struct sof_source *source[CONFIG_MODULE_MAX_CONNECTIONS]; + struct sof_sink *sink[CONFIG_MODULE_MAX_CONNECTIONS]; + } pipeline_state; + /* + * SOF_IPC4_MOD_INIT_INSTANCE: a by-value copy of the caller's + * ext_data. The original lives on the calling thread's stack, + * which the DP thread's memory domain doesn't have access to. + * The pointers it contains reference the IPC mailbox, which stays + * valid and is covered by the DP thread's SOF_DP_PART_CFG partition. + */ + struct module_ext_init_data init_instance; + struct { + uint32_t param_id; + enum module_cfg_fragment_position position; + uint32_t data_offset_size; + size_t fragment_size; + const char *data; + } set_config; + struct { + uint32_t param_id; + uint32_t data_offset_size; + size_t fragment_size; + char *data; + } get_config; + }; +}; + +/* memory allocation helper structure */ +struct scheduler_dp_task_memory { + struct task task; + struct task_dp_pdata pdata; + struct comp_driver drv; + struct ipc4_flat flat; +}; + +#endif /* CONFIG_SOF_USERSPACE_APPLICATION */ /** @brief Per-core DP scheduler locks, see scheduler_dp_lock(). */ extern struct k_sem dp_lock[CONFIG_MP_MAX_NUM_CPUS]; diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 4afc98e320bf..f49fc2d5f869 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -20,7 +20,11 @@ if(zephyr) zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER zephyr_dp_schedule-kernel.c ) - if(NOT CONFIG_SOF_USERSPACE_APPLICATION) + if(CONFIG_SOF_USERSPACE_APPLICATION) + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule_application-kernel.c + ) + else() zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER zephyr_dp_schedule_thread-kernel.c ) diff --git a/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c b/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c new file mode 100644 index 000000000000..a783b90b4ab3 --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; + +/* Kernel part of the application variant of the DP scheduler: the DP thread creation, + * its memory domain, the kernel-only record of its kernel objects and the + * scheduler_dp_internal_free() system call. The DP thread function and the IPC + * forwarding to it are in src/schedule/zephyr_dp_schedule_application.c + */ + +static struct objpool_head dp_mdom_head = {.list = LIST_INIT(dp_mdom_head.list)}; + +/* Synchronization semaphore for the scheduler thread to wait for DP startup */ +#define DP_SYNC_INIT(i, _) Z_SEM_INITIALIZER(dp_sync[i], 0, 1) +#define DP_SYNC_INIT_LIST LISTIFY(CONFIG_CORE_COUNT, DP_SYNC_INIT, (,)) +STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_sync, CONFIG_CORE_COUNT) = { DP_SYNC_INIT_LIST }; + +/* + * Kernel-only record of the kernel objects owned by a DP task. + * + * struct scheduler_dp_task_memory lives in the module's heap, which is mapped + * into the DP thread's memory domain, and is also writable by the LL and IPC + * user threads. Freeing must therefore never use the pointers stored there: + * scheduler_dp_internal_free() takes them from this record instead. The task + * pointer is only used as a lookup key. + */ +struct scheduler_dp_task_kobj { + struct list_item list; + struct task *task; + struct processing_module *mod; + struct k_event *event; + struct k_thread *thread; + struct k_mem_domain *mdom; + struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; + uint32_t comp_id; + uint32_t gen; /* unique per record, see dp_task_kobj_take() */ +}; + +static struct list_item dp_task_kobj_list = LIST_INIT(dp_task_kobj_list); +static struct k_spinlock dp_task_kobj_lock; +/* generation of the last added record, under dp_task_kobj_lock */ +static uint32_t dp_task_kobj_gen; + +static void dp_task_kobj_add(struct scheduler_dp_task_kobj *kobj) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + + kobj->gen = ++dp_task_kobj_gen; + list_item_append(&kobj->list, &dp_task_kobj_list); + k_spin_unlock(&dp_task_kobj_lock, key); +} + +/* Find the record for @task, caller holds dp_task_kobj_lock */ +static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task) +{ + struct scheduler_dp_task_kobj *kobj; + struct list_item *item; + + list_for_item(item, &dp_task_kobj_list) { + kobj = container_of(item, struct scheduler_dp_task_kobj, list); + if (kobj->task == task) + return kobj; + } + + return NULL; +} + +/* Copy the record for @task to @copy without unlinking it */ +static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); + + if (kobj) + *copy = *kobj; + + k_spin_unlock(&dp_task_kobj_lock, key); + + return kobj != NULL; +} + +/* + * Find the record for @task and unlink it, so it can only be freed once. + * If @expected is given, only take the record it was copied from, so a + * record replaced since @expected was peeked is left alone. + */ +static struct scheduler_dp_task_kobj * +dp_task_kobj_take(const struct task *task, const struct scheduler_dp_task_kobj *expected) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); + + /* + * Compare the generation, not the object addresses: the task could + * have been freed and recreated at the same addresses meanwhile. + */ + if (kobj && expected && kobj->gen != expected->gen) + kobj = NULL; + + if (kobj) + list_item_del(&kobj->list); + + k_spin_unlock(&dp_task_kobj_lock, key); + + return kobj; +} + +/* + * Safe to call with partial successful initialisation, + * k_mem_domain_remove_partition() then just returns -ENOENT + */ +static void scheduler_dp_domain_free(struct processing_module *pmod, struct k_mem_domain *mdom, + struct k_mem_partition *mpart, uint32_t comp_id) +{ + llext_manager_rm_domain(comp_id, mdom); + + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP_CACHE); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG_CACHE); + + if (mdom->num_partitions) { + /* Violation: all partitions should be freed by now */ + unsigned int n = mdom->num_partitions; + + LOG_WRN("%u domain partition(s) not freed, force-freeing them all", n); + + for (unsigned int i = 0; i < arch_mem_domain_max_partitions_get() && n; i++) { + struct k_mem_partition *dpart = &mdom->partitions[i]; + + if (dpart->size) { + k_mem_domain_remove_partition(mdom, dpart); + n--; + } + } + } + + /* All partitions removed, the domain can be freed now */ + pmod->mdom = NULL; + k_mem_domain_deinit(mdom); + objpool_free(&dp_mdom_head, mdom); +} + +/* + * Free the kernel objects of @task, using only the kernel-only record. + * + * @return false if @task has no record (unknown task or already freed) + */ +static bool scheduler_dp_internal_free_kobj(struct task *task, + const struct scheduler_dp_task_kobj *expected) +{ + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_take(task, expected); + + if (!kobj) + return false; + + /* normally already terminated by scheduler_dp_task_free(), no-op then */ + k_thread_abort(kobj->thread); + k_object_free(kobj->event); + k_object_free(kobj->thread); + scheduler_dp_domain_free(kobj->mod, kobj->mdom, kobj->mpart, kobj->comp_id); + + mod_free(kobj->mod, container_of(task, struct scheduler_dp_task_memory, task)); + rfree(kobj); + + return true; +} + +void z_impl_scheduler_dp_internal_free(struct task *task) +{ + if (!scheduler_dp_internal_free_kobj(task, NULL)) + tr_err(&dp_tr, "unknown DP task %p", task); +} + +#ifdef CONFIG_THREAD_NAME +static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_module *mod) +{ + char name[CONFIG_THREAD_MAX_NAME_LEN]; + + snprintf(name, sizeof(name), "DP:%#x", mod->dev->ipc_config.id); + + k_thread_name_set(thread_id, name); +} +#else +/* k_thread_name_set() is a no-op so skip constructing a thread name */ +#define scheduler_dp_thread_name_set(x, y) +#endif + +/* Called only in IPC context in kernel mode (this can change) */ +int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, + const struct task_ops *ops, struct processing_module *mod, + uint16_t core, size_t stack_size, uint32_t options) +{ + k_thread_stack_t *p_stack; + struct scheduler_dp_task_memory *task_memory; + uint8_t *ipc_config_data = NULL; + k_tid_t thread_id; + uint32_t comp_id; + + int ret; + + /* must be called on the same core the task will be bound to */ + assert(cpu_get_id() == core); + assert(!k_is_user_context()); + + /* + * allocate memory + * to avoid multiple malloc operations allocate all required memory as a single structure + * and return pointer to task_memory->task + * As the structure contains zephyr kernel specific data, it must be located in + * shared, non cached memory + */ + task_memory = mod_alloc_ext(mod, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*task_memory), 0); + if (!task_memory) { + tr_err(&dp_tr, "memory alloc failed"); + return -ENOMEM; + } + + memset(task_memory, 0, sizeof(*task_memory)); + + const struct comp_driver *drv = mod->dev->drv; + + task_memory->drv = *mod->dev->drv; + mod->dev->drv = &task_memory->drv; + + /* allocate stack - must be aligned and cached so a separate alloc */ + p_stack = user_stack_allocate(stack_size, options); + if (!p_stack) { + tr_err(&dp_tr, "stack alloc failed"); + ret = -ENOMEM; + goto e_tmem; + } + + struct task *ptask = &task_memory->task; + + /* internal SOF task init */ + ret = schedule_task_init(ptask, uid, SOF_SCHEDULE_DP, 0, ops->run, mod, core, options); + if (ret < 0) { + tr_err(&dp_tr, "schedule_task_init failed"); + goto e_stack; + } + + /* + * task_memory, including pdata, is in the module heap, which user + * threads can write while this function runs. Keep everything the + * kernel uses here or at teardown in kobj and locals, and only mirror + * it into pdata for the DP thread. + */ + struct task_dp_pdata *pdata = &task_memory->pdata; + struct scheduler_dp_task_kobj *kobj = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*kobj)); + + if (!kobj) { + tr_err(&dp_tr, "task record alloc failed"); + ret = -ENOMEM; + goto e_stack; + } + + kobj->task = ptask; + kobj->mod = mod; + comp_id = mod->dev->ipc_config.id; + kobj->comp_id = comp_id; + + if (mod->dev->drv->adapter_ops->get_configuration) { + ipc_config_data = mod_alloc_ext(mod, SOF_MEM_FLAG_USER, SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc_config_data) { + tr_err(&dp_tr, "Failed to allocate memory for get-config"); + ret = -ENOMEM; + goto e_kobj; + } + } + pdata->ipc_config_data = ipc_config_data; + + pdata->flat = &task_memory->flat; + + kobj->event = k_object_alloc(K_OBJ_EVENT); + if (!kobj->event) { + tr_err(&dp_tr, "Event object allocation failed"); + ret = -ENOMEM; + goto e_config; + } + + pdata->event = kobj->event; + + kobj->thread = k_object_alloc(K_OBJ_THREAD); + if (!kobj->thread) { + tr_err(&dp_tr, "Thread object allocation failed"); + ret = -ENOMEM; + goto e_event; + } + memset(&kobj->thread->arch, 0, sizeof(kobj->thread->arch)); + pdata->thread = kobj->thread; + + /* success, fill the structures */ + pdata->p_stack = p_stack; + pdata->mod = mod; + + /* initialize other task structures */ + ptask->ops.complete = ops->complete; + ptask->ops.get_deadline = ops->get_deadline; + ptask->priv_data = pdata; + list_init(&ptask->list); + + /* create a zephyr thread for the task */ + thread_id = k_thread_create(kobj->thread, p_stack, + stack_size, dp_thread_fn, ptask, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, ptask->flags, K_FOREVER); + pdata->thread_id = thread_id; + scheduler_dp_thread_name_set(thread_id, mod); + + unsigned int pidx; + size_t size; + uintptr_t start; + bool on_pool = false; + +#ifdef CONFIG_SCHED_CPU_MASK + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } +#endif + + k_thread_access_grant(thread_id, kobj->event, &dp_sync[core]); + scheduler_dp_grant(thread_id, core); +#if CONFIG_SOF_USERSPACE_LL + struct k_thread *thread_ipc = ipc_thread_user(core); + + k_thread_access_grant(thread_ipc, kobj->event, thread_id, p_stack, &dp_sync[core]); + scheduler_dp_grant(thread_ipc, core); + scheduler_dp_grant(zephyr_ll_domain_thread(), core); +#endif + + struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), + SOF_MEM_FLAG_COHERENT); + + if (!mdom) { + tr_err(&dp_tr, "objpool allocation failed"); + ret = -ENOMEM; + goto e_thread; + } + + on_pool = true; + + mod->mdom = mdom; + kobj->mdom = mdom; + + if (!mdom->arch.ptables) { + ret = k_mem_domain_init(mdom, 0, NULL); + if (ret < 0) + goto e_thread; + } + + /* Module heap partition */ + mod_heap_info(mod, &size, &start); + kobj->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)start), + .size = size, + .attr = K_MEM_PARTITION_P_RW_U_RW, + }; + kobj->mpart[SOF_DP_PART_HEAP_CACHE] = (struct k_mem_partition){ + .start = start, + .size = size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + /* Host mailbox partition for additional IPC parameters: read-only */ + kobj->mpart[SOF_DP_PART_CFG] = (struct k_mem_partition){ + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)MAILBOX_HOSTBOX_BASE), + .size = MAILBOX_HOSTBOX_SIZE, + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + kobj->mpart[SOF_DP_PART_CFG_CACHE] = (struct k_mem_partition){ + .start = (uintptr_t)MAILBOX_HOSTBOX_BASE, + .size = MAILBOX_HOSTBOX_SIZE, + .attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB, + }; + + for (pidx = 0; pidx < SOF_DP_PART_TYPE_COUNT; pidx++) { + ret = k_mem_domain_add_partition(mdom, kobj->mpart + pidx); + if (ret < 0) + goto e_dom; + } + + memcpy(pdata->mpart, kobj->mpart, sizeof(pdata->mpart)); + + ret = llext_manager_add_domain(comp_id, mdom); + if (ret < 0) { + tr_err(&dp_tr, "failed to add LLEXT to domain %d", ret); + goto e_dom; + } + + /* + * Keep this call last, able to fail, otherwise domain will be removed + * before its thread + */ + ret = k_mem_domain_add_thread(mdom, thread_id); + if (ret < 0) { + tr_err(&dp_tr, "failed to add thread to domain %d", ret); + goto e_dom; + } + + /* publish the record before the thread can run */ + dp_task_kobj_add(kobj); + + /* start the thread, it should immediately stop at the semaphore */ + k_event_init(kobj->event); + k_thread_start(thread_id); + + /* ptask points into task_memory, so only publish it once it cannot be freed */ + *task = ptask; + + return 0; + +e_dom: + scheduler_dp_domain_free(mod, mdom, kobj->mpart, comp_id); + on_pool = false; +e_thread: + if (on_pool) + objpool_free(&dp_mdom_head, mdom); + k_thread_abort(thread_id); + k_object_free(kobj->thread); +e_event: + k_object_free(kobj->event); +e_config: + mod_free(mod, ipc_config_data); +e_kobj: + rfree(kobj); +e_stack: + user_stack_free(p_stack); +e_tmem: + /* the copy lives in task_memory, so stop pointing at it before freeing */ + mod->dev->drv = drv; + mod_free(mod, task_memory); + return ret; +} + +#ifdef CONFIG_USERSPACE +#include +#include + +/* + * mod_free() follows the module's resource and allocator pointers in + * supervisor mode. They are user-writable, so at least check that they + * point to memory the caller can access. + */ +static void scheduler_dp_mod_vrfy(struct processing_module *mod) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); + K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); + + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc, sizeof(*alloc))); + if (alloc->heap) { + size_t h_size = 0; + uintptr_t h_start; + + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); + } + if (alloc->vreg) { + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(alloc->vreg, &vr_size, &vr_start); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); + } +} + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + struct scheduler_dp_task_kobj kobj; + + SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); + + /* + * Do not read task->priv_data: the kernel objects come from the + * kernel-only record, and an unknown task pointer is rejected. + */ + K_OOPS(!dp_task_kobj_peek(task, &kobj)); + + /* + * A DP thread holds its own thread and event objects, so it would pass + * the checks below for its own task. Freeing would then abort the + * caller after the record is unlinked, leaking everything it holds. + */ + K_OOPS(kobj.thread == k_current_get()); + + /* + * The caller must have been granted the task's objects. Check before + * unlinking the record, so an unauthorized call leaves it intact. + */ + K_OOPS(K_SYSCALL_OBJ(kobj.event, K_OBJ_EVENT)); + K_OOPS(K_SYSCALL_OBJ_INIT(kobj.thread, K_OBJ_THREAD)); + scheduler_dp_mod_vrfy(kobj.mod); + + K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj)); +} +#include +#endif /* CONFIG_USERSPACE */ diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 54885466e9eb..0d7b8a522cee 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -35,52 +35,9 @@ LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; -static struct objpool_head dp_mdom_head = {.list = LIST_INIT(dp_mdom_head.list)}; - -/* Synchronization semaphore for the scheduler thread to wait for DP startup */ -#define DP_SYNC_INIT(i, _) Z_SEM_INITIALIZER(dp_sync[i], 0, 1) -#define DP_SYNC_INIT_LIST LISTIFY(CONFIG_CORE_COUNT, DP_SYNC_INIT, (,)) -static STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_sync, CONFIG_CORE_COUNT) = { DP_SYNC_INIT_LIST }; - -struct ipc4_flat { - unsigned int cmd; - int ret; - union { - struct { - struct ipc4_module_bind_unbind bu; - enum bind_type type; - } bind; - struct { - unsigned int trigger_cmd; - enum ipc4_pipeline_state state; - int n_sources; - int n_sinks; - struct sof_source *source[CONFIG_MODULE_MAX_CONNECTIONS]; - struct sof_sink *sink[CONFIG_MODULE_MAX_CONNECTIONS]; - } pipeline_state; - /* - * SOF_IPC4_MOD_INIT_INSTANCE: a by-value copy of the caller's - * ext_data. The original lives on the calling thread's stack, - * which the DP thread's memory domain doesn't have access to. - * The pointers it contains reference the IPC mailbox, which stays - * valid and is covered by the DP thread's SOF_DP_PART_CFG partition. - */ - struct module_ext_init_data init_instance; - struct { - uint32_t param_id; - enum module_cfg_fragment_position position; - uint32_t data_offset_size; - size_t fragment_size; - const char *data; - } set_config; - struct { - uint32_t param_id; - uint32_t data_offset_size; - size_t fragment_size; - char *data; - } get_config; - }; -}; +/* The DP thread creation and the scheduler_dp_internal_free() system call, which run + * in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule_application-kernel.c + */ /* Pack IPC input data */ static int ipc_thread_flatten(unsigned int cmd, const union scheduler_dp_thread_ipc_param *param, @@ -143,104 +100,6 @@ static int ipc_thread_flatten(unsigned int cmd, const union scheduler_dp_thread_ return 0; } -/* memory allocation helper structure */ -struct scheduler_dp_task_memory { - struct task task; - struct task_dp_pdata pdata; - struct comp_driver drv; - struct ipc4_flat flat; -}; - -/* - * Kernel-only record of the kernel objects owned by a DP task. - * - * struct scheduler_dp_task_memory lives in the module's heap, which is mapped - * into the DP thread's memory domain, and is also writable by the LL and IPC - * user threads. Freeing must therefore never use the pointers stored there: - * scheduler_dp_internal_free() takes them from this record instead. The task - * pointer is only used as a lookup key. - */ -struct scheduler_dp_task_kobj { - struct list_item list; - struct task *task; - struct processing_module *mod; - struct k_event *event; - struct k_thread *thread; - struct k_mem_domain *mdom; - struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; - uint32_t comp_id; - uint32_t gen; /* unique per record, see dp_task_kobj_take() */ -}; - -static struct list_item dp_task_kobj_list = LIST_INIT(dp_task_kobj_list); -static struct k_spinlock dp_task_kobj_lock; -/* generation of the last added record, under dp_task_kobj_lock */ -static uint32_t dp_task_kobj_gen; - -static void dp_task_kobj_add(struct scheduler_dp_task_kobj *kobj) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - - kobj->gen = ++dp_task_kobj_gen; - list_item_append(&kobj->list, &dp_task_kobj_list); - k_spin_unlock(&dp_task_kobj_lock, key); -} - -/* Find the record for @task, caller holds dp_task_kobj_lock */ -static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task) -{ - struct scheduler_dp_task_kobj *kobj; - struct list_item *item; - - list_for_item(item, &dp_task_kobj_list) { - kobj = container_of(item, struct scheduler_dp_task_kobj, list); - if (kobj->task == task) - return kobj; - } - - return NULL; -} - -/* Copy the record for @task to @copy without unlinking it */ -static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); - - if (kobj) - *copy = *kobj; - - k_spin_unlock(&dp_task_kobj_lock, key); - - return kobj != NULL; -} - -/* - * Find the record for @task and unlink it, so it can only be freed once. - * If @expected is given, only take the record it was copied from, so a - * record replaced since @expected was peeked is left alone. - */ -static struct scheduler_dp_task_kobj * -dp_task_kobj_take(const struct task *task, const struct scheduler_dp_task_kobj *expected) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); - - /* - * Compare the generation, not the object addresses: the task could - * have been freed and recreated at the same addresses meanwhile. - */ - if (kobj && expected && kobj->gen != expected->gen) - kobj = NULL; - - if (kobj) - list_item_del(&kobj->list); - - k_spin_unlock(&dp_task_kobj_lock, key); - - return kobj; -} - /* Unpack IPC data and execute a callback */ static void ipc_thread_unflatten_run(struct processing_module *pmod, struct ipc4_flat *flat) { @@ -547,401 +406,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3) if (task->state == SOF_TASK_STATE_COMPLETED) task_complete(task); } - -/* - * Safe to call with partial successful initialisation, - * k_mem_domain_remove_partition() then just returns -ENOENT - */ -static void scheduler_dp_domain_free(struct processing_module *pmod, struct k_mem_domain *mdom, - struct k_mem_partition *mpart, uint32_t comp_id) -{ - llext_manager_rm_domain(comp_id, mdom); - - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP_CACHE); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG_CACHE); - - if (mdom->num_partitions) { - /* Violation: all partitions should be freed by now */ - unsigned int n = mdom->num_partitions; - - LOG_WRN("%u domain partition(s) not freed, force-freeing them all", n); - - for (unsigned int i = 0; i < arch_mem_domain_max_partitions_get() && n; i++) { - struct k_mem_partition *dpart = &mdom->partitions[i]; - - if (dpart->size) { - k_mem_domain_remove_partition(mdom, dpart); - n--; - } - } - } - - /* All partitions removed, the domain can be freed now */ - pmod->mdom = NULL; - k_mem_domain_deinit(mdom); - objpool_free(&dp_mdom_head, mdom); -} - -/* - * Free the kernel objects of @task, using only the kernel-only record. - * - * @return false if @task has no record (unknown task or already freed) - */ -static bool scheduler_dp_internal_free_kobj(struct task *task, - const struct scheduler_dp_task_kobj *expected) -{ - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_take(task, expected); - - if (!kobj) - return false; - - /* normally already terminated by scheduler_dp_task_free(), no-op then */ - k_thread_abort(kobj->thread); - k_object_free(kobj->event); - k_object_free(kobj->thread); - scheduler_dp_domain_free(kobj->mod, kobj->mdom, kobj->mpart, kobj->comp_id); - - mod_free(kobj->mod, container_of(task, struct scheduler_dp_task_memory, task)); - rfree(kobj); - - return true; -} - -void z_impl_scheduler_dp_internal_free(struct task *task) -{ - if (!scheduler_dp_internal_free_kobj(task, NULL)) - tr_err(&dp_tr, "unknown DP task %p", task); -} - -#ifdef CONFIG_THREAD_NAME -static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_module *mod) -{ - char name[CONFIG_THREAD_MAX_NAME_LEN]; - - snprintf(name, sizeof(name), "DP:%#x", mod->dev->ipc_config.id); - - k_thread_name_set(thread_id, name); -} -#else -/* k_thread_name_set() is a no-op so skip constructing a thread name */ -#define scheduler_dp_thread_name_set(x, y) -#endif - -/* Called only in IPC context in kernel mode (this can change) */ -int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, - const struct task_ops *ops, struct processing_module *mod, - uint16_t core, size_t stack_size, uint32_t options) -{ - k_thread_stack_t *p_stack; - struct scheduler_dp_task_memory *task_memory; - uint8_t *ipc_config_data = NULL; - k_tid_t thread_id; - uint32_t comp_id; - - int ret; - - /* must be called on the same core the task will be bound to */ - assert(cpu_get_id() == core); - assert(!k_is_user_context()); - - /* - * allocate memory - * to avoid multiple malloc operations allocate all required memory as a single structure - * and return pointer to task_memory->task - * As the structure contains zephyr kernel specific data, it must be located in - * shared, non cached memory - */ - task_memory = mod_alloc_ext(mod, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory), 0); - if (!task_memory) { - tr_err(&dp_tr, "memory alloc failed"); - return -ENOMEM; - } - - memset(task_memory, 0, sizeof(*task_memory)); - - const struct comp_driver *drv = mod->dev->drv; - - task_memory->drv = *mod->dev->drv; - mod->dev->drv = &task_memory->drv; - - /* allocate stack - must be aligned and cached so a separate alloc */ - p_stack = user_stack_allocate(stack_size, options); - if (!p_stack) { - tr_err(&dp_tr, "stack alloc failed"); - ret = -ENOMEM; - goto e_tmem; - } - - struct task *ptask = &task_memory->task; - - /* internal SOF task init */ - ret = schedule_task_init(ptask, uid, SOF_SCHEDULE_DP, 0, ops->run, mod, core, options); - if (ret < 0) { - tr_err(&dp_tr, "schedule_task_init failed"); - goto e_stack; - } - - /* - * task_memory, including pdata, is in the module heap, which user - * threads can write while this function runs. Keep everything the - * kernel uses here or at teardown in kobj and locals, and only mirror - * it into pdata for the DP thread. - */ - struct task_dp_pdata *pdata = &task_memory->pdata; - struct scheduler_dp_task_kobj *kobj = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*kobj)); - - if (!kobj) { - tr_err(&dp_tr, "task record alloc failed"); - ret = -ENOMEM; - goto e_stack; - } - - kobj->task = ptask; - kobj->mod = mod; - comp_id = mod->dev->ipc_config.id; - kobj->comp_id = comp_id; - - if (mod->dev->drv->adapter_ops->get_configuration) { - ipc_config_data = mod_alloc_ext(mod, SOF_MEM_FLAG_USER, SOF_IPC_MSG_MAX_SIZE, 0); - if (!ipc_config_data) { - tr_err(&dp_tr, "Failed to allocate memory for get-config"); - ret = -ENOMEM; - goto e_kobj; - } - } - pdata->ipc_config_data = ipc_config_data; - - pdata->flat = &task_memory->flat; - - kobj->event = k_object_alloc(K_OBJ_EVENT); - if (!kobj->event) { - tr_err(&dp_tr, "Event object allocation failed"); - ret = -ENOMEM; - goto e_config; - } - - pdata->event = kobj->event; - - kobj->thread = k_object_alloc(K_OBJ_THREAD); - if (!kobj->thread) { - tr_err(&dp_tr, "Thread object allocation failed"); - ret = -ENOMEM; - goto e_event; - } - memset(&kobj->thread->arch, 0, sizeof(kobj->thread->arch)); - pdata->thread = kobj->thread; - - /* success, fill the structures */ - pdata->p_stack = p_stack; - pdata->mod = mod; - - /* initialize other task structures */ - ptask->ops.complete = ops->complete; - ptask->ops.get_deadline = ops->get_deadline; - ptask->priv_data = pdata; - list_init(&ptask->list); - - /* create a zephyr thread for the task */ - thread_id = k_thread_create(kobj->thread, p_stack, - stack_size, dp_thread_fn, ptask, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, ptask->flags, K_FOREVER); - pdata->thread_id = thread_id; - scheduler_dp_thread_name_set(thread_id, mod); - - unsigned int pidx; - size_t size; - uintptr_t start; - bool on_pool = false; - -#ifdef CONFIG_SCHED_CPU_MASK - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(thread_id, core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto e_thread; - } -#endif - - k_thread_access_grant(thread_id, kobj->event, &dp_sync[core]); - scheduler_dp_grant(thread_id, core); -#if CONFIG_SOF_USERSPACE_LL - struct k_thread *thread_ipc = ipc_thread_user(core); - - k_thread_access_grant(thread_ipc, kobj->event, thread_id, p_stack, &dp_sync[core]); - scheduler_dp_grant(thread_ipc, core); - scheduler_dp_grant(zephyr_ll_domain_thread(), core); -#endif - - struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), - SOF_MEM_FLAG_COHERENT); - - if (!mdom) { - tr_err(&dp_tr, "objpool allocation failed"); - ret = -ENOMEM; - goto e_thread; - } - - on_pool = true; - - mod->mdom = mdom; - kobj->mdom = mdom; - - if (!mdom->arch.ptables) { - ret = k_mem_domain_init(mdom, 0, NULL); - if (ret < 0) - goto e_thread; - } - - /* Module heap partition */ - mod_heap_info(mod, &size, &start); - kobj->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ - .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)start), - .size = size, - .attr = K_MEM_PARTITION_P_RW_U_RW, - }; - kobj->mpart[SOF_DP_PART_HEAP_CACHE] = (struct k_mem_partition){ - .start = start, - .size = size, - .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, - }; - /* Host mailbox partition for additional IPC parameters: read-only */ - kobj->mpart[SOF_DP_PART_CFG] = (struct k_mem_partition){ - .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)MAILBOX_HOSTBOX_BASE), - .size = MAILBOX_HOSTBOX_SIZE, - .attr = K_MEM_PARTITION_P_RO_U_RO, - }; - kobj->mpart[SOF_DP_PART_CFG_CACHE] = (struct k_mem_partition){ - .start = (uintptr_t)MAILBOX_HOSTBOX_BASE, - .size = MAILBOX_HOSTBOX_SIZE, - .attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB, - }; - - for (pidx = 0; pidx < SOF_DP_PART_TYPE_COUNT; pidx++) { - ret = k_mem_domain_add_partition(mdom, kobj->mpart + pidx); - if (ret < 0) - goto e_dom; - } - - memcpy(pdata->mpart, kobj->mpart, sizeof(pdata->mpart)); - - ret = llext_manager_add_domain(comp_id, mdom); - if (ret < 0) { - tr_err(&dp_tr, "failed to add LLEXT to domain %d", ret); - goto e_dom; - } - - /* - * Keep this call last, able to fail, otherwise domain will be removed - * before its thread - */ - ret = k_mem_domain_add_thread(mdom, thread_id); - if (ret < 0) { - tr_err(&dp_tr, "failed to add thread to domain %d", ret); - goto e_dom; - } - - /* publish the record before the thread can run */ - dp_task_kobj_add(kobj); - - /* start the thread, it should immediately stop at the semaphore */ - k_event_init(kobj->event); - k_thread_start(thread_id); - - /* ptask points into task_memory, so only publish it once it cannot be freed */ - *task = ptask; - - return 0; - -e_dom: - scheduler_dp_domain_free(mod, mdom, kobj->mpart, comp_id); - on_pool = false; -e_thread: - if (on_pool) - objpool_free(&dp_mdom_head, mdom); - k_thread_abort(thread_id); - k_object_free(kobj->thread); -e_event: - k_object_free(kobj->event); -e_config: - mod_free(mod, ipc_config_data); -e_kobj: - rfree(kobj); -e_stack: - user_stack_free(p_stack); -e_tmem: - /* the copy lives in task_memory, so stop pointing at it before freeing */ - mod->dev->drv = drv; - mod_free(mod, task_memory); - return ret; -} - -#ifdef CONFIG_USERSPACE -#include -#include - -/* - * mod_free() follows the module's resource and allocator pointers in - * supervisor mode. They are user-writable, so at least check that they - * point to memory the caller can access. - */ -static void scheduler_dp_mod_vrfy(struct processing_module *mod) -{ - K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); - K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); - - struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc, sizeof(*alloc))); - if (alloc->heap) { - size_t h_size = 0; - uintptr_t h_start; - - mod_heap_info(mod, &h_size, &h_start); - if (h_size) - K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); - } - if (alloc->vreg) { - size_t vr_size = 0; - uintptr_t vr_start; - - vregion_mem_info(alloc->vreg, &vr_size, &vr_start); - K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); - } -} - -void z_vrfy_scheduler_dp_internal_free(struct task *task) -{ - struct scheduler_dp_task_kobj kobj; - - SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); - - /* - * Do not read task->priv_data: the kernel objects come from the - * kernel-only record, and an unknown task pointer is rejected. - */ - K_OOPS(!dp_task_kobj_peek(task, &kobj)); - - /* - * A DP thread holds its own thread and event objects, so it would pass - * the checks below for its own task. Freeing would then abort the - * caller after the record is unlinked, leaking everything it holds. - */ - K_OOPS(kobj.thread == k_current_get()); - - /* - * The caller must have been granted the task's objects. Check before - * unlinking the record, so an unauthorized call leaves it intact. - */ - K_OOPS(K_SYSCALL_OBJ(kobj.event, K_OBJ_EVENT)); - K_OOPS(K_SYSCALL_OBJ_INIT(kobj.thread, K_OBJ_THREAD)); - scheduler_dp_mod_vrfy(kobj.mod); - - K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj)); -} -#include -#endif /* CONFIG_USERSPACE */ From 1bc6ca633108ba8ad591a170e4ec76542c4b3133 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:31:13 +0000 Subject: [PATCH 7/9] schedule: move the TWB scheduler to src/kernel/schedule The thread-with-budget scheduler is kernel-only code. It serializes with irq_lock(), including in its thread function, changes the priority and time slice of its threads, whose budget callback runs in interrupt context, and allocates the thread stacks itself. Move it to src/kernel/schedule, which builds it into sof_kernel, and update the scheduler README. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/kernel/schedule/CMakeLists.txt | 4 ++++ src/{ => kernel}/schedule/zephyr_twb_schedule.c | 0 src/schedule/CMakeLists.txt | 4 ---- src/schedule/README.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename src/{ => kernel}/schedule/zephyr_twb_schedule.c (100%) diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index f49fc2d5f869..a0a65021878d 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -29,4 +29,8 @@ if(zephyr) zephyr_dp_schedule_thread-kernel.c ) endif() + + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER + zephyr_twb_schedule.c + ) endif() diff --git a/src/schedule/zephyr_twb_schedule.c b/src/kernel/schedule/zephyr_twb_schedule.c similarity index 100% rename from src/schedule/zephyr_twb_schedule.c rename to src/kernel/schedule/zephyr_twb_schedule.c diff --git a/src/schedule/CMakeLists.txt b/src/schedule/CMakeLists.txt index bc14c6ecff46..ee9ceb32ef9d 100644 --- a/src/schedule/CMakeLists.txt +++ b/src/schedule/CMakeLists.txt @@ -50,8 +50,4 @@ else() ) endif() -zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER - zephyr_twb_schedule.c -) - endif() diff --git a/src/schedule/README.md b/src/schedule/README.md index 0bef23248837..937f9034dc09 100644 --- a/src/schedule/README.md +++ b/src/schedule/README.md @@ -139,7 +139,7 @@ stateDiagram-v2 ## 3. TWB (Thread With Budget) Scheduler -The TWB scheduler (`zephyr_twb_schedule.c`) provides execution budget limits for specific tasks to prevent them from starving the CPU. This is useful for intensive workloads that shouldn't disrupt the overall systemic low-latency chain. +The TWB scheduler (`src/kernel/schedule/zephyr_twb_schedule.c`) provides execution budget limits for specific tasks to prevent them from starving the CPU. This is useful for intensive workloads that shouldn't disrupt the overall systemic low-latency chain. ### Architecture From 5bc90bb677b94fe6024a08582b347f57052989d7 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:32:06 +0000 Subject: [PATCH 8/9] schedule: split the scheduler registration out of schedule.c scheduler_init() and scheduler_register() allocate and register the schedulers at boot, on the kernel heap or, for the user-space schedulers with user-space LL, on the system user heap, and panic on failure. Move them to src/kernel/schedule/schedule-kernel.c. schedule.c keeps schedule_task_init() and the task info query. scheduler_is_user(), which both parts use, becomes a static inline in sof/schedule/schedule.h. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/schedule.h | 15 +++++ src/kernel/schedule/CMakeLists.txt | 1 + src/kernel/schedule/schedule-kernel.c | 81 +++++++++++++++++++++++++++ src/schedule/schedule.c | 67 +--------------------- 4 files changed, 100 insertions(+), 64 deletions(-) create mode 100644 src/kernel/schedule/schedule-kernel.c diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index f5f646cfbc50..362d15fe948d 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -235,6 +235,21 @@ struct schedulers **arch_user_schedulers_get(void); */ struct schedulers **arch_user_schedulers_get_for_core(int core); +/** + * @brief Check whether a scheduler type is managed in user space. + * @param[in] type Scheduler type identifier (SOF_SCHEDULE_*). + * @return true for the LL timer and DP schedulers. + */ +static inline bool scheduler_is_user(int type) +{ + /* + * currently only LL managed in user-space, but longterm + * goal is to move all audio application level scheduling + * to user-space and only keep Zephyr scheduler logic in + * kernel + */ + return type == SOF_SCHEDULE_LL_TIMER || type == SOF_SCHEDULE_DP; +} /** * @brief Lookup scheduler private context data by scheduler type from a schedulers container. * @param[in] schedulers Schedulers list container. diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index a0a65021878d..8711fde99b95 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -3,6 +3,7 @@ is_zephyr(zephyr) if(zephyr) zephyr_library_sources( + schedule-kernel.c zephyr_domain-kernel.c ) diff --git a/src/kernel/schedule/schedule-kernel.c b/src/kernel/schedule/schedule-kernel.c new file mode 100644 index 000000000000..bdefa7f35567 --- /dev/null +++ b/src/kernel/schedule/schedule-kernel.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019 Intel Corporation. All rights reserved. +// +// Author: Bartosz Kokoszko + +/* Generic scheduler */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(schedule, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx sch_tr; + +/* Kernel part of the generic scheduler: scheduler registration. The rest is in + * src/schedule/schedule.c + */ + +__cold static void scheduler_register(struct schedule_data *scheduler) +{ + struct schedulers **sch = arch_schedulers_get(); + struct k_heap *heap = NULL; + + assert_can_be_cold(); + + if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(scheduler->type)) { + sch = arch_user_schedulers_get(); + heap = sof_sys_user_heap_get(); + } + + if (!*sch) { + /* init schedulers list */ + *sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0); + if (!*sch) { + tr_err(&sch_tr, "allocation failed"); + return; + } + memset(*sch, 0, sizeof(**sch)); + list_init(&(*sch)->list); + } + + list_item_append(&scheduler->list, &(*sch)->list); +} + +__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data) +{ + struct schedule_data *sch; + struct k_heap *heap = NULL; + + assert_can_be_cold(); + + if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(type)) + heap = sof_sys_user_heap_get(); + + if (!ops || !ops->schedule_task || !ops->schedule_task_cancel || + !ops->schedule_task_free) + return; + + sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0); + if (!sch) { + tr_err(&sch_tr, "allocation failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + memset(sch, 0, sizeof(*sch)); + list_init(&sch->list); + sch->type = type; + sch->ops = ops; + sch->data = data; + + scheduler_register(sch); +} diff --git a/src/schedule/schedule.c b/src/schedule/schedule.c index fd3349a14902..62438be7c663 100644 --- a/src/schedule/schedule.c +++ b/src/schedule/schedule.c @@ -25,16 +25,9 @@ SOF_DEFINE_REG_UUID(schedule); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(sch_tr, SOF_UUID(schedule_uuid), LOG_LEVEL_INFO); -static inline bool scheduler_is_user(int type) -{ - /* - * currently only LL managed in user-space, but longterm - * goal is to move all audio application level scheduling - * to user-space and only keep Zephyr scheduler logic in - * kernel - */ - return type == SOF_SCHEDULE_LL_TIMER || type == SOF_SCHEDULE_DP; -} +/* Scheduler registration, which runs in kernel mode, is in + * src/kernel/schedule/schedule-kernel.c + */ int schedule_task_init(struct task *task, const struct sof_uuid_entry *uid, uint16_t type, @@ -84,60 +77,6 @@ int schedule_task_init(struct task *task, return 0; } -__cold static void scheduler_register(struct schedule_data *scheduler) -{ - struct schedulers **sch = arch_schedulers_get(); - struct k_heap *heap = NULL; - - assert_can_be_cold(); - - if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(scheduler->type)) { - sch = arch_user_schedulers_get(); - heap = sof_sys_user_heap_get(); - } - - if (!*sch) { - /* init schedulers list */ - *sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0); - if (!*sch) { - tr_err(&sch_tr, "allocation failed"); - return; - } - memset(*sch, 0, sizeof(**sch)); - list_init(&(*sch)->list); - } - - list_item_append(&scheduler->list, &(*sch)->list); -} - -__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data) -{ - struct schedule_data *sch; - struct k_heap *heap = NULL; - - assert_can_be_cold(); - - if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(type)) - heap = sof_sys_user_heap_get(); - - if (!ops || !ops->schedule_task || !ops->schedule_task_cancel || - !ops->schedule_task_free) - return; - - sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0); - if (!sch) { - tr_err(&sch_tr, "allocation failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - memset(sch, 0, sizeof(*sch)); - list_init(&sch->list); - sch->type = type; - sch->ops = ops; - sch->data = data; - - scheduler_register(sch); -} - /* Locks for the list here should be held by the caller, * as different schedulers use different locks */ From afd142899d0774342aef79caaa37f042671eef35 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:32:44 +0000 Subject: [PATCH 9/9] docs: schedule: describe the kernel parts of the schedulers Update the scheduler README for the split of the scheduler sources: the kernel parts are in src/kernel/schedule, the TWB scheduler is there in full, and the shared data structures are in sof/schedule/zephyr_ll.h and sof/schedule/zephyr_dp_schedule.h. Point the syscall table of the user-space README at the kernel files. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/userspace/README.md | 2 +- src/schedule/README.md | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index c3db997161ef..69d7b9ad95e8 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -124,7 +124,7 @@ syscall by subsystem: | CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` | | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_data_blob_set`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` | | Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` | -| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `schedule/zephyr_ll.c`, `schedule/zephyr_dp_schedule*.c` | +| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `kernel/schedule/zephyr_ll-kernel.c`, `kernel/schedule/zephyr_dp_schedule*-kernel.c` | | Debug | `debug_stream_slot_send_record` | `user/debug_stream_slot.h` | scattered: `debug/debug_stream/debug_stream_slot.c` | | KPB | `kpb_notifier_init` | `sof/audio/kpb.h` | scattered: `audio/kpb.c` | diff --git a/src/schedule/README.md b/src/schedule/README.md index 937f9034dc09..d2a86f6bf55c 100644 --- a/src/schedule/README.md +++ b/src/schedule/README.md @@ -1,12 +1,15 @@ # SOF Scheduling Architecture -> **Runs in:** Boundary / bridge — this directory owns the LL/DP schedulers and their thread + -> memory-domain setup. `zephyr_ll.c` and `zephyr_domain.c` are kernel-side; the LL user memory -> domain and heap are set up by `src/kernel/schedule/zephyr_ll_user.c`. -> `zephyr_dp_schedule_thread.c` is boundary code that can create and manage both kernel and -> `K_USER` DP threads depending on configuration; `zephyr_dp_schedule_application.c` is the -> DP application-mode path. This directory hosts the `zephyr_ll_task_sem_*` and -> `scheduler_dp_*` syscalls. See [User/kernel split](../include/sof/userspace/README.md). +> **Runs in:** Boundary / bridge — this directory holds the parts of the LL and DP schedulers +> that run in the scheduler and IPC threads, which are user threads with user-space LL. Their +> kernel parts are in `src/kernel/schedule`, one `*-kernel.c` file per source here: scheduler +> registration and initialization, the LL thread and timer setup and the timer interrupt, the +> DP thread creation and memory domains, the access grants, and the kernel halves of the +> `zephyr_ll_task_sem_*` and `scheduler_dp_*` syscalls. The LL user memory domain and heap are +> set up by `src/kernel/schedule/zephyr_ll_user.c`, and the TWB scheduler, which is kernel-only, +> is `src/kernel/schedule/zephyr_twb_schedule.c`. The scheduler data structures shared by both +> parts are in `sof/schedule/zephyr_ll.h` and `sof/schedule/zephyr_dp_schedule.h`. See +> [User/kernel split](../include/sof/userspace/README.md). This directory (`src/schedule`) contains the Sound Open Firmware (SOF) scheduling infrastructure, deeply integrated with the underlying Zephyr RTOS. SOF utilizes a multi-tiered scheduling approach to cater to different real-time constraints, ranging from hard real-time, low-latency requirements to more relaxed, compute-intensive data processing tasks.