diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index f5f646cfbc50..362d15fe948d 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -235,6 +235,21 @@ struct schedulers **arch_user_schedulers_get(void); */ struct schedulers **arch_user_schedulers_get_for_core(int core); +/** + * @brief Check whether a scheduler type is managed in user space. + * @param[in] type Scheduler type identifier (SOF_SCHEDULE_*). + * @return true for the LL timer and DP schedulers. + */ +static inline bool scheduler_is_user(int type) +{ + /* + * currently only LL managed in user-space, but longterm + * goal is to move all audio application level scheduling + * to user-space and only keep Zephyr scheduler logic in + * kernel + */ + return type == SOF_SCHEDULE_LL_TIMER || type == SOF_SCHEDULE_DP; +} /** * @brief Lookup scheduler private context data by scheduler type from a schedulers container. * @param[in] schedulers Schedulers list container. diff --git a/src/schedule/zephyr_dp_schedule.h b/src/include/sof/schedule/zephyr_dp_schedule.h similarity index 63% rename from src/schedule/zephyr_dp_schedule.h rename to src/include/sof/schedule/zephyr_dp_schedule.h index 07d8ee4a4754..582193a6a816 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/include/sof/schedule/zephyr_dp_schedule.h @@ -6,16 +6,26 @@ */ /** - * @file src/schedule/zephyr_dp_schedule.h + * @file include/sof/schedule/zephyr_dp_schedule.h * @brief Internal Data Processing (DP) scheduler structures, memory partitions, and thread helpers. * @ingroup scheduling_api + * + * Shared by the DP scheduler sources in src/schedule and their kernel parts in + * src/kernel/schedule. Not for use outside the scheduler. */ +#ifndef __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ +#define __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ + #include +#include #include #include #include +#include +#include + #include #include @@ -66,6 +76,73 @@ struct task_dp_pdata { #endif }; +#if CONFIG_SOF_USERSPACE_APPLICATION +/** @brief Per-core DP thread start and IPC completion semaphores. */ +extern struct k_sem dp_sync[CONFIG_CORE_COUNT]; + +struct ipc4_flat { + unsigned int cmd; + int ret; + union { + struct { + struct ipc4_module_bind_unbind bu; + enum bind_type type; + } bind; + struct { + unsigned int trigger_cmd; + enum ipc4_pipeline_state state; + int n_sources; + int n_sinks; + struct sof_source *source[CONFIG_MODULE_MAX_CONNECTIONS]; + struct sof_sink *sink[CONFIG_MODULE_MAX_CONNECTIONS]; + } pipeline_state; + /* + * SOF_IPC4_MOD_INIT_INSTANCE: a by-value copy of the caller's + * ext_data. The original lives on the calling thread's stack, + * which the DP thread's memory domain doesn't have access to. + * The pointers it contains reference the IPC mailbox, which stays + * valid and is covered by the DP thread's SOF_DP_PART_CFG partition. + */ + struct module_ext_init_data init_instance; + struct { + uint32_t param_id; + enum module_cfg_fragment_position position; + uint32_t data_offset_size; + size_t fragment_size; + const char *data; + } set_config; + struct { + uint32_t param_id; + uint32_t data_offset_size; + size_t fragment_size; + char *data; + } get_config; + }; +}; + +/* memory allocation helper structure */ +struct scheduler_dp_task_memory { + struct task task; + struct task_dp_pdata pdata; + struct comp_driver drv; + struct ipc4_flat flat; +}; + +#endif /* CONFIG_SOF_USERSPACE_APPLICATION */ +/** @brief Per-core DP scheduler locks, see scheduler_dp_lock(). */ +extern struct k_sem dp_lock[CONFIG_MP_MAX_NUM_CPUS]; + +/** + * @brief DP scheduler ops, see struct scheduler_ops. + */ +int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, + uint64_t period); +int scheduler_dp_task_stop(void *data, struct task *task); +int scheduler_dp_task_free(void *data, struct task *task); +#if CONFIG_SOF_USERSPACE_APPLICATION +int scheduler_dp_task_cancel(void *data, struct task *task); +#endif + /** * @brief Recalculate deadlines and evaluate readiness for all active DP tasks on an LL tick. * @param[in,out] dp_sch Pointer to per-core DP scheduler state container. @@ -102,3 +179,5 @@ void scheduler_dp_unlock(unsigned int key); * @param[in] core Target DSP core ID. */ void scheduler_dp_grant(k_tid_t thread_id, uint16_t core); + +#endif /* __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ */ diff --git a/src/include/sof/schedule/zephyr_ll.h b/src/include/sof/schedule/zephyr_ll.h new file mode 100644 index 000000000000..14a7b75c1821 --- /dev/null +++ b/src/include/sof/schedule/zephyr_ll.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2019-2026 Intel Corporation. + */ + +/* + * Private data of the Zephyr LL scheduler and its timer domain, shared by + * the parts in src/schedule, which also run in the LL scheduler threads, and + * the kernel parts in src/kernel/schedule. Not for use outside the scheduler. + */ + +#ifndef __SOF_SCHEDULE_ZEPHYR_LL_H__ +#define __SOF_SCHEDULE_ZEPHYR_LL_H__ + +#include +#include +#include +#include +#include +#include +#include + +/* per-scheduler data */ +struct zephyr_ll { + struct list_item tasks; /* list of ll tasks */ + unsigned int n_tasks; /* task counter */ + struct ll_schedule_domain *ll_domain; /* scheduling domain */ + unsigned int core; /* core ID of this instance */ +#if CONFIG_SOF_USERSPACE_LL + struct k_mutex *lock; /* mutex for userspace */ +#endif + struct k_heap *heap; +}; + +/* per-task scheduler data */ +struct zephyr_ll_pdata { + bool run; + bool freeing; + struct k_sem *sem_p; +#if !CONFIG_DYNAMIC_OBJECTS + struct k_sem sem; +#endif +}; + +static inline void zephyr_ll_assert_core(const struct zephyr_ll *sch) +{ + assert(CONFIG_CORE_COUNT == 1 || IS_ENABLED(CONFIG_SOF_USERSPACE_LL) || + sch->core == cpu_get_id()); +} + +/* LL scheduler ops, see zephyr_ll.c */ +int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, + uint64_t period); +int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *before); +int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *after); +int zephyr_ll_task_free(void *data, struct task *task); +int zephyr_ll_task_cancel(void *data, struct task *task); + +#if CONFIG_SOF_USERSPACE_LL +/* per-core LL scheduler locks, see zephyr_ll.c */ +extern struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; + +struct k_thread *zephyr_ll_init_context(void *data, struct task *task); +#endif + +struct zephyr_domain_thread { + struct k_thread *ll_thread; + struct k_sem *sem; +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_thread ll_thread_obj; + struct k_sem sem_obj; +#endif + void (*handler)(void *arg); + void *arg; +}; + +struct zephyr_domain { + struct k_timer *timer; +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_timer timer_obj; +#endif + struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; + struct ll_schedule_domain *ll_domain; +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL + atomic_t block; + struct k_mutex block_mutex; + struct k_condvar block_condvar; +#endif +}; + +/* LL scheduler thread function, p1 is the domain and p2 the core */ +void zephyr_domain_thread_fn(void *p1, void *p2, void *p3); + +#ifdef CONFIG_SOF_USERSPACE_LL +int zephyr_domain_register_user(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg); +int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks); +#endif + +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL +void zephyr_domain_block(struct ll_schedule_domain *domain); +void zephyr_domain_unblock(struct ll_schedule_domain *domain); +#endif + +#endif /* __SOF_SCHEDULE_ZEPHYR_LL_H__ */ diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index c3db997161ef..69d7b9ad95e8 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -124,7 +124,7 @@ syscall by subsystem: | CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` | | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_data_blob_set`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` | | Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` | -| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `schedule/zephyr_ll.c`, `schedule/zephyr_dp_schedule*.c` | +| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `kernel/schedule/zephyr_ll-kernel.c`, `kernel/schedule/zephyr_dp_schedule*-kernel.c` | | Debug | `debug_stream_slot_send_record` | `user/debug_stream_slot.h` | scattered: `debug/debug_stream/debug_stream_slot.c` | | KPB | `kpb_notifier_init` | `sof/audio/kpb.h` | scattered: `audio/kpb.c` | diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 277378449165..8711fde99b95 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -2,7 +2,36 @@ is_zephyr(zephyr) if(zephyr) + zephyr_library_sources( + schedule-kernel.c + zephyr_domain-kernel.c + ) + + # The kernel part of zephyr_ll.c, built with it, see src/schedule/CMakeLists.txt + if(NOT CONFIG_SOC_ACP_6_0 OR CONFIG_DMA_DOMAIN) + zephyr_library_sources( + zephyr_ll-kernel.c + ) + endif() + zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL zephyr_ll_user.c ) + + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule-kernel.c + ) + if(CONFIG_SOF_USERSPACE_APPLICATION) + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule_application-kernel.c + ) + else() + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule_thread-kernel.c + ) + endif() + + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER + zephyr_twb_schedule.c + ) endif() diff --git a/src/kernel/schedule/schedule-kernel.c b/src/kernel/schedule/schedule-kernel.c new file mode 100644 index 000000000000..bdefa7f35567 --- /dev/null +++ b/src/kernel/schedule/schedule-kernel.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019 Intel Corporation. All rights reserved. +// +// Author: Bartosz Kokoszko + +/* Generic scheduler */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(schedule, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx sch_tr; + +/* Kernel part of the generic scheduler: scheduler registration. The rest is in + * src/schedule/schedule.c + */ + +__cold static void scheduler_register(struct schedule_data *scheduler) +{ + struct schedulers **sch = arch_schedulers_get(); + struct k_heap *heap = NULL; + + assert_can_be_cold(); + + if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(scheduler->type)) { + sch = arch_user_schedulers_get(); + heap = sof_sys_user_heap_get(); + } + + if (!*sch) { + /* init schedulers list */ + *sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0); + if (!*sch) { + tr_err(&sch_tr, "allocation failed"); + return; + } + memset(*sch, 0, sizeof(**sch)); + list_init(&(*sch)->list); + } + + list_item_append(&scheduler->list, &(*sch)->list); +} + +__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data) +{ + struct schedule_data *sch; + struct k_heap *heap = NULL; + + assert_can_be_cold(); + + if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(type)) + heap = sof_sys_user_heap_get(); + + if (!ops || !ops->schedule_task || !ops->schedule_task_cancel || + !ops->schedule_task_free) + return; + + sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0); + if (!sch) { + tr_err(&sch_tr, "allocation failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + memset(sch, 0, sizeof(*sch)); + list_init(&sch->list); + sch->type = type; + sch->ops = ops; + sch->data = data; + + scheduler_register(sch); +} diff --git a/src/kernel/schedule/zephyr_domain-kernel.c b/src/kernel/schedule/zephyr_domain-kernel.c new file mode 100644 index 000000000000..6acbb5e00904 --- /dev/null +++ b/src/kernel/schedule/zephyr_domain-kernel.c @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019-2021 Intel Corporation. All rights reserved. +// +// Author: Tomasz Lauda + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); + +/* Kernel part of the Zephyr LL scheduler timer domain: the scheduler threads, the + * timer and its interrupt handler. The thread function and the parts that run in the + * LL scheduler threads are in src/schedule/zephyr_domain.c + */ + +/* + * Currently the Zephyr clock rate is part it's Kconfig known at build time. + * SOF on Intel CAVS platforms currently only aligns with Zephyr when both + * use the CAVS 19.2 MHz SSP clock. TODO - needs runtime alignment. + */ +#if CONFIG_XTENSA && CONFIG_CAVS && !CONFIG_INTEL_ADSP_TIMER +#error "Zephyr uses 19.2MHz clock derived from SSP which must be enabled." +#endif + +#define ZEPHYR_LL_STACK_SIZE 8192 + +#if CONFIG_SOF_USERSPACE_LL +K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); +#else +K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); +#endif + +/* Timer callback: runs in timer IRQ context */ +static void zephyr_domain_timer_fn(struct k_timer *timer) +{ + struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer); + int core; + + /* + * A race is possible when the Zephyr LL scheduling domain is being + * unregistered while a timer IRQ is processed on a different core. Then + * the timer is removed by the former but then re-added by the latter, + * but this time with no user data and no handler set. This leads to the + * timer continuing to trigger and then leading to a lock up when it is + * registered again next time. + */ + if (!zephyr_domain) { + k_timer_stop(timer); + return; + } + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + if (dt->handler) + k_sem_give(dt->sem); + } +} + +/* The normal kernel-space implementation for register/unregister */ +#ifndef CONFIG_SOF_USERSPACE_LL + +static int zephyr_domain_register(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + char thread_name[] = "ll_thread0"; + k_tid_t thread; + k_spinlock_key_t key; + + tr_dbg(&ll_tr, "entry"); + + /* domain work only needs registered once on each core */ + if (dt->handler) + return 0; + + dt->handler = handler; + dt->arg = arg; + + /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ + k_sem_init(dt->sem, 0, 10); + + thread_name[sizeof(thread_name) - 2] = '0' + core; + + /* not allocated dynamically when LL in kernel space */ + dt->ll_thread = &dt->ll_thread_obj; + + thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, + zephyr_domain_thread_fn, zephyr_domain, INT_TO_POINTER(core), + NULL, CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER); + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, core); +#endif + k_thread_name_set(thread, thread_name); + + k_thread_start(thread); + + key = k_spin_lock(&domain->lock); + + if (!k_timer_user_data_get(zephyr_domain->timer)) { + k_timeout_t start = {0}; + + k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); + k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); + + k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); + + /* Enable the watchdog */ + watchdog_enable(core); + } + + k_spin_unlock(&domain->lock, key); + + tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", + domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); + + return 0; +} + +static int zephyr_domain_unregister(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + k_spinlock_key_t key; + + tr_dbg(&ll_tr, "entry"); + + /* tasks still registered on this core */ + if (num_tasks) + return 0; + + key = k_spin_lock(&domain->lock); + + if (!atomic_read(&domain->total_num_tasks)) { + /* Disable the watchdog */ + watchdog_disable(core); + + k_timer_stop(zephyr_domain->timer); + k_timer_user_data_set(zephyr_domain->timer, NULL); + } + + zephyr_domain->domain_thread[core].handler = NULL; + + k_spin_unlock(&domain->lock, key); + + tr_info(&ll_tr, "domain->type %d domain->clk %d", + domain->type, domain->clk); + + /* + * If running in the context of the domain thread, k_thread_abort() will + * not return + */ + k_thread_abort(zephyr_domain->domain_thread[core].ll_thread); + + tr_dbg(&ll_tr, "exit"); + + return 0; +} + +#else /* CONFIG_SOF_USERSPACE_LL */ + +/* + * Kernel-owned per-core LL thread table. Populated from the privileged + * domain-thread init path and consulted by z_impl_zephyr_ll_task_sem_alloc() + * so that privileged code never has to traverse the user-accessible + * scheduler/domain objects to find the LL thread of a given core. + */ +static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT]; + +/* + * Privileged thread initialization for userspace LL scheduling. + * Creates the scheduling thread, sets up timer, grants access to kernel + * objects. Must be called from kernel context before any user-space + * domain_register() calls. + */ +static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, + struct task *task) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + struct zephyr_domain_thread *dt; + char thread_name[] = "ll_thread0"; + k_tid_t thread; + int core = task->core; + + tr_dbg(&ll_tr, "entry"); + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + dt = zephyr_domain->domain_thread + core; + + /* thread only needs to be created once per core */ + if (dt->ll_thread) + return 0; + + dt->handler = NULL; + dt->arg = NULL; + + dt->sem = k_object_alloc(K_OBJ_SEM); + if (!dt->sem) { + tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core); + return -ENOMEM; + } + + /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ + k_sem_init(dt->sem, 0, 10); + + thread_name[sizeof(thread_name) - 2] = '0' + core; + + /* + * Allocate thread structure dynamically. Keep the handle in a local: + * the domain is user-writable, so dt->ll_thread must not be read back + * for the kernel-only tables below. + */ + thread = k_object_alloc(K_OBJ_THREAD); + if (!thread) { + tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); + k_object_free(dt->sem); + dt->sem = NULL; + return -ENOMEM; + } + + k_thread_create(thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, + zephyr_domain_thread_fn, zephyr_domain, + INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, + K_USER, K_FOREVER); + + /* record in the kernel-only table for syscall-context lookups */ + ll_thread_tid[core] = thread; + sof_sysuser_thread_set(SOF_SYSUSER_LL, core, thread); + dt->ll_thread = thread; + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, core); +#endif + k_thread_name_set(thread, thread_name); + + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + user_grant_uaol_access_all(thread); + tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); + + k_thread_start(thread); + + k_mutex_lock(domain->lock, K_FOREVER); + if (!k_timer_user_data_get(zephyr_domain->timer)) { + k_timeout_t start = {0}; + + k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); + k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); + + k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); + + /* Enable the watchdog */ + watchdog_enable(core); + } + + k_mutex_unlock(domain->lock); + + tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", + domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); + + return 0; +} + +/* + * Free resources acquired by zephyr_domain_thread_init(). + * Stops the timer, aborts the scheduling thread and frees the thread object. + * Must be called from kernel context. + */ +static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, + uint32_t num_tasks) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + tr_dbg(&ll_tr, "thread_free entry, core %d, num_tasks %u", core, num_tasks); + + /* Still tasks on other cores, only clean up this core's thread */ + k_mutex_lock(domain->lock, K_FOREVER); + + if (!num_tasks && !atomic_read(&domain->total_num_tasks)) { + /* Last task globally: stop the timer and watchdog */ + watchdog_disable(core); + + k_timer_stop(zephyr_domain->timer); + k_timer_user_data_set(zephyr_domain->timer, NULL); + } + + dt->handler = NULL; + dt->arg = NULL; + + k_mutex_unlock(domain->lock); + + if (dt->ll_thread) { + sof_sysuser_thread_set(SOF_SYSUSER_LL, core, NULL); + ll_thread_tid[core] = NULL; + k_thread_abort(dt->ll_thread); + k_object_free(dt->ll_thread); + dt->ll_thread = NULL; + } + + if (dt->sem) { + k_object_free(dt->sem); + dt->sem = NULL; + } + + tr_info(&ll_tr, "thread_free done, core %d", core); +} + +struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + tr_dbg(&ll_tr, "entry"); + + return dt->ll_thread; +} + +/* + * Return the LL scheduling thread for an explicitly given core. + * + * Reads a kernel-only table keyed by core, so it is safe to call from a + * privileged syscall context without dereferencing any user-accessible + * scheduler or domain object, and without relying on cpu_get_id(). + */ +struct k_thread *zephyr_domain_thread_tid_for_core(int core) +{ + if (core < 0 || core >= CONFIG_CORE_COUNT) + return NULL; + + return ll_thread_tid[core]; +} + +struct k_thread *zephyr_ll_domain_thread(void) +{ + struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); + + if (!ll_domain) + return NULL; + + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); + + return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; +} + +#endif /* CONFIG_SOF_USERSPACE_LL */ + +APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { +#ifdef CONFIG_SOF_USERSPACE_LL + .domain_register = zephyr_domain_register_user, + .domain_unregister = zephyr_domain_unregister_user, + .domain_thread_init = zephyr_domain_thread_init, + .domain_thread_free = zephyr_domain_thread_free, +#else + .domain_register = zephyr_domain_register, + .domain_unregister = zephyr_domain_unregister, +#endif +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL + .domain_block = zephyr_domain_block, + .domain_unblock = zephyr_domain_unblock, +#endif +}; + +/* Core 0 only */ +struct ll_schedule_domain *zephyr_domain_init(int clk) +{ + struct ll_schedule_domain *domain; + struct zephyr_domain *zephyr_domain; + int core; + + domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, false, + &zephyr_domain_ops); + if (!domain) { + tr_err(&ll_tr, "domain init failed"); + return NULL; + } + +#if CONFIG_SOF_USERSPACE_LL + zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*zephyr_domain), sizeof(void *)); + if (zephyr_domain) + memset(zephyr_domain, 0, sizeof(*zephyr_domain)); +#else + zephyr_domain = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*zephyr_domain)); +#endif + if (!zephyr_domain) { + tr_err(&ll_tr, "domain allocation failed"); + k_panic(); + return NULL; + } + + zephyr_domain->ll_domain = domain; + +#if CONFIG_SOF_USERSPACE_LL + /* Allocate timer dynamically for userspace access */ + zephyr_domain->timer = k_object_alloc(K_OBJ_TIMER); + if (!zephyr_domain->timer) { + tr_err(&ll_tr, "timer allocation failed"); + k_panic(); + return NULL; + } +#else + /* not allocated dynamically when LL in kernel space */ + zephyr_domain->timer = &zephyr_domain->timer_obj; +#endif + +#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL + atomic_set(&zephyr_domain->block, 0); + k_mutex_init(&zephyr_domain->block_mutex); + k_condvar_init(&zephyr_domain->block_condvar); +#endif + + ll_sch_domain_set_pdata(domain, zephyr_domain); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { +#ifndef CONFIG_SOF_USERSPACE_LL + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + /* not allocated dynamically when LL in kernel space */ + dt->sem = &dt->sem_obj; +#endif + } + + return domain; +} diff --git a/src/kernel/schedule/zephyr_dp_schedule-kernel.c b/src/kernel/schedule/zephyr_dp_schedule-kernel.c new file mode 100644 index 000000000000..7f0d7c285d62 --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule-kernel.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2023 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); +SOF_DEFINE_REG_UUID(dp_sched); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); + +/* Kernel part of the DP scheduler: the scheduler locks, their access grants, the LL + * tick system call and the scheduler initialization. The rest is in + * src/schedule/zephyr_dp_schedule.c + */ + +#define DP_LOCK_INIT(i, _) Z_SEM_INITIALIZER(dp_lock[i], 1, 1) +#define DP_LOCK_INIT_LIST LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,)) + +/* User threads don't need access to this array. Access is performed from + * the kernel space via a syscall. Array must be placed in special section + * to be qualified as initialized by the gen_kobject_list.py script. + */ +STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST }; + +void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) +{ + if (thread_id) + k_thread_access_grant(thread_id, &dp_lock[core]); +} + +/* dummy LL task - to start LL on secondary cores */ +static enum task_state scheduler_dp_ll_tick_dummy(void *data) +{ + return SOF_TASK_STATE_RESCHEDULE; +} + +/* + * function called after every LL tick + * + * This function checks if the queued DP tasks are ready to processing (meaning + * the module run by the task has enough data at all sources and enough free space + * on all sinks) + * + * if the task becomes ready, a deadline is set allowing Zephyr to schedule threads + * in right order + * + * TODO: currently there's a limitation - DP module must be surrounded by LL modules. + * it simplifies algorithm - there's no need to browse through DP chains calculating + * deadlines for each module in function of all modules execution status. + * Now is simple - modules deadline is its start + tick time. + * + * example: + * Lets assume we do have a pipeline: + * + * LL1 -> DP1 -> LL2 -> DP2 -> LL3 -> DP3 -> LL4 + * + * all LLs starts in 1ms tick + * + * for simplification lets assume + * - all LLs are on primary core, all DPs on secondary (100% CPU is for DP) + * - context switching requires 0 cycles + * + * DP1 - starts every 1ms, needs 0.5ms to finish processing + * DP2 - starts every 2ms, needs 0.6ms to finish processing + * DP3 - starts every 10ms, needs 0.3ms to finish processing + * + * TICK0 + * only LL1 is ready to run + * LL1 processing (producing data chunk for DP1) + * + * TICK1 + * LL1 is ready to run + * DP1 is ready tu run (has data from LL1) set deadline to TICK2 + * LL1 processing (producing second data chunk for DP1) + * DP1 processing for 0.5ms (consuming first data chunk, producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK2 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK3 + * LL2 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK3 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK4 + * LL2 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK4 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK5 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK6 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of second data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * !!!!!! Note here - DP1 must do before DP2 as it MUST finish in this tick. DP2 can wait + * >>>>>>> this is what we call EDF - EARIEST DEADLINE FIRST <<<<<< + * + * TICK5 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK6 + * LL2 is ready to run + * DP2 is in progress, deadline is set to TICK6 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of second data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.1ms (producing TWO data chunks for LL3) + * CPU is idle for 0.4ms (60% used) + * + * TICK6 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK7 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK8 + * LL3 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of second data chunk for DP2) + * LL3 processing (producing 10% of first data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * + * + * (........ 9 more cycles - LL3 procuces 100% of data for DP3......) + * + * + * TICK15 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK16 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK17 + * LL3 is ready to run + * DP3 is ready to run set deadline to TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of data chunk for DP2) + * LL3 processing (producing 10% of second data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used - + * !!! note that DP3 is ready but has no chance to get CPU in this cycle + * + * TICK16 + * LL1 is ready to run set deadline to TICK17 + * DP1 is ready tu run + * LL2 is ready to run + * DP2 is in progress, deadline is set to TICK17 + * LL3 is ready to run + * DP3 is in progress, deadline is set to TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * LL3 processing (producing 10% of second data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.1ms (producing data) + * DP3 processing for 0.2ms (producing 10 data chunks for LL4) + * 90% CPU used + * + * TICK17 + * LL1 is ready to run + * DP1 is ready tu run + * LL2 is ready to run + * DP2 is ready to run + * LL3 is ready to run + * LL4 is ready to run + * !! NOTE that DP3 is not ready - it will be ready again in TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * LL3 processing (producing next 10% of second data chunk for DP3) + * LL4 processing (consuming 10% of data prepared by DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * + * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 + * needed 1.2ms for processing - but the example would be too complicated) + */ + +void z_impl_scheduler_dp_ll_tick(void) +{ + unsigned int lock_key; +#if CONFIG_SOF_USERSPACE_LL + struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); +#else + struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); +#endif + + if (!dp_sch) + return; + + /* remember current timestamp as "NOW" */ + dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); + + lock_key = scheduler_dp_lock(cpu_get_id()); + scheduler_dp_recalculate(dp_sch); + scheduler_dp_unlock(lock_key); +} + +#ifdef CONFIG_USERSPACE +#include +#include +void z_vrfy_scheduler_dp_ll_tick(void) +{ + SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_LL); + + z_impl_scheduler_dp_ll_tick(); +} +#include +#endif + +void z_impl_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) +{ + if (deadline_clock_ticks && + (int32_t)(k_cycle_get_32() - deadline_clock_ticks) > 0) + telemetry_notify_dp_deadline_miss(core); +} + +#ifdef CONFIG_USERSPACE +void z_vrfy_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) +{ + K_OOPS(K_SYSCALL_VERIFY(core < CONFIG_MP_MAX_NUM_CPUS)); + z_impl_scheduler_dp_check_deadline(core, deadline_clock_ticks); +} +#include +#endif + +APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { + .schedule_task = scheduler_dp_task_shedule, +#if CONFIG_SOF_USERSPACE_APPLICATION + .schedule_task_cancel = scheduler_dp_task_cancel, +#else + .schedule_task_cancel = scheduler_dp_task_stop, +#endif + .schedule_task_free = scheduler_dp_task_free, +}; + +/* Runs on each core */ +__cold int scheduler_dp_init(void) +{ + assert_can_be_cold(); + + struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); + if (!dp_sch) + return -ENOMEM; + + memset(dp_sch, 0, sizeof(*dp_sch)); + list_init(&dp_sch->tasks); + + scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); + + /* init src of DP tick */ + return schedule_task_init_ll(&dp_sch->ll_tick_src, + SOF_UUID(dp_sched_uuid), + SOF_SCHEDULE_LL_TIMER, + 0, scheduler_dp_ll_tick_dummy, dp_sch, + cpu_get_id(), 0); +} diff --git a/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c b/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c new file mode 100644 index 000000000000..a783b90b4ab3 --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; + +/* Kernel part of the application variant of the DP scheduler: the DP thread creation, + * its memory domain, the kernel-only record of its kernel objects and the + * scheduler_dp_internal_free() system call. The DP thread function and the IPC + * forwarding to it are in src/schedule/zephyr_dp_schedule_application.c + */ + +static struct objpool_head dp_mdom_head = {.list = LIST_INIT(dp_mdom_head.list)}; + +/* Synchronization semaphore for the scheduler thread to wait for DP startup */ +#define DP_SYNC_INIT(i, _) Z_SEM_INITIALIZER(dp_sync[i], 0, 1) +#define DP_SYNC_INIT_LIST LISTIFY(CONFIG_CORE_COUNT, DP_SYNC_INIT, (,)) +STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_sync, CONFIG_CORE_COUNT) = { DP_SYNC_INIT_LIST }; + +/* + * Kernel-only record of the kernel objects owned by a DP task. + * + * struct scheduler_dp_task_memory lives in the module's heap, which is mapped + * into the DP thread's memory domain, and is also writable by the LL and IPC + * user threads. Freeing must therefore never use the pointers stored there: + * scheduler_dp_internal_free() takes them from this record instead. The task + * pointer is only used as a lookup key. + */ +struct scheduler_dp_task_kobj { + struct list_item list; + struct task *task; + struct processing_module *mod; + struct k_event *event; + struct k_thread *thread; + struct k_mem_domain *mdom; + struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; + uint32_t comp_id; + uint32_t gen; /* unique per record, see dp_task_kobj_take() */ +}; + +static struct list_item dp_task_kobj_list = LIST_INIT(dp_task_kobj_list); +static struct k_spinlock dp_task_kobj_lock; +/* generation of the last added record, under dp_task_kobj_lock */ +static uint32_t dp_task_kobj_gen; + +static void dp_task_kobj_add(struct scheduler_dp_task_kobj *kobj) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + + kobj->gen = ++dp_task_kobj_gen; + list_item_append(&kobj->list, &dp_task_kobj_list); + k_spin_unlock(&dp_task_kobj_lock, key); +} + +/* Find the record for @task, caller holds dp_task_kobj_lock */ +static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task) +{ + struct scheduler_dp_task_kobj *kobj; + struct list_item *item; + + list_for_item(item, &dp_task_kobj_list) { + kobj = container_of(item, struct scheduler_dp_task_kobj, list); + if (kobj->task == task) + return kobj; + } + + return NULL; +} + +/* Copy the record for @task to @copy without unlinking it */ +static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); + + if (kobj) + *copy = *kobj; + + k_spin_unlock(&dp_task_kobj_lock, key); + + return kobj != NULL; +} + +/* + * Find the record for @task and unlink it, so it can only be freed once. + * If @expected is given, only take the record it was copied from, so a + * record replaced since @expected was peeked is left alone. + */ +static struct scheduler_dp_task_kobj * +dp_task_kobj_take(const struct task *task, const struct scheduler_dp_task_kobj *expected) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); + + /* + * Compare the generation, not the object addresses: the task could + * have been freed and recreated at the same addresses meanwhile. + */ + if (kobj && expected && kobj->gen != expected->gen) + kobj = NULL; + + if (kobj) + list_item_del(&kobj->list); + + k_spin_unlock(&dp_task_kobj_lock, key); + + return kobj; +} + +/* + * Safe to call with partial successful initialisation, + * k_mem_domain_remove_partition() then just returns -ENOENT + */ +static void scheduler_dp_domain_free(struct processing_module *pmod, struct k_mem_domain *mdom, + struct k_mem_partition *mpart, uint32_t comp_id) +{ + llext_manager_rm_domain(comp_id, mdom); + + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP_CACHE); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG_CACHE); + + if (mdom->num_partitions) { + /* Violation: all partitions should be freed by now */ + unsigned int n = mdom->num_partitions; + + LOG_WRN("%u domain partition(s) not freed, force-freeing them all", n); + + for (unsigned int i = 0; i < arch_mem_domain_max_partitions_get() && n; i++) { + struct k_mem_partition *dpart = &mdom->partitions[i]; + + if (dpart->size) { + k_mem_domain_remove_partition(mdom, dpart); + n--; + } + } + } + + /* All partitions removed, the domain can be freed now */ + pmod->mdom = NULL; + k_mem_domain_deinit(mdom); + objpool_free(&dp_mdom_head, mdom); +} + +/* + * Free the kernel objects of @task, using only the kernel-only record. + * + * @return false if @task has no record (unknown task or already freed) + */ +static bool scheduler_dp_internal_free_kobj(struct task *task, + const struct scheduler_dp_task_kobj *expected) +{ + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_take(task, expected); + + if (!kobj) + return false; + + /* normally already terminated by scheduler_dp_task_free(), no-op then */ + k_thread_abort(kobj->thread); + k_object_free(kobj->event); + k_object_free(kobj->thread); + scheduler_dp_domain_free(kobj->mod, kobj->mdom, kobj->mpart, kobj->comp_id); + + mod_free(kobj->mod, container_of(task, struct scheduler_dp_task_memory, task)); + rfree(kobj); + + return true; +} + +void z_impl_scheduler_dp_internal_free(struct task *task) +{ + if (!scheduler_dp_internal_free_kobj(task, NULL)) + tr_err(&dp_tr, "unknown DP task %p", task); +} + +#ifdef CONFIG_THREAD_NAME +static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_module *mod) +{ + char name[CONFIG_THREAD_MAX_NAME_LEN]; + + snprintf(name, sizeof(name), "DP:%#x", mod->dev->ipc_config.id); + + k_thread_name_set(thread_id, name); +} +#else +/* k_thread_name_set() is a no-op so skip constructing a thread name */ +#define scheduler_dp_thread_name_set(x, y) +#endif + +/* Called only in IPC context in kernel mode (this can change) */ +int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, + const struct task_ops *ops, struct processing_module *mod, + uint16_t core, size_t stack_size, uint32_t options) +{ + k_thread_stack_t *p_stack; + struct scheduler_dp_task_memory *task_memory; + uint8_t *ipc_config_data = NULL; + k_tid_t thread_id; + uint32_t comp_id; + + int ret; + + /* must be called on the same core the task will be bound to */ + assert(cpu_get_id() == core); + assert(!k_is_user_context()); + + /* + * allocate memory + * to avoid multiple malloc operations allocate all required memory as a single structure + * and return pointer to task_memory->task + * As the structure contains zephyr kernel specific data, it must be located in + * shared, non cached memory + */ + task_memory = mod_alloc_ext(mod, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*task_memory), 0); + if (!task_memory) { + tr_err(&dp_tr, "memory alloc failed"); + return -ENOMEM; + } + + memset(task_memory, 0, sizeof(*task_memory)); + + const struct comp_driver *drv = mod->dev->drv; + + task_memory->drv = *mod->dev->drv; + mod->dev->drv = &task_memory->drv; + + /* allocate stack - must be aligned and cached so a separate alloc */ + p_stack = user_stack_allocate(stack_size, options); + if (!p_stack) { + tr_err(&dp_tr, "stack alloc failed"); + ret = -ENOMEM; + goto e_tmem; + } + + struct task *ptask = &task_memory->task; + + /* internal SOF task init */ + ret = schedule_task_init(ptask, uid, SOF_SCHEDULE_DP, 0, ops->run, mod, core, options); + if (ret < 0) { + tr_err(&dp_tr, "schedule_task_init failed"); + goto e_stack; + } + + /* + * task_memory, including pdata, is in the module heap, which user + * threads can write while this function runs. Keep everything the + * kernel uses here or at teardown in kobj and locals, and only mirror + * it into pdata for the DP thread. + */ + struct task_dp_pdata *pdata = &task_memory->pdata; + struct scheduler_dp_task_kobj *kobj = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*kobj)); + + if (!kobj) { + tr_err(&dp_tr, "task record alloc failed"); + ret = -ENOMEM; + goto e_stack; + } + + kobj->task = ptask; + kobj->mod = mod; + comp_id = mod->dev->ipc_config.id; + kobj->comp_id = comp_id; + + if (mod->dev->drv->adapter_ops->get_configuration) { + ipc_config_data = mod_alloc_ext(mod, SOF_MEM_FLAG_USER, SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc_config_data) { + tr_err(&dp_tr, "Failed to allocate memory for get-config"); + ret = -ENOMEM; + goto e_kobj; + } + } + pdata->ipc_config_data = ipc_config_data; + + pdata->flat = &task_memory->flat; + + kobj->event = k_object_alloc(K_OBJ_EVENT); + if (!kobj->event) { + tr_err(&dp_tr, "Event object allocation failed"); + ret = -ENOMEM; + goto e_config; + } + + pdata->event = kobj->event; + + kobj->thread = k_object_alloc(K_OBJ_THREAD); + if (!kobj->thread) { + tr_err(&dp_tr, "Thread object allocation failed"); + ret = -ENOMEM; + goto e_event; + } + memset(&kobj->thread->arch, 0, sizeof(kobj->thread->arch)); + pdata->thread = kobj->thread; + + /* success, fill the structures */ + pdata->p_stack = p_stack; + pdata->mod = mod; + + /* initialize other task structures */ + ptask->ops.complete = ops->complete; + ptask->ops.get_deadline = ops->get_deadline; + ptask->priv_data = pdata; + list_init(&ptask->list); + + /* create a zephyr thread for the task */ + thread_id = k_thread_create(kobj->thread, p_stack, + stack_size, dp_thread_fn, ptask, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, ptask->flags, K_FOREVER); + pdata->thread_id = thread_id; + scheduler_dp_thread_name_set(thread_id, mod); + + unsigned int pidx; + size_t size; + uintptr_t start; + bool on_pool = false; + +#ifdef CONFIG_SCHED_CPU_MASK + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } +#endif + + k_thread_access_grant(thread_id, kobj->event, &dp_sync[core]); + scheduler_dp_grant(thread_id, core); +#if CONFIG_SOF_USERSPACE_LL + struct k_thread *thread_ipc = ipc_thread_user(core); + + k_thread_access_grant(thread_ipc, kobj->event, thread_id, p_stack, &dp_sync[core]); + scheduler_dp_grant(thread_ipc, core); + scheduler_dp_grant(zephyr_ll_domain_thread(), core); +#endif + + struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), + SOF_MEM_FLAG_COHERENT); + + if (!mdom) { + tr_err(&dp_tr, "objpool allocation failed"); + ret = -ENOMEM; + goto e_thread; + } + + on_pool = true; + + mod->mdom = mdom; + kobj->mdom = mdom; + + if (!mdom->arch.ptables) { + ret = k_mem_domain_init(mdom, 0, NULL); + if (ret < 0) + goto e_thread; + } + + /* Module heap partition */ + mod_heap_info(mod, &size, &start); + kobj->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)start), + .size = size, + .attr = K_MEM_PARTITION_P_RW_U_RW, + }; + kobj->mpart[SOF_DP_PART_HEAP_CACHE] = (struct k_mem_partition){ + .start = start, + .size = size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + /* Host mailbox partition for additional IPC parameters: read-only */ + kobj->mpart[SOF_DP_PART_CFG] = (struct k_mem_partition){ + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)MAILBOX_HOSTBOX_BASE), + .size = MAILBOX_HOSTBOX_SIZE, + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + kobj->mpart[SOF_DP_PART_CFG_CACHE] = (struct k_mem_partition){ + .start = (uintptr_t)MAILBOX_HOSTBOX_BASE, + .size = MAILBOX_HOSTBOX_SIZE, + .attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB, + }; + + for (pidx = 0; pidx < SOF_DP_PART_TYPE_COUNT; pidx++) { + ret = k_mem_domain_add_partition(mdom, kobj->mpart + pidx); + if (ret < 0) + goto e_dom; + } + + memcpy(pdata->mpart, kobj->mpart, sizeof(pdata->mpart)); + + ret = llext_manager_add_domain(comp_id, mdom); + if (ret < 0) { + tr_err(&dp_tr, "failed to add LLEXT to domain %d", ret); + goto e_dom; + } + + /* + * Keep this call last, able to fail, otherwise domain will be removed + * before its thread + */ + ret = k_mem_domain_add_thread(mdom, thread_id); + if (ret < 0) { + tr_err(&dp_tr, "failed to add thread to domain %d", ret); + goto e_dom; + } + + /* publish the record before the thread can run */ + dp_task_kobj_add(kobj); + + /* start the thread, it should immediately stop at the semaphore */ + k_event_init(kobj->event); + k_thread_start(thread_id); + + /* ptask points into task_memory, so only publish it once it cannot be freed */ + *task = ptask; + + return 0; + +e_dom: + scheduler_dp_domain_free(mod, mdom, kobj->mpart, comp_id); + on_pool = false; +e_thread: + if (on_pool) + objpool_free(&dp_mdom_head, mdom); + k_thread_abort(thread_id); + k_object_free(kobj->thread); +e_event: + k_object_free(kobj->event); +e_config: + mod_free(mod, ipc_config_data); +e_kobj: + rfree(kobj); +e_stack: + user_stack_free(p_stack); +e_tmem: + /* the copy lives in task_memory, so stop pointing at it before freeing */ + mod->dev->drv = drv; + mod_free(mod, task_memory); + return ret; +} + +#ifdef CONFIG_USERSPACE +#include +#include + +/* + * mod_free() follows the module's resource and allocator pointers in + * supervisor mode. They are user-writable, so at least check that they + * point to memory the caller can access. + */ +static void scheduler_dp_mod_vrfy(struct processing_module *mod) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); + K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); + + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc, sizeof(*alloc))); + if (alloc->heap) { + size_t h_size = 0; + uintptr_t h_start; + + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); + } + if (alloc->vreg) { + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(alloc->vreg, &vr_size, &vr_start); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); + } +} + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + struct scheduler_dp_task_kobj kobj; + + SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); + + /* + * Do not read task->priv_data: the kernel objects come from the + * kernel-only record, and an unknown task pointer is rejected. + */ + K_OOPS(!dp_task_kobj_peek(task, &kobj)); + + /* + * A DP thread holds its own thread and event objects, so it would pass + * the checks below for its own task. Freeing would then abort the + * caller after the record is unlinked, leaking everything it holds. + */ + K_OOPS(kobj.thread == k_current_get()); + + /* + * The caller must have been granted the task's objects. Check before + * unlinking the record, so an unauthorized call leaves it intact. + */ + K_OOPS(K_SYSCALL_OBJ(kobj.event, K_OBJ_EVENT)); + K_OOPS(K_SYSCALL_OBJ_INIT(kobj.thread, K_OBJ_THREAD)); + scheduler_dp_mod_vrfy(kobj.mod); + + K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj)); +} +#include +#endif /* CONFIG_USERSPACE */ diff --git a/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c b/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c new file mode 100644 index 000000000000..05c85ad98b1e --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + * Adrian Warecki + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; + +/* Kernel part of the thread variant of the DP scheduler: the DP thread creation and + * the scheduler_dp_internal_free() system call. The DP thread function is in + * src/schedule/zephyr_dp_schedule_thread.c + */ + +int scheduler_dp_task_init(struct task **task, + const struct sof_uuid_entry *uid, + const struct task_ops *ops, + struct processing_module *mod, + uint16_t core, + size_t stack_size, + uint32_t options) +{ + void __sparse_cache *p_stack = NULL; + struct k_heap *const user_heap = mod->dev->drv->user_heap; + + /* memory allocation helper structure */ + struct { + struct task task; /* keep first, used for freeing below */ + struct task_dp_pdata pdata; + } *task_memory; + + int ret; + + /* must be called on the same core the task will be binded to */ + assert(cpu_get_id() == core); + + /* + * allocate memory + * to avoid multiple malloc operations allocate all required memory as a single structure + * and return pointer to task_memory->task + * As the structure contains zephyr kernel specific data, it must be located in + * shared, non cached memory + */ + task_memory = sof_heap_alloc(user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*task_memory), 0); + if (!task_memory) { + tr_err(&dp_tr, "memory alloc failed"); + return -ENOMEM; + } + + memset(task_memory, 0, sizeof(*task_memory)); + /* allocate stack - must be aligned and cached so a separate alloc */ + p_stack = user_stack_allocate(stack_size, options); + if (!p_stack) { + tr_err(&dp_tr, "stack alloc failed"); + ret = -ENOMEM; + goto err; + } + + /* internal SOF task init */ + ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, + mod, core, options); + if (ret < 0) { + tr_err(&dp_tr, "schedule_task_init failed"); + goto err; + } + + struct task_dp_pdata *pdata = &task_memory->pdata; + + /* Point to event_struct event for kernel threads synchronization */ + /* It will be overwritten for K_USER threads to dynamic ones. */ + pdata->event = &pdata->event_struct; + pdata->thread = &pdata->thread_struct; + +#ifdef CONFIG_USERSPACE + if (options & K_USER) { + pdata->event = k_object_alloc(K_OBJ_EVENT); + if (!pdata->event) { + tr_err(&dp_tr, "Event object allocation failed"); + ret = -ENOMEM; + goto err; + } + + pdata->thread = k_object_alloc(K_OBJ_THREAD); + if (!pdata->thread) { + tr_err(&dp_tr, "Thread object allocation failed"); + ret = -ENOMEM; + goto err; + } + } +#endif /* CONFIG_USERSPACE */ + + /* initialize other task structures */ + task_memory->task.ops.complete = ops->complete; + task_memory->task.ops.get_deadline = ops->get_deadline; + task_memory->task.state = SOF_TASK_STATE_INIT; + task_memory->task.core = core; + task_memory->task.priv_data = pdata; + k_event_init(pdata->event); + + /* success, fill the structures */ + pdata->p_stack = p_stack; + pdata->mod = mod; + + /* create a zephyr thread for the task */ + pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)p_stack, + stack_size, dp_thread_fn, &task_memory->task, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, task_memory->task.flags, + K_FOREVER); + +#ifdef CONFIG_SCHED_CPU_MASK + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(pdata->thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } +#endif + +#ifdef CONFIG_USERSPACE + if (options & K_USER) { + k_thread_access_grant(pdata->thread_id, pdata->event); + scheduler_dp_grant(pdata->thread_id, core); + + ret = k_mem_domain_add_thread(pdata->mod->user_ctx->comp_dom, pdata->thread_id); + if (ret < 0) { + tr_err(&dp_tr, "k_mem_domain_add_thread() failed %d", ret); + goto e_thread; + } + +#if IS_ENABLED(CONFIG_SOF_USERSPACE_MOD_IPC_BY_DP_THREAD) + pdata->ipc_work_item = userspace_proxy_register_ipc_handler(mod, pdata->event); + assert(pdata->ipc_work_item); +#endif + } +#endif /* CONFIG_USERSPACE */ + + /* start the thread, it should immediately stop at an event */ + k_thread_start(pdata->thread_id); + + /* success, fill output parameter */ + *task = &task_memory->task; + return 0; + +e_thread: + k_thread_abort(pdata->thread_id); +err: + /* cleanup - free all allocated resources */ + if (user_stack_free((__sparse_force void *)p_stack)) + tr_err(&dp_tr, "user_stack_free failed!"); + + /* k_object_free looks for a pointer in the list, any invalid value can be passed */ + k_object_free(task_memory->pdata.event); + k_object_free(task_memory->pdata.thread); + sof_heap_free(user_heap, task_memory); + return ret; +} + +void z_impl_scheduler_dp_internal_free(struct task *task) +{ + struct task_dp_pdata *pdata = task->priv_data; + +#ifdef CONFIG_USERSPACE + if (pdata->event != &pdata->event_struct) + k_object_free(pdata->event); + if (pdata->thread != &pdata->thread_struct) + k_object_free(pdata->thread); +#endif + + /* task is the first member in task_memory above */ + sof_heap_free(pdata->mod->dev->drv->user_heap, task); +} + +#ifdef CONFIG_USERSPACE +#include + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + /* + * With the thread DP scheduler variant scheduler_dp_internal_free() is + * never called from the userspace context + */ + K_OOPS(true); +} +#include +#endif diff --git a/src/kernel/schedule/zephyr_ll-kernel.c b/src/kernel/schedule/zephyr_ll-kernel.c new file mode 100644 index 000000000000..c0a8398b3dc8 --- /dev/null +++ b/src/kernel/schedule/zephyr_ll-kernel.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Guennadi Liakhovetski + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); + +/* Kernel part of the Zephyr LL scheduler: the scheduler initialization, the task + * semaphore syscalls and the access grants to the scheduler objects. The rest is in + * src/schedule/zephyr_ll.c + */ + +#if CONFIG_SOF_USERSPACE_LL +/* + * Kernel-only copy of zephyr_ll_locks[]. That array is writable by user + * threads, so kernel code granting access to the locks must use this copy, + * otherwise a user thread could make it grant access to any kernel object. + */ +static struct k_mutex *zephyr_ll_klocks[CONFIG_CORE_COUNT]; +#endif + +#if CONFIG_DYNAMIC_OBJECTS +static struct list_item zephyr_ll_task_sem_list = LIST_INIT(zephyr_ll_task_sem_list); + +struct zephyr_ll_task_sem { + struct task *task; + struct k_sem *sem; + struct list_item list; +}; + +/* + * The syscall validators pass in pdata that they read once from user memory + * and validated, so these helpers must not read task->priv_data again. + */ +static int zephyr_ll_task_sem_alloc_pdata(struct task *task, struct zephyr_ll_pdata *pdata) +{ + struct zephyr_ll_task_sem *ts = rmalloc(SOF_MEM_FLAG_COHERENT, sizeof(*ts)); + + if (!ts) + return -ENOMEM; + + ts->sem = k_object_alloc(K_OBJ_SEM); + if (!ts->sem) { + rfree(ts); + return -ENOMEM; + } + + k_sem_init(ts->sem, 0, 1); + +#if CONFIG_SOF_USERSPACE_LL + /* + * The per-task semaphore is signalled from zephyr_ll_task_done(), + * which runs in the (unprivileged) LL scheduler thread when a task is + * freed while it is still running. k_object_alloc() only grants access + * to the calling thread (the IPC handler that creates the task), so the + * LL thread must be granted access explicitly, otherwise its + * k_sem_give() traps with a userspace permission fault. + * + * Resolve the LL thread from kernel-only per-core state keyed by the + * task's target core; never traverse the user-accessible scheduler or + * domain objects from privileged context. + */ + struct k_thread *ll_tid = zephyr_domain_thread_tid_for_core(task->core); + + if (ll_tid) + k_thread_access_grant(ll_tid, ts->sem); +#endif /* CONFIG_SOF_USERSPACE_LL */ + + ts->task = task; + pdata->sem_p = ts->sem; + /* List is protected by IPC serialization */ + list_item_append(&ts->list, &zephyr_ll_task_sem_list); + + return 0; +} + +int z_impl_zephyr_ll_task_sem_alloc(struct task *task) +{ + return zephyr_ll_task_sem_alloc_pdata(task, task->priv_data); +} + +static int zephyr_ll_task_sem_free_pdata(struct task *task, struct zephyr_ll_pdata *pdata) +{ + struct list_item *list; + struct zephyr_ll_task_sem *ts; + bool found = false; + + /* List is protected by IPC serialization */ + list_for_item(list, &zephyr_ll_task_sem_list) { + ts = container_of(list, struct zephyr_ll_task_sem, list); + if (ts->task == task) { + found = true; + break; + } + } + + if (!found) + return -ENOENT; + + if (pdata->sem_p != ts->sem) + return -EINVAL; + + list_item_del(list); + k_object_free(ts->sem); + rfree(ts); + + return 0; +} + +int z_impl_zephyr_ll_task_sem_free(struct task *task) +{ + return zephyr_ll_task_sem_free_pdata(task, task->priv_data); +} + +#ifdef CONFIG_USERSPACE +#include +#include +/* + * Read task->priv_data exactly once. The task lives in user memory and a + * thread on another core can change the pointer after it has been checked. + * + * @return validated pdata pointer, or NULL if the task has none + */ +static struct zephyr_ll_pdata *zephyr_ll_task_pdata_vrfy(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + K_OOPS(k_usermode_from_copy(&pdata, &task->priv_data, sizeof(pdata))); + if (pdata) + K_OOPS(K_SYSCALL_MEMORY_WRITE(pdata, sizeof(*pdata))); + + return pdata; +} + +static inline int z_vrfy_zephyr_ll_task_sem_alloc(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); + + if (!task) + return -EINVAL; + pdata = zephyr_ll_task_pdata_vrfy(task); + if (!pdata) + return -EINVAL; + + return zephyr_ll_task_sem_alloc_pdata(task, pdata); +} +#include + +static inline int z_vrfy_zephyr_ll_task_sem_free(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); + + if (!task) + return -EINVAL; + pdata = zephyr_ll_task_pdata_vrfy(task); + if (!pdata) + return -EINVAL; + + return zephyr_ll_task_sem_free_pdata(task, pdata); +} +#include +#endif /* CONFIG_USERSPACE */ +#endif /* CONFIG_DYNAMIC_OBJECTS */ + +#if CONFIG_SOF_USERSPACE_LL +struct k_thread *zephyr_ll_init_context(void *data, struct task *task) +{ + struct zephyr_ll *sch = data; + struct zephyr_ll_pdata *pdata = task->priv_data; + int ret; + + /* + * Use domain_thread_init() for privileged setup (thread creation, + * timer, access grants). domain_register() is now bookkeeping only + * and will be called later from user context when scheduling tasks. + */ + ret = domain_thread_init(sch->ll_domain, task); + if (ret < 0) { + tr_err(&ll_tr, "cannot init_context %d", ret); + return NULL; + } + + assert(!k_is_user_context()); + k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock, pdata->sem_p); + + tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + + return zephyr_domain_thread_tid(sch->ll_domain); +} + +void user_ll_grant_access(struct k_thread *thread, int core) +{ + assert(core < CONFIG_CORE_COUNT && zephyr_ll_klocks[core] != NULL); + + k_thread_access_grant(thread, zephyr_ll_klocks[core]); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY +static void zephyr_ll_scheduler_free(void *data, uint32_t flags) +{ + struct zephyr_ll *sch = data; + + zephyr_ll_assert_core(sch); + +#if CONFIG_SOF_USERSPACE_LL + zephyr_ll_locks[sch->core] = NULL; + zephyr_ll_klocks[sch->core] = NULL; + k_object_free(sch->lock); +#endif + + if (sch->n_tasks) + tr_err(&ll_tr, "%u tasks are still active!", + sch->n_tasks); + +#if CONFIG_SOF_USERSPACE_LL + domain_thread_free(sch->ll_domain, sch->n_tasks); +#endif + sof_heap_free(sch->heap, sch); +} +#endif /* CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY */ + +static const struct scheduler_ops zephyr_ll_ops = { + .schedule_task = zephyr_ll_task_schedule, + .schedule_task_before = zephyr_ll_task_schedule_before, + .schedule_task_after = zephyr_ll_task_schedule_after, + .schedule_task_free = zephyr_ll_task_free, + .schedule_task_cancel = zephyr_ll_task_cancel, +#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY + .scheduler_free = zephyr_ll_scheduler_free, +#endif +#if CONFIG_SOF_USERSPACE_LL + .scheduler_init_context = zephyr_ll_init_context, +#endif +}; + +/* TODO: low-power mode clock support */ +/* Runs on each core during initialisation with the same domain argument */ +__cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) +{ + struct zephyr_ll *sch; + int core = cpu_get_id(); + struct k_heap *heap = sof_sys_heap_get(); + int flags = SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT; + + assert_can_be_cold(); + +#if CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; +#endif + tr_dbg(&ll_tr, "init on core %d", core); + + /* initialize per-core scheduler private data */ + sch = sof_heap_alloc(heap, flags, sizeof(*sch), 0); + if (!sch) { + tr_err(&ll_tr, "allocation failed"); + return -ENOMEM; + } + + memset(sch, 0, sizeof(*sch)); + + list_init(&sch->tasks); + sch->ll_domain = domain; + sch->core = core; + sch->n_tasks = 0; + sch->heap = heap; + +#if CONFIG_SOF_USERSPACE_LL + /* Allocate mutex dynamically for userspace access */ + sch->lock = k_object_alloc(K_OBJ_MUTEX); + if (!sch->lock) { + tr_err(&ll_tr, "mutex allocation failed"); + sof_heap_free(sch->heap, sch); + return -ENOMEM; + } + assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); + zephyr_ll_locks[core] = sch->lock; + zephyr_ll_klocks[core] = sch->lock; + k_mutex_init(sch->lock); + + tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); +#endif + + scheduler_init(domain->type, &zephyr_ll_ops, sch); + + return 0; +} diff --git a/src/schedule/zephyr_twb_schedule.c b/src/kernel/schedule/zephyr_twb_schedule.c similarity index 100% rename from src/schedule/zephyr_twb_schedule.c rename to src/kernel/schedule/zephyr_twb_schedule.c diff --git a/src/schedule/CMakeLists.txt b/src/schedule/CMakeLists.txt index bc14c6ecff46..ee9ceb32ef9d 100644 --- a/src/schedule/CMakeLists.txt +++ b/src/schedule/CMakeLists.txt @@ -50,8 +50,4 @@ else() ) endif() -zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER - zephyr_twb_schedule.c -) - endif() diff --git a/src/schedule/README.md b/src/schedule/README.md index 0bef23248837..d2a86f6bf55c 100644 --- a/src/schedule/README.md +++ b/src/schedule/README.md @@ -1,12 +1,15 @@ # SOF Scheduling Architecture -> **Runs in:** Boundary / bridge — this directory owns the LL/DP schedulers and their thread + -> memory-domain setup. `zephyr_ll.c` and `zephyr_domain.c` are kernel-side; the LL user memory -> domain and heap are set up by `src/kernel/schedule/zephyr_ll_user.c`. -> `zephyr_dp_schedule_thread.c` is boundary code that can create and manage both kernel and -> `K_USER` DP threads depending on configuration; `zephyr_dp_schedule_application.c` is the -> DP application-mode path. This directory hosts the `zephyr_ll_task_sem_*` and -> `scheduler_dp_*` syscalls. See [User/kernel split](../include/sof/userspace/README.md). +> **Runs in:** Boundary / bridge — this directory holds the parts of the LL and DP schedulers +> that run in the scheduler and IPC threads, which are user threads with user-space LL. Their +> kernel parts are in `src/kernel/schedule`, one `*-kernel.c` file per source here: scheduler +> registration and initialization, the LL thread and timer setup and the timer interrupt, the +> DP thread creation and memory domains, the access grants, and the kernel halves of the +> `zephyr_ll_task_sem_*` and `scheduler_dp_*` syscalls. The LL user memory domain and heap are +> set up by `src/kernel/schedule/zephyr_ll_user.c`, and the TWB scheduler, which is kernel-only, +> is `src/kernel/schedule/zephyr_twb_schedule.c`. The scheduler data structures shared by both +> parts are in `sof/schedule/zephyr_ll.h` and `sof/schedule/zephyr_dp_schedule.h`. See +> [User/kernel split](../include/sof/userspace/README.md). This directory (`src/schedule`) contains the Sound Open Firmware (SOF) scheduling infrastructure, deeply integrated with the underlying Zephyr RTOS. SOF utilizes a multi-tiered scheduling approach to cater to different real-time constraints, ranging from hard real-time, low-latency requirements to more relaxed, compute-intensive data processing tasks. @@ -139,7 +142,7 @@ stateDiagram-v2 ## 3. TWB (Thread With Budget) Scheduler -The TWB scheduler (`zephyr_twb_schedule.c`) provides execution budget limits for specific tasks to prevent them from starving the CPU. This is useful for intensive workloads that shouldn't disrupt the overall systemic low-latency chain. +The TWB scheduler (`src/kernel/schedule/zephyr_twb_schedule.c`) provides execution budget limits for specific tasks to prevent them from starving the CPU. This is useful for intensive workloads that shouldn't disrupt the overall systemic low-latency chain. ### Architecture diff --git a/src/schedule/schedule.c b/src/schedule/schedule.c index fd3349a14902..62438be7c663 100644 --- a/src/schedule/schedule.c +++ b/src/schedule/schedule.c @@ -25,16 +25,9 @@ SOF_DEFINE_REG_UUID(schedule); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(sch_tr, SOF_UUID(schedule_uuid), LOG_LEVEL_INFO); -static inline bool scheduler_is_user(int type) -{ - /* - * currently only LL managed in user-space, but longterm - * goal is to move all audio application level scheduling - * to user-space and only keep Zephyr scheduler logic in - * kernel - */ - return type == SOF_SCHEDULE_LL_TIMER || type == SOF_SCHEDULE_DP; -} +/* Scheduler registration, which runs in kernel mode, is in + * src/kernel/schedule/schedule-kernel.c + */ int schedule_task_init(struct task *task, const struct sof_uuid_entry *uid, uint16_t type, @@ -84,60 +77,6 @@ int schedule_task_init(struct task *task, return 0; } -__cold static void scheduler_register(struct schedule_data *scheduler) -{ - struct schedulers **sch = arch_schedulers_get(); - struct k_heap *heap = NULL; - - assert_can_be_cold(); - - if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(scheduler->type)) { - sch = arch_user_schedulers_get(); - heap = sof_sys_user_heap_get(); - } - - if (!*sch) { - /* init schedulers list */ - *sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0); - if (!*sch) { - tr_err(&sch_tr, "allocation failed"); - return; - } - memset(*sch, 0, sizeof(**sch)); - list_init(&(*sch)->list); - } - - list_item_append(&scheduler->list, &(*sch)->list); -} - -__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data) -{ - struct schedule_data *sch; - struct k_heap *heap = NULL; - - assert_can_be_cold(); - - if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(type)) - heap = sof_sys_user_heap_get(); - - if (!ops || !ops->schedule_task || !ops->schedule_task_cancel || - !ops->schedule_task_free) - return; - - sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0); - if (!sch) { - tr_err(&sch_tr, "allocation failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - memset(sch, 0, sizeof(*sch)); - list_init(&sch->list); - sch->type = type; - sch->ops = ops; - sch->data = data; - - scheduler_register(sch); -} - /* Locks for the list here should be held by the caller, * as different schedulers use different locks */ diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 58d1f4b7c0fe..11e14dbc187c 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -4,19 +4,15 @@ // // Author: Tomasz Lauda -#include #include #include -#include #include -#include #include #include -#include #include #include +#include #include -#include #include #include #include @@ -29,47 +25,9 @@ LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); -/* - * Currently the Zephyr clock rate is part it's Kconfig known at build time. - * SOF on Intel CAVS platforms currently only aligns with Zephyr when both - * use the CAVS 19.2 MHz SSP clock. TODO - needs runtime alignment. +/* The thread and timer setup, which runs in kernel mode, is in + * src/kernel/schedule/zephyr_domain-kernel.c */ -#if CONFIG_XTENSA && CONFIG_CAVS && !CONFIG_INTEL_ADSP_TIMER -#error "Zephyr uses 19.2MHz clock derived from SSP which must be enabled." -#endif - -#define ZEPHYR_LL_STACK_SIZE 8192 - -#if CONFIG_SOF_USERSPACE_LL -K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); -#else -K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); -#endif - -struct zephyr_domain_thread { - struct k_thread *ll_thread; - struct k_sem *sem; -#ifndef CONFIG_SOF_USERSPACE_LL - struct k_thread ll_thread_obj; - struct k_sem sem_obj; -#endif - void (*handler)(void *arg); - void *arg; -}; - -struct zephyr_domain { - struct k_timer *timer; -#ifndef CONFIG_SOF_USERSPACE_LL - struct k_timer timer_obj; -#endif - struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; - struct ll_schedule_domain *ll_domain; -#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL - atomic_t block; - struct k_mutex block_mutex; - struct k_condvar block_condvar; -#endif -}; /* perf measurement windows size 2^x */ #define CYCLES_WINDOW_SIZE CONFIG_SCHEDULE_LL_STATS_LOG_WINDOW_SIZE @@ -87,7 +45,7 @@ static inline void stats_report(unsigned int runs, int core, unsigned int cycles } #endif /* CONFIG_SCHEDULE_LL_STATS_LOG */ -static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) +void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) { struct zephyr_domain *zephyr_domain = p1; int core = POINTER_TO_INT(p2); @@ -156,253 +114,14 @@ static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) } } -/* Timer callback: runs in timer IRQ context */ -static void zephyr_domain_timer_fn(struct k_timer *timer) -{ - struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer); - int core; - - /* - * A race is possible when the Zephyr LL scheduling domain is being - * unregistered while a timer IRQ is processed on a different core. Then - * the timer is removed by the former but then re-added by the latter, - * but this time with no user data and no handler set. This leads to the - * timer continuing to trigger and then leading to a lock up when it is - * registered again next time. - */ - if (!zephyr_domain) { - k_timer_stop(timer); - return; - } - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - if (dt->handler) - k_sem_give(dt->sem); - } -} - -/* The normal kernel-space implementation for register/unregister */ -#ifndef CONFIG_SOF_USERSPACE_LL - -static int zephyr_domain_register(struct ll_schedule_domain *domain, - struct task *task, - void (*handler)(void *arg), void *arg) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - char thread_name[] = "ll_thread0"; - k_tid_t thread; - k_spinlock_key_t key; - - tr_dbg(&ll_tr, "entry"); - - /* domain work only needs registered once on each core */ - if (dt->handler) - return 0; - - dt->handler = handler; - dt->arg = arg; - - /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ - k_sem_init(dt->sem, 0, 10); - - thread_name[sizeof(thread_name) - 2] = '0' + core; - - /* not allocated dynamically when LL in kernel space */ - dt->ll_thread = &dt->ll_thread_obj; - - thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, - zephyr_domain_thread_fn, zephyr_domain, INT_TO_POINTER(core), - NULL, CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER); - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, core); -#endif - k_thread_name_set(thread, thread_name); - - k_thread_start(thread); - - key = k_spin_lock(&domain->lock); - - if (!k_timer_user_data_get(zephyr_domain->timer)) { - k_timeout_t start = {0}; - - k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); - k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); - - k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); - - /* Enable the watchdog */ - watchdog_enable(core); - } - - k_spin_unlock(&domain->lock, key); - - tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", - domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); - - return 0; -} - -static int zephyr_domain_unregister(struct ll_schedule_domain *domain, - struct task *task, uint32_t num_tasks) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - k_spinlock_key_t key; - - tr_dbg(&ll_tr, "entry"); - - /* tasks still registered on this core */ - if (num_tasks) - return 0; - - key = k_spin_lock(&domain->lock); - - if (!atomic_read(&domain->total_num_tasks)) { - /* Disable the watchdog */ - watchdog_disable(core); - - k_timer_stop(zephyr_domain->timer); - k_timer_user_data_set(zephyr_domain->timer, NULL); - } - - zephyr_domain->domain_thread[core].handler = NULL; - - k_spin_unlock(&domain->lock, key); - - tr_info(&ll_tr, "domain->type %d domain->clk %d", - domain->type, domain->clk); - - /* - * If running in the context of the domain thread, k_thread_abort() will - * not return - */ - k_thread_abort(zephyr_domain->domain_thread[core].ll_thread); - - tr_dbg(&ll_tr, "exit"); - - return 0; -} - -#else /* CONFIG_SOF_USERSPACE_LL */ - -/* - * Kernel-owned per-core LL thread table. Populated from the privileged - * domain-thread init path and consulted by z_impl_zephyr_ll_task_sem_alloc() - * so that privileged code never has to traverse the user-accessible - * scheduler/domain objects to find the LL thread of a given core. - */ -static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT]; - -/* - * Privileged thread initialization for userspace LL scheduling. - * Creates the scheduling thread, sets up timer, grants access to kernel - * objects. Must be called from kernel context before any user-space - * domain_register() calls. - */ -static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, - struct task *task) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - struct zephyr_domain_thread *dt; - char thread_name[] = "ll_thread0"; - k_tid_t thread; - int core = task->core; - - tr_dbg(&ll_tr, "entry"); - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - dt = zephyr_domain->domain_thread + core; - - /* thread only needs to be created once per core */ - if (dt->ll_thread) - return 0; - - dt->handler = NULL; - dt->arg = NULL; - - dt->sem = k_object_alloc(K_OBJ_SEM); - if (!dt->sem) { - tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core); - return -ENOMEM; - } - - /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ - k_sem_init(dt->sem, 0, 10); - - thread_name[sizeof(thread_name) - 2] = '0' + core; - - /* - * Allocate thread structure dynamically. Keep the handle in a local: - * the domain is user-writable, so dt->ll_thread must not be read back - * for the kernel-only tables below. - */ - thread = k_object_alloc(K_OBJ_THREAD); - if (!thread) { - tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); - k_object_free(dt->sem); - dt->sem = NULL; - return -ENOMEM; - } - - k_thread_create(thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, - zephyr_domain_thread_fn, zephyr_domain, - INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, - K_USER, K_FOREVER); - - /* record in the kernel-only table for syscall-context lookups */ - ll_thread_tid[core] = thread; - sof_sysuser_thread_set(SOF_SYSUSER_LL, core, thread); - dt->ll_thread = thread; - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, core); -#endif - k_thread_name_set(thread, thread_name); - - k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); - user_grant_dai_access_all(thread); - user_grant_dma_access_all(thread); - user_grant_uaol_access_all(thread); - tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); - - k_thread_start(thread); - - k_mutex_lock(domain->lock, K_FOREVER); - if (!k_timer_user_data_get(zephyr_domain->timer)) { - k_timeout_t start = {0}; - - k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); - k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); - - k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); - - /* Enable the watchdog */ - watchdog_enable(core); - } - - k_mutex_unlock(domain->lock); - - tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", - domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); - - return 0; -} - +#ifdef CONFIG_SOF_USERSPACE_LL /* * User-space register: bookkeeping only. The privileged thread setup has * already been done by domain_thread_init() called from kernel context. */ -static int zephyr_domain_register_user(struct ll_schedule_domain *domain, - struct task *task, - void (*handler)(void *arg), void *arg) +int zephyr_domain_register_user(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); struct zephyr_domain_thread *dt; @@ -433,8 +152,8 @@ static int zephyr_domain_register_user(struct ll_schedule_domain *domain, return 0; } -static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, - struct task *task, uint32_t num_tasks) +int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); int core = task->core; @@ -461,95 +180,10 @@ static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, return 0; } - -/* - * Free resources acquired by zephyr_domain_thread_init(). - * Stops the timer, aborts the scheduling thread and frees the thread object. - * Must be called from kernel context. - */ -static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, - uint32_t num_tasks) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - tr_dbg(&ll_tr, "thread_free entry, core %d, num_tasks %u", core, num_tasks); - - /* Still tasks on other cores, only clean up this core's thread */ - k_mutex_lock(domain->lock, K_FOREVER); - - if (!num_tasks && !atomic_read(&domain->total_num_tasks)) { - /* Last task globally: stop the timer and watchdog */ - watchdog_disable(core); - - k_timer_stop(zephyr_domain->timer); - k_timer_user_data_set(zephyr_domain->timer, NULL); - } - - dt->handler = NULL; - dt->arg = NULL; - - k_mutex_unlock(domain->lock); - - if (dt->ll_thread) { - sof_sysuser_thread_set(SOF_SYSUSER_LL, core, NULL); - ll_thread_tid[core] = NULL; - k_thread_abort(dt->ll_thread); - k_object_free(dt->ll_thread); - dt->ll_thread = NULL; - } - - if (dt->sem) { - k_object_free(dt->sem); - dt->sem = NULL; - } - - tr_info(&ll_tr, "thread_free done, core %d", core); -} - -struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - tr_dbg(&ll_tr, "entry"); - - return dt->ll_thread; -} - -/* - * Return the LL scheduling thread for an explicitly given core. - * - * Reads a kernel-only table keyed by core, so it is safe to call from a - * privileged syscall context without dereferencing any user-accessible - * scheduler or domain object, and without relying on cpu_get_id(). - */ -struct k_thread *zephyr_domain_thread_tid_for_core(int core) -{ - if (core < 0 || core >= CONFIG_CORE_COUNT) - return NULL; - - return ll_thread_tid[core]; -} - -struct k_thread *zephyr_ll_domain_thread(void) -{ - struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); - - if (!ll_domain) - return NULL; - - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); - - return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; -} - #endif /* CONFIG_SOF_USERSPACE_LL */ #if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL -static void zephyr_domain_block(struct ll_schedule_domain *domain) +void zephyr_domain_block(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -560,7 +194,7 @@ static void zephyr_domain_block(struct ll_schedule_domain *domain) k_mutex_unlock(&zephyr_domain->block_mutex); } -static void zephyr_domain_unblock(struct ll_schedule_domain *domain) +void zephyr_domain_unblock(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -573,87 +207,6 @@ static void zephyr_domain_unblock(struct ll_schedule_domain *domain) } #endif /* CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL */ -APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { -#ifdef CONFIG_SOF_USERSPACE_LL - .domain_register = zephyr_domain_register_user, - .domain_unregister = zephyr_domain_unregister_user, - .domain_thread_init = zephyr_domain_thread_init, - .domain_thread_free = zephyr_domain_thread_free, -#else - .domain_register = zephyr_domain_register, - .domain_unregister = zephyr_domain_unregister, -#endif -#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL - .domain_block = zephyr_domain_block, - .domain_unblock = zephyr_domain_unblock, -#endif -}; - -/* Core 0 only */ -struct ll_schedule_domain *zephyr_domain_init(int clk) -{ - struct ll_schedule_domain *domain; - struct zephyr_domain *zephyr_domain; - int core; - - domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, false, - &zephyr_domain_ops); - if (!domain) { - tr_err(&ll_tr, "domain init failed"); - return NULL; - } - -#if CONFIG_SOF_USERSPACE_LL - zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*zephyr_domain), sizeof(void *)); - if (zephyr_domain) - memset(zephyr_domain, 0, sizeof(*zephyr_domain)); -#else - zephyr_domain = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*zephyr_domain)); -#endif - if (!zephyr_domain) { - tr_err(&ll_tr, "domain allocation failed"); - k_panic(); - return NULL; - } - - zephyr_domain->ll_domain = domain; - -#if CONFIG_SOF_USERSPACE_LL - /* Allocate timer dynamically for userspace access */ - zephyr_domain->timer = k_object_alloc(K_OBJ_TIMER); - if (!zephyr_domain->timer) { - tr_err(&ll_tr, "timer allocation failed"); - k_panic(); - return NULL; - } -#else - /* not allocated dynamically when LL in kernel space */ - zephyr_domain->timer = &zephyr_domain->timer_obj; -#endif - -#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL - atomic_set(&zephyr_domain->block, 0); - k_mutex_init(&zephyr_domain->block_mutex); - k_condvar_init(&zephyr_domain->block_condvar); -#endif - - ll_sch_domain_set_pdata(domain, zephyr_domain); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { -#ifndef CONFIG_SOF_USERSPACE_LL - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - /* not allocated dynamically when LL in kernel space */ - dt->sem = &dt->sem_obj; -#endif - } - - return domain; -} - /* Check if currently running in the LL scheduler thread context */ bool ll_sch_is_current(void) { diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 9ff0eb1928a5..ac50500abd7f 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -25,28 +24,19 @@ #include #include -#include "zephyr_dp_schedule.h" +#include #include #include #include -LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); -SOF_DEFINE_REG_UUID(dp_sched); +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); - -#define DP_LOCK_INIT(i, _) Z_SEM_INITIALIZER(dp_lock[i], 1, 1) -#define DP_LOCK_INIT_LIST LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,)) - -/* User threads don't need access to this array. Access is performed from - * the kernel space via a syscall. Array must be placed in special section - * to be qualified as initialized by the gen_kobject_list.py script. +/* The scheduler locks, the LL tick system call and the scheduler initialization, which + * run in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule-kernel.c */ -static -STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST }; /* Each per-core instance of DP scheduler has separate structures; hence, locks are per-core. * @@ -63,221 +53,8 @@ void scheduler_dp_unlock(unsigned int key) k_sem_give(&dp_lock[key]); } -void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) -{ - if (thread_id) - k_thread_access_grant(thread_id, &dp_lock[core]); -} - -/* dummy LL task - to start LL on secondary cores */ -static enum task_state scheduler_dp_ll_tick_dummy(void *data) -{ - return SOF_TASK_STATE_RESCHEDULE; -} - -/* - * function called after every LL tick - * - * This function checks if the queued DP tasks are ready to processing (meaning - * the module run by the task has enough data at all sources and enough free space - * on all sinks) - * - * if the task becomes ready, a deadline is set allowing Zephyr to schedule threads - * in right order - * - * TODO: currently there's a limitation - DP module must be surrounded by LL modules. - * it simplifies algorithm - there's no need to browse through DP chains calculating - * deadlines for each module in function of all modules execution status. - * Now is simple - modules deadline is its start + tick time. - * - * example: - * Lets assume we do have a pipeline: - * - * LL1 -> DP1 -> LL2 -> DP2 -> LL3 -> DP3 -> LL4 - * - * all LLs starts in 1ms tick - * - * for simplification lets assume - * - all LLs are on primary core, all DPs on secondary (100% CPU is for DP) - * - context switching requires 0 cycles - * - * DP1 - starts every 1ms, needs 0.5ms to finish processing - * DP2 - starts every 2ms, needs 0.6ms to finish processing - * DP3 - starts every 10ms, needs 0.3ms to finish processing - * - * TICK0 - * only LL1 is ready to run - * LL1 processing (producing data chunk for DP1) - * - * TICK1 - * LL1 is ready to run - * DP1 is ready tu run (has data from LL1) set deadline to TICK2 - * LL1 processing (producing second data chunk for DP1) - * DP1 processing for 0.5ms (consuming first data chunk, producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK2 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK3 - * LL2 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK3 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK4 - * LL2 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK4 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK5 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK6 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of second data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * !!!!!! Note here - DP1 must do before DP2 as it MUST finish in this tick. DP2 can wait - * >>>>>>> this is what we call EDF - EARIEST DEADLINE FIRST <<<<<< - * - * TICK5 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK6 - * LL2 is ready to run - * DP2 is in progress, deadline is set to TICK6 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of second data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.1ms (producing TWO data chunks for LL3) - * CPU is idle for 0.4ms (60% used) - * - * TICK6 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK7 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK8 - * LL3 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of second data chunk for DP2) - * LL3 processing (producing 10% of first data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * - * - * (........ 9 more cycles - LL3 procuces 100% of data for DP3......) - * - * - * TICK15 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK16 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK17 - * LL3 is ready to run - * DP3 is ready to run set deadline to TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of data chunk for DP2) - * LL3 processing (producing 10% of second data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - - * !!! note that DP3 is ready but has no chance to get CPU in this cycle - * - * TICK16 - * LL1 is ready to run set deadline to TICK17 - * DP1 is ready tu run - * LL2 is ready to run - * DP2 is in progress, deadline is set to TICK17 - * LL3 is ready to run - * DP3 is in progress, deadline is set to TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * LL3 processing (producing 10% of second data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.1ms (producing data) - * DP3 processing for 0.2ms (producing 10 data chunks for LL4) - * 90% CPU used - * - * TICK17 - * LL1 is ready to run - * DP1 is ready tu run - * LL2 is ready to run - * DP2 is ready to run - * LL3 is ready to run - * LL4 is ready to run - * !! NOTE that DP3 is not ready - it will be ready again in TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * LL3 processing (producing next 10% of second data chunk for DP3) - * LL4 processing (consuming 10% of data prepared by DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * - * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 - * needed 1.2ms for processing - but the example would be too complicated) - */ - -void z_impl_scheduler_dp_ll_tick(void) -{ - unsigned int lock_key; -#if CONFIG_SOF_USERSPACE_LL - struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); -#else - struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); -#endif - - if (!dp_sch) - return; - - /* remember current timestamp as "NOW" */ - dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); - - lock_key = scheduler_dp_lock(cpu_get_id()); - scheduler_dp_recalculate(dp_sch); - scheduler_dp_unlock(lock_key); -} - -#ifdef CONFIG_USERSPACE -#include -#include -void z_vrfy_scheduler_dp_ll_tick(void) -{ - SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_LL); - - z_impl_scheduler_dp_ll_tick(); -} -#include -#endif - -void z_impl_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) -{ - if (deadline_clock_ticks && - (int32_t)(k_cycle_get_32() - deadline_clock_ticks) > 0) - telemetry_notify_dp_deadline_miss(core); -} - -#ifdef CONFIG_USERSPACE -void z_vrfy_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) -{ - K_OOPS(K_SYSCALL_VERIFY(core < CONFIG_MP_MAX_NUM_CPUS)); - z_impl_scheduler_dp_check_deadline(core, deadline_clock_ticks); -} -#include -#endif - #if CONFIG_SOF_USERSPACE_APPLICATION -static int scheduler_dp_task_cancel(void *data, struct task *task) +int scheduler_dp_task_cancel(void *data, struct task *task) { /* Should never be called */ k_panic(); @@ -285,7 +62,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) } #endif -static int scheduler_dp_task_stop(void *data, struct task *task) +int scheduler_dp_task_stop(void *data, struct task *task) { unsigned int lock_key; struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; @@ -313,7 +90,7 @@ static int scheduler_dp_task_stop(void *data, struct task *task) return 0; } -static int scheduler_dp_task_free(void *data, struct task *task) +int scheduler_dp_task_free(void *data, struct task *task) { struct task_dp_pdata *pdata = task->priv_data; int ret; @@ -337,8 +114,8 @@ static int scheduler_dp_task_free(void *data, struct task *task) return ret; } -static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, - uint64_t period) +int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, + uint64_t period) { struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; struct task_dp_pdata *pdata = task->priv_data; @@ -374,39 +151,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return 0; } -APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { - .schedule_task = scheduler_dp_task_shedule, -#if CONFIG_SOF_USERSPACE_APPLICATION - .schedule_task_cancel = scheduler_dp_task_cancel, -#else - .schedule_task_cancel = scheduler_dp_task_stop, -#endif - .schedule_task_free = scheduler_dp_task_free, -}; - -/* Runs on each core */ -__cold int scheduler_dp_init(void) -{ - assert_can_be_cold(); - - struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); - if (!dp_sch) - return -ENOMEM; - - memset(dp_sch, 0, sizeof(*dp_sch)); - list_init(&dp_sch->tasks); - - scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); - - /* init src of DP tick */ - return schedule_task_init_ll(&dp_sch->ll_tick_src, - SOF_UUID(dp_sched_uuid), - SOF_SCHEDULE_LL_TIMER, - 0, scheduler_dp_ll_tick_dummy, dp_sch, - cpu_get_id(), 0); -} - void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { unsigned int lock_key; diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 523a1e0886f2..0d7b8a522cee 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -30,57 +30,14 @@ #include #include -#include "zephyr_dp_schedule.h" +#include LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; -static struct objpool_head dp_mdom_head = {.list = LIST_INIT(dp_mdom_head.list)}; - -/* Synchronization semaphore for the scheduler thread to wait for DP startup */ -#define DP_SYNC_INIT(i, _) Z_SEM_INITIALIZER(dp_sync[i], 0, 1) -#define DP_SYNC_INIT_LIST LISTIFY(CONFIG_CORE_COUNT, DP_SYNC_INIT, (,)) -static STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_sync, CONFIG_CORE_COUNT) = { DP_SYNC_INIT_LIST }; - -struct ipc4_flat { - unsigned int cmd; - int ret; - union { - struct { - struct ipc4_module_bind_unbind bu; - enum bind_type type; - } bind; - struct { - unsigned int trigger_cmd; - enum ipc4_pipeline_state state; - int n_sources; - int n_sinks; - struct sof_source *source[CONFIG_MODULE_MAX_CONNECTIONS]; - struct sof_sink *sink[CONFIG_MODULE_MAX_CONNECTIONS]; - } pipeline_state; - /* - * SOF_IPC4_MOD_INIT_INSTANCE: a by-value copy of the caller's - * ext_data. The original lives on the calling thread's stack, - * which the DP thread's memory domain doesn't have access to. - * The pointers it contains reference the IPC mailbox, which stays - * valid and is covered by the DP thread's SOF_DP_PART_CFG partition. - */ - struct module_ext_init_data init_instance; - struct { - uint32_t param_id; - enum module_cfg_fragment_position position; - uint32_t data_offset_size; - size_t fragment_size; - const char *data; - } set_config; - struct { - uint32_t param_id; - uint32_t data_offset_size; - size_t fragment_size; - char *data; - } get_config; - }; -}; +/* The DP thread creation and the scheduler_dp_internal_free() system call, which run + * in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule_application-kernel.c + */ /* Pack IPC input data */ static int ipc_thread_flatten(unsigned int cmd, const union scheduler_dp_thread_ipc_param *param, @@ -143,104 +100,6 @@ static int ipc_thread_flatten(unsigned int cmd, const union scheduler_dp_thread_ return 0; } -/* memory allocation helper structure */ -struct scheduler_dp_task_memory { - struct task task; - struct task_dp_pdata pdata; - struct comp_driver drv; - struct ipc4_flat flat; -}; - -/* - * Kernel-only record of the kernel objects owned by a DP task. - * - * struct scheduler_dp_task_memory lives in the module's heap, which is mapped - * into the DP thread's memory domain, and is also writable by the LL and IPC - * user threads. Freeing must therefore never use the pointers stored there: - * scheduler_dp_internal_free() takes them from this record instead. The task - * pointer is only used as a lookup key. - */ -struct scheduler_dp_task_kobj { - struct list_item list; - struct task *task; - struct processing_module *mod; - struct k_event *event; - struct k_thread *thread; - struct k_mem_domain *mdom; - struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; - uint32_t comp_id; - uint32_t gen; /* unique per record, see dp_task_kobj_take() */ -}; - -static struct list_item dp_task_kobj_list = LIST_INIT(dp_task_kobj_list); -static struct k_spinlock dp_task_kobj_lock; -/* generation of the last added record, under dp_task_kobj_lock */ -static uint32_t dp_task_kobj_gen; - -static void dp_task_kobj_add(struct scheduler_dp_task_kobj *kobj) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - - kobj->gen = ++dp_task_kobj_gen; - list_item_append(&kobj->list, &dp_task_kobj_list); - k_spin_unlock(&dp_task_kobj_lock, key); -} - -/* Find the record for @task, caller holds dp_task_kobj_lock */ -static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task) -{ - struct scheduler_dp_task_kobj *kobj; - struct list_item *item; - - list_for_item(item, &dp_task_kobj_list) { - kobj = container_of(item, struct scheduler_dp_task_kobj, list); - if (kobj->task == task) - return kobj; - } - - return NULL; -} - -/* Copy the record for @task to @copy without unlinking it */ -static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); - - if (kobj) - *copy = *kobj; - - k_spin_unlock(&dp_task_kobj_lock, key); - - return kobj != NULL; -} - -/* - * Find the record for @task and unlink it, so it can only be freed once. - * If @expected is given, only take the record it was copied from, so a - * record replaced since @expected was peeked is left alone. - */ -static struct scheduler_dp_task_kobj * -dp_task_kobj_take(const struct task *task, const struct scheduler_dp_task_kobj *expected) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); - - /* - * Compare the generation, not the object addresses: the task could - * have been freed and recreated at the same addresses meanwhile. - */ - if (kobj && expected && kobj->gen != expected->gen) - kobj = NULL; - - if (kobj) - list_item_del(&kobj->list); - - k_spin_unlock(&dp_task_kobj_lock, key); - - return kobj; -} - /* Unpack IPC data and execute a callback */ static void ipc_thread_unflatten_run(struct processing_module *pmod, struct ipc4_flat *flat) { @@ -547,401 +406,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3) if (task->state == SOF_TASK_STATE_COMPLETED) task_complete(task); } - -/* - * Safe to call with partial successful initialisation, - * k_mem_domain_remove_partition() then just returns -ENOENT - */ -static void scheduler_dp_domain_free(struct processing_module *pmod, struct k_mem_domain *mdom, - struct k_mem_partition *mpart, uint32_t comp_id) -{ - llext_manager_rm_domain(comp_id, mdom); - - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP_CACHE); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG_CACHE); - - if (mdom->num_partitions) { - /* Violation: all partitions should be freed by now */ - unsigned int n = mdom->num_partitions; - - LOG_WRN("%u domain partition(s) not freed, force-freeing them all", n); - - for (unsigned int i = 0; i < arch_mem_domain_max_partitions_get() && n; i++) { - struct k_mem_partition *dpart = &mdom->partitions[i]; - - if (dpart->size) { - k_mem_domain_remove_partition(mdom, dpart); - n--; - } - } - } - - /* All partitions removed, the domain can be freed now */ - pmod->mdom = NULL; - k_mem_domain_deinit(mdom); - objpool_free(&dp_mdom_head, mdom); -} - -/* - * Free the kernel objects of @task, using only the kernel-only record. - * - * @return false if @task has no record (unknown task or already freed) - */ -static bool scheduler_dp_internal_free_kobj(struct task *task, - const struct scheduler_dp_task_kobj *expected) -{ - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_take(task, expected); - - if (!kobj) - return false; - - /* normally already terminated by scheduler_dp_task_free(), no-op then */ - k_thread_abort(kobj->thread); - k_object_free(kobj->event); - k_object_free(kobj->thread); - scheduler_dp_domain_free(kobj->mod, kobj->mdom, kobj->mpart, kobj->comp_id); - - mod_free(kobj->mod, container_of(task, struct scheduler_dp_task_memory, task)); - rfree(kobj); - - return true; -} - -void z_impl_scheduler_dp_internal_free(struct task *task) -{ - if (!scheduler_dp_internal_free_kobj(task, NULL)) - tr_err(&dp_tr, "unknown DP task %p", task); -} - -#ifdef CONFIG_THREAD_NAME -static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_module *mod) -{ - char name[CONFIG_THREAD_MAX_NAME_LEN]; - - snprintf(name, sizeof(name), "DP:%#x", mod->dev->ipc_config.id); - - k_thread_name_set(thread_id, name); -} -#else -/* k_thread_name_set() is a no-op so skip constructing a thread name */ -#define scheduler_dp_thread_name_set(x, y) -#endif - -/* Called only in IPC context in kernel mode (this can change) */ -int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, - const struct task_ops *ops, struct processing_module *mod, - uint16_t core, size_t stack_size, uint32_t options) -{ - k_thread_stack_t *p_stack; - struct scheduler_dp_task_memory *task_memory; - uint8_t *ipc_config_data = NULL; - k_tid_t thread_id; - uint32_t comp_id; - - int ret; - - /* must be called on the same core the task will be bound to */ - assert(cpu_get_id() == core); - assert(!k_is_user_context()); - - /* - * allocate memory - * to avoid multiple malloc operations allocate all required memory as a single structure - * and return pointer to task_memory->task - * As the structure contains zephyr kernel specific data, it must be located in - * shared, non cached memory - */ - task_memory = mod_alloc_ext(mod, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory), 0); - if (!task_memory) { - tr_err(&dp_tr, "memory alloc failed"); - return -ENOMEM; - } - - memset(task_memory, 0, sizeof(*task_memory)); - - const struct comp_driver *drv = mod->dev->drv; - - task_memory->drv = *mod->dev->drv; - mod->dev->drv = &task_memory->drv; - - /* allocate stack - must be aligned and cached so a separate alloc */ - p_stack = user_stack_allocate(stack_size, options); - if (!p_stack) { - tr_err(&dp_tr, "stack alloc failed"); - ret = -ENOMEM; - goto e_tmem; - } - - struct task *ptask = &task_memory->task; - - /* internal SOF task init */ - ret = schedule_task_init(ptask, uid, SOF_SCHEDULE_DP, 0, ops->run, mod, core, options); - if (ret < 0) { - tr_err(&dp_tr, "schedule_task_init failed"); - goto e_stack; - } - - /* - * task_memory, including pdata, is in the module heap, which user - * threads can write while this function runs. Keep everything the - * kernel uses here or at teardown in kobj and locals, and only mirror - * it into pdata for the DP thread. - */ - struct task_dp_pdata *pdata = &task_memory->pdata; - struct scheduler_dp_task_kobj *kobj = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*kobj)); - - if (!kobj) { - tr_err(&dp_tr, "task record alloc failed"); - ret = -ENOMEM; - goto e_stack; - } - - kobj->task = ptask; - kobj->mod = mod; - comp_id = mod->dev->ipc_config.id; - kobj->comp_id = comp_id; - - if (mod->dev->drv->adapter_ops->get_configuration) { - ipc_config_data = mod_alloc_ext(mod, SOF_MEM_FLAG_USER, SOF_IPC_MSG_MAX_SIZE, 0); - if (!ipc_config_data) { - tr_err(&dp_tr, "Failed to allocate memory for get-config"); - ret = -ENOMEM; - goto e_kobj; - } - } - pdata->ipc_config_data = ipc_config_data; - - pdata->flat = &task_memory->flat; - - kobj->event = k_object_alloc(K_OBJ_EVENT); - if (!kobj->event) { - tr_err(&dp_tr, "Event object allocation failed"); - ret = -ENOMEM; - goto e_config; - } - - pdata->event = kobj->event; - - kobj->thread = k_object_alloc(K_OBJ_THREAD); - if (!kobj->thread) { - tr_err(&dp_tr, "Thread object allocation failed"); - ret = -ENOMEM; - goto e_event; - } - memset(&kobj->thread->arch, 0, sizeof(kobj->thread->arch)); - pdata->thread = kobj->thread; - - /* success, fill the structures */ - pdata->p_stack = p_stack; - pdata->mod = mod; - - /* initialize other task structures */ - ptask->ops.complete = ops->complete; - ptask->ops.get_deadline = ops->get_deadline; - ptask->priv_data = pdata; - list_init(&ptask->list); - - /* create a zephyr thread for the task */ - thread_id = k_thread_create(kobj->thread, p_stack, - stack_size, dp_thread_fn, ptask, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, ptask->flags, K_FOREVER); - pdata->thread_id = thread_id; - scheduler_dp_thread_name_set(thread_id, mod); - - unsigned int pidx; - size_t size; - uintptr_t start; - bool on_pool = false; - -#ifdef CONFIG_SCHED_CPU_MASK - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(thread_id, core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto e_thread; - } -#endif - - k_thread_access_grant(thread_id, kobj->event, &dp_sync[core]); - scheduler_dp_grant(thread_id, core); -#if CONFIG_SOF_USERSPACE_LL - struct k_thread *thread_ipc = ipc_thread_user(core); - - k_thread_access_grant(thread_ipc, kobj->event, thread_id, p_stack, &dp_sync[core]); - scheduler_dp_grant(thread_ipc, core); - scheduler_dp_grant(zephyr_ll_domain_thread(), core); -#endif - - struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), - SOF_MEM_FLAG_COHERENT); - - if (!mdom) { - tr_err(&dp_tr, "objpool allocation failed"); - ret = -ENOMEM; - goto e_thread; - } - - on_pool = true; - - mod->mdom = mdom; - kobj->mdom = mdom; - - if (!mdom->arch.ptables) { - ret = k_mem_domain_init(mdom, 0, NULL); - if (ret < 0) - goto e_thread; - } - - /* Module heap partition */ - mod_heap_info(mod, &size, &start); - kobj->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ - .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)start), - .size = size, - .attr = K_MEM_PARTITION_P_RW_U_RW, - }; - kobj->mpart[SOF_DP_PART_HEAP_CACHE] = (struct k_mem_partition){ - .start = start, - .size = size, - .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, - }; - /* Host mailbox partition for additional IPC parameters: read-only */ - kobj->mpart[SOF_DP_PART_CFG] = (struct k_mem_partition){ - .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)MAILBOX_HOSTBOX_BASE), - .size = MAILBOX_HOSTBOX_SIZE, - .attr = K_MEM_PARTITION_P_RO_U_RO, - }; - kobj->mpart[SOF_DP_PART_CFG_CACHE] = (struct k_mem_partition){ - .start = (uintptr_t)MAILBOX_HOSTBOX_BASE, - .size = MAILBOX_HOSTBOX_SIZE, - .attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB, - }; - - for (pidx = 0; pidx < SOF_DP_PART_TYPE_COUNT; pidx++) { - ret = k_mem_domain_add_partition(mdom, kobj->mpart + pidx); - if (ret < 0) - goto e_dom; - } - - memcpy(pdata->mpart, kobj->mpart, sizeof(pdata->mpart)); - - ret = llext_manager_add_domain(comp_id, mdom); - if (ret < 0) { - tr_err(&dp_tr, "failed to add LLEXT to domain %d", ret); - goto e_dom; - } - - /* - * Keep this call last, able to fail, otherwise domain will be removed - * before its thread - */ - ret = k_mem_domain_add_thread(mdom, thread_id); - if (ret < 0) { - tr_err(&dp_tr, "failed to add thread to domain %d", ret); - goto e_dom; - } - - /* publish the record before the thread can run */ - dp_task_kobj_add(kobj); - - /* start the thread, it should immediately stop at the semaphore */ - k_event_init(kobj->event); - k_thread_start(thread_id); - - /* ptask points into task_memory, so only publish it once it cannot be freed */ - *task = ptask; - - return 0; - -e_dom: - scheduler_dp_domain_free(mod, mdom, kobj->mpart, comp_id); - on_pool = false; -e_thread: - if (on_pool) - objpool_free(&dp_mdom_head, mdom); - k_thread_abort(thread_id); - k_object_free(kobj->thread); -e_event: - k_object_free(kobj->event); -e_config: - mod_free(mod, ipc_config_data); -e_kobj: - rfree(kobj); -e_stack: - user_stack_free(p_stack); -e_tmem: - /* the copy lives in task_memory, so stop pointing at it before freeing */ - mod->dev->drv = drv; - mod_free(mod, task_memory); - return ret; -} - -#ifdef CONFIG_USERSPACE -#include -#include - -/* - * mod_free() follows the module's resource and allocator pointers in - * supervisor mode. They are user-writable, so at least check that they - * point to memory the caller can access. - */ -static void scheduler_dp_mod_vrfy(struct processing_module *mod) -{ - K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); - K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); - - struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc, sizeof(*alloc))); - if (alloc->heap) { - size_t h_size = 0; - uintptr_t h_start; - - mod_heap_info(mod, &h_size, &h_start); - if (h_size) - K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); - } - if (alloc->vreg) { - size_t vr_size = 0; - uintptr_t vr_start; - - vregion_mem_info(alloc->vreg, &vr_size, &vr_start); - K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); - } -} - -void z_vrfy_scheduler_dp_internal_free(struct task *task) -{ - struct scheduler_dp_task_kobj kobj; - - SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); - - /* - * Do not read task->priv_data: the kernel objects come from the - * kernel-only record, and an unknown task pointer is rejected. - */ - K_OOPS(!dp_task_kobj_peek(task, &kobj)); - - /* - * A DP thread holds its own thread and event objects, so it would pass - * the checks below for its own task. Freeing would then abort the - * caller after the record is unlinked, leaking everything it holds. - */ - K_OOPS(kobj.thread == k_current_get()); - - /* - * The caller must have been granted the task's objects. Check before - * unlinking the record, so an unauthorized call leaves it intact. - */ - K_OOPS(K_SYSCALL_OBJ(kobj.event, K_OBJ_EVENT)); - K_OOPS(K_SYSCALL_OBJ_INIT(kobj.thread, K_OBJ_THREAD)); - scheduler_dp_mod_vrfy(kobj.mod); - - K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj)); -} -#include -#endif /* CONFIG_USERSPACE */ diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index 8e33ee1ab2ce..db2311c3d90a 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -25,11 +25,15 @@ #include #include -#include "zephyr_dp_schedule.h" +#include LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; +/* The DP thread creation and the scheduler_dp_internal_free() system call, which run + * in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c + */ + /* Go through all DP tasks and recalculate their readiness and deadlines * NOT REENTRANT, should be called with scheduler_dp_lock() */ @@ -214,176 +218,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3) if (task->state == SOF_TASK_STATE_COMPLETED) task_complete(task); } - -int scheduler_dp_task_init(struct task **task, - const struct sof_uuid_entry *uid, - const struct task_ops *ops, - struct processing_module *mod, - uint16_t core, - size_t stack_size, - uint32_t options) -{ - void __sparse_cache *p_stack = NULL; - struct k_heap *const user_heap = mod->dev->drv->user_heap; - - /* memory allocation helper structure */ - struct { - struct task task; /* keep first, used for freeing below */ - struct task_dp_pdata pdata; - } *task_memory; - - int ret; - - /* must be called on the same core the task will be binded to */ - assert(cpu_get_id() == core); - - /* - * allocate memory - * to avoid multiple malloc operations allocate all required memory as a single structure - * and return pointer to task_memory->task - * As the structure contains zephyr kernel specific data, it must be located in - * shared, non cached memory - */ - task_memory = sof_heap_alloc(user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory), 0); - if (!task_memory) { - tr_err(&dp_tr, "memory alloc failed"); - return -ENOMEM; - } - - memset(task_memory, 0, sizeof(*task_memory)); - /* allocate stack - must be aligned and cached so a separate alloc */ - p_stack = user_stack_allocate(stack_size, options); - if (!p_stack) { - tr_err(&dp_tr, "stack alloc failed"); - ret = -ENOMEM; - goto err; - } - - /* internal SOF task init */ - ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, - mod, core, options); - if (ret < 0) { - tr_err(&dp_tr, "schedule_task_init failed"); - goto err; - } - - struct task_dp_pdata *pdata = &task_memory->pdata; - - /* Point to event_struct event for kernel threads synchronization */ - /* It will be overwritten for K_USER threads to dynamic ones. */ - pdata->event = &pdata->event_struct; - pdata->thread = &pdata->thread_struct; - -#ifdef CONFIG_USERSPACE - if (options & K_USER) { - pdata->event = k_object_alloc(K_OBJ_EVENT); - if (!pdata->event) { - tr_err(&dp_tr, "Event object allocation failed"); - ret = -ENOMEM; - goto err; - } - - pdata->thread = k_object_alloc(K_OBJ_THREAD); - if (!pdata->thread) { - tr_err(&dp_tr, "Thread object allocation failed"); - ret = -ENOMEM; - goto err; - } - } -#endif /* CONFIG_USERSPACE */ - - /* initialize other task structures */ - task_memory->task.ops.complete = ops->complete; - task_memory->task.ops.get_deadline = ops->get_deadline; - task_memory->task.state = SOF_TASK_STATE_INIT; - task_memory->task.core = core; - task_memory->task.priv_data = pdata; - k_event_init(pdata->event); - - /* success, fill the structures */ - pdata->p_stack = p_stack; - pdata->mod = mod; - - /* create a zephyr thread for the task */ - pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)p_stack, - stack_size, dp_thread_fn, &task_memory->task, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, task_memory->task.flags, - K_FOREVER); - -#ifdef CONFIG_SCHED_CPU_MASK - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(pdata->thread_id, core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto e_thread; - } -#endif - -#ifdef CONFIG_USERSPACE - if (options & K_USER) { - k_thread_access_grant(pdata->thread_id, pdata->event); - scheduler_dp_grant(pdata->thread_id, core); - - ret = k_mem_domain_add_thread(pdata->mod->user_ctx->comp_dom, pdata->thread_id); - if (ret < 0) { - tr_err(&dp_tr, "k_mem_domain_add_thread() failed %d", ret); - goto e_thread; - } - -#if IS_ENABLED(CONFIG_SOF_USERSPACE_MOD_IPC_BY_DP_THREAD) - pdata->ipc_work_item = userspace_proxy_register_ipc_handler(mod, pdata->event); - assert(pdata->ipc_work_item); -#endif - } -#endif /* CONFIG_USERSPACE */ - - /* start the thread, it should immediately stop at an event */ - k_thread_start(pdata->thread_id); - - /* success, fill output parameter */ - *task = &task_memory->task; - return 0; - -e_thread: - k_thread_abort(pdata->thread_id); -err: - /* cleanup - free all allocated resources */ - if (user_stack_free((__sparse_force void *)p_stack)) - tr_err(&dp_tr, "user_stack_free failed!"); - - /* k_object_free looks for a pointer in the list, any invalid value can be passed */ - k_object_free(task_memory->pdata.event); - k_object_free(task_memory->pdata.thread); - sof_heap_free(user_heap, task_memory); - return ret; -} - -void z_impl_scheduler_dp_internal_free(struct task *task) -{ - struct task_dp_pdata *pdata = task->priv_data; - -#ifdef CONFIG_USERSPACE - if (pdata->event != &pdata->event_struct) - k_object_free(pdata->event); - if (pdata->thread != &pdata->thread_struct) - k_object_free(pdata->thread); -#endif - - /* task is the first member in task_memory above */ - sof_heap_free(pdata->mod->dev->drv->user_heap, task); -} - -#ifdef CONFIG_USERSPACE -#include - -void z_vrfy_scheduler_dp_internal_free(struct task *task) -{ - /* - * With the thread DP scheduler variant scheduler_dp_internal_free() is - * never called from the userspace context - */ - K_OOPS(true); -} -#include -#endif diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index e04a05edd742..7daa415b8fd1 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -26,27 +27,9 @@ SOF_DEFINE_REG_UUID(zll_sched); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(ll_tr, SOF_UUID(zll_sched_uuid), LOG_LEVEL_INFO); -/* per-scheduler data */ -struct zephyr_ll { - struct list_item tasks; /* list of ll tasks */ - unsigned int n_tasks; /* task counter */ - struct ll_schedule_domain *ll_domain; /* scheduling domain */ - unsigned int core; /* core ID of this instance */ -#if CONFIG_SOF_USERSPACE_LL - struct k_mutex *lock; /* mutex for userspace */ -#endif - struct k_heap *heap; -}; - -/* per-task scheduler data */ -struct zephyr_ll_pdata { - bool run; - bool freeing; - struct k_sem *sem_p; -#if !CONFIG_DYNAMIC_OBJECTS - struct k_sem sem; -#endif -}; +/* The scheduler syscalls and access grants, which run in kernel mode, are in + * src/kernel/schedule/zephyr_ll-kernel.c + */ #if CONFIG_SOF_USERSPACE_LL /* @@ -54,14 +37,7 @@ struct zephyr_ll_pdata { * can read the pointer for syscalls. The actual lock object resides * in kernel memory, there are just handles! */ -static APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; - -/* - * Kernel-only copy of zephyr_ll_locks[]. That array is writable by user - * threads, so kernel code granting access to the locks must use this copy, - * otherwise a user thread could make it grant access to any kernel object. - */ -static struct k_mutex *zephyr_ll_klocks[CONFIG_CORE_COUNT]; +APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; /* * Shadow ownership tracking for the per-core LL mutex, used by @@ -118,12 +94,6 @@ static void zephyr_ll_unlock(struct zephyr_ll *sch, uint32_t *flags) #endif } -static void zephyr_ll_assert_core(const struct zephyr_ll *sch) -{ - assert(CONFIG_CORE_COUNT == 1 || IS_ENABLED(CONFIG_SOF_USERSPACE_LL) || - sch->core == cpu_get_id()); -} - /* Locking: caller should hold the domain lock */ static void zephyr_ll_task_done(struct zephyr_ll *sch, struct task *task) @@ -433,183 +403,35 @@ static int zephyr_ll_task_schedule_common(struct zephyr_ll *sch, struct task *ta return 0; } -static int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, - uint64_t period) +int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, + uint64_t period) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, NULL, false); } -static int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, - uint64_t period, struct task *before) +int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *before) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, before, true); } -static int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, - uint64_t period, struct task *after) +int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *after) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, after, false); } -#if CONFIG_DYNAMIC_OBJECTS -static struct list_item zephyr_ll_task_sem_list = LIST_INIT(zephyr_ll_task_sem_list); - -struct zephyr_ll_task_sem { - struct task *task; - struct k_sem *sem; - struct list_item list; -}; - -/* - * The syscall validators pass in pdata that they read once from user memory - * and validated, so these helpers must not read task->priv_data again. - */ -static int zephyr_ll_task_sem_alloc_pdata(struct task *task, struct zephyr_ll_pdata *pdata) -{ - struct zephyr_ll_task_sem *ts = rmalloc(SOF_MEM_FLAG_COHERENT, sizeof(*ts)); - - if (!ts) - return -ENOMEM; - - ts->sem = k_object_alloc(K_OBJ_SEM); - if (!ts->sem) { - rfree(ts); - return -ENOMEM; - } - - k_sem_init(ts->sem, 0, 1); - -#if CONFIG_SOF_USERSPACE_LL - /* - * The per-task semaphore is signalled from zephyr_ll_task_done(), - * which runs in the (unprivileged) LL scheduler thread when a task is - * freed while it is still running. k_object_alloc() only grants access - * to the calling thread (the IPC handler that creates the task), so the - * LL thread must be granted access explicitly, otherwise its - * k_sem_give() traps with a userspace permission fault. - * - * Resolve the LL thread from kernel-only per-core state keyed by the - * task's target core; never traverse the user-accessible scheduler or - * domain objects from privileged context. - */ - struct k_thread *ll_tid = zephyr_domain_thread_tid_for_core(task->core); - - if (ll_tid) - k_thread_access_grant(ll_tid, ts->sem); -#endif /* CONFIG_SOF_USERSPACE_LL */ - - ts->task = task; - pdata->sem_p = ts->sem; - /* List is protected by IPC serialization */ - list_item_append(&ts->list, &zephyr_ll_task_sem_list); - - return 0; -} - -int z_impl_zephyr_ll_task_sem_alloc(struct task *task) -{ - return zephyr_ll_task_sem_alloc_pdata(task, task->priv_data); -} - -static int zephyr_ll_task_sem_free_pdata(struct task *task, struct zephyr_ll_pdata *pdata) -{ - struct list_item *list; - struct zephyr_ll_task_sem *ts; - bool found = false; - - /* List is protected by IPC serialization */ - list_for_item(list, &zephyr_ll_task_sem_list) { - ts = container_of(list, struct zephyr_ll_task_sem, list); - if (ts->task == task) { - found = true; - break; - } - } - - if (!found) - return -ENOENT; - - if (pdata->sem_p != ts->sem) - return -EINVAL; - - list_item_del(list); - k_object_free(ts->sem); - rfree(ts); - - return 0; -} - -int z_impl_zephyr_ll_task_sem_free(struct task *task) -{ - return zephyr_ll_task_sem_free_pdata(task, task->priv_data); -} - -#ifdef CONFIG_USERSPACE -#include -#include -/* - * Read task->priv_data exactly once. The task lives in user memory and a - * thread on another core can change the pointer after it has been checked. - * - * @return validated pdata pointer, or NULL if the task has none - */ -static struct zephyr_ll_pdata *zephyr_ll_task_pdata_vrfy(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); - K_OOPS(k_usermode_from_copy(&pdata, &task->priv_data, sizeof(pdata))); - if (pdata) - K_OOPS(K_SYSCALL_MEMORY_WRITE(pdata, sizeof(*pdata))); - - return pdata; -} - -static inline int z_vrfy_zephyr_ll_task_sem_alloc(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); - - if (!task) - return -EINVAL; - pdata = zephyr_ll_task_pdata_vrfy(task); - if (!pdata) - return -EINVAL; - - return zephyr_ll_task_sem_alloc_pdata(task, pdata); -} -#include - -static inline int z_vrfy_zephyr_ll_task_sem_free(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_INFRA); - - if (!task) - return -EINVAL; - pdata = zephyr_ll_task_pdata_vrfy(task); - if (!pdata) - return -EINVAL; - - return zephyr_ll_task_sem_free_pdata(task, pdata); -} -#include -#endif /* CONFIG_USERSPACE */ -#endif /* CONFIG_DYNAMIC_OBJECTS */ - /* * This is synchronous - after this returns the object can be destroyed! * Assertion: under Zephyr this is always called from a thread context! */ -static int zephyr_ll_task_free(void *data, struct task *task) +int zephyr_ll_task_free(void *data, struct task *task) { struct zephyr_ll *sch = data; uint32_t flags; @@ -677,7 +499,7 @@ static int zephyr_ll_task_free(void *data, struct task *task) } /* This is asynchronous */ -static int zephyr_ll_task_cancel(void *data, struct task *task) +int zephyr_ll_task_cancel(void *data, struct task *task) { struct zephyr_ll *sch = data; uint32_t flags; @@ -706,74 +528,6 @@ static int zephyr_ll_task_cancel(void *data, struct task *task) return 0; } -#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY -static void zephyr_ll_scheduler_free(void *data, uint32_t flags) -{ - struct zephyr_ll *sch = data; - - zephyr_ll_assert_core(sch); - -#if CONFIG_SOF_USERSPACE_LL - zephyr_ll_locks[sch->core] = NULL; - zephyr_ll_klocks[sch->core] = NULL; - k_object_free(sch->lock); -#endif - - if (sch->n_tasks) - tr_err(&ll_tr, "%u tasks are still active!", - sch->n_tasks); - -#if CONFIG_SOF_USERSPACE_LL - domain_thread_free(sch->ll_domain, sch->n_tasks); -#endif - sof_heap_free(sch->heap, sch); -} -#endif /* CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY */ - -#if CONFIG_SOF_USERSPACE_LL -struct k_thread *zephyr_ll_init_context(void *data, struct task *task) -{ - struct zephyr_ll *sch = data; - struct zephyr_ll_pdata *pdata = task->priv_data; - int ret; - - /* - * Use domain_thread_init() for privileged setup (thread creation, - * timer, access grants). domain_register() is now bookkeeping only - * and will be called later from user context when scheduling tasks. - */ - ret = domain_thread_init(sch->ll_domain, task); - if (ret < 0) { - tr_err(&ll_tr, "cannot init_context %d", ret); - return NULL; - } - - assert(!k_is_user_context()); - k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock, pdata->sem_p); - - tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, - zephyr_domain_thread_tid(sch->ll_domain)); - tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, - zephyr_domain_thread_tid(sch->ll_domain)); - - return zephyr_domain_thread_tid(sch->ll_domain); -} -#endif /* CONFIG_SOF_USERSPACE_LL */ - -static const struct scheduler_ops zephyr_ll_ops = { - .schedule_task = zephyr_ll_task_schedule, - .schedule_task_before = zephyr_ll_task_schedule_before, - .schedule_task_after = zephyr_ll_task_schedule_after, - .schedule_task_free = zephyr_ll_task_free, - .schedule_task_cancel = zephyr_ll_task_cancel, -#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY - .scheduler_free = zephyr_ll_scheduler_free, -#endif -#if CONFIG_SOF_USERSPACE_LL - .scheduler_init_context = zephyr_ll_init_context, -#endif -}; - #if CONFIG_SOF_USERSPACE_LL struct task *zephyr_ll_task_alloc(void) { @@ -787,13 +541,6 @@ struct task *zephyr_ll_task_alloc(void) return task; } -void user_ll_grant_access(struct k_thread *thread, int core) -{ - assert(core < CONFIG_CORE_COUNT && zephyr_ll_klocks[core] != NULL); - - k_thread_access_grant(thread, zephyr_ll_klocks[core]); -} - /** * Lock the LL scheduler to prevent it from processing tasks. * @@ -901,59 +648,6 @@ int zephyr_ll_task_init(struct task *task, } EXPORT_SYMBOL(zephyr_ll_task_init); -/* TODO: low-power mode clock support */ -/* Runs on each core during initialisation with the same domain argument */ -__cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) -{ - struct zephyr_ll *sch; - int core = cpu_get_id(); - struct k_heap *heap = sof_sys_heap_get(); - int flags = SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT; - - assert_can_be_cold(); - -#if CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); - flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; -#endif - tr_dbg(&ll_tr, "init on core %d", core); - - /* initialize per-core scheduler private data */ - sch = sof_heap_alloc(heap, flags, sizeof(*sch), 0); - if (!sch) { - tr_err(&ll_tr, "allocation failed"); - return -ENOMEM; - } - - memset(sch, 0, sizeof(*sch)); - - list_init(&sch->tasks); - sch->ll_domain = domain; - sch->core = core; - sch->n_tasks = 0; - sch->heap = heap; - -#if CONFIG_SOF_USERSPACE_LL - /* Allocate mutex dynamically for userspace access */ - sch->lock = k_object_alloc(K_OBJ_MUTEX); - if (!sch->lock) { - tr_err(&ll_tr, "mutex allocation failed"); - sof_heap_free(sch->heap, sch); - return -ENOMEM; - } - assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); - zephyr_ll_locks[core] = sch->lock; - zephyr_ll_klocks[core] = sch->lock; - k_mutex_init(sch->lock); - - tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); -#endif - - scheduler_init(domain->type, &zephyr_ll_ops, sch); - - return 0; -} - void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t *data_off_size) {