Phase 3 (part 2): split the schedulers, kernel parts to src/kernel/schedule

src/include/sof/schedule/schedule.h

@@ -235,6 +235,21 @@ struct schedulers **arch_user_schedulers_get(void);
  */
 struct schedulers **arch_user_schedulers_get_for_core(int core);
 
+/**
+ * @brief Check whether a scheduler type is managed in user space.
+ * @param[in] type Scheduler type identifier (SOF_SCHEDULE_*).
+ * @return true for the LL timer and DP schedulers.
+ */
+static inline bool scheduler_is_user(int type)
+{
+	/*
+	 * currently only LL managed in user-space, but longterm
+	 * goal is to move all audio application level scheduling
+	 * to user-space and only keep Zephyr scheduler logic in
+	 * kernel
+	 */
+	return type == SOF_SCHEDULE_LL_TIMER || type == SOF_SCHEDULE_DP;
+}
 /**
  * @brief Lookup scheduler private context data by scheduler type from a schedulers container.
  * @param[in] schedulers Schedulers list container.

src/schedule/zephyr_dp_schedule.h → src/include/sof/schedule/zephyr_dp_schedule.h

@@ -6,16 +6,26 @@
  */
 
 /**
- * @file src/schedule/zephyr_dp_schedule.h
+ * @file include/sof/schedule/zephyr_dp_schedule.h
  * @brief Internal Data Processing (DP) scheduler structures, memory partitions, and thread helpers.
  * @ingroup scheduling_api
+ *
+ * Shared by the DP scheduler sources in src/schedule and their kernel parts in
+ * src/kernel/schedule. Not for use outside the scheduler.
  */
 
+#ifndef __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__
+#define __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__
+
 #include <rtos/task.h>
+#include <sof/audio/component.h>
 #include <sof/audio/module_adapter/module/generic.h>
 #include <sof/list.h>
 #include <sof/compiler_attributes.h>
 
+#include <ipc4/module.h>
+#include <ipc4/pipeline.h>
+
 #include <zephyr/app_memory/mem_domain.h>
 
 #include <stdbool.h>
@@ -66,6 +76,73 @@ struct task_dp_pdata {
 #endif
 };
 
+#if CONFIG_SOF_USERSPACE_APPLICATION
+/** @brief Per-core DP thread start and IPC completion semaphores. */
+extern struct k_sem dp_sync[CONFIG_CORE_COUNT];
+
+struct ipc4_flat {
+	unsigned int cmd;
+	int ret;
+	union {
+		struct {
+			struct ipc4_module_bind_unbind bu;
+			enum bind_type type;
+		} bind;
+		struct {
+			unsigned int trigger_cmd;
+			enum ipc4_pipeline_state state;
+			int n_sources;
+			int n_sinks;
+			struct sof_source *source[CONFIG_MODULE_MAX_CONNECTIONS];
+			struct sof_sink *sink[CONFIG_MODULE_MAX_CONNECTIONS];
+		} pipeline_state;
+		/*
+		 * SOF_IPC4_MOD_INIT_INSTANCE: a by-value copy of the caller's
+		 * ext_data. The original lives on the calling thread's stack,
+		 * which the DP thread's memory domain doesn't have access to.
+		 * The pointers it contains reference the IPC mailbox, which stays
+		 * valid and is covered by the DP thread's SOF_DP_PART_CFG partition.
+		 */
+		struct module_ext_init_data init_instance;
+		struct {
+			uint32_t param_id;
+			enum module_cfg_fragment_position position;
+			uint32_t data_offset_size;
+			size_t fragment_size;
+			const char *data;
+		} set_config;
+		struct {
+			uint32_t param_id;
+			uint32_t data_offset_size;
+			size_t fragment_size;
+			char *data;
+		} get_config;
+	};
+};
+
+/* memory allocation helper structure */
+struct scheduler_dp_task_memory {
+	struct task task;
+	struct task_dp_pdata pdata;
+	struct comp_driver drv;
+	struct ipc4_flat flat;
+};
+
+#endif /* CONFIG_SOF_USERSPACE_APPLICATION */
+/** @brief Per-core DP scheduler locks, see scheduler_dp_lock(). */
+extern struct k_sem dp_lock[CONFIG_MP_MAX_NUM_CPUS];
+
+/**
+ * @brief DP scheduler ops, see struct scheduler_ops.
+ */
+int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start,
+			      uint64_t period);
+int scheduler_dp_task_stop(void *data, struct task *task);
+int scheduler_dp_task_free(void *data, struct task *task);
+#if CONFIG_SOF_USERSPACE_APPLICATION
+int scheduler_dp_task_cancel(void *data, struct task *task);
+#endif
+
 /**
  * @brief Recalculate deadlines and evaluate readiness for all active DP tasks on an LL tick.
  * @param[in,out] dp_sch Pointer to per-core DP scheduler state container.
@@ -102,3 +179,5 @@ void scheduler_dp_unlock(unsigned int key);
  * @param[in] core Target DSP core ID.
  */
 void scheduler_dp_grant(k_tid_t thread_id, uint16_t core);
+
+#endif /* __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ */

src/include/sof/schedule/zephyr_ll.h

@@ -0,0 +1,109 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2019-2026 Intel Corporation.
+ */
+
+/*
+ * Private data of the Zephyr LL scheduler and its timer domain, shared by
+ * the parts in src/schedule, which also run in the LL scheduler threads, and
+ * the kernel parts in src/kernel/schedule. Not for use outside the scheduler.
+ */
+
+#ifndef __SOF_SCHEDULE_ZEPHYR_LL_H__
+#define __SOF_SCHEDULE_ZEPHYR_LL_H__
+
+#include <sof/lib/cpu.h>
+#include <sof/list.h>
+#include <sof/schedule/ll_schedule_domain.h>
+#include <rtos/task.h>
+#include <zephyr/kernel.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+/* per-scheduler data */
+struct zephyr_ll {
+	struct list_item tasks;			/* list of ll tasks */
+	unsigned int n_tasks;			/* task counter */
+	struct ll_schedule_domain *ll_domain;	/* scheduling domain */
+	unsigned int core;			/* core ID of this instance */
+#if CONFIG_SOF_USERSPACE_LL
+	struct k_mutex *lock;			/* mutex for userspace */
+#endif
+	struct k_heap *heap;
+};
+
+/* per-task scheduler data */
+struct zephyr_ll_pdata {
+	bool run;
+	bool freeing;
+	struct k_sem *sem_p;
+#if !CONFIG_DYNAMIC_OBJECTS
+	struct k_sem sem;
+#endif
+};
+
+static inline void zephyr_ll_assert_core(const struct zephyr_ll *sch)
+{
+	assert(CONFIG_CORE_COUNT == 1 || IS_ENABLED(CONFIG_SOF_USERSPACE_LL) ||
+	       sch->core == cpu_get_id());
+}
+
+/* LL scheduler ops, see zephyr_ll.c */
+int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start,
+			    uint64_t period);
+int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start,
+				   uint64_t period, struct task *before);
+int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start,
+				  uint64_t period, struct task *after);
+int zephyr_ll_task_free(void *data, struct task *task);
+int zephyr_ll_task_cancel(void *data, struct task *task);
+
+#if CONFIG_SOF_USERSPACE_LL
+/* per-core LL scheduler locks, see zephyr_ll.c */
+extern struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT];
+
+struct k_thread *zephyr_ll_init_context(void *data, struct task *task);
+#endif
+
+struct zephyr_domain_thread {
+	struct k_thread *ll_thread;
+	struct k_sem *sem;
+#ifndef CONFIG_SOF_USERSPACE_LL
+	struct k_thread ll_thread_obj;
+	struct k_sem sem_obj;
+#endif
+	void (*handler)(void *arg);
+	void *arg;
+};
+
+struct zephyr_domain {
+	struct k_timer *timer;
+#ifndef CONFIG_SOF_USERSPACE_LL
+	struct k_timer timer_obj;
+#endif
+	struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT];
+	struct ll_schedule_domain *ll_domain;
+#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL
+	atomic_t block;
+	struct k_mutex block_mutex;
+	struct k_condvar block_condvar;
+#endif
+};
+
+/* LL scheduler thread function, p1 is the domain and p2 the core */
+void zephyr_domain_thread_fn(void *p1, void *p2, void *p3);
+
+#ifdef CONFIG_SOF_USERSPACE_LL
+int zephyr_domain_register_user(struct ll_schedule_domain *domain,
+				struct task *task,
+				void (*handler)(void *arg), void *arg);
+int zephyr_domain_unregister_user(struct ll_schedule_domain *domain,
+				  struct task *task, uint32_t num_tasks);
+#endif
+
+#if CONFIG_CROSS_CORE_STREAM && !CONFIG_SOF_USERSPACE_LL
+void zephyr_domain_block(struct ll_schedule_domain *domain);
+void zephyr_domain_unblock(struct ll_schedule_domain *domain);
+#endif
+
+#endif /* __SOF_SCHEDULE_ZEPHYR_LL_H__ */

src/include/sof/userspace/README.md

@@ -124,7 +124,7 @@ syscall by subsystem:
 | CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` |
 | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_data_blob_set`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` |
 | Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` |
-| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `schedule/zephyr_ll.c`, `schedule/zephyr_dp_schedule*.c` |
+| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `kernel/schedule/zephyr_ll-kernel.c`, `kernel/schedule/zephyr_dp_schedule*-kernel.c` |
 | Debug | `debug_stream_slot_send_record` | `user/debug_stream_slot.h` | scattered: `debug/debug_stream/debug_stream_slot.c` |
 | KPB | `kpb_notifier_init` | `sof/audio/kpb.h` | scattered: `audio/kpb.c` |
 

src/kernel/schedule/CMakeLists.txt

@@ -2,7 +2,36 @@
 
 is_zephyr(zephyr)
 if(zephyr)
+	zephyr_library_sources(
+		schedule-kernel.c
+		zephyr_domain-kernel.c
+	)
+
+	# The kernel part of zephyr_ll.c, built with it, see src/schedule/CMakeLists.txt
+	if(NOT CONFIG_SOC_ACP_6_0 OR CONFIG_DMA_DOMAIN)
+		zephyr_library_sources(
+			zephyr_ll-kernel.c
+		)
+	endif()
+
 	zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL
 		zephyr_ll_user.c
 	)
+
+	zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER
+		zephyr_dp_schedule-kernel.c
+	)
+	if(CONFIG_SOF_USERSPACE_APPLICATION)
+		zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER
+			zephyr_dp_schedule_application-kernel.c
+		)
+	else()
+		zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER
+			zephyr_dp_schedule_thread-kernel.c
+		)
+	endif()
+
+	zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER
+		zephyr_twb_schedule.c
+	)
 endif()

src/kernel/schedule/schedule-kernel.c

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// Copyright(c) 2019 Intel Corporation. All rights reserved.
+//
+// Author: Bartosz Kokoszko <bartoszx.kokoszko@linux.intel.com>
+
+/* Generic scheduler */
+
+#include <rtos/alloc.h>
+#include <sof/lib/cpu.h>
+#include <sof/lib/cold.h>
+#include <sof/lib/uuid.h>
+#include <sof/list.h>
+#include <sof/schedule/schedule.h>
+#include <rtos/task.h>
+#include <ipc/topology.h>
+#include <errno.h>
+#include <stdint.h>
+#include <ipc4/base_fw.h>
+
+LOG_MODULE_DECLARE(schedule, CONFIG_SOF_LOG_LEVEL);
+
+extern struct tr_ctx sch_tr;
+
+/* Kernel part of the generic scheduler: scheduler registration. The rest is in
+ * src/schedule/schedule.c
+ */
+
+__cold static void scheduler_register(struct schedule_data *scheduler)
+{
+	struct schedulers **sch = arch_schedulers_get();
+	struct k_heap *heap = NULL;
+
+	assert_can_be_cold();
+
+	if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(scheduler->type)) {
+		sch = arch_user_schedulers_get();
+		heap = sof_sys_user_heap_get();
+	}
+
+	if (!*sch) {
+		/* init schedulers list */
+		*sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0);
+		if (!*sch) {
+			tr_err(&sch_tr, "allocation failed");
+			return;
+		}
+		memset(*sch, 0, sizeof(**sch));
+		list_init(&(*sch)->list);
+	}
+
+	list_item_append(&scheduler->list, &(*sch)->list);
+}
+
+__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data)
+{
+	struct schedule_data *sch;
+	struct k_heap *heap = NULL;
+
+	assert_can_be_cold();
+
+	if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(type))
+		heap = sof_sys_user_heap_get();
+
+	if (!ops || !ops->schedule_task || !ops->schedule_task_cancel ||
+	    !ops->schedule_task_free)
+		return;
+
+	sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0);
+	if (!sch) {
+		tr_err(&sch_tr, "allocation failed");
+		sof_panic(SOF_IPC_PANIC_IPC);
+	}
+	memset(sch, 0, sizeof(*sch));
+	list_init(&sch->list);
+	sch->type = type;
+	sch->ops = ops;
+	sch->data = data;
+
+	scheduler_register(sch);
+}