diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index c5e2727ab3cb..0e7b344510cd 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -490,6 +490,44 @@ __cold static void ipc_user_init(void) if (ret < 0) LOG_WRN("ipc context partition add failed: %d", ret); + /* + * Grant user-space access to .cold (execute) and .coldrodata (read) + * sections in IMR. The prepare path walks component code that may + * reference __cold functions and __cold_rodata data. + */ +#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM + extern char __cold_start[], __cold_end[]; + extern char __coldrodata_start[]; + extern char _imr_end[]; + + if (&__cold_end[0] > &__cold_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__cold_start, + .size = ALIGN_UP((uintptr_t)__cold_end - (uintptr_t)__cold_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RX_U_RX, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold text partition add failed: %d", ret); + } + + if (&_imr_end[0] > &__coldrodata_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__coldrodata_start, + .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); + } +#endif + k_sem_init(ipc_user->sem, 0, 1); ret = ipc_user_init_thread(ipc_user); diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 12f1234aef7a..fbb68d809bd2 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1490,6 +1490,7 @@ __cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply) { + const struct ipc4_module_init_instance *mi; uint32_t type; int ret; @@ -1500,24 +1501,21 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, switch (type) { case SOF_IPC4_MOD_INIT_INSTANCE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= - sizeof(((struct ipc_user *)0)->init_drv_data), - "ipc_user.init_drv_data too small for driver copy"); - - /* User-space init: kernel does driver lookup only (requires - * access to IMR manifest and driver list in kernel memory). - * Component creation (drv->ops.create) runs in user thread - * so untrusted module code does not execute in kernel context. - * Cross-core creation stays fully in kernel. - */ - const struct ipc4_module_init_instance *mi = - (const struct ipc4_module_init_instance *)ipc4; + mi = (const struct ipc4_module_init_instance *)ipc4; - if (!cpu_is_me(mi->extension.r.core_id)) { - ret = ipc4_init_module_instance(ipc4); - } else { + if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain && + IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { +#ifdef CONFIG_SOF_USERSPACE_LL + BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= + sizeof(((struct ipc_user *)0)->init_drv_data), + "ipc_user.init_drv_data too small for driver copy"); + + /* User-space init: kernel does driver lookup only (requires + * access to IMR manifest and driver list in kernel memory). + * Component creation (drv->ops.create) runs in user thread + * so untrusted module code does not execute in kernel context. + * Cross-core creation stays fully in kernel. + */ struct ipc *ipc = ipc_get(); uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, mi->primary.r.instance_id); @@ -1550,11 +1548,15 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, pdata->init_drv = drv; ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); - } - } -#else - ret = ipc4_init_module_instance(ipc4); #endif + } else { + /* + * DP module creation starts running in kernel mode and + * switches to userspace later via a scheduler_dp_thread_ipc() + * call in module_init(). + */ + ret = ipc4_init_module_instance(ipc4); + } break; case SOF_IPC4_MOD_CONFIG_GET: #ifdef CONFIG_SOF_USERSPACE_LL diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 467a40a8d66d..0907bcb03ace 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -111,42 +111,65 @@ __cold static inline unsigned char *ipc4_get_comp_new_data(void) } #endif -/* Only called from ipc4_init_module_instance(), which is __cold */ -__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +__cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init) { - struct comp_ipc_config ipc_config; - const struct comp_driver *drv; - struct comp_dev *dev; - uint32_t comp_id; - unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); assert_can_be_cold(); - comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, - module_init->primary.r.instance_id); - if (ipc4_get_comp_dev(comp_id)) { tr_err(&ipc_tr, "comp 0x%x exists", comp_id); - return NULL; + return -EEXIST; } if (module_init->extension.r.core_id >= CONFIG_CORE_COUNT) { tr_err(&ipc_tr, "ipc: comp->core = %u", (uint32_t)module_init->extension.r.core_id); - return NULL; + return -EINVAL; } - memset(&ipc_config, 0, sizeof(ipc_config)); - ipc_config.id = comp_id; - ipc_config.pipeline_id = module_init->extension.r.ppl_instance_id; - ipc_config.core = module_init->extension.r.core_id; - ipc_config.ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); - ipc_config.ipc_extended_init = module_init->extension.r.extended_init; - if (ipc_config.ipc_config_size > MAILBOX_HOSTBOX_SIZE) { + memset(ipc_config, 0, sizeof(*ipc_config)); + ipc_config->id = comp_id; + ipc_config->pipeline_id = module_init->extension.r.ppl_instance_id; + ipc_config->core = module_init->extension.r.core_id; + ipc_config->ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); + ipc_config->ipc_extended_init = module_init->extension.r.extended_init; + if (ipc_config->ipc_config_size > MAILBOX_HOSTBOX_SIZE) { tr_err(&ipc_tr, "IPC payload size %u too big for the message window", - ipc_config.ipc_config_size); - return NULL; + ipc_config->ipc_config_size); + return -ENOSPC; + } + + if (!module_init->extension.r.proc_domain) { + ipc_config->proc_domain = COMP_PROCESSING_DOMAIN_LL; + } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { + ipc_config->proc_domain = COMP_PROCESSING_DOMAIN_DP; + } else { + tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); + return -EINVAL; } + return 0; +} + +/* Only called from ipc4_init_module_instance(), which is __cold */ +__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +{ + struct comp_ipc_config ipc_config; + const struct comp_driver *drv; + struct comp_dev *dev; + unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); + + assert_can_be_cold(); + + int ret = ipc4_comp_new_config(&ipc_config, module_init); + + if (ret < 0) + return NULL; + /* Reject a module naming a non-existent parent pipeline: otherwise * dev->pipeline stays NULL and a later init path (e.g. the copier) * dereferences it. IPC4_INVALID_PIPELINE_ID is exempt - it marks base FW @@ -160,6 +183,7 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo (uint32_t)ipc_config.pipeline_id); return NULL; } + #ifdef CONFIG_DCACHE_LINE_SIZE if (!IS_ENABLED(CONFIG_LIBRARY)) sys_cache_data_invd_range((__sparse_force void __sparse_cache *) @@ -177,19 +201,6 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo if (!drv) return NULL; -#if CONFIG_ZEPHYR_DP_SCHEDULER - if (module_init->extension.r.proc_domain) - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - else - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#else /* CONFIG_ZEPHYR_DP_SCHEDULER */ - if (module_init->extension.r.proc_domain) { - tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); - return NULL; - } - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ - if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { .data = data, @@ -242,43 +253,19 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, struct ipc4_module_init_instance module_init; struct comp_ipc_config ipc_config; struct comp_dev *dev; - uint32_t comp_id; unsigned char *data; - int ret; assert_can_be_cold(); - ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); - if (ret < 0) - return NULL; - - comp_id = IPC4_COMP_ID(module_init.primary.r.module_id, - module_init.primary.r.instance_id); + int ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); - if (ipc4_get_comp_dev(comp_id)) { - tr_err(&ipc_tr, "comp 0x%x exists", comp_id); + if (ret < 0) return NULL; - } - if (module_init.extension.r.core_id >= CONFIG_CORE_COUNT) { - tr_err(&ipc_tr, "ipc: comp->core = %u", - (uint32_t)module_init.extension.r.core_id); + ret = ipc4_comp_new_config(&ipc_config, &module_init); + if (ret < 0) return NULL; - } - memset(&ipc_config, 0, sizeof(ipc_config)); - ipc_config.id = comp_id; - ipc_config.pipeline_id = module_init.extension.r.ppl_instance_id; - ipc_config.core = module_init.extension.r.core_id; - ipc_config.ipc_config_size = - module_init.extension.r.param_block_size * sizeof(uint32_t); - ipc_config.ipc_extended_init = module_init.extension.r.extended_init; - if (ipc_config.ipc_config_size > MAILBOX_HOSTBOX_SIZE) { - tr_err(&ipc_tr, - "IPC payload size %u too big for the message window", - ipc_config.ipc_config_size); - return NULL; - } #ifdef CONFIG_DCACHE_LINE_SIZE if (!IS_ENABLED(CONFIG_LIBRARY)) sys_cache_data_invd_range( @@ -288,19 +275,6 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, #endif data = ipc4_get_comp_new_data(); -#if CONFIG_ZEPHYR_DP_SCHEDULER - if (module_init.extension.r.proc_domain) - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - else - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#else - if (module_init.extension.r.proc_domain) { - tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); - return NULL; - } - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#endif - if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { .data = data,