diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index b62ae38f415f..d937a8770cbc 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -28,8 +28,10 @@ /* The __ZEPHYR__ condition is to keep cmocka tests working */ #if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) #define MEM_API_CHECK_THREAD(res) do { \ - if ((res)->rsrc_mngr != k_current_get()) \ - LOG_WRN("mngr %p != cur %p", (res)->rsrc_mngr, k_current_get()); \ + k_tid_t tid = k_current_get(); \ + if (((res)->rsrc_mngr != NULL || (res)->dp_thread != NULL) && \ + (res)->rsrc_mngr != tid && (res)->dp_thread != tid) \ + LOG_WRN("cur %p != mngr %p or dp %p", tid, (res)->rsrc_mngr, (res)->dp_thread); \ } while (0) #else #define MEM_API_CHECK_THREAD(res) diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index d9518de7e13e..6db9389c5cd7 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -59,23 +59,37 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, #endif static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config, + const struct module_ext_init_data *ext_init, size_t *heap_size) { /* src-lite with 8 channels has been seen allocating 14k in one go */ - /* FIXME: the size will be derived from configuration */ - const size_t buf_size = 28 * 1024; + size_t buf_size = CONFIG_SOF_USERSPACE_DP_DEFAULT_HEAP_SIZE; - /* - * A 1-to-1 replacement of the original heap implementation would be to - * have "lifetime size" equal to 0. But (1) this is invalid for - * vregion_create() and (2) we gradually move objects, that are simple - * to move to the lifetime buffer. Make it 4k for the beginning. - */ - return vregion_create(4096, buf_size - 4096); +#if CONFIG_IPC_MAJOR_4 + if (config->ipc_extended_init && ext_init && ext_init->dp_data && + ext_init->dp_data->heap_bytes > 0) { + if (ext_init->dp_data->heap_bytes > 64*1024*1024) { + LOG_ERR("Bad heap size %u bytes for %#x", + ext_init->dp_data->heap_bytes, config->id); + return NULL; + } + + buf_size = ext_init->dp_data->heap_bytes; + + LOG_INF("%zu byte heap size requested in IPC for %#x", buf_size, config->id); + } +#endif + + *heap_size = buf_size; + + return vregion_create(buf_size); } -static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, - const struct comp_ipc_config *config) +static +struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const struct module_ext_init_data *ext_init, + struct mod_alloc_ctx *ppl_alloc) { struct k_heap *mod_heap; struct vregion *mod_vreg; @@ -91,15 +105,21 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; size_t heap_size; + bool use_ppl_alloc = ppl_alloc && + config->proc_domain == COMP_PROCESSING_DOMAIN_LL; if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) && IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_vreg = module_adapter_dp_heap_new(config, &heap_size); + mod_vreg = module_adapter_dp_heap_new(config, ext_init, &heap_size); if (!mod_vreg) { comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); return NULL; } mod_heap = NULL; + } else if (use_ppl_alloc) { + mod_vreg = ppl_alloc->vreg ? vregion_get(ppl_alloc->vreg) : NULL; + mod_heap = ppl_alloc->heap; + heap_size = 0; } else { #ifdef CONFIG_SOF_USERSPACE_LL mod_heap = sof_sys_user_heap_get(); @@ -111,26 +131,38 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv mod_vreg = NULL; } - if (!mod_vreg) + if (use_ppl_alloc) { + /* LL modules use the pipeline's alloc context */ + mod = sof_ctx_alloc(ppl_alloc, flags, sizeof(*mod), 0); + } else if (!mod_vreg) { mod = sof_heap_alloc(mod_heap, flags, sizeof(*mod), 0); - else if (flags & SOF_MEM_FLAG_COHERENT) - mod = vregion_alloc_coherent(mod_vreg, VREGION_MEM_TYPE_LIFETIME, sizeof(*mod)); - else - mod = vregion_alloc(mod_vreg, VREGION_MEM_TYPE_LIFETIME, sizeof(*mod)); + } else if (flags & SOF_MEM_FLAG_COHERENT) { + mod = vregion_alloc_coherent(mod_vreg, sizeof(*mod)); + } else { + mod = vregion_alloc(mod_vreg, sizeof(*mod)); + } if (!mod) { comp_cl_err(drv, "failed to allocate memory for module"); goto emod; } - struct mod_alloc_ctx *alloc = sof_heap_alloc(mod_heap, flags, sizeof(*alloc), 0); + struct mod_alloc_ctx *alloc; - if (!alloc) - goto ealloc; + if (use_ppl_alloc) { + /* LL modules share the pipeline's alloc context */ + alloc = ppl_alloc; + } else { + alloc = sof_heap_alloc(mod_heap, flags, sizeof(*alloc), 0); + if (!alloc) + goto ealloc; + + memset(alloc, 0, sizeof(*alloc)); + alloc->heap = mod_heap; + alloc->vreg = mod_vreg; + } memset(mod, 0, sizeof(*mod)); - alloc->heap = mod_heap; - alloc->vreg = mod_vreg; mod->priv.resources.alloc = alloc; mod_resource_init(mod); @@ -140,8 +172,10 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv * then it can be cached. Effectively it can be only cached in * single-core configurations. */ - if (mod_vreg) - dev = vregion_alloc_coherent(mod_vreg, VREGION_MEM_TYPE_LIFETIME, sizeof(*dev)); + if (use_ppl_alloc) + dev = sof_ctx_alloc(ppl_alloc, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0); + else if (mod_vreg) + dev = vregion_alloc_coherent(mod_vreg, sizeof(*dev)); else dev = sof_heap_alloc(mod_heap, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0); @@ -159,14 +193,20 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv return mod; edev: - sof_heap_free(mod_heap, alloc); + if (!use_ppl_alloc) + sof_heap_free(mod_heap, alloc); ealloc: - if (mod_vreg) + if (use_ppl_alloc) + sof_ctx_free(ppl_alloc, mod); + else if (mod_vreg) vregion_free(mod_vreg, mod); else sof_heap_free(mod_heap, mod); emod: - vregion_put(mod_vreg); + if (use_ppl_alloc) + vregion_put(ppl_alloc->vreg); + else + vregion_put(mod_vreg); return NULL; } @@ -174,26 +214,27 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv static void module_adapter_mem_free(struct processing_module *mod) { struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; - struct k_heap *mod_heap = alloc->heap; + bool ppl_alloc = mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL && + mod->dev->pipeline && mod->dev->pipeline->alloc == alloc; /* * In principle it shouldn't even be needed to free individual objects * on the module heap since we're freeing the heap itself too */ #if CONFIG_IPC_MAJOR_4 - sof_heap_free(mod_heap, mod->priv.cfg.input_pins); + sof_heap_free(alloc->heap, mod->priv.cfg.input_pins); #endif - if (alloc->vreg) { - struct vregion *mod_vreg = alloc->vreg; - - vregion_free(mod_vreg, mod->dev); - vregion_free(mod_vreg, mod); - if (!vregion_put(mod_vreg)) - sof_heap_free(alloc->heap, alloc); + sof_ctx_free(alloc, mod->dev); + sof_ctx_free(alloc, mod); + + if (ppl_alloc) { + /* alloc belongs to pipeline, just release vregion reference */ + vregion_put(alloc->vreg); + } else if (alloc->vreg) { + if (!vregion_put(alloc->vreg)) + rfree(alloc); } else { - sof_heap_free(mod_heap, mod->dev); - sof_heap_free(mod_heap, mod); - sof_heap_free(mod_heap, alloc); + rfree(alloc); } } @@ -236,8 +277,32 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, return NULL; } #endif + const struct module_ext_init_data *ext_init = +#if CONFIG_IPC_MAJOR_4 + &ext_data; +#else + NULL; +#endif + +#if CONFIG_IPC_MAJOR_4 + struct ipc_comp_dev *ipc_pipe; + struct ipc *ipc = ipc_get(); + struct mod_alloc_ctx *ppl_alloc = NULL; + + /* resolve the pipeline pointer early to pass its alloc to mem_alloc */ + ipc_pipe = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, config->pipeline_id, + IPC_COMP_IGNORE_REMOTE); + if (ipc_pipe && ipc_pipe->pipeline) + ppl_alloc = ipc_pipe->pipeline->alloc; +#endif - struct processing_module *mod = module_adapter_mem_alloc(drv, config); + struct processing_module *mod = module_adapter_mem_alloc(drv, config, ext_init, +#if CONFIG_IPC_MAJOR_4 + ppl_alloc +#else + NULL +#endif + ); if (!mod) return NULL; @@ -250,6 +315,18 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, struct comp_dev *dev = mod->dev; + dst = &mod->priv.cfg; + /* + * NOTE: dst->ext_data points to stack variable and contains + * pointers to IPC payload mailbox, so its only valid in + * functions that are called from this function. This is + * why the pointer is set to NULL before this function + * exits. + */ +#if CONFIG_IPC_MAJOR_4 + dst->ext_data = &ext_data; +#endif + #if CONFIG_ZEPHYR_DP_SCHEDULER /* create a task for DP processing */ if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) { @@ -262,16 +339,6 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, } #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ - dst = &mod->priv.cfg; - /* - * NOTE: dst->ext_data points to stack variable and contains - * pointers to IPC payload mailbox, so its only valid in - * functions that called from this function. This why - * the pointer is set NULL before this function exits. - */ -#if CONFIG_IPC_MAJOR_4 - dst->ext_data = &ext_data; -#endif ret = module_adapter_init_data(dev, dst, config, &spec); if (ret) { comp_err(dev, "%d: module init data failed", @@ -294,12 +361,7 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, goto err; #if CONFIG_IPC_MAJOR_4 - struct ipc_comp_dev *ipc_pipe; - struct ipc *ipc = ipc_get(); - /* set the pipeline pointer if ipc_pipe is valid */ - ipc_pipe = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, config->pipeline_id, - IPC_COMP_IGNORE_REMOTE); if (ipc_pipe) { dev->pipeline = ipc_pipe->pipeline; diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 8494f17998bd..9b834b31b18f 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -7,8 +7,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -23,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -112,6 +115,7 @@ void pipeline_posn_init(struct sof *sof) struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_t priority, uint32_t comp_id, struct create_pipeline_params *pparams) { + struct mod_alloc_ctx *alloc; struct sof_ipc_stream_posn posn; struct pipeline *p; int ret; @@ -122,17 +126,33 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ /* show heap status */ heap_trace_all(0); - /* allocate new pipeline */ - p = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, sizeof(*p), 0); - if (!p) { - pipe_cl_err("Out of Memory"); + alloc = rzalloc(SOF_MEM_FLAG_USER, sizeof(*alloc)); + if (!alloc) { + pipe_cl_err("Failed to allocate pipeline alloc context"); return NULL; } - memset(p, 0, sizeof(*p)); + alloc->heap = heap; + + /* Create vregion for pipeline and its modules if size info is available */ + if (IS_ENABLED(CONFIG_SOF_VREGIONS) && + pparams && pparams->mem_data && pparams->mem_data->heap_bytes) { + alloc->vreg = vregion_create(pparams->mem_data->heap_bytes); + if (!alloc->vreg) + pipe_cl_err("Failed to create pipeline vregion of %zu bytes, using heap", + pparams->mem_data->heap_bytes); + } + + p = sof_ctx_zalloc(alloc, SOF_MEM_FLAG_USER, sizeof(*p), 0); + if (!p) { + pipe_cl_err("Out of Memory"); + goto free_alloc; + } /* init pipeline */ p->heap = heap; + p->alloc = alloc; + p->comp_id = comp_id; p->priority = priority; p->pipeline_id = pipeline_id; @@ -165,7 +185,10 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ return p; free: - sof_heap_free(heap, p); + sof_ctx_free(alloc, p); +free_alloc: + vregion_put(alloc->vreg); + rfree(alloc); return NULL; } @@ -245,6 +268,8 @@ void pipeline_disconnect(struct comp_dev *comp, struct comp_buffer *buffer, int /* pipelines must be inactive */ int pipeline_free(struct pipeline *p) { + struct mod_alloc_ctx *alloc = p->alloc; + pipe_dbg(p, "entry"); /* @@ -265,7 +290,12 @@ int pipeline_free(struct pipeline *p) pipeline_posn_offset_put(p->posn_offset); /* now free the pipeline */ - sof_heap_free(p->heap, p); + sof_ctx_free(alloc, p); + + /* free alloc context and vregion */ + if (vregion_put(alloc->vreg)) + pipe_cl_warn("pipeline vregion still in use"); + rfree(alloc); /* show heap status */ heap_trace_all(0); @@ -294,8 +324,14 @@ static int pipeline_comp_complete(struct comp_dev *current, * It will be calculated during module prepare operation * either by the module or to default value based on module's OBS */ - if (current->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) + if (current->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) { current->period = ppl_data->p->period; + } else { + struct processing_module *mod = comp_mod(current); + + if (mod && mod->priv.resources.alloc) + vregion_set_interim(mod->priv.resources.alloc->vreg); + } current->priority = ppl_data->p->priority; @@ -336,6 +372,10 @@ int pipeline_complete(struct pipeline *p, struct comp_dev *source, p->source_comp = source; p->sink_comp = sink; + + if (p->alloc && p->alloc->vreg) + vregion_set_interim(p->alloc->vreg); + p->status = COMP_STATE_READY; /* show heap status */ diff --git a/src/audio/pipeline/pipeline-schedule.c b/src/audio/pipeline/pipeline-schedule.c index cb4ec8fd3c62..70ae1186b2a2 100644 --- a/src/audio/pipeline/pipeline-schedule.c +++ b/src/audio/pipeline/pipeline-schedule.c @@ -398,6 +398,7 @@ int pipeline_comp_dp_task_init(struct comp_dev *comp) { /* DP tasks are guaranteed to have a module_adapter */ struct processing_module *mod = comp_mod(comp); + size_t stack_size = TASK_DP_STACK_SIZE; struct task_ops ops = { .run = dp_task_run, .get_deadline = NULL, @@ -413,8 +414,17 @@ int pipeline_comp_dp_task_init(struct comp_dev *comp) unsigned int flags = IS_ENABLED(CONFIG_USERSPACE) ? K_USER : 0; #endif + if (mod->priv.cfg.ext_data && mod->priv.cfg.ext_data->dp_data && + mod->priv.cfg.ext_data->dp_data->stack_bytes > 0) { + stack_size = MAX(mod->priv.cfg.ext_data->dp_data->stack_bytes, + CONFIG_ZEPHYR_DP_SCHEDULER_MIN_STACK_SIZE); + comp_info(comp, "stack size set to %zu, %zu requested, min allowed %zu", + stack_size, mod->priv.cfg.ext_data->dp_data->stack_bytes, + CONFIG_ZEPHYR_DP_SCHEDULER_MIN_STACK_SIZE); + } + return scheduler_dp_task_init(&comp->task, SOF_UUID(dp_task_uuid), &ops, mod, - comp->ipc_config.core, TASK_DP_STACK_SIZE, flags); + comp->ipc_config.core, stack_size, flags); } #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ diff --git a/src/audio/src/src.c b/src/audio/src/src.c index 9870ad911ca8..b75ff5a4cce7 100644 --- a/src/audio/src/src.c +++ b/src/audio/src/src.c @@ -34,19 +34,16 @@ LOG_MODULE_DECLARE(src, CONFIG_SOF_LOG_LEVEL); -static int src_prepare(struct processing_module *mod, - struct sof_source **sources, int num_of_sources, - struct sof_sink **sinks, int num_of_sinks) +/* Set rate table pointers, compute rate indices, and copy filter stages. + * Must be in src.c because src_table1/2, src_in_fs, etc. come from + * the coefficient headers included by this file. + */ +static int src_setup_stages(struct processing_module *mod) { struct comp_data *cd = module_get_private_data(mod); struct src_param *a = &cd->param; int ret; - comp_info(mod->dev, "entry"); - - if (num_of_sources != 1 || num_of_sinks != 1) - return -EINVAL; - a->in_fs = src_in_fs; a->out_fs = src_out_fs; a->num_in_fs = NUM_IN_FS; @@ -54,27 +51,46 @@ static int src_prepare(struct processing_module *mod, a->max_fir_delay_size_xnch = (PLATFORM_MAX_CHANNELS * MAX_FIR_DELAY_SIZE); a->max_out_delay_size_xnch = (PLATFORM_MAX_CHANNELS * MAX_OUT_DELAY_SIZE); - src_get_source_sink_params(mod->dev, sources[0], sinks[0]); - ret = src_param_set(mod->dev, cd); if (ret < 0) return ret; - ret = src_allocate_copy_stages(mod, a, - src_table1[a->idx_out][a->idx_in], - src_table2[a->idx_out][a->idx_in]); - if (ret < 0) - return ret; + return src_allocate_copy_stages(mod, a, + src_table1[a->idx_out][a->idx_in], + src_table2[a->idx_out][a->idx_in]); +} - ret = src_params_general(mod, sources[0], sinks[0]); +static int src_do_init(struct processing_module *mod) +{ + struct comp_data *cd; + int ret; + + ret = src_init(mod); if (ret < 0) return ret; - return src_prepare_general(mod, sources[0], sinks[0]); + cd = module_get_private_data(mod); + cd->setup_stages = src_setup_stages; + + return src_init_stages(mod); +} + +static int src_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + comp_info(mod->dev, "entry"); + + if (num_of_sources != 1 || num_of_sinks != 1) + return -EINVAL; + + src_get_source_sink_params(mod->dev, sources[0], sinks[0]); + + return src_prepare_do(mod, sources[0], sinks[0]); } static const struct module_interface src_interface = { - .init = src_init, + .init = src_do_init, .prepare = src_prepare, .process = src_process, .is_ready_to_process = src_is_ready_to_process, diff --git a/src/audio/src/src_common.c b/src/audio/src/src_common.c index 3a9c7dac280f..e4ccf76ae3ba 100644 --- a/src/audio/src/src_common.c +++ b/src/audio/src/src_common.c @@ -574,6 +574,86 @@ int src_params_general(struct processing_module *mod, return 0; } +/* Allocate delay lines and initialize the polyphase SRC filter. + * Assumes that cd->param rate table pointers (in_fs, out_fs, etc.) + * and stage pointers (stage1, stage2) are already set up via + * cd->setup_stages(). + */ +int src_allocate_delay_lines(struct processing_module *mod) +{ + struct comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + size_t delay_lines_size; + int32_t *buffer_start; + int n; + int ret; + + /* For LL modules dev->period is already set from the pipeline. + * Compute dev->frames so buffer sizing works. + */ + if (!dev->frames) + component_set_nearest_period_frames(dev, cd->sink_rate); + + if (!cd->sink_rate) { + comp_err(dev, "zero sink rate"); + return -EINVAL; + } + + cd->source_frames = dev->frames * cd->source_rate / cd->sink_rate; + cd->sink_frames = dev->frames; + + /* Allocate needed memory for delay lines */ + ret = src_buffer_lengths(dev, cd, cd->channels_count); + if (ret < 0) { + comp_err(dev, "src_buffer_lengths() failed"); + return ret; + } + + delay_lines_size = ALIGN_UP(sizeof(int32_t) * cd->param.total, 8); + if (delay_lines_size == 0) { + comp_err(dev, "delay_lines_size = 0"); + return -EINVAL; + } + + mod_free(mod, cd->delay_lines); + + cd->delay_lines = mod_alloc(mod, delay_lines_size); + if (!cd->delay_lines) { + comp_err(dev, "failed to alloc cd->delay_lines, delay_lines_size = %zu", + delay_lines_size); + return -ENOMEM; + } + + memset(cd->delay_lines, 0, delay_lines_size); + buffer_start = cd->delay_lines + ALIGN_UP(cd->param.sbuf_length, 2); + + /* Initialize SRC for actual sample rate */ + n = src_polyphase_init(&cd->src, &cd->param, buffer_start); + + /* Reset stage buffer */ + cd->sbuf_r_ptr = cd->delay_lines; + cd->sbuf_w_ptr = cd->delay_lines; + cd->sbuf_avail = 0; + + switch (n) { + case 0: + cd->src_func = src_copy_sxx; + break; + case 1: + cd->src_func = src_1s; + break; + case 2: + cd->src_func = src_2s; + break; + default: + comp_info(dev, "missing coefficients for requested rates combination"); + cd->src_func = src_fallback; + return -EINVAL; + } + + return 0; +} + int src_param_set(struct comp_dev *dev, struct comp_data *cd) { struct src_param *a = &cd->param; diff --git a/src/audio/src/src_common.h b/src/audio/src/src_common.h index 98f29a263131..126b112737d3 100644 --- a/src/audio/src/src_common.h +++ b/src/audio/src/src_common.h @@ -167,6 +167,7 @@ struct comp_data { int (*src_func)(struct comp_data *cd, struct sof_source *source, struct sof_sink *sink); void (*polyphase_func)(struct src_stage_prm *s); + int (*setup_stages)(struct processing_module *mod); }; #if CONFIG_IPC_MAJOR_4 @@ -218,6 +219,7 @@ static inline int src_fallback(struct comp_data *cd, int src_allocate_copy_stages(struct processing_module *mod, struct src_param *prm, const struct src_stage *stage_src1, const struct src_stage *stage_src2); +int src_allocate_delay_lines(struct processing_module *mod); int src_rate_check(const void *spec); int src_set_params(struct processing_module *mod, struct sof_sink *sink); @@ -227,6 +229,9 @@ int src_prepare_general(struct processing_module *mod, struct sof_source *source, struct sof_sink *sink); int src_init(struct processing_module *mod); +int src_init_stages(struct processing_module *mod); +int src_prepare_do(struct processing_module *mod, + struct sof_source *source, struct sof_sink *sink); int src_copy_sxx(struct comp_data *cd, struct sof_source *source, struct sof_sink *sink); diff --git a/src/audio/src/src_ipc3.c b/src/audio/src/src_ipc3.c index 541efcdb24be..636c50bfdcea 100644 --- a/src/audio/src/src_ipc3.c +++ b/src/audio/src/src_ipc3.c @@ -195,3 +195,27 @@ int src_init(struct processing_module *mod) return 0; } +/* IPC3: No filter allocation at init, change ipc3 behavior as little as possible */ +int src_init_stages(struct processing_module *mod) +{ + return 0; +} + +/* IPC3: Full filter setup at prepare time */ +int src_prepare_do(struct processing_module *mod, + struct sof_source *source, struct sof_sink *sink) +{ + struct comp_data *cd = module_get_private_data(mod); + int ret; + + ret = cd->setup_stages(mod); + if (ret < 0) + return ret; + + ret = src_params_general(mod, source, sink); + if (ret < 0) + return ret; + + return src_prepare_general(mod, source, sink); +} + diff --git a/src/audio/src/src_ipc4.c b/src/audio/src/src_ipc4.c index 91f286347a5f..6e6f3c51b7be 100644 --- a/src/audio/src/src_ipc4.c +++ b/src/audio/src/src_ipc4.c @@ -246,3 +246,67 @@ __cold int src_init(struct processing_module *mod) return 0; } +/* Called after src_init() and setup_stages callback is set. + * Allocate filter stages and delay lines at init time. + */ +int src_init_stages(struct processing_module *mod) +{ + struct comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int ret; + + ret = cd->setup_stages(mod); + if (ret < 0) + return ret; + + /* For DP modules, dev->period is not yet set at init time (it's + * computed in src_set_params at prepare). Derive it here from the + * IPC config's output buffer size so that delay line allocation + * uses correct buffer sizes. + */ + if (dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP && !dev->frames) { + uint32_t frame_bytes = cd->channels_count * cd->sample_container_bytes; + + if (frame_bytes && cd->sink_rate) { + dev->period = 1000000ULL * + (cd->ipc_config.base.obs / frame_bytes) / + cd->sink_rate; + dev->period /= LL_TIMER_PERIOD_US; + dev->period *= LL_TIMER_PERIOD_US; + component_set_nearest_period_frames(dev, cd->sink_rate); + } + } + + return src_allocate_delay_lines(mod); +} + +/* At prepare time just verify rates and set downstream params */ +int src_prepare_do(struct processing_module *mod, + struct sof_source *source, struct sof_sink *sink) +{ + struct comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int ret; + + if (cd->source_rate != cd->ipc_config.base.audio_fmt.sampling_frequency || + cd->sink_rate != cd->ipc_config.sink_rate) { + comp_err(mod->dev, "rate mismatch: source %u/%u sink %u/%u", + cd->source_rate, + cd->ipc_config.base.audio_fmt.sampling_frequency, + cd->sink_rate, cd->ipc_config.sink_rate); + return -EINVAL; + } + + ret = src_set_params(mod, sink); + if (ret < 0) { + comp_err(mod->dev, "set params failed."); + return ret; + } + + /* Update frame counts with final dev->frames from src_set_params */ + cd->source_frames = dev->frames * cd->source_rate / cd->sink_rate; + cd->sink_frames = dev->frames; + + return src_prepare_general(mod, source, sink); +} + diff --git a/src/audio/src/src_lite.c b/src/audio/src/src_lite.c index 9d5593ff34ca..89d205fc7b85 100644 --- a/src/audio/src/src_lite.c +++ b/src/audio/src/src_lite.c @@ -14,24 +14,16 @@ LOG_MODULE_REGISTER(src_lite, CONFIG_SOF_LOG_LEVEL); -/* - * This function is 100% identical to src_prepare(), but it's - * assigning different coefficient arrays because it's including - * different headers. +/* Set rate table pointers, compute rate indices, and copy filter stages. + * Must be in src_lite.c because src_table1/2, src_in_fs, etc. come from + * the coefficient headers included by this file. */ -static int src_lite_prepare(struct processing_module *mod, - struct sof_source **sources, int num_of_sources, - struct sof_sink **sinks, int num_of_sinks) +static int src_lite_setup_stages(struct processing_module *mod) { struct comp_data *cd = module_get_private_data(mod); struct src_param *a = &cd->param; int ret; - comp_info(mod->dev, "entry"); - - if (num_of_sources != 1 || num_of_sinks != 1) - return -EINVAL; - a->in_fs = src_in_fs; a->out_fs = src_out_fs; a->num_in_fs = NUM_IN_FS; @@ -43,21 +35,42 @@ static int src_lite_prepare(struct processing_module *mod, if (ret < 0) return ret; - ret = src_allocate_copy_stages(mod, a, - src_table1[a->idx_out][a->idx_in], - src_table2[a->idx_out][a->idx_in]); - if (ret < 0) - return ret; + return src_allocate_copy_stages(mod, a, + src_table1[a->idx_out][a->idx_in], + src_table2[a->idx_out][a->idx_in]); +} + +static int src_lite_do_init(struct processing_module *mod) +{ + struct comp_data *cd; + int ret; - ret = src_params_general(mod, sources[0], sinks[0]); + ret = src_init(mod); if (ret < 0) return ret; - return src_prepare_general(mod, sources[0], sinks[0]); + cd = module_get_private_data(mod); + cd->setup_stages = src_lite_setup_stages; + + return src_init_stages(mod); +} + +static int src_lite_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + comp_info(mod->dev, "entry"); + + if (num_of_sources != 1 || num_of_sinks != 1) + return -EINVAL; + + src_get_source_sink_params(mod->dev, sources[0], sinks[0]); + + return src_prepare_do(mod, sources[0], sinks[0]); } const struct module_interface src_lite_interface = { - .init = src_init, + .init = src_lite_do_init, .prepare = src_lite_prepare, .process = src_process, .is_ready_to_process = src_is_ready_to_process, diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 94827d86cd9f..de22610b6251 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -135,6 +135,7 @@ struct module_resources { struct mod_alloc_ctx *alloc; #if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) k_tid_t rsrc_mngr; + k_tid_t dp_thread; #endif }; diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index 913a569c208c..3bd9d039a00d 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -26,6 +26,7 @@ struct comp_dev; struct ipc; struct ipc_msg; struct k_heap; +struct mod_alloc_ctx; /* * Pipeline status to stop execution of current path, but to keep the @@ -54,6 +55,7 @@ struct k_heap; */ struct pipeline { struct k_heap *heap; /**< heap used for allocating this pipeline */ + struct mod_alloc_ctx *alloc; /**< shared alloc context for pipeline modules */ uint32_t comp_id; /**< component id for pipeline */ uint32_t pipeline_id; /**< pipeline id */ uint32_t sched_id; /**< Scheduling component id */ diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index d64249ed64c8..5c066c90dbc8 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -24,6 +24,7 @@ struct vregion; enum vregion_mem_type { VREGION_MEM_TYPE_INTERIM, /* interim allocation that can be freed */ VREGION_MEM_TYPE_LIFETIME, /* lifetime allocation */ + VREGION_MEM_TYPE_INVALID, /* Interim heap initialization failed */ }; #if CONFIG_SOF_VREGIONS @@ -31,14 +32,24 @@ enum vregion_mem_type { /** * @brief Create a new virtual region instance. * - * Create a new virtual region instance with specified static and dynamic partitions. - * Total size is the sum of static and dynamic sizes. + * Create a new virtual region instance with specified memory size. + * Allocations start in LIFETIME mode. * - * @param[in] lifetime_size Size of the virtual region lifetime partition. - * @param[in] interim_size Size of the virtual region interim partition. + * @param[in] memsize Total size of the virtual region memory. * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. */ -struct vregion *vregion_create(size_t lifetime_size, size_t interim_size); +struct vregion *vregion_create(size_t memsize); + +/** + * @brief Switch virtual region allocations to interim mode. + * + * After this call, all allocations from this vregion will use the interim + * heap. The interim heap is created lazily from remaining lifetime space. + * Multiple calls are allowed but log a warning. + * + * @param[in] vr Pointer to the virtual region instance. + */ +void vregion_set_interim(struct vregion *vr); /** * @brief Increment virtual region's user count. @@ -66,36 +77,33 @@ struct vregion *vregion_put(struct vregion *vr); * @brief Allocate memory from the specified virtual region. * * @param[in] vr Pointer to the virtual region instance. - * @param[in] type Type of memory to allocate (lifetime or interim). * @param[in] size Size of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size); +void *vregion_alloc(struct vregion *vr, size_t size); /** * @brief like vregion_alloc() but allocates coherent memory */ -void *vregion_alloc_coherent(struct vregion *vr, enum vregion_mem_type type, size_t size); +void *vregion_alloc_coherent(struct vregion *vr, size_t size); /** * @brief Allocate aligned memory from the specified virtual region. * - * Allocate aligned memory from the specified virtual region based on the memory type. + * Allocate aligned memory from the specified virtual region using the + * current allocation mode (lifetime or interim). * * @param[in] vr Pointer to the virtual region instance. - * @param[in] type Type of memory to allocate (lifetime or interim). * @param[in] size Size of memory to allocate in bytes. * @param[in] alignment Alignment of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, - size_t size, size_t alignment); +void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); /** * @brief like vregion_alloc_align() but allocates coherent memory */ -void *vregion_alloc_coherent_align(struct vregion *vr, enum vregion_mem_type type, - size_t size, size_t alignment); +void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); /** * @brief Free memory allocated from the specified virtual region. @@ -129,10 +137,11 @@ struct vregion { unsigned int use_count; }; -static inline struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) +static inline struct vregion *vregion_create(size_t memsize) { return NULL; } +static inline void vregion_set_interim(struct vregion *vr) {} static inline struct vregion *vregion_get(struct vregion *vr) { return vr; @@ -141,22 +150,20 @@ static inline struct vregion *vregion_put(struct vregion *vr) { return vr; } -static inline void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) +static inline void *vregion_alloc(struct vregion *vr, size_t size) { return NULL; } -static inline void *vregion_alloc_coherent(struct vregion *vr, enum vregion_mem_type type, - size_t size) +static inline void *vregion_alloc_coherent(struct vregion *vr, size_t size) { return NULL; } -static inline void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, - size_t size, size_t alignment) +static inline void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) { return NULL; } -static inline void *vregion_alloc_coherent_align(struct vregion *vr, enum vregion_mem_type type, - size_t size, size_t alignment) +static inline void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, + size_t alignment) { return NULL; } diff --git a/src/lib/objpool.c b/src/lib/objpool.c index 6925e6f7070a..9636e018d3d1 100644 --- a/src/lib/objpool.c +++ b/src/lib/objpool.c @@ -44,10 +44,10 @@ static int objpool_add(struct objpool_head *head, unsigned int n, size_t size, u pobjpool = sof_heap_alloc(head->heap, flags, aligned_size + sizeof(*pobjpool), 0); else if (flags & SOF_MEM_FLAG_COHERENT) - pobjpool = vregion_alloc_coherent(head->vreg, VREGION_MEM_TYPE_INTERIM, + pobjpool = vregion_alloc_coherent(head->vreg, aligned_size + sizeof(*pobjpool)); else - pobjpool = vregion_alloc(head->vreg, VREGION_MEM_TYPE_INTERIM, + pobjpool = vregion_alloc(head->vreg, aligned_size + sizeof(*pobjpool)); if (!pobjpool) diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index ac03bb59fe5d..f1eee62d19ad 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -504,6 +504,10 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, stack_size, dp_thread_fn, ptask, NULL, NULL, CONFIG_DP_THREAD_PRIORITY, ptask->flags, K_FOREVER); scheduler_dp_thread_name_set(pdata->thread_id, mod); +#if CONFIG_MODULE_MEMORY_API_DEBUG + /* For a DP module the resource manager can also be the DP thread */ + mod->priv.resources.dp_thread = pdata->thread_id; +#endif unsigned int pidx; size_t size; diff --git a/tools/topology/topology2/cavs-nocodec.conf b/tools/topology/topology2/cavs-nocodec.conf index fea906f98741..52d603c0eb3c 100644 --- a/tools/topology/topology2/cavs-nocodec.conf +++ b/tools/topology/topology2/cavs-nocodec.conf @@ -698,9 +698,9 @@ IncludeByKey.PASSTHROUGH { IncludeByKey.SRC_DOMAIN { "DP" { core_id $DP_SRC_CORE_ID - domain_id 123 - stack_bytes_requirement 4096 - heap_bytes_requirement 8192 + domain_id 0 + stack_bytes_requirement 2048 + heap_bytes_requirement "$[(24 * 1024)]" } } } @@ -1382,9 +1382,9 @@ IncludeByKey.PASSTHROUGH { IncludeByKey.SRC_DOMAIN { "DP" { core_id $DP_SRC_CORE_ID - domain_id 123 - stack_bytes_requirement 4096 - heap_bytes_requirement 8192 + domain_id 0 + stack_bytes_requirement 2048 + heap_bytes_requirement "$[(24 * 1024)]" } } diff --git a/tools/topology/topology2/include/common/tokens.conf b/tools/topology/topology2/include/common/tokens.conf index 48b498280130..987dd86c5392 100644 --- a/tools/topology/topology2/include/common/tokens.conf +++ b/tools/topology/topology2/include/common/tokens.conf @@ -28,9 +28,7 @@ Object.Base.VendorToken { scheduler_domain 418 domain_id 419 stack_bytes_requirement 420 - interim_heap_bytes_requirement 421 - lifetime_heap_bytes_requirement 422 - shared_bytes_requirement 423 + heap_bytes_requirement 421 } "2" { @@ -68,7 +66,7 @@ Object.Base.VendorToken { } "7" { - name "asrc" + name "asrc" rate_out 321 operation_mode 323 } diff --git a/tools/topology/topology2/include/components/widget-common.conf b/tools/topology/topology2/include/components/widget-common.conf index 43ac5691c778..14ccd8fcf86a 100644 --- a/tools/topology/topology2/include/components/widget-common.conf +++ b/tools/topology/topology2/include/components/widget-common.conf @@ -149,33 +149,12 @@ DefineAttribute."stack_bytes_requirement" { token_ref "comp.word" } -## Interim Heap size requirement in bytes for this component. -## Used for resources that may change in size or be freed during the lifetime of the component. -## In practice this means anything allocated outside module's init() call-back. -DefineAttribute."interim_heap_bytes_requirement" { - # Token set reference name and type - token_ref "comp.word" -} - -## Lifetime heap memory allocation requirement in bytes for this component. -## This token's value indicates the amount of allocated memory needed at component init phase -## and used over the lifetime of the component until the component is destroyed. -## In practice this means anything allocated in init() call-back. -DefineAttribute."lifetime_heap_bytes_requirement" { - # Token set reference name and type - token_ref "comp.word" -} - -## Shared memory allocation requirement in bytes for this component. -## This token's value indicates the amount of shared memory the component may need to allocated. -## Shared memory may be shared to other components. -DefineAttribute."shared_bytes_requirement" { +## Heap size requirement in bytes for this component. +DefineAttribute."heap_bytes_requirement" { # Token set reference name and type token_ref "comp.word" } # These default values are here until we have measured module specific numbers stack_bytes_requirement 8192 -interim_heap_bytes_requirement 4096 -lifetime_heap_bytes_requirement 16384 -shared_bytes_requirement 4096 \ No newline at end of file +heap_bytes_requirement 24576 diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 27f37d829b4e..603e52aafebd 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -69,6 +69,15 @@ config SOF_ZEPHYR_HEAP_SIZE NOTE: Keep in mind that the heap size should not be greater than the physical memory size of the system defined in DT (and this includes baseFW text/data). +config SOF_USERSPACE_DP_DEFAULT_HEAP_SIZE + int "Default heap size for DP userspace threads" + default 20480 + help + Defines the default heap size for userspace DP processing + threads. The value can be overridden with IPC module init + ext_init module payload. The default is derived from what is + required for SRC module to produce all supported conversions. + config SOF_USERSPACE_USE_SHARED_HEAP bool "Use shared heap for SOF userspace modules" depends on USERSPACE @@ -232,6 +241,15 @@ config ZEPHYR_DP_SCHEDULER DP modules can be located in dieffrent cores than LL pipeline modules, may have different tick (i.e. 300ms for speech reccognition, etc.) +config ZEPHYR_DP_SCHEDULER_MIN_STACK_SIZE + int "Minimum stack size for DP processing thread" + default 512 + help + Defines the minimum stack size allowed for DP processing + threads despite what is requested in the module init IPC + ext_init payload. If the stack size requested in the IPC is + smaller than this, then the value defined here takes over. + config CROSS_CORE_STREAM bool "Enable cross-core connected pipelines" default y if IPC_MAJOR_4 diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index 98deb489f91d..b727d1e700fb 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -189,10 +189,9 @@ static inline void *sof_ctx_alloc(struct mod_alloc_ctx *ctx, uint32_t flags, return sof_heap_alloc(ctx ? ctx->heap : NULL, flags, size, alignment); if (flags & SOF_MEM_FLAG_COHERENT) - return vregion_alloc_coherent_align(ctx->vreg, VREGION_MEM_TYPE_INTERIM, - size, alignment); + return vregion_alloc_coherent_align(ctx->vreg, size, alignment); - return vregion_alloc_align(ctx->vreg, VREGION_MEM_TYPE_INTERIM, size, alignment); + return vregion_alloc_align(ctx->vreg, size, alignment); } /** diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index 0477129fd57c..cb0a2cfb07c7 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -203,7 +203,7 @@ const void *fast_get(struct mod_alloc_ctx *alloc, const void *dram_ptr, size_t s if (alloc && alloc->vreg && size <= FAST_GET_MAX_COPY_SIZE) /* A userspace allocation, that won't be shared */ - ret = vregion_alloc_align(alloc->vreg, VREGION_MEM_TYPE_INTERIM, alloc_size, + ret = vregion_alloc_align(alloc->vreg, alloc_size, alloc_align); else ret = sof_heap_alloc(heap, alloc_flags, alloc_size, alloc_align); diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 15683925196d..a5a61df4538e 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -5,6 +5,7 @@ * Author: Liam Girdwood */ +#include #include #include #include @@ -87,7 +88,10 @@ struct vregion { struct k_mutex lock; /* protect vregion heaps and use-count */ unsigned int use_count; - /* interim heap */ + /* current allocation mode */ + enum vregion_mem_type type; /* LIFETIME at creation, switch to INTERIM */ + + /* interim heap - created lazily on first interim allocation */ struct interim_heap interim; /* interim heap */ /* lifetime heap */ @@ -97,30 +101,31 @@ struct vregion { /** * @brief Create a new virtual region instance. * - * Create a new VIRTUAL REGION instance with specified static and dynamic - * sizes. Total size is their sum. + * Create a new VIRTUAL REGION instance with specified memory size. + * Allocations start in LIFETIME mode. * - * @param[in] lifetime_size Size of the virtual region lifetime partition. - * @param[in] interim_size Size of the virtual region interim partition. + * @param[in] memsize Total size of the virtual region memory. * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. */ -struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) +struct vregion *vregion_create(size_t memsize) { struct vregion *vr; unsigned int pages; size_t total_size; uint8_t *vregion_base; - if (!lifetime_size || !interim_size) { - LOG_ERR("error: invalid vregion lifetime size %zu or interim size %zu", - lifetime_size, interim_size); + if (!memsize) { + LOG_ERR("error: invalid vregion memsize %zu", memsize); return NULL; } - /* Align up lifetime sizes and interim sizes to nearest page */ - lifetime_size = ALIGN_UP(lifetime_size, CONFIG_MM_DRV_PAGE_SIZE); - interim_size = ALIGN_UP(interim_size, CONFIG_MM_DRV_PAGE_SIZE); - total_size = lifetime_size + interim_size; + /* + * Align up the memory size to nearest page. Lifetime + * allocations growing upward from the base. The interim + * k_heap is created lazily from the remaining space when + * initialization phase is ready. + */ + total_size = ALIGN_UP(memsize, CONFIG_MM_DRV_PAGE_SIZE); /* allocate vregion metadata separately to keep it inaccessible to the user */ vr = rmalloc(0, sizeof(*vr)); @@ -140,31 +145,23 @@ struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) vr->size = total_size; vr->pages = pages; - /* set partition sizes */ - vr->interim.heap.heap.init_bytes = interim_size; - vr->lifetime.size = lifetime_size; - - /* set base addresses for partitions */ - vr->interim.heap.heap.init_mem = vr->base; - vr->lifetime.base = vr->base + interim_size; - - /* set alloc ptr addresses for lifetime linear partitions */ - vr->lifetime.ptr = vr->lifetime.base; + /* lifetime linear allocator starts at the beginning of the vregion memory */ + vr->lifetime.base = vregion_base; + vr->lifetime.size = total_size; + vr->lifetime.ptr = vregion_base; vr->lifetime.used = 0; vr->lifetime.free_count = 0; - /* init interim heaps */ - k_heap_init(&vr->interim.heap, vr->interim.heap.heap.init_mem, interim_size); + /* start in lifetime allocation mode */ + vr->type = VREGION_MEM_TYPE_LIFETIME; k_mutex_init(&vr->lock); /* The creator is the first user */ vr->use_count = 1; /* log the new vregion */ - LOG_INF("new at base %p size %#zx pages %u struct embedded at %p", + LOG_INF("new at base %p size %#zx pages %u metadata at %p", (void *)vr->base, total_size, pages, (void *)vr); - LOG_DBG(" interim size %#zx at %p", interim_size, (void *)vr->interim.heap.heap.init_mem); - LOG_DBG(" lifetime size %#zx at %p", lifetime_size, (void *)vr->lifetime.base); return vr; } @@ -212,6 +209,64 @@ struct vregion *vregion_put(struct vregion *vr) return NULL; } +/** + * @brief Initialize the interim heap from remaining vregion space. + * + * Called on first interim allocation. Creates the k_heap from all buffer + * space not yet consumed by lifetime allocations. + * + * @param[in] vr Pointer to the virtual region instance. + */ +static void interim_heap_init(struct vregion *vr) +{ + uint8_t *interim_base; + ssize_t interim_size; + + if (vr->type == VREGION_MEM_TYPE_INTERIM) { + LOG_WRN("vregion %p already in interim mode", (void *)vr); + return; + } + + /* interim heap starts right after current lifetime pointer, page-aligned */ + interim_base = UINT_TO_POINTER(ALIGN_UP(POINTER_TO_UINT(vr->lifetime.ptr), + CONFIG_DCACHE_LINE_SIZE)); + interim_size = (vr->base + vr->size) - interim_base; + /* + * Calling k_heap_init() with too small size causes an assert + * failure. Exact limit is hard to deduce from sys_heap_init() + * code, but 1024 seems to be enough. + */ + if (interim_size < 1024) { + LOG_WRN("Too little memory, %zd bytes, left for interim heap", interim_size); + vr->type = VREGION_MEM_TYPE_INVALID; + return; + } + + LOG_INF("creating interim heap: lifetime used %zu, interim available %zu at %p", + vr->lifetime.used, interim_size, (void *)interim_base); + + /* cap lifetime so no more lifetime allocs can grow into interim */ + vr->lifetime.size = interim_base - vr->base; + + vr->interim.heap.heap.init_mem = interim_base; + vr->interim.heap.heap.init_bytes = interim_size; + + k_heap_init(&vr->interim.heap, interim_base, interim_size); + vr->type = VREGION_MEM_TYPE_INTERIM; +} + +void vregion_set_interim(struct vregion *vr) +{ + if (!vr) + return; + + k_mutex_lock(&vr->lock, K_FOREVER); + + interim_heap_init(vr); + + k_mutex_unlock(&vr->lock); +} + /** * @brief Allocate memory with alignment from the virtual region dynamic heap. * @@ -220,8 +275,7 @@ struct vregion *vregion_put(struct vregion *vr) * @param[in] align Alignment of the allocation. * @return void* Pointer to the allocated memory, or NULL on failure. */ -static void *interim_alloc(struct interim_heap *heap, - size_t size, size_t align) +static void *interim_alloc(struct interim_heap *heap, size_t size, size_t align) { void *ptr; @@ -247,14 +301,16 @@ static void interim_free(struct interim_heap *heap, void *ptr) /** * @brief Allocate memory from the virtual region lifetime allocator. * + * If the interim heap has already been created (i.e., an interim allocation + * was made), log a warning and fall back to interim allocation. + * * @param[in] heap Pointer to the linear heap to use. * @param[in] size Size of the allocation. * @param[in] align Alignment of the allocation. * * @return void* Pointer to the allocated memory, or NULL on failure. */ -static void *lifetime_alloc(struct vlinear_heap *heap, - size_t size, size_t align) +static void *lifetime_alloc(struct vlinear_heap *heap, size_t size, size_t align) { void *ptr; uint8_t *aligned_ptr; @@ -315,14 +371,15 @@ void vregion_free(struct vregion *vr, void *ptr) if (sys_cache_is_ptr_uncached(ptr)) ptr = sys_cache_cached_ptr_get(ptr); - if (ptr >= (void *)vr->interim.heap.heap.init_mem && + /* Check if pointer is in interim heap (if initialized) */ + if (vr->type == VREGION_MEM_TYPE_INTERIM && + ptr >= (void *)vr->interim.heap.heap.init_mem && ptr < (void *)((uint8_t *)vr->interim.heap.heap.init_mem + vr->interim.heap.heap.init_bytes)) - /* pointer is in interim heap */ interim_free(&vr->interim, ptr); else if (ptr >= (void *)vr->lifetime.base && ptr < (void *)(vr->lifetime.base + vr->lifetime.size)) - /* pointer is in lifetime heap */ + /* pointer is in lifetime area - no-op free */ lifetime_free(&vr->lifetime, ptr); else LOG_ERR("error: vregion free invalid pointer %p", ptr); @@ -332,17 +389,15 @@ void vregion_free(struct vregion *vr, void *ptr) EXPORT_SYMBOL(vregion_free); /** - * @brief Allocate memory type from the virtual region. + * @brief Allocate memory from the virtual region. * * @param[in] vr Pointer to the virtual region instance. - * @param[in] type Memory type to allocate. * @param[in] size Size of the allocation. * @param[in] alignment Alignment of the allocation. * * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, - size_t size, size_t alignment) +void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) { void *p; @@ -354,7 +409,7 @@ void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, k_mutex_lock(&vr->lock, K_FOREVER); - switch (type) { + switch (vr->type) { case VREGION_MEM_TYPE_INTERIM: p = interim_alloc(&vr->interim, size, alignment); break; @@ -362,7 +417,7 @@ void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, p = lifetime_alloc(&vr->lifetime, size, alignment); break; default: - LOG_ERR("error: invalid memory type %d", type); + LOG_ERR("error: invalid memory type %d", vr->type); p = NULL; } @@ -375,21 +430,20 @@ EXPORT_SYMBOL(vregion_alloc_align); /** * @brief Allocate memory from the virtual region. * @param[in] vr Pointer to the virtual region instance. - * @param[in] type Memory type to allocate. * @param[in] size Size of the allocation. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) +void *vregion_alloc(struct vregion *vr, size_t size) { - return vregion_alloc_align(vr, type, size, 0); + return vregion_alloc_align(vr, size, 0); } EXPORT_SYMBOL(vregion_alloc); -void *vregion_alloc_coherent(struct vregion *vr, enum vregion_mem_type type, size_t size) +void *vregion_alloc_coherent(struct vregion *vr, size_t size) { size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, type, size, CONFIG_DCACHE_LINE_SIZE); + void *p = vregion_alloc_align(vr, size, CONFIG_DCACHE_LINE_SIZE); if (!p) return NULL; @@ -399,14 +453,13 @@ void *vregion_alloc_coherent(struct vregion *vr, enum vregion_mem_type type, siz return sys_cache_uncached_ptr_get(p); } -void *vregion_alloc_coherent_align(struct vregion *vr, enum vregion_mem_type type, - size_t size, size_t alignment) +void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) { if (alignment < CONFIG_DCACHE_LINE_SIZE) alignment = CONFIG_DCACHE_LINE_SIZE; size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, type, size, alignment); + void *p = vregion_alloc_align(vr, size, alignment); if (!p) return NULL; diff --git a/zephyr/test/vregion.c b/zephyr/test/vregion.c index eb59f68a14d7..eb36721d582e 100644 --- a/zephyr/test/vregion.c +++ b/zephyr/test/vregion.c @@ -17,8 +17,11 @@ LOG_MODULE_DECLARE(sof_boot_test, CONFIG_SOF_LOG_LEVEL); static struct vregion *test_vreg_create(void) { - struct vregion *vreg = vregion_create(CONFIG_MM_DRV_PAGE_SIZE - 100, - CONFIG_MM_DRV_PAGE_SIZE); + /* + * Use a 3-page vregion so that after two lifetime allocations there is still + * enough remaining space to initialize and exercise the interim heap. + */ + struct vregion *vreg = vregion_create(3 * CONFIG_MM_DRV_PAGE_SIZE); zassert_not_null(vreg); @@ -27,40 +30,46 @@ static struct vregion *test_vreg_create(void) static void test_vreg_alloc_lifet(struct vregion *vreg) { - void *ptr = vregion_alloc(vreg, VREGION_MEM_TYPE_LIFETIME, 2000); + void *ptr = vregion_alloc(vreg, 6000); zassert_not_null(ptr); - void *ptr_align = vregion_alloc_align(vreg, VREGION_MEM_TYPE_LIFETIME, 1600, 16); + void *ptr_align = vregion_alloc_align(vreg, 5000, 16); zassert_not_null(ptr_align); zassert_equal((uintptr_t)ptr_align & 15, 0); - void *ptr_nomem = vregion_alloc(vreg, VREGION_MEM_TYPE_LIFETIME, 2000); + /* + * Seal lifetime, switch to interim. The interim heap is created + * lazily from the remaining ~1 page, so a 6000-byte alloc won't fit. + */ + vregion_set_interim(vreg); + + void *ptr_nomem = vregion_alloc(vreg, 6000); zassert_is_null(ptr_nomem); + /* Lifetime frees are no-ops; re-alloc from interim still fails */ vregion_free(vreg, ptr_align); vregion_free(vreg, ptr); - /* Freeing isn't possible with LIFETIME */ - ptr_nomem = vregion_alloc(vreg, VREGION_MEM_TYPE_LIFETIME, 2000); + ptr_nomem = vregion_alloc(vreg, 6000); zassert_is_null(ptr_nomem); } static void test_vreg_alloc_tmp(struct vregion *vreg) { - void *ptr = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, 20); + void *ptr = vregion_alloc(vreg, 20); zassert_not_null(ptr); - void *ptr_align = vregion_alloc_align(vreg, VREGION_MEM_TYPE_INTERIM, 2000, 16); + void *ptr_align = vregion_alloc_align(vreg, 1000, 16); zassert_not_null(ptr_align); zassert_equal((uintptr_t)ptr_align & 15, 0); - void *ptr_nomem = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, 2000); + void *ptr_nomem = vregion_alloc(vreg, 2000); zassert_is_null(ptr_nomem); @@ -68,7 +77,7 @@ static void test_vreg_alloc_tmp(struct vregion *vreg) vregion_free(vreg, ptr); /* Should be possible to allocate again */ - ptr = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, 2000); + ptr = vregion_alloc(vreg, 1000); zassert_not_null(ptr); } @@ -83,10 +92,10 @@ ZTEST(sof_boot, vregion) { struct vregion *vreg = test_vreg_create(); - /* Test interim allocations */ - test_vreg_alloc_tmp(vreg); - /* Test lifetime allocations */ + /* Test lifetime allocations (initial mode), then seal */ test_vreg_alloc_lifet(vreg); + /* Test interim allocations (already switched by lifet test) */ + test_vreg_alloc_tmp(vreg); test_vreg_destroy(vreg); }