[DNM][WiP][PoC] userspace LL scheduling: LLEXT & multicore

app/overlays/ptl/ll_userspace_overlay.conf

@@ -24,15 +24,12 @@ CONFIG_COLD_STORE_EXECUTE_DEBUG=n
 CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n
 CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n
 
-# disable loadable modules (hits privilege issues in user-space now)
-CONFIG_LLEXT_STORAGE_WRITABLE=n
-CONFIG_LLEXT_EXPERIMENTAL=n
-CONFIG_MODULES=n
-
 # some of current boot tests interfere with user-space setup
 CONFIG_SOF_BOOT_TEST_ALLOWED=n
 
 # misc features not yet compatible with user-space
 CONFIG_CROSS_CORE_STREAM=n
 CONFIG_INTEL_ADSP_MIC_PRIVACY=n
 CONFIG_XRUN_NOTIFICATIONS_ENABLE=n
+
+CONFIG_MAX_THREAD_BYTES=4

posix/include/rtos/mutex.h

@@ -62,4 +62,25 @@ static inline int sys_mutex_unlock(struct sys_mutex *mutex)
 	return 0;
 }
 
+/* provide a no-op implementation for zephyr/sys/sem.h */
+
+struct sys_sem {
+};
+
+static inline int sys_sem_init(struct sys_sem *sem, unsigned int initial_count,
+			       unsigned int limit)
+{
+	return 0;
+}
+
+static inline int sys_sem_give(struct sys_sem *sem)
+{
+	return 0;
+}
+
+static inline int sys_sem_take(struct sys_sem *sem, k_timeout_t timeout)
+{
+	return 0;
+}
+
 #endif

src/audio/Kconfig

@@ -42,6 +42,14 @@ config HOST_DMA_STREAM_SYNCHRONIZATION
 	  for each group, different than the default one determined by the system tick frequency.
 	  This feature will allow host lower power consumption in scenarios with deep buffering.
 
+config HOST_DMA_IPC_POSITION_UPDATES
+	bool "Support for stream position updates via IPC messages"
+	default y if IPC_MAJOR_3
+	help
+	  Support firmware functionality to report stream position updates
+	  by sending a IPC message whenever one period of audio is transfferred.
+	  Most platforms provide more efficient ways to query the DMA status.
+
 config COMP_CHAIN_DMA
 	  bool "Chain DMA component"
 	  depends on IPC_MAJOR_4

src/audio/chain_dma.c

@@ -15,7 +15,9 @@
 #include <ipc/dai.h>
 #include <ipc4/gateway.h>
 #include <sof/schedule/ll_schedule.h>
+#include <sof/schedule/ll_schedule_domain.h>
 #include <sof/schedule/schedule.h>
+#include <rtos/alloc.h>
 #include <rtos/task.h>
 #include <sof/lib/dma.h>
 #include <sof/lib/memory.h>
@@ -59,6 +61,15 @@ struct chain_dma_data {
 	bool xrun_notification_sent;
 #endif
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	/** Kernel workqueue scheduling for chain DMA in userspace builds.
+	 *  Chain DMA needs kernel context for DMA operations, so it cannot
+	 *  run on the user-space LL timer thread.
+	 */
+	struct k_work_delayable dma_work;
+	bool stopped;
+#endif
+
 	/* local host DMA config */
 	struct sof_dma *dma_host;
 	struct dma_chan_data *chan_host;
@@ -269,6 +280,25 @@ static enum task_state chain_task_run(void *data)
 	return SOF_TASK_STATE_RESCHEDULE;
 }
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+/** Kernel workqueue handler for chain DMA periodic task.
+ *  Runs chain_task_run() in kernel context and reschedules if needed.
+ */
+static void chain_dma_work_handler(struct k_work *work)
+{
+	struct k_work_delayable *dwork = k_work_delayable_from_work(work);
+	struct chain_dma_data *cd = CONTAINER_OF(dwork, struct chain_dma_data, dma_work);
+	enum task_state state;
+
+	if (cd->stopped)
+		return;
+
+	state = chain_task_run(cd);
+	if (state == SOF_TASK_STATE_RESCHEDULE && !cd->stopped)
+		k_work_reschedule(dwork, K_USEC(LL_TIMER_PERIOD_US));
+}
+#endif
+
 static int chain_task_start(struct comp_dev *dev)
 {
 	struct chain_dma_data *cd = comp_get_drvdata(dev);
@@ -310,6 +340,12 @@ static int chain_task_start(struct comp_dev *dev)
 		}
 	}
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	cd->stopped = false;
+	k_work_init_delayable(&cd->dma_work, chain_dma_work_handler);
+	k_work_reschedule(&cd->dma_work, K_NO_WAIT);
+	cd->chain_task.state = SOF_TASK_STATE_QUEUED;
+#else
 	ret = schedule_task_init_ll(&cd->chain_task, SOF_UUID(chain_dma_uuid),
 				    SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_HIGH,
 				    chain_task_run, cd, 0, 0);
@@ -324,27 +360,38 @@ static int chain_task_start(struct comp_dev *dev)
 		schedule_task_free(&cd->chain_task);
 		goto error_task;
 	}
+#endif
 
 	pm_policy_state_lock_get(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
 
 	return 0;
 
+#ifndef CONFIG_SOF_USERSPACE_LL
 error_task:
 	chain_host_stop(dev);
 	chain_link_stop(dev);
 
 	return ret;
+#endif
 }
 
 static int chain_task_pause(struct comp_dev *dev)
 {
 	struct chain_dma_data *cd = comp_get_drvdata(dev);
 	int ret, ret2;
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	if (cd->chain_task.state == SOF_TASK_STATE_FREE)
+		return 0;
+
+	cd->stopped = true;
+	cd->first_data_received = false;
+#else
 	if (cd->chain_task.state == SOF_TASK_STATE_FREE)
 		return 0;
 
 	cd->first_data_received = false;
+#endif
 	if (cd->stream_direction == SOF_IPC_STREAM_PLAYBACK) {
 		ret = chain_host_stop(dev);
 		ret2 = chain_link_stop(dev);
@@ -355,7 +402,12 @@ static int chain_task_pause(struct comp_dev *dev)
 	if (!ret)
 		ret = ret2;
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	k_work_cancel_delayable_sync(&cd->dma_work, &(struct k_work_sync){});
+	cd->chain_task.state = SOF_TASK_STATE_FREE;
+#else
 	schedule_task_free(&cd->chain_task);
+#endif
 	pm_policy_state_lock_put(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
 
 	return ret;
@@ -504,6 +556,7 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin
 				  uint32_t fifo_size)
 {
 	struct chain_dma_data *cd = comp_get_drvdata(dev);
+	struct mod_alloc_ctx *alloc_ctx = NULL;
 	uint32_t addr_align;
 	size_t buff_size;
 	void *buff_addr;
@@ -582,8 +635,14 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin
 	}
 
 	fifo_size = ALIGN_UP_INTERNAL(fifo_size, addr_align);
+
+#ifdef CONFIG_SOF_USERSPACE_LL
+	alloc_ctx = ipc_get()->ll_alloc;
+#endif
+
 	/* allocate not shared buffer */
-	cd->dma_buffer = buffer_alloc(NULL, fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA,
+	cd->dma_buffer = buffer_alloc(alloc_ctx, fifo_size,
+				      SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA,
 				      addr_align, BUFFER_USAGE_NOT_SHARED);
 
 	if (!cd->dma_buffer) {
@@ -643,14 +702,31 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv,
 	if (host_dma_id >= max_chain_number)
 		return NULL;
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	dev = sof_heap_alloc(sof_sys_user_heap_get(),
+			     SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
+			     sizeof(*dev), 0);
+	if (!dev)
+		return NULL;
+
+	memset(dev, 0, sizeof(*dev));
+	comp_init(drv, dev, sizeof(*dev));
+#else
 	dev = comp_alloc(drv, sizeof(*dev));
 	if (!dev)
 		return NULL;
+#endif
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	cd = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER, sizeof(*cd), 0);
+#else
 	cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd));
+#endif
 	if (!cd)
 		goto error;
 
+	memset(cd, 0, sizeof(*cd));
+
 	cd->first_data_received = false;
 	cd->cs = scs ? 2 : 4;
 	cd->chain_task.state = SOF_TASK_STATE_INIT;
@@ -661,9 +737,17 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv,
 	if (!ret)
 		return dev;
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(sof_sys_user_heap_get(), cd);
+#else
 	rfree(cd);
+#endif
 error:
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(sof_sys_user_heap_get(), dev);
+#else
 	comp_free_device(dev);
+#endif
 	return NULL;
 }
 
@@ -674,8 +758,16 @@ __cold static void chain_task_free(struct comp_dev *dev)
 	assert_can_be_cold();
 
 	chain_release(dev);
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(sof_sys_user_heap_get(), cd);
+#else
 	rfree(cd);
+#endif
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(sof_sys_user_heap_get(), dev);
+#else
 	comp_free_device(dev);
+#endif
 }
 
 static const struct comp_driver comp_chain_dma = {

src/audio/component.c

@@ -38,30 +38,39 @@ LOG_MODULE_REGISTER(component, CONFIG_SOF_LOG_LEVEL);
 
 static APP_SYSUSER_BSS SHARED_DATA struct comp_driver_list cd;
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+struct comp_driver_list *comp_drivers_get(void)
+{
+	return platform_shared_get(&cd, sizeof(cd));
+}
+EXPORT_SYMBOL(comp_drivers_get);
+#endif
+
 SOF_DEFINE_REG_UUID(component);
 
 DECLARE_TR_CTX(comp_tr, SOF_UUID(component_uuid), LOG_LEVEL_INFO);
 
 int comp_register(struct comp_driver_info *drv)
 {
 	struct comp_driver_list *drivers = comp_drivers_get();
-	k_spinlock_key_t key;
 
-	key = k_spin_lock(&drivers->lock);
+	/*
+	 * No locking needed: the driver list is only modified at FW boot,
+	 * where module init runs serially on the primary core, and at
+	 * runtime from the serialized IPC thread (library load). These
+	 * never overlap, so concurrent modification is not possible.
+	 */
 	list_item_prepend(&drv->list, &drivers->list);
-	k_spin_unlock(&drivers->lock, key);
 
 	return 0;
 }
 
 void comp_unregister(struct comp_driver_info *drv)
 {
 	struct comp_driver_list *drivers = comp_drivers_get();
-	k_spinlock_key_t key;
 
-	key = k_spin_lock(&drivers->lock);
+	/* see comp_register() on why no locking is needed */
 	list_item_del(&drv->list);
-	k_spin_unlock(&drivers->lock, key);
 }
 
 int comp_set_adapter_ops(const struct comp_driver *drv, const struct module_interface *ops)
@@ -190,7 +199,6 @@ void sys_comp_init(struct sof *sof)
 	sof->comp_drivers = platform_shared_get(&cd, sizeof(cd));
 
 	list_init(&sof->comp_drivers->list);
-	k_spinlock_init(&sof->comp_drivers->lock);
 }
 
 void comp_get_copy_limits(struct comp_buffer *source,

src/audio/copier/copier.c

@@ -13,6 +13,7 @@
 #include <rtos/interrupt.h>
 #include <sof/ipc/msg.h>
 #include <sof/ipc/topology.h>
+#include <sof/schedule/ll_schedule_domain.h>
 #include <rtos/interrupt.h>
 #include <rtos/timer.h>
 #include <rtos/cache.h>
@@ -824,7 +825,9 @@ __cold static int set_chmap(struct comp_dev *dev, const void *data, size_t data_
 	pcm_converter_func process;
 	pcm_converter_func converters[IPC4_COPIER_MODULE_OUTPUT_PINS_COUNT];
 	int i;
+#ifndef CONFIG_SOF_USERSPACE_LL
 	uint32_t irq_flags;
+#endif
 
 	assert_can_be_cold();
 
@@ -878,15 +881,26 @@ __cold static int set_chmap(struct comp_dev *dev, const void *data, size_t data_
 		}
 	}
 
-	/* Atomically update chmap, process and converters */
+	/* Atomically update chmap, process and converters.
+	 * In user-space builds irq_local_disable() is privileged,
+	 * use the LL scheduler lock instead.
+	 */
+#ifdef CONFIG_SOF_USERSPACE_LL
+	user_ll_lock_sched(dev->pipeline->core);
+#else
 	irq_local_disable(irq_flags);
+#endif
 
 	cd->dd[0]->chmap = chmap_cfg->channel_map;
 	cd->dd[0]->process = process;
 	for (i = 0; i < IPC4_COPIER_MODULE_OUTPUT_PINS_COUNT; i++)
 		cd->converter[i] = converters[i];
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	user_ll_unlock_sched(dev->pipeline->core);
+#else
 	irq_local_enable(irq_flags);
+#endif
 
 	return 0;
 }

src/audio/copier/host_copier.h

@@ -106,7 +106,9 @@ struct host_data {
 
 	/* stream info */
 	struct sof_ipc_stream_posn posn; /* TODO: update this */
+#if CONFIG_HOST_DMA_IPC_POSITION_UPDATES
 	struct ipc_msg *msg;	/**< host notification */
+#endif
 #if CONFIG_XRUN_NOTIFICATIONS_ENABLE
 	bool xrun_notification_sent;
 #endif