diff --git a/include/kernel.h b/include/kernel.h index a6bfbe065e2..ff695d38ed9 100644 --- a/include/kernel.h +++ b/include/kernel.h @@ -129,6 +129,7 @@ struct k_poll_event; struct k_poll_signal; struct k_mem_domain; struct k_mem_partition; +struct k_futex; /* This enumeration needs to be kept in sync with the lists of kernel objects * and subsystems in scripts/gen_kobject_list.py, as well as the otype_to_str() @@ -2088,6 +2089,94 @@ static inline void *z_impl_k_queue_peek_tail(struct k_queue *queue) /** @} */ +#ifdef CONFIG_USERSPACE +/** + * @brief futex structure + * + * A k_futex is a lightweight mutual exclusion primitive designed + * to minimize kernel involvement. Uncontended operation relies + * only on atomic access to shared memory. k_futex are tracked as + * kernel objects and can live in user memory so any access bypass + * the kernel object permission management mechanism. + */ +struct k_futex { + atomic_t val; +}; + +/** + * @brief futex kernel data structure + * + * z_futex_data are the helper data structure for k_futex to complete + * futex contended operation on kernel side, structure z_futex_data + * of every futex object is invisible in user mode. + */ +struct z_futex_data { + _wait_q_t wait_q; + struct k_spinlock lock; +}; + +#define Z_FUTEX_DATA_INITIALIZER(obj) \ + { \ + .wait_q = Z_WAIT_Q_INIT(&obj.wait_q) \ + } + +/** + * @defgroup futex_apis FUTEX APIs + * @ingroup kernel_apis + * @{ + */ + +/** + * @brief Initialize a futex. + * + * This routine initializes a futex object, prior to its first use. + * + * @param futex Address of the k_futex. + * + * @return N/A + */ +__syscall void k_futex_init(struct k_futex *futex); + +/** + * @brief Pend the current thread on a futex + * + * Tests that the supplied futex contains the expected value, and if so, + * goes to sleep until some other thread calls k_futex_wake() on it. + * + * @param futex Address of the futex. + * @param expected Expected value of the futex, if it is different the caller + * will not wait on it. + * @param timeout Waiting period on the futex, in milliseconds, or one of the + * special values K_NO_WAIT or K_FOREVER. + * @retval -EACCES Caller does not have read access to futex address. + * @retval -EAGAIN If the futex value did not match the expected parameter. + * @retval -EINVAL Futex parameter address not recognized by the kernel. + * @retval -ETIMEDOUT Thread woke up due to timeout and not a futex wakeup. + * @retval 0 if the caller went to sleep and was woken up. The caller + * should check the futex's value on wakeup to determine if it needs + * to block again. + */ +__syscall int k_futex_wait(struct k_futex *futex, int expected, s32_t timeout); + +/** + * @brief Wake one/all threads pending on a futex + * + * Wake up the highest priority thread pending on the supplied futex, or + * wakeup all the threads pending on the supplied futex, and the behavior + * depends on wake_all. + * + * @param futex Futex to wake up pending threads. + * @param wake_all If true, wake up all pending threads; If false, + * wakeup the highest priority thread. + * @retval -EACCES Caller does not have access to the futex address. + * @retval -EINVAL Futex parameter address not recognized by the kernel. + * @retval Number of threads that were woken up. + */ +__syscall int k_futex_wake(struct k_futex *futex, bool wake_all); + +/** @} */ +#endif + struct k_fifo { struct k_queue _queue; }; diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 37a27ba591b..72d848be961 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -46,6 +46,7 @@ target_sources_if_kconfig( kernel PRIVATE poll.c) target_sources_ifdef( CONFIG_USERSPACE kernel PRIVATE + futex.c mem_domain.c userspace_handler.c userspace.c diff --git a/kernel/futex.c b/kernel/futex.c new file mode 100644 index 00000000000..19f958f2784 --- /dev/null +++ b/kernel/futex.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2019 Intel corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include + +static struct z_futex_data *k_futex_find_data(struct k_futex *futex) +{ + struct _k_object *obj; + + obj = z_object_find(futex); + if (obj == NULL || obj->type != K_OBJ_FUTEX) { + return NULL; + } + + return (struct z_futex_data *)obj->data; +} + +void z_impl_k_futex_init(struct k_futex *futex) +{ + futex->val = 0U; + z_object_init(futex); +} + +Z_SYSCALL_HANDLER(k_futex_init, futex) +{ + if (Z_SYSCALL_MEMORY_WRITE(futex, sizeof(struct k_futex)) != 0) { + return -EACCES; + } + + z_impl_k_futex_init((struct k_futex *)futex); + + return 0; +} + +int z_impl_k_futex_wake(struct k_futex *futex, bool wake_all) +{ + k_spinlock_key_t key; + unsigned int woken = 0; + struct k_thread *thread; + struct z_futex_data *futex_data; + + futex_data = k_futex_find_data(futex); + if (futex_data == NULL) { + return -EINVAL; + } + + key = k_spin_lock(&futex_data->lock); + + do { + thread = z_unpend_first_thread(&futex_data->wait_q); + if (thread) { + z_ready_thread(thread); + z_set_thread_return_value(thread, 0); + woken++; + } + } while (thread && wake_all); + + z_reschedule(&futex_data->lock, key); + + return woken; +} + +Z_SYSCALL_HANDLER(k_futex_wake, futex, wake_all) +{ + if (Z_SYSCALL_MEMORY_WRITE(futex, sizeof(struct k_futex)) != 0) { + return -EACCES; + } + + return z_impl_k_futex_wake((struct k_futex *)futex, (bool)wake_all); +} + +int z_impl_k_futex_wait(struct k_futex *futex, int expected, s32_t timeout) +{ + int ret; + k_spinlock_key_t key; + struct z_futex_data *futex_data; + + futex_data = k_futex_find_data(futex); + if (futex_data == NULL) { + return -EINVAL; + } + + key = k_spin_lock(&futex_data->lock); + + if (atomic_get(&futex->val) != (atomic_val_t)expected) { + k_spin_unlock(&futex_data->lock, key); + return -EAGAIN; + } + + ret = z_pend_curr(&futex_data->lock, + key, &futex_data->wait_q, timeout); + if (ret == -EAGAIN) { + ret = -ETIMEDOUT; + } + + return ret; +} + +Z_SYSCALL_HANDLER(k_futex_wait, futex, expected, timeout) +{ + if (Z_SYSCALL_MEMORY_WRITE(futex, sizeof(struct k_futex)) != 0) { + return -EACCES; + } + + return z_impl_k_futex_wait((struct k_futex *)futex, + expected, (s32_t)timeout); +} diff --git a/scripts/elf_helper.py b/scripts/elf_helper.py index 005d9a21c72..6ed600d3261 100644 --- a/scripts/elf_helper.py +++ b/scripts/elf_helper.py @@ -38,6 +38,7 @@ DW_OP_fbreg = 0x91 STACK_TYPE = "_k_thread_stack_element" thread_counter = 0 sys_mutex_counter = 0 +futex_counter = 0 # Global type environment. Populated by pass 1. type_env = {} @@ -56,6 +57,7 @@ class KobjectInstance: def __init__(self, type_obj, addr): global thread_counter global sys_mutex_counter + global futex_counter self.addr = addr self.type_obj = type_obj @@ -72,6 +74,9 @@ class KobjectInstance: elif self.type_obj.name == "sys_mutex": self.data = "(u32_t)(&kernel_mutexes[%d])" % sys_mutex_counter sys_mutex_counter += 1 + elif self.type_obj.name == "k_futex": + self.data = "(u32_t)(&futex_data[%d])" % futex_counter + futex_counter += 1 else: self.data = 0 @@ -566,3 +571,6 @@ class ElfHelper: def get_sys_mutex_counter(self): return sys_mutex_counter + + def get_futex_counter(self): + return futex_counter diff --git a/scripts/gen_kobject_list.py b/scripts/gen_kobject_list.py index c5559407df4..d4aa54d7915 100755 --- a/scripts/gen_kobject_list.py +++ b/scripts/gen_kobject_list.py @@ -86,7 +86,8 @@ kobjects = OrderedDict ([ ("k_timer", (None, False)), ("_k_thread_stack_element", (None, False)), ("device", (None, False)), - ("sys_mutex", (None, True)) + ("sys_mutex", (None, True)), + ("k_futex", (None, True)) ]) @@ -170,6 +171,15 @@ def write_gperf_table(fp, eh, objs, static_begin, static_end): fp.write(", ") fp.write("};\n") + num_futex = eh.get_futex_counter() + if (num_futex != 0): + fp.write("static struct z_futex_data futex_data[%d] = {\n" % num_futex) + for i in range(num_futex): + fp.write("Z_FUTEX_DATA_INITIALIZER(futex_data[%d])" % i) + if (i != num_futex - 1): + fp.write(", ") + fp.write("};\n") + fp.write("%%\n") # Setup variables for mapping thread indexes syms = eh.get_symbols()