kernel: add futex support

A k_futex is a lightweight mutual exclusion primitive designed
to minimize kernel involvement. Uncontended operation relies
only on atomic access to shared memory. k_futex structure lives
in application memory. And when using futexes, the majority of
the synchronization operations are performed in user mode. A
user-mode thread employs the futex wait system call only when
it is likely that the program has to block for a longer time
until the condition becomes true. When the condition comes true,
futex wake operation will be used to wake up one or more threads
waiting on that futex.

This patch implements two futex operations: k_futex_wait and
k_futex_wake. For k_futex_wait, the comparison with the expected
value, and starting to sleep are performed atomically to prevent
lost wake-ups. If different context changed futex's value after
the calling use-mode thread decided to block himself based on
the old value, the comparison will help observing the value
change and will not start to sleep. And for k_futex_wake, it
will wake at most num_waiters of the waiters that are sleeping
on that futex. But no guarantees are made on which threads are
woken, that means scheduling priority is not taken into
consideration.

Fixes: #14493.

Signed-off-by: Wentong Wu <wentong.wu@intel.com>

include/kernel.h

@@ -129,6 +129,7 @@ struct k_poll_event;
 struct k_poll_signal;
 struct k_mem_domain;
 struct k_mem_partition;
+struct k_futex;
 
 /* This enumeration needs to be kept in sync with the lists of kernel objects
  * and subsystems in scripts/gen_kobject_list.py, as well as the otype_to_str()
@@ -2088,6 +2089,94 @@ static inline void *z_impl_k_queue_peek_tail(struct k_queue *queue)
 
 /** @} */
 
+#ifdef CONFIG_USERSPACE
+/**
+ * @brief futex structure
+ *
+ * A k_futex is a lightweight mutual exclusion primitive designed
+ * to minimize kernel involvement. Uncontended operation relies
+ * only on atomic access to shared memory. k_futex are tracked as
+ * kernel objects and can live in user memory so any access bypass
+ * the kernel object permission management mechanism.
+ */
+struct k_futex {
+	atomic_t val;
+};
+
+/**
+ * @brief futex kernel data structure
+ *
+ * z_futex_data are the helper data structure for k_futex to complete
+ * futex contended operation on kernel side, structure z_futex_data
+ * of every futex object is invisible in user mode.
+ */
+struct z_futex_data {
+	_wait_q_t wait_q;
+	struct k_spinlock lock;
+};
+
+#define Z_FUTEX_DATA_INITIALIZER(obj) \
+	{ \
+	.wait_q = Z_WAIT_Q_INIT(&obj.wait_q) \
+	}
+
+/**
+ * @defgroup futex_apis FUTEX APIs
+ * @ingroup kernel_apis
+ * @{
+ */
+
+/**
+ * @brief Initialize a futex.
+ *
+ * This routine initializes a futex object, prior to its first use.
+ *
+ * @param futex Address of the k_futex.
+ *
+ * @return N/A
+ */
+__syscall void k_futex_init(struct k_futex *futex);
+
+/**
+ * @brief Pend the current thread on a futex
+ *
+ * Tests that the supplied futex contains the expected value, and if so,
+ * goes to sleep until some other thread calls k_futex_wake() on it.
+ *
+ * @param futex Address of the futex.
+ * @param expected Expected value of the futex, if it is different the caller
+ *		   will not wait on it.
+ * @param timeout Waiting period on the futex, in milliseconds, or one of the
+ *		  special values K_NO_WAIT or K_FOREVER.
+ * @retval -EACCES Caller does not have read access to futex address.
+ * @retval -EAGAIN If the futex value did not match the expected parameter.
+ * @retval -EINVAL Futex parameter address not recognized by the kernel.
+ * @retval -ETIMEDOUT Thread woke up due to timeout and not a futex wakeup.
+ * @retval 0 if the caller went to sleep and was woken up. The caller
+ *	     should check the futex's value on wakeup to determine if it needs
+ *	     to block again.
+ */
+__syscall int k_futex_wait(struct k_futex *futex, int expected, s32_t timeout);
+
+/**
+ * @brief Wake one/all threads pending on a futex
+ *
+ * Wake up the highest priority thread pending on the supplied futex, or
+ * wakeup all the threads pending on the supplied futex, and the behavior
+ * depends on wake_all.
+ *
+ * @param futex Futex to wake up pending threads.
+ * @param wake_all If true, wake up all pending threads; If false,
+ *                 wakeup the highest priority thread.
+ * @retval -EACCES Caller does not have access to the futex address.
+ * @retval -EINVAL Futex parameter address not recognized by the kernel.
+ * @retval Number of threads that were woken up.
+ */
+__syscall int k_futex_wake(struct k_futex *futex, bool wake_all);
+
+/** @} */
+#endif
+
 struct k_fifo {
 	struct k_queue _queue;
 };

kernel/CMakeLists.txt

@@ -46,6 +46,7 @@ target_sources_if_kconfig(                        kernel PRIVATE poll.c)
 target_sources_ifdef(
   CONFIG_USERSPACE
   kernel PRIVATE
+  futex.c
   mem_domain.c
   userspace_handler.c
   userspace.c

kernel/futex.c

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2019 Intel corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <kernel.h>
+#include <kernel_structs.h>
+#include <spinlock.h>
+#include <kswap.h>
+#include <syscall_handler.h>
+#include <init.h>
+#include <ksched.h>
+
+static struct z_futex_data *k_futex_find_data(struct k_futex *futex)
+{
+	struct _k_object *obj;
+
+	obj = z_object_find(futex);
+	if (obj == NULL || obj->type != K_OBJ_FUTEX) {
+		return NULL;
+	}
+
+	return (struct z_futex_data *)obj->data;
+}
+
+void z_impl_k_futex_init(struct k_futex *futex)
+{
+	futex->val = 0U;
+	z_object_init(futex);
+}
+
+Z_SYSCALL_HANDLER(k_futex_init, futex)
+{
+	if (Z_SYSCALL_MEMORY_WRITE(futex, sizeof(struct k_futex)) != 0) {
+		return -EACCES;
+	}
+
+	z_impl_k_futex_init((struct k_futex *)futex);
+
+	return 0;
+}
+
+int z_impl_k_futex_wake(struct k_futex *futex, bool wake_all)
+{
+	k_spinlock_key_t key;
+	unsigned int woken = 0;
+	struct k_thread *thread;
+	struct z_futex_data *futex_data;
+
+	futex_data = k_futex_find_data(futex);
+	if (futex_data == NULL) {
+		return -EINVAL;
+	}
+
+	key = k_spin_lock(&futex_data->lock);
+
+	do {
+		thread = z_unpend_first_thread(&futex_data->wait_q);
+		if (thread) {
+			z_ready_thread(thread);
+			z_set_thread_return_value(thread, 0);
+			woken++;
+		}
+	} while (thread && wake_all);
+
+	z_reschedule(&futex_data->lock, key);
+
+	return woken;
+}
+
+Z_SYSCALL_HANDLER(k_futex_wake, futex, wake_all)
+{
+	if (Z_SYSCALL_MEMORY_WRITE(futex, sizeof(struct k_futex)) != 0) {
+		return -EACCES;
+	}
+
+	return z_impl_k_futex_wake((struct k_futex *)futex, (bool)wake_all);
+}
+
+int z_impl_k_futex_wait(struct k_futex *futex, int expected, s32_t timeout)
+{
+	int ret;
+	k_spinlock_key_t key;
+	struct z_futex_data *futex_data;
+
+	futex_data = k_futex_find_data(futex);
+	if (futex_data == NULL) {
+		return -EINVAL;
+	}
+
+	key = k_spin_lock(&futex_data->lock);
+
+	if (atomic_get(&futex->val) != (atomic_val_t)expected) {
+		k_spin_unlock(&futex_data->lock, key);
+		return -EAGAIN;
+	}
+
+	ret = z_pend_curr(&futex_data->lock,
+			key, &futex_data->wait_q, timeout);
+	if (ret == -EAGAIN) {
+		ret = -ETIMEDOUT;
+	}
+
+	return ret;
+}
+
+Z_SYSCALL_HANDLER(k_futex_wait, futex, expected, timeout)
+{
+	if (Z_SYSCALL_MEMORY_WRITE(futex, sizeof(struct k_futex)) != 0) {
+		return -EACCES;
+	}
+
+	return z_impl_k_futex_wait((struct k_futex *)futex,
+			expected, (s32_t)timeout);
+}

scripts/elf_helper.py

@@ -38,6 +38,7 @@ DW_OP_fbreg = 0x91
 STACK_TYPE = "_k_thread_stack_element"
 thread_counter = 0
 sys_mutex_counter = 0
+futex_counter = 0
 
 # Global type environment. Populated by pass 1.
 type_env = {}
@@ -56,6 +57,7 @@ class KobjectInstance:
     def __init__(self, type_obj, addr):
         global thread_counter
         global sys_mutex_counter
+        global futex_counter
 
         self.addr = addr
         self.type_obj = type_obj
@@ -72,6 +74,9 @@ class KobjectInstance:
         elif self.type_obj.name == "sys_mutex":
             self.data = "(u32_t)(&kernel_mutexes[%d])" % sys_mutex_counter
             sys_mutex_counter += 1
+        elif self.type_obj.name == "k_futex":
+            self.data = "(u32_t)(&futex_data[%d])" % futex_counter
+            futex_counter += 1
         else:
             self.data = 0
 
@@ -566,3 +571,6 @@ class ElfHelper:
 
     def get_sys_mutex_counter(self):
         return sys_mutex_counter
+
+    def get_futex_counter(self):
+        return futex_counter

scripts/gen_kobject_list.py

@@ -86,7 +86,8 @@ kobjects = OrderedDict ([
     ("k_timer", (None, False)),
     ("_k_thread_stack_element", (None, False)),
     ("device", (None, False)),
-    ("sys_mutex", (None, True))
+    ("sys_mutex", (None, True)),
+    ("k_futex", (None, True))
 ])
 
 
@@ -170,6 +171,15 @@ def write_gperf_table(fp, eh, objs, static_begin, static_end):
                 fp.write(", ")
         fp.write("};\n")
 
+    num_futex = eh.get_futex_counter()
+    if (num_futex != 0):
+        fp.write("static struct z_futex_data futex_data[%d] = {\n" % num_futex)
+        for i in range(num_futex):
+            fp.write("Z_FUTEX_DATA_INITIALIZER(futex_data[%d])" % i)
+            if (i != num_futex - 1):
+                fp.write(", ")
+        fp.write("};\n")
+
     fp.write("%%\n")
     # Setup variables for mapping thread indexes
     syms = eh.get_symbols()