From 5e7605fad2ffe3629e262bed70f8f5c971a14f26 Mon Sep 17 00:00:00 2001 From: Jonas 'Sortie' Termansen Date: Sat, 6 Feb 2021 21:23:43 +0100 Subject: [PATCH] Implement threading primitives that truly sleep. The idle thread is now actually run when the system is idle because it truly goes idle. The idle thread is made power efficient by using the hlt instruction rather than a busy loop. The new futex(2) system call is used to implement fast user-space mutexes, condition variables, and semaphores. The same backend and design is used as kutexes for truly sleeping kernel mutexes and condition variables. The new exit_thread(2) flag EXIT_THREAD_FUTEX_WAKE wakes a futex. Sleeping on clocks in the kernel now uses timers for true sleep. The interrupt worker thread now truly sleeps when idle. Kernel threads are now named. This is a compatible ABI change. --- Makefile | 2 +- kernel/Makefile | 1 - kernel/clock.cpp | 96 +++---- kernel/copy.cpp | 20 +- kernel/include/sortix/exit.h | 3 +- kernel/include/sortix/futex.h | 35 +++ kernel/include/sortix/kernel/copy.h | 3 +- kernel/include/sortix/kernel/interrupt.h | 6 +- kernel/include/sortix/kernel/kthread.h | 19 +- kernel/include/sortix/kernel/process.h | 7 +- kernel/include/sortix/kernel/scheduler.h | 20 +- kernel/include/sortix/kernel/syscall.h | 3 +- kernel/include/sortix/kernel/thread.h | 28 ++- kernel/include/sortix/syscall.h | 5 +- kernel/interlock.cpp | 2 + kernel/interrupt.cpp | 17 +- kernel/kernel.cpp | 21 +- kernel/kthread.cpp | 306 +++++++++++++++++++---- kernel/lfbtextbuffer.cpp | 8 +- kernel/process.cpp | 9 +- kernel/scheduler.cpp | 70 ++++-- kernel/signal.cpp | 2 +- kernel/syscall.cpp | 3 +- kernel/thread.cpp | 163 ++++++++++-- kernel/x64/kthread.S | 84 ------- kernel/x86/kthread.S | 87 ------- libc/Makefile | 1 + libc/include/__/pthread.h | 64 ++--- libc/include/pthread.h | 11 +- libc/include/semaphore.h | 4 +- libc/include/sys/futex.h | 30 +++ libc/pthread/pthread_cond_broadcast.c | 22 +- libc/pthread/pthread_cond_destroy.c | 4 +- libc/pthread/pthread_cond_init.c | 3 +- libc/pthread/pthread_cond_signal.c | 27 +- libc/pthread/pthread_cond_timedwait.c | 48 ++-- libc/pthread/pthread_cond_wait.c | 23 +- libc/pthread/pthread_exit.c | 4 +- libc/pthread/pthread_mutex_lock.c | 43 +++- libc/pthread/pthread_mutex_trylock.c | 19 +- libc/pthread/pthread_mutex_unlock.c | 20 +- libc/semaphore/sem_getvalue.c | 5 +- libc/semaphore/sem_init.c | 6 +- libc/semaphore/sem_post.c | 21 +- libc/semaphore/sem_timedwait.c | 64 ++--- libc/semaphore/sem_trywait.c | 12 +- libc/semaphore/sem_wait.c | 35 +-- libc/sys/futex/futex.c | 29 +++ share/man/man7/following-development.7 | 10 + 49 files changed, 967 insertions(+), 558 deletions(-) create mode 100644 kernel/include/sortix/futex.h delete mode 100644 kernel/x64/kthread.S delete mode 100644 kernel/x86/kthread.S create mode 100644 libc/include/sys/futex.h create mode 100644 libc/sys/futex/futex.c diff --git a/Makefile b/Makefile index 0332676f..9f827cee 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ sysroot-system: sysroot-fsh sysroot-base-headers echo 'ID=sortix' && \ echo 'VERSION_ID="$(VERSION)"' && \ echo 'PRETTY_NAME="Sortix $(VERSION)"' && \ - echo 'SORTIX_ABI=1.0' && \ + echo 'SORTIX_ABI=1.1' && \ true) > "$(SYSROOT)/etc/sortix-release" echo /etc/sortix-release >> "$(SYSROOT)/tix/manifest/system" ln -sf sortix-release "$(SYSROOT)/etc/os-release" diff --git a/kernel/Makefile b/kernel/Makefile index 7a6a61f3..69374bbb 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -82,7 +82,6 @@ alarm.o \ clock.o \ com.o \ copy.o \ -$(CPUDIR)/kthread.o \ descriptor.o \ disk/ahci/ahci.o \ disk/ahci/hba.o \ diff --git a/kernel/clock.cpp b/kernel/clock.cpp index f3cab609..81bb3da7 100644 --- a/kernel/clock.cpp +++ b/kernel/clock.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2016, 2017, 2018 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2016, 2017, 2018, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -260,60 +261,61 @@ bool Clock::TryCancel(Timer* timer) return active; } +static void timer_wakeup(Clock* /*clock*/, Timer* /*timer*/, void* ctx) +{ + Thread* thread = (Thread*) ctx; + thread->timer_woken = true; + kthread_wake_futex(thread); +} -// TODO: We need some method for threads to sleep for real but still be -// interrupted by signals. struct timespec Clock::SleepDelay(struct timespec duration) { - struct timespec start_advancement; - struct timespec elapsed = timespec_nul(); - bool start_advancement_set = false; - - while ( timespec_lt(elapsed, duration) ) - { - if ( start_advancement_set ) - { - if ( Signal::IsPending() ) - return duration; - - kthread_yield(); - } - - LockClock(); - - if ( !start_advancement_set ) - { - start_advancement = current_advancement; - start_advancement_set = true; - } - - elapsed = timespec_sub(current_advancement, start_advancement); - - UnlockClock(); - } - + LockClock(); + struct timespec start_advancement = current_advancement; + UnlockClock(); + Thread* thread = CurrentThread(); + thread->futex_woken = false; + thread->timer_woken = false; + Timer timer; + timer.Attach(this); + struct itimerspec timerspec; + timerspec.it_value = duration; + timerspec.it_interval.tv_sec = 0; + timerspec.it_interval.tv_nsec = 0; + int timer_flags = TIMER_FUNC_INTERRUPT_HANDLER; + timer.Set(&timerspec, NULL, timer_flags, timer_wakeup, thread); + kthread_wait_futex_signal(); + timer.Cancel(); + LockClock(); + struct timespec end_advancement = current_advancement; + UnlockClock(); + struct timespec elapsed = timespec_sub(end_advancement, start_advancement); + if ( timespec_lt(elapsed, duration) ) + return timespec_sub(duration, elapsed); return timespec_nul(); } -// TODO: We need some method for threads to sleep for real but still be -// interrupted by signals. struct timespec Clock::SleepUntil(struct timespec expiration) { - while ( true ) - { - LockClock(); - struct timespec now = current_time; - UnlockClock(); - - if ( timespec_le(expiration, now) ) - break; - - if ( Signal::IsPending() ) - return timespec_sub(expiration, now); - - kthread_yield(); - } - + Thread* thread = CurrentThread(); + thread->futex_woken = false; + thread->timer_woken = false; + Timer timer; + timer.Attach(this); + struct itimerspec timerspec; + timerspec.it_value = expiration; + timerspec.it_interval.tv_sec = 0; + timerspec.it_interval.tv_nsec = 0; + int timer_flags = TIMER_ABSOLUTE | TIMER_FUNC_INTERRUPT_HANDLER; + timer.Set(&timerspec, NULL, timer_flags, timer_wakeup, thread); + kthread_wait_futex_signal(); + timer.Cancel(); + LockClock(); + struct timespec now = current_time; + UnlockClock(); + struct timespec remaining = timespec_sub(expiration, now); + if ( timespec_lt(timespec_nul(), remaining) ) + return remaining; return timespec_nul(); } diff --git a/kernel/copy.cpp b/kernel/copy.cpp index f1bdd874..2b2c8c41 100644 --- a/kernel/copy.cpp +++ b/kernel/copy.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -129,6 +129,24 @@ bool CopyFromUser(void* kdst_ptr, const void* usersrc_ptr, size_t count) return result; } +bool ReadAtomicFromUser(int* kdst_ptr, const int* usersrc_ptr) +{ + uintptr_t usersrc = (uintptr_t) usersrc_ptr; + if ( usersrc & (sizeof(int) - 1) ) + return errno = EINVAL, false; + Process* process = CurrentProcess(); + assert(IsInProcessAddressSpace(process)); + ScopedLock lock(&process->segment_lock); + struct segment* segment = FindSegment(process, usersrc); + if ( !segment || !(segment->prot & PROT_READ) ) + return errno = EFAULT, false; + size_t segment_available = segment->addr + segment->size - usersrc; + if ( segment_available < sizeof(int) ) + return errno = EFAULT, false; + *kdst_ptr = __atomic_load_n(usersrc_ptr, __ATOMIC_SEQ_CST); + return true; +} + bool CopyToKernel(void* kdst, const void* ksrc, size_t count) { memcpy(kdst, ksrc, count); diff --git a/kernel/include/sortix/exit.h b/kernel/include/sortix/exit.h index 7e9a446f..9132b2c3 100644 --- a/kernel/include/sortix/exit.h +++ b/kernel/include/sortix/exit.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -51,6 +51,7 @@ struct exit_thread #define EXIT_THREAD_TLS_UNMAP (1<<3) #define EXIT_THREAD_PROCESS (1<<4) #define EXIT_THREAD_DUMP_CORE (1<<5) +#define EXIT_THREAD_FUTEX_WAKE (1<<6) #ifdef __cplusplus } /* extern "C" */ diff --git a/kernel/include/sortix/futex.h b/kernel/include/sortix/futex.h new file mode 100644 index 00000000..5b7f9886 --- /dev/null +++ b/kernel/include/sortix/futex.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021 Jonas 'Sortie' Termansen. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * sortix/futex.h + * Fast userspace mutexes. + */ + +#ifndef INCLUDE_SORTIX_FUTEX_H +#define INCLUDE_SORTIX_FUTEX_H + +#include + +#define FUTEX_WAIT 1 +#define FUTEX_WAKE 2 + +#define FUTEX_ABSOLUTE (1 << 8) + +#define FUTEX_CLOCK(clock) (clock << 24) + +#define FUTEX_GET_OP(op) ((op) & 0xFF) +#define FUTEX_GET_CLOCK(op) ((op) >> 24) + +#endif diff --git a/kernel/include/sortix/kernel/copy.h b/kernel/include/sortix/kernel/copy.h index 16663b63..ced4322f 100644 --- a/kernel/include/sortix/kernel/copy.h +++ b/kernel/include/sortix/kernel/copy.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -28,6 +28,7 @@ bool CopyToUser(void* userdst, const void* ksrc, size_t count); bool CopyFromUser(void* kdst, const void* usersrc, size_t count); bool CopyToKernel(void* kdst, const void* ksrc, size_t count); bool CopyFromKernel(void* kdst, const void* ksrc, size_t count); +bool ReadAtomicFromUser(int* kdst, const int* usersrc); bool ZeroKernel(void* kdst, size_t count); bool ZeroUser(void* userdst, size_t count); char* GetStringFromUser(const char* str); diff --git a/kernel/include/sortix/kernel/interrupt.h b/kernel/include/sortix/kernel/interrupt.h index 118380b4..fcf1bdf2 100644 --- a/kernel/include/sortix/kernel/interrupt.h +++ b/kernel/include/sortix/kernel/interrupt.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2011, 2012, 2013, 2014, 2017, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -110,12 +110,12 @@ inline bool IsCPUInterrupted() inline bool SetEnabled(bool is_enabled) { - bool wasenabled = IsEnabled(); + bool was_enabled = IsEnabled(); if ( is_enabled ) Enable(); else Disable(); - return wasenabled; + return was_enabled; } void RegisterHandler(unsigned int index, struct interrupt_handler* handler); diff --git a/kernel/include/sortix/kernel/kthread.h b/kernel/include/sortix/kernel/kthread.h index 6cfb354a..59a6b896 100644 --- a/kernel/include/sortix/kernel/kthread.h +++ b/kernel/include/sortix/kernel/kthread.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -26,15 +26,18 @@ namespace Sortix { -extern "C" { +class Thread; -inline static void kthread_yield(void) { asm volatile ("int $129"); } +void kthread_yield(); +void kthread_wait_futex(); +void kthread_wait_futex_signal(); +void kthread_wake_futex(Thread* thread); __attribute__((noreturn)) void kthread_exit(); -typedef unsigned kthread_mutex_t; +typedef int kthread_mutex_t; const kthread_mutex_t KTHREAD_MUTEX_INITIALIZER = 0; -unsigned kthread_mutex_trylock(kthread_mutex_t* mutex); +bool kthread_mutex_trylock(kthread_mutex_t* mutex); void kthread_mutex_lock(kthread_mutex_t* mutex); -unsigned long kthread_mutex_lock_signal(kthread_mutex_t* mutex); +bool kthread_mutex_lock_signal(kthread_mutex_t* mutex); void kthread_mutex_unlock(kthread_mutex_t* mutex); struct kthread_cond_elem; typedef struct kthread_cond_elem kthread_cond_elem_t; @@ -46,12 +49,10 @@ struct kthread_cond typedef struct kthread_cond kthread_cond_t; const kthread_cond_t KTHREAD_COND_INITIALIZER = { NULL, NULL }; void kthread_cond_wait(kthread_cond_t* cond, kthread_mutex_t* mutex); -unsigned long kthread_cond_wait_signal(kthread_cond_t* cond, kthread_mutex_t* mutex); +bool kthread_cond_wait_signal(kthread_cond_t* cond, kthread_mutex_t* mutex); void kthread_cond_signal(kthread_cond_t* cond); void kthread_cond_broadcast(kthread_cond_t* cond); -} /* extern "C" */ - class ScopedLock { public: diff --git a/kernel/include/sortix/kernel/process.h b/kernel/include/sortix/kernel/process.h index 089c1942..70ec1edc 100644 --- a/kernel/include/sortix/kernel/process.h +++ b/kernel/include/sortix/kernel/process.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 205, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -138,6 +138,11 @@ public: size_t threads_not_exiting_count; bool threads_exiting; +public: + kthread_mutex_t futex_lock; + Thread* futex_first_waiting; + Thread* futex_last_waiting; + public: struct segment* segments; size_t segments_used; diff --git a/kernel/include/sortix/kernel/scheduler.h b/kernel/include/sortix/kernel/scheduler.h index 1618c64f..139bd310 100644 --- a/kernel/include/sortix/kernel/scheduler.h +++ b/kernel/include/sortix/kernel/scheduler.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2011, 2012, 2013, 2014, 2017, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -29,28 +29,16 @@ class Thread; } // namespace Sortix namespace Sortix { -enum ThreadState { NONE, RUNNABLE, BLOCKING, DEAD }; +enum ThreadState { NONE, RUNNABLE, FUTEX_WAITING, DEAD }; } // namespace Sortix namespace Sortix { namespace Scheduler { -#if defined(__i386__) || defined(__x86_64__) -static inline void Yield() -{ - asm volatile ("int $129"); -} -__attribute__ ((noreturn)) -static inline void ExitThread() -{ - asm volatile ("int $132"); - __builtin_unreachable(); -} -#endif - void Switch(struct interrupt_context* intctx); void SwitchTo(struct interrupt_context* intctx, Thread* new_thread); -void SetThreadState(Thread* thread, ThreadState state); +void SetThreadState(Thread* thread, ThreadState state, bool wake_only = false); +void SetSignalPending(Thread* thread, unsigned long is_pending); ThreadState GetThreadState(Thread* thread); void SetIdleThread(Thread* thread); void SetInitProcess(Process* init); diff --git a/kernel/include/sortix/kernel/syscall.h b/kernel/include/sortix/kernel/syscall.h index b1ef64d8..9dd93c72 100644 --- a/kernel/include/sortix/kernel/syscall.h +++ b/kernel/include/sortix/kernel/syscall.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -89,6 +89,7 @@ int sys_fstatvfs(int, struct statvfs*); int sys_fstatvfsat(int, const char*, struct statvfs*, int); int sys_fsync(int); int sys_ftruncate(int, off_t); +int sys_futex(int*, int, int, const struct timespec*); int sys_futimens(int, const struct timespec*); int sys_getdnsconfig(struct dnsconfig*); gid_t sys_getegid(void); diff --git a/kernel/include/sortix/kernel/thread.h b/kernel/include/sortix/kernel/thread.h index ca44500d..829e87e6 100644 --- a/kernel/include/sortix/kernel/thread.h +++ b/kernel/include/sortix/kernel/thread.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2016, 2018 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2018, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -39,19 +39,30 @@ class Process; class Thread; // These functions create a new kernel process but doesn't start it. -Thread* CreateKernelThread(Process* process, struct thread_registers* regs); +Thread* CreateKernelThread(Process* process, struct thread_registers* regs, + const char* name); Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user, + const char* name, size_t stacksize = 0); +Thread* CreateKernelThread(void (*entry)(void*), void* user, const char* name, size_t stacksize = 0); -Thread* CreateKernelThread(void (*entry)(void*), void* user, size_t stacksize = 0); // This function can be used to start a thread from the above functions. void StartKernelThread(Thread* thread); // Alternatively, these functions both create and start the thread. -Thread* RunKernelThread(Process* process, struct thread_registers* regs); +Thread* RunKernelThread(Process* process, struct thread_registers* regs, + const char* name); Thread* RunKernelThread(Process* process, void (*entry)(void*), void* user, + const char* name, size_t stacksize = 0); +Thread* RunKernelThread(void (*entry)(void*), void* user, const char* name, size_t stacksize = 0); -Thread* RunKernelThread(void (*entry)(void*), void* user, size_t stacksize = 0); + +enum yield_operation +{ + YIELD_OPERATION_NONE, + YIELD_OPERATION_WAIT_FUTEX, + YIELD_OPERATION_WAIT_FUTEX_SIGNAL, +}; class Thread { @@ -60,6 +71,7 @@ public: ~Thread(); public: + const char* name; uintptr_t system_tid; uintptr_t yield_to_tid; struct thread_registers registers; @@ -87,6 +99,12 @@ public: bool has_saved_signal_mask; Clock execute_clock; Clock system_clock; + uintptr_t futex_address; + bool futex_woken; + bool timer_woken; + Thread* futex_prev_waiting; + Thread* futex_next_waiting; + enum yield_operation yield_operation; public: void HandleSignal(struct interrupt_context* intctx); diff --git a/kernel/include/sortix/syscall.h b/kernel/include/sortix/syscall.h index 73d137e5..194038a7 100644 --- a/kernel/include/sortix/syscall.h +++ b/kernel/include/sortix/syscall.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -188,6 +188,7 @@ #define SYSCALL_SOCKET 165 #define SYSCALL_GETDNSCONFIG 166 #define SYSCALL_SETDNSCONFIG 167 -#define SYSCALL_MAX_NUM 168 /* index of highest constant + 1 */ +#define SYSCALL_FUTEX 168 +#define SYSCALL_MAX_NUM 169 /* index of highest constant + 1 */ #endif diff --git a/kernel/interlock.cpp b/kernel/interlock.cpp index 00f4eefa..f11c0a2d 100644 --- a/kernel/interlock.cpp +++ b/kernel/interlock.cpp @@ -23,12 +23,14 @@ namespace Sortix { // TODO: This is likely not the most optimal way to perform these operations. +// TODO: Just discard this whole thing in favor of __atomic_? ilret_t InterlockedModify(unsigned long* ptr, ilockfunc f, unsigned long user) { unsigned long old_value, new_value; + // TODO: Migrate to __atomic. do { old_value = *((volatile unsigned long*) ptr); /* TODO: Need volatile? */ diff --git a/kernel/interrupt.cpp b/kernel/interrupt.cpp index 23f7ee66..f9e41151 100644 --- a/kernel/interrupt.cpp +++ b/kernel/interrupt.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2016, 2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2017, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -24,6 +24,7 @@ #include #include #include +#include namespace Sortix { namespace Interrupt { @@ -33,12 +34,16 @@ bool interrupt_worker_thread_boost = false; static struct interrupt_work* first; static struct interrupt_work* last; +static bool interrupt_worker_idle = false; void WorkerThread(void* /*user*/) { + Thread* thread = CurrentThread(); assert(Interrupt::IsEnabled()); while ( true ) { + thread->futex_woken = false; + thread->timer_woken = false; struct interrupt_work* work; Interrupt::Disable(); work = first; @@ -47,8 +52,9 @@ void WorkerThread(void* /*user*/) Interrupt::Enable(); if ( !work ) { - // TODO: Make this thread not run until work arrives. - kthread_yield(); + interrupt_worker_idle = true; + kthread_wait_futex(); + interrupt_worker_idle = false; continue; } while ( work ) @@ -67,6 +73,11 @@ void ScheduleWork(struct interrupt_work* work) work->next = NULL; last = work; interrupt_worker_thread_boost = true; + if ( interrupt_worker_idle ) + { + interrupt_worker_thread->futex_woken = true; + kthread_wake_futex(interrupt_worker_thread); + } } } // namespace Interrupt diff --git a/kernel/kernel.cpp b/kernel/kernel.cpp index 652c6f1d..430c2d57 100644 --- a/kernel/kernel.cpp +++ b/kernel/kernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2018, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -193,7 +193,6 @@ extern "C" void KernelInit(unsigned long magic, multiboot_info_t* bootinfo_p) // Display the logo. Log::PrintF("\e[37;41m\e[2J"); Log::Center(BRAND_LOGO); - #if defined(__x86_64__) // TODO: Remove this hack when qemu 1.4.x and 1.5.0 are obsolete. // Verify that we are not running under a buggy qemu where the instruction @@ -375,6 +374,7 @@ extern "C" void KernelInit(unsigned long magic, multiboot_info_t* bootinfo_p) // scheduler's set of runnable threads, but rather run whenever there is // _nothing_ else to run on this CPU. Thread* idlethread = AllocateThread(); + idlethread->name = "idle"; idlethread->process = system; idlethread->kernelstackpos = (addr_t) stack; idlethread->kernelstacksize = STACK_SIZE; @@ -387,7 +387,7 @@ extern "C" void KernelInit(unsigned long magic, multiboot_info_t* bootinfo_p) // Note that we don't do the work here: if all other threads are not running // and this thread isn't runnable, then there is nothing to run. Therefore // we must become the system idle thread. - RunKernelThread(BootThread, NULL); + RunKernelThread(BootThread, NULL, "boot"); // The time driver will run the scheduler on the next timer interrupt. Time::Start(); @@ -401,7 +401,14 @@ static void SystemIdleThread(void* /*user*/) // Alright, we are now the system idle thread. If there is nothing to do, // then we are run. Note that we must never do any real work here as the // idle thread must always be runnable. - while(true); + while ( true ) + { +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("hlt"); +#else +#warning "Implement a power efficient kernel idle thread" +#endif + } } static void BootThread(void* /*user*/) @@ -425,7 +432,7 @@ static void BootThread(void* /*user*/) // Let's create the interrupt worker thread that executes additional work // requested by interrupt handlers, where such work isn't safe. Interrupt::interrupt_worker_thread = - RunKernelThread(Interrupt::WorkerThread, NULL); + RunKernelThread(Interrupt::WorkerThread, NULL, "interrupt"); if ( !Interrupt::interrupt_worker_thread ) Panic("Could not create interrupt worker"); @@ -433,7 +440,7 @@ static void BootThread(void* /*user*/) Worker::Init(); // Create a general purpose worker thread. - Thread* worker_thread = RunKernelThread(Worker::Thread, NULL); + Thread* worker_thread = RunKernelThread(Worker::Thread, NULL, "worker"); if ( !worker_thread ) Panic("Unable to create general purpose worker thread"); @@ -644,7 +651,7 @@ static void BootThread(void* /*user*/) init->addrspace = initaddrspace; Scheduler::SetInitProcess(init); - Thread* initthread = RunKernelThread(init, InitThread, NULL); + Thread* initthread = RunKernelThread(init, InitThread, NULL, "main"); if ( !initthread ) Panic("Could not create init thread"); diff --git a/kernel/kthread.cpp b/kernel/kthread.cpp index cb8783be..fe822927 100644 --- a/kernel/kthread.cpp +++ b/kernel/kthread.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,6 +17,8 @@ * Utility and synchronization mechanisms for kernel threads. */ +#include + #include #include @@ -27,8 +29,195 @@ #include #include +#include "uart.h" + namespace Sortix { +void kthread_yield() +{ + Thread* thread = CurrentThread(); + thread->yield_operation = YIELD_OPERATION_NONE; +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("int $129"); +#else +#error "kthread_yield needs to be implemented" +#endif +} + +void kthread_wait_futex() +{ + Thread* thread = CurrentThread(); + thread->yield_operation = YIELD_OPERATION_WAIT_FUTEX; +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("int $129"); +#else +#error "kthread_wait_futex needs to be implemented" +#endif +} + +void kthread_wait_futex_signal() +{ + Thread* thread = CurrentThread(); + thread->yield_operation = YIELD_OPERATION_WAIT_FUTEX_SIGNAL; +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("int $129"); +#else +#error "kthread_wait_futex needs to be implemented" +#endif +} + +void kthread_wake_futex(Thread* thread) +{ + Scheduler::SetThreadState(thread, ThreadState::RUNNABLE, true); +} + +static const int UNLOCKED = 0; +static const int LOCKED = 1; +static const int CONTENDED = 2; + +void kthread_spinlock_lock(kthread_mutex_t* mutex) +{ + while ( true ) + { + int state = UNLOCKED; + int desired = LOCKED; + if ( __atomic_compare_exchange_n(mutex, &state, desired, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + break; + } +} + +void kthread_spinlock_unlock(kthread_mutex_t* mutex) +{ + __atomic_store_n(mutex, UNLOCKED, __ATOMIC_SEQ_CST); +} + +static bool kutex_wait(int* address, int value, bool signal) +{ + // TODO: Use a per-mutex wait queue instead. + Thread* thread = CurrentThread(); + Process* kernel_process = Scheduler::GetKernelProcess(); + bool was_enabled = Interrupt::SetEnabled(false); + kthread_spinlock_lock(&kernel_process->futex_lock); + thread->futex_address = (uintptr_t) address; + thread->futex_woken = false; + thread->futex_prev_waiting = kernel_process->futex_last_waiting; + thread->futex_next_waiting = NULL; + (kernel_process->futex_last_waiting ? + kernel_process->futex_last_waiting->futex_next_waiting : + kernel_process->futex_first_waiting) = thread; + kernel_process->futex_last_waiting = thread; + kthread_spinlock_unlock(&kernel_process->futex_lock); + Interrupt::SetEnabled(was_enabled); + thread->timer_woken = false; + bool result = true; + if ( __atomic_load_n(address, __ATOMIC_SEQ_CST) == value ) + { + if ( signal ) + kthread_wait_futex(); + else + kthread_wait_futex_signal(); + } + Interrupt::SetEnabled(false); + kthread_spinlock_lock(&kernel_process->futex_lock); + if ( result && !thread->futex_woken ) + { + if ( signal && Signal::IsPending() ) + result = false; + } + thread->futex_address = 0; + thread->futex_woken = false; + (thread->futex_prev_waiting ? + thread->futex_prev_waiting->futex_next_waiting : + kernel_process->futex_first_waiting) = thread->futex_next_waiting; + (thread->futex_next_waiting ? + thread->futex_next_waiting->futex_prev_waiting : + kernel_process->futex_last_waiting) = thread->futex_prev_waiting; + thread->futex_prev_waiting = NULL; + thread->futex_next_waiting = NULL; + kthread_spinlock_unlock(&kernel_process->futex_lock); + Interrupt::SetEnabled(was_enabled); + return result; +} + +static void kutex_wake(int* address, int count) +{ + Process* kernel_process = Scheduler::GetKernelProcess(); + bool was_enabled = Interrupt::SetEnabled(false); + kthread_spinlock_lock(&kernel_process->futex_lock); + for ( Thread* waiter = kernel_process->futex_first_waiting; + 0 < count && waiter; + waiter = waiter->futex_next_waiting ) + { + if ( waiter->futex_address == (uintptr_t) address ) + { + waiter->futex_woken = true; + kthread_wake_futex(waiter); + if ( count != INT_MAX ) + count--; + } + } + kthread_spinlock_unlock(&kernel_process->futex_lock); + Interrupt::SetEnabled(was_enabled); +} + +bool kthread_mutex_trylock(kthread_mutex_t* mutex) +{ + int state = UNLOCKED; + if ( !__atomic_compare_exchange_n(mutex, &state, LOCKED, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + return false; + return true; +} + +void kthread_mutex_lock(kthread_mutex_t* mutex) +{ + int state = UNLOCKED; + int desired = LOCKED; + while ( !__atomic_compare_exchange_n(mutex, &state, desired, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + { + if ( state == LOCKED && + !__atomic_compare_exchange_n(mutex, &state, CONTENDED, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + { + state = UNLOCKED; + continue; + } + desired = CONTENDED; + kutex_wait(mutex, CONTENDED, false); + state = UNLOCKED; + } +} + +bool kthread_mutex_lock_signal(kthread_mutex_t* mutex) +{ + int state = UNLOCKED; + int desired = LOCKED; + while ( !__atomic_compare_exchange_n(mutex, &state, desired, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + { + if ( state == LOCKED && + !__atomic_compare_exchange_n(mutex, &state, CONTENDED, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + { + state = UNLOCKED; + continue; + } + desired = CONTENDED; + if ( !kutex_wait(mutex, CONTENDED, true) ) + return false; + state = UNLOCKED; + } + return true; +} + +void kthread_mutex_unlock(kthread_mutex_t* mutex) +{ + if ( __atomic_exchange_n(mutex, UNLOCKED, __ATOMIC_SEQ_CST) == CONTENDED ) + kutex_wake(mutex, 1); +} + // The kernel thread needs another stack to delete its own stack. static void kthread_do_kill_thread(void* user) { @@ -38,7 +227,7 @@ static void kthread_do_kill_thread(void* user) FreeThread(thread); } -extern "C" void kthread_exit() +void kthread_exit() { Process* process = CurrentProcess(); // Note: This requires all threads in this process to have been made by @@ -55,84 +244,109 @@ extern "C" void kthread_exit() if ( is_last_to_exit ) process->OnLastThreadExit(); Worker::Schedule(kthread_do_kill_thread, CurrentThread()); - Scheduler::ExitThread(); +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("int $132"); +#else +#error "kthread_exit needs to be implemented" +#endif __builtin_unreachable(); } struct kthread_cond_elem { kthread_cond_elem_t* next; - volatile unsigned long woken; + kthread_cond_elem_t* prev; + int woken; }; -extern "C" void kthread_cond_wait(kthread_cond_t* cond, kthread_mutex_t* mutex) +void kthread_cond_wait(kthread_cond_t* cond, kthread_mutex_t* mutex) { kthread_cond_elem_t elem; elem.next = NULL; + elem.prev = cond->last; elem.woken = 0; - if ( cond->last ) { cond->last->next = &elem; } - if ( !cond->last ) { cond->first = &elem; } + if ( cond->last ) + cond->last->next = &elem; + if ( !cond->first ) + cond->first = &elem; cond->last = &elem; - while ( !elem.woken ) + kthread_mutex_unlock(mutex); + while ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) && + kutex_wait(&elem.woken, 0, false) < 0 ); + kthread_mutex_lock(mutex); + if ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) ) { - kthread_mutex_unlock(mutex); - Scheduler::Yield(); - kthread_mutex_lock(mutex); + if ( elem.next ) + elem.next->prev = elem.prev; + else + cond->last = elem.prev; + if ( elem.prev ) + elem.prev->next = elem.next; + else + cond->first = elem.next; } } -extern "C" unsigned long kthread_cond_wait_signal(kthread_cond_t* cond, - kthread_mutex_t* mutex) +bool kthread_cond_wait_signal(kthread_cond_t* cond, kthread_mutex_t* mutex) { if ( Signal::IsPending() ) - return 0; + return false; kthread_cond_elem_t elem; elem.next = NULL; + elem.prev = cond->last; elem.woken = 0; - if ( cond->last ) { cond->last->next = &elem; } - if ( !cond->last ) { cond->first = &elem; } + if ( cond->last ) + cond->last->next = &elem; + if ( !cond->first ) + cond->first = &elem; cond->last = &elem; - while ( !elem.woken ) + kthread_mutex_unlock(mutex); + bool result = true; + while ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) && + kutex_wait(&elem.woken, 0, false) < 0 ) { if ( Signal::IsPending() ) { - if ( cond->first == &elem ) - { - cond->first = elem.next; - if ( cond->last == &elem ) - cond->last = NULL; - } - else - { - kthread_cond_elem_t* prev = cond->first; - while ( prev->next != &elem ) - prev = prev->next; - prev->next = elem.next; - if ( cond->last == &elem ) - cond->last = prev; - } - // Note that the thread still owns the mutex. - return 0; + result = false; + break; } - kthread_mutex_unlock(mutex); - Scheduler::Yield(); - kthread_mutex_lock(mutex); } - return 1; + kthread_mutex_lock(mutex); + if ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) ) + { + if ( elem.next ) + elem.next->prev = elem.prev; + else + cond->last = elem.prev; + if ( elem.prev ) + elem.prev->next = elem.next; + else + cond->first = elem.next; + } + return result; } -extern "C" void kthread_cond_signal(kthread_cond_t* cond) +void kthread_cond_signal(kthread_cond_t* cond) { - kthread_cond_elem_t* elem = cond->first; - if ( !elem ) { return; } - if ( !(cond->first = elem->next) ) { cond->last = NULL; } - elem->next = NULL; - elem->woken = 1; + if ( cond->first ) + { + struct kthread_cond_elem* elem = cond->first; + if ( elem->next ) + elem->next->prev = elem->prev; + else + cond->last = elem->prev; + cond->first = elem->next; + elem->next = NULL; + elem->prev = NULL; + __atomic_store_n(&elem->woken, 1, __ATOMIC_SEQ_CST); + kutex_wake(&elem->woken, 1); + } } -extern "C" void kthread_cond_broadcast(kthread_cond_t* cond) +void kthread_cond_broadcast(kthread_cond_t* cond) { - while ( cond->first ) { kthread_cond_signal(cond); } + while ( cond->first ) + kthread_cond_signal(cond); } } // namespace Sortix diff --git a/kernel/lfbtextbuffer.cpp b/kernel/lfbtextbuffer.cpp index 421061b9..4c3b9ebf 100644 --- a/kernel/lfbtextbuffer.cpp +++ b/kernel/lfbtextbuffer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2013, 2014, 2015, 2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -119,7 +119,8 @@ LFBTextBuffer* CreateLFBTextBuffer(uint8_t* lfb, uint32_t lfbformat, return ret; ret->queue_thread = true; // Visible to new thread. - if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, ret) ) + if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, ret, + "console") ) ret->queue_thread = false; return ret; @@ -142,7 +143,8 @@ void LFBTextBuffer::SpawnThreads() return; Process* kernel_process = Scheduler::GetKernelProcess(); queue_thread = true; // Visible to new thread. - if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, this) ) + if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, this, + "console") ) queue_thread = false; } diff --git a/kernel/process.cpp b/kernel/process.cpp index 2f5301c9..e2be8c21 100644 --- a/kernel/process.cpp +++ b/kernel/process.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -141,6 +141,10 @@ Process::Process() threads_not_exiting_count = 0; threads_exiting = false; + futex_lock = KTHREAD_MUTEX_INITIALIZER; + futex_first_waiting = NULL; + futex_last_waiting = NULL; + segments = NULL; segments_used = 0; segments_length = 0; @@ -1550,7 +1554,8 @@ pid_t sys_tfork(int flags, struct tfork* user_regs) // If the thread could not be created, make the process commit suicide // in a manner such that we don't wait for its zombie. - Thread* thread = CreateKernelThread(child_process, &cpuregs); + Thread* thread = CreateKernelThread(child_process, &cpuregs, + making_process ? "main" : "second"); process_family_lock_lock.Reset(); if ( !thread ) { diff --git a/kernel/scheduler.cpp b/kernel/scheduler.cpp index 7be032d1..5a8167e5 100644 --- a/kernel/scheduler.cpp +++ b/kernel/scheduler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015 Jonas 'Sortie' Termansen. + * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -265,10 +265,17 @@ static void SwitchRegisters(struct interrupt_context* intctx, current_thread = next; } +static Thread* idle_thread; +static Thread* first_runnable_thread; +static Thread* true_current_thread; +static Process* init_process; + static void SwitchThread(struct interrupt_context* intctx, Thread* old_thread, Thread* new_thread) { + assert(new_thread->state == ThreadState::RUNNABLE || + new_thread == idle_thread); SwitchRegisters(intctx, old_thread, new_thread); if ( intctx->signal_pending && InUserspace(intctx) ) { @@ -290,14 +297,11 @@ static void SwitchThread(struct interrupt_context* intctx, } } -static Thread* idle_thread; -static Thread* first_runnable_thread; -static Thread* true_current_thread; -static Process* init_process; - static Thread* FindRunnableThreadWithSystemTid(uintptr_t system_tid) { - Thread* begun_thread = current_thread; + Thread* begun_thread = first_runnable_thread; + if ( !begun_thread ) + return NULL; Thread* iter = begun_thread; do { @@ -329,8 +333,6 @@ static Thread* PopNextThread(bool yielded) result = idle_thread; } - true_current_thread = result; - return result; } @@ -339,7 +341,10 @@ void SwitchTo(struct interrupt_context* intctx, Thread* new_thread) Thread* old_thread = CurrentThread(); if ( new_thread == old_thread ) return; - first_runnable_thread = old_thread; + if ( new_thread->state != ThreadState::RUNNABLE ) + return; + if ( old_thread != idle_thread ) + first_runnable_thread = old_thread; true_current_thread = new_thread; SwitchThread(intctx, old_thread, new_thread); } @@ -348,6 +353,7 @@ static void RealSwitch(struct interrupt_context* intctx, bool yielded) { Thread* old_thread = CurrentThread(); Thread* new_thread = PopNextThread(yielded); + true_current_thread = new_thread; SwitchThread(intctx, old_thread, new_thread); } @@ -358,7 +364,28 @@ void Switch(struct interrupt_context* intctx) void InterruptYieldCPU(struct interrupt_context* intctx, void* /*user*/) { - RealSwitch(intctx, true); + if ( current_thread->yield_operation == YIELD_OPERATION_NONE ) + RealSwitch(intctx, true); + else if ( current_thread->yield_operation == YIELD_OPERATION_WAIT_FUTEX ) + { + if ( !current_thread->futex_woken && + !current_thread->timer_woken && + !asm_signal_is_pending ) + { + SetThreadState(current_thread, ThreadState::FUTEX_WAITING); + RealSwitch(intctx, false); + } + } + else if ( current_thread->yield_operation == + YIELD_OPERATION_WAIT_FUTEX_SIGNAL ) + { + if ( !current_thread->futex_woken && + !current_thread->timer_woken ) + { + SetThreadState(current_thread, ThreadState::FUTEX_WAITING); + RealSwitch(intctx, false); + } + } } void ThreadExitCPU(struct interrupt_context* intctx, void* /*user*/) @@ -395,9 +422,13 @@ Process* GetKernelProcess() return idle_thread->process; } -void SetThreadState(Thread* thread, ThreadState state) +void SetThreadState(Thread* thread, ThreadState state, bool wake_only) { - bool wasenabled = Interrupt::SetEnabled(false); + bool was_enabled = Interrupt::SetEnabled(false); + + // Avoid transitioning to the RUNNABLE state from unintended states. + if ( wake_only && thread->state != ThreadState::FUTEX_WAITING ) + state = thread->state; // Remove the thread from the list of runnable threads. if ( thread->state == ThreadState::RUNNABLE && @@ -432,7 +463,14 @@ void SetThreadState(Thread* thread, ThreadState state) assert(thread->state != ThreadState::RUNNABLE || thread->scheduler_list_prev); assert(thread->state != ThreadState::RUNNABLE || thread->scheduler_list_next); - Interrupt::SetEnabled(wasenabled); + Interrupt::SetEnabled(was_enabled); +} + +void SetSignalPending(Thread* thread, unsigned long is_pending) +{ + thread->registers.signal_pending = is_pending; + if ( is_pending ) + Scheduler::SetThreadState(thread, ThreadState::RUNNABLE, true); } ThreadState GetThreadState(Thread* thread) @@ -457,14 +495,14 @@ namespace Scheduler { void ScheduleTrueThread() { - bool wasenabled = Interrupt::SetEnabled(false); + bool was_enabled = Interrupt::SetEnabled(false); if ( true_current_thread != current_thread ) { current_thread->yield_to_tid = 0; first_runnable_thread = true_current_thread; kthread_yield(); } - Interrupt::SetEnabled(wasenabled); + Interrupt::SetEnabled(was_enabled); } } // namespace Scheduler diff --git a/kernel/signal.cpp b/kernel/signal.cpp index 383d6782..993754b6 100644 --- a/kernel/signal.cpp +++ b/kernel/signal.cpp @@ -100,7 +100,7 @@ void UpdatePendingSignals(Thread* thread) // thread->process->signal_lock held if ( thread == CurrentThread() ) asm_signal_is_pending = is_pending; else - thread->registers.signal_pending = is_pending; + Scheduler::SetSignalPending(thread, is_pending); } void Thread::DoUpdatePendingSignal() diff --git a/kernel/syscall.cpp b/kernel/syscall.cpp index ebe8497a..d76f73ba 100644 --- a/kernel/syscall.cpp +++ b/kernel/syscall.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -202,6 +202,7 @@ void* syscall_list[SYSCALL_MAX_NUM + 1] = [SYSCALL_SOCKET] = (void*) sys_socket, [SYSCALL_GETDNSCONFIG] = (void*) sys_getdnsconfig, [SYSCALL_SETDNSCONFIG] = (void*) sys_setdnsconfig, + [SYSCALL_FUTEX] = (void*) sys_futex, [SYSCALL_MAX_NUM] = (void*) sys_bad_syscall, }; } /* extern "C" */ diff --git a/kernel/thread.cpp b/kernel/thread.cpp index 9245580e..2111be74 100644 --- a/kernel/thread.cpp +++ b/kernel/thread.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2016, 2018 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2018, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -21,16 +21,21 @@ #include #include +#include #include #include #include +#include +#include #include +#include #include #include #include #include +#include #include #include #include @@ -77,6 +82,7 @@ void FreeThread(Thread* thread) Thread::Thread() { assert(!((uintptr_t) registers.fpuenv & 0xFUL)); + name = ""; system_tid = (uintptr_t) this; yield_to_tid = 0; id = 0; // TODO: Make a thread id. @@ -105,6 +111,11 @@ Thread::Thread() // execute_clock initialized in member constructor. // system_clock initialized in member constructor. Time::InitializeThreadClocks(this); + futex_address = 0; + futex_woken = false; + futex_prev_waiting = NULL; + futex_next_waiting = NULL; + yield_operation = YIELD_OPERATION_NONE; } Thread::~Thread() @@ -116,7 +127,9 @@ Thread::~Thread() delete[] (uint8_t*) kernelstackpos; } -Thread* CreateKernelThread(Process* process, struct thread_registers* regs) +Thread* CreateKernelThread(Process* process, + struct thread_registers* regs, + const char* name) { assert(process && regs && process->addrspace); @@ -142,6 +155,7 @@ Thread* CreateKernelThread(Process* process, struct thread_registers* regs) Thread* thread = AllocateThread(); if ( !thread ) return NULL; + thread->name = name; memcpy(&thread->registers, regs, sizeof(struct thread_registers)); @@ -254,7 +268,7 @@ static void SetupKernelThreadRegs(struct thread_registers* regs, } Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user, - size_t stacksize) + const char* name, size_t stacksize) { const size_t DEFAULT_KERNEL_STACK_SIZE = 8 * 1024UL; if ( !stacksize ) @@ -264,9 +278,10 @@ Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user, return NULL; struct thread_registers regs; - SetupKernelThreadRegs(®s, process, entry, user, (uintptr_t) stack, stacksize); + SetupKernelThreadRegs(®s, process, entry, user, (uintptr_t) stack, + stacksize); - Thread* thread = CreateKernelThread(process, ®s); + Thread* thread = CreateKernelThread(process, ®s, name); if ( !thread ) { delete[] stack; return NULL; } thread->kernelstackpos = (uintptr_t) stack; @@ -276,9 +291,10 @@ Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user, return thread; } -Thread* CreateKernelThread(void (*entry)(void*), void* user, size_t stacksize) +Thread* CreateKernelThread(void (*entry)(void*), void* user, const char* name, + size_t stacksize) { - return CreateKernelThread(CurrentProcess(), entry, user, stacksize); + return CreateKernelThread(CurrentProcess(), entry, user, name, stacksize); } void StartKernelThread(Thread* thread) @@ -286,9 +302,10 @@ void StartKernelThread(Thread* thread) Scheduler::SetThreadState(thread, ThreadState::RUNNABLE); } -Thread* RunKernelThread(Process* process, struct thread_registers* regs) +Thread* RunKernelThread(Process* process, struct thread_registers* regs, + const char* name) { - Thread* thread = CreateKernelThread(process, regs); + Thread* thread = CreateKernelThread(process, regs, name); if ( !thread ) return NULL; StartKernelThread(thread); @@ -296,18 +313,19 @@ Thread* RunKernelThread(Process* process, struct thread_registers* regs) } Thread* RunKernelThread(Process* process, void (*entry)(void*), void* user, - size_t stacksize) + const char* name, size_t stacksize) { - Thread* thread = CreateKernelThread(process, entry, user, stacksize); + Thread* thread = CreateKernelThread(process, entry, user, name, stacksize); if ( !thread ) return NULL; StartKernelThread(thread); return thread; } -Thread* RunKernelThread(void (*entry)(void*), void* user, size_t stacksize) +Thread* RunKernelThread(void (*entry)(void*), void* user, const char* name, + size_t stacksize) { - Thread* thread = CreateKernelThread(entry, user, stacksize); + Thread* thread = CreateKernelThread(entry, user, name, stacksize); if ( !thread ) return NULL; StartKernelThread(thread); @@ -323,7 +341,8 @@ int sys_exit_thread(int requested_exit_code, EXIT_THREAD_ZERO | EXIT_THREAD_TLS_UNMAP | EXIT_THREAD_PROCESS | - EXIT_THREAD_DUMP_CORE) ) + EXIT_THREAD_DUMP_CORE | + EXIT_THREAD_FUTEX_WAKE) ) return errno = EINVAL, -1; if ( (flags & EXIT_THREAD_ONLY_IF_OTHERS) && (flags & EXIT_THREAD_PROCESS) ) @@ -395,6 +414,9 @@ int sys_exit_thread(int requested_exit_code, if ( flags & EXIT_THREAD_ZERO ) ZeroUser(extended.zero_from, extended.zero_size); + if ( flags & EXIT_THREAD_FUTEX_WAKE ) + sys_futex((int*) extended.zero_from, FUTEX_WAKE, 1, NULL); + if ( do_exit ) { // Validate the requested exit code such that the process can't exit @@ -431,4 +453,117 @@ int sys_exit_thread(int requested_exit_code, kthread_exit(); } +static void futex_timeout(Clock* /*clock*/, Timer* /*timer*/, void* ctx) +{ + Thread* thread = (Thread*) ctx; + thread->timer_woken = true; + kthread_wake_futex(thread); +} + +int sys_futex(int* user_address, + int op, + int value, + const struct timespec* user_timeout) +{ + ioctx_t ctx; SetupKernelIOCtx(&ctx); + Thread* thread = CurrentThread(); + Process* process = thread->process; + if ( FUTEX_GET_OP(op) == FUTEX_WAIT ) + { + kthread_mutex_lock(&process->futex_lock); + thread->futex_address = (uintptr_t) user_address; + thread->futex_woken = false; + thread->futex_prev_waiting = process->futex_last_waiting; + thread->futex_next_waiting = NULL; + (process->futex_last_waiting ? + process->futex_last_waiting->futex_next_waiting : + process->futex_first_waiting) = thread; + process->futex_last_waiting = thread; + kthread_mutex_unlock(&process->futex_lock); + thread->timer_woken = false; + Timer timer; + if ( user_timeout ) + { + clockid_t clockid = FUTEX_GET_CLOCK(op); + bool absolute = op & FUTEX_ABSOLUTE; + struct timespec timeout; + if ( !CopyFromUser(&timeout, user_timeout, sizeof(timeout)) ) + return -1; + if ( !timespec_is_canonical(timeout) ) + return errno = EINVAL, -1; + Clock* clock = Time::GetClock(clockid); + timer.Attach(clock); + struct itimerspec timerspec; + timerspec.it_value = timeout; + timerspec.it_interval.tv_sec = 0; + timerspec.it_interval.tv_nsec = 0; + int timer_flags = (absolute ? TIMER_ABSOLUTE : 0) | + TIMER_FUNC_INTERRUPT_HANDLER; + timer.Set(&timerspec, NULL, timer_flags, futex_timeout, thread); + } + int result = 0; + int current; + if ( !ReadAtomicFromUser(¤t, user_address) ) + result = -1; + else if ( current != value ) + { + errno = EAGAIN; + result = -1; + } + else + kthread_wait_futex_signal(); + if ( user_timeout ) + timer.Cancel(); + kthread_mutex_lock(&process->futex_lock); + if ( result == 0 && !thread->futex_woken ) + { + if ( Signal::IsPending() ) + { + errno = EINTR; + result = -1; + } + else if ( thread->timer_woken ) + { + errno = ETIMEDOUT; + result = -1; + } + } + thread->futex_address = 0; + thread->futex_woken = false; + (thread->futex_prev_waiting ? + thread->futex_prev_waiting->futex_next_waiting : + process->futex_first_waiting) = thread->futex_next_waiting; + (thread->futex_next_waiting ? + thread->futex_next_waiting->futex_prev_waiting : + process->futex_last_waiting) = thread->futex_prev_waiting; + thread->futex_prev_waiting = NULL; + thread->futex_next_waiting = NULL; + kthread_mutex_unlock(&process->futex_lock); + return result; + } + else if ( FUTEX_GET_OP(op) == FUTEX_WAKE ) + { + kthread_mutex_lock(&process->futex_lock); + int result = 0; + for ( Thread* waiter = process->futex_first_waiting; + 0 < value && waiter; + waiter = waiter->futex_next_waiting ) + { + if ( waiter->futex_address == (uintptr_t) user_address ) + { + waiter->futex_woken = true; + kthread_wake_futex(waiter); + if ( value != INT_MAX ) + value--; + if ( result != INT_MAX ) + result++; + } + } + kthread_mutex_unlock(&process->futex_lock); + return result; + } + else + return errno = EINVAL, -1; +} + } // namespace Sortix diff --git a/kernel/x64/kthread.S b/kernel/x64/kthread.S deleted file mode 100644 index 0556973b..00000000 --- a/kernel/x64/kthread.S +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2012 Jonas 'Sortie' Termansen. - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - * x64/kthread.S - * Utilities and synchronization mechanisms for x64 kernel threads. - */ - -.section .text - -.global kthread_mutex_trylock -.type kthread_mutex_trylock, @function -kthread_mutex_trylock: - pushq %rbp - movq %rsp, %rbp - movl $-1, %eax - xchgl (%rdi), %eax - not %eax - leaveq - retq -.size kthread_mutex_trylock, . - kthread_mutex_trylock - -.global kthread_mutex_lock -.type kthread_mutex_lock, @function -kthread_mutex_lock: - pushq %rbp - movq %rsp, %rbp -kthread_mutex_lock_retry: - movl $-1, %eax - xchgl (%rdi), %eax - testl %eax, %eax - jnz kthread_mutex_lock_failed - leaveq - retq -kthread_mutex_lock_failed: - int $0x81 # Yield the CPU. - jmp kthread_mutex_lock_retry -.size kthread_mutex_lock, . - kthread_mutex_lock - -.global kthread_mutex_lock_signal -.type kthread_mutex_lock_signal, @function -kthread_mutex_lock_signal: - pushq %rbp - movq %rsp, %rbp -kthread_mutex_lock_signal_retry: - movq asm_signal_is_pending, %rax - testq %rax, %rax - jnz kthread_mutex_lock_signal_pending - movl $-1, %eax - xchgl (%rdi), %eax - testl %eax, %eax - jnz kthread_mutex_lock_signal_failed - inc %eax -kthread_mutex_lock_signal_out: - leaveq - retq -kthread_mutex_lock_signal_failed: - int $0x81 # Yield the CPU. - jmp kthread_mutex_lock_signal_retry -kthread_mutex_lock_signal_pending: - xorl %eax, %eax - jmp kthread_mutex_lock_signal_out -.size kthread_mutex_lock_signal, . - kthread_mutex_lock_signal - -.global kthread_mutex_unlock -.type kthread_mutex_unlock, @function -kthread_mutex_unlock: - pushq %rbp - movq %rsp, %rbp - movl $0, (%rdi) - leaveq - retq -.size kthread_mutex_unlock, . - kthread_mutex_unlock diff --git a/kernel/x86/kthread.S b/kernel/x86/kthread.S deleted file mode 100644 index 12d348e3..00000000 --- a/kernel/x86/kthread.S +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2012 Jonas 'Sortie' Termansen. - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - * x86/kthread.S - * Utilities and synchronization mechanisms for x86 kernel threads. - */ - -.section .text - -.global kthread_mutex_trylock -.type kthread_mutex_trylock, @function -kthread_mutex_trylock: - pushl %ebp - movl %esp, %ebp - movl 8(%ebp), %edx - movl $-1, %eax - xchgl (%edx), %eax - not %eax - leavel - retl -.size kthread_mutex_trylock, . - kthread_mutex_trylock - -.global kthread_mutex_lock -.type kthread_mutex_lock, @function -kthread_mutex_lock: - pushl %ebp - movl %esp, %ebp - movl 8(%ebp), %edx -kthread_mutex_lock_retry: - movl $-1, %eax - xchgl (%edx), %eax - testl %eax, %eax - jnz kthread_mutex_lock_failed - leavel - retl -kthread_mutex_lock_failed: - int $0x81 # Yield the CPU. - jmp kthread_mutex_lock_retry -.size kthread_mutex_lock, . - kthread_mutex_lock - -.global kthread_mutex_lock_signal -.type kthread_mutex_lock_signal, @function -kthread_mutex_lock_signal: - pushl %ebp - movl %esp, %ebp - movl 8(%ebp), %edx -kthread_mutex_lock_signal_retry: - movl asm_signal_is_pending, %eax - testl %eax, %eax - jnz kthread_mutex_lock_signal_pending - movl $-1, %eax - xchgl (%edx), %eax - testl %eax, %eax - jnz kthread_mutex_lock_signal_failed - inc %eax -kthread_mutex_lock_signal_out: - leavel - retl -kthread_mutex_lock_signal_failed: - int $0x81 # Yield the CPU. - jmp kthread_mutex_lock_signal_retry -kthread_mutex_lock_signal_pending: - xorl %eax, %eax - jmp kthread_mutex_lock_signal_out - -.global kthread_mutex_unlock -.type kthread_mutex_unlock, @function -kthread_mutex_unlock: - pushl %ebp - movl %esp, %ebp - movl 8(%ebp), %edx - movl $0, (%edx) - leavel - retl -.size kthread_mutex_lock_signal, . - kthread_mutex_lock_signal diff --git a/libc/Makefile b/libc/Makefile index 8a32503d..372af5f6 100644 --- a/libc/Makefile +++ b/libc/Makefile @@ -561,6 +561,7 @@ sys/ioctl/ioctl.o \ sys/kernelinfo/kernelinfo.o \ syslog/closelog.o \ syslog/connectlog.o \ +sys/futex/futex.o \ syslog/openlog.o \ syslog/setlogmask.o \ syslog/syslog.o \ diff --git a/libc/include/__/pthread.h b/libc/include/__/pthread.h index 7b85f5e3..b935eaed 100644 --- a/libc/include/__/pthread.h +++ b/libc/include/__/pthread.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014, 2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2017, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -46,40 +46,12 @@ typedef int __pthread_barrier_t; typedef int __pthread_barrierattr_t; -#if defined(__is_sortix_libc) -typedef struct -{ - struct pthread_cond_elem* first; - struct pthread_cond_elem* last; - __clock_t clock; -} __pthread_cond_t; -#else -typedef struct -{ - void* __pthread_first; - void* __pthread_last; - __clock_t __pthread_clock; -} __pthread_cond_t; -#endif - -#if defined(__is_sortix_libc) -typedef struct -{ - __clock_t clock; -} __pthread_condattr_t; -#else -typedef struct -{ - __clock_t __pthread_clock; -} __pthread_condattr_t; -#endif - typedef __SIZE_TYPE__ __pthread_key_t; #if defined(__is_sortix_libc) typedef struct { - unsigned long lock; + int lock; unsigned long type; unsigned long owner; unsigned long recursion; @@ -87,7 +59,7 @@ typedef struct #else typedef struct { - unsigned long __pthread_lock; + int __pthread_lock; unsigned long __pthread_type; unsigned long __pthread_owner; unsigned long __pthread_recursion; @@ -106,6 +78,36 @@ typedef struct } __pthread_mutexattr_t; #endif +#if defined(__is_sortix_libc) +typedef struct +{ + __pthread_mutex_t lock; + struct pthread_cond_elem* first; + struct pthread_cond_elem* last; + __clockid_t clock; +} __pthread_cond_t; +#else +typedef struct +{ + __pthread_mutex_t __pthread_lock; + void* __pthread_first; + void* __pthread_last; + __clockid_t __pthread_clock; +} __pthread_cond_t; +#endif + +#if defined(__is_sortix_libc) +typedef struct +{ + __clockid_t clock; +} __pthread_condattr_t; +#else +typedef struct +{ + __clockid_t __pthread_clock; +} __pthread_condattr_t; +#endif + #if defined(__is_sortix_libc) typedef struct { diff --git a/libc/include/pthread.h b/libc/include/pthread.h index 34f2d165..8a3ad5bc 100644 --- a/libc/include/pthread.h +++ b/libc/include/pthread.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -150,18 +150,21 @@ struct pthread struct pthread_cond_elem { struct pthread_cond_elem* next; - volatile unsigned long woken; + struct pthread_cond_elem* prev; + int woken; }; #endif -#define PTHREAD_COND_INITIALIZER { NULL, NULL, CLOCK_REALTIME } +#define PTHREAD_COND_INITIALIZER { PTHREAD_NORMAL_MUTEX_INITIALIZER_NP, NULL, \ + NULL, CLOCK_REALTIME } #define PTHREAD_MUTEX_INITIALIZER { 0, PTHREAD_MUTEX_DEFAULT, 0, 0 } #define PTHREAD_RWLOCK_INITIALIZER { PTHREAD_COND_INITIALIZER, \ PTHREAD_COND_INITIALIZER, \ PTHREAD_MUTEX_INITIALIZER, 0, 0, 0, 0 } #define PTHREAD_NORMAL_MUTEX_INITIALIZER_NP { 0, PTHREAD_MUTEX_NORMAL, 0, 0 } -#define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP { 0, PTHREAD_MUTEX_RECURSIVE, 0, 0 } +#define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP { 0, PTHREAD_MUTEX_RECURSIVE, \ + 0, 0 } #define PTHREAD_ONCE_INIT { PTHREAD_NORMAL_MUTEX_INITIALIZER_NP, 0 } diff --git a/libc/include/semaphore.h b/libc/include/semaphore.h index 44118551..f0ab92f9 100644 --- a/libc/include/semaphore.h +++ b/libc/include/semaphore.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -32,8 +32,10 @@ typedef struct { #if defined(__is_sortix_libc) int value; + int waiters; #else int __value; + int __waiters; #endif } sem_t; diff --git a/libc/include/sys/futex.h b/libc/include/sys/futex.h new file mode 100644 index 00000000..053f1526 --- /dev/null +++ b/libc/include/sys/futex.h @@ -0,0 +1,30 @@ + /* + * Copyright (c) 2021 Jonas 'Sortie' Termansen. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * sys/futex.h + * Fast userspace mutexes. + */ + +#ifndef _SYS_FUTEX_H +#define _SYS_FUTEX_H 1 + +#include + +#include +#include + +int futex(int*, int, int, const struct timespec*); + +#endif diff --git a/libc/pthread/pthread_cond_broadcast.c b/libc/pthread/pthread_cond_broadcast.c index e2e08be3..af588d75 100644 --- a/libc/pthread/pthread_cond_broadcast.c +++ b/libc/pthread/pthread_cond_broadcast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,13 +17,27 @@ * Broadcasts a condition. */ +#include + +#include #include int pthread_cond_broadcast(pthread_cond_t* cond) { - int ret; + pthread_mutex_lock(&cond->lock); while ( cond->first ) - if ( (ret = pthread_cond_signal(cond)) ) - return ret; + { + struct pthread_cond_elem* elem = cond->first; + if ( elem->next ) + elem->next->prev = elem->prev; + else + cond->last = elem->prev; + cond->first = elem->next; + elem->next = NULL; + elem->prev = NULL; + __atomic_store_n(&elem->woken, 1, __ATOMIC_SEQ_CST); + futex(&elem->woken, FUTEX_WAKE, 1, NULL); + } + pthread_mutex_unlock(&cond->lock); return 0; } diff --git a/libc/pthread/pthread_cond_destroy.c b/libc/pthread/pthread_cond_destroy.c index 0ec78cd4..5c03297e 100644 --- a/libc/pthread/pthread_cond_destroy.c +++ b/libc/pthread/pthread_cond_destroy.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -21,6 +21,6 @@ int pthread_cond_destroy(pthread_cond_t* restrict cond) { - (void) cond; + pthread_mutex_destroy(&cond->lock); return 0; } diff --git a/libc/pthread/pthread_cond_init.c b/libc/pthread/pthread_cond_init.c index 1119e39f..1aaad448 100644 --- a/libc/pthread/pthread_cond_init.c +++ b/libc/pthread/pthread_cond_init.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -29,6 +29,7 @@ int pthread_cond_init(pthread_cond_t* restrict cond, attr = &default_attr; } + pthread_mutex_init(&cond->lock, NULL); cond->first = NULL; cond->last = NULL; cond->clock = attr->clock; diff --git a/libc/pthread/pthread_cond_signal.c b/libc/pthread/pthread_cond_signal.c index b8f5689d..c8171220 100644 --- a/libc/pthread/pthread_cond_signal.c +++ b/libc/pthread/pthread_cond_signal.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,16 +17,27 @@ * Signals a condition. */ +#include + +#include #include int pthread_cond_signal(pthread_cond_t* cond) { - struct pthread_cond_elem* elem = cond->first; - if ( !elem ) - return 0; - if ( !(cond->first = elem->next) ) - cond->last = NULL; - elem->next = NULL; - elem->woken = 1; + pthread_mutex_lock(&cond->lock); + if ( cond->first ) + { + struct pthread_cond_elem* elem = cond->first; + if ( elem->next ) + elem->next->prev = elem->prev; + else + cond->last = elem->prev; + cond->first = elem->next; + elem->next = NULL; + elem->prev = NULL; + __atomic_store_n(&elem->woken, 1, __ATOMIC_SEQ_CST); + futex(&elem->woken, FUTEX_WAKE, 1, NULL); + } + pthread_mutex_unlock(&cond->lock); return 0; } diff --git a/libc/pthread/pthread_cond_timedwait.c b/libc/pthread/pthread_cond_timedwait.c index 731d7c51..53d70149 100644 --- a/libc/pthread/pthread_cond_timedwait.c +++ b/libc/pthread/pthread_cond_timedwait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,35 +17,51 @@ * Waits on a condition or until a timeout happens. */ +#include + #include #include -#include -#include -#include -#include int pthread_cond_timedwait(pthread_cond_t* restrict cond, pthread_mutex_t* restrict mutex, const struct timespec* restrict abstime) { struct pthread_cond_elem elem; + pthread_mutex_lock(&cond->lock); elem.next = NULL; + elem.prev = cond->last; elem.woken = 0; if ( cond->last ) cond->last->next = &elem; - if ( !cond->last ) + if ( !cond->first ) cond->first = &elem; cond->last = &elem; - while ( !elem.woken ) + pthread_mutex_unlock(&cond->lock); + pthread_mutex_unlock(mutex); + int op = FUTEX_WAIT | FUTEX_ABSOLUTE | FUTEX_CLOCK(cond->clock); + int result = 0; + while ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) && + futex(&elem.woken, op, 0, abstime) < 0 ) { - struct timespec now; - if ( clock_gettime(cond->clock, &now) < 0 ) - return errno; - if ( timespec_le(*abstime, now) ) - return errno = ETIMEDOUT; - pthread_mutex_unlock(mutex); - sched_yield(); - pthread_mutex_lock(mutex); + if ( errno == EINTR ) + continue; + if ( errno != EAGAIN ) + result = errno; + break; } - return 0; + pthread_mutex_lock(mutex); + pthread_mutex_lock(&cond->lock); + if ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) ) + { + if ( elem.next ) + elem.next->prev = elem.prev; + else + cond->last = elem.prev; + if ( elem.prev ) + elem.prev->next = elem.next; + else + cond->first = elem.next; + } + pthread_mutex_unlock(&cond->lock); + return result; } diff --git a/libc/pthread/pthread_cond_wait.c b/libc/pthread/pthread_cond_wait.c index 17c8f87a..e30a10f6 100644 --- a/libc/pthread/pthread_cond_wait.c +++ b/libc/pthread/pthread_cond_wait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,28 +17,11 @@ * Waits on a condition. */ +#include #include -#include -#include -#include -#include int pthread_cond_wait(pthread_cond_t* restrict cond, pthread_mutex_t* restrict mutex) { - struct pthread_cond_elem elem; - elem.next = NULL; - elem.woken = 0; - if ( cond->last ) - cond->last->next = &elem; - if ( !cond->last ) - cond->first = &elem; - cond->last = &elem; - while ( !elem.woken ) - { - pthread_mutex_unlock(mutex); - sched_yield(); - pthread_mutex_lock(mutex); - } - return 0; + return pthread_cond_timedwait(cond, mutex, NULL); } diff --git a/libc/pthread/pthread_exit.c b/libc/pthread/pthread_exit.c index 7c93420e..c648269c 100644 --- a/libc/pthread/pthread_exit.c +++ b/libc/pthread/pthread_exit.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -62,7 +62,7 @@ void pthread_exit(void* return_value) { extended.zero_from = &thread->join_lock.lock; extended.zero_size = sizeof(thread->join_lock.lock); - exit_flags |= EXIT_THREAD_ZERO; + exit_flags |= EXIT_THREAD_ZERO | EXIT_THREAD_FUTEX_WAKE; } else { diff --git a/libc/pthread/pthread_mutex_lock.c b/libc/pthread/pthread_mutex_lock.c index 0fa28b4d..581c6bef 100644 --- a/libc/pthread/pthread_mutex_lock.c +++ b/libc/pthread/pthread_mutex_lock.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,22 +17,45 @@ * Locks a mutex. */ -#include -#include -#include -#include +#include -static const unsigned long UNLOCKED_VALUE = 0; -static const unsigned long LOCKED_VALUE = 1; +#include +#include +#include +#include + +static const int UNLOCKED = 0; +static const int LOCKED = 1; +static const int CONTENDED = 2; int pthread_mutex_lock(pthread_mutex_t* mutex) { - while ( !__sync_bool_compare_and_swap(&mutex->lock, UNLOCKED_VALUE, LOCKED_VALUE) ) + int state = UNLOCKED; + int desired = LOCKED; + while ( !__atomic_compare_exchange_n(&mutex->lock, &state, desired, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) { if ( mutex->type == PTHREAD_MUTEX_RECURSIVE && (pthread_t) mutex->owner == pthread_self() ) - return mutex->recursion++, 0; - sched_yield(); + { + if ( mutex->recursion == ULONG_MAX ) + return errno = EAGAIN; + mutex->recursion++; + return 0; + } + if ( state == LOCKED && + !__atomic_compare_exchange_n(&mutex->lock, &state, CONTENDED, + false, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST) ) + { + state = UNLOCKED; + continue; + } + desired = CONTENDED; + if ( futex(&mutex->lock, FUTEX_WAIT, CONTENDED, NULL) < 0 && + errno != EAGAIN && errno != EINTR ) + return errno; + state = UNLOCKED; } mutex->owner = (unsigned long) pthread_self(); mutex->recursion = 0; diff --git a/libc/pthread/pthread_mutex_trylock.c b/libc/pthread/pthread_mutex_trylock.c index b0aa8531..52f6d3fb 100644 --- a/libc/pthread/pthread_mutex_trylock.c +++ b/libc/pthread/pthread_mutex_trylock.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -18,18 +18,27 @@ */ #include +#include #include +#include -static const unsigned long UNLOCKED_VALUE = 0; -static const unsigned long LOCKED_VALUE = 1; +static const int UNLOCKED = 0; +static const int LOCKED = 1; int pthread_mutex_trylock(pthread_mutex_t* mutex) { - if ( !__sync_bool_compare_and_swap(&mutex->lock, UNLOCKED_VALUE, LOCKED_VALUE) ) + int state = UNLOCKED; + if ( !__atomic_compare_exchange_n(&mutex->lock, &state, LOCKED, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) { if ( mutex->type == PTHREAD_MUTEX_RECURSIVE && (pthread_t) mutex->owner == pthread_self() ) - return mutex->recursion++, 0; + { + if ( mutex->recursion == ULONG_MAX ) + return errno = EAGAIN; + mutex->recursion++; + return 0; + } return errno = EBUSY; } mutex->owner = (unsigned long) pthread_self(); diff --git a/libc/pthread/pthread_mutex_unlock.c b/libc/pthread/pthread_mutex_unlock.c index 25f29d90..0d78f0b5 100644 --- a/libc/pthread/pthread_mutex_unlock.c +++ b/libc/pthread/pthread_mutex_unlock.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,16 +17,24 @@ * Unlocks a mutex. */ -#include +#include -static const unsigned long UNLOCKED_VALUE = 0; -static const unsigned long LOCKED_VALUE = 1; +#include +#include + +static const int UNLOCKED = 0; +static const int CONTENDED = 2; int pthread_mutex_unlock(pthread_mutex_t* mutex) { if ( mutex->type == PTHREAD_MUTEX_RECURSIVE && mutex->recursion ) - return mutex->recursion--, 0; + { + mutex->recursion--; + return 0; + } mutex->owner = 0; - mutex->lock = UNLOCKED_VALUE; + if ( __atomic_exchange_n(&mutex->lock, UNLOCKED, + __ATOMIC_SEQ_CST) == CONTENDED ) + futex(&mutex->lock, FUTEX_WAKE, 1, NULL); return 0; } diff --git a/libc/semaphore/sem_getvalue.c b/libc/semaphore/sem_getvalue.c index 59d57944..cc983e67 100644 --- a/libc/semaphore/sem_getvalue.c +++ b/libc/semaphore/sem_getvalue.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -21,6 +21,7 @@ int sem_getvalue(sem_t* restrict sem, int* restrict value_ptr) { - *value_ptr = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + int value = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + *value_ptr = value == -1 ? 0 : value; return 0; } diff --git a/libc/semaphore/sem_init.c b/libc/semaphore/sem_init.c index a3448066..4301ac71 100644 --- a/libc/semaphore/sem_init.c +++ b/libc/semaphore/sem_init.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -25,11 +25,9 @@ int sem_init(sem_t* sem, int pshared, unsigned int value) { if ( pshared ) return errno = ENOSYS, -1; - if ( (unsigned int) INT_MAX < value ) return errno = EINVAL, -1; - sem->value = (int) value; - + sem->waiters = 0; return 0; } diff --git a/libc/semaphore/sem_post.c b/libc/semaphore/sem_post.c index fbd515d4..b7103de2 100644 --- a/libc/semaphore/sem_post.c +++ b/libc/semaphore/sem_post.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,25 +17,28 @@ * Unlock a semaphore. */ +#include + #include #include #include #include +#include int sem_post(sem_t* sem) { while ( true ) { - int old_value = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); - if ( old_value == INT_MAX ) - return errno = EOVERFLOW; - - int new_value = old_value + 1; - if ( !__atomic_compare_exchange_n(&sem->value, &old_value, new_value, - false, + int old = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + if ( old == INT_MAX ) + return errno = EOVERFLOW, -1; + int new = old == -1 ? 1 : old + 1; + int waiters = __atomic_load_n(&sem->waiters, __ATOMIC_SEQ_CST); + if ( !__atomic_compare_exchange_n(&sem->value, &old, new, false, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED) ) continue; - + if ( old == -1 || waiters ) + futex(&sem->value, FUTEX_WAKE, 1, NULL); return 0; } } diff --git a/libc/semaphore/sem_timedwait.c b/libc/semaphore/sem_timedwait.c index 04eb7bb9..d7fcbb28 100644 --- a/libc/semaphore/sem_timedwait.c +++ b/libc/semaphore/sem_timedwait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,57 +14,39 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * * semaphore/sem_timedwait.c - * Lock a semaphore. + * Lock a semaphore with a timeout. */ +#include + #include -#include #include -#include -#include +#include #include -#include int sem_timedwait(sem_t* restrict sem, const struct timespec* restrict abstime) { - if ( sem_trywait(sem) == 0 ) - return 0; - if ( errno != EAGAIN ) - return -1; - - sigset_t old_set_mask; - sigset_t old_set_allowed; - sigset_t all_signals; - sigfillset(&all_signals); - sigprocmask(SIG_SETMASK, &all_signals, &old_set_mask); - signotset(&old_set_allowed, &old_set_mask); - - while ( sem_trywait(sem) != 0 ) + while ( true ) { - // TODO: Using CLOCK_REALTIME for this is bad as it is not monotonic. We - // need to enchance the semaphore API so a better clock can be - // used instead. - if ( errno == EAGAIN ) + int old = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + int new = old != -1 ? old - 1 : -1; + bool waiting = new == -1; + if ( waiting ) + __atomic_add_fetch(&sem->waiters, 1, __ATOMIC_SEQ_CST); + if ( old != new && + !__atomic_compare_exchange_n(&sem->value, &old, new, false, + __ATOMIC_SEQ_CST, __ATOMIC_RELAXED) ) { - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - if ( timespec_le(*abstime, now) ) - errno = ETIMEDOUT; + if ( waiting ) + __atomic_sub_fetch(&sem->waiters, 1, __ATOMIC_SEQ_CST); + continue; } - - if ( errno == EAGAIN && sigpending(&old_set_allowed) ) - errno = EINTR; - - if ( errno != EAGAIN ) - { - sigprocmask(SIG_SETMASK, &old_set_mask, NULL); + if ( !waiting ) + return 0; + int op = FUTEX_WAIT | FUTEX_ABSOLUTE | FUTEX_CLOCK(CLOCK_REALTIME); + int ret = futex(&sem->value, op, -1, abstime); + __atomic_sub_fetch(&sem->waiters, 1, __ATOMIC_SEQ_CST); + if ( ret < 0 && errno != EAGAIN ) return -1; - } - - sched_yield(); } - - sigprocmask(SIG_SETMASK, &old_set_mask, NULL); - - return 0; } diff --git a/libc/semaphore/sem_trywait.c b/libc/semaphore/sem_trywait.c index 40a720f4..49f20e49 100644 --- a/libc/semaphore/sem_trywait.c +++ b/libc/semaphore/sem_trywait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,7 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * * semaphore/sem_trywait.c - * Lock a semaphore. + * Try to lock a semaphore. */ #include @@ -23,11 +23,11 @@ int sem_trywait(sem_t* sem) { - int old_value = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); - if ( old_value <= 0 ) + int old = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + if ( old <= 0 ) return errno = EAGAIN, -1; - int new_value = old_value - 1; - if ( !__atomic_compare_exchange_n(&sem->value, &old_value, new_value, false, + int new = old != -1 ? old - 1 : -1; + if ( !__atomic_compare_exchange_n(&sem->value, &old, new, false, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED) ) return errno = EAGAIN, -1; return 0; diff --git a/libc/semaphore/sem_wait.c b/libc/semaphore/sem_wait.c index c5155977..bbd0f2d0 100644 --- a/libc/semaphore/sem_wait.c +++ b/libc/semaphore/sem_wait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,41 +17,10 @@ * Lock a semaphore. */ -#include -#include #include -#include #include int sem_wait(sem_t* sem) { - if ( sem_trywait(sem) == 0 ) - return 0; - if ( errno != EAGAIN ) - return -1; - - sigset_t old_set_mask; - sigset_t old_set_allowed; - sigset_t all_signals; - sigfillset(&all_signals); - sigprocmask(SIG_SETMASK, &all_signals, &old_set_mask); - signotset(&old_set_allowed, &old_set_mask); - - while ( sem_trywait(sem) != 0 ) - { - if ( errno == EAGAIN && sigpending(&old_set_allowed) ) - errno = EINTR; - - if ( errno != EAGAIN ) - { - sigprocmask(SIG_SETMASK, &old_set_mask, NULL); - return -1; - } - - sched_yield(); - } - - sigprocmask(SIG_SETMASK, &old_set_mask, NULL); - - return 0; + return sem_timedwait(sem, NULL); } diff --git a/libc/sys/futex/futex.c b/libc/sys/futex/futex.c new file mode 100644 index 00000000..24f12e86 --- /dev/null +++ b/libc/sys/futex/futex.c @@ -0,0 +1,29 @@ + /* + * Copyright (c) 2021 Jonas 'Sortie' Termansen. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * sys/futex/futex.c + * Fast userspace mutexes. + */ + +#include +#include + +DEFN_SYSCALL4(int, sys_futex, SYSCALL_FUTEX, int*, int, int, + const struct timespec*); + +int futex(int* address, int op, int value, const struct timespec* timeout) +{ + return sys_futex(address, op, value, timeout); +} diff --git a/share/man/man7/following-development.7 b/share/man/man7/following-development.7 index a6cacd06..c3a9599a 100644 --- a/share/man/man7/following-development.7 +++ b/share/man/man7/following-development.7 @@ -69,6 +69,16 @@ releasing Sortix x.y, foo." to allow the maintainer to easily .Xr grep 1 for it after a release. .Sh CHANGES +.Ss Implement threading primitives that truly sleep +The +.Xr futex 2 +system system call for efficient thread waiting has been added. +The +.Xr exit_thread 2 +system call has gained a +.Dv EXIT_THREAD_FUTEX_WAKE +flag for waking a single waiter on a futex. +This is a minor compatible ABI change. .Ss Fix system upgrade leaking files .Xr sysupgrade 8 and