diff --git a/Makefile b/Makefile index 0332676f..9f827cee 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ sysroot-system: sysroot-fsh sysroot-base-headers echo 'ID=sortix' && \ echo 'VERSION_ID="$(VERSION)"' && \ echo 'PRETTY_NAME="Sortix $(VERSION)"' && \ - echo 'SORTIX_ABI=1.0' && \ + echo 'SORTIX_ABI=1.1' && \ true) > "$(SYSROOT)/etc/sortix-release" echo /etc/sortix-release >> "$(SYSROOT)/tix/manifest/system" ln -sf sortix-release "$(SYSROOT)/etc/os-release" diff --git a/kernel/Makefile b/kernel/Makefile index 7a6a61f3..69374bbb 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -82,7 +82,6 @@ alarm.o \ clock.o \ com.o \ copy.o \ -$(CPUDIR)/kthread.o \ descriptor.o \ disk/ahci/ahci.o \ disk/ahci/hba.o \ diff --git a/kernel/clock.cpp b/kernel/clock.cpp index f3cab609..81bb3da7 100644 --- a/kernel/clock.cpp +++ b/kernel/clock.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2016, 2017, 2018 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2016, 2017, 2018, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -260,60 +261,61 @@ bool Clock::TryCancel(Timer* timer) return active; } +static void timer_wakeup(Clock* /*clock*/, Timer* /*timer*/, void* ctx) +{ + Thread* thread = (Thread*) ctx; + thread->timer_woken = true; + kthread_wake_futex(thread); +} -// TODO: We need some method for threads to sleep for real but still be -// interrupted by signals. struct timespec Clock::SleepDelay(struct timespec duration) { - struct timespec start_advancement; - struct timespec elapsed = timespec_nul(); - bool start_advancement_set = false; - - while ( timespec_lt(elapsed, duration) ) - { - if ( start_advancement_set ) - { - if ( Signal::IsPending() ) - return duration; - - kthread_yield(); - } - - LockClock(); - - if ( !start_advancement_set ) - { - start_advancement = current_advancement; - start_advancement_set = true; - } - - elapsed = timespec_sub(current_advancement, start_advancement); - - UnlockClock(); - } - + LockClock(); + struct timespec start_advancement = current_advancement; + UnlockClock(); + Thread* thread = CurrentThread(); + thread->futex_woken = false; + thread->timer_woken = false; + Timer timer; + timer.Attach(this); + struct itimerspec timerspec; + timerspec.it_value = duration; + timerspec.it_interval.tv_sec = 0; + timerspec.it_interval.tv_nsec = 0; + int timer_flags = TIMER_FUNC_INTERRUPT_HANDLER; + timer.Set(&timerspec, NULL, timer_flags, timer_wakeup, thread); + kthread_wait_futex_signal(); + timer.Cancel(); + LockClock(); + struct timespec end_advancement = current_advancement; + UnlockClock(); + struct timespec elapsed = timespec_sub(end_advancement, start_advancement); + if ( timespec_lt(elapsed, duration) ) + return timespec_sub(duration, elapsed); return timespec_nul(); } -// TODO: We need some method for threads to sleep for real but still be -// interrupted by signals. struct timespec Clock::SleepUntil(struct timespec expiration) { - while ( true ) - { - LockClock(); - struct timespec now = current_time; - UnlockClock(); - - if ( timespec_le(expiration, now) ) - break; - - if ( Signal::IsPending() ) - return timespec_sub(expiration, now); - - kthread_yield(); - } - + Thread* thread = CurrentThread(); + thread->futex_woken = false; + thread->timer_woken = false; + Timer timer; + timer.Attach(this); + struct itimerspec timerspec; + timerspec.it_value = expiration; + timerspec.it_interval.tv_sec = 0; + timerspec.it_interval.tv_nsec = 0; + int timer_flags = TIMER_ABSOLUTE | TIMER_FUNC_INTERRUPT_HANDLER; + timer.Set(&timerspec, NULL, timer_flags, timer_wakeup, thread); + kthread_wait_futex_signal(); + timer.Cancel(); + LockClock(); + struct timespec now = current_time; + UnlockClock(); + struct timespec remaining = timespec_sub(expiration, now); + if ( timespec_lt(timespec_nul(), remaining) ) + return remaining; return timespec_nul(); } diff --git a/kernel/copy.cpp b/kernel/copy.cpp index f1bdd874..2b2c8c41 100644 --- a/kernel/copy.cpp +++ b/kernel/copy.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -129,6 +129,24 @@ bool CopyFromUser(void* kdst_ptr, const void* usersrc_ptr, size_t count) return result; } +bool ReadAtomicFromUser(int* kdst_ptr, const int* usersrc_ptr) +{ + uintptr_t usersrc = (uintptr_t) usersrc_ptr; + if ( usersrc & (sizeof(int) - 1) ) + return errno = EINVAL, false; + Process* process = CurrentProcess(); + assert(IsInProcessAddressSpace(process)); + ScopedLock lock(&process->segment_lock); + struct segment* segment = FindSegment(process, usersrc); + if ( !segment || !(segment->prot & PROT_READ) ) + return errno = EFAULT, false; + size_t segment_available = segment->addr + segment->size - usersrc; + if ( segment_available < sizeof(int) ) + return errno = EFAULT, false; + *kdst_ptr = __atomic_load_n(usersrc_ptr, __ATOMIC_SEQ_CST); + return true; +} + bool CopyToKernel(void* kdst, const void* ksrc, size_t count) { memcpy(kdst, ksrc, count); diff --git a/kernel/include/sortix/exit.h b/kernel/include/sortix/exit.h index 7e9a446f..9132b2c3 100644 --- a/kernel/include/sortix/exit.h +++ b/kernel/include/sortix/exit.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -51,6 +51,7 @@ struct exit_thread #define EXIT_THREAD_TLS_UNMAP (1<<3) #define EXIT_THREAD_PROCESS (1<<4) #define EXIT_THREAD_DUMP_CORE (1<<5) +#define EXIT_THREAD_FUTEX_WAKE (1<<6) #ifdef __cplusplus } /* extern "C" */ diff --git a/kernel/include/sortix/futex.h b/kernel/include/sortix/futex.h new file mode 100644 index 00000000..5b7f9886 --- /dev/null +++ b/kernel/include/sortix/futex.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021 Jonas 'Sortie' Termansen. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * sortix/futex.h + * Fast userspace mutexes. + */ + +#ifndef INCLUDE_SORTIX_FUTEX_H +#define INCLUDE_SORTIX_FUTEX_H + +#include + +#define FUTEX_WAIT 1 +#define FUTEX_WAKE 2 + +#define FUTEX_ABSOLUTE (1 << 8) + +#define FUTEX_CLOCK(clock) (clock << 24) + +#define FUTEX_GET_OP(op) ((op) & 0xFF) +#define FUTEX_GET_CLOCK(op) ((op) >> 24) + +#endif diff --git a/kernel/include/sortix/kernel/copy.h b/kernel/include/sortix/kernel/copy.h index 16663b63..ced4322f 100644 --- a/kernel/include/sortix/kernel/copy.h +++ b/kernel/include/sortix/kernel/copy.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -28,6 +28,7 @@ bool CopyToUser(void* userdst, const void* ksrc, size_t count); bool CopyFromUser(void* kdst, const void* usersrc, size_t count); bool CopyToKernel(void* kdst, const void* ksrc, size_t count); bool CopyFromKernel(void* kdst, const void* ksrc, size_t count); +bool ReadAtomicFromUser(int* kdst, const int* usersrc); bool ZeroKernel(void* kdst, size_t count); bool ZeroUser(void* userdst, size_t count); char* GetStringFromUser(const char* str); diff --git a/kernel/include/sortix/kernel/interrupt.h b/kernel/include/sortix/kernel/interrupt.h index 118380b4..fcf1bdf2 100644 --- a/kernel/include/sortix/kernel/interrupt.h +++ b/kernel/include/sortix/kernel/interrupt.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2011, 2012, 2013, 2014, 2017, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -110,12 +110,12 @@ inline bool IsCPUInterrupted() inline bool SetEnabled(bool is_enabled) { - bool wasenabled = IsEnabled(); + bool was_enabled = IsEnabled(); if ( is_enabled ) Enable(); else Disable(); - return wasenabled; + return was_enabled; } void RegisterHandler(unsigned int index, struct interrupt_handler* handler); diff --git a/kernel/include/sortix/kernel/kthread.h b/kernel/include/sortix/kernel/kthread.h index 6cfb354a..59a6b896 100644 --- a/kernel/include/sortix/kernel/kthread.h +++ b/kernel/include/sortix/kernel/kthread.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -26,15 +26,18 @@ namespace Sortix { -extern "C" { +class Thread; -inline static void kthread_yield(void) { asm volatile ("int $129"); } +void kthread_yield(); +void kthread_wait_futex(); +void kthread_wait_futex_signal(); +void kthread_wake_futex(Thread* thread); __attribute__((noreturn)) void kthread_exit(); -typedef unsigned kthread_mutex_t; +typedef int kthread_mutex_t; const kthread_mutex_t KTHREAD_MUTEX_INITIALIZER = 0; -unsigned kthread_mutex_trylock(kthread_mutex_t* mutex); +bool kthread_mutex_trylock(kthread_mutex_t* mutex); void kthread_mutex_lock(kthread_mutex_t* mutex); -unsigned long kthread_mutex_lock_signal(kthread_mutex_t* mutex); +bool kthread_mutex_lock_signal(kthread_mutex_t* mutex); void kthread_mutex_unlock(kthread_mutex_t* mutex); struct kthread_cond_elem; typedef struct kthread_cond_elem kthread_cond_elem_t; @@ -46,12 +49,10 @@ struct kthread_cond typedef struct kthread_cond kthread_cond_t; const kthread_cond_t KTHREAD_COND_INITIALIZER = { NULL, NULL }; void kthread_cond_wait(kthread_cond_t* cond, kthread_mutex_t* mutex); -unsigned long kthread_cond_wait_signal(kthread_cond_t* cond, kthread_mutex_t* mutex); +bool kthread_cond_wait_signal(kthread_cond_t* cond, kthread_mutex_t* mutex); void kthread_cond_signal(kthread_cond_t* cond); void kthread_cond_broadcast(kthread_cond_t* cond); -} /* extern "C" */ - class ScopedLock { public: diff --git a/kernel/include/sortix/kernel/process.h b/kernel/include/sortix/kernel/process.h index 089c1942..70ec1edc 100644 --- a/kernel/include/sortix/kernel/process.h +++ b/kernel/include/sortix/kernel/process.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 205, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -138,6 +138,11 @@ public: size_t threads_not_exiting_count; bool threads_exiting; +public: + kthread_mutex_t futex_lock; + Thread* futex_first_waiting; + Thread* futex_last_waiting; + public: struct segment* segments; size_t segments_used; diff --git a/kernel/include/sortix/kernel/scheduler.h b/kernel/include/sortix/kernel/scheduler.h index 1618c64f..139bd310 100644 --- a/kernel/include/sortix/kernel/scheduler.h +++ b/kernel/include/sortix/kernel/scheduler.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2011, 2012, 2013, 2014, 2017, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -29,28 +29,16 @@ class Thread; } // namespace Sortix namespace Sortix { -enum ThreadState { NONE, RUNNABLE, BLOCKING, DEAD }; +enum ThreadState { NONE, RUNNABLE, FUTEX_WAITING, DEAD }; } // namespace Sortix namespace Sortix { namespace Scheduler { -#if defined(__i386__) || defined(__x86_64__) -static inline void Yield() -{ - asm volatile ("int $129"); -} -__attribute__ ((noreturn)) -static inline void ExitThread() -{ - asm volatile ("int $132"); - __builtin_unreachable(); -} -#endif - void Switch(struct interrupt_context* intctx); void SwitchTo(struct interrupt_context* intctx, Thread* new_thread); -void SetThreadState(Thread* thread, ThreadState state); +void SetThreadState(Thread* thread, ThreadState state, bool wake_only = false); +void SetSignalPending(Thread* thread, unsigned long is_pending); ThreadState GetThreadState(Thread* thread); void SetIdleThread(Thread* thread); void SetInitProcess(Process* init); diff --git a/kernel/include/sortix/kernel/syscall.h b/kernel/include/sortix/kernel/syscall.h index b1ef64d8..9dd93c72 100644 --- a/kernel/include/sortix/kernel/syscall.h +++ b/kernel/include/sortix/kernel/syscall.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -89,6 +89,7 @@ int sys_fstatvfs(int, struct statvfs*); int sys_fstatvfsat(int, const char*, struct statvfs*, int); int sys_fsync(int); int sys_ftruncate(int, off_t); +int sys_futex(int*, int, int, const struct timespec*); int sys_futimens(int, const struct timespec*); int sys_getdnsconfig(struct dnsconfig*); gid_t sys_getegid(void); diff --git a/kernel/include/sortix/kernel/thread.h b/kernel/include/sortix/kernel/thread.h index ca44500d..829e87e6 100644 --- a/kernel/include/sortix/kernel/thread.h +++ b/kernel/include/sortix/kernel/thread.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2016, 2018 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2018, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -39,19 +39,30 @@ class Process; class Thread; // These functions create a new kernel process but doesn't start it. -Thread* CreateKernelThread(Process* process, struct thread_registers* regs); +Thread* CreateKernelThread(Process* process, struct thread_registers* regs, + const char* name); Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user, + const char* name, size_t stacksize = 0); +Thread* CreateKernelThread(void (*entry)(void*), void* user, const char* name, size_t stacksize = 0); -Thread* CreateKernelThread(void (*entry)(void*), void* user, size_t stacksize = 0); // This function can be used to start a thread from the above functions. void StartKernelThread(Thread* thread); // Alternatively, these functions both create and start the thread. -Thread* RunKernelThread(Process* process, struct thread_registers* regs); +Thread* RunKernelThread(Process* process, struct thread_registers* regs, + const char* name); Thread* RunKernelThread(Process* process, void (*entry)(void*), void* user, + const char* name, size_t stacksize = 0); +Thread* RunKernelThread(void (*entry)(void*), void* user, const char* name, size_t stacksize = 0); -Thread* RunKernelThread(void (*entry)(void*), void* user, size_t stacksize = 0); + +enum yield_operation +{ + YIELD_OPERATION_NONE, + YIELD_OPERATION_WAIT_FUTEX, + YIELD_OPERATION_WAIT_FUTEX_SIGNAL, +}; class Thread { @@ -60,6 +71,7 @@ public: ~Thread(); public: + const char* name; uintptr_t system_tid; uintptr_t yield_to_tid; struct thread_registers registers; @@ -87,6 +99,12 @@ public: bool has_saved_signal_mask; Clock execute_clock; Clock system_clock; + uintptr_t futex_address; + bool futex_woken; + bool timer_woken; + Thread* futex_prev_waiting; + Thread* futex_next_waiting; + enum yield_operation yield_operation; public: void HandleSignal(struct interrupt_context* intctx); diff --git a/kernel/include/sortix/syscall.h b/kernel/include/sortix/syscall.h index 73d137e5..194038a7 100644 --- a/kernel/include/sortix/syscall.h +++ b/kernel/include/sortix/syscall.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -188,6 +188,7 @@ #define SYSCALL_SOCKET 165 #define SYSCALL_GETDNSCONFIG 166 #define SYSCALL_SETDNSCONFIG 167 -#define SYSCALL_MAX_NUM 168 /* index of highest constant + 1 */ +#define SYSCALL_FUTEX 168 +#define SYSCALL_MAX_NUM 169 /* index of highest constant + 1 */ #endif diff --git a/kernel/interlock.cpp b/kernel/interlock.cpp index 00f4eefa..f11c0a2d 100644 --- a/kernel/interlock.cpp +++ b/kernel/interlock.cpp @@ -23,12 +23,14 @@ namespace Sortix { // TODO: This is likely not the most optimal way to perform these operations. +// TODO: Just discard this whole thing in favor of __atomic_? ilret_t InterlockedModify(unsigned long* ptr, ilockfunc f, unsigned long user) { unsigned long old_value, new_value; + // TODO: Migrate to __atomic. do { old_value = *((volatile unsigned long*) ptr); /* TODO: Need volatile? */ diff --git a/kernel/interrupt.cpp b/kernel/interrupt.cpp index 23f7ee66..f9e41151 100644 --- a/kernel/interrupt.cpp +++ b/kernel/interrupt.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2016, 2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2017, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -24,6 +24,7 @@ #include #include #include +#include namespace Sortix { namespace Interrupt { @@ -33,12 +34,16 @@ bool interrupt_worker_thread_boost = false; static struct interrupt_work* first; static struct interrupt_work* last; +static bool interrupt_worker_idle = false; void WorkerThread(void* /*user*/) { + Thread* thread = CurrentThread(); assert(Interrupt::IsEnabled()); while ( true ) { + thread->futex_woken = false; + thread->timer_woken = false; struct interrupt_work* work; Interrupt::Disable(); work = first; @@ -47,8 +52,9 @@ void WorkerThread(void* /*user*/) Interrupt::Enable(); if ( !work ) { - // TODO: Make this thread not run until work arrives. - kthread_yield(); + interrupt_worker_idle = true; + kthread_wait_futex(); + interrupt_worker_idle = false; continue; } while ( work ) @@ -67,6 +73,11 @@ void ScheduleWork(struct interrupt_work* work) work->next = NULL; last = work; interrupt_worker_thread_boost = true; + if ( interrupt_worker_idle ) + { + interrupt_worker_thread->futex_woken = true; + kthread_wake_futex(interrupt_worker_thread); + } } } // namespace Interrupt diff --git a/kernel/kernel.cpp b/kernel/kernel.cpp index 652c6f1d..430c2d57 100644 --- a/kernel/kernel.cpp +++ b/kernel/kernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2018, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -193,7 +193,6 @@ extern "C" void KernelInit(unsigned long magic, multiboot_info_t* bootinfo_p) // Display the logo. Log::PrintF("\e[37;41m\e[2J"); Log::Center(BRAND_LOGO); - #if defined(__x86_64__) // TODO: Remove this hack when qemu 1.4.x and 1.5.0 are obsolete. // Verify that we are not running under a buggy qemu where the instruction @@ -375,6 +374,7 @@ extern "C" void KernelInit(unsigned long magic, multiboot_info_t* bootinfo_p) // scheduler's set of runnable threads, but rather run whenever there is // _nothing_ else to run on this CPU. Thread* idlethread = AllocateThread(); + idlethread->name = "idle"; idlethread->process = system; idlethread->kernelstackpos = (addr_t) stack; idlethread->kernelstacksize = STACK_SIZE; @@ -387,7 +387,7 @@ extern "C" void KernelInit(unsigned long magic, multiboot_info_t* bootinfo_p) // Note that we don't do the work here: if all other threads are not running // and this thread isn't runnable, then there is nothing to run. Therefore // we must become the system idle thread. - RunKernelThread(BootThread, NULL); + RunKernelThread(BootThread, NULL, "boot"); // The time driver will run the scheduler on the next timer interrupt. Time::Start(); @@ -401,7 +401,14 @@ static void SystemIdleThread(void* /*user*/) // Alright, we are now the system idle thread. If there is nothing to do, // then we are run. Note that we must never do any real work here as the // idle thread must always be runnable. - while(true); + while ( true ) + { +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("hlt"); +#else +#warning "Implement a power efficient kernel idle thread" +#endif + } } static void BootThread(void* /*user*/) @@ -425,7 +432,7 @@ static void BootThread(void* /*user*/) // Let's create the interrupt worker thread that executes additional work // requested by interrupt handlers, where such work isn't safe. Interrupt::interrupt_worker_thread = - RunKernelThread(Interrupt::WorkerThread, NULL); + RunKernelThread(Interrupt::WorkerThread, NULL, "interrupt"); if ( !Interrupt::interrupt_worker_thread ) Panic("Could not create interrupt worker"); @@ -433,7 +440,7 @@ static void BootThread(void* /*user*/) Worker::Init(); // Create a general purpose worker thread. - Thread* worker_thread = RunKernelThread(Worker::Thread, NULL); + Thread* worker_thread = RunKernelThread(Worker::Thread, NULL, "worker"); if ( !worker_thread ) Panic("Unable to create general purpose worker thread"); @@ -644,7 +651,7 @@ static void BootThread(void* /*user*/) init->addrspace = initaddrspace; Scheduler::SetInitProcess(init); - Thread* initthread = RunKernelThread(init, InitThread, NULL); + Thread* initthread = RunKernelThread(init, InitThread, NULL, "main"); if ( !initthread ) Panic("Could not create init thread"); diff --git a/kernel/kthread.cpp b/kernel/kthread.cpp index cb8783be..fe822927 100644 --- a/kernel/kthread.cpp +++ b/kernel/kthread.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,6 +17,8 @@ * Utility and synchronization mechanisms for kernel threads. */ +#include + #include #include @@ -27,8 +29,195 @@ #include #include +#include "uart.h" + namespace Sortix { +void kthread_yield() +{ + Thread* thread = CurrentThread(); + thread->yield_operation = YIELD_OPERATION_NONE; +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("int $129"); +#else +#error "kthread_yield needs to be implemented" +#endif +} + +void kthread_wait_futex() +{ + Thread* thread = CurrentThread(); + thread->yield_operation = YIELD_OPERATION_WAIT_FUTEX; +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("int $129"); +#else +#error "kthread_wait_futex needs to be implemented" +#endif +} + +void kthread_wait_futex_signal() +{ + Thread* thread = CurrentThread(); + thread->yield_operation = YIELD_OPERATION_WAIT_FUTEX_SIGNAL; +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("int $129"); +#else +#error "kthread_wait_futex needs to be implemented" +#endif +} + +void kthread_wake_futex(Thread* thread) +{ + Scheduler::SetThreadState(thread, ThreadState::RUNNABLE, true); +} + +static const int UNLOCKED = 0; +static const int LOCKED = 1; +static const int CONTENDED = 2; + +void kthread_spinlock_lock(kthread_mutex_t* mutex) +{ + while ( true ) + { + int state = UNLOCKED; + int desired = LOCKED; + if ( __atomic_compare_exchange_n(mutex, &state, desired, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + break; + } +} + +void kthread_spinlock_unlock(kthread_mutex_t* mutex) +{ + __atomic_store_n(mutex, UNLOCKED, __ATOMIC_SEQ_CST); +} + +static bool kutex_wait(int* address, int value, bool signal) +{ + // TODO: Use a per-mutex wait queue instead. + Thread* thread = CurrentThread(); + Process* kernel_process = Scheduler::GetKernelProcess(); + bool was_enabled = Interrupt::SetEnabled(false); + kthread_spinlock_lock(&kernel_process->futex_lock); + thread->futex_address = (uintptr_t) address; + thread->futex_woken = false; + thread->futex_prev_waiting = kernel_process->futex_last_waiting; + thread->futex_next_waiting = NULL; + (kernel_process->futex_last_waiting ? + kernel_process->futex_last_waiting->futex_next_waiting : + kernel_process->futex_first_waiting) = thread; + kernel_process->futex_last_waiting = thread; + kthread_spinlock_unlock(&kernel_process->futex_lock); + Interrupt::SetEnabled(was_enabled); + thread->timer_woken = false; + bool result = true; + if ( __atomic_load_n(address, __ATOMIC_SEQ_CST) == value ) + { + if ( signal ) + kthread_wait_futex(); + else + kthread_wait_futex_signal(); + } + Interrupt::SetEnabled(false); + kthread_spinlock_lock(&kernel_process->futex_lock); + if ( result && !thread->futex_woken ) + { + if ( signal && Signal::IsPending() ) + result = false; + } + thread->futex_address = 0; + thread->futex_woken = false; + (thread->futex_prev_waiting ? + thread->futex_prev_waiting->futex_next_waiting : + kernel_process->futex_first_waiting) = thread->futex_next_waiting; + (thread->futex_next_waiting ? + thread->futex_next_waiting->futex_prev_waiting : + kernel_process->futex_last_waiting) = thread->futex_prev_waiting; + thread->futex_prev_waiting = NULL; + thread->futex_next_waiting = NULL; + kthread_spinlock_unlock(&kernel_process->futex_lock); + Interrupt::SetEnabled(was_enabled); + return result; +} + +static void kutex_wake(int* address, int count) +{ + Process* kernel_process = Scheduler::GetKernelProcess(); + bool was_enabled = Interrupt::SetEnabled(false); + kthread_spinlock_lock(&kernel_process->futex_lock); + for ( Thread* waiter = kernel_process->futex_first_waiting; + 0 < count && waiter; + waiter = waiter->futex_next_waiting ) + { + if ( waiter->futex_address == (uintptr_t) address ) + { + waiter->futex_woken = true; + kthread_wake_futex(waiter); + if ( count != INT_MAX ) + count--; + } + } + kthread_spinlock_unlock(&kernel_process->futex_lock); + Interrupt::SetEnabled(was_enabled); +} + +bool kthread_mutex_trylock(kthread_mutex_t* mutex) +{ + int state = UNLOCKED; + if ( !__atomic_compare_exchange_n(mutex, &state, LOCKED, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + return false; + return true; +} + +void kthread_mutex_lock(kthread_mutex_t* mutex) +{ + int state = UNLOCKED; + int desired = LOCKED; + while ( !__atomic_compare_exchange_n(mutex, &state, desired, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + { + if ( state == LOCKED && + !__atomic_compare_exchange_n(mutex, &state, CONTENDED, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + { + state = UNLOCKED; + continue; + } + desired = CONTENDED; + kutex_wait(mutex, CONTENDED, false); + state = UNLOCKED; + } +} + +bool kthread_mutex_lock_signal(kthread_mutex_t* mutex) +{ + int state = UNLOCKED; + int desired = LOCKED; + while ( !__atomic_compare_exchange_n(mutex, &state, desired, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + { + if ( state == LOCKED && + !__atomic_compare_exchange_n(mutex, &state, CONTENDED, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) + { + state = UNLOCKED; + continue; + } + desired = CONTENDED; + if ( !kutex_wait(mutex, CONTENDED, true) ) + return false; + state = UNLOCKED; + } + return true; +} + +void kthread_mutex_unlock(kthread_mutex_t* mutex) +{ + if ( __atomic_exchange_n(mutex, UNLOCKED, __ATOMIC_SEQ_CST) == CONTENDED ) + kutex_wake(mutex, 1); +} + // The kernel thread needs another stack to delete its own stack. static void kthread_do_kill_thread(void* user) { @@ -38,7 +227,7 @@ static void kthread_do_kill_thread(void* user) FreeThread(thread); } -extern "C" void kthread_exit() +void kthread_exit() { Process* process = CurrentProcess(); // Note: This requires all threads in this process to have been made by @@ -55,84 +244,109 @@ extern "C" void kthread_exit() if ( is_last_to_exit ) process->OnLastThreadExit(); Worker::Schedule(kthread_do_kill_thread, CurrentThread()); - Scheduler::ExitThread(); +#if defined(__i386__) || defined(__x86_64__) + asm volatile ("int $132"); +#else +#error "kthread_exit needs to be implemented" +#endif __builtin_unreachable(); } struct kthread_cond_elem { kthread_cond_elem_t* next; - volatile unsigned long woken; + kthread_cond_elem_t* prev; + int woken; }; -extern "C" void kthread_cond_wait(kthread_cond_t* cond, kthread_mutex_t* mutex) +void kthread_cond_wait(kthread_cond_t* cond, kthread_mutex_t* mutex) { kthread_cond_elem_t elem; elem.next = NULL; + elem.prev = cond->last; elem.woken = 0; - if ( cond->last ) { cond->last->next = &elem; } - if ( !cond->last ) { cond->first = &elem; } + if ( cond->last ) + cond->last->next = &elem; + if ( !cond->first ) + cond->first = &elem; cond->last = &elem; - while ( !elem.woken ) + kthread_mutex_unlock(mutex); + while ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) && + kutex_wait(&elem.woken, 0, false) < 0 ); + kthread_mutex_lock(mutex); + if ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) ) { - kthread_mutex_unlock(mutex); - Scheduler::Yield(); - kthread_mutex_lock(mutex); + if ( elem.next ) + elem.next->prev = elem.prev; + else + cond->last = elem.prev; + if ( elem.prev ) + elem.prev->next = elem.next; + else + cond->first = elem.next; } } -extern "C" unsigned long kthread_cond_wait_signal(kthread_cond_t* cond, - kthread_mutex_t* mutex) +bool kthread_cond_wait_signal(kthread_cond_t* cond, kthread_mutex_t* mutex) { if ( Signal::IsPending() ) - return 0; + return false; kthread_cond_elem_t elem; elem.next = NULL; + elem.prev = cond->last; elem.woken = 0; - if ( cond->last ) { cond->last->next = &elem; } - if ( !cond->last ) { cond->first = &elem; } + if ( cond->last ) + cond->last->next = &elem; + if ( !cond->first ) + cond->first = &elem; cond->last = &elem; - while ( !elem.woken ) + kthread_mutex_unlock(mutex); + bool result = true; + while ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) && + kutex_wait(&elem.woken, 0, false) < 0 ) { if ( Signal::IsPending() ) { - if ( cond->first == &elem ) - { - cond->first = elem.next; - if ( cond->last == &elem ) - cond->last = NULL; - } - else - { - kthread_cond_elem_t* prev = cond->first; - while ( prev->next != &elem ) - prev = prev->next; - prev->next = elem.next; - if ( cond->last == &elem ) - cond->last = prev; - } - // Note that the thread still owns the mutex. - return 0; + result = false; + break; } - kthread_mutex_unlock(mutex); - Scheduler::Yield(); - kthread_mutex_lock(mutex); } - return 1; + kthread_mutex_lock(mutex); + if ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) ) + { + if ( elem.next ) + elem.next->prev = elem.prev; + else + cond->last = elem.prev; + if ( elem.prev ) + elem.prev->next = elem.next; + else + cond->first = elem.next; + } + return result; } -extern "C" void kthread_cond_signal(kthread_cond_t* cond) +void kthread_cond_signal(kthread_cond_t* cond) { - kthread_cond_elem_t* elem = cond->first; - if ( !elem ) { return; } - if ( !(cond->first = elem->next) ) { cond->last = NULL; } - elem->next = NULL; - elem->woken = 1; + if ( cond->first ) + { + struct kthread_cond_elem* elem = cond->first; + if ( elem->next ) + elem->next->prev = elem->prev; + else + cond->last = elem->prev; + cond->first = elem->next; + elem->next = NULL; + elem->prev = NULL; + __atomic_store_n(&elem->woken, 1, __ATOMIC_SEQ_CST); + kutex_wake(&elem->woken, 1); + } } -extern "C" void kthread_cond_broadcast(kthread_cond_t* cond) +void kthread_cond_broadcast(kthread_cond_t* cond) { - while ( cond->first ) { kthread_cond_signal(cond); } + while ( cond->first ) + kthread_cond_signal(cond); } } // namespace Sortix diff --git a/kernel/lfbtextbuffer.cpp b/kernel/lfbtextbuffer.cpp index 421061b9..4c3b9ebf 100644 --- a/kernel/lfbtextbuffer.cpp +++ b/kernel/lfbtextbuffer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2012, 2013, 2014, 2015, 2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -119,7 +119,8 @@ LFBTextBuffer* CreateLFBTextBuffer(uint8_t* lfb, uint32_t lfbformat, return ret; ret->queue_thread = true; // Visible to new thread. - if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, ret) ) + if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, ret, + "console") ) ret->queue_thread = false; return ret; @@ -142,7 +143,8 @@ void LFBTextBuffer::SpawnThreads() return; Process* kernel_process = Scheduler::GetKernelProcess(); queue_thread = true; // Visible to new thread. - if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, this) ) + if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, this, + "console") ) queue_thread = false; } diff --git a/kernel/process.cpp b/kernel/process.cpp index 2f5301c9..e2be8c21 100644 --- a/kernel/process.cpp +++ b/kernel/process.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -141,6 +141,10 @@ Process::Process() threads_not_exiting_count = 0; threads_exiting = false; + futex_lock = KTHREAD_MUTEX_INITIALIZER; + futex_first_waiting = NULL; + futex_last_waiting = NULL; + segments = NULL; segments_used = 0; segments_length = 0; @@ -1550,7 +1554,8 @@ pid_t sys_tfork(int flags, struct tfork* user_regs) // If the thread could not be created, make the process commit suicide // in a manner such that we don't wait for its zombie. - Thread* thread = CreateKernelThread(child_process, &cpuregs); + Thread* thread = CreateKernelThread(child_process, &cpuregs, + making_process ? "main" : "second"); process_family_lock_lock.Reset(); if ( !thread ) { diff --git a/kernel/scheduler.cpp b/kernel/scheduler.cpp index 7be032d1..5a8167e5 100644 --- a/kernel/scheduler.cpp +++ b/kernel/scheduler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015 Jonas 'Sortie' Termansen. + * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -265,10 +265,17 @@ static void SwitchRegisters(struct interrupt_context* intctx, current_thread = next; } +static Thread* idle_thread; +static Thread* first_runnable_thread; +static Thread* true_current_thread; +static Process* init_process; + static void SwitchThread(struct interrupt_context* intctx, Thread* old_thread, Thread* new_thread) { + assert(new_thread->state == ThreadState::RUNNABLE || + new_thread == idle_thread); SwitchRegisters(intctx, old_thread, new_thread); if ( intctx->signal_pending && InUserspace(intctx) ) { @@ -290,14 +297,11 @@ static void SwitchThread(struct interrupt_context* intctx, } } -static Thread* idle_thread; -static Thread* first_runnable_thread; -static Thread* true_current_thread; -static Process* init_process; - static Thread* FindRunnableThreadWithSystemTid(uintptr_t system_tid) { - Thread* begun_thread = current_thread; + Thread* begun_thread = first_runnable_thread; + if ( !begun_thread ) + return NULL; Thread* iter = begun_thread; do { @@ -329,8 +333,6 @@ static Thread* PopNextThread(bool yielded) result = idle_thread; } - true_current_thread = result; - return result; } @@ -339,7 +341,10 @@ void SwitchTo(struct interrupt_context* intctx, Thread* new_thread) Thread* old_thread = CurrentThread(); if ( new_thread == old_thread ) return; - first_runnable_thread = old_thread; + if ( new_thread->state != ThreadState::RUNNABLE ) + return; + if ( old_thread != idle_thread ) + first_runnable_thread = old_thread; true_current_thread = new_thread; SwitchThread(intctx, old_thread, new_thread); } @@ -348,6 +353,7 @@ static void RealSwitch(struct interrupt_context* intctx, bool yielded) { Thread* old_thread = CurrentThread(); Thread* new_thread = PopNextThread(yielded); + true_current_thread = new_thread; SwitchThread(intctx, old_thread, new_thread); } @@ -358,7 +364,28 @@ void Switch(struct interrupt_context* intctx) void InterruptYieldCPU(struct interrupt_context* intctx, void* /*user*/) { - RealSwitch(intctx, true); + if ( current_thread->yield_operation == YIELD_OPERATION_NONE ) + RealSwitch(intctx, true); + else if ( current_thread->yield_operation == YIELD_OPERATION_WAIT_FUTEX ) + { + if ( !current_thread->futex_woken && + !current_thread->timer_woken && + !asm_signal_is_pending ) + { + SetThreadState(current_thread, ThreadState::FUTEX_WAITING); + RealSwitch(intctx, false); + } + } + else if ( current_thread->yield_operation == + YIELD_OPERATION_WAIT_FUTEX_SIGNAL ) + { + if ( !current_thread->futex_woken && + !current_thread->timer_woken ) + { + SetThreadState(current_thread, ThreadState::FUTEX_WAITING); + RealSwitch(intctx, false); + } + } } void ThreadExitCPU(struct interrupt_context* intctx, void* /*user*/) @@ -395,9 +422,13 @@ Process* GetKernelProcess() return idle_thread->process; } -void SetThreadState(Thread* thread, ThreadState state) +void SetThreadState(Thread* thread, ThreadState state, bool wake_only) { - bool wasenabled = Interrupt::SetEnabled(false); + bool was_enabled = Interrupt::SetEnabled(false); + + // Avoid transitioning to the RUNNABLE state from unintended states. + if ( wake_only && thread->state != ThreadState::FUTEX_WAITING ) + state = thread->state; // Remove the thread from the list of runnable threads. if ( thread->state == ThreadState::RUNNABLE && @@ -432,7 +463,14 @@ void SetThreadState(Thread* thread, ThreadState state) assert(thread->state != ThreadState::RUNNABLE || thread->scheduler_list_prev); assert(thread->state != ThreadState::RUNNABLE || thread->scheduler_list_next); - Interrupt::SetEnabled(wasenabled); + Interrupt::SetEnabled(was_enabled); +} + +void SetSignalPending(Thread* thread, unsigned long is_pending) +{ + thread->registers.signal_pending = is_pending; + if ( is_pending ) + Scheduler::SetThreadState(thread, ThreadState::RUNNABLE, true); } ThreadState GetThreadState(Thread* thread) @@ -457,14 +495,14 @@ namespace Scheduler { void ScheduleTrueThread() { - bool wasenabled = Interrupt::SetEnabled(false); + bool was_enabled = Interrupt::SetEnabled(false); if ( true_current_thread != current_thread ) { current_thread->yield_to_tid = 0; first_runnable_thread = true_current_thread; kthread_yield(); } - Interrupt::SetEnabled(wasenabled); + Interrupt::SetEnabled(was_enabled); } } // namespace Scheduler diff --git a/kernel/signal.cpp b/kernel/signal.cpp index 383d6782..993754b6 100644 --- a/kernel/signal.cpp +++ b/kernel/signal.cpp @@ -100,7 +100,7 @@ void UpdatePendingSignals(Thread* thread) // thread->process->signal_lock held if ( thread == CurrentThread() ) asm_signal_is_pending = is_pending; else - thread->registers.signal_pending = is_pending; + Scheduler::SetSignalPending(thread, is_pending); } void Thread::DoUpdatePendingSignal() diff --git a/kernel/syscall.cpp b/kernel/syscall.cpp index ebe8497a..d76f73ba 100644 --- a/kernel/syscall.cpp +++ b/kernel/syscall.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -202,6 +202,7 @@ void* syscall_list[SYSCALL_MAX_NUM + 1] = [SYSCALL_SOCKET] = (void*) sys_socket, [SYSCALL_GETDNSCONFIG] = (void*) sys_getdnsconfig, [SYSCALL_SETDNSCONFIG] = (void*) sys_setdnsconfig, + [SYSCALL_FUTEX] = (void*) sys_futex, [SYSCALL_MAX_NUM] = (void*) sys_bad_syscall, }; } /* extern "C" */ diff --git a/kernel/thread.cpp b/kernel/thread.cpp index 9245580e..2111be74 100644 --- a/kernel/thread.cpp +++ b/kernel/thread.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2016, 2018 Jonas 'Sortie' Termansen. + * Copyright (c) 2011-2016, 2018, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -21,16 +21,21 @@ #include #include +#include #include #include #include +#include +#include #include +#include #include #include #include #include +#include #include #include #include @@ -77,6 +82,7 @@ void FreeThread(Thread* thread) Thread::Thread() { assert(!((uintptr_t) registers.fpuenv & 0xFUL)); + name = ""; system_tid = (uintptr_t) this; yield_to_tid = 0; id = 0; // TODO: Make a thread id. @@ -105,6 +111,11 @@ Thread::Thread() // execute_clock initialized in member constructor. // system_clock initialized in member constructor. Time::InitializeThreadClocks(this); + futex_address = 0; + futex_woken = false; + futex_prev_waiting = NULL; + futex_next_waiting = NULL; + yield_operation = YIELD_OPERATION_NONE; } Thread::~Thread() @@ -116,7 +127,9 @@ Thread::~Thread() delete[] (uint8_t*) kernelstackpos; } -Thread* CreateKernelThread(Process* process, struct thread_registers* regs) +Thread* CreateKernelThread(Process* process, + struct thread_registers* regs, + const char* name) { assert(process && regs && process->addrspace); @@ -142,6 +155,7 @@ Thread* CreateKernelThread(Process* process, struct thread_registers* regs) Thread* thread = AllocateThread(); if ( !thread ) return NULL; + thread->name = name; memcpy(&thread->registers, regs, sizeof(struct thread_registers)); @@ -254,7 +268,7 @@ static void SetupKernelThreadRegs(struct thread_registers* regs, } Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user, - size_t stacksize) + const char* name, size_t stacksize) { const size_t DEFAULT_KERNEL_STACK_SIZE = 8 * 1024UL; if ( !stacksize ) @@ -264,9 +278,10 @@ Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user, return NULL; struct thread_registers regs; - SetupKernelThreadRegs(®s, process, entry, user, (uintptr_t) stack, stacksize); + SetupKernelThreadRegs(®s, process, entry, user, (uintptr_t) stack, + stacksize); - Thread* thread = CreateKernelThread(process, ®s); + Thread* thread = CreateKernelThread(process, ®s, name); if ( !thread ) { delete[] stack; return NULL; } thread->kernelstackpos = (uintptr_t) stack; @@ -276,9 +291,10 @@ Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user, return thread; } -Thread* CreateKernelThread(void (*entry)(void*), void* user, size_t stacksize) +Thread* CreateKernelThread(void (*entry)(void*), void* user, const char* name, + size_t stacksize) { - return CreateKernelThread(CurrentProcess(), entry, user, stacksize); + return CreateKernelThread(CurrentProcess(), entry, user, name, stacksize); } void StartKernelThread(Thread* thread) @@ -286,9 +302,10 @@ void StartKernelThread(Thread* thread) Scheduler::SetThreadState(thread, ThreadState::RUNNABLE); } -Thread* RunKernelThread(Process* process, struct thread_registers* regs) +Thread* RunKernelThread(Process* process, struct thread_registers* regs, + const char* name) { - Thread* thread = CreateKernelThread(process, regs); + Thread* thread = CreateKernelThread(process, regs, name); if ( !thread ) return NULL; StartKernelThread(thread); @@ -296,18 +313,19 @@ Thread* RunKernelThread(Process* process, struct thread_registers* regs) } Thread* RunKernelThread(Process* process, void (*entry)(void*), void* user, - size_t stacksize) + const char* name, size_t stacksize) { - Thread* thread = CreateKernelThread(process, entry, user, stacksize); + Thread* thread = CreateKernelThread(process, entry, user, name, stacksize); if ( !thread ) return NULL; StartKernelThread(thread); return thread; } -Thread* RunKernelThread(void (*entry)(void*), void* user, size_t stacksize) +Thread* RunKernelThread(void (*entry)(void*), void* user, const char* name, + size_t stacksize) { - Thread* thread = CreateKernelThread(entry, user, stacksize); + Thread* thread = CreateKernelThread(entry, user, name, stacksize); if ( !thread ) return NULL; StartKernelThread(thread); @@ -323,7 +341,8 @@ int sys_exit_thread(int requested_exit_code, EXIT_THREAD_ZERO | EXIT_THREAD_TLS_UNMAP | EXIT_THREAD_PROCESS | - EXIT_THREAD_DUMP_CORE) ) + EXIT_THREAD_DUMP_CORE | + EXIT_THREAD_FUTEX_WAKE) ) return errno = EINVAL, -1; if ( (flags & EXIT_THREAD_ONLY_IF_OTHERS) && (flags & EXIT_THREAD_PROCESS) ) @@ -395,6 +414,9 @@ int sys_exit_thread(int requested_exit_code, if ( flags & EXIT_THREAD_ZERO ) ZeroUser(extended.zero_from, extended.zero_size); + if ( flags & EXIT_THREAD_FUTEX_WAKE ) + sys_futex((int*) extended.zero_from, FUTEX_WAKE, 1, NULL); + if ( do_exit ) { // Validate the requested exit code such that the process can't exit @@ -431,4 +453,117 @@ int sys_exit_thread(int requested_exit_code, kthread_exit(); } +static void futex_timeout(Clock* /*clock*/, Timer* /*timer*/, void* ctx) +{ + Thread* thread = (Thread*) ctx; + thread->timer_woken = true; + kthread_wake_futex(thread); +} + +int sys_futex(int* user_address, + int op, + int value, + const struct timespec* user_timeout) +{ + ioctx_t ctx; SetupKernelIOCtx(&ctx); + Thread* thread = CurrentThread(); + Process* process = thread->process; + if ( FUTEX_GET_OP(op) == FUTEX_WAIT ) + { + kthread_mutex_lock(&process->futex_lock); + thread->futex_address = (uintptr_t) user_address; + thread->futex_woken = false; + thread->futex_prev_waiting = process->futex_last_waiting; + thread->futex_next_waiting = NULL; + (process->futex_last_waiting ? + process->futex_last_waiting->futex_next_waiting : + process->futex_first_waiting) = thread; + process->futex_last_waiting = thread; + kthread_mutex_unlock(&process->futex_lock); + thread->timer_woken = false; + Timer timer; + if ( user_timeout ) + { + clockid_t clockid = FUTEX_GET_CLOCK(op); + bool absolute = op & FUTEX_ABSOLUTE; + struct timespec timeout; + if ( !CopyFromUser(&timeout, user_timeout, sizeof(timeout)) ) + return -1; + if ( !timespec_is_canonical(timeout) ) + return errno = EINVAL, -1; + Clock* clock = Time::GetClock(clockid); + timer.Attach(clock); + struct itimerspec timerspec; + timerspec.it_value = timeout; + timerspec.it_interval.tv_sec = 0; + timerspec.it_interval.tv_nsec = 0; + int timer_flags = (absolute ? TIMER_ABSOLUTE : 0) | + TIMER_FUNC_INTERRUPT_HANDLER; + timer.Set(&timerspec, NULL, timer_flags, futex_timeout, thread); + } + int result = 0; + int current; + if ( !ReadAtomicFromUser(¤t, user_address) ) + result = -1; + else if ( current != value ) + { + errno = EAGAIN; + result = -1; + } + else + kthread_wait_futex_signal(); + if ( user_timeout ) + timer.Cancel(); + kthread_mutex_lock(&process->futex_lock); + if ( result == 0 && !thread->futex_woken ) + { + if ( Signal::IsPending() ) + { + errno = EINTR; + result = -1; + } + else if ( thread->timer_woken ) + { + errno = ETIMEDOUT; + result = -1; + } + } + thread->futex_address = 0; + thread->futex_woken = false; + (thread->futex_prev_waiting ? + thread->futex_prev_waiting->futex_next_waiting : + process->futex_first_waiting) = thread->futex_next_waiting; + (thread->futex_next_waiting ? + thread->futex_next_waiting->futex_prev_waiting : + process->futex_last_waiting) = thread->futex_prev_waiting; + thread->futex_prev_waiting = NULL; + thread->futex_next_waiting = NULL; + kthread_mutex_unlock(&process->futex_lock); + return result; + } + else if ( FUTEX_GET_OP(op) == FUTEX_WAKE ) + { + kthread_mutex_lock(&process->futex_lock); + int result = 0; + for ( Thread* waiter = process->futex_first_waiting; + 0 < value && waiter; + waiter = waiter->futex_next_waiting ) + { + if ( waiter->futex_address == (uintptr_t) user_address ) + { + waiter->futex_woken = true; + kthread_wake_futex(waiter); + if ( value != INT_MAX ) + value--; + if ( result != INT_MAX ) + result++; + } + } + kthread_mutex_unlock(&process->futex_lock); + return result; + } + else + return errno = EINVAL, -1; +} + } // namespace Sortix diff --git a/kernel/x64/kthread.S b/kernel/x64/kthread.S deleted file mode 100644 index 0556973b..00000000 --- a/kernel/x64/kthread.S +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2012 Jonas 'Sortie' Termansen. - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - * x64/kthread.S - * Utilities and synchronization mechanisms for x64 kernel threads. - */ - -.section .text - -.global kthread_mutex_trylock -.type kthread_mutex_trylock, @function -kthread_mutex_trylock: - pushq %rbp - movq %rsp, %rbp - movl $-1, %eax - xchgl (%rdi), %eax - not %eax - leaveq - retq -.size kthread_mutex_trylock, . - kthread_mutex_trylock - -.global kthread_mutex_lock -.type kthread_mutex_lock, @function -kthread_mutex_lock: - pushq %rbp - movq %rsp, %rbp -kthread_mutex_lock_retry: - movl $-1, %eax - xchgl (%rdi), %eax - testl %eax, %eax - jnz kthread_mutex_lock_failed - leaveq - retq -kthread_mutex_lock_failed: - int $0x81 # Yield the CPU. - jmp kthread_mutex_lock_retry -.size kthread_mutex_lock, . - kthread_mutex_lock - -.global kthread_mutex_lock_signal -.type kthread_mutex_lock_signal, @function -kthread_mutex_lock_signal: - pushq %rbp - movq %rsp, %rbp -kthread_mutex_lock_signal_retry: - movq asm_signal_is_pending, %rax - testq %rax, %rax - jnz kthread_mutex_lock_signal_pending - movl $-1, %eax - xchgl (%rdi), %eax - testl %eax, %eax - jnz kthread_mutex_lock_signal_failed - inc %eax -kthread_mutex_lock_signal_out: - leaveq - retq -kthread_mutex_lock_signal_failed: - int $0x81 # Yield the CPU. - jmp kthread_mutex_lock_signal_retry -kthread_mutex_lock_signal_pending: - xorl %eax, %eax - jmp kthread_mutex_lock_signal_out -.size kthread_mutex_lock_signal, . - kthread_mutex_lock_signal - -.global kthread_mutex_unlock -.type kthread_mutex_unlock, @function -kthread_mutex_unlock: - pushq %rbp - movq %rsp, %rbp - movl $0, (%rdi) - leaveq - retq -.size kthread_mutex_unlock, . - kthread_mutex_unlock diff --git a/kernel/x86/kthread.S b/kernel/x86/kthread.S deleted file mode 100644 index 12d348e3..00000000 --- a/kernel/x86/kthread.S +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2012 Jonas 'Sortie' Termansen. - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - * x86/kthread.S - * Utilities and synchronization mechanisms for x86 kernel threads. - */ - -.section .text - -.global kthread_mutex_trylock -.type kthread_mutex_trylock, @function -kthread_mutex_trylock: - pushl %ebp - movl %esp, %ebp - movl 8(%ebp), %edx - movl $-1, %eax - xchgl (%edx), %eax - not %eax - leavel - retl -.size kthread_mutex_trylock, . - kthread_mutex_trylock - -.global kthread_mutex_lock -.type kthread_mutex_lock, @function -kthread_mutex_lock: - pushl %ebp - movl %esp, %ebp - movl 8(%ebp), %edx -kthread_mutex_lock_retry: - movl $-1, %eax - xchgl (%edx), %eax - testl %eax, %eax - jnz kthread_mutex_lock_failed - leavel - retl -kthread_mutex_lock_failed: - int $0x81 # Yield the CPU. - jmp kthread_mutex_lock_retry -.size kthread_mutex_lock, . - kthread_mutex_lock - -.global kthread_mutex_lock_signal -.type kthread_mutex_lock_signal, @function -kthread_mutex_lock_signal: - pushl %ebp - movl %esp, %ebp - movl 8(%ebp), %edx -kthread_mutex_lock_signal_retry: - movl asm_signal_is_pending, %eax - testl %eax, %eax - jnz kthread_mutex_lock_signal_pending - movl $-1, %eax - xchgl (%edx), %eax - testl %eax, %eax - jnz kthread_mutex_lock_signal_failed - inc %eax -kthread_mutex_lock_signal_out: - leavel - retl -kthread_mutex_lock_signal_failed: - int $0x81 # Yield the CPU. - jmp kthread_mutex_lock_signal_retry -kthread_mutex_lock_signal_pending: - xorl %eax, %eax - jmp kthread_mutex_lock_signal_out - -.global kthread_mutex_unlock -.type kthread_mutex_unlock, @function -kthread_mutex_unlock: - pushl %ebp - movl %esp, %ebp - movl 8(%ebp), %edx - movl $0, (%edx) - leavel - retl -.size kthread_mutex_lock_signal, . - kthread_mutex_lock_signal diff --git a/libc/Makefile b/libc/Makefile index 8a32503d..372af5f6 100644 --- a/libc/Makefile +++ b/libc/Makefile @@ -561,6 +561,7 @@ sys/ioctl/ioctl.o \ sys/kernelinfo/kernelinfo.o \ syslog/closelog.o \ syslog/connectlog.o \ +sys/futex/futex.o \ syslog/openlog.o \ syslog/setlogmask.o \ syslog/syslog.o \ diff --git a/libc/include/__/pthread.h b/libc/include/__/pthread.h index 7b85f5e3..b935eaed 100644 --- a/libc/include/__/pthread.h +++ b/libc/include/__/pthread.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014, 2017 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2017, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -46,40 +46,12 @@ typedef int __pthread_barrier_t; typedef int __pthread_barrierattr_t; -#if defined(__is_sortix_libc) -typedef struct -{ - struct pthread_cond_elem* first; - struct pthread_cond_elem* last; - __clock_t clock; -} __pthread_cond_t; -#else -typedef struct -{ - void* __pthread_first; - void* __pthread_last; - __clock_t __pthread_clock; -} __pthread_cond_t; -#endif - -#if defined(__is_sortix_libc) -typedef struct -{ - __clock_t clock; -} __pthread_condattr_t; -#else -typedef struct -{ - __clock_t __pthread_clock; -} __pthread_condattr_t; -#endif - typedef __SIZE_TYPE__ __pthread_key_t; #if defined(__is_sortix_libc) typedef struct { - unsigned long lock; + int lock; unsigned long type; unsigned long owner; unsigned long recursion; @@ -87,7 +59,7 @@ typedef struct #else typedef struct { - unsigned long __pthread_lock; + int __pthread_lock; unsigned long __pthread_type; unsigned long __pthread_owner; unsigned long __pthread_recursion; @@ -106,6 +78,36 @@ typedef struct } __pthread_mutexattr_t; #endif +#if defined(__is_sortix_libc) +typedef struct +{ + __pthread_mutex_t lock; + struct pthread_cond_elem* first; + struct pthread_cond_elem* last; + __clockid_t clock; +} __pthread_cond_t; +#else +typedef struct +{ + __pthread_mutex_t __pthread_lock; + void* __pthread_first; + void* __pthread_last; + __clockid_t __pthread_clock; +} __pthread_cond_t; +#endif + +#if defined(__is_sortix_libc) +typedef struct +{ + __clockid_t clock; +} __pthread_condattr_t; +#else +typedef struct +{ + __clockid_t __pthread_clock; +} __pthread_condattr_t; +#endif + #if defined(__is_sortix_libc) typedef struct { diff --git a/libc/include/pthread.h b/libc/include/pthread.h index 34f2d165..8a3ad5bc 100644 --- a/libc/include/pthread.h +++ b/libc/include/pthread.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -150,18 +150,21 @@ struct pthread struct pthread_cond_elem { struct pthread_cond_elem* next; - volatile unsigned long woken; + struct pthread_cond_elem* prev; + int woken; }; #endif -#define PTHREAD_COND_INITIALIZER { NULL, NULL, CLOCK_REALTIME } +#define PTHREAD_COND_INITIALIZER { PTHREAD_NORMAL_MUTEX_INITIALIZER_NP, NULL, \ + NULL, CLOCK_REALTIME } #define PTHREAD_MUTEX_INITIALIZER { 0, PTHREAD_MUTEX_DEFAULT, 0, 0 } #define PTHREAD_RWLOCK_INITIALIZER { PTHREAD_COND_INITIALIZER, \ PTHREAD_COND_INITIALIZER, \ PTHREAD_MUTEX_INITIALIZER, 0, 0, 0, 0 } #define PTHREAD_NORMAL_MUTEX_INITIALIZER_NP { 0, PTHREAD_MUTEX_NORMAL, 0, 0 } -#define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP { 0, PTHREAD_MUTEX_RECURSIVE, 0, 0 } +#define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP { 0, PTHREAD_MUTEX_RECURSIVE, \ + 0, 0 } #define PTHREAD_ONCE_INIT { PTHREAD_NORMAL_MUTEX_INITIALIZER_NP, 0 } diff --git a/libc/include/semaphore.h b/libc/include/semaphore.h index 44118551..f0ab92f9 100644 --- a/libc/include/semaphore.h +++ b/libc/include/semaphore.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -32,8 +32,10 @@ typedef struct { #if defined(__is_sortix_libc) int value; + int waiters; #else int __value; + int __waiters; #endif } sem_t; diff --git a/libc/include/sys/futex.h b/libc/include/sys/futex.h new file mode 100644 index 00000000..053f1526 --- /dev/null +++ b/libc/include/sys/futex.h @@ -0,0 +1,30 @@ + /* + * Copyright (c) 2021 Jonas 'Sortie' Termansen. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * sys/futex.h + * Fast userspace mutexes. + */ + +#ifndef _SYS_FUTEX_H +#define _SYS_FUTEX_H 1 + +#include + +#include +#include + +int futex(int*, int, int, const struct timespec*); + +#endif diff --git a/libc/pthread/pthread_cond_broadcast.c b/libc/pthread/pthread_cond_broadcast.c index e2e08be3..af588d75 100644 --- a/libc/pthread/pthread_cond_broadcast.c +++ b/libc/pthread/pthread_cond_broadcast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,13 +17,27 @@ * Broadcasts a condition. */ +#include + +#include #include int pthread_cond_broadcast(pthread_cond_t* cond) { - int ret; + pthread_mutex_lock(&cond->lock); while ( cond->first ) - if ( (ret = pthread_cond_signal(cond)) ) - return ret; + { + struct pthread_cond_elem* elem = cond->first; + if ( elem->next ) + elem->next->prev = elem->prev; + else + cond->last = elem->prev; + cond->first = elem->next; + elem->next = NULL; + elem->prev = NULL; + __atomic_store_n(&elem->woken, 1, __ATOMIC_SEQ_CST); + futex(&elem->woken, FUTEX_WAKE, 1, NULL); + } + pthread_mutex_unlock(&cond->lock); return 0; } diff --git a/libc/pthread/pthread_cond_destroy.c b/libc/pthread/pthread_cond_destroy.c index 0ec78cd4..5c03297e 100644 --- a/libc/pthread/pthread_cond_destroy.c +++ b/libc/pthread/pthread_cond_destroy.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -21,6 +21,6 @@ int pthread_cond_destroy(pthread_cond_t* restrict cond) { - (void) cond; + pthread_mutex_destroy(&cond->lock); return 0; } diff --git a/libc/pthread/pthread_cond_init.c b/libc/pthread/pthread_cond_init.c index 1119e39f..1aaad448 100644 --- a/libc/pthread/pthread_cond_init.c +++ b/libc/pthread/pthread_cond_init.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -29,6 +29,7 @@ int pthread_cond_init(pthread_cond_t* restrict cond, attr = &default_attr; } + pthread_mutex_init(&cond->lock, NULL); cond->first = NULL; cond->last = NULL; cond->clock = attr->clock; diff --git a/libc/pthread/pthread_cond_signal.c b/libc/pthread/pthread_cond_signal.c index b8f5689d..c8171220 100644 --- a/libc/pthread/pthread_cond_signal.c +++ b/libc/pthread/pthread_cond_signal.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,16 +17,27 @@ * Signals a condition. */ +#include + +#include #include int pthread_cond_signal(pthread_cond_t* cond) { - struct pthread_cond_elem* elem = cond->first; - if ( !elem ) - return 0; - if ( !(cond->first = elem->next) ) - cond->last = NULL; - elem->next = NULL; - elem->woken = 1; + pthread_mutex_lock(&cond->lock); + if ( cond->first ) + { + struct pthread_cond_elem* elem = cond->first; + if ( elem->next ) + elem->next->prev = elem->prev; + else + cond->last = elem->prev; + cond->first = elem->next; + elem->next = NULL; + elem->prev = NULL; + __atomic_store_n(&elem->woken, 1, __ATOMIC_SEQ_CST); + futex(&elem->woken, FUTEX_WAKE, 1, NULL); + } + pthread_mutex_unlock(&cond->lock); return 0; } diff --git a/libc/pthread/pthread_cond_timedwait.c b/libc/pthread/pthread_cond_timedwait.c index 731d7c51..53d70149 100644 --- a/libc/pthread/pthread_cond_timedwait.c +++ b/libc/pthread/pthread_cond_timedwait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,35 +17,51 @@ * Waits on a condition or until a timeout happens. */ +#include + #include #include -#include -#include -#include -#include int pthread_cond_timedwait(pthread_cond_t* restrict cond, pthread_mutex_t* restrict mutex, const struct timespec* restrict abstime) { struct pthread_cond_elem elem; + pthread_mutex_lock(&cond->lock); elem.next = NULL; + elem.prev = cond->last; elem.woken = 0; if ( cond->last ) cond->last->next = &elem; - if ( !cond->last ) + if ( !cond->first ) cond->first = &elem; cond->last = &elem; - while ( !elem.woken ) + pthread_mutex_unlock(&cond->lock); + pthread_mutex_unlock(mutex); + int op = FUTEX_WAIT | FUTEX_ABSOLUTE | FUTEX_CLOCK(cond->clock); + int result = 0; + while ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) && + futex(&elem.woken, op, 0, abstime) < 0 ) { - struct timespec now; - if ( clock_gettime(cond->clock, &now) < 0 ) - return errno; - if ( timespec_le(*abstime, now) ) - return errno = ETIMEDOUT; - pthread_mutex_unlock(mutex); - sched_yield(); - pthread_mutex_lock(mutex); + if ( errno == EINTR ) + continue; + if ( errno != EAGAIN ) + result = errno; + break; } - return 0; + pthread_mutex_lock(mutex); + pthread_mutex_lock(&cond->lock); + if ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) ) + { + if ( elem.next ) + elem.next->prev = elem.prev; + else + cond->last = elem.prev; + if ( elem.prev ) + elem.prev->next = elem.next; + else + cond->first = elem.next; + } + pthread_mutex_unlock(&cond->lock); + return result; } diff --git a/libc/pthread/pthread_cond_wait.c b/libc/pthread/pthread_cond_wait.c index 17c8f87a..e30a10f6 100644 --- a/libc/pthread/pthread_cond_wait.c +++ b/libc/pthread/pthread_cond_wait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,28 +17,11 @@ * Waits on a condition. */ +#include #include -#include -#include -#include -#include int pthread_cond_wait(pthread_cond_t* restrict cond, pthread_mutex_t* restrict mutex) { - struct pthread_cond_elem elem; - elem.next = NULL; - elem.woken = 0; - if ( cond->last ) - cond->last->next = &elem; - if ( !cond->last ) - cond->first = &elem; - cond->last = &elem; - while ( !elem.woken ) - { - pthread_mutex_unlock(mutex); - sched_yield(); - pthread_mutex_lock(mutex); - } - return 0; + return pthread_cond_timedwait(cond, mutex, NULL); } diff --git a/libc/pthread/pthread_exit.c b/libc/pthread/pthread_exit.c index 7c93420e..c648269c 100644 --- a/libc/pthread/pthread_exit.c +++ b/libc/pthread/pthread_exit.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -62,7 +62,7 @@ void pthread_exit(void* return_value) { extended.zero_from = &thread->join_lock.lock; extended.zero_size = sizeof(thread->join_lock.lock); - exit_flags |= EXIT_THREAD_ZERO; + exit_flags |= EXIT_THREAD_ZERO | EXIT_THREAD_FUTEX_WAKE; } else { diff --git a/libc/pthread/pthread_mutex_lock.c b/libc/pthread/pthread_mutex_lock.c index 0fa28b4d..581c6bef 100644 --- a/libc/pthread/pthread_mutex_lock.c +++ b/libc/pthread/pthread_mutex_lock.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,22 +17,45 @@ * Locks a mutex. */ -#include -#include -#include -#include +#include -static const unsigned long UNLOCKED_VALUE = 0; -static const unsigned long LOCKED_VALUE = 1; +#include +#include +#include +#include + +static const int UNLOCKED = 0; +static const int LOCKED = 1; +static const int CONTENDED = 2; int pthread_mutex_lock(pthread_mutex_t* mutex) { - while ( !__sync_bool_compare_and_swap(&mutex->lock, UNLOCKED_VALUE, LOCKED_VALUE) ) + int state = UNLOCKED; + int desired = LOCKED; + while ( !__atomic_compare_exchange_n(&mutex->lock, &state, desired, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) { if ( mutex->type == PTHREAD_MUTEX_RECURSIVE && (pthread_t) mutex->owner == pthread_self() ) - return mutex->recursion++, 0; - sched_yield(); + { + if ( mutex->recursion == ULONG_MAX ) + return errno = EAGAIN; + mutex->recursion++; + return 0; + } + if ( state == LOCKED && + !__atomic_compare_exchange_n(&mutex->lock, &state, CONTENDED, + false, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST) ) + { + state = UNLOCKED; + continue; + } + desired = CONTENDED; + if ( futex(&mutex->lock, FUTEX_WAIT, CONTENDED, NULL) < 0 && + errno != EAGAIN && errno != EINTR ) + return errno; + state = UNLOCKED; } mutex->owner = (unsigned long) pthread_self(); mutex->recursion = 0; diff --git a/libc/pthread/pthread_mutex_trylock.c b/libc/pthread/pthread_mutex_trylock.c index b0aa8531..52f6d3fb 100644 --- a/libc/pthread/pthread_mutex_trylock.c +++ b/libc/pthread/pthread_mutex_trylock.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -18,18 +18,27 @@ */ #include +#include #include +#include -static const unsigned long UNLOCKED_VALUE = 0; -static const unsigned long LOCKED_VALUE = 1; +static const int UNLOCKED = 0; +static const int LOCKED = 1; int pthread_mutex_trylock(pthread_mutex_t* mutex) { - if ( !__sync_bool_compare_and_swap(&mutex->lock, UNLOCKED_VALUE, LOCKED_VALUE) ) + int state = UNLOCKED; + if ( !__atomic_compare_exchange_n(&mutex->lock, &state, LOCKED, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) { if ( mutex->type == PTHREAD_MUTEX_RECURSIVE && (pthread_t) mutex->owner == pthread_self() ) - return mutex->recursion++, 0; + { + if ( mutex->recursion == ULONG_MAX ) + return errno = EAGAIN; + mutex->recursion++; + return 0; + } return errno = EBUSY; } mutex->owner = (unsigned long) pthread_self(); diff --git a/libc/pthread/pthread_mutex_unlock.c b/libc/pthread/pthread_mutex_unlock.c index 25f29d90..0d78f0b5 100644 --- a/libc/pthread/pthread_mutex_unlock.c +++ b/libc/pthread/pthread_mutex_unlock.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,16 +17,24 @@ * Unlocks a mutex. */ -#include +#include -static const unsigned long UNLOCKED_VALUE = 0; -static const unsigned long LOCKED_VALUE = 1; +#include +#include + +static const int UNLOCKED = 0; +static const int CONTENDED = 2; int pthread_mutex_unlock(pthread_mutex_t* mutex) { if ( mutex->type == PTHREAD_MUTEX_RECURSIVE && mutex->recursion ) - return mutex->recursion--, 0; + { + mutex->recursion--; + return 0; + } mutex->owner = 0; - mutex->lock = UNLOCKED_VALUE; + if ( __atomic_exchange_n(&mutex->lock, UNLOCKED, + __ATOMIC_SEQ_CST) == CONTENDED ) + futex(&mutex->lock, FUTEX_WAKE, 1, NULL); return 0; } diff --git a/libc/semaphore/sem_getvalue.c b/libc/semaphore/sem_getvalue.c index 59d57944..cc983e67 100644 --- a/libc/semaphore/sem_getvalue.c +++ b/libc/semaphore/sem_getvalue.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -21,6 +21,7 @@ int sem_getvalue(sem_t* restrict sem, int* restrict value_ptr) { - *value_ptr = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + int value = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + *value_ptr = value == -1 ? 0 : value; return 0; } diff --git a/libc/semaphore/sem_init.c b/libc/semaphore/sem_init.c index a3448066..4301ac71 100644 --- a/libc/semaphore/sem_init.c +++ b/libc/semaphore/sem_init.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -25,11 +25,9 @@ int sem_init(sem_t* sem, int pshared, unsigned int value) { if ( pshared ) return errno = ENOSYS, -1; - if ( (unsigned int) INT_MAX < value ) return errno = EINVAL, -1; - sem->value = (int) value; - + sem->waiters = 0; return 0; } diff --git a/libc/semaphore/sem_post.c b/libc/semaphore/sem_post.c index fbd515d4..b7103de2 100644 --- a/libc/semaphore/sem_post.c +++ b/libc/semaphore/sem_post.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,25 +17,28 @@ * Unlock a semaphore. */ +#include + #include #include #include #include +#include int sem_post(sem_t* sem) { while ( true ) { - int old_value = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); - if ( old_value == INT_MAX ) - return errno = EOVERFLOW; - - int new_value = old_value + 1; - if ( !__atomic_compare_exchange_n(&sem->value, &old_value, new_value, - false, + int old = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + if ( old == INT_MAX ) + return errno = EOVERFLOW, -1; + int new = old == -1 ? 1 : old + 1; + int waiters = __atomic_load_n(&sem->waiters, __ATOMIC_SEQ_CST); + if ( !__atomic_compare_exchange_n(&sem->value, &old, new, false, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED) ) continue; - + if ( old == -1 || waiters ) + futex(&sem->value, FUTEX_WAKE, 1, NULL); return 0; } } diff --git a/libc/semaphore/sem_timedwait.c b/libc/semaphore/sem_timedwait.c index 04eb7bb9..d7fcbb28 100644 --- a/libc/semaphore/sem_timedwait.c +++ b/libc/semaphore/sem_timedwait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,57 +14,39 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * * semaphore/sem_timedwait.c - * Lock a semaphore. + * Lock a semaphore with a timeout. */ +#include + #include -#include #include -#include -#include +#include #include -#include int sem_timedwait(sem_t* restrict sem, const struct timespec* restrict abstime) { - if ( sem_trywait(sem) == 0 ) - return 0; - if ( errno != EAGAIN ) - return -1; - - sigset_t old_set_mask; - sigset_t old_set_allowed; - sigset_t all_signals; - sigfillset(&all_signals); - sigprocmask(SIG_SETMASK, &all_signals, &old_set_mask); - signotset(&old_set_allowed, &old_set_mask); - - while ( sem_trywait(sem) != 0 ) + while ( true ) { - // TODO: Using CLOCK_REALTIME for this is bad as it is not monotonic. We - // need to enchance the semaphore API so a better clock can be - // used instead. - if ( errno == EAGAIN ) + int old = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + int new = old != -1 ? old - 1 : -1; + bool waiting = new == -1; + if ( waiting ) + __atomic_add_fetch(&sem->waiters, 1, __ATOMIC_SEQ_CST); + if ( old != new && + !__atomic_compare_exchange_n(&sem->value, &old, new, false, + __ATOMIC_SEQ_CST, __ATOMIC_RELAXED) ) { - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - if ( timespec_le(*abstime, now) ) - errno = ETIMEDOUT; + if ( waiting ) + __atomic_sub_fetch(&sem->waiters, 1, __ATOMIC_SEQ_CST); + continue; } - - if ( errno == EAGAIN && sigpending(&old_set_allowed) ) - errno = EINTR; - - if ( errno != EAGAIN ) - { - sigprocmask(SIG_SETMASK, &old_set_mask, NULL); + if ( !waiting ) + return 0; + int op = FUTEX_WAIT | FUTEX_ABSOLUTE | FUTEX_CLOCK(CLOCK_REALTIME); + int ret = futex(&sem->value, op, -1, abstime); + __atomic_sub_fetch(&sem->waiters, 1, __ATOMIC_SEQ_CST); + if ( ret < 0 && errno != EAGAIN ) return -1; - } - - sched_yield(); } - - sigprocmask(SIG_SETMASK, &old_set_mask, NULL); - - return 0; } diff --git a/libc/semaphore/sem_trywait.c b/libc/semaphore/sem_trywait.c index 40a720f4..49f20e49 100644 --- a/libc/semaphore/sem_trywait.c +++ b/libc/semaphore/sem_trywait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,7 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * * semaphore/sem_trywait.c - * Lock a semaphore. + * Try to lock a semaphore. */ #include @@ -23,11 +23,11 @@ int sem_trywait(sem_t* sem) { - int old_value = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); - if ( old_value <= 0 ) + int old = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST); + if ( old <= 0 ) return errno = EAGAIN, -1; - int new_value = old_value - 1; - if ( !__atomic_compare_exchange_n(&sem->value, &old_value, new_value, false, + int new = old != -1 ? old - 1 : -1; + if ( !__atomic_compare_exchange_n(&sem->value, &old, new, false, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED) ) return errno = EAGAIN, -1; return 0; diff --git a/libc/semaphore/sem_wait.c b/libc/semaphore/sem_wait.c index c5155977..bbd0f2d0 100644 --- a/libc/semaphore/sem_wait.c +++ b/libc/semaphore/sem_wait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Jonas 'Sortie' Termansen. + * Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,41 +17,10 @@ * Lock a semaphore. */ -#include -#include #include -#include #include int sem_wait(sem_t* sem) { - if ( sem_trywait(sem) == 0 ) - return 0; - if ( errno != EAGAIN ) - return -1; - - sigset_t old_set_mask; - sigset_t old_set_allowed; - sigset_t all_signals; - sigfillset(&all_signals); - sigprocmask(SIG_SETMASK, &all_signals, &old_set_mask); - signotset(&old_set_allowed, &old_set_mask); - - while ( sem_trywait(sem) != 0 ) - { - if ( errno == EAGAIN && sigpending(&old_set_allowed) ) - errno = EINTR; - - if ( errno != EAGAIN ) - { - sigprocmask(SIG_SETMASK, &old_set_mask, NULL); - return -1; - } - - sched_yield(); - } - - sigprocmask(SIG_SETMASK, &old_set_mask, NULL); - - return 0; + return sem_timedwait(sem, NULL); } diff --git a/libc/sys/futex/futex.c b/libc/sys/futex/futex.c new file mode 100644 index 00000000..24f12e86 --- /dev/null +++ b/libc/sys/futex/futex.c @@ -0,0 +1,29 @@ + /* + * Copyright (c) 2021 Jonas 'Sortie' Termansen. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * sys/futex/futex.c + * Fast userspace mutexes. + */ + +#include +#include + +DEFN_SYSCALL4(int, sys_futex, SYSCALL_FUTEX, int*, int, int, + const struct timespec*); + +int futex(int* address, int op, int value, const struct timespec* timeout) +{ + return sys_futex(address, op, value, timeout); +} diff --git a/share/man/man7/following-development.7 b/share/man/man7/following-development.7 index a6cacd06..c3a9599a 100644 --- a/share/man/man7/following-development.7 +++ b/share/man/man7/following-development.7 @@ -69,6 +69,16 @@ releasing Sortix x.y, foo." to allow the maintainer to easily .Xr grep 1 for it after a release. .Sh CHANGES +.Ss Implement threading primitives that truly sleep +The +.Xr futex 2 +system system call for efficient thread waiting has been added. +The +.Xr exit_thread 2 +system call has gained a +.Dv EXIT_THREAD_FUTEX_WAKE +flag for waking a single waiter on a futex. +This is a minor compatible ABI change. .Ss Fix system upgrade leaking files .Xr sysupgrade 8 and