diff --git a/src/reflock.c b/src/reflock.c index ce3ea38..9463ee1 100644 --- a/src/reflock.c +++ b/src/reflock.c @@ -8,187 +8,90 @@ #include "util.h" #include "win.h" -typedef union _reflock_private { - volatile uint32_t counter; - uint64_t dummy; - struct { - uint32_t lock; - uint32_t delete; - } events; -} _reflock_private_t; - -static const uint32_t _CTR_REF = (uint32_t) 1; -static const uint32_t _CTR_LOCK = (uint32_t) 1 << 16; -static const uint32_t _CTR_DELETE = (uint32_t) 1 << 31; - -static const uint32_t _CTR_REF_MASK = ((uint32_t) 1 << 16) - 1; -static const uint32_t _CTR_LOCK_MASK = (((uint32_t) 1 << 15) - 1) << 16; - /* clang-format off */ - -#define _TOK_REFERENCED 0x01 -#define _TOK_LOCK_ANNOUNCED 0x02 -#define _TOK_DELETE_ANNOUNCED 0x04 -#define _TOK_MUST_WAIT 0x08 -#define _TOK_NO_WAIT 0x10 -#define _TOK_LOCKED 0x20 -#define _TOK_DELETED 0x40 - +static const uint32_t _REF = 0x00000001; +static const uint32_t _REF_MASK = 0x0fffffff; +static const uint32_t _DESTROY = 0x10000000; +static const uint32_t _DESTROY_MASK = 0xf0000000; +static const uint32_t _POISON = 0x300DEAD0; /* clang-format on */ -static HANDLE _global_event_handle = NULL; - -uint32_t _sync_add(volatile uint32_t* target, uint32_t addend) { - static_assert(sizeof(*target) == sizeof(long), ""); - return InterlockedAdd((volatile long*) target, addend) - addend; -} +static HANDLE _keyed_event = NULL; int reflock_global_init(void) { NTSTATUS status = - NtCreateKeyedEvent(&_global_event_handle, ~(ACCESS_MASK) 0, NULL, 0); + NtCreateKeyedEvent(&_keyed_event, ~(ACCESS_MASK) 0, NULL, 0); if (status != STATUS_SUCCESS) return_error(-1, RtlNtStatusToDosError(status)); return 0; } void reflock_init(reflock_t* reflock) { - memset(reflock, 0, sizeof *reflock); + reflock->state = 0; } -static _reflock_private_t* _reflock_private(reflock_t* reflock) { - static_assert(sizeof(reflock_t) == sizeof(_reflock_private_t), ""); - return (_reflock_private_t*) reflock; -} - -static void _wait_for_event(const uint32_t* address) { +static void _signal_event(const void* address) { NTSTATUS status = - NtWaitForKeyedEvent(_global_event_handle, (PVOID) address, FALSE, NULL); + NtReleaseKeyedEvent(_keyed_event, (PVOID) address, FALSE, NULL); if (status != STATUS_SUCCESS) abort(); } -static void _signal_event(const uint32_t* address) { +static void _await_event(const void* address) { NTSTATUS status = - NtReleaseKeyedEvent(_global_event_handle, (PVOID) address, FALSE, NULL); + NtWaitForKeyedEvent(_keyed_event, (PVOID) address, FALSE, NULL); if (status != STATUS_SUCCESS) abort(); } -reflock_token_t reflock_ref_and_announce_lock(reflock_t* reflock) { - _reflock_private_t* priv = _reflock_private(reflock); +static inline uint32_t _sync_add_and_fetch(volatile uint32_t* target, + uint32_t value) { + static_assert(sizeof(*target) == sizeof(long), ""); + return InterlockedAdd((volatile long*) target, value); +} - uint32_t prev = _sync_add(&priv->counter, _CTR_REF | _CTR_LOCK); - uint32_t prev_lock_count = prev & _CTR_LOCK_MASK; +static inline uint32_t _sync_sub_and_fetch(volatile uint32_t* target, + uint32_t value) { + uint32_t add_value = -(int32_t) value; + return _sync_add_and_fetch(target, add_value); +} - if (prev & _CTR_DELETE) - abort(); +static inline uint32_t _sync_fetch_and_set(volatile uint32_t* target, + uint32_t value) { + static_assert(sizeof(*target) == sizeof(long), ""); + return InterlockedExchange((volatile long*) target, value); +} - if (prev_lock_count == 0) - return _TOK_REFERENCED | _TOK_LOCK_ANNOUNCED | _TOK_NO_WAIT; +void reflock_ref(reflock_t* reflock) { + uint32_t state = _sync_add_and_fetch(&reflock->state, _REF); + unused(state); + assert((state & _DESTROY_MASK) == 0); /* Overflow or destroyed. */ +} + +void reflock_unref(reflock_t* reflock) { + uint32_t state = _sync_sub_and_fetch(&reflock->state, _REF); + uint32_t ref_count = state & _REF_MASK; + uint32_t destroy = state & _DESTROY_MASK; + + unused(ref_count); + unused(destroy); + + if (state == _DESTROY) + _signal_event(reflock); else - return _TOK_REFERENCED | _TOK_LOCK_ANNOUNCED | _TOK_MUST_WAIT; + assert(destroy == 0 || ref_count > 0); } -reflock_token_t reflock_lock(reflock_t* reflock, reflock_token_t token) { - _reflock_private_t* priv = _reflock_private(reflock); +void reflock_unref_and_destroy(reflock_t* reflock) { + uint32_t state = _sync_add_and_fetch(&reflock->state, _DESTROY - _REF); + uint32_t ref_count = state & _REF_MASK; - switch (token) { - case _TOK_REFERENCED: { - /* Lock hasn't been announced; increase lock count and wait if necessary. - */ - uint32_t prev_lock_count = - _sync_add(&priv->counter, _CTR_LOCK) & _CTR_LOCK_MASK; - if (prev_lock_count > 0) - _wait_for_event(&priv->events.lock); - break; - } + assert((state & _DESTROY_MASK) == + _DESTROY); /* Underflow or already destroyed. */ - case _TOK_REFERENCED | _TOK_LOCK_ANNOUNCED | _TOK_MUST_WAIT: - /* Lock has been announced; wait for the thread currently owning the lock - * to signal the event. */ - _wait_for_event(&priv->events.lock); - break; + if (ref_count != 0) + _await_event(reflock); - case _TOK_REFERENCED | _TOK_LOCK_ANNOUNCED | _TOK_NO_WAIT: - /* There were no waiters when the lock was announced; no need to wait for - * an event. */ - break; - - default: - abort(); - } - - return _TOK_REFERENCED | _TOK_LOCKED; -} - -void _unlock_signal_event(_reflock_private_t* priv, uint32_t counter) { - if (counter & _CTR_LOCK_MASK) { - /* Wake a thread that is waiting to acquire the lock. */ - _signal_event(&priv->events.lock); - } else if ((counter & _CTR_DELETE) && !(counter & _CTR_REF_MASK)) { - /* Wake the thread that is waiting to delete the lock. */ - _signal_event(&priv->events.delete); - } -} - -reflock_token_t reflock_unlock(reflock_t* reflock, reflock_token_t token) { - _reflock_private_t* priv = _reflock_private(reflock); - - uint32_t add = -(int32_t) _CTR_LOCK; - uint32_t prev_counter = _sync_add(&priv->counter, add); - - _unlock_signal_event(priv, prev_counter + add); - - unused(token); - assert(token == (_TOK_REFERENCED | _TOK_LOCKED)); - return _TOK_REFERENCED; -} - -void reflock_unref_and_unlock(reflock_t* reflock, reflock_token_t token) { - _reflock_private_t* priv = _reflock_private(reflock); - - uint32_t add = -(int32_t)(_CTR_REF | _CTR_LOCK); - uint32_t prev_counter = _sync_add(&priv->counter, add); - - _unlock_signal_event(priv, prev_counter + add); - - unused(token); - assert(token == (_TOK_REFERENCED | _TOK_LOCKED)); -} - -reflock_token_t reflock_announce_delete(reflock_t* reflock) { - _reflock_private_t* priv = _reflock_private(reflock); - - uint32_t prev = _sync_add(&priv->counter, _CTR_DELETE); - uint32_t ref_count = prev & _CTR_REF_MASK; - - if (prev & _CTR_DELETE) - abort(); - - if (ref_count == 0) - return _TOK_DELETE_ANNOUNCED | _TOK_NO_WAIT; - else - return _TOK_DELETE_ANNOUNCED | _TOK_MUST_WAIT; -} - -void reflock_delete(reflock_t* reflock, reflock_token_t token) { - _reflock_private_t* priv = _reflock_private(reflock); - - switch (token) { - case _TOK_DELETE_ANNOUNCED | _TOK_MUST_WAIT: - /* The lock was owned when deletion was announced; wait for the last lock - * owner to signal the delete event. */ - _wait_for_event(&priv->events.delete); - break; - - case _TOK_DELETE_ANNOUNCED | _TOK_NO_WAIT: - /* There were no other waiters when deletion was announced; there's no - * need to wait for an event. */ - break; - - default: - abort(); - } - - priv->counter = 0; + state = _sync_fetch_and_set(&reflock->state, _POISON); + assert(state == _DESTROY); } diff --git a/src/reflock.h b/src/reflock.h index fb9a389..fb548b4 100644 --- a/src/reflock.h +++ b/src/reflock.h @@ -1,23 +1,37 @@ +#ifndef REFLOCK_H_ +#define REFLOCK_H_ + +/* The reflock is a special kind of lock that normally prevents a chunk of + * memory from being freed, but does allow the chunk of memory to eventually be + * released in a coordinated fashion. + * + * Under normal operation, threads increase and decrease the reference count, + * which are wait-free operations. + * + * Exactly once during the reflock's lifecycle, a thread holding a reference to + * the lock may "destroy" the lock; this operation blocks until all other + * threads holding a reference to the lock have dereferenced it. After + * "destroy" returns, the calling thread may assume that no other threads have + * a reference to the lock. + * + * Attemmpting to lock or destroy a lock after reflock_unref_and_destroy() has + * been called is invalid and results in undefined behavior. Therefore the user + * should use another lock to guarantee that this can't happen. + */ + #include -typedef struct reflock { - uint64_t internal; -} reflock_t; +#include "internal.h" -typedef uint32_t reflock_token_t; +typedef struct reflock { + uint32_t state; +} reflock_t; EPOLL_INTERNAL int reflock_global_init(void); EPOLL_INTERNAL void reflock_init(reflock_t* reflock); +EPOLL_INTERNAL void reflock_ref(reflock_t* reflock); +EPOLL_INTERNAL void reflock_unref(reflock_t* reflock); +EPOLL_INTERNAL void reflock_unref_and_destroy(reflock_t* reflock); -EPOLL_INTERNAL reflock_token_t - reflock_ref_and_announce_lock(reflock_t* reflock); -EPOLL_INTERNAL reflock_token_t reflock_lock(reflock_t* reflock, - reflock_token_t token); -EPOLL_INTERNAL reflock_token_t reflock_unlock(reflock_t* reflock, - reflock_token_t token); -EPOLL_INTERNAL void reflock_unref_and_unlock(reflock_t* reflock, - reflock_token_t token); - -EPOLL_INTERNAL reflock_token_t reflock_announce_delete(reflock_t* reflock); -EPOLL_INTERNAL void reflock_delete(reflock_t* reflock, reflock_token_t token); +#endif /* REFLOCK_H_ */ diff --git a/test/test-reflock.c b/test/test-reflock.c index f63571c..4f93b73 100644 --- a/test/test-reflock.c +++ b/test/test-reflock.c @@ -10,133 +10,80 @@ #include "win.h" #define THREAD_COUNT 20 -#define TEST_ITERATIONS 10 -#define TEST_LENGTH 1000 - -typedef unsigned(__stdcall* thread_entry_t)(void* arg); +#define TEST_ITERATIONS 100 +#define TEST_LENGTH 100 typedef struct test_context { - reflock_t lock; - volatile bool ref_valid; - volatile bool content_valid; + reflock_t reflock; + SRWLOCK srwlock; volatile bool stop; } test_context_t; +static void init_context(test_context_t* context) { + reflock_init(&context->reflock); + InitializeSRWLock(&context->srwlock); + context->stop = false; +} + static void yield(void) { - int count = rand() % THREAD_COUNT; + int count = rand() % 3; while (count--) Sleep(0); } -static unsigned int __stdcall thread1(void* arg) { +static unsigned int __stdcall test_thread(void* arg) { test_context_t* context = arg; - reflock_token_t token; uint64_t lock_count = 0; - token = reflock_ref_and_announce_lock(&context->lock); - - yield(); - - token = reflock_lock(&context->lock, token); - - context->content_valid = false; + AcquireSRWLockShared(&context->srwlock); while (!context->stop) { + reflock_ref(&context->reflock); + ReleaseSRWLockShared(&context->srwlock); + lock_count++; + yield(); - assert(context->ref_valid); - assert(!context->content_valid); - context->content_valid = true; - - token = reflock_unlock(&context->lock, token); + reflock_unref(&context->reflock); yield(); - token = reflock_lock(&context->lock, token); - - assert(context->content_valid); - context->content_valid = false; + AcquireSRWLockShared(&context->srwlock); } - assert(context->ref_valid); - assert(!context->content_valid); - context->content_valid = true; - reflock_unref_and_unlock(&context->lock, token); + ReleaseSRWLockShared(&context->srwlock); - assert(lock_count > 100); /* Hopefully a lot more. */ + assert(lock_count > 100); /* Hopefully much more. */ + printf("%llu\n", lock_count); return 0; } -static unsigned int __stdcall thread2(void* arg) { - test_context_t* context = arg; - uint64_t lock_count = 0; - - while (!context->stop) { - reflock_token_t token = reflock_ref_and_announce_lock(&context->lock); - - yield(); - - token = reflock_lock(&context->lock, token); - - assert(context->ref_valid); - assert(context->content_valid); - context->content_valid = false; - - lock_count++; - - yield(); - - assert(context->ref_valid); - assert(!context->content_valid); - context->content_valid = true; - - reflock_unref_and_unlock(&context->lock, token); - } - - assert(lock_count > 100); /* Hopefully a lot more. */ - - return 0; -} - -static void delete_ref(test_context_t* context) { - reflock_token_t token; - - token = reflock_announce_delete(&context->lock); - +static void destroy_reflock(test_context_t* context) { + AcquireSRWLockExclusive(&context->srwlock); context->stop = true; + reflock_ref(&context->reflock); + ReleaseSRWLockExclusive(&context->srwlock); - yield(); - - reflock_delete(&context->lock, token); - - assert(context->content_valid); - context->content_valid = false; - assert(context->ref_valid); - context->ref_valid = false; + reflock_unref_and_destroy(&context->reflock); } static void run_test_iteration(void) { test_context_t context; HANDLE threads[THREAD_COUNT]; - reflock_init(&context.lock); - - context.ref_valid = true; - context.content_valid = true; - context.stop = false; + init_context(&context); for (size_t i = 0; i < array_count(threads); i++) { - thread_entry_t thread_entry = (i % 2 == 0) ? thread1 : thread2; HANDLE thread = - (HANDLE) _beginthreadex(NULL, 0, thread_entry, &context, 0, NULL); + (HANDLE) _beginthreadex(NULL, 0, test_thread, &context, 0, NULL); assert(thread != INVALID_HANDLE_VALUE); threads[i] = thread; } Sleep(TEST_LENGTH); - delete_ref(&context); + destroy_reflock(&context); for (size_t i = 0; i < array_count(threads); i++) { HANDLE thread = threads[i];