From 9b78dd977bb5f13b0fbe87c257d37eaf34b6aeb7 Mon Sep 17 00:00:00 2001 From: Ben Vanik Date: Sat, 12 Jul 2014 17:48:54 -0700 Subject: [PATCH] Migrating atomic ops to std::atomic where possible and poly. --- src/alloy/backend/ivm/ivm_intcode.cc | 8 +- src/alloy/core.h | 1 - src/alloy/tracing/tracer.cc | 6 +- src/poly/atomic.h | 127 +++++++++++++++++++++++++ src/poly/poly.h | 1 + src/poly/sources.gypi | 1 + src/xenia/atomic.h | 100 ------------------- src/xenia/common.h | 2 +- src/xenia/core/ref.cc | 4 +- src/xenia/cpu/xenon_thread_state.cc | 4 +- src/xenia/kernel/objects/xthread.cc | 2 +- src/xenia/kernel/objects/xthread.h | 4 +- src/xenia/kernel/xboxkrnl_rtl.cc | 12 +-- src/xenia/kernel/xboxkrnl_threading.cc | 22 +++-- src/xenia/kernel/xboxkrnl_threading.h | 4 +- src/xenia/kernel/xobject.cc | 8 +- src/xenia/kernel/xobject.h | 6 +- src/xenia/sources.gypi | 1 - 18 files changed, 174 insertions(+), 139 deletions(-) create mode 100644 src/poly/atomic.h delete mode 100644 src/xenia/atomic.h diff --git a/src/alloy/backend/ivm/ivm_intcode.cc b/src/alloy/backend/ivm/ivm_intcode.cc index 38fc97b06..cdc25f136 100644 --- a/src/alloy/backend/ivm/ivm_intcode.cc +++ b/src/alloy/backend/ivm/ivm_intcode.cc @@ -4020,16 +4020,16 @@ int Translate_UNPACK(TranslationContext& ctx, Instr* i) { } uint32_t IntCode_ATOMIC_EXCHANGE_I32(IntCodeState& ics, const IntCode* i) { - auto address = (uint8_t*)ics.rf[i->src1_reg].u64; + auto address = (uint32_t*)ics.rf[i->src1_reg].u64; auto new_value = ics.rf[i->src2_reg].u32; - auto old_value = xe_atomic_exchange_32(new_value, address); + auto old_value = poly::atomic_exchange(new_value, address); ics.rf[i->dest_reg].u32 = old_value; return IA_NEXT; } uint32_t IntCode_ATOMIC_EXCHANGE_I64(IntCodeState& ics, const IntCode* i) { - auto address = (uint8_t*)ics.rf[i->src1_reg].u64; + auto address = (uint64_t*)ics.rf[i->src1_reg].u64; auto new_value = ics.rf[i->src2_reg].u64; - auto old_value = xe_atomic_exchange_64(new_value, address); + auto old_value = poly::atomic_exchange(new_value, address); ics.rf[i->dest_reg].u64 = old_value; return IA_NEXT; } diff --git a/src/alloy/core.h b/src/alloy/core.h index 9e25f171d..3d60666da 100644 --- a/src/alloy/core.h +++ b/src/alloy/core.h @@ -11,7 +11,6 @@ #define ALLOY_CORE_H_ // TODO(benvanik): move the common stuff into here? -#include #include #include #include diff --git a/src/alloy/tracing/tracer.cc b/src/alloy/tracing/tracer.cc index fbb4c43bb..01ee501d3 100644 --- a/src/alloy/tracing/tracer.cc +++ b/src/alloy/tracing/tracer.cc @@ -9,15 +9,17 @@ #include +#include + #include namespace alloy { namespace tracing { -volatile int next_thread_id_ = 0x10000000; +std::atomic next_thread_id_(0x10000000); Tracer::Tracer(Channel* channel) : channel_(channel) { - thread_id_ = xe_atomic_inc_32(&next_thread_id_); + thread_id_ = ++next_thread_id_; } Tracer::~Tracer() {} diff --git a/src/poly/atomic.h b/src/poly/atomic.h new file mode 100644 index 000000000..dab7e39bc --- /dev/null +++ b/src/poly/atomic.h @@ -0,0 +1,127 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2014 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef POLY_ATOMIC_H_ +#define POLY_ATOMIC_H_ + +#include + +#include +#include + +namespace poly { + +// These functions are modeled off of the Apple OSAtomic routines +// http://developer.apple.com/library/mac/#documentation/DriversKernelHardware/Reference/libkern_ref/OSAtomic_h/ + +#if XE_LIKE_OSX +#include + +inline int32_t atomic_inc(volatile int32_t* value) { + return OSAtomicIncrement32Barrier(reinterpret_cast(value)); +} +inline int32_t atomic_dec(volatile int32_t* value) { + return OSAtomicDecrement32Barrier(reinterpret_cast(value)); +} + +inline int32_t atomic_exchange(int32_t new_value, volatile int32_t* value) { + // +} +inline int64_t atomic_exchange(int64_t new_value, volatile int64_t* value) { + // +} + +inline int32_t atomic_cas(int32_t old_value, int32_t new_value, + volatile int32_t* value) { + return OSAtomicCompareAndSwap32Barrier( + old_value, new_value, reinterpret_cast(value)); +} + +#elif XE_LIKE_WIN32 + +inline int32_t atomic_inc(volatile int32_t* value) { + return InterlockedIncrement(reinterpret_cast(value)); +} +inline int32_t atomic_dec(volatile int32_t* value) { + return InterlockedDecrement(reinterpret_cast(value)); +} + +inline int32_t atomic_exchange(int32_t new_value, volatile int32_t* value) { + return InterlockedExchange(reinterpret_cast(value), + new_value); +} +inline int64_t atomic_exchange(int64_t new_value, volatile int64_t* value) { + return InterlockedExchange64(reinterpret_cast(value), + new_value); +} + +inline int32_t atomic_cas(int32_t old_value, int32_t new_value, + volatile int32_t* value) { + return InterlockedCompareExchange(reinterpret_cast(value), + new_value, old_value) == old_value; +} + +#elif XE_LIKE_POSIX + +inline int32_t atomic_inc(volatile int32_t* value) { + return __sync_add_and_fetch(value, 1); +} +inline int32_t atomic_dec(volatile int32_t* value) { + return __sync_sub_and_fetch(value, 1); +} + +inline int32_t atomic_exchange(int32_t new_value, volatile int32_t* value) { + // +} +inline int64_t atomic_exchange(int64_t new_value, volatile int64_t* value) { + // +} + +inline int32_t atomic_cas(int32_t old_value, int32_t new_value, + volatile int32_t* value) { + return __sync_bool_compare_and_swap(reinterpret_cast(value), + old_value, new_value); +} + +#else + +#error No atomic primitives defined for this platform/cpu combination. + +#endif // OSX + +inline uint32_t atomic_inc(volatile uint32_t* value) { + return static_cast( + atomic_inc(reinterpret_cast(value))); +} +inline uint32_t atomic_dec(volatile uint32_t* value) { + return static_cast( + atomic_dec(reinterpret_cast(value))); +} + +inline uint32_t atomic_exchange(uint32_t new_value, volatile uint32_t* value) { + return static_cast( + atomic_exchange(static_cast(new_value), + reinterpret_cast(value))); +} +inline uint64_t atomic_exchange(uint64_t new_value, volatile uint64_t* value) { + return static_cast( + atomic_exchange(static_cast(new_value), + reinterpret_cast(value))); +} + +inline uint32_t atomic_cas(uint32_t old_value, uint32_t new_value, + volatile uint32_t* value) { + return static_cast(atomic_cas( + static_cast(old_value), static_cast(new_value), + reinterpret_cast(value))); +} + +} // namespace poly + +#endif // POLY_ATOMIC_H_ diff --git a/src/poly/poly.h b/src/poly/poly.h index 52cc74b79..b7af898ff 100644 --- a/src/poly/poly.h +++ b/src/poly/poly.h @@ -11,6 +11,7 @@ #define POLY_POLY_H_ #include +#include #include #include #include diff --git a/src/poly/sources.gypi b/src/poly/sources.gypi index 7a29bd85b..c93d66a5d 100644 --- a/src/poly/sources.gypi +++ b/src/poly/sources.gypi @@ -2,6 +2,7 @@ { 'sources': [ 'assert.h', + 'atomic.h', 'config.h', 'cxx_compat.h', 'math.h', diff --git a/src/xenia/atomic.h b/src/xenia/atomic.h deleted file mode 100644 index bd1ebc14d..000000000 --- a/src/xenia/atomic.h +++ /dev/null @@ -1,100 +0,0 @@ -/** - ****************************************************************************** - * Xenia : Xbox 360 Emulator Research Project * - ****************************************************************************** - * Copyright 2013 Ben Vanik. All rights reserved. * - * Released under the BSD license - see LICENSE in the root for more details. * - ****************************************************************************** - */ - -#ifndef XENIA_ATOMIC_H_ -#define XENIA_ATOMIC_H_ - -#include - -#include -#include - - -// These functions are modeled off of the Apple OSAtomic routines -// http://developer.apple.com/library/mac/#documentation/DriversKernelHardware/Reference/libkern_ref/OSAtomic_h/ - -#if XE_LIKE_OSX -#include - -#define xe_atomic_inc_32(value) \ - OSAtomicIncrement32Barrier(value) -#define xe_atomic_dec_32(value) \ - OSAtomicDecrement32Barrier(value) -#define xe_atomic_add_32(amount, value) \ - ((void)OSAtomicAdd32Barrier(amount, value)) -#define xe_atomic_sub_32(amount, value) \ - ((void)OSAtomicAdd32Barrier(-amount, value)) -#define xe_atomic_exchange_32(newValue, value) \ - TODOTODO -#define xe_atomic_cas_32(oldValue, newValue, value) \ - OSAtomicCompareAndSwap32Barrier(oldValue, newValue, value) - -typedef OSQueueHead xe_atomic_stack_t; -#define xe_atomic_stack_init(stack) \ - *(stack) = (OSQueueHead)OS_ATOMIC_QUEUE_INIT -#define xe_atomic_stack_enqueue(stack, item, offset) \ - OSAtomicEnqueue((OSQueueHead*)stack, item, offset) -#define xe_atomic_stack_dequeue(stack, offset) \ - OSAtomicDequeue((OSQueueHead*)stack, offset) - -#elif XE_LIKE_WIN32 - -#define xe_atomic_inc_32(value) \ - InterlockedIncrement((volatile LONG*)value) -#define xe_atomic_dec_32(value) \ - InterlockedDecrement((volatile LONG*)value) -#define xe_atomic_add_32(amount, value) \ - ((void)InterlockedExchangeAdd((volatile LONG*)value, amount)) -#define xe_atomic_sub_32(amount, value) \ - ((void)InterlockedExchangeSubtract((volatile unsigned*)value, amount)) -#define xe_atomic_exchange_32(newValue, value) \ - InterlockedExchange((volatile LONG*)value, newValue) -#define xe_atomic_exchange_64(newValue, value) \ - InterlockedExchange64((volatile LONGLONG*)value, newValue) -#define xe_atomic_cas_32(oldValue, newValue, value) \ - (InterlockedCompareExchange((volatile LONG*)value, newValue, oldValue) == oldValue) - -typedef SLIST_HEADER xe_atomic_stack_t; -#define xe_atomic_stack_init(stack) \ - InitializeSListHead((PSLIST_HEADER)stack) -#define xe_atomic_stack_enqueue(stack, item, offset) \ - XEIGNORE(InterlockedPushEntrySList((PSLIST_HEADER)stack, (PSLIST_ENTRY)((byte*)item + offset))) -XEFORCEINLINE void* xe_atomic_stack_dequeue(xe_atomic_stack_t* stack, - const size_t offset) { - void* ptr = (void*)InterlockedPopEntrySList((PSLIST_HEADER)stack); - if (ptr) { - return (void*)(((uint8_t*)ptr) - offset); - } else { - return NULL; - } -} - -#elif XE_LIKE_POSIX - -#define xe_atomic_inc_32(value) \ - __sync_add_and_fetch(value, 1) -#define xe_atomic_dec_32(value) \ - __sync_sub_and_fetch(value, 1) -#define xe_atomic_add_32(amount, value) \ - __sync_fetch_and_add(value, amount) -#define xe_atomic_sub_32(amount, value) \ - __sync_fetch_and_sub(value, amount) -#define xe_atomic_exchange_32(newValue, value) \ - TODOTODO -#define xe_atomic_cas_32(oldValue, newValue, value) \ - __sync_bool_compare_and_swap(value, oldValue, newValue) - -#else - -#error No atomic primitives defined for this platform/cpu combination. - -#endif // OSX - - -#endif // XENIA_ATOMIC_H_ diff --git a/src/xenia/common.h b/src/xenia/common.h index c76f9eae1..3e9010f26 100644 --- a/src/xenia/common.h +++ b/src/xenia/common.h @@ -11,8 +11,8 @@ #define XENIA_COMMON_H_ #include +#include -#include #include #include #include diff --git a/src/xenia/core/ref.cc b/src/xenia/core/ref.cc index 68a847d6b..c09c613eb 100644 --- a/src/xenia/core/ref.cc +++ b/src/xenia/core/ref.cc @@ -15,14 +15,14 @@ void xe_ref_init(xe_ref_t* ref) { } void xe_ref_retain(xe_ref_t* ref) { - xe_atomic_inc_32(&ref->count); + poly::atomic_inc(&ref->count); } void xe_ref_release(xe_ref_t* ref, xe_ref_dealloc_t dealloc) { if (!ref) { return; } - if (!xe_atomic_dec_32(&ref->count)) { + if (!poly::atomic_dec(&ref->count)) { if (dealloc) { dealloc(ref); } diff --git a/src/xenia/cpu/xenon_thread_state.cc b/src/xenia/cpu/xenon_thread_state.cc index e5ed16da5..57959e15f 100644 --- a/src/xenia/cpu/xenon_thread_state.cc +++ b/src/xenia/cpu/xenon_thread_state.cc @@ -76,7 +76,7 @@ volatile int* XenonThreadState::suspend_flag_address() const { int XenonThreadState::Suspend(uint32_t timeout_ms) { // Set suspend flag. // One of the checks should call in to OnSuspend() at some point. - xe_atomic_inc_32(&context_->suspend_flag); + poly::atomic_inc(&context_->suspend_flag); return 0; } @@ -86,7 +86,7 @@ int XenonThreadState::Resume(bool force) { context_->suspend_flag = 0; SetEvent(debug_break_); } else { - if (!xe_atomic_dec_32(&context_->suspend_flag)) { + if (!poly::atomic_dec(&context_->suspend_flag)) { SetEvent(debug_break_); } } diff --git a/src/xenia/kernel/objects/xthread.cc b/src/xenia/kernel/objects/xthread.cc index 47566c20d..ad58f0ff7 100644 --- a/src/xenia/kernel/objects/xthread.cc +++ b/src/xenia/kernel/objects/xthread.cc @@ -406,7 +406,7 @@ void XThread::LeaveCriticalRegion() { } uint32_t XThread::RaiseIrql(uint32_t new_irql) { - return xe_atomic_exchange_32(new_irql, &irql_); + return irql_.exchange(new_irql); } void XThread::LowerIrql(uint32_t new_irql) { diff --git a/src/xenia/kernel/objects/xthread.h b/src/xenia/kernel/objects/xthread.h index 8b403429b..cec3c4dce 100644 --- a/src/xenia/kernel/objects/xthread.h +++ b/src/xenia/kernel/objects/xthread.h @@ -10,6 +10,8 @@ #ifndef XENIA_KERNEL_XBOXKRNL_XTHREAD_H_ #define XENIA_KERNEL_XBOXKRNL_XTHREAD_H_ +#include + #include #include @@ -94,7 +96,7 @@ private: char* name_; - uint32_t irql_; + std::atomic irql_; xe_mutex_t* apc_lock_; NativeList* apc_list_; diff --git a/src/xenia/kernel/xboxkrnl_rtl.cc b/src/xenia/kernel/xboxkrnl_rtl.cc index 5b632a6b3..ebf874eb3 100644 --- a/src/xenia/kernel/xboxkrnl_rtl.cc +++ b/src/xenia/kernel/xboxkrnl_rtl.cc @@ -606,13 +606,13 @@ void xeRtlEnterCriticalSection(uint32_t cs_ptr, uint32_t thread_id) { uint32_t spin_wait_remaining = cs->spin_count_div_256 * 256; spin: - if (xe_atomic_inc_32(&cs->lock_count) != 0) { + if (poly::atomic_inc(&cs->lock_count) != 0) { // If this thread already owns the CS increment the recursion count. if (cs->owning_thread_id == thread_id) { cs->recursion_count++; return; } - xe_atomic_dec_32(&cs->lock_count); + poly::atomic_dec(&cs->lock_count); // Thread was locked - spin wait. if (spin_wait_remaining) { @@ -658,13 +658,13 @@ uint32_t xeRtlTryEnterCriticalSection(uint32_t cs_ptr, uint32_t thread_id) { X_RTL_CRITICAL_SECTION* cs = (X_RTL_CRITICAL_SECTION*)IMPL_MEM_ADDR(cs_ptr); - if (xe_atomic_cas_32(-1, 0, &cs->lock_count)) { + if (poly::atomic_cas(-1, 0, &cs->lock_count)) { // Able to steal the lock right away. cs->owning_thread_id = thread_id; cs->recursion_count = 1; return 1; } else if (cs->owning_thread_id == thread_id) { - xe_atomic_inc_32(&cs->lock_count); + poly::atomic_inc(&cs->lock_count); ++cs->recursion_count; return 1; } @@ -699,13 +699,13 @@ void xeRtlLeaveCriticalSection(uint32_t cs_ptr) { // Drop recursion count - if we are still not zero'ed return. uint32_t recursion_count = --cs->recursion_count; if (recursion_count) { - xe_atomic_dec_32(&cs->lock_count); + poly::atomic_dec(&cs->lock_count); return; } // Unlock! cs->owning_thread_id = 0; - if (xe_atomic_dec_32(&cs->lock_count) != -1) { + if (poly::atomic_dec(&cs->lock_count) != -1) { // There were waiters - wake one of them. // TODO(benvanik): wake a waiter. XELOGE("RtlLeaveCriticalSection would have woken a waiter"); diff --git a/src/xenia/kernel/xboxkrnl_threading.cc b/src/xenia/kernel/xboxkrnl_threading.cc index 8e10a84ce..6aaa8b78c 100644 --- a/src/xenia/kernel/xboxkrnl_threading.cc +++ b/src/xenia/kernel/xboxkrnl_threading.cc @@ -1258,9 +1258,9 @@ SHIM_CALL NtSignalAndWaitForSingleObjectEx_shim( } -uint32_t xeKfAcquireSpinLock(void* lock_ptr) { +uint32_t xeKfAcquireSpinLock(uint32_t* lock_ptr) { // Lock. - while (!xe_atomic_cas_32(0, 1, lock_ptr)) { + while (!poly::atomic_cas(0, 1, lock_ptr)) { // Spin! // TODO(benvanik): error on deadlock? } @@ -1279,19 +1279,20 @@ SHIM_CALL KfAcquireSpinLock_shim( "KfAcquireSpinLock(%.8X)", lock_ptr); - uint32_t old_irql = xeKfAcquireSpinLock(SHIM_MEM_ADDR(lock_ptr)); + auto lock = reinterpret_cast(SHIM_MEM_ADDR(lock_ptr)); + uint32_t old_irql = xeKfAcquireSpinLock(lock); SHIM_SET_RETURN_64(old_irql); } -void xeKfReleaseSpinLock(void* lock_ptr, uint32_t old_irql) { +void xeKfReleaseSpinLock(uint32_t* lock_ptr, uint32_t old_irql) { // Restore IRQL. XThread* thread = XThread::GetCurrentThread(); thread->LowerIrql(old_irql); // Unlock. - xe_atomic_dec_32(lock_ptr); + poly::atomic_dec(lock_ptr); } @@ -1305,7 +1306,8 @@ SHIM_CALL KfReleaseSpinLock_shim( lock_ptr, old_irql); - xeKfReleaseSpinLock(SHIM_MEM_ADDR(lock_ptr), old_irql); + xeKfReleaseSpinLock(reinterpret_cast(SHIM_MEM_ADDR(lock_ptr)), + old_irql); } @@ -1318,8 +1320,8 @@ SHIM_CALL KeAcquireSpinLockAtRaisedIrql_shim( lock_ptr); // Lock. - void* lock = SHIM_MEM_ADDR(lock_ptr); - while (!xe_atomic_cas_32(0, 1, lock)) { + auto lock = reinterpret_cast(SHIM_MEM_ADDR(lock_ptr)); + while (!poly::atomic_cas(0, 1, lock)) { // Spin! // TODO(benvanik): error on deadlock? } @@ -1335,8 +1337,8 @@ SHIM_CALL KeReleaseSpinLockFromRaisedIrql_shim( lock_ptr); // Unlock. - void* lock = SHIM_MEM_ADDR(lock_ptr); - xe_atomic_dec_32(lock); + auto lock = reinterpret_cast(SHIM_MEM_ADDR(lock_ptr)); + poly::atomic_dec(lock); } diff --git a/src/xenia/kernel/xboxkrnl_threading.h b/src/xenia/kernel/xboxkrnl_threading.h index 9b33964f9..62f807065 100644 --- a/src/xenia/kernel/xboxkrnl_threading.h +++ b/src/xenia/kernel/xboxkrnl_threading.h @@ -52,8 +52,8 @@ X_STATUS xeKeWaitForSingleObject( void* object_ptr, uint32_t wait_reason, uint32_t processor_mode, uint32_t alertable, uint64_t* opt_timeout); -uint32_t xeKfAcquireSpinLock(void* lock_ptr); -void xeKfReleaseSpinLock(void* lock_ptr, uint32_t old_irql); +uint32_t xeKfAcquireSpinLock(uint32_t* lock_ptr); +void xeKfReleaseSpinLock(uint32_t* lock_ptr, uint32_t old_irql); void xeKeEnterCriticalRegion(); void xeKeLeaveCriticalRegion(); diff --git a/src/xenia/kernel/xobject.cc b/src/xenia/kernel/xobject.cc index df99a1d2e..f3a48d8f7 100644 --- a/src/xenia/kernel/xobject.cc +++ b/src/xenia/kernel/xobject.cc @@ -46,22 +46,22 @@ X_HANDLE XObject::handle() const { } void XObject::RetainHandle() { - xe_atomic_inc_32(&handle_ref_count_); + ++handle_ref_count_; } bool XObject::ReleaseHandle() { - if (!xe_atomic_dec_32(&handle_ref_count_)) { + if (--handle_ref_count_ == 0) { return true; } return false; } void XObject::Retain() { - xe_atomic_inc_32(&pointer_ref_count_); + ++pointer_ref_count_; } void XObject::Release() { - if (!xe_atomic_dec_32(&pointer_ref_count_)) { + if (--pointer_ref_count_ == 0) { assert_true(pointer_ref_count_ >= handle_ref_count_); delete this; } diff --git a/src/xenia/kernel/xobject.h b/src/xenia/kernel/xobject.h index 1a3b8fae7..7733db4ab 100644 --- a/src/xenia/kernel/xobject.h +++ b/src/xenia/kernel/xobject.h @@ -10,6 +10,8 @@ #ifndef XENIA_KERNEL_XBOXKRNL_XOBJECT_H_ #define XENIA_KERNEL_XBOXKRNL_XOBJECT_H_ +#include + #include #include @@ -87,8 +89,8 @@ protected: KernelState* kernel_state_; private: - volatile int32_t handle_ref_count_; - volatile int32_t pointer_ref_count_; + std::atomic handle_ref_count_; + std::atomic pointer_ref_count_; Type type_; X_HANDLE handle_; diff --git a/src/xenia/sources.gypi b/src/xenia/sources.gypi index a60ec1749..48bb82f59 100644 --- a/src/xenia/sources.gypi +++ b/src/xenia/sources.gypi @@ -1,7 +1,6 @@ # Copyright 2013 Ben Vanik. All Rights Reserved. { 'sources': [ - 'atomic.h', 'byte_order.h', 'common.h', 'config.h',