diff --git a/lib/std/atomic.c3 b/lib/std/atomic.c3 index 415567fdb..0a3a7cc82 100644 --- a/lib/std/atomic.c3 +++ b/lib/std/atomic.c3 @@ -85,31 +85,31 @@ macro Type Atomic.min(&self, Type value, AtomicOrdering ordering = SEQ_CONSISTEN return @atomic_exec(atomic::fetch_min, data, value, ordering); } -macro Type Atomic.or(&self, ulong value, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) +macro Type Atomic.or(&self, uint value, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) { Type* data = &self.data; return @atomic_exec(atomic::fetch_or, data, value, ordering); } -fn Type Atomic.xor(&self, ulong value, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) +fn Type Atomic.xor(&self, uint value, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) { Type* data = &self.data; return @atomic_exec(atomic::fetch_xor, data, value, ordering); } -macro Type Atomic.and(&self, ulong value, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) +macro Type Atomic.and(&self, uint value, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) { Type* data = &self.data; return @atomic_exec(atomic::fetch_and, data, value, ordering); } -macro Type Atomic.shift_right(&self, ulong amount, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) +macro Type Atomic.shift_right(&self, uint amount, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) { Type* data = &self.data; return @atomic_exec(atomic::fetch_shift_right, data, amount, ordering); } -macro Type Atomic.shift_left(&self, ulong amount, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) +macro Type Atomic.shift_left(&self, uint amount, AtomicOrdering ordering = SEQ_CONSISTENT) @if(!types::is_float(Type)) { Type* data = &self.data; return @atomic_exec(atomic::fetch_shift_left, data, amount, ordering); @@ -139,31 +139,12 @@ module std::atomic; * @require types::is_int($typeof(*ptr)) || types::is_float($typeof(*ptr)) "Only integer/float pointers may be used." * @require $ordering != AtomicOrdering.NOT_ATOMIC && $ordering != AtomicOrdering.UNORDERED "Acquire ordering is not valid." **/ -macro fetch_add(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) +macro fetch_add(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT, bool $volatile = false, usz $alignment = 0) { - var $load_ordering = $ordering; - $if $ordering == RELEASE || $ordering == ACQUIRE_RELEASE: - $load_ordering = AtomicOrdering.SEQ_CONSISTENT; + $if $alignment == 0: + $alignment = $typeof(*ptr).sizeof; $endif - - var $StorageType = $typefrom(types::lower_to_atomic_compatible_type($typeof(*ptr))); - - $StorageType* storage_ptr = ($StorageType*)ptr; - - $typeof(*ptr) old_value; - $typeof(*ptr) new_value; - - $StorageType storage_old_value; - $StorageType storage_new_value; - - do { - storage_old_value = $$atomic_load(storage_ptr, false, $load_ordering.ordinal); - old_value = bitcast(storage_old_value, $typeof(*ptr)); - new_value = old_value + y; - storage_new_value = bitcast(new_value, $StorageType); - } while (mem::compare_exchange(storage_ptr, storage_old_value, storage_new_value, $ordering, $load_ordering) != storage_old_value); - - return old_value; + return $$atomic_fetch_add(ptr, y, $volatile, $ordering.ordinal, $alignment); } /** @@ -175,31 +156,12 @@ macro fetch_add(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) * @require types::is_int($typeof(*ptr)) || types::is_float($typeof(*ptr)) "Only integer/float pointers may be used." * @require $ordering != AtomicOrdering.NOT_ATOMIC && $ordering != AtomicOrdering.UNORDERED "Acquire ordering is not valid." **/ -macro fetch_sub(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) +macro fetch_sub(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT, bool $volatile = false, usz $alignment = 0) { - var $load_ordering = $ordering; - $if $ordering == RELEASE || $ordering == ACQUIRE_RELEASE: - $load_ordering = AtomicOrdering.SEQ_CONSISTENT; + $if $alignment == 0: + $alignment = $typeof(*ptr).sizeof; $endif - - var $StorageType = $typefrom(types::lower_to_atomic_compatible_type($typeof(*ptr))); - - $StorageType* storage_ptr = ($StorageType*)ptr; - - $typeof(*ptr) old_value; - $typeof(*ptr) new_value; - - $StorageType storage_old_value; - $StorageType storage_new_value; - - do { - storage_old_value = $$atomic_load(storage_ptr, false, $load_ordering.ordinal); - old_value = bitcast(storage_old_value, $typeof(*ptr)); - new_value = old_value - y; - storage_new_value = bitcast(new_value, $StorageType); - } while (mem::compare_exchange(storage_ptr, storage_old_value, storage_new_value, $ordering, $load_ordering) != storage_old_value); - - return old_value; + return $$atomic_fetch_sub(ptr, y, $volatile, $ordering.ordinal, $alignment); } /** @@ -285,8 +247,12 @@ macro fetch_div(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) * @require types::is_int($typeof(y)) "The value for or must be an int" * @require $ordering != AtomicOrdering.NOT_ATOMIC && $ordering != AtomicOrdering.UNORDERED "Acquire ordering is not valid." **/ -macro fetch_or(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) +macro fetch_or(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT, bool $volatile = false, usz $alignment = 0) { + $if types::is_int($typeof(*ptr)): + return $$atomic_fetch_or(ptr, y, $volatile, $ordering.ordinal, $alignment); + $endif + var $load_ordering = $ordering; $if $ordering == RELEASE || $ordering == ACQUIRE_RELEASE: $load_ordering = AtomicOrdering.SEQ_CONSISTENT; @@ -323,8 +289,12 @@ macro fetch_or(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) * @require types::is_int($typeof(y)) "The value for or must be an int" * @require $ordering != AtomicOrdering.NOT_ATOMIC && $ordering != AtomicOrdering.UNORDERED "Acquire ordering is not valid." **/ -macro fetch_xor(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) +macro fetch_xor(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT, bool $volatile = false, usz $alignment = 0) { + $if types::is_int($typeof(*ptr)): + return $$atomic_fetch_xor(ptr, y, $volatile, $ordering.ordinal, $alignment); + $endif + var $load_ordering = $ordering; $if $ordering == RELEASE || $ordering == ACQUIRE_RELEASE: $load_ordering = AtomicOrdering.SEQ_CONSISTENT; @@ -361,8 +331,12 @@ macro fetch_xor(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) * @require types::is_int($typeof(y)) "The value for or must be an int" * @require $ordering != AtomicOrdering.NOT_ATOMIC && $ordering != AtomicOrdering.UNORDERED "Acquire ordering is not valid." **/ -macro fetch_and(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) +macro fetch_and(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT, bool $volatile = false, usz $alignment = 0) { + $if types::is_int($typeof(*ptr)): + return $$atomic_fetch_and(ptr, y, $volatile, $ordering.ordinal, $alignment); + $endif + var $load_ordering = $ordering; $if $ordering == RELEASE || $ordering == ACQUIRE_RELEASE: $load_ordering = AtomicOrdering.SEQ_CONSISTENT; @@ -514,30 +488,12 @@ macro flag_clear(ptr, AtomicOrdering $ordering = SEQ_CONSISTENT) * @require types::is_int($typeof(*ptr)) || types::is_float($typeof(*ptr)) "Only integer/float pointers may be used." * @require $ordering != AtomicOrdering.NOT_ATOMIC && $ordering != AtomicOrdering.UNORDERED "Acquire ordering is not valid." **/ -macro fetch_max(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) +macro fetch_max(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT, bool $volatile = false, usz $alignment = 0) { - var $load_ordering = $ordering; - $if $ordering == RELEASE || $ordering == ACQUIRE_RELEASE: - $load_ordering = AtomicOrdering.SEQ_CONSISTENT; + $if $alignment == 0: + $alignment = $typeof(*ptr).sizeof; $endif - - var $StorageType = $typefrom(types::lower_to_atomic_compatible_type($typeof(*ptr))); - - $StorageType* storage_ptr = ($StorageType*)ptr; - - $typeof(*ptr) old_value; - $typeof(*ptr) new_value = y; - - $StorageType storage_old_value; - $StorageType storage_new_value = bitcast(new_value, $StorageType); - - do { - storage_old_value = $$atomic_load(storage_ptr, false, $load_ordering.ordinal); - old_value = bitcast(storage_old_value, $typeof(*ptr)); - if (old_value >= new_value) break; - } while (mem::compare_exchange(storage_ptr, storage_old_value, storage_new_value, $ordering, $load_ordering) != storage_old_value); - - return old_value; + return $$atomic_fetch_max(ptr, y, $volatile, $ordering.ordinal, $alignment); } /** @@ -549,28 +505,10 @@ macro fetch_max(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) * @require types::is_int($typeof(*ptr)) || types::is_float($typeof(*ptr)) "Only integer/float pointers may be used." * @require $ordering != AtomicOrdering.NOT_ATOMIC && $ordering != AtomicOrdering.UNORDERED "Acquire ordering is not valid." **/ -macro fetch_min(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT) +macro fetch_min(ptr, y, AtomicOrdering $ordering = SEQ_CONSISTENT, bool $volatile = false, usz $alignment = 0) { - var $load_ordering = $ordering; - $if $ordering == RELEASE || $ordering == ACQUIRE_RELEASE: - $load_ordering = AtomicOrdering.SEQ_CONSISTENT; + $if $alignment == 0: + $alignment = $typeof(*ptr).sizeof; $endif - - var $StorageType = $typefrom(types::lower_to_atomic_compatible_type($typeof(*ptr))); - - $StorageType* storage_ptr = ($StorageType*)ptr; - - $typeof(*ptr) old_value; - $typeof(*ptr) new_value = y; - - $StorageType storage_old_value; - $StorageType storage_new_value = bitcast(new_value, $StorageType); - - do { - storage_old_value = $$atomic_load(storage_ptr, false, $load_ordering.ordinal); - old_value = bitcast(storage_old_value, $typeof(*ptr)); - if (old_value <= new_value) break; - } while (mem::compare_exchange(storage_ptr, storage_old_value, storage_new_value, $ordering, $load_ordering) != storage_old_value); - - return old_value; + return $$atomic_fetch_min(ptr, y, $volatile, $ordering.ordinal, $alignment); } diff --git a/lib/std/atomic_nolibc.c3 b/lib/std/atomic_nolibc.c3 new file mode 100644 index 000000000..753bee9b2 --- /dev/null +++ b/lib/std/atomic_nolibc.c3 @@ -0,0 +1,63 @@ +// Copyright (c) 2023 Eduardo José Gómez Hernández. All rights reserved. +// Use of this source code is governed by the MIT license +// a copy of which can be found in the LICENSE_STDLIB file. +module std::atomic; + +macro @__atomic_compare_exchange_ordering_failure(ptr, expected, desired, $success, failure, $alignment) { + switch(failure) + { + case AtomicOrdering.RELAXED.ordinal: return $$compare_exchange(ptr, expected, desired, false, false, $success, AtomicOrdering.RELAXED.ordinal, $alignment); + case AtomicOrdering.ACQUIRE.ordinal: return $$compare_exchange(ptr, expected, desired, false, false, $success, AtomicOrdering.ACQUIRE.ordinal, $alignment); + case AtomicOrdering.SEQ_CONSISTENT.ordinal: return $$compare_exchange(ptr, expected, desired, false, false, $success, AtomicOrdering.SEQ_CONSISTENT.ordinal, $alignment); + default: assert(false, "Unrecognized failure ordering"); + } + return 0; +} + +macro @__atomic_compare_exchange_ordering_success(ptr, expected, desired, success, failure, $alignment) +{ + switch(success) + { + case AtomicOrdering.RELAXED.ordinal: return @__atomic_compare_exchange_ordering_failure(ptr, expected, desired, AtomicOrdering.RELAXED.ordinal, failure, $alignment); + case AtomicOrdering.ACQUIRE.ordinal: return @__atomic_compare_exchange_ordering_failure(ptr, expected, desired, AtomicOrdering.ACQUIRE.ordinal, failure, $alignment); + case AtomicOrdering.RELEASE.ordinal: return @__atomic_compare_exchange_ordering_failure(ptr, expected, desired, AtomicOrdering.RELEASE.ordinal, failure, $alignment); + case AtomicOrdering.ACQUIRE_RELEASE.ordinal: return @__atomic_compare_exchange_ordering_failure(ptr, expected, desired, AtomicOrdering.ACQUIRE_RELEASE.ordinal, failure, $alignment); + case AtomicOrdering.SEQ_CONSISTENT.ordinal: return @__atomic_compare_exchange_ordering_failure(ptr, expected, desired, AtomicOrdering.SEQ_CONSISTENT.ordinal, failure, $alignment); + default: assert(false, "Unrecognized success ordering"); + } + return 0; +} + +fn CInt __atomic_compare_exchange(CInt size, any* ptr, any* expected, any* desired, CInt success, CInt failure) @extern("__atomic_compare_exchange") @export +{ + switch (size) + { + case 1: + char* pt = (char*)ptr; + char ex = *(char*)expected; + char de = *(char*)desired; + if (ex == @__atomic_compare_exchange_ordering_success(pt, ex, de, success, failure, 1)) return 1; + case 2: + short* pt = (short*)ptr; + short ex = *(short*)expected; + short de = *(short*)desired; + if (ex == @__atomic_compare_exchange_ordering_success(pt, ex, de, success, failure, 2)) return 1; + case 4: + int* pt = (int*)ptr; + int ex = *(int*)expected; + int de = *(int*)desired; + if (ex == @__atomic_compare_exchange_ordering_success(pt, ex, de, success, failure, 4)) return 1; + case 8: + $if iptr.sizeof >= 8: + long* pt = (long*)ptr; + long ex = *(long*)expected; + long de = *(long*)desired; + if (ex == @__atomic_compare_exchange_ordering_success(pt, ex, de, success, failure, 8)) return 1; + $else + nextcase; + $endif + default: + assert(false, "Unsuported size (%d) for atomic_compare_exchange", size); + } + return 0; +} \ No newline at end of file diff --git a/src/compiler/llvm_codegen_builtins.c b/src/compiler/llvm_codegen_builtins.c index 27e3c2628..5c1e94889 100644 --- a/src/compiler/llvm_codegen_builtins.c +++ b/src/compiler/llvm_codegen_builtins.c @@ -199,7 +199,7 @@ INLINE void llvm_emit_atomic_fetch(GenContext *c, BuiltinFunction func, BEValue op = is_float ? LLVMAtomicRMWBinOpFAdd : LLVMAtomicRMWBinOpAdd; break; case BUILTIN_ATOMIC_FETCH_SUB: - op = is_float ? LLVMAtomicRMWBinOpFAdd : LLVMAtomicRMWBinOpAdd; + op = is_float ? LLVMAtomicRMWBinOpFSub : LLVMAtomicRMWBinOpSub; break; case BUILTIN_ATOMIC_FETCH_MAX: op = is_float ? LLVMAtomicRMWBinOpFMax : (is_unsigned ? LLVMAtomicRMWBinOpUMax : LLVMAtomicRMWBinOpMax);