keos/lang/slab/
atomic128.rs

1use core::cell::UnsafeCell;
2use core::fmt;
3use core::intrinsics;
4use core::intrinsics::AtomicOrdering;
5use core::sync::atomic::Ordering;
6
7macro_rules! atomic_int {
8    (
9        $atomic_type:ident,
10        $int_type:ident
11    ) => {
12        #[repr(align(16))]
13        pub struct $atomic_type {
14            v: UnsafeCell<$int_type>,
15        }
16
17        impl Default for $atomic_type {
18            #[inline]
19            fn default() -> Self {
20                Self::new(Default::default())
21            }
22        }
23
24        impl From<$int_type> for $atomic_type {
25            fn from(v: $int_type) -> Self {
26                Self::new(v)
27            }
28        }
29
30        impl fmt::Debug for $atomic_type {
31            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
32                fmt::Debug::fmt(&self.load(Ordering::SeqCst), f)
33            }
34        }
35
36        unsafe impl Sync for $atomic_type {}
37
38        impl $atomic_type {
39            #[inline]
40            pub const fn new(v: $int_type) -> Self {
41                Self {
42                    v: UnsafeCell::new(v),
43                }
44            }
45
46            #[inline]
47            pub fn get_mut(&mut self) -> &mut $int_type {
48                self.v.get_mut()
49            }
50
51            #[inline]
52            pub const fn into_inner(self) -> $int_type {
53                self.v.into_inner()
54            }
55
56            #[inline]
57            pub fn load(&self, order: Ordering) -> $int_type {
58                // SAFETY: data races are prevented by atomic intrinsics.
59                unsafe { atomic_load(self.v.get(), order) }
60            }
61
62            #[inline]
63            pub fn fetch_add(&self, v: $int_type, order: Ordering) -> $int_type {
64                // SAFETY: data races are prevented by atomic intrinsics.
65                unsafe { atomic_add(self.v.get(), v, order) }
66            }
67
68            pub fn compare_exchange(
69                &self,
70                current: $int_type,
71                new: $int_type,
72                success: Ordering,
73                failure: Ordering,
74            ) -> Result<$int_type, $int_type> {
75                // SAFETY: data races are prevented by atomic intrinsics.
76                unsafe { atomic_compare_exchange(self.v.get(), current, new, success, failure) }
77            }
78
79            #[inline]
80            pub fn store(&self, val: $int_type, order: Ordering) {
81                // SAFETY: data races are prevented by atomic intrinsics.
82                unsafe {
83                    atomic_store(self.v.get(), val, order);
84                }
85            }
86        }
87    };
88}
89
90atomic_int!(AtomicU128, u128);
91atomic_int!(AtomicI128, i128);
92
93#[inline]
94unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
95    unsafe {
96        // SAFETY: the caller must uphold the safety contract for `atomic_load`.
97        match order {
98            Ordering::Acquire => intrinsics::atomic_load::<T, { AtomicOrdering::Acquire }>(dst),
99            Ordering::Relaxed => intrinsics::atomic_load::<T, { AtomicOrdering::Relaxed }>(dst),
100            Ordering::SeqCst => intrinsics::atomic_load::<T, { AtomicOrdering::SeqCst }>(dst),
101            Ordering::Release => panic!("there is no such thing as a release load"),
102            Ordering::AcqRel => panic!("there is no such thing as an acquire/release load"),
103            _ => core::hint::unreachable_unchecked(),
104        }
105    }
106}
107
108/// Returns the previous value (like __sync_fetch_and_add).
109#[inline]
110unsafe fn atomic_add<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
111    unsafe {
112        // SAFETY: the caller must uphold the safety contract for `atomic_add`.
113        match order {
114            Ordering::Acquire => {
115                intrinsics::atomic_xadd::<T, T, { AtomicOrdering::Acquire }>(dst, val)
116            }
117            Ordering::Release => {
118                intrinsics::atomic_xadd::<T, T, { AtomicOrdering::Release }>(dst, val)
119            }
120            Ordering::AcqRel => {
121                intrinsics::atomic_xadd::<T, T, { AtomicOrdering::AcqRel }>(dst, val)
122            }
123            Ordering::Relaxed => {
124                intrinsics::atomic_xadd::<T, T, { AtomicOrdering::Relaxed }>(dst, val)
125            }
126            Ordering::SeqCst => {
127                intrinsics::atomic_xadd::<T, T, { AtomicOrdering::SeqCst }>(dst, val)
128            }
129            _ => core::hint::unreachable_unchecked(),
130        }
131    }
132}
133
134#[inline]
135unsafe fn atomic_compare_exchange<T: Copy>(
136    dst: *mut T,
137    old: T,
138    new: T,
139    success: Ordering,
140    failure: Ordering,
141) -> Result<T, T> {
142    // SAFETY: the caller must uphold the safety contract for
143    // `atomic_compare_exchange`.
144    let (val, ok) = unsafe {
145        match (success, failure) {
146            (Ordering::Relaxed, Ordering::Relaxed) => intrinsics::atomic_cxchg::<
147                T,
148                { AtomicOrdering::Relaxed },
149                { AtomicOrdering::Relaxed },
150            >(dst, old, new),
151            (Ordering::Relaxed, Ordering::Acquire) => intrinsics::atomic_cxchg::<
152                T,
153                { AtomicOrdering::Relaxed },
154                { AtomicOrdering::Acquire },
155            >(dst, old, new),
156            (Ordering::Relaxed, Ordering::SeqCst) => {
157                intrinsics::atomic_cxchg::<T, { AtomicOrdering::Relaxed }, { AtomicOrdering::SeqCst }>(
158                    dst, old, new,
159                )
160            }
161            (Ordering::Acquire, Ordering::Relaxed) => intrinsics::atomic_cxchg::<
162                T,
163                { AtomicOrdering::Acquire },
164                { AtomicOrdering::Relaxed },
165            >(dst, old, new),
166            (Ordering::Acquire, Ordering::Acquire) => intrinsics::atomic_cxchg::<
167                T,
168                { AtomicOrdering::Acquire },
169                { AtomicOrdering::Acquire },
170            >(dst, old, new),
171            (Ordering::Acquire, Ordering::SeqCst) => {
172                intrinsics::atomic_cxchg::<T, { AtomicOrdering::Acquire }, { AtomicOrdering::SeqCst }>(
173                    dst, old, new,
174                )
175            }
176            (Ordering::Release, Ordering::Relaxed) => intrinsics::atomic_cxchg::<
177                T,
178                { AtomicOrdering::Release },
179                { AtomicOrdering::Relaxed },
180            >(dst, old, new),
181            (Ordering::Release, Ordering::Acquire) => intrinsics::atomic_cxchg::<
182                T,
183                { AtomicOrdering::Release },
184                { AtomicOrdering::Acquire },
185            >(dst, old, new),
186            (Ordering::Release, Ordering::SeqCst) => {
187                intrinsics::atomic_cxchg::<T, { AtomicOrdering::Release }, { AtomicOrdering::SeqCst }>(
188                    dst, old, new,
189                )
190            }
191            (Ordering::AcqRel, Ordering::Relaxed) => {
192                intrinsics::atomic_cxchg::<T, { AtomicOrdering::AcqRel }, { AtomicOrdering::Relaxed }>(
193                    dst, old, new,
194                )
195            }
196            (Ordering::AcqRel, Ordering::Acquire) => {
197                intrinsics::atomic_cxchg::<T, { AtomicOrdering::AcqRel }, { AtomicOrdering::Acquire }>(
198                    dst, old, new,
199                )
200            }
201            (Ordering::AcqRel, Ordering::SeqCst) => {
202                intrinsics::atomic_cxchg::<T, { AtomicOrdering::AcqRel }, { AtomicOrdering::SeqCst }>(
203                    dst, old, new,
204                )
205            }
206            (Ordering::SeqCst, Ordering::Relaxed) => {
207                intrinsics::atomic_cxchg::<T, { AtomicOrdering::SeqCst }, { AtomicOrdering::Relaxed }>(
208                    dst, old, new,
209                )
210            }
211            (Ordering::SeqCst, Ordering::Acquire) => {
212                intrinsics::atomic_cxchg::<T, { AtomicOrdering::SeqCst }, { AtomicOrdering::Acquire }>(
213                    dst, old, new,
214                )
215            }
216            (Ordering::SeqCst, Ordering::SeqCst) => {
217                intrinsics::atomic_cxchg::<T, { AtomicOrdering::SeqCst }, { AtomicOrdering::SeqCst }>(
218                    dst, old, new,
219                )
220            }
221            (_, Ordering::AcqRel) => {
222                panic!("there is no such thing as an acquire-release failure ordering")
223            }
224            (_, Ordering::Release) => {
225                panic!("there is no such thing as a release failure ordering")
226            }
227            (_, _) => unreachable!(),
228        }
229    };
230    if ok { Ok(val) } else { Err(val) }
231}
232
233#[inline]
234unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
235    unsafe {
236        // SAFETY: the caller must uphold the safety contract for `atomic_store`.
237        match order {
238            Ordering::Release => {
239                intrinsics::atomic_store::<T, { AtomicOrdering::Release }>(dst, val)
240            }
241            Ordering::Relaxed => {
242                intrinsics::atomic_store::<T, { AtomicOrdering::Relaxed }>(dst, val)
243            }
244            Ordering::SeqCst => intrinsics::atomic_store::<T, { AtomicOrdering::SeqCst }>(dst, val),
245            Ordering::Acquire => panic!("there is no such thing as an acquire store"),
246            Ordering::AcqRel => panic!("there is no such thing as an acquire/release store"),
247            _ => core::hint::unreachable_unchecked(),
248        }
249    }
250}
keos/lang/slab/atomic128.rs

keos/lang/slab/
atomic128.rs
