portable_atomic/
utils.rs

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3#![cfg_attr(not(all(test, feature = "float")), allow(dead_code, unused_macros))]
4
5#[macro_use]
6#[path = "gen/utils.rs"]
7mod generated;
8
9use core::sync::atomic::Ordering;
10
11macro_rules! static_assert {
12    ($cond:expr $(,)?) => {{
13        let [] = [(); true as usize - $crate::utils::_assert_is_bool($cond) as usize];
14    }};
15}
16pub(crate) const fn _assert_is_bool(v: bool) -> bool {
17    v
18}
19
20macro_rules! static_assert_layout {
21    ($atomic_type:ty, $value_type:ty) => {
22        static_assert!(
23            core::mem::align_of::<$atomic_type>() == core::mem::size_of::<$atomic_type>()
24        );
25        static_assert!(core::mem::size_of::<$atomic_type>() == core::mem::size_of::<$value_type>());
26    };
27}
28
29// #[doc = concat!(...)] requires Rust 1.54
30macro_rules! doc_comment {
31    ($doc:expr, $($tt:tt)*) => {
32        #[doc = $doc]
33        $($tt)*
34    };
35}
36
37// Adapted from https://github.com/BurntSushi/memchr/blob/2.4.1/src/memchr/x86/mod.rs#L9-L71.
38/// # Safety
39///
40/// - the caller must uphold the safety contract for the function returned by $detect_body.
41/// - the memory pointed by the function pointer returned by $detect_body must be visible from any threads.
42///
43/// The second requirement is always met if the function pointer is to the function definition.
44/// (Currently, all uses of this macro in our code are in this case.)
45#[allow(unused_macros)]
46#[cfg(not(portable_atomic_no_outline_atomics))]
47#[cfg(any(
48    target_arch = "aarch64",
49    target_arch = "arm",
50    target_arch = "arm64ec",
51    target_arch = "powerpc64",
52    target_arch = "riscv32",
53    target_arch = "riscv64",
54    all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
55))]
56macro_rules! ifunc {
57    (unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)? { $($detect_body:tt)* }) => {{
58        type FnTy = unsafe fn($($arg_ty),*) $(-> $ret_ty)?;
59        static FUNC: core::sync::atomic::AtomicPtr<()>
60            = core::sync::atomic::AtomicPtr::new(detect as *mut ());
61        #[cold]
62        unsafe fn detect($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
63            let func: FnTy = { $($detect_body)* };
64            FUNC.store(func as *mut (), core::sync::atomic::Ordering::Relaxed);
65            // SAFETY: the caller must uphold the safety contract for the function returned by $detect_body.
66            unsafe { func($($arg_pat),*) }
67        }
68        // SAFETY: `FnTy` is a function pointer, which is always safe to transmute with a `*mut ()`.
69        // (To force the caller to use unsafe block for this macro, do not use
70        // unsafe block here.)
71        let func = {
72            core::mem::transmute::<*mut (), FnTy>(FUNC.load(core::sync::atomic::Ordering::Relaxed))
73        };
74        // SAFETY: the caller must uphold the safety contract for the function returned by $detect_body.
75        // (To force the caller to use unsafe block for this macro, do not use
76        // unsafe block here.)
77        func($($arg_pat),*)
78    }};
79}
80
81#[allow(unused_macros)]
82#[cfg(not(portable_atomic_no_outline_atomics))]
83#[cfg(any(
84    target_arch = "aarch64",
85    target_arch = "arm",
86    target_arch = "arm64ec",
87    target_arch = "powerpc64",
88    target_arch = "riscv32",
89    target_arch = "riscv64",
90    all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
91))]
92macro_rules! fn_alias {
93    (
94        $(#[$($fn_attr:tt)*])*
95        $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
96        $(#[$($alias_attr:tt)*])*
97        $new:ident = $from:ident($($last_args:tt)*);
98        $($rest:tt)*
99    ) => {
100        $(#[$($fn_attr)*])*
101        $(#[$($alias_attr)*])*
102        $vis unsafe fn $new($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
103            // SAFETY: the caller must uphold the safety contract.
104            unsafe { $from($($arg_pat,)* $($last_args)*) }
105        }
106        fn_alias! {
107            $(#[$($fn_attr)*])*
108            $vis unsafe fn($($arg_pat: $arg_ty),*) $(-> $ret_ty)?;
109            $($rest)*
110        }
111    };
112    (
113        $(#[$($attr:tt)*])*
114        $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
115    ) => {}
116}
117
118/// Make the given function const if the given condition is true.
119macro_rules! const_fn {
120    (
121        const_if: #[cfg($($cfg:tt)+)];
122        $(#[$($attr:tt)*])*
123        $vis:vis const $($rest:tt)*
124    ) => {
125        #[cfg($($cfg)+)]
126        $(#[$($attr)*])*
127        $vis const $($rest)*
128        #[cfg(not($($cfg)+))]
129        $(#[$($attr)*])*
130        $vis $($rest)*
131    };
132}
133
134/// Implements `core::fmt::Debug` and `serde::{Serialize, Deserialize}` (when serde
135/// feature is enabled) for atomic bool, integer, or float.
136macro_rules! impl_debug_and_serde {
137    // TODO(f16_and_f128): Implement serde traits for f16 & f128 once stabilized.
138    (AtomicF16) => {
139        impl_debug!(AtomicF16);
140    };
141    (AtomicF128) => {
142        impl_debug!(AtomicF128);
143    };
144    ($atomic_type:ident) => {
145        impl_debug!($atomic_type);
146        #[cfg(feature = "serde")]
147        #[cfg_attr(docsrs, doc(cfg(feature = "serde")))]
148        impl serde::ser::Serialize for $atomic_type {
149            #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl
150            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
151            where
152                S: serde::ser::Serializer,
153            {
154                // https://github.com/serde-rs/serde/blob/v1.0.152/serde/src/ser/impls.rs#L958-L959
155                self.load(Ordering::Relaxed).serialize(serializer)
156            }
157        }
158        #[cfg(feature = "serde")]
159        #[cfg_attr(docsrs, doc(cfg(feature = "serde")))]
160        impl<'de> serde::de::Deserialize<'de> for $atomic_type {
161            #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl
162            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
163            where
164                D: serde::de::Deserializer<'de>,
165            {
166                serde::de::Deserialize::deserialize(deserializer).map(Self::new)
167            }
168        }
169    };
170}
171macro_rules! impl_debug {
172    ($atomic_type:ident) => {
173        impl fmt::Debug for $atomic_type {
174            #[inline] // fmt is not hot path, but #[inline] on fmt seems to still be useful: https://github.com/rust-lang/rust/pull/117727
175            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
176                // std atomic types use Relaxed in Debug::fmt: https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L2188
177                fmt::Debug::fmt(&self.load(Ordering::Relaxed), f)
178            }
179        }
180    };
181}
182
183// We do not provide `nand` because it cannot be optimized on neither x86 nor MSP430.
184// https://godbolt.org/z/ahWejchbT
185macro_rules! impl_default_no_fetch_ops {
186    ($atomic_type:ident, bool) => {
187        impl $atomic_type {
188            #[inline]
189            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
190            pub(crate) fn and(&self, val: bool, order: Ordering) {
191                self.fetch_and(val, order);
192            }
193            #[inline]
194            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
195            pub(crate) fn or(&self, val: bool, order: Ordering) {
196                self.fetch_or(val, order);
197            }
198            #[inline]
199            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
200            pub(crate) fn xor(&self, val: bool, order: Ordering) {
201                self.fetch_xor(val, order);
202            }
203        }
204    };
205    ($atomic_type:ident, $int_type:ty) => {
206        impl $atomic_type {
207            #[inline]
208            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
209            pub(crate) fn add(&self, val: $int_type, order: Ordering) {
210                self.fetch_add(val, order);
211            }
212            #[inline]
213            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
214            pub(crate) fn sub(&self, val: $int_type, order: Ordering) {
215                self.fetch_sub(val, order);
216            }
217            #[inline]
218            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
219            pub(crate) fn and(&self, val: $int_type, order: Ordering) {
220                self.fetch_and(val, order);
221            }
222            #[inline]
223            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
224            pub(crate) fn or(&self, val: $int_type, order: Ordering) {
225                self.fetch_or(val, order);
226            }
227            #[inline]
228            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
229            pub(crate) fn xor(&self, val: $int_type, order: Ordering) {
230                self.fetch_xor(val, order);
231            }
232        }
233    };
234}
235macro_rules! impl_default_bit_opts {
236    ($atomic_type:ident, $int_type:ty) => {
237        impl $atomic_type {
238            #[inline]
239            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
240            pub(crate) fn bit_set(&self, bit: u32, order: Ordering) -> bool {
241                let mask = <$int_type>::wrapping_shl(1, bit);
242                self.fetch_or(mask, order) & mask != 0
243            }
244            #[inline]
245            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
246            pub(crate) fn bit_clear(&self, bit: u32, order: Ordering) -> bool {
247                let mask = <$int_type>::wrapping_shl(1, bit);
248                self.fetch_and(!mask, order) & mask != 0
249            }
250            #[inline]
251            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
252            pub(crate) fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {
253                let mask = <$int_type>::wrapping_shl(1, bit);
254                self.fetch_xor(mask, order) & mask != 0
255            }
256        }
257    };
258}
259
260// This just outputs the input as is, but can be used like an item-level block by using it with cfg.
261macro_rules! items {
262    ($($tt:tt)*) => {
263        $($tt)*
264    };
265}
266
267#[allow(dead_code)]
268#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
269// Stable version of https://doc.rust-lang.org/nightly/std/hint/fn.assert_unchecked.html.
270// TODO: use real core::hint::assert_unchecked on 1.81+ https://github.com/rust-lang/rust/pull/123588
271#[inline(always)]
272#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
273pub(crate) unsafe fn assert_unchecked(cond: bool) {
274    if !cond {
275        if cfg!(debug_assertions) {
276            unreachable!()
277        } else {
278            // SAFETY: the caller promised `cond` is true.
279            unsafe { core::hint::unreachable_unchecked() }
280        }
281    }
282}
283
284// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3338
285#[inline]
286#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
287pub(crate) fn assert_load_ordering(order: Ordering) {
288    match order {
289        Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
290        Ordering::Release => panic!("there is no such thing as a release load"),
291        Ordering::AcqRel => panic!("there is no such thing as an acquire-release load"),
292        _ => unreachable!(),
293    }
294}
295// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3323
296#[inline]
297#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
298pub(crate) fn assert_store_ordering(order: Ordering) {
299    match order {
300        Ordering::Release | Ordering::Relaxed | Ordering::SeqCst => {}
301        Ordering::Acquire => panic!("there is no such thing as an acquire store"),
302        Ordering::AcqRel => panic!("there is no such thing as an acquire-release store"),
303        _ => unreachable!(),
304    }
305}
306// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3404
307#[inline]
308#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
309pub(crate) fn assert_compare_exchange_ordering(success: Ordering, failure: Ordering) {
310    match success {
311        Ordering::AcqRel
312        | Ordering::Acquire
313        | Ordering::Relaxed
314        | Ordering::Release
315        | Ordering::SeqCst => {}
316        _ => unreachable!(),
317    }
318    match failure {
319        Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
320        Ordering::Release => panic!("there is no such thing as a release failure ordering"),
321        Ordering::AcqRel => panic!("there is no such thing as an acquire-release failure ordering"),
322        _ => unreachable!(),
323    }
324}
325
326// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0418r2.html
327// https://github.com/rust-lang/rust/pull/98383
328#[allow(dead_code)]
329#[inline]
330pub(crate) fn upgrade_success_ordering(success: Ordering, failure: Ordering) -> Ordering {
331    match (success, failure) {
332        (Ordering::Relaxed, Ordering::Acquire) => Ordering::Acquire,
333        (Ordering::Release, Ordering::Acquire) => Ordering::AcqRel,
334        (_, Ordering::SeqCst) => Ordering::SeqCst,
335        _ => success,
336    }
337}
338
339/// Zero-extends the given 32-bit pointer to `MaybeUninit<u64>`.
340/// This is used for 64-bit architecture's 32-bit ABI (e.g., AArch64 ILP32 ABI).
341/// See ptr_reg! macro in src/gen/utils.rs for details.
342#[cfg(not(portable_atomic_no_asm_maybe_uninit))]
343#[cfg(target_pointer_width = "32")]
344#[allow(dead_code)]
345#[inline]
346pub(crate) fn zero_extend64_ptr(v: *mut ()) -> core::mem::MaybeUninit<u64> {
347    #[repr(C)]
348    struct ZeroExtended {
349        #[cfg(target_endian = "big")]
350        pad: *mut (),
351        v: *mut (),
352        #[cfg(target_endian = "little")]
353        pad: *mut (),
354    }
355    // SAFETY: we can safely transmute any 64-bit value to MaybeUninit<u64>.
356    unsafe { core::mem::transmute(ZeroExtended { v, pad: core::ptr::null_mut() }) }
357}
358
359#[allow(dead_code)]
360#[cfg(any(
361    target_arch = "aarch64",
362    target_arch = "arm64ec",
363    target_arch = "powerpc64",
364    target_arch = "riscv64",
365    target_arch = "s390x",
366    target_arch = "x86_64",
367))]
368/// A 128-bit value represented as a pair of 64-bit values.
369///
370/// This type is `#[repr(C)]`, both fields have the same in-memory representation
371/// and are plain old data types, so access to the fields is always safe.
372#[derive(Clone, Copy)]
373#[repr(C)]
374pub(crate) union U128 {
375    pub(crate) whole: u128,
376    pub(crate) pair: Pair<u64>,
377}
378#[allow(dead_code)]
379#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
380/// A 64-bit value represented as a pair of 32-bit values.
381///
382/// This type is `#[repr(C)]`, both fields have the same in-memory representation
383/// and are plain old data types, so access to the fields is always safe.
384#[derive(Clone, Copy)]
385#[repr(C)]
386pub(crate) union U64 {
387    pub(crate) whole: u64,
388    pub(crate) pair: Pair<u32>,
389}
390#[allow(dead_code)]
391#[derive(Clone, Copy)]
392#[repr(C)]
393pub(crate) struct Pair<T: Copy> {
394    // little endian order
395    #[cfg(any(
396        target_endian = "little",
397        target_arch = "aarch64",
398        target_arch = "arm",
399        target_arch = "arm64ec",
400    ))]
401    pub(crate) lo: T,
402    pub(crate) hi: T,
403    // big endian order
404    #[cfg(not(any(
405        target_endian = "little",
406        target_arch = "aarch64",
407        target_arch = "arm",
408        target_arch = "arm64ec",
409    )))]
410    pub(crate) lo: T,
411}
412
413#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
414type MinWord = u32;
415#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
416type RetInt = u32;
417// Adapted from https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.6/src/utils.rs#L255.
418// Helper for implementing sub-word atomic operations using word-sized LL/SC loop or CAS loop.
419//
420// Refs: https://github.com/llvm/llvm-project/blob/llvmorg-20.1.0/llvm/lib/CodeGen/AtomicExpandPass.cpp#L799
421// (aligned_ptr, shift, mask)
422#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
423#[allow(dead_code)]
424#[inline]
425pub(crate) fn create_sub_word_mask_values<T>(ptr: *mut T) -> (*mut MinWord, RetInt, RetInt) {
426    #[cfg(portable_atomic_no_strict_provenance)]
427    use self::ptr::PtrExt as _;
428    use core::mem;
429    // RISC-V, MIPS, SPARC, LoongArch, Xtensa, BPF: shift amount of 32-bit shift instructions is 5 bits unsigned (0-31).
430    // PowerPC, C-SKY: shift amount of 32-bit shift instructions is 6 bits unsigned (0-63) and shift amount 32-63 means "clear".
431    // Arm: shift amount of 32-bit shift instructions is 8 bits unsigned (0-255).
432    // Hexagon: shift amount of 32-bit shift instructions is 7 bits signed (-64-63) and negative shift amount means "reverse the direction of the shift".
433    // (On s390x, we don't use the mask returned from this function.)
434    // (See also https://devblogs.microsoft.com/oldnewthing/20230904-00/?p=108704 for others)
435    const SHIFT_MASK: bool = !cfg!(any(
436        target_arch = "bpf",
437        target_arch = "loongarch32",
438        target_arch = "loongarch64",
439        target_arch = "mips",
440        target_arch = "mips32r6",
441        target_arch = "mips64",
442        target_arch = "mips64r6",
443        target_arch = "riscv32",
444        target_arch = "riscv64",
445        target_arch = "s390x",
446        target_arch = "sparc",
447        target_arch = "sparc64",
448        target_arch = "xtensa",
449    ));
450    let ptr_mask = mem::size_of::<MinWord>() - 1;
451    let aligned_ptr = ptr.with_addr(ptr.addr() & !ptr_mask) as *mut MinWord;
452    let ptr_lsb = if SHIFT_MASK {
453        ptr.addr() & ptr_mask
454    } else {
455        // We use 32-bit wrapping shift instructions in asm on these platforms.
456        ptr.addr()
457    };
458    let shift = if cfg!(any(target_endian = "little", target_arch = "s390x")) {
459        ptr_lsb.wrapping_mul(8)
460    } else {
461        (ptr_lsb ^ (mem::size_of::<MinWord>() - mem::size_of::<T>())).wrapping_mul(8)
462    };
463    let mut mask: RetInt = (1 << (mem::size_of::<T>() * 8)) - 1; // !(0 as T) as RetInt
464    if SHIFT_MASK {
465        mask <<= shift;
466    }
467    #[allow(clippy::cast_possible_truncation)]
468    {
469        (aligned_ptr, shift as RetInt, mask)
470    }
471}
472
473// This module provides core::ptr strict_provenance/exposed_provenance polyfill for pre-1.84 rustc.
474#[allow(dead_code)]
475pub(crate) mod ptr {
476    #[cfg(portable_atomic_no_strict_provenance)]
477    use core::mem;
478    #[cfg(not(portable_atomic_no_strict_provenance))]
479    #[allow(unused_imports)]
480    pub(crate) use core::ptr::{with_exposed_provenance, with_exposed_provenance_mut};
481
482    #[cfg(portable_atomic_no_strict_provenance)]
483    #[inline(always)]
484    #[must_use]
485    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
486    pub(crate) fn with_exposed_provenance<T>(addr: usize) -> *const T {
487        addr as *const T
488    }
489    #[cfg(portable_atomic_no_strict_provenance)]
490    #[inline(always)]
491    #[must_use]
492    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
493    pub(crate) fn with_exposed_provenance_mut<T>(addr: usize) -> *mut T {
494        addr as *mut T
495    }
496
497    #[cfg(portable_atomic_no_strict_provenance)]
498    pub(crate) trait PtrExt<T: ?Sized>: Copy {
499        #[must_use]
500        fn addr(self) -> usize;
501        #[must_use]
502        fn with_addr(self, addr: usize) -> Self
503        where
504            T: Sized;
505    }
506    #[cfg(portable_atomic_no_strict_provenance)]
507    impl<T: ?Sized> PtrExt<T> for *mut T {
508        #[inline(always)]
509        #[must_use]
510        fn addr(self) -> usize {
511            // A pointer-to-integer transmute currently has exactly the right semantics: it returns the
512            // address without exposing the provenance. Note that this is *not* a stable guarantee about
513            // transmute semantics, it relies on sysroot crates having special status.
514            // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the
515            // provenance).
516            #[allow(clippy::transmutes_expressible_as_ptr_casts)]
517            unsafe {
518                mem::transmute(self as *mut ())
519            }
520        }
521        #[allow(clippy::cast_possible_wrap)]
522        #[inline]
523        #[must_use]
524        fn with_addr(self, addr: usize) -> Self
525        where
526            T: Sized,
527        {
528            // This should probably be an intrinsic to avoid doing any sort of arithmetic, but
529            // meanwhile, we can implement it with `wrapping_offset`, which preserves the pointer's
530            // provenance.
531            let self_addr = self.addr() as isize;
532            let dest_addr = addr as isize;
533            let offset = dest_addr.wrapping_sub(self_addr);
534            (self as *mut u8).wrapping_offset(offset) as *mut T
535        }
536    }
537}
538
539// This module provides:
540// - core::ffi polyfill (c_* type aliases and CStr) for pre-1.64 rustc compatibility.
541//   (core::ffi::* (except c_void) requires Rust 1.64)
542// - safe abstraction (c! macro) for creating static C strings without runtime checks.
543//   (c"..." requires Rust 1.77)
544// - helper macros for defining FFI bindings.
545#[cfg(any(
546    test,
547    portable_atomic_test_no_std_static_assert_ffi,
548    not(any(target_arch = "x86", target_arch = "x86_64"))
549))]
550#[cfg(any(not(portable_atomic_no_asm), portable_atomic_unstable_asm))]
551#[allow(dead_code, non_camel_case_types, unused_macros)]
552#[macro_use]
553pub(crate) mod ffi {
554    pub(crate) type c_void = core::ffi::c_void;
555    // c_{,u}int is {i,u}16 on 16-bit targets, otherwise {i,u}32.
556    // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L156
557    #[cfg(target_pointer_width = "16")]
558    pub(crate) type c_int = i16;
559    #[cfg(target_pointer_width = "16")]
560    pub(crate) type c_uint = u16;
561    #[cfg(not(target_pointer_width = "16"))]
562    pub(crate) type c_int = i32;
563    #[cfg(not(target_pointer_width = "16"))]
564    pub(crate) type c_uint = u32;
565    // c_{,u}long is {i,u}64 on non-Windows 64-bit targets, otherwise {i,u}32.
566    // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L168
567    #[cfg(all(target_pointer_width = "64", not(windows)))]
568    pub(crate) type c_long = i64;
569    #[cfg(all(target_pointer_width = "64", not(windows)))]
570    pub(crate) type c_ulong = u64;
571    #[cfg(not(all(target_pointer_width = "64", not(windows))))]
572    pub(crate) type c_long = i32;
573    #[cfg(not(all(target_pointer_width = "64", not(windows))))]
574    pub(crate) type c_ulong = u32;
575    // c_size_t is currently always usize.
576    // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L76
577    pub(crate) type c_size_t = usize;
578    // c_char is u8 by default on non-Apple/non-Windows/non-Vita Arm/C-SKY/Hexagon/MSP430/PowerPC/RISC-V/s390x/Xtensa targets, otherwise i8 by default.
579    // See references in https://github.com/rust-lang/rust/issues/129945 for details.
580    #[cfg(all(
581        not(any(target_vendor = "apple", windows, target_os = "vita")),
582        any(
583            target_arch = "aarch64",
584            target_arch = "arm",
585            target_arch = "csky",
586            target_arch = "hexagon",
587            target_arch = "msp430",
588            target_arch = "powerpc",
589            target_arch = "powerpc64",
590            target_arch = "riscv32",
591            target_arch = "riscv64",
592            target_arch = "s390x",
593            target_arch = "xtensa",
594        ),
595    ))]
596    pub(crate) type c_char = u8;
597    #[cfg(not(all(
598        not(any(target_vendor = "apple", windows, target_os = "vita")),
599        any(
600            target_arch = "aarch64",
601            target_arch = "arm",
602            target_arch = "csky",
603            target_arch = "hexagon",
604            target_arch = "msp430",
605            target_arch = "powerpc",
606            target_arch = "powerpc64",
607            target_arch = "riscv32",
608            target_arch = "riscv64",
609            target_arch = "s390x",
610            target_arch = "xtensa",
611        ),
612    )))]
613    pub(crate) type c_char = i8;
614
615    // Static assertions for C type definitions.
616    #[cfg(test)]
617    const _: fn() = || {
618        let _: c_int = 0 as std::os::raw::c_int;
619        let _: c_uint = 0 as std::os::raw::c_uint;
620        let _: c_long = 0 as std::os::raw::c_long;
621        let _: c_ulong = 0 as std::os::raw::c_ulong;
622        #[cfg(unix)]
623        let _: c_size_t = 0 as libc::size_t; // std::os::raw::c_size_t is unstable
624        let _: c_char = 0 as std::os::raw::c_char;
625    };
626
627    #[repr(transparent)]
628    pub(crate) struct CStr([c_char]);
629    impl CStr {
630        #[inline]
631        #[must_use]
632        pub(crate) const fn as_ptr(&self) -> *const c_char {
633            self.0.as_ptr()
634        }
635        /// # Safety
636        ///
637        /// The provided slice **must** be nul-terminated and not contain any interior
638        /// nul bytes.
639        #[inline]
640        #[must_use]
641        pub(crate) unsafe fn from_bytes_with_nul_unchecked(bytes: &[u8]) -> &CStr {
642            // SAFETY: Casting to CStr is safe because *our* CStr is #[repr(transparent)]
643            // and its internal representation is a [u8] too. (Note that std's CStr
644            // is not #[repr(transparent)].)
645            // Dereferencing the obtained pointer is safe because it comes from a
646            // reference. Making a reference is then safe because its lifetime
647            // is bound by the lifetime of the given `bytes`.
648            unsafe { &*(bytes as *const [u8] as *const CStr) }
649        }
650        #[cfg(test)]
651        #[inline]
652        #[must_use]
653        pub(crate) fn to_bytes_with_nul(&self) -> &[u8] {
654            // SAFETY: Transmuting a slice of `c_char`s to a slice of `u8`s
655            // is safe on all supported targets.
656            #[allow(clippy::unnecessary_cast)] // triggered for targets that c_char is u8
657            unsafe {
658                &*(&self.0 as *const [c_char] as *const [u8])
659            }
660        }
661    }
662
663    macro_rules! c {
664        ($s:expr) => {{
665            const BYTES: &[u8] = concat!($s, "\0").as_bytes();
666            const _: () = static_assert!(crate::utils::ffi::_const_is_c_str(BYTES));
667            #[allow(unused_unsafe)]
668            // SAFETY: we've checked `BYTES` is a valid C string
669            unsafe {
670                crate::utils::ffi::CStr::from_bytes_with_nul_unchecked(BYTES)
671            }
672        }};
673    }
674
675    #[must_use]
676    pub(crate) const fn _const_is_c_str(bytes: &[u8]) -> bool {
677        #[cfg(portable_atomic_no_track_caller)]
678        {
679            // const_if_match/const_loop was stabilized (nightly-2020-06-30) 2 days before
680            // track_caller was stabilized (nightly-2020-07-02), so we reuse the cfg for
681            // track_caller here instead of emitting a cfg for const_if_match/const_loop.
682            // https://github.com/rust-lang/rust/pull/72437
683            // track_caller was stabilized 11 days after the oldest nightly version
684            // that uses this module, and is included in the same 1.46 stable release.
685            // The check here is insufficient in this case, but this is fine because this function
686            // is internal code that is not used to process input from the user and our CI checks
687            // all builtin targets and some custom targets with some versions of newer compilers.
688            !bytes.is_empty()
689        }
690        #[cfg(not(portable_atomic_no_track_caller))]
691        {
692            // Based on https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/c_str.rs#L417
693            // - bytes must be nul-terminated.
694            // - bytes must not contain any interior nul bytes.
695            if bytes.is_empty() {
696                return false;
697            }
698            let mut i = bytes.len() - 1;
699            if bytes[i] != 0 {
700                return false;
701            }
702            // Ending null byte exists, skip to the rest.
703            while i != 0 {
704                i -= 1;
705                if bytes[i] == 0 {
706                    return false;
707                }
708            }
709            true
710        }
711    }
712
713    /// Defines types with #[cfg(test)] static assertions which checks
714    /// types are the same as the platform's latest header files' ones.
715    // Note: This macro is sys_ty!({ }), not sys_ty! { }.
716    // An extra brace is used in input to make contents rustfmt-able.
717    macro_rules! sys_type {
718        ({$(
719            $(#[$attr:meta])*
720            $vis:vis type $([$($windows_path:ident)::+])? $name:ident = $ty:ty;
721        )*}) => {
722            $(
723                $(#[$attr])*
724                $vis type $name = $ty;
725            )*
726            // Static assertions for FFI bindings.
727            // This checks that FFI bindings defined in this crate and FFI bindings generated for
728            // the platform's latest header file using bindgen have the same types.
729            // Since this is static assertion, we can detect problems with
730            // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh)
731            // without actually running tests on these platforms.
732            // See also https://github.com/taiki-e/test-helper/blob/HEAD/tools/codegen/src/ffi.rs.
733            #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
734            #[allow(
735                unused_imports,
736                clippy::cast_possible_wrap,
737                clippy::cast_sign_loss,
738                clippy::cast_possible_truncation
739            )]
740            const _: fn() = || {
741                #[cfg(not(any(target_os = "aix", windows)))]
742                use test_helper::sys;
743                #[cfg(target_os = "aix")]
744                use libc as sys;
745                $(
746                    $(#[$attr])*
747                    {
748                        $(use windows_sys::$($windows_path)::+ as sys;)?
749                        let _: $name = 0 as sys::$name;
750                    }
751                )*
752            };
753        };
754    }
755    /// Defines #[repr(C)] structs with #[cfg(test)] static assertions which checks
756    /// fields are the same as the platform's latest header files' ones.
757    // Note: This macro is sys_struct!({ }), not sys_struct! { }.
758    // An extra brace is used in input to make contents rustfmt-able.
759    macro_rules! sys_struct {
760        ({$(
761            $(#[$attr:meta])*
762            $vis:vis struct $([$($windows_path:ident)::+])? $name:ident {$(
763                $(#[$field_attr:meta])*
764                $field_vis:vis $field_name:ident: $field_ty:ty,
765            )*}
766        )*}) => {
767            $(
768                $(#[$attr])*
769                #[derive(Clone, Copy)]
770                #[cfg_attr(
771                    any(test, portable_atomic_test_no_std_static_assert_ffi),
772                    derive(Debug, PartialEq)
773                )]
774                #[repr(C)]
775                $vis struct $name {$(
776                    $(#[$field_attr])*
777                    $field_vis $field_name: $field_ty,
778                )*}
779            )*
780            // Static assertions for FFI bindings.
781            // This checks that FFI bindings defined in this crate and FFI bindings generated for
782            // the platform's latest header file using bindgen have the same fields.
783            // Since this is static assertion, we can detect problems with
784            // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh)
785            // without actually running tests on these platforms.
786            // See also https://github.com/taiki-e/test-helper/blob/HEAD/tools/codegen/src/ffi.rs.
787            #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
788            #[allow(unused_imports, clippy::undocumented_unsafe_blocks)]
789            const _: fn() = || {
790                #[cfg(not(any(target_os = "aix", windows)))]
791                use test_helper::sys;
792                #[cfg(target_os = "aix")]
793                use libc as sys;
794                $(
795                    $(#[$attr])*
796                    {
797                        $(use windows_sys::$($windows_path)::+ as sys;)?
798                        static_assert!(
799                            core::mem::size_of::<$name>()
800                                == core::mem::size_of::<sys::$name>()
801                        );
802                        let s: $name = unsafe { core::mem::zeroed() };
803                        // field names and types
804                        let _ = sys::$name {$(
805                            $(#[$field_attr])*
806                            $field_name: s.$field_name,
807                        )*};
808                        // field offsets
809                        #[cfg(not(portable_atomic_no_offset_of))]
810                        {$(
811                            $(#[$field_attr])*
812                            static_assert!(
813                                core::mem::offset_of!($name, $field_name) ==
814                                    core::mem::offset_of!(sys::$name, $field_name),
815                            );
816                        )*}
817                    }
818                )*
819            };
820        };
821    }
822    /// Defines constants with #[cfg(test)] static assertions which checks
823    /// values are the same as the platform's latest header files' ones.
824    // Note: This macro is sys_const!({ }), not sys_const! { }.
825    // An extra brace is used in input to make contents rustfmt-able.
826    macro_rules! sys_const {
827        ({$(
828            $(#[$attr:meta])*
829            $vis:vis const $([$($windows_path:ident)::+])? $name:ident: $ty:ty = $val:expr;
830        )*}) => {
831            $(
832                $(#[$attr])*
833                $vis const $name: $ty = $val;
834            )*
835            // Static assertions for FFI bindings.
836            // This checks that FFI bindings defined in this crate and FFI bindings generated for
837            // the platform's latest header file using bindgen have the same values.
838            // Since this is static assertion, we can detect problems with
839            // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh)
840            // without actually running tests on these platforms.
841            // See also https://github.com/taiki-e/test-helper/blob/HEAD/tools/codegen/src/ffi.rs.
842            #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
843            #[allow(
844                unused_attributes, // for #[allow(..)] in $(#[$attr])*
845                unused_imports,
846                clippy::cast_possible_wrap,
847                clippy::cast_sign_loss,
848                clippy::cast_possible_truncation,
849            )]
850            const _: fn() = || {
851                #[cfg(not(any(target_os = "aix", windows)))]
852                use test_helper::sys;
853                #[cfg(target_os = "aix")]
854                use libc as sys;
855                $(
856                    $(#[$attr])*
857                    {
858                        $(use windows_sys::$($windows_path)::+ as sys;)?
859                        sys_const_cmp!($name, $ty);
860                    }
861                )*
862            };
863        };
864    }
865    #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
866    macro_rules! sys_const_cmp {
867        (RTLD_DEFAULT, $ty:ty) => {
868            // ptr comparison and ptr-to-int cast are not stable on const context, so use ptr-to-int
869            // transmute and compare its result.
870            static_assert!(
871                // SAFETY: Pointer-to-integer transmutes are valid (since we are okay with losing the
872                // provenance here). (Same as <pointer>::addr().)
873                unsafe {
874                    core::mem::transmute::<$ty, usize>(RTLD_DEFAULT)
875                        == core::mem::transmute::<$ty, usize>(sys::RTLD_DEFAULT)
876                }
877            );
878        };
879        ($name:ident, $ty:ty) => {
880            static_assert!($name == sys::$name as $ty);
881        };
882    }
883    /// Defines functions with #[cfg(test)] static assertions which checks
884    /// signatures are the same as the platform's latest header files' ones.
885    // Note: This macro is sys_fn!({ }), not sys_fn! { }.
886    // An extra brace is used in input to make contents rustfmt-able.
887    macro_rules! sys_fn {
888        ({
889            $(#[$extern_attr:meta])*
890            extern $abi:literal {$(
891                $(#[$fn_attr:meta])*
892                $vis:vis fn $([$($windows_path:ident)::+])? $name:ident(
893                    $($args:tt)*
894                ) $(-> $ret_ty:ty)?;
895            )*}
896        }) => {
897            $(#[$extern_attr])*
898            extern $abi {$(
899                $(#[$fn_attr])*
900                $vis fn $name($($args)*) $(-> $ret_ty)?;
901            )*}
902            // Static assertions for FFI bindings.
903            // This checks that FFI bindings defined in this crate and FFI bindings generated for
904            // the platform's latest header file using bindgen have the same signatures.
905            // Since this is static assertion, we can detect problems with
906            // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh)
907            // without actually running tests on these platforms.
908            // See also https://github.com/taiki-e/test-helper/blob/HEAD/tools/codegen/src/ffi.rs.
909            #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
910            #[allow(unused_imports)]
911            const _: fn() = || {
912                #[cfg(not(any(target_os = "aix", windows)))]
913                use test_helper::sys;
914                #[cfg(target_os = "aix")]
915                use libc as sys;
916                $(
917                    $(#[$fn_attr])*
918                    {
919                        $(use windows_sys::$($windows_path)::+ as sys;)?
920                        sys_fn_cmp!($abi fn $name($($args)*) $(-> $ret_ty)?);
921                    }
922                )*
923            };
924        };
925    }
926    #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
927    macro_rules! sys_fn_cmp {
928        (
929            $abi:literal fn $name:ident($($_arg_pat:ident: $arg_ty:ty),*, ...) $(-> $ret_ty:ty)?
930        ) => {
931            let mut _f: unsafe extern $abi fn($($arg_ty),*, ...) $(-> $ret_ty)? = $name;
932            _f = sys::$name;
933        };
934        (
935            $abi:literal fn $name:ident($($_arg_pat:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?
936        ) => {
937            let mut _f: unsafe extern $abi fn($($arg_ty),*) $(-> $ret_ty)? = $name;
938            _f = sys::$name;
939        };
940    }
941
942    #[allow(
943        clippy::alloc_instead_of_core,
944        clippy::std_instead_of_alloc,
945        clippy::std_instead_of_core,
946        clippy::undocumented_unsafe_blocks,
947        clippy::wildcard_imports
948    )]
949    #[cfg(test)]
950    mod tests {
951        #[test]
952        fn test_c_macro() {
953            #[track_caller]
954            fn t(s: &crate::utils::ffi::CStr, raw: &[u8]) {
955                assert_eq!(s.to_bytes_with_nul(), raw);
956            }
957            t(c!(""), b"\0");
958            t(c!("a"), b"a\0");
959            t(c!("abc"), b"abc\0");
960            t(c!(concat!("abc", "d")), b"abcd\0");
961        }
962
963        #[test]
964        fn test_is_c_str() {
965            #[track_caller]
966            fn t(bytes: &[u8]) {
967                assert_eq!(
968                    super::_const_is_c_str(bytes),
969                    std::ffi::CStr::from_bytes_with_nul(bytes).is_ok()
970                );
971            }
972            t(b"\0");
973            t(b"a\0");
974            t(b"abc\0");
975            t(b"");
976            t(b"a");
977            t(b"abc");
978            t(b"\0a");
979            t(b"\0a\0");
980            t(b"ab\0c\0");
981            t(b"\0\0");
982        }
983    }
984}
portable_atomic/utils.rs

portable_atomic/
utils.rs
