portable_atomic/imp/
mod.rs

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3// -----------------------------------------------------------------------------
4// Lock-free implementations
5
6#[cfg(not(any(
7    all(
8        portable_atomic_no_atomic_load_store,
9        not(all(target_arch = "bpf", not(feature = "critical-section"))),
10    ),
11    target_arch = "avr",
12    target_arch = "msp430",
13)))]
14#[cfg_attr(
15    portable_atomic_no_cfg_target_has_atomic,
16    cfg(not(all(
17        any(
18            target_arch = "riscv32",
19            target_arch = "riscv64",
20            feature = "critical-section",
21            portable_atomic_unsafe_assume_single_core,
22        ),
23        portable_atomic_no_atomic_cas,
24    )))
25)]
26#[cfg_attr(
27    not(portable_atomic_no_cfg_target_has_atomic),
28    cfg(not(all(
29        any(
30            target_arch = "riscv32",
31            target_arch = "riscv64",
32            feature = "critical-section",
33            portable_atomic_unsafe_assume_single_core,
34        ),
35        not(target_has_atomic = "ptr"),
36    )))
37)]
38mod core_atomic;
39
40// AVR
41#[cfg(target_arch = "avr")]
42#[cfg(not(portable_atomic_no_asm))]
43#[cfg(not(feature = "critical-section"))]
44mod avr;
45
46// MSP430
47#[cfg(target_arch = "msp430")]
48pub(crate) mod msp430;
49
50// RISC-V without A-extension
51#[cfg(any(test, not(feature = "critical-section")))]
52#[cfg_attr(
53    portable_atomic_no_cfg_target_has_atomic,
54    cfg(any(
55        all(test, not(any(miri, portable_atomic_sanitize_thread))),
56        portable_atomic_no_atomic_cas,
57    ))
58)]
59#[cfg_attr(
60    not(portable_atomic_no_cfg_target_has_atomic),
61    cfg(any(
62        all(test, not(any(miri, portable_atomic_sanitize_thread))),
63        not(target_has_atomic = "ptr"),
64    ))
65)]
66#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
67mod riscv;
68
69// x86-specific optimizations
70// Miri and Sanitizer do not support inline assembly.
71#[cfg(all(
72    any(target_arch = "x86", target_arch = "x86_64"),
73    not(any(miri, portable_atomic_sanitize_thread)),
74    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
75))]
76mod x86;
77
78// 64-bit atomic implementations on 32-bit architectures
79#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
80mod atomic64;
81
82// 128-bit atomic implementations on 64-bit architectures
83#[cfg(any(
84    target_arch = "aarch64",
85    target_arch = "arm64ec",
86    target_arch = "powerpc64",
87    target_arch = "riscv64",
88    target_arch = "s390x",
89    target_arch = "x86_64",
90))]
91mod atomic128;
92
93// -----------------------------------------------------------------------------
94// Lock-based fallback implementations
95
96#[cfg(feature = "fallback")]
97#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
98#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
99#[cfg(any(
100    test,
101    not(any(
102        all(
103            target_arch = "aarch64",
104            not(all(
105                any(miri, portable_atomic_sanitize_thread),
106                not(portable_atomic_atomic_intrinsics),
107            )),
108            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
109        ),
110        all(
111            target_arch = "arm64ec",
112            not(all(
113                any(miri, portable_atomic_sanitize_thread),
114                not(portable_atomic_atomic_intrinsics),
115            )),
116            not(portable_atomic_no_asm),
117        ),
118        all(
119            target_arch = "x86_64",
120            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
121            any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
122        ),
123        all(
124            target_arch = "riscv64",
125            not(any(miri, portable_atomic_sanitize_thread)),
126            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
127            any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
128        ),
129        all(
130            target_arch = "powerpc64",
131            not(all(
132                any(miri, portable_atomic_sanitize_thread),
133                not(portable_atomic_atomic_intrinsics),
134            )),
135            portable_atomic_unstable_asm_experimental_arch,
136            any(
137                target_feature = "quadword-atomics",
138                portable_atomic_target_feature = "quadword-atomics",
139            ),
140        ),
141        all(
142            target_arch = "s390x",
143            not(all(
144                any(miri, portable_atomic_sanitize_thread),
145                not(portable_atomic_atomic_intrinsics),
146            )),
147            not(portable_atomic_no_asm),
148        ),
149    ))
150))]
151mod fallback;
152
153// -----------------------------------------------------------------------------
154// Critical section based fallback implementations
155
156// On AVR, we always use critical section based fallback implementation.
157// AVR can be safely assumed to be single-core, so this is sound.
158// MSP430 as well.
159// See the module-level comments of interrupt module for more.
160#[cfg(any(
161    all(test, target_os = "none"),
162    portable_atomic_unsafe_assume_single_core,
163    feature = "critical-section",
164    target_arch = "avr",
165    target_arch = "msp430",
166))]
167#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
168#[cfg_attr(
169    not(portable_atomic_no_cfg_target_has_atomic),
170    cfg(any(test, not(target_has_atomic = "ptr")))
171)]
172#[cfg(any(
173    target_arch = "arm",
174    target_arch = "avr",
175    target_arch = "msp430",
176    target_arch = "riscv32",
177    target_arch = "riscv64",
178    target_arch = "xtensa",
179    feature = "critical-section",
180))]
181mod interrupt;
182
183// -----------------------------------------------------------------------------
184// Atomic float implementations
185
186#[cfg(feature = "float")]
187pub(crate) mod float;
188
189// -----------------------------------------------------------------------------
190
191// has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic
192#[cfg(not(any(
193    portable_atomic_no_atomic_load_store,
194    target_arch = "avr",
195    target_arch = "msp430",
196)))]
197#[cfg_attr(
198    portable_atomic_no_cfg_target_has_atomic,
199    cfg(not(all(
200        any(
201            target_arch = "riscv32",
202            target_arch = "riscv64",
203            feature = "critical-section",
204            portable_atomic_unsafe_assume_single_core,
205        ),
206        portable_atomic_no_atomic_cas,
207    )))
208)]
209#[cfg_attr(
210    not(portable_atomic_no_cfg_target_has_atomic),
211    cfg(not(all(
212        any(
213            target_arch = "riscv32",
214            target_arch = "riscv64",
215            feature = "critical-section",
216            portable_atomic_unsafe_assume_single_core,
217        ),
218        not(target_has_atomic = "ptr"),
219    )))
220)]
221items! {
222    pub(crate) use self::core_atomic::{
223        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
224        AtomicUsize,
225    };
226    #[cfg_attr(
227        portable_atomic_no_cfg_target_has_atomic,
228        cfg(any(
229            not(portable_atomic_no_atomic_64),
230            not(any(target_pointer_width = "16", target_pointer_width = "32")),
231        ))
232    )]
233    #[cfg_attr(
234        not(portable_atomic_no_cfg_target_has_atomic),
235        cfg(any(
236            target_has_atomic = "64",
237            not(any(target_pointer_width = "16", target_pointer_width = "32")),
238        ))
239    )]
240    pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
241}
242// bpf & !(critical section) => core atomic
243#[cfg(all(
244    target_arch = "bpf",
245    portable_atomic_no_atomic_load_store,
246    not(feature = "critical-section"),
247))]
248pub(crate) use self::core_atomic::{AtomicI64, AtomicIsize, AtomicPtr, AtomicU64, AtomicUsize};
249
250// RISC-V without A-extension & !(assume single core | critical section)
251#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
252#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
253#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
254#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
255items! {
256    pub(crate) use self::riscv::{
257        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
258        AtomicUsize,
259    };
260    #[cfg(target_arch = "riscv64")]
261    pub(crate) use self::riscv::{AtomicI64, AtomicU64};
262}
263
264// no core atomic CAS & (assume single core | critical section) => critical section based fallback
265#[cfg(any(
266    portable_atomic_unsafe_assume_single_core,
267    feature = "critical-section",
268    target_arch = "avr",
269    target_arch = "msp430",
270))]
271#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
272#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
273items! {
274    pub(crate) use self::interrupt::{
275        AtomicI16, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU8, AtomicUsize,
276    };
277    #[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
278    pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
279    #[cfg(any(
280        not(any(target_pointer_width = "16", target_pointer_width = "32")),
281        feature = "fallback",
282    ))]
283    pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
284    #[cfg(feature = "fallback")]
285    pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
286}
287
288// no core (64-bit | 128-bit) atomic & has CAS => use lock-base fallback
289#[cfg(feature = "fallback")]
290#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
291#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
292items! {
293    #[cfg(not(any(
294        all(
295            target_arch = "arm",
296            not(any(miri, portable_atomic_sanitize_thread)),
297            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
298            any(target_os = "linux", target_os = "android"),
299            not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
300            not(portable_atomic_no_outline_atomics),
301        ),
302        all(
303            target_arch = "riscv32",
304            not(any(miri, portable_atomic_sanitize_thread)),
305            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
306            any(
307                target_feature = "zacas",
308                portable_atomic_target_feature = "zacas",
309                all(
310                    feature = "fallback",
311                    not(portable_atomic_no_outline_atomics),
312                    any(target_os = "linux", target_os = "android"),
313                ),
314            ),
315        ),
316    )))]
317    #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
318    #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
319    pub(crate) use self::fallback::{AtomicI64, AtomicU64};
320    #[cfg(not(any(
321        all(
322            target_arch = "aarch64",
323            not(all(
324                any(miri, portable_atomic_sanitize_thread),
325                not(portable_atomic_atomic_intrinsics),
326            )),
327            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
328        ),
329        all(
330            target_arch = "arm64ec",
331            not(all(
332                any(miri, portable_atomic_sanitize_thread),
333                not(portable_atomic_atomic_intrinsics),
334            )),
335            not(portable_atomic_no_asm),
336        ),
337        all(
338            target_arch = "x86_64",
339            not(all(
340                any(miri, portable_atomic_sanitize_thread),
341                portable_atomic_no_cmpxchg16b_intrinsic,
342            )),
343            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
344            any(
345                target_feature = "cmpxchg16b",
346                portable_atomic_target_feature = "cmpxchg16b",
347                all(
348                    feature = "fallback",
349                    not(portable_atomic_no_outline_atomics),
350                    not(any(target_env = "sgx", miri)),
351                ),
352            ),
353        ),
354        all(
355            target_arch = "riscv64",
356            not(any(miri, portable_atomic_sanitize_thread)),
357            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
358            any(
359                target_feature = "zacas",
360                portable_atomic_target_feature = "zacas",
361                all(
362                    feature = "fallback",
363                    not(portable_atomic_no_outline_atomics),
364                    any(target_os = "linux", target_os = "android"),
365                ),
366            ),
367        ),
368        all(
369            target_arch = "powerpc64",
370            not(all(
371                any(miri, portable_atomic_sanitize_thread),
372                not(portable_atomic_atomic_intrinsics),
373            )),
374            portable_atomic_unstable_asm_experimental_arch,
375            any(
376                target_feature = "quadword-atomics",
377                portable_atomic_target_feature = "quadword-atomics",
378                all(
379                    feature = "fallback",
380                    not(portable_atomic_no_outline_atomics),
381                    any(
382                        all(
383                            target_os = "linux",
384                            any(
385                                all(
386                                    target_env = "gnu",
387                                    any(target_endian = "little", not(target_feature = "crt-static")),
388                                ),
389                                all(
390                                    target_env = "musl",
391                                    any(not(target_feature = "crt-static"), feature = "std"),
392                                ),
393                                target_env = "ohos",
394                                all(target_env = "uclibc", not(target_feature = "crt-static")),
395                                portable_atomic_outline_atomics,
396                            ),
397                        ),
398                        target_os = "android",
399                        all(
400                            target_os = "freebsd",
401                            any(
402                                target_endian = "little",
403                                not(target_feature = "crt-static"),
404                                portable_atomic_outline_atomics,
405                            ),
406                        ),
407                        target_os = "openbsd",
408                        all(
409                            target_os = "aix",
410                            not(portable_atomic_pre_llvm_20),
411                            portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
412                        ),
413                    ),
414                    not(any(miri, portable_atomic_sanitize_thread)),
415                ),
416            ),
417        ),
418        all(
419            target_arch = "s390x",
420            not(all(
421                any(miri, portable_atomic_sanitize_thread),
422                not(portable_atomic_atomic_intrinsics),
423            )),
424            not(portable_atomic_no_asm),
425        ),
426    )))]
427    pub(crate) use self::fallback::{AtomicI128, AtomicU128};
428}
429
430// 64-bit atomics (platform-specific)
431// pre-v6 Arm Linux
432#[cfg(feature = "fallback")]
433#[cfg(all(
434    target_arch = "arm",
435    not(any(miri, portable_atomic_sanitize_thread)),
436    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
437    any(target_os = "linux", target_os = "android"),
438    not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
439    not(portable_atomic_no_outline_atomics),
440))]
441#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
442#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
443pub(crate) use self::atomic64::arm_linux::{AtomicI64, AtomicU64};
444// riscv32 & (zacas | outline-atomics)
445#[cfg(all(
446    target_arch = "riscv32",
447    not(any(miri, portable_atomic_sanitize_thread)),
448    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
449    any(
450        target_feature = "zacas",
451        portable_atomic_target_feature = "zacas",
452        all(
453            feature = "fallback",
454            not(portable_atomic_no_outline_atomics),
455            any(target_os = "linux", target_os = "android"),
456        ),
457    ),
458))]
459pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64};
460
461// 128-bit atomics (platform-specific)
462// AArch64
463#[cfg(any(
464    all(
465        target_arch = "aarch64",
466        not(all(
467            any(miri, portable_atomic_sanitize_thread),
468            not(portable_atomic_atomic_intrinsics),
469        )),
470        any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
471    ),
472    all(
473        target_arch = "arm64ec",
474        not(all(
475            any(miri, portable_atomic_sanitize_thread),
476            not(portable_atomic_atomic_intrinsics),
477        )),
478        not(portable_atomic_no_asm),
479    ),
480))]
481pub(crate) use self::atomic128::aarch64::{AtomicI128, AtomicU128};
482// x86_64 & (cmpxchg16b | outline-atomics)
483#[cfg(all(
484    target_arch = "x86_64",
485    not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),
486    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
487    any(
488        target_feature = "cmpxchg16b",
489        portable_atomic_target_feature = "cmpxchg16b",
490        all(
491            feature = "fallback",
492            not(portable_atomic_no_outline_atomics),
493            not(any(target_env = "sgx", miri)),
494        ),
495    ),
496))]
497pub(crate) use self::atomic128::x86_64::{AtomicI128, AtomicU128};
498// riscv64 & (zacas | outline-atomics)
499#[cfg(all(
500    target_arch = "riscv64",
501    not(any(miri, portable_atomic_sanitize_thread)),
502    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
503    any(
504        target_feature = "zacas",
505        portable_atomic_target_feature = "zacas",
506        all(
507            feature = "fallback",
508            not(portable_atomic_no_outline_atomics),
509            any(target_os = "linux", target_os = "android"),
510        ),
511    ),
512))]
513pub(crate) use self::atomic128::riscv64::{AtomicI128, AtomicU128};
514// powerpc64 & (pwr8 | outline-atomics)
515#[cfg(all(
516    target_arch = "powerpc64",
517    not(all(
518        any(miri, portable_atomic_sanitize_thread),
519        not(portable_atomic_atomic_intrinsics),
520    )),
521    portable_atomic_unstable_asm_experimental_arch,
522    any(
523        target_feature = "quadword-atomics",
524        portable_atomic_target_feature = "quadword-atomics",
525        all(
526            feature = "fallback",
527            not(portable_atomic_no_outline_atomics),
528            any(
529                all(
530                    target_os = "linux",
531                    any(
532                        all(
533                            target_env = "gnu",
534                            any(target_endian = "little", not(target_feature = "crt-static")),
535                        ),
536                        all(
537                            target_env = "musl",
538                            any(not(target_feature = "crt-static"), feature = "std"),
539                        ),
540                        target_env = "ohos",
541                        all(target_env = "uclibc", not(target_feature = "crt-static")),
542                        portable_atomic_outline_atomics,
543                    ),
544                ),
545                target_os = "android",
546                all(
547                    target_os = "freebsd",
548                    any(
549                        target_endian = "little",
550                        not(target_feature = "crt-static"),
551                        portable_atomic_outline_atomics,
552                    ),
553                ),
554                target_os = "openbsd",
555                all(
556                    target_os = "aix",
557                    not(portable_atomic_pre_llvm_20),
558                    portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
559                ),
560            ),
561            not(any(miri, portable_atomic_sanitize_thread)),
562        ),
563    ),
564))]
565pub(crate) use self::atomic128::powerpc64::{AtomicI128, AtomicU128};
566// s390x
567#[cfg(all(
568    target_arch = "s390x",
569    not(all(any(miri, portable_atomic_sanitize_thread), not(portable_atomic_atomic_intrinsics))),
570    not(portable_atomic_no_asm),
571))]
572pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128};
portable_atomic/imp/mod.rs

portable_atomic/imp/
mod.rs
