17#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18#include <Kokkos_Macros.hpp>
20 "Including non-public Kokkos header files is not allowed.");
22#ifndef KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
23#define KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
24#include <Kokkos_Macros.hpp>
26#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS
27#include <Kokkos_Atomics_Desul_Config.hpp>
28#include <desul/atomics.hpp>
30#include <impl/Kokkos_Atomic_Memory_Order.hpp>
31#include <impl/Kokkos_Volatile_Load.hpp>
38inline const char* atomic_query_version() {
return "KOKKOS_DESUL_ATOMICS"; }
40#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
41 !defined(__CUDA_ARCH__)
43#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
44#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
48#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
49#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
54#ifdef KOKKOS_INTERNAL_NOT_PARALLEL
55#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
57#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
60template<
class T> KOKKOS_INLINE_FUNCTION
61T atomic_load(T*
const dest) {
return desul::atomic_load(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
63template<
class T> KOKKOS_INLINE_FUNCTION
64void atomic_store(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_store(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
66template<
class T> KOKKOS_INLINE_FUNCTION
67void atomic_assign(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { atomic_store(dest,val); }
71 desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE);
75void load_fence() {
return desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); }
78void store_fence() {
return desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); }
81template<
class T> KOKKOS_INLINE_FUNCTION
82T atomic_fetch_add (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
84template<
class T> KOKKOS_INLINE_FUNCTION
85T atomic_fetch_sub (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
87template<
class T> KOKKOS_INLINE_FUNCTION
88T atomic_fetch_max (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
90template<
class T> KOKKOS_INLINE_FUNCTION
91T atomic_fetch_min (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
93template<
class T> KOKKOS_INLINE_FUNCTION
94T atomic_fetch_mul (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
96template<
class T> KOKKOS_INLINE_FUNCTION
97T atomic_fetch_div (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
99template<
class T> KOKKOS_INLINE_FUNCTION
100T atomic_fetch_mod (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_mod (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
102template<
class T> KOKKOS_INLINE_FUNCTION
103T atomic_fetch_and (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
105template<
class T> KOKKOS_INLINE_FUNCTION
106T atomic_fetch_or (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
108template<
class T> KOKKOS_INLINE_FUNCTION
109T atomic_fetch_xor (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_xor (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
111template<
class T> KOKKOS_INLINE_FUNCTION
112T atomic_fetch_nand(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_nand(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
114template<
class T> KOKKOS_INLINE_FUNCTION
115T atomic_fetch_lshift(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_lshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
117template<
class T> KOKKOS_INLINE_FUNCTION
118T atomic_fetch_rshift(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_rshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
120template<
class T> KOKKOS_INLINE_FUNCTION
121T atomic_fetch_inc(T*
const dest) {
return desul::atomic_fetch_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
123template<
class T> KOKKOS_INLINE_FUNCTION
124T atomic_fetch_dec(T*
const dest) {
return desul::atomic_fetch_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
128template<
class T> KOKKOS_INLINE_FUNCTION
129T atomic_add_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_add_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
131template<
class T> KOKKOS_INLINE_FUNCTION
132T atomic_sub_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_sub_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
134template<
class T> KOKKOS_INLINE_FUNCTION
135T atomic_max_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_max_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
137template<
class T> KOKKOS_INLINE_FUNCTION
138T atomic_min_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_min_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
140template<
class T> KOKKOS_INLINE_FUNCTION
141T atomic_mul_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_mul_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
143template<
class T> KOKKOS_INLINE_FUNCTION
144T atomic_div_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_div_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
146template<
class T> KOKKOS_INLINE_FUNCTION
147T atomic_mod_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_mod_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
149template<
class T> KOKKOS_INLINE_FUNCTION
150T atomic_and_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_and_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
152template<
class T> KOKKOS_INLINE_FUNCTION
153T atomic_or_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_or_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
155template<
class T> KOKKOS_INLINE_FUNCTION
156T atomic_xor_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_xor_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
158template<
class T> KOKKOS_INLINE_FUNCTION
159T atomic_nand_fetch(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_nand_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
161template<
class T> KOKKOS_INLINE_FUNCTION
162T atomic_lshift_fetch(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_lshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
164template<
class T> KOKKOS_INLINE_FUNCTION
165T atomic_rshift_fetch(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_rshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
167template<
class T> KOKKOS_INLINE_FUNCTION
168T atomic_inc_fetch(T*
const dest) {
return desul::atomic_inc_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
170template<
class T> KOKKOS_INLINE_FUNCTION
171T atomic_dec_fetch(T*
const dest) {
return desul::atomic_dec_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
175template<
class T> KOKKOS_INLINE_FUNCTION
176void atomic_add(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
178template<
class T> KOKKOS_INLINE_FUNCTION
179void atomic_sub(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
181template<
class T> KOKKOS_INLINE_FUNCTION
182void atomic_mul(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
184template<
class T> KOKKOS_INLINE_FUNCTION
185void atomic_div(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
187template<
class T> KOKKOS_INLINE_FUNCTION
188void atomic_min(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
190template<
class T> KOKKOS_INLINE_FUNCTION
191void atomic_max(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
194template<
class T> KOKKOS_INLINE_FUNCTION
195void atomic_and(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
198template<
class T> KOKKOS_INLINE_FUNCTION
199void atomic_or(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
201template<
class T> KOKKOS_INLINE_FUNCTION
202void atomic_inc(T*
const dest) {
return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
204template<
class T> KOKKOS_INLINE_FUNCTION
205void atomic_dec(T*
const dest) {
return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
207template<
class T> KOKKOS_INLINE_FUNCTION
208void atomic_increment(T*
const dest) {
return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
210template<
class T> KOKKOS_INLINE_FUNCTION
211void atomic_decrement(T*
const dest) {
return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
215template<
class T> KOKKOS_INLINE_FUNCTION
216T atomic_exchange(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_exchange(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
218template<
class T> KOKKOS_INLINE_FUNCTION
219bool atomic_compare_exchange_strong(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> expected, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
220 T expected_ref = expected;
221 return desul::atomic_compare_exchange_strong(dest, expected_ref, desired,
222 desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
225template<
class T> KOKKOS_INLINE_FUNCTION
226T atomic_compare_exchange(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> compare, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
227 return desul::atomic_compare_exchange(dest, compare, desired,
228 desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
233 template<
class MemoryOrder>
234 struct KokkosToDesulMemoryOrder;
237 struct KokkosToDesulMemoryOrder<memory_order_seq_cst_t> {
238 using type = desul::MemoryOrderSeqCst;
241 struct KokkosToDesulMemoryOrder<memory_order_acquire_t> {
242 using type = desul::MemoryOrderAcquire;
245 struct KokkosToDesulMemoryOrder<memory_order_release_t> {
246 using type = desul::MemoryOrderRelease;
249 struct KokkosToDesulMemoryOrder<memory_order_acq_rel_t> {
250 using type = desul::MemoryOrderAcqRel;
253 struct KokkosToDesulMemoryOrder<memory_order_relaxed_t> {
254 using type = desul::MemoryOrderRelaxed;
256 template<
class T,
class MemOrderSuccess,
class MemOrderFailure> KOKKOS_INLINE_FUNCTION
257 bool atomic_compare_exchange_strong(T*
const dest, T& expected,
const T desired, MemOrderSuccess, MemOrderFailure) {
258 return desul::atomic_compare_exchange_strong(dest, expected, desired,
259 typename KokkosToDesulMemoryOrder<MemOrderSuccess>::type(),
260 typename KokkosToDesulMemoryOrder<MemOrderFailure>::type(),
261 KOKKOS_DESUL_MEM_SCOPE);
264 template<
class T,
class MemoryOrder>
265 KOKKOS_INLINE_FUNCTION
266 T atomic_load(
const T*
const src, MemoryOrder) {
267 return desul::atomic_load(src,
typename KokkosToDesulMemoryOrder<MemoryOrder>::type(), KOKKOS_DESUL_MEM_SCOPE);
269 template<
class T,
class MemoryOrder>
270 KOKKOS_INLINE_FUNCTION
271 void atomic_store(T*
const src,
const T val, MemoryOrder) {
272 return desul::atomic_store(src, val,
typename KokkosToDesulMemoryOrder<MemoryOrder>::type(), KOKKOS_DESUL_MEM_SCOPE);
278#undef KOKKOS_DESUL_MEM_SCOPE