Page MenuHomeFreeBSD
Files F108016878

D22963.diff
No OneTemporary
Actions

D22963.diff
View Options

Index: head/sys/amd64/include/atomic.h
===================================================================
--- head/sys/amd64/include/atomic.h
+++ head/sys/amd64/include/atomic.h
@@ -553,6 +553,7 @@
#define atomic_readandclear_int(p) atomic_swap_int(p, 0)
#define atomic_readandclear_long(p) atomic_swap_long(p, 0)
+#define atomic_testandset_acq_long atomic_testandset_long
/* Operations on 8-bit bytes. */
#define atomic_set_8 atomic_set_char
Index: head/sys/arm/include/atomic-v4.h
===================================================================
--- head/sys/arm/include/atomic-v4.h
+++ head/sys/arm/include/atomic-v4.h
@@ -521,8 +521,10 @@
#define atomic_fcmpset_rel_32 atomic_fcmpset_32
#define atomic_fcmpset_acq_32 atomic_fcmpset_32
#ifdef _KERNEL
+#define atomic_fcmpset_8 atomic_fcmpset_8
#define atomic_fcmpset_rel_8 atomic_fcmpset_8
#define atomic_fcmpset_acq_8 atomic_fcmpset_8
+#define atomic_fcmpset_16 atomic_fcmpset_16
#define atomic_fcmpset_rel_16 atomic_fcmpset_16
#define atomic_fcmpset_acq_16 atomic_fcmpset_16
#define atomic_fcmpset_rel_64 atomic_fcmpset_64
@@ -533,8 +535,10 @@
#define atomic_cmpset_rel_32 atomic_cmpset_32
#define atomic_cmpset_acq_32 atomic_cmpset_32
#ifdef _KERNEL
+#define atomic_cmpset_8 atomic_cmpset_8
#define atomic_cmpset_rel_8 atomic_cmpset_8
#define atomic_cmpset_acq_8 atomic_cmpset_8
+#define atomic_cmpset_16 atomic_cmpset_16
#define atomic_cmpset_rel_16 atomic_cmpset_16
#define atomic_cmpset_acq_16 atomic_cmpset_16
#define atomic_cmpset_rel_64 atomic_cmpset_64
Index: head/sys/arm/include/atomic-v6.h
===================================================================
--- head/sys/arm/include/atomic-v6.h
+++ head/sys/arm/include/atomic-v6.h
@@ -245,6 +245,7 @@
ATOMIC_FCMPSET_CODE(ret, uint8_t, "b");
return (ret);
}
+#define atomic_fcmpset_8 atomic_fcmpset_8
static __inline int
atomic_fcmpset_acq_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new)
@@ -274,6 +275,7 @@
ATOMIC_FCMPSET_CODE(ret, uint16_t, "h");
return (ret);
}
+#define atomic_fcmpset_16 atomic_fcmpset_16
static __inline int
atomic_fcmpset_acq_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new)
@@ -429,6 +431,7 @@
ATOMIC_CMPSET_CODE(ret, "b");
return (ret);
}
+#define atomic_cmpset_8 atomic_cmpset_8
static __inline int
atomic_cmpset_acq_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new)
@@ -458,6 +461,7 @@
ATOMIC_CMPSET_CODE(ret, "h");
return (ret);
}
+#define atomic_cmpset_16 atomic_cmpset_16
static __inline int
atomic_cmpset_acq_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new)
@@ -899,6 +903,7 @@
return (atomic_testandclear_32((volatile uint32_t *)p, v));
}
+#define atomic_testandclear_long atomic_testandclear_long
static __inline int
atomic_testandset_32(volatile uint32_t *ptr, u_int bit)
@@ -942,6 +947,7 @@
return (atomic_testandset_32((volatile uint32_t *)p, v));
}
+#define atomic_testandset_long atomic_testandset_long
static __inline int
atomic_testandset_64(volatile uint64_t *p, u_int v)
Index: head/sys/arm/include/atomic.h
===================================================================
--- head/sys/arm/include/atomic.h
+++ head/sys/arm/include/atomic.h
@@ -103,4 +103,6 @@
#define atomic_store_rel_int atomic_store_rel_32
#define atomic_swap_int atomic_swap_32
+#include <sys/_atomic_subword.h>
+
#endif /* _MACHINE_ATOMIC_H_ */
Index: head/sys/arm64/include/atomic.h
===================================================================
--- head/sys/arm64/include/atomic.h
+++ head/sys/arm64/include/atomic.h
@@ -257,6 +257,11 @@
_ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \
_ATOMIC_CMPSET_IMPL(64, , , bar, a, l)
+#define atomic_cmpset_8 atomic_cmpset_8
+#define atomic_fcmpset_8 atomic_fcmpset_8
+#define atomic_cmpset_16 atomic_cmpset_16
+#define atomic_fcmpset_16 atomic_fcmpset_16
+
_ATOMIC_CMPSET( , , )
_ATOMIC_CMPSET(acq_, a, )
_ATOMIC_CMPSET(rel_, ,l)
@@ -465,6 +470,8 @@
return (ret); \
}
+#define atomic_load_acq_8 atomic_load_acq_8
+#define atomic_load_acq_16 atomic_load_acq_16
_ATOMIC_LOAD_ACQ_IMPL(8, w, b)
_ATOMIC_LOAD_ACQ_IMPL(16, w, h)
_ATOMIC_LOAD_ACQ_IMPL(32, w, )
@@ -595,6 +602,8 @@
dmb(sy);
}
+
+#include <sys/_atomic_subword.h>
#endif /* KCSAN && !KCSAN_RUNTIME */
#endif /* _MACHINE_ATOMIC_H_ */
Index: head/sys/i386/include/atomic.h
===================================================================
--- head/sys/i386/include/atomic.h
+++ head/sys/i386/include/atomic.h
@@ -808,6 +808,7 @@
#define atomic_readandclear_int(p) atomic_swap_int(p, 0)
#define atomic_readandclear_long(p) atomic_swap_long(p, 0)
+#define atomic_testandset_acq_long atomic_testandset_long
/* Operations on 8-bit bytes. */
#define atomic_set_8 atomic_set_char
Index: head/sys/kern/subr_csan.c
===================================================================
--- head/sys/kern/subr_csan.c
+++ head/sys/kern/subr_csan.c
@@ -538,7 +538,7 @@
CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
-_CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
+CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
CSAN_ATOMIC_FUNC_SET(8, uint8_t)
CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
_CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
@@ -554,11 +554,7 @@
CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
-#if defined(__aarch64__)
-_CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
-#else
CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
-#endif
CSAN_ATOMIC_FUNC_SET(16, uint16_t)
CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
_CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
@@ -632,6 +628,7 @@
#if !defined(__aarch64__)
CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
+CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long)
#endif
CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
Index: head/sys/powerpc/include/atomic.h
===================================================================
--- head/sys/powerpc/include/atomic.h
+++ head/sys/powerpc/include/atomic.h
@@ -727,11 +727,15 @@
ATOMIC_CMPSET_ACQ_REL(long);
+#ifdef ISA_206_ATOMICS
#define atomic_cmpset_8 atomic_cmpset_char
+#endif
#define atomic_cmpset_acq_8 atomic_cmpset_acq_char
#define atomic_cmpset_rel_8 atomic_cmpset_rel_char
+#ifdef ISA_206_ATOMICS
#define atomic_cmpset_16 atomic_cmpset_short
+#endif
#define atomic_cmpset_acq_16 atomic_cmpset_acq_short
#define atomic_cmpset_rel_16 atomic_cmpset_rel_short
@@ -894,11 +898,15 @@
ATOMIC_FCMPSET_ACQ_REL(int);
ATOMIC_FCMPSET_ACQ_REL(long);
+#ifdef ISA_206_ATOMICS
#define atomic_fcmpset_8 atomic_fcmpset_char
+#endif
#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
+#ifdef ISA_206_ATOMICS
#define atomic_fcmpset_16 atomic_fcmpset_short
+#endif
#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
@@ -1018,6 +1026,10 @@
#ifndef ISA_206_ATOMICS
#include <sys/_atomic_subword.h>
+#define atomic_cmpset_char atomic_cmpset_8
+#define atomic_cmpset_short atomic_cmpset_16
+#define atomic_fcmpset_char atomic_fcmpset_8
+#define atomic_fcmpset_short atomic_fcmpset_16
#endif
/* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */
Index: head/sys/sys/_atomic_subword.h
===================================================================
--- head/sys/sys/_atomic_subword.h
+++ head/sys/sys/_atomic_subword.h
@@ -41,6 +41,9 @@
#endif
#include <machine/endian.h>
+#ifndef _KERNEL
+#include <stdbool.h>
+#endif
#ifndef NBBY
#define NBBY 8
@@ -113,6 +116,7 @@
}
#endif
+#ifndef atomic_cmpset_8
static __inline int
atomic_cmpset_8(__volatile uint8_t *addr, uint8_t old, uint8_t val)
{
@@ -123,7 +127,9 @@
return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
old << shift, val << shift, 0xff << shift));
}
+#endif
+#ifndef atomic_fcmpset_8
static __inline int
atomic_fcmpset_8(__volatile uint8_t *addr, uint8_t *old, uint8_t val)
{
@@ -138,7 +144,9 @@
*old = (wold >> shift) & 0xff;
return (ret);
}
+#endif
+#ifndef atomic_cmpset_16
static __inline int
atomic_cmpset_16(__volatile uint16_t *addr, uint16_t old, uint16_t val)
{
@@ -149,7 +157,9 @@
return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
old << shift, val << shift, 0xffff << shift));
}
+#endif
+#ifndef atomic_fcmpset_16
static __inline int
atomic_fcmpset_16(__volatile uint16_t *addr, uint16_t *old, uint16_t val)
{
@@ -164,9 +174,101 @@
*old = (wold >> shift) & 0xffff;
return (ret);
}
+#endif
+#ifndef atomic_load_acq_8
+static __inline uint8_t
+atomic_load_acq_8(volatile uint8_t *p)
+{
+ int shift;
+ uint8_t ret;
+
+ shift = _ATOMIC_BYTE_SHIFT(p);
+ ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) & 0xff;
+ return (ret);
+}
+#endif
+
+#ifndef atomic_load_acq_16
+static __inline uint16_t
+atomic_load_acq_16(volatile uint16_t *p)
+{
+ int shift;
+ uint16_t ret;
+
+ shift = _ATOMIC_HWORD_SHIFT(p);
+ ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) &
+ 0xffff;
+ return (ret);
+}
+#endif
+
#undef _ATOMIC_WORD_ALIGNED
#undef _ATOMIC_BYTE_SHIFT
#undef _ATOMIC_HWORD_SHIFT
+
+/*
+ * Provide generic testandset_long implementation based on fcmpset long
+ * primitive. It may not be ideal for any given arch, so machine/atomic.h
+ * should define the macro atomic_testandset_long to override with an
+ * MD-specific version.
+ *
+ * (Organizationally, this isn't really subword atomics. But atomic_common is
+ * included too early in machine/atomic.h, so it isn't a good place for derived
+ * primitives like this.)
+ */
+#ifndef atomic_testandset_acq_long
+static __inline int
+atomic_testandset_acq_long(volatile u_long *p, u_int v)
+{
+ u_long bit, old;
+ bool ret;
+
+ bit = (1ul << (v % (sizeof(*p) * NBBY)));
+
+ old = atomic_load_acq_long(p);
+ ret = false;
+ while (!ret && (old & bit) == 0)
+ ret = atomic_fcmpset_acq_long(p, &old, old | bit);
+
+ return (!ret);
+}
+#endif
+
+#ifndef atomic_testandset_long
+static __inline int
+atomic_testandset_long(volatile u_long *p, u_int v)
+{
+ u_long bit, old;
+ bool ret;
+
+ bit = (1ul << (v % (sizeof(*p) * NBBY)));
+
+ old = atomic_load_long(p);
+ ret = false;
+ while (!ret && (old & bit) == 0)
+ ret = atomic_fcmpset_long(p, &old, old | bit);
+
+ return (!ret);
+}
+#endif
+
+#ifndef atomic_testandclear_long
+static __inline int
+atomic_testandclear_long(volatile u_long *p, u_int v)
+{
+ u_long bit, old;
+ bool ret;
+
+ bit = (1ul << (v % (sizeof(*p) * NBBY)));
+
+ old = atomic_load_long(p);
+ ret = false;
+ while (!ret && (old & bit) != 0)
+ ret = atomic_fcmpset_long(p, &old, old & ~bit);
+
+ return (ret);
+}
+#endif
#endif /* _SYS__ATOMIC_SUBWORD_H_ */
Index: head/sys/sys/_cscan_atomic.h
===================================================================
--- head/sys/sys/_cscan_atomic.h
+++ head/sys/sys/_cscan_atomic.h
@@ -69,7 +69,8 @@
void kcsan_atomic_store_rel_##name(volatile type *, type)
#define KCSAN_ATOMIC_TEST(op, name, type) \
- int kcsan_atomic_##op##_##name(volatile type *, u_int)
+ int kcsan_atomic_##op##_##name(volatile type *, u_int); \
+ int kcsan_atomic_##op##_acq_##name(volatile type *, u_int)
#define KCSAN_ATOMIC_FUNCS(name, type) \
KCSAN_ATOMIC_FUNC_1(add, name, type); \
@@ -156,6 +157,7 @@
#define atomic_swap_long kcsan_atomic_swap_long
#define atomic_testandclear_long kcsan_atomic_testandclear_long
#define atomic_testandset_long kcsan_atomic_testandset_long
+#define atomic_testandset_acq_long kcsan_atomic_testandset_acq_long
#define atomic_add_ptr kcsan_atomic_add_ptr
#define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr

File Metadata

Mime Type
text/plain
Expires
Tue, Jan 21, 1:57 PM (21 h, 6 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16012452
Default Alt Text
D22963.diff (11 KB)

Event Timeline