1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_CMPXCHG_32_H
3 #define _ASM_X86_CMPXCHG_32_H
4
5 /*
6 * Note: if you use __cmpxchg64(), or their variants,
7 * you need to test for the feature in boot_cpu_data.
8 */
9
10 union __u64_halves {
11 u64 full;
12 struct {
13 u32 low, high;
14 };
15 };
16
17 #define __arch_cmpxchg64(_ptr, _old, _new, _lock) \
18 ({ \
19 union __u64_halves o = { .full = (_old), }, \
20 n = { .full = (_new), }; \
21 \
22 asm volatile(_lock "cmpxchg8b %[ptr]" \
23 : [ptr] "+m" (*(_ptr)), \
24 "+a" (o.low), "+d" (o.high) \
25 : "b" (n.low), "c" (n.high) \
26 : "memory"); \
27 \
28 o.full; \
29 })
30
31
32 static __always_inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
33 {
34 return __arch_cmpxchg64(ptr, old, new, LOCK_PREFIX);
35 }
36
37 static __always_inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
38 {
39 return __arch_cmpxchg64(ptr, old, new,);
40 }
41
42 #define __arch_try_cmpxchg64(_ptr, _oldp, _new, _lock) \
43 ({ \
44 union __u64_halves o = { .full = *(_oldp), }, \
45 n = { .full = (_new), }; \
46 bool ret; \
47 \
48 asm volatile(_lock "cmpxchg8b %[ptr]" \
49 CC_SET(e) \
50 : CC_OUT(e) (ret), \
51 [ptr] "+m" (*(_ptr)), \
52 "+a" (o.low), "+d" (o.high) \
53 : "b" (n.low), "c" (n.high) \
54 : "memory"); \
55 \
56 if (unlikely(!ret)) \
57 *(_oldp) = o.full; \
58 \
59 likely(ret); \
60 })
61
62 static __always_inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *oldp, u64 new)
63 {
64 return __arch_try_cmpxchg64(ptr, oldp, new, LOCK_PREFIX);
65 }
66
67 static __always_inline bool __try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp, u64 new)
68 {
69 return __arch_try_cmpxchg64(ptr, oldp, new,);
70 }
71
72 #ifdef CONFIG_X86_CMPXCHG64
73
74 #define arch_cmpxchg64 __cmpxchg64
75
76 #define arch_cmpxchg64_local __cmpxchg64_local
77
78 #define arch_try_cmpxchg64 __try_cmpxchg64
79
80 #define arch_try_cmpxchg64_local __try_cmpxchg64_local
81
82 #else
83
84 /*
85 * Building a kernel capable running on 80386 and 80486. It may be necessary
86 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
87 */
88
89 #define __arch_cmpxchg64_emu(_ptr, _old, _new, _lock_loc, _lock) \
90 ({ \
91 union __u64_halves o = { .full = (_old), }, \
92 n = { .full = (_new), }; \
93 \
94 asm volatile(ALTERNATIVE(_lock_loc \
95 "call cmpxchg8b_emu", \
96 _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
97 : "+a" (o.low), "+d" (o.high) \
98 : "b" (n.low), "c" (n.high), [ptr] "S" (_ptr) \
99 : "memory"); \
100 \
101 o.full; \
102 })
103
104 static __always_inline u64 arch_cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
105 {
106 return __arch_cmpxchg64_emu(ptr, old, new, LOCK_PREFIX_HERE, "lock; ");
107 }
108 #define arch_cmpxchg64 arch_cmpxchg64
109
110 static __always_inline u64 arch_cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
111 {
112 return __arch_cmpxchg64_emu(ptr, old, new, ,);
113 }
114 #define arch_cmpxchg64_local arch_cmpxchg64_local
115
116 #define __arch_try_cmpxchg64_emu(_ptr, _oldp, _new, _lock_loc, _lock) \
117 ({ \
118 union __u64_halves o = { .full = *(_oldp), }, \
119 n = { .full = (_new), }; \
120 bool ret; \
121 \
122 asm volatile(ALTERNATIVE(_lock_loc \
123 "call cmpxchg8b_emu", \
124 _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
125 CC_SET(e) \
126 : CC_OUT(e) (ret), \
127 "+a" (o.low), "+d" (o.high) \
128 : "b" (n.low), "c" (n.high), [ptr] "S" (_ptr) \
129 : "memory"); \
130 \
131 if (unlikely(!ret)) \
132 *(_oldp) = o.full; \
133 \
134 likely(ret); \
135 })
136
137 static __always_inline bool arch_try_cmpxchg64(volatile u64 *ptr, u64 *oldp, u64 new)
138 {
139 return __arch_try_cmpxchg64_emu(ptr, oldp, new, LOCK_PREFIX_HERE, "lock; ");
140 }
141 #define arch_try_cmpxchg64 arch_try_cmpxchg64
142
143 static __always_inline bool arch_try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp, u64 new)
144 {
145 return __arch_try_cmpxchg64_emu(ptr, oldp, new, ,);
146 }
147 #define arch_try_cmpxchg64_local arch_try_cmpxchg64_local
148
149 #endif
150
151 #define system_has_cmpxchg64() boot_cpu_has(X86_FEATURE_CX8)
152
153 #endif /* _ASM_X86_CMPXCHG_32_H */
154
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.