1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright IBM Corp. 1999, 2011
4 *
5 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
6 */
7
8 #ifndef __ASM_CMPXCHG_H
9 #define __ASM_CMPXCHG_H
10
11 #include <linux/mmdebug.h>
12 #include <linux/types.h>
13 #include <linux/bug.h>
14
15 void __xchg_called_with_bad_pointer(void);
16
17 static __always_inline unsigned long
18 __arch_xchg(unsigned long x, unsigned long address, int size)
19 {
20 unsigned long old;
21 int shift;
22
23 switch (size) {
24 case 1:
25 shift = (3 ^ (address & 3)) << 3;
26 address ^= address & 3;
27 asm volatile(
28 " l %0,%1\n"
29 "0: lr 0,%0\n"
30 " nr 0,%3\n"
31 " or 0,%2\n"
32 " cs %0,0,%1\n"
33 " jl 0b\n"
34 : "=&d" (old), "+Q" (*(int *) address)
35 : "d" ((x & 0xff) << shift), "d" (~(0xff << shift))
36 : "memory", "cc", "");
37 return old >> shift;
38 case 2:
39 shift = (2 ^ (address & 2)) << 3;
40 address ^= address & 2;
41 asm volatile(
42 " l %0,%1\n"
43 "0: lr 0,%0\n"
44 " nr 0,%3\n"
45 " or 0,%2\n"
46 " cs %0,0,%1\n"
47 " jl 0b\n"
48 : "=&d" (old), "+Q" (*(int *) address)
49 : "d" ((x & 0xffff) << shift), "d" (~(0xffff << shift))
50 : "memory", "cc", "");
51 return old >> shift;
52 case 4:
53 asm volatile(
54 " l %0,%1\n"
55 "0: cs %0,%2,%1\n"
56 " jl 0b\n"
57 : "=&d" (old), "+Q" (*(int *) address)
58 : "d" (x)
59 : "memory", "cc");
60 return old;
61 case 8:
62 asm volatile(
63 " lg %0,%1\n"
64 "0: csg %0,%2,%1\n"
65 " jl 0b\n"
66 : "=&d" (old), "+QS" (*(long *) address)
67 : "d" (x)
68 : "memory", "cc");
69 return old;
70 }
71 __xchg_called_with_bad_pointer();
72 return x;
73 }
74
75 #define arch_xchg(ptr, x) \
76 ({ \
77 __typeof__(*(ptr)) __ret; \
78 \
79 __ret = (__typeof__(*(ptr))) \
80 __arch_xchg((unsigned long)(x), (unsigned long)(ptr), \
81 sizeof(*(ptr))); \
82 __ret; \
83 })
84
85 void __cmpxchg_called_with_bad_pointer(void);
86
87 static __always_inline unsigned long __cmpxchg(unsigned long address,
88 unsigned long old,
89 unsigned long new, int size)
90 {
91 switch (size) {
92 case 1: {
93 unsigned int prev, shift, mask;
94
95 shift = (3 ^ (address & 3)) << 3;
96 address ^= address & 3;
97 old = (old & 0xff) << shift;
98 new = (new & 0xff) << shift;
99 mask = ~(0xff << shift);
100 asm volatile(
101 " l %[prev],%[address]\n"
102 " nr %[prev],%[mask]\n"
103 " xilf %[mask],0xffffffff\n"
104 " or %[new],%[prev]\n"
105 " or %[prev],%[tmp]\n"
106 "0: lr %[tmp],%[prev]\n"
107 " cs %[prev],%[new],%[address]\n"
108 " jnl 1f\n"
109 " xr %[tmp],%[prev]\n"
110 " xr %[new],%[tmp]\n"
111 " nr %[tmp],%[mask]\n"
112 " jz 0b\n"
113 "1:"
114 : [prev] "=&d" (prev),
115 [address] "+Q" (*(int *)address),
116 [tmp] "+&d" (old),
117 [new] "+&d" (new),
118 [mask] "+&d" (mask)
119 :: "memory", "cc");
120 return prev >> shift;
121 }
122 case 2: {
123 unsigned int prev, shift, mask;
124
125 shift = (2 ^ (address & 2)) << 3;
126 address ^= address & 2;
127 old = (old & 0xffff) << shift;
128 new = (new & 0xffff) << shift;
129 mask = ~(0xffff << shift);
130 asm volatile(
131 " l %[prev],%[address]\n"
132 " nr %[prev],%[mask]\n"
133 " xilf %[mask],0xffffffff\n"
134 " or %[new],%[prev]\n"
135 " or %[prev],%[tmp]\n"
136 "0: lr %[tmp],%[prev]\n"
137 " cs %[prev],%[new],%[address]\n"
138 " jnl 1f\n"
139 " xr %[tmp],%[prev]\n"
140 " xr %[new],%[tmp]\n"
141 " nr %[tmp],%[mask]\n"
142 " jz 0b\n"
143 "1:"
144 : [prev] "=&d" (prev),
145 [address] "+Q" (*(int *)address),
146 [tmp] "+&d" (old),
147 [new] "+&d" (new),
148 [mask] "+&d" (mask)
149 :: "memory", "cc");
150 return prev >> shift;
151 }
152 case 4: {
153 unsigned int prev = old;
154
155 asm volatile(
156 " cs %[prev],%[new],%[address]\n"
157 : [prev] "+&d" (prev),
158 [address] "+Q" (*(int *)address)
159 : [new] "d" (new)
160 : "memory", "cc");
161 return prev;
162 }
163 case 8: {
164 unsigned long prev = old;
165
166 asm volatile(
167 " csg %[prev],%[new],%[address]\n"
168 : [prev] "+&d" (prev),
169 [address] "+QS" (*(long *)address)
170 : [new] "d" (new)
171 : "memory", "cc");
172 return prev;
173 }
174 }
175 __cmpxchg_called_with_bad_pointer();
176 return old;
177 }
178
179 #define arch_cmpxchg(ptr, o, n) \
180 ({ \
181 __typeof__(*(ptr)) __ret; \
182 \
183 __ret = (__typeof__(*(ptr))) \
184 __cmpxchg((unsigned long)(ptr), (unsigned long)(o), \
185 (unsigned long)(n), sizeof(*(ptr))); \
186 __ret; \
187 })
188
189 #define arch_cmpxchg64 arch_cmpxchg
190 #define arch_cmpxchg_local arch_cmpxchg
191 #define arch_cmpxchg64_local arch_cmpxchg
192
193 #define system_has_cmpxchg128() 1
194
195 static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
196 {
197 asm volatile(
198 " cdsg %[old],%[new],%[ptr]\n"
199 : [old] "+d" (old), [ptr] "+QS" (*ptr)
200 : [new] "d" (new)
201 : "memory", "cc");
202 return old;
203 }
204
205 #define arch_cmpxchg128 arch_cmpxchg128
206
207 #endif /* __ASM_CMPXCHG_H */
208
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.