1 /* SPDX-License-Identifier: GPL-2.0-or-later *
2 /*
3 * atomic64_t for 586+
4 *
5 * Copyright © 2010 Luca Barbieri
6 */
7
8 #include <linux/linkage.h>
9 #include <asm/alternative.h>
10
11 .macro read64 reg
12 movl %ebx, %eax
13 movl %ecx, %edx
14 /* we need LOCK_PREFIX since otherwise cmpxchg
15 LOCK_PREFIX
16 cmpxchg8b (\reg)
17 .endm
18
19 .macro read64_nonatomic reg
20 movl (\reg), %eax
21 movl 4(\reg), %edx
22 .endm
23
24 SYM_FUNC_START(atomic64_read_cx8)
25 read64 %ecx
26 RET
27 SYM_FUNC_END(atomic64_read_cx8)
28
29 SYM_FUNC_START(atomic64_set_cx8)
30 1:
31 /* we don't need LOCK_PREFIX since aligned 64-
32 * are atomic on 586 and newer */
33 cmpxchg8b (%esi)
34 jne 1b
35
36 RET
37 SYM_FUNC_END(atomic64_set_cx8)
38
39 SYM_FUNC_START(atomic64_xchg_cx8)
40 1:
41 LOCK_PREFIX
42 cmpxchg8b (%esi)
43 jne 1b
44
45 RET
46 SYM_FUNC_END(atomic64_xchg_cx8)
47
48 .macro addsub_return func ins insc
49 SYM_FUNC_START(atomic64_\func\()_return_cx8)
50 pushl %ebp
51 pushl %ebx
52 pushl %esi
53 pushl %edi
54
55 movl %eax, %esi
56 movl %edx, %edi
57 movl %ecx, %ebp
58
59 read64_nonatomic %ecx
60 1:
61 movl %eax, %ebx
62 movl %edx, %ecx
63 \ins\()l %esi, %ebx
64 \insc\()l %edi, %ecx
65 LOCK_PREFIX
66 cmpxchg8b (%ebp)
67 jne 1b
68
69 10:
70 movl %ebx, %eax
71 movl %ecx, %edx
72 popl %edi
73 popl %esi
74 popl %ebx
75 popl %ebp
76 RET
77 SYM_FUNC_END(atomic64_\func\()_return_cx8)
78 .endm
79
80 addsub_return add add adc
81 addsub_return sub sub sbb
82
83 .macro incdec_return func ins insc
84 SYM_FUNC_START(atomic64_\func\()_return_cx8)
85 pushl %ebx
86
87 read64_nonatomic %esi
88 1:
89 movl %eax, %ebx
90 movl %edx, %ecx
91 \ins\()l $1, %ebx
92 \insc\()l $0, %ecx
93 LOCK_PREFIX
94 cmpxchg8b (%esi)
95 jne 1b
96
97 10:
98 movl %ebx, %eax
99 movl %ecx, %edx
100 popl %ebx
101 RET
102 SYM_FUNC_END(atomic64_\func\()_return_cx8)
103 .endm
104
105 incdec_return inc add adc
106 incdec_return dec sub sbb
107
108 SYM_FUNC_START(atomic64_dec_if_positive_cx8)
109 pushl %ebx
110
111 read64 %esi
112 1:
113 movl %eax, %ebx
114 movl %edx, %ecx
115 subl $1, %ebx
116 sbb $0, %ecx
117 js 2f
118 LOCK_PREFIX
119 cmpxchg8b (%esi)
120 jne 1b
121
122 2:
123 movl %ebx, %eax
124 movl %ecx, %edx
125 popl %ebx
126 RET
127 SYM_FUNC_END(atomic64_dec_if_positive_cx8)
128
129 SYM_FUNC_START(atomic64_add_unless_cx8)
130 pushl %ebp
131 pushl %ebx
132 /* these just push these two parameters on the
133 pushl %edi
134 pushl %ecx
135
136 movl %eax, %ebp
137 movl %edx, %edi
138
139 read64 %esi
140 1:
141 cmpl %eax, 0(%esp)
142 je 4f
143 2:
144 movl %eax, %ebx
145 movl %edx, %ecx
146 addl %ebp, %ebx
147 adcl %edi, %ecx
148 LOCK_PREFIX
149 cmpxchg8b (%esi)
150 jne 1b
151
152 movl $1, %eax
153 3:
154 addl $8, %esp
155 popl %ebx
156 popl %ebp
157 RET
158 4:
159 cmpl %edx, 4(%esp)
160 jne 2b
161 xorl %eax, %eax
162 jmp 3b
163 SYM_FUNC_END(atomic64_add_unless_cx8)
164
165 SYM_FUNC_START(atomic64_inc_not_zero_cx8)
166 pushl %ebx
167
168 read64 %esi
169 1:
170 movl %eax, %ecx
171 orl %edx, %ecx
172 jz 3f
173 movl %eax, %ebx
174 xorl %ecx, %ecx
175 addl $1, %ebx
176 adcl %edx, %ecx
177 LOCK_PREFIX
178 cmpxchg8b (%esi)
179 jne 1b
180
181 movl $1, %eax
182 3:
183 popl %ebx
184 RET
185 SYM_FUNC_END(atomic64_inc_not_zero_cx8)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.