1 /* SPDX-License-Identifier: GPL-2.0 */ 1
2 /*
3 * Copyright (C) 2020-2022 Loongson Technology
4 */
5 #ifndef __ASM_PERCPU_H
6 #define __ASM_PERCPU_H
7
8 #include <asm/cmpxchg.h>
9 #include <asm/loongarch.h>
10
11 /*
12 * The "address" (in fact, offset from $r21) o
13 * the loading address of main kernel image, b
14 * loaded. Tell the compiler this fact when us
15 */
16 #if defined(MODULE) && defined(CONFIG_AS_HAS_E
17 # if __has_attribute(model)
18 # define PER_CPU_ATTRIBUTES __attribute__((mo
19 # else
20 # error compiler support for the model attrib
21 # endif
22 #endif
23
24 /* Use r21 for fast access */
25 register unsigned long __my_cpu_offset __asm__
26
27 static inline void set_my_cpu_offset(unsigned
28 {
29 __my_cpu_offset = off;
30 csr_write64(off, PERCPU_BASE_KS);
31 }
32
33 #define __my_cpu_offset
34 ({
35 __asm__ __volatile__("":"+r"(__my_cpu_
36 __my_cpu_offset;
37 })
38
39 #define PERCPU_OP(op, asm_op, c_op)
40 static __always_inline unsigned long __percpu_
41 unsigned long val, int
42 {
43 unsigned long ret;
44
45 switch (size) {
46 case 4:
47 __asm__ __volatile__(
48 "am"#asm_op".w" " %[ret], %[va
49 : [ret] "=&r" (ret), [ptr] "+Z
50 : [val] "r" (val));
51 break;
52 case 8:
53 __asm__ __volatile__(
54 "am"#asm_op".d" " %[ret], %[va
55 : [ret] "=&r" (ret), [ptr] "+Z
56 : [val] "r" (val));
57 break;
58 default:
59 ret = 0;
60 BUILD_BUG();
61 }
62
63 return ret c_op val;
64 }
65
66 PERCPU_OP(add, add, +)
67 PERCPU_OP(and, and, &)
68 PERCPU_OP(or, or, |)
69 #undef PERCPU_OP
70
71 static __always_inline unsigned long __percpu_
72 {
73 switch (size) {
74 case 1:
75 case 2:
76 return __xchg_small((volatile
77
78 case 4:
79 return __xchg_asm("amswap.w",
80
81 case 8:
82 return __xchg_asm("amswap.d",
83
84 default:
85 BUILD_BUG();
86 }
87
88 return 0;
89 }
90
91 #define __pcpu_op_1(op) op ".b "
92 #define __pcpu_op_2(op) op ".h "
93 #define __pcpu_op_4(op) op ".w "
94 #define __pcpu_op_8(op) op ".d "
95
96 #define _percpu_read(size, _pcp)
97 ({
98 typeof(_pcp) __pcp_ret;
99
100 __asm__ __volatile__(
101 __pcpu_op_##size("ldx") "%[ret
102 : [ret] "=&r"(__pcp_ret)
103 : [ptr] "r"(&(_pcp))
104 : "memory");
105
106 __pcp_ret;
107 })
108
109 #define _percpu_write(size, _pcp, _val)
110 do {
111 __asm__ __volatile__(
112 __pcpu_op_##size("stx") "%[val
113 :
114 : [val] "r"(_val), [ptr] "r"(&
115 : "memory");
116 } while (0)
117
118 /* this_cpu_cmpxchg */
119 #define _protect_cmpxchg_local(pcp, o, n)
120 ({
121 typeof(*raw_cpu_ptr(&(pcp))) __ret;
122 preempt_disable_notrace();
123 __ret = cmpxchg_local(raw_cpu_ptr(&(pc
124 preempt_enable_notrace();
125 __ret;
126 })
127
128 #define _pcp_protect(operation, pcp, val)
129 ({
130 typeof(pcp) __retval;
131 preempt_disable_notrace();
132 __retval = (typeof(pcp))operation(raw_
133 (val
134 preempt_enable_notrace();
135 __retval;
136 })
137
138 #define _percpu_add(pcp, val) \
139 _pcp_protect(__percpu_add, pcp, val)
140
141 #define _percpu_add_return(pcp, val) _percpu_a
142
143 #define _percpu_and(pcp, val) \
144 _pcp_protect(__percpu_and, pcp, val)
145
146 #define _percpu_or(pcp, val) \
147 _pcp_protect(__percpu_or, pcp, val)
148
149 #define _percpu_xchg(pcp, val) ((typeof(pcp))
150 _pcp_protect(__percpu_xchg, pcp, (unsi
151
152 #define this_cpu_add_4(pcp, val) _percpu_add(p
153 #define this_cpu_add_8(pcp, val) _percpu_add(p
154
155 #define this_cpu_add_return_4(pcp, val) _percp
156 #define this_cpu_add_return_8(pcp, val) _percp
157
158 #define this_cpu_and_4(pcp, val) _percpu_and(p
159 #define this_cpu_and_8(pcp, val) _percpu_and(p
160
161 #define this_cpu_or_4(pcp, val) _percpu_or(pcp
162 #define this_cpu_or_8(pcp, val) _percpu_or(pcp
163
164 #define this_cpu_read_1(pcp) _percpu_read(1, p
165 #define this_cpu_read_2(pcp) _percpu_read(2, p
166 #define this_cpu_read_4(pcp) _percpu_read(4, p
167 #define this_cpu_read_8(pcp) _percpu_read(8, p
168
169 #define this_cpu_write_1(pcp, val) _percpu_wri
170 #define this_cpu_write_2(pcp, val) _percpu_wri
171 #define this_cpu_write_4(pcp, val) _percpu_wri
172 #define this_cpu_write_8(pcp, val) _percpu_wri
173
174 #define this_cpu_xchg_1(pcp, val) _percpu_xchg
175 #define this_cpu_xchg_2(pcp, val) _percpu_xchg
176 #define this_cpu_xchg_4(pcp, val) _percpu_xchg
177 #define this_cpu_xchg_8(pcp, val) _percpu_xchg
178
179 #define this_cpu_cmpxchg_1(ptr, o, n) _protect
180 #define this_cpu_cmpxchg_2(ptr, o, n) _protect
181 #define this_cpu_cmpxchg_4(ptr, o, n) _protect
182 #define this_cpu_cmpxchg_8(ptr, o, n) _protect
183
184 #include <asm-generic/percpu.h>
185
186 #endif /* __ASM_PERCPU_H */
187
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.