1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* atomic.S: These things are too big to do inline. 3 * 4 * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net) 5 */ 6 7 #include <linux/export.h> 8 #include <linux/linkage.h> 9 #include <asm/asi.h> 10 #include <asm/backoff.h> 11 12 .text 13 14 /* Three versions of the atomic routines, one that 15 * does not return a value and does not perform 16 * memory barriers, and a two which return 17 * a value, the new and old value resp. and does the 18 * barriers. 19 */ 20 21 #define ATOMIC_OP(op) \ 22 ENTRY(arch_atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ 23 BACKOFF_SETUP(%o2); \ 24 1: lduw [%o1], %g1; \ 25 op %g1, %o0, %g7; \ 26 cas [%o1], %g1, %g7; \ 27 cmp %g1, %g7; \ 28 bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ 29 nop; \ 30 retl; \ 31 nop; \ 32 2: BACKOFF_SPIN(%o2, %o3, 1b); \ 33 ENDPROC(arch_atomic_##op); \ 34 EXPORT_SYMBOL(arch_atomic_##op); 35 36 #define ATOMIC_OP_RETURN(op) \ 37 ENTRY(arch_atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */\ 38 BACKOFF_SETUP(%o2); \ 39 1: lduw [%o1], %g1; \ 40 op %g1, %o0, %g7; \ 41 cas [%o1], %g1, %g7; \ 42 cmp %g1, %g7; \ 43 bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ 44 op %g1, %o0, %g1; \ 45 retl; \ 46 sra %g1, 0, %o0; \ 47 2: BACKOFF_SPIN(%o2, %o3, 1b); \ 48 ENDPROC(arch_atomic_##op##_return); \ 49 EXPORT_SYMBOL(arch_atomic_##op##_return); 50 51 #define ATOMIC_FETCH_OP(op) \ 52 ENTRY(arch_atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ 53 BACKOFF_SETUP(%o2); \ 54 1: lduw [%o1], %g1; \ 55 op %g1, %o0, %g7; \ 56 cas [%o1], %g1, %g7; \ 57 cmp %g1, %g7; \ 58 bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ 59 nop; \ 60 retl; \ 61 sra %g1, 0, %o0; \ 62 2: BACKOFF_SPIN(%o2, %o3, 1b); \ 63 ENDPROC(arch_atomic_fetch_##op); \ 64 EXPORT_SYMBOL(arch_atomic_fetch_##op); 65 66 ATOMIC_OP(add) 67 ATOMIC_OP_RETURN(add) 68 ATOMIC_FETCH_OP(add) 69 70 ATOMIC_OP(sub) 71 ATOMIC_OP_RETURN(sub) 72 ATOMIC_FETCH_OP(sub) 73 74 ATOMIC_OP(and) 75 ATOMIC_FETCH_OP(and) 76 77 ATOMIC_OP(or) 78 ATOMIC_FETCH_OP(or) 79 80 ATOMIC_OP(xor) 81 ATOMIC_FETCH_OP(xor) 82 83 #undef ATOMIC_FETCH_OP 84 #undef ATOMIC_OP_RETURN 85 #undef ATOMIC_OP 86 87 #define ATOMIC64_OP(op) \ 88 ENTRY(arch_atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ 89 BACKOFF_SETUP(%o2); \ 90 1: ldx [%o1], %g1; \ 91 op %g1, %o0, %g7; \ 92 casx [%o1], %g1, %g7; \ 93 cmp %g1, %g7; \ 94 bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ 95 nop; \ 96 retl; \ 97 nop; \ 98 2: BACKOFF_SPIN(%o2, %o3, 1b); \ 99 ENDPROC(arch_atomic64_##op); \ 100 EXPORT_SYMBOL(arch_atomic64_##op); 101 102 #define ATOMIC64_OP_RETURN(op) \ 103 ENTRY(arch_atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ 104 BACKOFF_SETUP(%o2); \ 105 1: ldx [%o1], %g1; \ 106 op %g1, %o0, %g7; \ 107 casx [%o1], %g1, %g7; \ 108 cmp %g1, %g7; \ 109 bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ 110 nop; \ 111 retl; \ 112 op %g1, %o0, %o0; \ 113 2: BACKOFF_SPIN(%o2, %o3, 1b); \ 114 ENDPROC(arch_atomic64_##op##_return); \ 115 EXPORT_SYMBOL(arch_atomic64_##op##_return); 116 117 #define ATOMIC64_FETCH_OP(op) \ 118 ENTRY(arch_atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ 119 BACKOFF_SETUP(%o2); \ 120 1: ldx [%o1], %g1; \ 121 op %g1, %o0, %g7; \ 122 casx [%o1], %g1, %g7; \ 123 cmp %g1, %g7; \ 124 bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ 125 nop; \ 126 retl; \ 127 mov %g1, %o0; \ 128 2: BACKOFF_SPIN(%o2, %o3, 1b); \ 129 ENDPROC(arch_atomic64_fetch_##op); \ 130 EXPORT_SYMBOL(arch_atomic64_fetch_##op); 131 132 ATOMIC64_OP(add) 133 ATOMIC64_OP_RETURN(add) 134 ATOMIC64_FETCH_OP(add) 135 136 ATOMIC64_OP(sub) 137 ATOMIC64_OP_RETURN(sub) 138 ATOMIC64_FETCH_OP(sub) 139 140 ATOMIC64_OP(and) 141 ATOMIC64_FETCH_OP(and) 142 143 ATOMIC64_OP(or) 144 ATOMIC64_FETCH_OP(or) 145 146 ATOMIC64_OP(xor) 147 ATOMIC64_FETCH_OP(xor) 148 149 #undef ATOMIC64_FETCH_OP 150 #undef ATOMIC64_OP_RETURN 151 #undef ATOMIC64_OP 152 153 ENTRY(arch_atomic64_dec_if_positive) /* %o0 = atomic_ptr */ 154 BACKOFF_SETUP(%o2) 155 1: ldx [%o0], %g1 156 brlez,pn %g1, 3f 157 sub %g1, 1, %g7 158 casx [%o0], %g1, %g7 159 cmp %g1, %g7 160 bne,pn %xcc, BACKOFF_LABEL(2f, 1b) 161 nop 162 3: retl 163 sub %g1, 1, %o0 164 2: BACKOFF_SPIN(%o2, %o3, 1b) 165 ENDPROC(arch_atomic64_dec_if_positive) 166 EXPORT_SYMBOL(arch_atomic64_dec_if_positive)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.