1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <asm/ptrace.h> 2 #include <asm/ptrace.h>
3 3
4 #include "bpf_jit_32.h" 4 #include "bpf_jit_32.h"
5 5
6 #define SAVE_SZ 96 6 #define SAVE_SZ 96
7 #define SCRATCH_OFF 72 7 #define SCRATCH_OFF 72
8 #define BE_PTR(label) be label 8 #define BE_PTR(label) be label
9 #define SIGN_EXTEND(reg) 9 #define SIGN_EXTEND(reg)
10 10
11 #define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_ 11 #define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */
12 12
13 .text 13 .text
14 .globl bpf_jit_load_word 14 .globl bpf_jit_load_word
15 bpf_jit_load_word: 15 bpf_jit_load_word:
16 cmp r_OFF, 0 16 cmp r_OFF, 0
17 bl bpf_slow_path_word_neg 17 bl bpf_slow_path_word_neg
18 nop 18 nop
19 .globl bpf_jit_load_word_positive_off 19 .globl bpf_jit_load_word_positive_offset
20 bpf_jit_load_word_positive_offset: 20 bpf_jit_load_word_positive_offset:
21 sub r_HEADLEN, r_OFF, r_TMP 21 sub r_HEADLEN, r_OFF, r_TMP
22 cmp r_TMP, 3 22 cmp r_TMP, 3
23 ble bpf_slow_path_word 23 ble bpf_slow_path_word
24 add r_SKB_DATA, r_OFF, r_TMP 24 add r_SKB_DATA, r_OFF, r_TMP
25 andcc r_TMP, 3, %g0 25 andcc r_TMP, 3, %g0
26 bne load_word_unaligned 26 bne load_word_unaligned
27 nop 27 nop
28 retl 28 retl
29 ld [r_TMP], r_A 29 ld [r_TMP], r_A
30 load_word_unaligned: 30 load_word_unaligned:
31 ldub [r_TMP + 0x0], r_OFF 31 ldub [r_TMP + 0x0], r_OFF
32 ldub [r_TMP + 0x1], r_TMP2 32 ldub [r_TMP + 0x1], r_TMP2
33 sll r_OFF, 8, r_OFF 33 sll r_OFF, 8, r_OFF
34 or r_OFF, r_TMP2, r_OFF 34 or r_OFF, r_TMP2, r_OFF
35 ldub [r_TMP + 0x2], r_TMP2 35 ldub [r_TMP + 0x2], r_TMP2
36 sll r_OFF, 8, r_OFF 36 sll r_OFF, 8, r_OFF
37 or r_OFF, r_TMP2, r_OFF 37 or r_OFF, r_TMP2, r_OFF
38 ldub [r_TMP + 0x3], r_TMP2 38 ldub [r_TMP + 0x3], r_TMP2
39 sll r_OFF, 8, r_OFF 39 sll r_OFF, 8, r_OFF
40 retl 40 retl
41 or r_OFF, r_TMP2, r_A 41 or r_OFF, r_TMP2, r_A
42 42
43 .globl bpf_jit_load_half 43 .globl bpf_jit_load_half
44 bpf_jit_load_half: 44 bpf_jit_load_half:
45 cmp r_OFF, 0 45 cmp r_OFF, 0
46 bl bpf_slow_path_half_neg 46 bl bpf_slow_path_half_neg
47 nop 47 nop
48 .globl bpf_jit_load_half_positive_off 48 .globl bpf_jit_load_half_positive_offset
49 bpf_jit_load_half_positive_offset: 49 bpf_jit_load_half_positive_offset:
50 sub r_HEADLEN, r_OFF, r_TMP 50 sub r_HEADLEN, r_OFF, r_TMP
51 cmp r_TMP, 1 51 cmp r_TMP, 1
52 ble bpf_slow_path_half 52 ble bpf_slow_path_half
53 add r_SKB_DATA, r_OFF, r_TMP 53 add r_SKB_DATA, r_OFF, r_TMP
54 andcc r_TMP, 1, %g0 54 andcc r_TMP, 1, %g0
55 bne load_half_unaligned 55 bne load_half_unaligned
56 nop 56 nop
57 retl 57 retl
58 lduh [r_TMP], r_A 58 lduh [r_TMP], r_A
59 load_half_unaligned: 59 load_half_unaligned:
60 ldub [r_TMP + 0x0], r_OFF 60 ldub [r_TMP + 0x0], r_OFF
61 ldub [r_TMP + 0x1], r_TMP2 61 ldub [r_TMP + 0x1], r_TMP2
62 sll r_OFF, 8, r_OFF 62 sll r_OFF, 8, r_OFF
63 retl 63 retl
64 or r_OFF, r_TMP2, r_A 64 or r_OFF, r_TMP2, r_A
65 65
66 .globl bpf_jit_load_byte 66 .globl bpf_jit_load_byte
67 bpf_jit_load_byte: 67 bpf_jit_load_byte:
68 cmp r_OFF, 0 68 cmp r_OFF, 0
69 bl bpf_slow_path_byte_neg 69 bl bpf_slow_path_byte_neg
70 nop 70 nop
71 .globl bpf_jit_load_byte_positive_off 71 .globl bpf_jit_load_byte_positive_offset
72 bpf_jit_load_byte_positive_offset: 72 bpf_jit_load_byte_positive_offset:
73 cmp r_OFF, r_HEADLEN 73 cmp r_OFF, r_HEADLEN
74 bge bpf_slow_path_byte 74 bge bpf_slow_path_byte
75 nop 75 nop
76 retl 76 retl
77 ldub [r_SKB_DATA + r_OFF], r_A 77 ldub [r_SKB_DATA + r_OFF], r_A
78 78
79 .globl bpf_jit_load_byte_msh 79 .globl bpf_jit_load_byte_msh
80 bpf_jit_load_byte_msh: 80 bpf_jit_load_byte_msh:
81 cmp r_OFF, 0 81 cmp r_OFF, 0
82 bl bpf_slow_path_byte_msh_neg 82 bl bpf_slow_path_byte_msh_neg
83 nop 83 nop
84 .globl bpf_jit_load_byte_msh_positive 84 .globl bpf_jit_load_byte_msh_positive_offset
85 bpf_jit_load_byte_msh_positive_offset: 85 bpf_jit_load_byte_msh_positive_offset:
86 cmp r_OFF, r_HEADLEN 86 cmp r_OFF, r_HEADLEN
87 bge bpf_slow_path_byte_msh 87 bge bpf_slow_path_byte_msh
88 nop 88 nop
89 ldub [r_SKB_DATA + r_OFF], r_OFF 89 ldub [r_SKB_DATA + r_OFF], r_OFF
90 and r_OFF, 0xf, r_OFF 90 and r_OFF, 0xf, r_OFF
91 retl 91 retl
92 sll r_OFF, 2, r_X 92 sll r_OFF, 2, r_X
93 93
94 #define bpf_slow_path_common(LEN) \ 94 #define bpf_slow_path_common(LEN) \
95 save %sp, -SAVE_SZ, %sp; \ 95 save %sp, -SAVE_SZ, %sp; \
96 mov %i0, %o0; \ 96 mov %i0, %o0; \
97 mov r_OFF, %o1; \ 97 mov r_OFF, %o1; \
98 add %fp, SCRATCH_OFF, %o2; \ 98 add %fp, SCRATCH_OFF, %o2; \
99 call skb_copy_bits; \ 99 call skb_copy_bits; \
100 mov (LEN), %o3; \ 100 mov (LEN), %o3; \
101 cmp %o0, 0; \ 101 cmp %o0, 0; \
102 restore; 102 restore;
103 103
104 bpf_slow_path_word: 104 bpf_slow_path_word:
105 bpf_slow_path_common(4) 105 bpf_slow_path_common(4)
106 bl bpf_error 106 bl bpf_error
107 ld [%sp + SCRATCH_OFF], r_A 107 ld [%sp + SCRATCH_OFF], r_A
108 retl 108 retl
109 nop 109 nop
110 bpf_slow_path_half: 110 bpf_slow_path_half:
111 bpf_slow_path_common(2) 111 bpf_slow_path_common(2)
112 bl bpf_error 112 bl bpf_error
113 lduh [%sp + SCRATCH_OFF], r_A 113 lduh [%sp + SCRATCH_OFF], r_A
114 retl 114 retl
115 nop 115 nop
116 bpf_slow_path_byte: 116 bpf_slow_path_byte:
117 bpf_slow_path_common(1) 117 bpf_slow_path_common(1)
118 bl bpf_error 118 bl bpf_error
119 ldub [%sp + SCRATCH_OFF], r_A 119 ldub [%sp + SCRATCH_OFF], r_A
120 retl 120 retl
121 nop 121 nop
122 bpf_slow_path_byte_msh: 122 bpf_slow_path_byte_msh:
123 bpf_slow_path_common(1) 123 bpf_slow_path_common(1)
124 bl bpf_error 124 bl bpf_error
125 ldub [%sp + SCRATCH_OFF], r_A 125 ldub [%sp + SCRATCH_OFF], r_A
126 and r_OFF, 0xf, r_OFF 126 and r_OFF, 0xf, r_OFF
127 retl 127 retl
128 sll r_OFF, 2, r_X 128 sll r_OFF, 2, r_X
129 129
130 #define bpf_negative_common(LEN) 130 #define bpf_negative_common(LEN) \
131 save %sp, -SAVE_SZ, %sp; 131 save %sp, -SAVE_SZ, %sp; \
132 mov %i0, %o0; 132 mov %i0, %o0; \
133 mov r_OFF, %o1; 133 mov r_OFF, %o1; \
134 SIGN_EXTEND(%o1); 134 SIGN_EXTEND(%o1); \
135 call bpf_internal_load_pointer_neg_ 135 call bpf_internal_load_pointer_neg_helper; \
136 mov (LEN), %o2; 136 mov (LEN), %o2; \
137 mov %o0, r_TMP; 137 mov %o0, r_TMP; \
138 cmp %o0, 0; 138 cmp %o0, 0; \
139 BE_PTR(bpf_error); 139 BE_PTR(bpf_error); \
140 restore; 140 restore;
141 141
142 bpf_slow_path_word_neg: 142 bpf_slow_path_word_neg:
143 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 143 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
144 cmp r_OFF, r_TMP 144 cmp r_OFF, r_TMP
145 bl bpf_error 145 bl bpf_error
146 nop 146 nop
147 .globl bpf_jit_load_word_negative_off 147 .globl bpf_jit_load_word_negative_offset
148 bpf_jit_load_word_negative_offset: 148 bpf_jit_load_word_negative_offset:
149 bpf_negative_common(4) 149 bpf_negative_common(4)
150 andcc r_TMP, 3, %g0 150 andcc r_TMP, 3, %g0
151 bne load_word_unaligned 151 bne load_word_unaligned
152 nop 152 nop
153 retl 153 retl
154 ld [r_TMP], r_A 154 ld [r_TMP], r_A
155 155
156 bpf_slow_path_half_neg: 156 bpf_slow_path_half_neg:
157 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 157 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
158 cmp r_OFF, r_TMP 158 cmp r_OFF, r_TMP
159 bl bpf_error 159 bl bpf_error
160 nop 160 nop
161 .globl bpf_jit_load_half_negative_off 161 .globl bpf_jit_load_half_negative_offset
162 bpf_jit_load_half_negative_offset: 162 bpf_jit_load_half_negative_offset:
163 bpf_negative_common(2) 163 bpf_negative_common(2)
164 andcc r_TMP, 1, %g0 164 andcc r_TMP, 1, %g0
165 bne load_half_unaligned 165 bne load_half_unaligned
166 nop 166 nop
167 retl 167 retl
168 lduh [r_TMP], r_A 168 lduh [r_TMP], r_A
169 169
170 bpf_slow_path_byte_neg: 170 bpf_slow_path_byte_neg:
171 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 171 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
172 cmp r_OFF, r_TMP 172 cmp r_OFF, r_TMP
173 bl bpf_error 173 bl bpf_error
174 nop 174 nop
175 .globl bpf_jit_load_byte_negative_off 175 .globl bpf_jit_load_byte_negative_offset
176 bpf_jit_load_byte_negative_offset: 176 bpf_jit_load_byte_negative_offset:
177 bpf_negative_common(1) 177 bpf_negative_common(1)
178 retl 178 retl
179 ldub [r_TMP], r_A 179 ldub [r_TMP], r_A
180 180
181 bpf_slow_path_byte_msh_neg: 181 bpf_slow_path_byte_msh_neg:
182 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 182 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
183 cmp r_OFF, r_TMP 183 cmp r_OFF, r_TMP
184 bl bpf_error 184 bl bpf_error
185 nop 185 nop
186 .globl bpf_jit_load_byte_msh_negative 186 .globl bpf_jit_load_byte_msh_negative_offset
187 bpf_jit_load_byte_msh_negative_offset: 187 bpf_jit_load_byte_msh_negative_offset:
188 bpf_negative_common(1) 188 bpf_negative_common(1)
189 ldub [r_TMP], r_OFF 189 ldub [r_TMP], r_OFF
190 and r_OFF, 0xf, r_OFF 190 and r_OFF, 0xf, r_OFF
191 retl 191 retl
192 sll r_OFF, 2, r_X 192 sll r_OFF, 2, r_X
193 193
194 bpf_error: 194 bpf_error:
195 /* Make the JIT program return zero. 195 /* Make the JIT program return zero. The JIT epilogue
196 * stores away the original %o7 into r 196 * stores away the original %o7 into r_saved_O7. The
197 * normal leaf function return is to u 197 * normal leaf function return is to use "retl" which
198 * would evalute to "jmpl %o7 + 8, %g0 198 * would evalute to "jmpl %o7 + 8, %g0" but we want to
199 * use the saved value thus the sequen 199 * use the saved value thus the sequence you see here.
200 */ 200 */
201 jmpl r_saved_O7 + 8, %g0 201 jmpl r_saved_O7 + 8, %g0
202 clr %o0 202 clr %o0
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.