1 /* SPDX-License-Identifier: GPL-2.0-only */ 1
2 /*
3 * BPF JIT compiler for ARM64
4 *
5 * Copyright (C) 2014-2016 Zi Shen Lim <zlim.l
6 */
7 #ifndef _BPF_JIT_H
8 #define _BPF_JIT_H
9
10 #include <asm/insn.h>
11
12 /* 5-bit Register Operand */
13 #define A64_R(x) AARCH64_INSN_REG_##x
14 #define A64_FP AARCH64_INSN_REG_FP
15 #define A64_LR AARCH64_INSN_REG_LR
16 #define A64_ZR AARCH64_INSN_REG_ZR
17 #define A64_SP AARCH64_INSN_REG_SP
18
19 #define A64_VARIANT(sf) \
20 ((sf) ? AARCH64_INSN_VARIANT_64BIT : A
21
22 /* Compare & branch (immediate) */
23 #define A64_COMP_BRANCH(sf, Rt, offset, type)
24 aarch64_insn_gen_comp_branch_imm(0, of
25 AARCH64_INSN_BRANCH_COMP_##typ
26 #define A64_CBZ(sf, Rt, imm19) A64_COMP_BRANCH
27 #define A64_CBNZ(sf, Rt, imm19) A64_COMP_BRANC
28
29 /* Conditional branch (immediate) */
30 #define A64_COND_BRANCH(cond, offset) \
31 aarch64_insn_gen_cond_branch_imm(0, of
32 #define A64_COND_EQ AARCH64_INSN_COND_EQ /
33 #define A64_COND_NE AARCH64_INSN_COND_NE /
34 #define A64_COND_CS AARCH64_INSN_COND_CS /
35 #define A64_COND_HI AARCH64_INSN_COND_HI /
36 #define A64_COND_LS AARCH64_INSN_COND_LS /
37 #define A64_COND_CC AARCH64_INSN_COND_CC /
38 #define A64_COND_GE AARCH64_INSN_COND_GE /
39 #define A64_COND_GT AARCH64_INSN_COND_GT /
40 #define A64_COND_LE AARCH64_INSN_COND_LE /
41 #define A64_COND_LT AARCH64_INSN_COND_LT /
42 #define A64_B_(cond, imm19) A64_COND_BRANCH(co
43
44 /* Unconditional branch (immediate) */
45 #define A64_BRANCH(offset, type) aarch64_insn_
46 AARCH64_INSN_BRANCH_##type)
47 #define A64_B(imm26) A64_BRANCH((imm26) << 2,
48 #define A64_BL(imm26) A64_BRANCH((imm26) << 2,
49
50 /* Unconditional branch (register) */
51 #define A64_BR(Rn) aarch64_insn_gen_branch_re
52 #define A64_BLR(Rn) aarch64_insn_gen_branch_re
53 #define A64_RET(Rn) aarch64_insn_gen_branch_re
54
55 /* Load/store register (register offset) */
56 #define A64_LS_REG(Rt, Rn, Rm, size, type) \
57 aarch64_insn_gen_load_store_reg(Rt, Rn
58 AARCH64_INSN_SIZE_##size, \
59 AARCH64_INSN_LDST_##type##_REG
60 #define A64_STRB(Wt, Xn, Xm) A64_LS_REG(Wt, X
61 #define A64_LDRB(Wt, Xn, Xm) A64_LS_REG(Wt, X
62 #define A64_LDRSB(Xt, Xn, Xm) A64_LS_REG(Xt, X
63 #define A64_STRH(Wt, Xn, Xm) A64_LS_REG(Wt, X
64 #define A64_LDRH(Wt, Xn, Xm) A64_LS_REG(Wt, X
65 #define A64_LDRSH(Xt, Xn, Xm) A64_LS_REG(Xt, X
66 #define A64_STR32(Wt, Xn, Xm) A64_LS_REG(Wt, X
67 #define A64_LDR32(Wt, Xn, Xm) A64_LS_REG(Wt, X
68 #define A64_LDRSW(Xt, Xn, Xm) A64_LS_REG(Xt, X
69 #define A64_STR64(Xt, Xn, Xm) A64_LS_REG(Xt, X
70 #define A64_LDR64(Xt, Xn, Xm) A64_LS_REG(Xt, X
71
72 /* Load/store register (immediate offset) */
73 #define A64_LS_IMM(Rt, Rn, imm, size, type) \
74 aarch64_insn_gen_load_store_imm(Rt, Rn
75 AARCH64_INSN_SIZE_##size, \
76 AARCH64_INSN_LDST_##type##_IMM
77 #define A64_STRBI(Wt, Xn, imm) A64_LS_IMM(Wt,
78 #define A64_LDRBI(Wt, Xn, imm) A64_LS_IMM(Wt,
79 #define A64_LDRSBI(Xt, Xn, imm) A64_LS_IMM(Xt,
80 #define A64_STRHI(Wt, Xn, imm) A64_LS_IMM(Wt,
81 #define A64_LDRHI(Wt, Xn, imm) A64_LS_IMM(Wt,
82 #define A64_LDRSHI(Xt, Xn, imm) A64_LS_IMM(Xt,
83 #define A64_STR32I(Wt, Xn, imm) A64_LS_IMM(Wt,
84 #define A64_LDR32I(Wt, Xn, imm) A64_LS_IMM(Wt,
85 #define A64_LDRSWI(Xt, Xn, imm) A64_LS_IMM(Xt,
86 #define A64_STR64I(Xt, Xn, imm) A64_LS_IMM(Xt,
87 #define A64_LDR64I(Xt, Xn, imm) A64_LS_IMM(Xt,
88
89 /* LDR (literal) */
90 #define A64_LDR32LIT(Wt, offset) \
91 aarch64_insn_gen_load_literal(0, offse
92 #define A64_LDR64LIT(Xt, offset) \
93 aarch64_insn_gen_load_literal(0, offse
94
95 /* Load/store register pair */
96 #define A64_LS_PAIR(Rt, Rt2, Rn, offset, ls, t
97 aarch64_insn_gen_load_store_pair(Rt, R
98 AARCH64_INSN_VARIANT_64BIT, \
99 AARCH64_INSN_LDST_##ls##_PAIR_
100 /* Rn -= 16; Rn[0] = Rt; Rn[8] = Rt2; */
101 #define A64_PUSH(Rt, Rt2, Rn) A64_LS_PAIR(Rt,
102 /* Rt = Rn[0]; Rt2 = Rn[8]; Rn += 16; */
103 #define A64_POP(Rt, Rt2, Rn) A64_LS_PAIR(Rt,
104
105 /* Load/store exclusive */
106 #define A64_SIZE(sf) \
107 ((sf) ? AARCH64_INSN_SIZE_64 : AARCH64
108 #define A64_LSX(sf, Rt, Rn, Rs, type) \
109 aarch64_insn_gen_load_store_ex(Rt, Rn,
110 AARCH64
111 /* Rt = [Rn]; (atomic) */
112 #define A64_LDXR(sf, Rt, Rn) \
113 A64_LSX(sf, Rt, Rn, A64_ZR, LOAD_EX)
114 /* [Rn] = Rt; (atomic) Rs = [state] */
115 #define A64_STXR(sf, Rt, Rn, Rs) \
116 A64_LSX(sf, Rt, Rn, Rs, STORE_EX)
117 /* [Rn] = Rt (store release); (atomic) Rs = [s
118 #define A64_STLXR(sf, Rt, Rn, Rs) \
119 aarch64_insn_gen_load_store_ex(Rt, Rn,
120 AARCH64
121
122 /*
123 * LSE atomics
124 *
125 * ST{ADD,CLR,SET,EOR} is simply encoded as an
126 * LDD{ADD,CLR,SET,EOR} with XZR as the destin
127 */
128 #define A64_ST_OP(sf, Rn, Rs, op) \
129 aarch64_insn_gen_atomic_ld_op(A64_ZR,
130 A64_SIZE(sf), AARCH64_INSN_MEM
131 AARCH64_INSN_MEM_ORDER_NONE)
132 /* [Rn] <op>= Rs */
133 #define A64_STADD(sf, Rn, Rs) A64_ST_OP(sf, Rn
134 #define A64_STCLR(sf, Rn, Rs) A64_ST_OP(sf, Rn
135 #define A64_STEOR(sf, Rn, Rs) A64_ST_OP(sf, Rn
136 #define A64_STSET(sf, Rn, Rs) A64_ST_OP(sf, Rn
137
138 #define A64_LD_OP_AL(sf, Rt, Rn, Rs, op) \
139 aarch64_insn_gen_atomic_ld_op(Rt, Rn,
140 A64_SIZE(sf), AARCH64_INSN_MEM
141 AARCH64_INSN_MEM_ORDER_ACQREL)
142 /* Rt = [Rn] (load acquire); [Rn] <op>= Rs (st
143 #define A64_LDADDAL(sf, Rt, Rn, Rs) A64_LD_OP_
144 #define A64_LDCLRAL(sf, Rt, Rn, Rs) A64_LD_OP_
145 #define A64_LDEORAL(sf, Rt, Rn, Rs) A64_LD_OP_
146 #define A64_LDSETAL(sf, Rt, Rn, Rs) A64_LD_OP_
147 /* Rt = [Rn] (load acquire); [Rn] = Rs (store
148 #define A64_SWPAL(sf, Rt, Rn, Rs) A64_LD_OP_AL
149 /* Rs = CAS(Rn, Rs, Rt) (load acquire & store
150 #define A64_CASAL(sf, Rt, Rn, Rs) \
151 aarch64_insn_gen_cas(Rt, Rn, Rs, A64_S
152 AARCH64_INSN_MEM_ORDER_ACQREL)
153
154 /* Add/subtract (immediate) */
155 #define A64_ADDSUB_IMM(sf, Rd, Rn, imm12, type
156 aarch64_insn_gen_add_sub_imm(Rd, Rn, i
157 A64_VARIANT(sf), AARCH64_INSN_
158 /* Rd = Rn OP imm12 */
159 #define A64_ADD_I(sf, Rd, Rn, imm12) A64_ADDSU
160 #define A64_SUB_I(sf, Rd, Rn, imm12) A64_ADDSU
161 #define A64_ADDS_I(sf, Rd, Rn, imm12) \
162 A64_ADDSUB_IMM(sf, Rd, Rn, imm12, ADD_
163 #define A64_SUBS_I(sf, Rd, Rn, imm12) \
164 A64_ADDSUB_IMM(sf, Rd, Rn, imm12, SUB_
165 /* Rn + imm12; set condition flags */
166 #define A64_CMN_I(sf, Rn, imm12) A64_ADDS_I(sf
167 /* Rn - imm12; set condition flags */
168 #define A64_CMP_I(sf, Rn, imm12) A64_SUBS_I(sf
169 /* Rd = Rn */
170 #define A64_MOV(sf, Rd, Rn) A64_ADD_I(sf, Rd,
171
172 /* Bitfield move */
173 #define A64_BITFIELD(sf, Rd, Rn, immr, imms, t
174 aarch64_insn_gen_bitfield(Rd, Rn, immr
175 A64_VARIANT(sf), AARCH64_INSN_
176 /* Signed, with sign replication to left and z
177 #define A64_SBFM(sf, Rd, Rn, ir, is) A64_BITFI
178 /* Unsigned, with zeros to left and right */
179 #define A64_UBFM(sf, Rd, Rn, ir, is) A64_BITFI
180
181 /* Rd = Rn << shift */
182 #define A64_LSL(sf, Rd, Rn, shift) ({ \
183 int sz = (sf) ? 64 : 32; \
184 A64_UBFM(sf, Rd, Rn, (unsigned)-(shift
185 })
186 /* Rd = Rn >> shift */
187 #define A64_LSR(sf, Rd, Rn, shift) A64_UBFM(sf
188 /* Rd = Rn >> shift; signed */
189 #define A64_ASR(sf, Rd, Rn, shift) A64_SBFM(sf
190
191 /* Zero extend */
192 #define A64_UXTH(sf, Rd, Rn) A64_UBFM(sf, Rd,
193 #define A64_UXTW(sf, Rd, Rn) A64_UBFM(sf, Rd,
194
195 /* Sign extend */
196 #define A64_SXTB(sf, Rd, Rn) A64_SBFM(sf, Rd,
197 #define A64_SXTH(sf, Rd, Rn) A64_SBFM(sf, Rd,
198 #define A64_SXTW(sf, Rd, Rn) A64_SBFM(sf, Rd,
199
200 /* Move wide (immediate) */
201 #define A64_MOVEW(sf, Rd, imm16, shift, type)
202 aarch64_insn_gen_movewide(Rd, imm16, s
203 A64_VARIANT(sf), AARCH64_INSN_
204 /* Rd = Zeros (for MOVZ);
205 * Rd |= imm16 << shift (where shift is {0, 16
206 * Rd = ~Rd; (for MOVN); */
207 #define A64_MOVN(sf, Rd, imm16, shift) A64_MOV
208 #define A64_MOVZ(sf, Rd, imm16, shift) A64_MOV
209 #define A64_MOVK(sf, Rd, imm16, shift) A64_MOV
210
211 /* Add/subtract (shifted register) */
212 #define A64_ADDSUB_SREG(sf, Rd, Rn, Rm, type)
213 aarch64_insn_gen_add_sub_shifted_reg(R
214 A64_VARIANT(sf), AARCH64_INSN_
215 /* Rd = Rn OP Rm */
216 #define A64_ADD(sf, Rd, Rn, Rm) A64_ADDSUB_SR
217 #define A64_SUB(sf, Rd, Rn, Rm) A64_ADDSUB_SR
218 #define A64_SUBS(sf, Rd, Rn, Rm) A64_ADDSUB_SR
219 /* Rd = -Rm */
220 #define A64_NEG(sf, Rd, Rm) A64_SUB(sf, Rd, A6
221 /* Rn - Rm; set condition flags */
222 #define A64_CMP(sf, Rn, Rm) A64_SUBS(sf, A64_Z
223
224 /* Data-processing (1 source) */
225 #define A64_DATA1(sf, Rd, Rn, type) aarch64_in
226 A64_VARIANT(sf), AARCH64_INSN_DATA1_##
227 /* Rd = BSWAPx(Rn) */
228 #define A64_REV16(sf, Rd, Rn) A64_DATA1(sf, Rd
229 #define A64_REV32(sf, Rd, Rn) A64_DATA1(sf, Rd
230 #define A64_REV64(Rd, Rn) A64_DATA1(1, Rd,
231
232 /* Data-processing (2 source) */
233 /* Rd = Rn OP Rm */
234 #define A64_DATA2(sf, Rd, Rn, Rm, type) aarch6
235 A64_VARIANT(sf), AARCH64_INSN_DATA2_##
236 #define A64_UDIV(sf, Rd, Rn, Rm) A64_DATA2(sf,
237 #define A64_SDIV(sf, Rd, Rn, Rm) A64_DATA2(sf,
238 #define A64_LSLV(sf, Rd, Rn, Rm) A64_DATA2(sf,
239 #define A64_LSRV(sf, Rd, Rn, Rm) A64_DATA2(sf,
240 #define A64_ASRV(sf, Rd, Rn, Rm) A64_DATA2(sf,
241
242 /* Data-processing (3 source) */
243 /* Rd = Ra + Rn * Rm */
244 #define A64_MADD(sf, Rd, Ra, Rn, Rm) aarch64_i
245 A64_VARIANT(sf), AARCH64_INSN_DATA3_MA
246 /* Rd = Ra - Rn * Rm */
247 #define A64_MSUB(sf, Rd, Ra, Rn, Rm) aarch64_i
248 A64_VARIANT(sf), AARCH64_INSN_DATA3_MS
249 /* Rd = Rn * Rm */
250 #define A64_MUL(sf, Rd, Rn, Rm) A64_MADD(sf, R
251
252 /* Logical (shifted register) */
253 #define A64_LOGIC_SREG(sf, Rd, Rn, Rm, type) \
254 aarch64_insn_gen_logical_shifted_reg(R
255 A64_VARIANT(sf), AARCH64_INSN_
256 /* Rd = Rn OP Rm */
257 #define A64_AND(sf, Rd, Rn, Rm) A64_LOGIC_SRE
258 #define A64_ORR(sf, Rd, Rn, Rm) A64_LOGIC_SRE
259 #define A64_EOR(sf, Rd, Rn, Rm) A64_LOGIC_SRE
260 #define A64_ANDS(sf, Rd, Rn, Rm) A64_LOGIC_SRE
261 /* Rn & Rm; set condition flags */
262 #define A64_TST(sf, Rn, Rm) A64_ANDS(sf, A64_Z
263 /* Rd = ~Rm (alias of ORN with A64_ZR as Rn) *
264 #define A64_MVN(sf, Rd, Rm) \
265 A64_LOGIC_SREG(sf, Rd, A64_ZR, Rm, ORN
266
267 /* Logical (immediate) */
268 #define A64_LOGIC_IMM(sf, Rd, Rn, imm, type) (
269 u64 imm64 = (sf) ? (u64)imm : (u64)(u3
270 aarch64_insn_gen_logical_immediate(AAR
271 A64_VARIANT(sf), Rn, Rd, imm64
272 })
273 /* Rd = Rn OP imm */
274 #define A64_AND_I(sf, Rd, Rn, imm) A64_LOGIC_I
275 #define A64_ORR_I(sf, Rd, Rn, imm) A64_LOGIC_I
276 #define A64_EOR_I(sf, Rd, Rn, imm) A64_LOGIC_I
277 #define A64_ANDS_I(sf, Rd, Rn, imm) A64_LOGIC_
278 /* Rn & imm; set condition flags */
279 #define A64_TST_I(sf, Rn, imm) A64_ANDS_I(sf,
280
281 /* HINTs */
282 #define A64_HINT(x) aarch64_insn_gen_hint(x)
283
284 #define A64_PACIASP A64_HINT(AARCH64_INSN_HINT
285 #define A64_AUTIASP A64_HINT(AARCH64_INSN_HINT
286
287 /* BTI */
288 #define A64_BTI_C A64_HINT(AARCH64_INSN_HINT_
289 #define A64_BTI_J A64_HINT(AARCH64_INSN_HINT_
290 #define A64_BTI_JC A64_HINT(AARCH64_INSN_HINT_
291 #define A64_NOP A64_HINT(AARCH64_INSN_HINT_
292
293 /* DMB */
294 #define A64_DMB_ISH aarch64_insn_gen_dmb(AARCH
295
296 /* ADR */
297 #define A64_ADR(Rd, offset) \
298 aarch64_insn_gen_adr(0, offset, Rd, AA
299
300 /* MRS */
301 #define A64_MRS_TPIDR_EL1(Rt) \
302 aarch64_insn_gen_mrs(Rt, AARCH64_INSN_
303 #define A64_MRS_TPIDR_EL2(Rt) \
304 aarch64_insn_gen_mrs(Rt, AARCH64_INSN_
305 #define A64_MRS_SP_EL0(Rt) \
306 aarch64_insn_gen_mrs(Rt, AARCH64_INSN_
307
308 #endif /* _BPF_JIT_H */
309
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.