1 // SPDX-License-Identifier: GPL-2.0-only
2 // based on arch/arm/mm/alignment.c
3
4 #include <linux/compiler.h>
5 #include <linux/errno.h>
6 #include <linux/kernel.h>
7 #include <linux/init.h>
8 #include <linux/perf_event.h>
9 #include <linux/uaccess.h>
10
11 #include <asm/exception.h>
12 #include <asm/ptrace.h>
13 #include <asm/traps.h>
14
15 /*
16 * 32-bit misaligned trap handler (c) 1998 San Mehat (CCC) -July 1998
17 *
18 * Speed optimisations and better fault handling by Russell King.
19 */
20 #define CODING_BITS(i) (i & 0x0e000000)
21
22 #define LDST_P_BIT(i) (i & (1 << 24)) /* Preindex */
23 #define LDST_U_BIT(i) (i & (1 << 23)) /* Add offset */
24 #define LDST_W_BIT(i) (i & (1 << 21)) /* Writeback */
25 #define LDST_L_BIT(i) (i & (1 << 20)) /* Load */
26
27 #define LDST_P_EQ_U(i) ((((i) ^ ((i) >> 1)) & (1 << 23)) == 0)
28
29 #define LDSTHD_I_BIT(i) (i & (1 << 22)) /* double/half-word immed */
30
31 #define RN_BITS(i) ((i >> 16) & 15) /* Rn */
32 #define RD_BITS(i) ((i >> 12) & 15) /* Rd */
33 #define RM_BITS(i) (i & 15) /* Rm */
34
35 #define REGMASK_BITS(i) (i & 0xffff)
36
37 #define BAD_INSTR 0xdeadc0de
38
39 /* Thumb-2 32 bit format per ARMv7 DDI0406A A6.3, either f800h,e800h,f800h */
40 #define IS_T32(hi16) \
41 (((hi16) & 0xe000) == 0xe000 && ((hi16) & 0x1800))
42
43 union offset_union {
44 unsigned long un;
45 signed long sn;
46 };
47
48 #define TYPE_ERROR 0
49 #define TYPE_FAULT 1
50 #define TYPE_LDST 2
51 #define TYPE_DONE 3
52
53 static void
54 do_alignment_finish_ldst(unsigned long addr, u32 instr, struct pt_regs *regs,
55 union offset_union offset)
56 {
57 if (!LDST_U_BIT(instr))
58 offset.un = -offset.un;
59
60 if (!LDST_P_BIT(instr))
61 addr += offset.un;
62
63 if (!LDST_P_BIT(instr) || LDST_W_BIT(instr))
64 regs->regs[RN_BITS(instr)] = addr;
65 }
66
67 static int
68 do_alignment_ldrdstrd(unsigned long addr, u32 instr, struct pt_regs *regs)
69 {
70 unsigned int rd = RD_BITS(instr);
71 unsigned int rd2;
72 int load;
73
74 if ((instr & 0xfe000000) == 0xe8000000) {
75 /* ARMv7 Thumb-2 32-bit LDRD/STRD */
76 rd2 = (instr >> 8) & 0xf;
77 load = !!(LDST_L_BIT(instr));
78 } else if (((rd & 1) == 1) || (rd == 14)) {
79 return TYPE_ERROR;
80 } else {
81 load = ((instr & 0xf0) == 0xd0);
82 rd2 = rd + 1;
83 }
84
85 if (load) {
86 unsigned int val, val2;
87
88 if (get_user(val, (u32 __user *)addr) ||
89 get_user(val2, (u32 __user *)(addr + 4)))
90 return TYPE_FAULT;
91 regs->regs[rd] = val;
92 regs->regs[rd2] = val2;
93 } else {
94 if (put_user(regs->regs[rd], (u32 __user *)addr) ||
95 put_user(regs->regs[rd2], (u32 __user *)(addr + 4)))
96 return TYPE_FAULT;
97 }
98 return TYPE_LDST;
99 }
100
101 /*
102 * LDM/STM alignment handler.
103 *
104 * There are 4 variants of this instruction:
105 *
106 * B = rn pointer before instruction, A = rn pointer after instruction
107 * ------ increasing address ----->
108 * | | r0 | r1 | ... | rx | |
109 * PU = 01 B A
110 * PU = 11 B A
111 * PU = 00 A B
112 * PU = 10 A B
113 */
114 static int
115 do_alignment_ldmstm(unsigned long addr, u32 instr, struct pt_regs *regs)
116 {
117 unsigned int rd, rn, nr_regs, regbits;
118 unsigned long eaddr, newaddr;
119 unsigned int val;
120
121 /* count the number of registers in the mask to be transferred */
122 nr_regs = hweight16(REGMASK_BITS(instr)) * 4;
123
124 rn = RN_BITS(instr);
125 newaddr = eaddr = regs->regs[rn];
126
127 if (!LDST_U_BIT(instr))
128 nr_regs = -nr_regs;
129 newaddr += nr_regs;
130 if (!LDST_U_BIT(instr))
131 eaddr = newaddr;
132
133 if (LDST_P_EQ_U(instr)) /* U = P */
134 eaddr += 4;
135
136 for (regbits = REGMASK_BITS(instr), rd = 0; regbits;
137 regbits >>= 1, rd += 1)
138 if (regbits & 1) {
139 if (LDST_L_BIT(instr)) {
140 if (get_user(val, (u32 __user *)eaddr))
141 return TYPE_FAULT;
142 if (rd < 15)
143 regs->regs[rd] = val;
144 else
145 regs->pc = val;
146 } else {
147 /*
148 * The PC register has a bias of +8 in ARM mode
149 * and +4 in Thumb mode. This means that a read
150 * of the value of PC should account for this.
151 * Since Thumb does not permit STM instructions
152 * to refer to PC, just add 8 here.
153 */
154 val = (rd < 15) ? regs->regs[rd] : regs->pc + 8;
155 if (put_user(val, (u32 __user *)eaddr))
156 return TYPE_FAULT;
157 }
158 eaddr += 4;
159 }
160
161 if (LDST_W_BIT(instr))
162 regs->regs[rn] = newaddr;
163
164 return TYPE_DONE;
165 }
166
167 /*
168 * Convert Thumb multi-word load/store instruction forms to equivalent ARM
169 * instructions so we can reuse ARM userland alignment fault fixups for Thumb.
170 *
171 * This implementation was initially based on the algorithm found in
172 * gdb/sim/arm/thumbemu.c. It is basically just a code reduction of same
173 * to convert only Thumb ld/st instruction forms to equivalent ARM forms.
174 *
175 * NOTES:
176 * 1. Comments below refer to ARM ARM DDI0100E Thumb Instruction sections.
177 * 2. If for some reason we're passed an non-ld/st Thumb instruction to
178 * decode, we return 0xdeadc0de. This should never happen under normal
179 * circumstances but if it does, we've got other problems to deal with
180 * elsewhere and we obviously can't fix those problems here.
181 */
182
183 static unsigned long thumb2arm(u16 tinstr)
184 {
185 u32 L = (tinstr & (1<<11)) >> 11;
186
187 switch ((tinstr & 0xf800) >> 11) {
188 /* 6.6.1 Format 1: */
189 case 0xc000 >> 11: /* 7.1.51 STMIA */
190 case 0xc800 >> 11: /* 7.1.25 LDMIA */
191 {
192 u32 Rn = (tinstr & (7<<8)) >> 8;
193 u32 W = ((L<<Rn) & (tinstr&255)) ? 0 : 1<<21;
194
195 return 0xe8800000 | W | (L<<20) | (Rn<<16) |
196 (tinstr&255);
197 }
198
199 /* 6.6.1 Format 2: */
200 case 0xb000 >> 11: /* 7.1.48 PUSH */
201 case 0xb800 >> 11: /* 7.1.47 POP */
202 if ((tinstr & (3 << 9)) == 0x0400) {
203 static const u32 subset[4] = {
204 0xe92d0000, /* STMDB sp!,{registers} */
205 0xe92d4000, /* STMDB sp!,{registers,lr} */
206 0xe8bd0000, /* LDMIA sp!,{registers} */
207 0xe8bd8000 /* LDMIA sp!,{registers,pc} */
208 };
209 return subset[(L<<1) | ((tinstr & (1<<8)) >> 8)] |
210 (tinstr & 255); /* register_list */
211 }
212 fallthrough; /* for illegal instruction case */
213
214 default:
215 return BAD_INSTR;
216 }
217 }
218
219 /*
220 * Convert Thumb-2 32 bit LDM, STM, LDRD, STRD to equivalent instruction
221 * handlable by ARM alignment handler, also find the corresponding handler,
222 * so that we can reuse ARM userland alignment fault fixups for Thumb.
223 *
224 * @pinstr: original Thumb-2 instruction; returns new handlable instruction
225 * @regs: register context.
226 * @poffset: return offset from faulted addr for later writeback
227 *
228 * NOTES:
229 * 1. Comments below refer to ARMv7 DDI0406A Thumb Instruction sections.
230 * 2. Register name Rt from ARMv7 is same as Rd from ARMv6 (Rd is Rt)
231 */
232 static void *
233 do_alignment_t32_to_handler(u32 *pinstr, struct pt_regs *regs,
234 union offset_union *poffset)
235 {
236 u32 instr = *pinstr;
237 u16 tinst1 = (instr >> 16) & 0xffff;
238 u16 tinst2 = instr & 0xffff;
239
240 switch (tinst1 & 0xffe0) {
241 /* A6.3.5 Load/Store multiple */
242 case 0xe880: /* STM/STMIA/STMEA,LDM/LDMIA, PUSH/POP T2 */
243 case 0xe8a0: /* ...above writeback version */
244 case 0xe900: /* STMDB/STMFD, LDMDB/LDMEA */
245 case 0xe920: /* ...above writeback version */
246 /* no need offset decision since handler calculates it */
247 return do_alignment_ldmstm;
248
249 case 0xf840: /* POP/PUSH T3 (single register) */
250 if (RN_BITS(instr) == 13 && (tinst2 & 0x09ff) == 0x0904) {
251 u32 L = !!(LDST_L_BIT(instr));
252 const u32 subset[2] = {
253 0xe92d0000, /* STMDB sp!,{registers} */
254 0xe8bd0000, /* LDMIA sp!,{registers} */
255 };
256 *pinstr = subset[L] | (1<<RD_BITS(instr));
257 return do_alignment_ldmstm;
258 }
259 /* Else fall through for illegal instruction case */
260 break;
261
262 /* A6.3.6 Load/store double, STRD/LDRD(immed, lit, reg) */
263 case 0xe860:
264 case 0xe960:
265 case 0xe8e0:
266 case 0xe9e0:
267 poffset->un = (tinst2 & 0xff) << 2;
268 fallthrough;
269
270 case 0xe940:
271 case 0xe9c0:
272 return do_alignment_ldrdstrd;
273
274 /*
275 * No need to handle load/store instructions up to word size
276 * since ARMv6 and later CPUs can perform unaligned accesses.
277 */
278 default:
279 break;
280 }
281 return NULL;
282 }
283
284 static int alignment_get_arm(struct pt_regs *regs, __le32 __user *ip, u32 *inst)
285 {
286 __le32 instr = 0;
287 int fault;
288
289 fault = get_user(instr, ip);
290 if (fault)
291 return fault;
292
293 *inst = __le32_to_cpu(instr);
294 return 0;
295 }
296
297 static int alignment_get_thumb(struct pt_regs *regs, __le16 __user *ip, u16 *inst)
298 {
299 __le16 instr = 0;
300 int fault;
301
302 fault = get_user(instr, ip);
303 if (fault)
304 return fault;
305
306 *inst = __le16_to_cpu(instr);
307 return 0;
308 }
309
310 int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs)
311 {
312 union offset_union offset;
313 unsigned long instrptr;
314 int (*handler)(unsigned long addr, u32 instr, struct pt_regs *regs);
315 unsigned int type;
316 u32 instr = 0;
317 int isize = 4;
318 int thumb2_32b = 0;
319
320 instrptr = instruction_pointer(regs);
321
322 if (compat_thumb_mode(regs)) {
323 __le16 __user *ptr = (__le16 __user *)(instrptr & ~1);
324 u16 tinstr, tinst2;
325
326 if (alignment_get_thumb(regs, ptr, &tinstr))
327 return 1;
328
329 if (IS_T32(tinstr)) { /* Thumb-2 32-bit */
330 if (alignment_get_thumb(regs, ptr + 1, &tinst2))
331 return 1;
332 instr = ((u32)tinstr << 16) | tinst2;
333 thumb2_32b = 1;
334 } else {
335 isize = 2;
336 instr = thumb2arm(tinstr);
337 }
338 } else {
339 if (alignment_get_arm(regs, (__le32 __user *)instrptr, &instr))
340 return 1;
341 }
342
343 switch (CODING_BITS(instr)) {
344 case 0x00000000: /* 3.13.4 load/store instruction extensions */
345 if (LDSTHD_I_BIT(instr))
346 offset.un = (instr & 0xf00) >> 4 | (instr & 15);
347 else
348 offset.un = regs->regs[RM_BITS(instr)];
349
350 if ((instr & 0x001000f0) == 0x000000d0 || /* LDRD */
351 (instr & 0x001000f0) == 0x000000f0) /* STRD */
352 handler = do_alignment_ldrdstrd;
353 else
354 return 1;
355 break;
356
357 case 0x08000000: /* ldm or stm, or thumb-2 32bit instruction */
358 if (thumb2_32b) {
359 offset.un = 0;
360 handler = do_alignment_t32_to_handler(&instr, regs, &offset);
361 } else {
362 offset.un = 0;
363 handler = do_alignment_ldmstm;
364 }
365 break;
366
367 default:
368 return 1;
369 }
370
371 type = handler(addr, instr, regs);
372
373 if (type == TYPE_ERROR || type == TYPE_FAULT)
374 return 1;
375
376 if (type == TYPE_LDST)
377 do_alignment_finish_ldst(addr, instr, regs, offset);
378
379 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->pc);
380 arm64_skip_faulting_instruction(regs, isize);
381
382 return 0;
383 }
384
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.