1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Set up the VMAs to tell the VM about the vDSO.
4 * Copyright 2007 Andi Kleen, SUSE Labs.
5 */
6
7 /*
8 * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
9 */
10
11 #include <linux/mm.h>
12 #include <linux/err.h>
13 #include <linux/sched.h>
14 #include <linux/slab.h>
15 #include <linux/init.h>
16 #include <linux/linkage.h>
17 #include <linux/random.h>
18 #include <linux/elf.h>
19 #include <asm/cacheflush.h>
20 #include <asm/spitfire.h>
21 #include <asm/vdso.h>
22 #include <asm/vvar.h>
23 #include <asm/page.h>
24
25 unsigned int __read_mostly vdso_enabled = 1;
26
27 static struct vm_special_mapping vvar_mapping = {
28 .name = "[vvar]"
29 };
30
31 #ifdef CONFIG_SPARC64
32 static struct vm_special_mapping vdso_mapping64 = {
33 .name = "[vdso]"
34 };
35 #endif
36
37 #ifdef CONFIG_COMPAT
38 static struct vm_special_mapping vdso_mapping32 = {
39 .name = "[vdso]"
40 };
41 #endif
42
43 struct vvar_data *vvar_data;
44
45 struct vdso_elfinfo32 {
46 Elf32_Ehdr *hdr;
47 Elf32_Sym *dynsym;
48 unsigned long dynsymsize;
49 const char *dynstr;
50 unsigned long text;
51 };
52
53 struct vdso_elfinfo64 {
54 Elf64_Ehdr *hdr;
55 Elf64_Sym *dynsym;
56 unsigned long dynsymsize;
57 const char *dynstr;
58 unsigned long text;
59 };
60
61 struct vdso_elfinfo {
62 union {
63 struct vdso_elfinfo32 elf32;
64 struct vdso_elfinfo64 elf64;
65 } u;
66 };
67
68 static void *one_section64(struct vdso_elfinfo64 *e, const char *name,
69 unsigned long *size)
70 {
71 const char *snames;
72 Elf64_Shdr *shdrs;
73 unsigned int i;
74
75 shdrs = (void *)e->hdr + e->hdr->e_shoff;
76 snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
77 for (i = 1; i < e->hdr->e_shnum; i++) {
78 if (!strcmp(snames+shdrs[i].sh_name, name)) {
79 if (size)
80 *size = shdrs[i].sh_size;
81 return (void *)e->hdr + shdrs[i].sh_offset;
82 }
83 }
84 return NULL;
85 }
86
87 static int find_sections64(const struct vdso_image *image, struct vdso_elfinfo *_e)
88 {
89 struct vdso_elfinfo64 *e = &_e->u.elf64;
90
91 e->hdr = image->data;
92 e->dynsym = one_section64(e, ".dynsym", &e->dynsymsize);
93 e->dynstr = one_section64(e, ".dynstr", NULL);
94
95 if (!e->dynsym || !e->dynstr) {
96 pr_err("VDSO64: Missing symbol sections.\n");
97 return -ENODEV;
98 }
99 return 0;
100 }
101
102 static Elf64_Sym *find_sym64(const struct vdso_elfinfo64 *e, const char *name)
103 {
104 unsigned int i;
105
106 for (i = 0; i < (e->dynsymsize / sizeof(Elf64_Sym)); i++) {
107 Elf64_Sym *s = &e->dynsym[i];
108 if (s->st_name == 0)
109 continue;
110 if (!strcmp(e->dynstr + s->st_name, name))
111 return s;
112 }
113 return NULL;
114 }
115
116 static int patchsym64(struct vdso_elfinfo *_e, const char *orig,
117 const char *new)
118 {
119 struct vdso_elfinfo64 *e = &_e->u.elf64;
120 Elf64_Sym *osym = find_sym64(e, orig);
121 Elf64_Sym *nsym = find_sym64(e, new);
122
123 if (!nsym || !osym) {
124 pr_err("VDSO64: Missing symbols.\n");
125 return -ENODEV;
126 }
127 osym->st_value = nsym->st_value;
128 osym->st_size = nsym->st_size;
129 osym->st_info = nsym->st_info;
130 osym->st_other = nsym->st_other;
131 osym->st_shndx = nsym->st_shndx;
132
133 return 0;
134 }
135
136 static void *one_section32(struct vdso_elfinfo32 *e, const char *name,
137 unsigned long *size)
138 {
139 const char *snames;
140 Elf32_Shdr *shdrs;
141 unsigned int i;
142
143 shdrs = (void *)e->hdr + e->hdr->e_shoff;
144 snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
145 for (i = 1; i < e->hdr->e_shnum; i++) {
146 if (!strcmp(snames+shdrs[i].sh_name, name)) {
147 if (size)
148 *size = shdrs[i].sh_size;
149 return (void *)e->hdr + shdrs[i].sh_offset;
150 }
151 }
152 return NULL;
153 }
154
155 static int find_sections32(const struct vdso_image *image, struct vdso_elfinfo *_e)
156 {
157 struct vdso_elfinfo32 *e = &_e->u.elf32;
158
159 e->hdr = image->data;
160 e->dynsym = one_section32(e, ".dynsym", &e->dynsymsize);
161 e->dynstr = one_section32(e, ".dynstr", NULL);
162
163 if (!e->dynsym || !e->dynstr) {
164 pr_err("VDSO32: Missing symbol sections.\n");
165 return -ENODEV;
166 }
167 return 0;
168 }
169
170 static Elf32_Sym *find_sym32(const struct vdso_elfinfo32 *e, const char *name)
171 {
172 unsigned int i;
173
174 for (i = 0; i < (e->dynsymsize / sizeof(Elf32_Sym)); i++) {
175 Elf32_Sym *s = &e->dynsym[i];
176 if (s->st_name == 0)
177 continue;
178 if (!strcmp(e->dynstr + s->st_name, name))
179 return s;
180 }
181 return NULL;
182 }
183
184 static int patchsym32(struct vdso_elfinfo *_e, const char *orig,
185 const char *new)
186 {
187 struct vdso_elfinfo32 *e = &_e->u.elf32;
188 Elf32_Sym *osym = find_sym32(e, orig);
189 Elf32_Sym *nsym = find_sym32(e, new);
190
191 if (!nsym || !osym) {
192 pr_err("VDSO32: Missing symbols.\n");
193 return -ENODEV;
194 }
195 osym->st_value = nsym->st_value;
196 osym->st_size = nsym->st_size;
197 osym->st_info = nsym->st_info;
198 osym->st_other = nsym->st_other;
199 osym->st_shndx = nsym->st_shndx;
200
201 return 0;
202 }
203
204 static int find_sections(const struct vdso_image *image, struct vdso_elfinfo *e,
205 bool elf64)
206 {
207 if (elf64)
208 return find_sections64(image, e);
209 else
210 return find_sections32(image, e);
211 }
212
213 static int patch_one_symbol(struct vdso_elfinfo *e, const char *orig,
214 const char *new_target, bool elf64)
215 {
216 if (elf64)
217 return patchsym64(e, orig, new_target);
218 else
219 return patchsym32(e, orig, new_target);
220 }
221
222 static int stick_patch(const struct vdso_image *image, struct vdso_elfinfo *e, bool elf64)
223 {
224 int err;
225
226 err = find_sections(image, e, elf64);
227 if (err)
228 return err;
229
230 err = patch_one_symbol(e,
231 "__vdso_gettimeofday",
232 "__vdso_gettimeofday_stick", elf64);
233 if (err)
234 return err;
235
236 return patch_one_symbol(e,
237 "__vdso_clock_gettime",
238 "__vdso_clock_gettime_stick", elf64);
239 return 0;
240 }
241
242 /*
243 * Allocate pages for the vdso and vvar, and copy in the vdso text from the
244 * kernel image.
245 */
246 static int __init init_vdso_image(const struct vdso_image *image,
247 struct vm_special_mapping *vdso_mapping,
248 bool elf64)
249 {
250 int cnpages = (image->size) / PAGE_SIZE;
251 struct page *dp, **dpp = NULL;
252 struct page *cp, **cpp = NULL;
253 struct vdso_elfinfo ei;
254 int i, dnpages = 0;
255
256 if (tlb_type != spitfire) {
257 int err = stick_patch(image, &ei, elf64);
258 if (err)
259 return err;
260 }
261
262 /*
263 * First, the vdso text. This is initialied data, an integral number of
264 * pages long.
265 */
266 if (WARN_ON(image->size % PAGE_SIZE != 0))
267 goto oom;
268
269 cpp = kcalloc(cnpages, sizeof(struct page *), GFP_KERNEL);
270 vdso_mapping->pages = cpp;
271
272 if (!cpp)
273 goto oom;
274
275 for (i = 0; i < cnpages; i++) {
276 cp = alloc_page(GFP_KERNEL);
277 if (!cp)
278 goto oom;
279 cpp[i] = cp;
280 copy_page(page_address(cp), image->data + i * PAGE_SIZE);
281 }
282
283 /*
284 * Now the vvar page. This is uninitialized data.
285 */
286
287 if (vvar_data == NULL) {
288 dnpages = (sizeof(struct vvar_data) / PAGE_SIZE) + 1;
289 if (WARN_ON(dnpages != 1))
290 goto oom;
291 dpp = kcalloc(dnpages, sizeof(struct page *), GFP_KERNEL);
292 vvar_mapping.pages = dpp;
293
294 if (!dpp)
295 goto oom;
296
297 dp = alloc_page(GFP_KERNEL);
298 if (!dp)
299 goto oom;
300
301 dpp[0] = dp;
302 vvar_data = page_address(dp);
303 memset(vvar_data, 0, PAGE_SIZE);
304
305 vvar_data->seq = 0;
306 }
307
308 return 0;
309 oom:
310 if (cpp != NULL) {
311 for (i = 0; i < cnpages; i++) {
312 if (cpp[i] != NULL)
313 __free_page(cpp[i]);
314 }
315 kfree(cpp);
316 vdso_mapping->pages = NULL;
317 }
318
319 if (dpp != NULL) {
320 for (i = 0; i < dnpages; i++) {
321 if (dpp[i] != NULL)
322 __free_page(dpp[i]);
323 }
324 kfree(dpp);
325 vvar_mapping.pages = NULL;
326 }
327
328 pr_warn("Cannot allocate vdso\n");
329 vdso_enabled = 0;
330 return -ENOMEM;
331 }
332
333 static int __init init_vdso(void)
334 {
335 int err = 0;
336 #ifdef CONFIG_SPARC64
337 err = init_vdso_image(&vdso_image_64_builtin, &vdso_mapping64, true);
338 if (err)
339 return err;
340 #endif
341
342 #ifdef CONFIG_COMPAT
343 err = init_vdso_image(&vdso_image_32_builtin, &vdso_mapping32, false);
344 #endif
345 return err;
346
347 }
348 subsys_initcall(init_vdso);
349
350 struct linux_binprm;
351
352 /* Shuffle the vdso up a bit, randomly. */
353 static unsigned long vdso_addr(unsigned long start, unsigned int len)
354 {
355 unsigned int offset;
356
357 /* This loses some more bits than a modulo, but is cheaper */
358 offset = get_random_u32_below(PTRS_PER_PTE);
359 return start + (offset << PAGE_SHIFT);
360 }
361
362 static int map_vdso(const struct vdso_image *image,
363 struct vm_special_mapping *vdso_mapping)
364 {
365 struct mm_struct *mm = current->mm;
366 struct vm_area_struct *vma;
367 unsigned long text_start, addr = 0;
368 int ret = 0;
369
370 mmap_write_lock(mm);
371
372 /*
373 * First, get an unmapped region: then randomize it, and make sure that
374 * region is free.
375 */
376 if (current->flags & PF_RANDOMIZE) {
377 addr = get_unmapped_area(NULL, 0,
378 image->size - image->sym_vvar_start,
379 0, 0);
380 if (IS_ERR_VALUE(addr)) {
381 ret = addr;
382 goto up_fail;
383 }
384 addr = vdso_addr(addr, image->size - image->sym_vvar_start);
385 }
386 addr = get_unmapped_area(NULL, addr,
387 image->size - image->sym_vvar_start, 0, 0);
388 if (IS_ERR_VALUE(addr)) {
389 ret = addr;
390 goto up_fail;
391 }
392
393 text_start = addr - image->sym_vvar_start;
394 current->mm->context.vdso = (void __user *)text_start;
395
396 /*
397 * MAYWRITE to allow gdb to COW and set breakpoints
398 */
399 vma = _install_special_mapping(mm,
400 text_start,
401 image->size,
402 VM_READ|VM_EXEC|
403 VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
404 vdso_mapping);
405
406 if (IS_ERR(vma)) {
407 ret = PTR_ERR(vma);
408 goto up_fail;
409 }
410
411 vma = _install_special_mapping(mm,
412 addr,
413 -image->sym_vvar_start,
414 VM_READ|VM_MAYREAD,
415 &vvar_mapping);
416
417 if (IS_ERR(vma)) {
418 ret = PTR_ERR(vma);
419 do_munmap(mm, text_start, image->size, NULL);
420 }
421
422 up_fail:
423 if (ret)
424 current->mm->context.vdso = NULL;
425
426 mmap_write_unlock(mm);
427 return ret;
428 }
429
430 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
431 {
432
433 if (!vdso_enabled)
434 return 0;
435
436 #if defined CONFIG_COMPAT
437 if (!(is_32bit_task()))
438 return map_vdso(&vdso_image_64_builtin, &vdso_mapping64);
439 else
440 return map_vdso(&vdso_image_32_builtin, &vdso_mapping32);
441 #else
442 return map_vdso(&vdso_image_64_builtin, &vdso_mapping64);
443 #endif
444
445 }
446
447 static __init int vdso_setup(char *s)
448 {
449 int err;
450 unsigned long val;
451
452 err = kstrtoul(s, 10, &val);
453 if (!err)
454 vdso_enabled = val;
455 return 1;
456 }
457 __setup("vdso=", vdso_setup);
458
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.