>> 1 // SPDX-License-Identifier: GPL-2.0
1 /* 2 /*
2 * MIPS Huge TLB Page Support for Kernel. !! 3 * SPARC64 Huge TLB page support.
3 * 4 *
4 * This file is subject to the terms and condi !! 5 * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
5 * License. See the file "COPYING" in the mai <<
6 * for more details. <<
7 * <<
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@ <<
9 * Copyright 2005, Embedded Alley Solutions, I <<
10 * Matt Porter <mporter@embeddedalley.com> <<
11 * Copyright (C) 2008, 2009 Cavium Networks, I <<
12 */ 6 */
13 7
14 #include <linux/fs.h> 8 #include <linux/fs.h>
15 #include <linux/mm.h> 9 #include <linux/mm.h>
>> 10 #include <linux/sched/mm.h>
16 #include <linux/hugetlb.h> 11 #include <linux/hugetlb.h>
17 #include <linux/pagemap.h> 12 #include <linux/pagemap.h>
18 #include <linux/err.h> <<
19 #include <linux/sysctl.h> 13 #include <linux/sysctl.h>
>> 14
20 #include <asm/mman.h> 15 #include <asm/mman.h>
>> 16 #include <asm/pgalloc.h>
21 #include <asm/tlb.h> 17 #include <asm/tlb.h>
22 #include <asm/tlbflush.h> 18 #include <asm/tlbflush.h>
>> 19 #include <asm/cacheflush.h>
>> 20 #include <asm/mmu_context.h>
>> 21
>> 22 /* Slightly simplified from the non-hugepage variant because by
>> 23 * definition we don't have to worry about any page coloring stuff
>> 24 */
>> 25
>> 26 static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
>> 27 unsigned long addr,
>> 28 unsigned long len,
>> 29 unsigned long pgoff,
>> 30 unsigned long flags)
>> 31 {
>> 32 struct hstate *h = hstate_file(filp);
>> 33 unsigned long task_size = TASK_SIZE;
>> 34 struct vm_unmapped_area_info info = {};
>> 35
>> 36 if (test_thread_flag(TIF_32BIT))
>> 37 task_size = STACK_TOP32;
>> 38
>> 39 info.length = len;
>> 40 info.low_limit = TASK_UNMAPPED_BASE;
>> 41 info.high_limit = min(task_size, VA_EXCLUDE_START);
>> 42 info.align_mask = PAGE_MASK & ~huge_page_mask(h);
>> 43 addr = vm_unmapped_area(&info);
>> 44
>> 45 if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {
>> 46 VM_BUG_ON(addr != -ENOMEM);
>> 47 info.low_limit = VA_EXCLUDE_END;
>> 48 info.high_limit = task_size;
>> 49 addr = vm_unmapped_area(&info);
>> 50 }
>> 51
>> 52 return addr;
>> 53 }
>> 54
>> 55 static unsigned long
>> 56 hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
>> 57 const unsigned long len,
>> 58 const unsigned long pgoff,
>> 59 const unsigned long flags)
>> 60 {
>> 61 struct hstate *h = hstate_file(filp);
>> 62 struct mm_struct *mm = current->mm;
>> 63 unsigned long addr = addr0;
>> 64 struct vm_unmapped_area_info info = {};
>> 65
>> 66 /* This should only ever run for 32-bit processes. */
>> 67 BUG_ON(!test_thread_flag(TIF_32BIT));
>> 68
>> 69 info.flags = VM_UNMAPPED_AREA_TOPDOWN;
>> 70 info.length = len;
>> 71 info.low_limit = PAGE_SIZE;
>> 72 info.high_limit = mm->mmap_base;
>> 73 info.align_mask = PAGE_MASK & ~huge_page_mask(h);
>> 74 addr = vm_unmapped_area(&info);
>> 75
>> 76 /*
>> 77 * A failed mmap() very likely causes application failure,
>> 78 * so fall back to the bottom-up function here. This scenario
>> 79 * can happen with large stack limits and large mmap()
>> 80 * allocations.
>> 81 */
>> 82 if (addr & ~PAGE_MASK) {
>> 83 VM_BUG_ON(addr != -ENOMEM);
>> 84 info.flags = 0;
>> 85 info.low_limit = TASK_UNMAPPED_BASE;
>> 86 info.high_limit = STACK_TOP32;
>> 87 addr = vm_unmapped_area(&info);
>> 88 }
>> 89
>> 90 return addr;
>> 91 }
>> 92
>> 93 unsigned long
>> 94 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
>> 95 unsigned long len, unsigned long pgoff, unsigned long flags)
>> 96 {
>> 97 struct hstate *h = hstate_file(file);
>> 98 struct mm_struct *mm = current->mm;
>> 99 struct vm_area_struct *vma;
>> 100 unsigned long task_size = TASK_SIZE;
>> 101
>> 102 if (test_thread_flag(TIF_32BIT))
>> 103 task_size = STACK_TOP32;
>> 104
>> 105 if (len & ~huge_page_mask(h))
>> 106 return -EINVAL;
>> 107 if (len > task_size)
>> 108 return -ENOMEM;
>> 109
>> 110 if (flags & MAP_FIXED) {
>> 111 if (prepare_hugepage_range(file, addr, len))
>> 112 return -EINVAL;
>> 113 return addr;
>> 114 }
>> 115
>> 116 if (addr) {
>> 117 addr = ALIGN(addr, huge_page_size(h));
>> 118 vma = find_vma(mm, addr);
>> 119 if (task_size - len >= addr &&
>> 120 (!vma || addr + len <= vm_start_gap(vma)))
>> 121 return addr;
>> 122 }
>> 123 if (!test_bit(MMF_TOPDOWN, &mm->flags))
>> 124 return hugetlb_get_unmapped_area_bottomup(file, addr, len,
>> 125 pgoff, flags);
>> 126 else
>> 127 return hugetlb_get_unmapped_area_topdown(file, addr, len,
>> 128 pgoff, flags);
>> 129 }
>> 130
>> 131 static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
>> 132 {
>> 133 return entry;
>> 134 }
>> 135
>> 136 static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
>> 137 {
>> 138 unsigned long hugepage_size = _PAGE_SZ4MB_4V;
>> 139
>> 140 pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V;
>> 141
>> 142 switch (shift) {
>> 143 case HPAGE_16GB_SHIFT:
>> 144 hugepage_size = _PAGE_SZ16GB_4V;
>> 145 pte_val(entry) |= _PAGE_PUD_HUGE;
>> 146 break;
>> 147 case HPAGE_2GB_SHIFT:
>> 148 hugepage_size = _PAGE_SZ2GB_4V;
>> 149 pte_val(entry) |= _PAGE_PMD_HUGE;
>> 150 break;
>> 151 case HPAGE_256MB_SHIFT:
>> 152 hugepage_size = _PAGE_SZ256MB_4V;
>> 153 pte_val(entry) |= _PAGE_PMD_HUGE;
>> 154 break;
>> 155 case HPAGE_SHIFT:
>> 156 pte_val(entry) |= _PAGE_PMD_HUGE;
>> 157 break;
>> 158 case HPAGE_64K_SHIFT:
>> 159 hugepage_size = _PAGE_SZ64K_4V;
>> 160 break;
>> 161 default:
>> 162 WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
>> 163 }
>> 164
>> 165 pte_val(entry) = pte_val(entry) | hugepage_size;
>> 166 return entry;
>> 167 }
>> 168
>> 169 static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift)
>> 170 {
>> 171 if (tlb_type == hypervisor)
>> 172 return sun4v_hugepage_shift_to_tte(entry, shift);
>> 173 else
>> 174 return sun4u_hugepage_shift_to_tte(entry, shift);
>> 175 }
>> 176
>> 177 pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
>> 178 {
>> 179 pte_t pte;
>> 180
>> 181 entry = pte_mkhuge(entry);
>> 182 pte = hugepage_shift_to_tte(entry, shift);
>> 183
>> 184 #ifdef CONFIG_SPARC64
>> 185 /* If this vma has ADI enabled on it, turn on TTE.mcd
>> 186 */
>> 187 if (flags & VM_SPARC_ADI)
>> 188 return pte_mkmcd(pte);
>> 189 else
>> 190 return pte_mknotmcd(pte);
>> 191 #else
>> 192 return pte;
>> 193 #endif
>> 194 }
>> 195
>> 196 static unsigned int sun4v_huge_tte_to_shift(pte_t entry)
>> 197 {
>> 198 unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V;
>> 199 unsigned int shift;
>> 200
>> 201 switch (tte_szbits) {
>> 202 case _PAGE_SZ16GB_4V:
>> 203 shift = HPAGE_16GB_SHIFT;
>> 204 break;
>> 205 case _PAGE_SZ2GB_4V:
>> 206 shift = HPAGE_2GB_SHIFT;
>> 207 break;
>> 208 case _PAGE_SZ256MB_4V:
>> 209 shift = HPAGE_256MB_SHIFT;
>> 210 break;
>> 211 case _PAGE_SZ4MB_4V:
>> 212 shift = REAL_HPAGE_SHIFT;
>> 213 break;
>> 214 case _PAGE_SZ64K_4V:
>> 215 shift = HPAGE_64K_SHIFT;
>> 216 break;
>> 217 default:
>> 218 shift = PAGE_SHIFT;
>> 219 break;
>> 220 }
>> 221 return shift;
>> 222 }
>> 223
>> 224 static unsigned int sun4u_huge_tte_to_shift(pte_t entry)
>> 225 {
>> 226 unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U;
>> 227 unsigned int shift;
>> 228
>> 229 switch (tte_szbits) {
>> 230 case _PAGE_SZ256MB_4U:
>> 231 shift = HPAGE_256MB_SHIFT;
>> 232 break;
>> 233 case _PAGE_SZ4MB_4U:
>> 234 shift = REAL_HPAGE_SHIFT;
>> 235 break;
>> 236 case _PAGE_SZ64K_4U:
>> 237 shift = HPAGE_64K_SHIFT;
>> 238 break;
>> 239 default:
>> 240 shift = PAGE_SHIFT;
>> 241 break;
>> 242 }
>> 243 return shift;
>> 244 }
>> 245
>> 246 static unsigned long tte_to_shift(pte_t entry)
>> 247 {
>> 248 if (tlb_type == hypervisor)
>> 249 return sun4v_huge_tte_to_shift(entry);
>> 250
>> 251 return sun4u_huge_tte_to_shift(entry);
>> 252 }
>> 253
>> 254 static unsigned int huge_tte_to_shift(pte_t entry)
>> 255 {
>> 256 unsigned long shift = tte_to_shift(entry);
>> 257
>> 258 if (shift == PAGE_SHIFT)
>> 259 WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n",
>> 260 pte_val(entry));
>> 261
>> 262 return shift;
>> 263 }
>> 264
>> 265 static unsigned long huge_tte_to_size(pte_t pte)
>> 266 {
>> 267 unsigned long size = 1UL << huge_tte_to_shift(pte);
>> 268
>> 269 if (size == REAL_HPAGE_SIZE)
>> 270 size = HPAGE_SIZE;
>> 271 return size;
>> 272 }
>> 273
>> 274 unsigned long pud_leaf_size(pud_t pud) { return 1UL << tte_to_shift(*(pte_t *)&pud); }
>> 275 unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift(*(pte_t *)&pmd); }
>> 276 unsigned long pte_leaf_size(pte_t pte) { return 1UL << tte_to_shift(pte); }
23 277
24 pte_t *huge_pte_alloc(struct mm_struct *mm, st 278 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
25 unsigned long addr, unsi !! 279 unsigned long addr, unsigned long sz)
26 { 280 {
27 pgd_t *pgd; 281 pgd_t *pgd;
28 p4d_t *p4d; 282 p4d_t *p4d;
29 pud_t *pud; 283 pud_t *pud;
30 pte_t *pte = NULL; !! 284 pmd_t *pmd;
31 285
32 pgd = pgd_offset(mm, addr); 286 pgd = pgd_offset(mm, addr);
33 p4d = p4d_alloc(mm, pgd, addr); !! 287 p4d = p4d_offset(pgd, addr);
34 pud = pud_alloc(mm, p4d, addr); 288 pud = pud_alloc(mm, p4d, addr);
35 if (pud) !! 289 if (!pud)
36 pte = (pte_t *)pmd_alloc(mm, p !! 290 return NULL;
37 !! 291 if (sz >= PUD_SIZE)
38 return pte; !! 292 return (pte_t *)pud;
>> 293 pmd = pmd_alloc(mm, pud, addr);
>> 294 if (!pmd)
>> 295 return NULL;
>> 296 if (sz >= PMD_SIZE)
>> 297 return (pte_t *)pmd;
>> 298 return pte_alloc_huge(mm, pmd, addr);
39 } 299 }
40 300
41 pte_t *huge_pte_offset(struct mm_struct *mm, u !! 301 pte_t *huge_pte_offset(struct mm_struct *mm,
42 unsigned long sz) !! 302 unsigned long addr, unsigned long sz)
43 { 303 {
44 pgd_t *pgd; 304 pgd_t *pgd;
45 p4d_t *p4d; 305 p4d_t *p4d;
46 pud_t *pud; 306 pud_t *pud;
47 pmd_t *pmd = NULL; !! 307 pmd_t *pmd;
48 308
49 pgd = pgd_offset(mm, addr); 309 pgd = pgd_offset(mm, addr);
50 if (pgd_present(*pgd)) { !! 310 if (pgd_none(*pgd))
51 p4d = p4d_offset(pgd, addr); !! 311 return NULL;
52 if (p4d_present(*p4d)) { !! 312 p4d = p4d_offset(pgd, addr);
53 pud = pud_offset(p4d, !! 313 if (p4d_none(*p4d))
54 if (pud_present(*pud)) !! 314 return NULL;
55 pmd = pmd_offs !! 315 pud = pud_offset(p4d, addr);
56 } !! 316 if (pud_none(*pud))
>> 317 return NULL;
>> 318 if (is_hugetlb_pud(*pud))
>> 319 return (pte_t *)pud;
>> 320 pmd = pmd_offset(pud, addr);
>> 321 if (pmd_none(*pmd))
>> 322 return NULL;
>> 323 if (is_hugetlb_pmd(*pmd))
>> 324 return (pte_t *)pmd;
>> 325 return pte_offset_huge(pmd, addr);
>> 326 }
>> 327
>> 328 void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
>> 329 pte_t *ptep, pte_t entry)
>> 330 {
>> 331 unsigned int nptes, orig_shift, shift;
>> 332 unsigned long i, size;
>> 333 pte_t orig;
>> 334
>> 335 size = huge_tte_to_size(entry);
>> 336
>> 337 shift = PAGE_SHIFT;
>> 338 if (size >= PUD_SIZE)
>> 339 shift = PUD_SHIFT;
>> 340 else if (size >= PMD_SIZE)
>> 341 shift = PMD_SHIFT;
>> 342 else
>> 343 shift = PAGE_SHIFT;
>> 344
>> 345 nptes = size >> shift;
>> 346
>> 347 if (!pte_present(*ptep) && pte_present(entry))
>> 348 mm->context.hugetlb_pte_count += nptes;
>> 349
>> 350 addr &= ~(size - 1);
>> 351 orig = *ptep;
>> 352 orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig);
>> 353
>> 354 for (i = 0; i < nptes; i++)
>> 355 ptep[i] = __pte(pte_val(entry) + (i << shift));
>> 356
>> 357 maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);
>> 358 /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
>> 359 if (size == HPAGE_SIZE)
>> 360 maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,
>> 361 orig_shift);
>> 362 }
>> 363
>> 364 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
>> 365 pte_t *ptep, pte_t entry, unsigned long sz)
>> 366 {
>> 367 __set_huge_pte_at(mm, addr, ptep, entry);
>> 368 }
>> 369
>> 370 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
>> 371 pte_t *ptep)
>> 372 {
>> 373 unsigned int i, nptes, orig_shift, shift;
>> 374 unsigned long size;
>> 375 pte_t entry;
>> 376
>> 377 entry = *ptep;
>> 378 size = huge_tte_to_size(entry);
>> 379
>> 380 shift = PAGE_SHIFT;
>> 381 if (size >= PUD_SIZE)
>> 382 shift = PUD_SHIFT;
>> 383 else if (size >= PMD_SIZE)
>> 384 shift = PMD_SHIFT;
>> 385 else
>> 386 shift = PAGE_SHIFT;
>> 387
>> 388 nptes = size >> shift;
>> 389 orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry);
>> 390
>> 391 if (pte_present(entry))
>> 392 mm->context.hugetlb_pte_count -= nptes;
>> 393
>> 394 addr &= ~(size - 1);
>> 395 for (i = 0; i < nptes; i++)
>> 396 ptep[i] = __pte(0UL);
>> 397
>> 398 maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);
>> 399 /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
>> 400 if (size == HPAGE_SIZE)
>> 401 maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,
>> 402 orig_shift);
>> 403
>> 404 return entry;
>> 405 }
>> 406
>> 407 static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
>> 408 unsigned long addr)
>> 409 {
>> 410 pgtable_t token = pmd_pgtable(*pmd);
>> 411
>> 412 pmd_clear(pmd);
>> 413 pte_free_tlb(tlb, token, addr);
>> 414 mm_dec_nr_ptes(tlb->mm);
>> 415 }
>> 416
>> 417 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
>> 418 unsigned long addr, unsigned long end,
>> 419 unsigned long floor, unsigned long ceiling)
>> 420 {
>> 421 pmd_t *pmd;
>> 422 unsigned long next;
>> 423 unsigned long start;
>> 424
>> 425 start = addr;
>> 426 pmd = pmd_offset(pud, addr);
>> 427 do {
>> 428 next = pmd_addr_end(addr, end);
>> 429 if (pmd_none(*pmd))
>> 430 continue;
>> 431 if (is_hugetlb_pmd(*pmd))
>> 432 pmd_clear(pmd);
>> 433 else
>> 434 hugetlb_free_pte_range(tlb, pmd, addr);
>> 435 } while (pmd++, addr = next, addr != end);
>> 436
>> 437 start &= PUD_MASK;
>> 438 if (start < floor)
>> 439 return;
>> 440 if (ceiling) {
>> 441 ceiling &= PUD_MASK;
>> 442 if (!ceiling)
>> 443 return;
57 } 444 }
58 return (pte_t *) pmd; !! 445 if (end - 1 > ceiling - 1)
>> 446 return;
>> 447
>> 448 pmd = pmd_offset(pud, start);
>> 449 pud_clear(pud);
>> 450 pmd_free_tlb(tlb, pmd, start);
>> 451 mm_dec_nr_pmds(tlb->mm);
>> 452 }
>> 453
>> 454 static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
>> 455 unsigned long addr, unsigned long end,
>> 456 unsigned long floor, unsigned long ceiling)
>> 457 {
>> 458 pud_t *pud;
>> 459 unsigned long next;
>> 460 unsigned long start;
>> 461
>> 462 start = addr;
>> 463 pud = pud_offset(p4d, addr);
>> 464 do {
>> 465 next = pud_addr_end(addr, end);
>> 466 if (pud_none_or_clear_bad(pud))
>> 467 continue;
>> 468 if (is_hugetlb_pud(*pud))
>> 469 pud_clear(pud);
>> 470 else
>> 471 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
>> 472 ceiling);
>> 473 } while (pud++, addr = next, addr != end);
>> 474
>> 475 start &= PGDIR_MASK;
>> 476 if (start < floor)
>> 477 return;
>> 478 if (ceiling) {
>> 479 ceiling &= PGDIR_MASK;
>> 480 if (!ceiling)
>> 481 return;
>> 482 }
>> 483 if (end - 1 > ceiling - 1)
>> 484 return;
>> 485
>> 486 pud = pud_offset(p4d, start);
>> 487 p4d_clear(p4d);
>> 488 pud_free_tlb(tlb, pud, start);
>> 489 mm_dec_nr_puds(tlb->mm);
>> 490 }
>> 491
>> 492 void hugetlb_free_pgd_range(struct mmu_gather *tlb,
>> 493 unsigned long addr, unsigned long end,
>> 494 unsigned long floor, unsigned long ceiling)
>> 495 {
>> 496 pgd_t *pgd;
>> 497 p4d_t *p4d;
>> 498 unsigned long next;
>> 499
>> 500 addr &= PMD_MASK;
>> 501 if (addr < floor) {
>> 502 addr += PMD_SIZE;
>> 503 if (!addr)
>> 504 return;
>> 505 }
>> 506 if (ceiling) {
>> 507 ceiling &= PMD_MASK;
>> 508 if (!ceiling)
>> 509 return;
>> 510 }
>> 511 if (end - 1 > ceiling - 1)
>> 512 end -= PMD_SIZE;
>> 513 if (addr > end - 1)
>> 514 return;
>> 515
>> 516 pgd = pgd_offset(tlb->mm, addr);
>> 517 p4d = p4d_offset(pgd, addr);
>> 518 do {
>> 519 next = p4d_addr_end(addr, end);
>> 520 if (p4d_none_or_clear_bad(p4d))
>> 521 continue;
>> 522 hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
>> 523 } while (p4d++, addr = next, addr != end);
59 } 524 }
60 525
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.