1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com>
4 * Copyright (C) 2012 Regents of the University of California
5 */
6
7 #ifndef _ASM_RISCV_PGALLOC_H
8 #define _ASM_RISCV_PGALLOC_H
9
10 #include <linux/mm.h>
11 #include <asm/sbi.h>
12 #include <asm/tlb.h>
13
14 #ifdef CONFIG_MMU
15 #define __HAVE_ARCH_PUD_ALLOC_ONE
16 #define __HAVE_ARCH_PUD_FREE
17 #include <asm-generic/pgalloc.h>
18
19 static inline void riscv_tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt)
20 {
21 if (riscv_use_sbi_for_rfence())
22 tlb_remove_ptdesc(tlb, pt);
23 else
24 tlb_remove_page_ptdesc(tlb, pt);
25 }
26
27 static inline void pmd_populate_kernel(struct mm_struct *mm,
28 pmd_t *pmd, pte_t *pte)
29 {
30 unsigned long pfn = virt_to_pfn(pte);
31
32 set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
33 }
34
35 static inline void pmd_populate(struct mm_struct *mm,
36 pmd_t *pmd, pgtable_t pte)
37 {
38 unsigned long pfn = virt_to_pfn(page_address(pte));
39
40 set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
41 }
42
43 #ifndef __PAGETABLE_PMD_FOLDED
44 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
45 {
46 unsigned long pfn = virt_to_pfn(pmd);
47
48 set_pud(pud, __pud((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
49 }
50
51 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
52 {
53 if (pgtable_l4_enabled) {
54 unsigned long pfn = virt_to_pfn(pud);
55
56 set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
57 }
58 }
59
60 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d,
61 pud_t *pud)
62 {
63 if (pgtable_l4_enabled) {
64 unsigned long pfn = virt_to_pfn(pud);
65
66 set_p4d_safe(p4d,
67 __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
68 }
69 }
70
71 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
72 {
73 if (pgtable_l5_enabled) {
74 unsigned long pfn = virt_to_pfn(p4d);
75
76 set_pgd(pgd, __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
77 }
78 }
79
80 static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd,
81 p4d_t *p4d)
82 {
83 if (pgtable_l5_enabled) {
84 unsigned long pfn = virt_to_pfn(p4d);
85
86 set_pgd_safe(pgd,
87 __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
88 }
89 }
90
91 #define pud_alloc_one pud_alloc_one
92 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
93 {
94 if (pgtable_l4_enabled)
95 return __pud_alloc_one(mm, addr);
96
97 return NULL;
98 }
99
100 #define pud_free pud_free
101 static inline void pud_free(struct mm_struct *mm, pud_t *pud)
102 {
103 if (pgtable_l4_enabled)
104 __pud_free(mm, pud);
105 }
106
107 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
108 unsigned long addr)
109 {
110 if (pgtable_l4_enabled) {
111 struct ptdesc *ptdesc = virt_to_ptdesc(pud);
112
113 pagetable_pud_dtor(ptdesc);
114 riscv_tlb_remove_ptdesc(tlb, ptdesc);
115 }
116 }
117
118 #define p4d_alloc_one p4d_alloc_one
119 static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
120 {
121 if (pgtable_l5_enabled) {
122 gfp_t gfp = GFP_PGTABLE_USER;
123
124 if (mm == &init_mm)
125 gfp = GFP_PGTABLE_KERNEL;
126 return (p4d_t *)get_zeroed_page(gfp);
127 }
128
129 return NULL;
130 }
131
132 static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d)
133 {
134 BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
135 free_page((unsigned long)p4d);
136 }
137
138 #define p4d_free p4d_free
139 static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
140 {
141 if (pgtable_l5_enabled)
142 __p4d_free(mm, p4d);
143 }
144
145 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
146 unsigned long addr)
147 {
148 if (pgtable_l5_enabled)
149 riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d));
150 }
151 #endif /* __PAGETABLE_PMD_FOLDED */
152
153 static inline void sync_kernel_mappings(pgd_t *pgd)
154 {
155 memcpy(pgd + USER_PTRS_PER_PGD,
156 init_mm.pgd + USER_PTRS_PER_PGD,
157 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
158 }
159
160 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
161 {
162 pgd_t *pgd;
163
164 pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
165 if (likely(pgd != NULL)) {
166 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
167 /* Copy kernel mappings */
168 sync_kernel_mappings(pgd);
169 }
170 return pgd;
171 }
172
173 #ifndef __PAGETABLE_PMD_FOLDED
174
175 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
176 unsigned long addr)
177 {
178 struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
179
180 pagetable_pmd_dtor(ptdesc);
181 riscv_tlb_remove_ptdesc(tlb, ptdesc);
182 }
183
184 #endif /* __PAGETABLE_PMD_FOLDED */
185
186 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
187 unsigned long addr)
188 {
189 struct ptdesc *ptdesc = page_ptdesc(pte);
190
191 pagetable_pte_dtor(ptdesc);
192 riscv_tlb_remove_ptdesc(tlb, ptdesc);
193 }
194 #endif /* CONFIG_MMU */
195
196 #endif /* _ASM_RISCV_PGALLOC_H */
197
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.