1 /* SPDX-License-Identifier: GPL-2.0-or-later * 1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* 2 /*
3 * vma.h 3 * vma.h
4 * 4 *
5 * Core VMA manipulation API implemented in vm 5 * Core VMA manipulation API implemented in vma.c.
6 */ 6 */
7 #ifndef __MM_VMA_H 7 #ifndef __MM_VMA_H
8 #define __MM_VMA_H 8 #define __MM_VMA_H
9 9
10 /* 10 /*
11 * VMA lock generalization 11 * VMA lock generalization
12 */ 12 */
13 struct vma_prepare { 13 struct vma_prepare {
14 struct vm_area_struct *vma; 14 struct vm_area_struct *vma;
15 struct vm_area_struct *adj_next; 15 struct vm_area_struct *adj_next;
16 struct file *file; 16 struct file *file;
17 struct address_space *mapping; 17 struct address_space *mapping;
18 struct anon_vma *anon_vma; 18 struct anon_vma *anon_vma;
19 struct vm_area_struct *insert; 19 struct vm_area_struct *insert;
20 struct vm_area_struct *remove; 20 struct vm_area_struct *remove;
21 struct vm_area_struct *remove2; 21 struct vm_area_struct *remove2;
22 }; 22 };
23 23
24 struct unlink_vma_file_batch { 24 struct unlink_vma_file_batch {
25 int count; 25 int count;
26 struct vm_area_struct *vmas[8]; 26 struct vm_area_struct *vmas[8];
27 }; 27 };
28 28
29 /* 29 /*
30 * vma munmap operation 30 * vma munmap operation
31 */ 31 */
32 struct vma_munmap_struct { 32 struct vma_munmap_struct {
33 struct vma_iterator *vmi; 33 struct vma_iterator *vmi;
34 struct vm_area_struct *vma; /* The 34 struct vm_area_struct *vma; /* The first vma to munmap */
35 struct vm_area_struct *prev; /* vma 35 struct vm_area_struct *prev; /* vma before the munmap area */
36 struct vm_area_struct *next; /* vma 36 struct vm_area_struct *next; /* vma after the munmap area */
37 struct list_head *uf; /* Use 37 struct list_head *uf; /* Userfaultfd list_head */
38 unsigned long start; /* Ali 38 unsigned long start; /* Aligned start addr (inclusive) */
39 unsigned long end; /* Ali 39 unsigned long end; /* Aligned end addr (exclusive) */
40 unsigned long unmap_start; /* Unm 40 unsigned long unmap_start; /* Unmap PTE start */
41 unsigned long unmap_end; /* Unm 41 unsigned long unmap_end; /* Unmap PTE end */
42 int vma_count; /* Num 42 int vma_count; /* Number of vmas that will be removed */
43 bool unlock; /* Unl 43 bool unlock; /* Unlock after the munmap */
44 bool clear_ptes; /* If 44 bool clear_ptes; /* If there are outstanding PTE to be cleared */
45 /* 2 byte hole */ 45 /* 2 byte hole */
46 unsigned long nr_pages; /* Num 46 unsigned long nr_pages; /* Number of pages being removed */
47 unsigned long locked_vm; /* Num 47 unsigned long locked_vm; /* Number of locked pages */
48 unsigned long nr_accounted; /* Num 48 unsigned long nr_accounted; /* Number of VM_ACCOUNT pages */
49 unsigned long exec_vm; 49 unsigned long exec_vm;
50 unsigned long stack_vm; 50 unsigned long stack_vm;
51 unsigned long data_vm; 51 unsigned long data_vm;
52 }; 52 };
53 53
54 enum vma_merge_state { 54 enum vma_merge_state {
55 VMA_MERGE_START, 55 VMA_MERGE_START,
56 VMA_MERGE_ERROR_NOMEM, 56 VMA_MERGE_ERROR_NOMEM,
57 VMA_MERGE_NOMERGE, 57 VMA_MERGE_NOMERGE,
58 VMA_MERGE_SUCCESS, 58 VMA_MERGE_SUCCESS,
59 }; 59 };
60 60
61 enum vma_merge_flags { 61 enum vma_merge_flags {
62 VMG_FLAG_DEFAULT = 0, 62 VMG_FLAG_DEFAULT = 0,
63 /* 63 /*
64 * If we can expand, simply do so. We 64 * If we can expand, simply do so. We know there is nothing to merge to
65 * the right. Does not reset state upo 65 * the right. Does not reset state upon failure to merge. The VMA
66 * iterator is assumed to be positione 66 * iterator is assumed to be positioned at the previous VMA, rather than
67 * at the gap. 67 * at the gap.
68 */ 68 */
69 VMG_FLAG_JUST_EXPAND = 1 << 0, 69 VMG_FLAG_JUST_EXPAND = 1 << 0,
70 }; 70 };
71 71
72 /* Represents a VMA merge operation. */ 72 /* Represents a VMA merge operation. */
73 struct vma_merge_struct { 73 struct vma_merge_struct {
74 struct mm_struct *mm; 74 struct mm_struct *mm;
75 struct vma_iterator *vmi; 75 struct vma_iterator *vmi;
76 pgoff_t pgoff; 76 pgoff_t pgoff;
77 struct vm_area_struct *prev; 77 struct vm_area_struct *prev;
78 struct vm_area_struct *next; /* Modifi 78 struct vm_area_struct *next; /* Modified by vma_merge(). */
79 struct vm_area_struct *vma; /* Either 79 struct vm_area_struct *vma; /* Either a new VMA or the one being modified. */
80 unsigned long start; 80 unsigned long start;
81 unsigned long end; 81 unsigned long end;
82 unsigned long flags; 82 unsigned long flags;
83 struct file *file; 83 struct file *file;
84 struct anon_vma *anon_vma; 84 struct anon_vma *anon_vma;
85 struct mempolicy *policy; 85 struct mempolicy *policy;
86 struct vm_userfaultfd_ctx uffd_ctx; 86 struct vm_userfaultfd_ctx uffd_ctx;
87 struct anon_vma_name *anon_name; 87 struct anon_vma_name *anon_name;
88 enum vma_merge_flags merge_flags; 88 enum vma_merge_flags merge_flags;
89 enum vma_merge_state state; 89 enum vma_merge_state state;
90 }; 90 };
91 91
92 static inline bool vmg_nomem(struct vma_merge_ 92 static inline bool vmg_nomem(struct vma_merge_struct *vmg)
93 { 93 {
94 return vmg->state == VMA_MERGE_ERROR_N 94 return vmg->state == VMA_MERGE_ERROR_NOMEM;
95 } 95 }
96 96
97 /* Assumes addr >= vma->vm_start. */ 97 /* Assumes addr >= vma->vm_start. */
98 static inline pgoff_t vma_pgoff_offset(struct 98 static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma,
99 unsigne 99 unsigned long addr)
100 { 100 {
101 return vma->vm_pgoff + PHYS_PFN(addr - 101 return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start);
102 } 102 }
103 103
104 #define VMG_STATE(name, mm_, vmi_, start_, end 104 #define VMG_STATE(name, mm_, vmi_, start_, end_, flags_, pgoff_) \
105 struct vma_merge_struct name = { 105 struct vma_merge_struct name = { \
106 .mm = mm_, 106 .mm = mm_, \
107 .vmi = vmi_, 107 .vmi = vmi_, \
108 .start = start_, 108 .start = start_, \
109 .end = end_, 109 .end = end_, \
110 .flags = flags_, 110 .flags = flags_, \
111 .pgoff = pgoff_, 111 .pgoff = pgoff_, \
112 .state = VMA_MERGE_START, 112 .state = VMA_MERGE_START, \
113 .merge_flags = VMG_FLAG_DEFAUL 113 .merge_flags = VMG_FLAG_DEFAULT, \
114 } 114 }
115 115
116 #define VMG_VMA_STATE(name, vmi_, prev_, vma_, 116 #define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \
117 struct vma_merge_struct name = { 117 struct vma_merge_struct name = { \
118 .mm = vma_->vm_mm, 118 .mm = vma_->vm_mm, \
119 .vmi = vmi_, 119 .vmi = vmi_, \
120 .prev = prev_, 120 .prev = prev_, \
121 .next = NULL, 121 .next = NULL, \
122 .vma = vma_, 122 .vma = vma_, \
123 .start = start_, 123 .start = start_, \
124 .end = end_, 124 .end = end_, \
125 .flags = vma_->vm_flags, 125 .flags = vma_->vm_flags, \
126 .pgoff = vma_pgoff_offset(vma_ 126 .pgoff = vma_pgoff_offset(vma_, start_), \
127 .file = vma_->vm_file, 127 .file = vma_->vm_file, \
128 .anon_vma = vma_->anon_vma, 128 .anon_vma = vma_->anon_vma, \
129 .policy = vma_policy(vma_), 129 .policy = vma_policy(vma_), \
130 .uffd_ctx = vma_->vm_userfault 130 .uffd_ctx = vma_->vm_userfaultfd_ctx, \
131 .anon_name = anon_vma_name(vma 131 .anon_name = anon_vma_name(vma_), \
132 .state = VMA_MERGE_START, 132 .state = VMA_MERGE_START, \
133 .merge_flags = VMG_FLAG_DEFAUL 133 .merge_flags = VMG_FLAG_DEFAULT, \
134 } 134 }
135 135
136 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE 136 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
137 void validate_mm(struct mm_struct *mm); 137 void validate_mm(struct mm_struct *mm);
138 #else 138 #else
139 #define validate_mm(mm) do { } while (0) 139 #define validate_mm(mm) do { } while (0)
140 #endif 140 #endif
141 141
142 /* Required for expand_downwards(). */ 142 /* Required for expand_downwards(). */
143 void anon_vma_interval_tree_pre_update_vma(str 143 void anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma);
144 144
145 /* Required for expand_downwards(). */ 145 /* Required for expand_downwards(). */
146 void anon_vma_interval_tree_post_update_vma(st 146 void anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma);
147 147
148 int vma_expand(struct vma_merge_struct *vmg); 148 int vma_expand(struct vma_merge_struct *vmg);
149 int vma_shrink(struct vma_iterator *vmi, struc 149 int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
150 unsigned long start, unsigned l 150 unsigned long start, unsigned long end, pgoff_t pgoff);
151 151
152 static inline int vma_iter_store_gfp(struct vm 152 static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
153 struct vm_area_struct 153 struct vm_area_struct *vma, gfp_t gfp)
154 154
155 { 155 {
156 if (vmi->mas.status != ma_start && 156 if (vmi->mas.status != ma_start &&
157 ((vmi->mas.index > vma->vm_start) 157 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
158 vma_iter_invalidate(vmi); 158 vma_iter_invalidate(vmi);
159 159
160 __mas_set_range(&vmi->mas, vma->vm_sta 160 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
161 mas_store_gfp(&vmi->mas, vma, gfp); 161 mas_store_gfp(&vmi->mas, vma, gfp);
162 if (unlikely(mas_is_err(&vmi->mas))) 162 if (unlikely(mas_is_err(&vmi->mas)))
163 return -ENOMEM; 163 return -ENOMEM;
164 164
165 return 0; 165 return 0;
166 } 166 }
167 167
168 #ifdef CONFIG_MMU 168 #ifdef CONFIG_MMU
169 /* 169 /*
170 * init_vma_munmap() - Initializer wrapper for 170 * init_vma_munmap() - Initializer wrapper for vma_munmap_struct
171 * @vms: The vma munmap struct 171 * @vms: The vma munmap struct
172 * @vmi: The vma iterator 172 * @vmi: The vma iterator
173 * @vma: The first vm_area_struct to munmap 173 * @vma: The first vm_area_struct to munmap
174 * @start: The aligned start address to munmap 174 * @start: The aligned start address to munmap
175 * @end: The aligned end address to munmap 175 * @end: The aligned end address to munmap
176 * @uf: The userfaultfd list_head 176 * @uf: The userfaultfd list_head
177 * @unlock: Unlock after the operation. Only 177 * @unlock: Unlock after the operation. Only unlocked on success
178 */ 178 */
179 static inline void init_vma_munmap(struct vma_ 179 static inline void init_vma_munmap(struct vma_munmap_struct *vms,
180 struct vma_iterator *vmi, stru 180 struct vma_iterator *vmi, struct vm_area_struct *vma,
181 unsigned long start, unsigned 181 unsigned long start, unsigned long end, struct list_head *uf,
182 bool unlock) 182 bool unlock)
183 { 183 {
184 vms->vmi = vmi; 184 vms->vmi = vmi;
185 vms->vma = vma; 185 vms->vma = vma;
186 if (vma) { 186 if (vma) {
187 vms->start = start; 187 vms->start = start;
188 vms->end = end; 188 vms->end = end;
189 } else { 189 } else {
190 vms->start = vms->end = 0; 190 vms->start = vms->end = 0;
191 } 191 }
192 vms->unlock = unlock; 192 vms->unlock = unlock;
193 vms->uf = uf; 193 vms->uf = uf;
194 vms->vma_count = 0; 194 vms->vma_count = 0;
195 vms->nr_pages = vms->locked_vm = vms-> 195 vms->nr_pages = vms->locked_vm = vms->nr_accounted = 0;
196 vms->exec_vm = vms->stack_vm = vms->da 196 vms->exec_vm = vms->stack_vm = vms->data_vm = 0;
197 vms->unmap_start = FIRST_USER_ADDRESS; 197 vms->unmap_start = FIRST_USER_ADDRESS;
198 vms->unmap_end = USER_PGTABLES_CEILING 198 vms->unmap_end = USER_PGTABLES_CEILING;
199 vms->clear_ptes = false; 199 vms->clear_ptes = false;
200 } 200 }
201 #endif 201 #endif
202 202
203 int vms_gather_munmap_vmas(struct vma_munmap_s 203 int vms_gather_munmap_vmas(struct vma_munmap_struct *vms,
204 struct ma_state *mas_detach); 204 struct ma_state *mas_detach);
205 205
206 void vms_complete_munmap_vmas(struct vma_munma 206 void vms_complete_munmap_vmas(struct vma_munmap_struct *vms,
207 struct ma_state *mas_detach); 207 struct ma_state *mas_detach);
208 208
209 void vms_clean_up_area(struct vma_munmap_struc 209 void vms_clean_up_area(struct vma_munmap_struct *vms,
210 struct ma_state *mas_detach); 210 struct ma_state *mas_detach);
211 211
212 /* 212 /*
213 * reattach_vmas() - Undo any munmap work and 213 * reattach_vmas() - Undo any munmap work and free resources
214 * @mas_detach: The maple state with the detac 214 * @mas_detach: The maple state with the detached maple tree
215 * 215 *
216 * Reattach any detached vmas and free up the 216 * Reattach any detached vmas and free up the maple tree used to track the vmas.
217 */ 217 */
218 static inline void reattach_vmas(struct ma_sta 218 static inline void reattach_vmas(struct ma_state *mas_detach)
219 { 219 {
220 struct vm_area_struct *vma; 220 struct vm_area_struct *vma;
221 221
222 mas_set(mas_detach, 0); 222 mas_set(mas_detach, 0);
223 mas_for_each(mas_detach, vma, ULONG_MA 223 mas_for_each(mas_detach, vma, ULONG_MAX)
224 vma_mark_detached(vma, false); 224 vma_mark_detached(vma, false);
225 225
226 __mt_destroy(mas_detach->tree); 226 __mt_destroy(mas_detach->tree);
227 } 227 }
228 228
229 /* 229 /*
230 * vms_abort_munmap_vmas() - Undo as much as p 230 * vms_abort_munmap_vmas() - Undo as much as possible from an aborted munmap()
231 * operation. 231 * operation.
232 * @vms: The vma unmap structure 232 * @vms: The vma unmap structure
233 * @mas_detach: The maple state with the detac 233 * @mas_detach: The maple state with the detached maple tree
234 * 234 *
235 * Reattach any detached vmas, free up the map 235 * Reattach any detached vmas, free up the maple tree used to track the vmas.
236 * If that's not possible because the ptes are 236 * If that's not possible because the ptes are cleared (and vm_ops->closed() may
237 * have been called), then a NULL is written o 237 * have been called), then a NULL is written over the vmas and the vmas are
238 * removed (munmap() completed). 238 * removed (munmap() completed).
239 */ 239 */
240 static inline void vms_abort_munmap_vmas(struc 240 static inline void vms_abort_munmap_vmas(struct vma_munmap_struct *vms,
241 struct ma_state *mas_detach) 241 struct ma_state *mas_detach)
242 { 242 {
243 struct ma_state *mas = &vms->vmi->mas; 243 struct ma_state *mas = &vms->vmi->mas;
244 if (!vms->nr_pages) 244 if (!vms->nr_pages)
245 return; 245 return;
246 246
247 if (vms->clear_ptes) 247 if (vms->clear_ptes)
248 return reattach_vmas(mas_detac 248 return reattach_vmas(mas_detach);
249 249
250 /* 250 /*
251 * Aborting cannot just call the vm_op 251 * Aborting cannot just call the vm_ops open() because they are often
252 * not symmetrical and state data has 252 * not symmetrical and state data has been lost. Resort to the old
253 * failure method of leaving a gap whe 253 * failure method of leaving a gap where the MAP_FIXED mapping failed.
254 */ 254 */
255 mas_set_range(mas, vms->start, vms->en 255 mas_set_range(mas, vms->start, vms->end - 1);
256 mas_store_gfp(mas, NULL, GFP_KERNEL|__ 256 mas_store_gfp(mas, NULL, GFP_KERNEL|__GFP_NOFAIL);
257 /* Clean up the insertion of the unfor 257 /* Clean up the insertion of the unfortunate gap */
258 vms_complete_munmap_vmas(vms, mas_deta 258 vms_complete_munmap_vmas(vms, mas_detach);
259 } 259 }
260 260
261 int 261 int
262 do_vmi_align_munmap(struct vma_iterator *vmi, 262 do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
263 struct mm_struct *mm, unsi 263 struct mm_struct *mm, unsigned long start,
264 unsigned long end, struct 264 unsigned long end, struct list_head *uf, bool unlock);
265 265
266 int do_vmi_munmap(struct vma_iterator *vmi, st 266 int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
267 unsigned long start, size_t 267 unsigned long start, size_t len, struct list_head *uf,
268 bool unlock); 268 bool unlock);
269 269
270 void remove_vma(struct vm_area_struct *vma, bo 270 void remove_vma(struct vm_area_struct *vma, bool unreachable);
271 271
272 void unmap_region(struct ma_state *mas, struct 272 void unmap_region(struct ma_state *mas, struct vm_area_struct *vma,
273 struct vm_area_struct *prev, s 273 struct vm_area_struct *prev, struct vm_area_struct *next);
274 274
275 /* We are about to modify the VMA's flags. */ 275 /* We are about to modify the VMA's flags. */
276 struct vm_area_struct *vma_modify_flags(struct 276 struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi,
277 struct vm_area_struct *prev, s 277 struct vm_area_struct *prev, struct vm_area_struct *vma,
278 unsigned long start, unsigned 278 unsigned long start, unsigned long end,
279 unsigned long new_flags); 279 unsigned long new_flags);
280 280
281 /* We are about to modify the VMA's flags and/ 281 /* We are about to modify the VMA's flags and/or anon_name. */
282 struct vm_area_struct 282 struct vm_area_struct
283 *vma_modify_flags_name(struct vma_iterator *vm 283 *vma_modify_flags_name(struct vma_iterator *vmi,
284 struct vm_area_struct * 284 struct vm_area_struct *prev,
285 struct vm_area_struct * 285 struct vm_area_struct *vma,
286 unsigned long start, 286 unsigned long start,
287 unsigned long end, 287 unsigned long end,
288 unsigned long new_flags 288 unsigned long new_flags,
289 struct anon_vma_name *n 289 struct anon_vma_name *new_name);
290 290
291 /* We are about to modify the VMA's memory pol 291 /* We are about to modify the VMA's memory policy. */
292 struct vm_area_struct 292 struct vm_area_struct
293 *vma_modify_policy(struct vma_iterator *vmi, 293 *vma_modify_policy(struct vma_iterator *vmi,
294 struct vm_area_struct *prev 294 struct vm_area_struct *prev,
295 struct vm_area_struct *vma, 295 struct vm_area_struct *vma,
296 unsigned long start, unsign 296 unsigned long start, unsigned long end,
297 struct mempolicy *new_pol); 297 struct mempolicy *new_pol);
298 298
299 /* We are about to modify the VMA's flags and/ 299 /* We are about to modify the VMA's flags and/or uffd context. */
300 struct vm_area_struct 300 struct vm_area_struct
301 *vma_modify_flags_uffd(struct vma_iterator *vm 301 *vma_modify_flags_uffd(struct vma_iterator *vmi,
302 struct vm_area_struct * 302 struct vm_area_struct *prev,
303 struct vm_area_struct * 303 struct vm_area_struct *vma,
304 unsigned long start, un 304 unsigned long start, unsigned long end,
305 unsigned long new_flags 305 unsigned long new_flags,
306 struct vm_userfaultfd_c 306 struct vm_userfaultfd_ctx new_ctx);
307 307
308 struct vm_area_struct *vma_merge_new_range(str 308 struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg);
309 309
310 struct vm_area_struct *vma_merge_extend(struct 310 struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
311 struct 311 struct vm_area_struct *vma,
312 unsign 312 unsigned long delta);
313 313
314 void unlink_file_vma_batch_init(struct unlink_ 314 void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb);
315 315
316 void unlink_file_vma_batch_final(struct unlink 316 void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb);
317 317
318 void unlink_file_vma_batch_add(struct unlink_v 318 void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
319 struct vm_area_ 319 struct vm_area_struct *vma);
320 320
321 void unlink_file_vma(struct vm_area_struct *vm 321 void unlink_file_vma(struct vm_area_struct *vma);
322 322
323 void vma_link_file(struct vm_area_struct *vma) 323 void vma_link_file(struct vm_area_struct *vma);
324 324
325 int vma_link(struct mm_struct *mm, struct vm_a 325 int vma_link(struct mm_struct *mm, struct vm_area_struct *vma);
326 326
327 struct vm_area_struct *copy_vma(struct vm_area 327 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
328 unsigned long addr, unsigned long len, 328 unsigned long addr, unsigned long len, pgoff_t pgoff,
329 bool *need_rmap_locks); 329 bool *need_rmap_locks);
330 330
331 struct anon_vma *find_mergeable_anon_vma(struc 331 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma);
332 332
333 bool vma_needs_dirty_tracking(struct vm_area_s 333 bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
334 bool vma_wants_writenotify(struct vm_area_stru 334 bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
335 335
336 int mm_take_all_locks(struct mm_struct *mm); 336 int mm_take_all_locks(struct mm_struct *mm);
337 void mm_drop_all_locks(struct mm_struct *mm); 337 void mm_drop_all_locks(struct mm_struct *mm);
338 338
339 static inline bool vma_wants_manual_pte_write_ 339 static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
340 { 340 {
341 /* 341 /*
342 * We want to check manually if we can 342 * We want to check manually if we can change individual PTEs writable
343 * if we can't do that automatically f 343 * if we can't do that automatically for all PTEs in a mapping. For
344 * private mappings, that's always the 344 * private mappings, that's always the case when we have write
345 * permissions as we properly have to 345 * permissions as we properly have to handle COW.
346 */ 346 */
347 if (vma->vm_flags & VM_SHARED) 347 if (vma->vm_flags & VM_SHARED)
348 return vma_wants_writenotify(v 348 return vma_wants_writenotify(vma, vma->vm_page_prot);
349 return !!(vma->vm_flags & VM_WRITE); 349 return !!(vma->vm_flags & VM_WRITE);
350 } 350 }
351 351
352 #ifdef CONFIG_MMU 352 #ifdef CONFIG_MMU
353 static inline pgprot_t vm_pgprot_modify(pgprot 353 static inline pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
354 { 354 {
355 return pgprot_modify(oldprot, vm_get_p 355 return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
356 } 356 }
357 #endif 357 #endif
358 358
359 static inline struct vm_area_struct *vma_prev_ 359 static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
360 360 unsigned long min)
361 { 361 {
362 return mas_prev(&vmi->mas, min); 362 return mas_prev(&vmi->mas, min);
363 } 363 }
364 364
365 /* 365 /*
366 * These three helpers classifies VMAs for vir 366 * These three helpers classifies VMAs for virtual memory accounting.
367 */ 367 */
368 368
369 /* 369 /*
370 * Executable code area - executable, not writ 370 * Executable code area - executable, not writable, not stack
371 */ 371 */
372 static inline bool is_exec_mapping(vm_flags_t 372 static inline bool is_exec_mapping(vm_flags_t flags)
373 { 373 {
374 return (flags & (VM_EXEC | VM_WRITE | 374 return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
375 } 375 }
376 376
377 /* 377 /*
378 * Stack area (including shadow stacks) 378 * Stack area (including shadow stacks)
379 * 379 *
380 * VM_GROWSUP / VM_GROWSDOWN VMAs are always p 380 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
381 * do_mmap() forbids all other combinations. 381 * do_mmap() forbids all other combinations.
382 */ 382 */
383 static inline bool is_stack_mapping(vm_flags_t 383 static inline bool is_stack_mapping(vm_flags_t flags)
384 { 384 {
385 return ((flags & VM_STACK) == VM_STACK 385 return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
386 } 386 }
387 387
388 /* 388 /*
389 * Data area - private, writable, not stack 389 * Data area - private, writable, not stack
390 */ 390 */
391 static inline bool is_data_mapping(vm_flags_t 391 static inline bool is_data_mapping(vm_flags_t flags)
392 { 392 {
393 return (flags & (VM_WRITE | VM_SHARED 393 return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
394 } 394 }
395 395
396 396
397 static inline void vma_iter_config(struct vma_ 397 static inline void vma_iter_config(struct vma_iterator *vmi,
398 unsigned long index, unsigned 398 unsigned long index, unsigned long last)
399 { 399 {
400 __mas_set_range(&vmi->mas, index, last 400 __mas_set_range(&vmi->mas, index, last - 1);
401 } 401 }
402 402
403 static inline void vma_iter_reset(struct vma_i 403 static inline void vma_iter_reset(struct vma_iterator *vmi)
404 { 404 {
405 mas_reset(&vmi->mas); 405 mas_reset(&vmi->mas);
406 } 406 }
407 407
408 static inline 408 static inline
409 struct vm_area_struct *vma_iter_prev_range_lim 409 struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min)
410 { 410 {
411 return mas_prev_range(&vmi->mas, min); 411 return mas_prev_range(&vmi->mas, min);
412 } 412 }
413 413
414 static inline 414 static inline
415 struct vm_area_struct *vma_iter_next_range_lim 415 struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max)
416 { 416 {
417 return mas_next_range(&vmi->mas, max); 417 return mas_next_range(&vmi->mas, max);
418 } 418 }
419 419
420 static inline int vma_iter_area_lowest(struct 420 static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min,
421 unsigne 421 unsigned long max, unsigned long size)
422 { 422 {
423 return mas_empty_area(&vmi->mas, min, 423 return mas_empty_area(&vmi->mas, min, max - 1, size);
424 } 424 }
425 425
426 static inline int vma_iter_area_highest(struct 426 static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min,
427 unsign 427 unsigned long max, unsigned long size)
428 { 428 {
429 return mas_empty_area_rev(&vmi->mas, m 429 return mas_empty_area_rev(&vmi->mas, min, max - 1, size);
430 } 430 }
431 431
432 /* 432 /*
433 * VMA Iterator functions shared between nommu 433 * VMA Iterator functions shared between nommu and mmap
434 */ 434 */
435 static inline int vma_iter_prealloc(struct vma 435 static inline int vma_iter_prealloc(struct vma_iterator *vmi,
436 struct vm_area_struct *vma) 436 struct vm_area_struct *vma)
437 { 437 {
438 return mas_preallocate(&vmi->mas, vma, 438 return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
439 } 439 }
440 440
441 static inline void vma_iter_clear(struct vma_i 441 static inline void vma_iter_clear(struct vma_iterator *vmi)
442 { 442 {
443 mas_store_prealloc(&vmi->mas, NULL); 443 mas_store_prealloc(&vmi->mas, NULL);
444 } 444 }
445 445
446 static inline struct vm_area_struct *vma_iter_ 446 static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
447 { 447 {
448 return mas_walk(&vmi->mas); 448 return mas_walk(&vmi->mas);
449 } 449 }
450 450
451 /* Store a VMA with preallocated memory */ 451 /* Store a VMA with preallocated memory */
452 static inline void vma_iter_store(struct vma_i 452 static inline void vma_iter_store(struct vma_iterator *vmi,
453 struct vm_ar 453 struct vm_area_struct *vma)
454 { 454 {
455 455
456 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE) 456 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
457 if (MAS_WARN_ON(&vmi->mas, vmi->mas.st 457 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
458 vmi->mas.index > vma-> 458 vmi->mas.index > vma->vm_start)) {
459 pr_warn("%lx > %lx\n store vma 459 pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
460 vmi->mas.index, vma->v 460 vmi->mas.index, vma->vm_start, vma->vm_start,
461 vma->vm_end, vmi->mas. 461 vma->vm_end, vmi->mas.index, vmi->mas.last);
462 } 462 }
463 if (MAS_WARN_ON(&vmi->mas, vmi->mas.st 463 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
464 vmi->mas.last < vma-> 464 vmi->mas.last < vma->vm_start)) {
465 pr_warn("%lx < %lx\nstore vma 465 pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
466 vmi->mas.last, vma->vm_ 466 vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
467 vmi->mas.index, vmi->ma 467 vmi->mas.index, vmi->mas.last);
468 } 468 }
469 #endif 469 #endif
470 470
471 if (vmi->mas.status != ma_start && 471 if (vmi->mas.status != ma_start &&
472 ((vmi->mas.index > vma->vm_start) 472 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
473 vma_iter_invalidate(vmi); 473 vma_iter_invalidate(vmi);
474 474
475 __mas_set_range(&vmi->mas, vma->vm_sta 475 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
476 mas_store_prealloc(&vmi->mas, vma); 476 mas_store_prealloc(&vmi->mas, vma);
477 } 477 }
478 478
479 static inline unsigned long vma_iter_addr(stru 479 static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
480 { 480 {
481 return vmi->mas.index; 481 return vmi->mas.index;
482 } 482 }
483 483
484 static inline unsigned long vma_iter_end(struc 484 static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
485 { 485 {
486 return vmi->mas.last + 1; 486 return vmi->mas.last + 1;
487 } 487 }
488 488
489 static inline int vma_iter_bulk_alloc(struct v 489 static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
490 unsigned 490 unsigned long count)
491 { 491 {
492 return mas_expected_entries(&vmi->mas, 492 return mas_expected_entries(&vmi->mas, count);
493 } 493 }
494 494
495 static inline 495 static inline
496 struct vm_area_struct *vma_iter_prev_range(str 496 struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi)
497 { 497 {
498 return mas_prev_range(&vmi->mas, 0); 498 return mas_prev_range(&vmi->mas, 0);
499 } 499 }
500 500
501 /* 501 /*
502 * Retrieve the next VMA and rewind the iterat 502 * Retrieve the next VMA and rewind the iterator to end of the previous VMA, or
503 * if no previous VMA, to index 0. 503 * if no previous VMA, to index 0.
504 */ 504 */
505 static inline 505 static inline
506 struct vm_area_struct *vma_iter_next_rewind(st 506 struct vm_area_struct *vma_iter_next_rewind(struct vma_iterator *vmi,
507 struct vm_area_struct **pprev) 507 struct vm_area_struct **pprev)
508 { 508 {
509 struct vm_area_struct *next = vma_next 509 struct vm_area_struct *next = vma_next(vmi);
510 struct vm_area_struct *prev = vma_prev 510 struct vm_area_struct *prev = vma_prev(vmi);
511 511
512 /* 512 /*
513 * Consider the case where no previous 513 * Consider the case where no previous VMA exists. We advance to the
514 * next VMA, skipping any gap, then re 514 * next VMA, skipping any gap, then rewind to the start of the range.
515 * 515 *
516 * If we were to unconditionally advan 516 * If we were to unconditionally advance to the next range we'd wind up
517 * at the next VMA again, so we check 517 * at the next VMA again, so we check to ensure there is a previous VMA
518 * to skip over. 518 * to skip over.
519 */ 519 */
520 if (prev) 520 if (prev)
521 vma_iter_next_range(vmi); 521 vma_iter_next_range(vmi);
522 522
523 if (pprev) 523 if (pprev)
524 *pprev = prev; 524 *pprev = prev;
525 525
526 return next; 526 return next;
527 } 527 }
528 528
529 #ifdef CONFIG_64BIT 529 #ifdef CONFIG_64BIT
530 530
531 static inline bool vma_is_sealed(struct vm_are 531 static inline bool vma_is_sealed(struct vm_area_struct *vma)
532 { 532 {
533 return (vma->vm_flags & VM_SEALED); 533 return (vma->vm_flags & VM_SEALED);
534 } 534 }
535 535
536 /* 536 /*
537 * check if a vma is sealed for modification. 537 * check if a vma is sealed for modification.
538 * return true, if modification is allowed. 538 * return true, if modification is allowed.
539 */ 539 */
540 static inline bool can_modify_vma(struct vm_ar 540 static inline bool can_modify_vma(struct vm_area_struct *vma)
541 { 541 {
542 if (unlikely(vma_is_sealed(vma))) 542 if (unlikely(vma_is_sealed(vma)))
543 return false; 543 return false;
544 544
545 return true; 545 return true;
546 } 546 }
547 547
548 bool can_modify_vma_madv(struct vm_area_struct 548 bool can_modify_vma_madv(struct vm_area_struct *vma, int behavior);
549 549
550 #else 550 #else
551 551
552 static inline bool can_modify_vma(struct vm_ar 552 static inline bool can_modify_vma(struct vm_area_struct *vma)
553 { 553 {
554 return true; 554 return true;
555 } 555 }
556 556
557 static inline bool can_modify_vma_madv(struct 557 static inline bool can_modify_vma_madv(struct vm_area_struct *vma, int behavior)
558 { 558 {
559 return true; 559 return true;
560 } 560 }
561 561
562 #endif 562 #endif
563 563
564 #endif /* __MM_VMA_H */ 564 #endif /* __MM_VMA_H */
565 565
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.