1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* I/O iterator iteration building functions.
3 *
4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #ifndef _LINUX_IOV_ITER_H
9 #define _LINUX_IOV_ITER_H
10
11 #include <linux/uio.h>
12 #include <linux/bvec.h>
13
14 typedef size_t (*iov_step_f)(void *iter_base, size_t progress, size_t len,
15 void *priv, void *priv2);
16 typedef size_t (*iov_ustep_f)(void __user *iter_base, size_t progress, size_t len,
17 void *priv, void *priv2);
18
19 /*
20 * Handle ITER_UBUF.
21 */
22 static __always_inline
23 size_t iterate_ubuf(struct iov_iter *iter, size_t len, void *priv, void *priv2,
24 iov_ustep_f step)
25 {
26 void __user *base = iter->ubuf;
27 size_t progress = 0, remain;
28
29 remain = step(base + iter->iov_offset, 0, len, priv, priv2);
30 progress = len - remain;
31 iter->iov_offset += progress;
32 iter->count -= progress;
33 return progress;
34 }
35
36 /*
37 * Handle ITER_IOVEC.
38 */
39 static __always_inline
40 size_t iterate_iovec(struct iov_iter *iter, size_t len, void *priv, void *priv2,
41 iov_ustep_f step)
42 {
43 const struct iovec *p = iter->__iov;
44 size_t progress = 0, skip = iter->iov_offset;
45
46 do {
47 size_t remain, consumed;
48 size_t part = min(len, p->iov_len - skip);
49
50 if (likely(part)) {
51 remain = step(p->iov_base + skip, progress, part, priv, priv2);
52 consumed = part - remain;
53 progress += consumed;
54 skip += consumed;
55 len -= consumed;
56 if (skip < p->iov_len)
57 break;
58 }
59 p++;
60 skip = 0;
61 } while (len);
62
63 iter->nr_segs -= p - iter->__iov;
64 iter->__iov = p;
65 iter->iov_offset = skip;
66 iter->count -= progress;
67 return progress;
68 }
69
70 /*
71 * Handle ITER_KVEC.
72 */
73 static __always_inline
74 size_t iterate_kvec(struct iov_iter *iter, size_t len, void *priv, void *priv2,
75 iov_step_f step)
76 {
77 const struct kvec *p = iter->kvec;
78 size_t progress = 0, skip = iter->iov_offset;
79
80 do {
81 size_t remain, consumed;
82 size_t part = min(len, p->iov_len - skip);
83
84 if (likely(part)) {
85 remain = step(p->iov_base + skip, progress, part, priv, priv2);
86 consumed = part - remain;
87 progress += consumed;
88 skip += consumed;
89 len -= consumed;
90 if (skip < p->iov_len)
91 break;
92 }
93 p++;
94 skip = 0;
95 } while (len);
96
97 iter->nr_segs -= p - iter->kvec;
98 iter->kvec = p;
99 iter->iov_offset = skip;
100 iter->count -= progress;
101 return progress;
102 }
103
104 /*
105 * Handle ITER_BVEC.
106 */
107 static __always_inline
108 size_t iterate_bvec(struct iov_iter *iter, size_t len, void *priv, void *priv2,
109 iov_step_f step)
110 {
111 const struct bio_vec *p = iter->bvec;
112 size_t progress = 0, skip = iter->iov_offset;
113
114 do {
115 size_t remain, consumed;
116 size_t offset = p->bv_offset + skip, part;
117 void *kaddr = kmap_local_page(p->bv_page + offset / PAGE_SIZE);
118
119 part = min3(len,
120 (size_t)(p->bv_len - skip),
121 (size_t)(PAGE_SIZE - offset % PAGE_SIZE));
122 remain = step(kaddr + offset % PAGE_SIZE, progress, part, priv, priv2);
123 kunmap_local(kaddr);
124 consumed = part - remain;
125 len -= consumed;
126 progress += consumed;
127 skip += consumed;
128 if (skip >= p->bv_len) {
129 skip = 0;
130 p++;
131 }
132 if (remain)
133 break;
134 } while (len);
135
136 iter->nr_segs -= p - iter->bvec;
137 iter->bvec = p;
138 iter->iov_offset = skip;
139 iter->count -= progress;
140 return progress;
141 }
142
143 /*
144 * Handle ITER_XARRAY.
145 */
146 static __always_inline
147 size_t iterate_xarray(struct iov_iter *iter, size_t len, void *priv, void *priv2,
148 iov_step_f step)
149 {
150 struct folio *folio;
151 size_t progress = 0;
152 loff_t start = iter->xarray_start + iter->iov_offset;
153 pgoff_t index = start / PAGE_SIZE;
154 XA_STATE(xas, iter->xarray, index);
155
156 rcu_read_lock();
157 xas_for_each(&xas, folio, ULONG_MAX) {
158 size_t remain, consumed, offset, part, flen;
159
160 if (xas_retry(&xas, folio))
161 continue;
162 if (WARN_ON(xa_is_value(folio)))
163 break;
164 if (WARN_ON(folio_test_hugetlb(folio)))
165 break;
166
167 offset = offset_in_folio(folio, start + progress);
168 flen = min(folio_size(folio) - offset, len);
169
170 while (flen) {
171 void *base = kmap_local_folio(folio, offset);
172
173 part = min_t(size_t, flen,
174 PAGE_SIZE - offset_in_page(offset));
175 remain = step(base, progress, part, priv, priv2);
176 kunmap_local(base);
177
178 consumed = part - remain;
179 progress += consumed;
180 len -= consumed;
181
182 if (remain || len == 0)
183 goto out;
184 flen -= consumed;
185 offset += consumed;
186 }
187 }
188
189 out:
190 rcu_read_unlock();
191 iter->iov_offset += progress;
192 iter->count -= progress;
193 return progress;
194 }
195
196 /*
197 * Handle ITER_DISCARD.
198 */
199 static __always_inline
200 size_t iterate_discard(struct iov_iter *iter, size_t len, void *priv, void *priv2,
201 iov_step_f step)
202 {
203 size_t progress = len;
204
205 iter->count -= progress;
206 return progress;
207 }
208
209 /**
210 * iterate_and_advance2 - Iterate over an iterator
211 * @iter: The iterator to iterate over.
212 * @len: The amount to iterate over.
213 * @priv: Data for the step functions.
214 * @priv2: More data for the step functions.
215 * @ustep: Function for UBUF/IOVEC iterators; given __user addresses.
216 * @step: Function for other iterators; given kernel addresses.
217 *
218 * Iterate over the next part of an iterator, up to the specified length. The
219 * buffer is presented in segments, which for kernel iteration are broken up by
220 * physical pages and mapped, with the mapped address being presented.
221 *
222 * Two step functions, @step and @ustep, must be provided, one for handling
223 * mapped kernel addresses and the other is given user addresses which have the
224 * potential to fault since no pinning is performed.
225 *
226 * The step functions are passed the address and length of the segment, @priv,
227 * @priv2 and the amount of data so far iterated over (which can, for example,
228 * be added to @priv to point to the right part of a second buffer). The step
229 * functions should return the amount of the segment they didn't process (ie. 0
230 * indicates complete processsing).
231 *
232 * This function returns the amount of data processed (ie. 0 means nothing was
233 * processed and the value of @len means processes to completion).
234 */
235 static __always_inline
236 size_t iterate_and_advance2(struct iov_iter *iter, size_t len, void *priv,
237 void *priv2, iov_ustep_f ustep, iov_step_f step)
238 {
239 if (unlikely(iter->count < len))
240 len = iter->count;
241 if (unlikely(!len))
242 return 0;
243
244 if (likely(iter_is_ubuf(iter)))
245 return iterate_ubuf(iter, len, priv, priv2, ustep);
246 if (likely(iter_is_iovec(iter)))
247 return iterate_iovec(iter, len, priv, priv2, ustep);
248 if (iov_iter_is_bvec(iter))
249 return iterate_bvec(iter, len, priv, priv2, step);
250 if (iov_iter_is_kvec(iter))
251 return iterate_kvec(iter, len, priv, priv2, step);
252 if (iov_iter_is_xarray(iter))
253 return iterate_xarray(iter, len, priv, priv2, step);
254 return iterate_discard(iter, len, priv, priv2, step);
255 }
256
257 /**
258 * iterate_and_advance - Iterate over an iterator
259 * @iter: The iterator to iterate over.
260 * @len: The amount to iterate over.
261 * @priv: Data for the step functions.
262 * @ustep: Function for UBUF/IOVEC iterators; given __user addresses.
263 * @step: Function for other iterators; given kernel addresses.
264 *
265 * As iterate_and_advance2(), but priv2 is always NULL.
266 */
267 static __always_inline
268 size_t iterate_and_advance(struct iov_iter *iter, size_t len, void *priv,
269 iov_ustep_f ustep, iov_step_f step)
270 {
271 return iterate_and_advance2(iter, len, priv, NULL, ustep, step);
272 }
273
274 #endif /* _LINUX_IOV_ITER_H */
275
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.