1 // SPDX-License-Identifier: GPL-2.0-or-later 1
2 /*
3 * Copyright (C) 2024, Alibaba Cloud
4 */
5 #include "internal.h"
6 #include <trace/events/erofs.h>
7
8 struct erofs_fileio_rq {
9 struct bio_vec bvecs[BIO_MAX_VECS];
10 struct bio bio;
11 struct kiocb iocb;
12 };
13
14 struct erofs_fileio {
15 struct erofs_map_blocks map;
16 struct erofs_map_dev dev;
17 struct erofs_fileio_rq *rq;
18 };
19
20 static void erofs_fileio_ki_complete(struct ki
21 {
22 struct erofs_fileio_rq *rq =
23 container_of(iocb, str
24 struct folio_iter fi;
25
26 if (ret > 0) {
27 if (ret != rq->bio.bi_iter.bi_
28 bio_advance(&rq->bio,
29 zero_fill_bio(&rq->bio
30 }
31 ret = 0;
32 }
33 if (rq->bio.bi_end_io) {
34 rq->bio.bi_end_io(&rq->bio);
35 } else {
36 bio_for_each_folio_all(fi, &rq
37 DBG_BUGON(folio_test_u
38 erofs_onlinefolio_end(
39 }
40 }
41 bio_uninit(&rq->bio);
42 kfree(rq);
43 }
44
45 static void erofs_fileio_rq_submit(struct erof
46 {
47 struct iov_iter iter;
48 int ret;
49
50 if (!rq)
51 return;
52 rq->iocb.ki_pos = rq->bio.bi_iter.bi_s
53 rq->iocb.ki_ioprio = get_current_iopri
54 rq->iocb.ki_complete = erofs_fileio_ki
55 rq->iocb.ki_flags = (rq->iocb.ki_filp-
56 IOCB_DIRECT :
57 iov_iter_bvec(&iter, ITER_DEST, rq->bv
58 rq->bio.bi_iter.bi_size)
59 ret = vfs_iocb_iter_read(rq->iocb.ki_f
60 if (ret != -EIOCBQUEUED)
61 erofs_fileio_ki_complete(&rq->
62 }
63
64 static struct erofs_fileio_rq *erofs_fileio_rq
65 {
66 struct erofs_fileio_rq *rq = kzalloc(s
67 G
68
69 bio_init(&rq->bio, NULL, rq->bvecs, BI
70 rq->iocb.ki_filp = mdev->m_fp;
71 return rq;
72 }
73
74 struct bio *erofs_fileio_bio_alloc(struct erof
75 {
76 return &erofs_fileio_rq_alloc(mdev)->b
77 }
78
79 void erofs_fileio_submit_bio(struct bio *bio)
80 {
81 return erofs_fileio_rq_submit(containe
82
83 }
84
85 static int erofs_fileio_scan_folio(struct erof
86 {
87 struct inode *inode = folio_inode(foli
88 struct erofs_map_blocks *map = &io->ma
89 unsigned int cur = 0, end = folio_size
90 loff_t pos = folio_pos(folio), ofs;
91 struct iov_iter iter;
92 struct bio_vec bv;
93 int err = 0;
94
95 erofs_onlinefolio_init(folio);
96 while (cur < end) {
97 if (!in_range(pos + cur, map->
98 map->m_la = pos + cur;
99 map->m_llen = end - cu
100 err = erofs_map_blocks
101 if (err)
102 break;
103 }
104
105 ofs = folio_pos(folio) + cur -
106 len = min_t(loff_t, map->m_lle
107 if (map->m_flags & EROFS_MAP_M
108 struct erofs_buf buf =
109 void *src;
110
111 src = erofs_read_metab
112
113 if (IS_ERR(src)) {
114 err = PTR_ERR(
115 break;
116 }
117 bvec_set_folio(&bv, fo
118 iov_iter_bvec(&iter, I
119 if (copy_to_iter(src,
120 erofs_put_meta
121 err = -EIO;
122 break;
123 }
124 erofs_put_metabuf(&buf
125 } else if (!(map->m_flags & ER
126 folio_zero_segment(fol
127 attached = 0;
128 } else {
129 if (io->rq && (map->m_
130 map->m_
131 io_retry:
132 erofs_fileio_r
133 io->rq = NULL;
134 }
135
136 if (!io->rq) {
137 io->dev = (str
138 .m_pa
139 .m_dev
140 };
141 err = erofs_ma
142 if (err)
143 break;
144 io->rq = erofs
145 io->rq->bio.bi
146 attached = 0;
147 }
148 if (!attached++)
149 erofs_onlinefo
150 if (!bio_add_folio(&io
151 goto io_retry;
152 io->dev.m_pa += len;
153 }
154 cur += len;
155 }
156 erofs_onlinefolio_end(folio, err);
157 return err;
158 }
159
160 static int erofs_fileio_read_folio(struct file
161 {
162 struct erofs_fileio io = {};
163 int err;
164
165 trace_erofs_read_folio(folio, true);
166 err = erofs_fileio_scan_folio(&io, fol
167 erofs_fileio_rq_submit(io.rq);
168 return err;
169 }
170
171 static void erofs_fileio_readahead(struct read
172 {
173 struct inode *inode = rac->mapping->ho
174 struct erofs_fileio io = {};
175 struct folio *folio;
176 int err;
177
178 trace_erofs_readpages(inode, readahead
179 readahead_count(
180 while ((folio = readahead_folio(rac)))
181 err = erofs_fileio_scan_folio(
182 if (err && err != -EINTR)
183 erofs_err(inode->i_sb,
184 folio->index
185 }
186 erofs_fileio_rq_submit(io.rq);
187 }
188
189 const struct address_space_operations erofs_fi
190 .read_folio = erofs_fileio_read_folio,
191 .readahead = erofs_fileio_readahead,
192 };
193
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.