1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* 2 /*
3 * Copyright (C) 2024, Alibaba Cloud 3 * Copyright (C) 2024, Alibaba Cloud
4 */ 4 */
5 #include "internal.h" 5 #include "internal.h"
6 #include <trace/events/erofs.h> 6 #include <trace/events/erofs.h>
7 7
8 struct erofs_fileio_rq { 8 struct erofs_fileio_rq {
9 struct bio_vec bvecs[BIO_MAX_VECS]; 9 struct bio_vec bvecs[BIO_MAX_VECS];
10 struct bio bio; 10 struct bio bio;
11 struct kiocb iocb; 11 struct kiocb iocb;
12 }; 12 };
13 13
14 struct erofs_fileio { 14 struct erofs_fileio {
15 struct erofs_map_blocks map; 15 struct erofs_map_blocks map;
16 struct erofs_map_dev dev; 16 struct erofs_map_dev dev;
17 struct erofs_fileio_rq *rq; 17 struct erofs_fileio_rq *rq;
18 }; 18 };
19 19
20 static void erofs_fileio_ki_complete(struct ki 20 static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret)
21 { 21 {
22 struct erofs_fileio_rq *rq = 22 struct erofs_fileio_rq *rq =
23 container_of(iocb, str 23 container_of(iocb, struct erofs_fileio_rq, iocb);
24 struct folio_iter fi; 24 struct folio_iter fi;
25 25
26 if (ret > 0) { 26 if (ret > 0) {
27 if (ret != rq->bio.bi_iter.bi_ 27 if (ret != rq->bio.bi_iter.bi_size) {
28 bio_advance(&rq->bio, 28 bio_advance(&rq->bio, ret);
29 zero_fill_bio(&rq->bio 29 zero_fill_bio(&rq->bio);
30 } 30 }
31 ret = 0; 31 ret = 0;
32 } 32 }
33 if (rq->bio.bi_end_io) { 33 if (rq->bio.bi_end_io) {
34 rq->bio.bi_end_io(&rq->bio); 34 rq->bio.bi_end_io(&rq->bio);
35 } else { 35 } else {
36 bio_for_each_folio_all(fi, &rq 36 bio_for_each_folio_all(fi, &rq->bio) {
37 DBG_BUGON(folio_test_u 37 DBG_BUGON(folio_test_uptodate(fi.folio));
38 erofs_onlinefolio_end( 38 erofs_onlinefolio_end(fi.folio, ret);
39 } 39 }
40 } 40 }
41 bio_uninit(&rq->bio); 41 bio_uninit(&rq->bio);
42 kfree(rq); 42 kfree(rq);
43 } 43 }
44 44
45 static void erofs_fileio_rq_submit(struct erof 45 static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq)
46 { 46 {
47 struct iov_iter iter; 47 struct iov_iter iter;
48 int ret; 48 int ret;
49 49
50 if (!rq) 50 if (!rq)
51 return; 51 return;
52 rq->iocb.ki_pos = rq->bio.bi_iter.bi_s 52 rq->iocb.ki_pos = rq->bio.bi_iter.bi_sector << SECTOR_SHIFT;
53 rq->iocb.ki_ioprio = get_current_iopri 53 rq->iocb.ki_ioprio = get_current_ioprio();
54 rq->iocb.ki_complete = erofs_fileio_ki 54 rq->iocb.ki_complete = erofs_fileio_ki_complete;
55 rq->iocb.ki_flags = (rq->iocb.ki_filp- 55 rq->iocb.ki_flags = (rq->iocb.ki_filp->f_mode & FMODE_CAN_ODIRECT) ?
56 IOCB_DIRECT : 56 IOCB_DIRECT : 0;
57 iov_iter_bvec(&iter, ITER_DEST, rq->bv 57 iov_iter_bvec(&iter, ITER_DEST, rq->bvecs, rq->bio.bi_vcnt,
58 rq->bio.bi_iter.bi_size) 58 rq->bio.bi_iter.bi_size);
59 ret = vfs_iocb_iter_read(rq->iocb.ki_f 59 ret = vfs_iocb_iter_read(rq->iocb.ki_filp, &rq->iocb, &iter);
60 if (ret != -EIOCBQUEUED) 60 if (ret != -EIOCBQUEUED)
61 erofs_fileio_ki_complete(&rq-> 61 erofs_fileio_ki_complete(&rq->iocb, ret);
62 } 62 }
63 63
64 static struct erofs_fileio_rq *erofs_fileio_rq 64 static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev)
65 { 65 {
66 struct erofs_fileio_rq *rq = kzalloc(s 66 struct erofs_fileio_rq *rq = kzalloc(sizeof(*rq),
67 G 67 GFP_KERNEL | __GFP_NOFAIL);
68 68
69 bio_init(&rq->bio, NULL, rq->bvecs, BI 69 bio_init(&rq->bio, NULL, rq->bvecs, BIO_MAX_VECS, REQ_OP_READ);
70 rq->iocb.ki_filp = mdev->m_fp; 70 rq->iocb.ki_filp = mdev->m_fp;
71 return rq; 71 return rq;
72 } 72 }
73 73
74 struct bio *erofs_fileio_bio_alloc(struct erof 74 struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev)
75 { 75 {
76 return &erofs_fileio_rq_alloc(mdev)->b 76 return &erofs_fileio_rq_alloc(mdev)->bio;
77 } 77 }
78 78
79 void erofs_fileio_submit_bio(struct bio *bio) 79 void erofs_fileio_submit_bio(struct bio *bio)
80 { 80 {
81 return erofs_fileio_rq_submit(containe 81 return erofs_fileio_rq_submit(container_of(bio, struct erofs_fileio_rq,
82 82 bio));
83 } 83 }
84 84
85 static int erofs_fileio_scan_folio(struct erof 85 static int erofs_fileio_scan_folio(struct erofs_fileio *io, struct folio *folio)
86 { 86 {
87 struct inode *inode = folio_inode(foli 87 struct inode *inode = folio_inode(folio);
88 struct erofs_map_blocks *map = &io->ma 88 struct erofs_map_blocks *map = &io->map;
89 unsigned int cur = 0, end = folio_size 89 unsigned int cur = 0, end = folio_size(folio), len, attached = 0;
90 loff_t pos = folio_pos(folio), ofs; 90 loff_t pos = folio_pos(folio), ofs;
91 struct iov_iter iter; 91 struct iov_iter iter;
92 struct bio_vec bv; 92 struct bio_vec bv;
93 int err = 0; 93 int err = 0;
94 94
95 erofs_onlinefolio_init(folio); 95 erofs_onlinefolio_init(folio);
96 while (cur < end) { 96 while (cur < end) {
97 if (!in_range(pos + cur, map-> 97 if (!in_range(pos + cur, map->m_la, map->m_llen)) {
98 map->m_la = pos + cur; 98 map->m_la = pos + cur;
99 map->m_llen = end - cu 99 map->m_llen = end - cur;
100 err = erofs_map_blocks 100 err = erofs_map_blocks(inode, map);
101 if (err) 101 if (err)
102 break; 102 break;
103 } 103 }
104 104
105 ofs = folio_pos(folio) + cur - 105 ofs = folio_pos(folio) + cur - map->m_la;
106 len = min_t(loff_t, map->m_lle 106 len = min_t(loff_t, map->m_llen - ofs, end - cur);
107 if (map->m_flags & EROFS_MAP_M 107 if (map->m_flags & EROFS_MAP_META) {
108 struct erofs_buf buf = 108 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
109 void *src; 109 void *src;
110 110
111 src = erofs_read_metab 111 src = erofs_read_metabuf(&buf, inode->i_sb,
112 112 map->m_pa + ofs, EROFS_KMAP);
113 if (IS_ERR(src)) { 113 if (IS_ERR(src)) {
114 err = PTR_ERR( 114 err = PTR_ERR(src);
115 break; 115 break;
116 } 116 }
117 bvec_set_folio(&bv, fo 117 bvec_set_folio(&bv, folio, len, cur);
118 iov_iter_bvec(&iter, I 118 iov_iter_bvec(&iter, ITER_DEST, &bv, 1, len);
119 if (copy_to_iter(src, 119 if (copy_to_iter(src, len, &iter) != len) {
120 erofs_put_meta 120 erofs_put_metabuf(&buf);
121 err = -EIO; 121 err = -EIO;
122 break; 122 break;
123 } 123 }
124 erofs_put_metabuf(&buf 124 erofs_put_metabuf(&buf);
125 } else if (!(map->m_flags & ER 125 } else if (!(map->m_flags & EROFS_MAP_MAPPED)) {
126 folio_zero_segment(fol 126 folio_zero_segment(folio, cur, cur + len);
127 attached = 0; 127 attached = 0;
128 } else { 128 } else {
129 if (io->rq && (map->m_ 129 if (io->rq && (map->m_pa + ofs != io->dev.m_pa ||
130 map->m_ 130 map->m_deviceid != io->dev.m_deviceid)) {
131 io_retry: 131 io_retry:
132 erofs_fileio_r 132 erofs_fileio_rq_submit(io->rq);
133 io->rq = NULL; 133 io->rq = NULL;
134 } 134 }
135 135
136 if (!io->rq) { 136 if (!io->rq) {
137 io->dev = (str 137 io->dev = (struct erofs_map_dev) {
138 .m_pa 138 .m_pa = io->map.m_pa + ofs,
139 .m_dev 139 .m_deviceid = io->map.m_deviceid,
140 }; 140 };
141 err = erofs_ma 141 err = erofs_map_dev(inode->i_sb, &io->dev);
142 if (err) 142 if (err)
143 break; 143 break;
144 io->rq = erofs 144 io->rq = erofs_fileio_rq_alloc(&io->dev);
145 io->rq->bio.bi 145 io->rq->bio.bi_iter.bi_sector = io->dev.m_pa >> 9;
146 attached = 0; 146 attached = 0;
147 } 147 }
148 if (!attached++) 148 if (!attached++)
149 erofs_onlinefo 149 erofs_onlinefolio_split(folio);
150 if (!bio_add_folio(&io 150 if (!bio_add_folio(&io->rq->bio, folio, len, cur))
151 goto io_retry; 151 goto io_retry;
152 io->dev.m_pa += len; 152 io->dev.m_pa += len;
153 } 153 }
154 cur += len; 154 cur += len;
155 } 155 }
156 erofs_onlinefolio_end(folio, err); 156 erofs_onlinefolio_end(folio, err);
157 return err; 157 return err;
158 } 158 }
159 159
160 static int erofs_fileio_read_folio(struct file 160 static int erofs_fileio_read_folio(struct file *file, struct folio *folio)
161 { 161 {
162 struct erofs_fileio io = {}; 162 struct erofs_fileio io = {};
163 int err; 163 int err;
164 164
165 trace_erofs_read_folio(folio, true); 165 trace_erofs_read_folio(folio, true);
166 err = erofs_fileio_scan_folio(&io, fol 166 err = erofs_fileio_scan_folio(&io, folio);
167 erofs_fileio_rq_submit(io.rq); 167 erofs_fileio_rq_submit(io.rq);
168 return err; 168 return err;
169 } 169 }
170 170
171 static void erofs_fileio_readahead(struct read 171 static void erofs_fileio_readahead(struct readahead_control *rac)
172 { 172 {
173 struct inode *inode = rac->mapping->ho 173 struct inode *inode = rac->mapping->host;
174 struct erofs_fileio io = {}; 174 struct erofs_fileio io = {};
175 struct folio *folio; 175 struct folio *folio;
176 int err; 176 int err;
177 177
178 trace_erofs_readpages(inode, readahead 178 trace_erofs_readpages(inode, readahead_index(rac),
179 readahead_count( 179 readahead_count(rac), true);
180 while ((folio = readahead_folio(rac))) 180 while ((folio = readahead_folio(rac))) {
181 err = erofs_fileio_scan_folio( 181 err = erofs_fileio_scan_folio(&io, folio);
182 if (err && err != -EINTR) 182 if (err && err != -EINTR)
183 erofs_err(inode->i_sb, 183 erofs_err(inode->i_sb, "readahead error at folio %lu @ nid %llu",
184 folio->index 184 folio->index, EROFS_I(inode)->nid);
185 } 185 }
186 erofs_fileio_rq_submit(io.rq); 186 erofs_fileio_rq_submit(io.rq);
187 } 187 }
188 188
189 const struct address_space_operations erofs_fi 189 const struct address_space_operations erofs_fileio_aops = {
190 .read_folio = erofs_fileio_read_folio, 190 .read_folio = erofs_fileio_read_folio,
191 .readahead = erofs_fileio_readahead, 191 .readahead = erofs_fileio_readahead,
192 }; 192 };
193 193
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.