1 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
2 /* 2 /*
3 * Copyright (C) 2017-2018 HUAWEI, Inc. 3 * Copyright (C) 2017-2018 HUAWEI, Inc.
4 * https://www.huawei.com/ 4 * https://www.huawei.com/
5 * Copyright (C) 2021, Alibaba Cloud 5 * Copyright (C) 2021, Alibaba Cloud
6 */ 6 */
7 #include "internal.h" 7 #include "internal.h"
>> 8 #include <linux/prefetch.h>
8 #include <linux/sched/mm.h> 9 #include <linux/sched/mm.h>
>> 10 #include <linux/dax.h>
9 #include <trace/events/erofs.h> 11 #include <trace/events/erofs.h>
10 12
11 void erofs_unmap_metabuf(struct erofs_buf *buf 13 void erofs_unmap_metabuf(struct erofs_buf *buf)
12 { 14 {
13 if (buf->kmap_type == EROFS_KMAP) 15 if (buf->kmap_type == EROFS_KMAP)
14 kunmap_local(buf->base); 16 kunmap_local(buf->base);
15 buf->base = NULL; 17 buf->base = NULL;
16 buf->kmap_type = EROFS_NO_KMAP; 18 buf->kmap_type = EROFS_NO_KMAP;
17 } 19 }
18 20
19 void erofs_put_metabuf(struct erofs_buf *buf) 21 void erofs_put_metabuf(struct erofs_buf *buf)
20 { 22 {
21 if (!buf->page) 23 if (!buf->page)
22 return; 24 return;
23 erofs_unmap_metabuf(buf); 25 erofs_unmap_metabuf(buf);
24 folio_put(page_folio(buf->page)); !! 26 put_page(buf->page);
25 buf->page = NULL; 27 buf->page = NULL;
26 } 28 }
27 29
28 void *erofs_bread(struct erofs_buf *buf, erofs !! 30 void *erofs_bread(struct erofs_buf *buf, struct inode *inode,
29 enum erofs_kmap_type type) !! 31 erofs_blk_t blkaddr, enum erofs_kmap_type type)
30 { 32 {
>> 33 struct address_space *const mapping = inode->i_mapping;
>> 34 erofs_off_t offset = blknr_to_addr(blkaddr);
31 pgoff_t index = offset >> PAGE_SHIFT; 35 pgoff_t index = offset >> PAGE_SHIFT;
32 struct folio *folio = NULL; !! 36 struct page *page = buf->page;
>> 37 struct folio *folio;
>> 38 unsigned int nofs_flag;
33 39
34 if (buf->page) { !! 40 if (!page || page->index != index) {
35 folio = page_folio(buf->page); <<
36 if (folio_file_page(folio, ind <<
37 erofs_unmap_metabuf(bu <<
38 } <<
39 if (!folio || !folio_contains(folio, i <<
40 erofs_put_metabuf(buf); 41 erofs_put_metabuf(buf);
41 folio = read_mapping_folio(buf !! 42
>> 43 nofs_flag = memalloc_nofs_save();
>> 44 folio = read_cache_folio(mapping, index, NULL, NULL);
>> 45 memalloc_nofs_restore(nofs_flag);
42 if (IS_ERR(folio)) 46 if (IS_ERR(folio))
43 return folio; 47 return folio;
44 } <<
45 buf->page = folio_file_page(folio, ind <<
46 48
>> 49 /* should already be PageUptodate, no need to lock page */
>> 50 page = folio_file_page(folio, index);
>> 51 buf->page = page;
>> 52 }
47 if (buf->kmap_type == EROFS_NO_KMAP) { 53 if (buf->kmap_type == EROFS_NO_KMAP) {
48 if (type == EROFS_KMAP) 54 if (type == EROFS_KMAP)
49 buf->base = kmap_local !! 55 buf->base = kmap_local_page(page);
50 buf->kmap_type = type; 56 buf->kmap_type = type;
51 } else if (buf->kmap_type != type) { 57 } else if (buf->kmap_type != type) {
52 DBG_BUGON(1); 58 DBG_BUGON(1);
53 return ERR_PTR(-EFAULT); 59 return ERR_PTR(-EFAULT);
54 } 60 }
55 if (type == EROFS_NO_KMAP) 61 if (type == EROFS_NO_KMAP)
56 return NULL; 62 return NULL;
57 return buf->base + (offset & ~PAGE_MAS 63 return buf->base + (offset & ~PAGE_MASK);
58 } 64 }
59 65
60 void erofs_init_metabuf(struct erofs_buf *buf, <<
61 { <<
62 struct erofs_sb_info *sbi = EROFS_SB(s <<
63 <<
64 if (erofs_is_fileio_mode(sbi)) <<
65 buf->mapping = file_inode(sbi- <<
66 else if (erofs_is_fscache_mode(sb)) <<
67 buf->mapping = sbi->s_fscache- <<
68 else <<
69 buf->mapping = sb->s_bdev->bd_ <<
70 } <<
71 <<
72 void *erofs_read_metabuf(struct erofs_buf *buf 66 void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb,
73 erofs_off_t offset, e !! 67 erofs_blk_t blkaddr, enum erofs_kmap_type type)
74 { 68 {
75 erofs_init_metabuf(buf, sb); !! 69 if (erofs_is_fscache_mode(sb))
76 return erofs_bread(buf, offset, type); !! 70 return erofs_bread(buf, EROFS_SB(sb)->s_fscache->inode,
>> 71 blkaddr, type);
>> 72
>> 73 return erofs_bread(buf, sb->s_bdev->bd_inode, blkaddr, type);
77 } 74 }
78 75
79 static int erofs_map_blocks_flatmode(struct in 76 static int erofs_map_blocks_flatmode(struct inode *inode,
80 struct er 77 struct erofs_map_blocks *map)
81 { 78 {
>> 79 erofs_blk_t nblocks, lastblk;
>> 80 u64 offset = map->m_la;
82 struct erofs_inode *vi = EROFS_I(inode 81 struct erofs_inode *vi = EROFS_I(inode);
83 struct super_block *sb = inode->i_sb; <<
84 bool tailendpacking = (vi->datalayout 82 bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE);
85 erofs_blk_t lastblk = erofs_iblks(inod <<
86 83
87 map->m_flags = EROFS_MAP_MAPPED; !! 84 nblocks = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
88 if (map->m_la < erofs_pos(sb, lastblk) !! 85 lastblk = nblocks - tailendpacking;
89 map->m_pa = erofs_pos(sb, vi-> !! 86
90 map->m_plen = erofs_pos(sb, la !! 87 /* there is no hole in flatmode */
91 } else { !! 88 map->m_flags = EROFS_MAP_MAPPED;
92 DBG_BUGON(!tailendpacking); !! 89 if (offset < blknr_to_addr(lastblk)) {
>> 90 map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la;
>> 91 map->m_plen = blknr_to_addr(lastblk) - offset;
>> 92 } else if (tailendpacking) {
93 map->m_pa = erofs_iloc(inode) 93 map->m_pa = erofs_iloc(inode) + vi->inode_isize +
94 vi->xattr_isize + erof !! 94 vi->xattr_isize + erofs_blkoff(offset);
95 map->m_plen = inode->i_size - !! 95 map->m_plen = inode->i_size - offset;
96 96
97 /* inline data should be locat 97 /* inline data should be located in the same meta block */
98 if (erofs_blkoff(sb, map->m_pa !! 98 if (erofs_blkoff(map->m_pa) + map->m_plen > EROFS_BLKSIZ) {
99 erofs_err(sb, "inline !! 99 erofs_err(inode->i_sb,
>> 100 "inline data cross block boundary @ nid %llu",
>> 101 vi->nid);
100 DBG_BUGON(1); 102 DBG_BUGON(1);
101 return -EFSCORRUPTED; 103 return -EFSCORRUPTED;
102 } 104 }
103 map->m_flags |= EROFS_MAP_META 105 map->m_flags |= EROFS_MAP_META;
>> 106 } else {
>> 107 erofs_err(inode->i_sb,
>> 108 "internal error @ nid: %llu (size %llu), m_la 0x%llx",
>> 109 vi->nid, inode->i_size, map->m_la);
>> 110 DBG_BUGON(1);
>> 111 return -EIO;
104 } 112 }
105 return 0; 113 return 0;
106 } 114 }
107 115
108 int erofs_map_blocks(struct inode *inode, stru 116 int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map)
109 { 117 {
110 struct super_block *sb = inode->i_sb; 118 struct super_block *sb = inode->i_sb;
111 struct erofs_inode *vi = EROFS_I(inode 119 struct erofs_inode *vi = EROFS_I(inode);
112 struct erofs_inode_chunk_index *idx; 120 struct erofs_inode_chunk_index *idx;
113 struct erofs_buf buf = __EROFS_BUF_INI 121 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
114 u64 chunknr; 122 u64 chunknr;
115 unsigned int unit; 123 unsigned int unit;
116 erofs_off_t pos; 124 erofs_off_t pos;
117 void *kaddr; 125 void *kaddr;
118 int err = 0; 126 int err = 0;
119 127
120 trace_erofs_map_blocks_enter(inode, ma 128 trace_erofs_map_blocks_enter(inode, map, 0);
121 map->m_deviceid = 0; 129 map->m_deviceid = 0;
122 if (map->m_la >= inode->i_size) { 130 if (map->m_la >= inode->i_size) {
123 /* leave out-of-bound access u 131 /* leave out-of-bound access unmapped */
124 map->m_flags = 0; 132 map->m_flags = 0;
125 map->m_plen = map->m_llen; !! 133 map->m_plen = 0;
126 goto out; 134 goto out;
127 } 135 }
128 136
129 if (vi->datalayout != EROFS_INODE_CHUN 137 if (vi->datalayout != EROFS_INODE_CHUNK_BASED) {
130 err = erofs_map_blocks_flatmod 138 err = erofs_map_blocks_flatmode(inode, map);
131 goto out; 139 goto out;
132 } 140 }
133 141
134 if (vi->chunkformat & EROFS_CHUNK_FORM 142 if (vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES)
135 unit = sizeof(*idx); 143 unit = sizeof(*idx); /* chunk index */
136 else 144 else
137 unit = EROFS_BLOCK_MAP_ENTRY_S 145 unit = EROFS_BLOCK_MAP_ENTRY_SIZE; /* block map */
138 146
139 chunknr = map->m_la >> vi->chunkbits; 147 chunknr = map->m_la >> vi->chunkbits;
140 pos = ALIGN(erofs_iloc(inode) + vi->in 148 pos = ALIGN(erofs_iloc(inode) + vi->inode_isize +
141 vi->xattr_isize, unit) + u 149 vi->xattr_isize, unit) + unit * chunknr;
142 150
143 kaddr = erofs_read_metabuf(&buf, sb, p !! 151 kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(pos), EROFS_KMAP);
144 if (IS_ERR(kaddr)) { 152 if (IS_ERR(kaddr)) {
145 err = PTR_ERR(kaddr); 153 err = PTR_ERR(kaddr);
146 goto out; 154 goto out;
147 } 155 }
148 map->m_la = chunknr << vi->chunkbits; 156 map->m_la = chunknr << vi->chunkbits;
149 map->m_plen = min_t(erofs_off_t, 1UL < 157 map->m_plen = min_t(erofs_off_t, 1UL << vi->chunkbits,
150 round_up(inode->i_size !! 158 roundup(inode->i_size - map->m_la, EROFS_BLKSIZ));
151 159
152 /* handle block map */ 160 /* handle block map */
153 if (!(vi->chunkformat & EROFS_CHUNK_FO 161 if (!(vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES)) {
154 __le32 *blkaddr = kaddr; !! 162 __le32 *blkaddr = kaddr + erofs_blkoff(pos);
155 163
156 if (le32_to_cpu(*blkaddr) == E 164 if (le32_to_cpu(*blkaddr) == EROFS_NULL_ADDR) {
157 map->m_flags = 0; 165 map->m_flags = 0;
158 } else { 166 } else {
159 map->m_pa = erofs_pos( !! 167 map->m_pa = blknr_to_addr(le32_to_cpu(*blkaddr));
160 map->m_flags = EROFS_M 168 map->m_flags = EROFS_MAP_MAPPED;
161 } 169 }
162 goto out_unlock; 170 goto out_unlock;
163 } 171 }
164 /* parse chunk indexes */ 172 /* parse chunk indexes */
165 idx = kaddr; !! 173 idx = kaddr + erofs_blkoff(pos);
166 switch (le32_to_cpu(idx->blkaddr)) { 174 switch (le32_to_cpu(idx->blkaddr)) {
167 case EROFS_NULL_ADDR: 175 case EROFS_NULL_ADDR:
168 map->m_flags = 0; 176 map->m_flags = 0;
169 break; 177 break;
170 default: 178 default:
171 map->m_deviceid = le16_to_cpu( 179 map->m_deviceid = le16_to_cpu(idx->device_id) &
172 EROFS_SB(sb)->device_i 180 EROFS_SB(sb)->device_id_mask;
173 map->m_pa = erofs_pos(sb, le32 !! 181 map->m_pa = blknr_to_addr(le32_to_cpu(idx->blkaddr));
174 map->m_flags = EROFS_MAP_MAPPE 182 map->m_flags = EROFS_MAP_MAPPED;
175 break; 183 break;
176 } 184 }
177 out_unlock: 185 out_unlock:
178 erofs_put_metabuf(&buf); 186 erofs_put_metabuf(&buf);
179 out: 187 out:
180 if (!err) 188 if (!err)
181 map->m_llen = map->m_plen; 189 map->m_llen = map->m_plen;
182 trace_erofs_map_blocks_exit(inode, map 190 trace_erofs_map_blocks_exit(inode, map, 0, err);
183 return err; 191 return err;
184 } 192 }
185 193
186 static void erofs_fill_from_devinfo(struct ero <<
187 struct ero <<
188 { <<
189 map->m_bdev = NULL; <<
190 map->m_fp = NULL; <<
191 if (dif->file) { <<
192 if (S_ISBLK(file_inode(dif->fi <<
193 map->m_bdev = file_bde <<
194 else <<
195 map->m_fp = dif->file; <<
196 } <<
197 map->m_daxdev = dif->dax_dev; <<
198 map->m_dax_part_off = dif->dax_part_of <<
199 map->m_fscache = dif->fscache; <<
200 } <<
201 <<
202 int erofs_map_dev(struct super_block *sb, stru 194 int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *map)
203 { 195 {
204 struct erofs_dev_context *devs = EROFS 196 struct erofs_dev_context *devs = EROFS_SB(sb)->devs;
205 struct erofs_device_info *dif; 197 struct erofs_device_info *dif;
206 erofs_off_t startoff, length; <<
207 int id; 198 int id;
208 199
>> 200 /* primary device by default */
209 map->m_bdev = sb->s_bdev; 201 map->m_bdev = sb->s_bdev;
210 map->m_daxdev = EROFS_SB(sb)->dax_dev; 202 map->m_daxdev = EROFS_SB(sb)->dax_dev;
211 map->m_dax_part_off = EROFS_SB(sb)->da 203 map->m_dax_part_off = EROFS_SB(sb)->dax_part_off;
212 map->m_fscache = EROFS_SB(sb)->s_fscac 204 map->m_fscache = EROFS_SB(sb)->s_fscache;
213 map->m_fp = EROFS_SB(sb)->fdev; <<
214 205
215 if (map->m_deviceid) { 206 if (map->m_deviceid) {
216 down_read(&devs->rwsem); 207 down_read(&devs->rwsem);
217 dif = idr_find(&devs->tree, ma 208 dif = idr_find(&devs->tree, map->m_deviceid - 1);
218 if (!dif) { 209 if (!dif) {
219 up_read(&devs->rwsem); 210 up_read(&devs->rwsem);
220 return -ENODEV; 211 return -ENODEV;
221 } 212 }
222 if (devs->flatdev) { !! 213 map->m_bdev = dif->bdev;
223 map->m_pa += erofs_pos !! 214 map->m_daxdev = dif->dax_dev;
224 up_read(&devs->rwsem); !! 215 map->m_dax_part_off = dif->dax_part_off;
225 return 0; !! 216 map->m_fscache = dif->fscache;
226 } <<
227 erofs_fill_from_devinfo(map, d <<
228 up_read(&devs->rwsem); 217 up_read(&devs->rwsem);
229 } else if (devs->extra_devices && !dev !! 218 } else if (devs->extra_devices) {
230 down_read(&devs->rwsem); 219 down_read(&devs->rwsem);
231 idr_for_each_entry(&devs->tree 220 idr_for_each_entry(&devs->tree, dif, id) {
>> 221 erofs_off_t startoff, length;
>> 222
232 if (!dif->mapped_blkad 223 if (!dif->mapped_blkaddr)
233 continue; 224 continue;
>> 225 startoff = blknr_to_addr(dif->mapped_blkaddr);
>> 226 length = blknr_to_addr(dif->blocks);
234 227
235 startoff = erofs_pos(s <<
236 length = erofs_pos(sb, <<
237 if (map->m_pa >= start 228 if (map->m_pa >= startoff &&
238 map->m_pa < starto 229 map->m_pa < startoff + length) {
239 map->m_pa -= s 230 map->m_pa -= startoff;
240 erofs_fill_fro !! 231 map->m_bdev = dif->bdev;
>> 232 map->m_daxdev = dif->dax_dev;
>> 233 map->m_dax_part_off = dif->dax_part_off;
>> 234 map->m_fscache = dif->fscache;
241 break; 235 break;
242 } 236 }
243 } 237 }
244 up_read(&devs->rwsem); 238 up_read(&devs->rwsem);
245 } 239 }
246 return 0; 240 return 0;
247 } 241 }
248 242
249 /* <<
250 * bit 30: I/O error occurred on this folio <<
251 * bit 0 - 29: remaining parts to complete thi <<
252 */ <<
253 #define EROFS_ONLINEFOLIO_EIO <<
254 <<
255 void erofs_onlinefolio_init(struct folio *foli <<
256 { <<
257 union { <<
258 atomic_t o; <<
259 void *v; <<
260 } u = { .o = ATOMIC_INIT(1) }; <<
261 <<
262 folio->private = u.v; /* valid only <<
263 } <<
264 <<
265 void erofs_onlinefolio_split(struct folio *fol <<
266 { <<
267 atomic_inc((atomic_t *)&folio->private <<
268 } <<
269 <<
270 void erofs_onlinefolio_end(struct folio *folio <<
271 { <<
272 int orig, v; <<
273 <<
274 do { <<
275 orig = atomic_read((atomic_t * <<
276 v = (orig - 1) | (err ? EROFS_ <<
277 } while (atomic_cmpxchg((atomic_t *)&f <<
278 <<
279 if (v & ~EROFS_ONLINEFOLIO_EIO) <<
280 return; <<
281 folio->private = 0; <<
282 folio_end_read(folio, !(v & EROFS_ONLI <<
283 } <<
284 <<
285 static int erofs_iomap_begin(struct inode *ino 243 static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
286 unsigned int flags, struct iom 244 unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
287 { 245 {
288 int ret; 246 int ret;
289 struct super_block *sb = inode->i_sb; <<
290 struct erofs_map_blocks map; 247 struct erofs_map_blocks map;
291 struct erofs_map_dev mdev; 248 struct erofs_map_dev mdev;
292 249
293 map.m_la = offset; 250 map.m_la = offset;
294 map.m_llen = length; 251 map.m_llen = length;
295 252
296 ret = erofs_map_blocks(inode, &map); 253 ret = erofs_map_blocks(inode, &map);
297 if (ret < 0) 254 if (ret < 0)
298 return ret; 255 return ret;
299 256
300 mdev = (struct erofs_map_dev) { 257 mdev = (struct erofs_map_dev) {
301 .m_deviceid = map.m_deviceid, 258 .m_deviceid = map.m_deviceid,
302 .m_pa = map.m_pa, 259 .m_pa = map.m_pa,
303 }; 260 };
304 ret = erofs_map_dev(sb, &mdev); !! 261 ret = erofs_map_dev(inode->i_sb, &mdev);
305 if (ret) 262 if (ret)
306 return ret; 263 return ret;
307 264
308 iomap->offset = map.m_la; 265 iomap->offset = map.m_la;
309 if (flags & IOMAP_DAX) 266 if (flags & IOMAP_DAX)
310 iomap->dax_dev = mdev.m_daxdev 267 iomap->dax_dev = mdev.m_daxdev;
311 else 268 else
312 iomap->bdev = mdev.m_bdev; 269 iomap->bdev = mdev.m_bdev;
313 iomap->length = map.m_llen; 270 iomap->length = map.m_llen;
314 iomap->flags = 0; 271 iomap->flags = 0;
315 iomap->private = NULL; 272 iomap->private = NULL;
316 273
317 if (!(map.m_flags & EROFS_MAP_MAPPED)) 274 if (!(map.m_flags & EROFS_MAP_MAPPED)) {
318 iomap->type = IOMAP_HOLE; 275 iomap->type = IOMAP_HOLE;
319 iomap->addr = IOMAP_NULL_ADDR; 276 iomap->addr = IOMAP_NULL_ADDR;
320 if (!iomap->length) 277 if (!iomap->length)
321 iomap->length = length 278 iomap->length = length;
322 return 0; 279 return 0;
323 } 280 }
324 281
325 if (map.m_flags & EROFS_MAP_META) { 282 if (map.m_flags & EROFS_MAP_META) {
326 void *ptr; 283 void *ptr;
327 struct erofs_buf buf = __EROFS 284 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
328 285
329 iomap->type = IOMAP_INLINE; 286 iomap->type = IOMAP_INLINE;
330 ptr = erofs_read_metabuf(&buf, !! 287 ptr = erofs_read_metabuf(&buf, inode->i_sb,
>> 288 erofs_blknr(mdev.m_pa), EROFS_KMAP);
331 if (IS_ERR(ptr)) 289 if (IS_ERR(ptr))
332 return PTR_ERR(ptr); 290 return PTR_ERR(ptr);
333 iomap->inline_data = ptr; !! 291 iomap->inline_data = ptr + erofs_blkoff(mdev.m_pa);
334 iomap->private = buf.base; 292 iomap->private = buf.base;
335 } else { 293 } else {
336 iomap->type = IOMAP_MAPPED; 294 iomap->type = IOMAP_MAPPED;
337 iomap->addr = mdev.m_pa; 295 iomap->addr = mdev.m_pa;
338 if (flags & IOMAP_DAX) 296 if (flags & IOMAP_DAX)
339 iomap->addr += mdev.m_ 297 iomap->addr += mdev.m_dax_part_off;
340 } 298 }
341 return 0; 299 return 0;
342 } 300 }
343 301
344 static int erofs_iomap_end(struct inode *inode 302 static int erofs_iomap_end(struct inode *inode, loff_t pos, loff_t length,
345 ssize_t written, unsigned int 303 ssize_t written, unsigned int flags, struct iomap *iomap)
346 { 304 {
347 void *ptr = iomap->private; 305 void *ptr = iomap->private;
348 306
349 if (ptr) { 307 if (ptr) {
350 struct erofs_buf buf = { 308 struct erofs_buf buf = {
351 .page = kmap_to_page(p 309 .page = kmap_to_page(ptr),
352 .base = ptr, 310 .base = ptr,
353 .kmap_type = EROFS_KMA 311 .kmap_type = EROFS_KMAP,
354 }; 312 };
355 313
356 DBG_BUGON(iomap->type != IOMAP 314 DBG_BUGON(iomap->type != IOMAP_INLINE);
357 erofs_put_metabuf(&buf); 315 erofs_put_metabuf(&buf);
358 } else { 316 } else {
359 DBG_BUGON(iomap->type == IOMAP 317 DBG_BUGON(iomap->type == IOMAP_INLINE);
360 } 318 }
361 return written; 319 return written;
362 } 320 }
363 321
364 static const struct iomap_ops erofs_iomap_ops 322 static const struct iomap_ops erofs_iomap_ops = {
365 .iomap_begin = erofs_iomap_begin, 323 .iomap_begin = erofs_iomap_begin,
366 .iomap_end = erofs_iomap_end, 324 .iomap_end = erofs_iomap_end,
367 }; 325 };
368 326
369 int erofs_fiemap(struct inode *inode, struct f 327 int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
370 u64 start, u64 len) 328 u64 start, u64 len)
371 { 329 {
372 if (erofs_inode_is_data_compressed(ERO 330 if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) {
373 #ifdef CONFIG_EROFS_FS_ZIP 331 #ifdef CONFIG_EROFS_FS_ZIP
374 return iomap_fiemap(inode, fie 332 return iomap_fiemap(inode, fieinfo, start, len,
375 &z_erofs_i 333 &z_erofs_iomap_report_ops);
376 #else 334 #else
377 return -EOPNOTSUPP; 335 return -EOPNOTSUPP;
378 #endif 336 #endif
379 } 337 }
380 return iomap_fiemap(inode, fieinfo, st 338 return iomap_fiemap(inode, fieinfo, start, len, &erofs_iomap_ops);
381 } 339 }
382 340
383 /* 341 /*
384 * since we dont have write or truncate flows, 342 * since we dont have write or truncate flows, so no inode
385 * locking needs to be held at the moment. 343 * locking needs to be held at the moment.
386 */ 344 */
387 static int erofs_read_folio(struct file *file, 345 static int erofs_read_folio(struct file *file, struct folio *folio)
388 { 346 {
389 return iomap_read_folio(folio, &erofs_ 347 return iomap_read_folio(folio, &erofs_iomap_ops);
390 } 348 }
391 349
392 static void erofs_readahead(struct readahead_c 350 static void erofs_readahead(struct readahead_control *rac)
393 { 351 {
394 return iomap_readahead(rac, &erofs_iom 352 return iomap_readahead(rac, &erofs_iomap_ops);
395 } 353 }
396 354
397 static sector_t erofs_bmap(struct address_spac 355 static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
398 { 356 {
399 return iomap_bmap(mapping, block, &ero 357 return iomap_bmap(mapping, block, &erofs_iomap_ops);
400 } 358 }
401 359
402 static ssize_t erofs_file_read_iter(struct kio 360 static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
403 { 361 {
404 struct inode *inode = file_inode(iocb- 362 struct inode *inode = file_inode(iocb->ki_filp);
405 363
406 /* no need taking (shared) inode lock 364 /* no need taking (shared) inode lock since it's a ro filesystem */
407 if (!iov_iter_count(to)) 365 if (!iov_iter_count(to))
408 return 0; 366 return 0;
409 367
410 #ifdef CONFIG_FS_DAX 368 #ifdef CONFIG_FS_DAX
411 if (IS_DAX(inode)) 369 if (IS_DAX(inode))
412 return dax_iomap_rw(iocb, to, 370 return dax_iomap_rw(iocb, to, &erofs_iomap_ops);
413 #endif 371 #endif
414 if (iocb->ki_flags & IOCB_DIRECT) { 372 if (iocb->ki_flags & IOCB_DIRECT) {
415 struct block_device *bdev = in 373 struct block_device *bdev = inode->i_sb->s_bdev;
416 unsigned int blksize_mask; 374 unsigned int blksize_mask;
417 375
418 if (bdev) 376 if (bdev)
419 blksize_mask = bdev_lo 377 blksize_mask = bdev_logical_block_size(bdev) - 1;
420 else 378 else
421 blksize_mask = i_block 379 blksize_mask = i_blocksize(inode) - 1;
422 380
423 if ((iocb->ki_pos | iov_iter_c 381 if ((iocb->ki_pos | iov_iter_count(to) |
424 iov_iter_alignment(to)) & 382 iov_iter_alignment(to)) & blksize_mask)
425 return -EINVAL; 383 return -EINVAL;
426 384
427 return iomap_dio_rw(iocb, to, 385 return iomap_dio_rw(iocb, to, &erofs_iomap_ops,
428 NULL, 0, N 386 NULL, 0, NULL, 0);
429 } 387 }
430 return filemap_read(iocb, to, 0); 388 return filemap_read(iocb, to, 0);
431 } 389 }
432 390
433 /* for uncompressed (aligned) files and raw ac 391 /* for uncompressed (aligned) files and raw access for other files */
434 const struct address_space_operations erofs_ao !! 392 const struct address_space_operations erofs_raw_access_aops = {
435 .read_folio = erofs_read_folio, 393 .read_folio = erofs_read_folio,
436 .readahead = erofs_readahead, 394 .readahead = erofs_readahead,
437 .bmap = erofs_bmap, 395 .bmap = erofs_bmap,
438 .direct_IO = noop_direct_IO, 396 .direct_IO = noop_direct_IO,
439 .release_folio = iomap_release_folio, 397 .release_folio = iomap_release_folio,
440 .invalidate_folio = iomap_invalidate_f 398 .invalidate_folio = iomap_invalidate_folio,
441 }; 399 };
442 400
443 #ifdef CONFIG_FS_DAX 401 #ifdef CONFIG_FS_DAX
444 static vm_fault_t erofs_dax_huge_fault(struct 402 static vm_fault_t erofs_dax_huge_fault(struct vm_fault *vmf,
445 unsigned int order) !! 403 enum page_entry_size pe_size)
446 { 404 {
447 return dax_iomap_fault(vmf, order, NUL !! 405 return dax_iomap_fault(vmf, pe_size, NULL, NULL, &erofs_iomap_ops);
448 } 406 }
449 407
450 static vm_fault_t erofs_dax_fault(struct vm_fa 408 static vm_fault_t erofs_dax_fault(struct vm_fault *vmf)
451 { 409 {
452 return erofs_dax_huge_fault(vmf, 0); !! 410 return erofs_dax_huge_fault(vmf, PE_SIZE_PTE);
453 } 411 }
454 412
455 static const struct vm_operations_struct erofs 413 static const struct vm_operations_struct erofs_dax_vm_ops = {
456 .fault = erofs_dax_fault, 414 .fault = erofs_dax_fault,
457 .huge_fault = erofs_dax_huge_fault 415 .huge_fault = erofs_dax_huge_fault,
458 }; 416 };
459 417
460 static int erofs_file_mmap(struct file *file, 418 static int erofs_file_mmap(struct file *file, struct vm_area_struct *vma)
461 { 419 {
462 if (!IS_DAX(file_inode(file))) 420 if (!IS_DAX(file_inode(file)))
463 return generic_file_readonly_m 421 return generic_file_readonly_mmap(file, vma);
464 422
465 if ((vma->vm_flags & VM_SHARED) && (vm 423 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
466 return -EINVAL; 424 return -EINVAL;
467 425
468 vma->vm_ops = &erofs_dax_vm_ops; 426 vma->vm_ops = &erofs_dax_vm_ops;
469 vm_flags_set(vma, VM_HUGEPAGE); 427 vm_flags_set(vma, VM_HUGEPAGE);
470 return 0; 428 return 0;
471 } 429 }
472 #else 430 #else
473 #define erofs_file_mmap generic_file_readonly_ 431 #define erofs_file_mmap generic_file_readonly_mmap
474 #endif 432 #endif
475 433
476 const struct file_operations erofs_file_fops = 434 const struct file_operations erofs_file_fops = {
477 .llseek = generic_file_llseek, 435 .llseek = generic_file_llseek,
478 .read_iter = erofs_file_read_iter 436 .read_iter = erofs_file_read_iter,
479 .mmap = erofs_file_mmap, 437 .mmap = erofs_file_mmap,
480 .get_unmapped_area = thp_get_unmapped_ !! 438 .splice_read = generic_file_splice_read,
481 .splice_read = filemap_splice_read, <<
482 }; 439 };
483 440
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.