1 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
2 /* 2 /*
3 * Copyright (C) 2017-2018 HUAWEI, Inc. 3 * Copyright (C) 2017-2018 HUAWEI, Inc.
4 * https://www.huawei.com/ 4 * https://www.huawei.com/
5 * Copyright (C) 2021, Alibaba Cloud 5 * Copyright (C) 2021, Alibaba Cloud
6 */ 6 */
7 #include "internal.h" 7 #include "internal.h"
>> 8 #include <linux/prefetch.h>
8 #include <linux/sched/mm.h> 9 #include <linux/sched/mm.h>
>> 10 #include <linux/dax.h>
9 #include <trace/events/erofs.h> 11 #include <trace/events/erofs.h>
10 12
11 void erofs_unmap_metabuf(struct erofs_buf *buf 13 void erofs_unmap_metabuf(struct erofs_buf *buf)
12 { 14 {
13 if (buf->kmap_type == EROFS_KMAP) 15 if (buf->kmap_type == EROFS_KMAP)
14 kunmap_local(buf->base); !! 16 kunmap(buf->page);
>> 17 else if (buf->kmap_type == EROFS_KMAP_ATOMIC)
>> 18 kunmap_atomic(buf->base);
15 buf->base = NULL; 19 buf->base = NULL;
16 buf->kmap_type = EROFS_NO_KMAP; 20 buf->kmap_type = EROFS_NO_KMAP;
17 } 21 }
18 22
19 void erofs_put_metabuf(struct erofs_buf *buf) 23 void erofs_put_metabuf(struct erofs_buf *buf)
20 { 24 {
21 if (!buf->page) 25 if (!buf->page)
22 return; 26 return;
23 erofs_unmap_metabuf(buf); 27 erofs_unmap_metabuf(buf);
24 folio_put(page_folio(buf->page)); !! 28 put_page(buf->page);
25 buf->page = NULL; 29 buf->page = NULL;
26 } 30 }
27 31
28 void *erofs_bread(struct erofs_buf *buf, erofs !! 32 void *erofs_bread(struct erofs_buf *buf, struct inode *inode,
29 enum erofs_kmap_type type) !! 33 erofs_blk_t blkaddr, enum erofs_kmap_type type)
30 { 34 {
>> 35 struct address_space *const mapping = inode->i_mapping;
>> 36 erofs_off_t offset = blknr_to_addr(blkaddr);
31 pgoff_t index = offset >> PAGE_SHIFT; 37 pgoff_t index = offset >> PAGE_SHIFT;
32 struct folio *folio = NULL; !! 38 struct page *page = buf->page;
>> 39 struct folio *folio;
>> 40 unsigned int nofs_flag;
33 41
34 if (buf->page) { !! 42 if (!page || page->index != index) {
35 folio = page_folio(buf->page); <<
36 if (folio_file_page(folio, ind <<
37 erofs_unmap_metabuf(bu <<
38 } <<
39 if (!folio || !folio_contains(folio, i <<
40 erofs_put_metabuf(buf); 43 erofs_put_metabuf(buf);
41 folio = read_mapping_folio(buf !! 44
>> 45 nofs_flag = memalloc_nofs_save();
>> 46 folio = read_cache_folio(mapping, index, NULL, NULL);
>> 47 memalloc_nofs_restore(nofs_flag);
42 if (IS_ERR(folio)) 48 if (IS_ERR(folio))
43 return folio; 49 return folio;
44 } <<
45 buf->page = folio_file_page(folio, ind <<
46 50
>> 51 /* should already be PageUptodate, no need to lock page */
>> 52 page = folio_file_page(folio, index);
>> 53 buf->page = page;
>> 54 }
47 if (buf->kmap_type == EROFS_NO_KMAP) { 55 if (buf->kmap_type == EROFS_NO_KMAP) {
48 if (type == EROFS_KMAP) 56 if (type == EROFS_KMAP)
49 buf->base = kmap_local !! 57 buf->base = kmap(page);
>> 58 else if (type == EROFS_KMAP_ATOMIC)
>> 59 buf->base = kmap_atomic(page);
50 buf->kmap_type = type; 60 buf->kmap_type = type;
51 } else if (buf->kmap_type != type) { 61 } else if (buf->kmap_type != type) {
52 DBG_BUGON(1); 62 DBG_BUGON(1);
53 return ERR_PTR(-EFAULT); 63 return ERR_PTR(-EFAULT);
54 } 64 }
55 if (type == EROFS_NO_KMAP) 65 if (type == EROFS_NO_KMAP)
56 return NULL; 66 return NULL;
57 return buf->base + (offset & ~PAGE_MAS 67 return buf->base + (offset & ~PAGE_MASK);
58 } 68 }
59 69
60 void erofs_init_metabuf(struct erofs_buf *buf, <<
61 { <<
62 struct erofs_sb_info *sbi = EROFS_SB(s <<
63 <<
64 if (erofs_is_fileio_mode(sbi)) <<
65 buf->mapping = file_inode(sbi- <<
66 else if (erofs_is_fscache_mode(sb)) <<
67 buf->mapping = sbi->s_fscache- <<
68 else <<
69 buf->mapping = sb->s_bdev->bd_ <<
70 } <<
71 <<
72 void *erofs_read_metabuf(struct erofs_buf *buf 70 void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb,
73 erofs_off_t offset, e !! 71 erofs_blk_t blkaddr, enum erofs_kmap_type type)
74 { 72 {
75 erofs_init_metabuf(buf, sb); !! 73 if (erofs_is_fscache_mode(sb))
76 return erofs_bread(buf, offset, type); !! 74 return erofs_bread(buf, EROFS_SB(sb)->s_fscache->inode,
>> 75 blkaddr, type);
>> 76
>> 77 return erofs_bread(buf, sb->s_bdev->bd_inode, blkaddr, type);
77 } 78 }
78 79
79 static int erofs_map_blocks_flatmode(struct in 80 static int erofs_map_blocks_flatmode(struct inode *inode,
80 struct er !! 81 struct erofs_map_blocks *map,
>> 82 int flags)
81 { 83 {
>> 84 erofs_blk_t nblocks, lastblk;
>> 85 u64 offset = map->m_la;
82 struct erofs_inode *vi = EROFS_I(inode 86 struct erofs_inode *vi = EROFS_I(inode);
83 struct super_block *sb = inode->i_sb; <<
84 bool tailendpacking = (vi->datalayout 87 bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE);
85 erofs_blk_t lastblk = erofs_iblks(inod <<
86 88
87 map->m_flags = EROFS_MAP_MAPPED; !! 89 nblocks = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
88 if (map->m_la < erofs_pos(sb, lastblk) !! 90 lastblk = nblocks - tailendpacking;
89 map->m_pa = erofs_pos(sb, vi-> !! 91
90 map->m_plen = erofs_pos(sb, la !! 92 /* there is no hole in flatmode */
91 } else { !! 93 map->m_flags = EROFS_MAP_MAPPED;
92 DBG_BUGON(!tailendpacking); !! 94 if (offset < blknr_to_addr(lastblk)) {
93 map->m_pa = erofs_iloc(inode) !! 95 map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la;
94 vi->xattr_isize + erof !! 96 map->m_plen = blknr_to_addr(lastblk) - offset;
95 map->m_plen = inode->i_size - !! 97 } else if (tailendpacking) {
>> 98 /* 2 - inode inline B: inode, [xattrs], inline last blk... */
>> 99 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
>> 100
>> 101 map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize +
>> 102 vi->xattr_isize + erofs_blkoff(map->m_la);
>> 103 map->m_plen = inode->i_size - offset;
96 104
97 /* inline data should be locat 105 /* inline data should be located in the same meta block */
98 if (erofs_blkoff(sb, map->m_pa !! 106 if (erofs_blkoff(map->m_pa) + map->m_plen > EROFS_BLKSIZ) {
99 erofs_err(sb, "inline !! 107 erofs_err(inode->i_sb,
>> 108 "inline data cross block boundary @ nid %llu",
>> 109 vi->nid);
100 DBG_BUGON(1); 110 DBG_BUGON(1);
101 return -EFSCORRUPTED; 111 return -EFSCORRUPTED;
102 } 112 }
103 map->m_flags |= EROFS_MAP_META 113 map->m_flags |= EROFS_MAP_META;
>> 114 } else {
>> 115 erofs_err(inode->i_sb,
>> 116 "internal error @ nid: %llu (size %llu), m_la 0x%llx",
>> 117 vi->nid, inode->i_size, map->m_la);
>> 118 DBG_BUGON(1);
>> 119 return -EIO;
104 } 120 }
105 return 0; 121 return 0;
106 } 122 }
107 123
108 int erofs_map_blocks(struct inode *inode, stru !! 124 int erofs_map_blocks(struct inode *inode,
>> 125 struct erofs_map_blocks *map, int flags)
109 { 126 {
110 struct super_block *sb = inode->i_sb; 127 struct super_block *sb = inode->i_sb;
111 struct erofs_inode *vi = EROFS_I(inode 128 struct erofs_inode *vi = EROFS_I(inode);
112 struct erofs_inode_chunk_index *idx; 129 struct erofs_inode_chunk_index *idx;
113 struct erofs_buf buf = __EROFS_BUF_INI 130 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
114 u64 chunknr; 131 u64 chunknr;
115 unsigned int unit; 132 unsigned int unit;
116 erofs_off_t pos; 133 erofs_off_t pos;
117 void *kaddr; 134 void *kaddr;
118 int err = 0; 135 int err = 0;
119 136
120 trace_erofs_map_blocks_enter(inode, ma !! 137 trace_erofs_map_blocks_enter(inode, map, flags);
121 map->m_deviceid = 0; 138 map->m_deviceid = 0;
122 if (map->m_la >= inode->i_size) { 139 if (map->m_la >= inode->i_size) {
123 /* leave out-of-bound access u 140 /* leave out-of-bound access unmapped */
124 map->m_flags = 0; 141 map->m_flags = 0;
125 map->m_plen = map->m_llen; !! 142 map->m_plen = 0;
126 goto out; 143 goto out;
127 } 144 }
128 145
129 if (vi->datalayout != EROFS_INODE_CHUN 146 if (vi->datalayout != EROFS_INODE_CHUNK_BASED) {
130 err = erofs_map_blocks_flatmod !! 147 err = erofs_map_blocks_flatmode(inode, map, flags);
131 goto out; 148 goto out;
132 } 149 }
133 150
134 if (vi->chunkformat & EROFS_CHUNK_FORM 151 if (vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES)
135 unit = sizeof(*idx); 152 unit = sizeof(*idx); /* chunk index */
136 else 153 else
137 unit = EROFS_BLOCK_MAP_ENTRY_S 154 unit = EROFS_BLOCK_MAP_ENTRY_SIZE; /* block map */
138 155
139 chunknr = map->m_la >> vi->chunkbits; 156 chunknr = map->m_la >> vi->chunkbits;
140 pos = ALIGN(erofs_iloc(inode) + vi->in !! 157 pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
141 vi->xattr_isize, unit) + u 158 vi->xattr_isize, unit) + unit * chunknr;
142 159
143 kaddr = erofs_read_metabuf(&buf, sb, p !! 160 kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(pos), EROFS_KMAP);
144 if (IS_ERR(kaddr)) { 161 if (IS_ERR(kaddr)) {
145 err = PTR_ERR(kaddr); 162 err = PTR_ERR(kaddr);
146 goto out; 163 goto out;
147 } 164 }
148 map->m_la = chunknr << vi->chunkbits; 165 map->m_la = chunknr << vi->chunkbits;
149 map->m_plen = min_t(erofs_off_t, 1UL < 166 map->m_plen = min_t(erofs_off_t, 1UL << vi->chunkbits,
150 round_up(inode->i_size !! 167 roundup(inode->i_size - map->m_la, EROFS_BLKSIZ));
151 168
152 /* handle block map */ 169 /* handle block map */
153 if (!(vi->chunkformat & EROFS_CHUNK_FO 170 if (!(vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES)) {
154 __le32 *blkaddr = kaddr; !! 171 __le32 *blkaddr = kaddr + erofs_blkoff(pos);
155 172
156 if (le32_to_cpu(*blkaddr) == E 173 if (le32_to_cpu(*blkaddr) == EROFS_NULL_ADDR) {
157 map->m_flags = 0; 174 map->m_flags = 0;
158 } else { 175 } else {
159 map->m_pa = erofs_pos( !! 176 map->m_pa = blknr_to_addr(le32_to_cpu(*blkaddr));
160 map->m_flags = EROFS_M 177 map->m_flags = EROFS_MAP_MAPPED;
161 } 178 }
162 goto out_unlock; 179 goto out_unlock;
163 } 180 }
164 /* parse chunk indexes */ 181 /* parse chunk indexes */
165 idx = kaddr; !! 182 idx = kaddr + erofs_blkoff(pos);
166 switch (le32_to_cpu(idx->blkaddr)) { 183 switch (le32_to_cpu(idx->blkaddr)) {
167 case EROFS_NULL_ADDR: 184 case EROFS_NULL_ADDR:
168 map->m_flags = 0; 185 map->m_flags = 0;
169 break; 186 break;
170 default: 187 default:
171 map->m_deviceid = le16_to_cpu( 188 map->m_deviceid = le16_to_cpu(idx->device_id) &
172 EROFS_SB(sb)->device_i 189 EROFS_SB(sb)->device_id_mask;
173 map->m_pa = erofs_pos(sb, le32 !! 190 map->m_pa = blknr_to_addr(le32_to_cpu(idx->blkaddr));
174 map->m_flags = EROFS_MAP_MAPPE 191 map->m_flags = EROFS_MAP_MAPPED;
175 break; 192 break;
176 } 193 }
177 out_unlock: 194 out_unlock:
178 erofs_put_metabuf(&buf); 195 erofs_put_metabuf(&buf);
179 out: 196 out:
180 if (!err) 197 if (!err)
181 map->m_llen = map->m_plen; 198 map->m_llen = map->m_plen;
182 trace_erofs_map_blocks_exit(inode, map !! 199 trace_erofs_map_blocks_exit(inode, map, flags, 0);
183 return err; 200 return err;
184 } 201 }
185 202
186 static void erofs_fill_from_devinfo(struct ero <<
187 struct ero <<
188 { <<
189 map->m_bdev = NULL; <<
190 map->m_fp = NULL; <<
191 if (dif->file) { <<
192 if (S_ISBLK(file_inode(dif->fi <<
193 map->m_bdev = file_bde <<
194 else <<
195 map->m_fp = dif->file; <<
196 } <<
197 map->m_daxdev = dif->dax_dev; <<
198 map->m_dax_part_off = dif->dax_part_of <<
199 map->m_fscache = dif->fscache; <<
200 } <<
201 <<
202 int erofs_map_dev(struct super_block *sb, stru 203 int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *map)
203 { 204 {
204 struct erofs_dev_context *devs = EROFS 205 struct erofs_dev_context *devs = EROFS_SB(sb)->devs;
205 struct erofs_device_info *dif; 206 struct erofs_device_info *dif;
206 erofs_off_t startoff, length; <<
207 int id; 207 int id;
208 208
>> 209 /* primary device by default */
209 map->m_bdev = sb->s_bdev; 210 map->m_bdev = sb->s_bdev;
210 map->m_daxdev = EROFS_SB(sb)->dax_dev; 211 map->m_daxdev = EROFS_SB(sb)->dax_dev;
211 map->m_dax_part_off = EROFS_SB(sb)->da 212 map->m_dax_part_off = EROFS_SB(sb)->dax_part_off;
212 map->m_fscache = EROFS_SB(sb)->s_fscac 213 map->m_fscache = EROFS_SB(sb)->s_fscache;
213 map->m_fp = EROFS_SB(sb)->fdev; <<
214 214
215 if (map->m_deviceid) { 215 if (map->m_deviceid) {
216 down_read(&devs->rwsem); 216 down_read(&devs->rwsem);
217 dif = idr_find(&devs->tree, ma 217 dif = idr_find(&devs->tree, map->m_deviceid - 1);
218 if (!dif) { 218 if (!dif) {
219 up_read(&devs->rwsem); 219 up_read(&devs->rwsem);
220 return -ENODEV; 220 return -ENODEV;
221 } 221 }
222 if (devs->flatdev) { !! 222 map->m_bdev = dif->bdev;
223 map->m_pa += erofs_pos !! 223 map->m_daxdev = dif->dax_dev;
224 up_read(&devs->rwsem); !! 224 map->m_dax_part_off = dif->dax_part_off;
225 return 0; !! 225 map->m_fscache = dif->fscache;
226 } <<
227 erofs_fill_from_devinfo(map, d <<
228 up_read(&devs->rwsem); 226 up_read(&devs->rwsem);
229 } else if (devs->extra_devices && !dev !! 227 } else if (devs->extra_devices) {
230 down_read(&devs->rwsem); 228 down_read(&devs->rwsem);
231 idr_for_each_entry(&devs->tree 229 idr_for_each_entry(&devs->tree, dif, id) {
>> 230 erofs_off_t startoff, length;
>> 231
232 if (!dif->mapped_blkad 232 if (!dif->mapped_blkaddr)
233 continue; 233 continue;
>> 234 startoff = blknr_to_addr(dif->mapped_blkaddr);
>> 235 length = blknr_to_addr(dif->blocks);
234 236
235 startoff = erofs_pos(s <<
236 length = erofs_pos(sb, <<
237 if (map->m_pa >= start 237 if (map->m_pa >= startoff &&
238 map->m_pa < starto 238 map->m_pa < startoff + length) {
239 map->m_pa -= s 239 map->m_pa -= startoff;
240 erofs_fill_fro !! 240 map->m_bdev = dif->bdev;
>> 241 map->m_daxdev = dif->dax_dev;
>> 242 map->m_dax_part_off = dif->dax_part_off;
>> 243 map->m_fscache = dif->fscache;
241 break; 244 break;
242 } 245 }
243 } 246 }
244 up_read(&devs->rwsem); 247 up_read(&devs->rwsem);
245 } 248 }
246 return 0; 249 return 0;
247 } 250 }
248 251
249 /* <<
250 * bit 30: I/O error occurred on this folio <<
251 * bit 0 - 29: remaining parts to complete thi <<
252 */ <<
253 #define EROFS_ONLINEFOLIO_EIO <<
254 <<
255 void erofs_onlinefolio_init(struct folio *foli <<
256 { <<
257 union { <<
258 atomic_t o; <<
259 void *v; <<
260 } u = { .o = ATOMIC_INIT(1) }; <<
261 <<
262 folio->private = u.v; /* valid only <<
263 } <<
264 <<
265 void erofs_onlinefolio_split(struct folio *fol <<
266 { <<
267 atomic_inc((atomic_t *)&folio->private <<
268 } <<
269 <<
270 void erofs_onlinefolio_end(struct folio *folio <<
271 { <<
272 int orig, v; <<
273 <<
274 do { <<
275 orig = atomic_read((atomic_t * <<
276 v = (orig - 1) | (err ? EROFS_ <<
277 } while (atomic_cmpxchg((atomic_t *)&f <<
278 <<
279 if (v & ~EROFS_ONLINEFOLIO_EIO) <<
280 return; <<
281 folio->private = 0; <<
282 folio_end_read(folio, !(v & EROFS_ONLI <<
283 } <<
284 <<
285 static int erofs_iomap_begin(struct inode *ino 252 static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
286 unsigned int flags, struct iom 253 unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
287 { 254 {
288 int ret; 255 int ret;
289 struct super_block *sb = inode->i_sb; <<
290 struct erofs_map_blocks map; 256 struct erofs_map_blocks map;
291 struct erofs_map_dev mdev; 257 struct erofs_map_dev mdev;
292 258
293 map.m_la = offset; 259 map.m_la = offset;
294 map.m_llen = length; 260 map.m_llen = length;
295 261
296 ret = erofs_map_blocks(inode, &map); !! 262 ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
297 if (ret < 0) 263 if (ret < 0)
298 return ret; 264 return ret;
299 265
300 mdev = (struct erofs_map_dev) { 266 mdev = (struct erofs_map_dev) {
301 .m_deviceid = map.m_deviceid, 267 .m_deviceid = map.m_deviceid,
302 .m_pa = map.m_pa, 268 .m_pa = map.m_pa,
303 }; 269 };
304 ret = erofs_map_dev(sb, &mdev); !! 270 ret = erofs_map_dev(inode->i_sb, &mdev);
305 if (ret) 271 if (ret)
306 return ret; 272 return ret;
307 273
308 iomap->offset = map.m_la; 274 iomap->offset = map.m_la;
309 if (flags & IOMAP_DAX) 275 if (flags & IOMAP_DAX)
310 iomap->dax_dev = mdev.m_daxdev 276 iomap->dax_dev = mdev.m_daxdev;
311 else 277 else
312 iomap->bdev = mdev.m_bdev; 278 iomap->bdev = mdev.m_bdev;
313 iomap->length = map.m_llen; 279 iomap->length = map.m_llen;
314 iomap->flags = 0; 280 iomap->flags = 0;
315 iomap->private = NULL; 281 iomap->private = NULL;
316 282
317 if (!(map.m_flags & EROFS_MAP_MAPPED)) 283 if (!(map.m_flags & EROFS_MAP_MAPPED)) {
318 iomap->type = IOMAP_HOLE; 284 iomap->type = IOMAP_HOLE;
319 iomap->addr = IOMAP_NULL_ADDR; 285 iomap->addr = IOMAP_NULL_ADDR;
320 if (!iomap->length) 286 if (!iomap->length)
321 iomap->length = length 287 iomap->length = length;
322 return 0; 288 return 0;
323 } 289 }
324 290
325 if (map.m_flags & EROFS_MAP_META) { 291 if (map.m_flags & EROFS_MAP_META) {
326 void *ptr; 292 void *ptr;
327 struct erofs_buf buf = __EROFS 293 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
328 294
329 iomap->type = IOMAP_INLINE; 295 iomap->type = IOMAP_INLINE;
330 ptr = erofs_read_metabuf(&buf, !! 296 ptr = erofs_read_metabuf(&buf, inode->i_sb,
>> 297 erofs_blknr(mdev.m_pa), EROFS_KMAP);
331 if (IS_ERR(ptr)) 298 if (IS_ERR(ptr))
332 return PTR_ERR(ptr); 299 return PTR_ERR(ptr);
333 iomap->inline_data = ptr; !! 300 iomap->inline_data = ptr + erofs_blkoff(mdev.m_pa);
334 iomap->private = buf.base; 301 iomap->private = buf.base;
335 } else { 302 } else {
336 iomap->type = IOMAP_MAPPED; 303 iomap->type = IOMAP_MAPPED;
337 iomap->addr = mdev.m_pa; 304 iomap->addr = mdev.m_pa;
338 if (flags & IOMAP_DAX) 305 if (flags & IOMAP_DAX)
339 iomap->addr += mdev.m_ 306 iomap->addr += mdev.m_dax_part_off;
340 } 307 }
341 return 0; 308 return 0;
342 } 309 }
343 310
344 static int erofs_iomap_end(struct inode *inode 311 static int erofs_iomap_end(struct inode *inode, loff_t pos, loff_t length,
345 ssize_t written, unsigned int 312 ssize_t written, unsigned int flags, struct iomap *iomap)
346 { 313 {
347 void *ptr = iomap->private; 314 void *ptr = iomap->private;
348 315
349 if (ptr) { 316 if (ptr) {
350 struct erofs_buf buf = { 317 struct erofs_buf buf = {
351 .page = kmap_to_page(p 318 .page = kmap_to_page(ptr),
352 .base = ptr, 319 .base = ptr,
353 .kmap_type = EROFS_KMA 320 .kmap_type = EROFS_KMAP,
354 }; 321 };
355 322
356 DBG_BUGON(iomap->type != IOMAP 323 DBG_BUGON(iomap->type != IOMAP_INLINE);
357 erofs_put_metabuf(&buf); 324 erofs_put_metabuf(&buf);
358 } else { 325 } else {
359 DBG_BUGON(iomap->type == IOMAP 326 DBG_BUGON(iomap->type == IOMAP_INLINE);
360 } 327 }
361 return written; 328 return written;
362 } 329 }
363 330
364 static const struct iomap_ops erofs_iomap_ops 331 static const struct iomap_ops erofs_iomap_ops = {
365 .iomap_begin = erofs_iomap_begin, 332 .iomap_begin = erofs_iomap_begin,
366 .iomap_end = erofs_iomap_end, 333 .iomap_end = erofs_iomap_end,
367 }; 334 };
368 335
369 int erofs_fiemap(struct inode *inode, struct f 336 int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
370 u64 start, u64 len) 337 u64 start, u64 len)
371 { 338 {
372 if (erofs_inode_is_data_compressed(ERO 339 if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) {
373 #ifdef CONFIG_EROFS_FS_ZIP 340 #ifdef CONFIG_EROFS_FS_ZIP
374 return iomap_fiemap(inode, fie 341 return iomap_fiemap(inode, fieinfo, start, len,
375 &z_erofs_i 342 &z_erofs_iomap_report_ops);
376 #else 343 #else
377 return -EOPNOTSUPP; 344 return -EOPNOTSUPP;
378 #endif 345 #endif
379 } 346 }
380 return iomap_fiemap(inode, fieinfo, st 347 return iomap_fiemap(inode, fieinfo, start, len, &erofs_iomap_ops);
381 } 348 }
382 349
383 /* 350 /*
384 * since we dont have write or truncate flows, 351 * since we dont have write or truncate flows, so no inode
385 * locking needs to be held at the moment. 352 * locking needs to be held at the moment.
386 */ 353 */
387 static int erofs_read_folio(struct file *file, 354 static int erofs_read_folio(struct file *file, struct folio *folio)
388 { 355 {
389 return iomap_read_folio(folio, &erofs_ 356 return iomap_read_folio(folio, &erofs_iomap_ops);
390 } 357 }
391 358
392 static void erofs_readahead(struct readahead_c 359 static void erofs_readahead(struct readahead_control *rac)
393 { 360 {
394 return iomap_readahead(rac, &erofs_iom 361 return iomap_readahead(rac, &erofs_iomap_ops);
395 } 362 }
396 363
397 static sector_t erofs_bmap(struct address_spac 364 static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
398 { 365 {
399 return iomap_bmap(mapping, block, &ero 366 return iomap_bmap(mapping, block, &erofs_iomap_ops);
400 } 367 }
401 368
402 static ssize_t erofs_file_read_iter(struct kio 369 static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
403 { 370 {
404 struct inode *inode = file_inode(iocb- 371 struct inode *inode = file_inode(iocb->ki_filp);
405 372
406 /* no need taking (shared) inode lock 373 /* no need taking (shared) inode lock since it's a ro filesystem */
407 if (!iov_iter_count(to)) 374 if (!iov_iter_count(to))
408 return 0; 375 return 0;
409 376
410 #ifdef CONFIG_FS_DAX 377 #ifdef CONFIG_FS_DAX
411 if (IS_DAX(inode)) 378 if (IS_DAX(inode))
412 return dax_iomap_rw(iocb, to, 379 return dax_iomap_rw(iocb, to, &erofs_iomap_ops);
413 #endif 380 #endif
414 if (iocb->ki_flags & IOCB_DIRECT) { 381 if (iocb->ki_flags & IOCB_DIRECT) {
415 struct block_device *bdev = in 382 struct block_device *bdev = inode->i_sb->s_bdev;
416 unsigned int blksize_mask; 383 unsigned int blksize_mask;
417 384
418 if (bdev) 385 if (bdev)
419 blksize_mask = bdev_lo 386 blksize_mask = bdev_logical_block_size(bdev) - 1;
420 else 387 else
421 blksize_mask = i_block !! 388 blksize_mask = (1 << inode->i_blkbits) - 1;
422 389
423 if ((iocb->ki_pos | iov_iter_c 390 if ((iocb->ki_pos | iov_iter_count(to) |
424 iov_iter_alignment(to)) & 391 iov_iter_alignment(to)) & blksize_mask)
425 return -EINVAL; 392 return -EINVAL;
426 393
427 return iomap_dio_rw(iocb, to, 394 return iomap_dio_rw(iocb, to, &erofs_iomap_ops,
428 NULL, 0, N 395 NULL, 0, NULL, 0);
429 } 396 }
430 return filemap_read(iocb, to, 0); 397 return filemap_read(iocb, to, 0);
431 } 398 }
432 399
433 /* for uncompressed (aligned) files and raw ac 400 /* for uncompressed (aligned) files and raw access for other files */
434 const struct address_space_operations erofs_ao !! 401 const struct address_space_operations erofs_raw_access_aops = {
435 .read_folio = erofs_read_folio, 402 .read_folio = erofs_read_folio,
436 .readahead = erofs_readahead, 403 .readahead = erofs_readahead,
437 .bmap = erofs_bmap, 404 .bmap = erofs_bmap,
438 .direct_IO = noop_direct_IO, 405 .direct_IO = noop_direct_IO,
439 .release_folio = iomap_release_folio, <<
440 .invalidate_folio = iomap_invalidate_f <<
441 }; 406 };
442 407
443 #ifdef CONFIG_FS_DAX 408 #ifdef CONFIG_FS_DAX
444 static vm_fault_t erofs_dax_huge_fault(struct 409 static vm_fault_t erofs_dax_huge_fault(struct vm_fault *vmf,
445 unsigned int order) !! 410 enum page_entry_size pe_size)
446 { 411 {
447 return dax_iomap_fault(vmf, order, NUL !! 412 return dax_iomap_fault(vmf, pe_size, NULL, NULL, &erofs_iomap_ops);
448 } 413 }
449 414
450 static vm_fault_t erofs_dax_fault(struct vm_fa 415 static vm_fault_t erofs_dax_fault(struct vm_fault *vmf)
451 { 416 {
452 return erofs_dax_huge_fault(vmf, 0); !! 417 return erofs_dax_huge_fault(vmf, PE_SIZE_PTE);
453 } 418 }
454 419
455 static const struct vm_operations_struct erofs 420 static const struct vm_operations_struct erofs_dax_vm_ops = {
456 .fault = erofs_dax_fault, 421 .fault = erofs_dax_fault,
457 .huge_fault = erofs_dax_huge_fault 422 .huge_fault = erofs_dax_huge_fault,
458 }; 423 };
459 424
460 static int erofs_file_mmap(struct file *file, 425 static int erofs_file_mmap(struct file *file, struct vm_area_struct *vma)
461 { 426 {
462 if (!IS_DAX(file_inode(file))) 427 if (!IS_DAX(file_inode(file)))
463 return generic_file_readonly_m 428 return generic_file_readonly_mmap(file, vma);
464 429
465 if ((vma->vm_flags & VM_SHARED) && (vm 430 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
466 return -EINVAL; 431 return -EINVAL;
467 432
468 vma->vm_ops = &erofs_dax_vm_ops; 433 vma->vm_ops = &erofs_dax_vm_ops;
469 vm_flags_set(vma, VM_HUGEPAGE); !! 434 vma->vm_flags |= VM_HUGEPAGE;
470 return 0; 435 return 0;
471 } 436 }
472 #else 437 #else
473 #define erofs_file_mmap generic_file_readonly_ 438 #define erofs_file_mmap generic_file_readonly_mmap
474 #endif 439 #endif
475 440
476 const struct file_operations erofs_file_fops = 441 const struct file_operations erofs_file_fops = {
477 .llseek = generic_file_llseek, 442 .llseek = generic_file_llseek,
478 .read_iter = erofs_file_read_iter 443 .read_iter = erofs_file_read_iter,
479 .mmap = erofs_file_mmap, 444 .mmap = erofs_file_mmap,
480 .get_unmapped_area = thp_get_unmapped_ !! 445 .splice_read = generic_file_splice_read,
481 .splice_read = filemap_splice_read, <<
482 }; 446 };
483 447
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.