1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2012 Alexander Block. All rights reserved.
4 */
5
6 #include <linux/bsearch.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/sort.h>
10 #include <linux/mount.h>
11 #include <linux/xattr.h>
12 #include <linux/posix_acl_xattr.h>
13 #include <linux/radix-tree.h>
14 #include <linux/vmalloc.h>
15 #include <linux/string.h>
16 #include <linux/compat.h>
17 #include <linux/crc32c.h>
18 #include <linux/fsverity.h>
19
20 #include "send.h"
21 #include "ctree.h"
22 #include "backref.h"
23 #include "locking.h"
24 #include "disk-io.h"
25 #include "btrfs_inode.h"
26 #include "transaction.h"
27 #include "compression.h"
28 #include "print-tree.h"
29 #include "accessors.h"
30 #include "dir-item.h"
31 #include "file-item.h"
32 #include "ioctl.h"
33 #include "verity.h"
34 #include "lru_cache.h"
35
36 /*
37 * Maximum number of references an extent can have in order for us to attempt to
38 * issue clone operations instead of write operations. This currently exists to
39 * avoid hitting limitations of the backreference walking code (taking a lot of
40 * time and using too much memory for extents with large number of references).
41 */
42 #define SEND_MAX_EXTENT_REFS 1024
43
44 /*
45 * A fs_path is a helper to dynamically build path names with unknown size.
46 * It reallocates the internal buffer on demand.
47 * It allows fast adding of path elements on the right side (normal path) and
48 * fast adding to the left side (reversed path). A reversed path can also be
49 * unreversed if needed.
50 */
51 struct fs_path {
52 union {
53 struct {
54 char *start;
55 char *end;
56
57 char *buf;
58 unsigned short buf_len:15;
59 unsigned short reversed:1;
60 char inline_buf[];
61 };
62 /*
63 * Average path length does not exceed 200 bytes, we'll have
64 * better packing in the slab and higher chance to satisfy
65 * a allocation later during send.
66 */
67 char pad[256];
68 };
69 };
70 #define FS_PATH_INLINE_SIZE \
71 (sizeof(struct fs_path) - offsetof(struct fs_path, inline_buf))
72
73
74 /* reused for each extent */
75 struct clone_root {
76 struct btrfs_root *root;
77 u64 ino;
78 u64 offset;
79 u64 num_bytes;
80 bool found_ref;
81 };
82
83 #define SEND_MAX_NAME_CACHE_SIZE 256
84
85 /*
86 * Limit the root_ids array of struct backref_cache_entry to 17 elements.
87 * This makes the size of a cache entry to be exactly 192 bytes on x86_64, which
88 * can be satisfied from the kmalloc-192 slab, without wasting any space.
89 * The most common case is to have a single root for cloning, which corresponds
90 * to the send root. Having the user specify more than 16 clone roots is not
91 * common, and in such rare cases we simply don't use caching if the number of
92 * cloning roots that lead down to a leaf is more than 17.
93 */
94 #define SEND_MAX_BACKREF_CACHE_ROOTS 17
95
96 /*
97 * Max number of entries in the cache.
98 * With SEND_MAX_BACKREF_CACHE_ROOTS as 17, the size in bytes, excluding
99 * maple tree's internal nodes, is 24K.
100 */
101 #define SEND_MAX_BACKREF_CACHE_SIZE 128
102
103 /*
104 * A backref cache entry maps a leaf to a list of IDs of roots from which the
105 * leaf is accessible and we can use for clone operations.
106 * With SEND_MAX_BACKREF_CACHE_ROOTS as 12, each cache entry is 128 bytes (on
107 * x86_64).
108 */
109 struct backref_cache_entry {
110 struct btrfs_lru_cache_entry entry;
111 u64 root_ids[SEND_MAX_BACKREF_CACHE_ROOTS];
112 /* Number of valid elements in the root_ids array. */
113 int num_roots;
114 };
115
116 /* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
117 static_assert(offsetof(struct backref_cache_entry, entry) == 0);
118
119 /*
120 * Max number of entries in the cache that stores directories that were already
121 * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses
122 * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but
123 * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
124 */
125 #define SEND_MAX_DIR_CREATED_CACHE_SIZE 64
126
127 /*
128 * Max number of entries in the cache that stores directories that were already
129 * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses
130 * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but
131 * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
132 */
133 #define SEND_MAX_DIR_UTIMES_CACHE_SIZE 64
134
135 struct send_ctx {
136 struct file *send_filp;
137 loff_t send_off;
138 char *send_buf;
139 u32 send_size;
140 u32 send_max_size;
141 /*
142 * Whether BTRFS_SEND_A_DATA attribute was already added to current
143 * command (since protocol v2, data must be the last attribute).
144 */
145 bool put_data;
146 struct page **send_buf_pages;
147 u64 flags; /* 'flags' member of btrfs_ioctl_send_args is u64 */
148 /* Protocol version compatibility requested */
149 u32 proto;
150
151 struct btrfs_root *send_root;
152 struct btrfs_root *parent_root;
153 struct clone_root *clone_roots;
154 int clone_roots_cnt;
155
156 /* current state of the compare_tree call */
157 struct btrfs_path *left_path;
158 struct btrfs_path *right_path;
159 struct btrfs_key *cmp_key;
160
161 /*
162 * Keep track of the generation of the last transaction that was used
163 * for relocating a block group. This is periodically checked in order
164 * to detect if a relocation happened since the last check, so that we
165 * don't operate on stale extent buffers for nodes (level >= 1) or on
166 * stale disk_bytenr values of file extent items.
167 */
168 u64 last_reloc_trans;
169
170 /*
171 * infos of the currently processed inode. In case of deleted inodes,
172 * these are the values from the deleted inode.
173 */
174 u64 cur_ino;
175 u64 cur_inode_gen;
176 u64 cur_inode_size;
177 u64 cur_inode_mode;
178 u64 cur_inode_rdev;
179 u64 cur_inode_last_extent;
180 u64 cur_inode_next_write_offset;
181 bool cur_inode_new;
182 bool cur_inode_new_gen;
183 bool cur_inode_deleted;
184 bool ignore_cur_inode;
185 bool cur_inode_needs_verity;
186 void *verity_descriptor;
187
188 u64 send_progress;
189
190 struct list_head new_refs;
191 struct list_head deleted_refs;
192
193 struct btrfs_lru_cache name_cache;
194
195 /*
196 * The inode we are currently processing. It's not NULL only when we
197 * need to issue write commands for data extents from this inode.
198 */
199 struct inode *cur_inode;
200 struct file_ra_state ra;
201 u64 page_cache_clear_start;
202 bool clean_page_cache;
203
204 /*
205 * We process inodes by their increasing order, so if before an
206 * incremental send we reverse the parent/child relationship of
207 * directories such that a directory with a lower inode number was
208 * the parent of a directory with a higher inode number, and the one
209 * becoming the new parent got renamed too, we can't rename/move the
210 * directory with lower inode number when we finish processing it - we
211 * must process the directory with higher inode number first, then
212 * rename/move it and then rename/move the directory with lower inode
213 * number. Example follows.
214 *
215 * Tree state when the first send was performed:
216 *
217 * .
218 * |-- a (ino 257)
219 * |-- b (ino 258)
220 * |
221 * |
222 * |-- c (ino 259)
223 * | |-- d (ino 260)
224 * |
225 * |-- c2 (ino 261)
226 *
227 * Tree state when the second (incremental) send is performed:
228 *
229 * .
230 * |-- a (ino 257)
231 * |-- b (ino 258)
232 * |-- c2 (ino 261)
233 * |-- d2 (ino 260)
234 * |-- cc (ino 259)
235 *
236 * The sequence of steps that lead to the second state was:
237 *
238 * mv /a/b/c/d /a/b/c2/d2
239 * mv /a/b/c /a/b/c2/d2/cc
240 *
241 * "c" has lower inode number, but we can't move it (2nd mv operation)
242 * before we move "d", which has higher inode number.
243 *
244 * So we just memorize which move/rename operations must be performed
245 * later when their respective parent is processed and moved/renamed.
246 */
247
248 /* Indexed by parent directory inode number. */
249 struct rb_root pending_dir_moves;
250
251 /*
252 * Reverse index, indexed by the inode number of a directory that
253 * is waiting for the move/rename of its immediate parent before its
254 * own move/rename can be performed.
255 */
256 struct rb_root waiting_dir_moves;
257
258 /*
259 * A directory that is going to be rm'ed might have a child directory
260 * which is in the pending directory moves index above. In this case,
261 * the directory can only be removed after the move/rename of its child
262 * is performed. Example:
263 *
264 * Parent snapshot:
265 *
266 * . (ino 256)
267 * |-- a/ (ino 257)
268 * |-- b/ (ino 258)
269 * |-- c/ (ino 259)
270 * | |-- x/ (ino 260)
271 * |
272 * |-- y/ (ino 261)
273 *
274 * Send snapshot:
275 *
276 * . (ino 256)
277 * |-- a/ (ino 257)
278 * |-- b/ (ino 258)
279 * |-- YY/ (ino 261)
280 * |-- x/ (ino 260)
281 *
282 * Sequence of steps that lead to the send snapshot:
283 * rm -f /a/b/c/foo.txt
284 * mv /a/b/y /a/b/YY
285 * mv /a/b/c/x /a/b/YY
286 * rmdir /a/b/c
287 *
288 * When the child is processed, its move/rename is delayed until its
289 * parent is processed (as explained above), but all other operations
290 * like update utimes, chown, chgrp, etc, are performed and the paths
291 * that it uses for those operations must use the orphanized name of
292 * its parent (the directory we're going to rm later), so we need to
293 * memorize that name.
294 *
295 * Indexed by the inode number of the directory to be deleted.
296 */
297 struct rb_root orphan_dirs;
298
299 struct rb_root rbtree_new_refs;
300 struct rb_root rbtree_deleted_refs;
301
302 struct btrfs_lru_cache backref_cache;
303 u64 backref_cache_last_reloc_trans;
304
305 struct btrfs_lru_cache dir_created_cache;
306 struct btrfs_lru_cache dir_utimes_cache;
307 };
308
309 struct pending_dir_move {
310 struct rb_node node;
311 struct list_head list;
312 u64 parent_ino;
313 u64 ino;
314 u64 gen;
315 struct list_head update_refs;
316 };
317
318 struct waiting_dir_move {
319 struct rb_node node;
320 u64 ino;
321 /*
322 * There might be some directory that could not be removed because it
323 * was waiting for this directory inode to be moved first. Therefore
324 * after this directory is moved, we can try to rmdir the ino rmdir_ino.
325 */
326 u64 rmdir_ino;
327 u64 rmdir_gen;
328 bool orphanized;
329 };
330
331 struct orphan_dir_info {
332 struct rb_node node;
333 u64 ino;
334 u64 gen;
335 u64 last_dir_index_offset;
336 u64 dir_high_seq_ino;
337 };
338
339 struct name_cache_entry {
340 /*
341 * The key in the entry is an inode number, and the generation matches
342 * the inode's generation.
343 */
344 struct btrfs_lru_cache_entry entry;
345 u64 parent_ino;
346 u64 parent_gen;
347 int ret;
348 int need_later_update;
349 int name_len;
350 char name[];
351 };
352
353 /* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
354 static_assert(offsetof(struct name_cache_entry, entry) == 0);
355
356 #define ADVANCE 1
357 #define ADVANCE_ONLY_NEXT -1
358
359 enum btrfs_compare_tree_result {
360 BTRFS_COMPARE_TREE_NEW,
361 BTRFS_COMPARE_TREE_DELETED,
362 BTRFS_COMPARE_TREE_CHANGED,
363 BTRFS_COMPARE_TREE_SAME,
364 };
365
366 __cold
367 static void inconsistent_snapshot_error(struct send_ctx *sctx,
368 enum btrfs_compare_tree_result result,
369 const char *what)
370 {
371 const char *result_string;
372
373 switch (result) {
374 case BTRFS_COMPARE_TREE_NEW:
375 result_string = "new";
376 break;
377 case BTRFS_COMPARE_TREE_DELETED:
378 result_string = "deleted";
379 break;
380 case BTRFS_COMPARE_TREE_CHANGED:
381 result_string = "updated";
382 break;
383 case BTRFS_COMPARE_TREE_SAME:
384 ASSERT(0);
385 result_string = "unchanged";
386 break;
387 default:
388 ASSERT(0);
389 result_string = "unexpected";
390 }
391
392 btrfs_err(sctx->send_root->fs_info,
393 "Send: inconsistent snapshot, found %s %s for inode %llu without updated inode item, send root is %llu, parent root is %llu",
394 result_string, what, sctx->cmp_key->objectid,
395 btrfs_root_id(sctx->send_root),
396 (sctx->parent_root ? btrfs_root_id(sctx->parent_root) : 0));
397 }
398
399 __maybe_unused
400 static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd)
401 {
402 switch (sctx->proto) {
403 case 1: return cmd <= BTRFS_SEND_C_MAX_V1;
404 case 2: return cmd <= BTRFS_SEND_C_MAX_V2;
405 case 3: return cmd <= BTRFS_SEND_C_MAX_V3;
406 default: return false;
407 }
408 }
409
410 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino);
411
412 static struct waiting_dir_move *
413 get_waiting_dir_move(struct send_ctx *sctx, u64 ino);
414
415 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen);
416
417 static int need_send_hole(struct send_ctx *sctx)
418 {
419 return (sctx->parent_root && !sctx->cur_inode_new &&
420 !sctx->cur_inode_new_gen && !sctx->cur_inode_deleted &&
421 S_ISREG(sctx->cur_inode_mode));
422 }
423
424 static void fs_path_reset(struct fs_path *p)
425 {
426 if (p->reversed) {
427 p->start = p->buf + p->buf_len - 1;
428 p->end = p->start;
429 *p->start = 0;
430 } else {
431 p->start = p->buf;
432 p->end = p->start;
433 *p->start = 0;
434 }
435 }
436
437 static struct fs_path *fs_path_alloc(void)
438 {
439 struct fs_path *p;
440
441 p = kmalloc(sizeof(*p), GFP_KERNEL);
442 if (!p)
443 return NULL;
444 p->reversed = 0;
445 p->buf = p->inline_buf;
446 p->buf_len = FS_PATH_INLINE_SIZE;
447 fs_path_reset(p);
448 return p;
449 }
450
451 static struct fs_path *fs_path_alloc_reversed(void)
452 {
453 struct fs_path *p;
454
455 p = fs_path_alloc();
456 if (!p)
457 return NULL;
458 p->reversed = 1;
459 fs_path_reset(p);
460 return p;
461 }
462
463 static void fs_path_free(struct fs_path *p)
464 {
465 if (!p)
466 return;
467 if (p->buf != p->inline_buf)
468 kfree(p->buf);
469 kfree(p);
470 }
471
472 static int fs_path_len(struct fs_path *p)
473 {
474 return p->end - p->start;
475 }
476
477 static int fs_path_ensure_buf(struct fs_path *p, int len)
478 {
479 char *tmp_buf;
480 int path_len;
481 int old_buf_len;
482
483 len++;
484
485 if (p->buf_len >= len)
486 return 0;
487
488 if (len > PATH_MAX) {
489 WARN_ON(1);
490 return -ENOMEM;
491 }
492
493 path_len = p->end - p->start;
494 old_buf_len = p->buf_len;
495
496 /*
497 * Allocate to the next largest kmalloc bucket size, to let
498 * the fast path happen most of the time.
499 */
500 len = kmalloc_size_roundup(len);
501 /*
502 * First time the inline_buf does not suffice
503 */
504 if (p->buf == p->inline_buf) {
505 tmp_buf = kmalloc(len, GFP_KERNEL);
506 if (tmp_buf)
507 memcpy(tmp_buf, p->buf, old_buf_len);
508 } else {
509 tmp_buf = krealloc(p->buf, len, GFP_KERNEL);
510 }
511 if (!tmp_buf)
512 return -ENOMEM;
513 p->buf = tmp_buf;
514 p->buf_len = len;
515
516 if (p->reversed) {
517 tmp_buf = p->buf + old_buf_len - path_len - 1;
518 p->end = p->buf + p->buf_len - 1;
519 p->start = p->end - path_len;
520 memmove(p->start, tmp_buf, path_len + 1);
521 } else {
522 p->start = p->buf;
523 p->end = p->start + path_len;
524 }
525 return 0;
526 }
527
528 static int fs_path_prepare_for_add(struct fs_path *p, int name_len,
529 char **prepared)
530 {
531 int ret;
532 int new_len;
533
534 new_len = p->end - p->start + name_len;
535 if (p->start != p->end)
536 new_len++;
537 ret = fs_path_ensure_buf(p, new_len);
538 if (ret < 0)
539 goto out;
540
541 if (p->reversed) {
542 if (p->start != p->end)
543 *--p->start = '/';
544 p->start -= name_len;
545 *prepared = p->start;
546 } else {
547 if (p->start != p->end)
548 *p->end++ = '/';
549 *prepared = p->end;
550 p->end += name_len;
551 *p->end = 0;
552 }
553
554 out:
555 return ret;
556 }
557
558 static int fs_path_add(struct fs_path *p, const char *name, int name_len)
559 {
560 int ret;
561 char *prepared;
562
563 ret = fs_path_prepare_for_add(p, name_len, &prepared);
564 if (ret < 0)
565 goto out;
566 memcpy(prepared, name, name_len);
567
568 out:
569 return ret;
570 }
571
572 static int fs_path_add_path(struct fs_path *p, struct fs_path *p2)
573 {
574 int ret;
575 char *prepared;
576
577 ret = fs_path_prepare_for_add(p, p2->end - p2->start, &prepared);
578 if (ret < 0)
579 goto out;
580 memcpy(prepared, p2->start, p2->end - p2->start);
581
582 out:
583 return ret;
584 }
585
586 static int fs_path_add_from_extent_buffer(struct fs_path *p,
587 struct extent_buffer *eb,
588 unsigned long off, int len)
589 {
590 int ret;
591 char *prepared;
592
593 ret = fs_path_prepare_for_add(p, len, &prepared);
594 if (ret < 0)
595 goto out;
596
597 read_extent_buffer(eb, prepared, off, len);
598
599 out:
600 return ret;
601 }
602
603 static int fs_path_copy(struct fs_path *p, struct fs_path *from)
604 {
605 p->reversed = from->reversed;
606 fs_path_reset(p);
607
608 return fs_path_add_path(p, from);
609 }
610
611 static void fs_path_unreverse(struct fs_path *p)
612 {
613 char *tmp;
614 int len;
615
616 if (!p->reversed)
617 return;
618
619 tmp = p->start;
620 len = p->end - p->start;
621 p->start = p->buf;
622 p->end = p->start + len;
623 memmove(p->start, tmp, len + 1);
624 p->reversed = 0;
625 }
626
627 static struct btrfs_path *alloc_path_for_send(void)
628 {
629 struct btrfs_path *path;
630
631 path = btrfs_alloc_path();
632 if (!path)
633 return NULL;
634 path->search_commit_root = 1;
635 path->skip_locking = 1;
636 path->need_commit_sem = 1;
637 return path;
638 }
639
640 static int write_buf(struct file *filp, const void *buf, u32 len, loff_t *off)
641 {
642 int ret;
643 u32 pos = 0;
644
645 while (pos < len) {
646 ret = kernel_write(filp, buf + pos, len - pos, off);
647 if (ret < 0)
648 return ret;
649 if (ret == 0)
650 return -EIO;
651 pos += ret;
652 }
653
654 return 0;
655 }
656
657 static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len)
658 {
659 struct btrfs_tlv_header *hdr;
660 int total_len = sizeof(*hdr) + len;
661 int left = sctx->send_max_size - sctx->send_size;
662
663 if (WARN_ON_ONCE(sctx->put_data))
664 return -EINVAL;
665
666 if (unlikely(left < total_len))
667 return -EOVERFLOW;
668
669 hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size);
670 put_unaligned_le16(attr, &hdr->tlv_type);
671 put_unaligned_le16(len, &hdr->tlv_len);
672 memcpy(hdr + 1, data, len);
673 sctx->send_size += total_len;
674
675 return 0;
676 }
677
678 #define TLV_PUT_DEFINE_INT(bits) \
679 static int tlv_put_u##bits(struct send_ctx *sctx, \
680 u##bits attr, u##bits value) \
681 { \
682 __le##bits __tmp = cpu_to_le##bits(value); \
683 return tlv_put(sctx, attr, &__tmp, sizeof(__tmp)); \
684 }
685
686 TLV_PUT_DEFINE_INT(8)
687 TLV_PUT_DEFINE_INT(32)
688 TLV_PUT_DEFINE_INT(64)
689
690 static int tlv_put_string(struct send_ctx *sctx, u16 attr,
691 const char *str, int len)
692 {
693 if (len == -1)
694 len = strlen(str);
695 return tlv_put(sctx, attr, str, len);
696 }
697
698 static int tlv_put_uuid(struct send_ctx *sctx, u16 attr,
699 const u8 *uuid)
700 {
701 return tlv_put(sctx, attr, uuid, BTRFS_UUID_SIZE);
702 }
703
704 static int tlv_put_btrfs_timespec(struct send_ctx *sctx, u16 attr,
705 struct extent_buffer *eb,
706 struct btrfs_timespec *ts)
707 {
708 struct btrfs_timespec bts;
709 read_extent_buffer(eb, &bts, (unsigned long)ts, sizeof(bts));
710 return tlv_put(sctx, attr, &bts, sizeof(bts));
711 }
712
713
714 #define TLV_PUT(sctx, attrtype, data, attrlen) \
715 do { \
716 ret = tlv_put(sctx, attrtype, data, attrlen); \
717 if (ret < 0) \
718 goto tlv_put_failure; \
719 } while (0)
720
721 #define TLV_PUT_INT(sctx, attrtype, bits, value) \
722 do { \
723 ret = tlv_put_u##bits(sctx, attrtype, value); \
724 if (ret < 0) \
725 goto tlv_put_failure; \
726 } while (0)
727
728 #define TLV_PUT_U8(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 8, data)
729 #define TLV_PUT_U16(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 16, data)
730 #define TLV_PUT_U32(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 32, data)
731 #define TLV_PUT_U64(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 64, data)
732 #define TLV_PUT_STRING(sctx, attrtype, str, len) \
733 do { \
734 ret = tlv_put_string(sctx, attrtype, str, len); \
735 if (ret < 0) \
736 goto tlv_put_failure; \
737 } while (0)
738 #define TLV_PUT_PATH(sctx, attrtype, p) \
739 do { \
740 ret = tlv_put_string(sctx, attrtype, p->start, \
741 p->end - p->start); \
742 if (ret < 0) \
743 goto tlv_put_failure; \
744 } while(0)
745 #define TLV_PUT_UUID(sctx, attrtype, uuid) \
746 do { \
747 ret = tlv_put_uuid(sctx, attrtype, uuid); \
748 if (ret < 0) \
749 goto tlv_put_failure; \
750 } while (0)
751 #define TLV_PUT_BTRFS_TIMESPEC(sctx, attrtype, eb, ts) \
752 do { \
753 ret = tlv_put_btrfs_timespec(sctx, attrtype, eb, ts); \
754 if (ret < 0) \
755 goto tlv_put_failure; \
756 } while (0)
757
758 static int send_header(struct send_ctx *sctx)
759 {
760 struct btrfs_stream_header hdr;
761
762 strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
763 hdr.version = cpu_to_le32(sctx->proto);
764 return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
765 &sctx->send_off);
766 }
767
768 /*
769 * For each command/item we want to send to userspace, we call this function.
770 */
771 static int begin_cmd(struct send_ctx *sctx, int cmd)
772 {
773 struct btrfs_cmd_header *hdr;
774
775 if (WARN_ON(!sctx->send_buf))
776 return -EINVAL;
777
778 if (unlikely(sctx->send_size != 0)) {
779 btrfs_err(sctx->send_root->fs_info,
780 "send: command header buffer not empty cmd %d offset %llu",
781 cmd, sctx->send_off);
782 return -EINVAL;
783 }
784
785 sctx->send_size += sizeof(*hdr);
786 hdr = (struct btrfs_cmd_header *)sctx->send_buf;
787 put_unaligned_le16(cmd, &hdr->cmd);
788
789 return 0;
790 }
791
792 static int send_cmd(struct send_ctx *sctx)
793 {
794 int ret;
795 struct btrfs_cmd_header *hdr;
796 u32 crc;
797
798 hdr = (struct btrfs_cmd_header *)sctx->send_buf;
799 put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len);
800 put_unaligned_le32(0, &hdr->crc);
801
802 crc = crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size);
803 put_unaligned_le32(crc, &hdr->crc);
804
805 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
806 &sctx->send_off);
807
808 sctx->send_size = 0;
809 sctx->put_data = false;
810
811 return ret;
812 }
813
814 /*
815 * Sends a move instruction to user space
816 */
817 static int send_rename(struct send_ctx *sctx,
818 struct fs_path *from, struct fs_path *to)
819 {
820 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
821 int ret;
822
823 btrfs_debug(fs_info, "send_rename %s -> %s", from->start, to->start);
824
825 ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);
826 if (ret < 0)
827 goto out;
828
829 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, from);
830 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_TO, to);
831
832 ret = send_cmd(sctx);
833
834 tlv_put_failure:
835 out:
836 return ret;
837 }
838
839 /*
840 * Sends a link instruction to user space
841 */
842 static int send_link(struct send_ctx *sctx,
843 struct fs_path *path, struct fs_path *lnk)
844 {
845 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
846 int ret;
847
848 btrfs_debug(fs_info, "send_link %s -> %s", path->start, lnk->start);
849
850 ret = begin_cmd(sctx, BTRFS_SEND_C_LINK);
851 if (ret < 0)
852 goto out;
853
854 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
855 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, lnk);
856
857 ret = send_cmd(sctx);
858
859 tlv_put_failure:
860 out:
861 return ret;
862 }
863
864 /*
865 * Sends an unlink instruction to user space
866 */
867 static int send_unlink(struct send_ctx *sctx, struct fs_path *path)
868 {
869 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
870 int ret;
871
872 btrfs_debug(fs_info, "send_unlink %s", path->start);
873
874 ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);
875 if (ret < 0)
876 goto out;
877
878 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
879
880 ret = send_cmd(sctx);
881
882 tlv_put_failure:
883 out:
884 return ret;
885 }
886
887 /*
888 * Sends a rmdir instruction to user space
889 */
890 static int send_rmdir(struct send_ctx *sctx, struct fs_path *path)
891 {
892 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
893 int ret;
894
895 btrfs_debug(fs_info, "send_rmdir %s", path->start);
896
897 ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR);
898 if (ret < 0)
899 goto out;
900
901 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
902
903 ret = send_cmd(sctx);
904
905 tlv_put_failure:
906 out:
907 return ret;
908 }
909
910 struct btrfs_inode_info {
911 u64 size;
912 u64 gen;
913 u64 mode;
914 u64 uid;
915 u64 gid;
916 u64 rdev;
917 u64 fileattr;
918 u64 nlink;
919 };
920
921 /*
922 * Helper function to retrieve some fields from an inode item.
923 */
924 static int get_inode_info(struct btrfs_root *root, u64 ino,
925 struct btrfs_inode_info *info)
926 {
927 int ret;
928 struct btrfs_path *path;
929 struct btrfs_inode_item *ii;
930 struct btrfs_key key;
931
932 path = alloc_path_for_send();
933 if (!path)
934 return -ENOMEM;
935
936 key.objectid = ino;
937 key.type = BTRFS_INODE_ITEM_KEY;
938 key.offset = 0;
939 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
940 if (ret) {
941 if (ret > 0)
942 ret = -ENOENT;
943 goto out;
944 }
945
946 if (!info)
947 goto out;
948
949 ii = btrfs_item_ptr(path->nodes[0], path->slots[0],
950 struct btrfs_inode_item);
951 info->size = btrfs_inode_size(path->nodes[0], ii);
952 info->gen = btrfs_inode_generation(path->nodes[0], ii);
953 info->mode = btrfs_inode_mode(path->nodes[0], ii);
954 info->uid = btrfs_inode_uid(path->nodes[0], ii);
955 info->gid = btrfs_inode_gid(path->nodes[0], ii);
956 info->rdev = btrfs_inode_rdev(path->nodes[0], ii);
957 info->nlink = btrfs_inode_nlink(path->nodes[0], ii);
958 /*
959 * Transfer the unchanged u64 value of btrfs_inode_item::flags, that's
960 * otherwise logically split to 32/32 parts.
961 */
962 info->fileattr = btrfs_inode_flags(path->nodes[0], ii);
963
964 out:
965 btrfs_free_path(path);
966 return ret;
967 }
968
969 static int get_inode_gen(struct btrfs_root *root, u64 ino, u64 *gen)
970 {
971 int ret;
972 struct btrfs_inode_info info = { 0 };
973
974 ASSERT(gen);
975
976 ret = get_inode_info(root, ino, &info);
977 *gen = info.gen;
978 return ret;
979 }
980
981 typedef int (*iterate_inode_ref_t)(int num, u64 dir, int index,
982 struct fs_path *p,
983 void *ctx);
984
985 /*
986 * Helper function to iterate the entries in ONE btrfs_inode_ref or
987 * btrfs_inode_extref.
988 * The iterate callback may return a non zero value to stop iteration. This can
989 * be a negative value for error codes or 1 to simply stop it.
990 *
991 * path must point to the INODE_REF or INODE_EXTREF when called.
992 */
993 static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path,
994 struct btrfs_key *found_key, int resolve,
995 iterate_inode_ref_t iterate, void *ctx)
996 {
997 struct extent_buffer *eb = path->nodes[0];
998 struct btrfs_inode_ref *iref;
999 struct btrfs_inode_extref *extref;
1000 struct btrfs_path *tmp_path;
1001 struct fs_path *p;
1002 u32 cur = 0;
1003 u32 total;
1004 int slot = path->slots[0];
1005 u32 name_len;
1006 char *start;
1007 int ret = 0;
1008 int num = 0;
1009 int index;
1010 u64 dir;
1011 unsigned long name_off;
1012 unsigned long elem_size;
1013 unsigned long ptr;
1014
1015 p = fs_path_alloc_reversed();
1016 if (!p)
1017 return -ENOMEM;
1018
1019 tmp_path = alloc_path_for_send();
1020 if (!tmp_path) {
1021 fs_path_free(p);
1022 return -ENOMEM;
1023 }
1024
1025
1026 if (found_key->type == BTRFS_INODE_REF_KEY) {
1027 ptr = (unsigned long)btrfs_item_ptr(eb, slot,
1028 struct btrfs_inode_ref);
1029 total = btrfs_item_size(eb, slot);
1030 elem_size = sizeof(*iref);
1031 } else {
1032 ptr = btrfs_item_ptr_offset(eb, slot);
1033 total = btrfs_item_size(eb, slot);
1034 elem_size = sizeof(*extref);
1035 }
1036
1037 while (cur < total) {
1038 fs_path_reset(p);
1039
1040 if (found_key->type == BTRFS_INODE_REF_KEY) {
1041 iref = (struct btrfs_inode_ref *)(ptr + cur);
1042 name_len = btrfs_inode_ref_name_len(eb, iref);
1043 name_off = (unsigned long)(iref + 1);
1044 index = btrfs_inode_ref_index(eb, iref);
1045 dir = found_key->offset;
1046 } else {
1047 extref = (struct btrfs_inode_extref *)(ptr + cur);
1048 name_len = btrfs_inode_extref_name_len(eb, extref);
1049 name_off = (unsigned long)&extref->name;
1050 index = btrfs_inode_extref_index(eb, extref);
1051 dir = btrfs_inode_extref_parent(eb, extref);
1052 }
1053
1054 if (resolve) {
1055 start = btrfs_ref_to_path(root, tmp_path, name_len,
1056 name_off, eb, dir,
1057 p->buf, p->buf_len);
1058 if (IS_ERR(start)) {
1059 ret = PTR_ERR(start);
1060 goto out;
1061 }
1062 if (start < p->buf) {
1063 /* overflow , try again with larger buffer */
1064 ret = fs_path_ensure_buf(p,
1065 p->buf_len + p->buf - start);
1066 if (ret < 0)
1067 goto out;
1068 start = btrfs_ref_to_path(root, tmp_path,
1069 name_len, name_off,
1070 eb, dir,
1071 p->buf, p->buf_len);
1072 if (IS_ERR(start)) {
1073 ret = PTR_ERR(start);
1074 goto out;
1075 }
1076 if (unlikely(start < p->buf)) {
1077 btrfs_err(root->fs_info,
1078 "send: path ref buffer underflow for key (%llu %u %llu)",
1079 found_key->objectid,
1080 found_key->type,
1081 found_key->offset);
1082 ret = -EINVAL;
1083 goto out;
1084 }
1085 }
1086 p->start = start;
1087 } else {
1088 ret = fs_path_add_from_extent_buffer(p, eb, name_off,
1089 name_len);
1090 if (ret < 0)
1091 goto out;
1092 }
1093
1094 cur += elem_size + name_len;
1095 ret = iterate(num, dir, index, p, ctx);
1096 if (ret)
1097 goto out;
1098 num++;
1099 }
1100
1101 out:
1102 btrfs_free_path(tmp_path);
1103 fs_path_free(p);
1104 return ret;
1105 }
1106
1107 typedef int (*iterate_dir_item_t)(int num, struct btrfs_key *di_key,
1108 const char *name, int name_len,
1109 const char *data, int data_len,
1110 void *ctx);
1111
1112 /*
1113 * Helper function to iterate the entries in ONE btrfs_dir_item.
1114 * The iterate callback may return a non zero value to stop iteration. This can
1115 * be a negative value for error codes or 1 to simply stop it.
1116 *
1117 * path must point to the dir item when called.
1118 */
1119 static int iterate_dir_item(struct btrfs_root *root, struct btrfs_path *path,
1120 iterate_dir_item_t iterate, void *ctx)
1121 {
1122 int ret = 0;
1123 struct extent_buffer *eb;
1124 struct btrfs_dir_item *di;
1125 struct btrfs_key di_key;
1126 char *buf = NULL;
1127 int buf_len;
1128 u32 name_len;
1129 u32 data_len;
1130 u32 cur;
1131 u32 len;
1132 u32 total;
1133 int slot;
1134 int num;
1135
1136 /*
1137 * Start with a small buffer (1 page). If later we end up needing more
1138 * space, which can happen for xattrs on a fs with a leaf size greater
1139 * then the page size, attempt to increase the buffer. Typically xattr
1140 * values are small.
1141 */
1142 buf_len = PATH_MAX;
1143 buf = kmalloc(buf_len, GFP_KERNEL);
1144 if (!buf) {
1145 ret = -ENOMEM;
1146 goto out;
1147 }
1148
1149 eb = path->nodes[0];
1150 slot = path->slots[0];
1151 di = btrfs_item_ptr(eb, slot, struct btrfs_dir_item);
1152 cur = 0;
1153 len = 0;
1154 total = btrfs_item_size(eb, slot);
1155
1156 num = 0;
1157 while (cur < total) {
1158 name_len = btrfs_dir_name_len(eb, di);
1159 data_len = btrfs_dir_data_len(eb, di);
1160 btrfs_dir_item_key_to_cpu(eb, di, &di_key);
1161
1162 if (btrfs_dir_ftype(eb, di) == BTRFS_FT_XATTR) {
1163 if (name_len > XATTR_NAME_MAX) {
1164 ret = -ENAMETOOLONG;
1165 goto out;
1166 }
1167 if (name_len + data_len >
1168 BTRFS_MAX_XATTR_SIZE(root->fs_info)) {
1169 ret = -E2BIG;
1170 goto out;
1171 }
1172 } else {
1173 /*
1174 * Path too long
1175 */
1176 if (name_len + data_len > PATH_MAX) {
1177 ret = -ENAMETOOLONG;
1178 goto out;
1179 }
1180 }
1181
1182 if (name_len + data_len > buf_len) {
1183 buf_len = name_len + data_len;
1184 if (is_vmalloc_addr(buf)) {
1185 vfree(buf);
1186 buf = NULL;
1187 } else {
1188 char *tmp = krealloc(buf, buf_len,
1189 GFP_KERNEL | __GFP_NOWARN);
1190
1191 if (!tmp)
1192 kfree(buf);
1193 buf = tmp;
1194 }
1195 if (!buf) {
1196 buf = kvmalloc(buf_len, GFP_KERNEL);
1197 if (!buf) {
1198 ret = -ENOMEM;
1199 goto out;
1200 }
1201 }
1202 }
1203
1204 read_extent_buffer(eb, buf, (unsigned long)(di + 1),
1205 name_len + data_len);
1206
1207 len = sizeof(*di) + name_len + data_len;
1208 di = (struct btrfs_dir_item *)((char *)di + len);
1209 cur += len;
1210
1211 ret = iterate(num, &di_key, buf, name_len, buf + name_len,
1212 data_len, ctx);
1213 if (ret < 0)
1214 goto out;
1215 if (ret) {
1216 ret = 0;
1217 goto out;
1218 }
1219
1220 num++;
1221 }
1222
1223 out:
1224 kvfree(buf);
1225 return ret;
1226 }
1227
1228 static int __copy_first_ref(int num, u64 dir, int index,
1229 struct fs_path *p, void *ctx)
1230 {
1231 int ret;
1232 struct fs_path *pt = ctx;
1233
1234 ret = fs_path_copy(pt, p);
1235 if (ret < 0)
1236 return ret;
1237
1238 /* we want the first only */
1239 return 1;
1240 }
1241
1242 /*
1243 * Retrieve the first path of an inode. If an inode has more then one
1244 * ref/hardlink, this is ignored.
1245 */
1246 static int get_inode_path(struct btrfs_root *root,
1247 u64 ino, struct fs_path *path)
1248 {
1249 int ret;
1250 struct btrfs_key key, found_key;
1251 struct btrfs_path *p;
1252
1253 p = alloc_path_for_send();
1254 if (!p)
1255 return -ENOMEM;
1256
1257 fs_path_reset(path);
1258
1259 key.objectid = ino;
1260 key.type = BTRFS_INODE_REF_KEY;
1261 key.offset = 0;
1262
1263 ret = btrfs_search_slot_for_read(root, &key, p, 1, 0);
1264 if (ret < 0)
1265 goto out;
1266 if (ret) {
1267 ret = 1;
1268 goto out;
1269 }
1270 btrfs_item_key_to_cpu(p->nodes[0], &found_key, p->slots[0]);
1271 if (found_key.objectid != ino ||
1272 (found_key.type != BTRFS_INODE_REF_KEY &&
1273 found_key.type != BTRFS_INODE_EXTREF_KEY)) {
1274 ret = -ENOENT;
1275 goto out;
1276 }
1277
1278 ret = iterate_inode_ref(root, p, &found_key, 1,
1279 __copy_first_ref, path);
1280 if (ret < 0)
1281 goto out;
1282 ret = 0;
1283
1284 out:
1285 btrfs_free_path(p);
1286 return ret;
1287 }
1288
1289 struct backref_ctx {
1290 struct send_ctx *sctx;
1291
1292 /* number of total found references */
1293 u64 found;
1294
1295 /*
1296 * used for clones found in send_root. clones found behind cur_objectid
1297 * and cur_offset are not considered as allowed clones.
1298 */
1299 u64 cur_objectid;
1300 u64 cur_offset;
1301
1302 /* may be truncated in case it's the last extent in a file */
1303 u64 extent_len;
1304
1305 /* The bytenr the file extent item we are processing refers to. */
1306 u64 bytenr;
1307 /* The owner (root id) of the data backref for the current extent. */
1308 u64 backref_owner;
1309 /* The offset of the data backref for the current extent. */
1310 u64 backref_offset;
1311 };
1312
1313 static int __clone_root_cmp_bsearch(const void *key, const void *elt)
1314 {
1315 u64 root = (u64)(uintptr_t)key;
1316 const struct clone_root *cr = elt;
1317
1318 if (root < btrfs_root_id(cr->root))
1319 return -1;
1320 if (root > btrfs_root_id(cr->root))
1321 return 1;
1322 return 0;
1323 }
1324
1325 static int __clone_root_cmp_sort(const void *e1, const void *e2)
1326 {
1327 const struct clone_root *cr1 = e1;
1328 const struct clone_root *cr2 = e2;
1329
1330 if (btrfs_root_id(cr1->root) < btrfs_root_id(cr2->root))
1331 return -1;
1332 if (btrfs_root_id(cr1->root) > btrfs_root_id(cr2->root))
1333 return 1;
1334 return 0;
1335 }
1336
1337 /*
1338 * Called for every backref that is found for the current extent.
1339 * Results are collected in sctx->clone_roots->ino/offset.
1340 */
1341 static int iterate_backrefs(u64 ino, u64 offset, u64 num_bytes, u64 root_id,
1342 void *ctx_)
1343 {
1344 struct backref_ctx *bctx = ctx_;
1345 struct clone_root *clone_root;
1346
1347 /* First check if the root is in the list of accepted clone sources */
1348 clone_root = bsearch((void *)(uintptr_t)root_id, bctx->sctx->clone_roots,
1349 bctx->sctx->clone_roots_cnt,
1350 sizeof(struct clone_root),
1351 __clone_root_cmp_bsearch);
1352 if (!clone_root)
1353 return 0;
1354
1355 /* This is our own reference, bail out as we can't clone from it. */
1356 if (clone_root->root == bctx->sctx->send_root &&
1357 ino == bctx->cur_objectid &&
1358 offset == bctx->cur_offset)
1359 return 0;
1360
1361 /*
1362 * Make sure we don't consider clones from send_root that are
1363 * behind the current inode/offset.
1364 */
1365 if (clone_root->root == bctx->sctx->send_root) {
1366 /*
1367 * If the source inode was not yet processed we can't issue a
1368 * clone operation, as the source extent does not exist yet at
1369 * the destination of the stream.
1370 */
1371 if (ino > bctx->cur_objectid)
1372 return 0;
1373 /*
1374 * We clone from the inode currently being sent as long as the
1375 * source extent is already processed, otherwise we could try
1376 * to clone from an extent that does not exist yet at the
1377 * destination of the stream.
1378 */
1379 if (ino == bctx->cur_objectid &&
1380 offset + bctx->extent_len >
1381 bctx->sctx->cur_inode_next_write_offset)
1382 return 0;
1383 }
1384
1385 bctx->found++;
1386 clone_root->found_ref = true;
1387
1388 /*
1389 * If the given backref refers to a file extent item with a larger
1390 * number of bytes than what we found before, use the new one so that
1391 * we clone more optimally and end up doing less writes and getting
1392 * less exclusive, non-shared extents at the destination.
1393 */
1394 if (num_bytes > clone_root->num_bytes) {
1395 clone_root->ino = ino;
1396 clone_root->offset = offset;
1397 clone_root->num_bytes = num_bytes;
1398
1399 /*
1400 * Found a perfect candidate, so there's no need to continue
1401 * backref walking.
1402 */
1403 if (num_bytes >= bctx->extent_len)
1404 return BTRFS_ITERATE_EXTENT_INODES_STOP;
1405 }
1406
1407 return 0;
1408 }
1409
1410 static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx,
1411 const u64 **root_ids_ret, int *root_count_ret)
1412 {
1413 struct backref_ctx *bctx = ctx;
1414 struct send_ctx *sctx = bctx->sctx;
1415 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1416 const u64 key = leaf_bytenr >> fs_info->sectorsize_bits;
1417 struct btrfs_lru_cache_entry *raw_entry;
1418 struct backref_cache_entry *entry;
1419
1420 if (sctx->backref_cache.size == 0)
1421 return false;
1422
1423 /*
1424 * If relocation happened since we first filled the cache, then we must
1425 * empty the cache and can not use it, because even though we operate on
1426 * read-only roots, their leaves and nodes may have been reallocated and
1427 * now be used for different nodes/leaves of the same tree or some other
1428 * tree.
1429 *
1430 * We are called from iterate_extent_inodes() while either holding a
1431 * transaction handle or holding fs_info->commit_root_sem, so no need
1432 * to take any lock here.
1433 */
1434 if (fs_info->last_reloc_trans > sctx->backref_cache_last_reloc_trans) {
1435 btrfs_lru_cache_clear(&sctx->backref_cache);
1436 return false;
1437 }
1438
1439 raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key, 0);
1440 if (!raw_entry)
1441 return false;
1442
1443 entry = container_of(raw_entry, struct backref_cache_entry, entry);
1444 *root_ids_ret = entry->root_ids;
1445 *root_count_ret = entry->num_roots;
1446
1447 return true;
1448 }
1449
1450 static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids,
1451 void *ctx)
1452 {
1453 struct backref_ctx *bctx = ctx;
1454 struct send_ctx *sctx = bctx->sctx;
1455 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1456 struct backref_cache_entry *new_entry;
1457 struct ulist_iterator uiter;
1458 struct ulist_node *node;
1459 int ret;
1460
1461 /*
1462 * We're called while holding a transaction handle or while holding
1463 * fs_info->commit_root_sem (at iterate_extent_inodes()), so must do a
1464 * NOFS allocation.
1465 */
1466 new_entry = kmalloc(sizeof(struct backref_cache_entry), GFP_NOFS);
1467 /* No worries, cache is optional. */
1468 if (!new_entry)
1469 return;
1470
1471 new_entry->entry.key = leaf_bytenr >> fs_info->sectorsize_bits;
1472 new_entry->entry.gen = 0;
1473 new_entry->num_roots = 0;
1474 ULIST_ITER_INIT(&uiter);
1475 while ((node = ulist_next(root_ids, &uiter)) != NULL) {
1476 const u64 root_id = node->val;
1477 struct clone_root *root;
1478
1479 root = bsearch((void *)(uintptr_t)root_id, sctx->clone_roots,
1480 sctx->clone_roots_cnt, sizeof(struct clone_root),
1481 __clone_root_cmp_bsearch);
1482 if (!root)
1483 continue;
1484
1485 /* Too many roots, just exit, no worries as caching is optional. */
1486 if (new_entry->num_roots >= SEND_MAX_BACKREF_CACHE_ROOTS) {
1487 kfree(new_entry);
1488 return;
1489 }
1490
1491 new_entry->root_ids[new_entry->num_roots] = root_id;
1492 new_entry->num_roots++;
1493 }
1494
1495 /*
1496 * We may have not added any roots to the new cache entry, which means
1497 * none of the roots is part of the list of roots from which we are
1498 * allowed to clone. Cache the new entry as it's still useful to avoid
1499 * backref walking to determine which roots have a path to the leaf.
1500 *
1501 * Also use GFP_NOFS because we're called while holding a transaction
1502 * handle or while holding fs_info->commit_root_sem.
1503 */
1504 ret = btrfs_lru_cache_store(&sctx->backref_cache, &new_entry->entry,
1505 GFP_NOFS);
1506 ASSERT(ret == 0 || ret == -ENOMEM);
1507 if (ret) {
1508 /* Caching is optional, no worries. */
1509 kfree(new_entry);
1510 return;
1511 }
1512
1513 /*
1514 * We are called from iterate_extent_inodes() while either holding a
1515 * transaction handle or holding fs_info->commit_root_sem, so no need
1516 * to take any lock here.
1517 */
1518 if (sctx->backref_cache.size == 1)
1519 sctx->backref_cache_last_reloc_trans = fs_info->last_reloc_trans;
1520 }
1521
1522 static int check_extent_item(u64 bytenr, const struct btrfs_extent_item *ei,
1523 const struct extent_buffer *leaf, void *ctx)
1524 {
1525 const u64 refs = btrfs_extent_refs(leaf, ei);
1526 const struct backref_ctx *bctx = ctx;
1527 const struct send_ctx *sctx = bctx->sctx;
1528
1529 if (bytenr == bctx->bytenr) {
1530 const u64 flags = btrfs_extent_flags(leaf, ei);
1531
1532 if (WARN_ON(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK))
1533 return -EUCLEAN;
1534
1535 /*
1536 * If we have only one reference and only the send root as a
1537 * clone source - meaning no clone roots were given in the
1538 * struct btrfs_ioctl_send_args passed to the send ioctl - then
1539 * it's our reference and there's no point in doing backref
1540 * walking which is expensive, so exit early.
1541 */
1542 if (refs == 1 && sctx->clone_roots_cnt == 1)
1543 return -ENOENT;
1544 }
1545
1546 /*
1547 * Backreference walking (iterate_extent_inodes() below) is currently
1548 * too expensive when an extent has a large number of references, both
1549 * in time spent and used memory. So for now just fallback to write
1550 * operations instead of clone operations when an extent has more than
1551 * a certain amount of references.
1552 */
1553 if (refs > SEND_MAX_EXTENT_REFS)
1554 return -ENOENT;
1555
1556 return 0;
1557 }
1558
1559 static bool skip_self_data_ref(u64 root, u64 ino, u64 offset, void *ctx)
1560 {
1561 const struct backref_ctx *bctx = ctx;
1562
1563 if (ino == bctx->cur_objectid &&
1564 root == bctx->backref_owner &&
1565 offset == bctx->backref_offset)
1566 return true;
1567
1568 return false;
1569 }
1570
1571 /*
1572 * Given an inode, offset and extent item, it finds a good clone for a clone
1573 * instruction. Returns -ENOENT when none could be found. The function makes
1574 * sure that the returned clone is usable at the point where sending is at the
1575 * moment. This means, that no clones are accepted which lie behind the current
1576 * inode+offset.
1577 *
1578 * path must point to the extent item when called.
1579 */
1580 static int find_extent_clone(struct send_ctx *sctx,
1581 struct btrfs_path *path,
1582 u64 ino, u64 data_offset,
1583 u64 ino_size,
1584 struct clone_root **found)
1585 {
1586 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1587 int ret;
1588 int extent_type;
1589 u64 logical;
1590 u64 disk_byte;
1591 u64 num_bytes;
1592 struct btrfs_file_extent_item *fi;
1593 struct extent_buffer *eb = path->nodes[0];
1594 struct backref_ctx backref_ctx = { 0 };
1595 struct btrfs_backref_walk_ctx backref_walk_ctx = { 0 };
1596 struct clone_root *cur_clone_root;
1597 int compressed;
1598 u32 i;
1599
1600 /*
1601 * With fallocate we can get prealloc extents beyond the inode's i_size,
1602 * so we don't do anything here because clone operations can not clone
1603 * to a range beyond i_size without increasing the i_size of the
1604 * destination inode.
1605 */
1606 if (data_offset >= ino_size)
1607 return 0;
1608
1609 fi = btrfs_item_ptr(eb, path->slots[0], struct btrfs_file_extent_item);
1610 extent_type = btrfs_file_extent_type(eb, fi);
1611 if (extent_type == BTRFS_FILE_EXTENT_INLINE)
1612 return -ENOENT;
1613
1614 disk_byte = btrfs_file_extent_disk_bytenr(eb, fi);
1615 if (disk_byte == 0)
1616 return -ENOENT;
1617
1618 compressed = btrfs_file_extent_compression(eb, fi);
1619 num_bytes = btrfs_file_extent_num_bytes(eb, fi);
1620 logical = disk_byte + btrfs_file_extent_offset(eb, fi);
1621
1622 /*
1623 * Setup the clone roots.
1624 */
1625 for (i = 0; i < sctx->clone_roots_cnt; i++) {
1626 cur_clone_root = sctx->clone_roots + i;
1627 cur_clone_root->ino = (u64)-1;
1628 cur_clone_root->offset = 0;
1629 cur_clone_root->num_bytes = 0;
1630 cur_clone_root->found_ref = false;
1631 }
1632
1633 backref_ctx.sctx = sctx;
1634 backref_ctx.cur_objectid = ino;
1635 backref_ctx.cur_offset = data_offset;
1636 backref_ctx.bytenr = disk_byte;
1637 /*
1638 * Use the header owner and not the send root's id, because in case of a
1639 * snapshot we can have shared subtrees.
1640 */
1641 backref_ctx.backref_owner = btrfs_header_owner(eb);
1642 backref_ctx.backref_offset = data_offset - btrfs_file_extent_offset(eb, fi);
1643
1644 /*
1645 * The last extent of a file may be too large due to page alignment.
1646 * We need to adjust extent_len in this case so that the checks in
1647 * iterate_backrefs() work.
1648 */
1649 if (data_offset + num_bytes >= ino_size)
1650 backref_ctx.extent_len = ino_size - data_offset;
1651 else
1652 backref_ctx.extent_len = num_bytes;
1653
1654 /*
1655 * Now collect all backrefs.
1656 */
1657 backref_walk_ctx.bytenr = disk_byte;
1658 if (compressed == BTRFS_COMPRESS_NONE)
1659 backref_walk_ctx.extent_item_pos = btrfs_file_extent_offset(eb, fi);
1660 backref_walk_ctx.fs_info = fs_info;
1661 backref_walk_ctx.cache_lookup = lookup_backref_cache;
1662 backref_walk_ctx.cache_store = store_backref_cache;
1663 backref_walk_ctx.indirect_ref_iterator = iterate_backrefs;
1664 backref_walk_ctx.check_extent_item = check_extent_item;
1665 backref_walk_ctx.user_ctx = &backref_ctx;
1666
1667 /*
1668 * If have a single clone root, then it's the send root and we can tell
1669 * the backref walking code to skip our own backref and not resolve it,
1670 * since we can not use it for cloning - the source and destination
1671 * ranges can't overlap and in case the leaf is shared through a subtree
1672 * due to snapshots, we can't use those other roots since they are not
1673 * in the list of clone roots.
1674 */
1675 if (sctx->clone_roots_cnt == 1)
1676 backref_walk_ctx.skip_data_ref = skip_self_data_ref;
1677
1678 ret = iterate_extent_inodes(&backref_walk_ctx, true, iterate_backrefs,
1679 &backref_ctx);
1680 if (ret < 0)
1681 return ret;
1682
1683 down_read(&fs_info->commit_root_sem);
1684 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
1685 /*
1686 * A transaction commit for a transaction in which block group
1687 * relocation was done just happened.
1688 * The disk_bytenr of the file extent item we processed is
1689 * possibly stale, referring to the extent's location before
1690 * relocation. So act as if we haven't found any clone sources
1691 * and fallback to write commands, which will read the correct
1692 * data from the new extent location. Otherwise we will fail
1693 * below because we haven't found our own back reference or we
1694 * could be getting incorrect sources in case the old extent
1695 * was already reallocated after the relocation.
1696 */
1697 up_read(&fs_info->commit_root_sem);
1698 return -ENOENT;
1699 }
1700 up_read(&fs_info->commit_root_sem);
1701
1702 btrfs_debug(fs_info,
1703 "find_extent_clone: data_offset=%llu, ino=%llu, num_bytes=%llu, logical=%llu",
1704 data_offset, ino, num_bytes, logical);
1705
1706 if (!backref_ctx.found) {
1707 btrfs_debug(fs_info, "no clones found");
1708 return -ENOENT;
1709 }
1710
1711 cur_clone_root = NULL;
1712 for (i = 0; i < sctx->clone_roots_cnt; i++) {
1713 struct clone_root *clone_root = &sctx->clone_roots[i];
1714
1715 if (!clone_root->found_ref)
1716 continue;
1717
1718 /*
1719 * Choose the root from which we can clone more bytes, to
1720 * minimize write operations and therefore have more extent
1721 * sharing at the destination (the same as in the source).
1722 */
1723 if (!cur_clone_root ||
1724 clone_root->num_bytes > cur_clone_root->num_bytes) {
1725 cur_clone_root = clone_root;
1726
1727 /*
1728 * We found an optimal clone candidate (any inode from
1729 * any root is fine), so we're done.
1730 */
1731 if (clone_root->num_bytes >= backref_ctx.extent_len)
1732 break;
1733 }
1734 }
1735
1736 if (cur_clone_root) {
1737 *found = cur_clone_root;
1738 ret = 0;
1739 } else {
1740 ret = -ENOENT;
1741 }
1742
1743 return ret;
1744 }
1745
1746 static int read_symlink(struct btrfs_root *root,
1747 u64 ino,
1748 struct fs_path *dest)
1749 {
1750 int ret;
1751 struct btrfs_path *path;
1752 struct btrfs_key key;
1753 struct btrfs_file_extent_item *ei;
1754 u8 type;
1755 u8 compression;
1756 unsigned long off;
1757 int len;
1758
1759 path = alloc_path_for_send();
1760 if (!path)
1761 return -ENOMEM;
1762
1763 key.objectid = ino;
1764 key.type = BTRFS_EXTENT_DATA_KEY;
1765 key.offset = 0;
1766 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1767 if (ret < 0)
1768 goto out;
1769 if (ret) {
1770 /*
1771 * An empty symlink inode. Can happen in rare error paths when
1772 * creating a symlink (transaction committed before the inode
1773 * eviction handler removed the symlink inode items and a crash
1774 * happened in between or the subvol was snapshoted in between).
1775 * Print an informative message to dmesg/syslog so that the user
1776 * can delete the symlink.
1777 */
1778 btrfs_err(root->fs_info,
1779 "Found empty symlink inode %llu at root %llu",
1780 ino, btrfs_root_id(root));
1781 ret = -EIO;
1782 goto out;
1783 }
1784
1785 ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
1786 struct btrfs_file_extent_item);
1787 type = btrfs_file_extent_type(path->nodes[0], ei);
1788 if (unlikely(type != BTRFS_FILE_EXTENT_INLINE)) {
1789 ret = -EUCLEAN;
1790 btrfs_crit(root->fs_info,
1791 "send: found symlink extent that is not inline, ino %llu root %llu extent type %d",
1792 ino, btrfs_root_id(root), type);
1793 goto out;
1794 }
1795 compression = btrfs_file_extent_compression(path->nodes[0], ei);
1796 if (unlikely(compression != BTRFS_COMPRESS_NONE)) {
1797 ret = -EUCLEAN;
1798 btrfs_crit(root->fs_info,
1799 "send: found symlink extent with compression, ino %llu root %llu compression type %d",
1800 ino, btrfs_root_id(root), compression);
1801 goto out;
1802 }
1803
1804 off = btrfs_file_extent_inline_start(ei);
1805 len = btrfs_file_extent_ram_bytes(path->nodes[0], ei);
1806
1807 ret = fs_path_add_from_extent_buffer(dest, path->nodes[0], off, len);
1808
1809 out:
1810 btrfs_free_path(path);
1811 return ret;
1812 }
1813
1814 /*
1815 * Helper function to generate a file name that is unique in the root of
1816 * send_root and parent_root. This is used to generate names for orphan inodes.
1817 */
1818 static int gen_unique_name(struct send_ctx *sctx,
1819 u64 ino, u64 gen,
1820 struct fs_path *dest)
1821 {
1822 int ret = 0;
1823 struct btrfs_path *path;
1824 struct btrfs_dir_item *di;
1825 char tmp[64];
1826 int len;
1827 u64 idx = 0;
1828
1829 path = alloc_path_for_send();
1830 if (!path)
1831 return -ENOMEM;
1832
1833 while (1) {
1834 struct fscrypt_str tmp_name;
1835
1836 len = snprintf(tmp, sizeof(tmp), "o%llu-%llu-%llu",
1837 ino, gen, idx);
1838 ASSERT(len < sizeof(tmp));
1839 tmp_name.name = tmp;
1840 tmp_name.len = strlen(tmp);
1841
1842 di = btrfs_lookup_dir_item(NULL, sctx->send_root,
1843 path, BTRFS_FIRST_FREE_OBJECTID,
1844 &tmp_name, 0);
1845 btrfs_release_path(path);
1846 if (IS_ERR(di)) {
1847 ret = PTR_ERR(di);
1848 goto out;
1849 }
1850 if (di) {
1851 /* not unique, try again */
1852 idx++;
1853 continue;
1854 }
1855
1856 if (!sctx->parent_root) {
1857 /* unique */
1858 ret = 0;
1859 break;
1860 }
1861
1862 di = btrfs_lookup_dir_item(NULL, sctx->parent_root,
1863 path, BTRFS_FIRST_FREE_OBJECTID,
1864 &tmp_name, 0);
1865 btrfs_release_path(path);
1866 if (IS_ERR(di)) {
1867 ret = PTR_ERR(di);
1868 goto out;
1869 }
1870 if (di) {
1871 /* not unique, try again */
1872 idx++;
1873 continue;
1874 }
1875 /* unique */
1876 break;
1877 }
1878
1879 ret = fs_path_add(dest, tmp, strlen(tmp));
1880
1881 out:
1882 btrfs_free_path(path);
1883 return ret;
1884 }
1885
1886 enum inode_state {
1887 inode_state_no_change,
1888 inode_state_will_create,
1889 inode_state_did_create,
1890 inode_state_will_delete,
1891 inode_state_did_delete,
1892 };
1893
1894 static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen,
1895 u64 *send_gen, u64 *parent_gen)
1896 {
1897 int ret;
1898 int left_ret;
1899 int right_ret;
1900 u64 left_gen;
1901 u64 right_gen = 0;
1902 struct btrfs_inode_info info;
1903
1904 ret = get_inode_info(sctx->send_root, ino, &info);
1905 if (ret < 0 && ret != -ENOENT)
1906 goto out;
1907 left_ret = (info.nlink == 0) ? -ENOENT : ret;
1908 left_gen = info.gen;
1909 if (send_gen)
1910 *send_gen = ((left_ret == -ENOENT) ? 0 : info.gen);
1911
1912 if (!sctx->parent_root) {
1913 right_ret = -ENOENT;
1914 } else {
1915 ret = get_inode_info(sctx->parent_root, ino, &info);
1916 if (ret < 0 && ret != -ENOENT)
1917 goto out;
1918 right_ret = (info.nlink == 0) ? -ENOENT : ret;
1919 right_gen = info.gen;
1920 if (parent_gen)
1921 *parent_gen = ((right_ret == -ENOENT) ? 0 : info.gen);
1922 }
1923
1924 if (!left_ret && !right_ret) {
1925 if (left_gen == gen && right_gen == gen) {
1926 ret = inode_state_no_change;
1927 } else if (left_gen == gen) {
1928 if (ino < sctx->send_progress)
1929 ret = inode_state_did_create;
1930 else
1931 ret = inode_state_will_create;
1932 } else if (right_gen == gen) {
1933 if (ino < sctx->send_progress)
1934 ret = inode_state_did_delete;
1935 else
1936 ret = inode_state_will_delete;
1937 } else {
1938 ret = -ENOENT;
1939 }
1940 } else if (!left_ret) {
1941 if (left_gen == gen) {
1942 if (ino < sctx->send_progress)
1943 ret = inode_state_did_create;
1944 else
1945 ret = inode_state_will_create;
1946 } else {
1947 ret = -ENOENT;
1948 }
1949 } else if (!right_ret) {
1950 if (right_gen == gen) {
1951 if (ino < sctx->send_progress)
1952 ret = inode_state_did_delete;
1953 else
1954 ret = inode_state_will_delete;
1955 } else {
1956 ret = -ENOENT;
1957 }
1958 } else {
1959 ret = -ENOENT;
1960 }
1961
1962 out:
1963 return ret;
1964 }
1965
1966 static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen,
1967 u64 *send_gen, u64 *parent_gen)
1968 {
1969 int ret;
1970
1971 if (ino == BTRFS_FIRST_FREE_OBJECTID)
1972 return 1;
1973
1974 ret = get_cur_inode_state(sctx, ino, gen, send_gen, parent_gen);
1975 if (ret < 0)
1976 goto out;
1977
1978 if (ret == inode_state_no_change ||
1979 ret == inode_state_did_create ||
1980 ret == inode_state_will_delete)
1981 ret = 1;
1982 else
1983 ret = 0;
1984
1985 out:
1986 return ret;
1987 }
1988
1989 /*
1990 * Helper function to lookup a dir item in a dir.
1991 */
1992 static int lookup_dir_item_inode(struct btrfs_root *root,
1993 u64 dir, const char *name, int name_len,
1994 u64 *found_inode)
1995 {
1996 int ret = 0;
1997 struct btrfs_dir_item *di;
1998 struct btrfs_key key;
1999 struct btrfs_path *path;
2000 struct fscrypt_str name_str = FSTR_INIT((char *)name, name_len);
2001
2002 path = alloc_path_for_send();
2003 if (!path)
2004 return -ENOMEM;
2005
2006 di = btrfs_lookup_dir_item(NULL, root, path, dir, &name_str, 0);
2007 if (IS_ERR_OR_NULL(di)) {
2008 ret = di ? PTR_ERR(di) : -ENOENT;
2009 goto out;
2010 }
2011 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
2012 if (key.type == BTRFS_ROOT_ITEM_KEY) {
2013 ret = -ENOENT;
2014 goto out;
2015 }
2016 *found_inode = key.objectid;
2017
2018 out:
2019 btrfs_free_path(path);
2020 return ret;
2021 }
2022
2023 /*
2024 * Looks up the first btrfs_inode_ref of a given ino. It returns the parent dir,
2025 * generation of the parent dir and the name of the dir entry.
2026 */
2027 static int get_first_ref(struct btrfs_root *root, u64 ino,
2028 u64 *dir, u64 *dir_gen, struct fs_path *name)
2029 {
2030 int ret;
2031 struct btrfs_key key;
2032 struct btrfs_key found_key;
2033 struct btrfs_path *path;
2034 int len;
2035 u64 parent_dir;
2036
2037 path = alloc_path_for_send();
2038 if (!path)
2039 return -ENOMEM;
2040
2041 key.objectid = ino;
2042 key.type = BTRFS_INODE_REF_KEY;
2043 key.offset = 0;
2044
2045 ret = btrfs_search_slot_for_read(root, &key, path, 1, 0);
2046 if (ret < 0)
2047 goto out;
2048 if (!ret)
2049 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
2050 path->slots[0]);
2051 if (ret || found_key.objectid != ino ||
2052 (found_key.type != BTRFS_INODE_REF_KEY &&
2053 found_key.type != BTRFS_INODE_EXTREF_KEY)) {
2054 ret = -ENOENT;
2055 goto out;
2056 }
2057
2058 if (found_key.type == BTRFS_INODE_REF_KEY) {
2059 struct btrfs_inode_ref *iref;
2060 iref = btrfs_item_ptr(path->nodes[0], path->slots[0],
2061 struct btrfs_inode_ref);
2062 len = btrfs_inode_ref_name_len(path->nodes[0], iref);
2063 ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
2064 (unsigned long)(iref + 1),
2065 len);
2066 parent_dir = found_key.offset;
2067 } else {
2068 struct btrfs_inode_extref *extref;
2069 extref = btrfs_item_ptr(path->nodes[0], path->slots[0],
2070 struct btrfs_inode_extref);
2071 len = btrfs_inode_extref_name_len(path->nodes[0], extref);
2072 ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
2073 (unsigned long)&extref->name, len);
2074 parent_dir = btrfs_inode_extref_parent(path->nodes[0], extref);
2075 }
2076 if (ret < 0)
2077 goto out;
2078 btrfs_release_path(path);
2079
2080 if (dir_gen) {
2081 ret = get_inode_gen(root, parent_dir, dir_gen);
2082 if (ret < 0)
2083 goto out;
2084 }
2085
2086 *dir = parent_dir;
2087
2088 out:
2089 btrfs_free_path(path);
2090 return ret;
2091 }
2092
2093 static int is_first_ref(struct btrfs_root *root,
2094 u64 ino, u64 dir,
2095 const char *name, int name_len)
2096 {
2097 int ret;
2098 struct fs_path *tmp_name;
2099 u64 tmp_dir;
2100
2101 tmp_name = fs_path_alloc();
2102 if (!tmp_name)
2103 return -ENOMEM;
2104
2105 ret = get_first_ref(root, ino, &tmp_dir, NULL, tmp_name);
2106 if (ret < 0)
2107 goto out;
2108
2109 if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) {
2110 ret = 0;
2111 goto out;
2112 }
2113
2114 ret = !memcmp(tmp_name->start, name, name_len);
2115
2116 out:
2117 fs_path_free(tmp_name);
2118 return ret;
2119 }
2120
2121 /*
2122 * Used by process_recorded_refs to determine if a new ref would overwrite an
2123 * already existing ref. In case it detects an overwrite, it returns the
2124 * inode/gen in who_ino/who_gen.
2125 * When an overwrite is detected, process_recorded_refs does proper orphanizing
2126 * to make sure later references to the overwritten inode are possible.
2127 * Orphanizing is however only required for the first ref of an inode.
2128 * process_recorded_refs does an additional is_first_ref check to see if
2129 * orphanizing is really required.
2130 */
2131 static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen,
2132 const char *name, int name_len,
2133 u64 *who_ino, u64 *who_gen, u64 *who_mode)
2134 {
2135 int ret;
2136 u64 parent_root_dir_gen;
2137 u64 other_inode = 0;
2138 struct btrfs_inode_info info;
2139
2140 if (!sctx->parent_root)
2141 return 0;
2142
2143 ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen);
2144 if (ret <= 0)
2145 return 0;
2146
2147 /*
2148 * If we have a parent root we need to verify that the parent dir was
2149 * not deleted and then re-created, if it was then we have no overwrite
2150 * and we can just unlink this entry.
2151 *
2152 * @parent_root_dir_gen was set to 0 if the inode does not exist in the
2153 * parent root.
2154 */
2155 if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID &&
2156 parent_root_dir_gen != dir_gen)
2157 return 0;
2158
2159 ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len,
2160 &other_inode);
2161 if (ret == -ENOENT)
2162 return 0;
2163 else if (ret < 0)
2164 return ret;
2165
2166 /*
2167 * Check if the overwritten ref was already processed. If yes, the ref
2168 * was already unlinked/moved, so we can safely assume that we will not
2169 * overwrite anything at this point in time.
2170 */
2171 if (other_inode > sctx->send_progress ||
2172 is_waiting_for_move(sctx, other_inode)) {
2173 ret = get_inode_info(sctx->parent_root, other_inode, &info);
2174 if (ret < 0)
2175 return ret;
2176
2177 *who_ino = other_inode;
2178 *who_gen = info.gen;
2179 *who_mode = info.mode;
2180 return 1;
2181 }
2182
2183 return 0;
2184 }
2185
2186 /*
2187 * Checks if the ref was overwritten by an already processed inode. This is
2188 * used by __get_cur_name_and_parent to find out if the ref was orphanized and
2189 * thus the orphan name needs be used.
2190 * process_recorded_refs also uses it to avoid unlinking of refs that were
2191 * overwritten.
2192 */
2193 static int did_overwrite_ref(struct send_ctx *sctx,
2194 u64 dir, u64 dir_gen,
2195 u64 ino, u64 ino_gen,
2196 const char *name, int name_len)
2197 {
2198 int ret;
2199 u64 ow_inode;
2200 u64 ow_gen = 0;
2201 u64 send_root_dir_gen;
2202
2203 if (!sctx->parent_root)
2204 return 0;
2205
2206 ret = is_inode_existent(sctx, dir, dir_gen, &send_root_dir_gen, NULL);
2207 if (ret <= 0)
2208 return ret;
2209
2210 /*
2211 * @send_root_dir_gen was set to 0 if the inode does not exist in the
2212 * send root.
2213 */
2214 if (dir != BTRFS_FIRST_FREE_OBJECTID && send_root_dir_gen != dir_gen)
2215 return 0;
2216
2217 /* check if the ref was overwritten by another ref */
2218 ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len,
2219 &ow_inode);
2220 if (ret == -ENOENT) {
2221 /* was never and will never be overwritten */
2222 return 0;
2223 } else if (ret < 0) {
2224 return ret;
2225 }
2226
2227 if (ow_inode == ino) {
2228 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2229 if (ret < 0)
2230 return ret;
2231
2232 /* It's the same inode, so no overwrite happened. */
2233 if (ow_gen == ino_gen)
2234 return 0;
2235 }
2236
2237 /*
2238 * We know that it is or will be overwritten. Check this now.
2239 * The current inode being processed might have been the one that caused
2240 * inode 'ino' to be orphanized, therefore check if ow_inode matches
2241 * the current inode being processed.
2242 */
2243 if (ow_inode < sctx->send_progress)
2244 return 1;
2245
2246 if (ino != sctx->cur_ino && ow_inode == sctx->cur_ino) {
2247 if (ow_gen == 0) {
2248 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2249 if (ret < 0)
2250 return ret;
2251 }
2252 if (ow_gen == sctx->cur_inode_gen)
2253 return 1;
2254 }
2255
2256 return 0;
2257 }
2258
2259 /*
2260 * Same as did_overwrite_ref, but also checks if it is the first ref of an inode
2261 * that got overwritten. This is used by process_recorded_refs to determine
2262 * if it has to use the path as returned by get_cur_path or the orphan name.
2263 */
2264 static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen)
2265 {
2266 int ret = 0;
2267 struct fs_path *name = NULL;
2268 u64 dir;
2269 u64 dir_gen;
2270
2271 if (!sctx->parent_root)
2272 goto out;
2273
2274 name = fs_path_alloc();
2275 if (!name)
2276 return -ENOMEM;
2277
2278 ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name);
2279 if (ret < 0)
2280 goto out;
2281
2282 ret = did_overwrite_ref(sctx, dir, dir_gen, ino, gen,
2283 name->start, fs_path_len(name));
2284
2285 out:
2286 fs_path_free(name);
2287 return ret;
2288 }
2289
2290 static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx,
2291 u64 ino, u64 gen)
2292 {
2293 struct btrfs_lru_cache_entry *entry;
2294
2295 entry = btrfs_lru_cache_lookup(&sctx->name_cache, ino, gen);
2296 if (!entry)
2297 return NULL;
2298
2299 return container_of(entry, struct name_cache_entry, entry);
2300 }
2301
2302 /*
2303 * Used by get_cur_path for each ref up to the root.
2304 * Returns 0 if it succeeded.
2305 * Returns 1 if the inode is not existent or got overwritten. In that case, the
2306 * name is an orphan name. This instructs get_cur_path to stop iterating. If 1
2307 * is returned, parent_ino/parent_gen are not guaranteed to be valid.
2308 * Returns <0 in case of error.
2309 */
2310 static int __get_cur_name_and_parent(struct send_ctx *sctx,
2311 u64 ino, u64 gen,
2312 u64 *parent_ino,
2313 u64 *parent_gen,
2314 struct fs_path *dest)
2315 {
2316 int ret;
2317 int nce_ret;
2318 struct name_cache_entry *nce;
2319
2320 /*
2321 * First check if we already did a call to this function with the same
2322 * ino/gen. If yes, check if the cache entry is still up-to-date. If yes
2323 * return the cached result.
2324 */
2325 nce = name_cache_search(sctx, ino, gen);
2326 if (nce) {
2327 if (ino < sctx->send_progress && nce->need_later_update) {
2328 btrfs_lru_cache_remove(&sctx->name_cache, &nce->entry);
2329 nce = NULL;
2330 } else {
2331 *parent_ino = nce->parent_ino;
2332 *parent_gen = nce->parent_gen;
2333 ret = fs_path_add(dest, nce->name, nce->name_len);
2334 if (ret < 0)
2335 goto out;
2336 ret = nce->ret;
2337 goto out;
2338 }
2339 }
2340
2341 /*
2342 * If the inode is not existent yet, add the orphan name and return 1.
2343 * This should only happen for the parent dir that we determine in
2344 * record_new_ref_if_needed().
2345 */
2346 ret = is_inode_existent(sctx, ino, gen, NULL, NULL);
2347 if (ret < 0)
2348 goto out;
2349
2350 if (!ret) {
2351 ret = gen_unique_name(sctx, ino, gen, dest);
2352 if (ret < 0)
2353 goto out;
2354 ret = 1;
2355 goto out_cache;
2356 }
2357
2358 /*
2359 * Depending on whether the inode was already processed or not, use
2360 * send_root or parent_root for ref lookup.
2361 */
2362 if (ino < sctx->send_progress)
2363 ret = get_first_ref(sctx->send_root, ino,
2364 parent_ino, parent_gen, dest);
2365 else
2366 ret = get_first_ref(sctx->parent_root, ino,
2367 parent_ino, parent_gen, dest);
2368 if (ret < 0)
2369 goto out;
2370
2371 /*
2372 * Check if the ref was overwritten by an inode's ref that was processed
2373 * earlier. If yes, treat as orphan and return 1.
2374 */
2375 ret = did_overwrite_ref(sctx, *parent_ino, *parent_gen, ino, gen,
2376 dest->start, dest->end - dest->start);
2377 if (ret < 0)
2378 goto out;
2379 if (ret) {
2380 fs_path_reset(dest);
2381 ret = gen_unique_name(sctx, ino, gen, dest);
2382 if (ret < 0)
2383 goto out;
2384 ret = 1;
2385 }
2386
2387 out_cache:
2388 /*
2389 * Store the result of the lookup in the name cache.
2390 */
2391 nce = kmalloc(sizeof(*nce) + fs_path_len(dest) + 1, GFP_KERNEL);
2392 if (!nce) {
2393 ret = -ENOMEM;
2394 goto out;
2395 }
2396
2397 nce->entry.key = ino;
2398 nce->entry.gen = gen;
2399 nce->parent_ino = *parent_ino;
2400 nce->parent_gen = *parent_gen;
2401 nce->name_len = fs_path_len(dest);
2402 nce->ret = ret;
2403 strcpy(nce->name, dest->start);
2404
2405 if (ino < sctx->send_progress)
2406 nce->need_later_update = 0;
2407 else
2408 nce->need_later_update = 1;
2409
2410 nce_ret = btrfs_lru_cache_store(&sctx->name_cache, &nce->entry, GFP_KERNEL);
2411 if (nce_ret < 0) {
2412 kfree(nce);
2413 ret = nce_ret;
2414 }
2415
2416 out:
2417 return ret;
2418 }
2419
2420 /*
2421 * Magic happens here. This function returns the first ref to an inode as it
2422 * would look like while receiving the stream at this point in time.
2423 * We walk the path up to the root. For every inode in between, we check if it
2424 * was already processed/sent. If yes, we continue with the parent as found
2425 * in send_root. If not, we continue with the parent as found in parent_root.
2426 * If we encounter an inode that was deleted at this point in time, we use the
2427 * inodes "orphan" name instead of the real name and stop. Same with new inodes
2428 * that were not created yet and overwritten inodes/refs.
2429 *
2430 * When do we have orphan inodes:
2431 * 1. When an inode is freshly created and thus no valid refs are available yet
2432 * 2. When a directory lost all it's refs (deleted) but still has dir items
2433 * inside which were not processed yet (pending for move/delete). If anyone
2434 * tried to get the path to the dir items, it would get a path inside that
2435 * orphan directory.
2436 * 3. When an inode is moved around or gets new links, it may overwrite the ref
2437 * of an unprocessed inode. If in that case the first ref would be
2438 * overwritten, the overwritten inode gets "orphanized". Later when we
2439 * process this overwritten inode, it is restored at a new place by moving
2440 * the orphan inode.
2441 *
2442 * sctx->send_progress tells this function at which point in time receiving
2443 * would be.
2444 */
2445 static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen,
2446 struct fs_path *dest)
2447 {
2448 int ret = 0;
2449 struct fs_path *name = NULL;
2450 u64 parent_inode = 0;
2451 u64 parent_gen = 0;
2452 int stop = 0;
2453
2454 name = fs_path_alloc();
2455 if (!name) {
2456 ret = -ENOMEM;
2457 goto out;
2458 }
2459
2460 dest->reversed = 1;
2461 fs_path_reset(dest);
2462
2463 while (!stop && ino != BTRFS_FIRST_FREE_OBJECTID) {
2464 struct waiting_dir_move *wdm;
2465
2466 fs_path_reset(name);
2467
2468 if (is_waiting_for_rm(sctx, ino, gen)) {
2469 ret = gen_unique_name(sctx, ino, gen, name);
2470 if (ret < 0)
2471 goto out;
2472 ret = fs_path_add_path(dest, name);
2473 break;
2474 }
2475
2476 wdm = get_waiting_dir_move(sctx, ino);
2477 if (wdm && wdm->orphanized) {
2478 ret = gen_unique_name(sctx, ino, gen, name);
2479 stop = 1;
2480 } else if (wdm) {
2481 ret = get_first_ref(sctx->parent_root, ino,
2482 &parent_inode, &parent_gen, name);
2483 } else {
2484 ret = __get_cur_name_and_parent(sctx, ino, gen,
2485 &parent_inode,
2486 &parent_gen, name);
2487 if (ret)
2488 stop = 1;
2489 }
2490
2491 if (ret < 0)
2492 goto out;
2493
2494 ret = fs_path_add_path(dest, name);
2495 if (ret < 0)
2496 goto out;
2497
2498 ino = parent_inode;
2499 gen = parent_gen;
2500 }
2501
2502 out:
2503 fs_path_free(name);
2504 if (!ret)
2505 fs_path_unreverse(dest);
2506 return ret;
2507 }
2508
2509 /*
2510 * Sends a BTRFS_SEND_C_SUBVOL command/item to userspace
2511 */
2512 static int send_subvol_begin(struct send_ctx *sctx)
2513 {
2514 int ret;
2515 struct btrfs_root *send_root = sctx->send_root;
2516 struct btrfs_root *parent_root = sctx->parent_root;
2517 struct btrfs_path *path;
2518 struct btrfs_key key;
2519 struct btrfs_root_ref *ref;
2520 struct extent_buffer *leaf;
2521 char *name = NULL;
2522 int namelen;
2523
2524 path = btrfs_alloc_path();
2525 if (!path)
2526 return -ENOMEM;
2527
2528 name = kmalloc(BTRFS_PATH_NAME_MAX, GFP_KERNEL);
2529 if (!name) {
2530 btrfs_free_path(path);
2531 return -ENOMEM;
2532 }
2533
2534 key.objectid = btrfs_root_id(send_root);
2535 key.type = BTRFS_ROOT_BACKREF_KEY;
2536 key.offset = 0;
2537
2538 ret = btrfs_search_slot_for_read(send_root->fs_info->tree_root,
2539 &key, path, 1, 0);
2540 if (ret < 0)
2541 goto out;
2542 if (ret) {
2543 ret = -ENOENT;
2544 goto out;
2545 }
2546
2547 leaf = path->nodes[0];
2548 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2549 if (key.type != BTRFS_ROOT_BACKREF_KEY ||
2550 key.objectid != btrfs_root_id(send_root)) {
2551 ret = -ENOENT;
2552 goto out;
2553 }
2554 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
2555 namelen = btrfs_root_ref_name_len(leaf, ref);
2556 read_extent_buffer(leaf, name, (unsigned long)(ref + 1), namelen);
2557 btrfs_release_path(path);
2558
2559 if (parent_root) {
2560 ret = begin_cmd(sctx, BTRFS_SEND_C_SNAPSHOT);
2561 if (ret < 0)
2562 goto out;
2563 } else {
2564 ret = begin_cmd(sctx, BTRFS_SEND_C_SUBVOL);
2565 if (ret < 0)
2566 goto out;
2567 }
2568
2569 TLV_PUT_STRING(sctx, BTRFS_SEND_A_PATH, name, namelen);
2570
2571 if (!btrfs_is_empty_uuid(sctx->send_root->root_item.received_uuid))
2572 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2573 sctx->send_root->root_item.received_uuid);
2574 else
2575 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2576 sctx->send_root->root_item.uuid);
2577
2578 TLV_PUT_U64(sctx, BTRFS_SEND_A_CTRANSID,
2579 btrfs_root_ctransid(&sctx->send_root->root_item));
2580 if (parent_root) {
2581 if (!btrfs_is_empty_uuid(parent_root->root_item.received_uuid))
2582 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2583 parent_root->root_item.received_uuid);
2584 else
2585 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2586 parent_root->root_item.uuid);
2587 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
2588 btrfs_root_ctransid(&sctx->parent_root->root_item));
2589 }
2590
2591 ret = send_cmd(sctx);
2592
2593 tlv_put_failure:
2594 out:
2595 btrfs_free_path(path);
2596 kfree(name);
2597 return ret;
2598 }
2599
2600 static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size)
2601 {
2602 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2603 int ret = 0;
2604 struct fs_path *p;
2605
2606 btrfs_debug(fs_info, "send_truncate %llu size=%llu", ino, size);
2607
2608 p = fs_path_alloc();
2609 if (!p)
2610 return -ENOMEM;
2611
2612 ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE);
2613 if (ret < 0)
2614 goto out;
2615
2616 ret = get_cur_path(sctx, ino, gen, p);
2617 if (ret < 0)
2618 goto out;
2619 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2620 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size);
2621
2622 ret = send_cmd(sctx);
2623
2624 tlv_put_failure:
2625 out:
2626 fs_path_free(p);
2627 return ret;
2628 }
2629
2630 static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode)
2631 {
2632 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2633 int ret = 0;
2634 struct fs_path *p;
2635
2636 btrfs_debug(fs_info, "send_chmod %llu mode=%llu", ino, mode);
2637
2638 p = fs_path_alloc();
2639 if (!p)
2640 return -ENOMEM;
2641
2642 ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD);
2643 if (ret < 0)
2644 goto out;
2645
2646 ret = get_cur_path(sctx, ino, gen, p);
2647 if (ret < 0)
2648 goto out;
2649 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2650 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777);
2651
2652 ret = send_cmd(sctx);
2653
2654 tlv_put_failure:
2655 out:
2656 fs_path_free(p);
2657 return ret;
2658 }
2659
2660 static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr)
2661 {
2662 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2663 int ret = 0;
2664 struct fs_path *p;
2665
2666 if (sctx->proto < 2)
2667 return 0;
2668
2669 btrfs_debug(fs_info, "send_fileattr %llu fileattr=%llu", ino, fileattr);
2670
2671 p = fs_path_alloc();
2672 if (!p)
2673 return -ENOMEM;
2674
2675 ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR);
2676 if (ret < 0)
2677 goto out;
2678
2679 ret = get_cur_path(sctx, ino, gen, p);
2680 if (ret < 0)
2681 goto out;
2682 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2683 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr);
2684
2685 ret = send_cmd(sctx);
2686
2687 tlv_put_failure:
2688 out:
2689 fs_path_free(p);
2690 return ret;
2691 }
2692
2693 static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)
2694 {
2695 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2696 int ret = 0;
2697 struct fs_path *p;
2698
2699 btrfs_debug(fs_info, "send_chown %llu uid=%llu, gid=%llu",
2700 ino, uid, gid);
2701
2702 p = fs_path_alloc();
2703 if (!p)
2704 return -ENOMEM;
2705
2706 ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN);
2707 if (ret < 0)
2708 goto out;
2709
2710 ret = get_cur_path(sctx, ino, gen, p);
2711 if (ret < 0)
2712 goto out;
2713 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2714 TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid);
2715 TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid);
2716
2717 ret = send_cmd(sctx);
2718
2719 tlv_put_failure:
2720 out:
2721 fs_path_free(p);
2722 return ret;
2723 }
2724
2725 static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen)
2726 {
2727 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2728 int ret = 0;
2729 struct fs_path *p = NULL;
2730 struct btrfs_inode_item *ii;
2731 struct btrfs_path *path = NULL;
2732 struct extent_buffer *eb;
2733 struct btrfs_key key;
2734 int slot;
2735
2736 btrfs_debug(fs_info, "send_utimes %llu", ino);
2737
2738 p = fs_path_alloc();
2739 if (!p)
2740 return -ENOMEM;
2741
2742 path = alloc_path_for_send();
2743 if (!path) {
2744 ret = -ENOMEM;
2745 goto out;
2746 }
2747
2748 key.objectid = ino;
2749 key.type = BTRFS_INODE_ITEM_KEY;
2750 key.offset = 0;
2751 ret = btrfs_search_slot(NULL, sctx->send_root, &key, path, 0, 0);
2752 if (ret > 0)
2753 ret = -ENOENT;
2754 if (ret < 0)
2755 goto out;
2756
2757 eb = path->nodes[0];
2758 slot = path->slots[0];
2759 ii = btrfs_item_ptr(eb, slot, struct btrfs_inode_item);
2760
2761 ret = begin_cmd(sctx, BTRFS_SEND_C_UTIMES);
2762 if (ret < 0)
2763 goto out;
2764
2765 ret = get_cur_path(sctx, ino, gen, p);
2766 if (ret < 0)
2767 goto out;
2768 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2769 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime);
2770 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime);
2771 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime);
2772 if (sctx->proto >= 2)
2773 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime);
2774
2775 ret = send_cmd(sctx);
2776
2777 tlv_put_failure:
2778 out:
2779 fs_path_free(p);
2780 btrfs_free_path(path);
2781 return ret;
2782 }
2783
2784 /*
2785 * If the cache is full, we can't remove entries from it and do a call to
2786 * send_utimes() for each respective inode, because we might be finishing
2787 * processing an inode that is a directory and it just got renamed, and existing
2788 * entries in the cache may refer to inodes that have the directory in their
2789 * full path - in which case we would generate outdated paths (pre-rename)
2790 * for the inodes that the cache entries point to. Instead of prunning the
2791 * cache when inserting, do it after we finish processing each inode at
2792 * finish_inode_if_needed().
2793 */
2794 static int cache_dir_utimes(struct send_ctx *sctx, u64 dir, u64 gen)
2795 {
2796 struct btrfs_lru_cache_entry *entry;
2797 int ret;
2798
2799 entry = btrfs_lru_cache_lookup(&sctx->dir_utimes_cache, dir, gen);
2800 if (entry != NULL)
2801 return 0;
2802
2803 /* Caching is optional, don't fail if we can't allocate memory. */
2804 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2805 if (!entry)
2806 return send_utimes(sctx, dir, gen);
2807
2808 entry->key = dir;
2809 entry->gen = gen;
2810
2811 ret = btrfs_lru_cache_store(&sctx->dir_utimes_cache, entry, GFP_KERNEL);
2812 ASSERT(ret != -EEXIST);
2813 if (ret) {
2814 kfree(entry);
2815 return send_utimes(sctx, dir, gen);
2816 }
2817
2818 return 0;
2819 }
2820
2821 static int trim_dir_utimes_cache(struct send_ctx *sctx)
2822 {
2823 while (sctx->dir_utimes_cache.size > SEND_MAX_DIR_UTIMES_CACHE_SIZE) {
2824 struct btrfs_lru_cache_entry *lru;
2825 int ret;
2826
2827 lru = btrfs_lru_cache_lru_entry(&sctx->dir_utimes_cache);
2828 ASSERT(lru != NULL);
2829
2830 ret = send_utimes(sctx, lru->key, lru->gen);
2831 if (ret)
2832 return ret;
2833
2834 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, lru);
2835 }
2836
2837 return 0;
2838 }
2839
2840 /*
2841 * Sends a BTRFS_SEND_C_MKXXX or SYMLINK command to user space. We don't have
2842 * a valid path yet because we did not process the refs yet. So, the inode
2843 * is created as orphan.
2844 */
2845 static int send_create_inode(struct send_ctx *sctx, u64 ino)
2846 {
2847 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2848 int ret = 0;
2849 struct fs_path *p;
2850 int cmd;
2851 struct btrfs_inode_info info;
2852 u64 gen;
2853 u64 mode;
2854 u64 rdev;
2855
2856 btrfs_debug(fs_info, "send_create_inode %llu", ino);
2857
2858 p = fs_path_alloc();
2859 if (!p)
2860 return -ENOMEM;
2861
2862 if (ino != sctx->cur_ino) {
2863 ret = get_inode_info(sctx->send_root, ino, &info);
2864 if (ret < 0)
2865 goto out;
2866 gen = info.gen;
2867 mode = info.mode;
2868 rdev = info.rdev;
2869 } else {
2870 gen = sctx->cur_inode_gen;
2871 mode = sctx->cur_inode_mode;
2872 rdev = sctx->cur_inode_rdev;
2873 }
2874
2875 if (S_ISREG(mode)) {
2876 cmd = BTRFS_SEND_C_MKFILE;
2877 } else if (S_ISDIR(mode)) {
2878 cmd = BTRFS_SEND_C_MKDIR;
2879 } else if (S_ISLNK(mode)) {
2880 cmd = BTRFS_SEND_C_SYMLINK;
2881 } else if (S_ISCHR(mode) || S_ISBLK(mode)) {
2882 cmd = BTRFS_SEND_C_MKNOD;
2883 } else if (S_ISFIFO(mode)) {
2884 cmd = BTRFS_SEND_C_MKFIFO;
2885 } else if (S_ISSOCK(mode)) {
2886 cmd = BTRFS_SEND_C_MKSOCK;
2887 } else {
2888 btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o",
2889 (int)(mode & S_IFMT));
2890 ret = -EOPNOTSUPP;
2891 goto out;
2892 }
2893
2894 ret = begin_cmd(sctx, cmd);
2895 if (ret < 0)
2896 goto out;
2897
2898 ret = gen_unique_name(sctx, ino, gen, p);
2899 if (ret < 0)
2900 goto out;
2901
2902 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2903 TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino);
2904
2905 if (S_ISLNK(mode)) {
2906 fs_path_reset(p);
2907 ret = read_symlink(sctx->send_root, ino, p);
2908 if (ret < 0)
2909 goto out;
2910 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p);
2911 } else if (S_ISCHR(mode) || S_ISBLK(mode) ||
2912 S_ISFIFO(mode) || S_ISSOCK(mode)) {
2913 TLV_PUT_U64(sctx, BTRFS_SEND_A_RDEV, new_encode_dev(rdev));
2914 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode);
2915 }
2916
2917 ret = send_cmd(sctx);
2918 if (ret < 0)
2919 goto out;
2920
2921
2922 tlv_put_failure:
2923 out:
2924 fs_path_free(p);
2925 return ret;
2926 }
2927
2928 static void cache_dir_created(struct send_ctx *sctx, u64 dir)
2929 {
2930 struct btrfs_lru_cache_entry *entry;
2931 int ret;
2932
2933 /* Caching is optional, ignore any failures. */
2934 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2935 if (!entry)
2936 return;
2937
2938 entry->key = dir;
2939 entry->gen = 0;
2940 ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL);
2941 if (ret < 0)
2942 kfree(entry);
2943 }
2944
2945 /*
2946 * We need some special handling for inodes that get processed before the parent
2947 * directory got created. See process_recorded_refs for details.
2948 * This function does the check if we already created the dir out of order.
2949 */
2950 static int did_create_dir(struct send_ctx *sctx, u64 dir)
2951 {
2952 int ret = 0;
2953 int iter_ret = 0;
2954 struct btrfs_path *path = NULL;
2955 struct btrfs_key key;
2956 struct btrfs_key found_key;
2957 struct btrfs_key di_key;
2958 struct btrfs_dir_item *di;
2959
2960 if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir, 0))
2961 return 1;
2962
2963 path = alloc_path_for_send();
2964 if (!path)
2965 return -ENOMEM;
2966
2967 key.objectid = dir;
2968 key.type = BTRFS_DIR_INDEX_KEY;
2969 key.offset = 0;
2970
2971 btrfs_for_each_slot(sctx->send_root, &key, &found_key, path, iter_ret) {
2972 struct extent_buffer *eb = path->nodes[0];
2973
2974 if (found_key.objectid != key.objectid ||
2975 found_key.type != key.type) {
2976 ret = 0;
2977 break;
2978 }
2979
2980 di = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dir_item);
2981 btrfs_dir_item_key_to_cpu(eb, di, &di_key);
2982
2983 if (di_key.type != BTRFS_ROOT_ITEM_KEY &&
2984 di_key.objectid < sctx->send_progress) {
2985 ret = 1;
2986 cache_dir_created(sctx, dir);
2987 break;
2988 }
2989 }
2990 /* Catch error found during iteration */
2991 if (iter_ret < 0)
2992 ret = iter_ret;
2993
2994 btrfs_free_path(path);
2995 return ret;
2996 }
2997
2998 /*
2999 * Only creates the inode if it is:
3000 * 1. Not a directory
3001 * 2. Or a directory which was not created already due to out of order
3002 * directories. See did_create_dir and process_recorded_refs for details.
3003 */
3004 static int send_create_inode_if_needed(struct send_ctx *sctx)
3005 {
3006 int ret;
3007
3008 if (S_ISDIR(sctx->cur_inode_mode)) {
3009 ret = did_create_dir(sctx, sctx->cur_ino);
3010 if (ret < 0)
3011 return ret;
3012 else if (ret > 0)
3013 return 0;
3014 }
3015
3016 ret = send_create_inode(sctx, sctx->cur_ino);
3017
3018 if (ret == 0 && S_ISDIR(sctx->cur_inode_mode))
3019 cache_dir_created(sctx, sctx->cur_ino);
3020
3021 return ret;
3022 }
3023
3024 struct recorded_ref {
3025 struct list_head list;
3026 char *name;
3027 struct fs_path *full_path;
3028 u64 dir;
3029 u64 dir_gen;
3030 int name_len;
3031 struct rb_node node;
3032 struct rb_root *root;
3033 };
3034
3035 static struct recorded_ref *recorded_ref_alloc(void)
3036 {
3037 struct recorded_ref *ref;
3038
3039 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3040 if (!ref)
3041 return NULL;
3042 RB_CLEAR_NODE(&ref->node);
3043 INIT_LIST_HEAD(&ref->list);
3044 return ref;
3045 }
3046
3047 static void recorded_ref_free(struct recorded_ref *ref)
3048 {
3049 if (!ref)
3050 return;
3051 if (!RB_EMPTY_NODE(&ref->node))
3052 rb_erase(&ref->node, ref->root);
3053 list_del(&ref->list);
3054 fs_path_free(ref->full_path);
3055 kfree(ref);
3056 }
3057
3058 static void set_ref_path(struct recorded_ref *ref, struct fs_path *path)
3059 {
3060 ref->full_path = path;
3061 ref->name = (char *)kbasename(ref->full_path->start);
3062 ref->name_len = ref->full_path->end - ref->name;
3063 }
3064
3065 static int dup_ref(struct recorded_ref *ref, struct list_head *list)
3066 {
3067 struct recorded_ref *new;
3068
3069 new = recorded_ref_alloc();
3070 if (!new)
3071 return -ENOMEM;
3072
3073 new->dir = ref->dir;
3074 new->dir_gen = ref->dir_gen;
3075 list_add_tail(&new->list, list);
3076 return 0;
3077 }
3078
3079 static void __free_recorded_refs(struct list_head *head)
3080 {
3081 struct recorded_ref *cur;
3082
3083 while (!list_empty(head)) {
3084 cur = list_entry(head->next, struct recorded_ref, list);
3085 recorded_ref_free(cur);
3086 }
3087 }
3088
3089 static void free_recorded_refs(struct send_ctx *sctx)
3090 {
3091 __free_recorded_refs(&sctx->new_refs);
3092 __free_recorded_refs(&sctx->deleted_refs);
3093 }
3094
3095 /*
3096 * Renames/moves a file/dir to its orphan name. Used when the first
3097 * ref of an unprocessed inode gets overwritten and for all non empty
3098 * directories.
3099 */
3100 static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen,
3101 struct fs_path *path)
3102 {
3103 int ret;
3104 struct fs_path *orphan;
3105
3106 orphan = fs_path_alloc();
3107 if (!orphan)
3108 return -ENOMEM;
3109
3110 ret = gen_unique_name(sctx, ino, gen, orphan);
3111 if (ret < 0)
3112 goto out;
3113
3114 ret = send_rename(sctx, path, orphan);
3115
3116 out:
3117 fs_path_free(orphan);
3118 return ret;
3119 }
3120
3121 static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx,
3122 u64 dir_ino, u64 dir_gen)
3123 {
3124 struct rb_node **p = &sctx->orphan_dirs.rb_node;
3125 struct rb_node *parent = NULL;
3126 struct orphan_dir_info *entry, *odi;
3127
3128 while (*p) {
3129 parent = *p;
3130 entry = rb_entry(parent, struct orphan_dir_info, node);
3131 if (dir_ino < entry->ino)
3132 p = &(*p)->rb_left;
3133 else if (dir_ino > entry->ino)
3134 p = &(*p)->rb_right;
3135 else if (dir_gen < entry->gen)
3136 p = &(*p)->rb_left;
3137 else if (dir_gen > entry->gen)
3138 p = &(*p)->rb_right;
3139 else
3140 return entry;
3141 }
3142
3143 odi = kmalloc(sizeof(*odi), GFP_KERNEL);
3144 if (!odi)
3145 return ERR_PTR(-ENOMEM);
3146 odi->ino = dir_ino;
3147 odi->gen = dir_gen;
3148 odi->last_dir_index_offset = 0;
3149 odi->dir_high_seq_ino = 0;
3150
3151 rb_link_node(&odi->node, parent, p);
3152 rb_insert_color(&odi->node, &sctx->orphan_dirs);
3153 return odi;
3154 }
3155
3156 static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx,
3157 u64 dir_ino, u64 gen)
3158 {
3159 struct rb_node *n = sctx->orphan_dirs.rb_node;
3160 struct orphan_dir_info *entry;
3161
3162 while (n) {
3163 entry = rb_entry(n, struct orphan_dir_info, node);
3164 if (dir_ino < entry->ino)
3165 n = n->rb_left;
3166 else if (dir_ino > entry->ino)
3167 n = n->rb_right;
3168 else if (gen < entry->gen)
3169 n = n->rb_left;
3170 else if (gen > entry->gen)
3171 n = n->rb_right;
3172 else
3173 return entry;
3174 }
3175 return NULL;
3176 }
3177
3178 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen)
3179 {
3180 struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen);
3181
3182 return odi != NULL;
3183 }
3184
3185 static void free_orphan_dir_info(struct send_ctx *sctx,
3186 struct orphan_dir_info *odi)
3187 {
3188 if (!odi)
3189 return;
3190 rb_erase(&odi->node, &sctx->orphan_dirs);
3191 kfree(odi);
3192 }
3193
3194 /*
3195 * Returns 1 if a directory can be removed at this point in time.
3196 * We check this by iterating all dir items and checking if the inode behind
3197 * the dir item was already processed.
3198 */
3199 static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen)
3200 {
3201 int ret = 0;
3202 int iter_ret = 0;
3203 struct btrfs_root *root = sctx->parent_root;
3204 struct btrfs_path *path;
3205 struct btrfs_key key;
3206 struct btrfs_key found_key;
3207 struct btrfs_key loc;
3208 struct btrfs_dir_item *di;
3209 struct orphan_dir_info *odi = NULL;
3210 u64 dir_high_seq_ino = 0;
3211 u64 last_dir_index_offset = 0;
3212
3213 /*
3214 * Don't try to rmdir the top/root subvolume dir.
3215 */
3216 if (dir == BTRFS_FIRST_FREE_OBJECTID)
3217 return 0;
3218
3219 odi = get_orphan_dir_info(sctx, dir, dir_gen);
3220 if (odi && sctx->cur_ino < odi->dir_high_seq_ino)
3221 return 0;
3222
3223 path = alloc_path_for_send();
3224 if (!path)
3225 return -ENOMEM;
3226
3227 if (!odi) {
3228 /*
3229 * Find the inode number associated with the last dir index
3230 * entry. This is very likely the inode with the highest number
3231 * of all inodes that have an entry in the directory. We can
3232 * then use it to avoid future calls to can_rmdir(), when
3233 * processing inodes with a lower number, from having to search
3234 * the parent root b+tree for dir index keys.
3235 */
3236 key.objectid = dir;
3237 key.type = BTRFS_DIR_INDEX_KEY;
3238 key.offset = (u64)-1;
3239
3240 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3241 if (ret < 0) {
3242 goto out;
3243 } else if (ret > 0) {
3244 /* Can't happen, the root is never empty. */
3245 ASSERT(path->slots[0] > 0);
3246 if (WARN_ON(path->slots[0] == 0)) {
3247 ret = -EUCLEAN;
3248 goto out;
3249 }
3250 path->slots[0]--;
3251 }
3252
3253 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
3254 if (key.objectid != dir || key.type != BTRFS_DIR_INDEX_KEY) {
3255 /* No index keys, dir can be removed. */
3256 ret = 1;
3257 goto out;
3258 }
3259
3260 di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3261 struct btrfs_dir_item);
3262 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3263 dir_high_seq_ino = loc.objectid;
3264 if (sctx->cur_ino < dir_high_seq_ino) {
3265 ret = 0;
3266 goto out;
3267 }
3268
3269 btrfs_release_path(path);
3270 }
3271
3272 key.objectid = dir;
3273 key.type = BTRFS_DIR_INDEX_KEY;
3274 key.offset = (odi ? odi->last_dir_index_offset : 0);
3275
3276 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
3277 struct waiting_dir_move *dm;
3278
3279 if (found_key.objectid != key.objectid ||
3280 found_key.type != key.type)
3281 break;
3282
3283 di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3284 struct btrfs_dir_item);
3285 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3286
3287 dir_high_seq_ino = max(dir_high_seq_ino, loc.objectid);
3288 last_dir_index_offset = found_key.offset;
3289
3290 dm = get_waiting_dir_move(sctx, loc.objectid);
3291 if (dm) {
3292 dm->rmdir_ino = dir;
3293 dm->rmdir_gen = dir_gen;
3294 ret = 0;
3295 goto out;
3296 }
3297
3298 if (loc.objectid > sctx->cur_ino) {
3299 ret = 0;
3300 goto out;
3301 }
3302 }
3303 if (iter_ret < 0) {
3304 ret = iter_ret;
3305 goto out;
3306 }
3307 free_orphan_dir_info(sctx, odi);
3308
3309 ret = 1;
3310
3311 out:
3312 btrfs_free_path(path);
3313
3314 if (ret)
3315 return ret;
3316
3317 if (!odi) {
3318 odi = add_orphan_dir_info(sctx, dir, dir_gen);
3319 if (IS_ERR(odi))
3320 return PTR_ERR(odi);
3321
3322 odi->gen = dir_gen;
3323 }
3324
3325 odi->last_dir_index_offset = last_dir_index_offset;
3326 odi->dir_high_seq_ino = max(odi->dir_high_seq_ino, dir_high_seq_ino);
3327
3328 return 0;
3329 }
3330
3331 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino)
3332 {
3333 struct waiting_dir_move *entry = get_waiting_dir_move(sctx, ino);
3334
3335 return entry != NULL;
3336 }
3337
3338 static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized)
3339 {
3340 struct rb_node **p = &sctx->waiting_dir_moves.rb_node;
3341 struct rb_node *parent = NULL;
3342 struct waiting_dir_move *entry, *dm;
3343
3344 dm = kmalloc(sizeof(*dm), GFP_KERNEL);
3345 if (!dm)
3346 return -ENOMEM;
3347 dm->ino = ino;
3348 dm->rmdir_ino = 0;
3349 dm->rmdir_gen = 0;
3350 dm->orphanized = orphanized;
3351
3352 while (*p) {
3353 parent = *p;
3354 entry = rb_entry(parent, struct waiting_dir_move, node);
3355 if (ino < entry->ino) {
3356 p = &(*p)->rb_left;
3357 } else if (ino > entry->ino) {
3358 p = &(*p)->rb_right;
3359 } else {
3360 kfree(dm);
3361 return -EEXIST;
3362 }
3363 }
3364
3365 rb_link_node(&dm->node, parent, p);
3366 rb_insert_color(&dm->node, &sctx->waiting_dir_moves);
3367 return 0;
3368 }
3369
3370 static struct waiting_dir_move *
3371 get_waiting_dir_move(struct send_ctx *sctx, u64 ino)
3372 {
3373 struct rb_node *n = sctx->waiting_dir_moves.rb_node;
3374 struct waiting_dir_move *entry;
3375
3376 while (n) {
3377 entry = rb_entry(n, struct waiting_dir_move, node);
3378 if (ino < entry->ino)
3379 n = n->rb_left;
3380 else if (ino > entry->ino)
3381 n = n->rb_right;
3382 else
3383 return entry;
3384 }
3385 return NULL;
3386 }
3387
3388 static void free_waiting_dir_move(struct send_ctx *sctx,
3389 struct waiting_dir_move *dm)
3390 {
3391 if (!dm)
3392 return;
3393 rb_erase(&dm->node, &sctx->waiting_dir_moves);
3394 kfree(dm);
3395 }
3396
3397 static int add_pending_dir_move(struct send_ctx *sctx,
3398 u64 ino,
3399 u64 ino_gen,
3400 u64 parent_ino,
3401 struct list_head *new_refs,
3402 struct list_head *deleted_refs,
3403 const bool is_orphan)
3404 {
3405 struct rb_node **p = &sctx->pending_dir_moves.rb_node;
3406 struct rb_node *parent = NULL;
3407 struct pending_dir_move *entry = NULL, *pm;
3408 struct recorded_ref *cur;
3409 int exists = 0;
3410 int ret;
3411
3412 pm = kmalloc(sizeof(*pm), GFP_KERNEL);
3413 if (!pm)
3414 return -ENOMEM;
3415 pm->parent_ino = parent_ino;
3416 pm->ino = ino;
3417 pm->gen = ino_gen;
3418 INIT_LIST_HEAD(&pm->list);
3419 INIT_LIST_HEAD(&pm->update_refs);
3420 RB_CLEAR_NODE(&pm->node);
3421
3422 while (*p) {
3423 parent = *p;
3424 entry = rb_entry(parent, struct pending_dir_move, node);
3425 if (parent_ino < entry->parent_ino) {
3426 p = &(*p)->rb_left;
3427 } else if (parent_ino > entry->parent_ino) {
3428 p = &(*p)->rb_right;
3429 } else {
3430 exists = 1;
3431 break;
3432 }
3433 }
3434
3435 list_for_each_entry(cur, deleted_refs, list) {
3436 ret = dup_ref(cur, &pm->update_refs);
3437 if (ret < 0)
3438 goto out;
3439 }
3440 list_for_each_entry(cur, new_refs, list) {
3441 ret = dup_ref(cur, &pm->update_refs);
3442 if (ret < 0)
3443 goto out;
3444 }
3445
3446 ret = add_waiting_dir_move(sctx, pm->ino, is_orphan);
3447 if (ret)
3448 goto out;
3449
3450 if (exists) {
3451 list_add_tail(&pm->list, &entry->list);
3452 } else {
3453 rb_link_node(&pm->node, parent, p);
3454 rb_insert_color(&pm->node, &sctx->pending_dir_moves);
3455 }
3456 ret = 0;
3457 out:
3458 if (ret) {
3459 __free_recorded_refs(&pm->update_refs);
3460 kfree(pm);
3461 }
3462 return ret;
3463 }
3464
3465 static struct pending_dir_move *get_pending_dir_moves(struct send_ctx *sctx,
3466 u64 parent_ino)
3467 {
3468 struct rb_node *n = sctx->pending_dir_moves.rb_node;
3469 struct pending_dir_move *entry;
3470
3471 while (n) {
3472 entry = rb_entry(n, struct pending_dir_move, node);
3473 if (parent_ino < entry->parent_ino)
3474 n = n->rb_left;
3475 else if (parent_ino > entry->parent_ino)
3476 n = n->rb_right;
3477 else
3478 return entry;
3479 }
3480 return NULL;
3481 }
3482
3483 static int path_loop(struct send_ctx *sctx, struct fs_path *name,
3484 u64 ino, u64 gen, u64 *ancestor_ino)
3485 {
3486 int ret = 0;
3487 u64 parent_inode = 0;
3488 u64 parent_gen = 0;
3489 u64 start_ino = ino;
3490
3491 *ancestor_ino = 0;
3492 while (ino != BTRFS_FIRST_FREE_OBJECTID) {
3493 fs_path_reset(name);
3494
3495 if (is_waiting_for_rm(sctx, ino, gen))
3496 break;
3497 if (is_waiting_for_move(sctx, ino)) {
3498 if (*ancestor_ino == 0)
3499 *ancestor_ino = ino;
3500 ret = get_first_ref(sctx->parent_root, ino,
3501 &parent_inode, &parent_gen, name);
3502 } else {
3503 ret = __get_cur_name_and_parent(sctx, ino, gen,
3504 &parent_inode,
3505 &parent_gen, name);
3506 if (ret > 0) {
3507 ret = 0;
3508 break;
3509 }
3510 }
3511 if (ret < 0)
3512 break;
3513 if (parent_inode == start_ino) {
3514 ret = 1;
3515 if (*ancestor_ino == 0)
3516 *ancestor_ino = ino;
3517 break;
3518 }
3519 ino = parent_inode;
3520 gen = parent_gen;
3521 }
3522 return ret;
3523 }
3524
3525 static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
3526 {
3527 struct fs_path *from_path = NULL;
3528 struct fs_path *to_path = NULL;
3529 struct fs_path *name = NULL;
3530 u64 orig_progress = sctx->send_progress;
3531 struct recorded_ref *cur;
3532 u64 parent_ino, parent_gen;
3533 struct waiting_dir_move *dm = NULL;
3534 u64 rmdir_ino = 0;
3535 u64 rmdir_gen;
3536 u64 ancestor;
3537 bool is_orphan;
3538 int ret;
3539
3540 name = fs_path_alloc();
3541 from_path = fs_path_alloc();
3542 if (!name || !from_path) {
3543 ret = -ENOMEM;
3544 goto out;
3545 }
3546
3547 dm = get_waiting_dir_move(sctx, pm->ino);
3548 ASSERT(dm);
3549 rmdir_ino = dm->rmdir_ino;
3550 rmdir_gen = dm->rmdir_gen;
3551 is_orphan = dm->orphanized;
3552 free_waiting_dir_move(sctx, dm);
3553
3554 if (is_orphan) {
3555 ret = gen_unique_name(sctx, pm->ino,
3556 pm->gen, from_path);
3557 } else {
3558 ret = get_first_ref(sctx->parent_root, pm->ino,
3559 &parent_ino, &parent_gen, name);
3560 if (ret < 0)
3561 goto out;
3562 ret = get_cur_path(sctx, parent_ino, parent_gen,
3563 from_path);
3564 if (ret < 0)
3565 goto out;
3566 ret = fs_path_add_path(from_path, name);
3567 }
3568 if (ret < 0)
3569 goto out;
3570
3571 sctx->send_progress = sctx->cur_ino + 1;
3572 ret = path_loop(sctx, name, pm->ino, pm->gen, &ancestor);
3573 if (ret < 0)
3574 goto out;
3575 if (ret) {
3576 LIST_HEAD(deleted_refs);
3577 ASSERT(ancestor > BTRFS_FIRST_FREE_OBJECTID);
3578 ret = add_pending_dir_move(sctx, pm->ino, pm->gen, ancestor,
3579 &pm->update_refs, &deleted_refs,
3580 is_orphan);
3581 if (ret < 0)
3582 goto out;
3583 if (rmdir_ino) {
3584 dm = get_waiting_dir_move(sctx, pm->ino);
3585 ASSERT(dm);
3586 dm->rmdir_ino = rmdir_ino;
3587 dm->rmdir_gen = rmdir_gen;
3588 }
3589 goto out;
3590 }
3591 fs_path_reset(name);
3592 to_path = name;
3593 name = NULL;
3594 ret = get_cur_path(sctx, pm->ino, pm->gen, to_path);
3595 if (ret < 0)
3596 goto out;
3597
3598 ret = send_rename(sctx, from_path, to_path);
3599 if (ret < 0)
3600 goto out;
3601
3602 if (rmdir_ino) {
3603 struct orphan_dir_info *odi;
3604 u64 gen;
3605
3606 odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen);
3607 if (!odi) {
3608 /* already deleted */
3609 goto finish;
3610 }
3611 gen = odi->gen;
3612
3613 ret = can_rmdir(sctx, rmdir_ino, gen);
3614 if (ret < 0)
3615 goto out;
3616 if (!ret)
3617 goto finish;
3618
3619 name = fs_path_alloc();
3620 if (!name) {
3621 ret = -ENOMEM;
3622 goto out;
3623 }
3624 ret = get_cur_path(sctx, rmdir_ino, gen, name);
3625 if (ret < 0)
3626 goto out;
3627 ret = send_rmdir(sctx, name);
3628 if (ret < 0)
3629 goto out;
3630 }
3631
3632 finish:
3633 ret = cache_dir_utimes(sctx, pm->ino, pm->gen);
3634 if (ret < 0)
3635 goto out;
3636
3637 /*
3638 * After rename/move, need to update the utimes of both new parent(s)
3639 * and old parent(s).
3640 */
3641 list_for_each_entry(cur, &pm->update_refs, list) {
3642 /*
3643 * The parent inode might have been deleted in the send snapshot
3644 */
3645 ret = get_inode_info(sctx->send_root, cur->dir, NULL);
3646 if (ret == -ENOENT) {
3647 ret = 0;
3648 continue;
3649 }
3650 if (ret < 0)
3651 goto out;
3652
3653 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
3654 if (ret < 0)
3655 goto out;
3656 }
3657
3658 out:
3659 fs_path_free(name);
3660 fs_path_free(from_path);
3661 fs_path_free(to_path);
3662 sctx->send_progress = orig_progress;
3663
3664 return ret;
3665 }
3666
3667 static void free_pending_move(struct send_ctx *sctx, struct pending_dir_move *m)
3668 {
3669 if (!list_empty(&m->list))
3670 list_del(&m->list);
3671 if (!RB_EMPTY_NODE(&m->node))
3672 rb_erase(&m->node, &sctx->pending_dir_moves);
3673 __free_recorded_refs(&m->update_refs);
3674 kfree(m);
3675 }
3676
3677 static void tail_append_pending_moves(struct send_ctx *sctx,
3678 struct pending_dir_move *moves,
3679 struct list_head *stack)
3680 {
3681 if (list_empty(&moves->list)) {
3682 list_add_tail(&moves->list, stack);
3683 } else {
3684 LIST_HEAD(list);
3685 list_splice_init(&moves->list, &list);
3686 list_add_tail(&moves->list, stack);
3687 list_splice_tail(&list, stack);
3688 }
3689 if (!RB_EMPTY_NODE(&moves->node)) {
3690 rb_erase(&moves->node, &sctx->pending_dir_moves);
3691 RB_CLEAR_NODE(&moves->node);
3692 }
3693 }
3694
3695 static int apply_children_dir_moves(struct send_ctx *sctx)
3696 {
3697 struct pending_dir_move *pm;
3698 LIST_HEAD(stack);
3699 u64 parent_ino = sctx->cur_ino;
3700 int ret = 0;
3701
3702 pm = get_pending_dir_moves(sctx, parent_ino);
3703 if (!pm)
3704 return 0;
3705
3706 tail_append_pending_moves(sctx, pm, &stack);
3707
3708 while (!list_empty(&stack)) {
3709 pm = list_first_entry(&stack, struct pending_dir_move, list);
3710 parent_ino = pm->ino;
3711 ret = apply_dir_move(sctx, pm);
3712 free_pending_move(sctx, pm);
3713 if (ret)
3714 goto out;
3715 pm = get_pending_dir_moves(sctx, parent_ino);
3716 if (pm)
3717 tail_append_pending_moves(sctx, pm, &stack);
3718 }
3719 return 0;
3720
3721 out:
3722 while (!list_empty(&stack)) {
3723 pm = list_first_entry(&stack, struct pending_dir_move, list);
3724 free_pending_move(sctx, pm);
3725 }
3726 return ret;
3727 }
3728
3729 /*
3730 * We might need to delay a directory rename even when no ancestor directory
3731 * (in the send root) with a higher inode number than ours (sctx->cur_ino) was
3732 * renamed. This happens when we rename a directory to the old name (the name
3733 * in the parent root) of some other unrelated directory that got its rename
3734 * delayed due to some ancestor with higher number that got renamed.
3735 *
3736 * Example:
3737 *
3738 * Parent snapshot:
3739 * . (ino 256)
3740 * |---- a/ (ino 257)
3741 * | |---- file (ino 260)
3742 * |
3743 * |---- b/ (ino 258)
3744 * |---- c/ (ino 259)
3745 *
3746 * Send snapshot:
3747 * . (ino 256)
3748 * |---- a/ (ino 258)
3749 * |---- x/ (ino 259)
3750 * |---- y/ (ino 257)
3751 * |----- file (ino 260)
3752 *
3753 * Here we can not rename 258 from 'b' to 'a' without the rename of inode 257
3754 * from 'a' to 'x/y' happening first, which in turn depends on the rename of
3755 * inode 259 from 'c' to 'x'. So the order of rename commands the send stream
3756 * must issue is:
3757 *
3758 * 1 - rename 259 from 'c' to 'x'
3759 * 2 - rename 257 from 'a' to 'x/y'
3760 * 3 - rename 258 from 'b' to 'a'
3761 *
3762 * Returns 1 if the rename of sctx->cur_ino needs to be delayed, 0 if it can
3763 * be done right away and < 0 on error.
3764 */
3765 static int wait_for_dest_dir_move(struct send_ctx *sctx,
3766 struct recorded_ref *parent_ref,
3767 const bool is_orphan)
3768 {
3769 struct btrfs_fs_info *fs_info = sctx->parent_root->fs_info;
3770 struct btrfs_path *path;
3771 struct btrfs_key key;
3772 struct btrfs_key di_key;
3773 struct btrfs_dir_item *di;
3774 u64 left_gen;
3775 u64 right_gen;
3776 int ret = 0;
3777 struct waiting_dir_move *wdm;
3778
3779 if (RB_EMPTY_ROOT(&sctx->waiting_dir_moves))
3780 return 0;
3781
3782 path = alloc_path_for_send();
3783 if (!path)
3784 return -ENOMEM;
3785
3786 key.objectid = parent_ref->dir;
3787 key.type = BTRFS_DIR_ITEM_KEY;
3788 key.offset = btrfs_name_hash(parent_ref->name, parent_ref->name_len);
3789
3790 ret = btrfs_search_slot(NULL, sctx->parent_root, &key, path, 0, 0);
3791 if (ret < 0) {
3792 goto out;
3793 } else if (ret > 0) {
3794 ret = 0;
3795 goto out;
3796 }
3797
3798 di = btrfs_match_dir_item_name(fs_info, path, parent_ref->name,
3799 parent_ref->name_len);
3800 if (!di) {
3801 ret = 0;
3802 goto out;
3803 }
3804 /*
3805 * di_key.objectid has the number of the inode that has a dentry in the
3806 * parent directory with the same name that sctx->cur_ino is being
3807 * renamed to. We need to check if that inode is in the send root as
3808 * well and if it is currently marked as an inode with a pending rename,
3809 * if it is, we need to delay the rename of sctx->cur_ino as well, so
3810 * that it happens after that other inode is renamed.
3811 */
3812 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &di_key);
3813 if (di_key.type != BTRFS_INODE_ITEM_KEY) {
3814 ret = 0;
3815 goto out;
3816 }
3817
3818 ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen);
3819 if (ret < 0)
3820 goto out;
3821 ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen);
3822 if (ret < 0) {
3823 if (ret == -ENOENT)
3824 ret = 0;
3825 goto out;
3826 }
3827
3828 /* Different inode, no need to delay the rename of sctx->cur_ino */
3829 if (right_gen != left_gen) {
3830 ret = 0;
3831 goto out;
3832 }
3833
3834 wdm = get_waiting_dir_move(sctx, di_key.objectid);
3835 if (wdm && !wdm->orphanized) {
3836 ret = add_pending_dir_move(sctx,
3837 sctx->cur_ino,
3838 sctx->cur_inode_gen,
3839 di_key.objectid,
3840 &sctx->new_refs,
3841 &sctx->deleted_refs,
3842 is_orphan);
3843 if (!ret)
3844 ret = 1;
3845 }
3846 out:
3847 btrfs_free_path(path);
3848 return ret;
3849 }
3850
3851 /*
3852 * Check if inode ino2, or any of its ancestors, is inode ino1.
3853 * Return 1 if true, 0 if false and < 0 on error.
3854 */
3855 static int check_ino_in_path(struct btrfs_root *root,
3856 const u64 ino1,
3857 const u64 ino1_gen,
3858 const u64 ino2,
3859 const u64 ino2_gen,
3860 struct fs_path *fs_path)
3861 {
3862 u64 ino = ino2;
3863
3864 if (ino1 == ino2)
3865 return ino1_gen == ino2_gen;
3866
3867 while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3868 u64 parent;
3869 u64 parent_gen;
3870 int ret;
3871
3872 fs_path_reset(fs_path);
3873 ret = get_first_ref(root, ino, &parent, &parent_gen, fs_path);
3874 if (ret < 0)
3875 return ret;
3876 if (parent == ino1)
3877 return parent_gen == ino1_gen;
3878 ino = parent;
3879 }
3880 return 0;
3881 }
3882
3883 /*
3884 * Check if inode ino1 is an ancestor of inode ino2 in the given root for any
3885 * possible path (in case ino2 is not a directory and has multiple hard links).
3886 * Return 1 if true, 0 if false and < 0 on error.
3887 */
3888 static int is_ancestor(struct btrfs_root *root,
3889 const u64 ino1,
3890 const u64 ino1_gen,
3891 const u64 ino2,
3892 struct fs_path *fs_path)
3893 {
3894 bool free_fs_path = false;
3895 int ret = 0;
3896 int iter_ret = 0;
3897 struct btrfs_path *path = NULL;
3898 struct btrfs_key key;
3899
3900 if (!fs_path) {
3901 fs_path = fs_path_alloc();
3902 if (!fs_path)
3903 return -ENOMEM;
3904 free_fs_path = true;
3905 }
3906
3907 path = alloc_path_for_send();
3908 if (!path) {
3909 ret = -ENOMEM;
3910 goto out;
3911 }
3912
3913 key.objectid = ino2;
3914 key.type = BTRFS_INODE_REF_KEY;
3915 key.offset = 0;
3916
3917 btrfs_for_each_slot(root, &key, &key, path, iter_ret) {
3918 struct extent_buffer *leaf = path->nodes[0];
3919 int slot = path->slots[0];
3920 u32 cur_offset = 0;
3921 u32 item_size;
3922
3923 if (key.objectid != ino2)
3924 break;
3925 if (key.type != BTRFS_INODE_REF_KEY &&
3926 key.type != BTRFS_INODE_EXTREF_KEY)
3927 break;
3928
3929 item_size = btrfs_item_size(leaf, slot);
3930 while (cur_offset < item_size) {
3931 u64 parent;
3932 u64 parent_gen;
3933
3934 if (key.type == BTRFS_INODE_EXTREF_KEY) {
3935 unsigned long ptr;
3936 struct btrfs_inode_extref *extref;
3937
3938 ptr = btrfs_item_ptr_offset(leaf, slot);
3939 extref = (struct btrfs_inode_extref *)
3940 (ptr + cur_offset);
3941 parent = btrfs_inode_extref_parent(leaf,
3942 extref);
3943 cur_offset += sizeof(*extref);
3944 cur_offset += btrfs_inode_extref_name_len(leaf,
3945 extref);
3946 } else {
3947 parent = key.offset;
3948 cur_offset = item_size;
3949 }
3950
3951 ret = get_inode_gen(root, parent, &parent_gen);
3952 if (ret < 0)
3953 goto out;
3954 ret = check_ino_in_path(root, ino1, ino1_gen,
3955 parent, parent_gen, fs_path);
3956 if (ret)
3957 goto out;
3958 }
3959 }
3960 ret = 0;
3961 if (iter_ret < 0)
3962 ret = iter_ret;
3963
3964 out:
3965 btrfs_free_path(path);
3966 if (free_fs_path)
3967 fs_path_free(fs_path);
3968 return ret;
3969 }
3970
3971 static int wait_for_parent_move(struct send_ctx *sctx,
3972 struct recorded_ref *parent_ref,
3973 const bool is_orphan)
3974 {
3975 int ret = 0;
3976 u64 ino = parent_ref->dir;
3977 u64 ino_gen = parent_ref->dir_gen;
3978 u64 parent_ino_before, parent_ino_after;
3979 struct fs_path *path_before = NULL;
3980 struct fs_path *path_after = NULL;
3981 int len1, len2;
3982
3983 path_after = fs_path_alloc();
3984 path_before = fs_path_alloc();
3985 if (!path_after || !path_before) {
3986 ret = -ENOMEM;
3987 goto out;
3988 }
3989
3990 /*
3991 * Our current directory inode may not yet be renamed/moved because some
3992 * ancestor (immediate or not) has to be renamed/moved first. So find if
3993 * such ancestor exists and make sure our own rename/move happens after
3994 * that ancestor is processed to avoid path build infinite loops (done
3995 * at get_cur_path()).
3996 */
3997 while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3998 u64 parent_ino_after_gen;
3999
4000 if (is_waiting_for_move(sctx, ino)) {
4001 /*
4002 * If the current inode is an ancestor of ino in the
4003 * parent root, we need to delay the rename of the
4004 * current inode, otherwise don't delayed the rename
4005 * because we can end up with a circular dependency
4006 * of renames, resulting in some directories never
4007 * getting the respective rename operations issued in
4008 * the send stream or getting into infinite path build
4009 * loops.
4010 */
4011 ret = is_ancestor(sctx->parent_root,
4012 sctx->cur_ino, sctx->cur_inode_gen,
4013 ino, path_before);
4014 if (ret)
4015 break;
4016 }
4017
4018 fs_path_reset(path_before);
4019 fs_path_reset(path_after);
4020
4021 ret = get_first_ref(sctx->send_root, ino, &parent_ino_after,
4022 &parent_ino_after_gen, path_after);
4023 if (ret < 0)
4024 goto out;
4025 ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before,
4026 NULL, path_before);
4027 if (ret < 0 && ret != -ENOENT) {
4028 goto out;
4029 } else if (ret == -ENOENT) {
4030 ret = 0;
4031 break;
4032 }
4033
4034 len1 = fs_path_len(path_before);
4035 len2 = fs_path_len(path_after);
4036 if (ino > sctx->cur_ino &&
4037 (parent_ino_before != parent_ino_after || len1 != len2 ||
4038 memcmp(path_before->start, path_after->start, len1))) {
4039 u64 parent_ino_gen;
4040
4041 ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen);
4042 if (ret < 0)
4043 goto out;
4044 if (ino_gen == parent_ino_gen) {
4045 ret = 1;
4046 break;
4047 }
4048 }
4049 ino = parent_ino_after;
4050 ino_gen = parent_ino_after_gen;
4051 }
4052
4053 out:
4054 fs_path_free(path_before);
4055 fs_path_free(path_after);
4056
4057 if (ret == 1) {
4058 ret = add_pending_dir_move(sctx,
4059 sctx->cur_ino,
4060 sctx->cur_inode_gen,
4061 ino,
4062 &sctx->new_refs,
4063 &sctx->deleted_refs,
4064 is_orphan);
4065 if (!ret)
4066 ret = 1;
4067 }
4068
4069 return ret;
4070 }
4071
4072 static int update_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
4073 {
4074 int ret;
4075 struct fs_path *new_path;
4076
4077 /*
4078 * Our reference's name member points to its full_path member string, so
4079 * we use here a new path.
4080 */
4081 new_path = fs_path_alloc();
4082 if (!new_path)
4083 return -ENOMEM;
4084
4085 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, new_path);
4086 if (ret < 0) {
4087 fs_path_free(new_path);
4088 return ret;
4089 }
4090 ret = fs_path_add(new_path, ref->name, ref->name_len);
4091 if (ret < 0) {
4092 fs_path_free(new_path);
4093 return ret;
4094 }
4095
4096 fs_path_free(ref->full_path);
4097 set_ref_path(ref, new_path);
4098
4099 return 0;
4100 }
4101
4102 /*
4103 * When processing the new references for an inode we may orphanize an existing
4104 * directory inode because its old name conflicts with one of the new references
4105 * of the current inode. Later, when processing another new reference of our
4106 * inode, we might need to orphanize another inode, but the path we have in the
4107 * reference reflects the pre-orphanization name of the directory we previously
4108 * orphanized. For example:
4109 *
4110 * parent snapshot looks like:
4111 *
4112 * . (ino 256)
4113 * |----- f1 (ino 257)
4114 * |----- f2 (ino 258)
4115 * |----- d1/ (ino 259)
4116 * |----- d2/ (ino 260)
4117 *
4118 * send snapshot looks like:
4119 *
4120 * . (ino 256)
4121 * |----- d1 (ino 258)
4122 * |----- f2/ (ino 259)
4123 * |----- f2_link/ (ino 260)
4124 * | |----- f1 (ino 257)
4125 * |
4126 * |----- d2 (ino 258)
4127 *
4128 * When processing inode 257 we compute the name for inode 259 as "d1", and we
4129 * cache it in the name cache. Later when we start processing inode 258, when
4130 * collecting all its new references we set a full path of "d1/d2" for its new
4131 * reference with name "d2". When we start processing the new references we
4132 * start by processing the new reference with name "d1", and this results in
4133 * orphanizing inode 259, since its old reference causes a conflict. Then we
4134 * move on the next new reference, with name "d2", and we find out we must
4135 * orphanize inode 260, as its old reference conflicts with ours - but for the
4136 * orphanization we use a source path corresponding to the path we stored in the
4137 * new reference, which is "d1/d2" and not "o259-6-0/d2" - this makes the
4138 * receiver fail since the path component "d1/" no longer exists, it was renamed
4139 * to "o259-6-0/" when processing the previous new reference. So in this case we
4140 * must recompute the path in the new reference and use it for the new
4141 * orphanization operation.
4142 */
4143 static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
4144 {
4145 char *name;
4146 int ret;
4147
4148 name = kmemdup(ref->name, ref->name_len, GFP_KERNEL);
4149 if (!name)
4150 return -ENOMEM;
4151
4152 fs_path_reset(ref->full_path);
4153 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path);
4154 if (ret < 0)
4155 goto out;
4156
4157 ret = fs_path_add(ref->full_path, name, ref->name_len);
4158 if (ret < 0)
4159 goto out;
4160
4161 /* Update the reference's base name pointer. */
4162 set_ref_path(ref, ref->full_path);
4163 out:
4164 kfree(name);
4165 return ret;
4166 }
4167
4168 /*
4169 * This does all the move/link/unlink/rmdir magic.
4170 */
4171 static int process_recorded_refs(struct send_ctx *sctx, int *pending_move)
4172 {
4173 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
4174 int ret = 0;
4175 struct recorded_ref *cur;
4176 struct recorded_ref *cur2;
4177 LIST_HEAD(check_dirs);
4178 struct fs_path *valid_path = NULL;
4179 u64 ow_inode = 0;
4180 u64 ow_gen;
4181 u64 ow_mode;
4182 int did_overwrite = 0;
4183 int is_orphan = 0;
4184 u64 last_dir_ino_rm = 0;
4185 bool can_rename = true;
4186 bool orphanized_dir = false;
4187 bool orphanized_ancestor = false;
4188
4189 btrfs_debug(fs_info, "process_recorded_refs %llu", sctx->cur_ino);
4190
4191 /*
4192 * This should never happen as the root dir always has the same ref
4193 * which is always '..'
4194 */
4195 if (unlikely(sctx->cur_ino <= BTRFS_FIRST_FREE_OBJECTID)) {
4196 btrfs_err(fs_info,
4197 "send: unexpected inode %llu in process_recorded_refs()",
4198 sctx->cur_ino);
4199 ret = -EINVAL;
4200 goto out;
4201 }
4202
4203 valid_path = fs_path_alloc();
4204 if (!valid_path) {
4205 ret = -ENOMEM;
4206 goto out;
4207 }
4208
4209 /*
4210 * First, check if the first ref of the current inode was overwritten
4211 * before. If yes, we know that the current inode was already orphanized
4212 * and thus use the orphan name. If not, we can use get_cur_path to
4213 * get the path of the first ref as it would like while receiving at
4214 * this point in time.
4215 * New inodes are always orphan at the beginning, so force to use the
4216 * orphan name in this case.
4217 * The first ref is stored in valid_path and will be updated if it
4218 * gets moved around.
4219 */
4220 if (!sctx->cur_inode_new) {
4221 ret = did_overwrite_first_ref(sctx, sctx->cur_ino,
4222 sctx->cur_inode_gen);
4223 if (ret < 0)
4224 goto out;
4225 if (ret)
4226 did_overwrite = 1;
4227 }
4228 if (sctx->cur_inode_new || did_overwrite) {
4229 ret = gen_unique_name(sctx, sctx->cur_ino,
4230 sctx->cur_inode_gen, valid_path);
4231 if (ret < 0)
4232 goto out;
4233 is_orphan = 1;
4234 } else {
4235 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen,
4236 valid_path);
4237 if (ret < 0)
4238 goto out;
4239 }
4240
4241 /*
4242 * Before doing any rename and link operations, do a first pass on the
4243 * new references to orphanize any unprocessed inodes that may have a
4244 * reference that conflicts with one of the new references of the current
4245 * inode. This needs to happen first because a new reference may conflict
4246 * with the old reference of a parent directory, so we must make sure
4247 * that the path used for link and rename commands don't use an
4248 * orphanized name when an ancestor was not yet orphanized.
4249 *
4250 * Example:
4251 *
4252 * Parent snapshot:
4253 *
4254 * . (ino 256)
4255 * |----- testdir/ (ino 259)
4256 * | |----- a (ino 257)
4257 * |
4258 * |----- b (ino 258)
4259 *
4260 * Send snapshot:
4261 *
4262 * . (ino 256)
4263 * |----- testdir_2/ (ino 259)
4264 * | |----- a (ino 260)
4265 * |
4266 * |----- testdir (ino 257)
4267 * |----- b (ino 257)
4268 * |----- b2 (ino 258)
4269 *
4270 * Processing the new reference for inode 257 with name "b" may happen
4271 * before processing the new reference with name "testdir". If so, we
4272 * must make sure that by the time we send a link command to create the
4273 * hard link "b", inode 259 was already orphanized, since the generated
4274 * path in "valid_path" already contains the orphanized name for 259.
4275 * We are processing inode 257, so only later when processing 259 we do
4276 * the rename operation to change its temporary (orphanized) name to
4277 * "testdir_2".
4278 */
4279 list_for_each_entry(cur, &sctx->new_refs, list) {
4280 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4281 if (ret < 0)
4282 goto out;
4283 if (ret == inode_state_will_create)
4284 continue;
4285
4286 /*
4287 * Check if this new ref would overwrite the first ref of another
4288 * unprocessed inode. If yes, orphanize the overwritten inode.
4289 * If we find an overwritten ref that is not the first ref,
4290 * simply unlink it.
4291 */
4292 ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4293 cur->name, cur->name_len,
4294 &ow_inode, &ow_gen, &ow_mode);
4295 if (ret < 0)
4296 goto out;
4297 if (ret) {
4298 ret = is_first_ref(sctx->parent_root,
4299 ow_inode, cur->dir, cur->name,
4300 cur->name_len);
4301 if (ret < 0)
4302 goto out;
4303 if (ret) {
4304 struct name_cache_entry *nce;
4305 struct waiting_dir_move *wdm;
4306
4307 if (orphanized_dir) {
4308 ret = refresh_ref_path(sctx, cur);
4309 if (ret < 0)
4310 goto out;
4311 }
4312
4313 ret = orphanize_inode(sctx, ow_inode, ow_gen,
4314 cur->full_path);
4315 if (ret < 0)
4316 goto out;
4317 if (S_ISDIR(ow_mode))
4318 orphanized_dir = true;
4319
4320 /*
4321 * If ow_inode has its rename operation delayed
4322 * make sure that its orphanized name is used in
4323 * the source path when performing its rename
4324 * operation.
4325 */
4326 wdm = get_waiting_dir_move(sctx, ow_inode);
4327 if (wdm)
4328 wdm->orphanized = true;
4329
4330 /*
4331 * Make sure we clear our orphanized inode's
4332 * name from the name cache. This is because the
4333 * inode ow_inode might be an ancestor of some
4334 * other inode that will be orphanized as well
4335 * later and has an inode number greater than
4336 * sctx->send_progress. We need to prevent
4337 * future name lookups from using the old name
4338 * and get instead the orphan name.
4339 */
4340 nce = name_cache_search(sctx, ow_inode, ow_gen);
4341 if (nce)
4342 btrfs_lru_cache_remove(&sctx->name_cache,
4343 &nce->entry);
4344
4345 /*
4346 * ow_inode might currently be an ancestor of
4347 * cur_ino, therefore compute valid_path (the
4348 * current path of cur_ino) again because it
4349 * might contain the pre-orphanization name of
4350 * ow_inode, which is no longer valid.
4351 */
4352 ret = is_ancestor(sctx->parent_root,
4353 ow_inode, ow_gen,
4354 sctx->cur_ino, NULL);
4355 if (ret > 0) {
4356 orphanized_ancestor = true;
4357 fs_path_reset(valid_path);
4358 ret = get_cur_path(sctx, sctx->cur_ino,
4359 sctx->cur_inode_gen,
4360 valid_path);
4361 }
4362 if (ret < 0)
4363 goto out;
4364 } else {
4365 /*
4366 * If we previously orphanized a directory that
4367 * collided with a new reference that we already
4368 * processed, recompute the current path because
4369 * that directory may be part of the path.
4370 */
4371 if (orphanized_dir) {
4372 ret = refresh_ref_path(sctx, cur);
4373 if (ret < 0)
4374 goto out;
4375 }
4376 ret = send_unlink(sctx, cur->full_path);
4377 if (ret < 0)
4378 goto out;
4379 }
4380 }
4381
4382 }
4383
4384 list_for_each_entry(cur, &sctx->new_refs, list) {
4385 /*
4386 * We may have refs where the parent directory does not exist
4387 * yet. This happens if the parent directories inum is higher
4388 * than the current inum. To handle this case, we create the
4389 * parent directory out of order. But we need to check if this
4390 * did already happen before due to other refs in the same dir.
4391 */
4392 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4393 if (ret < 0)
4394 goto out;
4395 if (ret == inode_state_will_create) {
4396 ret = 0;
4397 /*
4398 * First check if any of the current inodes refs did
4399 * already create the dir.
4400 */
4401 list_for_each_entry(cur2, &sctx->new_refs, list) {
4402 if (cur == cur2)
4403 break;
4404 if (cur2->dir == cur->dir) {
4405 ret = 1;
4406 break;
4407 }
4408 }
4409
4410 /*
4411 * If that did not happen, check if a previous inode
4412 * did already create the dir.
4413 */
4414 if (!ret)
4415 ret = did_create_dir(sctx, cur->dir);
4416 if (ret < 0)
4417 goto out;
4418 if (!ret) {
4419 ret = send_create_inode(sctx, cur->dir);
4420 if (ret < 0)
4421 goto out;
4422 cache_dir_created(sctx, cur->dir);
4423 }
4424 }
4425
4426 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) {
4427 ret = wait_for_dest_dir_move(sctx, cur, is_orphan);
4428 if (ret < 0)
4429 goto out;
4430 if (ret == 1) {
4431 can_rename = false;
4432 *pending_move = 1;
4433 }
4434 }
4435
4436 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root &&
4437 can_rename) {
4438 ret = wait_for_parent_move(sctx, cur, is_orphan);
4439 if (ret < 0)
4440 goto out;
4441 if (ret == 1) {
4442 can_rename = false;
4443 *pending_move = 1;
4444 }
4445 }
4446
4447 /*
4448 * link/move the ref to the new place. If we have an orphan
4449 * inode, move it and update valid_path. If not, link or move
4450 * it depending on the inode mode.
4451 */
4452 if (is_orphan && can_rename) {
4453 ret = send_rename(sctx, valid_path, cur->full_path);
4454 if (ret < 0)
4455 goto out;
4456 is_orphan = 0;
4457 ret = fs_path_copy(valid_path, cur->full_path);
4458 if (ret < 0)
4459 goto out;
4460 } else if (can_rename) {
4461 if (S_ISDIR(sctx->cur_inode_mode)) {
4462 /*
4463 * Dirs can't be linked, so move it. For moved
4464 * dirs, we always have one new and one deleted
4465 * ref. The deleted ref is ignored later.
4466 */
4467 ret = send_rename(sctx, valid_path,
4468 cur->full_path);
4469 if (!ret)
4470 ret = fs_path_copy(valid_path,
4471 cur->full_path);
4472 if (ret < 0)
4473 goto out;
4474 } else {
4475 /*
4476 * We might have previously orphanized an inode
4477 * which is an ancestor of our current inode,
4478 * so our reference's full path, which was
4479 * computed before any such orphanizations, must
4480 * be updated.
4481 */
4482 if (orphanized_dir) {
4483 ret = update_ref_path(sctx, cur);
4484 if (ret < 0)
4485 goto out;
4486 }
4487 ret = send_link(sctx, cur->full_path,
4488 valid_path);
4489 if (ret < 0)
4490 goto out;
4491 }
4492 }
4493 ret = dup_ref(cur, &check_dirs);
4494 if (ret < 0)
4495 goto out;
4496 }
4497
4498 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_deleted) {
4499 /*
4500 * Check if we can already rmdir the directory. If not,
4501 * orphanize it. For every dir item inside that gets deleted
4502 * later, we do this check again and rmdir it then if possible.
4503 * See the use of check_dirs for more details.
4504 */
4505 ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen);
4506 if (ret < 0)
4507 goto out;
4508 if (ret) {
4509 ret = send_rmdir(sctx, valid_path);
4510 if (ret < 0)
4511 goto out;
4512 } else if (!is_orphan) {
4513 ret = orphanize_inode(sctx, sctx->cur_ino,
4514 sctx->cur_inode_gen, valid_path);
4515 if (ret < 0)
4516 goto out;
4517 is_orphan = 1;
4518 }
4519
4520 list_for_each_entry(cur, &sctx->deleted_refs, list) {
4521 ret = dup_ref(cur, &check_dirs);
4522 if (ret < 0)
4523 goto out;
4524 }
4525 } else if (S_ISDIR(sctx->cur_inode_mode) &&
4526 !list_empty(&sctx->deleted_refs)) {
4527 /*
4528 * We have a moved dir. Add the old parent to check_dirs
4529 */
4530 cur = list_entry(sctx->deleted_refs.next, struct recorded_ref,
4531 list);
4532 ret = dup_ref(cur, &check_dirs);
4533 if (ret < 0)
4534 goto out;
4535 } else if (!S_ISDIR(sctx->cur_inode_mode)) {
4536 /*
4537 * We have a non dir inode. Go through all deleted refs and
4538 * unlink them if they were not already overwritten by other
4539 * inodes.
4540 */
4541 list_for_each_entry(cur, &sctx->deleted_refs, list) {
4542 ret = did_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4543 sctx->cur_ino, sctx->cur_inode_gen,
4544 cur->name, cur->name_len);
4545 if (ret < 0)
4546 goto out;
4547 if (!ret) {
4548 /*
4549 * If we orphanized any ancestor before, we need
4550 * to recompute the full path for deleted names,
4551 * since any such path was computed before we
4552 * processed any references and orphanized any
4553 * ancestor inode.
4554 */
4555 if (orphanized_ancestor) {
4556 ret = update_ref_path(sctx, cur);
4557 if (ret < 0)
4558 goto out;
4559 }
4560 ret = send_unlink(sctx, cur->full_path);
4561 if (ret < 0)
4562 goto out;
4563 }
4564 ret = dup_ref(cur, &check_dirs);
4565 if (ret < 0)
4566 goto out;
4567 }
4568 /*
4569 * If the inode is still orphan, unlink the orphan. This may
4570 * happen when a previous inode did overwrite the first ref
4571 * of this inode and no new refs were added for the current
4572 * inode. Unlinking does not mean that the inode is deleted in
4573 * all cases. There may still be links to this inode in other
4574 * places.
4575 */
4576 if (is_orphan) {
4577 ret = send_unlink(sctx, valid_path);
4578 if (ret < 0)
4579 goto out;
4580 }
4581 }
4582
4583 /*
4584 * We did collect all parent dirs where cur_inode was once located. We
4585 * now go through all these dirs and check if they are pending for
4586 * deletion and if it's finally possible to perform the rmdir now.
4587 * We also update the inode stats of the parent dirs here.
4588 */
4589 list_for_each_entry(cur, &check_dirs, list) {
4590 /*
4591 * In case we had refs into dirs that were not processed yet,
4592 * we don't need to do the utime and rmdir logic for these dirs.
4593 * The dir will be processed later.
4594 */
4595 if (cur->dir > sctx->cur_ino)
4596 continue;
4597
4598 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4599 if (ret < 0)
4600 goto out;
4601
4602 if (ret == inode_state_did_create ||
4603 ret == inode_state_no_change) {
4604 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
4605 if (ret < 0)
4606 goto out;
4607 } else if (ret == inode_state_did_delete &&
4608 cur->dir != last_dir_ino_rm) {
4609 ret = can_rmdir(sctx, cur->dir, cur->dir_gen);
4610 if (ret < 0)
4611 goto out;
4612 if (ret) {
4613 ret = get_cur_path(sctx, cur->dir,
4614 cur->dir_gen, valid_path);
4615 if (ret < 0)
4616 goto out;
4617 ret = send_rmdir(sctx, valid_path);
4618 if (ret < 0)
4619 goto out;
4620 last_dir_ino_rm = cur->dir;
4621 }
4622 }
4623 }
4624
4625 ret = 0;
4626
4627 out:
4628 __free_recorded_refs(&check_dirs);
4629 free_recorded_refs(sctx);
4630 fs_path_free(valid_path);
4631 return ret;
4632 }
4633
4634 static int rbtree_ref_comp(const void *k, const struct rb_node *node)
4635 {
4636 const struct recorded_ref *data = k;
4637 const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node);
4638 int result;
4639
4640 if (data->dir > ref->dir)
4641 return 1;
4642 if (data->dir < ref->dir)
4643 return -1;
4644 if (data->dir_gen > ref->dir_gen)
4645 return 1;
4646 if (data->dir_gen < ref->dir_gen)
4647 return -1;
4648 if (data->name_len > ref->name_len)
4649 return 1;
4650 if (data->name_len < ref->name_len)
4651 return -1;
4652 result = strcmp(data->name, ref->name);
4653 if (result > 0)
4654 return 1;
4655 if (result < 0)
4656 return -1;
4657 return 0;
4658 }
4659
4660 static bool rbtree_ref_less(struct rb_node *node, const struct rb_node *parent)
4661 {
4662 const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node);
4663
4664 return rbtree_ref_comp(entry, parent) < 0;
4665 }
4666
4667 static int record_ref_in_tree(struct rb_root *root, struct list_head *refs,
4668 struct fs_path *name, u64 dir, u64 dir_gen,
4669 struct send_ctx *sctx)
4670 {
4671 int ret = 0;
4672 struct fs_path *path = NULL;
4673 struct recorded_ref *ref = NULL;
4674
4675 path = fs_path_alloc();
4676 if (!path) {
4677 ret = -ENOMEM;
4678 goto out;
4679 }
4680
4681 ref = recorded_ref_alloc();
4682 if (!ref) {
4683 ret = -ENOMEM;
4684 goto out;
4685 }
4686
4687 ret = get_cur_path(sctx, dir, dir_gen, path);
4688 if (ret < 0)
4689 goto out;
4690 ret = fs_path_add_path(path, name);
4691 if (ret < 0)
4692 goto out;
4693
4694 ref->dir = dir;
4695 ref->dir_gen = dir_gen;
4696 set_ref_path(ref, path);
4697 list_add_tail(&ref->list, refs);
4698 rb_add(&ref->node, root, rbtree_ref_less);
4699 ref->root = root;
4700 out:
4701 if (ret) {
4702 if (path && (!ref || !ref->full_path))
4703 fs_path_free(path);
4704 recorded_ref_free(ref);
4705 }
4706 return ret;
4707 }
4708
4709 static int record_new_ref_if_needed(int num, u64 dir, int index,
4710 struct fs_path *name, void *ctx)
4711 {
4712 int ret = 0;
4713 struct send_ctx *sctx = ctx;
4714 struct rb_node *node = NULL;
4715 struct recorded_ref data;
4716 struct recorded_ref *ref;
4717 u64 dir_gen;
4718
4719 ret = get_inode_gen(sctx->send_root, dir, &dir_gen);
4720 if (ret < 0)
4721 goto out;
4722
4723 data.dir = dir;
4724 data.dir_gen = dir_gen;
4725 set_ref_path(&data, name);
4726 node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp);
4727 if (node) {
4728 ref = rb_entry(node, struct recorded_ref, node);
4729 recorded_ref_free(ref);
4730 } else {
4731 ret = record_ref_in_tree(&sctx->rbtree_new_refs,
4732 &sctx->new_refs, name, dir, dir_gen,
4733 sctx);
4734 }
4735 out:
4736 return ret;
4737 }
4738
4739 static int record_deleted_ref_if_needed(int num, u64 dir, int index,
4740 struct fs_path *name, void *ctx)
4741 {
4742 int ret = 0;
4743 struct send_ctx *sctx = ctx;
4744 struct rb_node *node = NULL;
4745 struct recorded_ref data;
4746 struct recorded_ref *ref;
4747 u64 dir_gen;
4748
4749 ret = get_inode_gen(sctx->parent_root, dir, &dir_gen);
4750 if (ret < 0)
4751 goto out;
4752
4753 data.dir = dir;
4754 data.dir_gen = dir_gen;
4755 set_ref_path(&data, name);
4756 node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp);
4757 if (node) {
4758 ref = rb_entry(node, struct recorded_ref, node);
4759 recorded_ref_free(ref);
4760 } else {
4761 ret = record_ref_in_tree(&sctx->rbtree_deleted_refs,
4762 &sctx->deleted_refs, name, dir,
4763 dir_gen, sctx);
4764 }
4765 out:
4766 return ret;
4767 }
4768
4769 static int record_new_ref(struct send_ctx *sctx)
4770 {
4771 int ret;
4772
4773 ret = iterate_inode_ref(sctx->send_root, sctx->left_path,
4774 sctx->cmp_key, 0, record_new_ref_if_needed, sctx);
4775 if (ret < 0)
4776 goto out;
4777 ret = 0;
4778
4779 out:
4780 return ret;
4781 }
4782
4783 static int record_deleted_ref(struct send_ctx *sctx)
4784 {
4785 int ret;
4786
4787 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,
4788 sctx->cmp_key, 0, record_deleted_ref_if_needed,
4789 sctx);
4790 if (ret < 0)
4791 goto out;
4792 ret = 0;
4793
4794 out:
4795 return ret;
4796 }
4797
4798 static int record_changed_ref(struct send_ctx *sctx)
4799 {
4800 int ret = 0;
4801
4802 ret = iterate_inode_ref(sctx->send_root, sctx->left_path,
4803 sctx->cmp_key, 0, record_new_ref_if_needed, sctx);
4804 if (ret < 0)
4805 goto out;
4806 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,
4807 sctx->cmp_key, 0, record_deleted_ref_if_needed, sctx);
4808 if (ret < 0)
4809 goto out;
4810 ret = 0;
4811
4812 out:
4813 return ret;
4814 }
4815
4816 /*
4817 * Record and process all refs at once. Needed when an inode changes the
4818 * generation number, which means that it was deleted and recreated.
4819 */
4820 static int process_all_refs(struct send_ctx *sctx,
4821 enum btrfs_compare_tree_result cmd)
4822 {
4823 int ret = 0;
4824 int iter_ret = 0;
4825 struct btrfs_root *root;
4826 struct btrfs_path *path;
4827 struct btrfs_key key;
4828 struct btrfs_key found_key;
4829 iterate_inode_ref_t cb;
4830 int pending_move = 0;
4831
4832 path = alloc_path_for_send();
4833 if (!path)
4834 return -ENOMEM;
4835
4836 if (cmd == BTRFS_COMPARE_TREE_NEW) {
4837 root = sctx->send_root;
4838 cb = record_new_ref_if_needed;
4839 } else if (cmd == BTRFS_COMPARE_TREE_DELETED) {
4840 root = sctx->parent_root;
4841 cb = record_deleted_ref_if_needed;
4842 } else {
4843 btrfs_err(sctx->send_root->fs_info,
4844 "Wrong command %d in process_all_refs", cmd);
4845 ret = -EINVAL;
4846 goto out;
4847 }
4848
4849 key.objectid = sctx->cmp_key->objectid;
4850 key.type = BTRFS_INODE_REF_KEY;
4851 key.offset = 0;
4852 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
4853 if (found_key.objectid != key.objectid ||
4854 (found_key.type != BTRFS_INODE_REF_KEY &&
4855 found_key.type != BTRFS_INODE_EXTREF_KEY))
4856 break;
4857
4858 ret = iterate_inode_ref(root, path, &found_key, 0, cb, sctx);
4859 if (ret < 0)
4860 goto out;
4861 }
4862 /* Catch error found during iteration */
4863 if (iter_ret < 0) {
4864 ret = iter_ret;
4865 goto out;
4866 }
4867 btrfs_release_path(path);
4868
4869 /*
4870 * We don't actually care about pending_move as we are simply
4871 * re-creating this inode and will be rename'ing it into place once we
4872 * rename the parent directory.
4873 */
4874 ret = process_recorded_refs(sctx, &pending_move);
4875 out:
4876 btrfs_free_path(path);
4877 return ret;
4878 }
4879
4880 static int send_set_xattr(struct send_ctx *sctx,
4881 struct fs_path *path,
4882 const char *name, int name_len,
4883 const char *data, int data_len)
4884 {
4885 int ret = 0;
4886
4887 ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR);
4888 if (ret < 0)
4889 goto out;
4890
4891 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4892 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4893 TLV_PUT(sctx, BTRFS_SEND_A_XATTR_DATA, data, data_len);
4894
4895 ret = send_cmd(sctx);
4896
4897 tlv_put_failure:
4898 out:
4899 return ret;
4900 }
4901
4902 static int send_remove_xattr(struct send_ctx *sctx,
4903 struct fs_path *path,
4904 const char *name, int name_len)
4905 {
4906 int ret = 0;
4907
4908 ret = begin_cmd(sctx, BTRFS_SEND_C_REMOVE_XATTR);
4909 if (ret < 0)
4910 goto out;
4911
4912 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4913 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4914
4915 ret = send_cmd(sctx);
4916
4917 tlv_put_failure:
4918 out:
4919 return ret;
4920 }
4921
4922 static int __process_new_xattr(int num, struct btrfs_key *di_key,
4923 const char *name, int name_len, const char *data,
4924 int data_len, void *ctx)
4925 {
4926 int ret;
4927 struct send_ctx *sctx = ctx;
4928 struct fs_path *p;
4929 struct posix_acl_xattr_header dummy_acl;
4930
4931 /* Capabilities are emitted by finish_inode_if_needed */
4932 if (!strncmp(name, XATTR_NAME_CAPS, name_len))
4933 return 0;
4934
4935 p = fs_path_alloc();
4936 if (!p)
4937 return -ENOMEM;
4938
4939 /*
4940 * This hack is needed because empty acls are stored as zero byte
4941 * data in xattrs. Problem with that is, that receiving these zero byte
4942 * acls will fail later. To fix this, we send a dummy acl list that
4943 * only contains the version number and no entries.
4944 */
4945 if (!strncmp(name, XATTR_NAME_POSIX_ACL_ACCESS, name_len) ||
4946 !strncmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, name_len)) {
4947 if (data_len == 0) {
4948 dummy_acl.a_version =
4949 cpu_to_le32(POSIX_ACL_XATTR_VERSION);
4950 data = (char *)&dummy_acl;
4951 data_len = sizeof(dummy_acl);
4952 }
4953 }
4954
4955 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
4956 if (ret < 0)
4957 goto out;
4958
4959 ret = send_set_xattr(sctx, p, name, name_len, data, data_len);
4960
4961 out:
4962 fs_path_free(p);
4963 return ret;
4964 }
4965
4966 static int __process_deleted_xattr(int num, struct btrfs_key *di_key,
4967 const char *name, int name_len,
4968 const char *data, int data_len, void *ctx)
4969 {
4970 int ret;
4971 struct send_ctx *sctx = ctx;
4972 struct fs_path *p;
4973
4974 p = fs_path_alloc();
4975 if (!p)
4976 return -ENOMEM;
4977
4978 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
4979 if (ret < 0)
4980 goto out;
4981
4982 ret = send_remove_xattr(sctx, p, name, name_len);
4983
4984 out:
4985 fs_path_free(p);
4986 return ret;
4987 }
4988
4989 static int process_new_xattr(struct send_ctx *sctx)
4990 {
4991 int ret = 0;
4992
4993 ret = iterate_dir_item(sctx->send_root, sctx->left_path,
4994 __process_new_xattr, sctx);
4995
4996 return ret;
4997 }
4998
4999 static int process_deleted_xattr(struct send_ctx *sctx)
5000 {
5001 return iterate_dir_item(sctx->parent_root, sctx->right_path,
5002 __process_deleted_xattr, sctx);
5003 }
5004
5005 struct find_xattr_ctx {
5006 const char *name;
5007 int name_len;
5008 int found_idx;
5009 char *found_data;
5010 int found_data_len;
5011 };
5012
5013 static int __find_xattr(int num, struct btrfs_key *di_key, const char *name,
5014 int name_len, const char *data, int data_len, void *vctx)
5015 {
5016 struct find_xattr_ctx *ctx = vctx;
5017
5018 if (name_len == ctx->name_len &&
5019 strncmp(name, ctx->name, name_len) == 0) {
5020 ctx->found_idx = num;
5021 ctx->found_data_len = data_len;
5022 ctx->found_data = kmemdup(data, data_len, GFP_KERNEL);
5023 if (!ctx->found_data)
5024 return -ENOMEM;
5025 return 1;
5026 }
5027 return 0;
5028 }
5029
5030 static int find_xattr(struct btrfs_root *root,
5031 struct btrfs_path *path,
5032 struct btrfs_key *key,
5033 const char *name, int name_len,
5034 char **data, int *data_len)
5035 {
5036 int ret;
5037 struct find_xattr_ctx ctx;
5038
5039 ctx.name = name;
5040 ctx.name_len = name_len;
5041 ctx.found_idx = -1;
5042 ctx.found_data = NULL;
5043 ctx.found_data_len = 0;
5044
5045 ret = iterate_dir_item(root, path, __find_xattr, &ctx);
5046 if (ret < 0)
5047 return ret;
5048
5049 if (ctx.found_idx == -1)
5050 return -ENOENT;
5051 if (data) {
5052 *data = ctx.found_data;
5053 *data_len = ctx.found_data_len;
5054 } else {
5055 kfree(ctx.found_data);
5056 }
5057 return ctx.found_idx;
5058 }
5059
5060
5061 static int __process_changed_new_xattr(int num, struct btrfs_key *di_key,
5062 const char *name, int name_len,
5063 const char *data, int data_len,
5064 void *ctx)
5065 {
5066 int ret;
5067 struct send_ctx *sctx = ctx;
5068 char *found_data = NULL;
5069 int found_data_len = 0;
5070
5071 ret = find_xattr(sctx->parent_root, sctx->right_path,
5072 sctx->cmp_key, name, name_len, &found_data,
5073 &found_data_len);
5074 if (ret == -ENOENT) {
5075 ret = __process_new_xattr(num, di_key, name, name_len, data,
5076 data_len, ctx);
5077 } else if (ret >= 0) {
5078 if (data_len != found_data_len ||
5079 memcmp(data, found_data, data_len)) {
5080 ret = __process_new_xattr(num, di_key, name, name_len,
5081 data, data_len, ctx);
5082 } else {
5083 ret = 0;
5084 }
5085 }
5086
5087 kfree(found_data);
5088 return ret;
5089 }
5090
5091 static int __process_changed_deleted_xattr(int num, struct btrfs_key *di_key,
5092 const char *name, int name_len,
5093 const char *data, int data_len,
5094 void *ctx)
5095 {
5096 int ret;
5097 struct send_ctx *sctx = ctx;
5098
5099 ret = find_xattr(sctx->send_root, sctx->left_path, sctx->cmp_key,
5100 name, name_len, NULL, NULL);
5101 if (ret == -ENOENT)
5102 ret = __process_deleted_xattr(num, di_key, name, name_len, data,
5103 data_len, ctx);
5104 else if (ret >= 0)
5105 ret = 0;
5106
5107 return ret;
5108 }
5109
5110 static int process_changed_xattr(struct send_ctx *sctx)
5111 {
5112 int ret = 0;
5113
5114 ret = iterate_dir_item(sctx->send_root, sctx->left_path,
5115 __process_changed_new_xattr, sctx);
5116 if (ret < 0)
5117 goto out;
5118 ret = iterate_dir_item(sctx->parent_root, sctx->right_path,
5119 __process_changed_deleted_xattr, sctx);
5120
5121 out:
5122 return ret;
5123 }
5124
5125 static int process_all_new_xattrs(struct send_ctx *sctx)
5126 {
5127 int ret = 0;
5128 int iter_ret = 0;
5129 struct btrfs_root *root;
5130 struct btrfs_path *path;
5131 struct btrfs_key key;
5132 struct btrfs_key found_key;
5133
5134 path = alloc_path_for_send();
5135 if (!path)
5136 return -ENOMEM;
5137
5138 root = sctx->send_root;
5139
5140 key.objectid = sctx->cmp_key->objectid;
5141 key.type = BTRFS_XATTR_ITEM_KEY;
5142 key.offset = 0;
5143 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
5144 if (found_key.objectid != key.objectid ||
5145 found_key.type != key.type) {
5146 ret = 0;
5147 break;
5148 }
5149
5150 ret = iterate_dir_item(root, path, __process_new_xattr, sctx);
5151 if (ret < 0)
5152 break;
5153 }
5154 /* Catch error found during iteration */
5155 if (iter_ret < 0)
5156 ret = iter_ret;
5157
5158 btrfs_free_path(path);
5159 return ret;
5160 }
5161
5162 static int send_verity(struct send_ctx *sctx, struct fs_path *path,
5163 struct fsverity_descriptor *desc)
5164 {
5165 int ret;
5166
5167 ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY);
5168 if (ret < 0)
5169 goto out;
5170
5171 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
5172 TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM,
5173 le8_to_cpu(desc->hash_algorithm));
5174 TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE,
5175 1U << le8_to_cpu(desc->log_blocksize));
5176 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt,
5177 le8_to_cpu(desc->salt_size));
5178 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature,
5179 le32_to_cpu(desc->sig_size));
5180
5181 ret = send_cmd(sctx);
5182
5183 tlv_put_failure:
5184 out:
5185 return ret;
5186 }
5187
5188 static int process_verity(struct send_ctx *sctx)
5189 {
5190 int ret = 0;
5191 struct inode *inode;
5192 struct fs_path *p;
5193
5194 inode = btrfs_iget(sctx->cur_ino, sctx->send_root);
5195 if (IS_ERR(inode))
5196 return PTR_ERR(inode);
5197
5198 ret = btrfs_get_verity_descriptor(inode, NULL, 0);
5199 if (ret < 0)
5200 goto iput;
5201
5202 if (ret > FS_VERITY_MAX_DESCRIPTOR_SIZE) {
5203 ret = -EMSGSIZE;
5204 goto iput;
5205 }
5206 if (!sctx->verity_descriptor) {
5207 sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE,
5208 GFP_KERNEL);
5209 if (!sctx->verity_descriptor) {
5210 ret = -ENOMEM;
5211 goto iput;
5212 }
5213 }
5214
5215 ret = btrfs_get_verity_descriptor(inode, sctx->verity_descriptor, ret);
5216 if (ret < 0)
5217 goto iput;
5218
5219 p = fs_path_alloc();
5220 if (!p) {
5221 ret = -ENOMEM;
5222 goto iput;
5223 }
5224 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5225 if (ret < 0)
5226 goto free_path;
5227
5228 ret = send_verity(sctx, p, sctx->verity_descriptor);
5229 if (ret < 0)
5230 goto free_path;
5231
5232 free_path:
5233 fs_path_free(p);
5234 iput:
5235 iput(inode);
5236 return ret;
5237 }
5238
5239 static inline u64 max_send_read_size(const struct send_ctx *sctx)
5240 {
5241 return sctx->send_max_size - SZ_16K;
5242 }
5243
5244 static int put_data_header(struct send_ctx *sctx, u32 len)
5245 {
5246 if (WARN_ON_ONCE(sctx->put_data))
5247 return -EINVAL;
5248 sctx->put_data = true;
5249 if (sctx->proto >= 2) {
5250 /*
5251 * Since v2, the data attribute header doesn't include a length,
5252 * it is implicitly to the end of the command.
5253 */
5254 if (sctx->send_max_size - sctx->send_size < sizeof(__le16) + len)
5255 return -EOVERFLOW;
5256 put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size);
5257 sctx->send_size += sizeof(__le16);
5258 } else {
5259 struct btrfs_tlv_header *hdr;
5260
5261 if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len)
5262 return -EOVERFLOW;
5263 hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size);
5264 put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type);
5265 put_unaligned_le16(len, &hdr->tlv_len);
5266 sctx->send_size += sizeof(*hdr);
5267 }
5268 return 0;
5269 }
5270
5271 static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len)
5272 {
5273 struct btrfs_root *root = sctx->send_root;
5274 struct btrfs_fs_info *fs_info = root->fs_info;
5275 struct folio *folio;
5276 pgoff_t index = offset >> PAGE_SHIFT;
5277 pgoff_t last_index;
5278 unsigned pg_offset = offset_in_page(offset);
5279 struct address_space *mapping = sctx->cur_inode->i_mapping;
5280 int ret;
5281
5282 ret = put_data_header(sctx, len);
5283 if (ret)
5284 return ret;
5285
5286 last_index = (offset + len - 1) >> PAGE_SHIFT;
5287
5288 while (index <= last_index) {
5289 unsigned cur_len = min_t(unsigned, len,
5290 PAGE_SIZE - pg_offset);
5291
5292 folio = filemap_lock_folio(mapping, index);
5293 if (IS_ERR(folio)) {
5294 page_cache_sync_readahead(mapping,
5295 &sctx->ra, NULL, index,
5296 last_index + 1 - index);
5297
5298 folio = filemap_grab_folio(mapping, index);
5299 if (IS_ERR(folio)) {
5300 ret = PTR_ERR(folio);
5301 break;
5302 }
5303 }
5304
5305 WARN_ON(folio_order(folio));
5306
5307 if (folio_test_readahead(folio))
5308 page_cache_async_readahead(mapping, &sctx->ra, NULL, folio,
5309 last_index + 1 - index);
5310
5311 if (!folio_test_uptodate(folio)) {
5312 btrfs_read_folio(NULL, folio);
5313 folio_lock(folio);
5314 if (!folio_test_uptodate(folio)) {
5315 folio_unlock(folio);
5316 btrfs_err(fs_info,
5317 "send: IO error at offset %llu for inode %llu root %llu",
5318 folio_pos(folio), sctx->cur_ino,
5319 btrfs_root_id(sctx->send_root));
5320 folio_put(folio);
5321 ret = -EIO;
5322 break;
5323 }
5324 }
5325
5326 memcpy_from_folio(sctx->send_buf + sctx->send_size, folio,
5327 pg_offset, cur_len);
5328 folio_unlock(folio);
5329 folio_put(folio);
5330 index++;
5331 pg_offset = 0;
5332 len -= cur_len;
5333 sctx->send_size += cur_len;
5334 }
5335
5336 return ret;
5337 }
5338
5339 /*
5340 * Read some bytes from the current inode/file and send a write command to
5341 * user space.
5342 */
5343 static int send_write(struct send_ctx *sctx, u64 offset, u32 len)
5344 {
5345 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
5346 int ret = 0;
5347 struct fs_path *p;
5348
5349 p = fs_path_alloc();
5350 if (!p)
5351 return -ENOMEM;
5352
5353 btrfs_debug(fs_info, "send_write offset=%llu, len=%d", offset, len);
5354
5355 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5356 if (ret < 0)
5357 goto out;
5358
5359 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5360 if (ret < 0)
5361 goto out;
5362
5363 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5364 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5365 ret = put_file_data(sctx, offset, len);
5366 if (ret < 0)
5367 goto out;
5368
5369 ret = send_cmd(sctx);
5370
5371 tlv_put_failure:
5372 out:
5373 fs_path_free(p);
5374 return ret;
5375 }
5376
5377 /*
5378 * Send a clone command to user space.
5379 */
5380 static int send_clone(struct send_ctx *sctx,
5381 u64 offset, u32 len,
5382 struct clone_root *clone_root)
5383 {
5384 int ret = 0;
5385 struct fs_path *p;
5386 u64 gen;
5387
5388 btrfs_debug(sctx->send_root->fs_info,
5389 "send_clone offset=%llu, len=%d, clone_root=%llu, clone_inode=%llu, clone_offset=%llu",
5390 offset, len, btrfs_root_id(clone_root->root),
5391 clone_root->ino, clone_root->offset);
5392
5393 p = fs_path_alloc();
5394 if (!p)
5395 return -ENOMEM;
5396
5397 ret = begin_cmd(sctx, BTRFS_SEND_C_CLONE);
5398 if (ret < 0)
5399 goto out;
5400
5401 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5402 if (ret < 0)
5403 goto out;
5404
5405 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5406 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len);
5407 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5408
5409 if (clone_root->root == sctx->send_root) {
5410 ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen);
5411 if (ret < 0)
5412 goto out;
5413 ret = get_cur_path(sctx, clone_root->ino, gen, p);
5414 } else {
5415 ret = get_inode_path(clone_root->root, clone_root->ino, p);
5416 }
5417 if (ret < 0)
5418 goto out;
5419
5420 /*
5421 * If the parent we're using has a received_uuid set then use that as
5422 * our clone source as that is what we will look for when doing a
5423 * receive.
5424 *
5425 * This covers the case that we create a snapshot off of a received
5426 * subvolume and then use that as the parent and try to receive on a
5427 * different host.
5428 */
5429 if (!btrfs_is_empty_uuid(clone_root->root->root_item.received_uuid))
5430 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5431 clone_root->root->root_item.received_uuid);
5432 else
5433 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5434 clone_root->root->root_item.uuid);
5435 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
5436 btrfs_root_ctransid(&clone_root->root->root_item));
5437 TLV_PUT_PATH(sctx, BTRFS_SEND_A_CLONE_PATH, p);
5438 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_OFFSET,
5439 clone_root->offset);
5440
5441 ret = send_cmd(sctx);
5442
5443 tlv_put_failure:
5444 out:
5445 fs_path_free(p);
5446 return ret;
5447 }
5448
5449 /*
5450 * Send an update extent command to user space.
5451 */
5452 static int send_update_extent(struct send_ctx *sctx,
5453 u64 offset, u32 len)
5454 {
5455 int ret = 0;
5456 struct fs_path *p;
5457
5458 p = fs_path_alloc();
5459 if (!p)
5460 return -ENOMEM;
5461
5462 ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT);
5463 if (ret < 0)
5464 goto out;
5465
5466 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5467 if (ret < 0)
5468 goto out;
5469
5470 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5471 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5472 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
5473
5474 ret = send_cmd(sctx);
5475
5476 tlv_put_failure:
5477 out:
5478 fs_path_free(p);
5479 return ret;
5480 }
5481
5482 static int send_hole(struct send_ctx *sctx, u64 end)
5483 {
5484 struct fs_path *p = NULL;
5485 u64 read_size = max_send_read_size(sctx);
5486 u64 offset = sctx->cur_inode_last_extent;
5487 int ret = 0;
5488
5489 /*
5490 * A hole that starts at EOF or beyond it. Since we do not yet support
5491 * fallocate (for extent preallocation and hole punching), sending a
5492 * write of zeroes starting at EOF or beyond would later require issuing
5493 * a truncate operation which would undo the write and achieve nothing.
5494 */
5495 if (offset >= sctx->cur_inode_size)
5496 return 0;
5497
5498 /*
5499 * Don't go beyond the inode's i_size due to prealloc extents that start
5500 * after the i_size.
5501 */
5502 end = min_t(u64, end, sctx->cur_inode_size);
5503
5504 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5505 return send_update_extent(sctx, offset, end - offset);
5506
5507 p = fs_path_alloc();
5508 if (!p)
5509 return -ENOMEM;
5510 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5511 if (ret < 0)
5512 goto tlv_put_failure;
5513 while (offset < end) {
5514 u64 len = min(end - offset, read_size);
5515
5516 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5517 if (ret < 0)
5518 break;
5519 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5520 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5521 ret = put_data_header(sctx, len);
5522 if (ret < 0)
5523 break;
5524 memset(sctx->send_buf + sctx->send_size, 0, len);
5525 sctx->send_size += len;
5526 ret = send_cmd(sctx);
5527 if (ret < 0)
5528 break;
5529 offset += len;
5530 }
5531 sctx->cur_inode_next_write_offset = offset;
5532 tlv_put_failure:
5533 fs_path_free(p);
5534 return ret;
5535 }
5536
5537 static int send_encoded_inline_extent(struct send_ctx *sctx,
5538 struct btrfs_path *path, u64 offset,
5539 u64 len)
5540 {
5541 struct btrfs_root *root = sctx->send_root;
5542 struct btrfs_fs_info *fs_info = root->fs_info;
5543 struct inode *inode;
5544 struct fs_path *fspath;
5545 struct extent_buffer *leaf = path->nodes[0];
5546 struct btrfs_key key;
5547 struct btrfs_file_extent_item *ei;
5548 u64 ram_bytes;
5549 size_t inline_size;
5550 int ret;
5551
5552 inode = btrfs_iget(sctx->cur_ino, root);
5553 if (IS_ERR(inode))
5554 return PTR_ERR(inode);
5555
5556 fspath = fs_path_alloc();
5557 if (!fspath) {
5558 ret = -ENOMEM;
5559 goto out;
5560 }
5561
5562 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5563 if (ret < 0)
5564 goto out;
5565
5566 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5567 if (ret < 0)
5568 goto out;
5569
5570 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5571 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5572 ram_bytes = btrfs_file_extent_ram_bytes(leaf, ei);
5573 inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
5574
5575 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5576 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5577 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5578 min(key.offset + ram_bytes - offset, len));
5579 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes);
5580 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset);
5581 ret = btrfs_encoded_io_compression_from_extent(fs_info,
5582 btrfs_file_extent_compression(leaf, ei));
5583 if (ret < 0)
5584 goto out;
5585 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5586
5587 ret = put_data_header(sctx, inline_size);
5588 if (ret < 0)
5589 goto out;
5590 read_extent_buffer(leaf, sctx->send_buf + sctx->send_size,
5591 btrfs_file_extent_inline_start(ei), inline_size);
5592 sctx->send_size += inline_size;
5593
5594 ret = send_cmd(sctx);
5595
5596 tlv_put_failure:
5597 out:
5598 fs_path_free(fspath);
5599 iput(inode);
5600 return ret;
5601 }
5602
5603 static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path,
5604 u64 offset, u64 len)
5605 {
5606 struct btrfs_root *root = sctx->send_root;
5607 struct btrfs_fs_info *fs_info = root->fs_info;
5608 struct inode *inode;
5609 struct fs_path *fspath;
5610 struct extent_buffer *leaf = path->nodes[0];
5611 struct btrfs_key key;
5612 struct btrfs_file_extent_item *ei;
5613 u64 disk_bytenr, disk_num_bytes;
5614 u32 data_offset;
5615 struct btrfs_cmd_header *hdr;
5616 u32 crc;
5617 int ret;
5618
5619 inode = btrfs_iget(sctx->cur_ino, root);
5620 if (IS_ERR(inode))
5621 return PTR_ERR(inode);
5622
5623 fspath = fs_path_alloc();
5624 if (!fspath) {
5625 ret = -ENOMEM;
5626 goto out;
5627 }
5628
5629 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5630 if (ret < 0)
5631 goto out;
5632
5633 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5634 if (ret < 0)
5635 goto out;
5636
5637 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5638 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5639 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
5640 disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, ei);
5641
5642 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5643 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5644 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5645 min(key.offset + btrfs_file_extent_num_bytes(leaf, ei) - offset,
5646 len));
5647 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN,
5648 btrfs_file_extent_ram_bytes(leaf, ei));
5649 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET,
5650 offset - key.offset + btrfs_file_extent_offset(leaf, ei));
5651 ret = btrfs_encoded_io_compression_from_extent(fs_info,
5652 btrfs_file_extent_compression(leaf, ei));
5653 if (ret < 0)
5654 goto out;
5655 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5656 TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0);
5657
5658 ret = put_data_header(sctx, disk_num_bytes);
5659 if (ret < 0)
5660 goto out;
5661
5662 /*
5663 * We want to do I/O directly into the send buffer, so get the next page
5664 * boundary in the send buffer. This means that there may be a gap
5665 * between the beginning of the command and the file data.
5666 */
5667 data_offset = PAGE_ALIGN(sctx->send_size);
5668 if (data_offset > sctx->send_max_size ||
5669 sctx->send_max_size - data_offset < disk_num_bytes) {
5670 ret = -EOVERFLOW;
5671 goto out;
5672 }
5673
5674 /*
5675 * Note that send_buf is a mapping of send_buf_pages, so this is really
5676 * reading into send_buf.
5677 */
5678 ret = btrfs_encoded_read_regular_fill_pages(BTRFS_I(inode), offset,
5679 disk_bytenr, disk_num_bytes,
5680 sctx->send_buf_pages +
5681 (data_offset >> PAGE_SHIFT));
5682 if (ret)
5683 goto out;
5684
5685 hdr = (struct btrfs_cmd_header *)sctx->send_buf;
5686 hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr));
5687 hdr->crc = 0;
5688 crc = crc32c(0, sctx->send_buf, sctx->send_size);
5689 crc = crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes);
5690 hdr->crc = cpu_to_le32(crc);
5691
5692 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
5693 &sctx->send_off);
5694 if (!ret) {
5695 ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset,
5696 disk_num_bytes, &sctx->send_off);
5697 }
5698 sctx->send_size = 0;
5699 sctx->put_data = false;
5700
5701 tlv_put_failure:
5702 out:
5703 fs_path_free(fspath);
5704 iput(inode);
5705 return ret;
5706 }
5707
5708 static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path,
5709 const u64 offset, const u64 len)
5710 {
5711 const u64 end = offset + len;
5712 struct extent_buffer *leaf = path->nodes[0];
5713 struct btrfs_file_extent_item *ei;
5714 u64 read_size = max_send_read_size(sctx);
5715 u64 sent = 0;
5716
5717 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5718 return send_update_extent(sctx, offset, len);
5719
5720 ei = btrfs_item_ptr(leaf, path->slots[0],
5721 struct btrfs_file_extent_item);
5722 if ((sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) &&
5723 btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) {
5724 bool is_inline = (btrfs_file_extent_type(leaf, ei) ==
5725 BTRFS_FILE_EXTENT_INLINE);
5726
5727 /*
5728 * Send the compressed extent unless the compressed data is
5729 * larger than the decompressed data. This can happen if we're
5730 * not sending the entire extent, either because it has been
5731 * partially overwritten/truncated or because this is a part of
5732 * the extent that we couldn't clone in clone_range().
5733 */
5734 if (is_inline &&
5735 btrfs_file_extent_inline_item_len(leaf,
5736 path->slots[0]) <= len) {
5737 return send_encoded_inline_extent(sctx, path, offset,
5738 len);
5739 } else if (!is_inline &&
5740 btrfs_file_extent_disk_num_bytes(leaf, ei) <= len) {
5741 return send_encoded_extent(sctx, path, offset, len);
5742 }
5743 }
5744
5745 if (sctx->cur_inode == NULL) {
5746 struct btrfs_root *root = sctx->send_root;
5747
5748 sctx->cur_inode = btrfs_iget(sctx->cur_ino, root);
5749 if (IS_ERR(sctx->cur_inode)) {
5750 int err = PTR_ERR(sctx->cur_inode);
5751
5752 sctx->cur_inode = NULL;
5753 return err;
5754 }
5755 memset(&sctx->ra, 0, sizeof(struct file_ra_state));
5756 file_ra_state_init(&sctx->ra, sctx->cur_inode->i_mapping);
5757
5758 /*
5759 * It's very likely there are no pages from this inode in the page
5760 * cache, so after reading extents and sending their data, we clean
5761 * the page cache to avoid trashing the page cache (adding pressure
5762 * to the page cache and forcing eviction of other data more useful
5763 * for applications).
5764 *
5765 * We decide if we should clean the page cache simply by checking
5766 * if the inode's mapping nrpages is 0 when we first open it, and
5767 * not by using something like filemap_range_has_page() before
5768 * reading an extent because when we ask the readahead code to
5769 * read a given file range, it may (and almost always does) read
5770 * pages from beyond that range (see the documentation for
5771 * page_cache_sync_readahead()), so it would not be reliable,
5772 * because after reading the first extent future calls to
5773 * filemap_range_has_page() would return true because the readahead
5774 * on the previous extent resulted in reading pages of the current
5775 * extent as well.
5776 */
5777 sctx->clean_page_cache = (sctx->cur_inode->i_mapping->nrpages == 0);
5778 sctx->page_cache_clear_start = round_down(offset, PAGE_SIZE);
5779 }
5780
5781 while (sent < len) {
5782 u64 size = min(len - sent, read_size);
5783 int ret;
5784
5785 ret = send_write(sctx, offset + sent, size);
5786 if (ret < 0)
5787 return ret;
5788 sent += size;
5789 }
5790
5791 if (sctx->clean_page_cache && PAGE_ALIGNED(end)) {
5792 /*
5793 * Always operate only on ranges that are a multiple of the page
5794 * size. This is not only to prevent zeroing parts of a page in
5795 * the case of subpage sector size, but also to guarantee we evict
5796 * pages, as passing a range that is smaller than page size does
5797 * not evict the respective page (only zeroes part of its content).
5798 *
5799 * Always start from the end offset of the last range cleared.
5800 * This is because the readahead code may (and very often does)
5801 * reads pages beyond the range we request for readahead. So if
5802 * we have an extent layout like this:
5803 *
5804 * [ extent A ] [ extent B ] [ extent C ]
5805 *
5806 * When we ask page_cache_sync_readahead() to read extent A, it
5807 * may also trigger reads for pages of extent B. If we are doing
5808 * an incremental send and extent B has not changed between the
5809 * parent and send snapshots, some or all of its pages may end
5810 * up being read and placed in the page cache. So when truncating
5811 * the page cache we always start from the end offset of the
5812 * previously processed extent up to the end of the current
5813 * extent.
5814 */
5815 truncate_inode_pages_range(&sctx->cur_inode->i_data,
5816 sctx->page_cache_clear_start,
5817 end - 1);
5818 sctx->page_cache_clear_start = end;
5819 }
5820
5821 return 0;
5822 }
5823
5824 /*
5825 * Search for a capability xattr related to sctx->cur_ino. If the capability is
5826 * found, call send_set_xattr function to emit it.
5827 *
5828 * Return 0 if there isn't a capability, or when the capability was emitted
5829 * successfully, or < 0 if an error occurred.
5830 */
5831 static int send_capabilities(struct send_ctx *sctx)
5832 {
5833 struct fs_path *fspath = NULL;
5834 struct btrfs_path *path;
5835 struct btrfs_dir_item *di;
5836 struct extent_buffer *leaf;
5837 unsigned long data_ptr;
5838 char *buf = NULL;
5839 int buf_len;
5840 int ret = 0;
5841
5842 path = alloc_path_for_send();
5843 if (!path)
5844 return -ENOMEM;
5845
5846 di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino,
5847 XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), 0);
5848 if (!di) {
5849 /* There is no xattr for this inode */
5850 goto out;
5851 } else if (IS_ERR(di)) {
5852 ret = PTR_ERR(di);
5853 goto out;
5854 }
5855
5856 leaf = path->nodes[0];
5857 buf_len = btrfs_dir_data_len(leaf, di);
5858
5859 fspath = fs_path_alloc();
5860 buf = kmalloc(buf_len, GFP_KERNEL);
5861 if (!fspath || !buf) {
5862 ret = -ENOMEM;
5863 goto out;
5864 }
5865
5866 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5867 if (ret < 0)
5868 goto out;
5869
5870 data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di);
5871 read_extent_buffer(leaf, buf, data_ptr, buf_len);
5872
5873 ret = send_set_xattr(sctx, fspath, XATTR_NAME_CAPS,
5874 strlen(XATTR_NAME_CAPS), buf, buf_len);
5875 out:
5876 kfree(buf);
5877 fs_path_free(fspath);
5878 btrfs_free_path(path);
5879 return ret;
5880 }
5881
5882 static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path,
5883 struct clone_root *clone_root, const u64 disk_byte,
5884 u64 data_offset, u64 offset, u64 len)
5885 {
5886 struct btrfs_path *path;
5887 struct btrfs_key key;
5888 int ret;
5889 struct btrfs_inode_info info;
5890 u64 clone_src_i_size = 0;
5891
5892 /*
5893 * Prevent cloning from a zero offset with a length matching the sector
5894 * size because in some scenarios this will make the receiver fail.
5895 *
5896 * For example, if in the source filesystem the extent at offset 0
5897 * has a length of sectorsize and it was written using direct IO, then
5898 * it can never be an inline extent (even if compression is enabled).
5899 * Then this extent can be cloned in the original filesystem to a non
5900 * zero file offset, but it may not be possible to clone in the
5901 * destination filesystem because it can be inlined due to compression
5902 * on the destination filesystem (as the receiver's write operations are
5903 * always done using buffered IO). The same happens when the original
5904 * filesystem does not have compression enabled but the destination
5905 * filesystem has.
5906 */
5907 if (clone_root->offset == 0 &&
5908 len == sctx->send_root->fs_info->sectorsize)
5909 return send_extent_data(sctx, dst_path, offset, len);
5910
5911 path = alloc_path_for_send();
5912 if (!path)
5913 return -ENOMEM;
5914
5915 /*
5916 * There are inodes that have extents that lie behind its i_size. Don't
5917 * accept clones from these extents.
5918 */
5919 ret = get_inode_info(clone_root->root, clone_root->ino, &info);
5920 btrfs_release_path(path);
5921 if (ret < 0)
5922 goto out;
5923 clone_src_i_size = info.size;
5924
5925 /*
5926 * We can't send a clone operation for the entire range if we find
5927 * extent items in the respective range in the source file that
5928 * refer to different extents or if we find holes.
5929 * So check for that and do a mix of clone and regular write/copy
5930 * operations if needed.
5931 *
5932 * Example:
5933 *
5934 * mkfs.btrfs -f /dev/sda
5935 * mount /dev/sda /mnt
5936 * xfs_io -f -c "pwrite -S 0xaa 0K 100K" /mnt/foo
5937 * cp --reflink=always /mnt/foo /mnt/bar
5938 * xfs_io -c "pwrite -S 0xbb 50K 50K" /mnt/foo
5939 * btrfs subvolume snapshot -r /mnt /mnt/snap
5940 *
5941 * If when we send the snapshot and we are processing file bar (which
5942 * has a higher inode number than foo) we blindly send a clone operation
5943 * for the [0, 100K[ range from foo to bar, the receiver ends up getting
5944 * a file bar that matches the content of file foo - iow, doesn't match
5945 * the content from bar in the original filesystem.
5946 */
5947 key.objectid = clone_root->ino;
5948 key.type = BTRFS_EXTENT_DATA_KEY;
5949 key.offset = clone_root->offset;
5950 ret = btrfs_search_slot(NULL, clone_root->root, &key, path, 0, 0);
5951 if (ret < 0)
5952 goto out;
5953 if (ret > 0 && path->slots[0] > 0) {
5954 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
5955 if (key.objectid == clone_root->ino &&
5956 key.type == BTRFS_EXTENT_DATA_KEY)
5957 path->slots[0]--;
5958 }
5959
5960 while (true) {
5961 struct extent_buffer *leaf = path->nodes[0];
5962 int slot = path->slots[0];
5963 struct btrfs_file_extent_item *ei;
5964 u8 type;
5965 u64 ext_len;
5966 u64 clone_len;
5967 u64 clone_data_offset;
5968 bool crossed_src_i_size = false;
5969
5970 if (slot >= btrfs_header_nritems(leaf)) {
5971 ret = btrfs_next_leaf(clone_root->root, path);
5972 if (ret < 0)
5973 goto out;
5974 else if (ret > 0)
5975 break;
5976 continue;
5977 }
5978
5979 btrfs_item_key_to_cpu(leaf, &key, slot);
5980
5981 /*
5982 * We might have an implicit trailing hole (NO_HOLES feature
5983 * enabled). We deal with it after leaving this loop.
5984 */
5985 if (key.objectid != clone_root->ino ||
5986 key.type != BTRFS_EXTENT_DATA_KEY)
5987 break;
5988
5989 ei = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
5990 type = btrfs_file_extent_type(leaf, ei);
5991 if (type == BTRFS_FILE_EXTENT_INLINE) {
5992 ext_len = btrfs_file_extent_ram_bytes(leaf, ei);
5993 ext_len = PAGE_ALIGN(ext_len);
5994 } else {
5995 ext_len = btrfs_file_extent_num_bytes(leaf, ei);
5996 }
5997
5998 if (key.offset + ext_len <= clone_root->offset)
5999 goto next;
6000
6001 if (key.offset > clone_root->offset) {
6002 /* Implicit hole, NO_HOLES feature enabled. */
6003 u64 hole_len = key.offset - clone_root->offset;
6004
6005 if (hole_len > len)
6006 hole_len = len;
6007 ret = send_extent_data(sctx, dst_path, offset,
6008 hole_len);
6009 if (ret < 0)
6010 goto out;
6011
6012 len -= hole_len;
6013 if (len == 0)
6014 break;
6015 offset += hole_len;
6016 clone_root->offset += hole_len;
6017 data_offset += hole_len;
6018 }
6019
6020 if (key.offset >= clone_root->offset + len)
6021 break;
6022
6023 if (key.offset >= clone_src_i_size)
6024 break;
6025
6026 if (key.offset + ext_len > clone_src_i_size) {
6027 ext_len = clone_src_i_size - key.offset;
6028 crossed_src_i_size = true;
6029 }
6030
6031 clone_data_offset = btrfs_file_extent_offset(leaf, ei);
6032 if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte) {
6033 clone_root->offset = key.offset;
6034 if (clone_data_offset < data_offset &&
6035 clone_data_offset + ext_len > data_offset) {
6036 u64 extent_offset;
6037
6038 extent_offset = data_offset - clone_data_offset;
6039 ext_len -= extent_offset;
6040 clone_data_offset += extent_offset;
6041 clone_root->offset += extent_offset;
6042 }
6043 }
6044
6045 clone_len = min_t(u64, ext_len, len);
6046
6047 if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte &&
6048 clone_data_offset == data_offset) {
6049 const u64 src_end = clone_root->offset + clone_len;
6050 const u64 sectorsize = SZ_64K;
6051
6052 /*
6053 * We can't clone the last block, when its size is not
6054 * sector size aligned, into the middle of a file. If we
6055 * do so, the receiver will get a failure (-EINVAL) when
6056 * trying to clone or will silently corrupt the data in
6057 * the destination file if it's on a kernel without the
6058 * fix introduced by commit ac765f83f1397646
6059 * ("Btrfs: fix data corruption due to cloning of eof
6060 * block).
6061 *
6062 * So issue a clone of the aligned down range plus a
6063 * regular write for the eof block, if we hit that case.
6064 *
6065 * Also, we use the maximum possible sector size, 64K,
6066 * because we don't know what's the sector size of the
6067 * filesystem that receives the stream, so we have to
6068 * assume the largest possible sector size.
6069 */
6070 if (src_end == clone_src_i_size &&
6071 !IS_ALIGNED(src_end, sectorsize) &&
6072 offset + clone_len < sctx->cur_inode_size) {
6073 u64 slen;
6074
6075 slen = ALIGN_DOWN(src_end - clone_root->offset,
6076 sectorsize);
6077 if (slen > 0) {
6078 ret = send_clone(sctx, offset, slen,
6079 clone_root);
6080 if (ret < 0)
6081 goto out;
6082 }
6083 ret = send_extent_data(sctx, dst_path,
6084 offset + slen,
6085 clone_len - slen);
6086 } else {
6087 ret = send_clone(sctx, offset, clone_len,
6088 clone_root);
6089 }
6090 } else if (crossed_src_i_size && clone_len < len) {
6091 /*
6092 * If we are at i_size of the clone source inode and we
6093 * can not clone from it, terminate the loop. This is
6094 * to avoid sending two write operations, one with a
6095 * length matching clone_len and the final one after
6096 * this loop with a length of len - clone_len.
6097 *
6098 * When using encoded writes (BTRFS_SEND_FLAG_COMPRESSED
6099 * was passed to the send ioctl), this helps avoid
6100 * sending an encoded write for an offset that is not
6101 * sector size aligned, in case the i_size of the source
6102 * inode is not sector size aligned. That will make the
6103 * receiver fallback to decompression of the data and
6104 * writing it using regular buffered IO, therefore while
6105 * not incorrect, it's not optimal due decompression and
6106 * possible re-compression at the receiver.
6107 */
6108 break;
6109 } else {
6110 ret = send_extent_data(sctx, dst_path, offset,
6111 clone_len);
6112 }
6113
6114 if (ret < 0)
6115 goto out;
6116
6117 len -= clone_len;
6118 if (len == 0)
6119 break;
6120 offset += clone_len;
6121 clone_root->offset += clone_len;
6122
6123 /*
6124 * If we are cloning from the file we are currently processing,
6125 * and using the send root as the clone root, we must stop once
6126 * the current clone offset reaches the current eof of the file
6127 * at the receiver, otherwise we would issue an invalid clone
6128 * operation (source range going beyond eof) and cause the
6129 * receiver to fail. So if we reach the current eof, bail out
6130 * and fallback to a regular write.
6131 */
6132 if (clone_root->root == sctx->send_root &&
6133 clone_root->ino == sctx->cur_ino &&
6134 clone_root->offset >= sctx->cur_inode_next_write_offset)
6135 break;
6136
6137 data_offset += clone_len;
6138 next:
6139 path->slots[0]++;
6140 }
6141
6142 if (len > 0)
6143 ret = send_extent_data(sctx, dst_path, offset, len);
6144 else
6145 ret = 0;
6146 out:
6147 btrfs_free_path(path);
6148 return ret;
6149 }
6150
6151 static int send_write_or_clone(struct send_ctx *sctx,
6152 struct btrfs_path *path,
6153 struct btrfs_key *key,
6154 struct clone_root *clone_root)
6155 {
6156 int ret = 0;
6157 u64 offset = key->offset;
6158 u64 end;
6159 u64 bs = sctx->send_root->fs_info->sectorsize;
6160
6161 end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size);
6162 if (offset >= end)
6163 return 0;
6164
6165 if (clone_root && IS_ALIGNED(end, bs)) {
6166 struct btrfs_file_extent_item *ei;
6167 u64 disk_byte;
6168 u64 data_offset;
6169
6170 ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6171 struct btrfs_file_extent_item);
6172 disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei);
6173 data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
6174 ret = clone_range(sctx, path, clone_root, disk_byte,
6175 data_offset, offset, end - offset);
6176 } else {
6177 ret = send_extent_data(sctx, path, offset, end - offset);
6178 }
6179 sctx->cur_inode_next_write_offset = end;
6180 return ret;
6181 }
6182
6183 static int is_extent_unchanged(struct send_ctx *sctx,
6184 struct btrfs_path *left_path,
6185 struct btrfs_key *ekey)
6186 {
6187 int ret = 0;
6188 struct btrfs_key key;
6189 struct btrfs_path *path = NULL;
6190 struct extent_buffer *eb;
6191 int slot;
6192 struct btrfs_key found_key;
6193 struct btrfs_file_extent_item *ei;
6194 u64 left_disknr;
6195 u64 right_disknr;
6196 u64 left_offset;
6197 u64 right_offset;
6198 u64 left_offset_fixed;
6199 u64 left_len;
6200 u64 right_len;
6201 u64 left_gen;
6202 u64 right_gen;
6203 u8 left_type;
6204 u8 right_type;
6205
6206 path = alloc_path_for_send();
6207 if (!path)
6208 return -ENOMEM;
6209
6210 eb = left_path->nodes[0];
6211 slot = left_path->slots[0];
6212 ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6213 left_type = btrfs_file_extent_type(eb, ei);
6214
6215 if (left_type != BTRFS_FILE_EXTENT_REG) {
6216 ret = 0;
6217 goto out;
6218 }
6219 left_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6220 left_len = btrfs_file_extent_num_bytes(eb, ei);
6221 left_offset = btrfs_file_extent_offset(eb, ei);
6222 left_gen = btrfs_file_extent_generation(eb, ei);
6223
6224 /*
6225 * Following comments will refer to these graphics. L is the left
6226 * extents which we are checking at the moment. 1-8 are the right
6227 * extents that we iterate.
6228 *
6229 * |-----L-----|
6230 * |-1-|-2a-|-3-|-4-|-5-|-6-|
6231 *
6232 * |-----L-----|
6233 * |--1--|-2b-|...(same as above)
6234 *
6235 * Alternative situation. Happens on files where extents got split.
6236 * |-----L-----|
6237 * |-----------7-----------|-6-|
6238 *
6239 * Alternative situation. Happens on files which got larger.
6240 * |-----L-----|
6241 * |-8-|
6242 * Nothing follows after 8.
6243 */
6244
6245 key.objectid = ekey->objectid;
6246 key.type = BTRFS_EXTENT_DATA_KEY;
6247 key.offset = ekey->offset;
6248 ret = btrfs_search_slot_for_read(sctx->parent_root, &key, path, 0, 0);
6249 if (ret < 0)
6250 goto out;
6251 if (ret) {
6252 ret = 0;
6253 goto out;
6254 }
6255
6256 /*
6257 * Handle special case where the right side has no extents at all.
6258 */
6259 eb = path->nodes[0];
6260 slot = path->slots[0];
6261 btrfs_item_key_to_cpu(eb, &found_key, slot);
6262 if (found_key.objectid != key.objectid ||
6263 found_key.type != key.type) {
6264 /* If we're a hole then just pretend nothing changed */
6265 ret = (left_disknr) ? 0 : 1;
6266 goto out;
6267 }
6268
6269 /*
6270 * We're now on 2a, 2b or 7.
6271 */
6272 key = found_key;
6273 while (key.offset < ekey->offset + left_len) {
6274 ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6275 right_type = btrfs_file_extent_type(eb, ei);
6276 if (right_type != BTRFS_FILE_EXTENT_REG &&
6277 right_type != BTRFS_FILE_EXTENT_INLINE) {
6278 ret = 0;
6279 goto out;
6280 }
6281
6282 if (right_type == BTRFS_FILE_EXTENT_INLINE) {
6283 right_len = btrfs_file_extent_ram_bytes(eb, ei);
6284 right_len = PAGE_ALIGN(right_len);
6285 } else {
6286 right_len = btrfs_file_extent_num_bytes(eb, ei);
6287 }
6288
6289 /*
6290 * Are we at extent 8? If yes, we know the extent is changed.
6291 * This may only happen on the first iteration.
6292 */
6293 if (found_key.offset + right_len <= ekey->offset) {
6294 /* If we're a hole just pretend nothing changed */
6295 ret = (left_disknr) ? 0 : 1;
6296 goto out;
6297 }
6298
6299 /*
6300 * We just wanted to see if when we have an inline extent, what
6301 * follows it is a regular extent (wanted to check the above
6302 * condition for inline extents too). This should normally not
6303 * happen but it's possible for example when we have an inline
6304 * compressed extent representing data with a size matching
6305 * the page size (currently the same as sector size).
6306 */
6307 if (right_type == BTRFS_FILE_EXTENT_INLINE) {
6308 ret = 0;
6309 goto out;
6310 }
6311
6312 right_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6313 right_offset = btrfs_file_extent_offset(eb, ei);
6314 right_gen = btrfs_file_extent_generation(eb, ei);
6315
6316 left_offset_fixed = left_offset;
6317 if (key.offset < ekey->offset) {
6318 /* Fix the right offset for 2a and 7. */
6319 right_offset += ekey->offset - key.offset;
6320 } else {
6321 /* Fix the left offset for all behind 2a and 2b */
6322 left_offset_fixed += key.offset - ekey->offset;
6323 }
6324
6325 /*
6326 * Check if we have the same extent.
6327 */
6328 if (left_disknr != right_disknr ||
6329 left_offset_fixed != right_offset ||
6330 left_gen != right_gen) {
6331 ret = 0;
6332 goto out;
6333 }
6334
6335 /*
6336 * Go to the next extent.
6337 */
6338 ret = btrfs_next_item(sctx->parent_root, path);
6339 if (ret < 0)
6340 goto out;
6341 if (!ret) {
6342 eb = path->nodes[0];
6343 slot = path->slots[0];
6344 btrfs_item_key_to_cpu(eb, &found_key, slot);
6345 }
6346 if (ret || found_key.objectid != key.objectid ||
6347 found_key.type != key.type) {
6348 key.offset += right_len;
6349 break;
6350 }
6351 if (found_key.offset != key.offset + right_len) {
6352 ret = 0;
6353 goto out;
6354 }
6355 key = found_key;
6356 }
6357
6358 /*
6359 * We're now behind the left extent (treat as unchanged) or at the end
6360 * of the right side (treat as changed).
6361 */
6362 if (key.offset >= ekey->offset + left_len)
6363 ret = 1;
6364 else
6365 ret = 0;
6366
6367
6368 out:
6369 btrfs_free_path(path);
6370 return ret;
6371 }
6372
6373 static int get_last_extent(struct send_ctx *sctx, u64 offset)
6374 {
6375 struct btrfs_path *path;
6376 struct btrfs_root *root = sctx->send_root;
6377 struct btrfs_key key;
6378 int ret;
6379
6380 path = alloc_path_for_send();
6381 if (!path)
6382 return -ENOMEM;
6383
6384 sctx->cur_inode_last_extent = 0;
6385
6386 key.objectid = sctx->cur_ino;
6387 key.type = BTRFS_EXTENT_DATA_KEY;
6388 key.offset = offset;
6389 ret = btrfs_search_slot_for_read(root, &key, path, 0, 1);
6390 if (ret < 0)
6391 goto out;
6392 ret = 0;
6393 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
6394 if (key.objectid != sctx->cur_ino || key.type != BTRFS_EXTENT_DATA_KEY)
6395 goto out;
6396
6397 sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6398 out:
6399 btrfs_free_path(path);
6400 return ret;
6401 }
6402
6403 static int range_is_hole_in_parent(struct send_ctx *sctx,
6404 const u64 start,
6405 const u64 end)
6406 {
6407 struct btrfs_path *path;
6408 struct btrfs_key key;
6409 struct btrfs_root *root = sctx->parent_root;
6410 u64 search_start = start;
6411 int ret;
6412
6413 path = alloc_path_for_send();
6414 if (!path)
6415 return -ENOMEM;
6416
6417 key.objectid = sctx->cur_ino;
6418 key.type = BTRFS_EXTENT_DATA_KEY;
6419 key.offset = search_start;
6420 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6421 if (ret < 0)
6422 goto out;
6423 if (ret > 0 && path->slots[0] > 0)
6424 path->slots[0]--;
6425
6426 while (search_start < end) {
6427 struct extent_buffer *leaf = path->nodes[0];
6428 int slot = path->slots[0];
6429 struct btrfs_file_extent_item *fi;
6430 u64 extent_end;
6431
6432 if (slot >= btrfs_header_nritems(leaf)) {
6433 ret = btrfs_next_leaf(root, path);
6434 if (ret < 0)
6435 goto out;
6436 else if (ret > 0)
6437 break;
6438 continue;
6439 }
6440
6441 btrfs_item_key_to_cpu(leaf, &key, slot);
6442 if (key.objectid < sctx->cur_ino ||
6443 key.type < BTRFS_EXTENT_DATA_KEY)
6444 goto next;
6445 if (key.objectid > sctx->cur_ino ||
6446 key.type > BTRFS_EXTENT_DATA_KEY ||
6447 key.offset >= end)
6448 break;
6449
6450 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
6451 extent_end = btrfs_file_extent_end(path);
6452 if (extent_end <= start)
6453 goto next;
6454 if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) {
6455 search_start = extent_end;
6456 goto next;
6457 }
6458 ret = 0;
6459 goto out;
6460 next:
6461 path->slots[0]++;
6462 }
6463 ret = 1;
6464 out:
6465 btrfs_free_path(path);
6466 return ret;
6467 }
6468
6469 static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path,
6470 struct btrfs_key *key)
6471 {
6472 int ret = 0;
6473
6474 if (sctx->cur_ino != key->objectid || !need_send_hole(sctx))
6475 return 0;
6476
6477 /*
6478 * Get last extent's end offset (exclusive) if we haven't determined it
6479 * yet (we're processing the first file extent item that is new), or if
6480 * we're at the first slot of a leaf and the last extent's end is less
6481 * than the current extent's offset, because we might have skipped
6482 * entire leaves that contained only file extent items for our current
6483 * inode. These leaves have a generation number smaller (older) than the
6484 * one in the current leaf and the leaf our last extent came from, and
6485 * are located between these 2 leaves.
6486 */
6487 if ((sctx->cur_inode_last_extent == (u64)-1) ||
6488 (path->slots[0] == 0 && sctx->cur_inode_last_extent < key->offset)) {
6489 ret = get_last_extent(sctx, key->offset - 1);
6490 if (ret)
6491 return ret;
6492 }
6493
6494 if (sctx->cur_inode_last_extent < key->offset) {
6495 ret = range_is_hole_in_parent(sctx,
6496 sctx->cur_inode_last_extent,
6497 key->offset);
6498 if (ret < 0)
6499 return ret;
6500 else if (ret == 0)
6501 ret = send_hole(sctx, key->offset);
6502 else
6503 ret = 0;
6504 }
6505 sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6506 return ret;
6507 }
6508
6509 static int process_extent(struct send_ctx *sctx,
6510 struct btrfs_path *path,
6511 struct btrfs_key *key)
6512 {
6513 struct clone_root *found_clone = NULL;
6514 int ret = 0;
6515
6516 if (S_ISLNK(sctx->cur_inode_mode))
6517 return 0;
6518
6519 if (sctx->parent_root && !sctx->cur_inode_new) {
6520 ret = is_extent_unchanged(sctx, path, key);
6521 if (ret < 0)
6522 goto out;
6523 if (ret) {
6524 ret = 0;
6525 goto out_hole;
6526 }
6527 } else {
6528 struct btrfs_file_extent_item *ei;
6529 u8 type;
6530
6531 ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6532 struct btrfs_file_extent_item);
6533 type = btrfs_file_extent_type(path->nodes[0], ei);
6534 if (type == BTRFS_FILE_EXTENT_PREALLOC ||
6535 type == BTRFS_FILE_EXTENT_REG) {
6536 /*
6537 * The send spec does not have a prealloc command yet,
6538 * so just leave a hole for prealloc'ed extents until
6539 * we have enough commands queued up to justify rev'ing
6540 * the send spec.
6541 */
6542 if (type == BTRFS_FILE_EXTENT_PREALLOC) {
6543 ret = 0;
6544 goto out;
6545 }
6546
6547 /* Have a hole, just skip it. */
6548 if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0) {
6549 ret = 0;
6550 goto out;
6551 }
6552 }
6553 }
6554
6555 ret = find_extent_clone(sctx, path, key->objectid, key->offset,
6556 sctx->cur_inode_size, &found_clone);
6557 if (ret != -ENOENT && ret < 0)
6558 goto out;
6559
6560 ret = send_write_or_clone(sctx, path, key, found_clone);
6561 if (ret)
6562 goto out;
6563 out_hole:
6564 ret = maybe_send_hole(sctx, path, key);
6565 out:
6566 return ret;
6567 }
6568
6569 static int process_all_extents(struct send_ctx *sctx)
6570 {
6571 int ret = 0;
6572 int iter_ret = 0;
6573 struct btrfs_root *root;
6574 struct btrfs_path *path;
6575 struct btrfs_key key;
6576 struct btrfs_key found_key;
6577
6578 root = sctx->send_root;
6579 path = alloc_path_for_send();
6580 if (!path)
6581 return -ENOMEM;
6582
6583 key.objectid = sctx->cmp_key->objectid;
6584 key.type = BTRFS_EXTENT_DATA_KEY;
6585 key.offset = 0;
6586 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
6587 if (found_key.objectid != key.objectid ||
6588 found_key.type != key.type) {
6589 ret = 0;
6590 break;
6591 }
6592
6593 ret = process_extent(sctx, path, &found_key);
6594 if (ret < 0)
6595 break;
6596 }
6597 /* Catch error found during iteration */
6598 if (iter_ret < 0)
6599 ret = iter_ret;
6600
6601 btrfs_free_path(path);
6602 return ret;
6603 }
6604
6605 static int process_recorded_refs_if_needed(struct send_ctx *sctx, int at_end,
6606 int *pending_move,
6607 int *refs_processed)
6608 {
6609 int ret = 0;
6610
6611 if (sctx->cur_ino == 0)
6612 goto out;
6613 if (!at_end && sctx->cur_ino == sctx->cmp_key->objectid &&
6614 sctx->cmp_key->type <= BTRFS_INODE_EXTREF_KEY)
6615 goto out;
6616 if (list_empty(&sctx->new_refs) && list_empty(&sctx->deleted_refs))
6617 goto out;
6618
6619 ret = process_recorded_refs(sctx, pending_move);
6620 if (ret < 0)
6621 goto out;
6622
6623 *refs_processed = 1;
6624 out:
6625 return ret;
6626 }
6627
6628 static int finish_inode_if_needed(struct send_ctx *sctx, int at_end)
6629 {
6630 int ret = 0;
6631 struct btrfs_inode_info info;
6632 u64 left_mode;
6633 u64 left_uid;
6634 u64 left_gid;
6635 u64 left_fileattr;
6636 u64 right_mode;
6637 u64 right_uid;
6638 u64 right_gid;
6639 u64 right_fileattr;
6640 int need_chmod = 0;
6641 int need_chown = 0;
6642 bool need_fileattr = false;
6643 int need_truncate = 1;
6644 int pending_move = 0;
6645 int refs_processed = 0;
6646
6647 if (sctx->ignore_cur_inode)
6648 return 0;
6649
6650 ret = process_recorded_refs_if_needed(sctx, at_end, &pending_move,
6651 &refs_processed);
6652 if (ret < 0)
6653 goto out;
6654
6655 /*
6656 * We have processed the refs and thus need to advance send_progress.
6657 * Now, calls to get_cur_xxx will take the updated refs of the current
6658 * inode into account.
6659 *
6660 * On the other hand, if our current inode is a directory and couldn't
6661 * be moved/renamed because its parent was renamed/moved too and it has
6662 * a higher inode number, we can only move/rename our current inode
6663 * after we moved/renamed its parent. Therefore in this case operate on
6664 * the old path (pre move/rename) of our current inode, and the
6665 * move/rename will be performed later.
6666 */
6667 if (refs_processed && !pending_move)
6668 sctx->send_progress = sctx->cur_ino + 1;
6669
6670 if (sctx->cur_ino == 0 || sctx->cur_inode_deleted)
6671 goto out;
6672 if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino)
6673 goto out;
6674 ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info);
6675 if (ret < 0)
6676 goto out;
6677 left_mode = info.mode;
6678 left_uid = info.uid;
6679 left_gid = info.gid;
6680 left_fileattr = info.fileattr;
6681
6682 if (!sctx->parent_root || sctx->cur_inode_new) {
6683 need_chown = 1;
6684 if (!S_ISLNK(sctx->cur_inode_mode))
6685 need_chmod = 1;
6686 if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size)
6687 need_truncate = 0;
6688 } else {
6689 u64 old_size;
6690
6691 ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info);
6692 if (ret < 0)
6693 goto out;
6694 old_size = info.size;
6695 right_mode = info.mode;
6696 right_uid = info.uid;
6697 right_gid = info.gid;
6698 right_fileattr = info.fileattr;
6699
6700 if (left_uid != right_uid || left_gid != right_gid)
6701 need_chown = 1;
6702 if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode)
6703 need_chmod = 1;
6704 if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr)
6705 need_fileattr = true;
6706 if ((old_size == sctx->cur_inode_size) ||
6707 (sctx->cur_inode_size > old_size &&
6708 sctx->cur_inode_next_write_offset == sctx->cur_inode_size))
6709 need_truncate = 0;
6710 }
6711
6712 if (S_ISREG(sctx->cur_inode_mode)) {
6713 if (need_send_hole(sctx)) {
6714 if (sctx->cur_inode_last_extent == (u64)-1 ||
6715 sctx->cur_inode_last_extent <
6716 sctx->cur_inode_size) {
6717 ret = get_last_extent(sctx, (u64)-1);
6718 if (ret)
6719 goto out;
6720 }
6721 if (sctx->cur_inode_last_extent < sctx->cur_inode_size) {
6722 ret = range_is_hole_in_parent(sctx,
6723 sctx->cur_inode_last_extent,
6724 sctx->cur_inode_size);
6725 if (ret < 0) {
6726 goto out;
6727 } else if (ret == 0) {
6728 ret = send_hole(sctx, sctx->cur_inode_size);
6729 if (ret < 0)
6730 goto out;
6731 } else {
6732 /* Range is already a hole, skip. */
6733 ret = 0;
6734 }
6735 }
6736 }
6737 if (need_truncate) {
6738 ret = send_truncate(sctx, sctx->cur_ino,
6739 sctx->cur_inode_gen,
6740 sctx->cur_inode_size);
6741 if (ret < 0)
6742 goto out;
6743 }
6744 }
6745
6746 if (need_chown) {
6747 ret = send_chown(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6748 left_uid, left_gid);
6749 if (ret < 0)
6750 goto out;
6751 }
6752 if (need_chmod) {
6753 ret = send_chmod(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6754 left_mode);
6755 if (ret < 0)
6756 goto out;
6757 }
6758 if (need_fileattr) {
6759 ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6760 left_fileattr);
6761 if (ret < 0)
6762 goto out;
6763 }
6764
6765 if (proto_cmd_ok(sctx, BTRFS_SEND_C_ENABLE_VERITY)
6766 && sctx->cur_inode_needs_verity) {
6767 ret = process_verity(sctx);
6768 if (ret < 0)
6769 goto out;
6770 }
6771
6772 ret = send_capabilities(sctx);
6773 if (ret < 0)
6774 goto out;
6775
6776 /*
6777 * If other directory inodes depended on our current directory
6778 * inode's move/rename, now do their move/rename operations.
6779 */
6780 if (!is_waiting_for_move(sctx, sctx->cur_ino)) {
6781 ret = apply_children_dir_moves(sctx);
6782 if (ret)
6783 goto out;
6784 /*
6785 * Need to send that every time, no matter if it actually
6786 * changed between the two trees as we have done changes to
6787 * the inode before. If our inode is a directory and it's
6788 * waiting to be moved/renamed, we will send its utimes when
6789 * it's moved/renamed, therefore we don't need to do it here.
6790 */
6791 sctx->send_progress = sctx->cur_ino + 1;
6792
6793 /*
6794 * If the current inode is a non-empty directory, delay issuing
6795 * the utimes command for it, as it's very likely we have inodes
6796 * with an higher number inside it. We want to issue the utimes
6797 * command only after adding all dentries to it.
6798 */
6799 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_size > 0)
6800 ret = cache_dir_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6801 else
6802 ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6803
6804 if (ret < 0)
6805 goto out;
6806 }
6807
6808 out:
6809 if (!ret)
6810 ret = trim_dir_utimes_cache(sctx);
6811
6812 return ret;
6813 }
6814
6815 static void close_current_inode(struct send_ctx *sctx)
6816 {
6817 u64 i_size;
6818
6819 if (sctx->cur_inode == NULL)
6820 return;
6821
6822 i_size = i_size_read(sctx->cur_inode);
6823
6824 /*
6825 * If we are doing an incremental send, we may have extents between the
6826 * last processed extent and the i_size that have not been processed
6827 * because they haven't changed but we may have read some of their pages
6828 * through readahead, see the comments at send_extent_data().
6829 */
6830 if (sctx->clean_page_cache && sctx->page_cache_clear_start < i_size)
6831 truncate_inode_pages_range(&sctx->cur_inode->i_data,
6832 sctx->page_cache_clear_start,
6833 round_up(i_size, PAGE_SIZE) - 1);
6834
6835 iput(sctx->cur_inode);
6836 sctx->cur_inode = NULL;
6837 }
6838
6839 static int changed_inode(struct send_ctx *sctx,
6840 enum btrfs_compare_tree_result result)
6841 {
6842 int ret = 0;
6843 struct btrfs_key *key = sctx->cmp_key;
6844 struct btrfs_inode_item *left_ii = NULL;
6845 struct btrfs_inode_item *right_ii = NULL;
6846 u64 left_gen = 0;
6847 u64 right_gen = 0;
6848
6849 close_current_inode(sctx);
6850
6851 sctx->cur_ino = key->objectid;
6852 sctx->cur_inode_new_gen = false;
6853 sctx->cur_inode_last_extent = (u64)-1;
6854 sctx->cur_inode_next_write_offset = 0;
6855 sctx->ignore_cur_inode = false;
6856
6857 /*
6858 * Set send_progress to current inode. This will tell all get_cur_xxx
6859 * functions that the current inode's refs are not updated yet. Later,
6860 * when process_recorded_refs is finished, it is set to cur_ino + 1.
6861 */
6862 sctx->send_progress = sctx->cur_ino;
6863
6864 if (result == BTRFS_COMPARE_TREE_NEW ||
6865 result == BTRFS_COMPARE_TREE_CHANGED) {
6866 left_ii = btrfs_item_ptr(sctx->left_path->nodes[0],
6867 sctx->left_path->slots[0],
6868 struct btrfs_inode_item);
6869 left_gen = btrfs_inode_generation(sctx->left_path->nodes[0],
6870 left_ii);
6871 } else {
6872 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6873 sctx->right_path->slots[0],
6874 struct btrfs_inode_item);
6875 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6876 right_ii);
6877 }
6878 if (result == BTRFS_COMPARE_TREE_CHANGED) {
6879 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6880 sctx->right_path->slots[0],
6881 struct btrfs_inode_item);
6882
6883 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6884 right_ii);
6885
6886 /*
6887 * The cur_ino = root dir case is special here. We can't treat
6888 * the inode as deleted+reused because it would generate a
6889 * stream that tries to delete/mkdir the root dir.
6890 */
6891 if (left_gen != right_gen &&
6892 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6893 sctx->cur_inode_new_gen = true;
6894 }
6895
6896 /*
6897 * Normally we do not find inodes with a link count of zero (orphans)
6898 * because the most common case is to create a snapshot and use it
6899 * for a send operation. However other less common use cases involve
6900 * using a subvolume and send it after turning it to RO mode just
6901 * after deleting all hard links of a file while holding an open
6902 * file descriptor against it or turning a RO snapshot into RW mode,
6903 * keep an open file descriptor against a file, delete it and then
6904 * turn the snapshot back to RO mode before using it for a send
6905 * operation. The former is what the receiver operation does.
6906 * Therefore, if we want to send these snapshots soon after they're
6907 * received, we need to handle orphan inodes as well. Moreover, orphans
6908 * can appear not only in the send snapshot but also in the parent
6909 * snapshot. Here are several cases:
6910 *
6911 * Case 1: BTRFS_COMPARE_TREE_NEW
6912 * | send snapshot | action
6913 * --------------------------------
6914 * nlink | 0 | ignore
6915 *
6916 * Case 2: BTRFS_COMPARE_TREE_DELETED
6917 * | parent snapshot | action
6918 * ----------------------------------
6919 * nlink | 0 | as usual
6920 * Note: No unlinks will be sent because there're no paths for it.
6921 *
6922 * Case 3: BTRFS_COMPARE_TREE_CHANGED
6923 * | | parent snapshot | send snapshot | action
6924 * -----------------------------------------------------------------------
6925 * subcase 1 | nlink | 0 | 0 | ignore
6926 * subcase 2 | nlink | >0 | 0 | new_gen(deletion)
6927 * subcase 3 | nlink | 0 | >0 | new_gen(creation)
6928 *
6929 */
6930 if (result == BTRFS_COMPARE_TREE_NEW) {
6931 if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) {
6932 sctx->ignore_cur_inode = true;
6933 goto out;
6934 }
6935 sctx->cur_inode_gen = left_gen;
6936 sctx->cur_inode_new = true;
6937 sctx->cur_inode_deleted = false;
6938 sctx->cur_inode_size = btrfs_inode_size(
6939 sctx->left_path->nodes[0], left_ii);
6940 sctx->cur_inode_mode = btrfs_inode_mode(
6941 sctx->left_path->nodes[0], left_ii);
6942 sctx->cur_inode_rdev = btrfs_inode_rdev(
6943 sctx->left_path->nodes[0], left_ii);
6944 if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6945 ret = send_create_inode_if_needed(sctx);
6946 } else if (result == BTRFS_COMPARE_TREE_DELETED) {
6947 sctx->cur_inode_gen = right_gen;
6948 sctx->cur_inode_new = false;
6949 sctx->cur_inode_deleted = true;
6950 sctx->cur_inode_size = btrfs_inode_size(
6951 sctx->right_path->nodes[0], right_ii);
6952 sctx->cur_inode_mode = btrfs_inode_mode(
6953 sctx->right_path->nodes[0], right_ii);
6954 } else if (result == BTRFS_COMPARE_TREE_CHANGED) {
6955 u32 new_nlinks, old_nlinks;
6956
6957 new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii);
6958 old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii);
6959 if (new_nlinks == 0 && old_nlinks == 0) {
6960 sctx->ignore_cur_inode = true;
6961 goto out;
6962 } else if (new_nlinks == 0 || old_nlinks == 0) {
6963 sctx->cur_inode_new_gen = 1;
6964 }
6965 /*
6966 * We need to do some special handling in case the inode was
6967 * reported as changed with a changed generation number. This
6968 * means that the original inode was deleted and new inode
6969 * reused the same inum. So we have to treat the old inode as
6970 * deleted and the new one as new.
6971 */
6972 if (sctx->cur_inode_new_gen) {
6973 /*
6974 * First, process the inode as if it was deleted.
6975 */
6976 if (old_nlinks > 0) {
6977 sctx->cur_inode_gen = right_gen;
6978 sctx->cur_inode_new = false;
6979 sctx->cur_inode_deleted = true;
6980 sctx->cur_inode_size = btrfs_inode_size(
6981 sctx->right_path->nodes[0], right_ii);
6982 sctx->cur_inode_mode = btrfs_inode_mode(
6983 sctx->right_path->nodes[0], right_ii);
6984 ret = process_all_refs(sctx,
6985 BTRFS_COMPARE_TREE_DELETED);
6986 if (ret < 0)
6987 goto out;
6988 }
6989
6990 /*
6991 * Now process the inode as if it was new.
6992 */
6993 if (new_nlinks > 0) {
6994 sctx->cur_inode_gen = left_gen;
6995 sctx->cur_inode_new = true;
6996 sctx->cur_inode_deleted = false;
6997 sctx->cur_inode_size = btrfs_inode_size(
6998 sctx->left_path->nodes[0],
6999 left_ii);
7000 sctx->cur_inode_mode = btrfs_inode_mode(
7001 sctx->left_path->nodes[0],
7002 left_ii);
7003 sctx->cur_inode_rdev = btrfs_inode_rdev(
7004 sctx->left_path->nodes[0],
7005 left_ii);
7006 ret = send_create_inode_if_needed(sctx);
7007 if (ret < 0)
7008 goto out;
7009
7010 ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW);
7011 if (ret < 0)
7012 goto out;
7013 /*
7014 * Advance send_progress now as we did not get
7015 * into process_recorded_refs_if_needed in the
7016 * new_gen case.
7017 */
7018 sctx->send_progress = sctx->cur_ino + 1;
7019
7020 /*
7021 * Now process all extents and xattrs of the
7022 * inode as if they were all new.
7023 */
7024 ret = process_all_extents(sctx);
7025 if (ret < 0)
7026 goto out;
7027 ret = process_all_new_xattrs(sctx);
7028 if (ret < 0)
7029 goto out;
7030 }
7031 } else {
7032 sctx->cur_inode_gen = left_gen;
7033 sctx->cur_inode_new = false;
7034 sctx->cur_inode_new_gen = false;
7035 sctx->cur_inode_deleted = false;
7036 sctx->cur_inode_size = btrfs_inode_size(
7037 sctx->left_path->nodes[0], left_ii);
7038 sctx->cur_inode_mode = btrfs_inode_mode(
7039 sctx->left_path->nodes[0], left_ii);
7040 }
7041 }
7042
7043 out:
7044 return ret;
7045 }
7046
7047 /*
7048 * We have to process new refs before deleted refs, but compare_trees gives us
7049 * the new and deleted refs mixed. To fix this, we record the new/deleted refs
7050 * first and later process them in process_recorded_refs.
7051 * For the cur_inode_new_gen case, we skip recording completely because
7052 * changed_inode did already initiate processing of refs. The reason for this is
7053 * that in this case, compare_tree actually compares the refs of 2 different
7054 * inodes. To fix this, process_all_refs is used in changed_inode to handle all
7055 * refs of the right tree as deleted and all refs of the left tree as new.
7056 */
7057 static int changed_ref(struct send_ctx *sctx,
7058 enum btrfs_compare_tree_result result)
7059 {
7060 int ret = 0;
7061
7062 if (sctx->cur_ino != sctx->cmp_key->objectid) {
7063 inconsistent_snapshot_error(sctx, result, "reference");
7064 return -EIO;
7065 }
7066
7067 if (!sctx->cur_inode_new_gen &&
7068 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) {
7069 if (result == BTRFS_COMPARE_TREE_NEW)
7070 ret = record_new_ref(sctx);
7071 else if (result == BTRFS_COMPARE_TREE_DELETED)
7072 ret = record_deleted_ref(sctx);
7073 else if (result == BTRFS_COMPARE_TREE_CHANGED)
7074 ret = record_changed_ref(sctx);
7075 }
7076
7077 return ret;
7078 }
7079
7080 /*
7081 * Process new/deleted/changed xattrs. We skip processing in the
7082 * cur_inode_new_gen case because changed_inode did already initiate processing
7083 * of xattrs. The reason is the same as in changed_ref
7084 */
7085 static int changed_xattr(struct send_ctx *sctx,
7086 enum btrfs_compare_tree_result result)
7087 {
7088 int ret = 0;
7089
7090 if (sctx->cur_ino != sctx->cmp_key->objectid) {
7091 inconsistent_snapshot_error(sctx, result, "xattr");
7092 return -EIO;
7093 }
7094
7095 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7096 if (result == BTRFS_COMPARE_TREE_NEW)
7097 ret = process_new_xattr(sctx);
7098 else if (result == BTRFS_COMPARE_TREE_DELETED)
7099 ret = process_deleted_xattr(sctx);
7100 else if (result == BTRFS_COMPARE_TREE_CHANGED)
7101 ret = process_changed_xattr(sctx);
7102 }
7103
7104 return ret;
7105 }
7106
7107 /*
7108 * Process new/deleted/changed extents. We skip processing in the
7109 * cur_inode_new_gen case because changed_inode did already initiate processing
7110 * of extents. The reason is the same as in changed_ref
7111 */
7112 static int changed_extent(struct send_ctx *sctx,
7113 enum btrfs_compare_tree_result result)
7114 {
7115 int ret = 0;
7116
7117 /*
7118 * We have found an extent item that changed without the inode item
7119 * having changed. This can happen either after relocation (where the
7120 * disk_bytenr of an extent item is replaced at
7121 * relocation.c:replace_file_extents()) or after deduplication into a
7122 * file in both the parent and send snapshots (where an extent item can
7123 * get modified or replaced with a new one). Note that deduplication
7124 * updates the inode item, but it only changes the iversion (sequence
7125 * field in the inode item) of the inode, so if a file is deduplicated
7126 * the same amount of times in both the parent and send snapshots, its
7127 * iversion becomes the same in both snapshots, whence the inode item is
7128 * the same on both snapshots.
7129 */
7130 if (sctx->cur_ino != sctx->cmp_key->objectid)
7131 return 0;
7132
7133 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7134 if (result != BTRFS_COMPARE_TREE_DELETED)
7135 ret = process_extent(sctx, sctx->left_path,
7136 sctx->cmp_key);
7137 }
7138
7139 return ret;
7140 }
7141
7142 static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result)
7143 {
7144 int ret = 0;
7145
7146 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7147 if (result == BTRFS_COMPARE_TREE_NEW)
7148 sctx->cur_inode_needs_verity = true;
7149 }
7150 return ret;
7151 }
7152
7153 static int dir_changed(struct send_ctx *sctx, u64 dir)
7154 {
7155 u64 orig_gen, new_gen;
7156 int ret;
7157
7158 ret = get_inode_gen(sctx->send_root, dir, &new_gen);
7159 if (ret)
7160 return ret;
7161
7162 ret = get_inode_gen(sctx->parent_root, dir, &orig_gen);
7163 if (ret)
7164 return ret;
7165
7166 return (orig_gen != new_gen) ? 1 : 0;
7167 }
7168
7169 static int compare_refs(struct send_ctx *sctx, struct btrfs_path *path,
7170 struct btrfs_key *key)
7171 {
7172 struct btrfs_inode_extref *extref;
7173 struct extent_buffer *leaf;
7174 u64 dirid = 0, last_dirid = 0;
7175 unsigned long ptr;
7176 u32 item_size;
7177 u32 cur_offset = 0;
7178 int ref_name_len;
7179 int ret = 0;
7180
7181 /* Easy case, just check this one dirid */
7182 if (key->type == BTRFS_INODE_REF_KEY) {
7183 dirid = key->offset;
7184
7185 ret = dir_changed(sctx, dirid);
7186 goto out;
7187 }
7188
7189 leaf = path->nodes[0];
7190 item_size = btrfs_item_size(leaf, path->slots[0]);
7191 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
7192 while (cur_offset < item_size) {
7193 extref = (struct btrfs_inode_extref *)(ptr +
7194 cur_offset);
7195 dirid = btrfs_inode_extref_parent(leaf, extref);
7196 ref_name_len = btrfs_inode_extref_name_len(leaf, extref);
7197 cur_offset += ref_name_len + sizeof(*extref);
7198 if (dirid == last_dirid)
7199 continue;
7200 ret = dir_changed(sctx, dirid);
7201 if (ret)
7202 break;
7203 last_dirid = dirid;
7204 }
7205 out:
7206 return ret;
7207 }
7208
7209 /*
7210 * Updates compare related fields in sctx and simply forwards to the actual
7211 * changed_xxx functions.
7212 */
7213 static int changed_cb(struct btrfs_path *left_path,
7214 struct btrfs_path *right_path,
7215 struct btrfs_key *key,
7216 enum btrfs_compare_tree_result result,
7217 struct send_ctx *sctx)
7218 {
7219 int ret = 0;
7220
7221 /*
7222 * We can not hold the commit root semaphore here. This is because in
7223 * the case of sending and receiving to the same filesystem, using a
7224 * pipe, could result in a deadlock:
7225 *
7226 * 1) The task running send blocks on the pipe because it's full;
7227 *
7228 * 2) The task running receive, which is the only consumer of the pipe,
7229 * is waiting for a transaction commit (for example due to a space
7230 * reservation when doing a write or triggering a transaction commit
7231 * when creating a subvolume);
7232 *
7233 * 3) The transaction is waiting to write lock the commit root semaphore,
7234 * but can not acquire it since it's being held at 1).
7235 *
7236 * Down this call chain we write to the pipe through kernel_write().
7237 * The same type of problem can also happen when sending to a file that
7238 * is stored in the same filesystem - when reserving space for a write
7239 * into the file, we can trigger a transaction commit.
7240 *
7241 * Our caller has supplied us with clones of leaves from the send and
7242 * parent roots, so we're safe here from a concurrent relocation and
7243 * further reallocation of metadata extents while we are here. Below we
7244 * also assert that the leaves are clones.
7245 */
7246 lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem);
7247
7248 /*
7249 * We always have a send root, so left_path is never NULL. We will not
7250 * have a leaf when we have reached the end of the send root but have
7251 * not yet reached the end of the parent root.
7252 */
7253 if (left_path->nodes[0])
7254 ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7255 &left_path->nodes[0]->bflags));
7256 /*
7257 * When doing a full send we don't have a parent root, so right_path is
7258 * NULL. When doing an incremental send, we may have reached the end of
7259 * the parent root already, so we don't have a leaf at right_path.
7260 */
7261 if (right_path && right_path->nodes[0])
7262 ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7263 &right_path->nodes[0]->bflags));
7264
7265 if (result == BTRFS_COMPARE_TREE_SAME) {
7266 if (key->type == BTRFS_INODE_REF_KEY ||
7267 key->type == BTRFS_INODE_EXTREF_KEY) {
7268 ret = compare_refs(sctx, left_path, key);
7269 if (!ret)
7270 return 0;
7271 if (ret < 0)
7272 return ret;
7273 } else if (key->type == BTRFS_EXTENT_DATA_KEY) {
7274 return maybe_send_hole(sctx, left_path, key);
7275 } else {
7276 return 0;
7277 }
7278 result = BTRFS_COMPARE_TREE_CHANGED;
7279 ret = 0;
7280 }
7281
7282 sctx->left_path = left_path;
7283 sctx->right_path = right_path;
7284 sctx->cmp_key = key;
7285
7286 ret = finish_inode_if_needed(sctx, 0);
7287 if (ret < 0)
7288 goto out;
7289
7290 /* Ignore non-FS objects */
7291 if (key->objectid == BTRFS_FREE_INO_OBJECTID ||
7292 key->objectid == BTRFS_FREE_SPACE_OBJECTID)
7293 goto out;
7294
7295 if (key->type == BTRFS_INODE_ITEM_KEY) {
7296 ret = changed_inode(sctx, result);
7297 } else if (!sctx->ignore_cur_inode) {
7298 if (key->type == BTRFS_INODE_REF_KEY ||
7299 key->type == BTRFS_INODE_EXTREF_KEY)
7300 ret = changed_ref(sctx, result);
7301 else if (key->type == BTRFS_XATTR_ITEM_KEY)
7302 ret = changed_xattr(sctx, result);
7303 else if (key->type == BTRFS_EXTENT_DATA_KEY)
7304 ret = changed_extent(sctx, result);
7305 else if (key->type == BTRFS_VERITY_DESC_ITEM_KEY &&
7306 key->offset == 0)
7307 ret = changed_verity(sctx, result);
7308 }
7309
7310 out:
7311 return ret;
7312 }
7313
7314 static int search_key_again(const struct send_ctx *sctx,
7315 struct btrfs_root *root,
7316 struct btrfs_path *path,
7317 const struct btrfs_key *key)
7318 {
7319 int ret;
7320
7321 if (!path->need_commit_sem)
7322 lockdep_assert_held_read(&root->fs_info->commit_root_sem);
7323
7324 /*
7325 * Roots used for send operations are readonly and no one can add,
7326 * update or remove keys from them, so we should be able to find our
7327 * key again. The only exception is deduplication, which can operate on
7328 * readonly roots and add, update or remove keys to/from them - but at
7329 * the moment we don't allow it to run in parallel with send.
7330 */
7331 ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
7332 ASSERT(ret <= 0);
7333 if (ret > 0) {
7334 btrfs_print_tree(path->nodes[path->lowest_level], false);
7335 btrfs_err(root->fs_info,
7336 "send: key (%llu %u %llu) not found in %s root %llu, lowest_level %d, slot %d",
7337 key->objectid, key->type, key->offset,
7338 (root == sctx->parent_root ? "parent" : "send"),
7339 btrfs_root_id(root), path->lowest_level,
7340 path->slots[path->lowest_level]);
7341 return -EUCLEAN;
7342 }
7343
7344 return ret;
7345 }
7346
7347 static int full_send_tree(struct send_ctx *sctx)
7348 {
7349 int ret;
7350 struct btrfs_root *send_root = sctx->send_root;
7351 struct btrfs_key key;
7352 struct btrfs_fs_info *fs_info = send_root->fs_info;
7353 struct btrfs_path *path;
7354
7355 path = alloc_path_for_send();
7356 if (!path)
7357 return -ENOMEM;
7358 path->reada = READA_FORWARD_ALWAYS;
7359
7360 key.objectid = BTRFS_FIRST_FREE_OBJECTID;
7361 key.type = BTRFS_INODE_ITEM_KEY;
7362 key.offset = 0;
7363
7364 down_read(&fs_info->commit_root_sem);
7365 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7366 up_read(&fs_info->commit_root_sem);
7367
7368 ret = btrfs_search_slot_for_read(send_root, &key, path, 1, 0);
7369 if (ret < 0)
7370 goto out;
7371 if (ret)
7372 goto out_finish;
7373
7374 while (1) {
7375 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
7376
7377 ret = changed_cb(path, NULL, &key,
7378 BTRFS_COMPARE_TREE_NEW, sctx);
7379 if (ret < 0)
7380 goto out;
7381
7382 down_read(&fs_info->commit_root_sem);
7383 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7384 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7385 up_read(&fs_info->commit_root_sem);
7386 /*
7387 * A transaction used for relocating a block group was
7388 * committed or is about to finish its commit. Release
7389 * our path (leaf) and restart the search, so that we
7390 * avoid operating on any file extent items that are
7391 * stale, with a disk_bytenr that reflects a pre
7392 * relocation value. This way we avoid as much as
7393 * possible to fallback to regular writes when checking
7394 * if we can clone file ranges.
7395 */
7396 btrfs_release_path(path);
7397 ret = search_key_again(sctx, send_root, path, &key);
7398 if (ret < 0)
7399 goto out;
7400 } else {
7401 up_read(&fs_info->commit_root_sem);
7402 }
7403
7404 ret = btrfs_next_item(send_root, path);
7405 if (ret < 0)
7406 goto out;
7407 if (ret) {
7408 ret = 0;
7409 break;
7410 }
7411 }
7412
7413 out_finish:
7414 ret = finish_inode_if_needed(sctx, 1);
7415
7416 out:
7417 btrfs_free_path(path);
7418 return ret;
7419 }
7420
7421 static int replace_node_with_clone(struct btrfs_path *path, int level)
7422 {
7423 struct extent_buffer *clone;
7424
7425 clone = btrfs_clone_extent_buffer(path->nodes[level]);
7426 if (!clone)
7427 return -ENOMEM;
7428
7429 free_extent_buffer(path->nodes[level]);
7430 path->nodes[level] = clone;
7431
7432 return 0;
7433 }
7434
7435 static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen)
7436 {
7437 struct extent_buffer *eb;
7438 struct extent_buffer *parent = path->nodes[*level];
7439 int slot = path->slots[*level];
7440 const int nritems = btrfs_header_nritems(parent);
7441 u64 reada_max;
7442 u64 reada_done = 0;
7443
7444 lockdep_assert_held_read(&parent->fs_info->commit_root_sem);
7445 ASSERT(*level != 0);
7446
7447 eb = btrfs_read_node_slot(parent, slot);
7448 if (IS_ERR(eb))
7449 return PTR_ERR(eb);
7450
7451 /*
7452 * Trigger readahead for the next leaves we will process, so that it is
7453 * very likely that when we need them they are already in memory and we
7454 * will not block on disk IO. For nodes we only do readahead for one,
7455 * since the time window between processing nodes is typically larger.
7456 */
7457 reada_max = (*level == 1 ? SZ_128K : eb->fs_info->nodesize);
7458
7459 for (slot++; slot < nritems && reada_done < reada_max; slot++) {
7460 if (btrfs_node_ptr_generation(parent, slot) > reada_min_gen) {
7461 btrfs_readahead_node_child(parent, slot);
7462 reada_done += eb->fs_info->nodesize;
7463 }
7464 }
7465
7466 path->nodes[*level - 1] = eb;
7467 path->slots[*level - 1] = 0;
7468 (*level)--;
7469
7470 if (*level == 0)
7471 return replace_node_with_clone(path, 0);
7472
7473 return 0;
7474 }
7475
7476 static int tree_move_next_or_upnext(struct btrfs_path *path,
7477 int *level, int root_level)
7478 {
7479 int ret = 0;
7480 int nritems;
7481 nritems = btrfs_header_nritems(path->nodes[*level]);
7482
7483 path->slots[*level]++;
7484
7485 while (path->slots[*level] >= nritems) {
7486 if (*level == root_level) {
7487 path->slots[*level] = nritems - 1;
7488 return -1;
7489 }
7490
7491 /* move upnext */
7492 path->slots[*level] = 0;
7493 free_extent_buffer(path->nodes[*level]);
7494 path->nodes[*level] = NULL;
7495 (*level)++;
7496 path->slots[*level]++;
7497
7498 nritems = btrfs_header_nritems(path->nodes[*level]);
7499 ret = 1;
7500 }
7501 return ret;
7502 }
7503
7504 /*
7505 * Returns 1 if it had to move up and next. 0 is returned if it moved only next
7506 * or down.
7507 */
7508 static int tree_advance(struct btrfs_path *path,
7509 int *level, int root_level,
7510 int allow_down,
7511 struct btrfs_key *key,
7512 u64 reada_min_gen)
7513 {
7514 int ret;
7515
7516 if (*level == 0 || !allow_down) {
7517 ret = tree_move_next_or_upnext(path, level, root_level);
7518 } else {
7519 ret = tree_move_down(path, level, reada_min_gen);
7520 }
7521
7522 /*
7523 * Even if we have reached the end of a tree, ret is -1, update the key
7524 * anyway, so that in case we need to restart due to a block group
7525 * relocation, we can assert that the last key of the root node still
7526 * exists in the tree.
7527 */
7528 if (*level == 0)
7529 btrfs_item_key_to_cpu(path->nodes[*level], key,
7530 path->slots[*level]);
7531 else
7532 btrfs_node_key_to_cpu(path->nodes[*level], key,
7533 path->slots[*level]);
7534
7535 return ret;
7536 }
7537
7538 static int tree_compare_item(struct btrfs_path *left_path,
7539 struct btrfs_path *right_path,
7540 char *tmp_buf)
7541 {
7542 int cmp;
7543 int len1, len2;
7544 unsigned long off1, off2;
7545
7546 len1 = btrfs_item_size(left_path->nodes[0], left_path->slots[0]);
7547 len2 = btrfs_item_size(right_path->nodes[0], right_path->slots[0]);
7548 if (len1 != len2)
7549 return 1;
7550
7551 off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
7552 off2 = btrfs_item_ptr_offset(right_path->nodes[0],
7553 right_path->slots[0]);
7554
7555 read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
7556
7557 cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
7558 if (cmp)
7559 return 1;
7560 return 0;
7561 }
7562
7563 /*
7564 * A transaction used for relocating a block group was committed or is about to
7565 * finish its commit. Release our paths and restart the search, so that we are
7566 * not using stale extent buffers:
7567 *
7568 * 1) For levels > 0, we are only holding references of extent buffers, without
7569 * any locks on them, which does not prevent them from having been relocated
7570 * and reallocated after the last time we released the commit root semaphore.
7571 * The exception are the root nodes, for which we always have a clone, see
7572 * the comment at btrfs_compare_trees();
7573 *
7574 * 2) For leaves, level 0, we are holding copies (clones) of extent buffers, so
7575 * we are safe from the concurrent relocation and reallocation. However they
7576 * can have file extent items with a pre relocation disk_bytenr value, so we
7577 * restart the start from the current commit roots and clone the new leaves so
7578 * that we get the post relocation disk_bytenr values. Not doing so, could
7579 * make us clone the wrong data in case there are new extents using the old
7580 * disk_bytenr that happen to be shared.
7581 */
7582 static int restart_after_relocation(struct btrfs_path *left_path,
7583 struct btrfs_path *right_path,
7584 const struct btrfs_key *left_key,
7585 const struct btrfs_key *right_key,
7586 int left_level,
7587 int right_level,
7588 const struct send_ctx *sctx)
7589 {
7590 int root_level;
7591 int ret;
7592
7593 lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem);
7594
7595 btrfs_release_path(left_path);
7596 btrfs_release_path(right_path);
7597
7598 /*
7599 * Since keys can not be added or removed to/from our roots because they
7600 * are readonly and we do not allow deduplication to run in parallel
7601 * (which can add, remove or change keys), the layout of the trees should
7602 * not change.
7603 */
7604 left_path->lowest_level = left_level;
7605 ret = search_key_again(sctx, sctx->send_root, left_path, left_key);
7606 if (ret < 0)
7607 return ret;
7608
7609 right_path->lowest_level = right_level;
7610 ret = search_key_again(sctx, sctx->parent_root, right_path, right_key);
7611 if (ret < 0)
7612 return ret;
7613
7614 /*
7615 * If the lowest level nodes are leaves, clone them so that they can be
7616 * safely used by changed_cb() while not under the protection of the
7617 * commit root semaphore, even if relocation and reallocation happens in
7618 * parallel.
7619 */
7620 if (left_level == 0) {
7621 ret = replace_node_with_clone(left_path, 0);
7622 if (ret < 0)
7623 return ret;
7624 }
7625
7626 if (right_level == 0) {
7627 ret = replace_node_with_clone(right_path, 0);
7628 if (ret < 0)
7629 return ret;
7630 }
7631
7632 /*
7633 * Now clone the root nodes (unless they happen to be the leaves we have
7634 * already cloned). This is to protect against concurrent snapshotting of
7635 * the send and parent roots (see the comment at btrfs_compare_trees()).
7636 */
7637 root_level = btrfs_header_level(sctx->send_root->commit_root);
7638 if (root_level > 0) {
7639 ret = replace_node_with_clone(left_path, root_level);
7640 if (ret < 0)
7641 return ret;
7642 }
7643
7644 root_level = btrfs_header_level(sctx->parent_root->commit_root);
7645 if (root_level > 0) {
7646 ret = replace_node_with_clone(right_path, root_level);
7647 if (ret < 0)
7648 return ret;
7649 }
7650
7651 return 0;
7652 }
7653
7654 /*
7655 * This function compares two trees and calls the provided callback for
7656 * every changed/new/deleted item it finds.
7657 * If shared tree blocks are encountered, whole subtrees are skipped, making
7658 * the compare pretty fast on snapshotted subvolumes.
7659 *
7660 * This currently works on commit roots only. As commit roots are read only,
7661 * we don't do any locking. The commit roots are protected with transactions.
7662 * Transactions are ended and rejoined when a commit is tried in between.
7663 *
7664 * This function checks for modifications done to the trees while comparing.
7665 * If it detects a change, it aborts immediately.
7666 */
7667 static int btrfs_compare_trees(struct btrfs_root *left_root,
7668 struct btrfs_root *right_root, struct send_ctx *sctx)
7669 {
7670 struct btrfs_fs_info *fs_info = left_root->fs_info;
7671 int ret;
7672 int cmp;
7673 struct btrfs_path *left_path = NULL;
7674 struct btrfs_path *right_path = NULL;
7675 struct btrfs_key left_key;
7676 struct btrfs_key right_key;
7677 char *tmp_buf = NULL;
7678 int left_root_level;
7679 int right_root_level;
7680 int left_level;
7681 int right_level;
7682 int left_end_reached = 0;
7683 int right_end_reached = 0;
7684 int advance_left = 0;
7685 int advance_right = 0;
7686 u64 left_blockptr;
7687 u64 right_blockptr;
7688 u64 left_gen;
7689 u64 right_gen;
7690 u64 reada_min_gen;
7691
7692 left_path = btrfs_alloc_path();
7693 if (!left_path) {
7694 ret = -ENOMEM;
7695 goto out;
7696 }
7697 right_path = btrfs_alloc_path();
7698 if (!right_path) {
7699 ret = -ENOMEM;
7700 goto out;
7701 }
7702
7703 tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL);
7704 if (!tmp_buf) {
7705 ret = -ENOMEM;
7706 goto out;
7707 }
7708
7709 left_path->search_commit_root = 1;
7710 left_path->skip_locking = 1;
7711 right_path->search_commit_root = 1;
7712 right_path->skip_locking = 1;
7713
7714 /*
7715 * Strategy: Go to the first items of both trees. Then do
7716 *
7717 * If both trees are at level 0
7718 * Compare keys of current items
7719 * If left < right treat left item as new, advance left tree
7720 * and repeat
7721 * If left > right treat right item as deleted, advance right tree
7722 * and repeat
7723 * If left == right do deep compare of items, treat as changed if
7724 * needed, advance both trees and repeat
7725 * If both trees are at the same level but not at level 0
7726 * Compare keys of current nodes/leafs
7727 * If left < right advance left tree and repeat
7728 * If left > right advance right tree and repeat
7729 * If left == right compare blockptrs of the next nodes/leafs
7730 * If they match advance both trees but stay at the same level
7731 * and repeat
7732 * If they don't match advance both trees while allowing to go
7733 * deeper and repeat
7734 * If tree levels are different
7735 * Advance the tree that needs it and repeat
7736 *
7737 * Advancing a tree means:
7738 * If we are at level 0, try to go to the next slot. If that's not
7739 * possible, go one level up and repeat. Stop when we found a level
7740 * where we could go to the next slot. We may at this point be on a
7741 * node or a leaf.
7742 *
7743 * If we are not at level 0 and not on shared tree blocks, go one
7744 * level deeper.
7745 *
7746 * If we are not at level 0 and on shared tree blocks, go one slot to
7747 * the right if possible or go up and right.
7748 */
7749
7750 down_read(&fs_info->commit_root_sem);
7751 left_level = btrfs_header_level(left_root->commit_root);
7752 left_root_level = left_level;
7753 /*
7754 * We clone the root node of the send and parent roots to prevent races
7755 * with snapshot creation of these roots. Snapshot creation COWs the
7756 * root node of a tree, so after the transaction is committed the old
7757 * extent can be reallocated while this send operation is still ongoing.
7758 * So we clone them, under the commit root semaphore, to be race free.
7759 */
7760 left_path->nodes[left_level] =
7761 btrfs_clone_extent_buffer(left_root->commit_root);
7762 if (!left_path->nodes[left_level]) {
7763 ret = -ENOMEM;
7764 goto out_unlock;
7765 }
7766
7767 right_level = btrfs_header_level(right_root->commit_root);
7768 right_root_level = right_level;
7769 right_path->nodes[right_level] =
7770 btrfs_clone_extent_buffer(right_root->commit_root);
7771 if (!right_path->nodes[right_level]) {
7772 ret = -ENOMEM;
7773 goto out_unlock;
7774 }
7775 /*
7776 * Our right root is the parent root, while the left root is the "send"
7777 * root. We know that all new nodes/leaves in the left root must have
7778 * a generation greater than the right root's generation, so we trigger
7779 * readahead for those nodes and leaves of the left root, as we know we
7780 * will need to read them at some point.
7781 */
7782 reada_min_gen = btrfs_header_generation(right_root->commit_root);
7783
7784 if (left_level == 0)
7785 btrfs_item_key_to_cpu(left_path->nodes[left_level],
7786 &left_key, left_path->slots[left_level]);
7787 else
7788 btrfs_node_key_to_cpu(left_path->nodes[left_level],
7789 &left_key, left_path->slots[left_level]);
7790 if (right_level == 0)
7791 btrfs_item_key_to_cpu(right_path->nodes[right_level],
7792 &right_key, right_path->slots[right_level]);
7793 else
7794 btrfs_node_key_to_cpu(right_path->nodes[right_level],
7795 &right_key, right_path->slots[right_level]);
7796
7797 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7798
7799 while (1) {
7800 if (need_resched() ||
7801 rwsem_is_contended(&fs_info->commit_root_sem)) {
7802 up_read(&fs_info->commit_root_sem);
7803 cond_resched();
7804 down_read(&fs_info->commit_root_sem);
7805 }
7806
7807 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7808 ret = restart_after_relocation(left_path, right_path,
7809 &left_key, &right_key,
7810 left_level, right_level,
7811 sctx);
7812 if (ret < 0)
7813 goto out_unlock;
7814 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7815 }
7816
7817 if (advance_left && !left_end_reached) {
7818 ret = tree_advance(left_path, &left_level,
7819 left_root_level,
7820 advance_left != ADVANCE_ONLY_NEXT,
7821 &left_key, reada_min_gen);
7822 if (ret == -1)
7823 left_end_reached = ADVANCE;
7824 else if (ret < 0)
7825 goto out_unlock;
7826 advance_left = 0;
7827 }
7828 if (advance_right && !right_end_reached) {
7829 ret = tree_advance(right_path, &right_level,
7830 right_root_level,
7831 advance_right != ADVANCE_ONLY_NEXT,
7832 &right_key, reada_min_gen);
7833 if (ret == -1)
7834 right_end_reached = ADVANCE;
7835 else if (ret < 0)
7836 goto out_unlock;
7837 advance_right = 0;
7838 }
7839
7840 if (left_end_reached && right_end_reached) {
7841 ret = 0;
7842 goto out_unlock;
7843 } else if (left_end_reached) {
7844 if (right_level == 0) {
7845 up_read(&fs_info->commit_root_sem);
7846 ret = changed_cb(left_path, right_path,
7847 &right_key,
7848 BTRFS_COMPARE_TREE_DELETED,
7849 sctx);
7850 if (ret < 0)
7851 goto out;
7852 down_read(&fs_info->commit_root_sem);
7853 }
7854 advance_right = ADVANCE;
7855 continue;
7856 } else if (right_end_reached) {
7857 if (left_level == 0) {
7858 up_read(&fs_info->commit_root_sem);
7859 ret = changed_cb(left_path, right_path,
7860 &left_key,
7861 BTRFS_COMPARE_TREE_NEW,
7862 sctx);
7863 if (ret < 0)
7864 goto out;
7865 down_read(&fs_info->commit_root_sem);
7866 }
7867 advance_left = ADVANCE;
7868 continue;
7869 }
7870
7871 if (left_level == 0 && right_level == 0) {
7872 up_read(&fs_info->commit_root_sem);
7873 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7874 if (cmp < 0) {
7875 ret = changed_cb(left_path, right_path,
7876 &left_key,
7877 BTRFS_COMPARE_TREE_NEW,
7878 sctx);
7879 advance_left = ADVANCE;
7880 } else if (cmp > 0) {
7881 ret = changed_cb(left_path, right_path,
7882 &right_key,
7883 BTRFS_COMPARE_TREE_DELETED,
7884 sctx);
7885 advance_right = ADVANCE;
7886 } else {
7887 enum btrfs_compare_tree_result result;
7888
7889 WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
7890 ret = tree_compare_item(left_path, right_path,
7891 tmp_buf);
7892 if (ret)
7893 result = BTRFS_COMPARE_TREE_CHANGED;
7894 else
7895 result = BTRFS_COMPARE_TREE_SAME;
7896 ret = changed_cb(left_path, right_path,
7897 &left_key, result, sctx);
7898 advance_left = ADVANCE;
7899 advance_right = ADVANCE;
7900 }
7901
7902 if (ret < 0)
7903 goto out;
7904 down_read(&fs_info->commit_root_sem);
7905 } else if (left_level == right_level) {
7906 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7907 if (cmp < 0) {
7908 advance_left = ADVANCE;
7909 } else if (cmp > 0) {
7910 advance_right = ADVANCE;
7911 } else {
7912 left_blockptr = btrfs_node_blockptr(
7913 left_path->nodes[left_level],
7914 left_path->slots[left_level]);
7915 right_blockptr = btrfs_node_blockptr(
7916 right_path->nodes[right_level],
7917 right_path->slots[right_level]);
7918 left_gen = btrfs_node_ptr_generation(
7919 left_path->nodes[left_level],
7920 left_path->slots[left_level]);
7921 right_gen = btrfs_node_ptr_generation(
7922 right_path->nodes[right_level],
7923 right_path->slots[right_level]);
7924 if (left_blockptr == right_blockptr &&
7925 left_gen == right_gen) {
7926 /*
7927 * As we're on a shared block, don't
7928 * allow to go deeper.
7929 */
7930 advance_left = ADVANCE_ONLY_NEXT;
7931 advance_right = ADVANCE_ONLY_NEXT;
7932 } else {
7933 advance_left = ADVANCE;
7934 advance_right = ADVANCE;
7935 }
7936 }
7937 } else if (left_level < right_level) {
7938 advance_right = ADVANCE;
7939 } else {
7940 advance_left = ADVANCE;
7941 }
7942 }
7943
7944 out_unlock:
7945 up_read(&fs_info->commit_root_sem);
7946 out:
7947 btrfs_free_path(left_path);
7948 btrfs_free_path(right_path);
7949 kvfree(tmp_buf);
7950 return ret;
7951 }
7952
7953 static int send_subvol(struct send_ctx *sctx)
7954 {
7955 int ret;
7956
7957 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_STREAM_HEADER)) {
7958 ret = send_header(sctx);
7959 if (ret < 0)
7960 goto out;
7961 }
7962
7963 ret = send_subvol_begin(sctx);
7964 if (ret < 0)
7965 goto out;
7966
7967 if (sctx->parent_root) {
7968 ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx);
7969 if (ret < 0)
7970 goto out;
7971 ret = finish_inode_if_needed(sctx, 1);
7972 if (ret < 0)
7973 goto out;
7974 } else {
7975 ret = full_send_tree(sctx);
7976 if (ret < 0)
7977 goto out;
7978 }
7979
7980 out:
7981 free_recorded_refs(sctx);
7982 return ret;
7983 }
7984
7985 /*
7986 * If orphan cleanup did remove any orphans from a root, it means the tree
7987 * was modified and therefore the commit root is not the same as the current
7988 * root anymore. This is a problem, because send uses the commit root and
7989 * therefore can see inode items that don't exist in the current root anymore,
7990 * and for example make calls to btrfs_iget, which will do tree lookups based
7991 * on the current root and not on the commit root. Those lookups will fail,
7992 * returning a -ESTALE error, and making send fail with that error. So make
7993 * sure a send does not see any orphans we have just removed, and that it will
7994 * see the same inodes regardless of whether a transaction commit happened
7995 * before it started (meaning that the commit root will be the same as the
7996 * current root) or not.
7997 */
7998 static int ensure_commit_roots_uptodate(struct send_ctx *sctx)
7999 {
8000 struct btrfs_root *root = sctx->parent_root;
8001
8002 if (root && root->node != root->commit_root)
8003 return btrfs_commit_current_transaction(root);
8004
8005 for (int i = 0; i < sctx->clone_roots_cnt; i++) {
8006 root = sctx->clone_roots[i].root;
8007 if (root->node != root->commit_root)
8008 return btrfs_commit_current_transaction(root);
8009 }
8010
8011 return 0;
8012 }
8013
8014 /*
8015 * Make sure any existing dellaloc is flushed for any root used by a send
8016 * operation so that we do not miss any data and we do not race with writeback
8017 * finishing and changing a tree while send is using the tree. This could
8018 * happen if a subvolume is in RW mode, has delalloc, is turned to RO mode and
8019 * a send operation then uses the subvolume.
8020 * After flushing delalloc ensure_commit_roots_uptodate() must be called.
8021 */
8022 static int flush_delalloc_roots(struct send_ctx *sctx)
8023 {
8024 struct btrfs_root *root = sctx->parent_root;
8025 int ret;
8026 int i;
8027
8028 if (root) {
8029 ret = btrfs_start_delalloc_snapshot(root, false);
8030 if (ret)
8031 return ret;
8032 btrfs_wait_ordered_extents(root, U64_MAX, NULL);
8033 }
8034
8035 for (i = 0; i < sctx->clone_roots_cnt; i++) {
8036 root = sctx->clone_roots[i].root;
8037 ret = btrfs_start_delalloc_snapshot(root, false);
8038 if (ret)
8039 return ret;
8040 btrfs_wait_ordered_extents(root, U64_MAX, NULL);
8041 }
8042
8043 return 0;
8044 }
8045
8046 static void btrfs_root_dec_send_in_progress(struct btrfs_root* root)
8047 {
8048 spin_lock(&root->root_item_lock);
8049 root->send_in_progress--;
8050 /*
8051 * Not much left to do, we don't know why it's unbalanced and
8052 * can't blindly reset it to 0.
8053 */
8054 if (root->send_in_progress < 0)
8055 btrfs_err(root->fs_info,
8056 "send_in_progress unbalanced %d root %llu",
8057 root->send_in_progress, btrfs_root_id(root));
8058 spin_unlock(&root->root_item_lock);
8059 }
8060
8061 static void dedupe_in_progress_warn(const struct btrfs_root *root)
8062 {
8063 btrfs_warn_rl(root->fs_info,
8064 "cannot use root %llu for send while deduplications on it are in progress (%d in progress)",
8065 btrfs_root_id(root), root->dedupe_in_progress);
8066 }
8067
8068 long btrfs_ioctl_send(struct btrfs_inode *inode, const struct btrfs_ioctl_send_args *arg)
8069 {
8070 int ret = 0;
8071 struct btrfs_root *send_root = inode->root;
8072 struct btrfs_fs_info *fs_info = send_root->fs_info;
8073 struct btrfs_root *clone_root;
8074 struct send_ctx *sctx = NULL;
8075 u32 i;
8076 u64 *clone_sources_tmp = NULL;
8077 int clone_sources_to_rollback = 0;
8078 size_t alloc_size;
8079 int sort_clone_roots = 0;
8080 struct btrfs_lru_cache_entry *entry;
8081 struct btrfs_lru_cache_entry *tmp;
8082
8083 if (!capable(CAP_SYS_ADMIN))
8084 return -EPERM;
8085
8086 /*
8087 * The subvolume must remain read-only during send, protect against
8088 * making it RW. This also protects against deletion.
8089 */
8090 spin_lock(&send_root->root_item_lock);
8091 if (btrfs_root_readonly(send_root) && send_root->dedupe_in_progress) {
8092 dedupe_in_progress_warn(send_root);
8093 spin_unlock(&send_root->root_item_lock);
8094 return -EAGAIN;
8095 }
8096 send_root->send_in_progress++;
8097 spin_unlock(&send_root->root_item_lock);
8098
8099 /*
8100 * Userspace tools do the checks and warn the user if it's
8101 * not RO.
8102 */
8103 if (!btrfs_root_readonly(send_root)) {
8104 ret = -EPERM;
8105 goto out;
8106 }
8107
8108 /*
8109 * Check that we don't overflow at later allocations, we request
8110 * clone_sources_count + 1 items, and compare to unsigned long inside
8111 * access_ok. Also set an upper limit for allocation size so this can't
8112 * easily exhaust memory. Max number of clone sources is about 200K.
8113 */
8114 if (arg->clone_sources_count > SZ_8M / sizeof(struct clone_root)) {
8115 ret = -EINVAL;
8116 goto out;
8117 }
8118
8119 if (arg->flags & ~BTRFS_SEND_FLAG_MASK) {
8120 ret = -EOPNOTSUPP;
8121 goto out;
8122 }
8123
8124 sctx = kzalloc(sizeof(struct send_ctx), GFP_KERNEL);
8125 if (!sctx) {
8126 ret = -ENOMEM;
8127 goto out;
8128 }
8129
8130 INIT_LIST_HEAD(&sctx->new_refs);
8131 INIT_LIST_HEAD(&sctx->deleted_refs);
8132
8133 btrfs_lru_cache_init(&sctx->name_cache, SEND_MAX_NAME_CACHE_SIZE);
8134 btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE);
8135 btrfs_lru_cache_init(&sctx->dir_created_cache,
8136 SEND_MAX_DIR_CREATED_CACHE_SIZE);
8137 /*
8138 * This cache is periodically trimmed to a fixed size elsewhere, see
8139 * cache_dir_utimes() and trim_dir_utimes_cache().
8140 */
8141 btrfs_lru_cache_init(&sctx->dir_utimes_cache, 0);
8142
8143 sctx->pending_dir_moves = RB_ROOT;
8144 sctx->waiting_dir_moves = RB_ROOT;
8145 sctx->orphan_dirs = RB_ROOT;
8146 sctx->rbtree_new_refs = RB_ROOT;
8147 sctx->rbtree_deleted_refs = RB_ROOT;
8148
8149 sctx->flags = arg->flags;
8150
8151 if (arg->flags & BTRFS_SEND_FLAG_VERSION) {
8152 if (arg->version > BTRFS_SEND_STREAM_VERSION) {
8153 ret = -EPROTO;
8154 goto out;
8155 }
8156 /* Zero means "use the highest version" */
8157 sctx->proto = arg->version ?: BTRFS_SEND_STREAM_VERSION;
8158 } else {
8159 sctx->proto = 1;
8160 }
8161 if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) {
8162 ret = -EINVAL;
8163 goto out;
8164 }
8165
8166 sctx->send_filp = fget(arg->send_fd);
8167 if (!sctx->send_filp || !(sctx->send_filp->f_mode & FMODE_WRITE)) {
8168 ret = -EBADF;
8169 goto out;
8170 }
8171
8172 sctx->send_root = send_root;
8173 /*
8174 * Unlikely but possible, if the subvolume is marked for deletion but
8175 * is slow to remove the directory entry, send can still be started
8176 */
8177 if (btrfs_root_dead(sctx->send_root)) {
8178 ret = -EPERM;
8179 goto out;
8180 }
8181
8182 sctx->clone_roots_cnt = arg->clone_sources_count;
8183
8184 if (sctx->proto >= 2) {
8185 u32 send_buf_num_pages;
8186
8187 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V2;
8188 sctx->send_buf = vmalloc(sctx->send_max_size);
8189 if (!sctx->send_buf) {
8190 ret = -ENOMEM;
8191 goto out;
8192 }
8193 send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT;
8194 sctx->send_buf_pages = kcalloc(send_buf_num_pages,
8195 sizeof(*sctx->send_buf_pages),
8196 GFP_KERNEL);
8197 if (!sctx->send_buf_pages) {
8198 ret = -ENOMEM;
8199 goto out;
8200 }
8201 for (i = 0; i < send_buf_num_pages; i++) {
8202 sctx->send_buf_pages[i] =
8203 vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT));
8204 }
8205 } else {
8206 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1;
8207 sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL);
8208 }
8209 if (!sctx->send_buf) {
8210 ret = -ENOMEM;
8211 goto out;
8212 }
8213
8214 sctx->clone_roots = kvcalloc(arg->clone_sources_count + 1,
8215 sizeof(*sctx->clone_roots),
8216 GFP_KERNEL);
8217 if (!sctx->clone_roots) {
8218 ret = -ENOMEM;
8219 goto out;
8220 }
8221
8222 alloc_size = array_size(sizeof(*arg->clone_sources),
8223 arg->clone_sources_count);
8224
8225 if (arg->clone_sources_count) {
8226 clone_sources_tmp = kvmalloc(alloc_size, GFP_KERNEL);
8227 if (!clone_sources_tmp) {
8228 ret = -ENOMEM;
8229 goto out;
8230 }
8231
8232 ret = copy_from_user(clone_sources_tmp, arg->clone_sources,
8233 alloc_size);
8234 if (ret) {
8235 ret = -EFAULT;
8236 goto out;
8237 }
8238
8239 for (i = 0; i < arg->clone_sources_count; i++) {
8240 clone_root = btrfs_get_fs_root(fs_info,
8241 clone_sources_tmp[i], true);
8242 if (IS_ERR(clone_root)) {
8243 ret = PTR_ERR(clone_root);
8244 goto out;
8245 }
8246 spin_lock(&clone_root->root_item_lock);
8247 if (!btrfs_root_readonly(clone_root) ||
8248 btrfs_root_dead(clone_root)) {
8249 spin_unlock(&clone_root->root_item_lock);
8250 btrfs_put_root(clone_root);
8251 ret = -EPERM;
8252 goto out;
8253 }
8254 if (clone_root->dedupe_in_progress) {
8255 dedupe_in_progress_warn(clone_root);
8256 spin_unlock(&clone_root->root_item_lock);
8257 btrfs_put_root(clone_root);
8258 ret = -EAGAIN;
8259 goto out;
8260 }
8261 clone_root->send_in_progress++;
8262 spin_unlock(&clone_root->root_item_lock);
8263
8264 sctx->clone_roots[i].root = clone_root;
8265 clone_sources_to_rollback = i + 1;
8266 }
8267 kvfree(clone_sources_tmp);
8268 clone_sources_tmp = NULL;
8269 }
8270
8271 if (arg->parent_root) {
8272 sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root,
8273 true);
8274 if (IS_ERR(sctx->parent_root)) {
8275 ret = PTR_ERR(sctx->parent_root);
8276 goto out;
8277 }
8278
8279 spin_lock(&sctx->parent_root->root_item_lock);
8280 sctx->parent_root->send_in_progress++;
8281 if (!btrfs_root_readonly(sctx->parent_root) ||
8282 btrfs_root_dead(sctx->parent_root)) {
8283 spin_unlock(&sctx->parent_root->root_item_lock);
8284 ret = -EPERM;
8285 goto out;
8286 }
8287 if (sctx->parent_root->dedupe_in_progress) {
8288 dedupe_in_progress_warn(sctx->parent_root);
8289 spin_unlock(&sctx->parent_root->root_item_lock);
8290 ret = -EAGAIN;
8291 goto out;
8292 }
8293 spin_unlock(&sctx->parent_root->root_item_lock);
8294 }
8295
8296 /*
8297 * Clones from send_root are allowed, but only if the clone source
8298 * is behind the current send position. This is checked while searching
8299 * for possible clone sources.
8300 */
8301 sctx->clone_roots[sctx->clone_roots_cnt++].root =
8302 btrfs_grab_root(sctx->send_root);
8303
8304 /* We do a bsearch later */
8305 sort(sctx->clone_roots, sctx->clone_roots_cnt,
8306 sizeof(*sctx->clone_roots), __clone_root_cmp_sort,
8307 NULL);
8308 sort_clone_roots = 1;
8309
8310 ret = flush_delalloc_roots(sctx);
8311 if (ret)
8312 goto out;
8313
8314 ret = ensure_commit_roots_uptodate(sctx);
8315 if (ret)
8316 goto out;
8317
8318 ret = send_subvol(sctx);
8319 if (ret < 0)
8320 goto out;
8321
8322 btrfs_lru_cache_for_each_entry_safe(&sctx->dir_utimes_cache, entry, tmp) {
8323 ret = send_utimes(sctx, entry->key, entry->gen);
8324 if (ret < 0)
8325 goto out;
8326 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, entry);
8327 }
8328
8329 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) {
8330 ret = begin_cmd(sctx, BTRFS_SEND_C_END);
8331 if (ret < 0)
8332 goto out;
8333 ret = send_cmd(sctx);
8334 if (ret < 0)
8335 goto out;
8336 }
8337
8338 out:
8339 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->pending_dir_moves));
8340 while (sctx && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)) {
8341 struct rb_node *n;
8342 struct pending_dir_move *pm;
8343
8344 n = rb_first(&sctx->pending_dir_moves);
8345 pm = rb_entry(n, struct pending_dir_move, node);
8346 while (!list_empty(&pm->list)) {
8347 struct pending_dir_move *pm2;
8348
8349 pm2 = list_first_entry(&pm->list,
8350 struct pending_dir_move, list);
8351 free_pending_move(sctx, pm2);
8352 }
8353 free_pending_move(sctx, pm);
8354 }
8355
8356 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves));
8357 while (sctx && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) {
8358 struct rb_node *n;
8359 struct waiting_dir_move *dm;
8360
8361 n = rb_first(&sctx->waiting_dir_moves);
8362 dm = rb_entry(n, struct waiting_dir_move, node);
8363 rb_erase(&dm->node, &sctx->waiting_dir_moves);
8364 kfree(dm);
8365 }
8366
8367 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->orphan_dirs));
8368 while (sctx && !RB_EMPTY_ROOT(&sctx->orphan_dirs)) {
8369 struct rb_node *n;
8370 struct orphan_dir_info *odi;
8371
8372 n = rb_first(&sctx->orphan_dirs);
8373 odi = rb_entry(n, struct orphan_dir_info, node);
8374 free_orphan_dir_info(sctx, odi);
8375 }
8376
8377 if (sort_clone_roots) {
8378 for (i = 0; i < sctx->clone_roots_cnt; i++) {
8379 btrfs_root_dec_send_in_progress(
8380 sctx->clone_roots[i].root);
8381 btrfs_put_root(sctx->clone_roots[i].root);
8382 }
8383 } else {
8384 for (i = 0; sctx && i < clone_sources_to_rollback; i++) {
8385 btrfs_root_dec_send_in_progress(
8386 sctx->clone_roots[i].root);
8387 btrfs_put_root(sctx->clone_roots[i].root);
8388 }
8389
8390 btrfs_root_dec_send_in_progress(send_root);
8391 }
8392 if (sctx && !IS_ERR_OR_NULL(sctx->parent_root)) {
8393 btrfs_root_dec_send_in_progress(sctx->parent_root);
8394 btrfs_put_root(sctx->parent_root);
8395 }
8396
8397 kvfree(clone_sources_tmp);
8398
8399 if (sctx) {
8400 if (sctx->send_filp)
8401 fput(sctx->send_filp);
8402
8403 kvfree(sctx->clone_roots);
8404 kfree(sctx->send_buf_pages);
8405 kvfree(sctx->send_buf);
8406 kvfree(sctx->verity_descriptor);
8407
8408 close_current_inode(sctx);
8409
8410 btrfs_lru_cache_clear(&sctx->name_cache);
8411 btrfs_lru_cache_clear(&sctx->backref_cache);
8412 btrfs_lru_cache_clear(&sctx->dir_created_cache);
8413 btrfs_lru_cache_clear(&sctx->dir_utimes_cache);
8414
8415 kfree(sctx);
8416 }
8417
8418 return ret;
8419 }
8420
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.