1 // SPDX-License-Identifier: GPL-2.0 <<
2 #include <linux/ceph/ceph_debug.h> 1 #include <linux/ceph/ceph_debug.h>
3 2
4 #include <linux/module.h> 3 #include <linux/module.h>
5 #include <linux/sched.h> 4 #include <linux/sched.h>
6 #include <linux/slab.h> 5 #include <linux/slab.h>
7 #include <linux/file.h> 6 #include <linux/file.h>
8 #include <linux/namei.h> 7 #include <linux/namei.h>
9 #include <linux/writeback.h> 8 #include <linux/writeback.h>
10 9
11 #include <linux/ceph/libceph.h> 10 #include <linux/ceph/libceph.h>
12 11
>> 12 /*
>> 13 * build a vector of user pages
>> 14 */
>> 15 struct page **ceph_get_direct_page_vector(const void __user *data,
>> 16 int num_pages, bool write_page)
>> 17 {
>> 18 struct page **pages;
>> 19 int got = 0;
>> 20 int rc = 0;
>> 21
>> 22 pages = kmalloc(sizeof(*pages) * num_pages, GFP_NOFS);
>> 23 if (!pages)
>> 24 return ERR_PTR(-ENOMEM);
>> 25
>> 26 down_read(¤t->mm->mmap_sem);
>> 27 while (got < num_pages) {
>> 28 rc = get_user_pages(current, current->mm,
>> 29 (unsigned long)data + ((unsigned long)got * PAGE_SIZE),
>> 30 num_pages - got, write_page, 0, pages + got, NULL);
>> 31 if (rc < 0)
>> 32 break;
>> 33 BUG_ON(rc == 0);
>> 34 got += rc;
>> 35 }
>> 36 up_read(¤t->mm->mmap_sem);
>> 37 if (rc < 0)
>> 38 goto fail;
>> 39 return pages;
>> 40
>> 41 fail:
>> 42 ceph_put_page_vector(pages, got, false);
>> 43 return ERR_PTR(rc);
>> 44 }
>> 45 EXPORT_SYMBOL(ceph_get_direct_page_vector);
>> 46
13 void ceph_put_page_vector(struct page **pages, 47 void ceph_put_page_vector(struct page **pages, int num_pages, bool dirty)
14 { 48 {
15 int i; 49 int i;
16 50
17 for (i = 0; i < num_pages; i++) { 51 for (i = 0; i < num_pages; i++) {
18 if (dirty) 52 if (dirty)
19 set_page_dirty_lock(pa 53 set_page_dirty_lock(pages[i]);
20 put_page(pages[i]); 54 put_page(pages[i]);
21 } 55 }
22 kvfree(pages); !! 56 kfree(pages);
23 } 57 }
24 EXPORT_SYMBOL(ceph_put_page_vector); 58 EXPORT_SYMBOL(ceph_put_page_vector);
25 59
26 void ceph_release_page_vector(struct page **pa 60 void ceph_release_page_vector(struct page **pages, int num_pages)
27 { 61 {
28 int i; 62 int i;
29 63
30 for (i = 0; i < num_pages; i++) 64 for (i = 0; i < num_pages; i++)
31 __free_pages(pages[i], 0); 65 __free_pages(pages[i], 0);
32 kfree(pages); 66 kfree(pages);
33 } 67 }
34 EXPORT_SYMBOL(ceph_release_page_vector); 68 EXPORT_SYMBOL(ceph_release_page_vector);
35 69
36 /* 70 /*
37 * allocate a vector new pages 71 * allocate a vector new pages
38 */ 72 */
39 struct page **ceph_alloc_page_vector(int num_p 73 struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags)
40 { 74 {
41 struct page **pages; 75 struct page **pages;
42 int i; 76 int i;
43 77
44 pages = kmalloc_array(num_pages, sizeo !! 78 pages = kmalloc(sizeof(*pages) * num_pages, flags);
45 if (!pages) 79 if (!pages)
46 return ERR_PTR(-ENOMEM); 80 return ERR_PTR(-ENOMEM);
47 for (i = 0; i < num_pages; i++) { 81 for (i = 0; i < num_pages; i++) {
48 pages[i] = __page_cache_alloc( 82 pages[i] = __page_cache_alloc(flags);
49 if (pages[i] == NULL) { 83 if (pages[i] == NULL) {
50 ceph_release_page_vect 84 ceph_release_page_vector(pages, i);
51 return ERR_PTR(-ENOMEM 85 return ERR_PTR(-ENOMEM);
52 } 86 }
53 } 87 }
54 return pages; 88 return pages;
55 } 89 }
56 EXPORT_SYMBOL(ceph_alloc_page_vector); 90 EXPORT_SYMBOL(ceph_alloc_page_vector);
57 91
58 /* 92 /*
59 * copy user data into a page vector 93 * copy user data into a page vector
60 */ 94 */
61 int ceph_copy_user_to_page_vector(struct page 95 int ceph_copy_user_to_page_vector(struct page **pages,
62 const 96 const void __user *data,
63 loff_ 97 loff_t off, size_t len)
64 { 98 {
65 int i = 0; 99 int i = 0;
66 int po = off & ~PAGE_MASK; !! 100 int po = off & ~PAGE_CACHE_MASK;
67 int left = len; 101 int left = len;
68 int l, bad; 102 int l, bad;
69 103
70 while (left > 0) { 104 while (left > 0) {
71 l = min_t(int, PAGE_SIZE-po, l !! 105 l = min_t(int, PAGE_CACHE_SIZE-po, left);
72 bad = copy_from_user(page_addr 106 bad = copy_from_user(page_address(pages[i]) + po, data, l);
73 if (bad == l) 107 if (bad == l)
74 return -EFAULT; 108 return -EFAULT;
75 data += l - bad; 109 data += l - bad;
76 left -= l - bad; 110 left -= l - bad;
77 po += l - bad; 111 po += l - bad;
78 if (po == PAGE_SIZE) { !! 112 if (po == PAGE_CACHE_SIZE) {
79 po = 0; 113 po = 0;
80 i++; 114 i++;
81 } 115 }
82 } 116 }
83 return len; 117 return len;
84 } 118 }
85 EXPORT_SYMBOL(ceph_copy_user_to_page_vector); 119 EXPORT_SYMBOL(ceph_copy_user_to_page_vector);
86 120
87 void ceph_copy_to_page_vector(struct page **pa 121 void ceph_copy_to_page_vector(struct page **pages,
88 const void 122 const void *data,
89 loff_t off 123 loff_t off, size_t len)
90 { 124 {
91 int i = 0; 125 int i = 0;
92 size_t po = off & ~PAGE_MASK; !! 126 size_t po = off & ~PAGE_CACHE_MASK;
93 size_t left = len; 127 size_t left = len;
94 128
95 while (left > 0) { 129 while (left > 0) {
96 size_t l = min_t(size_t, PAGE_ !! 130 size_t l = min_t(size_t, PAGE_CACHE_SIZE-po, left);
97 131
98 memcpy(page_address(pages[i]) 132 memcpy(page_address(pages[i]) + po, data, l);
99 data += l; 133 data += l;
100 left -= l; 134 left -= l;
101 po += l; 135 po += l;
102 if (po == PAGE_SIZE) { !! 136 if (po == PAGE_CACHE_SIZE) {
103 po = 0; 137 po = 0;
104 i++; 138 i++;
105 } 139 }
106 } 140 }
107 } 141 }
108 EXPORT_SYMBOL(ceph_copy_to_page_vector); 142 EXPORT_SYMBOL(ceph_copy_to_page_vector);
109 143
110 void ceph_copy_from_page_vector(struct page ** 144 void ceph_copy_from_page_vector(struct page **pages,
111 void *data 145 void *data,
112 loff_t off 146 loff_t off, size_t len)
113 { 147 {
114 int i = 0; 148 int i = 0;
115 size_t po = off & ~PAGE_MASK; !! 149 size_t po = off & ~PAGE_CACHE_MASK;
116 size_t left = len; 150 size_t left = len;
117 151
118 while (left > 0) { 152 while (left > 0) {
119 size_t l = min_t(size_t, PAGE_ !! 153 size_t l = min_t(size_t, PAGE_CACHE_SIZE-po, left);
120 154
121 memcpy(data, page_address(page 155 memcpy(data, page_address(pages[i]) + po, l);
122 data += l; 156 data += l;
123 left -= l; 157 left -= l;
124 po += l; 158 po += l;
125 if (po == PAGE_SIZE) { !! 159 if (po == PAGE_CACHE_SIZE) {
126 po = 0; 160 po = 0;
127 i++; 161 i++;
128 } 162 }
129 } 163 }
130 } 164 }
131 EXPORT_SYMBOL(ceph_copy_from_page_vector); 165 EXPORT_SYMBOL(ceph_copy_from_page_vector);
132 166
133 /* 167 /*
>> 168 * copy user data from a page vector into a user pointer
>> 169 */
>> 170 int ceph_copy_page_vector_to_user(struct page **pages,
>> 171 void __user *data,
>> 172 loff_t off, size_t len)
>> 173 {
>> 174 int i = 0;
>> 175 int po = off & ~PAGE_CACHE_MASK;
>> 176 int left = len;
>> 177 int l, bad;
>> 178
>> 179 while (left > 0) {
>> 180 l = min_t(int, left, PAGE_CACHE_SIZE-po);
>> 181 bad = copy_to_user(data, page_address(pages[i]) + po, l);
>> 182 if (bad == l)
>> 183 return -EFAULT;
>> 184 data += l - bad;
>> 185 left -= l - bad;
>> 186 if (po) {
>> 187 po += l - bad;
>> 188 if (po == PAGE_CACHE_SIZE)
>> 189 po = 0;
>> 190 }
>> 191 i++;
>> 192 }
>> 193 return len;
>> 194 }
>> 195 EXPORT_SYMBOL(ceph_copy_page_vector_to_user);
>> 196
>> 197 /*
134 * Zero an extent within a page vector. Offse 198 * Zero an extent within a page vector. Offset is relative to the
135 * start of the first page. 199 * start of the first page.
136 */ 200 */
137 void ceph_zero_page_vector_range(int off, int 201 void ceph_zero_page_vector_range(int off, int len, struct page **pages)
138 { 202 {
139 int i = off >> PAGE_SHIFT; !! 203 int i = off >> PAGE_CACHE_SHIFT;
140 204
141 off &= ~PAGE_MASK; !! 205 off &= ~PAGE_CACHE_MASK;
142 206
143 dout("zero_page_vector_page %u~%u\n", 207 dout("zero_page_vector_page %u~%u\n", off, len);
144 208
145 /* leading partial page? */ 209 /* leading partial page? */
146 if (off) { 210 if (off) {
147 int end = min((int)PAGE_SIZE, !! 211 int end = min((int)PAGE_CACHE_SIZE, off + len);
148 dout("zeroing %d %p head from 212 dout("zeroing %d %p head from %d\n", i, pages[i],
149 (int)off); 213 (int)off);
150 zero_user_segment(pages[i], of 214 zero_user_segment(pages[i], off, end);
151 len -= (end - off); 215 len -= (end - off);
152 i++; 216 i++;
153 } 217 }
154 while (len >= PAGE_SIZE) { !! 218 while (len >= PAGE_CACHE_SIZE) {
155 dout("zeroing %d %p len=%d\n", 219 dout("zeroing %d %p len=%d\n", i, pages[i], len);
156 zero_user_segment(pages[i], 0, !! 220 zero_user_segment(pages[i], 0, PAGE_CACHE_SIZE);
157 len -= PAGE_SIZE; !! 221 len -= PAGE_CACHE_SIZE;
158 i++; 222 i++;
159 } 223 }
160 /* trailing partial page? */ 224 /* trailing partial page? */
161 if (len) { 225 if (len) {
162 dout("zeroing %d %p tail to %d 226 dout("zeroing %d %p tail to %d\n", i, pages[i], (int)len);
163 zero_user_segment(pages[i], 0, 227 zero_user_segment(pages[i], 0, len);
164 } 228 }
165 } 229 }
166 EXPORT_SYMBOL(ceph_zero_page_vector_range); 230 EXPORT_SYMBOL(ceph_zero_page_vector_range);
>> 231
167 232
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.