1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 #include <stdlib.h> 2 #include <stdlib.h>
3 #include <assert.h> 3 #include <assert.h>
4 #include <stdio.h> 4 #include <stdio.h>
5 #include <linux/types.h> 5 #include <linux/types.h>
6 #include <linux/kernel.h> 6 #include <linux/kernel.h>
7 #include <linux/bitops.h> 7 #include <linux/bitops.h>
8 8
9 #include "test.h" 9 #include "test.h"
10 10
11 struct item * 11 struct item *
12 item_tag_set(struct radix_tree_root *root, uns 12 item_tag_set(struct radix_tree_root *root, unsigned long index, int tag)
13 { 13 {
14 return radix_tree_tag_set(root, index, 14 return radix_tree_tag_set(root, index, tag);
15 } 15 }
16 16
17 struct item * 17 struct item *
18 item_tag_clear(struct radix_tree_root *root, u 18 item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag)
19 { 19 {
20 return radix_tree_tag_clear(root, inde 20 return radix_tree_tag_clear(root, index, tag);
21 } 21 }
22 22
23 int item_tag_get(struct radix_tree_root *root, 23 int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag)
24 { 24 {
25 return radix_tree_tag_get(root, index, 25 return radix_tree_tag_get(root, index, tag);
26 } 26 }
27 27
>> 28 int __item_insert(struct radix_tree_root *root, struct item *item)
>> 29 {
>> 30 return __radix_tree_insert(root, item->index, item->order, item);
>> 31 }
>> 32
28 struct item *item_create(unsigned long index, 33 struct item *item_create(unsigned long index, unsigned int order)
29 { 34 {
30 struct item *ret = malloc(sizeof(*ret) 35 struct item *ret = malloc(sizeof(*ret));
31 36
32 ret->index = index; 37 ret->index = index;
33 ret->order = order; 38 ret->order = order;
34 return ret; 39 return ret;
35 } 40 }
36 41
37 int item_insert(struct radix_tree_root *root, !! 42 int item_insert_order(struct radix_tree_root *root, unsigned long index,
>> 43 unsigned order)
38 { 44 {
39 struct item *item = item_create(index, !! 45 struct item *item = item_create(index, order);
40 int err = radix_tree_insert(root, item !! 46 int err = __item_insert(root, item);
41 if (err) 47 if (err)
42 free(item); 48 free(item);
43 return err; 49 return err;
44 } 50 }
45 51
>> 52 int item_insert(struct radix_tree_root *root, unsigned long index)
>> 53 {
>> 54 return item_insert_order(root, index, 0);
>> 55 }
>> 56
46 void item_sanity(struct item *item, unsigned l 57 void item_sanity(struct item *item, unsigned long index)
47 { 58 {
48 unsigned long mask; 59 unsigned long mask;
49 assert(!radix_tree_is_internal_node(it 60 assert(!radix_tree_is_internal_node(item));
50 assert(item->order < BITS_PER_LONG); 61 assert(item->order < BITS_PER_LONG);
51 mask = (1UL << item->order) - 1; 62 mask = (1UL << item->order) - 1;
52 assert((item->index | mask) == (index 63 assert((item->index | mask) == (index | mask));
53 } 64 }
54 65
55 void item_free(struct item *item, unsigned lon <<
56 { <<
57 item_sanity(item, index); <<
58 free(item); <<
59 } <<
60 <<
61 int item_delete(struct radix_tree_root *root, 66 int item_delete(struct radix_tree_root *root, unsigned long index)
62 { 67 {
63 struct item *item = radix_tree_delete( 68 struct item *item = radix_tree_delete(root, index);
64 69
65 if (!item) !! 70 if (item) {
66 return 0; !! 71 item_sanity(item, index);
67 !! 72 free(item);
68 item_free(item, index); !! 73 return 1;
69 return 1; !! 74 }
>> 75 return 0;
70 } 76 }
71 77
72 static void item_free_rcu(struct rcu_head *hea 78 static void item_free_rcu(struct rcu_head *head)
73 { 79 {
74 struct item *item = container_of(head, 80 struct item *item = container_of(head, struct item, rcu_head);
75 81
76 free(item); 82 free(item);
77 } 83 }
78 84
79 int item_delete_rcu(struct xarray *xa, unsigne !! 85 int item_delete_rcu(struct radix_tree_root *root, unsigned long index)
80 { 86 {
81 struct item *item = xa_erase(xa, index !! 87 struct item *item = radix_tree_delete(root, index);
82 88
83 if (item) { 89 if (item) {
84 item_sanity(item, index); 90 item_sanity(item, index);
85 call_rcu(&item->rcu_head, item 91 call_rcu(&item->rcu_head, item_free_rcu);
86 return 1; 92 return 1;
87 } 93 }
88 return 0; 94 return 0;
89 } 95 }
90 96
91 void item_check_present(struct radix_tree_root 97 void item_check_present(struct radix_tree_root *root, unsigned long index)
92 { 98 {
93 struct item *item; 99 struct item *item;
94 100
95 item = radix_tree_lookup(root, index); 101 item = radix_tree_lookup(root, index);
96 assert(item != NULL); 102 assert(item != NULL);
97 item_sanity(item, index); 103 item_sanity(item, index);
98 } 104 }
99 105
100 struct item *item_lookup(struct radix_tree_roo 106 struct item *item_lookup(struct radix_tree_root *root, unsigned long index)
101 { 107 {
102 return radix_tree_lookup(root, index); 108 return radix_tree_lookup(root, index);
103 } 109 }
104 110
105 void item_check_absent(struct radix_tree_root 111 void item_check_absent(struct radix_tree_root *root, unsigned long index)
106 { 112 {
107 struct item *item; 113 struct item *item;
108 114
109 item = radix_tree_lookup(root, index); 115 item = radix_tree_lookup(root, index);
110 assert(item == NULL); 116 assert(item == NULL);
111 } 117 }
112 118
113 /* 119 /*
114 * Scan only the passed (start, start+nr] for 120 * Scan only the passed (start, start+nr] for present items
115 */ 121 */
116 void item_gang_check_present(struct radix_tree 122 void item_gang_check_present(struct radix_tree_root *root,
117 unsigned long start, u 123 unsigned long start, unsigned long nr,
118 int chunk, int hop) 124 int chunk, int hop)
119 { 125 {
120 struct item *items[chunk]; 126 struct item *items[chunk];
121 unsigned long into; 127 unsigned long into;
122 128
123 for (into = 0; into < nr; ) { 129 for (into = 0; into < nr; ) {
124 int nfound; 130 int nfound;
125 int nr_to_find = chunk; 131 int nr_to_find = chunk;
126 int i; 132 int i;
127 133
128 if (nr_to_find > (nr - into)) 134 if (nr_to_find > (nr - into))
129 nr_to_find = nr - into 135 nr_to_find = nr - into;
130 136
131 nfound = radix_tree_gang_looku 137 nfound = radix_tree_gang_lookup(root, (void **)items,
132 138 start + into, nr_to_find);
133 assert(nfound == nr_to_find); 139 assert(nfound == nr_to_find);
134 for (i = 0; i < nfound; i++) 140 for (i = 0; i < nfound; i++)
135 assert(items[i]->index 141 assert(items[i]->index == start + into + i);
136 into += hop; 142 into += hop;
137 } 143 }
138 } 144 }
139 145
140 /* 146 /*
141 * Scan the entire tree, only expecting presen 147 * Scan the entire tree, only expecting present items (start, start+nr]
142 */ 148 */
143 void item_full_scan(struct radix_tree_root *ro 149 void item_full_scan(struct radix_tree_root *root, unsigned long start,
144 unsigned long nr, int 150 unsigned long nr, int chunk)
145 { 151 {
146 struct item *items[chunk]; 152 struct item *items[chunk];
147 unsigned long into = 0; 153 unsigned long into = 0;
148 unsigned long this_index = start; 154 unsigned long this_index = start;
149 int nfound; 155 int nfound;
150 int i; 156 int i;
151 157
152 // printf("%s(0x%08lx, 0x%08lx, %d)\n", _ 158 // printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk);
153 159
154 while ((nfound = radix_tree_gang_looku 160 while ((nfound = radix_tree_gang_lookup(root, (void **)items, into,
155 chunk) 161 chunk))) {
156 // printf("At 0x%08lx, nfound=%d\ 162 // printf("At 0x%08lx, nfound=%d\n", into, nfound);
157 for (i = 0; i < nfound; i++) { 163 for (i = 0; i < nfound; i++) {
158 assert(items[i]->index 164 assert(items[i]->index == this_index);
159 this_index++; 165 this_index++;
160 } 166 }
161 // printf("Found 0x%08lx->0x%08lx 167 // printf("Found 0x%08lx->0x%08lx\n",
162 // items[0]->index, items 168 // items[0]->index, items[nfound-1]->index);
163 into = this_index; 169 into = this_index;
164 } 170 }
165 if (chunk) 171 if (chunk)
166 assert(this_index == start + n 172 assert(this_index == start + nr);
167 nfound = radix_tree_gang_lookup(root, 173 nfound = radix_tree_gang_lookup(root, (void **)items,
168 this_i 174 this_index, chunk);
169 assert(nfound == 0); 175 assert(nfound == 0);
170 } 176 }
171 177
172 /* Use the same pattern as tag_pages_for_write 178 /* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */
173 int tag_tagged_items(struct xarray *xa, unsign !! 179 int tag_tagged_items(struct radix_tree_root *root, pthread_mutex_t *lock,
174 unsigned batch, xa_mark_t ifta !! 180 unsigned long start, unsigned long end, unsigned batch,
175 { !! 181 unsigned iftag, unsigned thentag)
176 XA_STATE(xas, xa, start); !! 182 {
177 unsigned int tagged = 0; !! 183 unsigned long tagged = 0;
178 struct item *item; !! 184 struct radix_tree_iter iter;
>> 185 void **slot;
179 186
180 if (batch == 0) 187 if (batch == 0)
181 batch = 1; 188 batch = 1;
182 189
183 xas_lock_irq(&xas); !! 190 if (lock)
184 xas_for_each_marked(&xas, item, end, i !! 191 pthread_mutex_lock(lock);
185 xas_set_mark(&xas, thentag); !! 192 radix_tree_for_each_tagged(slot, root, &iter, start, iftag) {
186 if (++tagged % batch) !! 193 if (iter.index > end)
>> 194 break;
>> 195 radix_tree_iter_tag_set(root, &iter, thentag);
>> 196 tagged++;
>> 197 if ((tagged % batch) != 0)
187 continue; 198 continue;
188 !! 199 slot = radix_tree_iter_resume(slot, &iter);
189 xas_pause(&xas); !! 200 if (lock) {
190 xas_unlock_irq(&xas); !! 201 pthread_mutex_unlock(lock);
191 rcu_barrier(); !! 202 rcu_barrier();
192 xas_lock_irq(&xas); !! 203 pthread_mutex_lock(lock);
>> 204 }
193 } 205 }
194 xas_unlock_irq(&xas); !! 206 if (lock)
>> 207 pthread_mutex_unlock(lock);
195 208
196 return tagged; 209 return tagged;
197 } 210 }
198 211
>> 212 /* Use the same pattern as find_swap_entry() in mm/shmem.c */
>> 213 unsigned long find_item(struct radix_tree_root *root, void *item)
>> 214 {
>> 215 struct radix_tree_iter iter;
>> 216 void **slot;
>> 217 unsigned long found = -1;
>> 218 unsigned long checked = 0;
>> 219
>> 220 radix_tree_for_each_slot(slot, root, &iter, 0) {
>> 221 if (*slot == item) {
>> 222 found = iter.index;
>> 223 break;
>> 224 }
>> 225 checked++;
>> 226 if ((checked % 4) != 0)
>> 227 continue;
>> 228 slot = radix_tree_iter_resume(slot, &iter);
>> 229 }
>> 230
>> 231 return found;
>> 232 }
>> 233
199 static int verify_node(struct radix_tree_node 234 static int verify_node(struct radix_tree_node *slot, unsigned int tag,
200 int tagged) 235 int tagged)
201 { 236 {
202 int anyset = 0; 237 int anyset = 0;
203 int i; 238 int i;
204 int j; 239 int j;
205 240
206 slot = entry_to_node(slot); 241 slot = entry_to_node(slot);
207 242
208 /* Verify consistency at this level */ 243 /* Verify consistency at this level */
209 for (i = 0; i < RADIX_TREE_TAG_LONGS; 244 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) {
210 if (slot->tags[tag][i]) { 245 if (slot->tags[tag][i]) {
211 anyset = 1; 246 anyset = 1;
212 break; 247 break;
213 } 248 }
214 } 249 }
215 if (tagged != anyset) { 250 if (tagged != anyset) {
216 printf("tag: %u, shift %u, tag 251 printf("tag: %u, shift %u, tagged: %d, anyset: %d\n",
217 tag, slot->shift, tagg 252 tag, slot->shift, tagged, anyset);
218 for (j = 0; j < RADIX_TREE_MAX 253 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
219 printf("tag %d: ", j); 254 printf("tag %d: ", j);
220 for (i = 0; i < RADIX_ 255 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
221 printf("%016lx 256 printf("%016lx ", slot->tags[j][i]);
222 printf("\n"); 257 printf("\n");
223 } 258 }
224 return 1; 259 return 1;
225 } 260 }
226 assert(tagged == anyset); 261 assert(tagged == anyset);
227 262
228 /* Go for next level */ 263 /* Go for next level */
229 if (slot->shift > 0) { 264 if (slot->shift > 0) {
230 for (i = 0; i < RADIX_TREE_MAP 265 for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
231 if (slot->slots[i]) 266 if (slot->slots[i])
232 if (verify_nod 267 if (verify_node(slot->slots[i], tag,
233 !! 268 !!test_bit(i, slot->tags[tag]))) {
234 printf 269 printf("Failure at off %d\n", i);
235 for (j 270 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
236 271 printf("tag %d: ", j);
237 272 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
238 273 printf("%016lx ", slot->tags[j][i]);
239 274 printf("\n");
240 } 275 }
241 return 276 return 1;
242 } 277 }
243 } 278 }
244 return 0; 279 return 0;
245 } 280 }
246 281
247 void verify_tag_consistency(struct radix_tree_ 282 void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
248 { 283 {
249 struct radix_tree_node *node = root->x !! 284 struct radix_tree_node *node = root->rnode;
250 if (!radix_tree_is_internal_node(node) 285 if (!radix_tree_is_internal_node(node))
251 return; 286 return;
252 verify_node(node, tag, !!root_tag_get( 287 verify_node(node, tag, !!root_tag_get(root, tag));
253 } 288 }
254 289
255 void item_kill_tree(struct xarray *xa) !! 290 void item_kill_tree(struct radix_tree_root *root)
256 { 291 {
257 XA_STATE(xas, xa, 0); !! 292 struct radix_tree_iter iter;
258 void *entry; !! 293 void **slot;
>> 294 struct item *items[32];
>> 295 int nfound;
259 296
260 xas_for_each(&xas, entry, ULONG_MAX) { !! 297 radix_tree_for_each_slot(slot, root, &iter, 0) {
261 if (!xa_is_value(entry)) { !! 298 if (radix_tree_exceptional_entry(*slot))
262 item_free(entry, xas.x !! 299 radix_tree_delete(root, iter.index);
263 } <<
264 xas_store(&xas, NULL); <<
265 } 300 }
266 301
267 assert(xa_empty(xa)); !! 302 while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) {
>> 303 int i;
>> 304
>> 305 for (i = 0; i < nfound; i++) {
>> 306 void *ret;
>> 307
>> 308 ret = radix_tree_delete(root, items[i]->index);
>> 309 assert(ret == items[i]);
>> 310 free(items[i]);
>> 311 }
>> 312 }
>> 313 assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0);
>> 314 assert(root->rnode == NULL);
268 } 315 }
269 316
270 void tree_verify_min_height(struct radix_tree_ 317 void tree_verify_min_height(struct radix_tree_root *root, int maxindex)
271 { 318 {
272 unsigned shift; 319 unsigned shift;
273 struct radix_tree_node *node = root->x !! 320 struct radix_tree_node *node = root->rnode;
274 if (!radix_tree_is_internal_node(node) 321 if (!radix_tree_is_internal_node(node)) {
275 assert(maxindex == 0); 322 assert(maxindex == 0);
276 return; 323 return;
277 } 324 }
278 325
279 node = entry_to_node(node); 326 node = entry_to_node(node);
280 assert(maxindex <= node_maxindex(node) 327 assert(maxindex <= node_maxindex(node));
281 328
282 shift = node->shift; 329 shift = node->shift;
283 if (shift > 0) 330 if (shift > 0)
284 assert(maxindex > shift_maxind 331 assert(maxindex > shift_maxindex(shift - RADIX_TREE_MAP_SHIFT));
285 else 332 else
286 assert(maxindex > 0); 333 assert(maxindex > 0);
287 } 334 }
288 335
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.