1 // SPDX-License-Identifier: GPL-2.0 <<
2 #include <stdlib.h> 1 #include <stdlib.h>
3 #include <assert.h> 2 #include <assert.h>
4 #include <stdio.h> 3 #include <stdio.h>
5 #include <linux/types.h> 4 #include <linux/types.h>
6 #include <linux/kernel.h> 5 #include <linux/kernel.h>
7 #include <linux/bitops.h> 6 #include <linux/bitops.h>
8 7
9 #include "test.h" 8 #include "test.h"
10 9
11 struct item * 10 struct item *
12 item_tag_set(struct radix_tree_root *root, uns 11 item_tag_set(struct radix_tree_root *root, unsigned long index, int tag)
13 { 12 {
14 return radix_tree_tag_set(root, index, 13 return radix_tree_tag_set(root, index, tag);
15 } 14 }
16 15
17 struct item * 16 struct item *
18 item_tag_clear(struct radix_tree_root *root, u 17 item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag)
19 { 18 {
20 return radix_tree_tag_clear(root, inde 19 return radix_tree_tag_clear(root, index, tag);
21 } 20 }
22 21
23 int item_tag_get(struct radix_tree_root *root, 22 int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag)
24 { 23 {
25 return radix_tree_tag_get(root, index, 24 return radix_tree_tag_get(root, index, tag);
26 } 25 }
27 26
28 struct item *item_create(unsigned long index, !! 27 int __item_insert(struct radix_tree_root *root, struct item *item,
>> 28 unsigned order)
29 { 29 {
30 struct item *ret = malloc(sizeof(*ret) !! 30 return __radix_tree_insert(root, item->index, order, item);
31 <<
32 ret->index = index; <<
33 ret->order = order; <<
34 return ret; <<
35 } 31 }
36 32
37 int item_insert(struct radix_tree_root *root, 33 int item_insert(struct radix_tree_root *root, unsigned long index)
38 { 34 {
39 struct item *item = item_create(index, !! 35 return __item_insert(root, item_create(index), 0);
40 int err = radix_tree_insert(root, item <<
41 if (err) <<
42 free(item); <<
43 return err; <<
44 } <<
45 <<
46 void item_sanity(struct item *item, unsigned l <<
47 { <<
48 unsigned long mask; <<
49 assert(!radix_tree_is_internal_node(it <<
50 assert(item->order < BITS_PER_LONG); <<
51 mask = (1UL << item->order) - 1; <<
52 assert((item->index | mask) == (index <<
53 } 36 }
54 37
55 void item_free(struct item *item, unsigned lon !! 38 int item_insert_order(struct radix_tree_root *root, unsigned long index,
>> 39 unsigned order)
56 { 40 {
57 item_sanity(item, index); !! 41 return __item_insert(root, item_create(index), order);
58 free(item); <<
59 } 42 }
60 43
61 int item_delete(struct radix_tree_root *root, 44 int item_delete(struct radix_tree_root *root, unsigned long index)
62 { 45 {
63 struct item *item = radix_tree_delete( 46 struct item *item = radix_tree_delete(root, index);
64 47
65 if (!item) <<
66 return 0; <<
67 <<
68 item_free(item, index); <<
69 return 1; <<
70 } <<
71 <<
72 static void item_free_rcu(struct rcu_head *hea <<
73 { <<
74 struct item *item = container_of(head, <<
75 <<
76 free(item); <<
77 } <<
78 <<
79 int item_delete_rcu(struct xarray *xa, unsigne <<
80 { <<
81 struct item *item = xa_erase(xa, index <<
82 <<
83 if (item) { 48 if (item) {
84 item_sanity(item, index); !! 49 assert(item->index == index);
85 call_rcu(&item->rcu_head, item !! 50 free(item);
86 return 1; 51 return 1;
87 } 52 }
88 return 0; 53 return 0;
89 } 54 }
90 55
>> 56 struct item *item_create(unsigned long index)
>> 57 {
>> 58 struct item *ret = malloc(sizeof(*ret));
>> 59
>> 60 ret->index = index;
>> 61 return ret;
>> 62 }
>> 63
91 void item_check_present(struct radix_tree_root 64 void item_check_present(struct radix_tree_root *root, unsigned long index)
92 { 65 {
93 struct item *item; 66 struct item *item;
94 67
95 item = radix_tree_lookup(root, index); 68 item = radix_tree_lookup(root, index);
96 assert(item != NULL); !! 69 assert(item != 0);
97 item_sanity(item, index); !! 70 assert(item->index == index);
98 } 71 }
99 72
100 struct item *item_lookup(struct radix_tree_roo 73 struct item *item_lookup(struct radix_tree_root *root, unsigned long index)
101 { 74 {
102 return radix_tree_lookup(root, index); 75 return radix_tree_lookup(root, index);
103 } 76 }
104 77
105 void item_check_absent(struct radix_tree_root 78 void item_check_absent(struct radix_tree_root *root, unsigned long index)
106 { 79 {
107 struct item *item; 80 struct item *item;
108 81
109 item = radix_tree_lookup(root, index); 82 item = radix_tree_lookup(root, index);
110 assert(item == NULL); !! 83 assert(item == 0);
111 } 84 }
112 85
113 /* 86 /*
114 * Scan only the passed (start, start+nr] for 87 * Scan only the passed (start, start+nr] for present items
115 */ 88 */
116 void item_gang_check_present(struct radix_tree 89 void item_gang_check_present(struct radix_tree_root *root,
117 unsigned long start, u 90 unsigned long start, unsigned long nr,
118 int chunk, int hop) 91 int chunk, int hop)
119 { 92 {
120 struct item *items[chunk]; 93 struct item *items[chunk];
121 unsigned long into; 94 unsigned long into;
122 95
123 for (into = 0; into < nr; ) { 96 for (into = 0; into < nr; ) {
124 int nfound; 97 int nfound;
125 int nr_to_find = chunk; 98 int nr_to_find = chunk;
126 int i; 99 int i;
127 100
128 if (nr_to_find > (nr - into)) 101 if (nr_to_find > (nr - into))
129 nr_to_find = nr - into 102 nr_to_find = nr - into;
130 103
131 nfound = radix_tree_gang_looku 104 nfound = radix_tree_gang_lookup(root, (void **)items,
132 105 start + into, nr_to_find);
133 assert(nfound == nr_to_find); 106 assert(nfound == nr_to_find);
134 for (i = 0; i < nfound; i++) 107 for (i = 0; i < nfound; i++)
135 assert(items[i]->index 108 assert(items[i]->index == start + into + i);
136 into += hop; 109 into += hop;
137 } 110 }
138 } 111 }
139 112
140 /* 113 /*
141 * Scan the entire tree, only expecting presen 114 * Scan the entire tree, only expecting present items (start, start+nr]
142 */ 115 */
143 void item_full_scan(struct radix_tree_root *ro 116 void item_full_scan(struct radix_tree_root *root, unsigned long start,
144 unsigned long nr, int 117 unsigned long nr, int chunk)
145 { 118 {
146 struct item *items[chunk]; 119 struct item *items[chunk];
147 unsigned long into = 0; 120 unsigned long into = 0;
148 unsigned long this_index = start; 121 unsigned long this_index = start;
149 int nfound; 122 int nfound;
150 int i; 123 int i;
151 124
152 // printf("%s(0x%08lx, 0x%08lx, %d)\n", _ 125 // printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk);
153 126
154 while ((nfound = radix_tree_gang_looku 127 while ((nfound = radix_tree_gang_lookup(root, (void **)items, into,
155 chunk) 128 chunk))) {
156 // printf("At 0x%08lx, nfound=%d\ 129 // printf("At 0x%08lx, nfound=%d\n", into, nfound);
157 for (i = 0; i < nfound; i++) { 130 for (i = 0; i < nfound; i++) {
158 assert(items[i]->index 131 assert(items[i]->index == this_index);
159 this_index++; 132 this_index++;
160 } 133 }
161 // printf("Found 0x%08lx->0x%08lx 134 // printf("Found 0x%08lx->0x%08lx\n",
162 // items[0]->index, items 135 // items[0]->index, items[nfound-1]->index);
163 into = this_index; 136 into = this_index;
164 } 137 }
165 if (chunk) 138 if (chunk)
166 assert(this_index == start + n 139 assert(this_index == start + nr);
167 nfound = radix_tree_gang_lookup(root, 140 nfound = radix_tree_gang_lookup(root, (void **)items,
168 this_i 141 this_index, chunk);
169 assert(nfound == 0); 142 assert(nfound == 0);
170 } 143 }
171 144
172 /* Use the same pattern as tag_pages_for_write <<
173 int tag_tagged_items(struct xarray *xa, unsign <<
174 unsigned batch, xa_mark_t ifta <<
175 { <<
176 XA_STATE(xas, xa, start); <<
177 unsigned int tagged = 0; <<
178 struct item *item; <<
179 <<
180 if (batch == 0) <<
181 batch = 1; <<
182 <<
183 xas_lock_irq(&xas); <<
184 xas_for_each_marked(&xas, item, end, i <<
185 xas_set_mark(&xas, thentag); <<
186 if (++tagged % batch) <<
187 continue; <<
188 <<
189 xas_pause(&xas); <<
190 xas_unlock_irq(&xas); <<
191 rcu_barrier(); <<
192 xas_lock_irq(&xas); <<
193 } <<
194 xas_unlock_irq(&xas); <<
195 <<
196 return tagged; <<
197 } <<
198 <<
199 static int verify_node(struct radix_tree_node 145 static int verify_node(struct radix_tree_node *slot, unsigned int tag,
200 int tagged) 146 int tagged)
201 { 147 {
202 int anyset = 0; 148 int anyset = 0;
203 int i; 149 int i;
204 int j; 150 int j;
205 151
206 slot = entry_to_node(slot); 152 slot = entry_to_node(slot);
207 153
208 /* Verify consistency at this level */ 154 /* Verify consistency at this level */
209 for (i = 0; i < RADIX_TREE_TAG_LONGS; 155 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) {
210 if (slot->tags[tag][i]) { 156 if (slot->tags[tag][i]) {
211 anyset = 1; 157 anyset = 1;
212 break; 158 break;
213 } 159 }
214 } 160 }
215 if (tagged != anyset) { 161 if (tagged != anyset) {
216 printf("tag: %u, shift %u, tag 162 printf("tag: %u, shift %u, tagged: %d, anyset: %d\n",
217 tag, slot->shift, tagg 163 tag, slot->shift, tagged, anyset);
218 for (j = 0; j < RADIX_TREE_MAX 164 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
219 printf("tag %d: ", j); 165 printf("tag %d: ", j);
220 for (i = 0; i < RADIX_ 166 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
221 printf("%016lx 167 printf("%016lx ", slot->tags[j][i]);
222 printf("\n"); 168 printf("\n");
223 } 169 }
224 return 1; 170 return 1;
225 } 171 }
226 assert(tagged == anyset); 172 assert(tagged == anyset);
227 173
228 /* Go for next level */ 174 /* Go for next level */
229 if (slot->shift > 0) { 175 if (slot->shift > 0) {
230 for (i = 0; i < RADIX_TREE_MAP 176 for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
231 if (slot->slots[i]) 177 if (slot->slots[i])
232 if (verify_nod 178 if (verify_node(slot->slots[i], tag,
233 !! 179 !!test_bit(i, slot->tags[tag]))) {
234 printf 180 printf("Failure at off %d\n", i);
235 for (j 181 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
236 182 printf("tag %d: ", j);
237 183 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
238 184 printf("%016lx ", slot->tags[j][i]);
239 185 printf("\n");
240 } 186 }
241 return 187 return 1;
242 } 188 }
243 } 189 }
244 return 0; 190 return 0;
245 } 191 }
246 192
247 void verify_tag_consistency(struct radix_tree_ 193 void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
248 { 194 {
249 struct radix_tree_node *node = root->x !! 195 struct radix_tree_node *node = root->rnode;
250 if (!radix_tree_is_internal_node(node) 196 if (!radix_tree_is_internal_node(node))
251 return; 197 return;
252 verify_node(node, tag, !!root_tag_get( 198 verify_node(node, tag, !!root_tag_get(root, tag));
253 } 199 }
254 200
255 void item_kill_tree(struct xarray *xa) !! 201 void item_kill_tree(struct radix_tree_root *root)
256 { 202 {
257 XA_STATE(xas, xa, 0); !! 203 struct item *items[32];
258 void *entry; !! 204 int nfound;
259 205
260 xas_for_each(&xas, entry, ULONG_MAX) { !! 206 while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) {
261 if (!xa_is_value(entry)) { !! 207 int i;
262 item_free(entry, xas.x !! 208
>> 209 for (i = 0; i < nfound; i++) {
>> 210 void *ret;
>> 211
>> 212 ret = radix_tree_delete(root, items[i]->index);
>> 213 assert(ret == items[i]);
>> 214 free(items[i]);
263 } 215 }
264 xas_store(&xas, NULL); <<
265 } 216 }
266 !! 217 assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0);
267 assert(xa_empty(xa)); !! 218 assert(root->rnode == NULL);
268 } 219 }
269 220
270 void tree_verify_min_height(struct radix_tree_ 221 void tree_verify_min_height(struct radix_tree_root *root, int maxindex)
271 { 222 {
272 unsigned shift; 223 unsigned shift;
273 struct radix_tree_node *node = root->x !! 224 struct radix_tree_node *node = root->rnode;
274 if (!radix_tree_is_internal_node(node) 225 if (!radix_tree_is_internal_node(node)) {
275 assert(maxindex == 0); 226 assert(maxindex == 0);
276 return; 227 return;
277 } 228 }
278 229
279 node = entry_to_node(node); 230 node = entry_to_node(node);
280 assert(maxindex <= node_maxindex(node) 231 assert(maxindex <= node_maxindex(node));
281 232
282 shift = node->shift; 233 shift = node->shift;
283 if (shift > 0) 234 if (shift > 0)
284 assert(maxindex > shift_maxind 235 assert(maxindex > shift_maxindex(shift - RADIX_TREE_MAP_SHIFT));
285 else 236 else
286 assert(maxindex > 0); 237 assert(maxindex > 0);
287 } 238 }
288 239
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.