1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
3
4 #include <linux/module.h>
5 #include <linux/slab.h>
6 #include <linux/rhashtable.h>
7 #include <linux/idr.h>
8 #include <linux/list.h>
9 #include <linux/sort.h>
10 #include <linux/objagg.h>
11
12 #define CREATE_TRACE_POINTS
13 #include <trace/events/objagg.h>
14
15 struct objagg_hints {
16 struct rhashtable node_ht;
17 struct rhashtable_params ht_params;
18 struct list_head node_list;
19 unsigned int node_count;
20 unsigned int root_count;
21 unsigned int refcount;
22 const struct objagg_ops *ops;
23 };
24
25 struct objagg_hints_node {
26 struct rhash_head ht_node; /* member of objagg_hints->node_ht */
27 struct list_head list; /* member of objagg_hints->node_list */
28 struct objagg_hints_node *parent;
29 unsigned int root_id;
30 struct objagg_obj_stats_info stats_info;
31 unsigned long obj[];
32 };
33
34 static struct objagg_hints_node *
35 objagg_hints_lookup(struct objagg_hints *objagg_hints, void *obj)
36 {
37 if (!objagg_hints)
38 return NULL;
39 return rhashtable_lookup_fast(&objagg_hints->node_ht, obj,
40 objagg_hints->ht_params);
41 }
42
43 struct objagg {
44 const struct objagg_ops *ops;
45 void *priv;
46 struct rhashtable obj_ht;
47 struct rhashtable_params ht_params;
48 struct list_head obj_list;
49 unsigned int obj_count;
50 struct ida root_ida;
51 struct objagg_hints *hints;
52 };
53
54 struct objagg_obj {
55 struct rhash_head ht_node; /* member of objagg->obj_ht */
56 struct list_head list; /* member of objagg->obj_list */
57 struct objagg_obj *parent; /* if the object is nested, this
58 * holds pointer to parent, otherwise NULL
59 */
60 union {
61 void *delta_priv; /* user delta private */
62 void *root_priv; /* user root private */
63 };
64 unsigned int root_id;
65 unsigned int refcount; /* counts number of users of this object
66 * including nested objects
67 */
68 struct objagg_obj_stats stats;
69 unsigned long obj[];
70 };
71
72 static unsigned int objagg_obj_ref_inc(struct objagg_obj *objagg_obj)
73 {
74 return ++objagg_obj->refcount;
75 }
76
77 static unsigned int objagg_obj_ref_dec(struct objagg_obj *objagg_obj)
78 {
79 return --objagg_obj->refcount;
80 }
81
82 static void objagg_obj_stats_inc(struct objagg_obj *objagg_obj)
83 {
84 objagg_obj->stats.user_count++;
85 objagg_obj->stats.delta_user_count++;
86 if (objagg_obj->parent)
87 objagg_obj->parent->stats.delta_user_count++;
88 }
89
90 static void objagg_obj_stats_dec(struct objagg_obj *objagg_obj)
91 {
92 objagg_obj->stats.user_count--;
93 objagg_obj->stats.delta_user_count--;
94 if (objagg_obj->parent)
95 objagg_obj->parent->stats.delta_user_count--;
96 }
97
98 static bool objagg_obj_is_root(const struct objagg_obj *objagg_obj)
99 {
100 /* Nesting is not supported, so we can use ->parent
101 * to figure out if the object is root.
102 */
103 return !objagg_obj->parent;
104 }
105
106 /**
107 * objagg_obj_root_priv - obtains root private for an object
108 * @objagg_obj: objagg object instance
109 *
110 * Note: all locking must be provided by the caller.
111 *
112 * Either the object is root itself when the private is returned
113 * directly, or the parent is root and its private is returned
114 * instead.
115 *
116 * Returns a user private root pointer.
117 */
118 const void *objagg_obj_root_priv(const struct objagg_obj *objagg_obj)
119 {
120 if (objagg_obj_is_root(objagg_obj))
121 return objagg_obj->root_priv;
122 WARN_ON(!objagg_obj_is_root(objagg_obj->parent));
123 return objagg_obj->parent->root_priv;
124 }
125 EXPORT_SYMBOL(objagg_obj_root_priv);
126
127 /**
128 * objagg_obj_delta_priv - obtains delta private for an object
129 * @objagg_obj: objagg object instance
130 *
131 * Note: all locking must be provided by the caller.
132 *
133 * Returns user private delta pointer or NULL in case the passed
134 * object is root.
135 */
136 const void *objagg_obj_delta_priv(const struct objagg_obj *objagg_obj)
137 {
138 if (objagg_obj_is_root(objagg_obj))
139 return NULL;
140 return objagg_obj->delta_priv;
141 }
142 EXPORT_SYMBOL(objagg_obj_delta_priv);
143
144 /**
145 * objagg_obj_raw - obtains object user private pointer
146 * @objagg_obj: objagg object instance
147 *
148 * Note: all locking must be provided by the caller.
149 *
150 * Returns user private pointer as was passed to objagg_obj_get() by "obj" arg.
151 */
152 const void *objagg_obj_raw(const struct objagg_obj *objagg_obj)
153 {
154 return objagg_obj->obj;
155 }
156 EXPORT_SYMBOL(objagg_obj_raw);
157
158 static struct objagg_obj *objagg_obj_lookup(struct objagg *objagg, void *obj)
159 {
160 return rhashtable_lookup_fast(&objagg->obj_ht, obj, objagg->ht_params);
161 }
162
163 static int objagg_obj_parent_assign(struct objagg *objagg,
164 struct objagg_obj *objagg_obj,
165 struct objagg_obj *parent,
166 bool take_parent_ref)
167 {
168 void *delta_priv;
169
170 if (WARN_ON(!objagg_obj_is_root(parent)))
171 return -EINVAL;
172
173 delta_priv = objagg->ops->delta_create(objagg->priv, parent->obj,
174 objagg_obj->obj);
175 if (IS_ERR(delta_priv))
176 return PTR_ERR(delta_priv);
177
178 /* User returned a delta private, that means that
179 * our object can be aggregated into the parent.
180 */
181 objagg_obj->parent = parent;
182 objagg_obj->delta_priv = delta_priv;
183 if (take_parent_ref)
184 objagg_obj_ref_inc(objagg_obj->parent);
185 trace_objagg_obj_parent_assign(objagg, objagg_obj,
186 parent,
187 parent->refcount);
188 return 0;
189 }
190
191 static int objagg_obj_parent_lookup_assign(struct objagg *objagg,
192 struct objagg_obj *objagg_obj)
193 {
194 struct objagg_obj *objagg_obj_cur;
195 int err;
196
197 list_for_each_entry(objagg_obj_cur, &objagg->obj_list, list) {
198 /* Nesting is not supported. In case the object
199 * is not root, it cannot be assigned as parent.
200 */
201 if (!objagg_obj_is_root(objagg_obj_cur))
202 continue;
203 err = objagg_obj_parent_assign(objagg, objagg_obj,
204 objagg_obj_cur, true);
205 if (!err)
206 return 0;
207 }
208 return -ENOENT;
209 }
210
211 static void __objagg_obj_put(struct objagg *objagg,
212 struct objagg_obj *objagg_obj);
213
214 static void objagg_obj_parent_unassign(struct objagg *objagg,
215 struct objagg_obj *objagg_obj)
216 {
217 trace_objagg_obj_parent_unassign(objagg, objagg_obj,
218 objagg_obj->parent,
219 objagg_obj->parent->refcount);
220 objagg->ops->delta_destroy(objagg->priv, objagg_obj->delta_priv);
221 __objagg_obj_put(objagg, objagg_obj->parent);
222 }
223
224 static int objagg_obj_root_id_alloc(struct objagg *objagg,
225 struct objagg_obj *objagg_obj,
226 struct objagg_hints_node *hnode)
227 {
228 unsigned int min, max;
229 int root_id;
230
231 /* In case there are no hints available, the root id is invalid. */
232 if (!objagg->hints) {
233 objagg_obj->root_id = OBJAGG_OBJ_ROOT_ID_INVALID;
234 return 0;
235 }
236
237 if (hnode) {
238 min = hnode->root_id;
239 max = hnode->root_id;
240 } else {
241 /* For objects with no hint, start after the last
242 * hinted root_id.
243 */
244 min = objagg->hints->root_count;
245 max = ~0;
246 }
247
248 root_id = ida_alloc_range(&objagg->root_ida, min, max, GFP_KERNEL);
249
250 if (root_id < 0)
251 return root_id;
252 objagg_obj->root_id = root_id;
253 return 0;
254 }
255
256 static void objagg_obj_root_id_free(struct objagg *objagg,
257 struct objagg_obj *objagg_obj)
258 {
259 if (!objagg->hints)
260 return;
261 ida_free(&objagg->root_ida, objagg_obj->root_id);
262 }
263
264 static int objagg_obj_root_create(struct objagg *objagg,
265 struct objagg_obj *objagg_obj,
266 struct objagg_hints_node *hnode)
267 {
268 int err;
269
270 err = objagg_obj_root_id_alloc(objagg, objagg_obj, hnode);
271 if (err)
272 return err;
273 objagg_obj->root_priv = objagg->ops->root_create(objagg->priv,
274 objagg_obj->obj,
275 objagg_obj->root_id);
276 if (IS_ERR(objagg_obj->root_priv)) {
277 err = PTR_ERR(objagg_obj->root_priv);
278 goto err_root_create;
279 }
280 trace_objagg_obj_root_create(objagg, objagg_obj);
281 return 0;
282
283 err_root_create:
284 objagg_obj_root_id_free(objagg, objagg_obj);
285 return err;
286 }
287
288 static void objagg_obj_root_destroy(struct objagg *objagg,
289 struct objagg_obj *objagg_obj)
290 {
291 trace_objagg_obj_root_destroy(objagg, objagg_obj);
292 objagg->ops->root_destroy(objagg->priv, objagg_obj->root_priv);
293 objagg_obj_root_id_free(objagg, objagg_obj);
294 }
295
296 static struct objagg_obj *__objagg_obj_get(struct objagg *objagg, void *obj);
297
298 static int objagg_obj_init_with_hints(struct objagg *objagg,
299 struct objagg_obj *objagg_obj,
300 bool *hint_found)
301 {
302 struct objagg_hints_node *hnode;
303 struct objagg_obj *parent;
304 int err;
305
306 hnode = objagg_hints_lookup(objagg->hints, objagg_obj->obj);
307 if (!hnode) {
308 *hint_found = false;
309 return 0;
310 }
311 *hint_found = true;
312
313 if (!hnode->parent)
314 return objagg_obj_root_create(objagg, objagg_obj, hnode);
315
316 parent = __objagg_obj_get(objagg, hnode->parent->obj);
317 if (IS_ERR(parent))
318 return PTR_ERR(parent);
319
320 err = objagg_obj_parent_assign(objagg, objagg_obj, parent, false);
321 if (err) {
322 *hint_found = false;
323 err = 0;
324 goto err_parent_assign;
325 }
326
327 return 0;
328
329 err_parent_assign:
330 objagg_obj_put(objagg, parent);
331 return err;
332 }
333
334 static int objagg_obj_init(struct objagg *objagg,
335 struct objagg_obj *objagg_obj)
336 {
337 bool hint_found;
338 int err;
339
340 /* First, try to use hints if they are available and
341 * if they provide result.
342 */
343 err = objagg_obj_init_with_hints(objagg, objagg_obj, &hint_found);
344 if (err)
345 return err;
346
347 if (hint_found)
348 return 0;
349
350 /* Try to find if the object can be aggregated under an existing one. */
351 err = objagg_obj_parent_lookup_assign(objagg, objagg_obj);
352 if (!err)
353 return 0;
354 /* If aggregation is not possible, make the object a root. */
355 return objagg_obj_root_create(objagg, objagg_obj, NULL);
356 }
357
358 static void objagg_obj_fini(struct objagg *objagg,
359 struct objagg_obj *objagg_obj)
360 {
361 if (!objagg_obj_is_root(objagg_obj))
362 objagg_obj_parent_unassign(objagg, objagg_obj);
363 else
364 objagg_obj_root_destroy(objagg, objagg_obj);
365 }
366
367 static struct objagg_obj *objagg_obj_create(struct objagg *objagg, void *obj)
368 {
369 struct objagg_obj *objagg_obj;
370 int err;
371
372 objagg_obj = kzalloc(sizeof(*objagg_obj) + objagg->ops->obj_size,
373 GFP_KERNEL);
374 if (!objagg_obj)
375 return ERR_PTR(-ENOMEM);
376 objagg_obj_ref_inc(objagg_obj);
377 memcpy(objagg_obj->obj, obj, objagg->ops->obj_size);
378
379 err = objagg_obj_init(objagg, objagg_obj);
380 if (err)
381 goto err_obj_init;
382
383 err = rhashtable_insert_fast(&objagg->obj_ht, &objagg_obj->ht_node,
384 objagg->ht_params);
385 if (err)
386 goto err_ht_insert;
387 list_add(&objagg_obj->list, &objagg->obj_list);
388 objagg->obj_count++;
389 trace_objagg_obj_create(objagg, objagg_obj);
390
391 return objagg_obj;
392
393 err_ht_insert:
394 objagg_obj_fini(objagg, objagg_obj);
395 err_obj_init:
396 kfree(objagg_obj);
397 return ERR_PTR(err);
398 }
399
400 static struct objagg_obj *__objagg_obj_get(struct objagg *objagg, void *obj)
401 {
402 struct objagg_obj *objagg_obj;
403
404 /* First, try to find the object exactly as user passed it,
405 * perhaps it is already in use.
406 */
407 objagg_obj = objagg_obj_lookup(objagg, obj);
408 if (objagg_obj) {
409 objagg_obj_ref_inc(objagg_obj);
410 return objagg_obj;
411 }
412
413 return objagg_obj_create(objagg, obj);
414 }
415
416 /**
417 * objagg_obj_get - gets an object within objagg instance
418 * @objagg: objagg instance
419 * @obj: user-specific private object pointer
420 *
421 * Note: all locking must be provided by the caller.
422 *
423 * Size of the "obj" memory is specified in "objagg->ops".
424 *
425 * There are 3 main options this function wraps:
426 * 1) The object according to "obj" already exist. In that case
427 * the reference counter is incremented and the object is returned.
428 * 2) The object does not exist, but it can be aggregated within
429 * another object. In that case, user ops->delta_create() is called
430 * to obtain delta data and a new object is created with returned
431 * user-delta private pointer.
432 * 3) The object does not exist and cannot be aggregated into
433 * any of the existing objects. In that case, user ops->root_create()
434 * is called to create the root and a new object is created with
435 * returned user-root private pointer.
436 *
437 * Returns a pointer to objagg object instance in case of success,
438 * otherwise it returns pointer error using ERR_PTR macro.
439 */
440 struct objagg_obj *objagg_obj_get(struct objagg *objagg, void *obj)
441 {
442 struct objagg_obj *objagg_obj;
443
444 objagg_obj = __objagg_obj_get(objagg, obj);
445 if (IS_ERR(objagg_obj))
446 return objagg_obj;
447 objagg_obj_stats_inc(objagg_obj);
448 trace_objagg_obj_get(objagg, objagg_obj, objagg_obj->refcount);
449 return objagg_obj;
450 }
451 EXPORT_SYMBOL(objagg_obj_get);
452
453 static void objagg_obj_destroy(struct objagg *objagg,
454 struct objagg_obj *objagg_obj)
455 {
456 trace_objagg_obj_destroy(objagg, objagg_obj);
457 --objagg->obj_count;
458 list_del(&objagg_obj->list);
459 rhashtable_remove_fast(&objagg->obj_ht, &objagg_obj->ht_node,
460 objagg->ht_params);
461 objagg_obj_fini(objagg, objagg_obj);
462 kfree(objagg_obj);
463 }
464
465 static void __objagg_obj_put(struct objagg *objagg,
466 struct objagg_obj *objagg_obj)
467 {
468 if (!objagg_obj_ref_dec(objagg_obj))
469 objagg_obj_destroy(objagg, objagg_obj);
470 }
471
472 /**
473 * objagg_obj_put - puts an object within objagg instance
474 * @objagg: objagg instance
475 * @objagg_obj: objagg object instance
476 *
477 * Note: all locking must be provided by the caller.
478 *
479 * Symmetric to objagg_obj_get().
480 */
481 void objagg_obj_put(struct objagg *objagg, struct objagg_obj *objagg_obj)
482 {
483 trace_objagg_obj_put(objagg, objagg_obj, objagg_obj->refcount);
484 objagg_obj_stats_dec(objagg_obj);
485 __objagg_obj_put(objagg, objagg_obj);
486 }
487 EXPORT_SYMBOL(objagg_obj_put);
488
489 /**
490 * objagg_create - creates a new objagg instance
491 * @ops: user-specific callbacks
492 * @objagg_hints: hints, can be NULL
493 * @priv: pointer to a private data passed to the ops
494 *
495 * Note: all locking must be provided by the caller.
496 *
497 * The purpose of the library is to provide an infrastructure to
498 * aggregate user-specified objects. Library does not care about the type
499 * of the object. User fills-up ops which take care of the specific
500 * user object manipulation.
501 *
502 * As a very stupid example, consider integer numbers. For example
503 * number 8 as a root object. That can aggregate number 9 with delta 1,
504 * number 10 with delta 2, etc. This example is implemented as
505 * a part of a testing module in test_objagg.c file.
506 *
507 * Each objagg instance contains multiple trees. Each tree node is
508 * represented by "an object". In the current implementation there can be
509 * only roots and leafs nodes. Leaf nodes are called deltas.
510 * But in general, this can be easily extended for intermediate nodes.
511 * In that extension, a delta would be associated with all non-root
512 * nodes.
513 *
514 * Returns a pointer to newly created objagg instance in case of success,
515 * otherwise it returns pointer error using ERR_PTR macro.
516 */
517 struct objagg *objagg_create(const struct objagg_ops *ops,
518 struct objagg_hints *objagg_hints, void *priv)
519 {
520 struct objagg *objagg;
521 int err;
522
523 if (WARN_ON(!ops || !ops->root_create || !ops->root_destroy ||
524 !ops->delta_check || !ops->delta_create ||
525 !ops->delta_destroy))
526 return ERR_PTR(-EINVAL);
527
528 objagg = kzalloc(sizeof(*objagg), GFP_KERNEL);
529 if (!objagg)
530 return ERR_PTR(-ENOMEM);
531 objagg->ops = ops;
532 if (objagg_hints) {
533 objagg->hints = objagg_hints;
534 objagg_hints->refcount++;
535 }
536 objagg->priv = priv;
537 INIT_LIST_HEAD(&objagg->obj_list);
538
539 objagg->ht_params.key_len = ops->obj_size;
540 objagg->ht_params.key_offset = offsetof(struct objagg_obj, obj);
541 objagg->ht_params.head_offset = offsetof(struct objagg_obj, ht_node);
542
543 err = rhashtable_init(&objagg->obj_ht, &objagg->ht_params);
544 if (err)
545 goto err_rhashtable_init;
546
547 ida_init(&objagg->root_ida);
548
549 trace_objagg_create(objagg);
550 return objagg;
551
552 err_rhashtable_init:
553 kfree(objagg);
554 return ERR_PTR(err);
555 }
556 EXPORT_SYMBOL(objagg_create);
557
558 /**
559 * objagg_destroy - destroys a new objagg instance
560 * @objagg: objagg instance
561 *
562 * Note: all locking must be provided by the caller.
563 */
564 void objagg_destroy(struct objagg *objagg)
565 {
566 trace_objagg_destroy(objagg);
567 ida_destroy(&objagg->root_ida);
568 WARN_ON(!list_empty(&objagg->obj_list));
569 rhashtable_destroy(&objagg->obj_ht);
570 if (objagg->hints)
571 objagg_hints_put(objagg->hints);
572 kfree(objagg);
573 }
574 EXPORT_SYMBOL(objagg_destroy);
575
576 static int objagg_stats_info_sort_cmp_func(const void *a, const void *b)
577 {
578 const struct objagg_obj_stats_info *stats_info1 = a;
579 const struct objagg_obj_stats_info *stats_info2 = b;
580
581 if (stats_info1->is_root != stats_info2->is_root)
582 return stats_info2->is_root - stats_info1->is_root;
583 if (stats_info1->stats.delta_user_count !=
584 stats_info2->stats.delta_user_count)
585 return stats_info2->stats.delta_user_count -
586 stats_info1->stats.delta_user_count;
587 return stats_info2->stats.user_count - stats_info1->stats.user_count;
588 }
589
590 /**
591 * objagg_stats_get - obtains stats of the objagg instance
592 * @objagg: objagg instance
593 *
594 * Note: all locking must be provided by the caller.
595 *
596 * The returned structure contains statistics of all object
597 * currently in use, ordered by following rules:
598 * 1) Root objects are always on lower indexes than the rest.
599 * 2) Objects with higher delta user count are always on lower
600 * indexes.
601 * 3) In case more objects have the same delta user count,
602 * the objects are ordered by user count.
603 *
604 * Returns a pointer to stats instance in case of success,
605 * otherwise it returns pointer error using ERR_PTR macro.
606 */
607 const struct objagg_stats *objagg_stats_get(struct objagg *objagg)
608 {
609 struct objagg_stats *objagg_stats;
610 struct objagg_obj *objagg_obj;
611 int i;
612
613 objagg_stats = kzalloc(struct_size(objagg_stats, stats_info,
614 objagg->obj_count), GFP_KERNEL);
615 if (!objagg_stats)
616 return ERR_PTR(-ENOMEM);
617
618 i = 0;
619 list_for_each_entry(objagg_obj, &objagg->obj_list, list) {
620 memcpy(&objagg_stats->stats_info[i].stats, &objagg_obj->stats,
621 sizeof(objagg_stats->stats_info[0].stats));
622 objagg_stats->stats_info[i].objagg_obj = objagg_obj;
623 objagg_stats->stats_info[i].is_root =
624 objagg_obj_is_root(objagg_obj);
625 if (objagg_stats->stats_info[i].is_root)
626 objagg_stats->root_count++;
627 i++;
628 }
629 objagg_stats->stats_info_count = i;
630
631 sort(objagg_stats->stats_info, objagg_stats->stats_info_count,
632 sizeof(struct objagg_obj_stats_info),
633 objagg_stats_info_sort_cmp_func, NULL);
634
635 return objagg_stats;
636 }
637 EXPORT_SYMBOL(objagg_stats_get);
638
639 /**
640 * objagg_stats_put - puts stats of the objagg instance
641 * @objagg_stats: objagg instance stats
642 *
643 * Note: all locking must be provided by the caller.
644 */
645 void objagg_stats_put(const struct objagg_stats *objagg_stats)
646 {
647 kfree(objagg_stats);
648 }
649 EXPORT_SYMBOL(objagg_stats_put);
650
651 static struct objagg_hints_node *
652 objagg_hints_node_create(struct objagg_hints *objagg_hints,
653 struct objagg_obj *objagg_obj, size_t obj_size,
654 struct objagg_hints_node *parent_hnode)
655 {
656 unsigned int user_count = objagg_obj->stats.user_count;
657 struct objagg_hints_node *hnode;
658 int err;
659
660 hnode = kzalloc(sizeof(*hnode) + obj_size, GFP_KERNEL);
661 if (!hnode)
662 return ERR_PTR(-ENOMEM);
663 memcpy(hnode->obj, &objagg_obj->obj, obj_size);
664 hnode->stats_info.stats.user_count = user_count;
665 hnode->stats_info.stats.delta_user_count = user_count;
666 if (parent_hnode) {
667 parent_hnode->stats_info.stats.delta_user_count += user_count;
668 } else {
669 hnode->root_id = objagg_hints->root_count++;
670 hnode->stats_info.is_root = true;
671 }
672 hnode->stats_info.objagg_obj = objagg_obj;
673
674 err = rhashtable_insert_fast(&objagg_hints->node_ht, &hnode->ht_node,
675 objagg_hints->ht_params);
676 if (err)
677 goto err_ht_insert;
678
679 list_add(&hnode->list, &objagg_hints->node_list);
680 hnode->parent = parent_hnode;
681 objagg_hints->node_count++;
682
683 return hnode;
684
685 err_ht_insert:
686 kfree(hnode);
687 return ERR_PTR(err);
688 }
689
690 static void objagg_hints_flush(struct objagg_hints *objagg_hints)
691 {
692 struct objagg_hints_node *hnode, *tmp;
693
694 list_for_each_entry_safe(hnode, tmp, &objagg_hints->node_list, list) {
695 list_del(&hnode->list);
696 rhashtable_remove_fast(&objagg_hints->node_ht, &hnode->ht_node,
697 objagg_hints->ht_params);
698 kfree(hnode);
699 }
700 }
701
702 struct objagg_tmp_node {
703 struct objagg_obj *objagg_obj;
704 bool crossed_out;
705 };
706
707 struct objagg_tmp_graph {
708 struct objagg_tmp_node *nodes;
709 unsigned long nodes_count;
710 unsigned long *edges;
711 };
712
713 static int objagg_tmp_graph_edge_index(struct objagg_tmp_graph *graph,
714 int parent_index, int index)
715 {
716 return index * graph->nodes_count + parent_index;
717 }
718
719 static void objagg_tmp_graph_edge_set(struct objagg_tmp_graph *graph,
720 int parent_index, int index)
721 {
722 int edge_index = objagg_tmp_graph_edge_index(graph, index,
723 parent_index);
724
725 __set_bit(edge_index, graph->edges);
726 }
727
728 static bool objagg_tmp_graph_is_edge(struct objagg_tmp_graph *graph,
729 int parent_index, int index)
730 {
731 int edge_index = objagg_tmp_graph_edge_index(graph, index,
732 parent_index);
733
734 return test_bit(edge_index, graph->edges);
735 }
736
737 static unsigned int objagg_tmp_graph_node_weight(struct objagg_tmp_graph *graph,
738 unsigned int index)
739 {
740 struct objagg_tmp_node *node = &graph->nodes[index];
741 unsigned int weight = node->objagg_obj->stats.user_count;
742 int j;
743
744 /* Node weight is sum of node users and all other nodes users
745 * that this node can represent with delta.
746 */
747
748 for (j = 0; j < graph->nodes_count; j++) {
749 if (!objagg_tmp_graph_is_edge(graph, index, j))
750 continue;
751 node = &graph->nodes[j];
752 if (node->crossed_out)
753 continue;
754 weight += node->objagg_obj->stats.user_count;
755 }
756 return weight;
757 }
758
759 static int objagg_tmp_graph_node_max_weight(struct objagg_tmp_graph *graph)
760 {
761 struct objagg_tmp_node *node;
762 unsigned int max_weight = 0;
763 unsigned int weight;
764 int max_index = -1;
765 int i;
766
767 for (i = 0; i < graph->nodes_count; i++) {
768 node = &graph->nodes[i];
769 if (node->crossed_out)
770 continue;
771 weight = objagg_tmp_graph_node_weight(graph, i);
772 if (weight >= max_weight) {
773 max_weight = weight;
774 max_index = i;
775 }
776 }
777 return max_index;
778 }
779
780 static struct objagg_tmp_graph *objagg_tmp_graph_create(struct objagg *objagg)
781 {
782 unsigned int nodes_count = objagg->obj_count;
783 struct objagg_tmp_graph *graph;
784 struct objagg_tmp_node *node;
785 struct objagg_tmp_node *pnode;
786 struct objagg_obj *objagg_obj;
787 int i, j;
788
789 graph = kzalloc(sizeof(*graph), GFP_KERNEL);
790 if (!graph)
791 return NULL;
792
793 graph->nodes = kcalloc(nodes_count, sizeof(*graph->nodes), GFP_KERNEL);
794 if (!graph->nodes)
795 goto err_nodes_alloc;
796 graph->nodes_count = nodes_count;
797
798 graph->edges = bitmap_zalloc(nodes_count * nodes_count, GFP_KERNEL);
799 if (!graph->edges)
800 goto err_edges_alloc;
801
802 i = 0;
803 list_for_each_entry(objagg_obj, &objagg->obj_list, list) {
804 node = &graph->nodes[i++];
805 node->objagg_obj = objagg_obj;
806 }
807
808 /* Assemble a temporary graph. Insert edge X->Y in case Y can be
809 * in delta of X.
810 */
811 for (i = 0; i < nodes_count; i++) {
812 for (j = 0; j < nodes_count; j++) {
813 if (i == j)
814 continue;
815 pnode = &graph->nodes[i];
816 node = &graph->nodes[j];
817 if (objagg->ops->delta_check(objagg->priv,
818 pnode->objagg_obj->obj,
819 node->objagg_obj->obj)) {
820 objagg_tmp_graph_edge_set(graph, i, j);
821
822 }
823 }
824 }
825 return graph;
826
827 err_edges_alloc:
828 kfree(graph->nodes);
829 err_nodes_alloc:
830 kfree(graph);
831 return NULL;
832 }
833
834 static void objagg_tmp_graph_destroy(struct objagg_tmp_graph *graph)
835 {
836 bitmap_free(graph->edges);
837 kfree(graph->nodes);
838 kfree(graph);
839 }
840
841 static int
842 objagg_opt_simple_greedy_fillup_hints(struct objagg_hints *objagg_hints,
843 struct objagg *objagg)
844 {
845 struct objagg_hints_node *hnode, *parent_hnode;
846 struct objagg_tmp_graph *graph;
847 struct objagg_tmp_node *node;
848 int index;
849 int j;
850 int err;
851
852 graph = objagg_tmp_graph_create(objagg);
853 if (!graph)
854 return -ENOMEM;
855
856 /* Find the nodes from the ones that can accommodate most users
857 * and cross them out of the graph. Save them to the hint list.
858 */
859 while ((index = objagg_tmp_graph_node_max_weight(graph)) != -1) {
860 node = &graph->nodes[index];
861 node->crossed_out = true;
862 hnode = objagg_hints_node_create(objagg_hints,
863 node->objagg_obj,
864 objagg->ops->obj_size,
865 NULL);
866 if (IS_ERR(hnode)) {
867 err = PTR_ERR(hnode);
868 goto out;
869 }
870 parent_hnode = hnode;
871 for (j = 0; j < graph->nodes_count; j++) {
872 if (!objagg_tmp_graph_is_edge(graph, index, j))
873 continue;
874 node = &graph->nodes[j];
875 if (node->crossed_out)
876 continue;
877 node->crossed_out = true;
878 hnode = objagg_hints_node_create(objagg_hints,
879 node->objagg_obj,
880 objagg->ops->obj_size,
881 parent_hnode);
882 if (IS_ERR(hnode)) {
883 err = PTR_ERR(hnode);
884 goto out;
885 }
886 }
887 }
888
889 err = 0;
890 out:
891 objagg_tmp_graph_destroy(graph);
892 return err;
893 }
894
895 struct objagg_opt_algo {
896 int (*fillup_hints)(struct objagg_hints *objagg_hints,
897 struct objagg *objagg);
898 };
899
900 static const struct objagg_opt_algo objagg_opt_simple_greedy = {
901 .fillup_hints = objagg_opt_simple_greedy_fillup_hints,
902 };
903
904
905 static const struct objagg_opt_algo *objagg_opt_algos[] = {
906 [OBJAGG_OPT_ALGO_SIMPLE_GREEDY] = &objagg_opt_simple_greedy,
907 };
908
909 /**
910 * objagg_hints_get - obtains hints instance
911 * @objagg: objagg instance
912 * @opt_algo_type: type of hints finding algorithm
913 *
914 * Note: all locking must be provided by the caller.
915 *
916 * According to the algo type, the existing objects of objagg instance
917 * are going to be went-through to assemble an optimal tree. We call this
918 * tree hints. These hints can be later on used for creation of
919 * a new objagg instance. There, the future object creations are going
920 * to be consulted with these hints in order to find out, where exactly
921 * the new object should be put as a root or delta.
922 *
923 * Returns a pointer to hints instance in case of success,
924 * otherwise it returns pointer error using ERR_PTR macro.
925 */
926 struct objagg_hints *objagg_hints_get(struct objagg *objagg,
927 enum objagg_opt_algo_type opt_algo_type)
928 {
929 const struct objagg_opt_algo *algo = objagg_opt_algos[opt_algo_type];
930 struct objagg_hints *objagg_hints;
931 int err;
932
933 objagg_hints = kzalloc(sizeof(*objagg_hints), GFP_KERNEL);
934 if (!objagg_hints)
935 return ERR_PTR(-ENOMEM);
936
937 objagg_hints->ops = objagg->ops;
938 objagg_hints->refcount = 1;
939
940 INIT_LIST_HEAD(&objagg_hints->node_list);
941
942 objagg_hints->ht_params.key_len = objagg->ops->obj_size;
943 objagg_hints->ht_params.key_offset =
944 offsetof(struct objagg_hints_node, obj);
945 objagg_hints->ht_params.head_offset =
946 offsetof(struct objagg_hints_node, ht_node);
947
948 err = rhashtable_init(&objagg_hints->node_ht, &objagg_hints->ht_params);
949 if (err)
950 goto err_rhashtable_init;
951
952 err = algo->fillup_hints(objagg_hints, objagg);
953 if (err)
954 goto err_fillup_hints;
955
956 if (WARN_ON(objagg_hints->node_count != objagg->obj_count)) {
957 err = -EINVAL;
958 goto err_node_count_check;
959 }
960
961 return objagg_hints;
962
963 err_node_count_check:
964 err_fillup_hints:
965 objagg_hints_flush(objagg_hints);
966 rhashtable_destroy(&objagg_hints->node_ht);
967 err_rhashtable_init:
968 kfree(objagg_hints);
969 return ERR_PTR(err);
970 }
971 EXPORT_SYMBOL(objagg_hints_get);
972
973 /**
974 * objagg_hints_put - puts hints instance
975 * @objagg_hints: objagg hints instance
976 *
977 * Note: all locking must be provided by the caller.
978 */
979 void objagg_hints_put(struct objagg_hints *objagg_hints)
980 {
981 if (--objagg_hints->refcount)
982 return;
983 objagg_hints_flush(objagg_hints);
984 rhashtable_destroy(&objagg_hints->node_ht);
985 kfree(objagg_hints);
986 }
987 EXPORT_SYMBOL(objagg_hints_put);
988
989 /**
990 * objagg_hints_stats_get - obtains stats of the hints instance
991 * @objagg_hints: hints instance
992 *
993 * Note: all locking must be provided by the caller.
994 *
995 * The returned structure contains statistics of all objects
996 * currently in use, ordered by following rules:
997 * 1) Root objects are always on lower indexes than the rest.
998 * 2) Objects with higher delta user count are always on lower
999 * indexes.
1000 * 3) In case multiple objects have the same delta user count,
1001 * the objects are ordered by user count.
1002 *
1003 * Returns a pointer to stats instance in case of success,
1004 * otherwise it returns pointer error using ERR_PTR macro.
1005 */
1006 const struct objagg_stats *
1007 objagg_hints_stats_get(struct objagg_hints *objagg_hints)
1008 {
1009 struct objagg_stats *objagg_stats;
1010 struct objagg_hints_node *hnode;
1011 int i;
1012
1013 objagg_stats = kzalloc(struct_size(objagg_stats, stats_info,
1014 objagg_hints->node_count),
1015 GFP_KERNEL);
1016 if (!objagg_stats)
1017 return ERR_PTR(-ENOMEM);
1018
1019 i = 0;
1020 list_for_each_entry(hnode, &objagg_hints->node_list, list) {
1021 memcpy(&objagg_stats->stats_info[i], &hnode->stats_info,
1022 sizeof(objagg_stats->stats_info[0]));
1023 if (objagg_stats->stats_info[i].is_root)
1024 objagg_stats->root_count++;
1025 i++;
1026 }
1027 objagg_stats->stats_info_count = i;
1028
1029 sort(objagg_stats->stats_info, objagg_stats->stats_info_count,
1030 sizeof(struct objagg_obj_stats_info),
1031 objagg_stats_info_sort_cmp_func, NULL);
1032
1033 return objagg_stats;
1034 }
1035 EXPORT_SYMBOL(objagg_hints_stats_get);
1036
1037 MODULE_LICENSE("Dual BSD/GPL");
1038 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
1039 MODULE_DESCRIPTION("Object aggregation manager");
1040
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.