lib/rbtree: avoid generating code twice for the cached versions
As was already noted in rbtree.h, the logic to cache rb_first (or rb_last) can easily be implemented externally to the core rbtree api. Change the implementation to do just that. Previously the update of rb_leftmost was wired deeper into the implmentation, but there were some disadvantages to that - mostly, lib/rbtree.c had separate instantiations for rb_insert_color() vs rb_insert_color_cached(), as well as rb_erase() vs rb_erase_cached(), which were doing exactly the same thing save for the rb_leftmost update at the start of either function. text data bss dec hex filename 5405 120 0 5525 1595 lib/rbtree.o-vanilla 3827 96 0 3923 f53 lib/rbtree.o-patch [dave@stgolabs.net: changelog addition] Link: http://lkml.kernel.org/r/20190628171416.by5gdizl3rcxk5h5@linux-r8p5 [akpm@linux-foundation.org: coding-style fixes] Link: http://lkml.kernel.org/r/20190628045008.39926-1-walken@google.com Signed-off-by: Michel Lespinasse <walken@google.com> Acked-by: Davidlohr Bueso <dbueso@suse.de> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Michel Lespinasse
Linus Torvalds
Родитель
4ab7ace465
Коммит
9f973cb380
|
|
@ -32,25 +32,9 @@ struct rb_root {
|
|||
struct rb_node *rb_node;
|
||||
};
|
||||
|
||||
/*
|
||||
* Leftmost-cached rbtrees.
|
||||
*
|
||||
* We do not cache the rightmost node based on footprint
|
||||
* size vs number of potential users that could benefit
|
||||
* from O(1) rb_last(). Just not worth it, users that want
|
||||
* this feature can always implement the logic explicitly.
|
||||
* Furthermore, users that want to cache both pointers may
|
||||
* find it a bit asymmetric, but that's ok.
|
||||
*/
|
||||
struct rb_root_cached {
|
||||
struct rb_root rb_root;
|
||||
struct rb_node *rb_leftmost;
|
||||
};
|
||||
|
||||
#define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3))
|
||||
|
||||
#define RB_ROOT (struct rb_root) { NULL, }
|
||||
#define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL }
|
||||
#define rb_entry(ptr, type, member) container_of(ptr, type, member)
|
||||
|
||||
#define RB_EMPTY_ROOT(root) (READ_ONCE((root)->rb_node) == NULL)
|
||||
|
|
@ -72,12 +56,6 @@ extern struct rb_node *rb_prev(const struct rb_node *);
|
|||
extern struct rb_node *rb_first(const struct rb_root *);
|
||||
extern struct rb_node *rb_last(const struct rb_root *);
|
||||
|
||||
extern void rb_insert_color_cached(struct rb_node *,
|
||||
struct rb_root_cached *, bool);
|
||||
extern void rb_erase_cached(struct rb_node *node, struct rb_root_cached *);
|
||||
/* Same as rb_first(), but O(1) */
|
||||
#define rb_first_cached(root) (root)->rb_leftmost
|
||||
|
||||
/* Postorder iteration - always visit the parent after its children */
|
||||
extern struct rb_node *rb_first_postorder(const struct rb_root *);
|
||||
extern struct rb_node *rb_next_postorder(const struct rb_node *);
|
||||
|
|
@ -87,8 +65,6 @@ extern void rb_replace_node(struct rb_node *victim, struct rb_node *new,
|
|||
struct rb_root *root);
|
||||
extern void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new,
|
||||
struct rb_root *root);
|
||||
extern void rb_replace_node_cached(struct rb_node *victim, struct rb_node *new,
|
||||
struct rb_root_cached *root);
|
||||
|
||||
static inline void rb_link_node(struct rb_node *node, struct rb_node *parent,
|
||||
struct rb_node **rb_link)
|
||||
|
|
@ -136,4 +112,50 @@ static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent
|
|||
typeof(*pos), field); 1; }); \
|
||||
pos = n)
|
||||
|
||||
/*
|
||||
* Leftmost-cached rbtrees.
|
||||
*
|
||||
* We do not cache the rightmost node based on footprint
|
||||
* size vs number of potential users that could benefit
|
||||
* from O(1) rb_last(). Just not worth it, users that want
|
||||
* this feature can always implement the logic explicitly.
|
||||
* Furthermore, users that want to cache both pointers may
|
||||
* find it a bit asymmetric, but that's ok.
|
||||
*/
|
||||
struct rb_root_cached {
|
||||
struct rb_root rb_root;
|
||||
struct rb_node *rb_leftmost;
|
||||
};
|
||||
|
||||
#define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL }
|
||||
|
||||
/* Same as rb_first(), but O(1) */
|
||||
#define rb_first_cached(root) (root)->rb_leftmost
|
||||
|
||||
static inline void rb_insert_color_cached(struct rb_node *node,
|
||||
struct rb_root_cached *root,
|
||||
bool leftmost)
|
||||
{
|
||||
if (leftmost)
|
||||
root->rb_leftmost = node;
|
||||
rb_insert_color(node, &root->rb_root);
|
||||
}
|
||||
|
||||
static inline void rb_erase_cached(struct rb_node *node,
|
||||
struct rb_root_cached *root)
|
||||
{
|
||||
if (root->rb_leftmost == node)
|
||||
root->rb_leftmost = rb_next(node);
|
||||
rb_erase(node, &root->rb_root);
|
||||
}
|
||||
|
||||
static inline void rb_replace_node_cached(struct rb_node *victim,
|
||||
struct rb_node *new,
|
||||
struct rb_root_cached *root)
|
||||
{
|
||||
if (root->rb_leftmost == victim)
|
||||
root->rb_leftmost = new;
|
||||
rb_replace_node(victim, new, &root->rb_root);
|
||||
}
|
||||
|
||||
#endif /* _LINUX_RBTREE_H */
|
||||
|
|
|
|||
|
|
@ -30,10 +30,9 @@ struct rb_augment_callbacks {
|
|||
void (*rotate)(struct rb_node *old, struct rb_node *new);
|
||||
};
|
||||
|
||||
extern void __rb_insert_augmented(struct rb_node *node,
|
||||
struct rb_root *root,
|
||||
bool newleft, struct rb_node **leftmost,
|
||||
extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
|
||||
void (*augment_rotate)(struct rb_node *old, struct rb_node *new));
|
||||
|
||||
/*
|
||||
* Fixup the rbtree and update the augmented information when rebalancing.
|
||||
*
|
||||
|
|
@ -48,7 +47,7 @@ static inline void
|
|||
rb_insert_augmented(struct rb_node *node, struct rb_root *root,
|
||||
const struct rb_augment_callbacks *augment)
|
||||
{
|
||||
__rb_insert_augmented(node, root, false, NULL, augment->rotate);
|
||||
__rb_insert_augmented(node, root, augment->rotate);
|
||||
}
|
||||
|
||||
static inline void
|
||||
|
|
@ -56,8 +55,9 @@ rb_insert_augmented_cached(struct rb_node *node,
|
|||
struct rb_root_cached *root, bool newleft,
|
||||
const struct rb_augment_callbacks *augment)
|
||||
{
|
||||
__rb_insert_augmented(node, &root->rb_root,
|
||||
newleft, &root->rb_leftmost, augment->rotate);
|
||||
if (newleft)
|
||||
root->rb_leftmost = node;
|
||||
rb_insert_augmented(node, &root->rb_root, augment);
|
||||
}
|
||||
|
||||
#define RB_DECLARE_CALLBACKS(rbstatic, rbname, rbstruct, rbfield, \
|
||||
|
|
@ -150,7 +150,6 @@ extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root,
|
|||
|
||||
static __always_inline struct rb_node *
|
||||
__rb_erase_augmented(struct rb_node *node, struct rb_root *root,
|
||||
struct rb_node **leftmost,
|
||||
const struct rb_augment_callbacks *augment)
|
||||
{
|
||||
struct rb_node *child = node->rb_right;
|
||||
|
|
@ -158,9 +157,6 @@ __rb_erase_augmented(struct rb_node *node, struct rb_root *root,
|
|||
struct rb_node *parent, *rebalance;
|
||||
unsigned long pc;
|
||||
|
||||
if (leftmost && node == *leftmost)
|
||||
*leftmost = rb_next(node);
|
||||
|
||||
if (!tmp) {
|
||||
/*
|
||||
* Case 1: node to erase has no more than 1 child (easy!)
|
||||
|
|
@ -260,8 +256,7 @@ static __always_inline void
|
|||
rb_erase_augmented(struct rb_node *node, struct rb_root *root,
|
||||
const struct rb_augment_callbacks *augment)
|
||||
{
|
||||
struct rb_node *rebalance = __rb_erase_augmented(node, root,
|
||||
NULL, augment);
|
||||
struct rb_node *rebalance = __rb_erase_augmented(node, root, augment);
|
||||
if (rebalance)
|
||||
__rb_erase_color(rebalance, root, augment->rotate);
|
||||
}
|
||||
|
|
@ -270,11 +265,9 @@ static __always_inline void
|
|||
rb_erase_augmented_cached(struct rb_node *node, struct rb_root_cached *root,
|
||||
const struct rb_augment_callbacks *augment)
|
||||
{
|
||||
struct rb_node *rebalance = __rb_erase_augmented(node, &root->rb_root,
|
||||
&root->rb_leftmost,
|
||||
augment);
|
||||
if (rebalance)
|
||||
__rb_erase_color(rebalance, &root->rb_root, augment->rotate);
|
||||
if (root->rb_leftmost == node)
|
||||
root->rb_leftmost = rb_next(node);
|
||||
rb_erase_augmented(node, &root->rb_root, augment);
|
||||
}
|
||||
|
||||
#endif /* _LINUX_RBTREE_AUGMENTED_H */
|
||||
|
|
|
|||
40
lib/rbtree.c
40
lib/rbtree.c
|
|
@ -83,14 +83,10 @@ __rb_rotate_set_parents(struct rb_node *old, struct rb_node *new,
|
|||
|
||||
static __always_inline void
|
||||
__rb_insert(struct rb_node *node, struct rb_root *root,
|
||||
bool newleft, struct rb_node **leftmost,
|
||||
void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
|
||||
{
|
||||
struct rb_node *parent = rb_red_parent(node), *gparent, *tmp;
|
||||
|
||||
if (newleft)
|
||||
*leftmost = node;
|
||||
|
||||
while (true) {
|
||||
/*
|
||||
* Loop invariant: node is red.
|
||||
|
|
@ -437,38 +433,19 @@ static const struct rb_augment_callbacks dummy_callbacks = {
|
|||
|
||||
void rb_insert_color(struct rb_node *node, struct rb_root *root)
|
||||
{
|
||||
__rb_insert(node, root, false, NULL, dummy_rotate);
|
||||
__rb_insert(node, root, dummy_rotate);
|
||||
}
|
||||
EXPORT_SYMBOL(rb_insert_color);
|
||||
|
||||
void rb_erase(struct rb_node *node, struct rb_root *root)
|
||||
{
|
||||
struct rb_node *rebalance;
|
||||
rebalance = __rb_erase_augmented(node, root,
|
||||
NULL, &dummy_callbacks);
|
||||
rebalance = __rb_erase_augmented(node, root, &dummy_callbacks);
|
||||
if (rebalance)
|
||||
____rb_erase_color(rebalance, root, dummy_rotate);
|
||||
}
|
||||
EXPORT_SYMBOL(rb_erase);
|
||||
|
||||
void rb_insert_color_cached(struct rb_node *node,
|
||||
struct rb_root_cached *root, bool leftmost)
|
||||
{
|
||||
__rb_insert(node, &root->rb_root, leftmost,
|
||||
&root->rb_leftmost, dummy_rotate);
|
||||
}
|
||||
EXPORT_SYMBOL(rb_insert_color_cached);
|
||||
|
||||
void rb_erase_cached(struct rb_node *node, struct rb_root_cached *root)
|
||||
{
|
||||
struct rb_node *rebalance;
|
||||
rebalance = __rb_erase_augmented(node, &root->rb_root,
|
||||
&root->rb_leftmost, &dummy_callbacks);
|
||||
if (rebalance)
|
||||
____rb_erase_color(rebalance, &root->rb_root, dummy_rotate);
|
||||
}
|
||||
EXPORT_SYMBOL(rb_erase_cached);
|
||||
|
||||
/*
|
||||
* Augmented rbtree manipulation functions.
|
||||
*
|
||||
|
|
@ -477,10 +454,9 @@ EXPORT_SYMBOL(rb_erase_cached);
|
|||
*/
|
||||
|
||||
void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
|
||||
bool newleft, struct rb_node **leftmost,
|
||||
void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
|
||||
{
|
||||
__rb_insert(node, root, newleft, leftmost, augment_rotate);
|
||||
__rb_insert(node, root, augment_rotate);
|
||||
}
|
||||
EXPORT_SYMBOL(__rb_insert_augmented);
|
||||
|
||||
|
|
@ -591,16 +567,6 @@ void rb_replace_node(struct rb_node *victim, struct rb_node *new,
|
|||
}
|
||||
EXPORT_SYMBOL(rb_replace_node);
|
||||
|
||||
void rb_replace_node_cached(struct rb_node *victim, struct rb_node *new,
|
||||
struct rb_root_cached *root)
|
||||
{
|
||||
rb_replace_node(victim, new, &root->rb_root);
|
||||
|
||||
if (root->rb_leftmost == victim)
|
||||
root->rb_leftmost = new;
|
||||
}
|
||||
EXPORT_SYMBOL(rb_replace_node_cached);
|
||||
|
||||
void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new,
|
||||
struct rb_root *root)
|
||||
{
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче