sbitmap: add __sbitmap_queue_get_batch()
The block layer tag allocation batching still calls into sbitmap to get each tag, but we can improve on that. Add __sbitmap_queue_get_batch(), which returns a mask of tags all at once, along with an offset for those tags. An example return would be 0xff, where bits 0..7 are set, with tag_offset == 128. The valid tags in this case would be 128..135. A batch is specific to an individual sbitmap_map, hence it cannot be larger than that. The requested number of tags is automatically reduced to the max that can be satisfied with a single map. On failure, 0 is returned. Caller should fall back to single tag allocation at that point/ Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Родитель
8971a3b7f1
Коммит
9672b0d437
|
@ -426,6 +426,19 @@ void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth);
|
||||||
*/
|
*/
|
||||||
int __sbitmap_queue_get(struct sbitmap_queue *sbq);
|
int __sbitmap_queue_get(struct sbitmap_queue *sbq);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* __sbitmap_queue_get_batch() - Try to allocate a batch of free bits
|
||||||
|
* @sbq: Bitmap queue to allocate from.
|
||||||
|
* @nr_tags: number of tags requested
|
||||||
|
* @offset: offset to add to returned bits
|
||||||
|
*
|
||||||
|
* Return: Mask of allocated tags, 0 if none are found. Each tag allocated is
|
||||||
|
* a bit in the mask returned, and the caller must add @offset to the value to
|
||||||
|
* get the absolute tag value.
|
||||||
|
*/
|
||||||
|
unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,
|
||||||
|
unsigned int *offset);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* __sbitmap_queue_get_shallow() - Try to allocate a free bit from a &struct
|
* __sbitmap_queue_get_shallow() - Try to allocate a free bit from a &struct
|
||||||
* sbitmap_queue, limiting the depth used from each word, with preemption
|
* sbitmap_queue, limiting the depth used from each word, with preemption
|
||||||
|
|
|
@ -489,6 +489,57 @@ int __sbitmap_queue_get(struct sbitmap_queue *sbq)
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(__sbitmap_queue_get);
|
EXPORT_SYMBOL_GPL(__sbitmap_queue_get);
|
||||||
|
|
||||||
|
unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,
|
||||||
|
unsigned int *offset)
|
||||||
|
{
|
||||||
|
struct sbitmap *sb = &sbq->sb;
|
||||||
|
unsigned int hint, depth;
|
||||||
|
unsigned long index, nr;
|
||||||
|
int i;
|
||||||
|
|
||||||
|
if (unlikely(sb->round_robin))
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
depth = READ_ONCE(sb->depth);
|
||||||
|
hint = update_alloc_hint_before_get(sb, depth);
|
||||||
|
|
||||||
|
index = SB_NR_TO_INDEX(sb, hint);
|
||||||
|
|
||||||
|
for (i = 0; i < sb->map_nr; i++) {
|
||||||
|
struct sbitmap_word *map = &sb->map[index];
|
||||||
|
unsigned long get_mask;
|
||||||
|
|
||||||
|
sbitmap_deferred_clear(map);
|
||||||
|
if (map->word == (1UL << (map->depth - 1)) - 1)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
nr = find_first_zero_bit(&map->word, map->depth);
|
||||||
|
if (nr + nr_tags <= map->depth) {
|
||||||
|
atomic_long_t *ptr = (atomic_long_t *) &map->word;
|
||||||
|
int map_tags = min_t(int, nr_tags, map->depth);
|
||||||
|
unsigned long val, ret;
|
||||||
|
|
||||||
|
get_mask = ((1UL << map_tags) - 1) << nr;
|
||||||
|
do {
|
||||||
|
val = READ_ONCE(map->word);
|
||||||
|
ret = atomic_long_cmpxchg(ptr, val, get_mask | val);
|
||||||
|
} while (ret != val);
|
||||||
|
get_mask = (get_mask & ~ret) >> nr;
|
||||||
|
if (get_mask) {
|
||||||
|
*offset = nr + (index << sb->shift);
|
||||||
|
update_alloc_hint_after_get(sb, depth, hint,
|
||||||
|
*offset + map_tags - 1);
|
||||||
|
return get_mask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Jump to next index. */
|
||||||
|
if (++index >= sb->map_nr)
|
||||||
|
index = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
|
int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
|
||||||
unsigned int shallow_depth)
|
unsigned int shallow_depth)
|
||||||
{
|
{
|
||||||
|
|
Загрузка…
Ссылка в новой задаче