bpf: Repurpose use_trace_rcu to reuse_now in bpf_local_storage
This patch re-purpose the use_trace_rcu to mean if the freed memory can be reused immediately or not. The use_trace_rcu is renamed to reuse_now. Other than the boolean test is reversed, it should be a no-op. The following explains the reason for the rename and how it will be used in a later patch. In a later patch, bpf_mem_cache_alloc/free will be used in the bpf_local_storage. The bpf mem allocator will reuse the freed memory immediately. Some of the free paths in bpf_local_storage does not support memory to be reused immediately. These paths are the "delete" elem cases from the bpf_*_storage_delete() helper and the map_delete_elem() syscall. Note that "delete" elem before the owner's (sk/task/cgrp/inode) lifetime ended is not the common usage for the local storage. The common free path, bpf_local_storage_destroy(), can reuse the memory immediately. This common path means the storage stays with its owner until the owner is destroyed. The above mentioned "delete" elem paths that cannot reuse immediately always has the 'use_trace_rcu == true'. The cases that is safe for immediate reuse always have 'use_trace_rcu == false'. Instead of adding another arg in a later patch, this patch re-purpose this arg to reuse_now and have the test logic reversed. In a later patch, 'reuse_now == true' will free to the bpf_mem_cache_free() where the memory can be reused immediately. 'reuse_now == false' will go through the call_rcu_tasks_trace(). Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org> Link: https://lore.kernel.org/r/20230308065936.1550103-7-martin.lau@linux.dev Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Родитель
fc6652aab6
Коммит
a47eabf216
|
@ -143,7 +143,7 @@ int bpf_local_storage_map_check_btf(const struct bpf_map *map,
|
|||
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
|
||||
struct bpf_local_storage_elem *selem);
|
||||
|
||||
void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu);
|
||||
void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now);
|
||||
|
||||
void bpf_selem_link_map(struct bpf_local_storage_map *smap,
|
||||
struct bpf_local_storage_elem *selem);
|
||||
|
|
|
@ -121,7 +121,7 @@ static int cgroup_storage_delete(struct cgroup *cgroup, struct bpf_map *map)
|
|||
if (!sdata)
|
||||
return -ENOENT;
|
||||
|
||||
bpf_selem_unlink(SELEM(sdata), true);
|
||||
bpf_selem_unlink(SELEM(sdata), false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
@ -122,7 +122,7 @@ static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
|
|||
if (!sdata)
|
||||
return -ENOENT;
|
||||
|
||||
bpf_selem_unlink(SELEM(sdata), true);
|
||||
bpf_selem_unlink(SELEM(sdata), false);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -147,7 +147,7 @@ static void bpf_selem_free_trace_rcu(struct rcu_head *rcu)
|
|||
*/
|
||||
static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
|
||||
struct bpf_local_storage_elem *selem,
|
||||
bool uncharge_mem, bool use_trace_rcu)
|
||||
bool uncharge_mem, bool reuse_now)
|
||||
{
|
||||
struct bpf_local_storage_map *smap;
|
||||
bool free_local_storage;
|
||||
|
@ -201,7 +201,7 @@ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_stor
|
|||
* any special fields.
|
||||
*/
|
||||
rec = smap->map.record;
|
||||
if (use_trace_rcu) {
|
||||
if (!reuse_now) {
|
||||
if (!IS_ERR_OR_NULL(rec))
|
||||
call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_fields_trace_rcu);
|
||||
else
|
||||
|
@ -220,7 +220,7 @@ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_stor
|
|||
}
|
||||
|
||||
static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem,
|
||||
bool use_trace_rcu)
|
||||
bool reuse_now)
|
||||
{
|
||||
struct bpf_local_storage *local_storage;
|
||||
bool free_local_storage = false;
|
||||
|
@ -235,11 +235,11 @@ static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem,
|
|||
raw_spin_lock_irqsave(&local_storage->lock, flags);
|
||||
if (likely(selem_linked_to_storage(selem)))
|
||||
free_local_storage = bpf_selem_unlink_storage_nolock(
|
||||
local_storage, selem, true, use_trace_rcu);
|
||||
local_storage, selem, true, reuse_now);
|
||||
raw_spin_unlock_irqrestore(&local_storage->lock, flags);
|
||||
|
||||
if (free_local_storage) {
|
||||
if (use_trace_rcu)
|
||||
if (!reuse_now)
|
||||
call_rcu_tasks_trace(&local_storage->rcu,
|
||||
bpf_local_storage_free_rcu);
|
||||
else
|
||||
|
@ -284,14 +284,14 @@ void bpf_selem_link_map(struct bpf_local_storage_map *smap,
|
|||
raw_spin_unlock_irqrestore(&b->lock, flags);
|
||||
}
|
||||
|
||||
void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu)
|
||||
void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now)
|
||||
{
|
||||
/* Always unlink from map before unlinking from local_storage
|
||||
* because selem will be freed after successfully unlinked from
|
||||
* the local_storage.
|
||||
*/
|
||||
bpf_selem_unlink_map(selem);
|
||||
bpf_selem_unlink_storage(selem, use_trace_rcu);
|
||||
bpf_selem_unlink_storage(selem, reuse_now);
|
||||
}
|
||||
|
||||
/* If cacheit_lockit is false, this lookup function is lockless */
|
||||
|
@ -538,7 +538,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
|
|||
if (old_sdata) {
|
||||
bpf_selem_unlink_map(SELEM(old_sdata));
|
||||
bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),
|
||||
false, true);
|
||||
false, false);
|
||||
}
|
||||
|
||||
unlock:
|
||||
|
@ -651,7 +651,7 @@ void bpf_local_storage_destroy(struct bpf_local_storage *local_storage)
|
|||
* of the loop will set the free_cgroup_storage to true.
|
||||
*/
|
||||
free_storage = bpf_selem_unlink_storage_nolock(
|
||||
local_storage, selem, false, false);
|
||||
local_storage, selem, false, true);
|
||||
}
|
||||
raw_spin_unlock_irqrestore(&local_storage->lock, flags);
|
||||
|
||||
|
@ -745,7 +745,7 @@ void bpf_local_storage_map_free(struct bpf_map *map,
|
|||
migrate_disable();
|
||||
this_cpu_inc(*busy_counter);
|
||||
}
|
||||
bpf_selem_unlink(selem, false);
|
||||
bpf_selem_unlink(selem, true);
|
||||
if (busy_counter) {
|
||||
this_cpu_dec(*busy_counter);
|
||||
migrate_enable();
|
||||
|
@ -783,8 +783,8 @@ void bpf_local_storage_map_free(struct bpf_map *map,
|
|||
/* We cannot skip rcu_barrier() when rcu_trace_implies_rcu_gp()
|
||||
* is true, because while call_rcu invocation is skipped in that
|
||||
* case in bpf_selem_free_fields_trace_rcu (and all local
|
||||
* storage maps pass use_trace_rcu = true), there can be
|
||||
* call_rcu callbacks based on use_trace_rcu = false in the
|
||||
* storage maps pass reuse_now = false), there can be
|
||||
* call_rcu callbacks based on reuse_now = true in the
|
||||
* while ((selem = ...)) loop above or when owner's free path
|
||||
* calls bpf_local_storage_unlink_nolock.
|
||||
*/
|
||||
|
|
|
@ -168,7 +168,7 @@ static int task_storage_delete(struct task_struct *task, struct bpf_map *map,
|
|||
if (!nobusy)
|
||||
return -EBUSY;
|
||||
|
||||
bpf_selem_unlink(SELEM(sdata), true);
|
||||
bpf_selem_unlink(SELEM(sdata), false);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -40,7 +40,7 @@ static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
|
|||
if (!sdata)
|
||||
return -ENOENT;
|
||||
|
||||
bpf_selem_unlink(SELEM(sdata), true);
|
||||
bpf_selem_unlink(SELEM(sdata), false);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
Загрузка…
Ссылка в новой задаче