diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index cd4a1ab0ea78..06f511fcbd8e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -47,7 +47,7 @@ mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en_tc.o en/rep/tc.o en/rep/neigh.o \ en/tc_tun_vxlan.o en/tc_tun_gre.o en/tc_tun_geneve.o \ en/tc_tun_mplsoudp.o diag/en_tc_tracepoint.o \ en/tc/post_act.o en/tc/int_port.o en/tc/meter.o \ - en/tc/post_meter.o + en/tc/post_meter.o en/tc/act_stats.o mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en/tc/act/act.o en/tc/act/drop.o en/tc/act/trap.o \ en/tc/act/accept.o en/tc/act/mark.o en/tc/act/goto.o \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c index b08339d986d5..3b590cfe33b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c @@ -589,7 +589,7 @@ mlx5e_rep_indr_stats_act(struct mlx5e_rep_priv *rpriv, act = mlx5e_tc_act_get(fl_act->id, ns_type); if (!act || !act->stats_action) - return -EOPNOTSUPP; + return mlx5e_tc_fill_action_stats(priv, fl_act); return act->stats_action(priv, fl_act); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.c new file mode 100644 index 000000000000..f71766dca660 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include +#include +#include "en/tc_priv.h" +#include "act_stats.h" +#include "en/fs.h" + +struct mlx5e_tc_act_stats_handle { + struct rhashtable ht; + spinlock_t ht_lock; /* protects hashtable */ +}; + +struct mlx5e_tc_act_stats { + unsigned long tc_act_cookie; + + struct mlx5_fc *counter; + u64 lastpackets; + u64 lastbytes; + + struct rhash_head hash; + struct rcu_head rcu_head; +}; + +static const struct rhashtable_params act_counters_ht_params = { + .head_offset = offsetof(struct mlx5e_tc_act_stats, hash), + .key_offset = 0, + .key_len = offsetof(struct mlx5e_tc_act_stats, counter), + .automatic_shrinking = true, +}; + +struct mlx5e_tc_act_stats_handle * +mlx5e_tc_act_stats_create(void) +{ + struct mlx5e_tc_act_stats_handle *handle; + int err; + + handle = kvzalloc(sizeof(*handle), GFP_KERNEL); + if (IS_ERR(handle)) + return ERR_PTR(-ENOMEM); + + err = rhashtable_init(&handle->ht, &act_counters_ht_params); + if (err) + goto err; + + spin_lock_init(&handle->ht_lock); + return handle; +err: + kvfree(handle); + return ERR_PTR(err); +} + +void mlx5e_tc_act_stats_free(struct mlx5e_tc_act_stats_handle *handle) +{ + rhashtable_destroy(&handle->ht); + kvfree(handle); +} + +static int +mlx5e_tc_act_stats_add(struct mlx5e_tc_act_stats_handle *handle, + unsigned long act_cookie, + struct mlx5_fc *counter) +{ + struct mlx5e_tc_act_stats *act_stats, *old_act_stats; + struct rhashtable *ht = &handle->ht; + int err = 0; + + act_stats = kvzalloc(sizeof(*act_stats), GFP_KERNEL); + if (!act_stats) + return -ENOMEM; + + act_stats->tc_act_cookie = act_cookie; + act_stats->counter = counter; + + rcu_read_lock(); + old_act_stats = rhashtable_lookup_get_insert_fast(ht, + &act_stats->hash, + act_counters_ht_params); + if (IS_ERR(old_act_stats)) { + err = PTR_ERR(old_act_stats); + goto err_hash_insert; + } else if (old_act_stats) { + err = -EEXIST; + goto err_hash_insert; + } + rcu_read_unlock(); + + return 0; + +err_hash_insert: + rcu_read_unlock(); + kvfree(act_stats); + return err; +} + +void +mlx5e_tc_act_stats_del_flow(struct mlx5e_tc_act_stats_handle *handle, + struct mlx5e_tc_flow *flow) +{ + struct mlx5_flow_attr *attr; + struct mlx5e_tc_act_stats *act_stats; + int i; + + if (!flow_flag_test(flow, USE_ACT_STATS)) + return; + + list_for_each_entry(attr, &flow->attrs, list) { + for (i = 0; i < attr->tc_act_cookies_count; i++) { + struct rhashtable *ht = &handle->ht; + + spin_lock(&handle->ht_lock); + act_stats = rhashtable_lookup_fast(ht, + &attr->tc_act_cookies[i], + act_counters_ht_params); + if (act_stats && + rhashtable_remove_fast(ht, &act_stats->hash, + act_counters_ht_params) == 0) + kvfree_rcu(act_stats, rcu_head); + + spin_unlock(&handle->ht_lock); + } + } +} + +int +mlx5e_tc_act_stats_add_flow(struct mlx5e_tc_act_stats_handle *handle, + struct mlx5e_tc_flow *flow) +{ + struct mlx5_fc *curr_counter = NULL; + unsigned long last_cookie = 0; + struct mlx5_flow_attr *attr; + int err; + int i; + + if (!flow_flag_test(flow, USE_ACT_STATS)) + return 0; + + list_for_each_entry(attr, &flow->attrs, list) { + if (attr->counter) + curr_counter = attr->counter; + + for (i = 0; i < attr->tc_act_cookies_count; i++) { + /* jump over identical ids (e.g. pedit)*/ + if (last_cookie == attr->tc_act_cookies[i]) + continue; + + err = mlx5e_tc_act_stats_add(handle, attr->tc_act_cookies[i], curr_counter); + if (err) + goto out_err; + last_cookie = attr->tc_act_cookies[i]; + } + } + + return 0; +out_err: + mlx5e_tc_act_stats_del_flow(handle, flow); + return err; +} + +int +mlx5e_tc_act_stats_fill_stats(struct mlx5e_tc_act_stats_handle *handle, + struct flow_offload_action *fl_act) +{ + struct rhashtable *ht = &handle->ht; + struct mlx5e_tc_act_stats *item; + struct mlx5e_tc_act_stats key; + u64 pkts, bytes, lastused; + int err = 0; + + key.tc_act_cookie = fl_act->cookie; + + rcu_read_lock(); + item = rhashtable_lookup(ht, &key, act_counters_ht_params); + if (!item) { + rcu_read_unlock(); + err = -ENOENT; + goto err_out; + } + + mlx5_fc_query_cached_raw(item->counter, + &bytes, &pkts, &lastused); + + flow_stats_update(&fl_act->stats, + bytes - item->lastbytes, + pkts - item->lastpackets, + 0, lastused, FLOW_ACTION_HW_STATS_DELAYED); + + item->lastpackets = pkts; + item->lastbytes = bytes; + rcu_read_unlock(); + + return 0; + +err_out: + return err; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.h new file mode 100644 index 000000000000..002292c2567c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_EN_ACT_STATS_H__ +#define __MLX5_EN_ACT_STATS_H__ + +#include +#include "en/tc_priv.h" + +struct mlx5e_tc_act_stats_handle; + +struct mlx5e_tc_act_stats_handle *mlx5e_tc_act_stats_create(void); +void mlx5e_tc_act_stats_free(struct mlx5e_tc_act_stats_handle *handle); + +int +mlx5e_tc_act_stats_add_flow(struct mlx5e_tc_act_stats_handle *handle, + struct mlx5e_tc_flow *flow); + +void +mlx5e_tc_act_stats_del_flow(struct mlx5e_tc_act_stats_handle *handle, + struct mlx5e_tc_flow *flow); + +int +mlx5e_tc_act_stats_fill_stats(struct mlx5e_tc_act_stats_handle *handle, + struct flow_offload_action *fl_act); + +#endif /* __MLX5_EN_ACT_STATS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h index f575646d2f50..451fd4342a5a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h @@ -30,6 +30,7 @@ enum { MLX5E_TC_FLOW_FLAG_TUN_RX = MLX5E_TC_FLOW_BASE + 9, MLX5E_TC_FLOW_FLAG_FAILED = MLX5E_TC_FLOW_BASE + 10, MLX5E_TC_FLOW_FLAG_SAMPLE = MLX5E_TC_FLOW_BASE + 11, + MLX5E_TC_FLOW_FLAG_USE_ACT_STATS = MLX5E_TC_FLOW_BASE + 12, }; struct mlx5e_tc_flow_parse_attr { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index b4e691760da9..0abe3313c673 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -100,6 +100,9 @@ struct mlx5_rep_uplink_priv { struct mlx5e_tc_int_port_priv *int_port_priv; struct mlx5e_flow_meters *flow_meters; + + /* tc action stats */ + struct mlx5e_tc_act_stats_handle *action_stats_handle; }; struct mlx5e_rep_priv { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index e2ec80ebde58..2d06b4412762 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -45,6 +45,7 @@ #include #include "en.h" #include "en/tc/post_act.h" +#include "en/tc/act_stats.h" #include "en_rep.h" #include "en/rep/tc.h" #include "en/rep/neigh.h" @@ -101,6 +102,9 @@ struct mlx5e_tc_table { struct mapping_ctx *mapping; struct mlx5e_hairpin_params hairpin_params; struct dentry *dfs_root; + + /* tc action stats */ + struct mlx5e_tc_act_stats_handle *action_stats_handle; }; struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { @@ -286,6 +290,24 @@ mlx5e_tc_match_to_reg_set_and_get_id(struct mlx5_core_dev *mdev, return err; } +static struct mlx5e_tc_act_stats_handle * +get_act_stats_handle(struct mlx5e_priv *priv) +{ + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); + struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + struct mlx5_rep_uplink_priv *uplink_priv; + struct mlx5e_rep_priv *uplink_rpriv; + + if (is_mdev_switchdev_mode(priv->mdev)) { + uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); + uplink_priv = &uplink_rpriv->uplink_priv; + + return uplink_priv->action_stats_handle; + } + + return tc->action_stats_handle; +} + struct mlx5e_tc_int_port_priv * mlx5e_get_int_port_priv(struct mlx5e_priv *priv) { @@ -2026,6 +2048,10 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, if (err) goto err_out; + err = mlx5e_tc_act_stats_add_flow(get_act_stats_handle(priv), flow); + if (err) + goto err_out; + /* we get here if one of the following takes place: * (1) there's no error * (2) there's an encap action and we don't have valid neigh @@ -2120,6 +2146,8 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv, if (flow_flag_test(flow, L3_TO_L2_DECAP)) mlx5e_detach_decap(priv, flow); + mlx5e_tc_act_stats_del_flow(get_act_stats_handle(priv), flow); + free_flow_post_acts(flow); free_branch_attr(flow, attr->branch_true); free_branch_attr(flow, attr->branch_false); @@ -3785,6 +3813,8 @@ mlx5e_clone_flow_attr_for_post_act(struct mlx5_flow_attr *attr, INIT_LIST_HEAD(&attr2->list); parse_attr->filter_dev = attr->parse_attr->filter_dev; attr2->action = 0; + attr2->counter = NULL; + attr->tc_act_cookies_count = 0; attr2->flags = 0; attr2->parse_attr = parse_attr; attr2->dest_chain = 0; @@ -4084,6 +4114,11 @@ parse_branch_ctrl(struct flow_action_entry *act, struct mlx5e_tc_act *tc_act, jump_state->jumping_attr = attr->branch_false; jump_state->jump_count = jump_count; + + /* branching action requires its own counter */ + attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; + flow_flag_set(flow, USE_ACT_STATS); + return 0; err_branch_false: @@ -4144,6 +4179,8 @@ parse_tc_actions(struct mlx5e_tc_act_parse_state *parse_state, goto out_free; parse_state->actions |= attr->action; + if (!tc_act->stats_action) + attr->tc_act_cookies[attr->tc_act_cookies_count++] = act->act_cookie; /* Split attr for multi table act if not the last act. */ if (jump_state.jump_target || @@ -4931,6 +4968,12 @@ errout: return err; } +int mlx5e_tc_fill_action_stats(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act) +{ + return mlx5e_tc_act_stats_fill_stats(get_act_stats_handle(priv), fl_act); +} + int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, struct flow_cls_offload *f, unsigned long flags) { @@ -4957,11 +5000,15 @@ int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, } if (mlx5e_is_offloaded_flow(flow) || flow_flag_test(flow, CT)) { - counter = mlx5e_tc_get_counter(flow); - if (!counter) - goto errout; + if (flow_flag_test(flow, USE_ACT_STATS)) { + f->use_act_stats = true; + } else { + counter = mlx5e_tc_get_counter(flow); + if (!counter) + goto errout; - mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse); + mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse); + } } /* Under multipath it's possible for one rule to be currently @@ -4977,14 +5024,18 @@ int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, u64 packets2; u64 lastuse2; - counter = mlx5e_tc_get_counter(flow->peer_flow); - if (!counter) - goto no_peer_counter; - mlx5_fc_query_cached(counter, &bytes2, &packets2, &lastuse2); + if (flow_flag_test(flow, USE_ACT_STATS)) { + f->use_act_stats = true; + } else { + counter = mlx5e_tc_get_counter(flow->peer_flow); + if (!counter) + goto no_peer_counter; + mlx5_fc_query_cached(counter, &bytes2, &packets2, &lastuse2); - bytes += bytes2; - packets += packets2; - lastuse = max_t(u64, lastuse, lastuse2); + bytes += bytes2; + packets += packets2; + lastuse = max_t(u64, lastuse, lastuse2); + } } no_peer_counter: @@ -5323,8 +5374,16 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv) mlx5e_tc_debugfs_init(tc, mlx5e_fs_get_debugfs_root(priv->fs)); + tc->action_stats_handle = mlx5e_tc_act_stats_create(); + if (IS_ERR(tc->action_stats_handle)) + goto err_act_stats; + return 0; +err_act_stats: + unregister_netdevice_notifier_dev_net(priv->netdev, + &tc->netdevice_nb, + &tc->netdevice_nn); err_reg: mlx5_tc_ct_clean(tc->ct); mlx5e_tc_post_act_destroy(tc->post_act); @@ -5374,6 +5433,7 @@ void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv) mapping_destroy(tc->mapping); mlx5_chains_destroy(tc->chains); mlx5e_tc_nic_destroy_miss_table(priv); + mlx5e_tc_act_stats_free(tc->action_stats_handle); } int mlx5e_tc_ht_init(struct rhashtable *tc_ht) @@ -5450,8 +5510,14 @@ int mlx5e_tc_esw_init(struct mlx5_rep_uplink_priv *uplink_priv) goto err_register_fib_notifier; } + uplink_priv->action_stats_handle = mlx5e_tc_act_stats_create(); + if (IS_ERR(uplink_priv->action_stats_handle)) + goto err_action_counter; + return 0; +err_action_counter: + mlx5e_tc_tun_cleanup(uplink_priv->encap); err_register_fib_notifier: mapping_destroy(uplink_priv->tunnel_enc_opts_mapping); err_enc_opts_mapping: @@ -5478,6 +5544,7 @@ void mlx5e_tc_esw_cleanup(struct mlx5_rep_uplink_priv *uplink_priv) mlx5_tc_ct_clean(uplink_priv->ct_priv); mlx5e_flow_meters_cleanup(uplink_priv->flow_meters); mlx5e_tc_post_act_destroy(uplink_priv->post_act); + mlx5e_tc_act_stats_free(uplink_priv->action_stats_handle); } int mlx5e_tc_num_filters(struct mlx5e_priv *priv, unsigned long flags) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index ce516dc7f3fd..e8e39fdcda73 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -69,6 +69,7 @@ struct mlx5_nic_flow_attr { struct mlx5_flow_attr { u32 action; + unsigned long tc_act_cookies[TCA_ACT_MAX_PRIO]; struct mlx5_fc *counter; struct mlx5_modify_hdr *modify_hdr; struct mlx5e_mod_hdr_handle *mh; /* attached mod header instance */ @@ -79,6 +80,7 @@ struct mlx5_flow_attr { struct mlx5e_tc_flow_parse_attr *parse_attr; u32 chain; u16 prio; + u16 tc_act_cookies_count; u32 dest_chain; struct mlx5_flow_table *ft; struct mlx5_flow_table *dest_ft; @@ -197,6 +199,8 @@ int mlx5e_delete_flower(struct net_device *dev, struct mlx5e_priv *priv, int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, struct flow_cls_offload *f, unsigned long flags); +int mlx5e_tc_fill_action_stats(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act); int mlx5e_tc_configure_matchall(struct mlx5e_priv *priv, struct tc_cls_matchall_offload *f); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c index b406e0367af6..17fe30a4c06c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c @@ -504,6 +504,16 @@ void mlx5_fc_query_cached(struct mlx5_fc *counter, counter->lastpackets = c.packets; } +void mlx5_fc_query_cached_raw(struct mlx5_fc *counter, + u64 *bytes, u64 *packets, u64 *lastuse) +{ + struct mlx5_fc_cache c = counter->cache; + + *bytes = c.bytes; + *packets = c.packets; + *lastuse = c.lastuse; +} + void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, struct delayed_work *dwork, unsigned long delay) diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index ba6958b49a8e..90a2fe5839fa 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -296,6 +296,8 @@ void mlx5_fc_destroy(struct mlx5_core_dev *dev, struct mlx5_fc *counter); u64 mlx5_fc_query_lastuse(struct mlx5_fc *counter); void mlx5_fc_query_cached(struct mlx5_fc *counter, u64 *bytes, u64 *packets, u64 *lastuse); +void mlx5_fc_query_cached_raw(struct mlx5_fc *counter, + u64 *bytes, u64 *packets, u64 *lastuse); int mlx5_fc_query(struct mlx5_core_dev *dev, struct mlx5_fc *counter, u64 *packets, u64 *bytes); u32 mlx5_fc_id(struct mlx5_fc *counter); diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h index 0400a0ac8a29..8c05455b1e34 100644 --- a/include/net/flow_offload.h +++ b/include/net/flow_offload.h @@ -228,6 +228,7 @@ void flow_action_cookie_destroy(struct flow_action_cookie *cookie); struct flow_action_entry { enum flow_action_id id; u32 hw_index; + unsigned long act_cookie; enum flow_action_hw_stats hw_stats; action_destr destructor; void *destructor_priv; @@ -593,6 +594,7 @@ struct flow_cls_common_offload { struct flow_cls_offload { struct flow_cls_common_offload common; enum flow_cls_command command; + bool use_act_stats; unsigned long cookie; struct flow_rule *rule; struct flow_stats stats; @@ -610,6 +612,7 @@ struct flow_offload_action { enum offload_act_command command; enum flow_action_id id; u32 index; + unsigned long cookie; struct flow_stats stats; struct flow_action action; }; diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h index cd410a87517b..ace437c6754b 100644 --- a/include/net/pkt_cls.h +++ b/include/net/pkt_cls.h @@ -292,10 +292,15 @@ static inline void tcf_exts_put_net(struct tcf_exts *exts) #define tcf_act_for_each_action(i, a, actions) \ for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = actions[i]); i++) +static inline bool tc_act_in_hw(struct tc_action *act) +{ + return !!act->in_hw_count; +} + static inline void tcf_exts_hw_stats_update(const struct tcf_exts *exts, - u64 bytes, u64 packets, u64 drops, u64 lastuse, - u8 used_hw_stats, bool used_hw_stats_valid) + struct flow_stats *stats, + bool use_act_stats) { #ifdef CONFIG_NET_CLS_ACT int i; @@ -303,16 +308,18 @@ tcf_exts_hw_stats_update(const struct tcf_exts *exts, for (i = 0; i < exts->nr_actions; i++) { struct tc_action *a = exts->actions[i]; - /* if stats from hw, just skip */ - if (tcf_action_update_hw_stats(a)) { - preempt_disable(); - tcf_action_stats_update(a, bytes, packets, drops, - lastuse, true); - preempt_enable(); - - a->used_hw_stats = used_hw_stats; - a->used_hw_stats_valid = used_hw_stats_valid; + if (use_act_stats || tc_act_in_hw(a)) { + if (!tcf_action_update_hw_stats(a)) + continue; } + + preempt_disable(); + tcf_action_stats_update(a, stats->bytes, stats->pkts, stats->drops, + stats->lastused, true); + preempt_enable(); + + a->used_hw_stats = stats->used_hw_stats; + a->used_hw_stats_valid = stats->used_hw_stats_valid; } #endif } @@ -770,6 +777,7 @@ struct tc_cls_matchall_offload { enum tc_matchall_command command; struct flow_rule *rule; struct flow_stats stats; + bool use_act_stats; unsigned long cookie; }; diff --git a/net/sched/act_api.c b/net/sched/act_api.c index cd09ef49df22..eda58b78da13 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -169,11 +169,6 @@ static bool tc_act_skip_sw(u32 flags) return (flags & TCA_ACT_FLAGS_SKIP_SW) ? true : false; } -static bool tc_act_in_hw(struct tc_action *act) -{ - return !!act->in_hw_count; -} - /* SKIP_HW and SKIP_SW are mutually exclusive flags. */ static bool tc_act_flags_valid(u32 flags) { @@ -192,6 +187,7 @@ static int offload_action_init(struct flow_offload_action *fl_action, fl_action->extack = extack; fl_action->command = cmd; fl_action->index = act->tcfa_index; + fl_action->cookie = (unsigned long)act; if (act->ops->offload_act_setup) { spin_lock_bh(&act->tcfa_lock); @@ -307,9 +303,6 @@ int tcf_action_update_hw_stats(struct tc_action *action) struct flow_offload_action fl_act = {}; int err; - if (!tc_act_in_hw(action)) - return -EOPNOTSUPP; - err = offload_action_init(&fl_act, action, FLOW_ACT_STATS, NULL); if (err) return err; @@ -539,6 +532,8 @@ static int tcf_dump_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, (unsigned long)p->tcfa_tm.lastuse)) continue; + tcf_action_update_hw_stats(p); + nest = nla_nest_start_noflag(skb, n_i); if (!nest) { index--; @@ -1539,9 +1534,6 @@ int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *p, if (p == NULL) goto errout; - /* update hw stats for this action */ - tcf_action_update_hw_stats(p); - /* compat_mode being true specifies a call that is supposed * to add additional backward compatibility statistic TLVs. */ diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index c42fcc47dd6d..35ebe5d5c261 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -545,7 +545,28 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, } *index_inc = k; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + u32 cmd = tcf_pedit_cmd(act, 0); + int k; + + switch (cmd) { + case TCA_PEDIT_KEY_EX_CMD_SET: + fl_action->id = FLOW_ACTION_MANGLE; + break; + case TCA_PEDIT_KEY_EX_CMD_ADD: + fl_action->id = FLOW_ACTION_ADD; + break; + default: + NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); + return -EOPNOTSUPP; + } + + for (k = 1; k < tcf_pedit_nkeys(act); k++) { + if (cmd != tcf_pedit_cmd(act, k)) { + NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); + return -EOPNOTSUPP; + } + } } return 0; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 5b4a95e8a1ee..bfabc9c95fa9 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -3577,6 +3577,7 @@ int tc_setup_action(struct flow_action *flow_action, for (k = 0; k < index ; k++) { entry[k].hw_stats = tc_act_hw_stats(act->hw_stats); entry[k].hw_index = act->tcfa_index; + entry[k].act_cookie = (unsigned long)act; } j += index; diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 0b15698b3531..885c95191ccf 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -502,12 +502,7 @@ static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f, tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false, rtnl_held); - tcf_exts_hw_stats_update(&f->exts, cls_flower.stats.bytes, - cls_flower.stats.pkts, - cls_flower.stats.drops, - cls_flower.stats.lastused, - cls_flower.stats.used_hw_stats, - cls_flower.stats.used_hw_stats_valid); + tcf_exts_hw_stats_update(&f->exts, &cls_flower.stats, cls_flower.use_act_stats); } static void __fl_put(struct cls_fl_filter *f) diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c index 705f63da2c21..fa3bbd187eb9 100644 --- a/net/sched/cls_matchall.c +++ b/net/sched/cls_matchall.c @@ -331,11 +331,7 @@ static void mall_stats_hw_filter(struct tcf_proto *tp, tc_setup_cb_call(block, TC_SETUP_CLSMATCHALL, &cls_mall, false, true); - tcf_exts_hw_stats_update(&head->exts, cls_mall.stats.bytes, - cls_mall.stats.pkts, cls_mall.stats.drops, - cls_mall.stats.lastused, - cls_mall.stats.used_hw_stats, - cls_mall.stats.used_hw_stats_valid); + tcf_exts_hw_stats_update(&head->exts, &cls_mall.stats, cls_mall.use_act_stats); } static int mall_dump(struct net *net, struct tcf_proto *tp, void *fh,