microsoft
/
WSL2-Linux-Kernel
зеркало из https://github.com/microsoft/WSL2-Linux-Kernel.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Extend packet offload to fully support libreswan 

The following patches are an outcome of Raed's work to add packet
 offload support to libreswan [1].
 
 The series includes:
  * Priority support to IPsec policies
  * Statistics per-SA (visible through "ip -s xfrm state ..." command)
  * Support to IKE policy holes
  * Fine tuning to acquire logic.
 
 Thanks
 
 [1] https://github.com/libreswan/libreswan/pull/986
 Link: https://lore.kernel.org/all/cover.1678714336.git.leon@kernel.org
 Signed-off-by: Leon Romanovsky <leon@kernel.org>
 -----BEGIN PGP SIGNATURE-----
 
 iHUEABYKAB0WIQT1m3YD37UfMCUQBNwp8NhrnBAZsQUCZBgoggAKCRAp8NhrnBAZ
 sQPvAP4/muXSlz26gU632wL+wjpFkvM5fX1iu8AwmdhOtX+gAAD/edtCVXj4QxD4
 0xGMjwrz4tH5WqdUzqDQjhZrBYNWmAk=
 =oNEQ
 -----END PGP SIGNATURE-----

Merge tag 'ipsec-libreswan-mlx5' of https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux

Leon Romanovsky says:

====================
Extend packet offload to fully support libreswan

The following patches are an outcome of Raed's work to add packet
offload support to libreswan [1].

The series includes:
 * Priority support to IPsec policies
 * Statistics per-SA (visible through "ip -s xfrm state ..." command)
 * Support to IKE policy holes
 * Fine tuning to acquire logic.

[1] https://github.com/libreswan/libreswan/pull/986
Link: https://lore.kernel.org/all/cover.1678714336.git.leon@kernel.org

* tag 'ipsec-libreswan-mlx5' of https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux:
  net/mlx5e: Update IPsec per SA packets/bytes count
  net/mlx5e: Use one rule to count all IPsec Tx offloaded traffic
  net/mlx5e: Support IPsec acquire default SA
  net/mlx5e: Allow policies with reqid 0, to support IKE policy holes
  xfrm: copy_to_user_state fetch offloaded SA packets/bytes statistics
  xfrm: add new device offload acquire flag
  net/mlx5e: Use chains for IPsec policy priority offload
  net/mlx5: fs_core: Allow ignore_flow_level on TX dest
  net/mlx5: fs_chains: Refactor to detach chains from tc usage
====================

Link: https://lore.kernel.org/r/20230320094722.1009304-1-leon@kernel.org
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski 2023-03-22 21:56:33 -07:00
Родитель 3079bfdbda 5a6cddb89b
Коммит e4d264e87a
12 изменённых файлов: 558 добавлений и 233 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

71
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
Просмотреть файл

@ -308,6 +308,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x,
struct net_device *netdev = x->xso.real_dev;
struct mlx5e_ipsec *ipsec;
struct mlx5e_priv *priv;
gfp_t gfp;
int err;
priv = netdev_priv(netdev);
@ -315,16 +316,20 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x,
return -EOPNOTSUPP;
ipsec = priv->ipsec;
err = mlx5e_xfrm_validate_state(priv->mdev, x, extack);
if (err)
return err;
sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
gfp = (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) ? GFP_ATOMIC : GFP_KERNEL;
sa_entry = kzalloc(sizeof(*sa_entry), gfp);
if (!sa_entry)
return -ENOMEM;
sa_entry->x = x;
sa_entry->ipsec = ipsec;
/* Check if this SA is originated from acquire flow temporary SA */
if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
goto out;
err = mlx5e_xfrm_validate_state(priv->mdev, x, extack);
if (err)
goto err_xfrm;
/* check esn */
mlx5e_ipsec_update_esn_state(sa_entry);
@ -353,6 +358,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x,
mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;
INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state);
out:
x->xso.offload_handle = (unsigned long)sa_entry;
return 0;
@ -372,6 +378,9 @@ static void mlx5e_xfrm_del_state(struct xfrm_state *x)
struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
struct mlx5e_ipsec_sa_entry *old;
if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
return;
old = xa_erase_bh(&ipsec->sadb, sa_entry->ipsec_obj_id);
WARN_ON(old != sa_entry);
}
@ -380,9 +389,13 @@ static void mlx5e_xfrm_free_state(struct xfrm_state *x)
{
struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
goto sa_entry_free;
cancel_work_sync(&sa_entry->modify_work.work);
mlx5e_accel_ipsec_fs_del_rule(sa_entry);
mlx5_ipsec_free_sa_ctx(sa_entry);
sa_entry_free:
kfree(sa_entry);
}
@ -482,26 +495,26 @@ static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x)
static void mlx5e_xfrm_update_curlft(struct xfrm_state *x)
{
struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
int err;
struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
u64 packets, bytes, lastuse;
lockdep_assert_held(&x->lock);
lockdep_assert(lockdep_is_held(&x->lock) ||
lockdep_is_held(&dev_net(x->xso.real_dev)->xfrm.xfrm_cfg_mutex));
if (sa_entry->attrs.soft_packet_limit == XFRM_INF)
/* Limits are not configured, as soft limit
* must be lowever than hard limit.
*/
if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
return;
err = mlx5e_ipsec_aso_query(sa_entry, NULL);
if (err)
return;
mlx5e_ipsec_aso_update_curlft(sa_entry, &x->curlft.packets);
mlx5_fc_query_cached(ipsec_rule->fc, &bytes, &packets, &lastuse);
x->curlft.packets += packets;
x->curlft.bytes += bytes;
}
static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x,
static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev,
struct xfrm_policy *x,
struct netlink_ext_ack *extack)
{
struct xfrm_selector *sel = &x->selector;
if (x->type != XFRM_POLICY_TYPE_MAIN) {
NL_SET_ERR_MSG_MOD(extack, "Cannot offload non-main policy types");
return -EINVAL;
@ -519,8 +532,9 @@ static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x,
return -EINVAL;
}
if (!x->xfrm_vec[0].reqid) {
NL_SET_ERR_MSG_MOD(extack, "Cannot offload policy without reqid");
if (!x->xfrm_vec[0].reqid && sel->proto == IPPROTO_IP &&
addr6_all_zero(sel->saddr.a6) && addr6_all_zero(sel->daddr.a6)) {
NL_SET_ERR_MSG_MOD(extack, "Unsupported policy with reqid 0 without at least one of upper protocol or ip addr(s) different than 0");
return -EINVAL;
}
@ -529,12 +543,24 @@ static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x,
return -EINVAL;
}
if (x->selector.proto != IPPROTO_IP &&
(x->selector.proto != IPPROTO_UDP || x->xdo.dir != XFRM_DEV_OFFLOAD_OUT)) {
if (sel->proto != IPPROTO_IP &&
(sel->proto != IPPROTO_UDP || x->xdo.dir != XFRM_DEV_OFFLOAD_OUT)) {
NL_SET_ERR_MSG_MOD(extack, "Device does not support upper protocol other than UDP, and only Tx direction");
return -EINVAL;
}
if (x->priority) {
if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) {
NL_SET_ERR_MSG_MOD(extack, "Device does not support policy priority");
return -EINVAL;
}
if (x->priority == U32_MAX) {
NL_SET_ERR_MSG_MOD(extack, "Device does not support requested policy priority");
return -EINVAL;
}
}
return 0;
}
@ -560,6 +586,7 @@ mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry,
attrs->upspec.sport = ntohs(sel->sport);
attrs->upspec.sport_mask = ntohs(sel->sport_mask);
attrs->upspec.proto = sel->proto;
attrs->prio = x->priority;
}
static int mlx5e_xfrm_add_policy(struct xfrm_policy *x,
@ -576,7 +603,7 @@ static int mlx5e_xfrm_add_policy(struct xfrm_policy *x,
return -EOPNOTSUPP;
}
err = mlx5e_xfrm_validate_policy(x, extack);
err = mlx5e_xfrm_validate_policy(priv->mdev, x, extack);
if (err)
return err;

13
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
Просмотреть файл

@ -94,6 +94,7 @@ enum mlx5_ipsec_cap {
MLX5_IPSEC_CAP_ESN = 1 << 1,
MLX5_IPSEC_CAP_PACKET_OFFLOAD = 1 << 2,
MLX5_IPSEC_CAP_ROCE = 1 << 3,
MLX5_IPSEC_CAP_PRIO = 1 << 4,
};
struct mlx5e_priv;
@ -161,6 +162,7 @@ struct mlx5e_ipsec_rule {
struct mlx5_flow_handle *rule;
struct mlx5_modify_hdr *modify_hdr;
struct mlx5_pkt_reformat *pkt_reformat;
struct mlx5_fc *fc;
};
struct mlx5e_ipsec_modify_state_work {
@ -198,6 +200,7 @@ struct mlx5_accel_pol_xfrm_attrs {
u8 type : 2;
u8 dir : 2;
u32 reqid;
u32 prio;
};
struct mlx5e_ipsec_pol_entry {
@ -233,9 +236,6 @@ void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec);
int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
struct mlx5_wqe_aso_ctrl_seg *data);
void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry,
u64 *packets);
void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv,
void *ipsec_stats);
@ -252,6 +252,13 @@ mlx5e_ipsec_pol2dev(struct mlx5e_ipsec_pol_entry *pol_entry)
{
return pol_entry->ipsec->mdev;
}
static inline bool addr6_all_zero(__be32 *addr6)
{
static const __be32 zaddr6[4] = {};
return !memcmp(addr6, zaddr6, sizeof(*zaddr6));
}
#else
static inline void mlx5e_ipsec_init(struct mlx5e_priv *priv)
{

538
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
Просмотреть файл

@ -7,6 +7,7 @@
#include "ipsec.h"
#include "fs_core.h"
#include "lib/ipsec_fs_roce.h"
#include "lib/fs_chains.h"
#define NUM_IPSEC_FTE BIT(15)
@ -34,13 +35,16 @@ struct mlx5e_ipsec_rx {
struct mlx5e_ipsec_miss sa;
struct mlx5e_ipsec_rule status;
struct mlx5e_ipsec_fc *fc;
struct mlx5_fs_chains *chains;
};
struct mlx5e_ipsec_tx {
struct mlx5e_ipsec_ft ft;
struct mlx5e_ipsec_miss pol;
struct mlx5e_ipsec_rule status;
struct mlx5_flow_namespace *ns;
struct mlx5e_ipsec_fc *fc;
struct mlx5_fs_chains *chains;
};
/* IPsec RX flow steering */
@ -51,6 +55,67 @@ static enum mlx5_traffic_types family2tt(u32 family)
return MLX5_TT_IPV6_IPSEC_ESP;
}
static struct mlx5e_ipsec_rx *ipsec_rx(struct mlx5e_ipsec *ipsec, u32 family)
{
if (family == AF_INET)
return ipsec->rx_ipv4;
return ipsec->rx_ipv6;
}
static struct mlx5_fs_chains *
ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft,
enum mlx5_flow_namespace_type ns, int base_prio,
int base_level, struct mlx5_flow_table **root_ft)
{
struct mlx5_chains_attr attr = {};
struct mlx5_fs_chains *chains;
struct mlx5_flow_table *ft;
int err;
attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED |
MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
attr.max_grp_num = 2;
attr.default_ft = miss_ft;
attr.ns = ns;
attr.fs_base_prio = base_prio;
attr.fs_base_level = base_level;
chains = mlx5_chains_create(mdev, &attr);
if (IS_ERR(chains))
return chains;
/* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */
ft = mlx5_chains_get_table(chains, 0, 1, 0);
if (IS_ERR(ft)) {
err = PTR_ERR(ft);
goto err_chains_get;
}
*root_ft = ft;
return chains;
err_chains_get:
mlx5_chains_destroy(chains);
return ERR_PTR(err);
}
static void ipsec_chains_destroy(struct mlx5_fs_chains *chains)
{
mlx5_chains_put_table(chains, 0, 1, 0);
mlx5_chains_destroy(chains);
}
static struct mlx5_flow_table *
ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio)
{
return mlx5_chains_get_table(chains, 0, prio + 1, 0);
}
static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio)
{
mlx5_chains_put_table(chains, 0, prio + 1, 0);
}
static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns,
int level, int prio,
int max_num_groups)
@ -170,9 +235,18 @@ out:
static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
struct mlx5e_ipsec_rx *rx, u32 family)
{
mlx5_del_flow_rules(rx->pol.rule);
mlx5_destroy_flow_group(rx->pol.group);
mlx5_destroy_flow_table(rx->ft.pol);
struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
/* disconnect */
mlx5_ttc_fwd_default_dest(ttc, family2tt(family));
if (rx->chains) {
ipsec_chains_destroy(rx->chains);
} else {
mlx5_del_flow_rules(rx->pol.rule);
mlx5_destroy_flow_group(rx->pol.group);
mlx5_destroy_flow_table(rx->ft.pol);
}
mlx5_del_flow_rules(rx->sa.rule);
mlx5_destroy_flow_group(rx->sa.group);
@ -238,6 +312,20 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
if (err)
goto err_fs;
if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
rx->chains = ipsec_chains_create(mdev, rx->ft.sa,
MLX5_FLOW_NAMESPACE_KERNEL,
MLX5E_NIC_PRIO,
MLX5E_ACCEL_FS_POL_FT_LEVEL,
&rx->ft.pol);
if (IS_ERR(rx->chains)) {
err = PTR_ERR(rx->chains);
goto err_pol_ft;
}
goto connect;
}
ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_POL_FT_LEVEL, MLX5E_NIC_PRIO,
2);
if (IS_ERR(ft)) {
@ -252,6 +340,12 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
if (err)
goto err_pol_miss;
connect:
/* connect */
memset(dest, 0x00, sizeof(*dest));
dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest[0].ft = rx->ft.pol;
mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest[0]);
return 0;
err_pol_miss:
@ -271,69 +365,147 @@ err_fs_ft_status:
return err;
}
static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev,
struct mlx5e_ipsec *ipsec, u32 family)
static int rx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
struct mlx5e_ipsec_rx *rx, u32 family)
{
struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
struct mlx5_flow_destination dest = {};
struct mlx5e_ipsec_rx *rx;
int err = 0;
int err;
if (family == AF_INET)
rx = ipsec->rx_ipv4;
else
rx = ipsec->rx_ipv6;
mutex_lock(&rx->ft.mutex);
if (rx->ft.refcnt)
goto skip;
/* create FT */
err = rx_create(mdev, ipsec, rx, family);
if (err)
goto out;
/* connect */
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest.ft = rx->ft.pol;
mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest);
return err;
skip:
rx->ft.refcnt++;
out:
return 0;
}
static void rx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx *rx,
u32 family)
{
if (--rx->ft.refcnt)
return;
rx_destroy(ipsec->mdev, ipsec, rx, family);
}
static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev,
struct mlx5e_ipsec *ipsec, u32 family)
{
struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family);
int err;
mutex_lock(&rx->ft.mutex);
err = rx_get(mdev, ipsec, rx, family);
mutex_unlock(&rx->ft.mutex);
if (err)
return ERR_PTR(err);
return rx;
}
static void rx_ft_put(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
u32 family)
static struct mlx5_flow_table *rx_ft_get_policy(struct mlx5_core_dev *mdev,
struct mlx5e_ipsec *ipsec,
u32 family, u32 prio)
{
struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
struct mlx5e_ipsec_rx *rx;
if (family == AF_INET)
rx = ipsec->rx_ipv4;
else
rx = ipsec->rx_ipv6;
struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family);
struct mlx5_flow_table *ft;
int err;
mutex_lock(&rx->ft.mutex);
rx->ft.refcnt--;
if (rx->ft.refcnt)
goto out;
err = rx_get(mdev, ipsec, rx, family);
if (err)
goto err_get;
/* disconnect */
mlx5_ttc_fwd_default_dest(ttc, family2tt(family));
ft = rx->chains ? ipsec_chains_get_table(rx->chains, prio) : rx->ft.pol;
if (IS_ERR(ft)) {
err = PTR_ERR(ft);
goto err_get_ft;
}
/* remove FT */
rx_destroy(mdev, ipsec, rx, family);
mutex_unlock(&rx->ft.mutex);
return ft;
out:
err_get_ft:
rx_put(ipsec, rx, family);
err_get:
mutex_unlock(&rx->ft.mutex);
return ERR_PTR(err);
}
static void rx_ft_put(struct mlx5e_ipsec *ipsec, u32 family)
{
struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family);
mutex_lock(&rx->ft.mutex);
rx_put(ipsec, rx, family);
mutex_unlock(&rx->ft.mutex);
}
static void rx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 family, u32 prio)
{
struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family);
mutex_lock(&rx->ft.mutex);
if (rx->chains)
ipsec_chains_put_table(rx->chains, prio);
rx_put(ipsec, rx, family);
mutex_unlock(&rx->ft.mutex);
}
static int ipsec_counter_rule_tx(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx)
{
struct mlx5_flow_destination dest = {};
struct mlx5_flow_act flow_act = {};
struct mlx5_flow_handle *fte;
struct mlx5_flow_spec *spec;
int err;
spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
if (!spec)
return -ENOMEM;
/* create fte */
flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW |
MLX5_FLOW_CONTEXT_ACTION_COUNT;
dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
dest.counter_id = mlx5_fc_id(tx->fc->cnt);
fte = mlx5_add_flow_rules(tx->ft.status, spec, &flow_act, &dest, 1);
if (IS_ERR(fte)) {
err = PTR_ERR(fte);
mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err);
goto err_rule;
}
kvfree(spec);
tx->status.rule = fte;
return 0;
err_rule:
kvfree(spec);
return err;
}
/* IPsec TX flow steering */
static void tx_destroy(struct mlx5e_ipsec_tx *tx, struct mlx5_ipsec_fs *roce)
{
mlx5_ipsec_fs_roce_tx_destroy(roce);
if (tx->chains) {
ipsec_chains_destroy(tx->chains);
} else {
mlx5_del_flow_rules(tx->pol.rule);
mlx5_destroy_flow_group(tx->pol.group);
mlx5_destroy_flow_table(tx->ft.pol);
}
mlx5_destroy_flow_table(tx->ft.sa);
mlx5_del_flow_rules(tx->status.rule);
mlx5_destroy_flow_table(tx->ft.status);
}
static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx,
struct mlx5_ipsec_fs *roce)
{
@ -341,12 +513,34 @@ static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx,
struct mlx5_flow_table *ft;
int err;
ft = ipsec_ft_create(tx->ns, 1, 0, 4);
ft = ipsec_ft_create(tx->ns, 2, 0, 1);
if (IS_ERR(ft))
return PTR_ERR(ft);
tx->ft.status = ft;
err = ipsec_counter_rule_tx(mdev, tx);
if (err)
goto err_status_rule;
ft = ipsec_ft_create(tx->ns, 1, 0, 4);
if (IS_ERR(ft)) {
err = PTR_ERR(ft);
goto err_sa_ft;
}
tx->ft.sa = ft;
if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
tx->chains = ipsec_chains_create(
mdev, tx->ft.sa, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, 0, 0,
&tx->ft.pol);
if (IS_ERR(tx->chains)) {
err = PTR_ERR(tx->chains);
goto err_pol_ft;
}
goto connect_roce;
}
ft = ipsec_ft_create(tx->ns, 0, 0, 2);
if (IS_ERR(ft)) {
err = PTR_ERR(ft);
@ -356,44 +550,100 @@ static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx,
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest.ft = tx->ft.sa;
err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest);
if (err)
goto err_pol_miss;
if (err) {
mlx5_destroy_flow_table(tx->ft.pol);
goto err_pol_ft;
}
connect_roce:
err = mlx5_ipsec_fs_roce_tx_create(mdev, roce, tx->ft.pol);
if (err)
goto err_roce;
return 0;
err_roce:
mlx5_del_flow_rules(tx->pol.rule);
mlx5_destroy_flow_group(tx->pol.group);
err_pol_miss:
mlx5_destroy_flow_table(tx->ft.pol);
if (tx->chains) {
ipsec_chains_destroy(tx->chains);
} else {
mlx5_del_flow_rules(tx->pol.rule);
mlx5_destroy_flow_group(tx->pol.group);
mlx5_destroy_flow_table(tx->ft.pol);
}
err_pol_ft:
mlx5_destroy_flow_table(tx->ft.sa);
err_sa_ft:
mlx5_del_flow_rules(tx->status.rule);
err_status_rule:
mlx5_destroy_flow_table(tx->ft.status);
return err;
}
static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
struct mlx5e_ipsec_tx *tx)
{
int err;
if (tx->ft.refcnt)
goto skip;
err = tx_create(mdev, tx, ipsec->roce);
if (err)
return err;
skip:
tx->ft.refcnt++;
return 0;
}
static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx)
{
if (--tx->ft.refcnt)
return;
tx_destroy(tx, ipsec->roce);
}
static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev,
struct mlx5e_ipsec *ipsec,
u32 prio)
{
struct mlx5e_ipsec_tx *tx = ipsec->tx;
struct mlx5_flow_table *ft;
int err;
mutex_lock(&tx->ft.mutex);
err = tx_get(mdev, ipsec, tx);
if (err)
goto err_get;
ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol;
if (IS_ERR(ft)) {
err = PTR_ERR(ft);
goto err_get_ft;
}
mutex_unlock(&tx->ft.mutex);
return ft;
err_get_ft:
tx_put(ipsec, tx);
err_get:
mutex_unlock(&tx->ft.mutex);
return ERR_PTR(err);
}
static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev,
struct mlx5e_ipsec *ipsec)
{
struct mlx5e_ipsec_tx *tx = ipsec->tx;
int err = 0;
int err;
mutex_lock(&tx->ft.mutex);
if (tx->ft.refcnt)
goto skip;
err = tx_create(mdev, tx, ipsec->roce);
if (err)
goto out;
skip:
tx->ft.refcnt++;
out:
err = tx_get(mdev, ipsec, tx);
mutex_unlock(&tx->ft.mutex);
if (err)
return ERR_PTR(err);
return tx;
}
@ -402,53 +652,72 @@ static void tx_ft_put(struct mlx5e_ipsec *ipsec)
struct mlx5e_ipsec_tx *tx = ipsec->tx;
mutex_lock(&tx->ft.mutex);
tx->ft.refcnt--;
if (tx->ft.refcnt)
goto out;
tx_put(ipsec, tx);
mutex_unlock(&tx->ft.mutex);
}
mlx5_ipsec_fs_roce_tx_destroy(ipsec->roce);
mlx5_del_flow_rules(tx->pol.rule);
mlx5_destroy_flow_group(tx->pol.group);
mlx5_destroy_flow_table(tx->ft.pol);
mlx5_destroy_flow_table(tx->ft.sa);
out:
static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio)
{
struct mlx5e_ipsec_tx *tx = ipsec->tx;
mutex_lock(&tx->ft.mutex);
if (tx->chains)
ipsec_chains_put_table(tx->chains, prio);
tx_put(ipsec, tx);
mutex_unlock(&tx->ft.mutex);
}
static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr,
__be32 *daddr)
{
if (!*saddr && !*daddr)
return;
spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4);
memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4);
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
if (*saddr) {
memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4);
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
}
if (*daddr) {
memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4);
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
}
}
static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr,
__be32 *daddr)
{
if (addr6_all_zero(saddr) && addr6_all_zero(daddr))
return;
spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16);
memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16);
memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16);
memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16);
if (!addr6_all_zero(saddr)) {
memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16);
memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16);
}
if (!addr6_all_zero(daddr)) {
memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16);
memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16);
}
}
static void setup_fte_esp(struct mlx5_flow_spec *spec)
@ -607,11 +876,12 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
struct mlx5_flow_destination dest = {};
struct mlx5_flow_destination dest[2];
struct mlx5_flow_act flow_act = {};
struct mlx5_flow_handle *rule;
struct mlx5_flow_spec *spec;
struct mlx5e_ipsec_rx *rx;
struct mlx5_fc *counter;
int err;
rx = rx_ft_get(mdev, ipsec, attrs->family);
@ -648,14 +918,22 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
break;
}
counter = mlx5_fc_create(mdev, true);
if (IS_ERR(counter)) {
err = PTR_ERR(counter);
goto err_add_cnt;
}
flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
flow_act.flags |= FLOW_ACT_NO_APPEND;
flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT;
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest.ft = rx->ft.status;
rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, &dest, 1);
MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
MLX5_FLOW_CONTEXT_ACTION_COUNT;
dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest[0].ft = rx->ft.status;
dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
dest[1].counter_id = mlx5_fc_id(counter);
rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2);
if (IS_ERR(rule)) {
err = PTR_ERR(rule);
mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err);
@ -665,10 +943,13 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
sa_entry->ipsec_rule.rule = rule;
sa_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr;
sa_entry->ipsec_rule.fc = counter;
sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat;
return 0;
err_add_flow:
mlx5_fc_destroy(mdev, counter);
err_add_cnt:
if (flow_act.pkt_reformat)
mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
err_pkt_reformat:
@ -676,7 +957,7 @@ err_pkt_reformat:
err_mod_header:
kvfree(spec);
err_alloc:
rx_ft_put(mdev, ipsec, attrs->family);
rx_ft_put(ipsec, attrs->family);
return err;
}
@ -685,12 +966,13 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
struct mlx5_flow_destination dest = {};
struct mlx5_flow_destination dest[2];
struct mlx5_flow_act flow_act = {};
struct mlx5_flow_handle *rule;
struct mlx5_flow_spec *spec;
struct mlx5e_ipsec_tx *tx;
int err = 0;
struct mlx5_fc *counter;
int err;
tx = tx_ft_get(mdev, ipsec);
if (IS_ERR(tx))
@ -717,7 +999,8 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
setup_fte_reg_a(spec);
break;
case XFRM_DEV_OFFLOAD_PACKET:
setup_fte_reg_c0(spec, attrs->reqid);
if (attrs->reqid)
setup_fte_reg_c0(spec, attrs->reqid);
err = setup_pkt_reformat(mdev, attrs, &flow_act);
if (err)
goto err_pkt_reformat;
@ -726,15 +1009,23 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
break;
}
counter = mlx5_fc_create(mdev, true);
if (IS_ERR(counter)) {
err = PTR_ERR(counter);
goto err_add_cnt;
}
flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
flow_act.flags |= FLOW_ACT_NO_APPEND;
flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW |
flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT |
MLX5_FLOW_CONTEXT_ACTION_COUNT;
dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
dest.counter_id = mlx5_fc_id(tx->fc->cnt);
rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, &dest, 1);
dest[0].ft = tx->ft.status;
dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
dest[1].counter_id = mlx5_fc_id(counter);
rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, dest, 2);
if (IS_ERR(rule)) {
err = PTR_ERR(rule);
mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err);
@ -743,10 +1034,13 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
kvfree(spec);
sa_entry->ipsec_rule.rule = rule;
sa_entry->ipsec_rule.fc = counter;
sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat;
return 0;
err_add_flow:
mlx5_fc_destroy(mdev, counter);
err_add_cnt:
if (flow_act.pkt_reformat)
mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
err_pkt_reformat:
@ -760,16 +1054,17 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
{
struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs;
struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
struct mlx5e_ipsec_tx *tx = pol_entry->ipsec->tx;
struct mlx5_flow_destination dest[2] = {};
struct mlx5_flow_act flow_act = {};
struct mlx5_flow_handle *rule;
struct mlx5_flow_spec *spec;
struct mlx5e_ipsec_tx *tx;
struct mlx5_flow_table *ft;
int err, dstn = 0;
tx = tx_ft_get(mdev, pol_entry->ipsec);
if (IS_ERR(tx))
return PTR_ERR(tx);
ft = tx_ft_get_policy(mdev, pol_entry->ipsec, attrs->prio);
if (IS_ERR(ft))
return PTR_ERR(ft);
spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
if (!spec) {
@ -785,10 +1080,12 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
setup_fte_no_frags(spec);
setup_fte_upper_proto_match(spec, &attrs->upspec);
err = setup_modify_header(mdev, attrs->reqid, XFRM_DEV_OFFLOAD_OUT,
&flow_act);
if (err)
goto err_mod_header;
if (attrs->reqid) {
err = setup_modify_header(mdev, attrs->reqid,
XFRM_DEV_OFFLOAD_OUT, &flow_act);
if (err)
goto err_mod_header;
}
switch (attrs->action) {
case XFRM_POLICY_ALLOW:
@ -811,7 +1108,7 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
dest[dstn].ft = tx->ft.sa;
dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dstn++;
rule = mlx5_add_flow_rules(tx->ft.pol, spec, &flow_act, dest, dstn);
rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
if (IS_ERR(rule)) {
err = PTR_ERR(rule);
mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err);
@ -824,11 +1121,12 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
return 0;
err_action:
mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
if (attrs->reqid)
mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
err_mod_header:
kvfree(spec);
err_alloc:
tx_ft_put(pol_entry->ipsec);
tx_ft_put_policy(pol_entry->ipsec, attrs->prio);
return err;
}
@ -840,12 +1138,15 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
struct mlx5_flow_act flow_act = {};
struct mlx5_flow_handle *rule;
struct mlx5_flow_spec *spec;
struct mlx5_flow_table *ft;
struct mlx5e_ipsec_rx *rx;
int err, dstn = 0;
rx = rx_ft_get(mdev, pol_entry->ipsec, attrs->family);
if (IS_ERR(rx))
return PTR_ERR(rx);
ft = rx_ft_get_policy(mdev, pol_entry->ipsec, attrs->family, attrs->prio);
if (IS_ERR(ft))
return PTR_ERR(ft);
rx = ipsec_rx(pol_entry->ipsec, attrs->family);
spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
if (!spec) {
@ -880,7 +1181,7 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest[dstn].ft = rx->ft.sa;
dstn++;
rule = mlx5_add_flow_rules(rx->ft.pol, spec, &flow_act, dest, dstn);
rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
if (IS_ERR(rule)) {
err = PTR_ERR(rule);
mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err);
@ -894,7 +1195,7 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
err_action:
kvfree(spec);
err_alloc:
rx_ft_put(mdev, pol_entry->ipsec, attrs->family);
rx_ft_put_policy(pol_entry->ipsec, attrs->family, attrs->prio);
return err;
}
@ -1022,7 +1323,7 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
mlx5_del_flow_rules(ipsec_rule->rule);
mlx5_fc_destroy(mdev, ipsec_rule->fc);
if (ipsec_rule->pkt_reformat)
mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat);
@ -1032,7 +1333,7 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
}
mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
rx_ft_put(mdev, sa_entry->ipsec, sa_entry->attrs.family);
rx_ft_put(sa_entry->ipsec, sa_entry->attrs.family);
}
int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
@ -1051,12 +1352,15 @@ void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
mlx5_del_flow_rules(ipsec_rule->rule);
if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) {
rx_ft_put(mdev, pol_entry->ipsec, pol_entry->attrs.family);
rx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.family,
pol_entry->attrs.prio);
return;
}
mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
tx_ft_put(pol_entry->ipsec);
if (ipsec_rule->modify_hdr)
mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio);
}
void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec)

32
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c
Просмотреть файл

@ -36,11 +36,18 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
MLX5_CAP_ETH(mdev, insert_trailer) && MLX5_CAP_ETH(mdev, swp))
caps |= MLX5_IPSEC_CAP_CRYPTO;
if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload) &&
MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) &&
MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) &&
MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap))
caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD;
if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload)) {
if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
reformat_add_esp_trasport) &&
MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
reformat_del_esp_trasport) &&
MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap))
caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD;
if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) &&
MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level))
caps |= MLX5_IPSEC_CAP_PRIO;
}
if (mlx5_get_roce_state(mdev) &&
MLX5_CAP_GEN_2(mdev, flow_table_type_2_type) & MLX5_FT_NIC_RX_2_NIC_RX_RDMA &&
@ -482,18 +489,3 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
spin_unlock_bh(&aso->lock);
return ret;
}
void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry,
u64 *packets)
{
struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
struct mlx5e_ipsec_aso *aso = ipsec->aso;
u64 hard_cnt;
hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt);
/* HW decresases the limit till it reaches zero to fire an avent.
* We need to fix the calculations, so the returned count is a total
* number of passed packets and not how much left.
*/
*packets = sa_entry->attrs.hard_packet_limit - hard_cnt;
}

20
drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
Просмотреть файл

@ -5181,22 +5181,6 @@ static int mlx5e_tc_netdev_event(struct notifier_block *this,
return NOTIFY_DONE;
}
static int mlx5e_tc_nic_get_ft_size(struct mlx5_core_dev *dev)
{
int tc_grp_size, tc_tbl_size;
u32 max_flow_counter;
max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
MLX5_CAP_GEN(dev, max_flow_counter_15_0);
tc_grp_size = min_t(int, max_flow_counter, MLX5E_TC_TABLE_MAX_GROUP_SIZE);
tc_tbl_size = min_t(int, tc_grp_size * MLX5E_TC_TABLE_NUM_GROUPS,
BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev, log_max_ft_size)));
return tc_tbl_size;
}
static int mlx5e_tc_nic_create_miss_table(struct mlx5e_priv *priv)
{
struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
@ -5269,10 +5253,10 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED |
MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
attr.ns = MLX5_FLOW_NAMESPACE_KERNEL;
attr.max_ft_sz = mlx5e_tc_nic_get_ft_size(dev);
attr.max_grp_num = MLX5E_TC_TABLE_NUM_GROUPS;
attr.default_ft = tc->miss_t;
attr.mapping = chains_mapping;
attr.fs_base_prio = MLX5E_TC_PRIO;
tc->chains = mlx5_chains_create(dev, &attr);
if (IS_ERR(tc->chains)) {
@ -5280,6 +5264,8 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
goto err_miss;
}
mlx5_chains_print_info(tc->chains);
tc->post_act = mlx5e_tc_post_act_init(priv, tc->chains, MLX5_FLOW_NAMESPACE_KERNEL);
tc->ct = mlx5_tc_ct_init(priv, tc->chains, &tc->mod_hdr,
MLX5_FLOW_NAMESPACE_KERNEL, tc->post_act);

6
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
Просмотреть файл

@ -1374,14 +1374,11 @@ esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
struct mlx5_flow_table *nf_ft, *ft;
struct mlx5_chains_attr attr = {};
struct mlx5_fs_chains *chains;
u32 fdb_max;
int err;
fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
esw_init_chains_offload_flags(esw, &attr.flags);
attr.ns = MLX5_FLOW_NAMESPACE_FDB;
attr.max_ft_sz = fdb_max;
attr.fs_base_prio = FDB_TC_OFFLOAD;
attr.max_grp_num = esw->params.large_group_num;
attr.default_ft = miss_fdb;
attr.mapping = esw->offloads.reg_c0_obj_pool;
@ -1392,6 +1389,7 @@ esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
return err;
}
mlx5_chains_print_info(chains);
esw->fdb_table.offloads.esw_chains_priv = chains;

5
drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
Просмотреть файл

@ -137,7 +137,7 @@
#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1)
#define KERNEL_TX_IPSEC_NUM_PRIOS 1
#define KERNEL_TX_IPSEC_NUM_LEVELS 2
#define KERNEL_TX_IPSEC_NUM_LEVELS 3
#define KERNEL_TX_IPSEC_MIN_LEVEL (KERNEL_TX_IPSEC_NUM_LEVELS)
#define KERNEL_TX_MACSEC_NUM_PRIOS 1
@ -1762,7 +1762,8 @@ static bool dest_is_valid(struct mlx5_flow_destination *dest,
if (ignore_level) {
if (ft->type != FS_FT_FDB &&
ft->type != FS_FT_NIC_RX)
ft->type != FS_FT_NIC_RX &&
ft->type != FS_FT_NIC_TX)
return false;
if (dest->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&

89
drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
Просмотреть файл

@ -14,10 +14,8 @@
#define chains_lock(chains) ((chains)->lock)
#define chains_ht(chains) ((chains)->chains_ht)
#define prios_ht(chains) ((chains)->prios_ht)
#define tc_default_ft(chains) ((chains)->tc_default_ft)
#define tc_end_ft(chains) ((chains)->tc_end_ft)
#define ns_to_chains_fs_prio(ns) ((ns) == MLX5_FLOW_NAMESPACE_FDB ? \
FDB_TC_OFFLOAD : MLX5E_TC_PRIO)
#define chains_default_ft(chains) ((chains)->chains_default_ft)
#define chains_end_ft(chains) ((chains)->chains_end_ft)
#define FT_TBL_SZ (64 * 1024)
struct mlx5_fs_chains {
@ -28,13 +26,15 @@ struct mlx5_fs_chains {
/* Protects above chains_ht and prios_ht */
struct mutex lock;
struct mlx5_flow_table *tc_default_ft;
struct mlx5_flow_table *tc_end_ft;
struct mlx5_flow_table *chains_default_ft;
struct mlx5_flow_table *chains_end_ft;
struct mapping_ctx *chains_mapping;
enum mlx5_flow_namespace_type ns;
u32 group_num;
u32 flags;
int fs_base_prio;
int fs_base_level;
};
struct fs_chain {
@ -145,7 +145,7 @@ void
mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
struct mlx5_flow_table *ft)
{
tc_end_ft(chains) = ft;
chains_end_ft(chains) = ft;
}
static struct mlx5_flow_table *
@ -164,11 +164,11 @@ mlx5_chains_create_table(struct mlx5_fs_chains *chains,
sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE;
ft_attr.max_fte = sz;
/* We use tc_default_ft(chains) as the table's next_ft till
/* We use chains_default_ft(chains) as the table's next_ft till
* ignore_flow_level is allowed on FT creation and not just for FTEs.
* Instead caller should add an explicit miss rule if needed.
*/
ft_attr.next_ft = tc_default_ft(chains);
ft_attr.next_ft = chains_default_ft(chains);
/* The root table(chain 0, prio 1, level 0) is required to be
* connected to the previous fs_core managed prio.
@ -177,22 +177,22 @@ mlx5_chains_create_table(struct mlx5_fs_chains *chains,
*/
if (!mlx5_chains_ignore_flow_level_supported(chains) ||
(chain == 0 && prio == 1 && level == 0)) {
ft_attr.level = level;
ft_attr.prio = prio - 1;
ft_attr.level = chains->fs_base_level;
ft_attr.prio = chains->fs_base_prio;
ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ?
mlx5_get_fdb_sub_ns(chains->dev, chain) :
mlx5_get_flow_namespace(chains->dev, chains->ns);
} else {
ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED;
ft_attr.prio = ns_to_chains_fs_prio(chains->ns);
ft_attr.prio = chains->fs_base_prio;
/* Firmware doesn't allow us to create another level 0 table,
* so we create all unmanaged tables as level 1.
* so we create all unmanaged tables as level 1 (base + 1).
*
* To connect them, we use explicit miss rules with
* ignore_flow_level. Caller is responsible to create
* these rules (if needed).
*/
ft_attr.level = 1;
ft_attr.level = chains->fs_base_level + 1;
ns = mlx5_get_flow_namespace(chains->dev, chains->ns);
}
@ -220,7 +220,8 @@ create_chain_restore(struct fs_chain *chain)
int err;
if (chain->chain == mlx5_chains_get_nf_ft_chain(chains) ||
!mlx5_chains_prios_supported(chains))
!mlx5_chains_prios_supported(chains) ||
!chains->chains_mapping)
return 0;
err = mlx5_chains_get_chain_mapping(chains, chain->chain, &index);
@ -380,7 +381,7 @@ mlx5_chains_add_miss_rule(struct fs_chain *chain,
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest.ft = next_ft;
if (next_ft == tc_end_ft(chains) &&
if (chains->chains_mapping && next_ft == chains_end_ft(chains) &&
chain->chain != mlx5_chains_get_nf_ft_chain(chains) &&
mlx5_chains_prios_supported(chains)) {
act.modify_hdr = chain->miss_modify_hdr;
@ -494,8 +495,8 @@ mlx5_chains_create_prio(struct mlx5_fs_chains *chains,
/* Default miss for each chain: */
next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
tc_default_ft(chains) :
tc_end_ft(chains);
chains_default_ft(chains) :
chains_end_ft(chains);
list_for_each(pos, &chain_s->prios_list) {
struct prio *p = list_entry(pos, struct prio, list);
@ -681,7 +682,7 @@ err_get_prio:
struct mlx5_flow_table *
mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains)
{
return tc_end_ft(chains);
return chains_end_ft(chains);
}
struct mlx5_flow_table *
@ -718,48 +719,38 @@ mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
static struct mlx5_fs_chains *
mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
{
struct mlx5_fs_chains *chains_priv;
u32 max_flow_counter;
struct mlx5_fs_chains *chains;
int err;
chains_priv = kzalloc(sizeof(*chains_priv), GFP_KERNEL);
if (!chains_priv)
chains = kzalloc(sizeof(*chains), GFP_KERNEL);
if (!chains)
return ERR_PTR(-ENOMEM);
max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
MLX5_CAP_GEN(dev, max_flow_counter_15_0);
chains->dev = dev;
chains->flags = attr->flags;
chains->ns = attr->ns;
chains->group_num = attr->max_grp_num;
chains->chains_mapping = attr->mapping;
chains->fs_base_prio = attr->fs_base_prio;
chains->fs_base_level = attr->fs_base_level;
chains_default_ft(chains) = chains_end_ft(chains) = attr->default_ft;
mlx5_core_dbg(dev,
"Init flow table chains, max counters(%d), groups(%d), max flow table size(%d)\n",
max_flow_counter, attr->max_grp_num, attr->max_ft_sz);
chains_priv->dev = dev;
chains_priv->flags = attr->flags;
chains_priv->ns = attr->ns;
chains_priv->group_num = attr->max_grp_num;
chains_priv->chains_mapping = attr->mapping;
tc_default_ft(chains_priv) = tc_end_ft(chains_priv) = attr->default_ft;
mlx5_core_info(dev, "Supported tc offload range - chains: %u, prios: %u\n",
mlx5_chains_get_chain_range(chains_priv),
mlx5_chains_get_prio_range(chains_priv));
err = rhashtable_init(&chains_ht(chains_priv), &chain_params);
err = rhashtable_init(&chains_ht(chains), &chain_params);
if (err)
goto init_chains_ht_err;
err = rhashtable_init(&prios_ht(chains_priv), &prio_params);
err = rhashtable_init(&prios_ht(chains), &prio_params);
if (err)
goto init_prios_ht_err;
mutex_init(&chains_lock(chains_priv));
mutex_init(&chains_lock(chains));
return chains_priv;
return chains;
init_prios_ht_err:
rhashtable_destroy(&chains_ht(chains_priv));
rhashtable_destroy(&chains_ht(chains));
init_chains_ht_err:
kfree(chains_priv);
kfree(chains);
return ERR_PTR(err);
}
@ -808,3 +799,9 @@ mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains, u32 chain_mapping)
return mapping_remove(ctx, chain_mapping);
}
void
mlx5_chains_print_info(struct mlx5_fs_chains *chains)
{
mlx5_core_dbg(chains->dev, "Flow table chains groups(%d)\n", chains->group_num);
}

9
drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h
Просмотреть файл

@ -17,8 +17,9 @@ enum mlx5_chains_flags {
struct mlx5_chains_attr {
enum mlx5_flow_namespace_type ns;
int fs_base_prio;
int fs_base_level;
u32 flags;
u32 max_ft_sz;
u32 max_grp_num;
struct mlx5_flow_table *default_ft;
struct mapping_ctx *mapping;
@ -68,6 +69,8 @@ void mlx5_chains_destroy(struct mlx5_fs_chains *chains);
void
mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
struct mlx5_flow_table *ft);
void
mlx5_chains_print_info(struct mlx5_fs_chains *chains);
#else /* CONFIG_MLX5_CLS_ACT */
@ -89,7 +92,9 @@ static inline struct mlx5_fs_chains *
mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
{ return NULL; }
static inline void
mlx5_chains_destroy(struct mlx5_fs_chains *chains) {};
mlx5_chains_destroy(struct mlx5_fs_chains *chains) {}
static inline void
mlx5_chains_print_info(struct mlx5_fs_chains *chains) {}
#endif /* CONFIG_MLX5_CLS_ACT */

5
include/net/xfrm.h
Просмотреть файл

@ -138,6 +138,10 @@ enum {
XFRM_DEV_OFFLOAD_PACKET,
};
enum {
XFRM_DEV_OFFLOAD_FLAG_ACQ = 1,
};
struct xfrm_dev_offload {
struct net_device *dev;
netdevice_tracker dev_tracker;
@ -145,6 +149,7 @@ struct xfrm_dev_offload {
unsigned long offload_handle;
u8 dir : 2;
u8 type : 2;
u8 flags : 2;
};
struct xfrm_mode {

1
net/xfrm/xfrm_state.c
Просмотреть файл

@ -1272,6 +1272,7 @@ found:
xso->dir = xdo->dir;
xso->dev = xdo->dev;
xso->real_dev = xdo->real_dev;
xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;
netdev_tracker_alloc(xso->dev, &xso->dev_tracker,
GFP_ATOMIC);
error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL);

2
net/xfrm/xfrm_user.c
Просмотреть файл

@ -901,6 +901,8 @@ static void copy_to_user_state(struct xfrm_state *x, struct xfrm_usersa_info *p)
memcpy(&p->id, &x->id, sizeof(p->id));
memcpy(&p->sel, &x->sel, sizeof(p->sel));
memcpy(&p->lft, &x->lft, sizeof(p->lft));
if (x->xso.dev)
xfrm_dev_state_update_curlft(x);
memcpy(&p->curlft, &x->curlft, sizeof(p->curlft));
put_unaligned(x->stats.replay_window, &p->stats.replay_window);
put_unaligned(x->stats.replay, &p->stats.replay);