Merge branch 'skbuff-micro-optimize-flow-dissection'
Alexander Lobakin says: ==================== skbuff: micro-optimize flow dissection This little number makes all of the flow dissection functions take raw input data pointer as const (1-5) and shuffles the branches in __skb_header_pointer() according to their hit probability. The result is +20 Mbps per flow/core with one Flow Dissector pass per packet. This affects RPS (with software hashing), drivers that use eth_get_headlen() on their Rx path and so on. From v2 [1]: - reword some commit messages as a potential fix for NIPA; - no functional changes. From v1 [0]: - rebase on top of the latest net-next. This was super-weird, but I double-checked that the series applies with no conflicts, and then on Patchwork it didn't; - no other changes. [0] https://lore.kernel.org/netdev/20210312194538.337504-1-alobakin@pm.me [1] https://lore.kernel.org/netdev/20210313113645.5949-1-alobakin@pm.me ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller
Коммит
c6baf7eeb0
|
|
@ -11,7 +11,7 @@
|
|||
* Authors: Ross Biro
|
||||
* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
|
||||
*
|
||||
* Relocated to include/linux where it belongs by Alan Cox
|
||||
* Relocated to include/linux where it belongs by Alan Cox
|
||||
* <gw4pts@gw4pts.ampr.org>
|
||||
*/
|
||||
#ifndef _LINUX_ETHERDEVICE_H
|
||||
|
|
@ -29,7 +29,7 @@ struct device;
|
|||
int eth_platform_get_mac_address(struct device *dev, u8 *mac_addr);
|
||||
unsigned char *arch_get_platform_mac_address(void);
|
||||
int nvmem_get_mac_address(struct device *dev, void *addrbuf);
|
||||
u32 eth_get_headlen(const struct net_device *dev, void *data, unsigned int len);
|
||||
u32 eth_get_headlen(const struct net_device *dev, const void *data, u32 len);
|
||||
__be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev);
|
||||
extern const struct header_ops eth_header_ops;
|
||||
|
||||
|
|
|
|||
|
|
@ -1292,10 +1292,10 @@ __skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
|
|||
void __skb_get_hash(struct sk_buff *skb);
|
||||
u32 __skb_get_hash_symmetric(const struct sk_buff *skb);
|
||||
u32 skb_get_poff(const struct sk_buff *skb);
|
||||
u32 __skb_get_poff(const struct sk_buff *skb, void *data,
|
||||
u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
|
||||
const struct flow_keys_basic *keys, int hlen);
|
||||
__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
|
||||
void *data, int hlen_proto);
|
||||
const void *data, int hlen_proto);
|
||||
|
||||
static inline __be32 skb_flow_get_ports(const struct sk_buff *skb,
|
||||
int thoff, u8 ip_proto)
|
||||
|
|
@ -1314,9 +1314,8 @@ bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
|
|||
bool __skb_flow_dissect(const struct net *net,
|
||||
const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container,
|
||||
void *data, __be16 proto, int nhoff, int hlen,
|
||||
unsigned int flags);
|
||||
void *target_container, const void *data,
|
||||
__be16 proto, int nhoff, int hlen, unsigned int flags);
|
||||
|
||||
static inline bool skb_flow_dissect(const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
|
|
@ -1338,9 +1337,9 @@ static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
|
|||
static inline bool
|
||||
skb_flow_dissect_flow_keys_basic(const struct net *net,
|
||||
const struct sk_buff *skb,
|
||||
struct flow_keys_basic *flow, void *data,
|
||||
__be16 proto, int nhoff, int hlen,
|
||||
unsigned int flags)
|
||||
struct flow_keys_basic *flow,
|
||||
const void *data, __be16 proto,
|
||||
int nhoff, int hlen, unsigned int flags)
|
||||
{
|
||||
memset(flow, 0, sizeof(*flow));
|
||||
return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow,
|
||||
|
|
@ -3678,14 +3677,13 @@ __wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
|
|||
__wsum csum);
|
||||
|
||||
static inline void * __must_check
|
||||
__skb_header_pointer(const struct sk_buff *skb, int offset,
|
||||
int len, void *data, int hlen, void *buffer)
|
||||
__skb_header_pointer(const struct sk_buff *skb, int offset, int len,
|
||||
const void *data, int hlen, void *buffer)
|
||||
{
|
||||
if (hlen - offset >= len)
|
||||
return data + offset;
|
||||
if (likely(hlen - offset >= len))
|
||||
return (void *)data + offset;
|
||||
|
||||
if (!skb ||
|
||||
skb_copy_bits(skb, offset, buffer, len) < 0)
|
||||
if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0))
|
||||
return NULL;
|
||||
|
||||
return buffer;
|
||||
|
|
|
|||
|
|
@ -350,7 +350,7 @@ static inline bool flow_keys_have_l4(const struct flow_keys *keys)
|
|||
u32 flow_hash_from_keys(struct flow_keys *keys);
|
||||
void skb_flow_get_icmp_tci(const struct sk_buff *skb,
|
||||
struct flow_dissector_key_icmp *key_icmp,
|
||||
void *data, int thoff, int hlen);
|
||||
const void *data, int thoff, int hlen);
|
||||
|
||||
static inline bool dissector_uses_key(const struct flow_dissector *flow_dissector,
|
||||
enum flow_dissector_key_id key_id)
|
||||
|
|
@ -368,8 +368,8 @@ static inline void *skb_flow_dissector_target(struct flow_dissector *flow_dissec
|
|||
struct bpf_flow_dissector {
|
||||
struct bpf_flow_keys *flow_keys;
|
||||
const struct sk_buff *skb;
|
||||
void *data;
|
||||
void *data_end;
|
||||
const void *data;
|
||||
const void *data_end;
|
||||
};
|
||||
|
||||
static inline void
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ int flow_dissector_bpf_prog_attach_check(struct net *net,
|
|||
* is the protocol port offset returned from proto_ports_offset
|
||||
*/
|
||||
__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
|
||||
void *data, int hlen)
|
||||
const void *data, int hlen)
|
||||
{
|
||||
int poff = proto_ports_offset(ip_proto);
|
||||
|
||||
|
|
@ -161,7 +161,7 @@ static bool icmp_has_id(u8 type)
|
|||
*/
|
||||
void skb_flow_get_icmp_tci(const struct sk_buff *skb,
|
||||
struct flow_dissector_key_icmp *key_icmp,
|
||||
void *data, int thoff, int hlen)
|
||||
const void *data, int thoff, int hlen)
|
||||
{
|
||||
struct icmphdr *ih, _ih;
|
||||
|
||||
|
|
@ -187,8 +187,8 @@ EXPORT_SYMBOL(skb_flow_get_icmp_tci);
|
|||
*/
|
||||
static void __skb_flow_dissect_icmp(const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container,
|
||||
void *data, int thoff, int hlen)
|
||||
void *target_container, const void *data,
|
||||
int thoff, int hlen)
|
||||
{
|
||||
struct flow_dissector_key_icmp *key_icmp;
|
||||
|
||||
|
|
@ -409,8 +409,8 @@ EXPORT_SYMBOL(skb_flow_dissect_hash);
|
|||
static enum flow_dissect_ret
|
||||
__skb_flow_dissect_mpls(const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container, void *data, int nhoff, int hlen,
|
||||
int lse_index, bool *entropy_label)
|
||||
void *target_container, const void *data, int nhoff,
|
||||
int hlen, int lse_index, bool *entropy_label)
|
||||
{
|
||||
struct mpls_label *hdr, _hdr;
|
||||
u32 entry, label, bos;
|
||||
|
|
@ -467,7 +467,8 @@ __skb_flow_dissect_mpls(const struct sk_buff *skb,
|
|||
static enum flow_dissect_ret
|
||||
__skb_flow_dissect_arp(const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container, void *data, int nhoff, int hlen)
|
||||
void *target_container, const void *data,
|
||||
int nhoff, int hlen)
|
||||
{
|
||||
struct flow_dissector_key_arp *key_arp;
|
||||
struct {
|
||||
|
|
@ -523,7 +524,7 @@ static enum flow_dissect_ret
|
|||
__skb_flow_dissect_gre(const struct sk_buff *skb,
|
||||
struct flow_dissector_key_control *key_control,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container, void *data,
|
||||
void *target_container, const void *data,
|
||||
__be16 *p_proto, int *p_nhoff, int *p_hlen,
|
||||
unsigned int flags)
|
||||
{
|
||||
|
|
@ -663,8 +664,8 @@ __skb_flow_dissect_gre(const struct sk_buff *skb,
|
|||
static enum flow_dissect_ret
|
||||
__skb_flow_dissect_batadv(const struct sk_buff *skb,
|
||||
struct flow_dissector_key_control *key_control,
|
||||
void *data, __be16 *p_proto, int *p_nhoff, int hlen,
|
||||
unsigned int flags)
|
||||
const void *data, __be16 *p_proto, int *p_nhoff,
|
||||
int hlen, unsigned int flags)
|
||||
{
|
||||
struct {
|
||||
struct batadv_unicast_packet batadv_unicast;
|
||||
|
|
@ -695,7 +696,8 @@ __skb_flow_dissect_batadv(const struct sk_buff *skb,
|
|||
static void
|
||||
__skb_flow_dissect_tcp(const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container, void *data, int thoff, int hlen)
|
||||
void *target_container, const void *data,
|
||||
int thoff, int hlen)
|
||||
{
|
||||
struct flow_dissector_key_tcp *key_tcp;
|
||||
struct tcphdr *th, _th;
|
||||
|
|
@ -719,8 +721,8 @@ __skb_flow_dissect_tcp(const struct sk_buff *skb,
|
|||
static void
|
||||
__skb_flow_dissect_ports(const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container, void *data, int nhoff,
|
||||
u8 ip_proto, int hlen)
|
||||
void *target_container, const void *data,
|
||||
int nhoff, u8 ip_proto, int hlen)
|
||||
{
|
||||
enum flow_dissector_key_id dissector_ports = FLOW_DISSECTOR_KEY_MAX;
|
||||
struct flow_dissector_key_ports *key_ports;
|
||||
|
|
@ -744,7 +746,8 @@ __skb_flow_dissect_ports(const struct sk_buff *skb,
|
|||
static void
|
||||
__skb_flow_dissect_ipv4(const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container, void *data, const struct iphdr *iph)
|
||||
void *target_container, const void *data,
|
||||
const struct iphdr *iph)
|
||||
{
|
||||
struct flow_dissector_key_ip *key_ip;
|
||||
|
||||
|
|
@ -761,7 +764,8 @@ __skb_flow_dissect_ipv4(const struct sk_buff *skb,
|
|||
static void
|
||||
__skb_flow_dissect_ipv6(const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container, void *data, const struct ipv6hdr *iph)
|
||||
void *target_container, const void *data,
|
||||
const struct ipv6hdr *iph)
|
||||
{
|
||||
struct flow_dissector_key_ip *key_ip;
|
||||
|
||||
|
|
@ -908,9 +912,8 @@ bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
|
|||
bool __skb_flow_dissect(const struct net *net,
|
||||
const struct sk_buff *skb,
|
||||
struct flow_dissector *flow_dissector,
|
||||
void *target_container,
|
||||
void *data, __be16 proto, int nhoff, int hlen,
|
||||
unsigned int flags)
|
||||
void *target_container, const void *data,
|
||||
__be16 proto, int nhoff, int hlen, unsigned int flags)
|
||||
{
|
||||
struct flow_dissector_key_control *key_control;
|
||||
struct flow_dissector_key_basic *key_basic;
|
||||
|
|
@ -1642,7 +1645,7 @@ __u32 skb_get_hash_perturb(const struct sk_buff *skb,
|
|||
}
|
||||
EXPORT_SYMBOL(skb_get_hash_perturb);
|
||||
|
||||
u32 __skb_get_poff(const struct sk_buff *skb, void *data,
|
||||
u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
|
||||
const struct flow_keys_basic *keys, int hlen)
|
||||
{
|
||||
u32 poff = keys->control.thoff;
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ EXPORT_SYMBOL(eth_header);
|
|||
* Make a best effort attempt to pull the length for all of the headers for
|
||||
* a given frame in a linear buffer.
|
||||
*/
|
||||
u32 eth_get_headlen(const struct net_device *dev, void *data, unsigned int len)
|
||||
u32 eth_get_headlen(const struct net_device *dev, const void *data, u32 len)
|
||||
{
|
||||
const unsigned int flags = FLOW_DISSECTOR_F_PARSE_1ST_FRAG;
|
||||
const struct ethhdr *eth = (const struct ethhdr *)data;
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче