Skip to content

Commit

Permalink
lib/net: add support to the PMD i40e
Browse files Browse the repository at this point in the history 
The PMD i40e does not support the RSS hash functions
ETH_RSS_IPV4 and ETH_RSS_IPV6 (i.e., RSS hash for IPv4 or IPv6
non-fragmented packets).  So Gatekeeper cannot work.

Nevertheless, i40e has an alternative way to set the needed
RSS hash.  This patch adds the parameter alternative_rss_hash to
struct gatekeeper_if to ask Gatekeeper to try this
alternative hash.
  • Loading branch information
@AltraMayor
AltraMayor committed Nov 4, 2022
1 parent 99b31a8 commit a601b26
Show file tree
Hide file tree
Showing 4 changed files with 174 additions and 36 deletions.
19 changes: 19 additions & 0 deletions include/gatekeeper_net.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,25 @@ struct gatekeeper_if {
*/
bool guarantee_random_entropy;

/*
* Some NICs do not support the RSS hash functions
* ETH_RSS_IPV4 amd ETH_RSS_IPV6 (i.e. RSS hash for IPv4 or IPv6
* non-fragmented packets). But they may support the hash functions
* ETH_RSS_NONFRAG_IPV4_TCP, ETH_RSS_NONFRAG_IPV4_UDP,
* ETH_RSS_NONFRAG_IPV6_TCP, and ETH_RSS_NONFRAG_IPV6_UDP, and
* setting the input set of the hash these hash functions.
* An example of this behavior is the PMD i40e.
*
* Enabling the parameter below, Gatekeeper will try the alternative
* RSS hash.
*
* Currently, this parameter only works for PMD i40e.
*
* If the interface is bonded, all ports in the bond must either
* need this parameter disabled or enabled.
*/
bool alternative_rss_hash;

/*
* The fields below are for internal use.
* Configuration files should not refer to them.
Expand Down
186 changes: 150 additions & 36 deletions lib/net.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include <rte_ethdev.h>
#include <rte_eth_bond.h>
#include <rte_malloc.h>
#include <rte_pmd_i40e.h>

#include "gatekeeper_acl.h"
#include "gatekeeper_main.h"
Expand Down Expand Up @@ -902,6 +903,105 @@ get_if_back(struct net_config *net_conf)
return net_conf->back_iface_enabled ? &net_conf->back : NULL;
}

static int
i40e_clear_inset_field(struct rte_pmd_i40e_inset *inset, uint8_t field_idx)
{
int ret = rte_pmd_i40e_inset_field_clear(&inset->inset, field_idx);
if (unlikely(ret < 0)) {
G_LOG(ERR, "%s(): cannot clear field %i\n",
__func__, field_idx);
}
return ret;
}

static int
i40e_disable_ports_from_inset(uint16_t port_id,
uint8_t pctype_id)
{
struct rte_pmd_i40e_inset inset;

/* Obtain the current RSS hash inset for @pctype_id. */
int ret = rte_pmd_i40e_inset_get(port_id, pctype_id, &inset,
INSET_HASH);
if (unlikely(ret < 0)) {
G_LOG(ERR, "%s(port_id=%i, pctype=%i): cannot get inset (errno=%i): %s\n",
__func__, port_id, pctype_id, -ret, rte_strerror(-ret));
return ret;
}

/*
* Remove source port (i.e. first two bytes) of @pctype_id.
*
* Field index obtained at "Table 7-12. Default field vector table"
* of "Intel Ethernet Controller X710/XXV710/XL710 Datasheet".
*
* The field index below works for @pctype_id equals to TCP or UDP.
*/
ret = i40e_clear_inset_field(&inset, 29);
if (unlikely(ret < 0))
return ret;

/*
* Remove destination port (i.e. third and forth bytes) of
* @pctype_id.
*/
ret = i40e_clear_inset_field(&inset, 30);
if (unlikely(ret < 0))
return ret;

ret = rte_pmd_i40e_inset_set(port_id, pctype_id, &inset, INSET_HASH);
if (unlikely(ret < 0)) {
G_LOG(ERR, "%s(port_id=%i, pctype=%i): cannot set inset (errno=%i): %s\n",
__func__, port_id, pctype_id, -ret, rte_strerror(-ret));
}
return ret;
}

static int
i40e_disable_pctypes_ports_from_inset(uint16_t port_id, uint8_t *pctypes,
uint8_t n)
{
int i;
for (i = 0; i < n; i++) {
int ret = i40e_disable_ports_from_inset(port_id, pctypes[i]);
if (unlikely(ret < 0))
return ret;
}
return 0;
}

static int
i40e_disable_ipv4_tcp_udp_ports_from_inset(uint16_t port_id)
{
/*
* PCTYPES obtained at "Table 7-5. Packet classifier types and
* its input sets" of "Intel Ethernet Controller X710/XXV710/XL710
* Datasheet".
*/
uint8_t pctypes[] = {
31, /* Non-fragmented IPv4, UDP. */
33, /* Non-fragmented IPv4, TCP. */
};
return i40e_disable_pctypes_ports_from_inset(port_id, pctypes,
RTE_DIM(pctypes));
}

static int
i40e_disable_ipv6_tcp_udp_ports_from_inset(uint16_t port_id)
{
/*
* PCTYPES obtained at "Table 7-5. Packet classifier types and
* its input sets" of "Intel Ethernet Controller X710/XXV710/XL710
* Datasheet".
*/
uint8_t pctypes[] = {
41, /* Non-fragmented IPv6, UDP. */
43, /* Non-fragmented IPv6, TCP. */
};
return i40e_disable_pctypes_ports_from_inset(port_id, pctypes,
RTE_DIM(pctypes));
}

/*
* Split up ETH_RSS_IP into IPv4-related and IPv6-related hash functions.
* For each type of IP being used in Gatekeeper, check the supported
Expand Down Expand Up @@ -934,21 +1034,21 @@ check_port_rss(struct gatekeeper_if *iface, unsigned int port_idx,
.rss_key_len = sizeof(rss_hash_key),
};
uint64_t rss_off = dev_info->flow_type_rss_offloads;
int ret = rte_eth_dev_rss_hash_conf_get(
iface->ports[port_idx], &__rss_conf);
uint16_t port_id = iface->ports[port_idx];
int ret = rte_eth_dev_rss_hash_conf_get(port_id, &__rss_conf);
if (ret == -ENOTSUP) {
G_LOG(WARNING, "%s(%s): port %hu (%s) does not support to get RSS configuration, disable RSS\n",
__func__, iface->name,
iface->ports[port_idx], iface->pci_addrs[port_idx]);
__func__, iface->name, port_id,
iface->pci_addrs[port_idx]);
goto disable_rss;
}

/* Do not use @rss_conf from now on. See issue #624 for details. */
/* Do not use @__rss_conf from now on. See issue #624 for details. */

if (ret < 0) {
G_LOG(ERR, "%s(%s): failed to get RSS hash configuration at port %hu (%s) (errno=%i): %s\n",
__func__, iface->name,
iface->ports[port_idx], iface->pci_addrs[port_idx],
__func__, iface->name, port_id,
iface->pci_addrs[port_idx],
-ret, rte_strerror(-ret));
return ret;
}
Expand All @@ -957,8 +1057,8 @@ check_port_rss(struct gatekeeper_if *iface, unsigned int port_idx,
/* This port doesn't support RSS, so disable RSS. */
if (rss_off == 0) {
G_LOG(WARNING, "%s(%s): port %hu (%s) does not support RSS\n",
__func__, iface->name,
iface->ports[port_idx], iface->pci_addrs[port_idx]);
__func__, iface->name, port_id,
iface->pci_addrs[port_idx]);
goto disable_rss;
}

Expand All @@ -967,8 +1067,8 @@ check_port_rss(struct gatekeeper_if *iface, unsigned int port_idx,
dev_info->hash_key_size > GATEKEEPER_RSS_MAX_KEY_LEN ||
dev_info->hash_key_size % 4 != 0) {
G_LOG(WARNING, "%s(%s): port %hu (%s) requires a RSS hash key of %i bytes; Gatekeeper only supports keys of [%i, %i] bytes long that are multiple of 4\n",
__func__, iface->name,
iface->ports[port_idx], iface->pci_addrs[port_idx],
__func__, iface->name, port_id,
iface->pci_addrs[port_idx],
dev_info->hash_key_size, GATEKEEPER_RSS_MIN_KEY_LEN,
GATEKEEPER_RSS_MAX_KEY_LEN);
goto disable_rss;
Expand All @@ -983,8 +1083,8 @@ check_port_rss(struct gatekeeper_if *iface, unsigned int port_idx,
if (iface->rss_key_len <= GATEKEEPER_RSS_MAX_KEY_LEN &&
iface->rss_key_len != dev_info->hash_key_size) {
G_LOG(WARNING, "%s(%s): port %hu (%s) requires a RSS hash key of %i bytes, but another port requires a key of %i bytes; all ports of the same interface must have the same key length\n",
__func__, iface->name,
iface->ports[port_idx], iface->pci_addrs[port_idx],
__func__, iface->name, port_id,
iface->pci_addrs[port_idx],
dev_info->hash_key_size, iface->rss_key_len);
goto disable_rss;
}
Expand All @@ -995,20 +1095,23 @@ check_port_rss(struct gatekeeper_if *iface, unsigned int port_idx,
/* No IPv4 hashes are supported, so disable RSS. */
if ((rss_off & GATEKEEPER_IPV4_RSS_HF) == 0) {
G_LOG(WARNING, "%s(%s): port %hu (%s) does not support any IPv4 related RSS hashes\n",
__func__, iface->name,
iface->ports[port_idx],
__func__, iface->name, port_id,
iface->pci_addrs[port_idx]);
goto disable_rss;
}

/*
* The IPv4 hash that we think is typically
* used is not supported, so warn the user.
*/
if ((rss_off & ETH_RSS_IPV4) == 0) {
G_LOG(WARNING, "%s(%s): port %hu (%s) does not support the ETH_RSS_IPV4 hash function; the device may not hash packets to the correct queues\n",
__func__, iface->name,
iface->ports[port_idx],
if (iface->alternative_rss_hash) {
ret = i40e_disable_ipv4_tcp_udp_ports_from_inset(
port_id);
if (ret < 0)
goto disable_rss;
} else if ((rss_off & ETH_RSS_IPV4) == 0) {
/*
* The IPv4 hash that we think is typically
* used is not supported, so warn the user.
*/
G_LOG(WARNING, "%s(%s): port %hu (%s) does not support the ETH_RSS_IPV4 hash function. The device may not hash packets to the correct queues. You may try parameter alternative_rss_hash.\n",
__func__, iface->name, port_id,
iface->pci_addrs[port_idx]);
}
}
Expand All @@ -1018,20 +1121,23 @@ check_port_rss(struct gatekeeper_if *iface, unsigned int port_idx,
/* No IPv6 hashes are supported, so disable RSS. */
if ((rss_off & GATEKEEPER_IPV6_RSS_HF) == 0) {
G_LOG(WARNING, "%s(%s): port %hu (%s) does not support any IPv6 related RSS hashes\n",
__func__, iface->name,
iface->ports[port_idx],
__func__, iface->name, port_id,
iface->pci_addrs[port_idx]);
goto disable_rss;
}

/*
* The IPv6 hash that we think is typically
* used is not supported, so warn the user.
*/
if ((rss_off & ETH_RSS_IPV6) == 0) {
G_LOG(WARNING, "%s(%s): port %hu (%s) does not support the ETH_RSS_IPV6 hash function; the device may not hash packets to the correct queues\n",
__func__, iface->name,
iface->ports[port_idx],
if (iface->alternative_rss_hash) {
ret = i40e_disable_ipv6_tcp_udp_ports_from_inset(
port_id);
if (ret < 0)
goto disable_rss;
} else if ((rss_off & ETH_RSS_IPV6) == 0) {
/*
* The IPv6 hash that we think is typically
* used is not supported, so warn the user.
*/
G_LOG(WARNING, "%s(%s): port %hu (%s) does not support the ETH_RSS_IPV6 hash function. The device may not hash packets to the correct queues. You may try parameter alternative_rss_hash.\n",
__func__, iface->name, port_id,
iface->pci_addrs[port_idx]);
}
}
Expand All @@ -1045,9 +1151,9 @@ check_port_rss(struct gatekeeper_if *iface, unsigned int port_idx,
if ((rss_off & port_conf->rx_adv_conf.rss_conf.rss_hf) !=
port_conf->rx_adv_conf.rss_conf.rss_hf) {
G_LOG(WARNING, "%s(%s): port %hu (%s) only supports RSS hash functions 0x%"PRIx64", but Gatekeeper asks for 0x%"PRIx64"\n",
__func__, iface->name,
iface->ports[port_idx], iface->pci_addrs[port_idx],
rss_off, port_conf->rx_adv_conf.rss_conf.rss_hf);
__func__, iface->name, port_id,
iface->pci_addrs[port_idx], rss_off,
port_conf->rx_adv_conf.rss_conf.rss_hf);
}

port_conf->rx_adv_conf.rss_conf.rss_hf &= rss_off;
Expand Down Expand Up @@ -1195,10 +1301,18 @@ check_port_offloads(struct gatekeeper_if *iface,
if (ipv4_if_configured(iface)) {
port_conf->rx_adv_conf.rss_conf.rss_hf |=
GATEKEEPER_IPV4_RSS_HF;
if (iface->alternative_rss_hash)
port_conf->rx_adv_conf.rss_conf.rss_hf |=
ETH_RSS_NONFRAG_IPV4_TCP |
ETH_RSS_NONFRAG_IPV4_UDP;
}
if (ipv6_if_configured(iface)) {
port_conf->rx_adv_conf.rss_conf.rss_hf |=
GATEKEEPER_IPV6_RSS_HF;
if (iface->alternative_rss_hash)
port_conf->rx_adv_conf.rss_conf.rss_hf |=
ETH_RSS_NONFRAG_IPV6_TCP |
ETH_RSS_NONFRAG_IPV6_UDP;
}

/*
Expand Down
1 change: 1 addition & 0 deletions lua/gatekeeper/staticlib.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,7 @@ struct gatekeeper_if {
bool ipv6_hw_udp_cksum;
bool ipv4_hw_cksum;
bool guarantee_random_entropy;
bool alternative_rss_hash;
/* This struct has hidden fields. */
};

Expand Down
4 changes: 4 additions & 0 deletions lua/net.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ return function (gatekeeper_server)
local back_ipv6_hw_udp_cksum = true
local front_ipv4_hw_cksum = true
local back_ipv4_hw_cksum = true
local front_alternative_rss_hash = false
local back_alternative_rss_hash = false

--
-- End configuration of the network.
Expand Down Expand Up @@ -76,6 +78,7 @@ return function (gatekeeper_server)
front_iface.ipv6_hw_udp_cksum = front_ipv6_hw_udp_cksum
front_iface.ipv4_hw_cksum = front_ipv4_hw_cksum
front_iface.guarantee_random_entropy = guarantee_random_entropy
front_iface.alternative_rss_hash = front_alternative_rss_hash
local ret = staticlib.init_iface(front_iface, "front",
front_ports, front_ips, front_ipv4_vlan_tag,
front_ipv6_vlan_tag)
Expand All @@ -99,6 +102,7 @@ return function (gatekeeper_server)
back_iface.ipv6_hw_udp_cksum = back_ipv6_hw_udp_cksum
back_iface.ipv4_hw_cksum = back_ipv4_hw_cksum
back_iface.guarantee_random_entropy = guarantee_random_entropy
back_iface.alternative_rss_hash = back_alternative_rss_hash
ret = staticlib.init_iface(back_iface, "back",
back_ports, back_ips, back_ipv4_vlan_tag,
back_ipv6_vlan_tag)
Expand Down

0 comments on commit a601b26

Please sign in to comment.