diff --git a/modules/infra/datapath/bond_output.c b/modules/infra/datapath/bond_output.c index cc33530b1..8ed8dac11 100644 --- a/modules/infra/datapath/bond_output.c +++ b/modules/infra/datapath/bond_output.c @@ -2,18 +2,12 @@ // Copyright (c) 2025 Robin Jarry #include "bond.h" +#include "flow_hash.h" #include "graph.h" #include "iface.h" #include "mbuf.h" #include "rxtx.h" -#include -#include -#include -#include -#include -#include - #include enum { @@ -33,33 +27,9 @@ static int bond_trace_format(char *buf, size_t len, const void *data, size_t /*d } static inline const struct iface * -hash_tx_member(const struct rte_mbuf *m, const struct iface_info_bond *bond) { - static const uint8_t rss_key[] = { - 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, - 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, - 0x80, 0x30, 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, - }; - union { - uint32_t u32; - struct { - struct rte_ether_addr mac; - rte_be16_t vlan_id; - } l2; - struct rte_ipv4_tuple v4; - struct rte_ipv6_tuple v6; - } tuple; - union { - const struct rte_ipv4_hdr *ip4; - const struct rte_ipv6_hdr *ip6; - } l3; - union { - const struct rte_udp_hdr *udp; - const struct rte_tcp_hdr *tcp; - } l4; - const struct rte_ether_hdr *eth; - const struct rte_vlan_hdr *vlan; - uint32_t l3_offset, len, hash; - rte_be16_t eth_type; +hash_tx_member(struct rte_mbuf *m, const struct iface_info_bond *bond) { + gr_mbuf_flow_hash_mode_t mode; + uint32_t hash; uint8_t member; if (bond->n_members == 0) @@ -67,115 +37,22 @@ hash_tx_member(const struct rte_mbuf *m, const struct iface_info_bond *bond) { switch (bond->algo) { case GR_BOND_ALGO_L2: - eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); - tuple.l2.mac = eth->dst_addr; - if (eth->ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN)) { - vlan = PAYLOAD(eth); - tuple.l2.vlan_id = vlan->vlan_tci; - } else { - tuple.l2.vlan_id = 0; - } - len = sizeof(tuple.l2); + mode = GR_MBUF_FLOW_HASH_L2; break; case GR_BOND_ALGO_RSS: - if (m->ol_flags & RTE_MBUF_F_RX_RSS_HASH) { - hash = m->hash.rss; - goto out; - } - // fallthrough + mode = GR_MBUF_FLOW_HASH_RSS; + break; case GR_BOND_ALGO_L3_L4: - eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); - tuple.l2.mac = eth->dst_addr; - if (eth->ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN)) { - vlan = PAYLOAD(eth); - tuple.l2.vlan_id = vlan->vlan_tci; - eth_type = vlan->eth_proto; - l3_offset = sizeof(*eth) + sizeof(*vlan); - } else { - tuple.l2.vlan_id = 0; - eth_type = eth->ether_type; - l3_offset = sizeof(*eth); - } - switch (eth_type) { - case RTE_BE16(RTE_ETHER_TYPE_IPV4): { - l3.ip4 = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, l3_offset); - tuple.v4.src_addr = l3.ip4->src_addr; - tuple.v4.dst_addr = l3.ip4->dst_addr; - switch (l3.ip4->next_proto_id) { - case IPPROTO_UDP: - if (l3.ip4->fragment_offset == 0) { - l4.udp = rte_pktmbuf_mtod_offset( - m, - struct rte_udp_hdr *, - l3_offset + rte_ipv4_hdr_len(l3.ip4) - ); - tuple.v4.sport = l4.udp->src_port; - tuple.v4.dport = l4.udp->dst_port; - } else { - // ignore UDP header for IP fragments - tuple.v4.sport = 0; - tuple.v4.dport = 0; - } - break; - case IPPROTO_TCP: - if (l3.ip4->fragment_offset == 0) { - l4.tcp = rte_pktmbuf_mtod_offset( - m, - struct rte_tcp_hdr *, - l3_offset + rte_ipv4_hdr_len(l3.ip4) - ); - tuple.v4.sport = l4.tcp->src_port; - tuple.v4.dport = l4.tcp->dst_port; - } else { - // ignore TCP header for IP fragments - tuple.v4.sport = 0; - tuple.v4.dport = 0; - } - break; - default: - tuple.v4.sport = 0; - tuple.v4.dport = 0; - } - len = sizeof(tuple.v4); - break; - } - case RTE_BE16(RTE_ETHER_TYPE_IPV6): { - l3.ip6 = rte_pktmbuf_mtod_offset(m, const struct rte_ipv6_hdr *, l3_offset); - tuple.v6.src_addr = l3.ip6->src_addr; - tuple.v6.dst_addr = l3.ip6->dst_addr; - switch (l3.ip6->proto) { - case IPPROTO_UDP: - l4.udp = rte_pktmbuf_mtod_offset( - m, struct rte_udp_hdr *, l3_offset + sizeof(*l3.ip6) - ); - tuple.v6.sport = l4.udp->src_port; - tuple.v6.dport = l4.udp->dst_port; - break; - case IPPROTO_TCP: - l4.tcp = rte_pktmbuf_mtod_offset( - m, struct rte_tcp_hdr *, l3_offset + sizeof(*l3.ip6) - ); - tuple.v6.sport = l4.tcp->src_port; - tuple.v6.dport = l4.tcp->dst_port; - break; - default: - tuple.v6.sport = 0; - tuple.v6.dport = 0; - } - len = sizeof(tuple.v6); - break; - } - default: - len = sizeof(tuple.l2); - break; - } + mode = GR_MBUF_FLOW_HASH_L3_L4; break; default: return NULL; } - hash = rte_softrss_be(&tuple.u32, len / sizeof(uint32_t), rss_key); -out: + if (mode == GR_MBUF_FLOW_HASH_RSS) + hash = gr_mbuf_flow_hash_get(m); + else + hash = gr_mbuf_flow_hash(m, mode); member = bond->redirection_table[hash % ARRAY_DIM(bond->redirection_table)]; if (member < bond->n_members) return bond->members[member].iface; @@ -183,7 +60,7 @@ hash_tx_member(const struct rte_mbuf *m, const struct iface_info_bond *bond) { } static inline const struct iface * -bond_select_tx_member(const struct rte_mbuf *m, const struct iface_info_bond *bond) { +bond_select_tx_member(struct rte_mbuf *m, const struct iface_info_bond *bond) { switch (bond->mode) { case GR_BOND_MODE_ACTIVE_BACKUP: { uint8_t active = bond->active_member; diff --git a/modules/infra/datapath/flow_hash.c b/modules/infra/datapath/flow_hash.c new file mode 100644 index 000000000..ba29dc3d9 --- /dev/null +++ b/modules/infra/datapath/flow_hash.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 Harrison Caldicott + +#include "flow_hash.h" +#include "mbuf.h" + +#include + +#include +#include +#include +#include +#include +#include + +static const uint8_t rss_key[] = { + 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, + 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, + 0x80, 0x30, 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, +}; + +static uint32_t flow_hash_l3(const struct rte_mbuf *m, uint32_t l3_offset, rte_be16_t eth_type) { + union { + uint32_t u32; + struct rte_ipv4_tuple v4; + struct rte_ipv6_tuple v6; + } tuple = {}; + union { + const struct rte_ipv4_hdr *ip4; + const struct rte_ipv6_hdr *ip6; + } l3; + union { + const struct rte_udp_hdr *udp; + const struct rte_tcp_hdr *tcp; + } l4; + uint32_t len; + + switch (eth_type) { + case RTE_BE16(RTE_ETHER_TYPE_IPV4): + l3.ip4 = rte_pktmbuf_mtod_offset(m, const struct rte_ipv4_hdr *, l3_offset); + tuple.v4.src_addr = l3.ip4->src_addr; + tuple.v4.dst_addr = l3.ip4->dst_addr; + switch (l3.ip4->next_proto_id) { + case IPPROTO_UDP: + if ((rte_be_to_cpu_16(l3.ip4->fragment_offset) + & (RTE_IPV4_HDR_OFFSET_MASK | RTE_IPV4_HDR_MF_FLAG)) + == 0) { + l4.udp = rte_pktmbuf_mtod_offset( + m, + const struct rte_udp_hdr *, + l3_offset + rte_ipv4_hdr_len(l3.ip4) + ); + tuple.v4.sport = l4.udp->src_port; + tuple.v4.dport = l4.udp->dst_port; + } else { + tuple.v4.sport = 0; + tuple.v4.dport = 0; + } + break; + case IPPROTO_TCP: + if ((rte_be_to_cpu_16(l3.ip4->fragment_offset) + & (RTE_IPV4_HDR_OFFSET_MASK | RTE_IPV4_HDR_MF_FLAG)) + == 0) { + l4.tcp = rte_pktmbuf_mtod_offset( + m, + const struct rte_tcp_hdr *, + l3_offset + rte_ipv4_hdr_len(l3.ip4) + ); + tuple.v4.sport = l4.tcp->src_port; + tuple.v4.dport = l4.tcp->dst_port; + } else { + tuple.v4.sport = 0; + tuple.v4.dport = 0; + } + break; + default: + tuple.v4.sport = 0; + tuple.v4.dport = 0; + } + len = sizeof(tuple.v4); + break; + case RTE_BE16(RTE_ETHER_TYPE_IPV6): + l3.ip6 = rte_pktmbuf_mtod_offset(m, const struct rte_ipv6_hdr *, l3_offset); + tuple.v6.src_addr = l3.ip6->src_addr; + tuple.v6.dst_addr = l3.ip6->dst_addr; + switch (l3.ip6->proto) { + case IPPROTO_UDP: + l4.udp = rte_pktmbuf_mtod_offset( + m, const struct rte_udp_hdr *, l3_offset + sizeof(*l3.ip6) + ); + tuple.v6.sport = l4.udp->src_port; + tuple.v6.dport = l4.udp->dst_port; + break; + case IPPROTO_TCP: + l4.tcp = rte_pktmbuf_mtod_offset( + m, const struct rte_tcp_hdr *, l3_offset + sizeof(*l3.ip6) + ); + tuple.v6.sport = l4.tcp->src_port; + tuple.v6.dport = l4.tcp->dst_port; + break; + default: + tuple.v6.sport = 0; + tuple.v6.dport = 0; + } + len = sizeof(tuple.v6); + break; + default: + return 0; + } + + return rte_softrss_be(&tuple.u32, len / sizeof(uint32_t), rss_key); +} + +uint32_t gr_mbuf_flow_hash(const struct rte_mbuf *m, gr_mbuf_flow_hash_mode_t mode) { + union { + uint32_t u32; + struct { + struct rte_ether_addr mac; + rte_be16_t vlan_id; + } l2; + } tuple; + const struct rte_ether_hdr *eth; + const struct rte_vlan_hdr *vlan; + uint32_t l3_offset; + rte_be16_t eth_type; + + if (mode == GR_MBUF_FLOW_HASH_RSS && (m->ol_flags & RTE_MBUF_F_RX_RSS_HASH)) + return m->hash.rss; + + eth = rte_pktmbuf_mtod(m, const struct rte_ether_hdr *); + tuple.l2.mac = eth->dst_addr; + if (eth->ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN)) { + vlan = PAYLOAD(eth); + tuple.l2.vlan_id = vlan->vlan_tci; + eth_type = vlan->eth_proto; + l3_offset = sizeof(*eth) + sizeof(*vlan); + } else { + tuple.l2.vlan_id = 0; + eth_type = eth->ether_type; + l3_offset = sizeof(*eth); + } + + if (mode == GR_MBUF_FLOW_HASH_L2 + || (eth_type != RTE_BE16(RTE_ETHER_TYPE_IPV4) + && eth_type != RTE_BE16(RTE_ETHER_TYPE_IPV6))) + return rte_softrss_be(&tuple.u32, sizeof(tuple.l2) / sizeof(uint32_t), rss_key); + + return flow_hash_l3(m, l3_offset, eth_type); +} + +uint32_t gr_mbuf_flow_hash_l3(const struct rte_mbuf *m, rte_be16_t eth_type) { + return flow_hash_l3(m, 0, eth_type); +} diff --git a/modules/infra/datapath/flow_hash.h b/modules/infra/datapath/flow_hash.h new file mode 100644 index 000000000..2a88cec06 --- /dev/null +++ b/modules/infra/datapath/flow_hash.h @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 Harrison Caldicott + +#pragma once + +#include + +#include +#include + +typedef enum : uint8_t { + GR_MBUF_FLOW_HASH_L2, + GR_MBUF_FLOW_HASH_L3_L4, + GR_MBUF_FLOW_HASH_RSS, +} gr_mbuf_flow_hash_mode_t; + +// Calculate a packet-flow hash without caching it. The packet data must start +// with an Ethernet header. RSS mode uses a hardware hash when present and falls +// back to a software L3/L4 hash for virtual devices without RSS. +uint32_t gr_mbuf_flow_hash(const struct rte_mbuf *, gr_mbuf_flow_hash_mode_t); + +static inline bool gr_mbuf_flow_hash_is_valid(const struct rte_mbuf *m) { + return m->ol_flags & RTE_MBUF_F_RX_RSS_HASH; +} + +// Store a software flow hash where a hardware driver would have put its RSS +// value, so every consumer reads m->hash.rss the same way. +static inline void gr_mbuf_flow_hash_set(struct rte_mbuf *m, uint32_t hash) { + m->hash.rss = hash; + m->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; +} + +// Drop a hash that no longer describes the packet, typically after tunnel +// decapsulation exposed an inner frame. The next consumer recomputes a +// software hash on demand. +static inline void gr_mbuf_flow_hash_invalidate(struct rte_mbuf *m) { + m->ol_flags &= ~RTE_MBUF_F_RX_RSS_HASH; +} + +// Same as gr_mbuf_flow_hash() in L3/L4 mode for packets whose data starts +// with an IPv4 or IPv6 header. eth_type must match that header. +uint32_t gr_mbuf_flow_hash_l3(const struct rte_mbuf *, rte_be16_t eth_type); + +// Return the flow hash of an Ethernet-framed packet. A hash already present +// in m->hash.rss is returned as is; otherwise a software L3/L4 hash is +// calculated once and cached there. +static inline uint32_t gr_mbuf_flow_hash_get(struct rte_mbuf *m) { + if (!gr_mbuf_flow_hash_is_valid(m)) + gr_mbuf_flow_hash_set(m, gr_mbuf_flow_hash(m, GR_MBUF_FLOW_HASH_L3_L4)); + return m->hash.rss; +} + +// Same as gr_mbuf_flow_hash_get() for packets whose data starts with an IPv4 +// or IPv6 header. eth_type must match that header. +static inline uint32_t gr_mbuf_flow_hash_get_l3(struct rte_mbuf *m, rte_be16_t eth_type) { + if (!gr_mbuf_flow_hash_is_valid(m)) + gr_mbuf_flow_hash_set(m, gr_mbuf_flow_hash_l3(m, eth_type)); + return m->hash.rss; +} diff --git a/modules/infra/datapath/flow_hash_test.c b/modules/infra/datapath/flow_hash_test.c new file mode 100644 index 000000000..db69c6823 --- /dev/null +++ b/modules/infra/datapath/flow_hash_test.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 Harrison Caldicott + +#include "_cmocka.h" +#include "flow_hash.h" + +#include +#include +#include +#include + +#include +#include + +struct test_packet { + struct rte_mbuf mbuf; + struct rte_ether_hdr eth; + struct rte_ipv4_hdr ip; + struct rte_udp_hdr udp; +}; + +struct test_packet6 { + struct rte_mbuf mbuf; + struct rte_ether_hdr eth; + struct rte_ipv6_hdr ip; + struct rte_udp_hdr udp; +}; + +static void packet_init(struct test_packet *p, rte_be16_t src_port) { + memset(p, 0, sizeof(*p)); + p->mbuf.buf_addr = &p->eth; + p->mbuf.data_len = sizeof(*p) - offsetof(struct test_packet, eth); + p->mbuf.pkt_len = p->mbuf.data_len; + p->eth.dst_addr = (struct rte_ether_addr) {{0x02, 0, 0, 0, 0, 2}}; + p->eth.src_addr = (struct rte_ether_addr) {{0x02, 0, 0, 0, 0, 1}}; + p->eth.ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4); + p->ip.version_ihl = RTE_IPV4_VHL_DEF; + p->ip.fragment_offset = RTE_BE16(RTE_IPV4_HDR_DF_FLAG); + p->ip.next_proto_id = IPPROTO_UDP; + p->ip.src_addr = RTE_BE32(0x0a000001); + p->ip.dst_addr = RTE_BE32(0x0a000002); + p->udp.src_port = src_port; + p->udp.dst_port = RTE_BE16(9000); +} + +static void packet6_init(struct test_packet6 *p, rte_be16_t src_port) { + memset(p, 0, sizeof(*p)); + p->mbuf.buf_addr = &p->eth; + p->mbuf.data_len = sizeof(*p) - offsetof(struct test_packet6, eth); + p->mbuf.pkt_len = p->mbuf.data_len; + p->eth.ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6); + p->ip.vtc_flow = RTE_BE32(6 << 28); + p->ip.proto = IPPROTO_UDP; + p->ip.src_addr = (struct rte_ipv6_addr)RTE_IPV6(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1); + p->ip.dst_addr = (struct rte_ipv6_addr)RTE_IPV6(0x2001, 0xdb8, 0, 0, 0, 0, 0, 2); + p->udp.src_port = src_port; + p->udp.dst_port = RTE_BE16(9000); +} + +static void software_hash_is_stable_per_flow(void **) { + struct test_packet a, b; + + packet_init(&a, RTE_BE16(20000)); + packet_init(&b, RTE_BE16(20000)); + assert_int_equal( + gr_mbuf_flow_hash(&a.mbuf, GR_MBUF_FLOW_HASH_RSS), + gr_mbuf_flow_hash(&b.mbuf, GR_MBUF_FLOW_HASH_RSS) + ); +} + +static void software_hash_distinguishes_udp_flows(void **) { + struct test_packet a, b; + + packet_init(&a, RTE_BE16(20000)); + packet_init(&b, RTE_BE16(20001)); + assert_int_not_equal( + gr_mbuf_flow_hash(&a.mbuf, GR_MBUF_FLOW_HASH_RSS), + gr_mbuf_flow_hash(&b.mbuf, GR_MBUF_FLOW_HASH_RSS) + ); + assert_int_equal( + gr_mbuf_flow_hash(&a.mbuf, GR_MBUF_FLOW_HASH_L2), + gr_mbuf_flow_hash(&b.mbuf, GR_MBUF_FLOW_HASH_L2) + ); +} + +static void hardware_rss_takes_precedence(void **) { + struct test_packet p; + + packet_init(&p, RTE_BE16(20000)); + p.mbuf.ol_flags = RTE_MBUF_F_RX_RSS_HASH; + p.mbuf.hash.rss = 0x12345678; + assert_int_equal(gr_mbuf_flow_hash(&p.mbuf, GR_MBUF_FLOW_HASH_RSS), 0x12345678); +} + +static void ipv4_fragments_share_one_hash(void **) { + struct test_packet first, later; + + packet_init(&first, RTE_BE16(20000)); + packet_init(&later, RTE_BE16(20001)); + first.ip.fragment_offset = RTE_BE16(RTE_IPV4_HDR_MF_FLAG); + later.ip.fragment_offset = RTE_BE16(1); + + assert_int_equal( + gr_mbuf_flow_hash(&first.mbuf, GR_MBUF_FLOW_HASH_RSS), + gr_mbuf_flow_hash(&later.mbuf, GR_MBUF_FLOW_HASH_RSS) + ); +} + +static void cached_hash_prefers_hardware_rss(void **) { + struct test_packet p; + + packet_init(&p, RTE_BE16(20000)); + p.mbuf.ol_flags = RTE_MBUF_F_RX_RSS_HASH; + p.mbuf.hash.rss = 0x12345678; + + assert_int_equal(gr_mbuf_flow_hash_get(&p.mbuf), 0x12345678); + assert_true(gr_mbuf_flow_hash_is_valid(&p.mbuf)); + assert_int_equal(p.mbuf.hash.rss, 0x12345678); +} + +static void software_hash_is_computed_once(void **) { + struct test_packet p; + uint32_t expected; + + packet_init(&p, RTE_BE16(20000)); + expected = gr_mbuf_flow_hash(&p.mbuf, GR_MBUF_FLOW_HASH_L3_L4); + + assert_false(gr_mbuf_flow_hash_is_valid(&p.mbuf)); + assert_int_equal(gr_mbuf_flow_hash_get(&p.mbuf), expected); + assert_true(gr_mbuf_flow_hash_is_valid(&p.mbuf)); + assert_int_equal(p.mbuf.hash.rss, expected); + + // the cached value is reused, not recomputed + p.udp.src_port = RTE_BE16(20001); + assert_int_equal(gr_mbuf_flow_hash_get(&p.mbuf), expected); +} + +static void cached_software_hash_distinguishes_udp_flows(void **) { + struct test_packet a, b; + + packet_init(&a, RTE_BE16(20000)); + packet_init(&b, RTE_BE16(20001)); + assert_int_not_equal(gr_mbuf_flow_hash_get(&a.mbuf), gr_mbuf_flow_hash_get(&b.mbuf)); +} + +static void l3_ipv4_hash_matches_ethernet_hash(void **) { + struct test_packet p; + uint32_t expected; + + packet_init(&p, RTE_BE16(20000)); + expected = gr_mbuf_flow_hash(&p.mbuf, GR_MBUF_FLOW_HASH_L3_L4); + assert_non_null(rte_pktmbuf_adj(&p.mbuf, sizeof(p.eth))); + + assert_int_equal( + gr_mbuf_flow_hash_get_l3(&p.mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4)), expected + ); + assert_true(gr_mbuf_flow_hash_is_valid(&p.mbuf)); +} + +static void l3_ipv6_hash_matches_ethernet_hash(void **) { + struct test_packet6 p; + uint32_t expected; + + packet6_init(&p, RTE_BE16(20000)); + expected = gr_mbuf_flow_hash(&p.mbuf, GR_MBUF_FLOW_HASH_L3_L4); + assert_non_null(rte_pktmbuf_adj(&p.mbuf, sizeof(p.eth))); + + assert_int_equal( + gr_mbuf_flow_hash_get_l3(&p.mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV6)), expected + ); + assert_true(gr_mbuf_flow_hash_is_valid(&p.mbuf)); +} + +static void l3_hash_imports_hardware_rss(void **) { + struct test_packet p; + + packet_init(&p, RTE_BE16(20000)); + assert_non_null(rte_pktmbuf_adj(&p.mbuf, sizeof(p.eth))); + p.mbuf.ol_flags = RTE_MBUF_F_RX_RSS_HASH; + p.mbuf.hash.rss = 0x12345678; + + assert_int_equal( + gr_mbuf_flow_hash_get_l3(&p.mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4)), 0x12345678 + ); + assert_true(gr_mbuf_flow_hash_is_valid(&p.mbuf)); +} + +static void cached_ipv4_fragments_share_one_hash(void **) { + struct test_packet first, later; + + packet_init(&first, RTE_BE16(20000)); + packet_init(&later, RTE_BE16(20001)); + first.ip.fragment_offset = RTE_BE16(RTE_IPV4_HDR_MF_FLAG); + later.ip.fragment_offset = RTE_BE16(1); + + assert_int_equal(gr_mbuf_flow_hash_get(&first.mbuf), gr_mbuf_flow_hash_get(&later.mbuf)); +} + +static void validity_is_cleared_on_reset(void **) { + struct test_packet p; + + packet_init(&p, RTE_BE16(20000)); + gr_mbuf_flow_hash_set(&p.mbuf, 0x12345678); + assert_true(gr_mbuf_flow_hash_is_valid(&p.mbuf)); + + rte_pktmbuf_reset(&p.mbuf); + assert_false(gr_mbuf_flow_hash_is_valid(&p.mbuf)); +} + +static void invalidate_forces_recompute_on_inner_frame(void **) { + struct test_packet p; + uint32_t expected; + + packet_init(&p, RTE_BE16(20000)); + expected = gr_mbuf_flow_hash(&p.mbuf, GR_MBUF_FLOW_HASH_L3_L4); + // pretend the NIC hashed a (since removed) outer tunnel header + gr_mbuf_flow_hash_set(&p.mbuf, ~expected); + + gr_mbuf_flow_hash_invalidate(&p.mbuf); + assert_false(gr_mbuf_flow_hash_is_valid(&p.mbuf)); + assert_int_equal(gr_mbuf_flow_hash_get(&p.mbuf), expected); + assert_true(gr_mbuf_flow_hash_is_valid(&p.mbuf)); +} + +int main(void) { + const struct CMUnitTest tests[] = { + cmocka_unit_test(software_hash_is_stable_per_flow), + cmocka_unit_test(software_hash_distinguishes_udp_flows), + cmocka_unit_test(hardware_rss_takes_precedence), + cmocka_unit_test(ipv4_fragments_share_one_hash), + cmocka_unit_test(cached_hash_prefers_hardware_rss), + cmocka_unit_test(software_hash_is_computed_once), + cmocka_unit_test(cached_software_hash_distinguishes_udp_flows), + cmocka_unit_test(l3_ipv4_hash_matches_ethernet_hash), + cmocka_unit_test(l3_ipv6_hash_matches_ethernet_hash), + cmocka_unit_test(l3_hash_imports_hardware_rss), + cmocka_unit_test(cached_ipv4_fragments_share_one_hash), + cmocka_unit_test(validity_is_cleared_on_reset), + cmocka_unit_test(invalidate_forces_recompute_on_inner_frame), + }; + return cmocka_run_group_tests(tests, NULL, NULL); +} diff --git a/modules/infra/datapath/meson.build b/modules/infra/datapath/meson.build index ce89bf2a2..c97bdd75d 100644 --- a/modules/infra/datapath/meson.build +++ b/modules/infra/datapath/meson.build @@ -8,6 +8,7 @@ src += files( 'drop.c', 'eth_input.c', 'eth_output.c', + 'flow_hash.c', 'iface_input.c', 'iface_output.c', 'l2_redirect.c', @@ -27,6 +28,10 @@ src += files( inc += include_directories('.') tests += [ + { + 'sources': files('flow_hash_test.c', 'flow_hash.c'), + 'link_args': [], + }, { 'sources': files('trace_test.c'), 'link_args': [], diff --git a/modules/ip/datapath/icmp_local_send.c b/modules/ip/datapath/icmp_local_send.c index 6464a9bf8..7ffc6d0d9 100644 --- a/modules/ip/datapath/icmp_local_send.c +++ b/modules/ip/datapath/icmp_local_send.c @@ -3,6 +3,7 @@ #include "clock.h" #include "control_input.h" +#include "flow_hash.h" #include "graph.h" #include "iface.h" #include "ip4.h" @@ -120,8 +121,7 @@ static uint16_t icmp_local_send_process( // Fake RSS to spread the traffic // for ECMP routes or active/active bonds. - mbuf->hash.rss = msg.ident; - mbuf->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; + gr_mbuf_flow_hash_set(mbuf, msg.ident); data = ip_local_mbuf_data(mbuf); data->proto = IPPROTO_ICMP; diff --git a/modules/ip/datapath/icmp_output.c b/modules/ip/datapath/icmp_output.c index 5d963c998..c0d80225c 100644 --- a/modules/ip/datapath/icmp_output.c +++ b/modules/ip/datapath/icmp_output.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2024 Robin Jarry +#include "flow_hash.h" #include "graph.h" #include "ip4.h" #include "ip4_datapath.h" @@ -44,7 +45,11 @@ icmp_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, goto next; } ip_set_fields(ip, local_data); - nh = fib4_lookup(local_data->vrf_id, local_data->dst, mbuf->hash.rss); + nh = fib4_lookup( + local_data->vrf_id, + local_data->dst, + gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4)) + ); if (nh == NULL) { // Do not let packets go to ip_output from icmp_output // with no available route to avoid loops of destination diff --git a/modules/ip/datapath/ip_error.c b/modules/ip/datapath/ip_error.c index d4cb27233..8729796ee 100644 --- a/modules/ip/datapath/ip_error.c +++ b/modules/ip/datapath/ip_error.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2024 Christophe Fontaine +#include "flow_hash.h" #include "graph.h" #include "ip4.h" #include "ip4_datapath.h" @@ -33,6 +34,7 @@ ip_error_process(struct rte_graph *graph, struct rte_node *node, void **objs, ui struct rte_ipv4_hdr *ip; struct rte_mbuf *mbuf; ip4_addr_t src, dst; + uint32_t hash; rte_edge_t edge; unsigned len; @@ -40,6 +42,7 @@ ip_error_process(struct rte_graph *graph, struct rte_node *node, void **objs, ui mbuf = objs[i]; ip = rte_pktmbuf_mtod(mbuf, struct rte_ipv4_hdr *); + hash = gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4)); src = ip->src_addr; // RFC792 payload size: ip header + 64 bits of original datagram len = rte_ipv4_hdr_len(ip) + 8; @@ -53,8 +56,7 @@ ip_error_process(struct rte_graph *graph, struct rte_node *node, void **objs, ui // Get the local router IP address from the input iface iface = l3_mbuf_data(mbuf)->iface; - if (iface == NULL - || (nh = fib4_lookup(iface->vrf_id, src, mbuf->hash.rss)) == NULL) { + if (iface == NULL || (nh = fib4_lookup(iface->vrf_id, src, hash)) == NULL) { edge = NO_IP; goto next; } diff --git a/modules/ip/datapath/ip_input.c b/modules/ip/datapath/ip_input.c index 68454c4b0..42ab0c5b8 100644 --- a/modules/ip/datapath/ip_input.c +++ b/modules/ip/datapath/ip_input.c @@ -3,6 +3,7 @@ #include "conntrack.h" #include "eth.h" +#include "flow_hash.h" #include "graph.h" #include "ip4.h" #include "ip4_datapath.h" @@ -144,7 +145,11 @@ ip_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, ui // A pre-resolved nexthop means the packet was handed over by a node // that already picked an adjacency, skip the route lookup. if (likely(e->nh == NULL)) - nh = fib4_lookup(iface->vrf_id, ip->dst_addr, mbuf->hash.rss); + nh = fib4_lookup( + iface->vrf_id, + ip->dst_addr, + gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4)) + ); else nh = e->nh; if (nh == NULL) { @@ -253,6 +258,7 @@ mock_func(int, drop_format(char *, size_t, const void *, size_t)); mock_func(int, trace_ip_format(char *, size_t, const struct rte_ipv4_hdr *, size_t)); mock_func(void, gr_eth_input_add_type(rte_be16_t, const char *)); mock_func(void, loopback_input_add_type(rte_be16_t, const char *)); +mock_func(uint32_t, gr_mbuf_flow_hash_l3(const struct rte_mbuf *, rte_be16_t)); mock_func( bool, gr_conn_parse_key( @@ -372,6 +378,7 @@ static void ip_input_conntrack_dnat(void **) { l3->flags = GR_NH_F_LOCAL; l3->ipv4 = fake_mbuf.ipv4_hdr.dst_addr; will_return(fib4_lookup, &nh); + will_return(gr_mbuf_flow_hash_l3, 0); iface.flags |= GR_IFACE_F_SNAT_DYNAMIC; struct conn conn; diff --git a/modules/ip6/datapath/icmp6_local_send.c b/modules/ip6/datapath/icmp6_local_send.c index 2292c2a5c..530f52475 100644 --- a/modules/ip6/datapath/icmp6_local_send.c +++ b/modules/ip6/datapath/icmp6_local_send.c @@ -3,6 +3,7 @@ #include "clock.h" #include "control_input.h" +#include "flow_hash.h" #include "graph.h" #include "icmp6.h" #include "iface.h" @@ -120,8 +121,7 @@ static uint16_t icmp6_local_send_process( // Fake RSS to spread the traffic // for ECMP routes or active/active bonds. - mbuf->hash.rss = msg.ident; - mbuf->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; + gr_mbuf_flow_hash_set(mbuf, msg.ident); payload = PAYLOAD(icmp6_echo); *payload = clock_ns(); diff --git a/modules/ip6/datapath/icmp6_output.c b/modules/ip6/datapath/icmp6_output.c index 26abe0213..6912c7e14 100644 --- a/modules/ip6/datapath/icmp6_output.c +++ b/modules/ip6/datapath/icmp6_output.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2024 Robin Jarry +#include "flow_hash.h" #include "graph.h" #include "icmp6.h" #include "ip6.h" @@ -58,7 +59,12 @@ static uint16_t icmp6_output_process( if (rte_ipv6_addr_is_mcast(&d->dst)) nh = nh6_lookup(d->iface->vrf_id, d->iface->id, &d->src); else - nh = fib6_lookup(d->iface->vrf_id, d->iface->id, &d->dst, mbuf->hash.rss); + nh = fib6_lookup( + d->iface->vrf_id, + d->iface->id, + &d->dst, + gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV6)) + ); if (nh == NULL) { edge = NO_ROUTE; diff --git a/modules/ip6/datapath/ip6_input.c b/modules/ip6/datapath/ip6_input.c index 0d635e463..383a3d35d 100644 --- a/modules/ip6/datapath/ip6_input.c +++ b/modules/ip6/datapath/ip6_input.c @@ -2,6 +2,7 @@ // Copyright (c) 2024 Robin Jarry #include "eth.h" +#include "flow_hash.h" #include "graph.h" #include "ip6.h" #include "ip6_datapath.h" @@ -122,7 +123,12 @@ ip6_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, u // A pre-resolved nexthop means the packet was handed over by a node // that already picked an adjacency, skip the route lookup. if (likely(e->nh == NULL)) - nh = fib6_lookup(iface->vrf_id, iface->id, &ip->dst_addr, mbuf->hash.rss); + nh = fib6_lookup( + iface->vrf_id, + iface->id, + &ip->dst_addr, + gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV6)) + ); else nh = e->nh; if (nh == NULL) { @@ -217,6 +223,7 @@ mock_func(int, drop_format(char *, size_t, const void *, size_t)); mock_func(int, trace_ip6_format(char *, size_t, const struct rte_ipv6_hdr *, size_t)); mock_func(void, gr_eth_input_add_type(rte_be16_t, const char *)); mock_func(void, loopback_input_add_type(rte_be16_t, const char *)); +mock_func(uint32_t, gr_mbuf_flow_hash_l3(const struct rte_mbuf *, rte_be16_t)); mock_func(struct nexthop *, mcast6_get_member(uint16_t, const struct rte_ipv6_addr *)); struct fake_mbuf { diff --git a/modules/ipip/datapath_out.c b/modules/ipip/datapath_out.c index fd4408fc9..991d8e889 100644 --- a/modules/ipip/datapath_out.c +++ b/modules/ipip/datapath_out.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2024 Robin Jarry +#include "flow_hash.h" #include "graph.h" #include "ip4.h" #include "ip4_datapath.h" @@ -32,6 +33,7 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, struct rte_ipv4_hdr *outer; const struct iface *iface; struct rte_mbuf *mbuf; + uint32_t hash; rte_edge_t edge; IFACE_STATS_VARS(tx, self); @@ -59,6 +61,7 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, // Encapsulate with another IPv4 header. inner = rte_pktmbuf_mtod(mbuf, const struct rte_ipv4_hdr *); + hash = gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4)); tunnel.src = ipip->local; tunnel.dst = ipip->remote; tunnel.len = rte_be_to_cpu_16(inner->total_length); @@ -75,7 +78,7 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, IFACE_STATS_INC(tx, self, mbuf, iface); // Resolve nexthop for the encapsulated packet. - ip_data->nh = fib4_lookup(iface->vrf_id, ipip->remote, mbuf->hash.rss); + ip_data->nh = fib4_lookup(iface->vrf_id, ipip->remote, hash); edge = IP_OUTPUT; next: diff --git a/modules/l2/datapath/vxlan_output.c b/modules/l2/datapath/vxlan_output.c index d853d5c1f..9020c86d7 100644 --- a/modules/l2/datapath/vxlan_output.c +++ b/modules/l2/datapath/vxlan_output.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2026 Robin Jarry +#include "flow_hash.h" #include "graph.h" #include "ip4.h" #include "ip6.h" @@ -46,6 +47,7 @@ static uint16_t vxlan_output_process( const struct nexthop *nh; struct rte_mbuf *m; rte_edge_t edge; + uint32_t hash; uint16_t len; for (uint16_t i = 0; i < nb_objs; i++) { @@ -72,10 +74,11 @@ static uint16_t vxlan_output_process( } len = rte_pktmbuf_pkt_len(m); + hash = gr_mbuf_flow_hash_get(m); switch (d->vtep.af) { case GR_AF_IP4: - nh = fib4_lookup(vxlan->encap_vrf_id, d->vtep.ipv4, m->hash.rss); + nh = fib4_lookup(vxlan->encap_vrf_id, d->vtep.ipv4, hash); if (nh == NULL) { edge = NO_ROUTE; goto next; @@ -87,7 +90,7 @@ static uint16_t vxlan_output_process( goto next; } *vh4 = vxlan->template.ipv4; - vh4->udp.src_port = vxlan_src_port(m->hash.rss); + vh4->udp.src_port = vxlan_src_port(hash); vh4->udp.dgram_len = rte_cpu_to_be_16( len + sizeof(vh4->udp) + sizeof(vh4->vxlan) ); @@ -98,9 +101,7 @@ static uint16_t vxlan_output_process( edge = IP_OUTPUT; break; case GR_AF_IP6: - nh = fib6_lookup( - vxlan->encap_vrf_id, d->iface->id, &d->vtep.ipv6, m->hash.rss - ); + nh = fib6_lookup(vxlan->encap_vrf_id, d->iface->id, &d->vtep.ipv6, hash); if (nh == NULL) { edge = NO_ROUTE; goto next; @@ -112,7 +113,7 @@ static uint16_t vxlan_output_process( goto next; } *vh6 = vxlan->template.ipv6; - vh6->udp.src_port = vxlan_src_port(m->hash.rss); + vh6->udp.src_port = vxlan_src_port(hash); vh6->udp.dgram_len = rte_cpu_to_be_16( len + sizeof(vh6->udp) + sizeof(vh6->vxlan) ); diff --git a/modules/policy/datapath/dnat44_dynamic.c b/modules/policy/datapath/dnat44_dynamic.c index 2cb45a161..c887bb21d 100644 --- a/modules/policy/datapath/dnat44_dynamic.c +++ b/modules/policy/datapath/dnat44_dynamic.c @@ -2,6 +2,7 @@ // Copyright (c) 2025 Robin Jarry #include "conntrack.h" +#include "flow_hash.h" #include "graph.h" #include "ip4.h" #include "l3.h" @@ -91,7 +92,11 @@ static uint16_t dnat44_dynamic_process( ); o = l3_mbuf_data(m); - o->nh = fib4_lookup(o->iface->vrf_id, ip->dst_addr, m->hash.rss); + o->nh = fib4_lookup( + o->iface->vrf_id, + ip->dst_addr, + gr_mbuf_flow_hash_get_l3(m, RTE_BE16(RTE_ETHER_TYPE_IPV4)) + ); if (o->nh == NULL) edge = NO_ROUTE; diff --git a/modules/policy/datapath/dnat44_static.c b/modules/policy/datapath/dnat44_static.c index 3733818d6..cf2ce54ad 100644 --- a/modules/policy/datapath/dnat44_static.c +++ b/modules/policy/datapath/dnat44_static.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2025 Robin Jarry +#include "flow_hash.h" #include "graph.h" #include "ip4.h" #include "ip4_datapath.h" @@ -74,7 +75,11 @@ static uint16_t dnat44_static_process( // We need the old address value to fixup the checksum properly. ip->dst_addr = dnat->replace; - d->nh = fib4_lookup(d->iface->vrf_id, ip->dst_addr, mbuf->hash.rss); + d->nh = fib4_lookup( + d->iface->vrf_id, + ip->dst_addr, + gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4)) + ); if (d->nh == NULL) edge = NO_ROUTE; diff --git a/modules/srv6/datapath/srv6_local.c b/modules/srv6/datapath/srv6_local.c index c2b446bae..5c2339d67 100644 --- a/modules/srv6/datapath/srv6_local.c +++ b/modules/srv6/datapath/srv6_local.c @@ -2,6 +2,7 @@ // Copyright (c) 2025 Olivier Gournet #include "eth.h" +#include "flow_hash.h" #include "graph.h" #include "ip6_datapath.h" #include "l3.h" @@ -260,6 +261,7 @@ static int process_behav_decap( ) { struct rte_ipv6_routing_ext *sr = ip6_info->sr; struct eth_input_mbuf_data *id; + uint32_t flow_label; rte_edge_t edge; // transit is not allowed @@ -287,9 +289,15 @@ static int process_behav_decap( return process_upper_layer(m, ip6_info); } - // Update the mbuf hash with ip6 flow id - // to avoid computing a soft RSS - m->hash.usr = rte_be_to_cpu_32(ip6_info->ip6_hdr->vtc_flow) & RTE_IPV6_HDR_FL_MASK; + // Use the outer flow label as the exposed flow's entropy when the + // encapsulating node filled it in (RFC 6438). A zero label carries + // no entropy: drop the outer hash instead so the first consumer + // hashes the inner packet. + flow_label = rte_be_to_cpu_32(ip6_info->ip6_hdr->vtc_flow) & RTE_IPV6_HDR_FL_MASK; + if (flow_label != 0) + gr_mbuf_flow_hash_set(m, flow_label); + else + gr_mbuf_flow_hash_invalidate(m); // remove tunnel ipv6 + ext headers decap_outer(m, ip6_info); @@ -805,6 +813,39 @@ static void srv6_decap_dt4_behind_dop(void **) { assert_int_equal(fm.mbuf.data_len, 0); } +// Decap must leave a coherent flow hash: a non-zero outer flow label +// becomes the hash, a zero label invalidates any stale outer RSS value. +static void srv6_decap_flow_hash(void **) { + struct nexthop_info_srv6_local sr_d = { + .base = {.behavior = SR_BEHAVIOR_END_DT4, .out_vrf_id = GR_VRF_ID_UNDEF}, + }; + struct ip6_info info = {0}, expect; + struct fake_mbuf fm; + + // zero flow label: the stale outer hardware hash must be dropped + fm_init_ipv6(&fm, &expect); + push_srh_1sid(&fm, &expect); + *fm.prev_next = IPPROTO_IPIP; + fm.mbuf.hash.rss = 0xdeadbeef; + fm.mbuf.ol_flags |= RTE_MBUF_F_RX_RSS_HASH; + + assert_int_equal(ip6_parse_to_srh(&fm.mbuf, &info), 0); + assert_int_equal(process_behav_decap(&fm.mbuf, &sr_d, &info), IP_INPUT); + assert_false(gr_mbuf_flow_hash_is_valid(&fm.mbuf)); + + // non-zero flow label: it becomes the flow hash + memset(&info, 0, sizeof(info)); + fm_init_ipv6(&fm, &expect); + fm.ip6.vtc_flow = rte_cpu_to_be_32((6u << 28) | 0xabcde); + push_srh_1sid(&fm, &expect); + *fm.prev_next = IPPROTO_IPIP; + + assert_int_equal(ip6_parse_to_srh(&fm.mbuf, &info), 0); + assert_int_equal(process_behav_decap(&fm.mbuf, &sr_d, &info), IP_INPUT); + assert_true(gr_mbuf_flow_hash_is_valid(&fm.mbuf)); + assert_int_equal(fm.mbuf.hash.rss, 0xabcde); +} + // End.X with USD hands the exposed packet to the L3 adjacency instead of the // FIB, but still lets the input node validate it. static void srv6_end_x_usd(bool inner_v4, rte_edge_t expected_edge, uint16_t expected_len) { @@ -971,6 +1012,7 @@ int main(void) { cmocka_unit_test(srv6_parse_ipv6_srv6_dop), cmocka_unit_test(srv6_parse_ipv6_hop_srv6_dop), cmocka_unit_test(srv6_decap_dt4_behind_dop), + cmocka_unit_test(srv6_decap_flow_hash), cmocka_unit_test(srv6_end_x_usd_inner_ipv4), cmocka_unit_test(srv6_end_x_usd_inner_ipv6), cmocka_unit_test(srv6_end_x_usd_no_inner), diff --git a/modules/srv6/datapath/srv6_output.c b/modules/srv6/datapath/srv6_output.c index 390687759..266bfef5c 100644 --- a/modules/srv6/datapath/srv6_output.c +++ b/modules/srv6/datapath/srv6_output.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2025 Olivier Gournet +#include "flow_hash.h" #include "graph.h" #include "ip4_datapath.h" #include "ip6.h" @@ -51,6 +52,8 @@ srv6_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, struct rte_ipv6_routing_ext *srh; struct rte_ipv6_hdr *outer_ip6; const struct nexthop *nh; + rte_be16_t eth_type; + uint32_t hash; uint32_t optlen, plen; struct rte_mbuf *m; uint8_t proto, n_omitted_segs; @@ -76,17 +79,20 @@ srv6_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, inner_ip4 = rte_pktmbuf_mtod(m, struct rte_ipv4_hdr *); plen = rte_be_to_cpu_16(inner_ip4->total_length); proto = IPPROTO_IPIP; + eth_type = RTE_BE16(RTE_ETHER_TYPE_IPV4); } else if (m->packet_type & RTE_PTYPE_L3_IPV6) { struct rte_ipv6_hdr *inner_ip6; inner_ip6 = rte_pktmbuf_mtod(m, struct rte_ipv6_hdr *); plen = rte_be_to_cpu_16(inner_ip6->payload_len) + sizeof(*inner_ip6); proto = IPPROTO_IPV6; + eth_type = RTE_BE16(RTE_ETHER_TYPE_IPV6); } else { edge = INVALID; goto next; } + hash = gr_mbuf_flow_hash_get_l3(m, eth_type); // Encapsulate with another IPv6 header optlen = 0; @@ -122,7 +128,7 @@ srv6_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, } // Resolve nexthop for the encapsulated packet. - nh = fib6_lookup(nh->vrf_id, GR_IFACE_ID_UNDEF, d->seglist, m->hash.rss); + nh = fib6_lookup(nh->vrf_id, GR_IFACE_ID_UNDEF, d->seglist, hash); if (nh == NULL) { edge = NO_ROUTE; goto next; diff --git a/smoke/ip_loadbalance_test.sh b/smoke/ip_loadbalance_test.sh index 8ecca1b20..b013962be 100755 --- a/smoke/ip_loadbalance_test.sh +++ b/smoke/ip_loadbalance_test.sh @@ -45,3 +45,25 @@ grcli ping 192.200.0.2 count 1 ident 2 delay 10 # Externally generated ICMP requests ip netns exec n0 ping -i0.01 -c3 -n 192.200.0.2 + +# Distinct flows entering through a port without RSS must spread across both +# group members instead of all collapsing onto one nexthop. +ip netns exec n0 ping -i0.01 -c1 -W1 -n -I x-p0 172.16.0.1 +ip netns exec n0 ping -i0.01 -c1 -W1 -n -I x-p1 172.16.1.1 + +rx_pkts() { + ip -n n0 -j -s link show "$1" | jq '.[0].stats64.rx.packets' +} +p0_before=$(rx_pkts x-p0) +p1_before=$(rx_pkts x-p1) +ip netns exec n1 bash -c \ + 'for i in $(seq 64); do echo flow > /dev/udp/192.200.0.2/7777; done' +sleep 0.3 +p0_delta=$(($(rx_pkts x-p0) - p0_before)) +p1_delta=$(($(rx_pkts x-p1) - p1_before)) +[ $((p0_delta + p1_delta)) -ge 32 ] || + fail "expected forwarded UDP flows on the members, saw $((p0_delta + p1_delta))" +[ "$p0_delta" -ge 8 ] || + fail "member p0 carried $p0_delta of 64 distinct flows" +[ "$p1_delta" -ge 8 ] || + fail "member p1 carried $p1_delta of 64 distinct flows" diff --git a/smoke/vxlan_test.sh b/smoke/vxlan_test.sh index 1af723393..06112d1e4 100755 --- a/smoke/vxlan_test.sh +++ b/smoke/vxlan_test.sh @@ -29,3 +29,29 @@ bridge -n n1 fdb add 00:00:00:00:00:00 dev vxlan100 self vni 100 dst 10.0.0.1 ip netns exec n1 ping -i0.01 -c3 -W1 192.168.100.1 grcli fdb show + +# Bridged traffic entering through a port without RSS must spread the vxlan +# UDP source port per flow instead of reusing one stale hash for every flow. +port_add p1 domain br100 +netns_add n2 +move_to_netns x-p1 n2 +ip -n n2 addr add 192.168.100.3/24 dev x-p1 +ip netns exec n2 ping -i0.01 -c3 -W1 192.168.100.2 + +ip netns exec n1 timeout 3 tcpdump -Q in -U -qn -i x-p0 \ + -w $tmp/vxlan-encap.pcap udp dst port 4789 & +capture_pid=$! +sleep 0.2 +ip netns exec n2 bash -c \ + 'for i in $(seq 32); do echo flow > /dev/udp/192.168.100.2/2152; done' +wait $capture_pid || true + +tcpdump -nn -r $tmp/vxlan-encap.pcap 2>/dev/null | awk \ + '$4 == ">" && $5 == "10.0.0.2.4789:" {n = split($3, a, "."); print a[n]}' \ + > $tmp/vxlan-src-ports.txt +encap_count=$(wc -l < $tmp/vxlan-src-ports.txt) +[ "$encap_count" -ge 16 ] || + fail "expected encapsulated UDP flows on the underlay, captured $encap_count" +uniq_ports=$(sort -u $tmp/vxlan-src-ports.txt | wc -l) +[ "$uniq_ports" -ge 4 ] || + fail "32 flows shared $uniq_ports vxlan source ports"