ip: use canonical hashes for L3 flow identity

The fib4/fib6 ECMP lookups in input, ICMP, error, NAT, IP-in-IP and
SRv6 paths read m->hash.rss directly. On ports without RSS the field
holds stale or zero data, so multipath selection is either unstable
or collapses onto a single nexthop, and locally generated packets
never had a meaningful hash at all.

Use gr_mbuf_flow_hash_get_l3() in these nodes so every lookup shares
the packet's cached canonical hash regardless of how it entered the
graph.

Extend the load balance smoke test to send distinct UDP flows through
an ECMP route and check that both group members carry traffic.

Signed-off-by: Harrison Caldicott <harrison@itsfubar.com.au>

Author: Harrison Caldicott

modules/ip/datapath/icmp_local_send.c

@@ -2,6 +2,7 @@
 // Copyright (c) 2024 Christophe Fontaine
 
 #include "control_input.h"
+#include "flow_hash.h"
 #include "graph.h"
 #include "iface.h"
 #include "ip4.h"
@@ -109,10 +110,9 @@ static uint16_t icmp_local_send_process(
 		icmp->icmp_seq_nb = rte_cpu_to_be_16(msg.seq_num);
 		icmp->icmp_ident = rte_cpu_to_be_16(msg.ident);
 
-		// Fake RSS to spread the traffic
-		// for ECMP routes or active/active bonds.
-		mbuf->hash.rss = msg.ident;
-		mbuf->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
+		// Seed canonical flow identity to spread locally generated traffic
+		// across ECMP routes or active/active bonds.
+		gr_mbuf_flow_hash_set(mbuf, msg.ident);
 
 		data = ip_local_mbuf_data(mbuf);
 		data->proto = IPPROTO_ICMP;

modules/ip/datapath/icmp_output.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2024 Robin Jarry
 
+#include "flow_hash.h"
 #include "graph.h"
 #include "ip4.h"
 #include "ip4_datapath.h"
@@ -44,7 +45,11 @@ icmp_output_process(struct rte_graph *graph, struct rte_node *node, void **objs,
 			goto next;
 		}
 		ip_set_fields(ip, local_data);
-		nh = fib4_lookup(local_data->vrf_id, local_data->dst, mbuf->hash.rss);
+		nh = fib4_lookup(
+			local_data->vrf_id,
+			local_data->dst,
+			gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4))
+		);
 		if (nh == NULL) {
 			// Do not let packets go to ip_output from icmp_output
 			// with no available route to avoid loops of destination

modules/ip/datapath/ip_error.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2024 Christophe Fontaine
 
+#include "flow_hash.h"
 #include "graph.h"
 #include "ip4.h"
 #include "ip4_datapath.h"
@@ -33,13 +34,15 @@ ip_error_process(struct rte_graph *graph, struct rte_node *node, void **objs, ui
 	struct rte_ipv4_hdr *ip;
 	struct rte_mbuf *mbuf;
 	ip4_addr_t src, dst;
+	uint32_t hash;
 	rte_edge_t edge;
 	unsigned len;
 
 	for (uint16_t i = 0; i < nb_objs; i++) {
 		mbuf = objs[i];
 
 		ip = rte_pktmbuf_mtod(mbuf, struct rte_ipv4_hdr *);
+		hash = gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4));
 		src = ip->src_addr;
 		// RFC792 payload size: ip header + 64 bits of original datagram
 		len = rte_ipv4_hdr_len(ip) + 8;
@@ -53,8 +56,7 @@ ip_error_process(struct rte_graph *graph, struct rte_node *node, void **objs, ui
 
 		// Get the local router IP address from the input iface
 		iface = l3_mbuf_data(mbuf)->iface;
-		if (iface == NULL
-		    || (nh = fib4_lookup(iface->vrf_id, src, mbuf->hash.rss)) == NULL) {
+		if (iface == NULL || (nh = fib4_lookup(iface->vrf_id, src, hash)) == NULL) {
 			edge = NO_IP;
 			goto next;
 		}

modules/ip/datapath/ip_input.c

@@ -3,6 +3,7 @@
 
 #include "conntrack.h"
 #include "eth.h"
+#include "flow_hash.h"
 #include "graph.h"
 #include "ip4.h"
 #include "ip4_datapath.h"
@@ -141,7 +142,11 @@ ip_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, ui
 			goto next;
 		}
 
-		nh = fib4_lookup(iface->vrf_id, ip->dst_addr, mbuf->hash.rss);
+		nh = fib4_lookup(
+			iface->vrf_id,
+			ip->dst_addr,
+			gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4))
+		);
 		if (nh == NULL) {
 			edge = NO_ROUTE;
 			goto next;
@@ -248,6 +253,7 @@ mock_func(int, drop_format(char *, size_t, const void *, size_t));
 mock_func(int, trace_ip_format(char *, size_t, const struct rte_ipv4_hdr *, size_t));
 mock_func(void, gr_eth_input_add_type(rte_be16_t, const char *));
 mock_func(void, loopback_input_add_type(rte_be16_t, const char *));
+mock_func(uint32_t, gr_mbuf_flow_hash_get_l3(struct rte_mbuf *, rte_be16_t));
 mock_func(
 	bool,
 	gr_conn_parse_key(
@@ -366,6 +372,7 @@ static void ip_input_conntrack_dnat(void **) {
 	l3->flags = GR_NH_F_LOCAL;
 	l3->ipv4 = fake_mbuf.ipv4_hdr.dst_addr;
 	will_return(fib4_lookup, &nh);
+	will_return(gr_mbuf_flow_hash_get_l3, 0);
 
 	iface.flags |= GR_IFACE_F_SNAT_DYNAMIC;
 	struct conn conn;

modules/ip6/datapath/icmp6_local_send.c

@@ -2,6 +2,7 @@
 // Copyright (c) 2025 Olivier Gournet
 
 #include "control_input.h"
+#include "flow_hash.h"
 #include "graph.h"
 #include "icmp6.h"
 #include "iface.h"
@@ -105,10 +106,9 @@ static uint16_t icmp6_local_send_process(
 		icmp6_echo->ident = rte_cpu_to_be_16(msg.ident);
 		icmp6_echo->seqnum = rte_cpu_to_be_16(msg.seq_num);
 
-		// Fake RSS to spread the traffic
-		// for ECMP routes or active/active bonds.
-		mbuf->hash.rss = msg.ident;
-		mbuf->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
+		// Seed canonical flow identity to spread locally generated traffic
+		// across ECMP routes or active/active bonds.
+		gr_mbuf_flow_hash_set(mbuf, msg.ident);
 
 		payload = PAYLOAD(icmp6_echo);
 		*payload = gr_clock_ns();

modules/ip6/datapath/icmp6_output.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2024 Robin Jarry
 
+#include "flow_hash.h"
 #include "graph.h"
 #include "icmp6.h"
 #include "ip6.h"
@@ -58,7 +59,12 @@ static uint16_t icmp6_output_process(
 		if (rte_ipv6_addr_is_mcast(&d->dst))
 			nh = nh6_lookup(d->iface->vrf_id, d->iface->id, &d->src);
 		else
-			nh = fib6_lookup(d->iface->vrf_id, d->iface->id, &d->dst, mbuf->hash.rss);
+			nh = fib6_lookup(
+				d->iface->vrf_id,
+				d->iface->id,
+				&d->dst,
+				gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV6))
+			);
 
 		if (nh == NULL) {
 			edge = NO_ROUTE;

modules/ip6/datapath/ip6_input.c

@@ -2,6 +2,7 @@
 // Copyright (c) 2024 Robin Jarry
 
 #include "eth.h"
+#include "flow_hash.h"
 #include "graph.h"
 #include "ip6.h"
 #include "ip6_datapath.h"
@@ -119,7 +120,12 @@ ip6_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, u
 			goto next;
 		}
 
-		nh = fib6_lookup(iface->vrf_id, iface->id, &ip->dst_addr, mbuf->hash.rss);
+		nh = fib6_lookup(
+			iface->vrf_id,
+			iface->id,
+			&ip->dst_addr,
+			gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV6))
+		);
 		if (nh == NULL) {
 			edge = DEST_UNREACH;
 			goto next;
@@ -212,6 +218,7 @@ mock_func(int, drop_format(char *, size_t, const void *, size_t));
 mock_func(int, trace_ip6_format(char *, size_t, const struct rte_ipv6_hdr *, size_t));
 mock_func(void, gr_eth_input_add_type(rte_be16_t, const char *));
 mock_func(void, loopback_input_add_type(rte_be16_t, const char *));
+mock_func(uint32_t, gr_mbuf_flow_hash_get_l3(struct rte_mbuf *, rte_be16_t));
 mock_func(struct nexthop *, mcast6_get_member(uint16_t, const struct rte_ipv6_addr *));
 
 struct fake_mbuf {

modules/ipip/datapath_out.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2024 Robin Jarry
 
+#include "flow_hash.h"
 #include "graph.h"
 #include "ip4.h"
 #include "ip4_datapath.h"
@@ -32,6 +33,7 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs,
 	struct rte_ipv4_hdr *outer;
 	const struct iface *iface;
 	struct rte_mbuf *mbuf;
+	uint32_t hash;
 	rte_edge_t edge;
 
 	IFACE_STATS_VARS(tx, self);
@@ -59,6 +61,7 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs,
 
 		// Encapsulate with another IPv4 header.
 		inner = rte_pktmbuf_mtod(mbuf, const struct rte_ipv4_hdr *);
+		hash = gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4));
 		tunnel.src = ipip->local;
 		tunnel.dst = ipip->remote;
 		tunnel.len = rte_be_to_cpu_16(inner->total_length);
@@ -75,7 +78,7 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs,
 		IFACE_STATS_INC(tx, self, mbuf, iface);
 
 		// Resolve nexthop for the encapsulated packet.
-		ip_data->nh = fib4_lookup(iface->vrf_id, ipip->remote, mbuf->hash.rss);
+		ip_data->nh = fib4_lookup(iface->vrf_id, ipip->remote, hash);
 		edge = IP_OUTPUT;
 
 next:

modules/policy/datapath/dnat44_dynamic.c

@@ -2,6 +2,7 @@
 // Copyright (c) 2025 Robin Jarry
 
 #include "conntrack.h"
+#include "flow_hash.h"
 #include "graph.h"
 #include "ip4.h"
 #include "l3.h"
@@ -91,7 +92,11 @@ static uint16_t dnat44_dynamic_process(
 		);
 
 		o = l3_mbuf_data(m);
-		o->nh = fib4_lookup(o->iface->vrf_id, ip->dst_addr, m->hash.rss);
+		o->nh = fib4_lookup(
+			o->iface->vrf_id,
+			ip->dst_addr,
+			gr_mbuf_flow_hash_get_l3(m, RTE_BE16(RTE_ETHER_TYPE_IPV4))
+		);
 
 		if (o->nh == NULL)
 			edge = NO_ROUTE;

modules/policy/datapath/dnat44_static.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2025 Robin Jarry
 
+#include "flow_hash.h"
 #include "graph.h"
 #include "ip4.h"
 #include "ip4_datapath.h"
@@ -74,7 +75,11 @@ static uint16_t dnat44_static_process(
 		// We need the old address value to fixup the checksum properly.
 		ip->dst_addr = dnat->replace;
 
-		d->nh = fib4_lookup(d->iface->vrf_id, ip->dst_addr, mbuf->hash.rss);
+		d->nh = fib4_lookup(
+			d->iface->vrf_id,
+			ip->dst_addr,
+			gr_mbuf_flow_hash_get_l3(mbuf, RTE_BE16(RTE_ETHER_TYPE_IPV4))
+		);
 
 		if (d->nh == NULL)
 			edge = NO_ROUTE;