Skip to content

Commit 6d74d59

Browse files
jfb8856606jfb8856606
committed
CM6: KNI owner thread + msg_ring per-thread + tools thread_id routing
- Add ff_kni_is_owner_thread() to gate KNI on owner thread (index 0) in thread_mode=1, replacing RTE_PROC_PRIMARY checks (8 sites). - Create msg_ring per thread (nb_threads) and assign proc_id=thread index for all lcores in thread_mode=1. - Update tools usage text to show proc_id|thread_id semantics. - Document ff_rss_self_queue_info thread_mode semantics in ff_api.h. - All new logic gated by if(thread_mode); thread_mode=0 zero-regression.
1 parent 7495e70 commit 6d74d59

15 files changed

Lines changed: 74 additions & 53 deletions

File tree

‎lib/ff_api.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,8 @@ int ff_getpeername(int s, struct linux_sockaddr *name,
121121
int ff_getsockname(int s, struct linux_sockaddr *name,
122122
socklen_t *namelen);
123123

124-
/* Read-only: this process's RSS queue info (for self-check tools). */
124+
/* Read-only: this process/thread's RSS queue info (for self-check tools).
125+
* In thread_mode=1, proc_id is the thread index. */
125126
int ff_rss_self_queue_info(uint16_t *proc_id, uint16_t *queueid,
126127
uint16_t *nb_queues, uint16_t *reta_size);
127128

‎lib/ff_dpdk_if.c‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,13 @@ init_lcore_conf(void)
415415
}
416416

417417
ff_cur_lcore_conf()->port_cfgs = ff_global_cfg.dpdk.port_cfgs;
418-
ff_cur_lcore_conf()->proc_id = ff_global_cfg.dpdk.proc_id;
418+
if (ff_global_cfg.dpdk.thread_mode) {
419+
int ti;
420+
for (ti = 0; ti < ff_global_cfg.dpdk.nb_threads; ti++)
421+
lcore_conf[ff_global_cfg.dpdk.proc_lcore[ti]].proc_id = ti;
422+
} else {
423+
ff_cur_lcore_conf()->proc_id = ff_global_cfg.dpdk.proc_id;
424+
}
419425

420426
uint16_t socket_id = 0;
421427
if (numa_on) {
@@ -649,13 +655,15 @@ static int
649655
init_msg_ring(void)
650656
{
651657
uint16_t i, j;
652-
uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs;
658+
uint16_t nb_rings = ff_global_cfg.dpdk.thread_mode
659+
? ff_global_cfg.dpdk.nb_threads
660+
: ff_global_cfg.dpdk.nb_procs;
653661
unsigned socketid = ff_cur_lcore_conf()->socket_id;
654662

655663
/* Create message buffer pool */
656664
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
657665
message_pool = rte_mempool_create(FF_MSG_POOL,
658-
MSG_RING_SIZE * 2 * nb_procs,
666+
MSG_RING_SIZE * 2 * nb_rings,
659667
MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0,
660668
NULL, NULL, ff_msg_init, NULL,
661669
socketid, 0);
@@ -667,7 +675,7 @@ init_msg_ring(void)
667675
rte_panic("Create msg mempool failed\n");
668676
}
669677

670-
for(i = 0; i < nb_procs; ++i) {
678+
for(i = 0; i < nb_rings; ++i) {
671679
snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE,
672680
"%s%u", FF_MSG_RING_IN, i);
673681
msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0],
@@ -771,7 +779,7 @@ init_port_start(void)
771779

772780
total_nb_ports = nb_ports;
773781
#ifdef FF_KNI
774-
if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) {
782+
if (enable_kni && ff_kni_is_owner_thread()) {
775783
total_nb_ports *= 2; /* one more virtio_user port for kernel per port */
776784
}
777785
#endif
@@ -2007,7 +2015,7 @@ process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs,
20072015
}
20082016

20092017
#ifdef FF_KNI
2010-
if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) {
2018+
if (enable_kni && ff_kni_is_owner_thread()) {
20112019
mbuf_pool = pktmbuf_pool[qconf->socket_id];
20122020
mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool);
20132021
if(mbuf_clone) {
@@ -2018,7 +2026,8 @@ process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs,
20182026
#endif
20192027
ff_veth_input(ctx, rtem);
20202028
#ifdef FF_KNI
2021-
} else if (enable_kni) {
2029+
} else if (enable_kni &&
2030+
(!ff_global_cfg.dpdk.thread_mode || ff_kni_is_owner_thread())) {
20222031
if (knictl_action == FF_KNICTL_ACTION_ALL_TO_KNI){
20232032
ff_add_vlan_tag(rtem);
20242033
ff_kni_enqueue(filter, port_id, rtem);
@@ -2666,7 +2675,7 @@ main_loop(void *arg)
26662675
ctx = veth_ctx[port_id];
26672676

26682677
#ifdef FF_KNI
2669-
if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) {
2678+
if (enable_kni && ff_kni_is_owner_thread()) {
26702679
ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST);
26712680
}
26722681
#endif

‎lib/ff_dpdk_kni.c‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,14 @@ struct kni_interface_stats {
8989
struct rte_ring **kni_rp;
9090
struct kni_interface_stats **kni_stat;
9191

92+
int
93+
ff_kni_is_owner_thread(void)
94+
{
95+
if (ff_global_cfg.dpdk.thread_mode)
96+
return rte_lcore_id() == ff_global_cfg.dpdk.proc_lcore[0];
97+
return rte_eal_process_type() == RTE_PROC_PRIMARY;
98+
}
99+
92100
struct kni_ratelimit kni_rate_limt = {0, 0, 0};
93101

94102
static void
@@ -376,7 +384,7 @@ ff_kni_proto_filter(const void *data, uint16_t len, uint16_t eth_frame_type)
376384
void
377385
ff_kni_init(uint16_t nb_ports, const char *tcp_ports, const char *udp_ports)
378386
{
379-
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
387+
if (ff_kni_is_owner_thread()) {
380388
kni_stat = rte_zmalloc("kni:stat",
381389
sizeof(struct kni_interface_stats *) * nb_ports,
382390
RTE_CACHE_LINE_SIZE);
@@ -423,7 +431,7 @@ void
423431
ff_kni_alloc(uint16_t port_id, unsigned socket_id, int port_idx,
424432
unsigned ring_queue_size)
425433
{
426-
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
434+
if (ff_kni_is_owner_thread()) {
427435
struct rte_ether_addr addr = {{0}};
428436
int ret;
429437

@@ -473,7 +481,7 @@ ff_kni_alloc(uint16_t port_id, unsigned socket_id, int port_idx,
473481
char ring_name[RTE_KNI_NAMESIZE];
474482
snprintf((char*)ring_name, RTE_KNI_NAMESIZE, "kni_ring_%u", port_id);
475483

476-
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
484+
if (ff_kni_is_owner_thread()) {
477485
kni_rp[port_id] = rte_ring_create(ring_name, ring_queue_size,
478486
socket_id, RING_F_SC_DEQ);
479487

@@ -526,7 +534,7 @@ ff_kni_enqueue(enum FilterReturn filter, uint16_t port_id, struct rte_mbuf *pkt)
526534
return 0;
527535

528536
error:
529-
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
537+
if (ff_kni_is_owner_thread()) {
530538
kni_stat[port_id]->rx_dropped++;
531539
}
532540
rte_pktmbuf_free(pkt);

‎lib/ff_dpdk_kni.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,7 @@ enum FilterReturn ff_kni_proto_filter(const void *data, uint16_t len, uint16_t e
7171

7272
int ff_kni_enqueue(enum FilterReturn filter, uint16_t port_id, struct rte_mbuf *pkt);
7373

74+
int ff_kni_is_owner_thread(void);
75+
7476

7577
#endif /* ifndef _FSTACK_DPDK_KNI_H */

‎tools/arp/arp.c‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -745,13 +745,13 @@ usage(void)
745745
" arp -S hostname ether_addr [temp] [reject | blackhole] [pub [only]]",
746746
" arp -f filename");
747747
#else
748-
"usage: arp -p <f-stack proc_id> [-n] [-i interface] hostname",
749-
" arp -p <f-stack proc_id> [-n] [-i interface] -a",
750-
" arp -p <f-stack proc_id> -d hostname [pub]",
751-
" arp -p <f-stack proc_id> -d [-i interface] -a",
752-
" arp -p <f-stack proc_id> -s hostname ether_addr [temp] [reject | blackhole] [pub [only]]",
753-
" arp -p <f-stack proc_id> -S hostname ether_addr [temp] [reject | blackhole] [pub [only]]",
754-
" arp -p <f-stack proc_id> -f filename");
748+
"usage: arp -p <f-stack proc_id|thread_id> [-n] [-i interface] hostname",
749+
" arp -p <f-stack proc_id|thread_id> [-n] [-i interface] -a",
750+
" arp -p <f-stack proc_id|thread_id> -d hostname [pub]",
751+
" arp -p <f-stack proc_id|thread_id> -d [-i interface] -a",
752+
" arp -p <f-stack proc_id|thread_id> -s hostname ether_addr [temp] [reject | blackhole] [pub [only]]",
753+
" arp -p <f-stack proc_id|thread_id> -S hostname ether_addr [temp] [reject | blackhole] [pub [only]]",
754+
" arp -p <f-stack proc_id|thread_id> -f filename");
755755
#endif
756756
#ifdef FSTACK
757757
ff_ipc_exit();

‎tools/ifconfig/ifconfig.c‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -240,12 +240,12 @@ usage(void)
240240
" ifconfig -l [-d] [-u] [address_family]\n"
241241
" ifconfig %s[-d] [-m] [-u] [-v]\n",
242242
#else
243-
"usage: ifconfig -p <f-stack proc_id> [-f type:format] %sinterface address_family\n"
243+
"usage: ifconfig -p <f-stack proc_id|thread_id> [-f type:format] %sinterface address_family\n"
244244
" [address [dest_address]] [parameters]\n"
245-
" ifconfig -p <f-stack proc_id> interface create\n"
246-
" ifconfig -p <f-stack proc_id> -a %s[-d] [-m] [-u] [-v] [address_family]\n"
247-
" ifconfig -p <f-stack proc_id> -l [-d] [-u] [address_family]\n"
248-
" ifconfig -p <f-stack proc_id> %s[-d] [-m] [-u] [-v]\n",
245+
" ifconfig -p <f-stack proc_id|thread_id> interface create\n"
246+
" ifconfig -p <f-stack proc_id|thread_id> -a %s[-d] [-m] [-u] [-v] [address_family]\n"
247+
" ifconfig -p <f-stack proc_id|thread_id> -l [-d] [-u] [address_family]\n"
248+
" ifconfig -p <f-stack proc_id|thread_id> %s[-d] [-m] [-u] [-v]\n",
249249
#endif
250250
options, options, options);
251251

@@ -1736,7 +1736,8 @@ ifmaybeload(const char *name)
17361736
* infer the names of all drivers (eg mlx4en(4)).
17371737
*/
17381738
(void) kldload(ifkind);
1739-
#endif
1739+
1740+
#endif
17401741
}
17411742

17421743
static struct cmd basic_cmds[] = {

‎tools/ipfw/main.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ help(void)
4848
#ifndef FSTACK
4949
"\tipfw [-abcdefhnNqStTv] <command>\n\n"
5050
#else
51-
"\tipfw -P <f-stack proc_id> [-abcdefhnNqStTv] <command>\n\n"
51+
"\tipfw -P <f-stack proc_id|thread_id> [-abcdefhnNqStTv] <command>\n\n"
5252
#endif
5353
"where <command> is one of the following:\n\n"
5454
"add [num] [set N] [prob x] RULE-BODY\n"

‎tools/knictl/knictl.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ void
66
usage(void)
77
{
88
printf("Usage:\n");
9-
printf(" knictl [-p <f-stack proc_id>] [-P <max proc_id>] "
9+
printf(" knictl [-p <f-stack proc_id|thread_id>] [-P <max proc_id|thread_id>] "
1010
"[-a alltokni/alltoff/default][-n]\n use `-a` to set kni action\n use `-n` to show \n");
1111
}
1212

‎tools/ndp/ndp.c‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -918,16 +918,16 @@ usage()
918918
#endif
919919
printf(" ndp [-nt] -s nodename etheraddr [temp] [proxy]\n");
920920
#else
921-
printf("usage: ndp -C <f-stack proc_id> [-nt] hostname\n");
922-
printf(" ndp -C <f-stack proc_id> [-nt] -a | -c | -p | -r | -H | -P | -R\n");
923-
printf(" ndp -C <f-stack proc_id> [-nt] -A wait\n");
924-
printf(" ndp -C <f-stack proc_id> [-nt] -d hostname\n");
925-
printf(" ndp -C <f-stack proc_id> [-nt] -f filename\n");
926-
printf(" ndp -C <f-stack proc_id> [-nt] -i interface [flags...]\n");
921+
printf("usage: ndp -C <f-stack proc_id|thread_id> [-nt] hostname\n");
922+
printf(" ndp -C <f-stack proc_id|thread_id> [-nt] -a | -c | -p | -r | -H | -P | -R\n");
923+
printf(" ndp -C <f-stack proc_id|thread_id> [-nt] -A wait\n");
924+
printf(" ndp -C <f-stack proc_id|thread_id> [-nt] -d hostname\n");
925+
printf(" ndp -C <f-stack proc_id|thread_id> [-nt] -f filename\n");
926+
printf(" ndp -C <f-stack proc_id|thread_id> [-nt] -i interface [flags...]\n");
927927
#ifdef SIOCSDEFIFACE_IN6
928-
printf(" ndp -C <f-stack proc_id> [-nt] -I [interface|delete]\n");
928+
printf(" ndp -C <f-stack proc_id|thread_id> [-nt] -I [interface|delete]\n");
929929
#endif
930-
printf(" ndp -C <f-stack proc_id> [-nt] -s nodename etheraddr [temp] [proxy]\n");
930+
printf(" ndp -C <f-stack proc_id|thread_id> [-nt] -s nodename etheraddr [temp] [proxy]\n");
931931
#endif
932932
exit(1);
933933
}

‎tools/netstat/main.c‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1009,18 +1009,18 @@ usage(void)
10091009
" netstat -gs [-46s] [-f address_family] [-M core] [-N system]",
10101010
" netstat -Q");
10111011
#else
1012-
"usage: netstat -t <f-stack proc_id> [-46AaLnRSTWx] [-f protocol_family | -p protocol]",
1013-
" netstat -t <f-stack proc_id> -i | -I interface [-46abdhnW] [-f address_family]",
1014-
" netstat -t <f-stack proc_id> -w wait [-I interface] [-46d] [-q howmany]",
1015-
" netstat -t <f-stack proc_id> -s [-46sz] [-f protocol_family | -p protocol]",
1016-
" netstat -t <f-stack proc_id> -i | -I interface -s [-46s]\n"
1012+
"usage: netstat -t <f-stack proc_id|thread_id> [-46AaLnRSTWx] [-f protocol_family | -p protocol]",
1013+
" netstat -t <f-stack proc_id|thread_id> -i | -I interface [-46abdhnW] [-f address_family]",
1014+
" netstat -t <f-stack proc_id|thread_id> -w wait [-I interface] [-46d] [-q howmany]",
1015+
" netstat -t <f-stack proc_id|thread_id> -s [-46sz] [-f protocol_family | -p protocol]",
1016+
" netstat -t <f-stack proc_id|thread_id> -i | -I interface -s [-46s]\n"
10171017
" [-f protocol_family | -p protocol]",
1018-
" netstat -t <f-stack proc_id> -B [-z] [-I interface]",
1019-
" netstat -t <f-stack proc_id> -r [-46AnW] [-F fibnum] [-f address_family]",
1020-
" netstat -t <f-stack proc_id> -rs [-s]",
1021-
" netstat -t <f-stack proc_id> -g [-46W] [-f address_family]",
1022-
" netstat -t <f-stack proc_id> -gs [-46s] [-f address_family]",
1023-
" netstat -t <f-stack proc_id> -Q");
1018+
" netstat -t <f-stack proc_id|thread_id> -B [-z] [-I interface]",
1019+
" netstat -t <f-stack proc_id|thread_id> -r [-46AnW] [-F fibnum] [-f address_family]",
1020+
" netstat -t <f-stack proc_id|thread_id> -rs [-s]",
1021+
" netstat -t <f-stack proc_id|thread_id> -g [-46W] [-f address_family]",
1022+
" netstat -t <f-stack proc_id|thread_id> -gs [-46s] [-f address_family]",
1023+
" netstat -t <f-stack proc_id|thread_id> -Q");
10241024

10251025
#endif
10261026
xo_finish();

0 commit comments

Comments
 (0)