CM4: VIMAGE feasibility PoC - enable VNET-based per-stack isolation (route B validated)

- opt_global.h: enable VIMAGE; compile net/vnet.c + netinet/tcp_ecn.c
- ff_vimage_glue.c: minimal prison_owns_vnet (verbatim from kern_jail.c),
  avoids pulling the 141KB kern_jail.c and its jail subsystem deps
- ff_stub_14_extra.c: drop ECN stub globals/no-op stubs; real tcp_ecn.c takes over
- ff_freebsd_init.c: pin main thread td_vnet=vnet0 after mi_startup so V_*
  resolve under curvnet (fixes SIGSEGV at sysctl on curvnet=NULL)
- ECN switch: new [freebsd] tcp_ecn config (default 0=off, zero-regression;
  1=passive ECN). V_tcp_do_ecn set after td_vnet bind.

Runtime PoC (thread_mode=0): vnet0 auto-bootstrap OK, vnet data segment
relocation OK, lo 127.0.0.1 OK, DPDK 9.134.214.176:80 curl HTTP 200.
Gates: build PASS (0 error, 51 warning == baseline), review PASS
(zero-regression), unit tests 199/199 PASS. Per-thread td_vnet is CM5.

Author: jfb8856606

lib/Makefile

@@ -282,7 +282,8 @@ FF_SRCS+=                     \
 	ff_vfs_ops.c          \
 	ff_veth.c             \
 	ff_route.c            \
-	ff_stub_14_extra.c
+	ff_stub_14_extra.c    \
+	ff_vimage_glue.c
 
 ifdef FF_NETGRAPH
 FF_SRCS+=                   \
@@ -440,7 +441,8 @@ NET_SRCS+=		 \
 	nhop.c		 \
 	nhop_ctl.c	 \
 	rtsock.c         \
-	slcompress.c
+	slcompress.c	 \
+	vnet.c
 
 ifdef FF_NETGRAPH
 NETGRAPH_SRCS +=         \
@@ -520,6 +522,7 @@ NETINET_SRCS+=		\
 	tcp_timer.c	\
 	tcp_timewait.c	\
 	tcp_usrreq.c	\
+	tcp_ecn.c	\
 	udp_usrreq.c	\
 	cc.c		\
 	cc_newreno.c    \

lib/ff_config.c

@@ -1108,6 +1108,8 @@ ini_parse_handler(void* user, const char* section, const char* name,
             pconfig->freebsd.fd_reserve = atoi(value);
         } else if (strcmp(name, "memsz_MB") == 0) {
             pconfig->freebsd.mem_size = atoi(value);
+        } else if (strcmp(name, "tcp_ecn") == 0) {
+            pconfig->freebsd.tcp_ecn = atoi(value);
         } else {
             return freebsd_conf_handler(pconfig, "boot", name, value);
         }
@@ -1514,6 +1516,7 @@ ff_default_config(struct ff_config *cfg)
     cfg->freebsd.physmem = 1048576*256;
     cfg->freebsd.fd_reserve = 0;
     cfg->freebsd.mem_size = 256;
+    cfg->freebsd.tcp_ecn = 0;
 
     cfg->log.level = FF_LOG_DISABLE;
     cfg->log.dir = strdup(FF_LOG_FILENAME_PREFIX);

lib/ff_config.h

@@ -359,6 +359,7 @@ struct ff_config {
         int hz;
         int fd_reserve;
         int mem_size;
+        int tcp_ecn;    /* 0=disable ECN (default, zero-regression); 1=passive ECN (upstream default 2) */
     } freebsd;
 
     struct {

lib/ff_freebsd_init.c

@@ -52,7 +52,11 @@
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_private.h>
+#include <net/vnet.h>
 #include <netinet/in_var.h>
+#include <netinet/in_pcb.h>
+#include <netinet/tcp.h>
+#include <netinet/tcp_var.h>
 
 int lo_set_defaultaddr(void);
 
@@ -178,6 +182,22 @@ ff_freebsd_init(void)
 
     mutex_init();
     mi_startup();
+
+    /*
+     * CM4 PoC: vnet_init_done clears curvnet at boot end (FreeBSD design).
+     * Under VIMAGE all V_* go through curvnet, so pin the main thread to
+     * vnet0 for the whole thread_mode=0 single-thread path (sysctl loop,
+     * lo_set_defaultaddr, ff_run/main_loop). Per-thread td_vnet is CM5.
+     */
+    curthread->td_vnet = vnet0;
+
+    /*
+     * ECN switch (default off for zero-regression vs CM0-3). Real tcp_ecn.c
+     * defaults V_tcp_do_ecn=2 (passive ECN); apply the config value here,
+     * after td_vnet=vnet0 so V_* resolves. 0=off, 1=passive(=2).
+     */
+    V_tcp_do_ecn = ff_global_cfg.freebsd.tcp_ecn ? 2 : 0;
+
     sx_init(&proctree_lock, "proctree");
     ff_fdused_range(ff_global_cfg.freebsd.fd_reserve);
 

lib/ff_stub_14_extra.c

@@ -572,57 +572,11 @@ int sendfile_wait_generic(struct sendfile_sync *sfs, int *err)
     return (0);
 }
 
-int tcp_do_ecn = 2;
-
-void tcp_ecn_input_parallel_syn(struct tcpcb *tp, uint16_t thflags, int iptos);
-void tcp_ecn_input_parallel_syn(struct tcpcb *tp, uint16_t thflags, int iptos)
-{
-    
-}
-
-int tcp_ecn_input_segment(struct tcpcb *tp, uint16_t thflags, int tlen, int pkts_acked, int iptos);
-int tcp_ecn_input_segment(struct tcpcb *tp, uint16_t thflags, int tlen, int pkts_acked, int iptos)
-{
-    return (0);
-}
-
-void tcp_ecn_input_syn_sent(struct tcpcb *tp, uint16_t thflags, int iptos);
-void tcp_ecn_input_syn_sent(struct tcpcb *tp, uint16_t thflags, int iptos)
-{
-    
-}
-
-int tcp_ecn_maxretries = 1;
-
-int tcp_ecn_output_established(struct tcpcb *tp, uint16_t *thflags, int len, bool rxmit);
-int tcp_ecn_output_established(struct tcpcb *tp, uint16_t *thflags, int len, bool rxmit)
-{
-    return (0);
-}
-
-uint16_t tcp_ecn_output_syn_sent(struct tcpcb *tp);
-uint16_t tcp_ecn_output_syn_sent(struct tcpcb *tp)
-{
-    return (0);
-}
-
-void tcp_ecn_syncache_add(struct syncache *sc, int iptos, uint16_t thflags);
-void tcp_ecn_syncache_add(struct syncache *sc, int iptos, uint16_t thflags)
-{
-    
-}
-
-uint16_t tcp_ecn_syncache_respond(uint16_t thflags, struct syncache *sc);
-uint16_t tcp_ecn_syncache_respond(uint16_t thflags, struct syncache *sc)
-{
-    return (thflags);
-}
-
-void tcp_ecn_syncache_socket(struct tcpcb *tp, struct syncache *sc);
-void tcp_ecn_syncache_socket(struct tcpcb *tp, struct syncache *sc)
-{
-    
-}
+/*
+ * ECN stubs (tcp_do_ecn/tcp_ecn_maxretries + 8 tcp_ecn_* no-ops) removed:
+ * under VIMAGE V_tcp_do_ecn needs the real VNET_DEFINE segment symbols,
+ * provided by the now-compiled freebsd/netinet/tcp_ecn.c.
+ */
 
 void (*tcp_hpts_softclock)(void) = NULL;
 

lib/ff_vimage_glue.c

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2017-2021 THL A29 Limited, a Tencent company.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/*
+ * VIMAGE glue layer (CM4).
+ *
+ * Enabling VIMAGE makes V_* macros resolve through the set_vnet segment and
+ * pulls in a small number of symbols from kern_jail.c's #ifdef VIMAGE block
+ * that are not otherwise compiled into libfstack. Compiling the whole 141KB
+ * kern_jail.c is infeasible here (it #includes opt_nfs.h and drags the entire
+ * jail/osd/racct subsystem). This file provides only the minimal necessary
+ * symbol, copied verbatim from FreeBSD kern/kern_jail.c:4048, to avoid that
+ * connected dependency explosion.
+ */
+
+#include <sys/param.h>
+#include <sys/jail.h>
+
+/*
+ * Determine whether the prison owns its VNET.
+ */
+bool
+prison_owns_vnet(struct prison *pr)
+{
+
+	/*
+	 * vnets cannot be added/removed after jail creation,
+	 * so no need to lock here.
+	 */
+	return ((pr->pr_flags & PR_VNET) != 0);
+}

lib/opt/opt_global.h

@@ -2,4 +2,5 @@
 #define RWLOCK_NOINLINE 1
 #define SX_NOINLINE 1
 #define DEV_RANDOM 1
-#define NO_EVENTTIMERS 1
+#define NO_EVENTTIMERS 1
+#define VIMAGE 1

tests/unit/fixtures/valid_tcp_ecn.ini

@@ -0,0 +1,24 @@
+; CM4 ECN config switch: tcp_ecn lives in [freebsd.boot] (ff_config.c:1111).
+; tcp_ecn=1 -> freebsd.tcp_ecn parsed as 1 (runtime maps to V_tcp_do_ecn=2).
+[dpdk]
+lcore_mask=1
+channel=4
+promiscuous=1
+numa_on=1
+port_list=0
+
+[port0]
+addr=192.168.1.10
+netmask=255.255.255.0
+broadcast=192.168.1.255
+gateway=192.168.1.1
+
+[freebsd.boot]
+hz=100
+physmem=268435456
+fd_reserve=1024
+mem_size=256
+tcp_ecn=1
+
+[freebsd.sysctl]
+kern.ipc.maxsockbuf=16777216

tests/unit/test_ff_config.c

@@ -1206,6 +1206,37 @@ test_ff_load_config_thread_mode_secondary_conflict(void **state)
     assert_int_equal(rv, -1);
 }
 
+/* ======================================================================== */
+/* CM4 (native-mt): ECN config switch (freebsd.tcp_ecn, [freebsd.boot] key)  */
+/* Spec: native_mt_spec/zh_cn. Parse @ff_config.c:1111, default 0 @:1519.    */
+/* Runtime maps tcp_ecn -> V_tcp_do_ecn (0=off / 1=passive) in              */
+/* ff_freebsd_init.c; that runtime effect is out of unit-test scope.        */
+/* ======================================================================== */
+
+/* TC-CM4-01: no tcp_ecn key -> defaults to 0 (zero-regression). */
+static void
+test_ff_load_config_tcp_ecn_default_off(void **state)
+{
+    (void)state;
+    /* valid_minimal.ini has [freebsd.boot] but NO tcp_ecn key. */
+    int rv = load_with_fixture(FIXTURE_PATH("valid_minimal.ini"));
+    (void)rv;
+    assert_int_equal(ff_global_cfg.freebsd.tcp_ecn, 0);
+}
+
+/* TC-CM4-02: explicit tcp_ecn=1 in [freebsd.boot] -> parsed as 1. */
+static void
+test_ff_load_config_tcp_ecn_enabled(void **state)
+{
+    (void)state;
+    int rv = load_with_fixture(FIXTURE_PATH("valid_tcp_ecn.ini"));
+    (void)rv;
+    assert_int_equal(ff_global_cfg.freebsd.tcp_ecn, 1);
+    /* New key must not disturb sibling [freebsd.boot] parsing. */
+    assert_int_equal(ff_global_cfg.freebsd.hz, 100);
+    assert_int_equal(ff_global_cfg.freebsd.mem_size, 256);
+}
+
 int
 main(void)
 {
@@ -1280,6 +1311,9 @@ main(void)
         cmocka_unit_test_setup_teardown(test_ff_load_config_thread_mode_enabled,        test_setup, NULL),
         cmocka_unit_test_setup_teardown(test_ff_load_config_thread_mode_default_off,     test_setup, NULL),
         cmocka_unit_test_setup_teardown(test_ff_load_config_thread_mode_secondary_conflict, test_setup, NULL),
+        /* CM4 (native-mt) ECN config switch (freebsd.tcp_ecn) */
+        cmocka_unit_test_setup_teardown(test_ff_load_config_tcp_ecn_default_off, test_setup, NULL),
+        cmocka_unit_test_setup_teardown(test_ff_load_config_tcp_ecn_enabled,     test_setup, NULL),
 #ifdef FF_KERNEL_COEXIST
         /* kernel_event_support: [stack] kernel_coexist */
         cmocka_unit_test_setup_teardown(test_ff_load_config_stack_coexist_enabled,         test_setup, NULL),