Support more than one TLS cert

CMakeLists.txt

@@ -397,6 +397,9 @@ add_library(moqx_core STATIC
   src/relay/PublisherCrossExecFilter.cpp
   src/relay/SubscriberCrossExecFilter.cpp
   src/logging/LogSetup.cpp
+  src/tls/CertDirScanner.cpp
+  src/tls/SniCertManager.cpp
+  src/tls/FizzContextBuilder.cpp
 )
 
 target_include_directories(moqx_core

RUNNING.md

@@ -154,7 +154,7 @@ docker compose up -d
 | `MOQX_VERBOSE` | `0` | VLOG level (0-4) |
 | `MOQX_LOG_PORT` | `9999` | Dozzle log viewer port (localhost only) |
 | `GLOG_vmodule` | -- | Per-module verbose level (passed through) |
-| `MOQX_INSECURE` | `false` | Use built-in dev cert |
+| `MOQX_INSECURE` | `false` | Use built-in dev cert; the entrypoint exits when set with `MOQX_CERT`/`MOQX_KEY` |
 
 The entrypoint maps `MOQX_LOG_LEVEL` and `MOQX_VERBOSE` to their `GLOG_*`
 equivalents and forces `GLOG_logtostderr=1`.

config.example.yaml

@@ -8,8 +8,8 @@ listeners:
         address: "::"        # Bind address (default: "::" = all interfaces)
         port: 9668            # Listen port (1-65535)
     tls:
-      # cert_file: /path/to/cert.pem   # Required when insecure: false
-      # key_file: /path/to/key.pem     # Required when insecure: false
+      # cert_file: /path/to/cert.pem   # Required when insecure: false (with key_file)
+      # key_file: /path/to/key.pem     # Required when insecure: false (with cert_file)
       # --- PKCS#12 bundle (alternative to cert_file/key_file) ---
       # A single .p12/.pfx carrying the cert chain + private key. Decrypted in
       # memory; the key is never written to disk. Mutually exclusive with
@@ -18,7 +18,18 @@ listeners:
       # pkcs12_password_file: /path/to/password.txt  # Preferred: keep the secret off the YAML
       # pkcs12_password_env: MOQX_PKCS12_PASSWORD    # Cloud-vault friendly: read the password from this env var
       # pkcs12_password: "inline-secret"             # Discouraged: persists in the config at rest
-      insecure: true          # Skip TLS for local development
+      # --- Multi-cert / SNI (fizz TLS stack: mvfst + proxygen_qmux) ---
+      # Serve per-hostname certs from a directory of <base>.pem + <base>.key pairs,
+      # selected by SNI (identities: DNS SANs, or CN when there are no DNS SANs).
+      # cert_file/key_file or pkcs12_file, if also set, serve connections whose SNI
+      # is absent or matches nothing. See docs/config.md.
+      # fizz:
+      #   cert_dir: /path/to/certs
+      #   cert_reload_interval_s: 60   # background rescan; 0 = scan once at startup
+      #   ticket_seeds_file: /path/to/ticket-seeds  # hex seeds, one per line; share across relays so resumption survives restarts
+      # --- Development-only mode ---
+      # insecure is rejected alongside cert_file/key_file/pkcs12_file/fizz.
+      insecure: true          # Serve the compiled-in dev cert; no real credentials (development only)
     endpoint: "/moq-relay"    # WebTransport endpoint path
     # moqt_versions: [14, 16] # MOQT draft versions (empty = default 14,16)
     # quic:                   # Per-listener QUIC overrides (optional; inherits listener_defaults.quic)

docker/entrypoint.sh

@@ -101,7 +101,21 @@ export MOQX_PORT="${MOQX_PORT:-4433}"
 export MOQX_ADMIN_PORT="${MOQX_ADMIN_PORT:-8000}"
 export MOQX_CERT="${MOQX_CERT:-}"
 export MOQX_KEY="${MOQX_KEY:-}"
-export MOQX_INSECURE="${MOQX_INSECURE:-false}"
+# Canonicalized before use: YAML accepts yes/on/1 as true, but the shell tests
+# below and in the picoquic block compare against the literal "true".
+case "$(printf %s "${MOQX_INSECURE:-false}" | tr '[:upper:]' '[:lower:]')" in
+  true|yes|on|1)   MOQX_INSECURE=true ;;
+  false|no|off|0)  MOQX_INSECURE=false ;;
+  *) echo "invalid boolean for MOQX_INSECURE (want true/false)" >&2; exit 2 ;;
+esac
+export MOQX_INSECURE
+# Both defaulted to empty above, so a value here is one the operator set.
+# Clearing it would serve the built-in dev cert instead, which is the silent
+# swap the config schema exists to refuse.
+if [ "$MOQX_INSECURE" = "true" ] && { [ -n "$MOQX_CERT" ] || [ -n "$MOQX_KEY" ]; }; then
+  echo "MOQX_INSECURE=true is mutually exclusive with MOQX_CERT/MOQX_KEY" >&2
+  exit 2
+fi
 export MOQX_ENDPOINT="${MOQX_ENDPOINT:-/moq-relay}"
 export MOQX_MOQT_VERSIONS="${MOQX_MOQT_VERSIONS:-[16, 14, 18]}"
 export MOQX_MAX_TRACKS="${MOQX_MAX_TRACKS:-1000}"

docs/config.md

@@ -61,14 +61,80 @@ listeners:
     tls:
       cert_file: /etc/moqx/cert.pem
       key_file:  /etc/moqx/key.pem
+      # fizz:                        # optional; mvfst/proxygen_qmux only
+      #   cert_dir: /etc/moqx/certs
+      #   cert_reload_interval_s: 60
     endpoint: /moq-relay
     quic_stack: mvfst         # optional; default mvfst
     moqt_versions: []         # optional; empty = default [14, 16]
     quic: { ... }             # optional; overrides listener_defaults.quic
 ```
 
-**TLS:** For development only, `tls: {insecure: true}` skips certificate
-verification. This is incompatible with `quic_stack: picoquic`.
+**TLS:** `tls: {insecure: true}` is for development only.
+
+- `insecure: true` serves a compiled-in certificate and does not request a
+  client certificate.
+- `insecure: true` alongside `cert_file`, `key_file`, `pkcs12_file`, or
+  `fizz.cert_dir` is rejected at config load.
+- `insecure: true` alongside `fizz.ticket_seeds_file` or
+  `fizz.cert_reload_interval_s` warns: neither reaches the compiled-in
+  certificate path.
+- `quic_stack: picoquic` rejects `insecure: true`.
+
+**Multi-certificate / SNI (`tls.fizz`):** options for the fizz TLS stack
+(`quic_stack: mvfst` and `proxygen_qmux`). `picoquic` rejects any option set in
+the block; an empty `fizz: {}` is accepted.
+
+| Field | Default | Notes |
+|---|---|---|
+| `cert_dir` | unset | Directory of certificate pairs, `<base>.pem` + `<base>.key` (arbitrary basenames, non-recursive). The certificate served is selected by the client's SNI. |
+| `cert_reload_interval_s` | 60 | Seconds between background rescans of `cert_dir`. `0` = scan once at startup, never rescan. Warns when set without `cert_dir`. |
+| `ticket_seeds_file` | unset | Session-ticket seeds: one hex-encoded seed (≥64 hex chars) per line, `#` starts a comment. Read once at startup. |
+
+- Identities come from each certificate's DNS SANs, or from its CN when the
+  certificate carries no DNS SANs. Other SAN types (IP, email) are ignored.
+- Wildcards match one label: `*.example.com` covers `a.example.com`, not
+  `b.a.example.com`.
+- Nested wildcards are rejected.
+- Certificates and keys are read up front: at startup, and on the rescan
+  thread afterwards. No handshake ever waits on disk.
+- Deferring the key read to first use is not an option: fizz's cert-selection
+  hook is synchronous, so the read would land on the connection's IO thread
+  and a slow `cert_dir` mount would stall every handshake sharing that thread.
+- `cert_file`/`key_file` (or `pkcs12_file`), if also set, become the fallback
+  certificate, serving connections with no SNI or no matching identity.
+- With `cert_dir` and no fallback certificate, a connection with no SNI or no
+  matching identity fails the handshake.
+- A PKCS#12 fallback is decrypted once at config load and is never refreshed.
+- Rescans pick up added, removed, and changed pairs, and refresh a changed
+  `cert_file`/`key_file` fallback.
+- Startup is strict: an orphan `.pem`/`.key`, an unparsable certificate, a
+  duplicate identity, or a key that fails to load aborts the relay.
+- A rescan is not strict: its errors are logged, and every certificate already
+  loaded keeps serving.
+- Removing either file of a pair retires it on the next rescan; a lone `.pem`
+  or `.key` left behind is not served.
+- A changed pair that fails to load keeps serving its previous version until a
+  later rescan succeeds.
+- A newly added pair that fails to load is dropped, and its identities fall to
+  the fallback certificate or to a handshake miss.
+- A new pair claiming an identity another pair already serves is dropped with
+  a warning, and the incumbent keeps that identity until its own files are
+  removed. At startup there is no incumbent: sorted filename order decides,
+  and the duplicate is fatal.
+
+**`ticket_seeds_file`:** a moqx format, not one shared with nginx or HAProxy.
+
+- The first seed encrypts new session tickets.
+- Every listed seed still decrypts, so rotate by prepending a new seed.
+- Generate a seed with `openssl rand -hex 32`.
+- Point every relay instance at the same file so resumption survives restarts
+  and works across relays.
+- The file is read once at startup, so a rotation takes effect on restart.
+- Without the option each process uses a random seed, and resumption dies with
+  the process.
+- An empty file, a comments-only file, a non-hex line, or a seed under 32
+  bytes is a config-load error.
 
 **moqt_versions:**: Currently supports 14 and 16.
 
@@ -530,7 +596,8 @@ or a restart.
 
 | Lifecycle | When changes apply | Examples |
 |---|---|---|
-| **Static** | Process restart required | listeners, admin, relay_id, TLS certificates, QUIC stack |
+| **Static** | Process restart required | listeners, admin, relay_id, `cert_file`/`key_file`/PKCS#12 certificates, QUIC stack |
+| **Automatic** | Background rescan, no reload needed | certificates under `tls.fizz.cert_dir` (see `cert_reload_interval_s`) |
 | **Reload:NewConn** | New connections/sessions only | service match rules, upstream URL |
 | **Reload:NewSub** | New subscriptions/tracks only | cache settings |
 | **Reload:Immediate** | All connections immediately | (reserved for future fields) |

scripts/moqx-run.sh

@@ -279,6 +279,12 @@ else
 fi
 export MOQX_INSECURE
 
+# The config schema rejects insecure alongside a cert source, so drop the paths
+# here rather than rendering a config the relay refuses to load.
+if [[ "$MOQX_INSECURE" == true ]]; then
+  export MOQX_CERT="" MOQX_KEY=""
+fi
+
 # ── Resolve placeholders into a temp config ───────────────────────────────
 # Fixed path by default; override (e.g. per perf run, to avoid concurrent
 # clobber) with MOQX_RESOLVED_CONFIG.

src/MoqxPicoRelayServer.cpp

@@ -42,7 +42,7 @@ std::string resolveCert(const config::ListenerConfig& cfg) {
         if constexpr (std::is_same_v<T, config::Insecure>) {
           return "";
         } else {
-          return tls.certFile;
+          return tls.tls.certFile;
         }
       },
       cfg.tlsMode
@@ -56,7 +56,7 @@ std::string resolveKey(const config::ListenerConfig& cfg) {
         if constexpr (std::is_same_v<T, config::Insecure>) {
           return "";
         } else {
-          return tls.keyFile;
+          return tls.tls.keyFile;
         }
       },
       cfg.tlsMode

src/MoqxQmuxRelayServer.cpp

@@ -7,9 +7,9 @@
 #include "MoqxQmuxRelayServer.h"
 
 #include "stats/EventBaseStatsCollector.h"
+#include "tls/FizzContextBuilder.h"
 #include <moxygen/MoQRelaySession.h>
 #include <moxygen/QmuxUtils.h>
-#include <proxygen/httpserver/samples/hq/FizzContext.h>
 
 #include <folly/logging/xlog.h>
 #include <quic/state/TransportSettings.h>
@@ -26,32 +26,6 @@ std::vector<std::string> buildQmuxAlpns(const std::string& versions) {
   return getMoqtProtocols(versions, /*useStandard=*/true);
 }
 
-std::shared_ptr<const fizz::server::FizzServerContext>
-buildFizzContext(const config::ListenerConfig& cfg) {
-  auto alpns = buildQmuxAlpns(cfg.moqtVersions);
-  return std::visit(
-      [&alpns](const auto& tls) -> std::shared_ptr<const fizz::server::FizzServerContext> {
-        using T = std::decay_t<decltype(tls)>;
-        if constexpr (std::is_same_v<T, config::Insecure>) {
-          return quic::samples::createFizzServerContextWithInsecureDefault(
-              alpns,
-              fizz::server::ClientAuthMode::None,
-              "",
-              ""
-          );
-        } else {
-          return quic::samples::createFizzServerContext(
-              alpns,
-              fizz::server::ClientAuthMode::Optional,
-              tls.certFile,
-              tls.keyFile
-          );
-        }
-      },
-      cfg.tlsMode
-  );
-}
-
 // Translate the QUIC flow-control / idle-timeout knobs into the QMUX transport
 // params advertised to the peer. mvfst-only tunables don't apply over TCP.
 MoQQmuxServer::Config buildQmuxConfig(const config::QuicConfig& quic) {
@@ -78,7 +52,10 @@ MoqxQmuxRelayServer::MoqxQmuxRelayServer(
 )
     : MoQQmuxServer(
           listenerCfg.endpoint,
-          buildFizzContext(listenerCfg),
+          tls::buildFizzServerContext(
+              listenerCfg.tlsMode,
+              {.alpns = buildQmuxAlpns(listenerCfg.moqtVersions)}
+          ),
           buildQmuxConfig(listenerCfg.quic)
       ),
       listenerCfg_(listenerCfg), context_(std::move(context)), ioExecutor_(ioExecutor) {

src/MoqxRelayServer.cpp

@@ -7,24 +7,15 @@
 #include "MoqxRelayServer.h"
 #include "stats/EventBaseStatsCollector.h"
 #include "stats/QuicStatsCollector.h"
+#include "tls/FizzContextBuilder.h"
 #include <moxygen/MoQRelaySession.h>
 #include <moxygen/events/MoQFollyExecutorImpl.h>
 #include <moxygen/util/InsecureVerifierDangerousDoNotUseInProduction.h>
-#include <proxygen/httpserver/samples/hq/FizzContext.h>
 
-#include <fizz/backend/openssl/certificate/CertUtils.h>
-#include <fizz/server/AeadTicketCipher.h>
-#include <fizz/server/DefaultCertManager.h>
-#include <fizz/server/ReplayCache.h>
-#include <fizz/server/TicketCodec.h>
-#include <fizz/util/Status.h>
-#include <folly/Random.h>
 #include <folly/logging/xlog.h>
 #include <quic/QuicConstants.h>
 #include <quic/logging/FileQLogger.h>
 
-#include <array>
-
 using namespace moxygen;
 
 namespace openmoq::moqx {
@@ -38,97 +29,6 @@ std::vector<std::string> buildAlpns(const std::string& versions) {
   return alpns;
 }
 
-// Build a FizzServerContext from in-memory PEM buffers so a PKCS#12-derived key
-// never touches disk. Mirrors quic::samples::createFizzServerContextImpl (the
-// path-based sample helper) to match the PEM-file path's TLS behavior.
-// TODO(#482): replace with a buffer-based helper in the moxygen fork.
-std::shared_ptr<const fizz::server::FizzServerContext> buildFizzContextFromMaterial(
-    const std::vector<std::string>& alpns,
-    fizz::server::ClientAuthMode clientAuth,
-    const std::string& certChainPem,
-    const std::string& keyPem
-) {
-  std::unique_ptr<fizz::SelfCert> cert;
-  fizz::Error err;
-  FIZZ_THROW_ON_ERROR(fizz::openssl::CertUtils::makeSelfCert(cert, err, certChainPem, keyPem), err);
-  auto certManager = std::make_shared<fizz::server::DefaultCertManager>();
-  certManager->addCertAndSetDefault(std::move(cert));
-
-  auto ctx = std::make_shared<fizz::server::FizzServerContext>();
-  ctx->setCertManager(certManager);
-  auto ticketCipher = std::make_shared<fizz::server::Aead128GCMTicketCipher<
-      fizz::server::TicketCodec<fizz::server::CertificateStorage::X509>>>(
-      ctx->getFactoryPtr(),
-      std::move(certManager)
-  );
-  std::array<uint8_t, 32> ticketSeed;
-  folly::Random::secureRandom(ticketSeed.data(), ticketSeed.size());
-  ticketCipher->setTicketSecrets({{folly::range(ticketSeed)}});
-  ctx->setTicketCipher(ticketCipher);
-  ctx->setClientAuthMode(clientAuth);
-  ctx->setSupportedAlpns(alpns);
-  ctx->setAlpnMode(fizz::server::AlpnMode::Required);
-  ctx->setSendNewSessionTicket(true);
-  ctx->setEarlyDataFbOnly(false);
-  ctx->setVersionFallbackEnabled(false);
-
-  fizz::server::ClockSkewTolerance tolerance;
-  tolerance.before = std::chrono::minutes(-5);
-  tolerance.after = std::chrono::minutes(5);
-  std::shared_ptr<fizz::server::ReplayCache> replayCache =
-      std::make_shared<fizz::server::AllowAllReplayReplayCache>();
-  ctx->setEarlyDataSettings(true, tolerance, std::move(replayCache));
-  return ctx;
-}
-
-std::shared_ptr<const fizz::server::FizzServerContext>
-buildFizzContext(const config::ListenerConfig& cfg) {
-  auto alpns = buildAlpns(cfg.moqtVersions);
-  return std::visit(
-      [&alpns](const auto& tls) -> std::shared_ptr<const fizz::server::FizzServerContext> {
-        using T = std::decay_t<decltype(tls)>;
-        if constexpr (std::is_same_v<T, config::Insecure>) {
-          return quic::samples::createFizzServerContextWithInsecureDefault(
-              alpns,
-              fizz::server::ClientAuthMode::None,
-              "",
-              ""
-          );
-        } else {
-          // PKCS#12 source: build the context from the in-memory PEM material
-          // (cert/key never written to disk).
-          if (tls.material.has_value()) {
-            return buildFizzContextFromMaterial(
-                alpns,
-                fizz::server::ClientAuthMode::Optional,
-                tls.material->certChainPem,
-                tls.material->keyPem
-            );
-          }
-          // createFizzServerContext throws (deep in fizz) when the cert/key
-          // can't be read or contain no certificate. Enrich the message with
-          // the offending paths so the caller can report it cleanly instead of
-          // letting an opaque "no certificates read" escape to std::terminate.
-          try {
-            return quic::samples::createFizzServerContext(
-                alpns,
-                fizz::server::ClientAuthMode::Optional,
-                tls.certFile,
-                tls.keyFile
-            );
-          } catch (const std::exception& e) {
-            throw std::runtime_error(
-                "failed to load TLS certificate/key (cert='" + tls.certFile + "', key='" +
-                tls.keyFile + "'): " + e.what() +
-                " - check the paths exist and are readable by this process"
-            );
-          }
-        }
-      },
-      cfg.tlsMode
-  );
-}
-
 quic::TransportSettings
 buildTransportSettings(const config::QuicConfig& quic, const config::MvfstConfig& mvfst) {
   // Start with MoQServer's optimized defaults, then apply config overrides.
@@ -211,7 +111,10 @@ MoqxRelayServer::MoqxRelayServer(
     folly::IOThreadPoolExecutor* ioExecutor
 )
     : MoQServer(
-          buildFizzContext(listenerCfg),
+          tls::buildFizzServerContext(
+              listenerCfg.tlsMode,
+              {.alpns = buildAlpns(listenerCfg.moqtVersions)}
+          ),
           listenerCfg.endpoint,
           MoQServer::Options{
               .transportSettings = buildTransportSettings(listenerCfg.quic, listenerCfg.mvfst),