From e7e382ebe48d5f06aa6d3f505bc3777a16b2975b Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Thu, 3 Sep 2026 12:58:58 -0700 Subject: [PATCH 01/10] feature(security): per-job certificates for job processes Job processes (SJ/CJ) get a short-lived credential issued at deploy time by a constrained job CA (CA:TRUE pathlen:0, marker extension) provisioned into the server kit by default. Leaf certs keep CN= and carry the job ID; site-scope identity checks reject anything the job CA signed, and cellnet binds a job cert to its job's FQCNs. In secure mode jobs never run on site certificates: missing/expiring job CA fails the deploy, CP rejects a deploy without a credential, job cells refuse to start without one. Docker, K8s and Slurm launchers never ship *.key files to job processes. Closes #5249 --- docs/design/job_launcher_and_job_handle.md | 2 +- .../no_shared_pvc_k8s_workspace_design.md | 16 +- docs/design/per_job_certs_design.md | 266 ++++++++++++++++++ docs/index.rst | 1 + docs/troubleshooting/common_errors.rst | 18 ++ .../admin_guide/deployment/helm_chart.rst | 9 +- .../deployment/slurm_job_launcher.rst | 8 + .../admin_guide/nvflare_security.rst | 1 + .../security/identity_security.rst | 7 +- .../security/per_job_certificates.rst | 232 +++++++++++++++ .../user_guide/nvflare_cli/deploy_command.rst | 9 + .../nvflare_cli/distributed_provisioning.rst | 10 + .../nvflare_cli/provision_command.rst | 22 ++ nvflare/apis/fl_constant.py | 3 + nvflare/apis/job_launcher_spec.py | 10 +- nvflare/apis/workspace.py | 9 +- .../app_opt/job_launcher/docker_launcher.py | 23 +- nvflare/app_opt/job_launcher/k8s_launcher.py | 31 +- nvflare/app_opt/job_launcher/slurm/batch.py | 2 +- .../app_opt/job_launcher/slurm/launcher.py | 5 + nvflare/app_opt/job_launcher/slurm/manager.py | 12 +- nvflare/app_opt/job_launcher/study_runtime.py | 4 +- .../job_launcher/workspace_cell_transfer.py | 77 +++-- nvflare/fuel/f3/cellnet/credential_manager.py | 12 +- nvflare/fuel/f3/cellnet/identity.py | 9 +- nvflare/fuel/f3/drivers/aio_conn.py | 3 +- nvflare/fuel/f3/drivers/aio_grpc_driver.py | 4 +- nvflare/fuel/f3/drivers/aio_http_driver.py | 3 +- nvflare/fuel/f3/drivers/driver_params.py | 1 + nvflare/fuel/f3/drivers/grpc_driver.py | 4 +- nvflare/fuel/f3/drivers/net_utils.py | 53 ++++ nvflare/fuel/f3/drivers/socket_conn.py | 3 +- nvflare/fuel/f3/sfm/conn_manager.py | 7 +- nvflare/lighter/constants.py | 12 + nvflare/lighter/impl/cert.py | 100 +++++-- nvflare/lighter/utils.py | 23 +- nvflare/private/defs.py | 1 + .../private/fed/app/client/worker_process.py | 2 + nvflare/private/fed/app/fl_conf.py | 9 + .../private/fed/app/server/runner_process.py | 2 + nvflare/private/fed/client/fed_client_base.py | 6 + nvflare/private/fed/client/training_cmds.py | 15 +- nvflare/private/fed/server/job_runner.py | 47 +++- nvflare/private/fed/utils/fed_utils.py | 16 +- nvflare/private/fed/utils/identity_utils.py | 17 +- nvflare/private/fed/utils/job_cert_utils.py | 231 +++++++++++++++ .../job_launcher/docker_launcher_test.py | 101 ++++++- .../app_opt/job_launcher/k8s_launcher_test.py | 73 ++++- .../job_launcher/slurm_launcher_test.py | 15 + .../job_launcher/slurm_manager_test.py | 23 +- .../workspace_cell_transfer_test.py | 63 ++++- .../fuel/f3/cellnet/identity_binding_test.py | 98 ++++++- tests/unit_test/lighter/cert_builder_test.py | 106 ++++++- .../fed/client/fed_client_base_test.py | 67 +++++ .../client/test_training_cmds_sig_verify.py | 58 +++- .../fed/server/job_runner_deploy_test.py | 14 + tests/unit_test/private/fed/test_fed_utils.py | 6 + .../private/fed/utils/fed_utils_test.py | 30 ++ .../private/fed/utils/identity_utils_test.py | 122 ++++++++ .../private/fed/utils/job_cert_utils_test.py | 241 ++++++++++++++++ .../tool/package/package_commands_test.py | 14 +- 61 files changed, 2259 insertions(+), 129 deletions(-) create mode 100644 docs/design/per_job_certs_design.md create mode 100644 docs/user_guide/admin_guide/security/per_job_certificates.rst create mode 100644 nvflare/private/fed/utils/job_cert_utils.py create mode 100644 tests/unit_test/private/fed/utils/job_cert_utils_test.py diff --git a/docs/design/job_launcher_and_job_handle.md b/docs/design/job_launcher_and_job_handle.md index 8335bde931..1d29c5bb92 100644 --- a/docs/design/job_launcher_and_job_handle.md +++ b/docs/design/job_launcher_and_job_handle.md @@ -414,7 +414,7 @@ Launch sequence: | 1 | Sanitize job ID via `uuid4_to_rfc1123`. Extract `site_name`, `job_image` from `get_job_launcher_spec(job_meta, site_name, "k8s")`. Raise if `WORKSPACE_OBJECT` missing. | | 2 | Read `JOB_PROCESS_ARGS`; raise if absent or `EXE_MODULE` missing. If `/local/study_runtime.yaml` exists, parse it (strict v2) and resolve the job study's datasets, env, secret_env, secret_mounts, container name, and pod template from it; coexistence with a v1 study data file is a hard error. Otherwise resolve dataset PVC mounts from `study_data_pvc_file_path` when configured and the YAML file contains entries for the job study. | | 3 | Build `job_config`: name, image, args from `get_module_args()`. Use `launcher_spec[site][k8s].python_path` for the pod command when present, falling back to `default_python_path`. Mount the job workspace at `workspace_mount_path`, mount the startup-kit Secret at `/startup`, and set custom-code `PYTHONPATH` under `workspace_mount_path`. Add the workspace `emptyDir.sizeLimit` and `resources.requests/limits["ephemeral-storage"]` from `launcher_spec[site][k8s].ephemeral_storage` when present, falling back to the launcher default. Add K8s CPU and memory limits from `launcher_spec`; add GPU limits from the flat `resource_spec[site].num_of_gpus` GPU resource requirement. Apply `launcher_spec[site][k8s].pending_timeout` when present. Missing study entries skip data PVC mounts and log a warning. If a pod template is resolved, preserve template pod fields and sidecars while replacing NVFlare-owned fields such as pod name, job container image/command/args, workspace mounts, transfer env vars, image pull secrets, and resources. | -| 3.5 | Create the per-job credential Secret `nvflare-cred-` (bootstrap credentials plus the workspace transfer token) and reference it from the job container via `env[].valueFrom.secretKeyRef` — credential values never appear in the pod object. | +| 3.5 | Create the per-job credential Secret `nvflare-cred-` (bootstrap credentials, the workspace transfer token, and — when the job has one — the per-job certificate and key as `NVFLARE_JOB_CERT` / `NVFLARE_JOB_KEY`) and reference it from the job container via `env[].valueFrom.secretKeyRef` — credential values never appear in the pod object. The startup Secret never includes `*.key`; in secure mode a job without a credential is refused. | | 4 | Create `K8sJobHandle` (carries the credential Secret name). | | 5 | `core_v1.create_namespaced_pod()`, then patch the credential Secret with an ownerReference to the created pod (GC backstop). On any exception: delete the credential Secret if the pod was never created, set `terminal_state = TERMINATED`, preserve `EXCEPTION` as the return code, and return handle. | | 6 | `job_handle.enter_states([RUNNING])`. On any `BaseException`: `terminate()` then re-raise. | diff --git a/docs/design/no_shared_pvc_k8s_workspace_design.md b/docs/design/no_shared_pvc_k8s_workspace_design.md index 6fa096e3f7..2796521aca 100644 --- a/docs/design/no_shared_pvc_k8s_workspace_design.md +++ b/docs/design/no_shared_pvc_k8s_workspace_design.md @@ -55,9 +55,14 @@ For each launched job pod, the launcher creates a pod manifest with: - optional study-data PVC mounts at `/data//` The launcher also creates or updates a startup Secret for the participant site. -That Secret contains the startup-kit files needed by the launched process, such -as certificates, keys, and JSON config files, and those files appear in the -pod under `/var/tmp/nvflare/workspace/startup` via the Secret mount. +That Secret contains the startup-kit files needed by the launched process — +certificates, `rootCA.pem`, and JSON config files — and those files appear in +the pod under `/var/tmp/nvflare/workspace/startup` via the Secret mount. Site +private keys (`*.key`) are never included: the job's own certificate and key +travel in the per-job credential Secret (`NVFLARE_JOB_CERT` / +`NVFLARE_JOB_KEY`) and the job process writes them into its run directory +before creating the bootstrap cell. In secure mode the launcher refuses to +start a job that has no credential (see `per_job_certs_design.md`). ## Transfer Architecture @@ -203,8 +208,9 @@ artifacts. The key protections are: - `startup/` is mounted read-only from a Kubernetes Secret -- secure mode bootstrap cells use `rootCA.pem` plus the available startup cert - and key pair +- secure mode bootstrap cells use `rootCA.pem` plus the job credential the + launcher delivered; there is no fallback to the site key +- workspace bundles and result uploads never include `job_cert/` - the launcher passes the parent listener's connection-security setting into the child process args, and the bootstrap cell installs its CellNet auth headers before `cell.start()` so the parent accepts the initial registration diff --git a/docs/design/per_job_certs_design.md b/docs/design/per_job_certs_design.md new file mode 100644 index 0000000000..e60ad54f77 --- /dev/null +++ b/docs/design/per_job_certs_design.md @@ -0,0 +1,266 @@ +# Per-Job Certificates for Job Cells + +This document describes per-job TLS credentials for job processes (SJ: server +job process, CJ: client job process). Before this change both job cells loaded +the same provisioned site certificates as their parent processes (SP: server +parent, CP: client parent), so every job process held the site's long-lived +private key. + +## Problem + +Job processes run job-supplied code (custom Executors, Controllers, third-party +training code). The cell creation paths gave them the site identity keys: + +- SJ: `BaseServer.create_job_cell()` read `server.crt` / `server.key` from the + server startup kit — identical to SP. +- CJ: `FederatedClientBase._create_cell()` read `client.crt` / `client.key` + from the client startup kit — identical to CP. + +Any code running inside a job could therefore read the site's private key and +impersonate the site indefinitely: register as a CP, decrypt message-level +traffic, or authenticate as the site after the job ends. + +## Goal + +Give each job process its own short-lived credential, scoped to one job, make +that credential the only one the job process refers to, and have container and +scheduler launchers withhold the site private keys from the job entirely. + +Non-goals: certificate revocation (bounded validity plus job-workspace teardown +limit the exposure window instead) and HA deployments with multiple servers. + +## Trust Model + +```text +rootCA (private key exists only during provisioning) +├── server.crt / server.key server startup kit (unchanged) +├── client.crt / client.key client startup kits (unchanged) +└── job_ca.crt / job_ca.key server startup kit ONLY (new) + CA:TRUE, pathlen:0, job-CA marker extension + └── per-job leaf certs, issued at job deploy time + CN=, job_id extension, bounded validity +``` + +Because the job CA chains to the existing root, no participant needs a new +trust anchor: a job cert presented together with `job_ca.crt` validates against +`rootCA.pem` with standard X.509 path validation, both in TLS handshakes and in +`verify_cert_chain()`. + +Leaf certs keep `CN=` so every existing common-name-based identity +check continues to pass. The job binding is carried in a certificate extension +holding the job ID. + +Only SP ever holds the job CA key. CP receives issued certificates; it does no +signing. + +### Extension OIDs + +Both extensions live under NVIDIA's IANA private enterprise arc +(`1.3.6.1.4.1.5703`), sub-arc `300`: + +| OID | Placed on | Meaning | +| --- | --------- | ------- | +| `1.3.6.1.4.1.5703.300.1` | job leaf certs | the job ID the credential is bound to | +| `1.3.6.1.4.1.5703.300.2` | the job CA cert | "issued by the job CA" marker | + +Both are non-critical, so standard TLS stacks ignore them; only FLARE code reads +them. Neither can be stripped: the marker is inside the root-signed job CA cert, +the job ID inside the job-CA-signed leaf. + +## Provisioning + +`CertBuilder` gains an `enable_job_ca` option, on by default so every newly +provisioned project (including POC) gets the job CA. It can be turned off: + +```yaml +builders: + - path: nvflare.lighter.impl.cert.CertBuilder + args: + enable_job_ca: false +``` + +Provisioning generates one additional pair signed by the root: +`job_ca.crt` / `job_ca.key`, written to the **server** startup kit only, with +the key at mode 0600. The pair is persisted in the certificate state file so +re-provisioning reuses it (an expired stored job CA is regenerated). `pathlen:0` +prevents the job CA from issuing further CAs. + +Client startup kits are unchanged. A server kit provisioned before this feature +has no job CA, so in secure mode every job deploy fails until the project is +re-provisioned; there is no mode in which jobs run on site certificates. +Re-provisioning an existing project adds the job CA (the root is reused from +the state file, so no other cert changes) and only the server kit needs to be +redeployed. + +A customer-supplied root passed to `nvflare provision` is covered: `CertBuilder` +signs the job CA with whatever root key it holds. The distributed flow +(`nvflare cert` at the CA side, `nvflare package` at the site) is not: the +packaging step has no root key, so those kits have no job CA and cannot run +jobs in secure mode. Extending that flow (job-CA CSR signed by `nvflare cert`, +installed by `nvflare package`) is addressed separately. + +## Runtime Issuance (SP) + +A `JobCertIssuer` in the SP process loads `job_ca.crt` / `job_ca.key` from the +startup kit. In secure mode every job deploy needs it: if the kit has no job CA, +or the job CA has less than a minimum remaining validity (so jobs never get +certs that expire mid-run), the deploy fails with the reason recorded in the +job's deploy detail. Secure jobs never fall back to site certificates — a +downgrade would only be visible in a log line nobody reads. + +During job deployment (`JobRunner._deploy_job`), for each participant the +issuer generates an RSA keypair and a leaf certificate: + +- the subject CN matches the CN the site's own certificate presents: for the + SJ it is read from the server certificate in the startup kit, and for each + CJ it is the registered client name (registration enforces that this equals + the client cert's CN) — so whatever identity enforcement passed with site + certs passes with job certs +- a job-ID extension identifying the job +- `notBefore` backdated a few minutes to tolerate clock skew between the + issuing server and the sites that validate the cert seconds later +- `notAfter` = issue time + `job_cert_valid_days` (server startup config + `fed_server.json`, `--set`, or `NVFLARE_JOB_CERT_VALID_DAYS`; default 30), + clamped to the job CA's own expiry. There is no renewal, so this is the + maximum job duration; set it for the longest job the server runs. + +The issued credential is a PEM bundle: leaf cert followed by `job_ca.crt` +(so TLS peers can build the chain to the root), plus the private key PEM. + +## Distribution + +**SJ (local write).** After all apps are deployed to the server workspace +(`AppDeployer` recreates the run directory), SP writes the SJ credential into +the job run directory before the SJ process is launched. + +**CJ (push).** The job deploy message becomes per-site: the shared app bytes +stay a single payload reference, but each site's message carries an additional +header with that site's cert bundle and private key. The message travels over +the existing authenticated CP–SP channel (mTLS in secure mode). On the client, +the deploy processor writes the credential into the job run directory (key at +mode 0600) after the app passes signature verification and deploys. + +No new channels, topics, or handshakes are introduced. SP generates the CJ +keypair and sends the private key together with the certificate. A CSR exchange +(CP generates the key, SP signs) would not change the trust model — SP holds the +job CA key and can already mint a certificate for any key it chooses — so it +would add a deploy round trip for nothing. + +In secure mode the CP rejects a deploy request that carries no valid job +credential; the job is not deployed on site certificates. + +## Workspace Layout + +```text +/ + startup/ site certs (unchanged); server kit adds job_ca.* + / + app_/ deployed app (unchanged) + job_cert/ + job.crt leaf + job_ca.crt PEM bundle + job.key per-job private key, 0600 +``` + +## Job Process Changes + +At job-process startup, `security_init_for_job` refuses to start a secure job +whose run directory has no `job_cert/job.crt` + `job_cert/job.key`. The starter +configers then make those paths the process's `ssl_cert` / `ssl_private_key`, +so nothing in the job process refers to the site key any more; there is no +separate "job credential" configuration key. `ssl_root_cert` remains +`rootCA.pem`. + +- `BaseServer.create_job_cell()` and `FederatedClientBase._create_cell()` use + the job credential for the SJ/CJ cell. The CJ also pins its server-role + credential to the job cert; otherwise, on listener-enabled sites, the site's + server cert would be back-filled from the startup kit and preferred by + message-level crypto. +- The startup content-integrity check (`signature.json` kits) no longer + requires the site private key in job processes. +- The job process never registers with the server (it receives the CP's auth + token and signature) and never asserts site identity, so it has no other use + for the site key. + +Non-secure mode and the simulator use no certificates at all, so the job +process keeps its site config untouched there. + +## Site-Scope Rejection + +A job cert is a valid `CN=` certificate chaining to the project root, so +without extra checks job credentials could be replayed at site scope — most +notably to register a rogue CP (client registration accepts a caller-supplied +cert chain). All site-scope identity assertions funnel through +`IdentityVerifier.verify_common_name()` (client registration, admin login, and +the client's verification of the server), and no job cell ever legitimately +asserts identity there. Two rejections cover two distinct threats: + +1. **Leaked job leaf key**: any certificate carrying the job-ID extension is + rejected. This is keyed on the extension, not the issuer, so it holds + regardless of which CA issued the certificate (which also keeps future HA + setups with multiple job CAs simple). +2. **Stolen job CA key**: an attacker holding `job_ca.key` can mint a clean + site-named leaf *without* the extension. The job CA certificate therefore + carries a root-signed marker extension, and any presented chain containing + a marked CA is rejected. The attacker cannot strip the marker (the job CA + cert is signed by the root) and cannot validate without presenting it. + +With both checks, compromise of the job CA key no longer escalates to site or +admin identity; its blast radius is job cells only. + +## Job Binding in Cellnet + +Site-scope rejection stops a job credential from acting as a site. Job binding +stops one job's credential from acting as another job's cell: + +- Every TLS driver exposes the peer certificate's job-ID extension as the + `PEER_JOB_ID` connection property next to `PEER_CN`. +- `CellIdentityResolver.require_match()` rejects a peer whose certificate is + bound to job X unless the FQCN it claims contains the segment X + (`site-1.X`, `server.X`, and their descendants). The check runs at the + connection handshake (`ConnManager`) and again on the certificate exchanged + for message-level crypto (`CredentialManager`), which is the certificate + later used to decrypt that peer's messages. + +The rule is one-directional on purpose: it constrains what a job cert may +claim, not which cert a job FQCN must present. Refusing to start a job cell +without a job credential is enforced at job startup (`security_init_for_job`), +not in the connection handshake. + +## Site-Key Isolation by Launcher + +With the job process no longer referring to the site key, launchers never ship +the `*.key` files of the startup kit (this includes `job_ca.key` on the server) +to a job process, and in secure mode they refuse to launch a job that has no +credential. + +| Launcher | Startup kit delivered to the job | Job credential | +| -------- | -------------------------------- | -------------- | +| Process | same host workspace, same user — no filesystem isolation possible | run dir | +| Docker | `startup/` bound file by file, read-only, `*.key` omitted | read-write job workspace bind | +| Kubernetes | startup Secret without `*.key` | `NVFLARE_JOB_CERT` / `NVFLARE_JOB_KEY` in the per-pod credential Secret (`secretKeyRef` env) | +| Slurm (apptainer / pyxis) | keyless staged copy under the 0700 job dir, bound at `/startup` | run dir bind | +| Slurm (`sandbox: none`) | bare host process — no isolation possible | run dir | + +Kubernetes needs the environment route because the pod's bootstrap cell, which +downloads the run directory, exists before the run directory does. The job +process pops both variables and writes the credential into the run directory +(`_install_job_cert_from_env`) before creating the bootstrap cell, which +authenticates with the job credential only. Workspace bundles and result +uploads exclude `job_cert/` so the key travels only once. +A Secret volume at `/job_cert` was rejected because kubelet would +create the run directory root-owned and break extraction for non-root pods. + +## Compatibility + +| Deployment | Behavior | +| ---------- | -------- | +| Server kit without job CA (pre-feature, `enable_job_ca: false`, `nvflare package`) | Secure mode: every job deploy fails with an explicit error until the project is re-provisioned. Non-secure mode: unaffected | +| Server kit with job CA, matching client release (default for new provision and POC) | CJ certs pushed and used; SJ cert used; launchers withhold site keys | +| Server kit with job CA, older client release | Not supported: the old CJ would run on site certificates. Run matching releases | +| Non-secure mode / simulator | No certificates in use; feature inactive | +| Relays | Relays keep their provisioned site certs; CJs behind a relay use job certs | +| Sub-worker cells, Client API trainers | Unchanged: sub-workers use unauthenticated local internal links; CellPipe trainers connect with the root CA only. Neither holds a site key, and if they are ever given a credential it must be the job credential, never the site's. | + +(Message-level encryption was verified to already support certificate chains: +`CredentialManager` loads multi-cert PEMs and `SimpleCellCipher` validates +leaf + intermediates against the root, so no changes were needed there.) diff --git a/docs/index.rst b/docs/index.rst index c492bfbdd2..81e9c95d88 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -107,6 +107,7 @@ NVIDIA FLARE system_architecture/security_overview user_guide/admin_guide/security/terminologies_and_roles Identity & Access Control + Per-Job Certificates user_guide/admin_guide/security/site_policy_management Network & Communication Data Privacy & Filters diff --git a/docs/troubleshooting/common_errors.rst b/docs/troubleshooting/common_errors.rst index b4db9c7452..6bc8d3b4a4 100644 --- a/docs/troubleshooting/common_errors.rst +++ b/docs/troubleshooting/common_errors.rst @@ -47,6 +47,24 @@ Job Submission & Execution Errors - Out-of-memory errors (see also :doc:`/programming_guide/memory_management`) - Model serialization errors +Job fails with ``FAILED_TO_RUN`` and a job credential message +--------------------------------------------------------------- + +In secure mode every job runs on a :ref:`per-job certificate ` +and there is no fallback to the site certificates. The reason is recorded in +the job's ``job_deploy_detail``: + +- ``server startup kit has no job CA (job_ca.crt / job_ca.key)``: the server was + provisioned before this feature or with ``enable_job_ca: false``. + Re-provision the project and redeploy the server startup kit. +- ``job CA expires at ...``: the job CA has less than one hour of validity + left. Re-provision the project to renew it. +- ``deploy request carries no valid job credential`` (reported by a client): + the server did not send a job credential. The server and client releases do + not match; run the same release on all participants. +- ``has no job credential`` (reported by the Docker, Kubernetes, or Slurm + launcher): same cause as above. + Training & Convergence Issues ============================== diff --git a/docs/user_guide/admin_guide/deployment/helm_chart.rst b/docs/user_guide/admin_guide/deployment/helm_chart.rst index 48e8d5a451..c8e24c32bd 100644 --- a/docs/user_guide/admin_guide/deployment/helm_chart.rst +++ b/docs/user_guide/admin_guide/deployment/helm_chart.rst @@ -982,9 +982,12 @@ Role/RoleBinding by default. The launcher needs permission to: The Secret permission is required because the launcher creates or updates a per-site startup-kit Secret for dynamically launched job pods, and a per-job credential Secret (``nvflare-cred-``) delivering the job bootstrap -credentials as env vars via ``secretKeyRef``. The credential Secret is patched -with an ownerReference to its pod and deleted when the job ends. Job pods mount -the startup-kit Secret read-only at ``/startup``. The +credentials and the job's own certificate and key (see +:ref:`per_job_certificates`) as env vars via ``secretKeyRef``. The credential +Secret is patched with an ownerReference to its pod and deleted when the job +ends. Job pods mount the startup-kit Secret read-only at +``/startup``; that Secret contains the certificates, +``rootCA.pem``, and JSON configuration of the kit but never a private key. The startup-kit Secret name uses this pattern: .. code-block:: text diff --git a/docs/user_guide/admin_guide/deployment/slurm_job_launcher.rst b/docs/user_guide/admin_guide/deployment/slurm_job_launcher.rst index 33c7d4897f..604a84e2c4 100644 --- a/docs/user_guide/admin_guide/deployment/slurm_job_launcher.rst +++ b/docs/user_guide/admin_guide/deployment/slurm_job_launcher.rst @@ -516,6 +516,14 @@ insecure opt-out, and the shared-file transport is clear file I/O (see isolated site network or filesystem. None of this changes the configured security of the external NVFlare federation channel. +Job processes run on :ref:`per-job certificates `. With +``sandbox: apptainer`` or ``sandbox: pyxis`` the job sees a copy of the startup +kit without any private key, staged under the job's transient artifact +directory and bound at ``/startup``; the job's own certificate and +key live in its run directory. With ``sandbox: none`` the job is a plain +process on the shared filesystem and no such isolation is possible. In secure +mode the launcher refuses to start a job that has no job credential. + Working accounting is mandatory. The parent refuses to start if ``sacct`` is unavailable. A later scheduler or accounting outage leaves affected jobs non-terminal and retries; it never assumes that a missing observation means a diff --git a/docs/user_guide/admin_guide/nvflare_security.rst b/docs/user_guide/admin_guide/nvflare_security.rst index 1112ae4da1..c0bbe69b72 100644 --- a/docs/user_guide/admin_guide/nvflare_security.rst +++ b/docs/user_guide/admin_guide/nvflare_security.rst @@ -31,6 +31,7 @@ NVFLARE implements security measures in the following areas (see each section be security/terminologies_and_roles security/identity_security + security/per_job_certificates security/site_policy_management security/authorization_policy_previewer security/communication_security diff --git a/docs/user_guide/admin_guide/security/identity_security.rst b/docs/user_guide/admin_guide/security/identity_security.rst index 8b5d55f6c4..5131efa580 100644 --- a/docs/user_guide/admin_guide/security/identity_security.rst +++ b/docs/user_guide/admin_guide/security/identity_security.rst @@ -31,7 +31,12 @@ The security of the system comes from the PKI credentials in the Startup Kits. A - The Project Admin, who is responsible for the provisioning process of the study, should protect the study's configuration files and store created Startup Kits securely. - When distributing Startup Kits, the Project Admin should use trusted communication methods, and never send passwords of the Startup Kits in the same communication. It is preferred to send the Kits and passwords with different communication methods. - Org Admin and users must protect their Startup Kits and only use them for intended purposes. - + +Job processes do not use the site's certificate and private key. In secure mode +every job process runs on a short-lived certificate issued for that job alone, +and container and scheduler launchers withhold the site private keys from job +processes. See :ref:`per_job_certificates`. + .. note:: The provisioning tools generate X.509 certificates with 2048-bit RSA diff --git a/docs/user_guide/admin_guide/security/per_job_certificates.rst b/docs/user_guide/admin_guide/security/per_job_certificates.rst new file mode 100644 index 0000000000..663d1d26ef --- /dev/null +++ b/docs/user_guide/admin_guide/security/per_job_certificates.rst @@ -0,0 +1,232 @@ +.. _per_job_certificates: + +#################### +Per-Job Certificates +#################### + +In secure mode (``secure_train=true``, the normal production setting where all +participants authenticate with TLS), every job process — the server job process +(SJ) and each client job process (CJ) — runs on its own short-lived X.509 +certificate, issued for that job alone. Job processes run job-supplied code (custom Controllers and +Executors, third-party training code). Before this feature that code had access +to the site's long-lived certificate and private key: whatever the site could +do, the job could do, for as long as the site key stayed valid. + +With per-job certificates: + +- a job process never refers to the site's private key; its TLS and + message-level credential is the job certificate; +- the Docker, Kubernetes, and Slurm job launchers do not give the job any site + private key; +- a job certificate cannot register a site, log in as an administrator, + impersonate the server, or act as another job's process, even if it leaks; +- there is no fallback: in secure mode a job either gets its certificate or it + does not run. + +Non-secure mode (``secure_train=false``) and the simulator use no certificates +and are not affected. + +How it works +============ + +Provisioning creates a job-signing intermediate CA — ``job_ca.crt`` and +``job_ca.key`` — in the **server** startup kit only. It is signed by the project +root CA, so no participant needs a new trust anchor. When a job is deployed, the +server issues one certificate per participating site (``CN=``, with +an extension carrying the job ID), writes the server job's credential into the +job's run directory, and sends each client its own credential inside the deploy +message over the existing mutually authenticated channel. Every site verifies +these certificates against the ``rootCA.pem`` it already has. + +Job certificates are valid for 30 days by default (see `Certificate validity`_) +and are not renewed: a job cannot run longer than its certificate. + +What to expect after provisioning +================================= + +The server startup kit gains two files:: + + startup/ + job_ca.crt job-signing CA certificate, signed by the project root + job_ca.key its private key, mode 0600 — protect it like server.key + +Client and admin kits are unchanged. Keep ``job_ca.key`` with the server's other +secrets. Whoever holds it can issue job certificates, and nothing else: +site-level authentication (client registration, admin login, server identity) +rejects any certificate the job CA signed. + +Each running job has its credential at:: + + //job_cert/job.crt certificate followed by job_ca.crt + //job_cert/job.key private key, mode 0600 + +Enabling and disabling +====================== + +The job CA is created by default. To turn it off, set ``enable_job_ca`` on +``CertBuilder`` in ``project.yml``: + +.. code-block:: yaml + + builders: + - path: nvflare.lighter.impl.cert.CertBuilder + args: + enable_job_ca: false + +Only do this for non-secure deployments (``secure_train=false``, e.g. local +testing). A production server provisioned without the job CA cannot run jobs: +every job deploy fails with ``server startup kit has no job CA``, because jobs +are never started on the site's own certificate. + +Upgrading an existing project +============================= + +Server startup kits provisioned before this feature have no job CA. Once the +server runs a release with per-job certificates, every job deploy fails until +the project is re-provisioned: + +#. Run ``nvflare provision`` again on the same provisioning workspace. The root + CA and all participant certificates are reused from the workspace's + ``state`` directory; only ``job_ca.crt`` / ``job_ca.key`` are added to the + server kit. +#. Redeploy the server startup kit. Client and admin kits do not change. + +Run the same NVFlare release on the server and on all clients. An older client +would run its job process on the site certificate, which a server with per-job +certificates does not allow; the client reports ``deploy request carries no +valid job credential`` and the job fails. + +Certificate validity +==================== + +Job certificates are issued for ``job_cert_valid_days`` (default 30), clamped +to the job CA's own expiry. There is no renewal, so this value is the maximum +duration of a job; set it for the longest job the server runs. It is a server +setting, resolved in this order: + +#. ``--set job_cert_valid_days=`` on the server start command; +#. a top-level ``"job_cert_valid_days": `` entry in the server's + ``fed_server.json``; +#. the environment variable ``NVFLARE_JOB_CERT_VALID_DAYS``. + +The job CA itself is valid for 360 days, bounded by the root CA. When it has +less than one hour of validity left, job deploys fail with ``job CA expires at +...``; re-provision the project to renew it (``nvflare provision`` regenerates +an expired job CA automatically). + +Job launchers +============= + +Job processes never need a site private key, so the launchers do not give them +one. In secure mode each launcher refuses to start a job that has no job +credential instead of falling back to site certificates. + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Launcher + - What the job process sees + * - Process (in-process) + - The site workspace as-is. The job process runs as the same user on the + same host, so the site key is readable on disk; only the job's own + configuration no longer refers to it. + * - Docker + - ``startup/`` bound into the container file by file, read-only, without + ``*.key`` files. The job credential arrives with the job's read-write + workspace bind. + * - Kubernetes + - A startup-kit Secret without ``*.key`` files. The job certificate and + key are delivered through the per-pod credential Secret + (``nvflare-cred-``) as the ``NVFLARE_JOB_CERT`` and + ``NVFLARE_JOB_KEY`` variables, which the job process writes to its run + directory and removes from its environment before anything else starts. + Workspace bundles never contain the job credential. + * - Slurm, ``sandbox: apptainer`` or ``pyxis`` + - A copy of the startup kit without ``*.key`` files, staged under the + job's transient artifact directory and bound at ``/startup``. + The job credential is in the job's run directory. + * - Slurm, ``sandbox: none`` + - The shared filesystem as-is, like the process launcher. + +See :ref:`deploy_prepare_command`, :ref:`helm_chart`, and +:ref:`slurm_job_launcher`. + +.. _per_job_certificates_external_ca: + +Distributed provisioning and external PKI +========================================= + +Kits assembled with the distributed ``nvflare cert`` / ``nvflare package`` +workflow (:ref:`distributed_provisioning`) have no job CA, because the packaging +step never has the root CA key. A server running from such a kit cannot run +jobs in secure mode. Extending that workflow is addressed separately. + +If you operate your own PKI you can issue the job CA yourself. The server only +requires that ``startup/job_ca.crt`` and ``startup/job_ca.key`` exist and that +the certificate chains to the ``rootCA.pem`` in the kits. The job CA must be an +intermediate CA certificate with: + +- ``basicConstraints = critical, CA:TRUE, pathlen:0``; +- ``keyUsage = critical, digitalSignature, keyCertSign, cRLSign``; +- the job-CA marker: a **non-critical** extension with OID + ``1.3.6.1.4.1.5703.300.2`` (any value). Sites use it to reject anything the + job CA signed when a site identity is asserted, so it must be present. + +The root CA certificate in ``rootCA.pem`` must itself carry a ``keyUsage`` +extension that includes ``keyCertSign``. NVFlare validates certificate chains +strictly per RFC 5280 and rejects a root without it; roots created by +``nvflare provision`` or ``nvflare cert init`` always have it, but the default +OpenSSL ``v3_ca`` profile leaves ``keyUsage`` commented out. + +An OpenSSL extension section that produces this: + +.. code-block:: ini + + [ job_ca_ext ] + basicConstraints = critical, CA:TRUE, pathlen:0 + keyUsage = critical, digitalSignature, keyCertSign, cRLSign + subjectKeyIdentifier = hash + authorityKeyIdentifier = keyid:always + 1.3.6.1.4.1.5703.300.2 = ASN1:UTF8String:job_ca + +.. code-block:: bash + + openssl req -new -newkey rsa:2048 -nodes -keyout job_ca.key \ + -subj "/CN=job_ca." -out job_ca.csr + openssl x509 -req -in job_ca.csr -CA rootCA.pem -CAkey rootCA.key \ + -CAcreateserial -days 360 -extfile job_ca.cnf -extensions job_ca_ext \ + -out job_ca.crt + chmod 600 job_ca.key + +Copy ``job_ca.crt`` and ``job_ca.key`` into the server's ``startup/`` directory +before starting the server. + +Troubleshooting +=============== + +Failures are recorded in the job's ``job_deploy_detail`` (shown by +``list_jobs`` and in the job metadata) and in the server or client log. + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - Message + - Cause and fix + * - ``server startup kit has no job CA (job_ca.crt / job_ca.key)`` + - The server kit was provisioned before this feature, with + ``enable_job_ca: false``, or by ``nvflare package``. Re-provision the + project and redeploy the server kit. + * - ``job CA expires at ... (less than 1:00:00 left)`` + - Re-provision the project to renew the job CA. + * - ``deploy request carries no valid job credential`` (client) + - The server sent no credential: the server and client releases differ. + Run the same release everywhere. + * - ``has no job credential; secure jobs run only on per-job certificates`` + (Docker, Kubernetes, or Slurm launcher) + - Same cause as above. + * - ``authenticated with a certificate bound to job '...' but claimed + endpoint ... is not part of that job`` + - A process presented another job's certificate. This does not happen in + normal operation; investigate the site. diff --git a/docs/user_guide/nvflare_cli/deploy_command.rst b/docs/user_guide/nvflare_cli/deploy_command.rst index 2f65eac18b..da5d75b0d5 100644 --- a/docs/user_guide/nvflare_cli/deploy_command.rst +++ b/docs/user_guide/nvflare_cli/deploy_command.rst @@ -25,6 +25,15 @@ Kubernetes, or Slurm. carrying only ``clientAuth`` or only ``serverAuth`` — must be re-provisioned before using these runtimes. Kits whose certificates carry no EKU (unrestricted) remain compatible and do not need re-provisioning. +.. note:: + + In secure mode, jobs run on :ref:`per-job certificates ` + issued by the server, and these runtimes never ship the site private keys + to job processes. The **server** startup kit must contain the job CA + (``job_ca.crt`` / ``job_ca.key``), which ``nvflare provision`` adds by + default. Server kits provisioned before this feature must be re-provisioned + or every job deploy fails. + For Kubernetes deployment workflow, see :ref:`helm_chart`. For the Slurm deployment workflow and security checklist, see :ref:`slurm_job_launcher`. For job-level runtime settings, see :ref:`launcher_spec`. diff --git a/docs/user_guide/nvflare_cli/distributed_provisioning.rst b/docs/user_guide/nvflare_cli/distributed_provisioning.rst index 9fedd9239e..a8dc4d810f 100644 --- a/docs/user_guide/nvflare_cli/distributed_provisioning.rst +++ b/docs/user_guide/nvflare_cli/distributed_provisioning.rst @@ -52,6 +52,16 @@ startup kit. homomorphic encryption) are not directly supported; use centralized ``nvflare provision`` for those deployments. +.. note:: + + Kits assembled with ``nvflare package`` do not contain the job CA that + :ref:`per-job certificates ` require, because the + packaging step has no access to the root CA key. A server running from such a + kit cannot run jobs in secure mode. Extending this workflow is addressed + separately; until then, either provision the server centrally with + ``nvflare provision`` or add a job CA issued by your own PKI as described in + :ref:`per_job_certificates_external_ca`. + ******************************************* Before You Start: Record Connection Details ******************************************* diff --git a/docs/user_guide/nvflare_cli/provision_command.rst b/docs/user_guide/nvflare_cli/provision_command.rst index 3c9c6fa4a3..bb6c350bbe 100644 --- a/docs/user_guide/nvflare_cli/provision_command.rst +++ b/docs/user_guide/nvflare_cli/provision_command.rst @@ -63,6 +63,28 @@ normal 360-day validity unless the root expires sooner. Changing an established root requires a separate multi-root rollover; ``root_valid_days`` never extends or replaces it. +Job CA +====== + +Provisioning also creates a job-signing intermediate CA, ``job_ca.crt`` / +``job_ca.key``, in the **server** startup kit only. The server uses it to issue +a short-lived certificate to every job process (see +:ref:`per_job_certificates`); in secure mode jobs cannot run without it. It is +on by default and can be turned off: + +.. code-block:: yaml + + builders: + - path: nvflare.lighter.impl.cert.CertBuilder + args: + enable_job_ca: false + +Like the participant certificates, the job CA is kept in the workspace's +``state`` directory and reused on later provisioning runs; an expired one is +regenerated. Running ``nvflare provision`` again on a workspace created before +this feature adds the job CA to the server kit without changing any other +certificate. + Certificate Identity Overrides ============================== diff --git a/nvflare/apis/fl_constant.py b/nvflare/apis/fl_constant.py index 66f57f7f8f..eaceea728c 100644 --- a/nvflare/apis/fl_constant.py +++ b/nvflare/apis/fl_constant.py @@ -572,6 +572,9 @@ class ConfigVarName: # server: max time to wait for client terminal outcomes after the server process exits CLIENT_OUTCOME_WAIT_TIMEOUT = "client_outcome_wait_timeout" + # server: validity in days of the per-job certificates issued at job deploy + JOB_CERT_VALID_DAYS = "job_cert_valid_days" + # customized nvflare decomposers module name DECOMPOSER_MODULE = "nvflare_decomposers" diff --git a/nvflare/apis/job_launcher_spec.py b/nvflare/apis/job_launcher_spec.py index a448a2975a..d71dc74c33 100644 --- a/nvflare/apis/job_launcher_spec.py +++ b/nvflare/apis/job_launcher_spec.py @@ -51,6 +51,13 @@ class JobProcessEnv: TOKEN_SIGNATURE = "NVFLARE_JOB_TOKEN_SIGNATURE" SSID = "NVFLARE_JOB_SSID" + # PEM job credential for launchers whose job process has no filesystem path to the + # run dir at startup (K8s workspace transfer); written to the run dir by download_workspace(). + JOB_CERT = "NVFLARE_JOB_CERT" + JOB_KEY = "NVFLARE_JOB_KEY" + + ALL = (AUTH_TOKEN, TOKEN_SIGNATURE, SSID, JOB_CERT, JOB_KEY) + def pop_credential_env() -> dict: """Remove every JobProcessEnv credential from the environment and return it. @@ -59,8 +66,7 @@ def pop_credential_env() -> dict: job-spawned children from inheriting credentials. Empty values count as absent so a blank env var fails parsing like a missing one. """ - names = (JobProcessEnv.AUTH_TOKEN, JobProcessEnv.TOKEN_SIGNATURE, JobProcessEnv.SSID) - return {name: os.environ.pop(name, None) or None for name in names} + return {name: os.environ.pop(name, None) or None for name in JobProcessEnv.ALL} class JobReturnCode(ProcessExitCode): diff --git a/nvflare/apis/workspace.py b/nvflare/apis/workspace.py index 202764c800..9e7504d1b9 100644 --- a/nvflare/apis/workspace.py +++ b/nvflare/apis/workspace.py @@ -229,9 +229,14 @@ def get_log_root(self, job_id=None) -> str: def get_root_dir(self) -> str: return self.root_dir + @staticmethod + def run_dir_path(root_dir: str, job_id: str) -> str: + """Run directory of job_id under root_dir; usable before the workspace exists on disk.""" + job_id = Workspace._check_job_id(job_id) + return Workspace._join_under_root(root_dir, WorkspaceConstants.WORKSPACE_PREFIX + job_id) + def get_run_dir(self, job_id: str) -> str: - job_id = self._check_job_id(job_id) - return self._join_under_root(self.root_dir, WorkspaceConstants.WORKSPACE_PREFIX + job_id) + return self.run_dir_path(self.root_dir, job_id) def get_app_dir(self, job_id: str) -> str: return os.path.join(self.get_run_dir(job_id), WorkspaceConstants.APP_PREFIX + self.site_name) diff --git a/nvflare/app_opt/job_launcher/docker_launcher.py b/nvflare/app_opt/job_launcher/docker_launcher.py index 615155b879..7e322f58df 100644 --- a/nvflare/app_opt/job_launcher/docker_launcher.py +++ b/nvflare/app_opt/job_launcher/docker_launcher.py @@ -50,6 +50,7 @@ from nvflare.fuel.f3.comm_error import CommError from nvflare.fuel.f3.drivers.file_driver import SCHEME as SHARED_FILE_SCHEME from nvflare.fuel.f3.drivers.file_driver import parse_file_url +from nvflare.private.fed.utils.job_cert_utils import job_startup_files, require_job_cert from nvflare.utils.job_launcher_utils import ( DOCKER_JOB_CONTAINER_KWARGS, get_client_job_args, @@ -720,8 +721,23 @@ def launch_job(self, job_meta: dict, fl_ctx: FLContext) -> JobHandleSpec: self.logger.info(f"launching job {job_id} as container {container_name} using image {job_image}") + if workspace_obj is None: + raise RuntimeError(f"missing {FLContextKey.WORKSPACE_OBJECT} in FLContext") + require_job_cert(fl_ctx, workspace_obj.get_run_dir(job_id)) + docker_client = self._get_docker_client() try: + # bind the kit file by file: site private keys never enter the job container; + # the job credential arrives through the read-write job workspace bind + startup_mounts = [ + docker.types.Mount( + target=posixpath.join(container_startup_dir, fname), + source=os.path.join(host_startup_dir, fname), + type="bind", + read_only=True, + ) + for fname in job_startup_files(workspace_obj.get_startup_kit_dir()) + ] mounts = [ docker.types.Mount( target=self.WORKSPACE_MOUNT, @@ -730,12 +746,7 @@ def launch_job(self, job_meta: dict, fl_ctx: FLContext) -> JobHandleSpec: read_only=False, tmpfs_mode=_WORKSPACE_TMPFS_MODE, ), - docker.types.Mount( - target=container_startup_dir, - source=host_startup_dir, - type="bind", - read_only=True, - ), + *startup_mounts, docker.types.Mount(target=container_local_dir, source=host_local_dir, type="bind", read_only=True), docker.types.Mount( target=container_job_workspace, diff --git a/nvflare/app_opt/job_launcher/k8s_launcher.py b/nvflare/app_opt/job_launcher/k8s_launcher.py index 1611787b23..5dd0f38f70 100644 --- a/nvflare/app_opt/job_launcher/k8s_launcher.py +++ b/nvflare/app_opt/job_launcher/k8s_launcher.py @@ -28,7 +28,14 @@ from nvflare.apis.fl_constant import FLContextKey, JobConstants from nvflare.apis.fl_context import FLContext from nvflare.apis.job_def import JobMetaKey -from nvflare.apis.job_launcher_spec import JobHandleSpec, JobLauncherSpec, JobProcessArgs, JobReturnCode, add_launcher +from nvflare.apis.job_launcher_spec import ( + JobHandleSpec, + JobLauncherSpec, + JobProcessArgs, + JobProcessEnv, + JobReturnCode, + add_launcher, +) from nvflare.app_opt.job_launcher.study_data import ( load_study_data_file, resolve_study_dataset_mounts, @@ -45,6 +52,7 @@ WorkspaceTransferManager, ) from nvflare.fuel.common.exit_codes import ProcessExitCode +from nvflare.private.fed.utils.job_cert_utils import job_startup_files, read_job_cert, require_job_cert from nvflare.utils.job_launcher_utils import ( get_client_job_args, get_credential_env, @@ -125,9 +133,10 @@ class PendingPodAction(Enum): "NetworkNotReady", } # Files actually read from startup/ by the job pod at runtime. Others in -# startup/ are dropped to shrink the Secret. local/ is bundled whole with each -# job workspace so job resource files and local custom code keep working. -_STARTUP_KEEP_SUFFIXES = (".crt", ".key", ".pem", ".json") +# startup/ are dropped to shrink the Secret (job_startup_files() already withholds +# private keys). local/ is bundled whole with each job workspace so job resource +# files and local custom code keep working. +_STARTUP_KEEP_SUFFIXES = (".crt", ".pem", ".json") def _keep_startup_file(fname: str) -> bool: @@ -963,13 +972,11 @@ def _ensure_startup_secret(self, site_name: str, startup_dir: str) -> str: """ data = {} if os.path.isdir(startup_dir): - for fname in os.listdir(startup_dir): + for fname in job_startup_files(startup_dir): if not _keep_startup_file(fname): continue - fpath = os.path.join(startup_dir, fname) - if os.path.isfile(fpath): - with open(fpath, "rb") as f: - data[fname] = base64.b64encode(f.read()).decode() + with open(os.path.join(startup_dir, fname), "rb") as f: + data[fname] = base64.b64encode(f.read()).decode() return self._create_or_replace_secret(f"nvflare-startup-{site_name_to_rfc1123(site_name)}", {"data": data}) @@ -1115,6 +1122,8 @@ def launch_job(self, job_meta: dict, fl_ctx: FLContext) -> JobHandleSpec: ) startup_dir = workspace_obj.get_startup_kit_dir() + run_dir = workspace_obj.get_run_dir(raw_job_id) + job_cert = read_job_cert(run_dir) if require_job_cert(fl_ctx, run_dir) else None engine = fl_ctx.get_engine() owner_cell = getattr(engine, "cell", None) if engine else None if owner_cell is None: @@ -1131,6 +1140,10 @@ def launch_job(self, job_meta: dict, fl_ctx: FLContext) -> JobHandleSpec: # would be readable in the pod object by anyone with pods/get. credential_env = get_credential_env(job_args) credential_env[ENV_WORKSPACE_TRANSFER_TOKEN] = workspace_transfer_token + if job_cert is not None: + # the pod's bootstrap cell needs the credential before the run dir is downloaded + credential_env[JobProcessEnv.JOB_CERT] = job_cert[0].decode("ascii") + credential_env[JobProcessEnv.JOB_KEY] = job_cert[1].decode("ascii") credential_secret_name = self._ensure_job_credential_secret(pod_name, credential_env) env[ENV_WORKSPACE_OWNER_FQCN] = workspace_transfer.owner_fqcn diff --git a/nvflare/app_opt/job_launcher/slurm/batch.py b/nvflare/app_opt/job_launcher/slurm/batch.py index 01ec0ed207..125f6cbc03 100644 --- a/nvflare/app_opt/job_launcher/slurm/batch.py +++ b/nvflare/app_opt/job_launcher/slurm/batch.py @@ -218,7 +218,7 @@ def _render_node_script(plan: LaunchPlan, config: SlurmConfig) -> str: """ worker_words = _build_worker_words(plan) node_words = [shlex.quote(word) for word in plan.additional_node_command] - credential_names = (JobProcessEnv.AUTH_TOKEN, JobProcessEnv.TOKEN_SIGNATURE, JobProcessEnv.SSID) + credential_names = JobProcessEnv.ALL nonzero_setup = [ f" unset {' '.join(credential_names)}", f" export {CLIENT_API_TYPE_KEY}={CELL_API_TYPE}", diff --git a/nvflare/app_opt/job_launcher/slurm/launcher.py b/nvflare/app_opt/job_launcher/slurm/launcher.py index 05f0b9f954..e6d3ea5860 100644 --- a/nvflare/app_opt/job_launcher/slurm/launcher.py +++ b/nvflare/app_opt/job_launcher/slurm/launcher.py @@ -63,6 +63,7 @@ from nvflare.fuel.f3.drivers.file_driver import parse_file_url from nvflare.fuel.utils.config_service import ConfigService from nvflare.fuel.utils.secret_utils import has_secret_refs +from nvflare.private.fed.utils.job_cert_utils import JobCertError, require_job_cert from nvflare.utils.job_launcher_utils import ( get_client_job_args, get_credential_env, @@ -447,6 +448,10 @@ def _build_launch_plan(self, job_meta: dict, fl_ctx: FLContext) -> LaunchPlan: if os.path.realpath(workspace.get_root_dir()) != self.config.workspace_path: raise SlurmLauncherError("FLContext workspace does not match configured Slurm workspace_path") run_dir = _validate_run_dir(self.config.workspace_path, workspace.get_run_dir(job_id)) + try: + require_job_cert(fl_ctx, run_dir) + except JobCertError as e: + raise SlurmLauncherError(str(e)) from e raw_job_args = fl_ctx.get_prop(FLContextKey.JOB_PROCESS_ARGS) if not isinstance(raw_job_args, dict) or not raw_job_args: diff --git a/nvflare/app_opt/job_launcher/slurm/manager.py b/nvflare/app_opt/job_launcher/slurm/manager.py index fd13734c62..914de339c9 100644 --- a/nvflare/app_opt/job_launcher/slurm/manager.py +++ b/nvflare/app_opt/job_launcher/slurm/manager.py @@ -54,6 +54,7 @@ ) from nvflare.app_opt.job_launcher.slurm.scheduler_client import _command_diagnostic, _SlurmCliAdapter from nvflare.fuel.common.exit_codes import PROCESS_EXIT_REASON, ProcessExitCode +from nvflare.private.fed.utils.job_cert_utils import stage_job_startup_dir _HEALTHY_MISSES = 5 _ACCOUNTING_RETRY_INTERVAL = 6.0 @@ -209,14 +210,15 @@ def _prepare_job_dir(self, job_id: str, needs_sandbox_root: bool) -> tuple[str, def _write_job_files(self, plan: LaunchPlan, job_dir: str) -> None: if plan.sandbox != "none": workspace = self.config.workspace_path + # a keyless copy of the kit keeps the site private keys out of the sandbox; + # the job credential is inside the run dir bind + startup_source = stage_job_startup_dir( + os.path.realpath(os.path.join(workspace, "startup")), os.path.join(job_dir, "startup") + ) launcher_mounts = ( BindMount(os.path.join(job_dir, SANDBOX_ROOT), workspace, "rw"), BindMount(plan.run_dir, plan.run_dir, "rw"), - BindMount( - os.path.realpath(os.path.join(workspace, "startup")), - os.path.join(workspace, "startup"), - "ro", - ), + BindMount(startup_source, os.path.join(workspace, "startup"), "ro"), BindMount( os.path.realpath(os.path.join(workspace, "local")), os.path.join(workspace, "local"), diff --git a/nvflare/app_opt/job_launcher/study_runtime.py b/nvflare/app_opt/job_launcher/study_runtime.py index 5d436ed421..2fcaa4ef14 100644 --- a/nvflare/app_opt/job_launcher/study_runtime.py +++ b/nvflare/app_opt/job_launcher/study_runtime.py @@ -68,9 +68,7 @@ "PYTHONPATH", "NVFL_WORKSPACE_OWNER_FQCN", "NVFL_WORKSPACE_TRANSFER_TOKEN", - JobProcessEnv.AUTH_TOKEN, - JobProcessEnv.TOKEN_SIGNATURE, - JobProcessEnv.SSID, + *JobProcessEnv.ALL, } ) SLURM_RESERVED_ENV_NAMES = _RESERVED_ENV_NAMES.union( diff --git a/nvflare/app_opt/job_launcher/workspace_cell_transfer.py b/nvflare/app_opt/job_launcher/workspace_cell_transfer.py index a9b3796880..8fcd407ec9 100644 --- a/nvflare/app_opt/job_launcher/workspace_cell_transfer.py +++ b/nvflare/app_opt/job_launcher/workspace_cell_transfer.py @@ -43,7 +43,8 @@ import yaml -from nvflare.apis.fl_constant import ConnPropKey +from nvflare.apis.fl_constant import ConnPropKey, WorkspaceConstants +from nvflare.apis.workspace import Workspace from nvflare.fuel.f3.cellnet.cell import Cell from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.fqcn import FQCN @@ -56,6 +57,13 @@ from nvflare.fuel.f3.streaming.obj_downloader import ObjectDownloader from nvflare.fuel.sec.authn import set_add_auth_headers_filters from nvflare.private.defs import AUTH_CLIENT_NAME_FOR_SJ +from nvflare.private.fed.utils.job_cert_utils import ( + JOB_CERT_DIR_NAME, + JOB_CERT_FILE_NAME, + JOB_KEY_FILE_NAME, + find_job_cert, + write_job_cert, +) from nvflare.security.logging import secure_format_exception logger = logging.getLogger(__name__) @@ -195,16 +203,34 @@ def _write_dir_to_zip(zf: zipfile.ZipFile, src: str, root: str, excluded_paths: zf.write(abs_path, rel_path) +def _run_dir_name(job_id: str) -> str: + return WorkspaceConstants.WORKSPACE_PREFIX + job_id + + +def _run_dir(workspace_root: str, job_id: str) -> str: + # pod side, before startup/local exist (so no Workspace instance): validated and resolved + return Workspace.run_dir_path(workspace_root, job_id) + + +def _job_cert_excludes(job_id: str) -> frozenset[str]: + # the job credential is delivered through the credential Secret, never inside a bundle + cert_dir = posixpath.join(_run_dir_name(job_id), JOB_CERT_DIR_NAME) + return frozenset(posixpath.join(cert_dir, fname) for fname in (JOB_CERT_FILE_NAME, JOB_KEY_FILE_NAME)) + + def _zip_workspace_to_file(workspace_root: str, job_id: str, file_path: str) -> None: + # entry names are relative to workspace_root, so the run dir is joined without resolving symlinks excluded_paths = _workspace_download_excludes(workspace_root) + run_dir = os.path.join(workspace_root, _run_dir_name(job_id)) with zipfile.ZipFile(file_path, "w", zipfile.ZIP_DEFLATED) as zf: _write_dir_to_zip(zf, os.path.join(workspace_root, "local"), workspace_root, excluded_paths) - _write_dir_to_zip(zf, os.path.join(workspace_root, job_id), workspace_root) + _write_dir_to_zip(zf, run_dir, workspace_root, _job_cert_excludes(job_id)) def _zip_results_to_file(workspace_root: str, job_id: str, file_path: str) -> None: + run_dir = os.path.join(workspace_root, _run_dir_name(job_id)) with zipfile.ZipFile(file_path, "w", zipfile.ZIP_DEFLATED) as zf: - _write_dir_to_zip(zf, os.path.join(workspace_root, job_id), workspace_root) + _write_dir_to_zip(zf, run_dir, workspace_root, _job_cert_excludes(job_id)) def _validate_relative_zip_members(zf: zipfile.ZipFile) -> None: @@ -494,25 +520,18 @@ def _get_root_url(args) -> str: raise RuntimeError("unable to determine root_url for workspace transfer bootstrap cell") -def _get_bootstrap_tls_pair(startup_dir: str, owner_fqcn: str) -> tuple[str, str, str, str]: - prefer_server = FQCN.get_root(owner_fqcn) == FQCN.ROOT_SERVER - if prefer_server: - candidates = [ - ("server.crt", "server.key", DriverParams.SERVER_CERT.value, DriverParams.SERVER_KEY.value), - ("client.crt", "client.key", DriverParams.CLIENT_CERT.value, DriverParams.CLIENT_KEY.value), - ] - else: - candidates = [ - ("client.crt", "client.key", DriverParams.CLIENT_CERT.value, DriverParams.CLIENT_KEY.value), - ("server.crt", "server.key", DriverParams.SERVER_CERT.value, DriverParams.SERVER_KEY.value), - ] - - for cert_name, key_name, cert_key, key_key in candidates: - cert_path = os.path.join(startup_dir, cert_name) - key_path = os.path.join(startup_dir, key_name) - if os.path.exists(cert_path) and os.path.exists(key_path): - return cert_path, key_path, cert_key, key_key - raise RuntimeError(f"workspace transfer requires cert/key files in startup dir: {startup_dir}") +def _get_bootstrap_tls_pair(run_dir: str, owner_fqcn: str) -> tuple[str, str, str, str]: + """The job credential, in the TLS role the bootstrap cell plays (server-side jobs dial as a server).""" + job_cert = find_job_cert(run_dir) + if not job_cert: + raise RuntimeError( + f"workspace transfer requires the job credential in {run_dir}/{JOB_CERT_DIR_NAME}; " + "secure jobs run only on per-job certificates" + ) + cert_path, key_path = job_cert + if FQCN.get_root(owner_fqcn) == FQCN.ROOT_SERVER: + return cert_path, key_path, DriverParams.SERVER_CERT.value, DriverParams.SERVER_KEY.value + return cert_path, key_path, DriverParams.CLIENT_CERT.value, DriverParams.CLIENT_KEY.value def _load_startup_json(startup_dir: str, filename: str) -> dict | None: @@ -577,7 +596,9 @@ def _create_bootstrap_cell(args, owner_fqcn: str, secure_mode: bool) -> tuple[Ce root_ca = os.path.join(startup_dir, "rootCA.pem") if not os.path.exists(root_ca): raise RuntimeError(f"workspace transfer requires rootCA.pem in startup dir: {startup_dir}") - cert_path, key_path, cert_key, key_key = _get_bootstrap_tls_pair(startup_dir, owner_fqcn) + cert_path, key_path, cert_key, key_key = _get_bootstrap_tls_pair( + _run_dir(args.workspace, args.job_id), owner_fqcn + ) credentials = { DriverParams.CA_CERT.value: root_ca, cert_key: cert_path, @@ -653,6 +674,14 @@ def _request_workspace_bundle(cell: Cell, owner_fqcn: str, job_id: str, transfer return payload +def _install_job_cert(args) -> None: + """Write the job credential the launcher passed through the environment (parsed into args) into the run dir.""" + cert_pem = getattr(args, "job_cert_pem", None) + key_pem = getattr(args, "job_key_pem", None) + if cert_pem and key_pem: + write_job_cert(_run_dir(args.workspace, args.job_id), cert_pem.encode("ascii"), key_pem.encode("ascii")) + + def download_workspace(args, secure_mode: bool) -> None: owner_fqcn = os.environ.get(ENV_WORKSPACE_OWNER_FQCN, "") if not owner_fqcn: @@ -662,6 +691,8 @@ def download_workspace(args, secure_mode: bool) -> None: raise RuntimeError(f"workspace transfer requires env var {ENV_WORKSPACE_TRANSFER_TOKEN}") os.makedirs(args.workspace, exist_ok=True) + # the bootstrap cell authenticates with the job credential, so install it before creating the cell + _install_job_cert(args) temp_dir = tempfile.mkdtemp(prefix="workspace-download-") try: cell = _get_bootstrap_cell(args, owner_fqcn, secure_mode) diff --git a/nvflare/fuel/f3/cellnet/credential_manager.py b/nvflare/fuel/f3/cellnet/credential_manager.py index 0a57eb528e..4b58733a1a 100644 --- a/nvflare/fuel/f3/cellnet/credential_manager.py +++ b/nvflare/fuel/f3/cellnet/credential_manager.py @@ -21,8 +21,9 @@ from nvflare.fuel.f3.cellnet.cell_cipher import SimpleCellCipher from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey -from nvflare.fuel.f3.cellnet.identity import CellIdentityResolver, get_cert_common_name_from_pem +from nvflare.fuel.f3.cellnet.identity import CellIdentityResolver, get_cert_common_name from nvflare.fuel.f3.drivers.driver_params import DriverParams +from nvflare.fuel.f3.drivers.net_utils import get_cert_job_id from nvflare.fuel.f3.endpoint import Endpoint from nvflare.fuel.f3.message import Message @@ -113,9 +114,14 @@ def _cache_cert(self, fqcn: str, cert: bytes): raise RuntimeError(f"missing certificate for {fqcn}") if self.enforce_identity: - cn = get_cert_common_name_from_pem(cert) + peer_cert = x509.load_pem_x509_certificate(cert) try: - self.identity_resolver.require_match(fqcn, cn, f"certificate for {fqcn}") + self.identity_resolver.require_match( + fqcn, + get_cert_common_name(peer_cert), + f"certificate for {fqcn}", + peer_job_id=get_cert_job_id(peer_cert), + ) except ValueError as ex: raise RuntimeError(str(ex)) diff --git a/nvflare/fuel/f3/cellnet/identity.py b/nvflare/fuel/f3/cellnet/identity.py index d924f469cd..cd7a2d6ceb 100644 --- a/nvflare/fuel/f3/cellnet/identity.py +++ b/nvflare/fuel/f3/cellnet/identity.py @@ -156,7 +156,7 @@ def resolve(self, fqcn: str) -> Optional[str]: return parts[0] if parts else fqcn - def require_match(self, fqcn: str, peer_cn: str, peer_desc: str): + def require_match(self, fqcn: str, peer_cn: str, peer_desc: str, peer_job_id: Optional[str] = None): expected_cn = self.resolve(fqcn) if not expected_cn: raise ValueError(f"{peer_desc} claimed endpoint '{fqcn}' does not resolve to an expected identity") @@ -164,6 +164,13 @@ def require_match(self, fqcn: str, peer_cn: str, peer_desc: str): if not peer_cn or peer_cn == "N/A": raise ValueError(f"{peer_desc} does not have an authenticated mTLS peer common name") + # A per-job certificate may only authenticate cells of that job. + if peer_job_id is not None and peer_job_id not in FQCN.split(FQCN.normalize(fqcn)): + raise ValueError( + f"{peer_desc} authenticated with a certificate bound to job '{peer_job_id}' " + f"but claimed endpoint '{fqcn}' is not part of that job" + ) + # Admin client cell names are per-session random IDs; the authenticated user is the cert CN. if is_valid_admin_client_name(fqcn): return diff --git a/nvflare/fuel/f3/drivers/aio_conn.py b/nvflare/fuel/f3/drivers/aio_conn.py index 9d9e471ccb..4e5767cfbf 100644 --- a/nvflare/fuel/f3/drivers/aio_conn.py +++ b/nvflare/fuel/f3/drivers/aio_conn.py @@ -19,7 +19,7 @@ from nvflare.fuel.f3.drivers.aio_context import AioContext from nvflare.fuel.f3.drivers.connector_info import ConnectorInfo from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE +from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE, add_peer_job_id from nvflare.fuel.f3.sfm.prefix import PREFIX_LEN, Prefix from nvflare.fuel.hci.security import get_certificate_common_name from nvflare.security.logging import secure_format_exception @@ -134,6 +134,7 @@ def _get_aio_properties(self) -> dict: peer_cert = self.writer.get_extra_info("peercert") if peer_cert: cn = get_certificate_common_name(peer_cert) + add_peer_job_id(conn_props, self.writer.get_extra_info("ssl_object")) else: if self.secure: cn = "N/A" diff --git a/nvflare/fuel/f3/drivers/aio_grpc_driver.py b/nvflare/fuel/f3/drivers/aio_grpc_driver.py index ceda910c3a..16498f80e3 100644 --- a/nvflare/fuel/f3/drivers/aio_grpc_driver.py +++ b/nvflare/fuel/f3/drivers/aio_grpc_driver.py @@ -196,9 +196,11 @@ async def Stream(self, request_iterator, context): DriverParams.LOCAL_ADDR.value: get_address(self.server.connector.params), } - cn_names = context.auth_context().get("x509_common_name") + auth_context = context.auth_context() + cn_names = auth_context.get("x509_common_name") if cn_names: conn_props[DriverParams.PEER_CN.value] = cn_names[0].decode("utf-8") + add_grpc_peer_job_id(conn_props, auth_context) connection = AioStreamSession( aio_ctx=self.aio_ctx, diff --git a/nvflare/fuel/f3/drivers/aio_http_driver.py b/nvflare/fuel/f3/drivers/aio_http_driver.py index 0494a5af13..db6c13c1f5 100644 --- a/nvflare/fuel/f3/drivers/aio_http_driver.py +++ b/nvflare/fuel/f3/drivers/aio_http_driver.py @@ -26,7 +26,7 @@ from nvflare.fuel.f3.drivers.base_driver import BaseDriver from nvflare.fuel.f3.drivers.driver import ConnectorInfo from nvflare.fuel.f3.drivers.driver_params import DriverCap, DriverParams -from nvflare.fuel.f3.drivers.net_utils import get_tcp_urls +from nvflare.fuel.f3.drivers.net_utils import add_peer_job_id, get_tcp_urls from nvflare.fuel.hci.security import get_certificate_common_name from nvflare.security.logging import secure_format_exception @@ -72,6 +72,7 @@ def _get_ws_properties(self) -> dict: peer_cert = self.websocket.get_extra_info("peercert") if peer_cert: cn = get_certificate_common_name(peer_cert) + add_peer_job_id(conn_props, self.websocket.get_extra_info("ssl_object")) else: cn = "N/A" if self.ssl_context else None diff --git a/nvflare/fuel/f3/drivers/driver_params.py b/nvflare/fuel/f3/drivers/driver_params.py index 2cb82eeaff..2f561bb92a 100644 --- a/nvflare/fuel/f3/drivers/driver_params.py +++ b/nvflare/fuel/f3/drivers/driver_params.py @@ -42,6 +42,7 @@ class DriverParams(str, Enum): LOCAL_ADDR = "local_addr" PEER_ADDR = "peer_addr" PEER_CN = "peer_cn" + PEER_JOB_ID = "peer_job_id" IMPLEMENTED_CONN_SEC = "implemented_conn_sec" diff --git a/nvflare/fuel/f3/drivers/grpc_driver.py b/nvflare/fuel/f3/drivers/grpc_driver.py index 66962b3bb8..ca70f0925f 100644 --- a/nvflare/fuel/f3/drivers/grpc_driver.py +++ b/nvflare/fuel/f3/drivers/grpc_driver.py @@ -136,9 +136,11 @@ def Stream(self, request_iterator, context): DriverParams.PEER_ADDR.value: context.peer(), DriverParams.LOCAL_ADDR.value: get_address(self.server.connector.params), } - cn_names = context.auth_context().get("x509_common_name") + auth_context = context.auth_context() + cn_names = auth_context.get("x509_common_name") if cn_names: conn_props[DriverParams.PEER_CN.value] = cn_names[0].decode("utf-8") + add_grpc_peer_job_id(conn_props, auth_context) try: self.logger.debug(f"SERVER started Stream CB in thread {ct.name}") diff --git a/nvflare/fuel/f3/drivers/net_utils.py b/nvflare/fuel/f3/drivers/net_utils.py index c4d328938f..b7e78a3757 100644 --- a/nvflare/fuel/f3/drivers/net_utils.py +++ b/nvflare/fuel/f3/drivers/net_utils.py @@ -21,14 +21,19 @@ from typing import Any, Optional from urllib.parse import parse_qsl, urlencode, urlparse +from cryptography import x509 + from nvflare.apis.fl_constant import ConnectionSecurity from nvflare.fuel.f3.comm_error import CommError from nvflare.fuel.f3.drivers.driver_params import DriverParams from nvflare.fuel.utils.argument_utils import str2bool +from nvflare.lighter.constants import CertExtensionOID from nvflare.security.logging import secure_format_exception log = logging.getLogger(__name__) +JOB_ID_EXTENSION_OID = x509.ObjectIdentifier(CertExtensionOID.JOB_ID) + LO_PORT = 1025 HI_PORT = 65535 MAX_ITER_SIZE = 10 @@ -117,6 +122,54 @@ def get_ssl_context(params: dict, ssl_server: bool) -> Optional[SSLContext]: return ctx +def get_cert_job_id(cert: x509.Certificate) -> Optional[str]: + """Job ID a per-job certificate is bound to; None for a site certificate. + + An undecodable extension value comes back with replacement characters so it + can never equal a real job ID. + """ + try: + ext = cert.extensions.get_extension_for_oid(JOB_ID_EXTENSION_OID) + except x509.ExtensionNotFound: + return None + return ext.value.value.decode("utf-8", errors="replace") + + +def get_cert_job_id_from_pem(cert_bytes: bytes) -> Optional[str]: + if not cert_bytes: + return None + return get_cert_job_id(x509.load_pem_x509_certificate(cert_bytes)) + + +def get_peer_job_id(ssl_object) -> Optional[str]: + """Job ID from the peer certificate of an established TLS connection (SSLSocket or SSLObject).""" + if not ssl_object: + return None + der = ssl_object.getpeercert(binary_form=True) + if not der: + return None + return get_cert_job_id(x509.load_der_x509_certificate(der)) + + +def get_grpc_peer_job_id(auth_context: dict) -> Optional[str]: + pem_certs = auth_context.get("x509_pem_cert") if auth_context else None + return get_cert_job_id_from_pem(pem_certs[0]) if pem_certs else None + + +def add_peer_job_id(conn_props: dict, ssl_object) -> None: + """Record the peer certificate's job binding next to PEER_CN (TLS socket connections).""" + job_id = get_peer_job_id(ssl_object) + if job_id: + conn_props[DriverParams.PEER_JOB_ID.value] = job_id + + +def add_grpc_peer_job_id(conn_props: dict, auth_context: dict) -> None: + """Record the peer certificate's job binding next to PEER_CN (gRPC server-side connections).""" + job_id = get_grpc_peer_job_id(auth_context) + if job_id: + conn_props[DriverParams.PEER_JOB_ID.value] = job_id + + def get_address(params: dict) -> str: host = params.get(DriverParams.HOST.value, "0.0.0.0") port = params.get(DriverParams.PORT.value, 0) diff --git a/nvflare/fuel/f3/drivers/socket_conn.py b/nvflare/fuel/f3/drivers/socket_conn.py index 97ea549f16..db63ec538b 100644 --- a/nvflare/fuel/f3/drivers/socket_conn.py +++ b/nvflare/fuel/f3/drivers/socket_conn.py @@ -24,7 +24,7 @@ from nvflare.fuel.f3.connection import BytesAlike, Connection from nvflare.fuel.f3.drivers.driver import ConnectorInfo from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE +from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE, add_peer_job_id from nvflare.fuel.f3.sfm.prefix import PREFIX_LEN, Prefix from nvflare.fuel.hci.security import get_certificate_common_name from nvflare.security.logging import secure_format_exception @@ -223,6 +223,7 @@ def _get_socket_properties(self) -> dict: cert = self.sock.getpeercert() if cert: cn = get_certificate_common_name(cert) + add_peer_job_id(conn_props, self.sock) else: cn = "N/A" conn_props[DriverParams.PEER_CN.value] = cn diff --git a/nvflare/fuel/f3/sfm/conn_manager.py b/nvflare/fuel/f3/sfm/conn_manager.py index 9fe1004f31..67ec4e48d5 100644 --- a/nvflare/fuel/f3/sfm/conn_manager.py +++ b/nvflare/fuel/f3/sfm/conn_manager.py @@ -435,7 +435,12 @@ def update_endpoint(self, sfm_conn: SfmConnection, data: dict): f"Admin endpoint '{endpoint_name}' can only connect through an admin listener", ) try: - self.identity_resolver.require_match(endpoint_name, peer_cn, f"connection {sfm_conn.get_name()}") + self.identity_resolver.require_match( + endpoint_name, + peer_cn, + f"connection {sfm_conn.get_name()}", + peer_job_id=get_param(conn_props, DriverParams.PEER_JOB_ID), + ) except ValueError as ex: sfm_conn.conn.close() raise CommError(CommError.BAD_DATA, str(ex)) diff --git a/nvflare/lighter/constants.py b/nvflare/lighter/constants.py index d264826912..9dceb10865 100644 --- a/nvflare/lighter/constants.py +++ b/nvflare/lighter/constants.py @@ -191,6 +191,8 @@ class ProvFileName: AWS_START_SH = "aws_start.sh" EDGE_RESOURCES_JSON = "edge__p_resources.json" START_ALL_SH = "start_all.sh" + JOB_CA_CERT = "job_ca.crt" + JOB_CA_KEY = "job_ca.key" class CertFileBasename: @@ -198,6 +200,16 @@ class CertFileBasename: SERVER = "server" +class CertExtensionOID: + """Private extensions under NVIDIA's IANA enterprise arc (1.3.6.1.4.1.5703).""" + + # on job leaf certs: the job ID the certificate is bound to + JOB_ID = "1.3.6.1.4.1.5703.300.1" + # on the job CA cert: marks certs it issues as job-scoped, so site-scope + # verification can reject them by issuer even without the job-ID extension + JOB_CA_MARKER = "1.3.6.1.4.1.5703.300.2" + + class CommConfigArg: SCHEME = "scheme" HOST = "host" diff --git a/nvflare/lighter/impl/cert.py b/nvflare/lighter/impl/cert.py index 1f26cc5cd1..45b1ec6e75 100644 --- a/nvflare/lighter/impl/cert.py +++ b/nvflare/lighter/impl/cert.py @@ -21,11 +21,20 @@ from cryptography.hazmat.primitives import serialization from cryptography.x509.oid import NameOID -from nvflare.lighter.constants import CertFileBasename, CtxKey, ParticipantType, PropKey +from nvflare.lighter.constants import CertExtensionOID, CertFileBasename, CtxKey, ParticipantType, PropKey, ProvFileName from nvflare.lighter.ctx import ProvisionContext from nvflare.lighter.entity import Participant, Project from nvflare.lighter.spec import Builder -from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert, serialize_pri_key +from nvflare.lighter.utils import ( + Identity, + bounded_validity, + generate_cert, + generate_keys, + load_crt_bytes, + serialize_cert, + serialize_pri_key, + write_pri_key_file, +) MAX_CN_LENGTH = 64 DEFAULT_CERT_VALID_DAYS = 360 @@ -97,7 +106,7 @@ def persist(self): class CertBuilder(Builder): - def __init__(self, root_valid_days=DEFAULT_CERT_VALID_DAYS): + def __init__(self, root_valid_days=DEFAULT_CERT_VALID_DAYS, enable_job_ca=True): """Build certificate chain for every participant. Handles building (creating and self-signing) the root CA certificates, creating server, client and @@ -107,14 +116,20 @@ def __init__(self, root_valid_days=DEFAULT_CERT_VALID_DAYS): Args: root_valid_days: validity period in days for a newly generated root CA certificate. This value does not renew or replace a root CA already stored in the provisioning state. + enable_job_ca: also generate a job-signing intermediate CA (job_ca.crt/job_ca.key) in the server + startup kit. The server uses it at job deploy time to issue short-lived per-job certificates; + without it, secure-mode jobs refuse to deploy. Set False only for non-secure deployments. """ if isinstance(root_valid_days, bool) or not isinstance(root_valid_days, int) or root_valid_days <= 0: raise ValueError( f"root_valid_days must be a positive integer, got {root_valid_days!r} " f"({type(root_valid_days).__name__})" ) + if not isinstance(enable_job_ca, bool): + raise ValueError(f"enable_job_ca must be a bool, got {enable_job_ca!r} ({type(enable_job_ca).__name__})") self.root_valid_days = root_valid_days + self.enable_job_ca = enable_job_ca self.root_cert = None self.persistent_state = None self.serialized_cert = None @@ -257,10 +272,7 @@ def _build_write_cert_pair(self, participant: Participant, base_name, ctx: Provi dest_dir = ctx.get_kit_dir(participant) with open(os.path.join(dest_dir, f"{base_name}.crt"), "wb") as f: f.write(serialize_cert(cert)) - key_path = os.path.join(dest_dir, f"{base_name}.key") - fd = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - with os.fdopen(fd, "wb") as f: - f.write(serialize_pri_key(pri_key)) + write_pri_key_file(os.path.join(dest_dir, f"{base_name}.key"), serialize_pri_key(pri_key)) if participant.type in [ParticipantType.CLIENT, ParticipantType.RELAY]: self._build_internal_listener_cert(participant, ctx) @@ -308,10 +320,7 @@ def _build_internal_listener_cert(self, participant: Participant, ctx: Provision bn = CertFileBasename.SERVER with open(os.path.join(dest_dir, f"{bn}.crt"), "wb") as f: f.write(serialize_cert(tmp_cert)) - key_path_bn = os.path.join(dest_dir, f"{bn}.key") - fd = os.open(key_path_bn, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - with os.fdopen(fd, "wb") as f: - f.write(serialize_pri_key(tmp_pri_key)) + write_pri_key_file(os.path.join(dest_dir, f"{bn}.key"), serialize_pri_key(tmp_pri_key)) def build(self, project: Project, ctx: ProvisionContext): self._build_root(project.name, subject_org=None) @@ -325,6 +334,8 @@ def build(self, project: Project, ctx: ProvisionContext): server = project.get_server() if server: self._build_write_cert_pair(server, CertFileBasename.SERVER, ctx) + if self.enable_job_ca: + self._build_write_job_ca(project, server, ctx) for client in project.get_clients(): self._build_write_cert_pair(client, CertFileBasename.CLIENT, ctx) @@ -335,6 +346,61 @@ def build(self, project: Project, ctx: ProvisionContext): for admin in project.get_admins(): self._build_write_cert_pair(admin, CertFileBasename.CLIENT, ctx) + def _build_write_job_ca(self, project: Project, server: Participant, ctx: ProvisionContext): + """Generate the job-signing intermediate CA and write it to the server startup kit. + + The job CA is signed by the root, constrained to issue only leaf certs (pathlen:0), and its + private key goes only to the server kit. The server uses it at job deploy time to issue + short-lived per-job certificates that chain to the project root. + """ + assert isinstance(self.persistent_state, _CertState) + subject = f"job_ca.{project.name}"[:MAX_CN_LENGTH] + cert_pem = None + key_pem = None + if self.persistent_state.has_subject(subject): + stored_cert_pem = self.persistent_state.get_subject_cert(subject).encode("ascii") + stored_cert = load_crt_bytes(stored_cert_pem) + if stored_cert.not_valid_after_utc > datetime.datetime.now(datetime.timezone.utc): + cert_pem = stored_cert_pem + key_pem = self.persistent_state.get_subject_pri_key(subject).encode("ascii") + else: + ctx.info(f"stored job CA expired at {stored_cert.not_valid_after_utc.isoformat()}; regenerating") + + if cert_pem is None: + pri_key, pub_key = generate_keys() + now, not_valid_after = self._bounded_not_valid_after("job CA") + # the marker lets site-scope verification reject anything this CA issues by + # issuer, even a cert minted without the job-ID extension by a stolen CA key + marker = x509.UnrecognizedExtension(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER), b"job_ca") + cert = self._generate_cert( + subject, + None, + self.issuer, + self.pri_key, + pub_key, + ca=True, + ca_path_length=0, + not_valid_before=now, + not_valid_after=not_valid_after, + extra_extensions=[(marker, False)], + ) + cert_pem = serialize_cert(cert) + key_pem = serialize_pri_key(pri_key) + self.persistent_state.add_subject_cert(subject, cert_pem.decode("ascii")) + self.persistent_state.add_subject_pri_key(subject, key_pem.decode("ascii")) + + dest_dir = ctx.get_kit_dir(server) + with open(os.path.join(dest_dir, ProvFileName.JOB_CA_CERT), "wb") as f: + f.write(cert_pem) + write_pri_key_file(os.path.join(dest_dir, ProvFileName.JOB_CA_KEY), key_pem) + + def _bounded_not_valid_after(self, subject_desc: str): + """Validity window for a cert signed by the root: now until DEFAULT_CERT_VALID_DAYS, clamped to the root.""" + try: + return bounded_validity(self.root_cert, DEFAULT_CERT_VALID_DAYS) + except ValueError as e: + raise RuntimeError(f"cannot generate certificate for '{subject_desc}': root CA {e}") from e + def get_pri_key_cert(self, participant: Participant): pri_key, pub_key = generate_keys() subject = participant.subject @@ -344,15 +410,7 @@ def get_pri_key_cert(self, participant: Participant): else: role = None - now = datetime.datetime.now(datetime.timezone.utc) - root_not_after = self.root_cert.not_valid_after_utc - if root_not_after <= now: - raise RuntimeError( - f"cannot generate certificate for '{participant.name}': root CA expired at {root_not_after.isoformat()}" - ) - not_valid_after = ( - root_not_after if root_not_after < now + datetime.timedelta(days=DEFAULT_CERT_VALID_DAYS) else None - ) + now, not_valid_after = self._bounded_not_valid_after(participant.name) server = participant if participant.type == ParticipantType.SERVER else None cert = self._generate_cert( @@ -377,6 +435,7 @@ def _generate_cert( subject_pub_key, valid_days=DEFAULT_CERT_VALID_DAYS, ca=False, + ca_path_length=None, role=None, server: Participant = None, server_default_host=None, @@ -398,6 +457,7 @@ def _generate_cert( subject_pub_key=subject_pub_key, valid_days=valid_days, ca=ca, + ca_path_length=ca_path_length, server_default_host=server_default_host, server_additional_hosts=server_additional_hosts, extra_extensions=extra_extensions, diff --git a/nvflare/lighter/utils.py b/nvflare/lighter/utils.py index 986d8a5518..18ae2c2328 100644 --- a/nvflare/lighter/utils.py +++ b/nvflare/lighter/utils.py @@ -83,6 +83,7 @@ def generate_cert( not_valid_before=None, not_valid_after=None, extra_extensions=None, + ca_path_length=None, ): now = not_valid_before or datetime.datetime.now(datetime.timezone.utc) cert_not_valid_after = not_valid_after or now + datetime.timedelta(days=valid_days) @@ -109,7 +110,9 @@ def generate_cert( ) if ca: - builder = builder.add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True).add_extension( + builder = builder.add_extension( + x509.BasicConstraints(ca=True, path_length=ca_path_length), critical=True + ).add_extension( x509.KeyUsage( digital_signature=True, content_commitment=False, @@ -164,6 +167,24 @@ def serialize_cert(cert): return cert.public_bytes(serialization.Encoding.PEM) +def write_pri_key_file(path: str, pri_key_pem: bytes): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + # os.open's mode only applies on creation; re-tighten in case the file pre-existed + os.chmod(path, 0o600) + with os.fdopen(fd, "wb") as f: + f.write(pri_key_pem) + + +def bounded_validity(issuer_cert, valid_days: int, backdate=datetime.timedelta(0)): + """(not_valid_before, not_valid_after) for a cert signed by issuer_cert: now minus backdate until + valid_days from now, clamped to the issuer's own expiry. Raises ValueError if the issuer has expired.""" + now = datetime.datetime.now(datetime.timezone.utc) + issuer_not_after = issuer_cert.not_valid_after_utc + if issuer_not_after <= now: + raise ValueError(f"expired at {issuer_not_after.isoformat()}") + return now - backdate, min(issuer_not_after, now + datetime.timedelta(days=valid_days)) + + def generate_keys(): pri_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) pub_key = pri_key.public_key() diff --git a/nvflare/private/defs.py b/nvflare/private/defs.py index 0c8282af25..174992fd00 100644 --- a/nvflare/private/defs.py +++ b/nvflare/private/defs.py @@ -79,6 +79,7 @@ class RequestHeader(object): JOB_STATUS = "job_status" TOPIC = "topic" JOB_META = "job_meta" + JOB_CERT = "job_cert" APP_NAME = "app_name" CONTROL_COMMAND = "control_command" CALL_NAME = "call_name" diff --git a/nvflare/private/fed/app/client/worker_process.py b/nvflare/private/fed/app/client/worker_process.py index 45bfb940e8..915f807db1 100644 --- a/nvflare/private/fed/app/client/worker_process.py +++ b/nvflare/private/fed/app/client/worker_process.py @@ -191,6 +191,8 @@ def parse_arguments(): parser.add_argument("--set", metavar="KEY=VALUE", nargs="*") parser.add_argument("--local_rank", type=int, default=0) args = parser.parse_args() + args.job_cert_pem = creds[JobProcessEnv.JOB_CERT] + args.job_key_pem = creds[JobProcessEnv.JOB_KEY] return args diff --git a/nvflare/private/fed/app/fl_conf.py b/nvflare/private/fed/app/fl_conf.py index 845d7086a9..f48c199bec 100644 --- a/nvflare/private/fed/app/fl_conf.py +++ b/nvflare/private/fed/app/fl_conf.py @@ -30,6 +30,7 @@ from nvflare.fuel.utils.url_utils import make_url from nvflare.fuel.utils.wfconf import ConfigContext, ConfigError from nvflare.private.defs import ClientRegMsgKey, SSLConstants +from nvflare.private.fed.utils.job_cert_utils import apply_job_cert_config from nvflare.private.fed.utils.site_config import project_site_config from nvflare.private.json_configer import JsonConfigurator from nvflare.private.privacy_manager import PrivacyManager, Scope @@ -115,6 +116,11 @@ def start_config(self, config_ctx: ConfigContext): except Exception: raise ValueError(f"Server config error: '{self.server_config_file_names}'") + if self.args.job_id: + run_dir = self.workspace.get_run_dir(self.args.job_id) + for server in self.config_data["servers"]: + apply_job_cert_config(server, run_dir) + def build_component(self, config_dict): t = super().build_component(config_dict) if isinstance(t, FLComponent): @@ -404,6 +410,9 @@ def start_config(self, config_ctx: ConfigContext): except Exception: raise ValueError(f"Client config error: '{self.client_config_file_names}'") + if self.args.job_id: + apply_job_cert_config(self.config_data["client"], self.workspace.get_run_dir(self.args.job_id)) + def finalize_config(self, config_ctx: ConfigContext): """Finalize the config process. diff --git a/nvflare/private/fed/app/server/runner_process.py b/nvflare/private/fed/app/server/runner_process.py index 6083b0b748..36c72b08ee 100644 --- a/nvflare/private/fed/app/server/runner_process.py +++ b/nvflare/private/fed/app/server/runner_process.py @@ -189,6 +189,8 @@ def parse_arguments(): ) parser.add_argument("--set", metavar="KEY=VALUE", nargs="*") args = parser.parse_args() + args.job_cert_pem = creds[JobProcessEnv.JOB_CERT] + args.job_key_pem = creds[JobProcessEnv.JOB_KEY] return args diff --git a/nvflare/private/fed/client/fed_client_base.py b/nvflare/private/fed/client/fed_client_base.py index c41531a5f8..b0af33e18c 100644 --- a/nvflare/private/fed/client/fed_client_base.py +++ b/nvflare/private/fed/client/fed_client_base.py @@ -210,6 +210,12 @@ def _create_cell(self, location, scheme): DriverParams.CLIENT_CERT.value: ssl_cert, DriverParams.CLIENT_KEY.value: private_key, } + if self.args.job_id: + # the CJ's ssl_cert is its job credential; pin the server-role credential to it + # too: otherwise, on listener-enabled sites, the site's server cert gets + # back-filled from the startup kit and message crypto prefers it over CLIENT_CERT + credentials[DriverParams.SERVER_CERT.value] = ssl_cert + credentials[DriverParams.SERVER_KEY.value] = private_key else: credentials = {} diff --git a/nvflare/private/fed/client/training_cmds.py b/nvflare/private/fed/client/training_cmds.py index b43e718a1d..413bbdc25f 100644 --- a/nvflare/private/fed/client/training_cmds.py +++ b/nvflare/private/fed/client/training_cmds.py @@ -18,7 +18,7 @@ import tempfile from typing import List -from nvflare.apis.fl_constant import WorkspaceConstants +from nvflare.apis.fl_constant import FLContextKey, WorkspaceConstants from nvflare.apis.workspace import Workspace from nvflare.fuel.hci.proto import MetaStatusValue, make_meta from nvflare.fuel.utils.zip_utils import unzip_all_from_bytes @@ -29,6 +29,7 @@ from nvflare.private.fed.client.admin import RequestProcessor from nvflare.private.fed.client.client_engine_internal_spec import ClientEngineInternalSpec from nvflare.private.fed.utils.fed_utils import get_scope_info, require_signed_jobs +from nvflare.private.fed.utils.job_cert_utils import unpack_job_cert_header, write_job_cert from nvflare.security.logging import secure_format_exception logger = logging.getLogger(__name__) @@ -109,6 +110,15 @@ def process(self, req: Message, app_ctx) -> Message: if not job_meta: return error_reply("missing job meta") + job_cert = req.get_header(RequestHeader.JOB_CERT) + job_creds = unpack_job_cert_header(job_cert) if job_cert else None + if job_creds is None and engine.new_context().get_prop(FLContextKey.SECURE_MODE, False): + # secure jobs run only on per-job credentials; never deploy a job that would use site certs + return error_reply( + f"job {job_id} deploy request carries no valid job credential; " + "the server must be provisioned with a job CA" + ) + workspace = Workspace(root_dir=engine.args.workspace, site_name=client_name) root_ca_path = os.path.join(workspace.get_startup_kit_dir(), "rootCA.pem") # Verify the received bytes before deploying them. AppDeployer will @@ -137,6 +147,9 @@ def process(self, req: Message, app_ctx) -> Message: if err: return error_reply(err) + if job_creds: + write_job_cert(workspace.get_run_dir(job_id), *job_creds) + return ok_reply(body=f"deployed {app_name} to {client_name}") diff --git a/nvflare/private/fed/server/job_runner.py b/nvflare/private/fed/server/job_runner.py index e07c873c52..33cc34c23b 100644 --- a/nvflare/private/fed/server/job_runner.py +++ b/nvflare/private/fed/server/job_runner.py @@ -27,6 +27,7 @@ ConfigVarName, FLContextKey, RunProcessKey, + SecureTrainConst, SiteType, SystemComponents, SystemConfigs, @@ -37,6 +38,7 @@ from nvflare.apis.job_scheduler_spec import DispatchInfo from nvflare.apis.workspace import Workspace from nvflare.fuel.common.exit_codes import ProcessExitCode +from nvflare.fuel.f3.cellnet.identity import get_cert_common_name_from_file from nvflare.fuel.utils.config_service import ConfigService from nvflare.lighter.tool_consts import NVFLARE_SIG_FILE from nvflare.lighter.utils import verify_folder_signature @@ -46,6 +48,13 @@ from nvflare.private.fed.server.server_state import HotState from nvflare.private.fed.utils.app_deployer import AppDeployer from nvflare.private.fed.utils.fed_utils import extract_participants, require_signed_jobs, set_message_security_data +from nvflare.private.fed.utils.job_cert_utils import ( + JOB_CERT_VALID_DAYS, + JobCertError, + load_job_cert_issuer, + pack_job_cert_header, + write_job_cert, +) from nvflare.security.logging import secure_format_exception WORKSPACE_SAVE_RETRY_GRACE_TIME = 60 @@ -109,6 +118,11 @@ def __init__(self, workspace_root: str) -> None: self.client_outcome_wait_timeout = ConfigService.get_float_var( name=ConfigVarName.CLIENT_OUTCOME_WAIT_TIMEOUT, conf=SystemConfigs.APPLICATION_CONF, default=900.0 ) + self.job_cert_valid_days = ConfigService.get_int_var( + name=ConfigVarName.JOB_CERT_VALID_DAYS, conf=SystemConfigs.APPLICATION_CONF, default=JOB_CERT_VALID_DAYS + ) + if self.job_cert_valid_days <= 0: + raise ValueError(f"{ConfigVarName.JOB_CERT_VALID_DAYS} must be positive, got {self.job_cert_valid_days}") self.lock = threading.Lock() def is_client_outcome_pending(self, job_id: str, client_name: str) -> bool: @@ -170,6 +184,15 @@ def _deploy_job(self, job: Job, sites: dict, fl_ctx: FLContext) -> Tuple[str, li deploy_detail = [] fl_ctx.set_prop(FLContextKey.JOB_DEPLOY_DETAIL, deploy_detail) + job_cert_issuer = None + if fl_ctx.get_prop(FLContextKey.SECURE_MODE, False): + # secure jobs run only on per-job credentials: a missing or expiring job CA fails the deploy + try: + job_cert_issuer = load_job_cert_issuer(workspace.get_startup_kit_dir()) + except JobCertError as e: + deploy_detail.append(f"server: {e}") + raise RuntimeError(f"cannot issue job credentials: {e}") from e + for app_name, participants in job.get_deployment().items(): app_data = job.get_application(app_name, fl_ctx) participants = extract_participants(participants) @@ -218,17 +241,27 @@ def _deploy_job(self, job: Job, sites: dict, fl_ctx: FLContext) -> Tuple[str, li if client_sites: self.fire_event(EventType.DEPLOY_JOB_TO_CLIENT, fl_ctx) - message = self._make_deploy_message(job, app_data, app_name, fl_ctx) clients, invalid_inputs = engine.validate_targets(client_sites) if invalid_inputs: deploy_detail.append("invalid_clients: {}".format(",".join(invalid_inputs))) raise RuntimeError(f"unknown clients: {invalid_inputs}.") + # each site receives only its own job credential, so the deploy message is per site + # (the app bytes stay shared). c.name equals the CN of the client's registered + # cert (registration enforces CN == client name) + job_creds = {} + if job_cert_issuer: + site_names = [c.name for c in clients] + job_creds = job_cert_issuer.issue_many(site_names, job.job_id, self.job_cert_valid_days) + for c in clients: assert isinstance(c, Client) client_token_to_name[c.token] = c.name - client_deploy_requests[c.token] = message + client_request = self._make_deploy_message(job, app_data, app_name, fl_ctx) + if c.name in job_creds: + client_request.set_header(RequestHeader.JOB_CERT, pack_job_cert_header(*job_creds[c.name])) + client_deploy_requests[c.token] = client_request client_token_to_reply[c.token] = None display_sites = ",".join(client_sites) @@ -238,6 +271,16 @@ def _deploy_job(self, job: Job, sites: dict, fl_ctx: FLContext) -> Tuple[str, li fire_event=False, ) + if job_cert_issuer: + # write the SJ credential only after all apps are deployed: AppDeployer wipes the + # run dir. The SJ cert must present the CN of the site's server cert — that is the + # identity peers expect for server FQCNs (fl_ctx.get_identity_name() is the literal + # "server") + server_cert_path = fl_ctx.get_prop(FLContextKey.SERVER_CONFIG)[0][SecureTrainConst.SSL_CERT] + server_cn = get_cert_common_name_from_file(server_cert_path) + cert_pem, key_pem = job_cert_issuer.issue(server_cn, job.job_id, self.job_cert_valid_days) + write_job_cert(workspace.get_run_dir(job.job_id), cert_pem, key_pem) + abort_job = False failed_clients = [] if client_deploy_requests: diff --git a/nvflare/private/fed/utils/fed_utils.py b/nvflare/private/fed/utils/fed_utils.py index 15a09cfda6..2a51cb583c 100644 --- a/nvflare/private/fed/utils/fed_utils.py +++ b/nvflare/private/fed/utils/fed_utils.py @@ -44,6 +44,7 @@ from nvflare.private.defs import RequestHeader, SSLConstants from nvflare.private.event import fire_event from nvflare.private.fed.utils.decomposers import private_decomposers +from nvflare.private.fed.utils.job_cert_utils import NO_JOB_CREDENTIAL, JobCertError, find_job_cert from nvflare.private.privacy_manager import PrivacyManager, PrivacyService from nvflare.security.logging import secure_format_exception from nvflare.security.security import EmptyAuthorizer, FLAuthorizer @@ -140,11 +141,13 @@ def require_signed_jobs(workspace: Workspace, startup_config: str = WorkspaceCon return value -def _check_secure_content(site_type: str) -> List[str]: +def _check_secure_content(site_type: str, check_private_key: bool = True) -> List[str]: """To check the security contents. Args: site_type (str): "server" or "client" + check_private_key: whether the site private key must be present and signed; job processes + run on their job credential and launchers may withhold the site key from them. Returns: A list of insecure content. @@ -161,8 +164,11 @@ def _check_secure_content(site_type: str) -> List[str]: sites_to_check = data["servers"] if site_type == SiteType.SERVER else [data["client"]] + filenames = [SSLConstants.CERT, SSLConstants.ROOT_CERT] + if check_private_key: + filenames.append(SSLConstants.PRIVATE_KEY) for site in sites_to_check: - for filename in [SSLConstants.CERT, SSLConstants.PRIVATE_KEY, SSLConstants.ROOT_CERT]: + for filename in filenames: content, sig = SecurityContentService.load_content(site.get(filename)) if sig != LoadResult.OK: insecure_list.append(site.get(filename)) @@ -258,6 +264,10 @@ def security_init_for_job(secure_train: bool, workspace: Workspace, site_type: s workspace: the workspace object. site_type (str): server or client. fed_client.json or fed_server.json """ + # secure jobs run only on their per-job credential; never start one on site certificates + if secure_train and not find_job_cert(workspace.get_run_dir(job_id)): + raise JobCertError(NO_JOB_CREDENTIAL) + # initialize the SecurityContentService. # must do this before initializing other services since it may be needed by them! startup_dir = workspace.get_startup_kit_dir() @@ -267,7 +277,7 @@ def security_init_for_job(secure_train: bool, workspace: Workspace, site_type: s # for standard mTLS kits without a startup content-integrity manifest; TLS credentials # remain the trust anchor. if secure_train and SecurityContentService.security_content_manager.valid_config: - insecure_list = _check_secure_content(site_type=site_type) + insecure_list = _check_secure_content(site_type=site_type, check_private_key=False) if len(insecure_list): print("The following files are not secure content.") for item in insecure_list: diff --git a/nvflare/private/fed/utils/identity_utils.py b/nvflare/private/fed/utils/identity_utils.py index 536df1a042..8037cedfe9 100644 --- a/nvflare/private/fed/utils/identity_utils.py +++ b/nvflare/private/fed/utils/identity_utils.py @@ -16,6 +16,7 @@ from cryptography import x509 from cryptography.x509.oid import NameOID +from nvflare.fuel.f3.drivers.net_utils import get_cert_job_id from nvflare.fuel.utils.log_utils import get_obj_logger from nvflare.lighter.utils import ( load_crt, @@ -27,6 +28,7 @@ verify_cert_chain, verify_content, ) +from nvflare.private.fed.utils.job_cert_utils import has_job_ca_marker from nvflare.security.logging import secure_format_exception @@ -138,7 +140,7 @@ def verify_common_name( cert_to_be_verified=asserter_cert, root_ca_public_key=self.root_public_key, ) - _validate_identity_cert_usage(asserter_cert, expected_eku) + _validate_identity_cert_usage(asserter_cert, expected_eku, intermediate_certs) except Exception as ex: raise InvalidAsserterCert(str(ex)) from ex @@ -157,13 +159,24 @@ def verify_common_name( return True -def _validate_identity_cert_usage(cert, expected_eku): +def _validate_identity_cert_usage(cert, expected_eku, intermediate_certs=None): """Enforce certificate usage restrictions for the common-name challenge. + Per-job certificates are scoped to one job's cells and must never assert site, admin, + or server identity: a leaf carrying the job-ID extension is rejected, and so is any + chain containing the job-CA marker (a stolen job CA key can mint leaves without the + extension, but cannot strip the root-signed marker off the CA cert it must present). + Legacy FLARE certificates may omit KeyUsage and ExtendedKeyUsage, so absent extensions remain unrestricted. When present, they must allow the signing operation and the caller-specific authentication purpose. """ + if get_cert_job_id(cert) is not None: + raise ValueError("job-scoped certificate cannot be used to assert site identity") + for chain_cert in (cert, *(intermediate_certs or ())): + if has_job_ca_marker(chain_cert): + raise ValueError("certificate issued by the job CA cannot assert site identity") + try: key_usage = cert.extensions.get_extension_for_class(x509.KeyUsage).value except x509.ExtensionNotFound: diff --git a/nvflare/private/fed/utils/job_cert_utils.py b/nvflare/private/fed/utils/job_cert_utils.py new file mode 100644 index 0000000000..9f1aa23098 --- /dev/null +++ b/nvflare/private/fed/utils/job_cert_utils.py @@ -0,0 +1,231 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import shutil +from concurrent.futures import ThreadPoolExecutor +from typing import Dict, Iterable, List, Optional, Tuple + +from cryptography import x509 + +from nvflare.apis.fl_constant import FLContextKey, SecureTrainConst +from nvflare.fuel.f3.cellnet.identity import get_cert_common_name +from nvflare.fuel.f3.drivers.net_utils import JOB_ID_EXTENSION_OID +from nvflare.lighter.constants import CertExtensionOID, ProvFileName +from nvflare.lighter.utils import ( + Identity, + bounded_validity, + generate_cert, + generate_keys, + load_crt_bytes, + load_private_key_file, + serialize_cert, + serialize_pri_key, + write_pri_key_file, +) + +JOB_CERT_DIR_NAME = "job_cert" +JOB_CERT_FILE_NAME = "job.crt" +JOB_KEY_FILE_NAME = "job.key" + +JOB_CERT_VALID_DAYS = 30 + +# leaf notBefore is backdated to tolerate clock skew between the issuing +# server and the sites that validate the cert seconds later +JOB_CERT_BACKDATE = datetime.timedelta(minutes=5) + +# below this remaining job-CA validity, refuse to issue (and so to deploy) rather +# than hand out certs that expire mid-run +JOB_CA_MIN_REMAINING = datetime.timedelta(hours=1) + +NO_JOB_CREDENTIAL = ( + "job has no job credential; secure jobs run only on per-job certificates " + "(the server must be provisioned with a job CA)" +) + +# keys of the credential dict pushed to clients in the deploy message +_PROP_CERT = "cert" +_PROP_KEY = "key" + +JOB_CA_MARKER_OID = x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER) + + +class JobCertError(RuntimeError): + """A secure-mode job cannot get or use its per-job credential; there is no fallback to site certs.""" + + +def job_cert_paths(run_dir: str) -> Tuple[str, str]: + cert_dir = os.path.join(run_dir, JOB_CERT_DIR_NAME) + return os.path.join(cert_dir, JOB_CERT_FILE_NAME), os.path.join(cert_dir, JOB_KEY_FILE_NAME) + + +def write_job_cert(run_dir: str, cert_chain_pem: bytes, key_pem: bytes): + cert_path, key_path = job_cert_paths(run_dir) + os.makedirs(os.path.dirname(cert_path), exist_ok=True) + with open(cert_path, "wb") as f: + f.write(cert_chain_pem) + write_pri_key_file(key_path, key_pem) + + +def find_job_cert(run_dir: str) -> Optional[Tuple[str, str]]: + cert_path, key_path = job_cert_paths(run_dir) + if os.path.isfile(cert_path) and os.path.isfile(key_path): + return cert_path, key_path + return None + + +def read_job_cert(run_dir: str) -> Optional[Tuple[bytes, bytes]]: + paths = find_job_cert(run_dir) + if not paths: + return None + with open(paths[0], "rb") as f: + cert_pem = f.read() + with open(paths[1], "rb") as f: + key_pem = f.read() + return cert_pem, key_pem + + +def require_job_cert(fl_ctx, run_dir: str) -> Optional[Tuple[str, str]]: + """The job credential paths, or None only in non-secure mode. + + Launchers call this before starting a job process: a secure job without its credential + is refused instead of being started on site certificates. + """ + paths = find_job_cert(run_dir) + if paths is None and fl_ctx.get_prop(FLContextKey.SECURE_MODE, False): + raise JobCertError(NO_JOB_CREDENTIAL) + return paths + + +def apply_job_cert_config(site_config: dict, run_dir: str) -> None: + """Make the job credential the job process's ssl_cert / ssl_private_key. + + A job process refers to no other credential. Left untouched when the job has none: in + non-secure mode no certificate is used, and a secure job has already refused to start. + """ + paths = find_job_cert(run_dir) + if paths: + site_config[SecureTrainConst.SSL_CERT], site_config[SecureTrainConst.PRIVATE_KEY] = paths + + +def job_startup_files(startup_dir: str) -> List[str]: + """Startup-kit files a job process may see: every regular file except private keys.""" + return sorted( + f for f in os.listdir(startup_dir) if not f.endswith(".key") and os.path.isfile(os.path.join(startup_dir, f)) + ) + + +def stage_job_startup_dir(startup_dir: str, dest_dir: str) -> str: + """Copy the startup kit without its private keys to dest_dir, for a launcher to mount into the job.""" + os.makedirs(dest_dir, mode=0o700, exist_ok=True) + for fname in job_startup_files(startup_dir): + shutil.copy2(os.path.join(startup_dir, fname), os.path.join(dest_dir, fname)) + return dest_dir + + +def pack_job_cert_header(cert_chain_pem: bytes, key_pem: bytes) -> dict: + return {_PROP_CERT: cert_chain_pem.decode("ascii"), _PROP_KEY: key_pem.decode("ascii")} + + +def unpack_job_cert_header(header) -> Optional[Tuple[bytes, bytes]]: + """Decode a pushed job credential; None for anything malformed (e.g. version skew).""" + if not isinstance(header, dict): + return None + cert_pem = header.get(_PROP_CERT) + key_pem = header.get(_PROP_KEY) + if not (isinstance(cert_pem, str) and isinstance(key_pem, str) and cert_pem and key_pem): + return None + try: + return cert_pem.encode("ascii"), key_pem.encode("ascii") + except UnicodeEncodeError: + return None + + +def has_job_ca_marker(cert: x509.Certificate) -> bool: + try: + cert.extensions.get_extension_for_oid(JOB_CA_MARKER_OID) + return True + except x509.ExtensionNotFound: + return False + + +class JobCertIssuer: + """Issues short-lived per-job certificates signed by the provisioned job CA. + + Only the server parent process issues job certs. Use load_job_cert_issuer() to create one + from a startup kit. + """ + + def __init__(self, ca_cert_pem: bytes, ca_key): + self.ca_cert_pem = ca_cert_pem + self.ca_cert = load_crt_bytes(ca_cert_pem) + self.ca_key = ca_key + self.ca_cn = get_cert_common_name(self.ca_cert) + + def issue(self, site_name: str, job_id: str, valid_days: int = JOB_CERT_VALID_DAYS) -> Tuple[bytes, bytes]: + """Issue a per-job credential for one site. + + Returns: + (cert_chain_pem, key_pem): leaf cert followed by the job CA cert, and the private key. + """ + pri_key, pub_key = generate_keys() + not_valid_before, not_valid_after = bounded_validity(self.ca_cert, valid_days, backdate=JOB_CERT_BACKDATE) + job_id_ext = x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, job_id.encode("utf-8")) + cert = generate_cert( + subject=Identity(site_name), + issuer=Identity(self.ca_cn), + signing_pri_key=self.ca_key, + subject_pub_key=pub_key, + not_valid_before=not_valid_before, + not_valid_after=not_valid_after, + extra_extensions=[(job_id_ext, False)], + ) + return serialize_cert(cert) + self.ca_cert_pem, serialize_pri_key(pri_key) + + def issue_many( + self, site_names: Iterable[str], job_id: str, valid_days: int = JOB_CERT_VALID_DAYS + ) -> Dict[str, Tuple[bytes, bytes]]: + """Issue credentials for several sites at once; RSA key generation dominates and runs in parallel.""" + names = list(site_names) + if not names: + return {} + with ThreadPoolExecutor(max_workers=min(8, len(names))) as pool: + return dict(zip(names, pool.map(lambda name: self.issue(name, job_id, valid_days), names))) + + +def load_job_cert_issuer(startup_dir: str) -> JobCertIssuer: + """Create a JobCertIssuer from the startup kit's job CA. + + Raises JobCertError when the kit has no job CA (provisioned before this feature or with + CertBuilder's enable_job_ca off) or the job CA is about to expire; secure-mode jobs must + not run without a per-job credential. + """ + cert_path = os.path.join(startup_dir, ProvFileName.JOB_CA_CERT) + key_path = os.path.join(startup_dir, ProvFileName.JOB_CA_KEY) + if not (os.path.isfile(cert_path) and os.path.isfile(key_path)): + raise JobCertError( + f"server startup kit has no job CA ({ProvFileName.JOB_CA_CERT} / {ProvFileName.JOB_CA_KEY}); " + "re-provision the project (CertBuilder enable_job_ca) to run jobs in secure mode" + ) + + with open(cert_path, "rb") as f: + ca_cert_pem = f.read() + ca_cert = load_crt_bytes(ca_cert_pem) + if ca_cert.not_valid_after_utc <= datetime.datetime.now(datetime.timezone.utc) + JOB_CA_MIN_REMAINING: + raise JobCertError( + f"job CA expires at {ca_cert.not_valid_after_utc.isoformat()} (less than {JOB_CA_MIN_REMAINING} left); " + "re-provision the project to renew it" + ) + return JobCertIssuer(ca_cert_pem, load_private_key_file(key_path)) diff --git a/tests/unit_test/app_opt/job_launcher/docker_launcher_test.py b/tests/unit_test/app_opt/job_launcher/docker_launcher_test.py index 5335e61012..4ea5f22395 100644 --- a/tests/unit_test/app_opt/job_launcher/docker_launcher_test.py +++ b/tests/unit_test/app_opt/job_launcher/docker_launcher_test.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import os import sys import threading from types import ModuleType @@ -506,6 +507,31 @@ def test_raises_if_network_not_found(self): # --------------------------------------------------------------------------- +_FAKE_STARTUP_FILES = ("rootCA.pem", "client.crt", "client.key", "fed_client.json") +_fake_startup_dir = None + + +@pytest.fixture(scope="module", autouse=True) +def _shared_startup_kit(tmp_path_factory): + """A real startup kit directory, shared by the module, so the launcher can list files to bind.""" + global _fake_startup_dir + _fake_startup_dir = str(tmp_path_factory.mktemp("startup")) + for name in _FAKE_STARTUP_FILES: + with open(os.path.join(_fake_startup_dir, name), "w") as f: + f.write(name) + yield + _fake_startup_dir = None + + +def _make_workspace_obj(startup_dir=None, run_dir="/ws/job-1"): + workspace_obj = MagicMock() + workspace_obj.get_startup_kit_dir.return_value = startup_dir or _fake_startup_dir + workspace_obj.get_run_dir.return_value = run_dir + workspace_obj.get_app_custom_dir.return_value = "" + workspace_obj.get_site_custom_dir.return_value = "" + return workspace_obj + + def _make_fl_ctx( job_id="job-1", exe_module="nvflare.private.fed.app.client.worker_process", @@ -515,7 +541,11 @@ def _make_fl_ctx( workspace_path="/ws", set_list=None, num_of_gpus=None, + workspace_obj=None, + secure_mode=False, ): + if workspace_obj is None: + workspace_obj = _make_workspace_obj() fl_ctx = MagicMock(spec=FLContext) fl_ctx.get_identity_name.return_value = identity_name @@ -530,8 +560,9 @@ def _make_fl_ctx( job_args[JobProcessArgs.PARENT_CONN_SEC] = ("--parent_conn_sec", parent_conn_sec) fl_ctx.get_prop.side_effect = lambda key, *a, **kw: { FLContextKey.JOB_PROCESS_ARGS: job_args, - FLContextKey.WORKSPACE_OBJECT: None, + FLContextKey.WORKSPACE_OBJECT: workspace_obj, FLContextKey.ARGS: None, + FLContextKey.SECURE_MODE: secure_mode, }.get(key) return fl_ctx, job_args @@ -720,9 +751,13 @@ def test_launch_workspace_bind_mounted(self): call_kwargs = dc.containers.run.call_args[1] mounts = call_kwargs["mounts"] assert mounts[0]["Target"] == "/var/tmp/nvflare/workspace" - assert mounts[1]["Target"] == "/var/tmp/nvflare/workspace/startup" - assert mounts[2]["Target"] == "/var/tmp/nvflare/workspace/local" - assert mounts[3]["Target"] == "/var/tmp/nvflare/workspace/job-1" + startup_targets = [m["Target"] for m in mounts if m["Target"].startswith("/var/tmp/nvflare/workspace/startup")] + # the kit is bound file by file, without private keys, ahead of local/ and the job workspace + assert startup_targets == [ + f"/var/tmp/nvflare/workspace/startup/{name}" for name in ("client.crt", "fed_client.json", "rootCA.pem") + ] + assert mounts[len(startup_targets) + 1]["Target"] == "/var/tmp/nvflare/workspace/local" + assert mounts[len(startup_targets) + 2]["Target"] == "/var/tmp/nvflare/workspace/job-1" mounts_by_target = _mounts_by_target(call_kwargs["mounts"]) assert mounts_by_target["/var/tmp/nvflare/workspace"] == { @@ -732,12 +767,13 @@ def test_launch_workspace_bind_mounted(self): "ReadOnly": False, "tmpfs_mode": 0o1777, } - assert mounts_by_target["/var/tmp/nvflare/workspace/startup"] == { - "Target": "/var/tmp/nvflare/workspace/startup", - "Source": "/host/workspace/startup", + assert mounts_by_target["/var/tmp/nvflare/workspace/startup/rootCA.pem"] == { + "Target": "/var/tmp/nvflare/workspace/startup/rootCA.pem", + "Source": "/host/workspace/startup/rootCA.pem", "Type": "bind", "ReadOnly": True, } + assert "/var/tmp/nvflare/workspace/startup/client.key" not in mounts_by_target assert mounts_by_target["/var/tmp/nvflare/workspace/local"] == { "Target": "/var/tmp/nvflare/workspace/local", "Source": "/host/workspace/local", @@ -751,6 +787,49 @@ def test_launch_workspace_bind_mounted(self): "ReadOnly": False, } + def test_secure_launch_with_job_credential_binds_startup_files_without_keys(self, tmp_path): + startup = tmp_path / "startup" + startup.mkdir() + for name in ("rootCA.pem", "client.crt", "client.key", "fed_client.json"): + (startup / name).write_text(name) + run_dir = tmp_path / "job-1" + (run_dir / "job_cert").mkdir(parents=True) + (run_dir / "job_cert" / "job.crt").write_text("cert") + (run_dir / "job_cert" / "job.key").write_text("key") + workspace_obj = _make_workspace_obj(startup_dir=str(startup), run_dir=str(run_dir)) + + launcher = _make_launcher(workspace="/host/workspace") + dc = launcher._docker_client + container = MagicMock() + container.id = "abc123" + dc.containers.run.return_value = container + dc.containers.get.return_value = _make_container("running") + fl_ctx, _ = _make_fl_ctx(workspace_obj=workspace_obj, secure_mode=True) + + launcher.launch_job(_make_job_meta(), fl_ctx) + + mounts_by_target = _mounts_by_target(dc.containers.run.call_args[1]["mounts"]) + assert "/var/tmp/nvflare/workspace/startup" not in mounts_by_target + assert "/var/tmp/nvflare/workspace/startup/client.key" not in mounts_by_target + for name in ("rootCA.pem", "client.crt", "fed_client.json"): + assert mounts_by_target[f"/var/tmp/nvflare/workspace/startup/{name}"] == { + "Target": f"/var/tmp/nvflare/workspace/startup/{name}", + "Source": f"/host/workspace/startup/{name}", + "Type": "bind", + "ReadOnly": True, + } + assert mounts_by_target["/var/tmp/nvflare/workspace/job-1"]["ReadOnly"] is False + + def test_secure_launch_without_job_credential_is_refused(self, tmp_path): + launcher = _make_launcher(workspace="/host/workspace") + dc = launcher._docker_client + fl_ctx, _ = _make_fl_ctx(workspace_obj=_make_workspace_obj(run_dir=str(tmp_path / "job-1")), secure_mode=True) + + with pytest.raises(RuntimeError, match="no job credential"): + launcher.launch_job(_make_job_meta(), fl_ctx) + + dc.containers.run.assert_not_called() + def test_launch_rejects_job_workspace_path_escape(self): launcher = _make_launcher(workspace="/host/workspace") dc = launcher._docker_client @@ -880,7 +959,9 @@ def test_launch_default_study_without_mapping_does_not_mount_data(self): mounts_by_target = _mounts_by_target(dc.containers.run.call_args[1]["mounts"]) assert set(mounts_by_target) == { "/var/tmp/nvflare/workspace", - "/var/tmp/nvflare/workspace/startup", + "/var/tmp/nvflare/workspace/startup/client.crt", + "/var/tmp/nvflare/workspace/startup/fed_client.json", + "/var/tmp/nvflare/workspace/startup/rootCA.pem", "/var/tmp/nvflare/workspace/local", "/var/tmp/nvflare/workspace/job-1", } @@ -927,7 +1008,9 @@ def test_launch_omits_data_mount_when_study_mapping_is_missing(self, caplog): mounts_by_target = _mounts_by_target(dc.containers.run.call_args[1]["mounts"]) assert set(mounts_by_target) == { "/var/tmp/nvflare/workspace", - "/var/tmp/nvflare/workspace/startup", + "/var/tmp/nvflare/workspace/startup/client.crt", + "/var/tmp/nvflare/workspace/startup/fed_client.json", + "/var/tmp/nvflare/workspace/startup/rootCA.pem", "/var/tmp/nvflare/workspace/local", "/var/tmp/nvflare/workspace/job-1", } diff --git a/tests/unit_test/app_opt/job_launcher/k8s_launcher_test.py b/tests/unit_test/app_opt/job_launcher/k8s_launcher_test.py index f8835ac78d..12e57738db 100644 --- a/tests/unit_test/app_opt/job_launcher/k8s_launcher_test.py +++ b/tests/unit_test/app_opt/job_launcher/k8s_launcher_test.py @@ -56,7 +56,7 @@ def __init__(self, status=None, reason=None, http_resp=None): from nvflare.apis.fl_constant import FLContextKey, JobConstants, ReservedKey from nvflare.apis.fl_context import FLContext from nvflare.apis.job_def import JobMetaKey -from nvflare.apis.job_launcher_spec import JobProcessArgs, JobReturnCode +from nvflare.apis.job_launcher_spec import JobProcessArgs, JobProcessEnv, JobReturnCode from nvflare.app_opt.job_launcher.k8s_launcher import ( JOB_RETURN_CODE_MAPPING, POD_STATE_MAPPING, @@ -1707,9 +1707,11 @@ def _make_launch_fl_ctx( app_custom_folder="", workspace_arg="/var/tmp/nvflare/workspace", workspace="/fake/workspace", + secure_mode=False, ): fl_ctx = FLContext() fl_ctx.set_prop(ReservedKey.IDENTITY_NAME, site_name, private=False, sticky=True) + fl_ctx.set_prop(FLContextKey.SECURE_MODE, secure_mode, private=True, sticky=True) job_args = { JobProcessArgs.EXE_MODULE: ("-m", _WORKER_MODULE), JobProcessArgs.WORKSPACE: ("-w", workspace_arg), @@ -1724,6 +1726,7 @@ def _make_launch_fl_ctx( workspace_obj.get_app_custom_dir.return_value = app_custom_folder workspace_obj.get_startup_kit_dir.return_value = "/fake/startup" workspace_obj.get_site_config_dir.return_value = "/fake/local" + workspace_obj.get_run_dir.return_value = "/fake/run" fl_ctx.set_prop(FLContextKey.WORKSPACE_OBJECT, workspace_obj, private=True, sticky=False) engine = Mock() engine.cell = Mock() @@ -3481,8 +3484,8 @@ def test_security_context_empty_dict_not_applied(self): _EXPECTED_CRED_SECRET_NAME = f"nvflare-cred-{_EXPECTED_POD_NAME}" -def _make_cred_fl_ctx(): - fl_ctx = _make_launch_fl_ctx() +def _make_cred_fl_ctx(secure_mode=False): + fl_ctx = _make_launch_fl_ctx(secure_mode=secure_mode) fl_ctx.get_prop(FLContextKey.JOB_PROCESS_ARGS).update( { JobProcessArgs.AUTH_TOKEN: ("-t", "secret-token"), @@ -3511,12 +3514,12 @@ def _setup(self, patches): return launcher, mock_api @pytest.mark.parametrize(("cert_name", "key_name"), [("client.crt", "client.key"), ("server.crt", "server.key")]) - def test_startup_secret_contains_complete_participant_tls_credentials(self, tmp_path, cert_name, key_name): + def test_startup_secret_never_ships_private_keys(self, tmp_path, cert_name, key_name): from nvflare.app_opt.job_launcher.k8s_launcher import ClientK8sJobLauncher startup_dir = tmp_path / "startup" startup_dir.mkdir() - for name in ("rootCA.pem", cert_name, key_name): + for name in ("rootCA.pem", cert_name, key_name, "job_ca.key", "fed_client.json"): (startup_dir / name).write_text(name) launcher = ClientK8sJobLauncher(config_file_path=None) launcher.core_v1 = MagicMock() @@ -3524,7 +3527,65 @@ def test_startup_secret_contains_complete_participant_tls_credentials(self, tmp_ launcher._ensure_startup_secret("site-1", str(startup_dir)) body = launcher.core_v1.create_namespaced_secret.call_args.kwargs["body"] - assert set(body["data"]) == {"rootCA.pem", cert_name, key_name} + assert set(body["data"]) == {"rootCA.pem", cert_name, "fed_client.json"} + + def test_job_credential_rides_credential_secret(self): + patches = _make_k8s_launcher_patches() + launcher, mock_api = self._setup(patches) + launcher._ensure_startup_secret = MagicMock(return_value="nvflare-startup-site-1") + try: + with ( + patch( + "nvflare.app_opt.job_launcher.k8s_launcher.require_job_cert", return_value=("job.crt", "job.key") + ), + patch( + "nvflare.app_opt.job_launcher.k8s_launcher.read_job_cert", + return_value=(b"JOB-CERT-PEM", b"JOB-KEY-PEM"), + ), + ): + launcher.launch_job(_make_launch_job_meta(), _make_cred_fl_ctx(secure_mode=True)) + + (body,) = _cred_secret_bodies(mock_api) + assert body["stringData"] == { + **_CREDENTIAL_ENV, + ENV_WORKSPACE_TRANSFER_TOKEN: "transfer-token", + JobProcessEnv.JOB_CERT: "JOB-CERT-PEM", + JobProcessEnv.JOB_KEY: "JOB-KEY-PEM", + } + + manifest = mock_api.create_namespaced_pod.call_args.kwargs["body"] + env_by_name = {item["name"]: item for item in manifest["spec"]["containers"][0]["env"]} + for env_name in (JobProcessEnv.JOB_CERT, JobProcessEnv.JOB_KEY): + ref = env_by_name[env_name]["valueFrom"]["secretKeyRef"] + assert ref == {"name": _EXPECTED_CRED_SECRET_NAME, "key": env_name} + assert "JOB-KEY-PEM" not in str(manifest) + finally: + _exit_patches(patches) + + def test_secure_job_without_credential_is_refused(self): + patches = _make_k8s_launcher_patches() + launcher, mock_api = self._setup(patches) + try: + with pytest.raises(RuntimeError, match="no job credential"): + launcher.launch_job(_make_launch_job_meta(), _make_cred_fl_ctx(secure_mode=True)) + + mock_api.create_namespaced_secret.assert_not_called() + mock_api.create_namespaced_pod.assert_not_called() + finally: + _exit_patches(patches) + + def test_non_secure_job_launches_without_credential(self): + patches = _make_k8s_launcher_patches() + launcher, mock_api = self._setup(patches) + launcher._ensure_startup_secret = MagicMock(return_value="nvflare-startup-site-1") + try: + launcher.launch_job(_make_launch_job_meta(), _make_cred_fl_ctx()) + + (body,) = _cred_secret_bodies(mock_api) + assert JobProcessEnv.JOB_CERT not in body["stringData"] + assert JobProcessEnv.JOB_KEY not in body["stringData"] + finally: + _exit_patches(patches) def test_secret_created_and_pod_references_it_without_values(self): patches = _make_k8s_launcher_patches() diff --git a/tests/unit_test/app_opt/job_launcher/slurm_launcher_test.py b/tests/unit_test/app_opt/job_launcher/slurm_launcher_test.py index bddb2a65cb..2257ecdf47 100644 --- a/tests/unit_test/app_opt/job_launcher/slurm_launcher_test.py +++ b/tests/unit_test/app_opt/job_launcher/slurm_launcher_test.py @@ -579,6 +579,21 @@ def test_launch_plan_rejects_different_context_workspace(tmp_path): launcher._build_launch_plan({JobConstants.JOB_ID: "job-1"}, _fl_ctx(context_workspace)) +def test_launch_plan_rejects_secure_job_without_credential(tmp_path): + workspace = _workspace(tmp_path) + launcher = _launcher(tmp_path, workspace) + fl_ctx = _fl_ctx(workspace) + fl_ctx.set_prop(FLContextKey.SECURE_MODE, True, private=True, sticky=True) + + with pytest.raises(SlurmLauncherError, match="no job credential"): + launcher._build_launch_plan({JobConstants.JOB_ID: "job-1"}, fl_ctx) + + (workspace / "job-1" / "job_cert").mkdir() + (workspace / "job-1" / "job_cert" / "job.crt").write_text("cert") + (workspace / "job-1" / "job_cert" / "job.key").write_text("key") + assert launcher._build_launch_plan({JobConstants.JOB_ID: "job-1"}, fl_ctx).run_dir == str(workspace / "job-1") + + @pytest.mark.parametrize("launcher_class", [ClientSlurmJobLauncher, ServerSlurmJobLauncher]) def test_launch_plan_preserves_mtls_parent_args(tmp_path, launcher_class): workspace = _workspace(tmp_path) diff --git a/tests/unit_test/app_opt/job_launcher/slurm_manager_test.py b/tests/unit_test/app_opt/job_launcher/slurm_manager_test.py index dd95777505..1233fc0f03 100644 --- a/tests/unit_test/app_opt/job_launcher/slurm_manager_test.py +++ b/tests/unit_test/app_opt/job_launcher/slurm_manager_test.py @@ -138,6 +138,7 @@ def _manager(tmp_path, adapter=None, monotonic=None, **config_kwargs): def _plan(tmp_path, pending_timeout=5, setup="", sandbox="none", image=None): run_dir = tmp_path / "job-1" run_dir.mkdir(exist_ok=True) + (tmp_path / "startup").mkdir(exist_ok=True) return LaunchPlan( job_id="job-1", site_name="site-1", @@ -353,7 +354,23 @@ def test_pyxis_node_group_writes_node_script_and_mounts_job_artifacts(tmp_path): assert "python3 -m trainer" in node_script.read_text(encoding="utf-8") batch = Path(handle.job_dir, "batch.sh").read_text(encoding="utf-8") assert f"{handle.job_dir}:{handle.job_dir}:ro" in batch - assert f"{tmp_path / 'startup'}:{tmp_path / 'startup'}:ro" in batch + assert f"{Path(handle.job_dir, 'startup')}:{tmp_path / 'startup'}:ro" in batch + + +def test_sandbox_gets_keyless_staged_startup_kit(tmp_path): + startup = tmp_path / "startup" + startup.mkdir() + for name in ("rootCA.pem", "client.crt", "client.key"): + (startup / name).write_text(name) + adapter = Adapter() + manager = _manager(tmp_path, adapter) + + handle = manager.launch(_plan(tmp_path, sandbox="apptainer", image="/image.sif")) + + staged = Path(handle.job_dir, "startup") + assert sorted(p.name for p in staged.iterdir()) == ["client.crt", "rootCA.pem"] + assert f"{staged}:{tmp_path / 'startup'}:ro" in adapter.submitted_batch + assert f"{tmp_path / 'startup'}:{tmp_path / 'startup'}:ro" not in adapter.submitted_batch def test_submission_uses_site_timeout(tmp_path): @@ -574,9 +591,9 @@ def parent_realpath(path): adapter = Adapter() manager = _manager(tmp_path, adapter) - manager.launch(_plan(tmp_path, sandbox="apptainer", image="/image.sif")) + handle = manager.launch(_plan(tmp_path, sandbox="apptainer", image="/image.sif")) - assert f"{tmp_path / 'startup'}:{tmp_path / 'startup'}:ro" in adapter.submitted_batch + assert f"{Path(handle.job_dir, 'startup')}:{tmp_path / 'startup'}:ro" in adapter.submitted_batch assert f"{CONTAINER_RESOLV_CONF}:{CONTAINER_RESOLV_CONF}:ro" in adapter.submitted_batch assert parent_target not in adapter.submitted_batch diff --git a/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py b/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py index 29f615a1f2..829df230d6 100644 --- a/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py +++ b/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py @@ -29,8 +29,11 @@ WorkspaceTransferManager, _bootstrap_auth_identity_map, _create_bootstrap_cell, + _get_bootstrap_tls_pair, _hash_file, + _install_job_cert, _wait_for_bootstrap_ready, + _zip_results_to_file, _zip_workspace_to_file, download_workspace, make_workspace_transfer_fqcn, @@ -40,6 +43,7 @@ from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.fqcn import FQCN from nvflare.fuel.f3.cellnet.utils import make_reply, new_cell_message +from nvflare.fuel.f3.drivers.driver_params import DriverParams JOB_ID = "abc12345-dead-beef-0000-111122223333" @@ -195,6 +199,21 @@ def test_workspace_bundle_excludes_study_runtime_pod_templates(self): assert "local/study_runtime.yaml" not in names assert "local/pod_specs/h100-pod.yaml" not in names + @pytest.mark.parametrize("zip_fn", [_zip_workspace_to_file, _zip_results_to_file]) + def test_bundles_exclude_job_credential(self, zip_fn): + with tempfile.TemporaryDirectory() as ws_root, tempfile.TemporaryDirectory() as tmp: + _make_workspace(ws_root, JOB_ID) + _write_file(os.path.join(ws_root, JOB_ID, "job_cert", "job.crt"), b"cert") + _write_file(os.path.join(ws_root, JOB_ID, "job_cert", "job.key"), b"key") + zip_path = os.path.join(tmp, "bundle.zip") + + zip_fn(ws_root, JOB_ID, zip_path) + + with zipfile.ZipFile(zip_path) as zf: + names = set(zf.namelist()) + assert f"{JOB_ID}/app/config/config_train.json" in names + assert not any(name.startswith(f"{JOB_ID}/job_cert/") for name in names) + def test_prepare_download_returns_ref_for_valid_token(self, monkeypatch): with tempfile.TemporaryDirectory() as ws_root: _make_workspace(ws_root, JOB_ID) @@ -811,11 +830,11 @@ def test_create_bootstrap_cell_passes_identity_map(self, monkeypatch, tmp_path): startup = tmp_path / "startup" startup.mkdir() (startup / "rootCA.pem").write_text("ca") - (startup / "client.crt").write_text("cert") - (startup / "client.key").write_text("key") (startup / "fed_client.json").write_text( json.dumps({"servers": [{"name": "project", "identity": "gcp-server"}]}) ) + _write_file(str(tmp_path / JOB_ID / "job_cert" / "job.crt"), b"job-cert") + _write_file(str(tmp_path / JOB_ID / "job_cert" / "job.key"), b"job-key") captured = {} @@ -848,3 +867,43 @@ def start(self): assert captured["auth_identity_map"] == {FQCN.ROOT_SERVER: "gcp-server"} assert captured["secure"] is True + job_cert_dir = str(tmp_path / JOB_ID / "job_cert") + assert captured["credentials"][DriverParams.CLIENT_CERT.value] == os.path.join(job_cert_dir, "job.crt") + assert captured["credentials"][DriverParams.CLIENT_KEY.value] == os.path.join(job_cert_dir, "job.key") + + def test_bootstrap_tls_pair_uses_job_credential_in_the_peer_role(self, tmp_path): + run_dir = tmp_path / JOB_ID + job_crt = run_dir / "job_cert" / "job.crt" + job_key = run_dir / "job_cert" / "job.key" + _write_file(str(job_crt), b"job-cert") + _write_file(str(job_key), b"job-key") + + cert_path, key_path, cert_key, key_key = _get_bootstrap_tls_pair(str(run_dir), "site-1") + + assert (cert_path, key_path) == (str(job_crt), str(job_key)) + assert (cert_key, key_key) == (DriverParams.CLIENT_CERT.value, DriverParams.CLIENT_KEY.value) + + _, _, cert_key, key_key = _get_bootstrap_tls_pair(str(run_dir), FQCN.ROOT_SERVER) + assert (cert_key, key_key) == (DriverParams.SERVER_CERT.value, DriverParams.SERVER_KEY.value) + + def test_bootstrap_tls_pair_requires_job_credential(self, tmp_path): + with pytest.raises(RuntimeError, match="requires the job credential"): + _get_bootstrap_tls_pair(str(tmp_path / JOB_ID), "site-1") + + def test_install_job_cert_writes_run_dir(self, tmp_path): + args = SimpleNamespace(workspace=str(tmp_path), job_id=JOB_ID, job_cert_pem="cert-pem", job_key_pem="key-pem") + + _install_job_cert(args) + + cert_path = tmp_path / JOB_ID / "job_cert" / "job.crt" + key_path = tmp_path / JOB_ID / "job_cert" / "job.key" + assert cert_path.read_bytes() == b"cert-pem" + assert key_path.read_bytes() == b"key-pem" + assert stat.S_IMODE(os.stat(key_path).st_mode) == 0o600 + + def test_install_job_cert_noop_without_complete_credential(self, tmp_path): + args = SimpleNamespace(workspace=str(tmp_path), job_id=JOB_ID, job_cert_pem=None, job_key_pem="key-only") + + _install_job_cert(args) + + assert not (tmp_path / JOB_ID).exists() diff --git a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py index 4faf2e71b0..1eac5cbbe2 100644 --- a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py +++ b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py @@ -39,6 +39,12 @@ from nvflare.fuel.f3.cellnet.utils import make_reply from nvflare.fuel.f3.comm_error import CommError from nvflare.fuel.f3.drivers.driver_params import DriverParams +from nvflare.fuel.f3.drivers.net_utils import ( + JOB_ID_EXTENSION_OID, + get_cert_job_id_from_pem, + get_grpc_peer_job_id, + get_peer_job_id, +) from nvflare.fuel.f3.endpoint import Endpoint from nvflare.fuel.f3.message import Message from nvflare.fuel.f3.sfm.conn_manager import ConnManager @@ -49,7 +55,14 @@ class _FakeConnection: - def __init__(self, peer_cn, conn_security=ConnectionSecurity.MTLS, mode=Mode.PASSIVE, admin_listener=False): + def __init__( + self, + peer_cn, + conn_security=ConnectionSecurity.MTLS, + mode=Mode.PASSIVE, + admin_listener=False, + peer_job_id=None, + ): self.name = "CN-test" self.closed = False self.connector = SimpleNamespace( @@ -64,6 +77,8 @@ def __init__(self, peer_cn, conn_security=ConnectionSecurity.MTLS, mode=Mode.PAS self.conn_props = {} if peer_cn is not None: self.conn_props[DriverParams.PEER_CN.value] = peer_cn + if peer_job_id is not None: + self.conn_props[DriverParams.PEER_JOB_ID.value] = peer_job_id def get_conn_properties(self): return self.conn_props @@ -89,6 +104,19 @@ def _cert_pem(common_name: str): return cert.public_bytes(serialization.Encoding.PEM) +def _job_cert_pem(common_name: str, job_id): + key, pub_key = generate_keys() + job_id_bytes = job_id if isinstance(job_id, bytes) else job_id.encode("utf-8") + cert = generate_cert( + subject=Identity(common_name), + issuer=Identity(common_name), + signing_pri_key=key, + subject_pub_key=pub_key, + extra_extensions=[(x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, job_id_bytes), False)], + ) + return cert.public_bytes(serialization.Encoding.PEM) + + def _make_chained_cell_cipher_cert(): root_key, root_pub_key = generate_keys() root_cert = generate_cert( @@ -421,6 +449,74 @@ def test_mtls_certificate_cache_accepts_configured_auth_identity_for_site_cert_c assert manager.cert_cache["site-1.job-123"] == cert +def test_identity_resolver_binds_job_cert_to_job_fqcn(): + resolver = CellIdentityResolver(local_fqcn="server", prefix_identity_map={"site-1": "site-1"}) + + resolver.require_match("site-1.job-123", "site-1", "connection cj", peer_job_id="job-123") + resolver.require_match("site-1.job-123.sub-1", "site-1", "connection sub", peer_job_id="job-123") + + with pytest.raises(ValueError, match="bound to job 'job-123'"): + resolver.require_match("site-1.job-999", "site-1", "connection cj", peer_job_id="job-123") + with pytest.raises(ValueError, match="bound to job 'job-123'"): + resolver.require_match("site-1", "site-1", "connection cp", peer_job_id="job-123") + with pytest.raises(ValueError, match="bound to job 'job-123'"): + resolver.require_match( + "_admin_9af49fef-235f-41bd-9296-12fd09eacb2a", "admin@nvidia.com", "connection admin", peer_job_id="job-123" + ) + + +def test_mtls_handshake_accepts_job_cert_for_own_job(): + manager = _conn_manager(identity_map={"site-1": "site-1"}) + conn = _FakeConnection(peer_cn="site-1", peer_job_id="job-123") + sfm_conn = SfmConnection(conn, Endpoint("server")) + + manager.update_endpoint(sfm_conn, {HandshakeKeys.ENDPOINT_NAME: "site-1.job-123"}) + + assert "site-1.job-123" in manager.sfm_endpoints + assert not conn.closed + + +@pytest.mark.parametrize("endpoint_name", ["site-1.job-999", "site-1"]) +def test_mtls_handshake_rejects_job_cert_outside_its_job(endpoint_name): + manager = _conn_manager(identity_map={"site-1": "site-1"}) + conn = _FakeConnection(peer_cn="site-1", peer_job_id="job-123") + sfm_conn = SfmConnection(conn, Endpoint("server")) + + with pytest.raises(CommError) as ex: + manager.update_endpoint(sfm_conn, {HandshakeKeys.ENDPOINT_NAME: endpoint_name}) + + assert ex.value.code == CommError.BAD_DATA + assert endpoint_name not in manager.sfm_endpoints + assert conn.closed + + +def test_mtls_certificate_cache_binds_job_cert_to_job_fqcn(): + resolver = CellIdentityResolver(local_fqcn="server", prefix_identity_map={"site-1": "site-1"}) + manager = CredentialManager(Endpoint("server"), identity_resolver=resolver, enforce_identity=True) + cert = _job_cert_pem("site-1", "job-123") + + own_job = Message(headers={MessageHeaderKey.ORIGIN: "site-1.job-123"}, payload={CERT_CONTENT: cert}) + assert manager.process_response(own_job) == cert + + for origin in ("site-1", "site-1.job-999"): + with pytest.raises(RuntimeError, match="bound to job 'job-123'"): + manager.process_response(Message(headers={MessageHeaderKey.ORIGIN: origin}, payload={CERT_CONTENT: cert})) + assert origin not in manager.cert_cache + + +def test_peer_job_id_extraction(): + assert get_cert_job_id_from_pem(_cert_pem("site-1")) is None + assert get_cert_job_id_from_pem(_job_cert_pem("site-1", "job-123")) == "job-123" + assert "�" in get_cert_job_id_from_pem(_job_cert_pem("site-1", b"\xff\xfe")) + + der = x509.load_pem_x509_certificate(_job_cert_pem("site-1", "job-123")).public_bytes(serialization.Encoding.DER) + assert get_peer_job_id(SimpleNamespace(getpeercert=lambda binary_form=False: der)) == "job-123" + assert get_peer_job_id(None) is None + + assert get_grpc_peer_job_id({"x509_pem_cert": [_job_cert_pem("site-1", "job-123")]}) == "job-123" + assert get_grpc_peer_job_id({"x509_common_name": [b"site-1"]}) is None + + def test_cell_cipher_accepts_leaf_certificate_with_intermediate_chain(): root_cert, leaf_key, leaf_cert, intermediate_cert = _make_chained_cell_cipher_cert() diff --git a/tests/unit_test/lighter/cert_builder_test.py b/tests/unit_test/lighter/cert_builder_test.py index 6301881be8..379af43365 100644 --- a/tests/unit_test/lighter/cert_builder_test.py +++ b/tests/unit_test/lighter/cert_builder_test.py @@ -13,17 +13,29 @@ # limitations under the License. import datetime +import json +import os +import stat from unittest.mock import patch import pytest +from cryptography import x509 -from nvflare.lighter.constants import CtxKey, ParticipantType +from nvflare.lighter.constants import CertExtensionOID, CtxKey, ParticipantType, ProvFileName from nvflare.lighter.entity import Participant, Project from nvflare.lighter.impl.cert import CertBuilder from nvflare.lighter.impl.workspace import WorkspaceBuilder from nvflare.lighter.prov_utils import prepare_builders from nvflare.lighter.provisioner import Provisioner -from nvflare.lighter.utils import load_crt +from nvflare.lighter.utils import ( + Identity, + generate_cert, + generate_keys, + load_crt, + load_private_key, + serialize_cert, + verify_cert, +) def _make_project(): @@ -54,8 +66,12 @@ def _provision(workspace, cert_builder): return provisioner.provision(_make_project()) +def _kit_dir(workspace, participant_name): + return workspace / "test-project" / "prod_00" / participant_name / "startup" + + def _server_cert(workspace): - return load_crt(str(workspace / "test-project" / "prod_00" / "server1" / "startup" / "server.crt")) + return load_crt(str(_kit_dir(workspace, "server1") / "server.crt")) def test_default_root_validity_is_reported(tmp_path, capsys): @@ -131,3 +147,87 @@ def test_leaf_validity_is_bounded_by_shorter_root(tmp_path): assert leaf_cert.not_valid_before_utc >= root_cert.not_valid_before_utc assert leaf_cert.not_valid_after_utc == root_cert.not_valid_after_utc assert leaf_cert.not_valid_after_utc - leaf_cert.not_valid_before_utc <= datetime.timedelta(days=1) + + +def test_job_ca_can_be_disabled(tmp_path): + workspace = tmp_path / "workspace" + + ctx = _provision(workspace, CertBuilder(enable_job_ca=False)) + + assert not ctx.get(CtxKey.BUILD_ERROR) + assert not (_kit_dir(workspace, "server1") / ProvFileName.JOB_CA_CERT).exists() + assert not (_kit_dir(workspace, "server1") / ProvFileName.JOB_CA_KEY).exists() + + +def test_job_ca_written_to_server_kit_by_default(tmp_path): + workspace = tmp_path / "workspace" + + ctx = _provision(workspace, CertBuilder()) + + assert not ctx.get(CtxKey.BUILD_ERROR) + job_ca_cert_path = _kit_dir(workspace, "server1") / ProvFileName.JOB_CA_CERT + job_ca_key_path = _kit_dir(workspace, "server1") / ProvFileName.JOB_CA_KEY + assert job_ca_cert_path.exists() + assert job_ca_key_path.exists() + assert stat.S_IMODE(os.stat(job_ca_key_path).st_mode) == 0o600 + assert not (_kit_dir(workspace, "site-1") / ProvFileName.JOB_CA_CERT).exists() + assert not (_kit_dir(workspace, "site-1") / ProvFileName.JOB_CA_KEY).exists() + + job_ca_cert = load_crt(str(job_ca_cert_path)) + basic_constraints = job_ca_cert.extensions.get_extension_for_class(x509.BasicConstraints) + assert basic_constraints.value.ca is True + assert basic_constraints.value.path_length == 0 + key_usage = job_ca_cert.extensions.get_extension_for_class(x509.KeyUsage) + assert key_usage.value.key_cert_sign is True + job_ca_cert.extensions.get_extension_for_oid(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER)) + verify_cert(job_ca_cert, ctx[CtxKey.ROOT_CERT].public_key()) + assert job_ca_cert.not_valid_after_utc <= ctx[CtxKey.ROOT_CERT].not_valid_after_utc + + +def test_job_ca_reused_on_reprovision(tmp_path): + workspace = tmp_path / "workspace" + _provision(workspace, CertBuilder(enable_job_ca=True)) + first_cert = load_crt(str(_kit_dir(workspace, "server1") / ProvFileName.JOB_CA_CERT)) + + second_ctx = _provision(workspace, CertBuilder(enable_job_ca=True)) + + assert not second_ctx.get(CtxKey.BUILD_ERROR) + second_cert = load_crt(str(_kit_dir(workspace, "server1") / ProvFileName.JOB_CA_CERT)) + assert first_cert.serial_number == second_cert.serial_number + + +@pytest.mark.parametrize("value", [0, 1, "true", None, [], {}]) +def test_rejects_invalid_enable_job_ca(value): + with pytest.raises(ValueError, match="enable_job_ca must be a bool"): + CertBuilder(enable_job_ca=value) + + +def test_expired_job_ca_regenerated_on_reprovision(tmp_path): + workspace = tmp_path / "workspace" + _provision(workspace, CertBuilder(enable_job_ca=True)) + + state_file = next(workspace.glob("**/cert.json")) + state = json.loads(state_file.read_text()) + root_key = load_private_key(state[CtxKey.ROOT_PRI_KEY]) + subject = "job_ca.test-project" + now = datetime.datetime.now(datetime.timezone.utc) + _, expired_pub = generate_keys() + expired_cert = generate_cert( + Identity(subject), + Identity("test-project"), + root_key, + expired_pub, + ca=True, + ca_path_length=0, + not_valid_before=now - datetime.timedelta(days=2), + not_valid_after=now - datetime.timedelta(days=1), + ) + state[subject]["cert"] = serialize_cert(expired_cert).decode("ascii") + state_file.write_text(json.dumps(state)) + + second_ctx = _provision(workspace, CertBuilder(enable_job_ca=True)) + + assert not second_ctx.get(CtxKey.BUILD_ERROR) + new_cert = load_crt(str(_kit_dir(workspace, "server1") / ProvFileName.JOB_CA_CERT)) + assert new_cert.serial_number != expired_cert.serial_number + assert new_cert.not_valid_after_utc > now diff --git a/tests/unit_test/private/fed/client/fed_client_base_test.py b/tests/unit_test/private/fed/client/fed_client_base_test.py index 73c8037b49..ba05e02ad6 100644 --- a/tests/unit_test/private/fed/client/fed_client_base_test.py +++ b/tests/unit_test/private/fed/client/fed_client_base_test.py @@ -16,8 +16,23 @@ from types import SimpleNamespace from unittest.mock import MagicMock +from nvflare.apis.fl_constant import ConnPropKey, SecureTrainConst +from nvflare.fuel.f3.drivers.driver_params import DriverParams +from nvflare.private.fed.client import fed_client_base from nvflare.private.fed.client.fed_client_base import FederatedClientBase +_SITE_ARGS = { + SecureTrainConst.SSL_ROOT_CERT: "rootCA.pem", + SecureTrainConst.SSL_CERT: "client.crt", + SecureTrainConst.PRIVATE_KEY: "client.key", +} +# in a job process the configer has already made the job credential the ssl_cert / ssl_private_key +_JOB_ARGS = { + SecureTrainConst.SSL_ROOT_CERT: "rootCA.pem", + SecureTrainConst.SSL_CERT: "job.crt", + SecureTrainConst.PRIVATE_KEY: "job.key", +} + def _make_client(): client = FederatedClientBase.__new__(FederatedClientBase) @@ -32,6 +47,58 @@ def _make_client(): return client +def _create_cell_credentials(monkeypatch, job_id, client_args): + captured = {} + + class _FakeCell: + def __init__(self, **kwargs): + captured.update(kwargs) + + def start(self): + pass + + def stop(self): + pass + + conn_props = {ConnPropKey.CP_CONN_PROPS: {ConnPropKey.FQCN: "site-1", ConnPropKey.URL: "tcp://cp:1"}} + monkeypatch.setattr(fed_client_base, "Cell", _FakeCell) + monkeypatch.setattr(fed_client_base, "NetAgent", lambda cell: MagicMock()) + monkeypatch.setattr(fed_client_base.mpm, "add_cleanup_cb", lambda cb: None) + monkeypatch.setattr( + fed_client_base, "get_scope_property", lambda name, key, default=None: conn_props.get(key, default) + ) + + client = _make_client() + client.secure_train = True + client.client_args = dict(client_args) + client.args = SimpleNamespace(job_id=job_id) + client.communicator = MagicMock() + client.engine_create_timeout = 1.0 + client.cell_check_frequency = 0.001 + client.engine = MagicMock() + client.client_runner = MagicMock() + + client._create_cell("localhost:8002", "grpc") + return captured["credentials"] + + +def test_cp_cell_uses_site_credential(monkeypatch): + credentials = _create_cell_credentials(monkeypatch, None, _SITE_ARGS) + + assert credentials[DriverParams.CLIENT_CERT.value] == "client.crt" + assert credentials[DriverParams.CLIENT_KEY.value] == "client.key" + assert DriverParams.SERVER_CERT.value not in credentials + + +def test_cj_cell_uses_job_credential_in_both_tls_roles(monkeypatch): + credentials = _create_cell_credentials(monkeypatch, "job-1", _JOB_ARGS) + + assert credentials[DriverParams.CLIENT_CERT.value] == "job.crt" + assert credentials[DriverParams.CLIENT_KEY.value] == "job.key" + assert credentials[DriverParams.SERVER_CERT.value] == "job.crt" + assert credentials[DriverParams.SERVER_KEY.value] == "job.key" + + def test_send_request_before_shutdown_skips_after_close(): client = _make_client() reply = MagicMock() diff --git a/tests/unit_test/private/fed/client/test_training_cmds_sig_verify.py b/tests/unit_test/private/fed/client/test_training_cmds_sig_verify.py index afb08e70b1..b85015fd8c 100644 --- a/tests/unit_test/private/fed/client/test_training_cmds_sig_verify.py +++ b/tests/unit_test/private/fed/client/test_training_cmds_sig_verify.py @@ -23,6 +23,10 @@ from unittest.mock import MagicMock, patch from zipfile import ZipFile +import pytest + +from nvflare.apis.fl_constant import FLContextKey +from nvflare.apis.fl_context import FLContext from nvflare.lighter.tool_consts import NVFLARE_SIG_FILE from nvflare.private.admin_defs import Message from nvflare.private.defs import RequestHeader @@ -37,10 +41,11 @@ class _StubEngine(ClientEngineInternalSpec): """Minimal concrete implementation of ClientEngineInternalSpec for tests.""" - def __init__(self, workspace_dir="/fake/workspace", client_name="site-1", deploy_result=""): + def __init__(self, workspace_dir="/fake/workspace", client_name="site-1", deploy_result="", secure_mode=False): self._client_name = client_name self._workspace_dir = workspace_dir self._deploy_result = deploy_result + self._secure_mode = secure_mode self._deploy_calls = [] # args mock @@ -96,7 +101,9 @@ def configure_job_log(self, job_id, log_config): # Optional methods that may be needed by parent classes def new_context(self): - return MagicMock() + fl_ctx = FLContext() + fl_ctx.set_prop(FLContextKey.SECURE_MODE, self._secure_mode, private=True, sticky=True) + return fl_ctx def fire_event(self, event_type, fl_ctx): pass @@ -119,7 +126,7 @@ def _make_app_zip(signed=True) -> bytes: return zip_bytes.getvalue() -def _make_request(job_id="job-1", app_name="test-app", job_meta=None, body=None): +def _make_request(job_id="job-1", app_name="test-app", job_meta=None, body=None, job_cert=None): """Build a minimal deploy Message. job_meta defaults to a sentinel dict with one key so it passes the "if not job_meta" check. @@ -137,6 +144,7 @@ def get_header(key, default=None): RequestHeader.JOB_ID: job_id, RequestHeader.APP_NAME: app_name, RequestHeader.JOB_META: job_meta, + RequestHeader.JOB_CERT: job_cert, } return mapping.get(key, default) @@ -196,6 +204,50 @@ def test_signed_valid_calls_deploy_app_with_verified_bytes(self, tmp_path): assert engine._deploy_calls[0][4] == req.body +# --------------------------------------------------------------------------- +# Job credential — secure mode never deploys a job onto site certificates +# --------------------------------------------------------------------------- + + +class TestJobCredential: + @pytest.mark.parametrize("job_cert", [None, {"cert": "only-cert"}, "garbage"]) + def test_secure_deploy_without_valid_job_credential_is_rejected(self, tmp_path, job_cert): + req = _make_request(job_cert=job_cert) + engine = _StubEngine(workspace_dir=str(tmp_path), secure_mode=True) + _write_root_ca(tmp_path) + + reply = _run_process(req, engine, str(tmp_path)) + + assert "no valid job credential" in reply.body + assert engine._deploy_calls == [] + + def test_secure_deploy_writes_job_credential_after_deploy(self, tmp_path): + req = _make_request(job_cert={"cert": "CERT-PEM", "key": "KEY-PEM"}) + engine = _StubEngine(workspace_dir=str(tmp_path), secure_mode=True) + _write_root_ca(tmp_path) + + with ( + patch("nvflare.private.fed.client.training_cmds.verify_folder_signature", return_value=True), + patch("nvflare.private.fed.client.training_cmds.write_job_cert") as write_mock, + ): + reply = _run_process(req, engine, str(tmp_path)) + + assert "deployed" in reply.body + assert len(engine._deploy_calls) == 1 + write_mock.assert_called_once() + assert write_mock.call_args.args[1:] == (b"CERT-PEM", b"KEY-PEM") + + def test_non_secure_deploy_needs_no_job_credential(self, tmp_path): + req = _make_request() + engine = _StubEngine(workspace_dir=str(tmp_path)) + _write_root_ca(tmp_path) + + with patch("nvflare.private.fed.client.training_cmds.verify_folder_signature", return_value=True): + reply = _run_process(req, engine, str(tmp_path)) + + assert "deployed" in reply.body + + # --------------------------------------------------------------------------- # Signed, invalid (tampered) — should return error_reply # --------------------------------------------------------------------------- diff --git a/tests/unit_test/private/fed/server/job_runner_deploy_test.py b/tests/unit_test/private/fed/server/job_runner_deploy_test.py index 77841d1743..4cd48429a1 100644 --- a/tests/unit_test/private/fed/server/job_runner_deploy_test.py +++ b/tests/unit_test/private/fed/server/job_runner_deploy_test.py @@ -108,6 +108,20 @@ def _build_fl_ctx(token_to_reply: dict, job_id="job-1", min_sites=None, required return runner, fl_ctx, engine, job, sites +def test_secure_deploy_fails_without_job_ca(tmp_path): + runner, fl_ctx, engine, job, sites = _build_fl_ctx({"tok-1": _ok_reply()}) + (tmp_path / "startup").mkdir() + (tmp_path / "local").mkdir() + runner.workspace_root = str(tmp_path) + deploy_detail = fl_ctx.get_prop.return_value + fl_ctx.get_prop.side_effect = lambda key, default=None: True if key == FLContextKey.SECURE_MODE else deploy_detail + + with pytest.raises(RuntimeError, match="no job CA"): + runner._deploy_job(job, sites, fl_ctx) + + engine.server.admin_server.send_requests_and_get_reply_dict.assert_not_called() + + # --------------------------------------------------------------------------- # Deployment timeout classified as failure # --------------------------------------------------------------------------- diff --git a/tests/unit_test/private/fed/test_fed_utils.py b/tests/unit_test/private/fed/test_fed_utils.py index 1d8bbbb9c6..44ad850572 100644 --- a/tests/unit_test/private/fed/test_fed_utils.py +++ b/tests/unit_test/private/fed/test_fed_utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys from unittest.mock import MagicMock, patch @@ -19,6 +20,7 @@ from nvflare.fuel.sec.security_content_service import SecurityContentService from nvflare.private.fed.utils.fed_utils import security_init, security_init_for_job +from nvflare.private.fed.utils.job_cert_utils import write_job_cert @pytest.fixture(autouse=True) @@ -37,6 +39,10 @@ def _make_workspace(startup_dir): ws.get_audit_file_path.return_value = startup_dir + "/audit.log" ws.get_authorization_file_path.return_value = None ws.get_study_registry_file_path.return_value = startup_dir + "/study_registry.json" + # secure job processes refuse to start without their per-job credential + run_dir = os.path.join(os.path.dirname(startup_dir), "job-1") + write_job_cert(run_dir, b"job-cert", b"job-key") + ws.get_run_dir.return_value = run_dir return ws diff --git a/tests/unit_test/private/fed/utils/fed_utils_test.py b/tests/unit_test/private/fed/utils/fed_utils_test.py index e5515274c9..0205008b63 100644 --- a/tests/unit_test/private/fed/utils/fed_utils_test.py +++ b/tests/unit_test/private/fed/utils/fed_utils_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock, patch @@ -19,15 +20,20 @@ from nvflare.apis.app_validation import AppValidationKey from nvflare.apis.fl_constant import ConfigVarName +from nvflare.apis.workspace import Workspace +from nvflare.fuel.sec.security_content_service import LoadResult, SecurityContentService from nvflare.fuel.utils import fobs from nvflare.fuel.utils.fobs import Decomposer from nvflare.fuel.utils.fobs.datum import DatumManager from nvflare.fuel.utils.fobs.fobs import register_custom_folder +from nvflare.private.fed.utils import fed_utils from nvflare.private.fed.utils.fed_utils import ( create_job_processing_context_properties, custom_fobs_initialize, extract_participants, + security_init_for_job, ) +from nvflare.private.fed.utils.job_cert_utils import JobCertError class ExampleTestClass: @@ -150,3 +156,27 @@ def test_job_decomposers_ignored_when_meta_unavailable(self, mock_exists, mock_g custom_decomposer_dir = os.path.join("job", "custom", ConfigVarName.DECOMPOSER_MODULE) assert custom_decomposer_dir not in registered_dirs + + +def test_security_init_for_job_refuses_secure_job_without_credential(tmp_path): + (tmp_path / "startup").mkdir() + (tmp_path / "local").mkdir() + workspace = Workspace(str(tmp_path), site_name="site-1") + + with pytest.raises(JobCertError, match="no job credential"): + security_init_for_job(True, workspace, "client", "job-1") + + +def test_check_secure_content_can_skip_site_private_key(monkeypatch): + config = {"client": {"ssl_cert": "client.crt", "ssl_private_key": "client.key", "ssl_root_cert": "rootCA.pem"}} + + def load_content(cls, name): + return b"", LoadResult.NO_SUCH_CONTENT if name == "client.key" else LoadResult.OK + + monkeypatch.setattr(SecurityContentService, "load_json", classmethod(lambda cls, name: (config, LoadResult.OK))) + monkeypatch.setattr(SecurityContentService, "load_content", classmethod(load_content)) + monkeypatch.setattr(SecurityContentService, "check_json_files", classmethod(lambda cls, patterns: [])) + monkeypatch.setattr(SecurityContentService, "security_content_manager", SimpleNamespace(signature={})) + + assert fed_utils._check_secure_content("client") == ["client.key"] + assert fed_utils._check_secure_content("client", check_private_key=False) == [] diff --git a/tests/unit_test/private/fed/utils/identity_utils_test.py b/tests/unit_test/private/fed/utils/identity_utils_test.py index 19b4564f0a..f09d82f96b 100644 --- a/tests/unit_test/private/fed/utils/identity_utils_test.py +++ b/tests/unit_test/private/fed/utils/identity_utils_test.py @@ -16,9 +16,11 @@ from cryptography import x509 from cryptography.x509.oid import ExtendedKeyUsageOID +from nvflare.fuel.f3.drivers.net_utils import JOB_ID_EXTENSION_OID from nvflare.lighter.impl.cert import serialize_cert from nvflare.lighter.utils import Identity, generate_cert, generate_keys, sign_content from nvflare.private.fed.utils.identity_utils import IdentityVerifier, InvalidAsserterCert, get_parent_site_name +from nvflare.private.fed.utils.job_cert_utils import JOB_CA_MARKER_OID class TestIdentityUtils: @@ -131,6 +133,126 @@ def test_identity_verifier_accepts_expected_extended_key_usage(tmp_path): ) +def test_identity_verifier_rejects_job_scoped_cert_chain(tmp_path): + # a leaked job cert (leaf + job CA chaining to root, CN=site) must not be + # usable to register as the site — the rogue-CP scenario + root_key, root_pub_key = generate_keys() + root_cert = generate_cert( + subject=Identity("root", "nvidia"), + issuer=Identity("root", "nvidia"), + signing_pri_key=root_key, + subject_pub_key=root_pub_key, + ca=True, + ) + job_ca_key, job_ca_pub_key = generate_keys() + job_ca_cert = generate_cert( + subject=Identity("job_ca.test", "nvidia"), + issuer=Identity("root", "nvidia"), + signing_pri_key=root_key, + subject_pub_key=job_ca_pub_key, + ca=True, + ca_path_length=0, + ) + leaf_key, leaf_pub_key = generate_keys() + job_id_ext = x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, b"job-123") + leaf_cert = generate_cert( + subject=Identity("client", "nvidia"), + issuer=Identity("job_ca.test", "nvidia"), + signing_pri_key=job_ca_key, + subject_pub_key=leaf_pub_key, + extra_extensions=[(job_id_ext, False)], + ) + root_cert_path = tmp_path / "root.crt" + root_cert_path.write_bytes(serialize_cert(root_cert)) + verifier = IdentityVerifier(str(root_cert_path)) + signature = sign_content("client" + "nonce", leaf_key, return_str=False) + + with pytest.raises(InvalidAsserterCert, match="job-scoped"): + verifier.verify_common_name( + "client", + "nonce", + leaf_cert, + signature, + intermediate_certs=[job_ca_cert], + expected_eku=ExtendedKeyUsageOID.CLIENT_AUTH, + ) + + +def test_identity_verifier_rejects_leaf_minted_by_job_ca_without_extension(tmp_path): + # stolen job-CA-key attack: mint a clean site cert with NO job-id extension; + # rejection must key on the root-signed marker in the presented chain + root_key, root_pub_key = generate_keys() + root_cert = generate_cert( + subject=Identity("root", "nvidia"), + issuer=Identity("root", "nvidia"), + signing_pri_key=root_key, + subject_pub_key=root_pub_key, + ca=True, + ) + marker = x509.UnrecognizedExtension(JOB_CA_MARKER_OID, b"job_ca") + job_ca_key, job_ca_pub_key = generate_keys() + job_ca_cert = generate_cert( + subject=Identity("job_ca.test", "nvidia"), + issuer=Identity("root", "nvidia"), + signing_pri_key=root_key, + subject_pub_key=job_ca_pub_key, + ca=True, + ca_path_length=0, + extra_extensions=[(marker, False)], + ) + leaf_key, leaf_pub_key = generate_keys() + leaf_cert = generate_cert( + subject=Identity("client", "nvidia"), + issuer=Identity("job_ca.test", "nvidia"), + signing_pri_key=job_ca_key, + subject_pub_key=leaf_pub_key, + ) + root_cert_path = tmp_path / "root.crt" + root_cert_path.write_bytes(serialize_cert(root_cert)) + verifier = IdentityVerifier(str(root_cert_path)) + signature = sign_content("client" + "nonce", leaf_key, return_str=False) + + with pytest.raises(InvalidAsserterCert, match="issued by the job CA"): + verifier.verify_common_name( + "client", + "nonce", + leaf_cert, + signature, + intermediate_certs=[job_ca_cert], + expected_eku=ExtendedKeyUsageOID.CLIENT_AUTH, + ) + + +def test_identity_verifier_rejects_job_extension_with_non_utf8_value(tmp_path): + # rejection must not depend on decoding the extension value + job_id_ext = x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, b"\xff\xfe\xfd") + root_cert, _root_key, client_cert, client_key = _make_root_and_client_certs(extra_extensions=[(job_id_ext, False)]) + root_cert_path = tmp_path / "root.crt" + root_cert_path.write_bytes(serialize_cert(root_cert)) + verifier = IdentityVerifier(str(root_cert_path)) + signature = sign_content("client" + "nonce", client_key, return_str=False) + + with pytest.raises(InvalidAsserterCert, match="job-scoped"): + verifier.verify_common_name( + "client", "nonce", client_cert, signature, expected_eku=ExtendedKeyUsageOID.CLIENT_AUTH + ) + + +def test_identity_verifier_rejects_job_extension_even_when_root_issued(tmp_path): + # the rejection is keyed on the job-id extension, not the issuer + job_id_ext = x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, b"job-123") + root_cert, _root_key, client_cert, client_key = _make_root_and_client_certs(extra_extensions=[(job_id_ext, False)]) + root_cert_path = tmp_path / "root.crt" + root_cert_path.write_bytes(serialize_cert(root_cert)) + verifier = IdentityVerifier(str(root_cert_path)) + signature = sign_content("client" + "nonce", client_key, return_str=False) + + with pytest.raises(InvalidAsserterCert, match="job-scoped"): + verifier.verify_common_name( + "client", "nonce", client_cert, signature, expected_eku=ExtendedKeyUsageOID.CLIENT_AUTH + ) + + def test_identity_verifier_wraps_invalid_cert_chain(tmp_path): root_cert, _root_key, _client_cert, _client_key = _make_root_and_client_certs() other_root_key, _other_root_pub_key = generate_keys() diff --git a/tests/unit_test/private/fed/utils/job_cert_utils_test.py b/tests/unit_test/private/fed/utils/job_cert_utils_test.py new file mode 100644 index 0000000000..b41feff6a0 --- /dev/null +++ b/tests/unit_test/private/fed/utils/job_cert_utils_test.py @@ -0,0 +1,241 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import stat + +import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.x509.oid import NameOID + +from nvflare.apis.fl_constant import SecureTrainConst +from nvflare.fuel.f3.cellnet.cell_cipher import SimpleCellCipher +from nvflare.fuel.f3.drivers.net_utils import get_cert_job_id +from nvflare.lighter.constants import ProvFileName +from nvflare.lighter.utils import ( + Identity, + generate_cert, + generate_keys, + serialize_cert, + serialize_pri_key, + verify_cert_chain, +) +from nvflare.private.fed.utils.job_cert_utils import ( + JOB_CA_MARKER_OID, + JOB_CERT_BACKDATE, + JOB_CERT_FILE_NAME, + JOB_CERT_VALID_DAYS, + JOB_KEY_FILE_NAME, + JobCertError, + apply_job_cert_config, + find_job_cert, + has_job_ca_marker, + job_startup_files, + load_job_cert_issuer, + pack_job_cert_header, + read_job_cert, + stage_job_startup_dir, + unpack_job_cert_header, + write_job_cert, +) + + +def _write_job_ca(startup_dir, ca_lifetime=datetime.timedelta(days=360), expired=False): + os.makedirs(startup_dir, exist_ok=True) + root_key, root_pub = generate_keys() + root_cert = generate_cert(Identity("root"), Identity("root"), root_key, root_pub, ca=True) + + now = datetime.datetime.now(datetime.timezone.utc) + if expired: + not_valid_before = now - datetime.timedelta(days=2) + not_valid_after = now - datetime.timedelta(days=1) + else: + not_valid_before = now + not_valid_after = now + ca_lifetime + + ca_key, ca_pub = generate_keys() + marker = x509.UnrecognizedExtension(JOB_CA_MARKER_OID, b"job_ca") + ca_cert = generate_cert( + Identity("job_ca.test"), + Identity("root"), + root_key, + ca_pub, + ca=True, + ca_path_length=0, + not_valid_before=not_valid_before, + not_valid_after=not_valid_after, + extra_extensions=[(marker, False)], + ) + + with open(os.path.join(startup_dir, ProvFileName.JOB_CA_CERT), "wb") as f: + f.write(serialize_cert(ca_cert)) + with open(os.path.join(startup_dir, ProvFileName.JOB_CA_KEY), "wb") as f: + f.write(serialize_pri_key(ca_key)) + return root_cert, ca_cert + + +def test_issuer_requires_job_ca(tmp_path): + with pytest.raises(JobCertError, match="no job CA"): + load_job_cert_issuer(str(tmp_path)) + + +def test_issuer_rejects_expired_job_ca(tmp_path): + _write_job_ca(str(tmp_path), expired=True) + with pytest.raises(JobCertError, match="expires at"): + load_job_cert_issuer(str(tmp_path)) + + +def test_issuer_rejects_job_ca_near_expiry(tmp_path): + _write_job_ca(str(tmp_path), ca_lifetime=datetime.timedelta(minutes=30)) + with pytest.raises(JobCertError, match="expires at"): + load_job_cert_issuer(str(tmp_path)) + + +def test_issued_cert_chains_to_root_and_carries_job_id(tmp_path): + root_cert, ca_cert = _write_job_ca(str(tmp_path)) + issuer = load_job_cert_issuer(str(tmp_path)) + assert issuer is not None + + cert_pem, key_pem = issuer.issue("site-1", "job-123") + + chain = x509.load_pem_x509_certificates(cert_pem) + assert len(chain) == 2 + leaf, intermediate = chain + assert intermediate == ca_cert + verify_cert_chain(leaf_cert=leaf, intermediate_certs=[intermediate], root_ca_cert=root_cert) + assert leaf.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value == "site-1" + assert get_cert_job_id(leaf) == "job-123" + assert has_job_ca_marker(intermediate) and not has_job_ca_marker(leaf) + expected_lifetime = datetime.timedelta(days=JOB_CERT_VALID_DAYS) + JOB_CERT_BACKDATE + assert leaf.not_valid_after_utc - leaf.not_valid_before_utc == expected_lifetime + assert b"PRIVATE KEY" in key_pem + + +def test_issued_cert_validity_clamped_to_job_ca(tmp_path): + _, ca_cert = _write_job_ca(str(tmp_path), ca_lifetime=datetime.timedelta(days=1)) + issuer = load_job_cert_issuer(str(tmp_path)) + + cert_pem, _ = issuer.issue("site-1", "job-123") + + leaf = x509.load_pem_x509_certificates(cert_pem)[0] + assert leaf.not_valid_after_utc == ca_cert.not_valid_after_utc.replace(microsecond=0) + + +def test_issue_honors_valid_days(tmp_path): + _write_job_ca(str(tmp_path)) + issuer = load_job_cert_issuer(str(tmp_path)) + + cert_pem, _ = issuer.issue("site-1", "job-123", valid_days=3) + + leaf = x509.load_pem_x509_certificates(cert_pem)[0] + assert leaf.not_valid_after_utc - leaf.not_valid_before_utc == datetime.timedelta(days=3) + JOB_CERT_BACKDATE + + +def test_issue_many_issues_one_credential_per_site(tmp_path): + _write_job_ca(str(tmp_path)) + issuer = load_job_cert_issuer(str(tmp_path)) + + creds = issuer.issue_many(["site-1", "site-2"], "job-123") + + assert set(creds) == {"site-1", "site-2"} + leaves = {name: x509.load_pem_x509_certificates(cert_pem)[0] for name, (cert_pem, _) in creds.items()} + assert {leaf.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value for leaf in leaves.values()} == set(creds) + assert leaves["site-1"].public_key() != leaves["site-2"].public_key() + assert issuer.issue_many([], "job-123") == {} + + +def test_pack_unpack_job_cert_header_round_trip(): + header = pack_job_cert_header(b"cert-bytes", b"key-bytes") + assert unpack_job_cert_header(header) == (b"cert-bytes", b"key-bytes") + + +@pytest.mark.parametrize( + "header", [None, "not-a-dict", b"bytes", 5, {}, {"cert": "x"}, {"key": "y"}, {"cert": "", "key": "k"}] +) +def test_unpack_job_cert_header_rejects_malformed(header): + assert unpack_job_cert_header(header) is None + + +def test_job_id_absent_from_site_cert(): + root_key, root_pub = generate_keys() + root_cert = generate_cert(Identity("root"), Identity("root"), root_key, root_pub, ca=True) + assert get_cert_job_id(root_cert) is None + + +def test_write_find_read_job_cert(tmp_path): + run_dir = str(tmp_path / "run_1") + assert find_job_cert(run_dir) is None + assert read_job_cert(run_dir) is None + + write_job_cert(run_dir, b"cert-1", b"key-1") + write_job_cert(run_dir, b"cert-2", b"key-2") + + cert_path, key_path = find_job_cert(run_dir) + assert cert_path.endswith(JOB_CERT_FILE_NAME) and key_path.endswith(JOB_KEY_FILE_NAME) + assert read_job_cert(run_dir) == (b"cert-2", b"key-2") + assert stat.S_IMODE(os.stat(key_path).st_mode) == 0o600 + + +def test_apply_job_cert_config_replaces_site_credential(tmp_path): + run_dir = str(tmp_path / "run_1") + site_only = {SecureTrainConst.SSL_CERT: "site.crt", SecureTrainConst.PRIVATE_KEY: "site.key"} + config = dict(site_only) + + apply_job_cert_config(config, run_dir) + assert config == site_only + + write_job_cert(run_dir, b"c", b"k") + apply_job_cert_config(config, run_dir) + cert_path, key_path = find_job_cert(run_dir) + assert config == {SecureTrainConst.SSL_CERT: cert_path, SecureTrainConst.PRIVATE_KEY: key_path} + + +def test_job_startup_files_and_staging_exclude_private_keys(tmp_path): + startup = tmp_path / "startup" + startup.mkdir() + for name in ("rootCA.pem", "client.crt", "client.key", "fed_client.json", "job_ca.key", "start.sh"): + (startup / name).write_text(name) + (startup / "subdir").mkdir() + + assert job_startup_files(str(startup)) == ["client.crt", "fed_client.json", "rootCA.pem", "start.sh"] + + staged = stage_job_startup_dir(str(startup), str(tmp_path / "job" / "startup")) + + assert sorted(os.listdir(staged)) == ["client.crt", "fed_client.json", "rootCA.pem", "start.sh"] + assert stat.S_IMODE(os.stat(staged).st_mode) == 0o700 + assert (tmp_path / "job" / "startup" / "rootCA.pem").read_text() == "rootCA.pem" + + +def test_cell_cipher_works_with_job_cert_chains(tmp_path): + root_cert, _ = _write_job_ca(str(tmp_path)) + issuer = load_job_cert_issuer(str(tmp_path)) + + sj_cert_pem, sj_key_pem = issuer.issue("server", "job-123") + cj_cert_pem, cj_key_pem = issuer.issue("site-1", "job-123") + + sj_cipher = SimpleCellCipher( + root_cert, + serialization.load_pem_private_key(sj_key_pem, password=None), + x509.load_pem_x509_certificates(sj_cert_pem), + ) + cj_cipher = SimpleCellCipher( + root_cert, + serialization.load_pem_private_key(cj_key_pem, password=None), + x509.load_pem_x509_certificates(cj_cert_pem), + ) + + cipher_text = sj_cipher.encrypt(b"task data", x509.load_pem_x509_certificates(cj_cert_pem)) + assert cj_cipher.decrypt(cipher_text, x509.load_pem_x509_certificates(sj_cert_pem)) == b"task data" diff --git a/tests/unit_test/tool/package/package_commands_test.py b/tests/unit_test/tool/package/package_commands_test.py index ad9bc2a35a..f98ab32422 100644 --- a/tests/unit_test/tool/package/package_commands_test.py +++ b/tests/unit_test/tool/package/package_commands_test.py @@ -30,7 +30,7 @@ import yaml from cryptography.hazmat.primitives import serialization -from nvflare.lighter.constants import CtxKey, PropKey +from nvflare.lighter.constants import CtxKey, PropKey, ProvFileName from nvflare.lighter.entity import Project from nvflare.lighter.impl.cert import CertBuilder from nvflare.lighter.impl.signature import SignatureBuilder @@ -2301,6 +2301,10 @@ def _capture(result): # Cert-related keys inside JSON configs — content differs by design (different CAs). _CERT_VALUE_KEYS = frozenset(["ssl_private_key", "ssl_cert", "ssl_root_cert", "client_key", "client_cert", "ca_cert"]) +# The provisioner signs a job-issuing CA into the server kit; ``nvflare package`` never holds +# the root key, so it cannot produce one. +_JOB_CA_FILES = frozenset(os.path.join("startup", f) for f in (ProvFileName.JOB_CA_CERT, ProvFileName.JOB_CA_KEY)) + def _provision_project(workspace: str) -> str: """Run the standard provisioner and return the prod_NN directory path. @@ -2423,9 +2427,13 @@ def _compare_kit_dirs(prov_dir: str, pkg_dir: str, participant_type: str) -> lis assert os.path.isdir(os.path.join(prov_dir, subdir)), f"prov missing subdir: {subdir}" assert os.path.isdir(os.path.join(pkg_dir, subdir)), f"pkg missing subdir: {subdir}" - # 3. File sets must match exactly. + # 3. File sets must match exactly, except for the server kit's job CA. prov_files = set(_file_tree(prov_dir).keys()) pkg_files = set(_file_tree(pkg_dir).keys()) + if participant_type == "server": + for rel in sorted(_JOB_CA_FILES & prov_files): + intentional_diffs.append(f"job-ca-prov-only: {rel}") + prov_files -= _JOB_CA_FILES prov_only = prov_files - pkg_files pkg_only = pkg_files - prov_files assert not prov_only, f"Files only in provision output: {sorted(prov_only)}" @@ -2533,6 +2541,8 @@ class TestProvisionPackageParity: "cert-value-skipped:", # CertBuilder does not chmod key files; PrebuiltCertBuilder always sets 0o600. "key-perm-prov-", + # Only the provisioner has the root key needed to sign the job CA. + "job-ca-prov-only:", ) @staticmethod From 194bd75f934d65c835651c758621db547aa07338 Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Thu, 3 Sep 2026 14:58:03 -0700 Subject: [PATCH 02/10] fix(security): accept job-bound certs on workspace-transfer bootstrap cells The K8s workspace-transfer bootstrap cell is named .ws_transfer_, so the job id is embedded in a segment instead of being one and the job-cert binding rejected the job pod's own credential at the parent ("claimed endpoint ... is not part of that job"), failing every secure K8s job. FQCN.belongs_to_job now defines a job's cells as the job cell and its descendants plus auxiliary cells named _; the binding check and the bootstrap FQCN builder both use it. Adds a cross-process mTLS regression test that connects a real bootstrap cell to a real parent with a job-bound cert and verifies another job's cert is still rejected. --- .../no_shared_pvc_k8s_workspace_design.md | 3 +- docs/design/per_job_certs_design.md | 7 +- .../job_launcher/workspace_cell_transfer.py | 4 +- nvflare/fuel/f3/cellnet/fqcn.py | 15 ++ nvflare/fuel/f3/cellnet/identity.py | 2 +- .../workspace_cell_transfer_test.py | 10 + .../workspace_transfer_bootstrap_mtls_test.py | 185 ++++++++++++++++++ .../fuel/f3/cellnet/identity_binding_test.py | 37 +++- 8 files changed, 253 insertions(+), 10 deletions(-) create mode 100644 tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py diff --git a/docs/design/no_shared_pvc_k8s_workspace_design.md b/docs/design/no_shared_pvc_k8s_workspace_design.md index 2796521aca..2442a7bf2a 100644 --- a/docs/design/no_shared_pvc_k8s_workspace_design.md +++ b/docs/design/no_shared_pvc_k8s_workspace_design.md @@ -123,7 +123,8 @@ The download sequence is: `NVFL_WORKSPACE_TRANSFER_TOKEN`. 2. It creates a short-lived bootstrap child cell using the startup kit and the existing parent connection settings. The bootstrap FQCN is - `.ws_transfer_`. When the child process is a client + `.ws_transfer_`, which `FQCN.belongs_to_job` recognizes as + part of the job so the job-bound certificate is accepted. When the child process is a client worker, the bootstrap cell reuses that worker's `client_name`, auth token, token signature, and `ssid`. When the child process is a server runner, the bootstrap cell uses the same server-job auth identity as the main runner. diff --git a/docs/design/per_job_certs_design.md b/docs/design/per_job_certs_design.md index e60ad54f77..b8914c045b 100644 --- a/docs/design/per_job_certs_design.md +++ b/docs/design/per_job_certs_design.md @@ -215,8 +215,11 @@ stops one job's credential from acting as another job's cell: - Every TLS driver exposes the peer certificate's job-ID extension as the `PEER_JOB_ID` connection property next to `PEER_CN`. - `CellIdentityResolver.require_match()` rejects a peer whose certificate is - bound to job X unless the FQCN it claims contains the segment X - (`site-1.X`, `server.X`, and their descendants). The check runs at the + bound to job X unless the FQCN it claims belongs to that job + (`FQCN.belongs_to_job`): the job cell `.X` and its descendants, or an + auxiliary job cell named `_X` directly under the site, such as the + Kubernetes workspace-transfer bootstrap cell `server.ws_transfer_X`, which + authenticates with the job credential before the job cell exists. The check runs at the connection handshake (`ConnManager`) and again on the certificate exchanged for message-level crypto (`CredentialManager`), which is the certificate later used to decrypt that peer's messages. diff --git a/nvflare/app_opt/job_launcher/workspace_cell_transfer.py b/nvflare/app_opt/job_launcher/workspace_cell_transfer.py index 8fcd407ec9..862db11755 100644 --- a/nvflare/app_opt/job_launcher/workspace_cell_transfer.py +++ b/nvflare/app_opt/job_launcher/workspace_cell_transfer.py @@ -80,7 +80,7 @@ BOOTSTRAP_CONNECT_TIMEOUT = 30.0 BOOTSTRAP_CONNECT_POLL_INTERVAL = 0.1 -_BOOTSTRAP_CELL_PREFIX = "ws_transfer_" +_BOOTSTRAP_CELL_NAME = "ws_transfer" _DEFAULT_WORKSPACE_DOWNLOAD_EXCLUDES = frozenset({"local/study_data.yaml", "local/study_runtime.yaml"}) _RESOURCE_CONFIG_NAMES = ("resources.json", "resources.json.default") _K8S_LAUNCHER_COMPONENT_ID = "k8s_launcher" @@ -259,7 +259,7 @@ def _hash_file(path: str) -> str: def make_workspace_transfer_fqcn(owner_fqcn: str, job_id: str) -> str: - return FQCN.join([owner_fqcn, f"{_BOOTSTRAP_CELL_PREFIX}{job_id}"]) + return FQCN.join([owner_fqcn, FQCN.job_aux_name(_BOOTSTRAP_CELL_NAME, job_id)]) def _cleanup_files(paths) -> None: diff --git a/nvflare/fuel/f3/cellnet/fqcn.py b/nvflare/fuel/f3/cellnet/fqcn.py index 762928acb6..80f82eac13 100644 --- a/nvflare/fuel/f3/cellnet/fqcn.py +++ b/nvflare/fuel/f3/cellnet/fqcn.py @@ -17,6 +17,21 @@ class FQCN(FQN): VALID_PATTERN = "^[A-Za-z0-9_.~-]*$" + # A job's cells are the job cell (.) and its descendants, plus auxiliary cells + # named _ directly under the site (e.g. the workspace-transfer bootstrap cell). + JOB_AUX_SEPARATOR = "_" + + @staticmethod + def job_aux_name(name: str, job_id: str) -> str: + return f"{name}{FQCN.JOB_AUX_SEPARATOR}{job_id}" + + @staticmethod + def belongs_to_job(fqcn: str, job_id: str) -> bool: + if not job_id: + return False + aux_suffix = FQCN.JOB_AUX_SEPARATOR + job_id + return any(seg == job_id or seg.endswith(aux_suffix) for seg in FQCN.split(FQCN.normalize(fqcn))) + # A network Attach trainer connects beneath the stable site CP and authenticates # with that physical parent's provisioned identity. diff --git a/nvflare/fuel/f3/cellnet/identity.py b/nvflare/fuel/f3/cellnet/identity.py index cd7a2d6ceb..dd2ceaea3f 100644 --- a/nvflare/fuel/f3/cellnet/identity.py +++ b/nvflare/fuel/f3/cellnet/identity.py @@ -165,7 +165,7 @@ def require_match(self, fqcn: str, peer_cn: str, peer_desc: str, peer_job_id: Op raise ValueError(f"{peer_desc} does not have an authenticated mTLS peer common name") # A per-job certificate may only authenticate cells of that job. - if peer_job_id is not None and peer_job_id not in FQCN.split(FQCN.normalize(fqcn)): + if peer_job_id is not None and not FQCN.belongs_to_job(fqcn, peer_job_id): raise ValueError( f"{peer_desc} authenticated with a certificate bound to job '{peer_job_id}' " f"but claimed endpoint '{fqcn}' is not part of that job" diff --git a/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py b/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py index 829df230d6..5b17c63ce8 100644 --- a/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py +++ b/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py @@ -42,6 +42,7 @@ ) from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.fqcn import FQCN +from nvflare.fuel.f3.cellnet.identity import CellIdentityResolver from nvflare.fuel.f3.cellnet.utils import make_reply, new_cell_message from nvflare.fuel.f3.drivers.driver_params import DriverParams @@ -111,6 +112,15 @@ def stop(self): class TestGetOrCreate: + @pytest.mark.parametrize("owner_fqcn, owner_cn", [("server", "server"), ("site-1", "site-1")]) + def test_bootstrap_fqcn_is_accepted_by_job_cert_binding(self, owner_fqcn, owner_cn): + resolver = CellIdentityResolver(local_fqcn=owner_fqcn, prefix_identity_map={owner_fqcn: owner_cn}) + fqcn = make_workspace_transfer_fqcn(owner_fqcn, JOB_ID) + + resolver.require_match(fqcn, owner_cn, "bootstrap", peer_job_id=JOB_ID) + with pytest.raises(ValueError, match="bound to job"): + resolver.require_match(fqcn, owner_cn, "bootstrap", peer_job_id="other-job") + def test_returns_same_manager_for_same_cell(self): owner_cell = _FakeCell(fqcn="site-1.parent") first = WorkspaceTransferManager.get_or_create(owner_cell) diff --git a/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py b/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py new file mode 100644 index 0000000000..0aeb7bd2f2 --- /dev/null +++ b/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py @@ -0,0 +1,185 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Live mTLS regression: the workspace-transfer bootstrap cell authenticates to its parent with a job-bound cert.""" + +import multiprocessing as mp +import os +import socket +import time +import traceback +import uuid + +import pytest +from cryptography import x509 + +from nvflare.app_opt.job_launcher.workspace_cell_transfer import make_workspace_transfer_fqcn +from nvflare.fuel.f3.cellnet.cell import Cell +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode +from nvflare.fuel.f3.drivers.driver_params import DriverParams +from nvflare.fuel.f3.message import Message +from nvflare.lighter.constants import CertExtensionOID +from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert, serialize_pri_key +from nvflare.private.fed.utils.job_cert_utils import JobCertIssuer + +_CHANNEL = "ws_bootstrap_test" +_TOPIC = "echo" +_CONNECT_TIMEOUT = 10.0 +_REJECT_WAIT = 3.0 +_JOB_ID = str(uuid.uuid4()) +_OTHER_JOB_ID = str(uuid.uuid4()) + + +def _write_pki(out_dir: str) -> dict: + root_key, root_pub = generate_keys() + root_cert = generate_cert(Identity("rootCA"), Identity("rootCA"), root_key, root_pub, ca=True) + srv_key, srv_pub = generate_keys() + srv_cert = generate_cert(Identity("server"), Identity("rootCA"), root_key, srv_pub, server_default_host="localhost") + jca_key, jca_pub = generate_keys() + marker = x509.UnrecognizedExtension(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER), b"job_ca") + jca_cert = generate_cert( + Identity("job_ca"), + Identity("rootCA"), + root_key, + jca_pub, + ca=True, + ca_path_length=0, + extra_extensions=[(marker, False)], + ) + issuer = JobCertIssuer(serialize_cert(jca_cert), jca_key) + job_crt, job_key = issuer.issue("server", _JOB_ID) + other_crt, other_key = issuer.issue("server", _OTHER_JOB_ID) + files = { + "rootCA.pem": serialize_cert(root_cert), + "server.crt": serialize_cert(srv_cert), + "server.key": serialize_pri_key(srv_key), + "job.crt": job_crt, + "job.key": job_key, + "other_job.crt": other_crt, + "other_job.key": other_key, + } + paths = {} + for name, data in files.items(): + paths[name] = os.path.join(out_dir, name) + with open(paths[name], "wb") as f: + f.write(data) + return paths + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _run_parent(root_url, pki, ready_q, stop_ev): + cell = None + try: + credentials = { + DriverParams.CA_CERT.value: pki["rootCA.pem"], + DriverParams.SERVER_CERT.value: pki["server.crt"], + DriverParams.SERVER_KEY.value: pki["server.key"], + DriverParams.CONNECTION_SECURITY.value: "mtls", + } + cell = Cell("server", root_url, secure=True, credentials=credentials, create_internal_listener=False) + cell.register_request_cb(_CHANNEL, _TOPIC, lambda request: Message(payload=request.payload)) + cell.start() + ready_q.put("ready") + stop_ev.wait(120) + except Exception: + ready_q.put(traceback.format_exc()) + finally: + if cell: + cell.stop() + + +def _run_bootstrap(root_url, pki, cert_name, fqcn, wait, result_q): + cell = None + try: + credentials = { + DriverParams.CA_CERT.value: pki["rootCA.pem"], + DriverParams.SERVER_CERT.value: pki[f"{cert_name}.crt"], + DriverParams.SERVER_KEY.value: pki[f"{cert_name}.key"], + DriverParams.CONNECTION_SECURITY.value: "mtls", + } + cell = Cell( + fqcn, + root_url, + secure=True, + credentials=credentials, + create_internal_listener=False, + parent_url=root_url, + parent_resources={DriverParams.CONNECTION_SECURITY.value: "mtls"}, + auth_identity_map={"server": "server"}, + ) + cell.start() + deadline = time.time() + wait + while time.time() < deadline and not cell.is_cell_connected("server"): + time.sleep(0.1) + connected = cell.is_cell_connected("server") + rc = None + if connected: + reply = cell.send_request(_CHANNEL, _TOPIC, "server", Message(payload="hello"), timeout=5.0) + rc = reply.get_header(MessageHeaderKey.RETURN_CODE) + result_q.put({"connected": connected, "rc": rc}) + except Exception: + result_q.put({"error": traceback.format_exc()}) + finally: + if cell: + cell.stop() + + +@pytest.fixture(scope="module") +def parent(tmp_path_factory): + ctx = mp.get_context("spawn") + pki = _write_pki(str(tmp_path_factory.mktemp("pki"))) + root_url = f"stcp://localhost:{_free_port()}" + ready_q, stop_ev = ctx.Queue(), ctx.Event() + proc = ctx.Process(target=_run_parent, args=(root_url, pki, ready_q, stop_ev)) + proc.start() + try: + status = ready_q.get(timeout=30) + assert status == "ready", status + yield root_url, pki + finally: + stop_ev.set() + proc.join(15) + + +def _bootstrap(parent, cert_name, wait): + root_url, pki = parent + ctx = mp.get_context("spawn") + result_q = ctx.Queue() + fqcn = make_workspace_transfer_fqcn("server", _JOB_ID) + proc = ctx.Process(target=_run_bootstrap, args=(root_url, pki, cert_name, fqcn, wait, result_q)) + proc.start() + try: + result = result_q.get(timeout=wait + 30) + finally: + proc.join(15) + assert "error" not in result, result.get("error") + return result + + +def test_bootstrap_cell_authenticates_with_its_jobs_cert(parent): + result = _bootstrap(parent, "job", _CONNECT_TIMEOUT) + + assert result["connected"] is True + assert result["rc"] == ReturnCode.OK + + +def test_bootstrap_cell_rejected_with_another_jobs_cert(parent): + result = _bootstrap(parent, "other_job", _REJECT_WAIT) + + assert result["connected"] is False diff --git a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py index 1eac5cbbe2..fb7d9a712a 100644 --- a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py +++ b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py @@ -35,6 +35,7 @@ from nvflare.fuel.f3.cellnet.core_cell import CoreCell from nvflare.fuel.f3.cellnet.credential_manager import CERT_CONTENT, CredentialManager from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, MessageType, ReturnCode +from nvflare.fuel.f3.cellnet.fqcn import FQCN from nvflare.fuel.f3.cellnet.identity import ADMIN_LISTENER_KEY, CellIdentityResolver from nvflare.fuel.f3.cellnet.utils import make_reply from nvflare.fuel.f3.comm_error import CommError @@ -449,14 +450,41 @@ def test_mtls_certificate_cache_accepts_configured_auth_identity_for_site_cert_c assert manager.cert_cache["site-1.job-123"] == cert +@pytest.mark.parametrize( + "fqcn, expected", + [ + ("site-1.job-123", True), + ("server.job-123", True), + ("site-1.job-123.sub-1", True), + ("site-1.ws_transfer_job-123", True), + ("server.ws_transfer_job-123", True), + ("site-1", False), + ("site-1.job-999", False), + ("site-1.ws_transfer_job-999", False), + ("site-1.ws_transferjob-123", False), + ("site-1.job-1234", False), + ("site-1.xjob-123", False), + ], +) +def test_fqcn_belongs_to_job(fqcn, expected): + assert FQCN.belongs_to_job(fqcn, "job-123") is expected + + +def test_fqcn_belongs_to_job_rejects_empty_job_id(): + assert FQCN.belongs_to_job("site-1.ws_transfer_", "") is False + + def test_identity_resolver_binds_job_cert_to_job_fqcn(): resolver = CellIdentityResolver(local_fqcn="server", prefix_identity_map={"site-1": "site-1"}) resolver.require_match("site-1.job-123", "site-1", "connection cj", peer_job_id="job-123") resolver.require_match("site-1.job-123.sub-1", "site-1", "connection sub", peer_job_id="job-123") + resolver.require_match("site-1.ws_transfer_job-123", "site-1", "connection bootstrap", peer_job_id="job-123") with pytest.raises(ValueError, match="bound to job 'job-123'"): resolver.require_match("site-1.job-999", "site-1", "connection cj", peer_job_id="job-123") + with pytest.raises(ValueError, match="bound to job 'job-123'"): + resolver.require_match("site-1.ws_transfer_job-999", "site-1", "connection bootstrap", peer_job_id="job-123") with pytest.raises(ValueError, match="bound to job 'job-123'"): resolver.require_match("site-1", "site-1", "connection cp", peer_job_id="job-123") with pytest.raises(ValueError, match="bound to job 'job-123'"): @@ -465,18 +493,19 @@ def test_identity_resolver_binds_job_cert_to_job_fqcn(): ) -def test_mtls_handshake_accepts_job_cert_for_own_job(): +@pytest.mark.parametrize("endpoint_name", ["site-1.job-123", "site-1.ws_transfer_job-123"]) +def test_mtls_handshake_accepts_job_cert_for_own_job(endpoint_name): manager = _conn_manager(identity_map={"site-1": "site-1"}) conn = _FakeConnection(peer_cn="site-1", peer_job_id="job-123") sfm_conn = SfmConnection(conn, Endpoint("server")) - manager.update_endpoint(sfm_conn, {HandshakeKeys.ENDPOINT_NAME: "site-1.job-123"}) + manager.update_endpoint(sfm_conn, {HandshakeKeys.ENDPOINT_NAME: endpoint_name}) - assert "site-1.job-123" in manager.sfm_endpoints + assert endpoint_name in manager.sfm_endpoints assert not conn.closed -@pytest.mark.parametrize("endpoint_name", ["site-1.job-999", "site-1"]) +@pytest.mark.parametrize("endpoint_name", ["site-1.job-999", "site-1", "site-1.ws_transfer_job-999"]) def test_mtls_handshake_rejects_job_cert_outside_its_job(endpoint_name): manager = _conn_manager(identity_map={"site-1": "site-1"}) conn = _FakeConnection(peer_cn="site-1", peer_job_id="job-123") From a77e1544f72b7fbc66dc73f801c8efba4da05cf2 Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Fri, 4 Sep 2026 08:04:58 -0700 Subject: [PATCH 03/10] fix(security): destroy job credentials at job end; never archive them The server archived the whole SJ run directory into the job store when a job finished, so download_job returned job_cert/job.crt and job_cert/job.key (found by the Docker e2e validation). Clients kept the CJ credential on disk after the CJ exited. The SP now destroys the SJ credential before the run directory is archived, and the CP destroys the CJ credential as soon as the CJ process has exited. The credential is dead at that point: no renewal, no reuse. Verified with a full POC job: the stored workspace contains no job_cert entries or key files and client run dirs keep the app but no credential. --- docs/design/per_job_certs_design.md | 5 ++++ .../security/per_job_certificates.rst | 3 ++ nvflare/private/fed/client/client_executor.py | 2 ++ nvflare/private/fed/server/job_runner.py | 4 +++ nvflare/private/fed/utils/job_cert_utils.py | 11 ++++++++ .../fed/client/client_executor_test.py | 28 +++++++++++++++++++ .../private/fed/server/job_runner_test.py | 26 +++++++++++++++++ 7 files changed, 79 insertions(+) diff --git a/docs/design/per_job_certs_design.md b/docs/design/per_job_certs_design.md index b8914c045b..dc19cc259c 100644 --- a/docs/design/per_job_certs_design.md +++ b/docs/design/per_job_certs_design.md @@ -133,6 +133,11 @@ The issued credential is a PEM bundle: leaf cert followed by `job_ca.crt` (`AppDeployer` recreates the run directory), SP writes the SJ credential into the job run directory before the SJ process is launched. +When the job finishes, SP destroys the SJ credential before the run directory +is archived to the job store, so the workspace served by `download_job` never +contains it; CP destroys the CJ credential as soon as the CJ process has +exited. The credential is dead at that point: there is no renewal or reuse. + **CJ (push).** The job deploy message becomes per-site: the shared app bytes stay a single payload reference, but each site's message carries an additional header with that site's cert bundle and private key. The message travels over diff --git a/docs/user_guide/admin_guide/security/per_job_certificates.rst b/docs/user_guide/admin_guide/security/per_job_certificates.rst index 663d1d26ef..28e7f8d02b 100644 --- a/docs/user_guide/admin_guide/security/per_job_certificates.rst +++ b/docs/user_guide/admin_guide/security/per_job_certificates.rst @@ -60,6 +60,9 @@ Each running job has its credential at:: //job_cert/job.crt certificate followed by job_ca.crt //job_cert/job.key private key, mode 0600 +Both files are deleted when the job process exits. The job workspace archived +on the server (``download_job``) never contains them. + Enabling and disabling ====================== diff --git a/nvflare/private/fed/client/client_executor.py b/nvflare/private/fed/client/client_executor.py index aff148a370..b1a096a2ae 100644 --- a/nvflare/private/fed/client/client_executor.py +++ b/nvflare/private/fed/client/client_executor.py @@ -34,6 +34,7 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from nvflare.private.defs import CellChannel, CellChannelTopic, JobFailureMsgKey, new_cell_message from nvflare.private.fed.utils.fed_utils import get_job_launcher, get_return_code +from nvflare.private.fed.utils.job_cert_utils import remove_job_cert from nvflare.security.logging import secure_format_exception, secure_log_traceback from .client_status import ClientStatus, get_status_message @@ -626,6 +627,7 @@ def _wait_child_process_finish( job_handle = self.run_processes.get(job_id, {}).get(RunProcessKey.JOB_HANDLE) if job_handle: job_handle.wait() + remove_job_cert(Workspace.run_dir_path(workspace, job_id)) return_code = get_return_code(job_handle, job_id, workspace, self.logger) diff --git a/nvflare/private/fed/server/job_runner.py b/nvflare/private/fed/server/job_runner.py index 33cc34c23b..75f1bd6068 100644 --- a/nvflare/private/fed/server/job_runner.py +++ b/nvflare/private/fed/server/job_runner.py @@ -53,6 +53,7 @@ JobCertError, load_job_cert_issuer, pack_job_cert_header, + remove_job_cert, write_job_cert, ) from nvflare.security.logging import secure_format_exception @@ -637,6 +638,9 @@ def _save_workspace( else: workspace = fl_ctx.get_workspace() run_dir = workspace.get_run_dir(job_id) + # the archive is served by download_job; the job credential is dead and must not be in it + if os.path.isdir(run_dir): + remove_job_cert(run_dir) result_root = workspace.get_result_root(job_id) log_root = workspace.get_log_root(job_id) audit_root = workspace.get_audit_root(job_id) diff --git a/nvflare/private/fed/utils/job_cert_utils.py b/nvflare/private/fed/utils/job_cert_utils.py index 9f1aa23098..b9f88e78ec 100644 --- a/nvflare/private/fed/utils/job_cert_utils.py +++ b/nvflare/private/fed/utils/job_cert_utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import datetime import os import shutil @@ -79,6 +80,16 @@ def write_job_cert(run_dir: str, cert_chain_pem: bytes, key_pem: bytes): write_pri_key_file(key_path, key_pem) +def remove_job_cert(run_dir: str) -> None: + """Destroy a job's credential; it is dead once the job process has exited and must never be archived.""" + cert_path, key_path = job_cert_paths(run_dir) + for path in (cert_path, key_path): + with contextlib.suppress(FileNotFoundError): + os.remove(path) + with contextlib.suppress(OSError): + os.rmdir(os.path.dirname(cert_path)) + + def find_job_cert(run_dir: str) -> Optional[Tuple[str, str]]: cert_path, key_path = job_cert_paths(run_dir) if os.path.isfile(cert_path) and os.path.isfile(key_path): diff --git a/tests/unit_test/private/fed/client/client_executor_test.py b/tests/unit_test/private/fed/client/client_executor_test.py index af3a8b1540..1109c753e8 100644 --- a/tests/unit_test/private/fed/client/client_executor_test.py +++ b/tests/unit_test/private/fed/client/client_executor_test.py @@ -38,6 +38,7 @@ ) from nvflare.private.fed.client.client_status import ClientStatus from nvflare.private.fed.client.communicator import Communicator +from nvflare.private.fed.utils.job_cert_utils import find_job_cert, write_job_cert EXPECTED_REPORTABLE_JOB_FAILURES = { ProcessExitCode.EXCEPTION: "exception", @@ -587,6 +588,33 @@ def test_wait_child_process_reports_failure_return_code_to_server(return_code, r engine.fire_event.assert_called_once_with(EventType.JOB_COMPLETED, fl_ctx) +def test_wait_child_process_destroys_job_credential_when_worker_exits(tmp_path): + client = MagicMock() + client.client_name = "site-1" + client.send_request_before_shutdown.return_value.get_header.return_value = ReturnCode.OK + job_executor = JobExecutor(client=client, startup="startup") + job_handle = MagicMock() + job_handle.poll.return_value = JobReturnCode.SUCCESS + job_executor.run_processes = {"job-1": {RunProcessKey.JOB_HANDLE: job_handle}} + run_dir = tmp_path / "job-1" + run_dir.mkdir() + (run_dir / "app_site-1").mkdir() + write_job_cert(str(run_dir), b"cert", b"key") + + job_executor._wait_child_process_finish( + client=client, + job_id="job-1", + allocated_resource=None, + token=None, + resource_manager=MagicMock(), + workspace=str(tmp_path), + fl_ctx=MagicMock(), + ) + + assert find_job_cert(str(run_dir)) is None + assert (run_dir / "app_site-1").is_dir() + + def test_wait_child_process_preserves_launcher_infrastructure_error_over_rc_file(tmp_path): client = MagicMock() client.client_name = "site-1" diff --git a/tests/unit_test/private/fed/server/job_runner_test.py b/tests/unit_test/private/fed/server/job_runner_test.py index cece8252b7..8a884d079e 100644 --- a/tests/unit_test/private/fed/server/job_runner_test.py +++ b/tests/unit_test/private/fed/server/job_runner_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os from contextlib import nullcontext from unittest.mock import ANY, MagicMock, call, patch @@ -25,9 +26,11 @@ from nvflare.apis.utils.event import fire_event_to_components from nvflare.app_common.job_schedulers.job_scheduler import DefaultJobScheduler from nvflare.fuel.common.exit_codes import ProcessExitCode +from nvflare.fuel.utils.zip_utils import get_all_file_paths from nvflare.private.admin_defs import Message, MsgHeader, ReturnCode from nvflare.private.fed.server.job_runner import JobRunner, _FinishedJobState from nvflare.private.fed.server.message_send import ClientReply +from nvflare.private.fed.utils.job_cert_utils import write_job_cert def _patch_job_runner_sleep(side_effect): @@ -328,6 +331,29 @@ def test_save_workspace_archives_only_existing_deduplicated_sources(tmp_path): assert not result_root.exists() +def test_save_workspace_destroys_job_credential_before_archiving(tmp_path): + run_dir = tmp_path / "run" + run_dir.mkdir() + (run_dir / "app_server").mkdir() + (run_dir / "app_server" / "result.txt").write_text("2") + write_job_cert(str(run_dir), b"cert", b"key") + runner, fl_ctx, job_manager = _make_workspace_save_inputs(str(run_dir), str(run_dir), str(run_dir), str(run_dir)) + archived = {} + + def _capture(job_id, sources, ctx): + archived[job_id] = sorted( + os.path.relpath(f, sources[0]) for f in get_all_file_paths(sources[0]) if os.path.isfile(f) + ) + return "/store/job-1/workspace" + + job_manager.save_workspace.side_effect = _capture + + runner._save_workspace(fl_ctx) + + assert archived["job-1"] == [os.path.join("app_server", "result.txt")] + assert not run_dir.exists() + + def test_save_workspace_tolerates_source_disappearing_during_cleanup(tmp_path): run_dir = tmp_path / "run" run_dir.mkdir() From 4bc36f68f41d6253d31dd6e327002caef9b68636 Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Fri, 4 Sep 2026 08:40:29 -0700 Subject: [PATCH 04/10] fix(security): require mTLS child links in secure mode; read job_cert_valid_days from server config QA on Slurm found a "cross-job replay" accepted at the site parent and the documented fed_server.json job_cert_valid_days ignored. - Docker and Slurm launchers refuse a clear-text parent connection in secure mode: job-cert binding exists only on mTLS links, and a clear internal listener lets any process that reaches it claim any job FQCN. The shared-file transport is exempt (filesystem permissions). A live test now connects real job cells to a site parent's mTLS internal listener and verifies another job's certificate is rejected there. - SJ and the K8s bootstrap cell pin both TLS roles to the job credential so cellnet's directory-based back-fill can never present a site certificate. - JobRunner reads job_cert_valid_days from the server's fed_server.json / resources.json sections; the SP never loads the application section. --- docs/design/per_job_certs_design.md | 23 +- .../security/per_job_certificates.rst | 15 ++ .../app_opt/job_launcher/docker_launcher.py | 11 +- .../app_opt/job_launcher/slurm/launcher.py | 6 + .../job_launcher/workspace_cell_transfer.py | 27 +- nvflare/private/fed/server/fed_server.py | 3 + nvflare/private/fed/server/job_runner.py | 4 +- .../job_launcher/docker_launcher_test.py | 21 +- .../job_launcher/slurm_launcher_test.py | 16 ++ .../workspace_cell_transfer_test.py | 28 +- .../f3/cellnet/job_cert_binding_live_test.py | 241 ++++++++++++++++++ .../private/fed/server/fed_server_test.py | 36 ++- .../private/fed/server/job_runner_test.py | 18 +- 13 files changed, 411 insertions(+), 38 deletions(-) create mode 100644 tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py diff --git a/docs/design/per_job_certs_design.md b/docs/design/per_job_certs_design.md index dc19cc259c..891b330b12 100644 --- a/docs/design/per_job_certs_design.md +++ b/docs/design/per_job_certs_design.md @@ -119,8 +119,8 @@ issuer generates an RSA keypair and a leaf certificate: - a job-ID extension identifying the job - `notBefore` backdated a few minutes to tolerate clock skew between the issuing server and the sites that validate the cert seconds later -- `notAfter` = issue time + `job_cert_valid_days` (server startup config - `fed_server.json`, `--set`, or `NVFLARE_JOB_CERT_VALID_DAYS`; default 30), +- `notAfter` = issue time + `job_cert_valid_days` (server `fed_server.json` or + `resources.json`, `--set`, or `NVFLARE_JOB_CERT_VALID_DAYS`; default 30), clamped to the job CA's own expiry. There is no renewal, so this is the maximum job duration; set it for the longest job the server runs. @@ -176,10 +176,12 @@ separate "job credential" configuration key. `ssl_root_cert` remains `rootCA.pem`. - `BaseServer.create_job_cell()` and `FederatedClientBase._create_cell()` use - the job credential for the SJ/CJ cell. The CJ also pins its server-role - credential to the job cert; otherwise, on listener-enabled sites, the site's - server cert would be back-filled from the startup kit and preferred by - message-level crypto. + the job credential for the SJ/CJ cell, in both TLS roles (client and server + certificate). Cellnet back-fills a missing role from `client.crt` / + `server.crt` next to `rootCA.pem`; pinning both roles means no site + certificate found in the startup directory can ever be presented by a job + cell, in TLS or in message-level crypto. The Kubernetes bootstrap cell does + the same. - The startup content-integrity check (`signature.json` kits) no longer requires the site private key in job processes. - The job process never registers with the server (it receives the CP's auth @@ -249,6 +251,15 @@ credential. | Slurm (apptainer / pyxis) | keyless staged copy under the 0700 job dir, bound at `/startup` | run dir bind | | Slurm (`sandbox: none`) | bare host process — no isolation possible | run dir | +Docker and Slurm job processes reach the CP's internal listener over the +network. Job binding exists only on mTLS connections, and a clear-text listener +lets any process that can reach it claim any job FQCN, so in secure mode both +launchers refuse a parent link that is not mTLS (the client needs a +`listening_host` with scheme `stcp` and connection security `mtls`). The +shared-file parent transport is exempt: it carries no certificates and is +guarded by filesystem permissions, like the in-process launcher's local link. +Kubernetes job pods connect to the parent's external `stcp` listener with mTLS. + Kubernetes needs the environment route because the pod's bootstrap cell, which downloads the run directory, exists before the run directory does. The job process pops both variables and writes the credential into the run directory diff --git a/docs/user_guide/admin_guide/security/per_job_certificates.rst b/docs/user_guide/admin_guide/security/per_job_certificates.rst index 28e7f8d02b..4e544df22d 100644 --- a/docs/user_guide/admin_guide/security/per_job_certificates.rst +++ b/docs/user_guide/admin_guide/security/per_job_certificates.rst @@ -110,6 +110,7 @@ setting, resolved in this order: #. ``--set job_cert_valid_days=`` on the server start command; #. a top-level ``"job_cert_valid_days": `` entry in the server's ``fed_server.json``; +#. the same entry in the server's ``resources.json``; #. the environment variable ``NVFLARE_JOB_CERT_VALID_DAYS``. The job CA itself is valid for 360 days, bounded by the root CA. When it has @@ -124,6 +125,16 @@ Job processes never need a site private key, so the launchers do not give them one. In secure mode each launcher refuses to start a job that has no job credential instead of falling back to site certificates. +Docker and Slurm job processes connect to the client's internal listener over +the network. The job binding described above exists only on an mTLS link, so in +secure mode both launchers refuse a clear-text parent connection with ``secure +mode requires an mTLS parent connection``. Give the client a ``listening_host`` +in ``project.yml`` with ``scheme: stcp`` and ``conn_sec: mtls`` (a client +without one gets the default clear-text ``tcp`` listener). Kubernetes job pods +already connect to the parent's external ``stcp`` listener with mTLS. The +shared-file parent transport is exempt: it carries no certificates and relies on +filesystem permissions. + .. list-table:: :header-rows: 1 :widths: 22 78 @@ -229,6 +240,10 @@ Failures are recorded in the job's ``job_deploy_detail`` (shown by * - ``has no job credential; secure jobs run only on per-job certificates`` (Docker, Kubernetes, or Slurm launcher) - Same cause as above. + * - ``secure mode requires an mTLS parent connection`` (Docker or Slurm + launcher) + - The client's internal listener is clear text. Provision the client with + a ``listening_host`` using ``scheme: stcp`` and ``conn_sec: mtls``. * - ``authenticated with a certificate bound to job '...' but claimed endpoint ... is not part of that job`` - A process presented another job's certificate. This does not happen in diff --git a/nvflare/app_opt/job_launcher/docker_launcher.py b/nvflare/app_opt/job_launcher/docker_launcher.py index 7e322f58df..67929916a0 100644 --- a/nvflare/app_opt/job_launcher/docker_launcher.py +++ b/nvflare/app_opt/job_launcher/docker_launcher.py @@ -86,7 +86,7 @@ class DockerStatus: _RESERVED_DEFAULT_KWARGS = RESERVED_DOCKER_KWARGS -def _rewrite_parent_url(job_args: dict, site_name: str) -> tuple[dict, str | None]: +def _rewrite_parent_url(job_args: dict, site_name: str, secure_mode: bool = False) -> tuple[dict, str | None]: """Rewrite a parent URL to Docker DNS while preserving its transport security.""" entry = job_args.get(JobProcessArgs.PARENT_URL) if not entry: @@ -123,6 +123,11 @@ def _rewrite_parent_url(job_args: dict, site_name: str) -> tuple[dict, str | Non raise ValueError(f"invalid parent URL {original_url!r}") from e if parsed.scheme not in ("tcp", "stcp") or not host or not port: raise ValueError(f"parent URL must use {SHARED_FILE_SCHEME}, tcp, or stcp with a host and port") + if secure_mode and connection_security != ConnectionSecurity.MTLS: + raise ValueError( + "secure mode requires an mTLS parent connection for Docker jobs: configure the client's internal " + "listener (listening_host) with scheme stcp and connection security mtls" + ) if (parsed.scheme == "stcp") != (connection_security == ConnectionSecurity.MTLS): raise ValueError("parent URL scheme does not match parent connection security") @@ -605,7 +610,9 @@ def launch_job(self, job_meta: dict, fl_ctx: FLContext) -> JobHandleSpec: # Derive parent_url at runtime: site name (= container name on Docker DNS) + port # from the original PARENT_URL in job_args. This avoids baking parent_url into # resources.json at provision time. - job_args, file_parent_dir = _rewrite_parent_url(job_args, site_name) + job_args, file_parent_dir = _rewrite_parent_url( + job_args, site_name, secure_mode=fl_ctx.get_prop(FLContextKey.SECURE_MODE, False) + ) if file_parent_dir and file_parent_dir.startswith(self.WORKSPACE_MOUNT): raise ValueError(f"shared-file parent directory {file_parent_dir} overlaps the container workspace mount") diff --git a/nvflare/app_opt/job_launcher/slurm/launcher.py b/nvflare/app_opt/job_launcher/slurm/launcher.py index e6d3ea5860..8ccc020abe 100644 --- a/nvflare/app_opt/job_launcher/slurm/launcher.py +++ b/nvflare/app_opt/job_launcher/slurm/launcher.py @@ -468,6 +468,12 @@ def _build_launch_plan(self, job_meta: dict, fl_ctx: FLContext) -> LaunchPlan: internal_port=self.config.internal_port, ) parent_scheme = urlsplit(str(job_args[JobProcessArgs.PARENT_URL][1])).scheme + secure_mode = fl_ctx.get_prop(FLContextKey.SECURE_MODE, False) + if secure_mode and parent_scheme != SHARED_FILE_SCHEME and process_connection_security != "mtls": + raise SlurmLauncherError( + "secure mode requires an mTLS parent connection for Slurm jobs: configure the client's internal " + "listener (listening_host) with scheme stcp and connection security mtls" + ) if (parent_scheme == "stcp") != (process_connection_security == "mtls"): raise SlurmLauncherError("parent URL scheme does not match parent connection security") diff --git a/nvflare/app_opt/job_launcher/workspace_cell_transfer.py b/nvflare/app_opt/job_launcher/workspace_cell_transfer.py index 862db11755..90f9ec9fe9 100644 --- a/nvflare/app_opt/job_launcher/workspace_cell_transfer.py +++ b/nvflare/app_opt/job_launcher/workspace_cell_transfer.py @@ -520,8 +520,12 @@ def _get_root_url(args) -> str: raise RuntimeError("unable to determine root_url for workspace transfer bootstrap cell") -def _get_bootstrap_tls_pair(run_dir: str, owner_fqcn: str) -> tuple[str, str, str, str]: - """The job credential, in the TLS role the bootstrap cell plays (server-side jobs dial as a server).""" +def _bootstrap_credentials(run_dir: str, root_ca: str) -> dict: + """TLS credentials of the bootstrap cell: the job credential in both roles. + + Pinning both roles keeps cellnet's directory-based credential back-fill from ever + substituting a site certificate found next to rootCA.pem. + """ job_cert = find_job_cert(run_dir) if not job_cert: raise RuntimeError( @@ -529,9 +533,13 @@ def _get_bootstrap_tls_pair(run_dir: str, owner_fqcn: str) -> tuple[str, str, st "secure jobs run only on per-job certificates" ) cert_path, key_path = job_cert - if FQCN.get_root(owner_fqcn) == FQCN.ROOT_SERVER: - return cert_path, key_path, DriverParams.SERVER_CERT.value, DriverParams.SERVER_KEY.value - return cert_path, key_path, DriverParams.CLIENT_CERT.value, DriverParams.CLIENT_KEY.value + return { + DriverParams.CA_CERT.value: root_ca, + DriverParams.SERVER_CERT.value: cert_path, + DriverParams.SERVER_KEY.value: key_path, + DriverParams.CLIENT_CERT.value: cert_path, + DriverParams.CLIENT_KEY.value: key_path, + } def _load_startup_json(startup_dir: str, filename: str) -> dict | None: @@ -596,14 +604,7 @@ def _create_bootstrap_cell(args, owner_fqcn: str, secure_mode: bool) -> tuple[Ce root_ca = os.path.join(startup_dir, "rootCA.pem") if not os.path.exists(root_ca): raise RuntimeError(f"workspace transfer requires rootCA.pem in startup dir: {startup_dir}") - cert_path, key_path, cert_key, key_key = _get_bootstrap_tls_pair( - _run_dir(args.workspace, args.job_id), owner_fqcn - ) - credentials = { - DriverParams.CA_CERT.value: root_ca, - cert_key: cert_path, - key_key: key_path, - } + credentials = _bootstrap_credentials(_run_dir(args.workspace, args.job_id), root_ca) auth_identity_map = _bootstrap_auth_identity_map(startup_dir) parent_resources = {} diff --git a/nvflare/private/fed/server/fed_server.py b/nvflare/private/fed/server/fed_server.py index 4e41f77976..d70523d2f0 100644 --- a/nvflare/private/fed/server/fed_server.py +++ b/nvflare/private/fed/server/fed_server.py @@ -635,10 +635,13 @@ def create_job_cell(self, job_id, root_url, parent_url, secure_train, server_con ssl_cert = server_config[SecureTrainConst.SSL_CERT] private_key = server_config[SecureTrainConst.PRIVATE_KEY] + # both TLS roles use the job credential so directory-based back-fill never picks a site cert credentials = { DriverParams.CA_CERT.value: root_cert, DriverParams.SERVER_CERT.value: ssl_cert, DriverParams.SERVER_KEY.value: private_key, + DriverParams.CLIENT_CERT.value: ssl_cert, + DriverParams.CLIENT_KEY.value: private_key, } conn_security = server_config.get(ConnPropKey.CONNECTION_SECURITY) diff --git a/nvflare/private/fed/server/job_runner.py b/nvflare/private/fed/server/job_runner.py index 75f1bd6068..4a243d635f 100644 --- a/nvflare/private/fed/server/job_runner.py +++ b/nvflare/private/fed/server/job_runner.py @@ -120,7 +120,9 @@ def __init__(self, workspace_root: str) -> None: name=ConfigVarName.CLIENT_OUTCOME_WAIT_TIMEOUT, conf=SystemConfigs.APPLICATION_CONF, default=900.0 ) self.job_cert_valid_days = ConfigService.get_int_var( - name=ConfigVarName.JOB_CERT_VALID_DAYS, conf=SystemConfigs.APPLICATION_CONF, default=JOB_CERT_VALID_DAYS + name=ConfigVarName.JOB_CERT_VALID_DAYS, + conf=[SystemConfigs.STARTUP_CONF, SystemConfigs.RESOURCES_CONF], + default=JOB_CERT_VALID_DAYS, ) if self.job_cert_valid_days <= 0: raise ValueError(f"{ConfigVarName.JOB_CERT_VALID_DAYS} must be positive, got {self.job_cert_valid_days}") diff --git a/tests/unit_test/app_opt/job_launcher/docker_launcher_test.py b/tests/unit_test/app_opt/job_launcher/docker_launcher_test.py index 4ea5f22395..f3929c7a49 100644 --- a/tests/unit_test/app_opt/job_launcher/docker_launcher_test.py +++ b/tests/unit_test/app_opt/job_launcher/docker_launcher_test.py @@ -804,7 +804,9 @@ def test_secure_launch_with_job_credential_binds_startup_files_without_keys(self container.id = "abc123" dc.containers.run.return_value = container dc.containers.get.return_value = _make_container("running") - fl_ctx, _ = _make_fl_ctx(workspace_obj=workspace_obj, secure_mode=True) + fl_ctx, _ = _make_fl_ctx( + workspace_obj=workspace_obj, parent_url="stcp://localhost:8002", parent_conn_sec="mtls", secure_mode=True + ) launcher.launch_job(_make_job_meta(), fl_ctx) @@ -823,13 +825,28 @@ def test_secure_launch_with_job_credential_binds_startup_files_without_keys(self def test_secure_launch_without_job_credential_is_refused(self, tmp_path): launcher = _make_launcher(workspace="/host/workspace") dc = launcher._docker_client - fl_ctx, _ = _make_fl_ctx(workspace_obj=_make_workspace_obj(run_dir=str(tmp_path / "job-1")), secure_mode=True) + fl_ctx, _ = _make_fl_ctx( + workspace_obj=_make_workspace_obj(run_dir=str(tmp_path / "job-1")), + parent_url="stcp://localhost:8002", + parent_conn_sec="mtls", + secure_mode=True, + ) with pytest.raises(RuntimeError, match="no job credential"): launcher.launch_job(_make_job_meta(), fl_ctx) dc.containers.run.assert_not_called() + @pytest.mark.parametrize("parent_conn_sec", [None, "clear"]) + def test_secure_launch_rejects_clear_parent_link(self, parent_conn_sec): + launcher = _make_launcher(workspace="/host/workspace") + fl_ctx, _ = _make_fl_ctx(parent_url="tcp://localhost:8002", parent_conn_sec=parent_conn_sec, secure_mode=True) + + with pytest.raises(ValueError, match="requires an mTLS parent connection"): + launcher.launch_job(_make_job_meta(), fl_ctx) + + launcher._docker_client.containers.run.assert_not_called() + def test_launch_rejects_job_workspace_path_escape(self): launcher = _make_launcher(workspace="/host/workspace") dc = launcher._docker_client diff --git a/tests/unit_test/app_opt/job_launcher/slurm_launcher_test.py b/tests/unit_test/app_opt/job_launcher/slurm_launcher_test.py index 2257ecdf47..7ec66bf09f 100644 --- a/tests/unit_test/app_opt/job_launcher/slurm_launcher_test.py +++ b/tests/unit_test/app_opt/job_launcher/slurm_launcher_test.py @@ -584,6 +584,9 @@ def test_launch_plan_rejects_secure_job_without_credential(tmp_path): launcher = _launcher(tmp_path, workspace) fl_ctx = _fl_ctx(workspace) fl_ctx.set_prop(FLContextKey.SECURE_MODE, True, private=True, sticky=True) + job_args = fl_ctx.get_prop(FLContextKey.JOB_PROCESS_ARGS) + job_args[JobProcessArgs.PARENT_URL] = ("-p", "stcp://old-host:8102") + job_args[JobProcessArgs.PARENT_CONN_SEC] = ("--parent_conn_sec", "mtls") with pytest.raises(SlurmLauncherError, match="no job credential"): launcher._build_launch_plan({JobConstants.JOB_ID: "job-1"}, fl_ctx) @@ -594,6 +597,19 @@ def test_launch_plan_rejects_secure_job_without_credential(tmp_path): assert launcher._build_launch_plan({JobConstants.JOB_ID: "job-1"}, fl_ctx).run_dir == str(workspace / "job-1") +def test_launch_plan_rejects_clear_parent_link_in_secure_mode(tmp_path): + workspace = _workspace(tmp_path) + launcher = _launcher(tmp_path, workspace) + fl_ctx = _fl_ctx(workspace) + fl_ctx.set_prop(FLContextKey.SECURE_MODE, True, private=True, sticky=True) + (workspace / "job-1" / "job_cert").mkdir() + (workspace / "job-1" / "job_cert" / "job.crt").write_text("cert") + (workspace / "job-1" / "job_cert" / "job.key").write_text("key") + + with pytest.raises(SlurmLauncherError, match="requires an mTLS parent connection"): + launcher._build_launch_plan({JobConstants.JOB_ID: "job-1"}, fl_ctx) + + @pytest.mark.parametrize("launcher_class", [ClientSlurmJobLauncher, ServerSlurmJobLauncher]) def test_launch_plan_preserves_mtls_parent_args(tmp_path, launcher_class): workspace = _workspace(tmp_path) diff --git a/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py b/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py index 5b17c63ce8..e038fe8df3 100644 --- a/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py +++ b/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py @@ -28,8 +28,8 @@ ENV_WORKSPACE_TRANSFER_TOKEN, WorkspaceTransferManager, _bootstrap_auth_identity_map, + _bootstrap_credentials, _create_bootstrap_cell, - _get_bootstrap_tls_pair, _hash_file, _install_job_cert, _wait_for_bootstrap_ready, @@ -878,27 +878,31 @@ def start(self): assert captured["auth_identity_map"] == {FQCN.ROOT_SERVER: "gcp-server"} assert captured["secure"] is True job_cert_dir = str(tmp_path / JOB_ID / "job_cert") - assert captured["credentials"][DriverParams.CLIENT_CERT.value] == os.path.join(job_cert_dir, "job.crt") - assert captured["credentials"][DriverParams.CLIENT_KEY.value] == os.path.join(job_cert_dir, "job.key") + for role in (DriverParams.CLIENT_CERT, DriverParams.SERVER_CERT): + assert captured["credentials"][role.value] == os.path.join(job_cert_dir, "job.crt") + for role in (DriverParams.CLIENT_KEY, DriverParams.SERVER_KEY): + assert captured["credentials"][role.value] == os.path.join(job_cert_dir, "job.key") - def test_bootstrap_tls_pair_uses_job_credential_in_the_peer_role(self, tmp_path): + def test_bootstrap_credentials_pin_both_tls_roles_to_job_credential(self, tmp_path): run_dir = tmp_path / JOB_ID job_crt = run_dir / "job_cert" / "job.crt" job_key = run_dir / "job_cert" / "job.key" _write_file(str(job_crt), b"job-cert") _write_file(str(job_key), b"job-key") - cert_path, key_path, cert_key, key_key = _get_bootstrap_tls_pair(str(run_dir), "site-1") + credentials = _bootstrap_credentials(str(run_dir), "/startup/rootCA.pem") - assert (cert_path, key_path) == (str(job_crt), str(job_key)) - assert (cert_key, key_key) == (DriverParams.CLIENT_CERT.value, DriverParams.CLIENT_KEY.value) + assert credentials == { + DriverParams.CA_CERT.value: "/startup/rootCA.pem", + DriverParams.SERVER_CERT.value: str(job_crt), + DriverParams.SERVER_KEY.value: str(job_key), + DriverParams.CLIENT_CERT.value: str(job_crt), + DriverParams.CLIENT_KEY.value: str(job_key), + } - _, _, cert_key, key_key = _get_bootstrap_tls_pair(str(run_dir), FQCN.ROOT_SERVER) - assert (cert_key, key_key) == (DriverParams.SERVER_CERT.value, DriverParams.SERVER_KEY.value) - - def test_bootstrap_tls_pair_requires_job_credential(self, tmp_path): + def test_bootstrap_credentials_require_job_credential(self, tmp_path): with pytest.raises(RuntimeError, match="requires the job credential"): - _get_bootstrap_tls_pair(str(tmp_path / JOB_ID), "site-1") + _bootstrap_credentials(str(tmp_path / JOB_ID), "/startup/rootCA.pem") def test_install_job_cert_writes_run_dir(self, tmp_path): args = SimpleNamespace(workspace=str(tmp_path), job_id=JOB_ID, job_cert_pem="cert-pem", job_key_pem="key-pem") diff --git a/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py b/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py new file mode 100644 index 0000000000..e8b4a06df9 --- /dev/null +++ b/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py @@ -0,0 +1,241 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Live mTLS regression: a site parent's internal listener binds job certificates to their job's FQCNs.""" + +import json +import multiprocessing as mp +import os +import socket +import time +import traceback +import uuid + +import pytest +from cryptography import x509 + +from nvflare.fuel.f3.cellnet.cell import Cell +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode +from nvflare.fuel.f3.drivers.driver_params import DriverParams +from nvflare.fuel.f3.message import Message +from nvflare.fuel.utils.config_service import ConfigService +from nvflare.lighter.constants import CertExtensionOID +from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert, serialize_pri_key +from nvflare.private.fed.utils.job_cert_utils import JobCertIssuer + +_CHANNEL = "job_binding_test" +_TOPIC = "echo" +_CONNECT_TIMEOUT = 10.0 +_REJECT_WAIT = 3.0 +_JOB_A = str(uuid.uuid4()) +_JOB_B = str(uuid.uuid4()) + + +def _write_pki(out_dir: str) -> dict: + root_key, root_pub = generate_keys() + root_cert = generate_cert(Identity("rootCA"), Identity("rootCA"), root_key, root_pub, ca=True) + server_key, server_pub = generate_keys() + server_cert = generate_cert( + Identity("server"), Identity("rootCA"), root_key, server_pub, server_default_host="localhost" + ) + site_key, site_pub = generate_keys() + site_cert = generate_cert( + Identity("site-1"), Identity("rootCA"), root_key, site_pub, server_default_host="localhost" + ) + jca_key, jca_pub = generate_keys() + marker = x509.UnrecognizedExtension(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER), b"job_ca") + jca_cert = generate_cert( + Identity("job_ca"), + Identity("rootCA"), + root_key, + jca_pub, + ca=True, + ca_path_length=0, + extra_extensions=[(marker, False)], + ) + issuer = JobCertIssuer(serialize_cert(jca_cert), jca_key) + job_a_crt, job_a_key = issuer.issue("site-1", _JOB_A) + job_b_crt, job_b_key = issuer.issue("site-1", _JOB_B) + files = { + "rootCA.pem": serialize_cert(root_cert), + "server.crt": serialize_cert(server_cert), + "server.key": serialize_pri_key(server_key), + "site-1.crt": serialize_cert(site_cert), + "site-1.key": serialize_pri_key(site_key), + "job_a.crt": job_a_crt, + "job_a.key": job_a_key, + "job_b.crt": job_b_crt, + "job_b.key": job_b_key, + } + paths = {} + for name, data in files.items(): + paths[name] = os.path.join(out_dir, name) + with open(paths[name], "wb") as f: + f.write(data) + return paths + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _run_server(root_url, pki, ready_q, stop_ev): + cell = None + try: + credentials = { + DriverParams.CA_CERT.value: pki["rootCA.pem"], + DriverParams.SERVER_CERT.value: pki["server.crt"], + DriverParams.SERVER_KEY.value: pki["server.key"], + DriverParams.CONNECTION_SECURITY.value: "mtls", + } + cell = Cell("server", root_url, secure=True, credentials=credentials, create_internal_listener=False) + cell.start() + ready_q.put("ready") + stop_ev.wait(120) + except Exception: + ready_q.put(traceback.format_exc()) + finally: + if cell: + cell.stop() + + +def _run_site_parent(root_url, pki, config_dir, ready_q, stop_ev): + cell = None + try: + # the internal listener scheme and security come from comm_config.json, as in a provisioned kit + ConfigService.initialize(section_files={}, config_path=[config_dir]) + credentials = { + DriverParams.CA_CERT.value: pki["rootCA.pem"], + DriverParams.CLIENT_CERT.value: pki["site-1.crt"], + DriverParams.CLIENT_KEY.value: pki["site-1.key"], + DriverParams.SERVER_CERT.value: pki["site-1.crt"], + DriverParams.SERVER_KEY.value: pki["site-1.key"], + DriverParams.CONNECTION_SECURITY.value: "mtls", + } + cell = Cell( + "site-1", + root_url, + secure=True, + credentials=credentials, + create_internal_listener=True, + auth_identity="site-1", + auth_identity_map={"site-1": "site-1", "server": "server"}, + ) + cell.register_request_cb(_CHANNEL, _TOPIC, lambda request: Message(payload=request.payload)) + cell.start() + deadline = time.time() + _CONNECT_TIMEOUT + while time.time() < deadline and not cell.is_cell_connected("server"): + time.sleep(0.1) + ready_q.put(cell.get_internal_listener_url()) + stop_ev.wait(120) + except Exception: + ready_q.put(traceback.format_exc()) + finally: + if cell: + cell.stop() + + +def _run_job_cell(parent_url, pki, cert_name, fqcn, wait, result_q): + cell = None + try: + credentials = { + DriverParams.CA_CERT.value: pki["rootCA.pem"], + DriverParams.CLIENT_CERT.value: pki[f"{cert_name}.crt"], + DriverParams.CLIENT_KEY.value: pki[f"{cert_name}.key"], + DriverParams.CONNECTION_SECURITY.value: "mtls", + } + cell = Cell( + fqcn, + parent_url, + secure=True, + credentials=credentials, + create_internal_listener=False, + parent_url=parent_url, + parent_resources={DriverParams.CONNECTION_SECURITY.value: "mtls"}, + auth_identity_map={"site-1": "site-1"}, + ) + cell.start() + deadline = time.time() + wait + while time.time() < deadline and not cell.is_cell_connected("site-1"): + time.sleep(0.1) + connected = cell.is_cell_connected("site-1") + rc = None + if connected: + reply = cell.send_request(_CHANNEL, _TOPIC, "site-1", Message(payload="hello"), timeout=5.0) + rc = reply.get_header(MessageHeaderKey.RETURN_CODE) + result_q.put({"connected": connected, "rc": rc}) + except Exception: + result_q.put({"error": traceback.format_exc()}) + finally: + if cell: + cell.stop() + + +@pytest.fixture(scope="module") +def site_parent(tmp_path_factory): + ctx = mp.get_context("spawn") + pki = _write_pki(str(tmp_path_factory.mktemp("pki"))) + config_dir = str(tmp_path_factory.mktemp("config")) + with open(os.path.join(config_dir, "comm_config.json"), "w") as f: + json.dump( + {"internal": {"scheme": "stcp", "resources": {"host": "localhost", "connection_security": "mtls"}}}, f + ) + root_url = f"stcp://localhost:{_free_port()}" + stop_ev = ctx.Event() + server_q, parent_q = ctx.Queue(), ctx.Queue() + server = ctx.Process(target=_run_server, args=(root_url, pki, server_q, stop_ev)) + server.start() + parent = None + try: + status = server_q.get(timeout=30) + assert status == "ready", status + parent = ctx.Process(target=_run_site_parent, args=(root_url, pki, config_dir, parent_q, stop_ev)) + parent.start() + internal_url = parent_q.get(timeout=40) + assert internal_url.startswith("stcp://"), internal_url + yield internal_url, pki + finally: + stop_ev.set() + if parent: + parent.join(15) + server.join(15) + + +def _job_cell(site_parent, cert_name, fqcn, wait): + internal_url, pki = site_parent + ctx = mp.get_context("spawn") + result_q = ctx.Queue() + proc = ctx.Process(target=_run_job_cell, args=(internal_url, pki, cert_name, fqcn, wait, result_q)) + proc.start() + try: + result = result_q.get(timeout=wait + 30) + finally: + proc.join(15) + assert "error" not in result, result.get("error") + return result + + +def test_site_parent_accepts_job_cell_with_its_own_job_cert(site_parent): + result = _job_cell(site_parent, "job_b", f"site-1.{_JOB_B}", _CONNECT_TIMEOUT) + + assert result["connected"] is True + assert result["rc"] == ReturnCode.OK + + +def test_site_parent_rejects_another_jobs_cert_on_job_fqcn(site_parent): + result = _job_cell(site_parent, "job_a", f"site-1.{_JOB_B}", _REJECT_WAIT) + + assert result["connected"] is False diff --git a/tests/unit_test/private/fed/server/fed_server_test.py b/tests/unit_test/private/fed/server/fed_server_test.py index 9444fde09d..b94350cc28 100644 --- a/tests/unit_test/private/fed/server/fed_server_test.py +++ b/tests/unit_test/private/fed/server/fed_server_test.py @@ -17,7 +17,7 @@ import pytest -from nvflare.apis.fl_constant import ConnPropKey, RunProcessKey +from nvflare.apis.fl_constant import ConnPropKey, RunProcessKey, SecureTrainConst from nvflare.apis.job_def import JobMetaKey, RunStatus from nvflare.apis.job_launcher_spec import JobReturnCode from nvflare.apis.shareable import Shareable @@ -26,6 +26,7 @@ from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey from nvflare.fuel.f3.cellnet.defs import ReturnCode as F3ReturnCode from nvflare.fuel.f3.cellnet.identity import ADMIN_LISTENER_KEY +from nvflare.fuel.f3.drivers.driver_params import DriverParams from nvflare.private.defs import CellChannel, CellMessageHeaderKeys, ClientRegMsgKey, JobFailureMsgKey, new_cell_message from nvflare.private.fed.authenticator import MISSING_CLIENT_FQCN from nvflare.private.fed.server.fed_server import BaseServer, FederatedServer @@ -441,6 +442,39 @@ def test_create_job_cell_uses_auth_identity_from_server_config(self): assert cell_cls.call_args.kwargs["auth_identity"] == "server-cn" assert cell_cls.call_args.kwargs["auth_identity_map"] == auth_identity_map + def test_create_job_cell_pins_both_tls_roles_to_job_credential(self): + server = object.__new__(FederatedServer) + server.engine = MagicMock() + + with ( + patch("nvflare.private.fed.server.fed_server.Cell") as cell_cls, + patch("nvflare.private.fed.server.fed_server.NetAgent") as net_agent_cls, + patch("nvflare.private.fed.server.fed_server.ServerCommandAgent") as command_agent_cls, + patch("nvflare.private.fed.server.fed_server.mpm.add_cleanup_cb"), + ): + cell_cls.return_value = MagicMock() + net_agent_cls.return_value = MagicMock() + command_agent_cls.return_value = MagicMock() + + server.create_job_cell( + "job-1", + "tcp://root", + "tcp://parent", + True, + { + SecureTrainConst.SSL_ROOT_CERT: "/ws/startup/rootCA.pem", + SecureTrainConst.SSL_CERT: "/ws/job-1/job_cert/job.crt", + SecureTrainConst.PRIVATE_KEY: "/ws/job-1/job_cert/job.key", + }, + ) + + credentials = cell_cls.call_args.kwargs["credentials"] + assert credentials[DriverParams.CA_CERT.value] == "/ws/startup/rootCA.pem" + for role in (DriverParams.SERVER_CERT, DriverParams.CLIENT_CERT): + assert credentials[role.value] == "/ws/job-1/job_cert/job.crt" + for role in (DriverParams.SERVER_KEY, DriverParams.CLIENT_KEY): + assert credentials[role.value] == "/ws/job-1/job_cert/job.key" + def test_set_cell_preserves_server_command_agent_aux_callback(self): server, engine, cell, aux_callback = self._create_job_cell_with_command_agent(HotState(ssid="ssid")) diff --git a/tests/unit_test/private/fed/server/job_runner_test.py b/tests/unit_test/private/fed/server/job_runner_test.py index 8a884d079e..a494145ace 100644 --- a/tests/unit_test/private/fed/server/job_runner_test.py +++ b/tests/unit_test/private/fed/server/job_runner_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os from contextlib import nullcontext from unittest.mock import ANY, MagicMock, call, patch @@ -19,13 +20,14 @@ import pytest from nvflare.apis.event_type import EventType -from nvflare.apis.fl_constant import FLContextKey, RunProcessKey +from nvflare.apis.fl_constant import ConfigVarName, FLContextKey, RunProcessKey, SystemConfigs from nvflare.apis.fl_context import FLContextManager from nvflare.apis.job_def import JobMetaKey, RunStatus from nvflare.apis.job_launcher_spec import JobReturnCode from nvflare.apis.utils.event import fire_event_to_components from nvflare.app_common.job_schedulers.job_scheduler import DefaultJobScheduler from nvflare.fuel.common.exit_codes import ProcessExitCode +from nvflare.fuel.utils.config_service import ConfigService from nvflare.fuel.utils.zip_utils import get_all_file_paths from nvflare.private.admin_defs import Message, MsgHeader, ReturnCode from nvflare.private.fed.server.job_runner import JobRunner, _FinishedJobState @@ -299,6 +301,20 @@ def _start_client_job_side_effect(passed_job, passed_client_sites, passed_fl_ctx assert seen_job_clients_meta["value"] == [{"name": "site-1"}, {"name": "site-2"}] +@pytest.mark.parametrize( + ("section", "file_name"), + [(SystemConfigs.STARTUP_CONF, "fed_server.json"), (SystemConfigs.RESOURCES_CONF, "resources.json")], +) +def test_job_cert_valid_days_read_from_server_config(tmp_path, section, file_name): + (tmp_path / file_name).write_text(json.dumps({ConfigVarName.JOB_CERT_VALID_DAYS: 7})) + ConfigService.reset() + ConfigService.initialize(section_files={section: file_name}, config_path=[str(tmp_path)]) + try: + assert JobRunner(workspace_root=str(tmp_path)).job_cert_valid_days == 7 + finally: + ConfigService.reset() + + def test_save_workspace_skips_duplicate_missing_sources(tmp_path): missing = str(tmp_path / "missing") runner, fl_ctx, job_manager = _make_workspace_save_inputs(missing, missing, missing, missing) From 2435538e74fca06a557c7bd7241c77e5f3cb461b Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Fri, 4 Sep 2026 09:03:19 -0700 Subject: [PATCH 05/10] fix(security): destroy CJ credential even when waiting on the job process fails The client removed the job credential only after job_handle.wait() returned normally. It is now removed in a finally block, and also when no job handle was registered, so an exceptional wait or a launch that never produced a handle cannot leave the credential on disk. --- nvflare/private/fed/client/client_executor.py | 10 ++-- .../fed/client/client_executor_test.py | 47 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/nvflare/private/fed/client/client_executor.py b/nvflare/private/fed/client/client_executor.py index b1a096a2ae..74f4718446 100644 --- a/nvflare/private/fed/client/client_executor.py +++ b/nvflare/private/fed/client/client_executor.py @@ -625,10 +625,14 @@ def _wait_child_process_finish( ): self.logger.info(f"run ({job_id}): waiting for child worker process to finish.") job_handle = self.run_processes.get(job_id, {}).get(RunProcessKey.JOB_HANDLE) + run_dir = Workspace.run_dir_path(workspace, job_id) + try: + if job_handle: + job_handle.wait() + finally: + # the job process is gone, or never started: its credential is dead either way + remove_job_cert(run_dir) if job_handle: - job_handle.wait() - remove_job_cert(Workspace.run_dir_path(workspace, job_id)) - return_code = get_return_code(job_handle, job_id, workspace, self.logger) with self.lock: diff --git a/tests/unit_test/private/fed/client/client_executor_test.py b/tests/unit_test/private/fed/client/client_executor_test.py index 1109c753e8..9815b059bc 100644 --- a/tests/unit_test/private/fed/client/client_executor_test.py +++ b/tests/unit_test/private/fed/client/client_executor_test.py @@ -615,6 +615,53 @@ def test_wait_child_process_destroys_job_credential_when_worker_exits(tmp_path): assert (run_dir / "app_site-1").is_dir() +def test_wait_child_process_destroys_job_credential_when_wait_raises(tmp_path): + client = MagicMock() + client.client_name = "site-1" + job_executor = JobExecutor(client=client, startup="startup") + job_handle = MagicMock() + job_handle.wait.side_effect = RuntimeError("launcher lost the job") + job_executor.run_processes = {"job-1": {RunProcessKey.JOB_HANDLE: job_handle}} + run_dir = tmp_path / "job-1" + run_dir.mkdir() + write_job_cert(str(run_dir), b"cert", b"key") + + with pytest.raises(RuntimeError, match="lost the job"): + job_executor._wait_child_process_finish( + client=client, + job_id="job-1", + allocated_resource=None, + token=None, + resource_manager=MagicMock(), + workspace=str(tmp_path), + fl_ctx=MagicMock(), + ) + + assert find_job_cert(str(run_dir)) is None + + +def test_wait_child_process_destroys_job_credential_without_job_handle(tmp_path): + client = MagicMock() + client.client_name = "site-1" + job_executor = JobExecutor(client=client, startup="startup") + job_executor.run_processes = {} + run_dir = tmp_path / "job-1" + run_dir.mkdir() + write_job_cert(str(run_dir), b"cert", b"key") + + job_executor._wait_child_process_finish( + client=client, + job_id="job-1", + allocated_resource=None, + token=None, + resource_manager=MagicMock(), + workspace=str(tmp_path), + fl_ctx=MagicMock(), + ) + + assert find_job_cert(str(run_dir)) is None + + def test_wait_child_process_preserves_launcher_infrastructure_error_over_rc_file(tmp_path): client = MagicMock() client.client_name = "site-1" From 04c2a9ec7e63c031b9224eb06e55ff20707913a7 Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Fri, 4 Sep 2026 10:50:28 -0700 Subject: [PATCH 06/10] test(security): cover secure deploy issuance and job-credential configer hooks - _deploy_job in secure mode with a real job CA: SJ credential written to the run dir (CN of the server cert, job id, leaf + job CA chain, key 0600) and the CJ credential carried in the per-site deploy message. - FLServerStarterConfiger / FLClientStarterConfiger point the job process at the job credential when one exists, and leave site paths alone otherwise. --- .../unit_test/private/fed/app/fl_conf_test.py | 90 +++++++++++++++++- .../fed/server/job_runner_deploy_test.py | 91 ++++++++++++++++++- 2 files changed, 178 insertions(+), 3 deletions(-) diff --git a/tests/unit_test/private/fed/app/fl_conf_test.py b/tests/unit_test/private/fed/app/fl_conf_test.py index 6ea3f476d2..3769a173b7 100644 --- a/tests/unit_test/private/fed/app/fl_conf_test.py +++ b/tests/unit_test/private/fed/app/fl_conf_test.py @@ -13,15 +13,19 @@ # limitations under the License. import logging +import os from types import SimpleNamespace +from unittest.mock import MagicMock, patch import pytest -from nvflare.apis.fl_constant import ConnectionSecurity, ConnPropKey +from nvflare.apis.fl_constant import ConnectionSecurity, ConnPropKey, SecureTrainConst, SiteType +from nvflare.apis.workspace import Workspace from nvflare.fuel.data_event.data_bus import DataBus from nvflare.fuel.data_event.utils import get_scope_property from nvflare.fuel.f3.cellnet.fqcn import FQCN -from nvflare.private.fed.app.fl_conf import FLClientStarterConfiger +from nvflare.private.fed.app.fl_conf import FLClientStarterConfiger, FLServerStarterConfiger +from nvflare.private.fed.utils.job_cert_utils import job_cert_paths, write_job_cert @pytest.fixture(autouse=True) @@ -66,3 +70,85 @@ def test_cp_conn_props_include_root_auth_identity(): cp_conn_props = get_scope_property("site-1", ConnPropKey.CP_CONN_PROPS) assert cp_conn_props[ConnPropKey.AUTH_IDENTITY] == "custom-site-cn" + + +def _workspace_with_kit(tmp_path, site_name): + (tmp_path / "startup").mkdir() + (tmp_path / "local").mkdir() + return Workspace(str(tmp_path), site_name=site_name) + + +@pytest.mark.parametrize( + ("job_id", "with_job_cert", "expect_job_cert"), + [("job-1", True, True), ("job-1", False, False), (None, True, False)], +) +def test_server_configer_points_job_process_at_job_credential(tmp_path, job_id, with_job_cert, expect_job_cert): + workspace = _workspace_with_kit(tmp_path, SiteType.SERVER) + run_dir = workspace.get_run_dir("job-1") + if with_job_cert: + write_job_cert(run_dir, b"cert", b"key") + configer = FLServerStarterConfiger.__new__(FLServerStarterConfiger) + configer.args = SimpleNamespace(job_id=job_id) + configer.workspace = workspace + configer.server_config_file_names = ["fed_server.json"] + server = { + SecureTrainConst.SSL_ROOT_CERT: "rootCA.pem", + SecureTrainConst.SSL_CERT: "server.crt", + SecureTrainConst.PRIVATE_KEY: "server.key", + } + configer.config_data = {"servers": [server]} + + with patch("nvflare.private.fed.app.fl_conf.JsonConfigurator.start_config"): + configer.start_config(MagicMock()) + + startup = workspace.get_startup_kit_dir() + assert server[SecureTrainConst.SSL_ROOT_CERT] == os.path.join(startup, "rootCA.pem") + if expect_job_cert: + assert (server[SecureTrainConst.SSL_CERT], server[SecureTrainConst.PRIVATE_KEY]) == job_cert_paths(run_dir) + else: + assert server[SecureTrainConst.SSL_CERT] == os.path.join(startup, "server.crt") + assert server[SecureTrainConst.PRIVATE_KEY] == os.path.join(startup, "server.key") + + +@pytest.mark.parametrize( + ("job_id", "with_job_cert", "expect_job_cert"), + [("job-1", True, True), ("job-1", False, False), (None, True, False)], +) +def test_client_configer_points_job_process_at_job_credential(tmp_path, job_id, with_job_cert, expect_job_cert): + workspace = _workspace_with_kit(tmp_path, "site-1") + run_dir = workspace.get_run_dir("job-1") + if with_job_cert: + write_job_cert(run_dir, b"cert", b"key") + configer = FLClientStarterConfiger.__new__(FLClientStarterConfiger) + configer.args = SimpleNamespace( + job_id=job_id, + sp_scheme="grpc", + sp_target="server.example.com:8002", + parent_url="stcp://localhost:8102", + parent_conn_sec="mtls", + ) + configer.workspace = workspace + configer.logger = logging.getLogger(__name__) + configer.cmd_vars = {"uid": "site-1"} + configer.client_config_file_names = ["fed_client.json"] + client = { + ConnPropKey.IDENTITY: "site-1", + SecureTrainConst.SSL_ROOT_CERT: "rootCA.pem", + SecureTrainConst.SSL_CERT: "client.crt", + SecureTrainConst.PRIVATE_KEY: "client.key", + } + configer.config_data = { + "servers": [{"service": {"scheme": "grpc", "target": "server.example.com:8002"}}], + "client": client, + } + + with patch("nvflare.private.fed.app.fl_conf.JsonConfigurator.start_config"): + configer.start_config(MagicMock()) + + startup = workspace.get_startup_kit_dir() + assert client[SecureTrainConst.SSL_ROOT_CERT] == os.path.join(startup, "rootCA.pem") + if expect_job_cert: + assert (client[SecureTrainConst.SSL_CERT], client[SecureTrainConst.PRIVATE_KEY]) == job_cert_paths(run_dir) + else: + assert client[SecureTrainConst.SSL_CERT] == os.path.join(startup, "client.crt") + assert client[SecureTrainConst.PRIVATE_KEY] == os.path.join(startup, "client.key") diff --git a/tests/unit_test/private/fed/server/job_runner_deploy_test.py b/tests/unit_test/private/fed/server/job_runner_deploy_test.py index 4cd48429a1..7e4aaefcc1 100644 --- a/tests/unit_test/private/fed/server/job_runner_deploy_test.py +++ b/tests/unit_test/private/fed/server/job_runner_deploy_test.py @@ -18,15 +18,25 @@ The test infrastructure stubs out all engine/fl_ctx interaction so that only _deploy_job()'s own logic is exercised.""" +import os from unittest.mock import MagicMock, patch import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.x509.oid import NameOID from nvflare.apis.client import Client -from nvflare.apis.fl_constant import FLContextKey +from nvflare.apis.fl_constant import FLContextKey, SecureTrainConst, SiteType from nvflare.apis.job_def import Job +from nvflare.apis.workspace import Workspace +from nvflare.fuel.f3.drivers.net_utils import get_cert_job_id +from nvflare.lighter.constants import CertExtensionOID, ProvFileName +from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert, serialize_pri_key from nvflare.private.admin_defs import Message, MsgHeader, ReturnCode +from nvflare.private.defs import RequestHeader from nvflare.private.fed.server.job_runner import JobRunner +from nvflare.private.fed.utils.job_cert_utils import job_cert_paths, read_job_cert, unpack_job_cert_header # --------------------------------------------------------------------------- # Helpers @@ -122,6 +132,85 @@ def test_secure_deploy_fails_without_job_ca(tmp_path): engine.server.admin_server.send_requests_and_get_reply_dict.assert_not_called() +def _write_server_kit_with_job_ca(startup): + root_key, root_pub = generate_keys() + root_cert = generate_cert(Identity("rootCA"), Identity("rootCA"), root_key, root_pub, ca=True) + server_key, server_pub = generate_keys() + server_cert = generate_cert(Identity("server-cn"), Identity("rootCA"), root_key, server_pub) + job_ca_key, job_ca_pub = generate_keys() + marker = x509.UnrecognizedExtension(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER), b"job_ca") + job_ca_cert = generate_cert( + Identity("job_ca"), + Identity("rootCA"), + root_key, + job_ca_pub, + ca=True, + ca_path_length=0, + extra_extensions=[(marker, False)], + ) + (startup / "rootCA.pem").write_bytes(serialize_cert(root_cert)) + (startup / "server.crt").write_bytes(serialize_cert(server_cert)) + (startup / "server.key").write_bytes(serialize_pri_key(server_key)) + (startup / ProvFileName.JOB_CA_CERT).write_bytes(serialize_cert(job_ca_cert)) + (startup / ProvFileName.JOB_CA_KEY).write_bytes(serialize_pri_key(job_ca_key)) + + +def _common_name(cert): + return cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + + +def test_secure_deploy_issues_server_and_client_job_credentials(tmp_path): + startup = tmp_path / "startup" + startup.mkdir() + (tmp_path / "local").mkdir() + _write_server_kit_with_job_ca(startup) + runner, fl_ctx, engine, job, sites = _build_fl_ctx({"tok-1": _ok_reply()}) + runner.workspace_root = str(tmp_path) + job.get_deployment.return_value = {"app": [SiteType.SERVER, "site-1"]} + props = { + FLContextKey.SECURE_MODE: True, + FLContextKey.SERVER_CONFIG: [{SecureTrainConst.SSL_CERT: str(startup / "server.crt")}], + } + fl_ctx.get_prop.side_effect = lambda key, default=None: props.get(key, default) + fl_ctx.set_prop.side_effect = lambda key, val, **kw: props.__setitem__(key, val) + deploy_requests = [] + + def make_deploy_message(*_args, **_kwargs): + request = Message(topic="deploy", body=b"app") + deploy_requests.append(request) + return request + + with ( + patch.object(runner, "_make_deploy_message", side_effect=make_deploy_message), + patch("nvflare.private.fed.server.job_runner.AppDeployer") as deployer_cls, + patch("nvflare.private.fed.server.job_runner.require_signed_jobs", return_value=False), + ): + deployer_cls.return_value.deploy.return_value = "" + job_id, failed_clients = runner._deploy_job(job, sites, fl_ctx) + + assert (job_id, failed_clients) == ("job-1", []) + assert "server: OK" in props[FLContextKey.JOB_DEPLOY_DETAIL] + + run_dir = Workspace(root_dir=str(tmp_path), site_name=SiteType.SERVER).get_run_dir("job-1") + sj_cert_pem, sj_key_pem = read_job_cert(run_dir) + sj_cert = x509.load_pem_x509_certificate(sj_cert_pem) + assert _common_name(sj_cert) == "server-cn" + assert get_cert_job_id(sj_cert) == "job-1" + assert sj_cert_pem.count(b"BEGIN CERTIFICATE") == 2 # leaf + job CA, chains to the root + assert oct(os.stat(job_cert_paths(run_dir)[1]).st_mode & 0o777) == "0o600" + assert serialization.load_pem_private_key(sj_key_pem, None).public_key() == sj_cert.public_key() + + [deploy_request] = deploy_requests + cj_cert_pem, cj_key_pem = unpack_job_cert_header(deploy_request.get_header(RequestHeader.JOB_CERT)) + cj_cert = x509.load_pem_x509_certificate(cj_cert_pem) + assert _common_name(cj_cert) == "site-1" + assert get_cert_job_id(cj_cert) == "job-1" + assert cj_cert.issuer == x509.load_pem_x509_certificate((startup / ProvFileName.JOB_CA_CERT).read_bytes()).subject + assert cj_cert.public_key() != sj_cert.public_key() + assert serialization.load_pem_private_key(cj_key_pem, None).public_key() == cj_cert.public_key() + engine.server.admin_server.send_requests_and_get_reply_dict.assert_called_once() + + # --------------------------------------------------------------------------- # Deployment timeout classified as failure # --------------------------------------------------------------------------- From 3336583becb4019a7a4864a625676f07b9bb98e0 Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Fri, 4 Sep 2026 12:49:02 -0700 Subject: [PATCH 07/10] fix(security): import gRPC peer job-id helper; bind job certs to the owner-relative job segment Review follow-ups on #5263: - grpc_driver.py / aio_grpc_driver.py called add_grpc_peer_job_id without importing it, so every authenticated gRPC stream failed with NameError. Both servicers now import it; new tests drive their authenticated stream paths with site and job certificates. setup.cfg ignores F821, which is why flake8 never reported the undefined name. - FQCN.belongs_to_job matched the job id in any segment, so job A's cert was accepted on site-1..ws_transfer_. The job segment must now be the one right after the owner's prefix, which the identity resolver reports alongside the expected identity (resolve_owner). Covered in the resolver, handshake, certificate-cache and live mTLS tests. - The live tests asserted the child's transient connected flag, which can read True before the parent rejects the handshake. They now assert that an application request fails and that the parent logged the rejection. --- docs/design/per_job_certs_design.md | 9 ++- nvflare/fuel/f3/cellnet/fqcn.py | 15 ++-- nvflare/fuel/f3/cellnet/identity.py | 28 ++++--- nvflare/fuel/f3/drivers/aio_grpc_driver.py | 2 +- nvflare/fuel/f3/drivers/grpc_driver.py | 2 +- .../workspace_transfer_bootstrap_mtls_test.py | 48 ++++++++--- .../fuel/f3/cellnet/identity_binding_test.py | 35 +++++++- .../f3/cellnet/job_cert_binding_live_test.py | 62 +++++++++++---- .../fuel/f3/drivers/grpc_driver_test.py | 79 +++++++++++++++++++ 9 files changed, 230 insertions(+), 50 deletions(-) create mode 100644 tests/unit_test/fuel/f3/drivers/grpc_driver_test.py diff --git a/docs/design/per_job_certs_design.md b/docs/design/per_job_certs_design.md index 891b330b12..15fca392ea 100644 --- a/docs/design/per_job_certs_design.md +++ b/docs/design/per_job_certs_design.md @@ -223,10 +223,13 @@ stops one job's credential from acting as another job's cell: `PEER_JOB_ID` connection property next to `PEER_CN`. - `CellIdentityResolver.require_match()` rejects a peer whose certificate is bound to job X unless the FQCN it claims belongs to that job - (`FQCN.belongs_to_job`): the job cell `.X` and its descendants, or an - auxiliary job cell named `_X` directly under the site, such as the + (`FQCN.belongs_to_job`): the segment right after the owning site's prefix + (as resolved for the identity check) must be X — the job cell `.X` and + its descendants — or an auxiliary job cell named `_X`, such as the Kubernetes workspace-transfer bootstrap cell `server.ws_transfer_X`, which - authenticates with the job credential before the job cell exists. The check runs at the + authenticates with the job credential before the job cell exists. The + position matters: `.Y.ws_transfer_X` is a cell of job Y and is rejected + for job X's certificate. The check runs at the connection handshake (`ConnManager`) and again on the certificate exchanged for message-level crypto (`CredentialManager`), which is the certificate later used to decrypt that peer's messages. diff --git a/nvflare/fuel/f3/cellnet/fqcn.py b/nvflare/fuel/f3/cellnet/fqcn.py index 80f82eac13..a4528f7b9c 100644 --- a/nvflare/fuel/f3/cellnet/fqcn.py +++ b/nvflare/fuel/f3/cellnet/fqcn.py @@ -17,8 +17,9 @@ class FQCN(FQN): VALID_PATTERN = "^[A-Za-z0-9_.~-]*$" - # A job's cells are the job cell (.) and its descendants, plus auxiliary cells - # named _ directly under the site (e.g. the workspace-transfer bootstrap cell). + # A job's cells hang directly off the owning site: . (the job cell) and its + # descendants, or an auxiliary cell ._ (e.g. the workspace-transfer + # bootstrap cell). Nothing deeper counts: a cell under another job's cell is that job's. JOB_AUX_SEPARATOR = "_" @staticmethod @@ -26,11 +27,15 @@ def job_aux_name(name: str, job_id: str) -> str: return f"{name}{FQCN.JOB_AUX_SEPARATOR}{job_id}" @staticmethod - def belongs_to_job(fqcn: str, job_id: str) -> bool: + def belongs_to_job(fqcn: str, job_id: str, owner_segments: int = 1) -> bool: + """True if the segment right after the owner's prefix is the job id or a _ auxiliary name.""" if not job_id: return False - aux_suffix = FQCN.JOB_AUX_SEPARATOR + job_id - return any(seg == job_id or seg.endswith(aux_suffix) for seg in FQCN.split(FQCN.normalize(fqcn))) + parts = FQCN.split(FQCN.normalize(fqcn)) + if len(parts) <= owner_segments: + return False + job_segment = parts[owner_segments] + return job_segment == job_id or job_segment.endswith(FQCN.JOB_AUX_SEPARATOR + job_id) # A network Attach trainer connects beneath the stable site CP and authenticates diff --git a/nvflare/fuel/f3/cellnet/identity.py b/nvflare/fuel/f3/cellnet/identity.py index dd2ceaea3f..9fb999f792 100644 --- a/nvflare/fuel/f3/cellnet/identity.py +++ b/nvflare/fuel/f3/cellnet/identity.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional +from typing import Optional, Tuple from cryptography import x509 from cryptography.x509.oid import NameOID @@ -125,15 +125,23 @@ def _resolve_local_child_identity(self, fqcn: str) -> Optional[str]: return parts[0] if parts else None def resolve(self, fqcn: str) -> Optional[str]: + return self.resolve_owner(fqcn)[0] + + def resolve_owner(self, fqcn: str) -> Tuple[Optional[str], int]: + """The identity expected for an FQCN and how many leading segments name that identity's owner. + + A job's cells hang directly off their owner, so the job segment is the first one after the + owner's prefix (see FQCN.belongs_to_job). + """ if not fqcn: - return None + return None, 0 fqcn = FQCN.normalize(fqcn) + parts = FQCN.split(fqcn) identity = self.exact_identity_map.get(fqcn) if identity: - return identity + return identity, len(parts) - parts = FQCN.split(fqcn) leaf = parts[-1] for i in range(len(parts), 0, -1): prefix = FQCN.join(parts[:i]) @@ -142,22 +150,22 @@ def resolve(self, fqcn: str) -> Optional[str]: identity = self.prefix_identity_map.get(prefix) if identity: - return identity + return identity, i # Network Attach trainers are children of the stable site CP and use # that site's provisioned certificate. The dynamic CJ is a sibling and # remains the application-level task/result boundary. if len(parts) > 1 and leaf.startswith(CLIENT_API_ATTACH_LEAF_PREFIX): - return self.resolve(FQCN.join(parts[:-1])) + return self.resolve_owner(FQCN.join(parts[:-1])) identity = self._resolve_local_child_identity(fqcn) if identity: - return identity + return identity, len(FQCN.split(self.local_fqcn)) - return parts[0] if parts else fqcn + return parts[0], 1 def require_match(self, fqcn: str, peer_cn: str, peer_desc: str, peer_job_id: Optional[str] = None): - expected_cn = self.resolve(fqcn) + expected_cn, owner_segments = self.resolve_owner(fqcn) if not expected_cn: raise ValueError(f"{peer_desc} claimed endpoint '{fqcn}' does not resolve to an expected identity") @@ -165,7 +173,7 @@ def require_match(self, fqcn: str, peer_cn: str, peer_desc: str, peer_job_id: Op raise ValueError(f"{peer_desc} does not have an authenticated mTLS peer common name") # A per-job certificate may only authenticate cells of that job. - if peer_job_id is not None and not FQCN.belongs_to_job(fqcn, peer_job_id): + if peer_job_id is not None and not FQCN.belongs_to_job(fqcn, peer_job_id, owner_segments): raise ValueError( f"{peer_desc} authenticated with a certificate bound to job '{peer_job_id}' " f"but claimed endpoint '{fqcn}' is not part of that job" diff --git a/nvflare/fuel/f3/drivers/aio_grpc_driver.py b/nvflare/fuel/f3/drivers/aio_grpc_driver.py index 16498f80e3..aa60d35c8c 100644 --- a/nvflare/fuel/f3/drivers/aio_grpc_driver.py +++ b/nvflare/fuel/f3/drivers/aio_grpc_driver.py @@ -39,7 +39,7 @@ from .driver_params import DriverCap, DriverParams from .grpc.streamer_pb2 import Frame from .grpc.utils import get_grpc_client_credentials, get_grpc_server_credentials, use_aio_grpc -from .net_utils import MAX_FRAME_SIZE, get_address, get_tcp_urls, ssl_required +from .net_utils import MAX_FRAME_SIZE, add_grpc_peer_job_id, get_address, get_tcp_urls, ssl_required GRPC_DEFAULT_OPTIONS = [ ("grpc.max_send_message_length", MAX_FRAME_SIZE), diff --git a/nvflare/fuel/f3/drivers/grpc_driver.py b/nvflare/fuel/f3/drivers/grpc_driver.py index ca70f0925f..fc61189f63 100644 --- a/nvflare/fuel/f3/drivers/grpc_driver.py +++ b/nvflare/fuel/f3/drivers/grpc_driver.py @@ -37,7 +37,7 @@ from .grpc.qq import QQ from .grpc.streamer_pb2 import Frame from .grpc.utils import get_grpc_client_credentials, get_grpc_server_credentials, use_aio_grpc -from .net_utils import MAX_FRAME_SIZE, get_address, get_tcp_urls, ssl_required +from .net_utils import MAX_FRAME_SIZE, add_grpc_peer_job_id, get_address, get_tcp_urls, ssl_required GRPC_DEFAULT_OPTIONS = [ ("grpc.max_send_message_length", MAX_FRAME_SIZE), diff --git a/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py b/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py index 0aeb7bd2f2..86a8c8950e 100644 --- a/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py +++ b/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py @@ -14,6 +14,7 @@ """Live mTLS regression: the workspace-transfer bootstrap cell authenticates to its parent with a job-bound cert.""" +import logging import multiprocessing as mp import os import socket @@ -37,10 +38,25 @@ _TOPIC = "echo" _CONNECT_TIMEOUT = 10.0 _REJECT_WAIT = 3.0 +_REQUEST_TIMEOUT = 3.0 +_REJECTION_LOG_WAIT = 10.0 _JOB_ID = str(uuid.uuid4()) _OTHER_JOB_ID = str(uuid.uuid4()) +class _RejectionRecorder(logging.Handler): + """Forwards the parent's job-binding rejections to the test process.""" + + def __init__(self, queue): + super().__init__(level=logging.ERROR) + self.queue = queue + + def emit(self, record): + message = record.getMessage() + if "bound to job" in message: + self.queue.put(message) + + def _write_pki(out_dir: str) -> dict: root_key, root_pub = generate_keys() root_cert = generate_cert(Identity("rootCA"), Identity("rootCA"), root_key, root_pub, ca=True) @@ -83,9 +99,10 @@ def _free_port() -> int: return s.getsockname()[1] -def _run_parent(root_url, pki, ready_q, stop_ev): +def _run_parent(root_url, pki, ready_q, stop_ev, reject_q): cell = None try: + logging.getLogger().addHandler(_RejectionRecorder(reject_q)) credentials = { DriverParams.CA_CERT.value: pki["rootCA.pem"], DriverParams.SERVER_CERT.value: pki["server.crt"], @@ -127,12 +144,10 @@ def _run_bootstrap(root_url, pki, cert_name, fqcn, wait, result_q): deadline = time.time() + wait while time.time() < deadline and not cell.is_cell_connected("server"): time.sleep(0.1) - connected = cell.is_cell_connected("server") - rc = None - if connected: - reply = cell.send_request(_CHANNEL, _TOPIC, "server", Message(payload="hello"), timeout=5.0) - rc = reply.get_header(MessageHeaderKey.RETURN_CODE) - result_q.put({"connected": connected, "rc": rc}) + # the client-side connected flag is transient while the parent is still validating the handshake; + # only a completed application request proves the parent accepted this cell + reply = cell.send_request(_CHANNEL, _TOPIC, "server", Message(payload="hello"), timeout=_REQUEST_TIMEOUT) + result_q.put({"rc": reply.get_header(MessageHeaderKey.RETURN_CODE)}) except Exception: result_q.put({"error": traceback.format_exc()}) finally: @@ -145,20 +160,20 @@ def parent(tmp_path_factory): ctx = mp.get_context("spawn") pki = _write_pki(str(tmp_path_factory.mktemp("pki"))) root_url = f"stcp://localhost:{_free_port()}" - ready_q, stop_ev = ctx.Queue(), ctx.Event() - proc = ctx.Process(target=_run_parent, args=(root_url, pki, ready_q, stop_ev)) + ready_q, reject_q, stop_ev = ctx.Queue(), ctx.Queue(), ctx.Event() + proc = ctx.Process(target=_run_parent, args=(root_url, pki, ready_q, stop_ev, reject_q)) proc.start() try: status = ready_q.get(timeout=30) assert status == "ready", status - yield root_url, pki + yield root_url, pki, reject_q finally: stop_ev.set() proc.join(15) def _bootstrap(parent, cert_name, wait): - root_url, pki = parent + root_url, pki, _ = parent ctx = mp.get_context("spawn") result_q = ctx.Queue() fqcn = make_workspace_transfer_fqcn("server", _JOB_ID) @@ -175,11 +190,18 @@ def _bootstrap(parent, cert_name, wait): def test_bootstrap_cell_authenticates_with_its_jobs_cert(parent): result = _bootstrap(parent, "job", _CONNECT_TIMEOUT) - assert result["connected"] is True assert result["rc"] == ReturnCode.OK def test_bootstrap_cell_rejected_with_another_jobs_cert(parent): + fqcn = make_workspace_transfer_fqcn("server", _JOB_ID) result = _bootstrap(parent, "other_job", _REJECT_WAIT) - assert result["connected"] is False + assert result["rc"] != ReturnCode.OK + reject_q = parent[2] + deadline = time.time() + _REJECTION_LOG_WAIT + while True: + message = reject_q.get(timeout=max(0.1, deadline - time.time())) + if fqcn in message: + break + assert f"bound to job '{_OTHER_JOB_ID}'" in message diff --git a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py index fb7d9a712a..8a5b179a68 100644 --- a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py +++ b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py @@ -464,6 +464,9 @@ def test_mtls_certificate_cache_accepts_configured_auth_identity_for_site_cert_c ("site-1.ws_transferjob-123", False), ("site-1.job-1234", False), ("site-1.xjob-123", False), + ("site-1.job-999.ws_transfer_job-123", False), + ("site-1.job-999.job-123", False), + ("site-1.job-123.ws_transfer_job-999", True), ], ) def test_fqcn_belongs_to_job(fqcn, expected): @@ -474,6 +477,32 @@ def test_fqcn_belongs_to_job_rejects_empty_job_id(): assert FQCN.belongs_to_job("site-1.ws_transfer_", "") is False +def test_fqcn_belongs_to_job_places_the_job_after_the_owner_prefix(): + assert FQCN.belongs_to_job("relay-1.site-1.job-123", "job-123", owner_segments=2) is True + assert FQCN.belongs_to_job("relay-1.site-1.ws_transfer_job-123", "job-123", owner_segments=2) is True + assert FQCN.belongs_to_job("relay-1.site-1.job-123", "job-123", owner_segments=1) is False + assert FQCN.belongs_to_job("relay-1.site-1", "job-123", owner_segments=2) is False + + +def test_identity_resolver_binds_job_cert_below_the_owning_site_only(): + resolver = CellIdentityResolver(local_fqcn="server", prefix_identity_map={"site-1": "site-1"}) + + with pytest.raises(ValueError, match="bound to job 'job-123'"): + resolver.require_match( + "site-1.job-999.ws_transfer_job-123", "site-1", "connection bootstrap", peer_job_id="job-123" + ) + with pytest.raises(ValueError, match="bound to job 'job-123'"): + resolver.require_match("site-1.job-999.job-123", "site-1", "connection sub", peer_job_id="job-123") + + +def test_identity_resolver_binds_job_cert_behind_relay(): + resolver = CellIdentityResolver(local_fqcn="relay-1", prefix_identity_map={"relay-1.site-1": "site-1"}) + + resolver.require_match("relay-1.site-1.job-123", "site-1", "connection cj", peer_job_id="job-123") + with pytest.raises(ValueError, match="bound to job 'job-123'"): + resolver.require_match("relay-1.site-1", "site-1", "connection cp", peer_job_id="job-123") + + def test_identity_resolver_binds_job_cert_to_job_fqcn(): resolver = CellIdentityResolver(local_fqcn="server", prefix_identity_map={"site-1": "site-1"}) @@ -505,7 +534,9 @@ def test_mtls_handshake_accepts_job_cert_for_own_job(endpoint_name): assert not conn.closed -@pytest.mark.parametrize("endpoint_name", ["site-1.job-999", "site-1", "site-1.ws_transfer_job-999"]) +@pytest.mark.parametrize( + "endpoint_name", ["site-1.job-999", "site-1", "site-1.ws_transfer_job-999", "site-1.job-999.ws_transfer_job-123"] +) def test_mtls_handshake_rejects_job_cert_outside_its_job(endpoint_name): manager = _conn_manager(identity_map={"site-1": "site-1"}) conn = _FakeConnection(peer_cn="site-1", peer_job_id="job-123") @@ -527,7 +558,7 @@ def test_mtls_certificate_cache_binds_job_cert_to_job_fqcn(): own_job = Message(headers={MessageHeaderKey.ORIGIN: "site-1.job-123"}, payload={CERT_CONTENT: cert}) assert manager.process_response(own_job) == cert - for origin in ("site-1", "site-1.job-999"): + for origin in ("site-1", "site-1.job-999", "site-1.job-999.ws_transfer_job-123"): with pytest.raises(RuntimeError, match="bound to job 'job-123'"): manager.process_response(Message(headers={MessageHeaderKey.ORIGIN: origin}, payload={CERT_CONTENT: cert})) assert origin not in manager.cert_cache diff --git a/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py b/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py index e8b4a06df9..8575549268 100644 --- a/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py +++ b/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py @@ -15,6 +15,7 @@ """Live mTLS regression: a site parent's internal listener binds job certificates to their job's FQCNs.""" import json +import logging import multiprocessing as mp import os import socket @@ -38,10 +39,34 @@ _TOPIC = "echo" _CONNECT_TIMEOUT = 10.0 _REJECT_WAIT = 3.0 +_REQUEST_TIMEOUT = 3.0 +_REJECTION_LOG_WAIT = 10.0 _JOB_A = str(uuid.uuid4()) _JOB_B = str(uuid.uuid4()) +class _RejectionRecorder(logging.Handler): + """Forwards the parent's job-binding rejections to the test process.""" + + def __init__(self, queue): + super().__init__(level=logging.ERROR) + self.queue = queue + + def emit(self, record): + message = record.getMessage() + if "bound to job" in message: + self.queue.put(message) + + +def _await_rejection_of(reject_q, fqcn: str) -> str: + deadline = time.time() + _REJECTION_LOG_WAIT + while time.time() < deadline: + message = reject_q.get(timeout=max(0.1, deadline - time.time())) + if fqcn in message: + return message + raise AssertionError(f"parent logged no binding rejection for {fqcn}") + + def _write_pki(out_dir: str) -> dict: root_key, root_pub = generate_keys() root_cert = generate_cert(Identity("rootCA"), Identity("rootCA"), root_key, root_pub, ca=True) @@ -112,9 +137,10 @@ def _run_server(root_url, pki, ready_q, stop_ev): cell.stop() -def _run_site_parent(root_url, pki, config_dir, ready_q, stop_ev): +def _run_site_parent(root_url, pki, config_dir, ready_q, stop_ev, reject_q): cell = None try: + logging.getLogger().addHandler(_RejectionRecorder(reject_q)) # the internal listener scheme and security come from comm_config.json, as in a provisioned kit ConfigService.initialize(section_files={}, config_path=[config_dir]) credentials = { @@ -171,12 +197,10 @@ def _run_job_cell(parent_url, pki, cert_name, fqcn, wait, result_q): deadline = time.time() + wait while time.time() < deadline and not cell.is_cell_connected("site-1"): time.sleep(0.1) - connected = cell.is_cell_connected("site-1") - rc = None - if connected: - reply = cell.send_request(_CHANNEL, _TOPIC, "site-1", Message(payload="hello"), timeout=5.0) - rc = reply.get_header(MessageHeaderKey.RETURN_CODE) - result_q.put({"connected": connected, "rc": rc}) + # the client-side connected flag is transient while the parent is still validating the handshake; + # only a completed application request proves the parent accepted this cell + reply = cell.send_request(_CHANNEL, _TOPIC, "site-1", Message(payload="hello"), timeout=_REQUEST_TIMEOUT) + result_q.put({"rc": reply.get_header(MessageHeaderKey.RETURN_CODE)}) except Exception: result_q.put({"error": traceback.format_exc()}) finally: @@ -195,18 +219,18 @@ def site_parent(tmp_path_factory): ) root_url = f"stcp://localhost:{_free_port()}" stop_ev = ctx.Event() - server_q, parent_q = ctx.Queue(), ctx.Queue() + server_q, parent_q, reject_q = ctx.Queue(), ctx.Queue(), ctx.Queue() server = ctx.Process(target=_run_server, args=(root_url, pki, server_q, stop_ev)) server.start() parent = None try: status = server_q.get(timeout=30) assert status == "ready", status - parent = ctx.Process(target=_run_site_parent, args=(root_url, pki, config_dir, parent_q, stop_ev)) + parent = ctx.Process(target=_run_site_parent, args=(root_url, pki, config_dir, parent_q, stop_ev, reject_q)) parent.start() internal_url = parent_q.get(timeout=40) assert internal_url.startswith("stcp://"), internal_url - yield internal_url, pki + yield internal_url, pki, reject_q finally: stop_ev.set() if parent: @@ -215,7 +239,7 @@ def site_parent(tmp_path_factory): def _job_cell(site_parent, cert_name, fqcn, wait): - internal_url, pki = site_parent + internal_url, pki, _ = site_parent ctx = mp.get_context("spawn") result_q = ctx.Queue() proc = ctx.Process(target=_run_job_cell, args=(internal_url, pki, cert_name, fqcn, wait, result_q)) @@ -231,11 +255,19 @@ def _job_cell(site_parent, cert_name, fqcn, wait): def test_site_parent_accepts_job_cell_with_its_own_job_cert(site_parent): result = _job_cell(site_parent, "job_b", f"site-1.{_JOB_B}", _CONNECT_TIMEOUT) - assert result["connected"] is True assert result["rc"] == ReturnCode.OK -def test_site_parent_rejects_another_jobs_cert_on_job_fqcn(site_parent): - result = _job_cell(site_parent, "job_a", f"site-1.{_JOB_B}", _REJECT_WAIT) +@pytest.mark.parametrize( + "claimed_fqcn", + [ + f"site-1.{_JOB_B}", # job B's own cell + f"site-1.{_JOB_B}.ws_transfer_{_JOB_A}", # an auxiliary name below job B's cell is still job B's + ], +) +def test_site_parent_rejects_another_jobs_cert_on_job_fqcn(site_parent, claimed_fqcn): + result = _job_cell(site_parent, "job_a", claimed_fqcn, _REJECT_WAIT) - assert result["connected"] is False + assert result["rc"] != ReturnCode.OK + rejection = _await_rejection_of(site_parent[2], claimed_fqcn) + assert f"bound to job '{_JOB_A}'" in rejection diff --git a/tests/unit_test/fuel/f3/drivers/grpc_driver_test.py b/tests/unit_test/fuel/f3/drivers/grpc_driver_test.py new file mode 100644 index 0000000000..8801801626 --- /dev/null +++ b/tests/unit_test/fuel/f3/drivers/grpc_driver_test.py @@ -0,0 +1,79 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from cryptography import x509 + +from nvflare.fuel.f3.drivers import aio_grpc_driver, grpc_driver +from nvflare.fuel.f3.drivers.driver_params import DriverParams +from nvflare.fuel.f3.drivers.net_utils import JOB_ID_EXTENSION_OID +from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert + + +def _cert_pem(common_name: str, job_id=None) -> bytes: + key, pub_key = generate_keys() + extensions = None + if job_id: + extensions = [(x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, job_id.encode("utf-8")), False)] + cert = generate_cert(Identity(common_name), Identity(common_name), key, pub_key, extra_extensions=extensions) + return serialize_cert(cert) + + +def _authenticated_context(cert_pem: bytes): + context = MagicMock() + context.peer.return_value = "ipv4:127.0.0.1:50001" + context.auth_context.return_value = {"x509_common_name": [b"site-1"], "x509_pem_cert": [cert_pem]} + return context + + +def _server(): + server = MagicMock() + server.connector.params = {DriverParams.HOST.value: "localhost", DriverParams.PORT.value: "8002"} + return server + + +@pytest.mark.parametrize("job_id", ["job-1", None]) +def test_grpc_servicer_records_peer_identity_of_authenticated_stream(job_id): + servicer = grpc_driver.Servicer(_server()) + + with ( + patch.object(grpc_driver, "StreamConnection") as connection_cls, + patch.object(grpc_driver.threading, "Thread"), + ): + connection_cls.return_value.generate_output.return_value = iter([]) + list(servicer.Stream(iter([]), _authenticated_context(_cert_pem("site-1", job_id)))) + + conn_props = connection_cls.call_args.args[2] + assert conn_props[DriverParams.PEER_CN.value] == "site-1" + assert conn_props.get(DriverParams.PEER_JOB_ID.value) == job_id + + +@pytest.mark.parametrize("job_id", ["job-1", None]) +def test_aio_grpc_servicer_records_peer_identity_of_authenticated_stream(job_id): + servicer = aio_grpc_driver.Servicer(_server(), aio_ctx=MagicMock()) + + async def consume(): + async for _ in servicer.Stream(iter([]), _authenticated_context(_cert_pem("site-1", job_id))): + pass + + with patch.object(aio_grpc_driver, "AioStreamSession") as session_cls: + session_cls.return_value.read_oq = AsyncMock(side_effect=asyncio.CancelledError()) + asyncio.run(consume()) + + conn_props = session_cls.call_args.kwargs["conn_props"] + assert conn_props[DriverParams.PEER_CN.value] == "site-1" + assert conn_props.get(DriverParams.PEER_JOB_ID.value) == job_id From e86f265a6e7b92e40a557c6b2644bb05e4301ceb Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Wed, 9 Sep 2026 14:46:44 -0700 Subject: [PATCH 08/10] test(security): fix xdist collection mismatch in live mTLS tests The live tests generated job ids with uuid4() at import time, so each pytest-xdist worker collected different parametrized test ids and CI failed with "Different tests were collected". Use fixed ids. --- .../workspace_transfer_bootstrap_mtls_test.py | 6 +++--- .../fuel/f3/cellnet/job_cert_binding_live_test.py | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py b/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py index 86a8c8950e..3d532070a6 100644 --- a/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py +++ b/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py @@ -20,7 +20,6 @@ import socket import time import traceback -import uuid import pytest from cryptography import x509 @@ -40,8 +39,9 @@ _REJECT_WAIT = 3.0 _REQUEST_TIMEOUT = 3.0 _REJECTION_LOG_WAIT = 10.0 -_JOB_ID = str(uuid.uuid4()) -_OTHER_JOB_ID = str(uuid.uuid4()) +# fixed ids: pytest-xdist requires identical test ids on every worker +_JOB_ID = "11111111-0000-4000-8000-000000000001" +_OTHER_JOB_ID = "22222222-0000-4000-8000-000000000002" class _RejectionRecorder(logging.Handler): diff --git a/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py b/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py index 8575549268..07e9fdfc18 100644 --- a/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py +++ b/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py @@ -21,7 +21,6 @@ import socket import time import traceback -import uuid import pytest from cryptography import x509 @@ -41,8 +40,9 @@ _REJECT_WAIT = 3.0 _REQUEST_TIMEOUT = 3.0 _REJECTION_LOG_WAIT = 10.0 -_JOB_A = str(uuid.uuid4()) -_JOB_B = str(uuid.uuid4()) +# fixed ids: pytest-xdist requires identical test ids on every worker +_JOB_A = "aaaaaaaa-0000-4000-8000-00000000000a" +_JOB_B = "bbbbbbbb-0000-4000-8000-00000000000b" class _RejectionRecorder(logging.Handler): @@ -261,9 +261,10 @@ def test_site_parent_accepts_job_cell_with_its_own_job_cert(site_parent): @pytest.mark.parametrize( "claimed_fqcn", [ - f"site-1.{_JOB_B}", # job B's own cell + f"site-1.{_JOB_B}", f"site-1.{_JOB_B}.ws_transfer_{_JOB_A}", # an auxiliary name below job B's cell is still job B's ], + ids=["job_b_cell", "aux_cell_below_job_b"], ) def test_site_parent_rejects_another_jobs_cert_on_job_fqcn(site_parent, claimed_fqcn): result = _job_cell(site_parent, "job_a", claimed_fqcn, _REJECT_WAIT) From 15c4cdb784227dd429645ed7fbbe2fb7eb9c437b Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Wed, 9 Sep 2026 15:41:09 -0700 Subject: [PATCH 09/10] refactor(security): keep F3 job-free; carry job certificate attributes as URI SANs Review follow-up on #5263: job_id belongs above cellnet, and private OIDs under NVIDIA's enterprise arc cannot be allocated. F3 now knows only that a certificate may restrict the cells it can claim: drivers expose the authenticated peer certificate (PEER_CERT), and CellIdentityResolver rejects a claimed FQCN outside the certificate's cell-scope URIs (FQCN.is_ancestor). PEER_JOB_ID, the job-id helpers in net_utils, FQCN.belongs_to_job/job_aux_name and resolve_owner are gone. Certificate attributes are https URI Subject Alternative Names under https://nvidia.com/nvflare/v1/ (nvflare/fuel/sec/cert_uri.py): job/ and cell/ on job leaves, ca/job on the job CA. The three private OIDs are removed. The FL layer decides the cells when it issues a credential: the job cell and the bootstrap cell under the owning CP or server FQCN, which also covers clients behind relays. --- .../no_shared_pvc_k8s_workspace_design.md | 4 +- docs/design/per_job_certs_design.md | 73 ++++---- .../security/per_job_certificates.rst | 13 +- .../job_launcher/workspace_cell_transfer.py | 4 +- nvflare/fuel/f3/cellnet/credential_manager.py | 3 +- nvflare/fuel/f3/cellnet/fqcn.py | 20 --- nvflare/fuel/f3/cellnet/identity.py | 53 +++--- nvflare/fuel/f3/drivers/aio_conn.py | 4 +- nvflare/fuel/f3/drivers/aio_grpc_driver.py | 4 +- nvflare/fuel/f3/drivers/aio_http_driver.py | 4 +- nvflare/fuel/f3/drivers/driver_params.py | 2 +- nvflare/fuel/f3/drivers/grpc_driver.py | 4 +- nvflare/fuel/f3/drivers/net_utils.py | 58 ++----- nvflare/fuel/f3/drivers/socket_conn.py | 4 +- nvflare/fuel/f3/sfm/conn_manager.py | 8 +- nvflare/fuel/sec/cert_uri.py | 71 ++++++++ nvflare/lighter/constants.py | 10 -- nvflare/lighter/impl/cert.py | 10 +- nvflare/lighter/utils.py | 11 +- nvflare/private/fed/server/job_runner.py | 7 +- nvflare/private/fed/utils/identity_utils.py | 7 +- nvflare/private/fed/utils/job_cert_utils.py | 76 ++++++--- .../workspace_cell_transfer_test.py | 14 +- .../workspace_transfer_bootstrap_mtls_test.py | 16 +- .../fuel/f3/cellnet/identity_binding_test.py | 161 ++++++++++-------- .../f3/cellnet/job_cert_binding_live_test.py | 16 +- .../fuel/f3/drivers/grpc_driver_test.py | 33 ++-- tests/unit_test/lighter/cert_builder_test.py | 5 +- .../fed/server/job_runner_deploy_test.py | 18 +- .../private/fed/utils/identity_utils_test.py | 30 ++-- .../private/fed/utils/job_cert_utils_test.py | 50 ++++-- 31 files changed, 467 insertions(+), 326 deletions(-) create mode 100644 nvflare/fuel/sec/cert_uri.py diff --git a/docs/design/no_shared_pvc_k8s_workspace_design.md b/docs/design/no_shared_pvc_k8s_workspace_design.md index 2442a7bf2a..c2903548e3 100644 --- a/docs/design/no_shared_pvc_k8s_workspace_design.md +++ b/docs/design/no_shared_pvc_k8s_workspace_design.md @@ -123,8 +123,8 @@ The download sequence is: `NVFL_WORKSPACE_TRANSFER_TOKEN`. 2. It creates a short-lived bootstrap child cell using the startup kit and the existing parent connection settings. The bootstrap FQCN is - `.ws_transfer_`, which `FQCN.belongs_to_job` recognizes as - part of the job so the job-bound certificate is accepted. When the child process is a client + `.ws_transfer_`, one of the cells the job credential lists + as its own, so the certificate is accepted there. When the child process is a client worker, the bootstrap cell reuses that worker's `client_name`, auth token, token signature, and `ssid`. When the child process is a server runner, the bootstrap cell uses the same server-job auth identity as the main runner. diff --git a/docs/design/per_job_certs_design.md b/docs/design/per_job_certs_design.md index 15fca392ea..4b006c0985 100644 --- a/docs/design/per_job_certs_design.md +++ b/docs/design/per_job_certs_design.md @@ -36,9 +36,9 @@ rootCA (private key exists only during provisioning) ├── server.crt / server.key server startup kit (unchanged) ├── client.crt / client.key client startup kits (unchanged) └── job_ca.crt / job_ca.key server startup kit ONLY (new) - CA:TRUE, pathlen:0, job-CA marker extension + CA:TRUE, pathlen:0, job-CA marker URI └── per-job leaf certs, issued at job deploy time - CN=, job_id extension, bounded validity + CN=, job and cell-scope URIs, bounded validity ``` Because the job CA chains to the existing root, no participant needs a new @@ -47,25 +47,31 @@ trust anchor: a job cert presented together with `job_ca.crt` validates against `verify_cert_chain()`. Leaf certs keep `CN=` so every existing common-name-based identity -check continues to pass. The job binding is carried in a certificate extension -holding the job ID. +check continues to pass. The job binding is carried in URI Subject Alternative +Names (see below). Only SP ever holds the job CA key. CP receives issued certificates; it does no signing. -### Extension OIDs +### Certificate URIs -Both extensions live under NVIDIA's IANA private enterprise arc -(`1.3.6.1.4.1.5703`), sub-arc `300`: +Certificate attributes are https URI Subject Alternative Names under a root the +project owns, `https://nvidia.com/nvflare/v1/`. A private X.509 extension would +need an OID allocated under NVIDIA's enterprise arc (no reachable registrar), +and UUID-based `2.25` OIDs break Go's x509 parser; a domain-owned URI is +globally unique without a registry and is parsed natively by Go, `cryptography` +and OpenSSL. Readers match the root exactly, ignore URIs on other hosts, and +fail closed on a malformed URI under the root (`nvflare/fuel/sec/cert_uri.py`). -| OID | Placed on | Meaning | +| URI | Placed on | Read by | | --- | --------- | ------- | -| `1.3.6.1.4.1.5703.300.1` | job leaf certs | the job ID the credential is bound to | -| `1.3.6.1.4.1.5703.300.2` | the job CA cert | "issued by the job CA" marker | +| `.../v1/job/` | job leaf certs | FL layer: site-scope rejection, log messages | +| `.../v1/ca/job` | the job CA cert | FL layer: rejects anything the job CA issued at site scope | +| `.../v1/cell/` (one per allowed cell) | job leaf certs | cellnet: the cells the certificate may claim | -Both are non-critical, so standard TLS stacks ignore them; only FLARE code reads -them. Neither can be stripped: the marker is inside the root-signed job CA cert, -the job ID inside the job-CA-signed leaf. +None of them is critical, so standard TLS stacks ignore them. None can be +stripped: the marker is inside the root-signed job CA cert, the job and cell +URIs inside the job-CA-signed leaf. ## Provisioning @@ -116,7 +122,10 @@ issuer generates an RSA keypair and a leaf certificate: CJ it is the registered client name (registration enforces that this equals the client cert's CN) — so whatever identity enforcement passed with site certs passes with job certs -- a job-ID extension identifying the job +- URI SANs naming the job and the cells the credential may claim: `.` + (the job cell and, implicitly, its descendants) and `.ws_transfer_` + (the workspace-transfer bootstrap cell), where `` is the site's CP FQCN or + `server`; behind a relay the CP FQCN already carries the relay prefix - `notBefore` backdated a few minutes to tolerate clock skew between the issuing server and the sites that validate the cert seconds later - `notAfter` = issue time + `job_cert_valid_days` (server `fed_server.json` or @@ -201,13 +210,13 @@ cert chain). All site-scope identity assertions funnel through the client's verification of the server), and no job cell ever legitimately asserts identity there. Two rejections cover two distinct threats: -1. **Leaked job leaf key**: any certificate carrying the job-ID extension is - rejected. This is keyed on the extension, not the issuer, so it holds +1. **Leaked job leaf key**: any certificate carrying the job URI is + rejected. This is keyed on the URI, not the issuer, so it holds regardless of which CA issued the certificate (which also keeps future HA setups with multiple job CAs simple). 2. **Stolen job CA key**: an attacker holding `job_ca.key` can mint a clean - site-named leaf *without* the extension. The job CA certificate therefore - carries a root-signed marker extension, and any presented chain containing + site-named leaf *without* the job URI. The job CA certificate therefore + carries a root-signed marker URI, and any presented chain containing a marked CA is rejected. The attacker cannot strip the marker (the job CA cert is signed by the root) and cannot validate without presenting it. @@ -219,20 +228,20 @@ admin identity; its blast radius is job cells only. Site-scope rejection stops a job credential from acting as a site. Job binding stops one job's credential from acting as another job's cell: -- Every TLS driver exposes the peer certificate's job-ID extension as the - `PEER_JOB_ID` connection property next to `PEER_CN`. -- `CellIdentityResolver.require_match()` rejects a peer whose certificate is - bound to job X unless the FQCN it claims belongs to that job - (`FQCN.belongs_to_job`): the segment right after the owning site's prefix - (as resolved for the identity check) must be X — the job cell `.X` and - its descendants — or an auxiliary job cell named `_X`, such as the - Kubernetes workspace-transfer bootstrap cell `server.ws_transfer_X`, which - authenticates with the job credential before the job cell exists. The - position matters: `.Y.ws_transfer_X` is a cell of job Y and is rejected - for job X's certificate. The check runs at the - connection handshake (`ConnManager`) and again on the certificate exchanged - for message-level crypto (`CredentialManager`), which is the certificate - later used to decrypt that peer's messages. +- Every TLS driver exposes the authenticated peer certificate as the + `PEER_CERT` connection property next to `PEER_CN`. Drivers parse nothing. +- Cellnet knows one generic rule, in its own vocabulary: a certificate may carry + cell-scope URIs, and a peer presenting one may only claim an FQCN equal to or + under one of those cells (`cell_scopes` / `fqcn_in_scopes` in + `nvflare/fuel/f3/cellnet/identity.py`, built on `FQCN.is_ancestor`). The + check runs at the connection handshake (`ConnManager`) and again on the + certificate exchanged for message-level crypto (`CredentialManager`). A + certificate without cell URIs is unrestricted, as site certificates are. +- Which cells a job credential lists is decided where it is issued + (`job_cell_scopes` in `nvflare/private/fed/utils/job_cert_utils.py`): the job + cell and the bootstrap cell under the owning CP or server. Cellnet carries no + job vocabulary; `site-1..ws_transfer_` is under job B's cell + and is rejected for job A's certificate. The rule is one-directional on purpose: it constrains what a job cert may claim, not which cert a job FQCN must present. Refusing to start a job cell diff --git a/docs/user_guide/admin_guide/security/per_job_certificates.rst b/docs/user_guide/admin_guide/security/per_job_certificates.rst index 4e544df22d..94501e938c 100644 --- a/docs/user_guide/admin_guide/security/per_job_certificates.rst +++ b/docs/user_guide/admin_guide/security/per_job_certificates.rst @@ -33,7 +33,8 @@ Provisioning creates a job-signing intermediate CA — ``job_ca.crt`` and ``job_ca.key`` — in the **server** startup kit only. It is signed by the project root CA, so no participant needs a new trust anchor. When a job is deployed, the server issues one certificate per participating site (``CN=``, with -an extension carrying the job ID), writes the server job's credential into the +URI Subject Alternative Names naming the job and the cells it may act as), +writes the server job's credential into the job's run directory, and sends each client its own credential inside the deploy message over the existing mutually authenticated channel. Every site verifies these certificates against the ``rootCA.pem`` it already has. @@ -183,8 +184,8 @@ intermediate CA certificate with: - ``basicConstraints = critical, CA:TRUE, pathlen:0``; - ``keyUsage = critical, digitalSignature, keyCertSign, cRLSign``; -- the job-CA marker: a **non-critical** extension with OID - ``1.3.6.1.4.1.5703.300.2`` (any value). Sites use it to reject anything the +- the job-CA marker: a URI Subject Alternative Name + ``https://nvidia.com/nvflare/v1/ca/job``. Sites use it to reject anything the job CA signed when a site identity is asserted, so it must be present. The root CA certificate in ``rootCA.pem`` must itself carry a ``keyUsage`` @@ -202,7 +203,7 @@ An OpenSSL extension section that produces this: keyUsage = critical, digitalSignature, keyCertSign, cRLSign subjectKeyIdentifier = hash authorityKeyIdentifier = keyid:always - 1.3.6.1.4.1.5703.300.2 = ASN1:UTF8String:job_ca + subjectAltName = URI:https://nvidia.com/nvflare/v1/ca/job .. code-block:: bash @@ -244,7 +245,7 @@ Failures are recorded in the job's ``job_deploy_detail`` (shown by launcher) - The client's internal listener is clear text. Provision the client with a ``listening_host`` using ``scheme: stcp`` and ``conn_sec: mtls``. - * - ``authenticated with a certificate bound to job '...' but claimed - endpoint ... is not part of that job`` + * - ``authenticated with a certificate restricted to cells [...] but claimed + endpoint ... is outside that scope`` - A process presented another job's certificate. This does not happen in normal operation; investigate the site. diff --git a/nvflare/app_opt/job_launcher/workspace_cell_transfer.py b/nvflare/app_opt/job_launcher/workspace_cell_transfer.py index 90f9ec9fe9..897162f0d7 100644 --- a/nvflare/app_opt/job_launcher/workspace_cell_transfer.py +++ b/nvflare/app_opt/job_launcher/workspace_cell_transfer.py @@ -62,6 +62,7 @@ JOB_CERT_FILE_NAME, JOB_KEY_FILE_NAME, find_job_cert, + workspace_transfer_cell_name, write_job_cert, ) from nvflare.security.logging import secure_format_exception @@ -80,7 +81,6 @@ BOOTSTRAP_CONNECT_TIMEOUT = 30.0 BOOTSTRAP_CONNECT_POLL_INTERVAL = 0.1 -_BOOTSTRAP_CELL_NAME = "ws_transfer" _DEFAULT_WORKSPACE_DOWNLOAD_EXCLUDES = frozenset({"local/study_data.yaml", "local/study_runtime.yaml"}) _RESOURCE_CONFIG_NAMES = ("resources.json", "resources.json.default") _K8S_LAUNCHER_COMPONENT_ID = "k8s_launcher" @@ -259,7 +259,7 @@ def _hash_file(path: str) -> str: def make_workspace_transfer_fqcn(owner_fqcn: str, job_id: str) -> str: - return FQCN.join([owner_fqcn, FQCN.job_aux_name(_BOOTSTRAP_CELL_NAME, job_id)]) + return FQCN.join([owner_fqcn, workspace_transfer_cell_name(job_id)]) def _cleanup_files(paths) -> None: diff --git a/nvflare/fuel/f3/cellnet/credential_manager.py b/nvflare/fuel/f3/cellnet/credential_manager.py index 4b58733a1a..77a0d769dc 100644 --- a/nvflare/fuel/f3/cellnet/credential_manager.py +++ b/nvflare/fuel/f3/cellnet/credential_manager.py @@ -23,7 +23,6 @@ from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey from nvflare.fuel.f3.cellnet.identity import CellIdentityResolver, get_cert_common_name from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import get_cert_job_id from nvflare.fuel.f3.endpoint import Endpoint from nvflare.fuel.f3.message import Message @@ -120,7 +119,7 @@ def _cache_cert(self, fqcn: str, cert: bytes): fqcn, get_cert_common_name(peer_cert), f"certificate for {fqcn}", - peer_job_id=get_cert_job_id(peer_cert), + peer_cert=peer_cert, ) except ValueError as ex: raise RuntimeError(str(ex)) diff --git a/nvflare/fuel/f3/cellnet/fqcn.py b/nvflare/fuel/f3/cellnet/fqcn.py index a4528f7b9c..762928acb6 100644 --- a/nvflare/fuel/f3/cellnet/fqcn.py +++ b/nvflare/fuel/f3/cellnet/fqcn.py @@ -17,26 +17,6 @@ class FQCN(FQN): VALID_PATTERN = "^[A-Za-z0-9_.~-]*$" - # A job's cells hang directly off the owning site: . (the job cell) and its - # descendants, or an auxiliary cell ._ (e.g. the workspace-transfer - # bootstrap cell). Nothing deeper counts: a cell under another job's cell is that job's. - JOB_AUX_SEPARATOR = "_" - - @staticmethod - def job_aux_name(name: str, job_id: str) -> str: - return f"{name}{FQCN.JOB_AUX_SEPARATOR}{job_id}" - - @staticmethod - def belongs_to_job(fqcn: str, job_id: str, owner_segments: int = 1) -> bool: - """True if the segment right after the owner's prefix is the job id or a _ auxiliary name.""" - if not job_id: - return False - parts = FQCN.split(FQCN.normalize(fqcn)) - if len(parts) <= owner_segments: - return False - job_segment = parts[owner_segments] - return job_segment == job_id or job_segment.endswith(FQCN.JOB_AUX_SEPARATOR + job_id) - # A network Attach trainer connects beneath the stable site CP and authenticates # with that physical parent's provisioned identity. diff --git a/nvflare/fuel/f3/cellnet/identity.py b/nvflare/fuel/f3/cellnet/identity.py index 9fb999f792..73d37aef11 100644 --- a/nvflare/fuel/f3/cellnet/identity.py +++ b/nvflare/fuel/f3/cellnet/identity.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple +from typing import List, Optional from cryptography import x509 from cryptography.x509.oid import NameOID @@ -21,6 +21,7 @@ from nvflare.fuel.f3.cellnet.fqcn import CLIENT_API_ATTACH_LEAF_PREFIX, FQCN from nvflare.fuel.f3.drivers.driver_params import DriverParams from nvflare.fuel.f3.drivers.net_utils import SECURE_SCHEMES +from nvflare.fuel.sec.cert_uri import CELL_URI_KIND, cert_uri_values from nvflare.fuel.utils.admin_name_utils import is_valid_admin_client_name from nvflare.fuel.utils.argument_utils import str2bool @@ -37,6 +38,16 @@ def get_param(params: dict, key: DriverParams, default=None): return value +def cell_scopes(cert: x509.Certificate) -> List[str]: + """FQCNs a certificate is restricted to, each with its descendants; empty means unrestricted.""" + return cert_uri_values(cert, CELL_URI_KIND) + + +def fqcn_in_scopes(fqcn: str, scopes: List[str]) -> bool: + fqcn = FQCN.normalize(fqcn) + return any(fqcn == scope or FQCN.is_ancestor(scope, fqcn) for scope in scopes) + + def is_mtls_connection(params: dict) -> bool: if not params: return False @@ -125,23 +136,15 @@ def _resolve_local_child_identity(self, fqcn: str) -> Optional[str]: return parts[0] if parts else None def resolve(self, fqcn: str) -> Optional[str]: - return self.resolve_owner(fqcn)[0] - - def resolve_owner(self, fqcn: str) -> Tuple[Optional[str], int]: - """The identity expected for an FQCN and how many leading segments name that identity's owner. - - A job's cells hang directly off their owner, so the job segment is the first one after the - owner's prefix (see FQCN.belongs_to_job). - """ if not fqcn: - return None, 0 + return None fqcn = FQCN.normalize(fqcn) - parts = FQCN.split(fqcn) identity = self.exact_identity_map.get(fqcn) if identity: - return identity, len(parts) + return identity + parts = FQCN.split(fqcn) leaf = parts[-1] for i in range(len(parts), 0, -1): prefix = FQCN.join(parts[:i]) @@ -150,34 +153,36 @@ def resolve_owner(self, fqcn: str) -> Tuple[Optional[str], int]: identity = self.prefix_identity_map.get(prefix) if identity: - return identity, i + return identity # Network Attach trainers are children of the stable site CP and use # that site's provisioned certificate. The dynamic CJ is a sibling and # remains the application-level task/result boundary. if len(parts) > 1 and leaf.startswith(CLIENT_API_ATTACH_LEAF_PREFIX): - return self.resolve_owner(FQCN.join(parts[:-1])) + return self.resolve(FQCN.join(parts[:-1])) identity = self._resolve_local_child_identity(fqcn) if identity: - return identity, len(FQCN.split(self.local_fqcn)) + return identity - return parts[0], 1 + return parts[0] if parts else fqcn - def require_match(self, fqcn: str, peer_cn: str, peer_desc: str, peer_job_id: Optional[str] = None): - expected_cn, owner_segments = self.resolve_owner(fqcn) + def require_match(self, fqcn: str, peer_cn: str, peer_desc: str, peer_cert: Optional[x509.Certificate] = None): + expected_cn = self.resolve(fqcn) if not expected_cn: raise ValueError(f"{peer_desc} claimed endpoint '{fqcn}' does not resolve to an expected identity") if not peer_cn or peer_cn == "N/A": raise ValueError(f"{peer_desc} does not have an authenticated mTLS peer common name") - # A per-job certificate may only authenticate cells of that job. - if peer_job_id is not None and not FQCN.belongs_to_job(fqcn, peer_job_id, owner_segments): - raise ValueError( - f"{peer_desc} authenticated with a certificate bound to job '{peer_job_id}' " - f"but claimed endpoint '{fqcn}' is not part of that job" - ) + # A certificate may restrict which cells it can claim (see cell_scopes). + if peer_cert is not None: + scopes = cell_scopes(peer_cert) + if scopes and not fqcn_in_scopes(fqcn, scopes): + raise ValueError( + f"{peer_desc} authenticated with a certificate restricted to cells {scopes} " + f"but claimed endpoint '{fqcn}' is outside that scope" + ) # Admin client cell names are per-session random IDs; the authenticated user is the cert CN. if is_valid_admin_client_name(fqcn): diff --git a/nvflare/fuel/f3/drivers/aio_conn.py b/nvflare/fuel/f3/drivers/aio_conn.py index 4e5767cfbf..217a310010 100644 --- a/nvflare/fuel/f3/drivers/aio_conn.py +++ b/nvflare/fuel/f3/drivers/aio_conn.py @@ -19,7 +19,7 @@ from nvflare.fuel.f3.drivers.aio_context import AioContext from nvflare.fuel.f3.drivers.connector_info import ConnectorInfo from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE, add_peer_job_id +from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE, add_peer_cert from nvflare.fuel.f3.sfm.prefix import PREFIX_LEN, Prefix from nvflare.fuel.hci.security import get_certificate_common_name from nvflare.security.logging import secure_format_exception @@ -134,7 +134,7 @@ def _get_aio_properties(self) -> dict: peer_cert = self.writer.get_extra_info("peercert") if peer_cert: cn = get_certificate_common_name(peer_cert) - add_peer_job_id(conn_props, self.writer.get_extra_info("ssl_object")) + add_peer_cert(conn_props, self.writer.get_extra_info("ssl_object")) else: if self.secure: cn = "N/A" diff --git a/nvflare/fuel/f3/drivers/aio_grpc_driver.py b/nvflare/fuel/f3/drivers/aio_grpc_driver.py index aa60d35c8c..320419bf7f 100644 --- a/nvflare/fuel/f3/drivers/aio_grpc_driver.py +++ b/nvflare/fuel/f3/drivers/aio_grpc_driver.py @@ -39,7 +39,7 @@ from .driver_params import DriverCap, DriverParams from .grpc.streamer_pb2 import Frame from .grpc.utils import get_grpc_client_credentials, get_grpc_server_credentials, use_aio_grpc -from .net_utils import MAX_FRAME_SIZE, add_grpc_peer_job_id, get_address, get_tcp_urls, ssl_required +from .net_utils import MAX_FRAME_SIZE, add_grpc_peer_cert, get_address, get_tcp_urls, ssl_required GRPC_DEFAULT_OPTIONS = [ ("grpc.max_send_message_length", MAX_FRAME_SIZE), @@ -200,7 +200,7 @@ async def Stream(self, request_iterator, context): cn_names = auth_context.get("x509_common_name") if cn_names: conn_props[DriverParams.PEER_CN.value] = cn_names[0].decode("utf-8") - add_grpc_peer_job_id(conn_props, auth_context) + add_grpc_peer_cert(conn_props, auth_context) connection = AioStreamSession( aio_ctx=self.aio_ctx, diff --git a/nvflare/fuel/f3/drivers/aio_http_driver.py b/nvflare/fuel/f3/drivers/aio_http_driver.py index db6c13c1f5..b961735301 100644 --- a/nvflare/fuel/f3/drivers/aio_http_driver.py +++ b/nvflare/fuel/f3/drivers/aio_http_driver.py @@ -26,7 +26,7 @@ from nvflare.fuel.f3.drivers.base_driver import BaseDriver from nvflare.fuel.f3.drivers.driver import ConnectorInfo from nvflare.fuel.f3.drivers.driver_params import DriverCap, DriverParams -from nvflare.fuel.f3.drivers.net_utils import add_peer_job_id, get_tcp_urls +from nvflare.fuel.f3.drivers.net_utils import add_peer_cert, get_tcp_urls from nvflare.fuel.hci.security import get_certificate_common_name from nvflare.security.logging import secure_format_exception @@ -72,7 +72,7 @@ def _get_ws_properties(self) -> dict: peer_cert = self.websocket.get_extra_info("peercert") if peer_cert: cn = get_certificate_common_name(peer_cert) - add_peer_job_id(conn_props, self.websocket.get_extra_info("ssl_object")) + add_peer_cert(conn_props, self.websocket.get_extra_info("ssl_object")) else: cn = "N/A" if self.ssl_context else None diff --git a/nvflare/fuel/f3/drivers/driver_params.py b/nvflare/fuel/f3/drivers/driver_params.py index 2f561bb92a..c11541878f 100644 --- a/nvflare/fuel/f3/drivers/driver_params.py +++ b/nvflare/fuel/f3/drivers/driver_params.py @@ -42,7 +42,7 @@ class DriverParams(str, Enum): LOCAL_ADDR = "local_addr" PEER_ADDR = "peer_addr" PEER_CN = "peer_cn" - PEER_JOB_ID = "peer_job_id" + PEER_CERT = "peer_cert" # DER of the authenticated peer certificate IMPLEMENTED_CONN_SEC = "implemented_conn_sec" diff --git a/nvflare/fuel/f3/drivers/grpc_driver.py b/nvflare/fuel/f3/drivers/grpc_driver.py index fc61189f63..25e3d388bb 100644 --- a/nvflare/fuel/f3/drivers/grpc_driver.py +++ b/nvflare/fuel/f3/drivers/grpc_driver.py @@ -37,7 +37,7 @@ from .grpc.qq import QQ from .grpc.streamer_pb2 import Frame from .grpc.utils import get_grpc_client_credentials, get_grpc_server_credentials, use_aio_grpc -from .net_utils import MAX_FRAME_SIZE, add_grpc_peer_job_id, get_address, get_tcp_urls, ssl_required +from .net_utils import MAX_FRAME_SIZE, add_grpc_peer_cert, get_address, get_tcp_urls, ssl_required GRPC_DEFAULT_OPTIONS = [ ("grpc.max_send_message_length", MAX_FRAME_SIZE), @@ -140,7 +140,7 @@ def Stream(self, request_iterator, context): cn_names = auth_context.get("x509_common_name") if cn_names: conn_props[DriverParams.PEER_CN.value] = cn_names[0].decode("utf-8") - add_grpc_peer_job_id(conn_props, auth_context) + add_grpc_peer_cert(conn_props, auth_context) try: self.logger.debug(f"SERVER started Stream CB in thread {ct.name}") diff --git a/nvflare/fuel/f3/drivers/net_utils.py b/nvflare/fuel/f3/drivers/net_utils.py index b7e78a3757..9cbc08c9a4 100644 --- a/nvflare/fuel/f3/drivers/net_utils.py +++ b/nvflare/fuel/f3/drivers/net_utils.py @@ -22,18 +22,16 @@ from urllib.parse import parse_qsl, urlencode, urlparse from cryptography import x509 +from cryptography.hazmat.primitives import serialization from nvflare.apis.fl_constant import ConnectionSecurity from nvflare.fuel.f3.comm_error import CommError from nvflare.fuel.f3.drivers.driver_params import DriverParams from nvflare.fuel.utils.argument_utils import str2bool -from nvflare.lighter.constants import CertExtensionOID from nvflare.security.logging import secure_format_exception log = logging.getLogger(__name__) -JOB_ID_EXTENSION_OID = x509.ObjectIdentifier(CertExtensionOID.JOB_ID) - LO_PORT = 1025 HI_PORT = 65535 MAX_ITER_SIZE = 10 @@ -122,52 +120,26 @@ def get_ssl_context(params: dict, ssl_server: bool) -> Optional[SSLContext]: return ctx -def get_cert_job_id(cert: x509.Certificate) -> Optional[str]: - """Job ID a per-job certificate is bound to; None for a site certificate. - - An undecodable extension value comes back with replacement characters so it - can never equal a real job ID. - """ - try: - ext = cert.extensions.get_extension_for_oid(JOB_ID_EXTENSION_OID) - except x509.ExtensionNotFound: - return None - return ext.value.value.decode("utf-8", errors="replace") - - -def get_cert_job_id_from_pem(cert_bytes: bytes) -> Optional[str]: - if not cert_bytes: - return None - return get_cert_job_id(x509.load_pem_x509_certificate(cert_bytes)) - - -def get_peer_job_id(ssl_object) -> Optional[str]: - """Job ID from the peer certificate of an established TLS connection (SSLSocket or SSLObject).""" +def get_peer_cert_der(ssl_object) -> Optional[bytes]: + """DER of the peer certificate of an established TLS connection (SSLSocket or SSLObject).""" if not ssl_object: return None - der = ssl_object.getpeercert(binary_form=True) - if not der: - return None - return get_cert_job_id(x509.load_der_x509_certificate(der)) + return ssl_object.getpeercert(binary_form=True) or None -def get_grpc_peer_job_id(auth_context: dict) -> Optional[str]: - pem_certs = auth_context.get("x509_pem_cert") if auth_context else None - return get_cert_job_id_from_pem(pem_certs[0]) if pem_certs else None +def add_peer_cert(conn_props: dict, ssl_object) -> None: + """Record the authenticated peer certificate next to PEER_CN (TLS socket connections).""" + der = get_peer_cert_der(ssl_object) + if der: + conn_props[DriverParams.PEER_CERT.value] = der -def add_peer_job_id(conn_props: dict, ssl_object) -> None: - """Record the peer certificate's job binding next to PEER_CN (TLS socket connections).""" - job_id = get_peer_job_id(ssl_object) - if job_id: - conn_props[DriverParams.PEER_JOB_ID.value] = job_id - - -def add_grpc_peer_job_id(conn_props: dict, auth_context: dict) -> None: - """Record the peer certificate's job binding next to PEER_CN (gRPC server-side connections).""" - job_id = get_grpc_peer_job_id(auth_context) - if job_id: - conn_props[DriverParams.PEER_JOB_ID.value] = job_id +def add_grpc_peer_cert(conn_props: dict, auth_context: dict) -> None: + """Record the authenticated peer certificate next to PEER_CN (gRPC server-side connections).""" + pem_certs = auth_context.get("x509_pem_cert") if auth_context else None + if pem_certs: + cert = x509.load_pem_x509_certificate(pem_certs[0]) + conn_props[DriverParams.PEER_CERT.value] = cert.public_bytes(serialization.Encoding.DER) def get_address(params: dict) -> str: diff --git a/nvflare/fuel/f3/drivers/socket_conn.py b/nvflare/fuel/f3/drivers/socket_conn.py index db63ec538b..cdffe50df6 100644 --- a/nvflare/fuel/f3/drivers/socket_conn.py +++ b/nvflare/fuel/f3/drivers/socket_conn.py @@ -24,7 +24,7 @@ from nvflare.fuel.f3.connection import BytesAlike, Connection from nvflare.fuel.f3.drivers.driver import ConnectorInfo from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE, add_peer_job_id +from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE, add_peer_cert from nvflare.fuel.f3.sfm.prefix import PREFIX_LEN, Prefix from nvflare.fuel.hci.security import get_certificate_common_name from nvflare.security.logging import secure_format_exception @@ -223,7 +223,7 @@ def _get_socket_properties(self) -> dict: cert = self.sock.getpeercert() if cert: cn = get_certificate_common_name(cert) - add_peer_job_id(conn_props, self.sock) + add_peer_cert(conn_props, self.sock) else: cn = "N/A" conn_props[DriverParams.PEER_CN.value] = cn diff --git a/nvflare/fuel/f3/sfm/conn_manager.py b/nvflare/fuel/f3/sfm/conn_manager.py index 67ec4e48d5..c6e7693765 100644 --- a/nvflare/fuel/f3/sfm/conn_manager.py +++ b/nvflare/fuel/f3/sfm/conn_manager.py @@ -19,6 +19,7 @@ from typing import Dict, List, Optional import msgpack +from cryptography import x509 from nvflare.fuel.f3.cellnet.fqcn import FQCN from nvflare.fuel.f3.cellnet.identity import CellIdentityResolver, get_param, is_admin_listener, is_mtls_connection @@ -53,6 +54,11 @@ handle_count = 0 +def _peer_cert(conn_props: dict): + der = get_param(conn_props, DriverParams.PEER_CERT) + return x509.load_der_x509_certificate(der) if der else None + + def get_handle(): global handle_lock, handle_count with handle_lock: @@ -439,7 +445,7 @@ def update_endpoint(self, sfm_conn: SfmConnection, data: dict): endpoint_name, peer_cn, f"connection {sfm_conn.get_name()}", - peer_job_id=get_param(conn_props, DriverParams.PEER_JOB_ID), + peer_cert=_peer_cert(conn_props), ) except ValueError as ex: sfm_conn.conn.close() diff --git a/nvflare/fuel/sec/cert_uri.py b/nvflare/fuel/sec/cert_uri.py new file mode 100644 index 0000000000..9697fc0333 --- /dev/null +++ b/nvflare/fuel/sec/cert_uri.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""NVFlare certificate URIs. + +Certificate attributes travel as https URI Subject Alternative Names under a root the project +owns rather than as private X.509 extensions: a URI under a controlled domain is globally +unique without an OID allocation, and Go, cryptography and OpenSSL parse it natively. +Readers match the root exactly, ignore URIs on other hosts, and reject malformed URIs +under the root. +""" + +from typing import Iterable, List +from urllib.parse import quote, unquote + +from cryptography import x509 + +NVFLARE_CERT_URI_ROOT = "https://nvidia.com/nvflare/" +_V1_PREFIX = NVFLARE_CERT_URI_ROOT + "v1/" + +# https://nvidia.com/nvflare/v1// +JOB_URI_KIND = "job" # leaf: the job the credential belongs to +CA_URI_KIND = "ca" # CA certificate: the role of the CA +CELL_URI_KIND = "cell" # leaf: an FQCN the certificate may claim in cellnet (with its descendants) +JOB_CA_URI_VALUE = "job" + + +def cert_uri(kind: str, value: str) -> str: + return f"{_V1_PREFIX}{kind}/{quote(value, safe='')}" + + +def job_ca_marker_uri() -> str: + return cert_uri(CA_URI_KIND, JOB_CA_URI_VALUE) + + +def uri_general_names(uris: Iterable[str]) -> List[x509.UniformResourceIdentifier]: + return [x509.UniformResourceIdentifier(uri) for uri in uris] + + +def cert_uri_values(cert: x509.Certificate, kind: str) -> List[str]: + """Values of one kind carried by the certificate's NVFlare URI SANs. + + Raises ValueError for a URI under the NVFlare root that is not a well-formed v1 entry. + """ + try: + san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + except x509.ExtensionNotFound: + return [] + values = [] + for uri in san.get_values_for_type(x509.UniformResourceIdentifier): + if not uri.startswith(NVFLARE_CERT_URI_ROOT): + continue + if not uri.startswith(_V1_PREFIX): + raise ValueError(f"unsupported NVFlare certificate URI: {uri}") + uri_kind, separator, encoded_value = uri[len(_V1_PREFIX) :].partition("/") + if not separator or not uri_kind or not encoded_value or "/" in encoded_value: + raise ValueError(f"malformed NVFlare certificate URI: {uri}") + if uri_kind == kind: + values.append(unquote(encoded_value)) + return values diff --git a/nvflare/lighter/constants.py b/nvflare/lighter/constants.py index 9dceb10865..26ee9f8936 100644 --- a/nvflare/lighter/constants.py +++ b/nvflare/lighter/constants.py @@ -200,16 +200,6 @@ class CertFileBasename: SERVER = "server" -class CertExtensionOID: - """Private extensions under NVIDIA's IANA enterprise arc (1.3.6.1.4.1.5703).""" - - # on job leaf certs: the job ID the certificate is bound to - JOB_ID = "1.3.6.1.4.1.5703.300.1" - # on the job CA cert: marks certs it issues as job-scoped, so site-scope - # verification can reject them by issuer even without the job-ID extension - JOB_CA_MARKER = "1.3.6.1.4.1.5703.300.2" - - class CommConfigArg: SCHEME = "scheme" HOST = "host" diff --git a/nvflare/lighter/impl/cert.py b/nvflare/lighter/impl/cert.py index 45b1ec6e75..e7fee2dbfc 100644 --- a/nvflare/lighter/impl/cert.py +++ b/nvflare/lighter/impl/cert.py @@ -21,7 +21,8 @@ from cryptography.hazmat.primitives import serialization from cryptography.x509.oid import NameOID -from nvflare.lighter.constants import CertExtensionOID, CertFileBasename, CtxKey, ParticipantType, PropKey, ProvFileName +from nvflare.fuel.sec.cert_uri import job_ca_marker_uri +from nvflare.lighter.constants import CertFileBasename, CtxKey, ParticipantType, PropKey, ProvFileName from nvflare.lighter.ctx import ProvisionContext from nvflare.lighter.entity import Participant, Project from nvflare.lighter.spec import Builder @@ -370,8 +371,7 @@ def _build_write_job_ca(self, project: Project, server: Participant, ctx: Provis pri_key, pub_key = generate_keys() now, not_valid_after = self._bounded_not_valid_after("job CA") # the marker lets site-scope verification reject anything this CA issues by - # issuer, even a cert minted without the job-ID extension by a stolen CA key - marker = x509.UnrecognizedExtension(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER), b"job_ca") + # issuer, even a cert minted without the job URI by a stolen CA key cert = self._generate_cert( subject, None, @@ -382,7 +382,7 @@ def _build_write_job_ca(self, project: Project, server: Participant, ctx: Provis ca_path_length=0, not_valid_before=now, not_valid_after=not_valid_after, - extra_extensions=[(marker, False)], + uri_names=[job_ca_marker_uri()], ) cert_pem = serialize_cert(cert) key_pem = serialize_pri_key(pri_key) @@ -443,6 +443,7 @@ def _generate_cert( extra_extensions=None, not_valid_before=None, not_valid_after=None, + uri_names=None, ): if server: # This is to generate a server cert. @@ -463,6 +464,7 @@ def _generate_cert( extra_extensions=extra_extensions, not_valid_before=not_valid_before, not_valid_after=not_valid_after, + uri_names=uri_names, ) def finalize(self, project: Project, ctx: ProvisionContext): diff --git a/nvflare/lighter/utils.py b/nvflare/lighter/utils.py index 18ae2c2328..3a457d1251 100644 --- a/nvflare/lighter/utils.py +++ b/nvflare/lighter/utils.py @@ -84,6 +84,7 @@ def generate_cert( not_valid_after=None, extra_extensions=None, ca_path_length=None, + uri_names=None, ): now = not_valid_before or datetime.datetime.now(datetime.timezone.utc) cert_not_valid_after = not_valid_after or now + datetime.timedelta(days=valid_days) @@ -137,12 +138,10 @@ def generate_cert( seen_extension_oids.add(extension.oid) builder = builder.add_extension(extension, critical=critical) - builder = builder.add_extension( - x509.SubjectAlternativeName( - build_subject_alt_names(server_default_host, server_additional_hosts, subject.name) - ), - critical=False, - ) + subject_alt_names = build_subject_alt_names(server_default_host, server_additional_hosts, subject.name) + if uri_names: + subject_alt_names.extend(x509.UniformResourceIdentifier(uri) for uri in uri_names) + builder = builder.add_extension(x509.SubjectAlternativeName(subject_alt_names), critical=False) return builder.sign(signing_pri_key, hashes.SHA256(), default_backend()) diff --git a/nvflare/private/fed/server/job_runner.py b/nvflare/private/fed/server/job_runner.py index 4a243d635f..3cf80d6bd6 100644 --- a/nvflare/private/fed/server/job_runner.py +++ b/nvflare/private/fed/server/job_runner.py @@ -38,6 +38,7 @@ from nvflare.apis.job_scheduler_spec import DispatchInfo from nvflare.apis.workspace import Workspace from nvflare.fuel.common.exit_codes import ProcessExitCode +from nvflare.fuel.f3.cellnet.fqcn import FQCN from nvflare.fuel.f3.cellnet.identity import get_cert_common_name_from_file from nvflare.fuel.utils.config_service import ConfigService from nvflare.lighter.tool_consts import NVFLARE_SIG_FILE @@ -255,8 +256,8 @@ def _deploy_job(self, job: Job, sites: dict, fl_ctx: FLContext) -> Tuple[str, li # cert (registration enforces CN == client name) job_creds = {} if job_cert_issuer: - site_names = [c.name for c in clients] - job_creds = job_cert_issuer.issue_many(site_names, job.job_id, self.job_cert_valid_days) + site_owners = {c.name: c.get_fqcn() or c.name for c in clients} + job_creds = job_cert_issuer.issue_many(site_owners, job.job_id, self.job_cert_valid_days) for c in clients: assert isinstance(c, Client) @@ -281,7 +282,7 @@ def _deploy_job(self, job: Job, sites: dict, fl_ctx: FLContext) -> Tuple[str, li # "server") server_cert_path = fl_ctx.get_prop(FLContextKey.SERVER_CONFIG)[0][SecureTrainConst.SSL_CERT] server_cn = get_cert_common_name_from_file(server_cert_path) - cert_pem, key_pem = job_cert_issuer.issue(server_cn, job.job_id, self.job_cert_valid_days) + cert_pem, key_pem = job_cert_issuer.issue(server_cn, job.job_id, FQCN.ROOT_SERVER, self.job_cert_valid_days) write_job_cert(workspace.get_run_dir(job.job_id), cert_pem, key_pem) abort_job = False diff --git a/nvflare/private/fed/utils/identity_utils.py b/nvflare/private/fed/utils/identity_utils.py index 8037cedfe9..fe00bc81dd 100644 --- a/nvflare/private/fed/utils/identity_utils.py +++ b/nvflare/private/fed/utils/identity_utils.py @@ -16,7 +16,6 @@ from cryptography import x509 from cryptography.x509.oid import NameOID -from nvflare.fuel.f3.drivers.net_utils import get_cert_job_id from nvflare.fuel.utils.log_utils import get_obj_logger from nvflare.lighter.utils import ( load_crt, @@ -28,7 +27,7 @@ verify_cert_chain, verify_content, ) -from nvflare.private.fed.utils.job_cert_utils import has_job_ca_marker +from nvflare.private.fed.utils.job_cert_utils import get_cert_job_id, has_job_ca_marker from nvflare.security.logging import secure_format_exception @@ -163,9 +162,9 @@ def _validate_identity_cert_usage(cert, expected_eku, intermediate_certs=None): """Enforce certificate usage restrictions for the common-name challenge. Per-job certificates are scoped to one job's cells and must never assert site, admin, - or server identity: a leaf carrying the job-ID extension is rejected, and so is any + or server identity: a leaf carrying the job URI is rejected, and so is any chain containing the job-CA marker (a stolen job CA key can mint leaves without the - extension, but cannot strip the root-signed marker off the CA cert it must present). + extension, but cannot strip the root-signed marker URI off the CA cert it must present). Legacy FLARE certificates may omit KeyUsage and ExtendedKeyUsage, so absent extensions remain unrestricted. When present, they must allow the signing diff --git a/nvflare/private/fed/utils/job_cert_utils.py b/nvflare/private/fed/utils/job_cert_utils.py index b9f88e78ec..a2d542eaa7 100644 --- a/nvflare/private/fed/utils/job_cert_utils.py +++ b/nvflare/private/fed/utils/job_cert_utils.py @@ -17,14 +17,22 @@ import os import shutil from concurrent.futures import ThreadPoolExecutor -from typing import Dict, Iterable, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple from cryptography import x509 from nvflare.apis.fl_constant import FLContextKey, SecureTrainConst +from nvflare.fuel.f3.cellnet.fqcn import FQCN from nvflare.fuel.f3.cellnet.identity import get_cert_common_name -from nvflare.fuel.f3.drivers.net_utils import JOB_ID_EXTENSION_OID -from nvflare.lighter.constants import CertExtensionOID, ProvFileName +from nvflare.fuel.sec.cert_uri import ( + CA_URI_KIND, + CELL_URI_KIND, + JOB_CA_URI_VALUE, + JOB_URI_KIND, + cert_uri, + cert_uri_values, +) +from nvflare.lighter.constants import ProvFileName from nvflare.lighter.utils import ( Identity, bounded_validity, @@ -60,7 +68,36 @@ _PROP_CERT = "cert" _PROP_KEY = "key" -JOB_CA_MARKER_OID = x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER) +WORKSPACE_TRANSFER_CELL_NAME = "ws_transfer" + + +def workspace_transfer_cell_name(job_id: str) -> str: + """Name of the job's workspace-transfer bootstrap cell, directly under the site that owns the job.""" + return f"{WORKSPACE_TRANSFER_CELL_NAME}_{job_id}" + + +def job_cell_scopes(owner_fqcn: str, job_id: str) -> List[str]: + """Cells a job credential may claim: the job cell (with its descendants) and the bootstrap cell.""" + return [FQCN.join([owner_fqcn, job_id]), FQCN.join([owner_fqcn, workspace_transfer_cell_name(job_id)])] + + +def job_cert_uris(owner_fqcn: str, job_id: str) -> List[str]: + return [cert_uri(JOB_URI_KIND, job_id)] + [cert_uri(CELL_URI_KIND, s) for s in job_cell_scopes(owner_fqcn, job_id)] + + +def get_cert_job_id(cert: x509.Certificate) -> Optional[str]: + """Job a per-job certificate belongs to; None for a site certificate.""" + job_ids = cert_uri_values(cert, JOB_URI_KIND) + if not job_ids: + return None + if len(job_ids) > 1: + raise ValueError(f"certificate claims several jobs: {job_ids}") + return job_ids[0] + + +def has_job_ca_marker(cert: x509.Certificate) -> bool: + """True for the job CA certificate: anything it issued is job-scoped and never a site identity.""" + return JOB_CA_URI_VALUE in cert_uri_values(cert, CA_URI_KIND) class JobCertError(RuntimeError): @@ -164,14 +201,6 @@ def unpack_job_cert_header(header) -> Optional[Tuple[bytes, bytes]]: return None -def has_job_ca_marker(cert: x509.Certificate) -> bool: - try: - cert.extensions.get_extension_for_oid(JOB_CA_MARKER_OID) - return True - except x509.ExtensionNotFound: - return False - - class JobCertIssuer: """Issues short-lived per-job certificates signed by the provisioned job CA. @@ -185,15 +214,22 @@ def __init__(self, ca_cert_pem: bytes, ca_key): self.ca_key = ca_key self.ca_cn = get_cert_common_name(self.ca_cert) - def issue(self, site_name: str, job_id: str, valid_days: int = JOB_CERT_VALID_DAYS) -> Tuple[bytes, bytes]: + def issue( + self, site_name: str, job_id: str, owner_fqcn: str, valid_days: int = JOB_CERT_VALID_DAYS + ) -> Tuple[bytes, bytes]: """Issue a per-job credential for one site. + Args: + site_name: CN of the site, as its own certificate presents it + job_id: the job the credential belongs to + owner_fqcn: FQCN of the site's parent cell (CP or server) under which the job's cells live + valid_days: validity, clamped to the job CA's own + Returns: (cert_chain_pem, key_pem): leaf cert followed by the job CA cert, and the private key. """ pri_key, pub_key = generate_keys() not_valid_before, not_valid_after = bounded_validity(self.ca_cert, valid_days, backdate=JOB_CERT_BACKDATE) - job_id_ext = x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, job_id.encode("utf-8")) cert = generate_cert( subject=Identity(site_name), issuer=Identity(self.ca_cn), @@ -201,19 +237,21 @@ def issue(self, site_name: str, job_id: str, valid_days: int = JOB_CERT_VALID_DA subject_pub_key=pub_key, not_valid_before=not_valid_before, not_valid_after=not_valid_after, - extra_extensions=[(job_id_ext, False)], + uri_names=job_cert_uris(owner_fqcn, job_id), ) return serialize_cert(cert) + self.ca_cert_pem, serialize_pri_key(pri_key) def issue_many( - self, site_names: Iterable[str], job_id: str, valid_days: int = JOB_CERT_VALID_DAYS + self, site_owners: Dict[str, str], job_id: str, valid_days: int = JOB_CERT_VALID_DAYS ) -> Dict[str, Tuple[bytes, bytes]]: - """Issue credentials for several sites at once; RSA key generation dominates and runs in parallel.""" - names = list(site_names) + """Issue credentials for several sites (name -> owner FQCN); RSA key generation runs in parallel.""" + names = list(site_owners) if not names: return {} with ThreadPoolExecutor(max_workers=min(8, len(names))) as pool: - return dict(zip(names, pool.map(lambda name: self.issue(name, job_id, valid_days), names))) + return dict( + zip(names, pool.map(lambda name: self.issue(name, job_id, site_owners[name], valid_days), names)) + ) def load_job_cert_issuer(startup_dir: str) -> JobCertIssuer: diff --git a/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py b/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py index e038fe8df3..7f925fa7b0 100644 --- a/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py +++ b/tests/unit_test/app_opt/job_launcher/workspace_cell_transfer_test.py @@ -45,6 +45,8 @@ from nvflare.fuel.f3.cellnet.identity import CellIdentityResolver from nvflare.fuel.f3.cellnet.utils import make_reply, new_cell_message from nvflare.fuel.f3.drivers.driver_params import DriverParams +from nvflare.lighter.utils import Identity, generate_cert, generate_keys +from nvflare.private.fed.utils.job_cert_utils import job_cert_uris JOB_ID = "abc12345-dead-beef-0000-111122223333" @@ -117,9 +119,15 @@ def test_bootstrap_fqcn_is_accepted_by_job_cert_binding(self, owner_fqcn, owner_ resolver = CellIdentityResolver(local_fqcn=owner_fqcn, prefix_identity_map={owner_fqcn: owner_cn}) fqcn = make_workspace_transfer_fqcn(owner_fqcn, JOB_ID) - resolver.require_match(fqcn, owner_cn, "bootstrap", peer_job_id=JOB_ID) - with pytest.raises(ValueError, match="bound to job"): - resolver.require_match(fqcn, owner_cn, "bootstrap", peer_job_id="other-job") + def job_credential(job_id): + key, pub_key = generate_keys() + return generate_cert( + Identity(owner_cn), Identity(owner_cn), key, pub_key, uri_names=job_cert_uris(owner_fqcn, job_id) + ) + + resolver.require_match(fqcn, owner_cn, "bootstrap", peer_cert=job_credential(JOB_ID)) + with pytest.raises(ValueError, match="outside that scope"): + resolver.require_match(fqcn, owner_cn, "bootstrap", peer_cert=job_credential("other-job")) def test_returns_same_manager_for_same_cell(self): owner_cell = _FakeCell(fqcn="site-1.parent") diff --git a/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py b/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py index 3d532070a6..a31211b325 100644 --- a/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py +++ b/tests/unit_test/app_opt/job_launcher/workspace_transfer_bootstrap_mtls_test.py @@ -22,14 +22,13 @@ import traceback import pytest -from cryptography import x509 from nvflare.app_opt.job_launcher.workspace_cell_transfer import make_workspace_transfer_fqcn from nvflare.fuel.f3.cellnet.cell import Cell from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.drivers.driver_params import DriverParams from nvflare.fuel.f3.message import Message -from nvflare.lighter.constants import CertExtensionOID +from nvflare.fuel.sec.cert_uri import job_ca_marker_uri from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert, serialize_pri_key from nvflare.private.fed.utils.job_cert_utils import JobCertIssuer @@ -45,7 +44,7 @@ class _RejectionRecorder(logging.Handler): - """Forwards the parent's job-binding rejections to the test process.""" + """Forwards the parent's certificate-scope rejections to the test process.""" def __init__(self, queue): super().__init__(level=logging.ERROR) @@ -53,7 +52,7 @@ def __init__(self, queue): def emit(self, record): message = record.getMessage() - if "bound to job" in message: + if "outside that scope" in message: self.queue.put(message) @@ -63,7 +62,6 @@ def _write_pki(out_dir: str) -> dict: srv_key, srv_pub = generate_keys() srv_cert = generate_cert(Identity("server"), Identity("rootCA"), root_key, srv_pub, server_default_host="localhost") jca_key, jca_pub = generate_keys() - marker = x509.UnrecognizedExtension(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER), b"job_ca") jca_cert = generate_cert( Identity("job_ca"), Identity("rootCA"), @@ -71,11 +69,11 @@ def _write_pki(out_dir: str) -> dict: jca_pub, ca=True, ca_path_length=0, - extra_extensions=[(marker, False)], + uri_names=[job_ca_marker_uri()], ) issuer = JobCertIssuer(serialize_cert(jca_cert), jca_key) - job_crt, job_key = issuer.issue("server", _JOB_ID) - other_crt, other_key = issuer.issue("server", _OTHER_JOB_ID) + job_crt, job_key = issuer.issue("server", _JOB_ID, "server") + other_crt, other_key = issuer.issue("server", _OTHER_JOB_ID, "server") files = { "rootCA.pem": serialize_cert(root_cert), "server.crt": serialize_cert(srv_cert), @@ -204,4 +202,4 @@ def test_bootstrap_cell_rejected_with_another_jobs_cert(parent): message = reject_q.get(timeout=max(0.1, deadline - time.time())) if fqcn in message: break - assert f"bound to job '{_OTHER_JOB_ID}'" in message + assert f"server.{_OTHER_JOB_ID}" in message # the certificate's own scope is named in the rejection diff --git a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py index 8a5b179a68..f49ea1979e 100644 --- a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py +++ b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py @@ -35,22 +35,17 @@ from nvflare.fuel.f3.cellnet.core_cell import CoreCell from nvflare.fuel.f3.cellnet.credential_manager import CERT_CONTENT, CredentialManager from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, MessageType, ReturnCode -from nvflare.fuel.f3.cellnet.fqcn import FQCN -from nvflare.fuel.f3.cellnet.identity import ADMIN_LISTENER_KEY, CellIdentityResolver +from nvflare.fuel.f3.cellnet.identity import ADMIN_LISTENER_KEY, CellIdentityResolver, cell_scopes, fqcn_in_scopes from nvflare.fuel.f3.cellnet.utils import make_reply from nvflare.fuel.f3.comm_error import CommError from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import ( - JOB_ID_EXTENSION_OID, - get_cert_job_id_from_pem, - get_grpc_peer_job_id, - get_peer_job_id, -) +from nvflare.fuel.f3.drivers.net_utils import add_grpc_peer_cert, add_peer_cert from nvflare.fuel.f3.endpoint import Endpoint from nvflare.fuel.f3.message import Message from nvflare.fuel.f3.sfm.conn_manager import ConnManager from nvflare.fuel.f3.sfm.constants import HandshakeKeys from nvflare.fuel.f3.sfm.sfm_conn import SfmConnection +from nvflare.fuel.sec.cert_uri import CELL_URI_KIND, cert_uri from nvflare.fuel.utils.constants import Mode from nvflare.lighter.utils import Identity, generate_cert, generate_keys @@ -62,7 +57,7 @@ def __init__( conn_security=ConnectionSecurity.MTLS, mode=Mode.PASSIVE, admin_listener=False, - peer_job_id=None, + peer_cert=None, ): self.name = "CN-test" self.closed = False @@ -78,8 +73,8 @@ def __init__( self.conn_props = {} if peer_cn is not None: self.conn_props[DriverParams.PEER_CN.value] = peer_cn - if peer_job_id is not None: - self.conn_props[DriverParams.PEER_JOB_ID.value] = peer_job_id + if peer_cert is not None: + self.conn_props[DriverParams.PEER_CERT.value] = peer_cert def get_conn_properties(self): return self.conn_props @@ -105,19 +100,30 @@ def _cert_pem(common_name: str): return cert.public_bytes(serialization.Encoding.PEM) -def _job_cert_pem(common_name: str, job_id): +_JOB_SCOPES = ["site-1.job-123", "site-1.ws_transfer_job-123"] + + +def _scoped_cert_pem(common_name: str, scopes=None, uris=None): key, pub_key = generate_keys() - job_id_bytes = job_id if isinstance(job_id, bytes) else job_id.encode("utf-8") + uri_names = list(uris or []) + [cert_uri(CELL_URI_KIND, scope) for scope in (scopes or [])] cert = generate_cert( subject=Identity(common_name), issuer=Identity(common_name), signing_pri_key=key, subject_pub_key=pub_key, - extra_extensions=[(x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, job_id_bytes), False)], + uri_names=uri_names, ) return cert.public_bytes(serialization.Encoding.PEM) +def _cert(pem: bytes) -> x509.Certificate: + return x509.load_pem_x509_certificate(pem) + + +def _der(pem: bytes) -> bytes: + return _cert(pem).public_bytes(serialization.Encoding.DER) + + def _make_chained_cell_cipher_cert(): root_key, root_pub_key = generate_keys() root_cert = generate_cert( @@ -454,78 +460,89 @@ def test_mtls_certificate_cache_accepts_configured_auth_identity_for_site_cert_c "fqcn, expected", [ ("site-1.job-123", True), - ("server.job-123", True), ("site-1.job-123.sub-1", True), ("site-1.ws_transfer_job-123", True), - ("server.ws_transfer_job-123", True), ("site-1", False), ("site-1.job-999", False), ("site-1.ws_transfer_job-999", False), - ("site-1.ws_transferjob-123", False), ("site-1.job-1234", False), - ("site-1.xjob-123", False), ("site-1.job-999.ws_transfer_job-123", False), ("site-1.job-999.job-123", False), - ("site-1.job-123.ws_transfer_job-999", True), + ("relay-1.site-1.job-123", False), ], ) -def test_fqcn_belongs_to_job(fqcn, expected): - assert FQCN.belongs_to_job(fqcn, "job-123") is expected +def test_fqcn_in_scopes(fqcn, expected): + assert fqcn_in_scopes(fqcn, _JOB_SCOPES) is expected -def test_fqcn_belongs_to_job_rejects_empty_job_id(): - assert FQCN.belongs_to_job("site-1.ws_transfer_", "") is False +def test_cell_scopes_of_unrestricted_and_scoped_certs(): + assert cell_scopes(_cert(_cert_pem("site-1"))) == [] + assert cell_scopes(_cert(_scoped_cert_pem("site-1", _JOB_SCOPES))) == _JOB_SCOPES + # URIs on other hosts, and NVFlare URIs of other kinds, are not cell scopes + other = _scoped_cert_pem( + "site-1", uris=["https://example.com/nvflare/v1/cell/site-1", "https://nvidia.com/nvflare/v1/job/job-123"] + ) + assert cell_scopes(_cert(other)) == [] -def test_fqcn_belongs_to_job_places_the_job_after_the_owner_prefix(): - assert FQCN.belongs_to_job("relay-1.site-1.job-123", "job-123", owner_segments=2) is True - assert FQCN.belongs_to_job("relay-1.site-1.ws_transfer_job-123", "job-123", owner_segments=2) is True - assert FQCN.belongs_to_job("relay-1.site-1.job-123", "job-123", owner_segments=1) is False - assert FQCN.belongs_to_job("relay-1.site-1", "job-123", owner_segments=2) is False +@pytest.mark.parametrize( + "uri", + [ + "https://nvidia.com/nvflare/v1/cell/", + "https://nvidia.com/nvflare/v1/cell/a/b", + "https://nvidia.com/nvflare/v2/cell/a", + "https://nvidia.com/nvflare/cell", + ], +) +def test_cell_scopes_reject_malformed_nvflare_uri(uri): + with pytest.raises(ValueError): + cell_scopes(_cert(_scoped_cert_pem("site-1", uris=[uri]))) -def test_identity_resolver_binds_job_cert_below_the_owning_site_only(): +def test_identity_resolver_enforces_certificate_scope(): resolver = CellIdentityResolver(local_fqcn="server", prefix_identity_map={"site-1": "site-1"}) - - with pytest.raises(ValueError, match="bound to job 'job-123'"): + scoped = _cert(_scoped_cert_pem("site-1", _JOB_SCOPES)) + + for fqcn in ("site-1.job-123", "site-1.job-123.sub-1", "site-1.ws_transfer_job-123"): + resolver.require_match(fqcn, "site-1", "connection", peer_cert=scoped) + for fqcn in ("site-1", "site-1.job-999", "site-1.ws_transfer_job-999", "site-1.job-999.ws_transfer_job-123"): + with pytest.raises(ValueError, match="outside that scope"): + resolver.require_match(fqcn, "site-1", "connection", peer_cert=scoped) + with pytest.raises(ValueError, match="outside that scope"): resolver.require_match( - "site-1.job-999.ws_transfer_job-123", "site-1", "connection bootstrap", peer_job_id="job-123" + "_admin_9af49fef-235f-41bd-9296-12fd09eacb2a", "admin@nvidia.com", "connection admin", peer_cert=scoped ) - with pytest.raises(ValueError, match="bound to job 'job-123'"): - resolver.require_match("site-1.job-999.job-123", "site-1", "connection sub", peer_job_id="job-123") -def test_identity_resolver_binds_job_cert_behind_relay(): +def test_identity_resolver_leaves_unrestricted_cert_alone(): + resolver = CellIdentityResolver(local_fqcn="server", prefix_identity_map={"site-1": "site-1"}) + site_cert = _cert(_cert_pem("site-1")) + + resolver.require_match("site-1", "site-1", "connection cp", peer_cert=site_cert) + resolver.require_match("site-1.job-123", "site-1", "connection cj", peer_cert=site_cert) + + +def test_identity_resolver_enforces_scope_behind_relay(): resolver = CellIdentityResolver(local_fqcn="relay-1", prefix_identity_map={"relay-1.site-1": "site-1"}) + scoped = _cert(_scoped_cert_pem("site-1", ["relay-1.site-1.job-123"])) - resolver.require_match("relay-1.site-1.job-123", "site-1", "connection cj", peer_job_id="job-123") - with pytest.raises(ValueError, match="bound to job 'job-123'"): - resolver.require_match("relay-1.site-1", "site-1", "connection cp", peer_job_id="job-123") + resolver.require_match("relay-1.site-1.job-123", "site-1", "connection cj", peer_cert=scoped) + with pytest.raises(ValueError, match="outside that scope"): + resolver.require_match("relay-1.site-1", "site-1", "connection cp", peer_cert=scoped) -def test_identity_resolver_binds_job_cert_to_job_fqcn(): +def test_identity_resolver_rejects_malformed_scope_uri(): resolver = CellIdentityResolver(local_fqcn="server", prefix_identity_map={"site-1": "site-1"}) + malformed = _cert(_scoped_cert_pem("site-1", uris=["https://nvidia.com/nvflare/v1/cell/"])) - resolver.require_match("site-1.job-123", "site-1", "connection cj", peer_job_id="job-123") - resolver.require_match("site-1.job-123.sub-1", "site-1", "connection sub", peer_job_id="job-123") - resolver.require_match("site-1.ws_transfer_job-123", "site-1", "connection bootstrap", peer_job_id="job-123") - - with pytest.raises(ValueError, match="bound to job 'job-123'"): - resolver.require_match("site-1.job-999", "site-1", "connection cj", peer_job_id="job-123") - with pytest.raises(ValueError, match="bound to job 'job-123'"): - resolver.require_match("site-1.ws_transfer_job-999", "site-1", "connection bootstrap", peer_job_id="job-123") - with pytest.raises(ValueError, match="bound to job 'job-123'"): - resolver.require_match("site-1", "site-1", "connection cp", peer_job_id="job-123") - with pytest.raises(ValueError, match="bound to job 'job-123'"): - resolver.require_match( - "_admin_9af49fef-235f-41bd-9296-12fd09eacb2a", "admin@nvidia.com", "connection admin", peer_job_id="job-123" - ) + with pytest.raises(ValueError, match="malformed"): + resolver.require_match("site-1.job-123", "site-1", "connection cj", peer_cert=malformed) @pytest.mark.parametrize("endpoint_name", ["site-1.job-123", "site-1.ws_transfer_job-123"]) -def test_mtls_handshake_accepts_job_cert_for_own_job(endpoint_name): +def test_mtls_handshake_accepts_scoped_cert_inside_its_scope(endpoint_name): manager = _conn_manager(identity_map={"site-1": "site-1"}) - conn = _FakeConnection(peer_cn="site-1", peer_job_id="job-123") + conn = _FakeConnection(peer_cn="site-1", peer_cert=_der(_scoped_cert_pem("site-1", _JOB_SCOPES))) sfm_conn = SfmConnection(conn, Endpoint("server")) manager.update_endpoint(sfm_conn, {HandshakeKeys.ENDPOINT_NAME: endpoint_name}) @@ -537,9 +554,9 @@ def test_mtls_handshake_accepts_job_cert_for_own_job(endpoint_name): @pytest.mark.parametrize( "endpoint_name", ["site-1.job-999", "site-1", "site-1.ws_transfer_job-999", "site-1.job-999.ws_transfer_job-123"] ) -def test_mtls_handshake_rejects_job_cert_outside_its_job(endpoint_name): +def test_mtls_handshake_rejects_scoped_cert_outside_its_scope(endpoint_name): manager = _conn_manager(identity_map={"site-1": "site-1"}) - conn = _FakeConnection(peer_cn="site-1", peer_job_id="job-123") + conn = _FakeConnection(peer_cn="site-1", peer_cert=_der(_scoped_cert_pem("site-1", _JOB_SCOPES))) sfm_conn = SfmConnection(conn, Endpoint("server")) with pytest.raises(CommError) as ex: @@ -550,31 +567,37 @@ def test_mtls_handshake_rejects_job_cert_outside_its_job(endpoint_name): assert conn.closed -def test_mtls_certificate_cache_binds_job_cert_to_job_fqcn(): +def test_mtls_certificate_cache_enforces_certificate_scope(): resolver = CellIdentityResolver(local_fqcn="server", prefix_identity_map={"site-1": "site-1"}) manager = CredentialManager(Endpoint("server"), identity_resolver=resolver, enforce_identity=True) - cert = _job_cert_pem("site-1", "job-123") + cert = _scoped_cert_pem("site-1", _JOB_SCOPES) own_job = Message(headers={MessageHeaderKey.ORIGIN: "site-1.job-123"}, payload={CERT_CONTENT: cert}) assert manager.process_response(own_job) == cert for origin in ("site-1", "site-1.job-999", "site-1.job-999.ws_transfer_job-123"): - with pytest.raises(RuntimeError, match="bound to job 'job-123'"): + with pytest.raises(RuntimeError, match="outside that scope"): manager.process_response(Message(headers={MessageHeaderKey.ORIGIN: origin}, payload={CERT_CONTENT: cert})) assert origin not in manager.cert_cache -def test_peer_job_id_extraction(): - assert get_cert_job_id_from_pem(_cert_pem("site-1")) is None - assert get_cert_job_id_from_pem(_job_cert_pem("site-1", "job-123")) == "job-123" - assert "�" in get_cert_job_id_from_pem(_job_cert_pem("site-1", b"\xff\xfe")) +def test_peer_cert_exposure(): + pem = _scoped_cert_pem("site-1", _JOB_SCOPES) + der = _der(pem) - der = x509.load_pem_x509_certificate(_job_cert_pem("site-1", "job-123")).public_bytes(serialization.Encoding.DER) - assert get_peer_job_id(SimpleNamespace(getpeercert=lambda binary_form=False: der)) == "job-123" - assert get_peer_job_id(None) is None + props = {} + add_peer_cert(props, SimpleNamespace(getpeercert=lambda binary_form=False: der)) + assert props[DriverParams.PEER_CERT.value] == der + props = {} + add_peer_cert(props, None) + assert props == {} - assert get_grpc_peer_job_id({"x509_pem_cert": [_job_cert_pem("site-1", "job-123")]}) == "job-123" - assert get_grpc_peer_job_id({"x509_common_name": [b"site-1"]}) is None + props = {} + add_grpc_peer_cert(props, {"x509_pem_cert": [pem]}) + assert props[DriverParams.PEER_CERT.value] == der + props = {} + add_grpc_peer_cert(props, {"x509_common_name": [b"site-1"]}) + assert props == {} def test_cell_cipher_accepts_leaf_certificate_with_intermediate_chain(): diff --git a/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py b/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py index 07e9fdfc18..d7ed9ced03 100644 --- a/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py +++ b/tests/unit_test/fuel/f3/cellnet/job_cert_binding_live_test.py @@ -23,14 +23,13 @@ import traceback import pytest -from cryptography import x509 from nvflare.fuel.f3.cellnet.cell import Cell from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.drivers.driver_params import DriverParams from nvflare.fuel.f3.message import Message +from nvflare.fuel.sec.cert_uri import job_ca_marker_uri from nvflare.fuel.utils.config_service import ConfigService -from nvflare.lighter.constants import CertExtensionOID from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert, serialize_pri_key from nvflare.private.fed.utils.job_cert_utils import JobCertIssuer @@ -46,7 +45,7 @@ class _RejectionRecorder(logging.Handler): - """Forwards the parent's job-binding rejections to the test process.""" + """Forwards the parent's certificate-scope rejections to the test process.""" def __init__(self, queue): super().__init__(level=logging.ERROR) @@ -54,7 +53,7 @@ def __init__(self, queue): def emit(self, record): message = record.getMessage() - if "bound to job" in message: + if "outside that scope" in message: self.queue.put(message) @@ -79,7 +78,6 @@ def _write_pki(out_dir: str) -> dict: Identity("site-1"), Identity("rootCA"), root_key, site_pub, server_default_host="localhost" ) jca_key, jca_pub = generate_keys() - marker = x509.UnrecognizedExtension(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER), b"job_ca") jca_cert = generate_cert( Identity("job_ca"), Identity("rootCA"), @@ -87,11 +85,11 @@ def _write_pki(out_dir: str) -> dict: jca_pub, ca=True, ca_path_length=0, - extra_extensions=[(marker, False)], + uri_names=[job_ca_marker_uri()], ) issuer = JobCertIssuer(serialize_cert(jca_cert), jca_key) - job_a_crt, job_a_key = issuer.issue("site-1", _JOB_A) - job_b_crt, job_b_key = issuer.issue("site-1", _JOB_B) + job_a_crt, job_a_key = issuer.issue("site-1", _JOB_A, "site-1") + job_b_crt, job_b_key = issuer.issue("site-1", _JOB_B, "site-1") files = { "rootCA.pem": serialize_cert(root_cert), "server.crt": serialize_cert(server_cert), @@ -271,4 +269,4 @@ def test_site_parent_rejects_another_jobs_cert_on_job_fqcn(site_parent, claimed_ assert result["rc"] != ReturnCode.OK rejection = _await_rejection_of(site_parent[2], claimed_fqcn) - assert f"bound to job '{_JOB_A}'" in rejection + assert f"site-1.{_JOB_A}" in rejection # the certificate's own scope is named in the rejection diff --git a/tests/unit_test/fuel/f3/drivers/grpc_driver_test.py b/tests/unit_test/fuel/f3/drivers/grpc_driver_test.py index 8801801626..82b74911ea 100644 --- a/tests/unit_test/fuel/f3/drivers/grpc_driver_test.py +++ b/tests/unit_test/fuel/f3/drivers/grpc_driver_test.py @@ -17,22 +17,25 @@ import pytest from cryptography import x509 +from cryptography.hazmat.primitives import serialization from nvflare.fuel.f3.drivers import aio_grpc_driver, grpc_driver from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import JOB_ID_EXTENSION_OID +from nvflare.fuel.sec.cert_uri import CELL_URI_KIND, cert_uri from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert -def _cert_pem(common_name: str, job_id=None) -> bytes: +def _cert_pem(common_name: str, scoped: bool) -> bytes: key, pub_key = generate_keys() - extensions = None - if job_id: - extensions = [(x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, job_id.encode("utf-8")), False)] - cert = generate_cert(Identity(common_name), Identity(common_name), key, pub_key, extra_extensions=extensions) + uri_names = [cert_uri(CELL_URI_KIND, "site-1.job-1")] if scoped else None + cert = generate_cert(Identity(common_name), Identity(common_name), key, pub_key, uri_names=uri_names) return serialize_cert(cert) +def _der(pem: bytes) -> bytes: + return x509.load_pem_x509_certificate(pem).public_bytes(serialization.Encoding.DER) + + def _authenticated_context(cert_pem: bytes): context = MagicMock() context.peer.return_value = "ipv4:127.0.0.1:50001" @@ -46,28 +49,30 @@ def _server(): return server -@pytest.mark.parametrize("job_id", ["job-1", None]) -def test_grpc_servicer_records_peer_identity_of_authenticated_stream(job_id): +@pytest.mark.parametrize("scoped", [True, False]) +def test_grpc_servicer_records_peer_identity_of_authenticated_stream(scoped): servicer = grpc_driver.Servicer(_server()) + pem = _cert_pem("site-1", scoped) with ( patch.object(grpc_driver, "StreamConnection") as connection_cls, patch.object(grpc_driver.threading, "Thread"), ): connection_cls.return_value.generate_output.return_value = iter([]) - list(servicer.Stream(iter([]), _authenticated_context(_cert_pem("site-1", job_id)))) + list(servicer.Stream(iter([]), _authenticated_context(pem))) conn_props = connection_cls.call_args.args[2] assert conn_props[DriverParams.PEER_CN.value] == "site-1" - assert conn_props.get(DriverParams.PEER_JOB_ID.value) == job_id + assert conn_props[DriverParams.PEER_CERT.value] == _der(pem) -@pytest.mark.parametrize("job_id", ["job-1", None]) -def test_aio_grpc_servicer_records_peer_identity_of_authenticated_stream(job_id): +@pytest.mark.parametrize("scoped", [True, False]) +def test_aio_grpc_servicer_records_peer_identity_of_authenticated_stream(scoped): servicer = aio_grpc_driver.Servicer(_server(), aio_ctx=MagicMock()) + pem = _cert_pem("site-1", scoped) async def consume(): - async for _ in servicer.Stream(iter([]), _authenticated_context(_cert_pem("site-1", job_id))): + async for _ in servicer.Stream(iter([]), _authenticated_context(pem)): pass with patch.object(aio_grpc_driver, "AioStreamSession") as session_cls: @@ -76,4 +81,4 @@ async def consume(): conn_props = session_cls.call_args.kwargs["conn_props"] assert conn_props[DriverParams.PEER_CN.value] == "site-1" - assert conn_props.get(DriverParams.PEER_JOB_ID.value) == job_id + assert conn_props[DriverParams.PEER_CERT.value] == _der(pem) diff --git a/tests/unit_test/lighter/cert_builder_test.py b/tests/unit_test/lighter/cert_builder_test.py index 379af43365..2f957f536d 100644 --- a/tests/unit_test/lighter/cert_builder_test.py +++ b/tests/unit_test/lighter/cert_builder_test.py @@ -21,7 +21,8 @@ import pytest from cryptography import x509 -from nvflare.lighter.constants import CertExtensionOID, CtxKey, ParticipantType, ProvFileName +from nvflare.fuel.sec.cert_uri import CA_URI_KIND, JOB_CA_URI_VALUE, cert_uri_values +from nvflare.lighter.constants import CtxKey, ParticipantType, ProvFileName from nvflare.lighter.entity import Participant, Project from nvflare.lighter.impl.cert import CertBuilder from nvflare.lighter.impl.workspace import WorkspaceBuilder @@ -179,7 +180,7 @@ def test_job_ca_written_to_server_kit_by_default(tmp_path): assert basic_constraints.value.path_length == 0 key_usage = job_ca_cert.extensions.get_extension_for_class(x509.KeyUsage) assert key_usage.value.key_cert_sign is True - job_ca_cert.extensions.get_extension_for_oid(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER)) + assert cert_uri_values(job_ca_cert, CA_URI_KIND) == [JOB_CA_URI_VALUE] verify_cert(job_ca_cert, ctx[CtxKey.ROOT_CERT].public_key()) assert job_ca_cert.not_valid_after_utc <= ctx[CtxKey.ROOT_CERT].not_valid_after_utc diff --git a/tests/unit_test/private/fed/server/job_runner_deploy_test.py b/tests/unit_test/private/fed/server/job_runner_deploy_test.py index 7e4aaefcc1..003ab348f8 100644 --- a/tests/unit_test/private/fed/server/job_runner_deploy_test.py +++ b/tests/unit_test/private/fed/server/job_runner_deploy_test.py @@ -30,13 +30,19 @@ from nvflare.apis.fl_constant import FLContextKey, SecureTrainConst, SiteType from nvflare.apis.job_def import Job from nvflare.apis.workspace import Workspace -from nvflare.fuel.f3.drivers.net_utils import get_cert_job_id -from nvflare.lighter.constants import CertExtensionOID, ProvFileName +from nvflare.fuel.sec.cert_uri import CELL_URI_KIND, cert_uri_values, job_ca_marker_uri +from nvflare.lighter.constants import ProvFileName from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert, serialize_pri_key from nvflare.private.admin_defs import Message, MsgHeader, ReturnCode from nvflare.private.defs import RequestHeader from nvflare.private.fed.server.job_runner import JobRunner -from nvflare.private.fed.utils.job_cert_utils import job_cert_paths, read_job_cert, unpack_job_cert_header +from nvflare.private.fed.utils.job_cert_utils import ( + get_cert_job_id, + job_cell_scopes, + job_cert_paths, + read_job_cert, + unpack_job_cert_header, +) # --------------------------------------------------------------------------- # Helpers @@ -82,6 +88,7 @@ def _build_fl_ctx(token_to_reply: dict, job_id="job-1", min_sites=None, required c = MagicMock(spec=Client) c.token = token c.name = client_name + c.get_fqcn.return_value = client_name client_objects.append(c) sites[client_name] = MagicMock() @@ -138,7 +145,6 @@ def _write_server_kit_with_job_ca(startup): server_key, server_pub = generate_keys() server_cert = generate_cert(Identity("server-cn"), Identity("rootCA"), root_key, server_pub) job_ca_key, job_ca_pub = generate_keys() - marker = x509.UnrecognizedExtension(x509.ObjectIdentifier(CertExtensionOID.JOB_CA_MARKER), b"job_ca") job_ca_cert = generate_cert( Identity("job_ca"), Identity("rootCA"), @@ -146,7 +152,7 @@ def _write_server_kit_with_job_ca(startup): job_ca_pub, ca=True, ca_path_length=0, - extra_extensions=[(marker, False)], + uri_names=[job_ca_marker_uri()], ) (startup / "rootCA.pem").write_bytes(serialize_cert(root_cert)) (startup / "server.crt").write_bytes(serialize_cert(server_cert)) @@ -196,6 +202,7 @@ def make_deploy_message(*_args, **_kwargs): sj_cert = x509.load_pem_x509_certificate(sj_cert_pem) assert _common_name(sj_cert) == "server-cn" assert get_cert_job_id(sj_cert) == "job-1" + assert cert_uri_values(sj_cert, CELL_URI_KIND) == job_cell_scopes(SiteType.SERVER, "job-1") assert sj_cert_pem.count(b"BEGIN CERTIFICATE") == 2 # leaf + job CA, chains to the root assert oct(os.stat(job_cert_paths(run_dir)[1]).st_mode & 0o777) == "0o600" assert serialization.load_pem_private_key(sj_key_pem, None).public_key() == sj_cert.public_key() @@ -205,6 +212,7 @@ def make_deploy_message(*_args, **_kwargs): cj_cert = x509.load_pem_x509_certificate(cj_cert_pem) assert _common_name(cj_cert) == "site-1" assert get_cert_job_id(cj_cert) == "job-1" + assert cert_uri_values(cj_cert, CELL_URI_KIND) == job_cell_scopes("site-1", "job-1") assert cj_cert.issuer == x509.load_pem_x509_certificate((startup / ProvFileName.JOB_CA_CERT).read_bytes()).subject assert cj_cert.public_key() != sj_cert.public_key() assert serialization.load_pem_private_key(cj_key_pem, None).public_key() == cj_cert.public_key() diff --git a/tests/unit_test/private/fed/utils/identity_utils_test.py b/tests/unit_test/private/fed/utils/identity_utils_test.py index f09d82f96b..870039d4f4 100644 --- a/tests/unit_test/private/fed/utils/identity_utils_test.py +++ b/tests/unit_test/private/fed/utils/identity_utils_test.py @@ -16,11 +16,10 @@ from cryptography import x509 from cryptography.x509.oid import ExtendedKeyUsageOID -from nvflare.fuel.f3.drivers.net_utils import JOB_ID_EXTENSION_OID +from nvflare.fuel.sec.cert_uri import JOB_URI_KIND, cert_uri, job_ca_marker_uri from nvflare.lighter.impl.cert import serialize_cert from nvflare.lighter.utils import Identity, generate_cert, generate_keys, sign_content from nvflare.private.fed.utils.identity_utils import IdentityVerifier, InvalidAsserterCert, get_parent_site_name -from nvflare.private.fed.utils.job_cert_utils import JOB_CA_MARKER_OID class TestIdentityUtils: @@ -40,7 +39,7 @@ def test_get_parent_site_name(self, fqsn, result): assert get_parent_site_name(fqsn) == result -def _make_root_and_client_certs(extra_extensions=None): +def _make_root_and_client_certs(extra_extensions=None, uri_names=None): root_key, root_pub_key = generate_keys() root_cert = generate_cert( subject=Identity("root", "nvidia"), @@ -56,6 +55,7 @@ def _make_root_and_client_certs(extra_extensions=None): signing_pri_key=root_key, subject_pub_key=client_pub_key, extra_extensions=extra_extensions, + uri_names=uri_names, ) return root_cert, root_key, client_cert, client_key @@ -154,13 +154,12 @@ def test_identity_verifier_rejects_job_scoped_cert_chain(tmp_path): ca_path_length=0, ) leaf_key, leaf_pub_key = generate_keys() - job_id_ext = x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, b"job-123") leaf_cert = generate_cert( subject=Identity("client", "nvidia"), issuer=Identity("job_ca.test", "nvidia"), signing_pri_key=job_ca_key, subject_pub_key=leaf_pub_key, - extra_extensions=[(job_id_ext, False)], + uri_names=[cert_uri(JOB_URI_KIND, "job-123")], ) root_cert_path = tmp_path / "root.crt" root_cert_path.write_bytes(serialize_cert(root_cert)) @@ -189,7 +188,6 @@ def test_identity_verifier_rejects_leaf_minted_by_job_ca_without_extension(tmp_p subject_pub_key=root_pub_key, ca=True, ) - marker = x509.UnrecognizedExtension(JOB_CA_MARKER_OID, b"job_ca") job_ca_key, job_ca_pub_key = generate_keys() job_ca_cert = generate_cert( subject=Identity("job_ca.test", "nvidia"), @@ -198,7 +196,7 @@ def test_identity_verifier_rejects_leaf_minted_by_job_ca_without_extension(tmp_p subject_pub_key=job_ca_pub_key, ca=True, ca_path_length=0, - extra_extensions=[(marker, False)], + uri_names=[job_ca_marker_uri()], ) leaf_key, leaf_pub_key = generate_keys() leaf_cert = generate_cert( @@ -223,25 +221,27 @@ def test_identity_verifier_rejects_leaf_minted_by_job_ca_without_extension(tmp_p ) -def test_identity_verifier_rejects_job_extension_with_non_utf8_value(tmp_path): - # rejection must not depend on decoding the extension value - job_id_ext = x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, b"\xff\xfe\xfd") - root_cert, _root_key, client_cert, client_key = _make_root_and_client_certs(extra_extensions=[(job_id_ext, False)]) +def test_identity_verifier_rejects_malformed_job_uri(tmp_path): + # a malformed NVFlare URI fails closed instead of being ignored + root_cert, _root_key, client_cert, client_key = _make_root_and_client_certs( + uri_names=["https://nvidia.com/nvflare/v1/job/"] + ) root_cert_path = tmp_path / "root.crt" root_cert_path.write_bytes(serialize_cert(root_cert)) verifier = IdentityVerifier(str(root_cert_path)) signature = sign_content("client" + "nonce", client_key, return_str=False) - with pytest.raises(InvalidAsserterCert, match="job-scoped"): + with pytest.raises(InvalidAsserterCert, match="malformed"): verifier.verify_common_name( "client", "nonce", client_cert, signature, expected_eku=ExtendedKeyUsageOID.CLIENT_AUTH ) def test_identity_verifier_rejects_job_extension_even_when_root_issued(tmp_path): - # the rejection is keyed on the job-id extension, not the issuer - job_id_ext = x509.UnrecognizedExtension(JOB_ID_EXTENSION_OID, b"job-123") - root_cert, _root_key, client_cert, client_key = _make_root_and_client_certs(extra_extensions=[(job_id_ext, False)]) + # the rejection is keyed on the job URI, not the issuer + root_cert, _root_key, client_cert, client_key = _make_root_and_client_certs( + uri_names=[cert_uri(JOB_URI_KIND, "job-123")] + ) root_cert_path = tmp_path / "root.crt" root_cert_path.write_bytes(serialize_cert(root_cert)) verifier = IdentityVerifier(str(root_cert_path)) diff --git a/tests/unit_test/private/fed/utils/job_cert_utils_test.py b/tests/unit_test/private/fed/utils/job_cert_utils_test.py index b41feff6a0..551385388b 100644 --- a/tests/unit_test/private/fed/utils/job_cert_utils_test.py +++ b/tests/unit_test/private/fed/utils/job_cert_utils_test.py @@ -23,7 +23,7 @@ from nvflare.apis.fl_constant import SecureTrainConst from nvflare.fuel.f3.cellnet.cell_cipher import SimpleCellCipher -from nvflare.fuel.f3.drivers.net_utils import get_cert_job_id +from nvflare.fuel.sec.cert_uri import CELL_URI_KIND, cert_uri_values, job_ca_marker_uri from nvflare.lighter.constants import ProvFileName from nvflare.lighter.utils import ( Identity, @@ -34,7 +34,6 @@ verify_cert_chain, ) from nvflare.private.fed.utils.job_cert_utils import ( - JOB_CA_MARKER_OID, JOB_CERT_BACKDATE, JOB_CERT_FILE_NAME, JOB_CERT_VALID_DAYS, @@ -42,13 +41,17 @@ JobCertError, apply_job_cert_config, find_job_cert, + get_cert_job_id, has_job_ca_marker, + job_cell_scopes, + job_cert_uris, job_startup_files, load_job_cert_issuer, pack_job_cert_header, read_job_cert, stage_job_startup_dir, unpack_job_cert_header, + workspace_transfer_cell_name, write_job_cert, ) @@ -67,7 +70,6 @@ def _write_job_ca(startup_dir, ca_lifetime=datetime.timedelta(days=360), expired not_valid_after = now + ca_lifetime ca_key, ca_pub = generate_keys() - marker = x509.UnrecognizedExtension(JOB_CA_MARKER_OID, b"job_ca") ca_cert = generate_cert( Identity("job_ca.test"), Identity("root"), @@ -77,7 +79,7 @@ def _write_job_ca(startup_dir, ca_lifetime=datetime.timedelta(days=360), expired ca_path_length=0, not_valid_before=not_valid_before, not_valid_after=not_valid_after, - extra_extensions=[(marker, False)], + uri_names=[job_ca_marker_uri()], ) with open(os.path.join(startup_dir, ProvFileName.JOB_CA_CERT), "wb") as f: @@ -109,7 +111,7 @@ def test_issued_cert_chains_to_root_and_carries_job_id(tmp_path): issuer = load_job_cert_issuer(str(tmp_path)) assert issuer is not None - cert_pem, key_pem = issuer.issue("site-1", "job-123") + cert_pem, key_pem = issuer.issue("site-1", "job-123", "site-1") chain = x509.load_pem_x509_certificates(cert_pem) assert len(chain) == 2 @@ -118,6 +120,7 @@ def test_issued_cert_chains_to_root_and_carries_job_id(tmp_path): verify_cert_chain(leaf_cert=leaf, intermediate_certs=[intermediate], root_ca_cert=root_cert) assert leaf.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value == "site-1" assert get_cert_job_id(leaf) == "job-123" + assert cert_uri_values(leaf, CELL_URI_KIND) == job_cell_scopes("site-1", "job-123") assert has_job_ca_marker(intermediate) and not has_job_ca_marker(leaf) expected_lifetime = datetime.timedelta(days=JOB_CERT_VALID_DAYS) + JOB_CERT_BACKDATE assert leaf.not_valid_after_utc - leaf.not_valid_before_utc == expected_lifetime @@ -128,7 +131,7 @@ def test_issued_cert_validity_clamped_to_job_ca(tmp_path): _, ca_cert = _write_job_ca(str(tmp_path), ca_lifetime=datetime.timedelta(days=1)) issuer = load_job_cert_issuer(str(tmp_path)) - cert_pem, _ = issuer.issue("site-1", "job-123") + cert_pem, _ = issuer.issue("site-1", "job-123", "site-1") leaf = x509.load_pem_x509_certificates(cert_pem)[0] assert leaf.not_valid_after_utc == ca_cert.not_valid_after_utc.replace(microsecond=0) @@ -138,7 +141,7 @@ def test_issue_honors_valid_days(tmp_path): _write_job_ca(str(tmp_path)) issuer = load_job_cert_issuer(str(tmp_path)) - cert_pem, _ = issuer.issue("site-1", "job-123", valid_days=3) + cert_pem, _ = issuer.issue("site-1", "job-123", "site-1", valid_days=3) leaf = x509.load_pem_x509_certificates(cert_pem)[0] assert leaf.not_valid_after_utc - leaf.not_valid_before_utc == datetime.timedelta(days=3) + JOB_CERT_BACKDATE @@ -148,13 +151,38 @@ def test_issue_many_issues_one_credential_per_site(tmp_path): _write_job_ca(str(tmp_path)) issuer = load_job_cert_issuer(str(tmp_path)) - creds = issuer.issue_many(["site-1", "site-2"], "job-123") + creds = issuer.issue_many({"site-1": "site-1", "site-2": "relay-1.site-2"}, "job-123") assert set(creds) == {"site-1", "site-2"} leaves = {name: x509.load_pem_x509_certificates(cert_pem)[0] for name, (cert_pem, _) in creds.items()} assert {leaf.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value for leaf in leaves.values()} == set(creds) assert leaves["site-1"].public_key() != leaves["site-2"].public_key() - assert issuer.issue_many([], "job-123") == {} + assert cert_uri_values(leaves["site-2"], CELL_URI_KIND) == job_cell_scopes("relay-1.site-2", "job-123") + assert issuer.issue_many({}, "job-123") == {} + + +def test_job_cert_uris_name_the_job_and_its_cells(): + assert workspace_transfer_cell_name("job-1") == "ws_transfer_job-1" + assert job_cell_scopes("relay-1.site-1", "job-1") == ["relay-1.site-1.job-1", "relay-1.site-1.ws_transfer_job-1"] + assert job_cert_uris("site-1", "job-1") == [ + "https://nvidia.com/nvflare/v1/job/job-1", + "https://nvidia.com/nvflare/v1/cell/site-1.job-1", + "https://nvidia.com/nvflare/v1/cell/site-1.ws_transfer_job-1", + ] + + +def test_get_cert_job_id_rejects_a_cert_claiming_several_jobs(): + key, pub_key = generate_keys() + cert = generate_cert( + Identity("site-1"), + Identity("site-1"), + key, + pub_key, + uri_names=["https://nvidia.com/nvflare/v1/job/job-1", "https://nvidia.com/nvflare/v1/job/job-2"], + ) + + with pytest.raises(ValueError, match="several jobs"): + get_cert_job_id(cert) def test_pack_unpack_job_cert_header_round_trip(): @@ -223,8 +251,8 @@ def test_cell_cipher_works_with_job_cert_chains(tmp_path): root_cert, _ = _write_job_ca(str(tmp_path)) issuer = load_job_cert_issuer(str(tmp_path)) - sj_cert_pem, sj_key_pem = issuer.issue("server", "job-123") - cj_cert_pem, cj_key_pem = issuer.issue("site-1", "job-123") + sj_cert_pem, sj_key_pem = issuer.issue("server", "job-123", "server") + cj_cert_pem, cj_key_pem = issuer.issue("site-1", "job-123", "site-1") sj_cipher = SimpleCellCipher( root_cert, From 1dbde5a68928361aafd8b09ef777560351ded9fa Mon Sep 17 00:00:00 2001 From: Peter Cnudde Date: Mon, 14 Sep 2026 13:58:02 -0700 Subject: [PATCH 10/10] refactor(f3): derive peer identity once in Connection.record_peer Review follow-up on #5263: the per-driver steps that derive PEER_CN from the peer certificate and record PEER_CERT were duplicated across the socket, asyncio, websocket and both gRPC transports, each with its own fallback convention, and pulled nvflare.fuel.hci.security into the transport layer. Connection.record_peer (F3 base class) now takes the raw peer certificate, DER or PEM as the transport provides it, and sets PEER_CERT and PEER_CN once; a secure connection with no peer certificate still reports PEER_CN "N/A". Each driver hands over the certificate in one line. Consumers of PEER_CN and PEER_CERT are unchanged. Driver files are now smaller than upstream. --- docs/design/per_job_certs_design.md | 5 ++-- nvflare/fuel/f3/connection.py | 25 ++++++++++++++++++- nvflare/fuel/f3/drivers/aio_conn.py | 17 +++---------- nvflare/fuel/f3/drivers/aio_grpc_driver.py | 9 +++---- nvflare/fuel/f3/drivers/aio_http_driver.py | 16 ++++-------- nvflare/fuel/f3/drivers/grpc_driver.py | 9 +++---- nvflare/fuel/f3/drivers/net_utils.py | 25 ------------------- nvflare/fuel/f3/drivers/socket_conn.py | 11 ++------ .../fuel/f3/cellnet/identity_binding_test.py | 20 +++++++-------- 9 files changed, 52 insertions(+), 85 deletions(-) diff --git a/docs/design/per_job_certs_design.md b/docs/design/per_job_certs_design.md index 4b006c0985..6ab55f1ab4 100644 --- a/docs/design/per_job_certs_design.md +++ b/docs/design/per_job_certs_design.md @@ -228,8 +228,9 @@ admin identity; its blast radius is job cells only. Site-scope rejection stops a job credential from acting as a site. Job binding stops one job's credential from acting as another job's cell: -- Every TLS driver exposes the authenticated peer certificate as the - `PEER_CERT` connection property next to `PEER_CN`. Drivers parse nothing. +- Every TLS driver hands the raw peer certificate to `Connection.record_peer` + (the F3 base class), which derives `PEER_CN` and stores the DER as + `PEER_CERT` once for all transports. Drivers parse nothing. - Cellnet knows one generic rule, in its own vocabulary: a certificate may carry cell-scope URIs, and a peer presenting one may only claim an FQCN equal to or under one of those cells (`cell_scopes` / `fqcn_in_scopes` in diff --git a/nvflare/fuel/f3/connection.py b/nvflare/fuel/f3/connection.py index d88ffdcb78..d5d387b688 100644 --- a/nvflare/fuel/f3/connection.py +++ b/nvflare/fuel/f3/connection.py @@ -15,7 +15,11 @@ import threading from abc import ABC, abstractmethod from enum import Enum -from typing import Union +from typing import Optional, Union + +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.x509.oid import NameOID from nvflare.fuel.f3.drivers.connector_info import ConnectorInfo, Mode from nvflare.fuel.f3.drivers.driver_params import DriverParams @@ -72,6 +76,25 @@ def get_conn_properties(self) -> dict: """ pass + @staticmethod + def record_peer(conn_props: dict, peer_cert: Optional[bytes], secure: bool = False) -> None: + """Record the authenticated peer: its certificate (PEER_CERT, DER) and the CN derived from it (PEER_CN). + + peer_cert is DER or PEM, as the transport provides it. A secure connection with no peer + certificate (TLS without client authentication) reports PEER_CN "N/A". + """ + if not peer_cert: + if secure: + conn_props[DriverParams.PEER_CN.value] = "N/A" + return + if peer_cert.startswith(b"-----BEGIN"): + cert = x509.load_pem_x509_certificate(peer_cert) + else: + cert = x509.load_der_x509_certificate(peer_cert) + common_names = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) + conn_props[DriverParams.PEER_CERT.value] = cert.public_bytes(serialization.Encoding.DER) + conn_props[DriverParams.PEER_CN.value] = common_names[0].value if common_names else "N/A" + @abstractmethod def close(self): """Close connection diff --git a/nvflare/fuel/f3/drivers/aio_conn.py b/nvflare/fuel/f3/drivers/aio_conn.py index 217a310010..1964a1642c 100644 --- a/nvflare/fuel/f3/drivers/aio_conn.py +++ b/nvflare/fuel/f3/drivers/aio_conn.py @@ -19,9 +19,8 @@ from nvflare.fuel.f3.drivers.aio_context import AioContext from nvflare.fuel.f3.drivers.connector_info import ConnectorInfo from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE, add_peer_cert +from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE from nvflare.fuel.f3.sfm.prefix import PREFIX_LEN, Prefix -from nvflare.fuel.hci.security import get_certificate_common_name from nvflare.security.logging import secure_format_exception log = logging.getLogger(__name__) @@ -131,17 +130,7 @@ def _get_aio_properties(self) -> dict: conn_props[DriverParams.LOCAL_ADDR.value] = local_addr conn_props[DriverParams.PEER_ADDR.value] = peer_addr - peer_cert = self.writer.get_extra_info("peercert") - if peer_cert: - cn = get_certificate_common_name(peer_cert) - add_peer_cert(conn_props, self.writer.get_extra_info("ssl_object")) - else: - if self.secure: - cn = "N/A" - else: - cn = None - - if cn: - conn_props[DriverParams.PEER_CN.value] = cn + ssl_object = self.writer.get_extra_info("ssl_object") + self.record_peer(conn_props, ssl_object.getpeercert(binary_form=True) if ssl_object else None, self.secure) return conn_props diff --git a/nvflare/fuel/f3/drivers/aio_grpc_driver.py b/nvflare/fuel/f3/drivers/aio_grpc_driver.py index 320419bf7f..5708971349 100644 --- a/nvflare/fuel/f3/drivers/aio_grpc_driver.py +++ b/nvflare/fuel/f3/drivers/aio_grpc_driver.py @@ -39,7 +39,7 @@ from .driver_params import DriverCap, DriverParams from .grpc.streamer_pb2 import Frame from .grpc.utils import get_grpc_client_credentials, get_grpc_server_credentials, use_aio_grpc -from .net_utils import MAX_FRAME_SIZE, add_grpc_peer_cert, get_address, get_tcp_urls, ssl_required +from .net_utils import MAX_FRAME_SIZE, get_address, get_tcp_urls, ssl_required GRPC_DEFAULT_OPTIONS = [ ("grpc.max_send_message_length", MAX_FRAME_SIZE), @@ -196,11 +196,8 @@ async def Stream(self, request_iterator, context): DriverParams.LOCAL_ADDR.value: get_address(self.server.connector.params), } - auth_context = context.auth_context() - cn_names = auth_context.get("x509_common_name") - if cn_names: - conn_props[DriverParams.PEER_CN.value] = cn_names[0].decode("utf-8") - add_grpc_peer_cert(conn_props, auth_context) + pem_certs = context.auth_context().get("x509_pem_cert") + Connection.record_peer(conn_props, pem_certs[0] if pem_certs else None) connection = AioStreamSession( aio_ctx=self.aio_ctx, diff --git a/nvflare/fuel/f3/drivers/aio_http_driver.py b/nvflare/fuel/f3/drivers/aio_http_driver.py index b961735301..1959bb06d3 100644 --- a/nvflare/fuel/f3/drivers/aio_http_driver.py +++ b/nvflare/fuel/f3/drivers/aio_http_driver.py @@ -26,8 +26,7 @@ from nvflare.fuel.f3.drivers.base_driver import BaseDriver from nvflare.fuel.f3.drivers.driver import ConnectorInfo from nvflare.fuel.f3.drivers.driver_params import DriverCap, DriverParams -from nvflare.fuel.f3.drivers.net_utils import add_peer_cert, get_tcp_urls -from nvflare.fuel.hci.security import get_certificate_common_name +from nvflare.fuel.f3.drivers.net_utils import get_tcp_urls from nvflare.security.logging import secure_format_exception log = logging.getLogger(__name__) @@ -69,15 +68,10 @@ def _get_ws_properties(self) -> dict: if peer_sock: conn_props[DriverParams.PEER_ADDR.value] = f"{peer_sock[0]}:{peer_sock[1]}" - peer_cert = self.websocket.get_extra_info("peercert") - if peer_cert: - cn = get_certificate_common_name(peer_cert) - add_peer_cert(conn_props, self.websocket.get_extra_info("ssl_object")) - else: - cn = "N/A" if self.ssl_context else None - - if cn: - conn_props[DriverParams.PEER_CN.value] = cn + ssl_object = self.websocket.get_extra_info("ssl_object") + self.record_peer( + conn_props, ssl_object.getpeercert(binary_form=True) if ssl_object else None, bool(self.ssl_context) + ) return conn_props diff --git a/nvflare/fuel/f3/drivers/grpc_driver.py b/nvflare/fuel/f3/drivers/grpc_driver.py index 25e3d388bb..136673918b 100644 --- a/nvflare/fuel/f3/drivers/grpc_driver.py +++ b/nvflare/fuel/f3/drivers/grpc_driver.py @@ -37,7 +37,7 @@ from .grpc.qq import QQ from .grpc.streamer_pb2 import Frame from .grpc.utils import get_grpc_client_credentials, get_grpc_server_credentials, use_aio_grpc -from .net_utils import MAX_FRAME_SIZE, add_grpc_peer_cert, get_address, get_tcp_urls, ssl_required +from .net_utils import MAX_FRAME_SIZE, get_address, get_tcp_urls, ssl_required GRPC_DEFAULT_OPTIONS = [ ("grpc.max_send_message_length", MAX_FRAME_SIZE), @@ -136,11 +136,8 @@ def Stream(self, request_iterator, context): DriverParams.PEER_ADDR.value: context.peer(), DriverParams.LOCAL_ADDR.value: get_address(self.server.connector.params), } - auth_context = context.auth_context() - cn_names = auth_context.get("x509_common_name") - if cn_names: - conn_props[DriverParams.PEER_CN.value] = cn_names[0].decode("utf-8") - add_grpc_peer_cert(conn_props, auth_context) + pem_certs = context.auth_context().get("x509_pem_cert") + Connection.record_peer(conn_props, pem_certs[0] if pem_certs else None) try: self.logger.debug(f"SERVER started Stream CB in thread {ct.name}") diff --git a/nvflare/fuel/f3/drivers/net_utils.py b/nvflare/fuel/f3/drivers/net_utils.py index 9cbc08c9a4..c4d328938f 100644 --- a/nvflare/fuel/f3/drivers/net_utils.py +++ b/nvflare/fuel/f3/drivers/net_utils.py @@ -21,9 +21,6 @@ from typing import Any, Optional from urllib.parse import parse_qsl, urlencode, urlparse -from cryptography import x509 -from cryptography.hazmat.primitives import serialization - from nvflare.apis.fl_constant import ConnectionSecurity from nvflare.fuel.f3.comm_error import CommError from nvflare.fuel.f3.drivers.driver_params import DriverParams @@ -120,28 +117,6 @@ def get_ssl_context(params: dict, ssl_server: bool) -> Optional[SSLContext]: return ctx -def get_peer_cert_der(ssl_object) -> Optional[bytes]: - """DER of the peer certificate of an established TLS connection (SSLSocket or SSLObject).""" - if not ssl_object: - return None - return ssl_object.getpeercert(binary_form=True) or None - - -def add_peer_cert(conn_props: dict, ssl_object) -> None: - """Record the authenticated peer certificate next to PEER_CN (TLS socket connections).""" - der = get_peer_cert_der(ssl_object) - if der: - conn_props[DriverParams.PEER_CERT.value] = der - - -def add_grpc_peer_cert(conn_props: dict, auth_context: dict) -> None: - """Record the authenticated peer certificate next to PEER_CN (gRPC server-side connections).""" - pem_certs = auth_context.get("x509_pem_cert") if auth_context else None - if pem_certs: - cert = x509.load_pem_x509_certificate(pem_certs[0]) - conn_props[DriverParams.PEER_CERT.value] = cert.public_bytes(serialization.Encoding.DER) - - def get_address(params: dict) -> str: host = params.get(DriverParams.HOST.value, "0.0.0.0") port = params.get(DriverParams.PORT.value, 0) diff --git a/nvflare/fuel/f3/drivers/socket_conn.py b/nvflare/fuel/f3/drivers/socket_conn.py index cdffe50df6..635b88e079 100644 --- a/nvflare/fuel/f3/drivers/socket_conn.py +++ b/nvflare/fuel/f3/drivers/socket_conn.py @@ -24,9 +24,8 @@ from nvflare.fuel.f3.connection import BytesAlike, Connection from nvflare.fuel.f3.drivers.driver import ConnectorInfo from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE, add_peer_cert +from nvflare.fuel.f3.drivers.net_utils import MAX_FRAME_SIZE from nvflare.fuel.f3.sfm.prefix import PREFIX_LEN, Prefix -from nvflare.fuel.hci.security import get_certificate_common_name from nvflare.security.logging import secure_format_exception log = logging.getLogger(__name__) @@ -220,13 +219,7 @@ def _get_socket_properties(self) -> dict: conn_props[DriverParams.LOCAL_ADDR.value] = self._format_address(local, fileno) if self.secure: - cert = self.sock.getpeercert() - if cert: - cn = get_certificate_common_name(cert) - add_peer_cert(conn_props, self.sock) - else: - cn = "N/A" - conn_props[DriverParams.PEER_CN.value] = cn + self.record_peer(conn_props, self.sock.getpeercert(binary_form=True), secure=True) return conn_props diff --git a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py index f49ea1979e..7d981c83e4 100644 --- a/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py +++ b/tests/unit_test/fuel/f3/cellnet/identity_binding_test.py @@ -38,8 +38,8 @@ from nvflare.fuel.f3.cellnet.identity import ADMIN_LISTENER_KEY, CellIdentityResolver, cell_scopes, fqcn_in_scopes from nvflare.fuel.f3.cellnet.utils import make_reply from nvflare.fuel.f3.comm_error import CommError +from nvflare.fuel.f3.connection import Connection from nvflare.fuel.f3.drivers.driver_params import DriverParams -from nvflare.fuel.f3.drivers.net_utils import add_grpc_peer_cert, add_peer_cert from nvflare.fuel.f3.endpoint import Endpoint from nvflare.fuel.f3.message import Message from nvflare.fuel.f3.sfm.conn_manager import ConnManager @@ -581,22 +581,20 @@ def test_mtls_certificate_cache_enforces_certificate_scope(): assert origin not in manager.cert_cache -def test_peer_cert_exposure(): +def test_connection_records_peer_cert_and_cn(): pem = _scoped_cert_pem("site-1", _JOB_SCOPES) der = _der(pem) - props = {} - add_peer_cert(props, SimpleNamespace(getpeercert=lambda binary_form=False: der)) - assert props[DriverParams.PEER_CERT.value] == der - props = {} - add_peer_cert(props, None) - assert props == {} + for peer_cert in (der, pem): # sockets hand over DER, gRPC hands over PEM + props = {} + Connection.record_peer(props, peer_cert) + assert props == {DriverParams.PEER_CERT.value: der, DriverParams.PEER_CN.value: "site-1"} props = {} - add_grpc_peer_cert(props, {"x509_pem_cert": [pem]}) - assert props[DriverParams.PEER_CERT.value] == der + Connection.record_peer(props, None, secure=True) + assert props == {DriverParams.PEER_CN.value: "N/A"} # TLS without client authentication props = {} - add_grpc_peer_cert(props, {"x509_common_name": [b"site-1"]}) + Connection.record_peer(props, None, secure=False) assert props == {}