Skip to content

Commit ea74f2b

Browse files
authored
fix(vm): bind cached registry images to verified manifest digests (#4342)
Signed-off-by: Adrien Langou <alangou@nvidia.com>
1 parent c608c13 commit ea74f2b

4 files changed

Lines changed: 575 additions & 35 deletions

File tree

‎crates/openshell-driver-vm/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,13 @@ prepared inside the bootstrap VM. The driver checks that a prepared disk
208208
contains the unpacked rootfs before caching it; on failure it caches nothing
209209
and reports the image-prep console tail. Set `OPENSHELL_VM_IMAGE_PULL_CONCURRENCY` to
210210
tune registry layer download parallelism (default `4`, maximum `16`).
211+
Registry pulls in both caches use the resolved digest as an immutable reference.
212+
The driver verifies the downloaded manifest against that digest before downloading
213+
layers. For multi-platform images, it verifies the index and then the selected
214+
platform manifest. Digest-pinned requests retain the caller's digest.
215+
Cache keys hash source identities to keep cache and staging filenames bounded.
216+
Bootstrap cache layout v6 and prepared-image layout v4 rebuild older entries on
217+
first use; older cache files remain on disk but are not reused for registry images.
211218
Both caches are scoped by source image identity and OpenShell version, so an
212219
OpenShell upgrade builds a fresh guest rootfs instead of reusing one with an old
213220
embedded supervisor. Host-side bootstrap layer assembly preserves relative

‎crates/openshell-driver-vm/src/driver.rs‎

Lines changed: 86 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
#[path = "preparation.rs"]
77
mod preparation;
88

9+
#[cfg(test)]
10+
#[path = "registry_cache_tests.rs"]
11+
mod registry_cache_tests;
12+
913
use crate::gpu::{GpuInventory, allocate_vsock_cid};
1014

1115
use crate::isolation::VmBoundarySpec;
@@ -200,8 +204,9 @@ const GUEST_IMAGE_CONFIG_DIR: &str = "openshell-image";
200204
const GUEST_IMAGE_OCI_LAYOUT_DIR: &str = "oci";
201205
const GUEST_IMAGE_OCI_REF: &str = "openshell";
202206
const IMAGE_EXPORT_ROOTFS_ARCHIVE: &str = "source-rootfs.tar";
203-
const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v5";
204-
const PREPARED_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-prepared-rootfs-ext4-umoci-v3";
207+
// Older registry cache entries were not bound to the downloaded manifest.
208+
const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v6";
209+
const PREPARED_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-prepared-rootfs-ext4-umoci-v4";
205210
const IMAGE_IDENTITY_FILE: &str = "image-identity";
206211
const IMAGE_REFERENCE_FILE: &str = "image-reference";
207212
const IMAGE_PREP_INIT_MODE: &str = "image-prep";
@@ -3191,10 +3196,27 @@ impl VmDriver {
31913196
return span_status.finish(result);
31923197
}
31933198

3199+
span_status.finish(
3200+
self.ensure_cached_registry_rootfs_image(
3201+
sandbox_id,
3202+
image_ref,
3203+
&registry_client(),
3204+
&registry_auth(image_ref)?,
3205+
)
3206+
.await,
3207+
)
3208+
}
3209+
3210+
async fn ensure_cached_registry_rootfs_image(
3211+
&self,
3212+
sandbox_id: &str,
3213+
image_ref: &str,
3214+
client: &OciClient,
3215+
auth: &RegistryAuth,
3216+
) -> Result<String, Status> {
3217+
let span_status = openshell_otel::ErrorStatusGuard::current();
31943218
info!(image_ref = %image_ref, "vm driver: ensuring cached root disk image (registry)");
31953219
let reference = parse_registry_reference(image_ref)?;
3196-
let client = registry_client();
3197-
let auth = registry_auth(image_ref)?;
31983220
info!(image_ref = %image_ref, "vm driver: authenticating with registry");
31993221
self.publish_vm_progress(
32003222
sandbox_id,
@@ -3206,7 +3228,7 @@ impl VmDriver {
32063228
]),
32073229
);
32083230
retry_registry_request("authenticate with registry", || {
3209-
client.auth(&reference, &auth, RegistryOperation::Pull)
3231+
client.auth(&reference, auth, RegistryOperation::Pull)
32103232
})
32113233
.await
32123234
.map_err(|err| {
@@ -3224,15 +3246,13 @@ impl VmDriver {
32243246
("image_source".to_string(), "registry".to_string()),
32253247
]),
32263248
);
3227-
let source_image_identity = retry_registry_request("fetch manifest digest", || {
3228-
client.fetch_manifest_digest(&reference, &auth)
3229-
})
3230-
.await
3231-
.map_err(|err| {
3232-
Status::failed_precondition(format!(
3233-
"failed to resolve vm sandbox image '{image_ref}': {err}"
3234-
))
3235-
})?;
3249+
let (reference, source_image_identity) = pin_registry_reference(client, &reference, auth)
3250+
.await
3251+
.map_err(|err| {
3252+
Status::failed_precondition(format!(
3253+
"failed to resolve vm sandbox image '{image_ref}': {err}"
3254+
))
3255+
})?;
32363256
info!(
32373257
image_ref = %image_ref,
32383258
image_identity = %source_image_identity,
@@ -3327,9 +3347,9 @@ impl VmDriver {
33273347

33283348
self.build_cached_registry_image_rootfs_image(
33293349
sandbox_id,
3330-
&client,
3350+
client,
33313351
&reference,
3332-
&auth,
3352+
auth,
33333353
image_ref,
33343354
&image_identity,
33353355
)
@@ -3514,8 +3534,14 @@ impl VmDriver {
35143534
.await;
35153535
}
35163536

3517-
self.ensure_prepared_registry_image_disk(sandbox_id, image_ref, bootstrap_root_disk)
3518-
.await
3537+
self.ensure_prepared_registry_image_disk(
3538+
sandbox_id,
3539+
image_ref,
3540+
bootstrap_root_disk,
3541+
&registry_client(),
3542+
&registry_auth(image_ref)?,
3543+
)
3544+
.await
35193545
}
35203546

35213547
async fn ensure_prepared_local_image_disk(
@@ -3738,10 +3764,10 @@ impl VmDriver {
37383764
sandbox_id: &str,
37393765
image_ref: &str,
37403766
bootstrap_root_disk: &Path,
3767+
client: &OciClient,
3768+
auth: &RegistryAuth,
37413769
) -> Result<PreparedImageDisk, Status> {
37423770
let reference = parse_registry_reference(image_ref)?;
3743-
let client = registry_client();
3744-
let auth = registry_auth(image_ref)?;
37453771

37463772
self.publish_vm_progress(
37473773
sandbox_id,
@@ -3753,7 +3779,7 @@ impl VmDriver {
37533779
]),
37543780
);
37553781
retry_registry_request("authenticate with registry", || {
3756-
client.auth(&reference, &auth, RegistryOperation::Pull)
3782+
client.auth(&reference, auth, RegistryOperation::Pull)
37573783
})
37583784
.await
37593785
.map_err(|err| {
@@ -3771,15 +3797,13 @@ impl VmDriver {
37713797
("image_source".to_string(), "registry".to_string()),
37723798
]),
37733799
);
3774-
let source_image_identity = retry_registry_request("fetch manifest digest", || {
3775-
client.fetch_manifest_digest(&reference, &auth)
3776-
})
3777-
.await
3778-
.map_err(|err| {
3779-
Status::failed_precondition(format!(
3780-
"failed to resolve vm sandbox image '{image_ref}': {err}"
3781-
))
3782-
})?;
3800+
let (reference, source_image_identity) = pin_registry_reference(client, &reference, auth)
3801+
.await
3802+
.map_err(|err| {
3803+
Status::failed_precondition(format!(
3804+
"failed to resolve vm sandbox image '{image_ref}': {err}"
3805+
))
3806+
})?;
37833807
let cache_identity = prepared_image_cache_identity(&source_image_identity, &self.config);
37843808
let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity);
37853809

@@ -3806,7 +3830,7 @@ impl VmDriver {
38063830
let layout_dir = staging_dir.join(GUEST_IMAGE_OCI_LAYOUT_DIR);
38073831

38083832
let (manifest, _) = retry_registry_request("pull image manifest", || {
3809-
client.pull_image_manifest(&reference, &auth)
3833+
client.pull_image_manifest(&reference, auth)
38103834
})
38113835
.await
38123836
.map_err(|err| {
@@ -3819,7 +3843,7 @@ impl VmDriver {
38193843
.map_err(|err| Status::internal(format!("create guest OCI layout failed: {err}")))?;
38203844

38213845
download_registry_descriptor_blob_file(
3822-
&client,
3846+
client,
38233847
&reference,
38243848
image_ref,
38253849
&layout_dir,
@@ -5194,6 +5218,28 @@ fn registry_client() -> OciClient {
51945218
})
51955219
}
51965220

5221+
async fn pin_registry_reference(
5222+
client: &OciClient,
5223+
reference: &Reference,
5224+
auth: &RegistryAuth,
5225+
) -> Result<(Reference, String), OciDistributionError> {
5226+
// Preserve caller pins, including their digest algorithm. A HEAD response
5227+
// is only a resolution hint for tags, never proof of the image's contents.
5228+
let digest = match reference.digest() {
5229+
Some(digest) => digest.to_string(),
5230+
None => {
5231+
retry_registry_request("fetch manifest digest", || {
5232+
client.fetch_manifest_digest(reference, auth)
5233+
})
5234+
.await?
5235+
}
5236+
};
5237+
// oci-client verifies GET bytes against this pin. For an index it verifies
5238+
// the index first, then the selected platform manifest against its entry.
5239+
// All retries must use this same reference before publishing to the cache.
5240+
Ok((reference.clone_with_digest(digest.clone()), digest))
5241+
}
5242+
51975243
async fn retry_registry_request<T, F, Fut>(
51985244
operation: &str,
51995245
request: F,
@@ -6457,6 +6503,9 @@ fn write_oci_layout_for_manifest(
64576503
}
64586504

64596505
fn bootstrap_image_cache_identity(image_identity: &str) -> String {
6506+
// Bound new cache/staging filenames even for long source digests such as
6507+
// SHA-512. Persisted identities still use the unchanged path lookup.
6508+
let image_identity = compute_bytes_sha256_hex(image_identity.as_bytes());
64606509
format!(
64616510
"{BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:guest-{}:{image_identity}",
64626511
openshell_core::VERSION,
@@ -6472,6 +6521,7 @@ fn configured_sandbox_identity(config: &VmDriverConfig) -> Option<(u32, u32)> {
64726521
}
64736522

64746523
fn prepared_image_cache_identity(image_identity: &str, config: &VmDriverConfig) -> String {
6524+
let image_identity = compute_bytes_sha256_hex(image_identity.as_bytes());
64756525
let identity = configured_sandbox_identity(config).map_or_else(
64766526
|| "image-account".to_string(),
64776527
|(uid, gid)| format!("configured-{uid}-{gid}"),
@@ -11485,11 +11535,12 @@ mod tests {
1148511535
#[test]
1148611536
fn prepared_image_cache_identity_includes_rootfs_layout_and_openshell_version() {
1148711537
let image = "sha256:local-image";
11538+
let image_hash = compute_bytes_sha256_hex(image.as_bytes());
1148811539
let image_account = prepared_image_cache_identity(image, &VmDriverConfig::default());
1148911540
assert_eq!(
1149011541
image_account,
1149111542
format!(
11492-
"sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:image-account:{image}",
11543+
"sandbox-prepared-rootfs-ext4-umoci-v4:openshell-{}:image-account:{image_hash}",
1149311544
openshell_core::VERSION
1149411545
)
1149511546
);
@@ -11531,10 +11582,10 @@ mod tests {
1153111582
fn bootstrap_image_cache_identity_includes_rootfs_layout_version_and_guest_runtime() {
1153211583
let identity = bootstrap_image_cache_identity("sha256:bootstrap-image");
1153311584
assert!(identity.starts_with(&format!(
11534-
"sandbox-bootstrap-rootfs-ext4-v5:openshell-{}:guest-",
11585+
"sandbox-bootstrap-rootfs-ext4-v6:openshell-{}:guest-",
1153511586
openshell_core::VERSION
1153611587
)));
11537-
assert!(identity.ends_with(":sha256:bootstrap-image"));
11588+
assert!(identity.ends_with(&compute_bytes_sha256_hex(b"sha256:bootstrap-image")));
1153811589
assert!(identity.contains(&sandbox_guest_runtime_identity()));
1153911590
}
1154011591

0 commit comments

Comments
 (0)