diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index fd8c60ae..050834c7 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -21,7 +21,6 @@ use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::RequestFilterEffects; use trusted_server_core::platform::PlatformGeo as _; -use trusted_server_core::platform::RuntimeServices; use trusted_server_core::proxy::{stream_asset_body, AssetProxyCachePolicy}; use trusted_server_core::settings::Settings; @@ -312,11 +311,44 @@ fn run_edgezero_pull_sync_after_send( partner_registry: &PartnerRegistry, ec_state: &EcFinalizeState, ) { - if ec_state.is_real_browser { - if let Some(context) = build_pull_sync_context(&ec_state.ec_context) { - run_pull_sync_after_send(settings, partner_registry, &context, &ec_state.services); - } + if !ec_state.is_real_browser { + return; } + + let prepared_context = build_pull_sync_context(&ec_state.ec_context, partner_registry); + let Some((context, kv)) = + prepare_pull_sync_after_send(prepared_context, || require_identity_graph(settings)) + else { + return; + }; + + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + dispatch_pull_sync( + settings, + &kv, + partner_registry, + &limiter, + &context, + &ec_state.services, + ); +} + +fn prepare_pull_sync_after_send( + context: Option, + graph_factory: F, +) -> Option<(PullSyncContext, KvIdentityGraph)> +where + F: FnOnce() -> Result>, +{ + let context = context?; + let kv = match graph_factory() { + Ok(kv) => kv, + Err(err) => { + log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); + return None; + } + }; + Some((context, kv)) } /// Sends a finalized `EdgeZero` response to the client. @@ -421,24 +453,6 @@ pub(crate) fn maybe_identity_graph(settings: &Settings) -> Option kv, - Err(err) => { - log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); - return; - } - }; - - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - dispatch_pull_sync(settings, &kv, partner_registry, &limiter, context, services); -} - /// Constructs a `KvIdentityGraph` from settings, or returns an error if the /// `ec_store` config is not set. pub(crate) fn require_identity_graph( @@ -513,6 +527,26 @@ mod tests { .expect("should parse test settings") } + #[test] + fn pull_sync_noop_states_skip_post_send_graph_factory() { + for reason in ["no partners", "complete snapshot", "unread marker state"] { + let calls = std::cell::Cell::new(0); + let result = prepare_pull_sync_after_send(None, || { + calls.set(calls.get() + 1); + Err(Report::new(TrustedServerError::KvStore { + store_name: "unexpected".to_owned(), + message: "graph factory should not run".to_owned(), + })) + }); + assert!(result.is_none(), "{reason} preparation should return none"); + assert_eq!( + calls.get(), + 0, + "{reason} should not invoke the graph factory" + ); + } + } + #[test] fn health_response_short_circuits_get_health() { let req = FastlyRequest::get("https://example.com/health"); diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index ffcf4f03..07d94de2 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -1,6 +1,8 @@ use http::header::HeaderName; pub const COOKIE_TS_EC: &str = "ts-ec"; +/// Short-lived signed proof that the current EC row has every pull-partner UID. +pub const COOKIE_TS_EC_PULL_COMPLETE: &str = "ts-ec-pull-complete"; /// Cookie written by the Trusted Server JS SDK containing a standard-base64-encoded /// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 8c72f978..33b8fbe2 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -16,6 +16,7 @@ use super::generation::{generate_ec_id, is_valid_ec_id}; use super::kv::{apply_partner_id_updates, CreateIfAbsentOutcome, KvIdentityGraph}; use super::kv_types::KvEntry; use super::prebid_eids::collect_eid_cookie_updates; +use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; use super::{current_timestamp, EcContext, EcKvSnapshot}; @@ -47,6 +48,7 @@ pub fn ec_finalize_response( sharedid_cookie: Option<&str>, response: &mut Response, ) { + ec_context.validate_pull_sync_marker(settings, registry); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); @@ -57,8 +59,12 @@ pub fn ec_finalize_response( // consent input. clear_ec_headers_on_response(response, Some(registry)); - // Only expire the browser cookie and tombstone the identity-graph row - // when the request carries an explicit withdrawal signal. + if consent_withdrawn { + expire_marker(ec_context.pull_sync_marker_mut(), response); + } + + // Only expire the EC cookie and tombstone the identity-graph row when + // explicit withdrawal accompanies an EC cookie. if consent_withdrawn && ec_context.cookie_was_present() { expire_ec_cookie(settings, response); @@ -102,6 +108,8 @@ pub fn ec_finalize_response( } } + reconcile_pull_sync_marker(settings, registry, ec_context, response); + // Ordinary returning-user page views no longer refresh the browser // cookie, emit the EC header, or update KV TTL. return; @@ -113,6 +121,7 @@ pub fn ec_finalize_response( if ec_context.ec_generated() { let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { log::info!("Skipping generated EC response write because KV graph is unavailable"); + reconcile_pull_sync_marker(settings, registry, ec_context, response); return; }; @@ -129,6 +138,26 @@ pub fn ec_finalize_response( log::warn!("Skipping generated EC cookie because backing row is not authoritative"); } } + + reconcile_pull_sync_marker(settings, registry, ec_context, response); +} + +fn reconcile_pull_sync_marker( + settings: &Settings, + registry: &PartnerRegistry, + ec_context: &mut EcContext, + response: &mut Response, +) { + let ec_id = ec_context.ec_value().map(str::to_owned); + let snapshot = ec_context.kv_snapshot().clone(); + reconcile_marker( + settings, + registry, + ec_id.as_deref(), + &snapshot, + ec_context.pull_sync_marker_mut(), + response, + ); } fn recover_orphaned_ec( @@ -982,6 +1011,104 @@ mod tests { ); } + #[test] + fn finalize_sets_marker_for_complete_pull_partner_snapshot() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("compl1"); + let mut partner = make_partner("ssp.example.com"); + partner.pull_sync_enabled = true; + partner.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); + partner.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; + partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); + let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); + let mut ec_context = make_context( + Some(&ec_id), + Some(&ec_id), + true, + false, + Jurisdiction::NonRegulated, + ); + let mut entry = live_entry(); + entry.ids.insert( + "ssp.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "partner-uid".to_owned(), + }, + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Present { + ec_id, + entry: Box::new(entry), + generation: Some(1), + }); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + None, + ®istry, + None, + None, + &mut response, + ); + + let cookies = response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>(); + assert!( + cookies + .iter() + .any(|cookie| cookie.starts_with("ts-ec-pull-complete=v1.")), + "complete snapshot should issue the marker" + ); + } + + #[test] + fn explicit_withdrawal_expires_marker_without_ec_cookie() { + let settings = create_test_settings(); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = make_context_with_consent(None, None, false, false, consent); + ec_context.set_pull_sync_marker_for_test( + crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + None, + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let cookies = response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>(); + assert!( + cookies.iter().any(|cookie| { + cookie.starts_with("ts-ec-pull-complete=;") && cookie.contains("Max-Age=0") + }), + "withdrawal should expire the marker independently of EC cookie state" + ); + assert!( + cookies.iter().all(|cookie| !cookie.starts_with("ts-ec=;")), + "missing EC cookie should not add an EC-cookie expiry" + ); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 4423eb3e..584bd919 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -45,6 +45,7 @@ pub mod kv_types; pub mod partner; pub mod prebid_eids; pub mod pull_sync; +pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; @@ -65,7 +66,7 @@ use error_stack::Report; use http::Request; use crate::consent::{self as consent_mod, ConsentContext, ConsentPipelineInput}; -use crate::constants::COOKIE_TS_EC; +use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_PULL_COMPLETE}; use crate::cookies::handle_request_cookies; use crate::ec::cookies::ec_id_has_only_allowed_chars; use crate::error::TrustedServerError; @@ -76,6 +77,7 @@ use device::DeviceSignals; use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph}; use self::kv_types::KvEntry; +use self::pull_sync_marker::{validate_marker_state, PullSyncMarkerState}; /// Request-scoped view of one EC identity-graph lookup. /// @@ -147,6 +149,8 @@ pub use generation::{ struct RequestEc { /// EC ID from the `ts-ec` cookie, if present. cookie_ec: Option, + /// Pull-sync completeness marker, if present. + pull_sync_marker: Option, /// The parsed cookie jar (retained for consent pipeline input). jar: Option, } @@ -163,8 +167,17 @@ fn parse_ec_from_request(req: &Request) -> Result Option { @@ -226,6 +239,8 @@ pub struct EcContext { kv_snapshot: EcKvSnapshot, /// Whether this request may rotate an orphaned EC identity. recovery_eligible: bool, + /// Browser-carried proof of recent pull-partner completeness. + pull_sync_marker: PullSyncMarkerState, } impl EcContext { @@ -307,6 +322,7 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, + pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), }) } @@ -393,6 +409,7 @@ impl EcContext { self.ec_value = Some(ec_id); self.ec_generated = true; + self.pull_sync_marker.invalidate_for_replaced_ec(); return Ok(()); } @@ -495,11 +512,43 @@ impl EcContext { self.recovery_eligible } + /// Validates a browser completeness marker against the active EC and partner set. + pub(crate) fn validate_pull_sync_marker( + &mut self, + settings: &Settings, + registry: ®istry::PartnerRegistry, + ) { + validate_marker_state( + &mut self.pull_sync_marker, + settings, + registry, + self.ec_value.as_deref(), + ); + } + + /// Returns the current pull-sync marker state. + #[must_use] + pub(crate) fn pull_sync_marker(&self) -> &PullSyncMarkerState { + &self.pull_sync_marker + } + + /// Returns mutable pull-sync marker state for response reconciliation. + pub(crate) fn pull_sync_marker_mut(&mut self) -> &mut PullSyncMarkerState { + &mut self.pull_sync_marker + } + + /// Sets pull-sync marker state in focused unit tests. + #[cfg(test)] + pub(crate) fn set_pull_sync_marker_for_test(&mut self, state: PullSyncMarkerState) { + self.pull_sync_marker = state; + } + /// Replaces an orphaned active ID after its new backing row is persisted. pub(crate) fn replace_with_generated(&mut self, ec_id: String, snapshot: EcKvSnapshot) { self.ec_value = Some(ec_id); self.ec_generated = true; self.kv_snapshot = snapshot; + self.pull_sync_marker.invalidate_for_replaced_ec(); } /// Returns whether EC creation is permitted by consent for this request. @@ -549,6 +598,7 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, + pull_sync_marker: PullSyncMarkerState::Absent, } } @@ -571,6 +621,7 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, + pull_sync_marker: PullSyncMarkerState::Absent, } } @@ -596,6 +647,7 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, + pull_sync_marker: PullSyncMarkerState::Absent, } } } diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 89df8fa4..5b312fd5 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -22,6 +22,7 @@ use crate::settings::Settings; use super::generation::{ec_hash, is_valid_ec_id}; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; +use super::pull_sync_marker::entry_is_pull_complete; use super::rate_limiter::RateLimiter; use super::registry::{PartnerConfig, PartnerRegistry}; @@ -59,8 +60,11 @@ struct PullSyncResponse { /// /// Returns `None` when consent denies EC or there is no active EC ID. #[must_use] -pub fn build_pull_sync_context(ec_context: &EcContext) -> Option { - if !ec_context.ec_allowed() { +pub fn build_pull_sync_context( + ec_context: &EcContext, + registry: &PartnerRegistry, +) -> Option { + if registry.pull_enabled_partners().is_empty() || !ec_context.ec_allowed() { return None; } @@ -70,6 +74,11 @@ pub fn build_pull_sync_context(ec_context: &EcContext) -> Option; + +const MARKER_VERSION: &str = "v1"; +const MARKER_KEY_LABEL: &[u8] = b"trusted-server/ec-pull-complete/key/v1"; +const MARKER_MAX_AGE_SECS: u64 = 60 * 60; +const MAX_MARKER_LENGTH: usize = 256; + +/// Request-local validation state for the pull-sync completeness marker. +#[derive(Clone, Default)] +pub(crate) enum PullSyncMarkerState { + /// No marker cookie was present. + #[default] + Absent, + /// A marker was present but has not been checked against the active EC and partner set. + Unvalidated(Redacted), + /// A present marker failed validation or was disproved by authoritative KV state. + Invalid, + /// The marker is valid until the given Unix timestamp. + Valid { expires_at: u64 }, +} + +impl fmt::Debug for PullSyncMarkerState { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Absent => formatter.write_str("Absent"), + Self::Unvalidated(_) => formatter.write_str("Unvalidated()"), + Self::Invalid => formatter.write_str("Invalid"), + Self::Valid { expires_at } => formatter + .debug_struct("Valid") + .field("expires_at", expires_at) + .finish(), + } + } +} + +impl PullSyncMarkerState { + /// Creates marker state from an optional request cookie value. + #[must_use] + pub(crate) fn from_cookie(value: Option) -> Self { + value + .map(Redacted::new) + .map_or(Self::Absent, Self::Unvalidated) + } + + /// Returns whether a marker cookie was present on the request. + #[must_use] + pub(crate) fn was_present(&self) -> bool { + !matches!(self, Self::Absent) + } + + /// Returns whether the marker is currently valid. + #[must_use] + pub(crate) fn is_valid(&self) -> bool { + matches!(self, Self::Valid { .. }) + } + + /// Invalidates state bound to a replaced active EC ID. + pub(crate) fn invalidate_for_replaced_ec(&mut self) { + if self.was_present() { + *self = Self::Invalid; + } + } +} + +/// Validates an unvalidated marker against the active EC and current pull-partner set. +pub(crate) fn validate_marker_state( + state: &mut PullSyncMarkerState, + settings: &Settings, + registry: &PartnerRegistry, + ec_id: Option<&str>, +) { + let PullSyncMarkerState::Unvalidated(value) = state else { + return; + }; + let valid_until = ec_id.and_then(|ec_id| { + validate_marker( + value.expose(), + settings, + registry, + ec_id, + current_timestamp(), + ) + }); + *state = valid_until.map_or(PullSyncMarkerState::Invalid, |expires_at| { + PullSyncMarkerState::Valid { expires_at } + }); +} + +/// Reconciles browser marker state against finalized authoritative KV state. +pub(crate) fn reconcile_marker( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: Option<&str>, + snapshot: &EcKvSnapshot, + state: &mut PullSyncMarkerState, + response: &mut Response, +) { + let pull_partners = sorted_pull_partner_domains(registry); + if pull_partners.is_empty() { + expire_if_present(state, response); + return; + } + + let Some(ec_id) = ec_id else { + expire_if_present(state, response); + return; + }; + + if !matches!(snapshot, EcKvSnapshot::NotRead) && !snapshot.belongs_to(ec_id) { + expire_if_present(state, response); + return; + } + + match snapshot { + EcKvSnapshot::Present { .. } => { + let entry = snapshot + .entry_for(ec_id) + .expect("snapshot binding should be checked before marker reconciliation"); + if !entry.consent.ok { + expire_if_present(state, response); + } else if entry_has_all_pull_partner_ids(entry, &pull_partners) { + if !state.is_valid() { + set_marker(settings, registry, ec_id, state, response); + } + } else { + expire_if_present(state, response); + } + } + EcKvSnapshot::Missing { .. } => { + expire_if_present(state, response); + } + EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => { + if matches!(state, PullSyncMarkerState::Invalid) { + expire_if_present(state, response); + } + } + } +} + +/// Expires the completeness marker regardless of KV state. +pub(crate) fn expire_marker(state: &mut PullSyncMarkerState, response: &mut Response) { + append_cookie(response, &format_marker_cookie("", 0)); + *state = PullSyncMarkerState::Absent; +} + +/// Returns whether a live entry contains every pull-enabled partner ID. +#[must_use] +pub(crate) fn entry_is_pull_complete(entry: &KvEntry, registry: &PartnerRegistry) -> bool { + let pull_partners = sorted_pull_partner_domains(registry); + !pull_partners.is_empty() && entry_has_all_pull_partner_ids(entry, &pull_partners) +} + +fn entry_has_all_pull_partner_ids(entry: &KvEntry, pull_partners: &[String]) -> bool { + entry.consent.ok + && pull_partners + .iter() + .all(|source_domain| entry.ids.contains_key(source_domain)) +} + +fn set_marker( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, + state: &mut PullSyncMarkerState, + response: &mut Response, +) { + let now = current_timestamp(); + let expires_at = now.saturating_add(MARKER_MAX_AGE_SECS); + let Some(value) = create_marker(settings, registry, ec_id, expires_at) else { + return; + }; + append_cookie(response, &format_marker_cookie(&value, MARKER_MAX_AGE_SECS)); + *state = PullSyncMarkerState::Valid { expires_at }; +} + +fn expire_if_present(state: &mut PullSyncMarkerState, response: &mut Response) { + if state.was_present() { + expire_marker(state, response); + } +} + +fn append_cookie(response: &mut Response, value: &str) { + match HeaderValue::from_str(value) { + Ok(value) => { + response.headers_mut().append(header::SET_COOKIE, value); + } + Err(err) => { + log::warn!("Skipping pull-sync marker cookie: invalid header value: {err}"); + } + } +} + +fn format_marker_cookie(value: &str, max_age: u64) -> String { + format!( + "{COOKIE_TS_EC_PULL_COMPLETE}={value}; Path=/; Secure; SameSite=Lax; Max-Age={max_age}; HttpOnly" + ) +} + +#[cfg(test)] +pub(crate) fn create_marker_for_test( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, +) -> String { + create_marker_for_test_with_expiry( + settings, + registry, + ec_id, + current_timestamp().saturating_add(MARKER_MAX_AGE_SECS), + ) +} + +#[cfg(test)] +pub(crate) fn create_marker_for_test_with_expiry( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, + expires_at: u64, +) -> String { + create_marker(settings, registry, ec_id, expires_at) + .expect("should create marker for non-empty test registry") +} + +fn create_marker( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, + expires_at: u64, +) -> Option { + let fingerprint = partner_set_fingerprint(registry)?; + let payload = marker_payload(ec_id, expires_at, &fingerprint); + let key = marker_key(settings); + let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); + mac.update(payload.as_bytes()); + let tag = hex::encode(mac.finalize().into_bytes()); + Some(format!("{MARKER_VERSION}.{expires_at}.{fingerprint}.{tag}")) +} + +fn validate_marker( + value: &str, + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, + now: u64, +) -> Option { + if value.len() > MAX_MARKER_LENGTH { + return None; + } + + let mut segments = value.split('.'); + let version = segments.next()?; + let expires = segments.next()?; + let fingerprint = segments.next()?; + let tag = segments.next()?; + if segments.next().is_some() || version != MARKER_VERSION { + return None; + } + + let expires_at = expires.parse::().ok()?; + if expires_at <= now || expires_at > now.saturating_add(MARKER_MAX_AGE_SECS) { + return None; + } + + let expected_fingerprint = partner_set_fingerprint(registry)?; + if fingerprint != expected_fingerprint { + return None; + } + + let tag = hex::decode(tag).ok()?; + if tag.len() != 32 { + return None; + } + + let payload = marker_payload(ec_id, expires_at, fingerprint); + let key = marker_key(settings); + let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); + mac.update(payload.as_bytes()); + mac.verify_slice(&tag).ok()?; + Some(expires_at) +} + +fn marker_key(settings: &Settings) -> [u8; 32] { + let mut mac = HmacSha256::new_from_slice(settings.ec.passphrase.expose().as_bytes()) + .expect("should create marker key HMAC"); + mac.update(MARKER_KEY_LABEL); + mac.finalize().into_bytes().into() +} + +fn marker_payload(ec_id: &str, expires_at: u64, fingerprint: &str) -> String { + format!("{MARKER_VERSION}\0{ec_id}\0{expires_at}\0{fingerprint}") +} + +fn partner_set_fingerprint(registry: &PartnerRegistry) -> Option { + let domains = sorted_pull_partner_domains(registry); + if domains.is_empty() { + return None; + } + + let mut hasher = Sha256::new(); + hasher.update(b"trusted-server/ec-pull-partners/v1\0"); + for domain in domains { + hasher.update((domain.len() as u64).to_be_bytes()); + hasher.update(domain.as_bytes()); + } + Some(hex::encode(hasher.finalize())) +} + +fn sorted_pull_partner_domains(registry: &PartnerRegistry) -> Vec { + let mut domains = registry + .pull_enabled_partners() + .into_iter() + .map(|partner| partner.source_domain.clone()) + .collect::>(); + domains.sort(); + domains +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::kv_types::KvEntry; + use crate::redacted::Redacted; + use crate::settings::{EcPartner, Settings}; + use crate::test_support::tests::create_test_settings; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn pull_partner(source_domain: &str) -> EcPartner { + EcPartner { + name: format!("Partner {source_domain}"), + source_domain: source_domain.to_owned(), + openrtb_atype: EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: Redacted::new(format!("token-{source_domain}-32-bytes-minimum-value")), + batch_rate_limit: EcPartner::default_batch_rate_limit(), + pull_sync_enabled: true, + pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), + pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], + pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), + pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), + ts_pull_token: Some(Redacted::new("pull-token".to_owned())), + } + } + + fn settings_and_registry(domains: &[&str]) -> (Settings, PartnerRegistry) { + let mut settings = create_test_settings(); + settings.ec.partners = domains.iter().map(|domain| pull_partner(domain)).collect(); + let registry = PartnerRegistry::from_config(&settings.ec.partners) + .expect("should build pull partner registry"); + (settings, registry) + } + + fn empty_response() -> Response { + Response::builder() + .status(200) + .body(EdgeBody::empty()) + .expect("should build response") + } + + fn live_snapshot(ec_id: &str, domains: &[&str]) -> EcKvSnapshot { + let mut entry = KvEntry::tombstone(1_000); + entry.consent.ok = true; + for domain in domains { + entry.ids.insert( + (*domain).to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: format!("uid-{domain}"), + }, + ); + } + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry), + generation: Some(1), + } + } + + fn marker_cookies(response: &Response) -> Vec<&str> { + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect() + } + + fn assert_expired(state: &PullSyncMarkerState, response: &Response) { + assert!(matches!(state, PullSyncMarkerState::Absent)); + assert_eq!( + marker_cookies(response), + vec!["ts-ec-pull-complete=; Path=/; Secure; SameSite=Lax; Max-Age=0; HttpOnly"], + "should emit the exact host-only expiration cookie" + ); + } + + #[test] + fn marker_round_trip_binds_ec_and_partner_set() { + let (settings, registry) = settings_and_registry(&["a.example.com", "b.example.com"]); + let now = 1_000; + let marker = + create_marker(&settings, ®istry, EC_ID, now + 3_600).expect("should create marker"); + + assert_eq!( + validate_marker(&marker, &settings, ®istry, EC_ID, now), + Some(now + 3_600), + "should validate the issued marker" + ); + assert!( + validate_marker(&marker, &settings, ®istry, "wrong", now).is_none(), + "should reject a marker for another EC" + ); + } + + #[test] + fn marker_fingerprint_is_order_independent_and_set_sensitive() { + let (settings, first) = settings_and_registry(&["a.example.com", "b.example.com"]); + let (_, reordered) = settings_and_registry(&["b.example.com", "a.example.com"]); + let (_, changed) = settings_and_registry(&["a.example.com", "c.example.com"]); + let marker = create_marker(&settings, &first, EC_ID, 4_600).expect("should create marker"); + + assert!( + validate_marker(&marker, &settings, &reordered, EC_ID, 1_000).is_some(), + "config ordering should not change the marker" + ); + assert!( + validate_marker(&marker, &settings, &changed, EC_ID, 1_000).is_none(), + "partner-set changes should invalidate the marker" + ); + } + + #[test] + fn fingerprint_changes_for_enable_disable_add_and_remove() { + let (_, enabled_one) = settings_and_registry(&["a.example.com"]); + let (_, enabled_two) = settings_and_registry(&["a.example.com", "b.example.com"]); + let mut disabled_config = pull_partner("a.example.com"); + disabled_config.pull_sync_enabled = false; + let disabled = PartnerRegistry::from_config(&[disabled_config]) + .expect("should build disabled registry"); + let removed = PartnerRegistry::empty(); + + let base = partner_set_fingerprint(&enabled_one); + assert_ne!( + base, + partner_set_fingerprint(&enabled_two), + "adding a pull partner should change the fingerprint" + ); + assert_ne!( + base, + partner_set_fingerprint(&disabled), + "disabling a pull partner should change the fingerprint" + ); + assert_ne!( + base, + partner_set_fingerprint(&removed), + "removing a pull partner should change the fingerprint" + ); + assert_ne!( + partner_set_fingerprint(&disabled), + partner_set_fingerprint(&enabled_one), + "enabling a partner should change the fingerprint" + ); + } + + #[test] + fn reconcile_rejects_snapshot_bound_to_different_ec() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let snapshot = live_snapshot("different-ec", &["a.example.com"]); + let mut valid_state = PullSyncMarkerState::Valid { expires_at: 4_600 }; + let mut valid_response = empty_response(); + + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &snapshot, + &mut valid_state, + &mut valid_response, + ); + assert_expired(&valid_state, &valid_response); + + let mut absent_state = PullSyncMarkerState::Absent; + let mut absent_response = empty_response(); + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &snapshot, + &mut absent_state, + &mut absent_response, + ); + assert!(marker_cookies(&absent_response).is_empty()); + assert!(matches!(absent_state, PullSyncMarkerState::Absent)); + } + + #[test] + fn authoritative_incomplete_states_expire_valid_marker() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let mut tombstone = KvEntry::tombstone(1_000); + tombstone.ids.insert( + "a.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "stale".to_owned(), + }, + ); + let snapshots = [ + live_snapshot(EC_ID, &[]), + EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }, + EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(tombstone), + generation: Some(1), + }, + ]; + + for snapshot in snapshots { + let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; + let mut response = empty_response(); + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &snapshot, + &mut state, + &mut response, + ); + assert_expired(&state, &response); + } + } + + #[test] + fn non_authoritative_states_preserve_valid_fixed_expiry_marker() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let snapshots = [ + EcKvSnapshot::Failed { + ec_id: EC_ID.to_owned(), + }, + EcKvSnapshot::NotRead, + ]; + + for snapshot in snapshots { + let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; + let mut response = empty_response(); + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &snapshot, + &mut state, + &mut response, + ); + assert!(matches!( + state, + PullSyncMarkerState::Valid { expires_at: 4_600 } + )); + assert!(marker_cookies(&response).is_empty()); + } + } + + #[test] + fn invalid_marker_is_cleared_without_authoritative_snapshot() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let mut state = PullSyncMarkerState::Invalid; + let mut response = empty_response(); + + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &EcKvSnapshot::NotRead, + &mut state, + &mut response, + ); + + assert_expired(&state, &response); + } + + #[test] + fn marker_rejects_expired_overlong_and_tampered_values() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let marker = + create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); + let mut tampered = marker.clone(); + tampered.push('0'); + + assert!( + validate_marker(&marker, &settings, ®istry, EC_ID, 4_600).is_none(), + "expired marker should fail" + ); + assert!( + validate_marker(&marker, &settings, ®istry, EC_ID, 999).is_none(), + "marker more than one hour in the future should fail" + ); + assert!( + validate_marker(&tampered, &settings, ®istry, EC_ID, 1_000).is_none(), + "tampering should fail" + ); + assert!( + validate_marker( + &"x".repeat(MAX_MARKER_LENGTH + 1), + &settings, + ®istry, + EC_ID, + 1_000 + ) + .is_none(), + "overlong marker should fail" + ); + } + + #[test] + fn marker_rejects_wrong_passphrase_and_empty_partner_set() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let marker = + create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); + let mut changed_settings = settings.clone(); + changed_settings.ec.passphrase = + Redacted::new("different-secret-key-32-bytes-minimum".to_owned()); + let empty = PartnerRegistry::empty(); + + assert!( + validate_marker(&marker, &changed_settings, ®istry, EC_ID, 1_000).is_none(), + "passphrase rotation should invalidate the marker" + ); + assert!( + create_marker(&settings, &empty, EC_ID, 4_600).is_none(), + "empty partner sets should not produce a marker" + ); + } + + #[test] + fn marker_cookie_is_host_only_and_secure() { + let cookie = format_marker_cookie("value", MARKER_MAX_AGE_SECS); + assert_eq!( + cookie, + "ts-ec-pull-complete=value; Path=/; Secure; SameSite=Lax; Max-Age=3600; HttpOnly" + ); + assert!(!cookie.contains("Domain="), "marker should be host-only"); + } + + #[test] + fn completeness_requires_all_pull_partner_ids() { + let (_, registry) = settings_and_registry(&["a.example.com", "b.example.com"]); + let mut entry = KvEntry::minimal("a.example.com", "uid-a", 1_000); + assert!(!entry_is_pull_complete(&entry, ®istry)); + + entry.ids.insert( + "b.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "uid-b".to_owned(), + }, + ); + assert!(entry_is_pull_complete(&entry, ®istry)); + } +} diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 7d87d66e..b2d9fe3d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -41,7 +41,7 @@ use crate::auction::types::{ AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; -use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; +use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; @@ -765,13 +765,25 @@ fn request_head_snapshot(req: &Request) -> Request { snapshot } -fn should_preload_ec_snapshot( +#[derive(Clone, Copy)] +struct EcSnapshotPreloadInput { is_navigation: bool, is_get: bool, has_ec_id: bool, has_kv: bool, -) -> bool { - is_navigation && is_get && has_ec_id && has_kv + marker_valid: bool, + auction_needs_row: bool, + eid_cookie_may_need_persistence: bool, + snapshot_already_read: bool, +} + +fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { + let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; + let marker_can_skip = input.marker_valid + && !input.auction_needs_row + && !input.eid_cookie_may_need_persistence + && !input.snapshot_already_read; + eligible && !marker_can_skip } /// Rewrites a downstream request into an outbound publisher-origin request. @@ -1398,6 +1410,9 @@ pub async fn handle_publisher_request( .map(str::to_owned); let ec_id = ec_id_owned.as_deref(); let cookie_jar = handle_request_cookies(&req)?; + if let Some(registry) = auction.registry { + ec_context.validate_pull_sync_marker(settings, registry); + } let geo = ec_context.geo_info().cloned(); let parsed_origin = url::Url::parse(&settings.publisher.origin_url).change_context( @@ -1505,8 +1520,25 @@ pub async fn handle_publisher_request( let auction_client_request = request_head_snapshot(&req); let mut origin_request = Some(req); - let should_preload_ec = - should_preload_ec_snapshot(is_navigation, is_get, ec_id.is_some(), kv.is_some()); + let eid_cookie_may_need_persistence = cookie_jar + .as_ref() + .is_some_and(|jar| jar.get(COOKIE_TS_EIDS).is_some() || jar.get(COOKIE_SHAREDID).is_some()); + let should_preload_ec = should_preload_ec_snapshot(&EcSnapshotPreloadInput { + is_navigation, + is_get, + has_ec_id: ec_id.is_some(), + has_kv: kv.is_some(), + marker_valid: ec_context.pull_sync_marker().is_valid(), + auction_needs_row: should_run_auction + && auction + .registry + .is_some_and(|registry| !registry.is_empty()), + eid_cookie_may_need_persistence, + snapshot_already_read: !matches!( + ec_context.kv_snapshot(), + crate::ec::EcKvSnapshot::NotRead + ), + }); let mut pending_origin = None; if should_preload_ec && services.http_client().supports_concurrent_fanout() { let mut origin_req = origin_request.take().ok_or_else(|| { @@ -2550,11 +2582,60 @@ mod tests { #[test] fn ec_snapshot_preload_requires_navigation_get_ec_and_kv() { - assert!(should_preload_ec_snapshot(true, true, true, true)); - assert!(!should_preload_ec_snapshot(false, true, true, true)); - assert!(!should_preload_ec_snapshot(true, false, true, true)); - assert!(!should_preload_ec_snapshot(true, true, false, true)); - assert!(!should_preload_ec_snapshot(true, true, true, false)); + let baseline = EcSnapshotPreloadInput { + is_navigation: true, + is_get: true, + has_ec_id: true, + has_kv: true, + marker_valid: false, + auction_needs_row: false, + eid_cookie_may_need_persistence: false, + snapshot_already_read: false, + }; + assert!(should_preload_ec_snapshot(&baseline)); + assert!(!should_preload_ec_snapshot(&EcSnapshotPreloadInput { + is_navigation: false, + ..baseline + })); + assert!(!should_preload_ec_snapshot(&EcSnapshotPreloadInput { + is_get: false, + ..baseline + })); + assert!(!should_preload_ec_snapshot(&EcSnapshotPreloadInput { + has_ec_id: false, + ..baseline + })); + assert!(!should_preload_ec_snapshot(&EcSnapshotPreloadInput { + has_kv: false, + ..baseline + })); + } + + #[test] + fn valid_marker_skips_only_when_no_other_consumer_needs_snapshot() { + let marker_only = EcSnapshotPreloadInput { + is_navigation: true, + is_get: true, + has_ec_id: true, + has_kv: true, + marker_valid: true, + auction_needs_row: false, + eid_cookie_may_need_persistence: false, + snapshot_already_read: false, + }; + assert!(!should_preload_ec_snapshot(&marker_only)); + assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { + auction_needs_row: true, + ..marker_only + })); + assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { + eid_cookie_may_need_persistence: true, + ..marker_only + })); + assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { + snapshot_already_read: true, + ..marker_only + })); } use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::consent::ConsentContext; @@ -2684,6 +2765,142 @@ mod tests { ) } + #[derive(Clone, Copy)] + enum MarkerProbe { + Valid, + Malformed, + WrongEc, + Expired, + PartnerSetMismatch, + } + + fn marker_partner(source_domain: &str) -> crate::settings::EcPartner { + crate::settings::EcPartner { + name: format!("Marker partner {source_domain}"), + source_domain: source_domain.to_owned(), + openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: crate::redacted::Redacted::new(format!( + "marker-{source_domain}-api-token-32-bytes-minimum" + )), + batch_rate_limit: crate::settings::EcPartner::default_batch_rate_limit(), + pull_sync_enabled: true, + pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), + pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], + pull_sync_ttl_sec: crate::settings::EcPartner::default_pull_sync_ttl_sec(), + pull_sync_rate_limit: crate::settings::EcPartner::default_pull_sync_rate_limit(), + ts_pull_token: Some(crate::redacted::Redacted::new("pull-token".to_owned())), + } + } + + async fn run_marker_lookup_probe(probe: MarkerProbe, has_eid_cookie: bool) -> usize { + let mut settings = create_test_settings(); + settings.ec.partners = vec![marker_partner("ssp.example.com")]; + let registry = PartnerRegistry::from_config(&settings.ec.partners) + .expect("should build pull partner registry"); + let http = Arc::new(StubHttpClient::new()); + http.set_concurrent_fanout(true); + http.push_response(200, b"ok".to_vec()); + let lookups = Arc::new(AtomicUsize::new(0)); + let graph = KvIdentityGraph::new(OrderRecordingKv { + inner: InMemoryEcKv::new("marker-store"), + http: Arc::clone(&http), + http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), + lookups: Arc::clone(&lookups), + }); + let services = build_services_with_http_client( + Arc::clone(&http) as Arc + ); + let ec_id = format!("{}.CkId01", "b".repeat(64)); + let marker = match probe { + MarkerProbe::Valid => { + crate::ec::pull_sync_marker::create_marker_for_test(&settings, ®istry, &ec_id) + } + MarkerProbe::Malformed => "not-a-marker".to_owned(), + MarkerProbe::WrongEc => crate::ec::pull_sync_marker::create_marker_for_test( + &settings, + ®istry, + &format!("{}.Other1", "c".repeat(64)), + ), + MarkerProbe::Expired => { + crate::ec::pull_sync_marker::create_marker_for_test_with_expiry( + &settings, + ®istry, + &ec_id, + crate::ec::current_timestamp().saturating_sub(1), + ) + } + MarkerProbe::PartnerSetMismatch => { + let other_registry = + PartnerRegistry::from_config(&[marker_partner("other.example.com")]) + .expect("should build alternate registry"); + crate::ec::pull_sync_marker::create_marker_for_test( + &settings, + &other_registry, + &ec_id, + ) + } + }; + let mut ec_context = EcContext::new_for_test_with_ip( + Some(ec_id), + scheduling_consent(), + Some("203.0.113.7".to_owned()), + ); + ec_context.set_pull_sync_marker_for_test( + crate::ec::pull_sync_marker::PullSyncMarkerState::from_cookie(Some(marker)), + ); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut request = navigation_request(); + if has_eid_cookie { + request + .headers_mut() + .insert(header::COOKIE, HeaderValue::from_static("ts-eids=present")); + } + + let result = handle_publisher_request( + &settings, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: Some(®istry), + }, + request, + ) + .await; + + assert!(result.is_ok(), "should proxy the origin response"); + lookups.load(Ordering::SeqCst) + } + + #[tokio::test] + async fn valid_completeness_marker_skips_pull_only_snapshot_lookup() { + assert_eq!(run_marker_lookup_probe(MarkerProbe::Valid, false).await, 0); + } + + #[tokio::test] + async fn invalid_markers_fall_back_to_snapshot_lookup() { + for probe in [ + MarkerProbe::Malformed, + MarkerProbe::WrongEc, + MarkerProbe::Expired, + MarkerProbe::PartnerSetMismatch, + ] { + assert_eq!( + run_marker_lookup_probe(probe, false).await, + 1, + "invalid marker should retain the normal preload" + ); + } + } + + #[tokio::test] + async fn valid_marker_does_not_skip_eid_cookie_persistence_lookup() { + assert_eq!(run_marker_lookup_probe(MarkerProbe::Valid, true).await, 1); + } + #[tokio::test] async fn concurrent_client_starts_origin_before_ec_lookup() { let (lookups, http_calls_at_lookup, ok) = run_scheduling_probe(true, true).await; diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index dd9cbbd9..ce060a60 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -230,6 +230,14 @@ The relevant OpenRTB structure forwarded to Prebid Server and downstream partner Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. When a partner UID already exists in KV, pull sync does not periodically refresh it; browser-side Prebid sync can still replace the stored UID if a later `ts-eids` cookie carries a different value for the same configured partner source. +### Pull-Sync Completeness Marker + +When the identity graph contains a UID for every pull-enabled partner, Trusted Server sets a signed, host-only `ts-ec-pull-complete` cookie. The cookie contains no partner UID or EC ID. It authenticates a one-hour expiration and a fingerprint of the current pull-partner source-domain set, bound to the active EC ID with key material derived from `ec.passphrase`. + +A valid marker avoids a KV lookup only when pull-sync completeness is the sole reason to inspect the row. Auctions that need stored EIDs, browser EID-cookie ingestion, explicit withdrawal, generation, and detected orphan recovery continue to use KV. Partner-set changes, passphrase rotation, malformed values, and expiration invalidate the marker and fall back to the normal KV path. The one-hour bound also limits how long deletion of a previously complete row can go undetected. + +Pull sync runs after response delivery, so a partner response that fills the last missing UID cannot set the marker on that already-sent response. A later eligible request verifies the completed row and issues the marker. Explicit withdrawal expires both `ts-ec` and `ts-ec-pull-complete` even when KV is unavailable. + ## Configuration Configure EC settings in `trusted-server.toml`. See the full [Configuration Reference](/guide/configuration) for the `[ec]` section and environment variable overrides. diff --git a/docs/superpowers/plans/2026-07-13-issue-880-no-op-pull-sync-kv-reads.md b/docs/superpowers/plans/2026-07-13-issue-880-no-op-pull-sync-kv-reads.md new file mode 100644 index 00000000..f5cca07f --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-issue-880-no-op-pull-sync-kv-reads.md @@ -0,0 +1,446 @@ +# Issue #880: No-Op Pull-Sync KV Read Elimination Plan + +- **Date:** 2026-07-13 +- **Status:** Implemented and verified +- **Issue:** [#880 — Avoid no-op EC KV reads in post-send pull sync](https://github.com/IABTechLab/trusted-server/issues/880) +- **Stack base:** [PR #885 — Thread EC KV read through the request and recover orphaned cookies](https://github.com/IABTechLab/trusted-server/pull/885) + +## Goal + +Avoid EC identity-graph work whose only purpose is determining that post-send +pull sync has nothing to do. Return before constructing the post-send graph when +there are no pull-enabled partners, and let browsers carry a short-lived signed +proof that their EC row was recently complete for the current pull-partner set. + +The change must preserve PR #885's single request-scoped [`EcKvSnapshot`], +auction EID resolution, request EID ingestion, orphan recovery, withdrawal, +missing-UID-only dispatch, rate limits, CAS protection, and best-effort +post-send failure policy. + +## Baseline After PR #885 + +PR #885 changes the original issue context: + +- Publisher navigations can load one EC KV snapshot before response delivery, + after starting a truly asynchronous publisher-origin request. +- Auction EID resolution, EC finalization, and pull sync reuse that snapshot. +- Pull sync no longer performs its own initial `kv.get`; missing, failed, + unread, and tombstoned snapshots do not dispatch. +- Fastly still constructs a post-send `KvIdentityGraph` before core proves that + there are no pull partners or no missing partner IDs. +- Publisher snapshot preload remains unconditional for eligible returning-user + GET navigations, including requests where no auction runs and pull-sync + completeness is the only reason to inspect partner IDs. + +Accordingly, “zero EC KV operations” in this plan means **zero operations caused +solely by pull sync**. A request still reads KV when auction EIDs, EID-cookie +persistence, withdrawal, generation, orphan detection, or another identity +lifecycle consumer needs the row. In particular, a zero-partner registry does +not by itself disable PR #885's navigation preload: doing so would indefinitely +suppress orphan detection on sites without pull partners. The no-partner fast +path instead eliminates the entire post-send pull-sync graph/operation path. + +## Clarified Contract + +- The first no-partner decision occurs before the post-send graph is + constructed or opened. +- The completeness marker is a signed, host-only, `HttpOnly`, `Secure`, + `SameSite=Lax` cookie with a maximum one-hour lifetime. +- The marker is bound to the active EC ID, its expiration, and a deterministic + fingerprint of the sorted canonical pull-enabled partner source domains. +- The existing `ec.passphrase` supplies keying material through explicit + marker-specific key separation; no new secret or configuration field is + introduced. +- Missing, malformed, expired, overlong, wrongly bound, incorrectly signed, or + partner-set-mismatched markers fall back safely to normal KV behavior. +- A valid marker suppresses snapshot preload only when no other current-request + consumer needs the identity row. +- A row deleted after marker issuance can remain undetected until the marker + expires. This bounded delay is accepted; once a `Missing` snapshot is + actually observed, the marker cannot suppress PR #885's recovery flow. +- An authoritative partial, missing, or tombstoned snapshot disproves the + marker and clears it. +- Explicit consent withdrawal clears both `ts-ec` and the completeness marker, + regardless of KV success. +- A post-send pull result cannot mutate the already-delivered response. If pull + sync fills the final missing UID, a later request must verify completeness + pre-send before issuing the marker. + +## Non-Goals + +- Do not introduce another request-scoped EC cache or duplicate `EcKvSnapshot`. +- Do not promise zero request-wide KV reads when another EC consumer needs data. +- Do not store partner UIDs, consent data, or the full EC ID in the marker. +- Do not refresh existing partner UIDs or revive the legacy per-partner sync TTL. +- Do not change batch-sync behavior, withdrawal tombstone semantics, auction + consent gates, or partner rate limits. +- Do not move partner HTTP dispatch before response delivery. +- Do not add an operator-facing marker setting or signing secret. +- Do not refactor all Fastly partner-registry construction as part of this fix. + +## Proposed Design + +### 1. Versioned signed marker + +Add `crates/trusted-server-core/src/ec/pull_sync_marker.rs` and define the cookie +name in `crates/trusted-server-core/src/constants.rs`, using the private name +`ts-ec-pull-complete`. + +Use a cookie-safe versioned wire form: + +```text +v1... +``` + +The protocol will: + +1. Take only pull-enabled partners from [`PartnerRegistry`]. +2. Sort their normalized `source_domain` values lexicographically. +3. Hash an unambiguous, versioned encoding of that list with SHA-256. +4. Derive a marker-only HMAC subkey from `ec.passphrase` with a fixed domain + label such as `trusted-server/ec-pull-complete/key/v1`. +5. Authenticate the protocol version, active EC ID, expiration, and partner-set + fingerprint with HMAC-SHA256. +6. Verify the tag through the HMAC API's constant-time verification method. +7. Accept only `now < expires <= now + 3600`; malformed numbers, unexpected + segment counts, invalid digest lengths, and invalid hex fail closed. + +The live cookie is host-only and uses: + +```text +Path=/; Secure; SameSite=Lax; Max-Age=3600; HttpOnly +``` + +It deliberately omits `Domain`. Expiration uses the same attributes with an +empty value and `Max-Age=0`. EC passphrase rotation therefore invalidates every +outstanding marker and safely falls back to KV. + +The marker module also owns the authoritative completeness predicate: a live +snapshot is complete only when its `ids` map contains every non-empty, +pull-enabled source domain in the current registry. An empty pull-partner set +never produces a marker. + +### 2. Carry marker state without carrying KV state twice + +Extend `EcContext` with a small non-KV marker state. The state should distinguish +at least: + +- no marker on the request; +- a present marker not yet validated; +- invalid/stale marker; +- valid marker, including its bounded expiration. + +`parse_ec_from_request` captures the raw marker from the already-parsed +`CookieJar`. The raw value must not appear in logs or an unrestricted `Debug` +representation. + +Validate before a publisher snapshot decision, once the active EC ID and +partner registry are available. Finalization performs the same validation if a +route did not pass through publisher handling. If no validated registry is +available, the conservative result is invalid/fallback rather than skip. + +The state travels naturally with the existing `EcContext` through +`EcRequestState` and `EcFinalizeState`; it is not a second identity snapshot. +Generating or replacing the active EC ID invalidates marker state bound to the +old ID. + +### 3. Gate publisher snapshot preload by actual consumers + +Replace the four-boolean `should_preload_ec_snapshot` decision with a named input +or requirement structure that remains readable as the conditions grow. + +The existing prerequisites remain: + +- document navigation; +- `GET` request; +- consent-allowed active EC ID; +- available EC graph. + +Without a valid marker, retain PR #885's normal preload, including its bounded +orphan-detection responsibility. With a valid marker, skip only when no +current-request operation needs the row itself. The marker's signed recent +existence proof is what permits orphan detection to be deferred for at most one +hour; the absence of pull partners alone is not such proof. Force the lookup +when any of these apply: + +- a server-side auction will run with a registry that can resolve stored EIDs; +- `ts-eids` or `sharedId` is present and may require persistence; +- an existing authoritative snapshot already requires mutation or recovery; +- a privacy path requires authoritative withdrawal state; +- EC generation/replacement requires persisted-state confirmation. + +For a plain returning-user navigation with a valid marker and no such consumer, +treat the marker as recent proof that the row existed and was complete. Leave +the snapshot `NotRead` and defer any newly orphaned-row discovery for at most one +hour. + +Whenever a lookup remains necessary, preserve PR #885's ordering exactly: + +```text +concurrent client: origin start -> KV snapshot -> auction dispatch -> origin wait +eager client: KV snapshot -> auction dispatch -> origin execution +``` + +### 4. Reconcile marker state during pre-send finalization + +After finalization finishes any EID ingestion, generation, or recovery mutation, +reconcile the marker against the resulting snapshot and current registry: + +| Snapshot / marker state | Response action | +| ---------------------------------------- | ------------------------------------------------------------- | +| Live and complete; marker absent/invalid | Set a fresh one-hour marker | +| Live and complete; marker already valid | Keep it without unnecessary refresh | +| Live but partial | Expire any present marker | +| Missing or tombstoned | Expire any present marker | +| Failed | Do not create a marker; do not turn failure into completeness | +| `NotRead` with a valid marker | Preserve it until its fixed expiration | +| `NotRead` without a valid marker | Do nothing | +| No pull-enabled partners | Never issue; remove a stale marker if present | + +After setting or clearing a marker, update the in-request status so post-send +logic sees the same decision. Refreshing is allowed only after a current +snapshot again proves completeness; a skipped request must not slide the marker +expiration indefinitely. + +On every explicit withdrawal, append the host-only marker-expiration cookie +before KV work, even when no usable `ts-ec` cookie is present. Keep EC-cookie +expiration and tombstone creation under their existing cookie/valid-ID gates. +KV failure remains best-effort and cannot prevent either applicable browser +cookie deletion. Do not change PR #885's existing-key-only conditional tombstone +operation. + +### 5. Plan pull sync before constructing the post-send graph + +Change the pull-sync preparation boundary so core receives the finalized +`EcContext` and `PartnerRegistry` before Fastly calls `require_identity_graph`. +The preparation step returns `None` when: + +- consent or active-EC validation fails; +- no pull-enabled partner exists; +- the finalized request retains a valid completeness marker and no + authoritative partial snapshot supersedes it; +- the snapshot is unread, failed, missing, tombstoned, or already complete. + +Only a present, live, partial snapshot produces `PullSyncContext`. Fastly then +constructs the graph and rate limiter and calls `dispatch_pull_sync`. + +Keep a defensive no-partner check at the start of `dispatch_pull_sync`. Use the +registry's deterministic pull-partner ordering before applying the existing +hourly rotation. Do not alter URL validation, token handling, response limits, +HTTP concurrency, missing-UID filtering, request-wide bulk CAS, tombstone +rejection, or best-effort logging. + +## File Map + +### New + +- `crates/trusted-server-core/src/ec/pull_sync_marker.rs` + - Marker protocol, key separation, fingerprinting, cookie formatting, + validation, completeness predicate, and focused unit tests. + +### Modify + +- `crates/trusted-server-core/src/constants.rs` + - Add the private completeness-cookie name. +- `crates/trusted-server-core/src/ec/mod.rs` + - Register the marker module; parse and carry marker state in `EcContext`; + invalidate it when the active EC changes. +- `crates/trusted-server-core/src/ec/registry.rs` + - Add deterministic pull-enabled partner/source-domain helpers. +- `crates/trusted-server-core/src/publisher.rs` + - Validate the marker before preload and retain reads required by auction, + EID ingestion, recovery, or privacy behavior. +- `crates/trusted-server-core/src/ec/finalize.rs` + - Reconcile marker issuance/expiration and clear it on withdrawal. +- `crates/trusted-server-core/src/ec/pull_sync.rs` + - Plan no-op/partial pull work from marker plus finalized snapshot before any + graph requirement. +- `crates/trusted-server-adapter-fastly/src/main.rs` + - Move `require_identity_graph` after pull-sync preparation succeeds. +- `docs/guide/edge-cookies.md` + - Document the internal cookie, bounded stale window, invalidation, and + limits of the optimization. + +No dependency or configuration-file changes are expected: `hmac`, `sha2`, and +`hex` are already direct core dependencies. + +## Implementation Tasks + +### Task 1 — Marker protocol and registry ordering + +**Files:** `constants.rs`, `registry.rs`, new `pull_sync_marker.rs` + +- [ ] Add failing tests for deterministic partner fingerprints across config + order and for changes caused by enabling, disabling, adding, or removing + a pull partner. +- [ ] Add failing marker tests for valid round-trip, malformed payload, invalid + hex/length, tampering, wrong EC ID, expiration, expiration beyond one + hour, passphrase mismatch, and partner-set mismatch. +- [ ] Add failing cookie tests for the exact live/expired security attributes + and absence of `Domain`. +- [ ] Implement the smallest versioned protocol and deterministic registry + helpers needed to make those tests pass. +- [ ] Run `cargo test-fastly pull_sync_marker` and registry tests. + +### Task 2 — Request marker state + +**Files:** `ec/mod.rs`, `pull_sync_marker.rs` + +- [ ] Add failing tests for absent, unvalidated, invalid, and valid state. +- [ ] Prove active-EC replacement cannot retain a marker validated for the old + EC ID. +- [ ] Parse the marker through the existing cookie jar and add crate-private + validation/status accessors. +- [ ] Ensure marker values and full EC IDs never enter logs. +- [ ] Run focused EC context and marker tests. + +### Task 3 — Preload decision + +**File:** `publisher.rs` + +- [ ] Extend the existing `OrderRecordingKv` tests to prove a valid marker plus + no auction/EID consumer performs zero lookups. +- [ ] Prove absent, malformed, expired, wrong-EC, and stale-partner-set markers + retain the normal lookup. +- [ ] Prove a valid marker does not suppress auction EID lookup or EID-cookie + persistence. +- [ ] Prove marker expiration restores normal missing-row detection and orphan + recovery. +- [ ] Refactor the preload decision and implement validation before + `load_snapshot`. +- [ ] Re-run concurrent/eager ordering and origin-error tests unchanged. + +### Task 4 — Finalize marker lifecycle + +**Files:** `finalize.rs`, `pull_sync_marker.rs` + +- [ ] Add failing tests for complete, partial, missing, tombstoned, failed, and + unread snapshots. +- [ ] Prove a pre-send complete snapshot sets the marker, while post-send + completion cannot modify the current response. +- [ ] Prove an authoritative partial snapshot clears stale valid state so its + missing partners remain eligible after send. +- [ ] Prove withdrawal expires both cookies even when KV fails, expires a + marker even when no usable `ts-ec` cookie is present, and still handles + differing active/cookie EC IDs under PR #885's rules. +- [ ] Implement one centralized marker reconciliation step at every relevant + finalize exit. +- [ ] Run focused finalize and withdrawal tests. + +### Task 5 — Pre-graph pull planning + +**Files:** `pull_sync.rs`, Fastly `main.rs` + +- [ ] Add tests showing no partners, a retained valid marker, a complete + snapshot, a tombstone, and failed/missing/unread snapshots produce no + dispatch context. +- [ ] Add a minimal test seam around the Fastly post-send graph factory and + prove it is not invoked for no-partner, valid-marker, complete-snapshot, + or otherwise no-dispatch preparation outcomes. +- [ ] Separately use a counting/failing core KV collaborator to prove those + paths perform no EC KV operation if dispatch is called defensively. +- [ ] Preserve a partial snapshot path that dispatches only missing eligible + partners and persists returned UIDs through PR #885's request-wide CAS. +- [ ] Move Fastly graph construction after successful preparation. +- [ ] Re-run existing multi-batch, response-validation, rate-limit, and CAS + conflict tests. + +### Task 6 — Documentation and regression verification + +**File:** `docs/guide/edge-cookies.md` + +- [ ] Document that `ts-ec-pull-complete` is signed, contains no UID, is + host-only, expires after one hour, and is deleted on withdrawal. +- [ ] Explain partner-set fingerprint invalidation and the bounded orphan + detection delay. +- [ ] State explicitly that auctions, EID ingestion, recovery, and withdrawal + may still require KV. +- [ ] Explain why sync completed after send can be marked only on a later + request. +- [ ] Run docs formatting and the full verification contract below. + +## Acceptance Test Matrix + +| Issue acceptance criterion | Planned evidence | +| --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| No pull-enabled partners causes zero pull-sync EC KV operations | Core no-partner preparation/counting-KV tests plus a Fastly graph-factory seam proving construction is skipped | +| Complete current marker skips initial pull-only read | Publisher counting test with valid marker and no other row consumer | +| Missing/expired/invalid state falls back | Marker unit tests plus publisher lookup-count tests | +| Partner changes cannot permanently preserve old completeness | Deterministic fingerprint mismatch tests plus one-hour maximum validation | +| Partial users dispatch only eligible partners | Existing and expanded partial-snapshot pull tests | +| Returned UIDs retain CAS protection | Existing PR #885 bulk snapshot/CAS tests remain green | +| KV failure stays best-effort | Publisher/pull/finalize failure tests; no marker is minted from failed state | +| Withdrawal clears browser state | Finalize test asserting both cookie-expiration headers despite KV failure | + +## Verification Contract + +Run focused tests during implementation: + +```bash +cargo test-fastly pull_sync_marker +cargo test-fastly ec_snapshot_preload +cargo test-fastly pull_sync +cargo test-fastly withdrawal +``` + +Before committing and opening the PR, run the repository gates: + +```bash +cargo fmt --all -- --check +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cd crates/trusted-server-js/lib && npx vitest run +cd crates/trusted-server-js/lib && npm run format +cd docs && npm run format +cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 +git diff --check +``` + +Record every command and result in the PR test plan. A failed environment-only +check must be reported accurately rather than marked complete. + +## Definition of Done + +- No-partner and recently complete pull-only paths do not construct the + post-send graph or perform EC KV operations. +- Marker validation precedes any snapshot read it is allowed to suppress. +- Other identity consumers still obtain the row when required. +- Invalid or stale markers fail safely; configuration changes and the one-hour + maximum prevent permanent stale completeness. +- Withdrawal deletes the marker independently of KV success. +- Incomplete users preserve current partner eligibility, HTTP, rate-limit, and + CAS behavior. +- No second request-scoped EC cache, new secret, new setting, or batch-sync + behavior is introduced. +- Documentation and all applicable CI gates pass. + +## Risks and Mitigations + +- **Dependency on PR #885:** Reconcile any changes to its snapshot or preload + contract before implementation continues; do not recreate superseded APIs. +- **Bounded orphan-detection delay:** A row removed after marker issuance may be + discovered up to one hour later. The signed expiration is enforced server-side + and cannot slide on a skipped request. +- **Marker cannot help every request:** Auction EIDs and request EID ingestion + still require actual row contents. Tests and docs must avoid broader claims. +- **Cookie churn and cache privacy:** Set the marker only when absent/invalid or + newly proven; existing cache-privacy middleware already downgrades responses + carrying `Set-Cookie`. +- **Host-only scope:** Different serving hostnames establish independent + markers. This is conservative and avoids widening cookie reach. +- **Partner configuration changes:** Source-set changes invalidate immediately; + other same-source configuration changes remain bounded by the one-hour + expiration and do not alter fill-missing completeness semantics. +- **Key rotation:** Passphrase rotation intentionally invalidates all markers and + falls back to KV. +- **Post-send limitation:** The last successful partner response cannot mark the + response already sent, so one later verification request is expected.