From 5c0b5ea1beb9bd29344c0df4b2646fe9ab7c823d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:50:45 +0530 Subject: [PATCH 01/12] Run pull sync as an identity provider on navigations Pull-sync partners become provider-owned: browser ingest skips their sources and batch sync rejects them. The post-send dispatcher and the completeness marker are removed, and leftover marker cookies are expired. Signed-off-by: dhruv8sh --- Cargo.lock | 5 + .../trusted-server-adapter-fastly/src/app.rs | 21 +- .../trusted-server-adapter-fastly/src/main.rs | 71 - crates/trusted-server-core/src/ec/cookies.rs | 13 + crates/trusted-server-core/src/ec/finalize.rs | 178 +-- crates/trusted-server-core/src/ec/kv.rs | 699 --------- crates/trusted-server-core/src/ec/mod.rs | 91 +- .../trusted-server-core/src/ec/pull_sync.rs | 1245 ----------------- .../src/ec/pull_sync_marker.rs | 689 --------- crates/trusted-server-core/src/ec/registry.rs | 54 +- .../src/migration_guards.rs | 1 - crates/trusted-server-core/src/publisher.rs | 225 +-- crates/trusted-server-core/src/settings.rs | 35 +- .../Cargo.toml | 5 + .../src/lib.rs | 107 +- .../src/pull_sync.rs | 234 ++++ 16 files changed, 500 insertions(+), 3173 deletions(-) delete mode 100644 crates/trusted-server-core/src/ec/pull_sync.rs delete mode 100644 crates/trusted-server-core/src/ec/pull_sync_marker.rs create mode 100644 crates/trusted-server-identity-providers/src/pull_sync.rs diff --git a/Cargo.lock b/Cargo.lock index 116d45a14c..e243b3c3a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6742,10 +6742,15 @@ dependencies = [ name = "trusted-server-identity-providers" version = "0.1.0" dependencies = [ + "bytes", "error-stack", + "http", "log", + "serde", + "serde_json", "toml", "trusted-server-core", + "url", ] [[package]] diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 5364bfc182..3faa735398 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -152,7 +152,9 @@ use trusted_server_core::settings::{CollectTiming, ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; -use trusted_server_identity_providers::{build_coordinator_with, partner_registry}; +use trusted_server_identity_providers::{ + build_coordinator_with, partner_registry, pull_sync_providers, +}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -361,6 +363,7 @@ pub(crate) struct EcFinalizeState { pub(crate) use_finalize_kv: bool, pub(crate) eids_cookie: Option, pub(crate) sharedid_cookie: Option, + #[cfg_attr(not(test), allow(dead_code))] pub(crate) is_real_browser: bool, /// Per-request services carried to the entry point so the pull-sync /// dispatcher can reuse the same platform HTTP client. @@ -435,8 +438,9 @@ impl<'a> ProviderRequestSignals<'a> { /// Captures the request signals for after-response identity lookups on a /// permitted publisher navigation. /// -/// Returns `None` when no after-response provider is enabled, the request has -/// no EC, or the identity graph is unavailable. +/// Returns `None` when no after-response provider (including a pull-sync +/// partner) is enabled, the request has no EC, or the identity graph is +/// unavailable. fn prepare_after_response_lookups( state: &AppState, req: &Request, @@ -447,7 +451,8 @@ fn prepare_after_response_lookups( .ec .identity_providers .iter() - .any(|config| config.enabled && config.collect == CollectTiming::AfterResponse); + .any(|config| config.enabled && config.collect == CollectTiming::AfterResponse) + || pull_sync_providers(&state.settings.ec).next().is_some(); if !has_after_response_provider { return None; } @@ -472,13 +477,15 @@ async fn dispatch_identity_lookups( req: &Request, ec: &mut EcRequestState, ) -> Option { - if !state + // Pull-sync partners run after the response, so only page providers + // count here. + let any_provider = state .settings .ec .identity_providers .iter() - .any(|config| config.enabled) - { + .any(|config| config.enabled && config.collect == CollectTiming::Page); + if !any_provider { return None; } let partners = match state.partner_registry() { diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 134889694b..a4474fc118 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -26,9 +26,6 @@ use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::provider::ClientSignals; -use trusted_server_core::ec::pull_sync::{ - PullSyncContext, build_pull_sync_context, dispatch_pull_sync, -}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::RequestFilterEffects; @@ -576,7 +573,6 @@ fn edgezero_main( request_filter_effects.as_ref(), counters.as_ref(), ); - run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); run_after_response_lookups( settings, &partner_registry, @@ -607,11 +603,6 @@ fn edgezero_main( request_filter_effects.as_ref(), counters.as_ref(), ); - run_edgezero_pull_sync_after_send( - &settings, - &partner_registry, - &ec_state, - ); run_after_response_lookups( &settings, &partner_registry, @@ -730,33 +721,6 @@ fn apply_edgezero_ec_finalize( Ok(partner_registry) } -fn run_edgezero_pull_sync_after_send( - settings: &Settings, - partner_registry: &PartnerRegistry, - ec_state: &EcFinalizeState, -) { - if !ec_state.is_real_browser { - return; - } - - let prepared_context = build_pull_sync_context(&ec_state.ec_context, partner_registry); - let Some((context, kv)) = - prepare_pull_sync_after_send(prepared_context, || require_identity_graph(settings)) - else { - return; - }; - - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - dispatch_pull_sync( - settings, - &kv, - partner_registry, - &limiter, - &context, - &ec_state.services, - ); -} - /// Runs after-response identity lookups and applies their follow-up write. /// /// Called once the response has been sent, so provider latency never delays @@ -831,24 +795,6 @@ fn run_after_response_lookups( log::info!("After-response lookups: write outcome={outcome}"); } -fn prepare_pull_sync_after_send( - context: Option, - graph_factory: F, -) -> Option<(PullSyncContext, KvIdentityGraph)> -where - F: FnOnce() -> Result>, -{ - let context = context?; - let kv = match graph_factory() { - Ok(kv) => kv, - Err(err) => { - log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); - return None; - } - }; - Some((context, kv)) -} - /// Sends a finalized `EdgeZero` response to the client. /// /// Streaming `EdgeZero` bodies commit headers first, then pipe chunks to Fastly's @@ -1291,23 +1237,6 @@ mod tests { } } - #[test] - fn pull_sync_noop_states_skip_post_send_graph_factory() { - let calls = std::cell::Cell::new(0); - let result = prepare_pull_sync_after_send(None, || { - calls.set(calls.get() + 1); - Err(Report::new(TrustedServerError::KvStore { - store_name: "unexpected".to_owned(), - message: "graph factory should not run".to_owned(), - })) - }); - assert!( - result.is_none(), - "a skipped pull-sync plan should return none" - ); - assert_eq!(calls.get(), 0, "should not invoke the graph factory"); - } - #[test] fn trace_path_classification_survives_core_request_conversion() { // `main` skips client-IP sanitization when the native path is a trace diff --git a/crates/trusted-server-core/src/ec/cookies.rs b/crates/trusted-server-core/src/ec/cookies.rs index ac0e0c05ba..71e5acb109 100644 --- a/crates/trusted-server-core/src/ec/cookies.rs +++ b/crates/trusted-server-core/src/ec/cookies.rs @@ -174,6 +174,19 @@ pub fn expire_ec_cookie(settings: &Settings, response: &mut Response) } } +/// Appends a raw `Set-Cookie` value, skipping values that are not valid +/// header values. +pub(crate) fn append_cookie(response: &mut Response, value: &str) { + match HeaderValue::from_str(value) { + Ok(value) => { + response.headers_mut().append(header::SET_COOKIE, value); + } + Err(err) => { + log::warn!("Skipping Set-Cookie: invalid header value: {err}"); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 2e28f92a23..f87c47bee3 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -10,17 +10,17 @@ use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; use crate::consent::{ConsentContext, allows_eid_persistence}; +use crate::constants::COOKIE_TS_EC_PULL_COMPLETE; use crate::settings::Settings; use super::EcContext; -use super::cookies::{expire_ec_cookie, set_ec_cookie}; +use super::cookies::{append_cookie, expire_ec_cookie, set_ec_cookie}; use super::eid_types::UnixSeconds; use super::generation::generate_ec_id; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; use super::page_load::{Generation, PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; -use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; use super::write_plan::{WritePlanInput, plan_write}; use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; @@ -59,7 +59,39 @@ pub fn ec_finalize_response( sharedid_cookie: Option<&str>, response: &mut Response, ) { - ec_context.validate_pull_sync_marker(settings, registry); + finalize_response_inner( + settings, + ec_context, + kv, + registry, + eids_cookie, + sharedid_cookie, + response, + ); + expire_retired_pull_marker(ec_context, response); +} + +/// Expires a leftover pull-sync completeness marker; the marker is retired. +fn expire_retired_pull_marker(ec_context: &EcContext, response: &mut Response) { + if ec_context.pull_marker_present() { + append_cookie( + response, + &format!( + "{COOKIE_TS_EC_PULL_COMPLETE}=; Path=/; Secure; SameSite=Lax; Max-Age=0; HttpOnly" + ), + ); + } +} + +fn finalize_response_inner( + settings: &Settings, + ec_context: &mut EcContext, + kv: Option<&KvIdentityGraph>, + registry: &PartnerRegistry, + eids_cookie: Option<&str>, + sharedid_cookie: Option<&str>, + response: &mut Response, +) { let (permit, after_response) = ec_context .take_navigation_permit() .and_then(|permit| permit.confirm_with_after_response(response.status())) @@ -68,12 +100,6 @@ pub fn ec_finalize_response( let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); if !consent_allows_ec { - // Expire the request-local marker independently of the EC cookie: a - // withdrawal must stop any pending pull-sync disclosure window. - if consent_withdrawn && ec_context.pull_sync_marker().was_present() { - expire_marker(ec_context.pull_sync_marker_mut(), response); - } - finalize_unusable_consent( settings, ec_context, @@ -124,8 +150,6 @@ pub fn ec_finalize_response( // mint a browser cookie with no backing identity-graph row. log::info!("Skipping generated EC response write because KV graph is unavailable"); } - - reconcile_pull_sync_marker(settings, registry, ec_context, response); } /// Shared inputs for the navigation's single write. @@ -275,24 +299,6 @@ fn update_returning( } } -fn reconcile_pull_sync_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_context: &mut EcContext, - response: &mut Response, -) { - let ec_id = ec_context.ec_value().map(str::to_owned); - let snapshot = ec_context.kv_snapshot().clone(); - reconcile_marker( - settings, - registry, - ec_id.as_deref(), - &snapshot, - ec_context.pull_sync_marker_mut(), - response, - ); -} - /// Withholds EID-derived KV updates when consent does not allow EID /// persistence, on top of the Purpose 1 (EC) gate `ec_finalize_response` /// already enforces before reaching this point. @@ -1236,59 +1242,6 @@ mod tests { ); } - #[test] - fn valid_marker_with_unread_snapshot_defers_orphan_recovery() { - let settings = create_test_settings(); - let orphaned_ec = sample_ec_id("orphn2"); - let consent = ConsentContext { - jurisdiction: Jurisdiction::NonRegulated, - source: ConsentSource::Cookie, - ..Default::default() - }; - let mut ec_context = EcContext::new_for_test_with_ip( - Some(orphaned_ec.clone()), - consent, - Some("192.0.2.10".to_owned()), - ); - ec_context.set_recovery_eligible(true); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::Valid { expires_at: 4_600 }, - ); - let mut partner = make_partner("pull.example.com"); - partner.pull_sync_enabled = true; - partner.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); - partner.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; - partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); - let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); - let graph = KvIdentityGraph::in_memory("test_store"); - ec_context.set_navigation_permit(NavigationPermit::for_test()); - let mut response = empty_response(); - - ec_finalize_response( - &settings, - &mut ec_context, - Some(&graph), - ®istry, - None, - None, - &mut response, - ); - - assert_eq!( - ec_context.ec_value(), - Some(orphaned_ec.as_str()), - "an unread snapshot should defer orphan rotation until marker expiry" - ); - assert!( - matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead), - "a marker-skipped request should leave the snapshot unread" - ); - assert!( - response.headers().get(http::header::SET_COOKIE).is_none(), - "bounded orphan deferral should not rewrite browser identity state" - ); - } - #[test] fn finalize_returning_user_subresource_does_not_persist_eid_updates() { let settings = create_test_settings(); @@ -2354,9 +2307,7 @@ mod tests { }; let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, - ); + ec_context.set_pull_marker_present_for_test(true); ec_context.set_navigation_permit(NavigationPermit::for_test()); let graph = KvIdentityGraph::failing("unavailable-store"); let mut response = empty_response(); @@ -2394,59 +2345,6 @@ mod tests { ); } - #[test] - fn finalize_sets_marker_for_complete_pull_partner_snapshot() { - let settings = create_test_settings(); - let ec_id = sample_ec_id("compl1"); - let mut partner = make_partner("ssp.example.com"); - partner.pull_sync_enabled = true; - partner.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); - partner.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; - partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); - let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); - let mut ec_context = make_context( - Some(&ec_id), - Some(&ec_id), - true, - false, - Jurisdiction::NonRegulated, - ); - let mut entry = live_entry(); - entry.ids.insert( - "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("partner-uid".to_owned()), - ); - ec_context.set_kv_snapshot(EcKvSnapshot::Present { - ec_id, - entry: Box::new(entry), - generation: Some(1), - }); - let mut response = empty_response(); - - ec_finalize_response( - &settings, - &mut ec_context, - None, - ®istry, - None, - None, - &mut response, - ); - - let cookies = response - .headers() - .get_all(http::header::SET_COOKIE) - .iter() - .filter_map(|value| value.to_str().ok()) - .collect::>(); - assert!( - cookies - .iter() - .any(|cookie| cookie.starts_with("ts-ec-pull-complete=v1.")), - "complete snapshot should issue the marker" - ); - } - #[test] fn explicit_withdrawal_without_marker_or_ec_cookie_does_not_set_cookie() { let settings = create_test_settings(); @@ -2485,9 +2383,7 @@ mod tests { ..Default::default() }; let mut ec_context = make_context_with_consent(None, None, false, false, consent); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, - ); + ec_context.set_pull_marker_present_for_test(true); let mut response = empty_response(); ec_finalize_response( diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index b0dc976387..e83ffb9140 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -10,7 +10,6 @@ //! (organic request paths) or propagate them (sync endpoints). See the //! per-operation error handling policy in the spec §7.5. -use std::collections::BTreeMap; use std::fmt; use std::sync::Arc; use std::time::Duration; @@ -81,32 +80,6 @@ impl PartnerIdUpdate { } } -pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - let mut changed = false; - for (partner_id, uid) in latest_updates { - if entry - .ids - .get(partner_id) - .is_some_and(|existing| existing.uid == uid) - { - continue; - } - - entry.ids.insert( - partner_id.to_owned(), - super::kv_types::KvPartnerId::single(uid.to_owned()), - ); - changed = true; - } - - changed -} - /// EC identity graph on top of the platform KV store primitives. /// /// Each EC ID (`{64hex}.{6alnum}`) maps to a JSON-encoded [`KvEntry`] @@ -546,167 +519,6 @@ impl KvIdentityGraph { ))) } - /// Merges partner IDs using request-scoped persisted state as the first CAS input. - /// - /// A caller-supplied `Present` snapshot is *proof* that the row exists — - /// either an `Add`-confirmed create from [`generate_if_needed`] or an - /// earlier authoritative read in the same request. Partner-ID enrichment is - /// best effort, so a refresh that reads absent or unreadable never - /// downgrades that proof: the update is skipped and logged, and the proven - /// snapshot is returned so cookie issuance continues from the confirmed - /// write. A failed *write* still reports [`EcKvSnapshot::Failed`] — that is - /// an operation failure, not an ambiguous read. - /// - /// A caller-supplied `Missing` snapshot is revalidated once before the - /// updates are dropped, because a point read on an eventually-consistent - /// store cannot prove absence and routes without orphan recovery have no - /// later chance to retry. - /// - /// [`generate_if_needed`]: super::generate_if_needed - pub(crate) fn upsert_partner_ids_from_snapshot( - &self, - ec_id: &str, - updates: &[PartnerIdUpdate], - snapshot: EcKvSnapshot, - ) -> EcKvSnapshot { - if updates.is_empty() { - return snapshot; - } - - // Existence proof carried by the incoming snapshot. Retained across - // every refresh so a best-effort enrichment read can never retract it. - let proven = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - .. - } if snapshot_id == ec_id => Some(snapshot.clone()), - _ => None, - }; - - // Resolve the initial usable snapshot without spending a CAS attempt. - // Every snapshot except a usable `Present` for this EC ID and a read - // that already failed is refreshed once. A `Missing` recorded earlier - // in the request is not proof of absence: edge KV point reads are - // eventually consistent, and named routes such as `/auction` and - // `/_ts/page-bids` never run orphan recovery, so short-circuiting on a - // stale miss there would drop the request's collected partner IDs - // outright. A refresh that still misses keeps the no-create behavior. - // A `Failed` read is returned as-is — the hot path never retries a - // lookup that already errored. Resolving here keeps all - // `MAX_CAS_RETRIES` iterations available for actual writes. - let mut current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - EcKvSnapshot::Failed { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => return snapshot, - _ => self.load_snapshot(ec_id), - }; - - for _attempt in 0..MAX_CAS_RETRIES { - let (mut entry, generation) = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - generation: Some(generation), - } if snapshot_id == ec_id => (entry.as_ref().clone(), generation), - // A refreshed read that is absent or unreadable is authoritative - // for this write: never create or overwrite a missing root. It - // is not authoritative for *existence* though, so a snapshot - // that already proved the row exists survives the refresh. - EcKvSnapshot::Missing { .. } | EcKvSnapshot::Failed { .. } => { - return Self::keep_proven(ec_id, current, proven.as_ref()); - } - // `load_snapshot` never yields `NotRead` or a generation-less - // `Present`; fail closed if that invariant is ever violated. - EcKvSnapshot::Present { .. } | EcKvSnapshot::NotRead => { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - }; - - if !entry.consent.ok { - return current; - } - if !apply_partner_id_updates(&mut entry, updates) { - return current; - } - let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => { - return EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: None, - }; - } - Ok(EcKvWriteOutcome::PreconditionFailed) => { - current = self.load_snapshot(ec_id); - } - Err(err) => { - log::warn!( - "snapshot partner upsert failed for '{}': {err:?}", - log_id(ec_id) - ); - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - } - } - - log::warn!( - "snapshot partner upsert for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; \ - {} partner updates were not persisted", - log_id(ec_id), - updates.len(), - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - - /// Returns `proven` instead of a read outcome that cannot disprove it. - /// - /// Partner-ID enrichment is best effort. When the caller already held proof - /// that the row exists — an `Add`-confirmed create or an earlier - /// authoritative read in the same request — a refresh that misses or fails - /// says nothing about existence, so the proof is kept and the skipped - /// update is logged. Write failures are *not* routed here: they are real - /// operation failures and stay [`EcKvSnapshot::Failed`] so callers can - /// report them. - fn keep_proven( - ec_id: &str, - downgraded: EcKvSnapshot, - proven: Option<&EcKvSnapshot>, - ) -> EcKvSnapshot { - match proven { - Some(proven) => { - log::warn!( - "snapshot partner upsert skipped for '{}': refresh was not authoritative; \ - keeping the confirmed row", - log_id(ec_id) - ); - proven.clone() - } - None => downgraded, - } - } - /// Atomically merges a partner ID into the existing entry. /// /// Uses CAS (generation markers) to avoid clobbering concurrent writes @@ -1603,93 +1415,6 @@ mod tests { ); } - #[test] - fn apply_partner_id_updates_returns_unchanged_for_empty_updates() { - let mut entry = live_entry(); - - let changed = apply_partner_id_updates(&mut entry, &[]); - - assert!(!changed, "should not change entry for empty updates"); - assert!(entry.ids.is_empty(), "should not add partner IDs"); - } - - #[test] - fn apply_partner_id_updates_skips_matching_existing_uid() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("uid-1".to_owned()), - ); - let updates = vec![PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!(!changed, "should not change when UID already matches"); - assert_eq!(entry.ids["ssp_x"].uid, "uid-1"); - } - - #[test] - fn apply_partner_id_updates_inserts_new_partner_uid() { - let mut entry = live_entry(); - let updates = vec![PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!(changed, "should report changed entry"); - assert_eq!(entry.ids["ssp_x"].uid, "uid-1"); - } - - #[test] - fn apply_partner_id_updates_overwrites_different_uid() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("old-uid".to_owned()), - ); - let updates = vec![PartnerIdUpdate::new("ssp_x", "new-uid")]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!(changed, "should report changed entry"); - assert_eq!(entry.ids["ssp_x"].uid, "new-uid"); - } - - #[test] - fn apply_partner_id_updates_applies_multiple_updates() { - let mut entry = live_entry(); - let updates = vec![ - PartnerIdUpdate::new("ssp_x", "uid-x"), - PartnerIdUpdate::new("ssp_y", "uid-y"), - ]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!(changed, "should report changed entry"); - assert_eq!(entry.ids["ssp_x"].uid, "uid-x"); - assert_eq!(entry.ids["ssp_y"].uid, "uid-y"); - } - - #[test] - fn apply_partner_id_updates_uses_last_duplicate_value() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("original".to_owned()), - ); - let updates = vec![ - PartnerIdUpdate::new("ssp_x", "intermediate"), - PartnerIdUpdate::new("ssp_x", "original"), - ]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!( - !changed, - "should not write when the final duplicate value matches existing state" - ); - assert_eq!(entry.ids["ssp_x"].uid, "original"); - } - #[test] fn evaluate_cluster_returns_stored_value_without_store_io() { let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); @@ -2062,51 +1787,6 @@ mod tests { ); } - #[test] - fn snapshot_bulk_upsert_returns_persisted_entry_without_stale_generation() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); - let snapshot = kv.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - let entry = outcome - .entry_for(&ec_id) - .expect("should retain persisted entry"); - assert_eq!( - entry.ids.get("ssp_x").map(|id| id.uid.as_str()), - Some("uid-1") - ); - assert_eq!( - outcome.generation_for(&ec_id), - None, - "backend does not return the post-write generation" - ); - } - - #[test] - fn snapshot_bulk_upsert_does_not_create_missing_root() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "missing snapshot should remain missing" - ); - assert!(kv.get(&ec_id).expect("should read store").is_none()); - } - #[test] fn write_withdrawal_tombstone_overwrites_live_entry() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -2343,321 +2023,6 @@ mod tests { } } - /// [`EcKvStore`] whose reads succeed but every write fails, simulating a - /// store that becomes unwritable mid-request. - struct WriteFailingEcKv { - inner: InMemoryEcKv, - } - - impl WriteFailingEcKv { - fn new() -> Self { - Self { - inner: InMemoryEcKv::new("write-failing-store"), - } - } - } - - impl EcKvStore for WriteFailingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - _key: &str, - _write: EcKvWrite<'_>, - ) -> Result> { - Err(Report::new(TrustedServerError::KvStore { - store_name: self.inner.store_name().to_owned(), - message: "write failing test store".to_owned(), - })) - } - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - - #[test] - fn snapshot_upsert_with_generation_writes_without_reading() { - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 0, - "a usable generation must avoid the initial read" - ); - assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("uid-1") - ); - assert_eq!( - outcome.generation_for(&ec_id), - None, - "a successful write should clear the stale generation" - ); - } - - #[test] - fn snapshot_upsert_unchanged_updates_preserve_generation() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - let mut seeded = live_entry(); - apply_partner_id_updates(&mut seeded, &[PartnerIdUpdate::new("ssp_x", "uid-1")]); - kv.create(&ec_id, &seeded).expect("should seed"); - let snapshot = kv.load_snapshot(&ec_id); - assert_eq!( - snapshot.generation_for(&ec_id), - Some(1), - "seeded snapshot should carry its stored generation" - ); - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "uid-1")], - snapshot, - ); - - assert_eq!( - outcome.generation_for(&ec_id), - Some(1), - "an unchanged merge preserves the usable generation and performs no write" - ); - } - - #[test] - fn snapshot_upsert_refreshes_unavailable_generation_exactly_once() { - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - // Finalize-written style snapshot: entry known, generation unavailable. - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 1, - "an unavailable generation refreshes exactly once before CAS" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|e| e.ids.contains_key("ssp_x")), - "refreshing an unavailable generation should persist the partner update" - ); - } - - #[test] - fn snapshot_upsert_gen_unavailable_survives_four_conflicts_then_writes() { - // A generation-unavailable snapshot (finalize-written style) refreshes - // once to obtain a usable generation. That refresh must not consume a - // CAS attempt, so all five write attempts remain: four conflicts - // followed by a successful fifth write still persist the update. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("uid-1"), - "the fifth CAS attempt must still succeed after a refresh and four conflicts" - ); - } - - #[test] - fn snapshot_upsert_cas_conflict_remerges_concurrent_data() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, true)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - let entry = outcome - .entry_for(&ec_id) - .expect("should persist re-merged entry"); - assert_eq!( - entry.ids.get("ssp_x").map(|id| id.uid.as_str()), - Some("uid-1"), - "conflict must re-merge our update onto the concurrently revived row" - ); - assert!(entry.consent.ok, "concurrent revive keeps the row live"); - } - - #[test] - fn snapshot_upsert_revalidates_transient_missing_and_persists() { - // An eventually-consistent point read earlier in the request missed a - // row that exists. Named routes such as `/auction` never run orphan - // recovery, so this refresh is the request's only chance to persist the - // collected partner IDs. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x").map(|id| id.uid.clone())), - Some("uid-1".to_owned()), - "a stale miss must be revalidated before the updates are dropped" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should keep the row"); - assert_eq!( - stored.ids.get("ssp_x").map(|id| id.uid.as_str()), - Some("uid-1"), - "the revalidated update must reach the store" - ); - } - - #[test] - fn snapshot_upsert_confirmed_missing_still_never_creates() { - // Revalidation only changes what a *stale* miss does. A row that is - // genuinely absent on the refresh must stay absent. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a confirmed miss must stay missing" - ); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "must not create a root entry for a missing key" - ); - } - - #[test] - fn snapshot_upsert_failed_snapshot_is_not_revalidated() { - // A lookup that already errored is not retried on the hot path, even - // though the row exists. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a failed lookup must not be retried by partner enrichment" - ); - } - - #[test] - fn snapshot_upsert_rejects_tombstone() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &KvEntry::tombstone(1000)) - .expect("should seed tombstone"); - let snapshot = kv.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| entry.ids.is_empty()), - "a tombstone must reject partner enrichment" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should keep the tombstone"); - assert!(stored.ids.is_empty(), "no update should reach the store"); - } - - #[test] - fn snapshot_upsert_store_failure_returns_failed_not_request_local() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a store write failure must not claim request-local IDs were persisted" - ); - } - #[test] fn upsert_partner_id_rejects_tombstone() { let graph = KvIdentityGraph::in_memory("test-store"); @@ -2708,70 +2073,6 @@ mod tests { ); } - #[test] - fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_misses() { - // `generate_if_needed` records a successful `Add` as `Present` without a - // generation. EID ingestion refreshes that snapshot to obtain one; on an - // eventually-consistent store the refresh can miss. Enrichment is best - // effort, so the miss must not retract the confirmed create — otherwise - // finalization suppresses the `ts-ec` cookie for a root that was written. - let graph = KvIdentityGraph::in_memory("empty-store"); - let ec_id = snapshot_ec_id(); - let add_confirmed = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); - - assert!( - outcome.entry_for(&ec_id).is_some(), - "an Add-confirmed row must survive a non-authoritative refresh miss" - ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "a missed refresh must not create or overwrite a root" - ); - } - - #[test] - fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_fails() { - let graph = KvIdentityGraph::failing("failing-store"); - let ec_id = snapshot_ec_id(); - let add_confirmed = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); - - assert!( - outcome.entry_for(&ec_id).is_some(), - "a read failure is not evidence of absence and must not retract the create" - ); - } - - #[test] - fn snapshot_upsert_without_proof_still_reports_a_refresh_miss() { - // No prior proof of existence: a `NotRead` snapshot that refreshes into - // a miss must stay `Missing` so finalization can run orphan recovery. - let graph = KvIdentityGraph::in_memory("empty-store"); - let ec_id = snapshot_ec_id(); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = - graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, EcKvSnapshot::NotRead); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "an unproven refresh miss must remain a miss" - ); - } - #[test] fn a_store_error_never_carries_the_whole_identifier() { // Every message in this module goes through `log_id`, so a report that diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 84c3147393..b426e869b9 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -29,7 +29,6 @@ //! - [`eids`] — Shared EID resolution and formatting helpers //! - [`eid_types`] — Value types for full-fidelity provider EIDs //! - [`batch_sync`] — S2S batch sync endpoint (`POST /_ts/api/v1/batch-sync`) -//! - [`pull_sync`] — Background pull-sync dispatcher for organic routes //! - [`page_load`] — Single-use page-load write permit and row changes //! - [`write_plan`] — Plans a navigation's single row change //! - [`provider`] — Identity provider interface and configuration @@ -56,8 +55,6 @@ pub mod page_load; pub mod partner; pub mod prebid_eids; pub mod provider; -pub mod pull_sync; -pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; pub mod write_plan; @@ -99,7 +96,6 @@ use self::coordinator::results::LookupResults; use self::kv::KvIdentityGraph; use self::kv_types::KvEntry; use self::page_load::{AfterResponseWrite, NavigationPermit}; -use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; /// Bounded request classifications that may persist browser EID cookies. /// @@ -196,8 +192,8 @@ pub use generation::{ struct RequestEc { /// EC ID from the `ts-ec` cookie, if present. cookie_ec: Option, - /// Pull-sync completeness marker, if present. - pull_sync_marker: Option, + /// Whether the retired pull-sync completeness marker is present. + pull_marker_present: bool, /// The parsed cookie jar (retained for consent pipeline input). jar: Option, } @@ -214,15 +210,13 @@ fn parse_ec_from_request(req: &Request) -> Result>, - /// Browser-carried proof of recent pull-partner completeness. - pull_sync_marker: PullSyncMarkerState, + /// Whether the request carried the retired pull-sync marker cookie. + pull_marker_present: bool, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, /// Navigation permit issued by the router for page-load navigations. @@ -407,7 +401,7 @@ impl EcContext { kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, client_eids: None, - pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), + pull_marker_present: parsed.pull_marker_present, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -479,7 +473,6 @@ impl EcContext { } self.ec_value = Some(ec_id); self.ec_generated = true; - self.pull_sync_marker.invalidate_for_replaced_ec(); Ok(()) } @@ -598,35 +591,16 @@ impl EcContext { self.client_eids.as_deref() } - /// Validates a browser completeness marker against the active EC and partner set. - pub(crate) fn validate_pull_sync_marker( - &mut self, - settings: &Settings, - registry: ®istry::PartnerRegistry, - ) { - validate_marker_state( - &mut self.pull_sync_marker, - settings, - registry, - self.ec_value.as_deref(), - ); - } - - /// Returns the current pull-sync marker state. + /// Whether the request carried the retired pull-sync marker cookie. #[must_use] - pub(crate) fn pull_sync_marker(&self) -> &PullSyncMarkerState { - &self.pull_sync_marker + pub(crate) fn pull_marker_present(&self) -> bool { + self.pull_marker_present } - /// Returns mutable pull-sync marker state for response reconciliation. - pub(crate) fn pull_sync_marker_mut(&mut self) -> &mut PullSyncMarkerState { - &mut self.pull_sync_marker - } - - /// Sets pull-sync marker state in focused unit tests. + /// Marks the retired pull-sync marker cookie as present, for tests. #[cfg(test)] - pub(crate) fn set_pull_sync_marker_for_test(&mut self, state: PullSyncMarkerState) { - self.pull_sync_marker = state; + pub(crate) fn set_pull_marker_present_for_test(&mut self, present: bool) { + self.pull_marker_present = present; } /// Stores the navigation permit issued by the router. @@ -682,7 +656,6 @@ impl EcContext { self.ec_value = Some(ec_id); self.ec_generated = true; self.kv_snapshot = snapshot; - self.pull_sync_marker.invalidate_for_replaced_ec(); } /// Returns whether EC creation is permitted by consent for this request. @@ -733,7 +706,7 @@ impl EcContext { kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, client_eids: None, - pull_sync_marker: PullSyncMarkerState::Absent, + pull_marker_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -762,7 +735,7 @@ impl EcContext { kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, client_eids: None, - pull_sync_marker: PullSyncMarkerState::Absent, + pull_marker_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -794,7 +767,7 @@ impl EcContext { kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, client_eids: None, - pull_sync_marker: PullSyncMarkerState::Absent, + pull_marker_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -1006,6 +979,36 @@ mod tests { assert_eq!(snapshot.generation_for("ec-1"), None); } + #[test] + fn leftover_pull_marker_is_expired() { + let settings = create_test_settings(); + let req = create_test_request(&[("cookie", "ts-ec-pull-complete=x")]); + let mut ec = EcContext::read_from_request(&settings, &req, &noop_services()) + .expect("should read EC context"); + let mut response = http::Response::new(EdgeBody::empty()); + + finalize::ec_finalize_response( + &settings, + &mut ec, + None, + ®istry::PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .any(|cookie| cookie.starts_with("ts-ec-pull-complete=;") + && cookie.contains("Max-Age=0")), + "should expire the leftover marker" + ); + } + #[test] fn read_from_request_ignores_header_ec() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs deleted file mode 100644 index 4fac813dd5..0000000000 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ /dev/null @@ -1,1245 +0,0 @@ -//! Pull sync background dispatch. -//! -//! Launches partner pull-sync requests for organic traffic after the client -//! response has been sent. Dispatch is best-effort and never affects client -//! response status. -//! -//! Pull sync currently fills missing partner UIDs only. Once a partner UID is -//! present in the EC identity entry, it is not periodically refreshed because -//! the entry no longer stores per-partner sync timestamps. - -use edgezero_core::body::Body as EdgeBody; -use http::{Method, StatusCode, header}; -use serde::Deserialize; -use url::Url; - -use crate::consent::allows_eid_persistence; -use crate::platform::{ - DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, PlatformHttpRequest, PlatformPendingRequest, - PlatformResponse, RuntimeServices, -}; -use crate::settings::Settings; - -use super::generation::{ec_hash, is_valid_ec_id}; -use super::kv::{KvIdentityGraph, PartnerIdUpdate}; -use super::kv_types::KvEntry; -use super::rate_limiter::RateLimiter; -use super::registry::{PartnerConfig, PartnerRegistry}; - -// `current_timestamp` is defined in the parent `ec` module. -use super::EcContext; -use super::EcKvSnapshot; -use super::current_timestamp; - -/// Inputs needed to dispatch pull sync after response flush. -#[derive(Debug, Clone)] -pub struct PullSyncContext { - ec_id: String, - snapshot: EcKvSnapshot, -} - -impl PullSyncContext { - /// Returns the EC ID for the request. - #[must_use] - pub fn ec_id(&self) -> &str { - &self.ec_id - } -} - -struct InFlightPull { - source_domain: String, - pending: PlatformPendingRequest, -} - -#[derive(Debug, Deserialize)] -struct PullSyncResponse { - uid: Option, -} - -/// Builds post-send pull-sync context from the route EC context. -/// -/// Returns `None` when there are no pull-enabled partners, when consent denies -/// EC, when there is no valid active EC ID, or when the request snapshot holds -/// no consented row that is still missing at least one pull-partner UID. -#[must_use] -pub fn build_pull_sync_context( - ec_context: &EcContext, - registry: &PartnerRegistry, -) -> Option { - // Without EID-persistence consent (TCF Purpose 1 + 4), finalization - // withholds browser-supplied partner UIDs. That leaves partner slots empty, - // which pull sync would otherwise read as eligible and disclose the EC ID - // to partners for exactly the users who opted out. - if registry.pull_enabled_partners().is_empty() - || !ec_context.ec_allowed() - || !allows_eid_persistence(ec_context.consent()) - { - return None; - } - - let ec_id_ref = ec_context.ec_value()?; - if !is_valid_ec_id(ec_id_ref) { - log::debug!("Pull sync: skipping dispatch because active EC ID is invalid format"); - return None; - } - - let entry = ec_context.kv_snapshot().entry_for(ec_id_ref)?; - if !entry.consent.ok || entry_is_pull_complete(entry, registry) { - return None; - } - - let ec_id = ec_id_ref.to_owned(); - let snapshot = ec_context.kv_snapshot().clone(); - Some(PullSyncContext { ec_id, snapshot }) -} - -/// Dispatches partner pull-sync requests in the background. -/// -/// This function is best-effort: all errors are logged and swallowed. -/// -/// # Consent revalidation -/// -/// Pull sync runs *after* the page response has been flushed to the browser and -/// discloses the raw `ec_id` to partners, so it cannot be authorized by the -/// request snapshot alone: a concurrent CMP withdrawal can tombstone the row in -/// the window between snapshot capture and dispatch. The live row is re-read -/// immediately before the first partner request and anything short of a live, -/// consenting entry cancels the dispatch. The read is deferred until after the -/// snapshot-based eligibility filter, so a request with nothing to pull still -/// costs no KV operation, and the fresh snapshot is reused for the write-back -/// instead of the stale one. -/// -/// # Panics -/// -/// Panics if the HTTP request builder produces an invalid request, which -/// cannot happen with the hardcoded method and well-formed URI used here. -pub fn dispatch_pull_sync( - settings: &Settings, - kv: &KvIdentityGraph, - registry: &PartnerRegistry, - rate_limiter: &dyn RateLimiter, - context: &PullSyncContext, - services: &RuntimeServices, -) { - let now = current_timestamp(); - let mut pull_partners = registry.pull_enabled_partners(); - - if pull_partners.is_empty() { - return; - } - - let Some(request_entry) = context.snapshot.entry_for(context.ec_id()) else { - log::debug!( - "Pull sync: skipping dispatch for '{}' because the request captured no usable \ - identity snapshot", - super::log_id(context.ec_id()) - ); - return; - }; - if !request_entry.consent.ok { - return; - } - - // Sort by source domain for deterministic ordering, then apply a rotating - // hourly offset so that different partners get dispatch priority (§10.3). - pull_partners.sort_by(|a, b| a.source_domain.cmp(&b.source_domain)); - - log::debug!( - "Pull sync: {} pull-enabled partners after filtering", - pull_partners.len(), - ); - - // Rotate the partner list so that the starting partner changes each - // hour. This ensures fair distribution when max_concurrency limits - // how many partners are dispatched per request. - let offset = (now / 3600) as usize % pull_partners.len(); - pull_partners.rotate_left(offset); - - // Drop partners the request snapshot already has a UID for before paying - // for the revalidation read: a dispatch with nothing to pull must not add a - // KV operation. - pull_partners.retain(|partner| is_partner_pull_eligible(partner, Some(request_entry))); - if pull_partners.is_empty() { - return; - } - - let live_snapshot = kv.load_snapshot(context.ec_id()); - let Some(live_entry) = live_snapshot.entry_for(context.ec_id()) else { - log::warn!( - "Pull sync: skipping dispatch for '{}' because the live identity row could not be \ - confirmed", - super::log_id(context.ec_id()) - ); - return; - }; - if !live_entry.consent.ok { - log::info!( - "Pull sync: skipping dispatch for '{}' because consent was withdrawn after the \ - request snapshot was captured", - super::log_id(context.ec_id()) - ); - return; - } - - let max_concurrency = settings.ec.pull_sync_concurrency.max(1); - let mut in_flight: Vec = Vec::new(); - let mut updates = Vec::new(); - - for partner in pull_partners { - // Re-checked against the live row: a concurrent request may have filled - // this partner's UID since the request snapshot was captured. - if !is_partner_pull_eligible(partner, Some(live_entry)) { - continue; - } - - let Some(url) = validated_pull_sync_url(partner) else { - continue; - }; - - let rate_key = pull_rate_limit_key(&partner.source_domain, context.ec_id()); - match rate_limiter.exceeded(&rate_key, partner.pull_sync_rate_limit) { - Ok(true) => { - log::debug!( - "Pull sync: rate-limited partner '{}' for ec_id '{}'", - partner.source_domain, - super::log_id(context.ec_id()) - ); - continue; - } - Ok(false) => {} - Err(err) => { - log::warn!( - "Pull sync: failed to read rate limit for partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - } - - let Some(token) = partner.ts_pull_token.as_ref() else { - log::warn!( - "Pull sync: partner '{}' enabled but missing ts_pull_token", - partner.source_domain - ); - continue; - }; - - let request_url = build_pull_request_url(url, context.ec_id()); - let scheme = request_url.scheme().to_string(); - let host = request_url.host_str().unwrap_or_default().to_string(); - let port = request_url.port(); - - let backend_name = match services.backend().ensure(&PlatformBackendSpec { - scheme, - host, - port, - host_header_override: None, - certificate_check: settings.proxy.certificate_check, - first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - discriminator: None, - }) { - Ok(name) => name, - Err(err) => { - log::warn!( - "Pull sync: failed to resolve backend for partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - }; - - let request = http::Request::builder() - .method(Method::GET) - .uri(request_url.as_str()) - .header("authorization", format!("Bearer {}", token.expose())) - .body(EdgeBody::empty()) - .expect("should build pull sync request"); - - let pending = match futures::executor::block_on( - services - .http_client() - .send_async(PlatformHttpRequest::new(request, backend_name)), - ) { - Ok(pending) => pending, - Err(err) => { - log::warn!( - "Pull sync: failed to dispatch partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - }; - - in_flight.push(InFlightPull { - source_domain: partner.source_domain.clone(), - pending, - }); - - if in_flight.len() >= max_concurrency { - drain_pull_batch(&mut in_flight, services, &mut updates); - } - } - - drain_pull_batch(&mut in_flight, services, &mut updates); - if !updates.is_empty() { - // Write back from the revalidated snapshot, not the request one: its - // generation is current, so the first CAS attempt is not spent losing a - // conflict against the read that authorized this dispatch. - let outcome = - kv.upsert_partner_ids_from_snapshot(context.ec_id(), &updates, live_snapshot.clone()); - if matches!(outcome, EcKvSnapshot::Failed { .. }) { - log::warn!( - "Pull sync: failed to persist partner updates for '{}'", - super::log_id(context.ec_id()) - ); - } - } -} - -/// Returns whether a consented live entry holds an ID for every pull-enabled -/// partner. Always returns `false` when there are no pull-enabled partners. -#[must_use] -pub(crate) fn entry_is_pull_complete(entry: &KvEntry, registry: &PartnerRegistry) -> bool { - let pull_partners = registry.pull_enabled_partners(); - !pull_partners.is_empty() - && entry.consent.ok - && pull_partners - .iter() - .all(|partner| !is_partner_pull_eligible(partner, Some(entry))) -} - -fn is_partner_pull_eligible(partner: &PartnerConfig, kv_entry: Option<&KvEntry>) -> bool { - kv_entry - .and_then(|entry| entry.ids.get(&partner.source_domain)) - .is_none() -} - -fn validated_pull_sync_url(partner: &PartnerConfig) -> Option { - let pull_sync_url = partner.pull_sync_url.as_deref()?; - let parsed = match Url::parse(pull_sync_url) { - Ok(url) => url, - Err(err) => { - log::error!( - "Pull sync: partner '{}' has invalid pull_sync_url '{}': {err}", - partner.source_domain, - pull_sync_url - ); - return None; - } - }; - - if parsed.scheme() != "https" { - log::error!( - "Pull sync: partner '{}' pull_sync_url must use HTTPS, got scheme '{}'", - partner.source_domain, - parsed.scheme() - ); - return None; - } - - let Some(hostname) = parsed.host_str() else { - log::error!( - "Pull sync: partner '{}' pull_sync_url has no hostname: {}", - partner.source_domain, - pull_sync_url - ); - return None; - }; - - let hostname = hostname.trim_end_matches('.').to_ascii_lowercase(); - if !partner.pull_sync_allowed_domains.iter().any(|domain| { - domain - .trim() - .trim_end_matches('.') - .eq_ignore_ascii_case(&hostname) - }) { - log::error!( - "Pull sync: partner '{}' URL host '{}' not in pull_sync_allowed_domains", - partner.source_domain, - hostname - ); - return None; - } - - Some(parsed) -} - -fn build_pull_request_url(mut base_url: Url, ec_id: &str) -> Url { - base_url.query_pairs_mut().append_pair("ec_id", ec_id); - base_url -} - -fn pull_rate_limit_key(source_domain: &str, ec_id: &str) -> String { - format!("pull:{source_domain}:{}", ec_hash(ec_id)) -} - -fn drain_pull_batch( - in_flight: &mut Vec, - services: &RuntimeServices, - updates: &mut Vec, -) { - for pending in in_flight.drain(..) { - let source_domain = pending.source_domain; - // All requests were dispatched up front via send_async, so waiting on - // each in turn does not change concurrency. - let response = - match futures::executor::block_on(services.http_client().wait(pending.pending)) { - Ok(response) => response, - Err(err) => { - log::warn!( - "Pull sync: request failed for partner '{}': {err:?}", - source_domain - ); - continue; - } - }; - - let Some(uid) = extract_pull_uid(response, &source_domain) else { - continue; - }; - - updates.push(PartnerIdUpdate::new(source_domain, uid)); - } -} - -/// Maximum response body size accepted from pull sync partners (64 KiB). -/// -/// The expected response is `{"uid":""}`, so 64 KiB is generous. -/// This prevents a misbehaving partner from exhausting WASM memory. -const MAX_PULL_RESPONSE_BYTES: usize = 64 * 1024; - -fn response_content_length_exceeds_limit(response: &PlatformResponse, source_domain: &str) -> bool { - let Some(value) = response.response.headers().get(header::CONTENT_LENGTH) else { - return false; - }; - - let Some(value) = value.to_str().ok() else { - log::warn!( - "Pull sync: partner '{}' returned invalid Content-Length header, rejecting", - source_domain - ); - return true; - }; - - let Ok(length) = value.parse::() else { - log::warn!( - "Pull sync: partner '{}' returned malformed Content-Length header, rejecting", - source_domain - ); - return true; - }; - - if length > MAX_PULL_RESPONSE_BYTES { - log::warn!( - "Pull sync: partner '{}' returned oversized Content-Length ({} bytes), rejecting", - source_domain, - length - ); - return true; - } - - false -} - -fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option { - let status = response.response.status(); - - if status == StatusCode::NOT_FOUND { - log::debug!( - "Pull sync: partner '{}' returned 404, treating as no-op", - source_domain - ); - return None; - } - - if !status.is_success() { - log::warn!( - "Pull sync: partner '{}' returned non-success status {}", - source_domain, - status - ); - return None; - } - - if response_content_length_exceeds_limit(&response, source_domain) { - return None; - } - - let body = response - .response - .into_body() - .into_bytes() - .unwrap_or_default(); - if body.len() > MAX_PULL_RESPONSE_BYTES { - log::warn!( - "Pull sync: partner '{}' returned oversized response ({} bytes), rejecting", - source_domain, - body.len() - ); - return None; - } - let payload = match serde_json::from_slice::(&body) { - Ok(payload) => payload, - Err(err) => { - log::warn!( - "Pull sync: partner '{}' returned invalid JSON body: {err}", - source_domain - ); - return None; - } - }; - - use super::kv_types::MAX_UID_LENGTH; - - let uid = payload.uid.filter(|value| !value.trim().is_empty()); - match uid { - None => { - log::debug!( - "Pull sync: partner '{}' returned null/empty uid, treating as no-op", - source_domain - ); - None - } - Some(ref value) if value.len() > MAX_UID_LENGTH => { - log::warn!( - "Pull sync: partner '{}' returned uid exceeding {} bytes (got {}), rejecting", - source_domain, - MAX_UID_LENGTH, - value.len() - ); - None - } - _ => uid, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::consent::types::ConsentContext; - use crate::ec::kv_types::KvEntry; - use crate::platform::PlatformResponse; - use crate::redacted::Redacted; - use crate::test_support::tests::purpose_one_only_consent; - - fn make_response(status: u16, body: &[u8]) -> PlatformResponse { - PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .body(EdgeBody::from(body.to_vec())) - .expect("should build test response"), - ) - } - - fn make_response_with_content_length( - status: u16, - content_length: usize, - body: &[u8], - ) -> PlatformResponse { - PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .header(header::CONTENT_LENGTH, content_length.to_string()) - .body(EdgeBody::from(body.to_vec())) - .expect("should build test response"), - ) - } - - fn pull_partner(ttl_sec: u64) -> PartnerConfig { - PartnerConfig { - name: "SSP X".to_owned(), - api_key_hash: Some("deadbeef".to_owned()), - bidstream_enabled: true, - source_domain: "ssp.example.com".to_owned(), - openrtb_atype: 3, - batch_rate_limit: 60, - pull_sync_enabled: true, - pull_sync_url: Some("https://sync.partner.test/pull".to_owned()), - pull_sync_allowed_domains: vec!["sync.partner.test".to_owned()], - pull_sync_ttl_sec: ttl_sec, - pull_sync_rate_limit: 20, - ts_pull_token: Some(Redacted::new("token".to_owned())), - provider_owned: false, - } - } - - #[test] - fn build_pull_sync_context_returns_context_when_valid() { - let consent = ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..ConsentContext::default() - }; - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); - let graph = KvIdentityGraph::in_memory("pull_store"); - ec_context.set_kv_snapshot(seed_present_snapshot(&graph, &ec_id)); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - - let context = build_pull_sync_context(&ec_context, ®istry) - .expect("should build pull sync context for valid EC"); - assert_eq!( - context.ec_id(), - ec_context.ec_value().expect("ec should be present"), - "should capture the EC ID from context" - ); - } - - #[test] - fn build_pull_sync_context_rejects_invalid_ec_id() { - let consent = ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..ConsentContext::default() - }; - let ec_context = EcContext::new_for_test(Some("invalid-ec".to_owned()), consent); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - - let context = build_pull_sync_context(&ec_context, ®istry); - assert!( - context.is_none(), - "should reject pull sync context when EC ID format is invalid" - ); - } - - #[test] - fn build_pull_sync_context_skips_when_eid_persistence_is_denied() { - // Purpose 1 granted keeps the EC, but Purpose 4 denied withholds EID - // writes and leaves partner slots empty. Pull sync must not read those - // empty slots as eligible and disclose the EC ID to partners. - let consent = purpose_one_only_consent(); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); - let graph = KvIdentityGraph::in_memory("pull_store"); - ec_context.set_kv_snapshot(seed_present_snapshot(&graph, &ec_id)); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - assert!( - ec_context.ec_allowed(), - "should keep the EC allowed when Purpose 1 is granted" - ); - - let context = build_pull_sync_context(&ec_context, ®istry); - - assert!( - context.is_none(), - "should skip pull sync when TCF Purpose 4 is denied" - ); - } - - #[test] - fn build_pull_sync_context_skips_empty_registry_and_complete_snapshot() { - let consent = ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..ConsentContext::default() - }; - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); - let graph = KvIdentityGraph::in_memory("pull_store"); - let mut snapshot = seed_present_snapshot(&graph, &ec_id); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - - assert!( - build_pull_sync_context(&ec_context, &PartnerRegistry::empty()).is_none(), - "no pull partners should skip before graph construction" - ); - assert!( - build_pull_sync_context(&ec_context, ®istry).is_none(), - "an unread snapshot should skip before graph construction" - ); - - if let EcKvSnapshot::Present { entry, .. } = &mut snapshot { - entry.ids.insert( - "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("uid".to_owned()), - ); - } - ec_context.set_kv_snapshot(snapshot); - assert!( - build_pull_sync_context(&ec_context, ®istry).is_none(), - "a complete snapshot should skip post-send work" - ); - } - - #[test] - fn partner_is_eligible_when_missing_from_entry() { - let partner = pull_partner(3600); - let entry = KvEntry::minimal("other_partner", "uid-1", 100); - - assert!( - is_partner_pull_eligible(&partner, Some(&entry)), - "should dispatch when partner has no stored UID" - ); - } - - #[test] - fn partner_is_not_eligible_when_already_present() { - let partner = pull_partner(3600); - let entry = KvEntry::minimal("ssp.example.com", "uid-1", 1000); - - assert!( - !is_partner_pull_eligible(&partner, Some(&entry)), - "should skip dispatch when partner already has a stored UID" - ); - } - - #[test] - fn completeness_requires_all_pull_partner_ids() { - let registry = PartnerRegistry::from_config(&[ - pull_enabled_ec_partner("a.example.com"), - pull_enabled_ec_partner("b.example.com"), - ]) - .expect("should build registry"); - let mut entry = KvEntry::minimal("a.example.com", "uid-a", 1_000); - - assert!( - !entry_is_pull_complete(&entry, ®istry), - "entry missing a pull partner ID should be incomplete" - ); - - entry.ids.insert( - "b.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("uid-b".to_owned()), - ); - - assert!( - entry_is_pull_complete(&entry, ®istry), - "entry with every pull partner ID should be complete" - ); - } - - #[test] - fn validated_pull_sync_url_rejects_http_scheme() { - let mut partner = pull_partner(3600); - partner.pull_sync_url = Some("http://sync.partner.test/pull".to_owned()); - - let validated = validated_pull_sync_url(&partner); - assert!( - validated.is_none(), - "should reject pull_sync_url with HTTP scheme" - ); - } - - #[test] - fn validated_pull_sync_url_rejects_non_allowlisted_host() { - let mut partner = pull_partner(3600); - partner.pull_sync_url = Some("https://evil.test/pull".to_owned()); - - let validated = validated_pull_sync_url(&partner); - assert!( - validated.is_none(), - "should reject runtime pull_sync_url host outside allowlist" - ); - } - - #[test] - fn validated_pull_sync_url_accepts_normalized_allowlist_match() { - let mut partner = pull_partner(3600); - partner.pull_sync_url = Some("https://SYNC.PARTNER.TEST./pull".to_owned()); - partner.pull_sync_allowed_domains = vec!["sync.partner.test".to_owned()]; - - let validated = validated_pull_sync_url(&partner); - assert!( - validated.is_some(), - "should accept allowlist match after hostname normalization" - ); - } - - #[test] - fn build_pull_request_url_appends_ec_id() { - let url = Url::parse("https://sync.partner.test/pull?x=1").expect("should parse URL"); - let result = build_pull_request_url(url, "ecid123"); - - let query = result.query().expect("should have query string"); - assert!(query.contains("x=1"), "should preserve existing query"); - assert!(query.contains("ec_id=ecid123"), "should append ec_id"); - assert!( - !query.contains("ip="), - "should not forward client IP to partners" - ); - } - - #[test] - fn pull_rate_limit_key_uses_ec_hash_only() { - let first_ec_id = format!("{}.ABC123", "a".repeat(64)); - let second_ec_id = format!("{}.XYZ789", "a".repeat(64)); - - let first_key = pull_rate_limit_key("ssp.example.com", &first_ec_id); - let second_key = pull_rate_limit_key("ssp.example.com", &second_ec_id); - - assert_eq!( - first_key, second_key, - "should bucket different suffixes for the same EC hash together" - ); - assert_eq!( - first_key, - format!("pull:ssp.example.com:{}", "a".repeat(64)), - "should key pull-sync rate limiting by source domain and EC hash" - ); - } - - #[test] - fn extract_pull_uid_treats_404_as_noop() { - let response = make_response(404, b""); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should treat 404 as no-op"); - } - - #[test] - fn extract_pull_uid_treats_uid_null_as_noop() { - let response = make_response(200, b"{\"uid\":null}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should treat uid=null as no-op"); - } - - #[test] - fn extract_pull_uid_rejects_oversized_uid() { - let long_uid = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); - let body = format!("{{\"uid\":\"{long_uid}\"}}"); - let response = make_response(200, body.as_bytes()); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject uid exceeding MAX_UID_LENGTH"); - } - - #[test] - fn extract_pull_uid_reads_uid_from_success_body() { - let response = make_response(200, b"{\"uid\":\"abc123\"}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert_eq!( - uid.as_deref(), - Some("abc123"), - "should parse uid from 200 body" - ); - } - - #[test] - fn extract_pull_uid_rejects_oversized_content_length_before_body_read() { - let response = make_response_with_content_length( - 200, - MAX_PULL_RESPONSE_BYTES + 1, - b"{\"uid\":\"abc123\"}", - ); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!( - uid.is_none(), - "should reject oversized Content-Length before parsing body" - ); - } - - #[test] - fn extract_pull_uid_accepts_small_body_without_content_length() { - let response = make_response(200, b"{\"uid\":\"abc123\"}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert_eq!( - uid.as_deref(), - Some("abc123"), - "should accept small valid response without Content-Length" - ); - } - - #[test] - fn extract_pull_uid_rejects_body_larger_than_limit() { - let body = format!("{{\"uid\":\"{}\"}}", "x".repeat(MAX_PULL_RESPONSE_BYTES)); - let response = make_response(200, body.as_bytes()); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject body larger than limit"); - } - - #[test] - fn rotating_offset_distributes_partners_across_hours() { - // Simulate 3 partners sorted by source domain: alpha, beta, gamma. - let ids = vec!["alpha.example.com", "beta.example.com", "gamma.example.com"]; - - // Hour 0: offset = 0 % 3 = 0 → [alpha, beta, gamma] - let ts_h0: u64 = 100; // within hour 0 - let offset_h0 = (ts_h0 / 3600) as usize % ids.len(); - assert_eq!(offset_h0, 0, "hour 0 should start at index 0"); - - // Hour 1: offset = (3600 / 3600) % 3 = 1 → [beta, gamma, alpha] - let offset_h1 = (3600u64 / 3600) as usize % ids.len(); - assert_eq!(offset_h1, 1, "hour 1 should start at index 1"); - - // Hour 2: offset = (7200 / 3600) % 3 = 2 → [gamma, alpha, beta] - let offset_h2 = (7200u64 / 3600) as usize % ids.len(); - assert_eq!(offset_h2, 2, "hour 2 should start at index 2"); - - // Hour 3: offset = (10800 / 3600) % 3 = 0 → wraps back to [alpha, beta, gamma] - let offset_h3 = (10800u64 / 3600) as usize % ids.len(); - assert_eq!(offset_h3, 0, "hour 3 should wrap back to index 0"); - - // Verify rotate_left produces expected ordering - let mut rotated = ids.clone(); - rotated.rotate_left(offset_h1); - assert_eq!( - rotated, - vec!["beta.example.com", "gamma.example.com", "alpha.example.com"], - "hour 1 rotation should move beta to front" - ); - } - - // ----------------------------------------------------------------------- - // Snapshot-driven eligibility and request-wide aggregation - // ----------------------------------------------------------------------- - - use crate::ec::kv::TombstoneOutcome; - use crate::error::TrustedServerError; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; - use crate::settings::EcPartner; - use crate::test_support::tests::create_test_settings; - use error_stack::Report; - use std::sync::Arc; - - struct AllowAllRateLimiter; - - impl RateLimiter for AllowAllRateLimiter { - fn exceeded( - &self, - _key: &str, - _hourly_limit: u32, - ) -> Result> { - Ok(false) - } - } - - fn pull_enabled_ec_partner(source_domain: &str) -> EcPartner { - EcPartner { - name: format!("Partner {source_domain}"), - source_domain: source_domain.to_owned(), - openrtb_atype: EcPartner::default_openrtb_atype(), - bidstream_enabled: true, - api_token: Some(Redacted::new(format!( - "{source_domain}-api-token-32-bytes-minimum" - ))), - batch_rate_limit: EcPartner::default_batch_rate_limit(), - pull_sync_enabled: true, - pull_sync_url: Some(format!("https://{source_domain}/sync")), - pull_sync_allowed_domains: vec![source_domain.to_owned()], - pull_sync_ttl_sec: 3600, - pull_sync_rate_limit: 100, - ts_pull_token: Some(Redacted::new("outbound-token".to_owned())), - } - } - - fn snapshot_ec_id() -> String { - format!("{}.ABC123", "a".repeat(64)) - } - - fn seed_present_snapshot(graph: &KvIdentityGraph, ec_id: &str) -> EcKvSnapshot { - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - graph.create(ec_id, &entry).expect("should seed live entry"); - graph.load_snapshot(ec_id) - } - - #[test] - fn dispatch_pull_sync_aggregates_batches_into_one_bulk_write() { - let mut settings = create_test_settings(); - // Force one partner per concurrency batch so responses span batches. - settings.ec.pull_sync_concurrency = 1; - let registry = PartnerRegistry::from_config(&[ - pull_enabled_ec_partner("alpha.example.com"), - pull_enabled_ec_partner("beta.example.com"), - ]) - .expect("should build pull registry"); - - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let snapshot = seed_present_snapshot(&graph, &ec_id); - - let stub = Arc::new(StubHttpClient::new()); - // One JSON response per partner, drained across two concurrency batches. - stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); - stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - let (entry, generation) = graph - .get(&ec_id) - .expect("should read store") - .expect("entry should exist"); - assert_eq!( - entry.ids.get("alpha.example.com").map(|id| id.uid.as_str()), - Some("synced-uid"), - "first partner UID should persist" - ); - assert_eq!( - entry.ids.get("beta.example.com").map(|id| id.uid.as_str()), - Some("synced-uid"), - "second partner UID should persist" - ); - assert_eq!( - generation, 2, - "two partner responses across batches must persist in exactly one bulk write" - ); - } - - #[test] - fn dispatch_pull_sync_calls_and_persists_only_missing_partner() { - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = PartnerRegistry::from_config(&[ - pull_enabled_ec_partner("alpha.example.com"), - pull_enabled_ec_partner("beta.example.com"), - ]) - .expect("should build pull registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - graph - .create( - &ec_id, - &KvEntry::minimal("alpha.example.com", "existing-alpha", 1_000), - ) - .expect("should seed partial entry"); - let snapshot = graph.load_snapshot(&ec_id); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"uid":"new-beta"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert_eq!( - stub.recorded_backend_names().len(), - 1, - "should call only the missing partner" - ); - let (entry, generation) = graph - .get(&ec_id) - .expect("should read store") - .expect("entry should exist"); - assert_eq!( - entry - .ids - .get("alpha.example.com") - .map(|partner_id| partner_id.uid.as_str()), - Some("existing-alpha"), - "existing UID should remain unchanged" - ); - assert_eq!( - entry - .ids - .get("beta.example.com") - .map(|partner_id| partner_id.uid.as_str()), - Some("new-beta"), - "missing UID should persist" - ); - assert_eq!(generation, 2, "missing UID should use one bulk CAS write"); - } - - #[test] - fn dispatch_pull_sync_skips_non_present_snapshots() { - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); - - for snapshot in [ - EcKvSnapshot::NotRead, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ] { - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - } - - assert!( - stub.recorded_backend_names().is_empty(), - "not-read, missing, and failed snapshots must not dispatch pull sync" - ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "no snapshot state should create a missing root" - ); - } - - #[test] - fn dispatch_pull_sync_skips_tombstone_snapshot() { - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1000)), - generation: Some(1), - }; - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert!( - stub.recorded_backend_names().is_empty(), - "a tombstone snapshot must not dispatch pull sync" - ); - } - #[test] - fn dispatch_pull_sync_skips_dispatch_when_ec_is_tombstoned_after_snapshot_capture() { - // The request snapshot authorized pull sync, then a concurrent request - // completed a CMP withdrawal while the page response was in flight. - // Pull sync runs post-send and discloses the raw `ec_id` to partners, so - // it must revalidate the live row and cancel rather than leak an - // identity the user just withdrew. - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let snapshot = seed_present_snapshot(&graph, &ec_id); - - // Concurrent withdrawal lands after the snapshot was captured. - assert_eq!( - graph - .write_withdrawal_tombstone(&ec_id, drop) - .expect("should tombstone the row"), - TombstoneOutcome::Written, - "should tombstone the row the snapshot still reports as present" - ); - - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"uid":"leaked-uid"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert!( - stub.recorded_backend_names().is_empty(), - "a withdrawal that lands after snapshot capture must cancel post-send pull sync" - ); - let (entry, _) = graph - .get(&ec_id) - .expect("should read store") - .expect("should keep the tombstone"); - assert!( - entry.ids.is_empty(), - "no partner UID may be written back onto a tombstoned row" - ); - } - - #[test] - fn dispatch_pull_sync_skips_revalidation_read_when_no_partner_is_eligible() { - // Every pull-enabled partner already has a UID in the request snapshot, - // so there is nothing to dispatch. The revalidation read exists to - // authorize outbound calls; with no calls to authorize it must not cost - // a KV operation. - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - entry.ids.insert( - "alpha.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("already-known".to_owned()), - ); - graph - .create(&ec_id, &entry) - .expect("should seed live entry"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: Some(1), - }; - - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert!( - stub.recorded_backend_names().is_empty(), - "a fully synced entry must not dispatch pull sync" - ); - } -} diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs deleted file mode 100644 index 7cc86898b2..0000000000 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ /dev/null @@ -1,689 +0,0 @@ -//! Short-lived browser marker for complete pull-sync partner state. - -use core::fmt; - -use edgezero_core::body::Body as EdgeBody; -use hmac::{Hmac, KeyInit as _, Mac as _}; -use http::{HeaderValue, Response, header}; -use sha2::{Digest as _, Sha256}; - -use crate::constants::COOKIE_TS_EC_PULL_COMPLETE; -use crate::redacted::Redacted; -use crate::settings::Settings; - -use super::pull_sync::entry_is_pull_complete; -use super::registry::PartnerRegistry; -use super::{EcKvSnapshot, current_timestamp}; - -type HmacSha256 = Hmac; - -const MARKER_VERSION: &str = "v1"; -const MARKER_KEY_LABEL: &[u8] = b"trusted-server/ec-pull-complete/key/v1"; -const MARKER_MAX_AGE_SECS: u64 = 60 * 60; -const MARKER_CLOCK_SKEW_SECS: u64 = 60; -const MAX_MARKER_LENGTH: usize = 256; - -/// Request-local validation state for the pull-sync completeness marker. -#[derive(Clone, Default)] -pub(crate) enum PullSyncMarkerState { - /// No marker cookie was present. - #[default] - Absent, - /// A marker was present but has not been checked against the active EC and partner set. - Unvalidated(Redacted), - /// A present marker failed validation or was disproved by authoritative KV state. - Invalid, - /// The marker is valid until the given Unix timestamp. - Valid { expires_at: u64 }, -} - -impl fmt::Debug for PullSyncMarkerState { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Absent => formatter.write_str("Absent"), - Self::Unvalidated(_) => formatter.write_str("Unvalidated()"), - Self::Invalid => formatter.write_str("Invalid"), - Self::Valid { expires_at } => formatter - .debug_struct("Valid") - .field("expires_at", expires_at) - .finish(), - } - } -} - -impl PullSyncMarkerState { - /// Creates marker state from an optional request cookie value. - #[must_use] - pub(crate) fn from_cookie(value: Option) -> Self { - value - .map(Redacted::new) - .map_or(Self::Absent, Self::Unvalidated) - } - - /// Returns whether a marker cookie was present on the request. - #[must_use] - pub(crate) fn was_present(&self) -> bool { - !matches!(self, Self::Absent) - } - - /// Returns whether the marker is currently valid. - #[must_use] - pub(crate) fn is_valid(&self) -> bool { - matches!(self, Self::Valid { .. }) - } - - /// Invalidates state bound to a replaced active EC ID. - pub(crate) fn invalidate_for_replaced_ec(&mut self) { - if self.was_present() { - *self = Self::Invalid; - } - } -} - -/// Validates an unvalidated marker against the active EC and current pull-partner set. -pub(crate) fn validate_marker_state( - state: &mut PullSyncMarkerState, - settings: &Settings, - registry: &PartnerRegistry, - ec_id: Option<&str>, -) { - let PullSyncMarkerState::Unvalidated(value) = state else { - return; - }; - let valid_until = ec_id.and_then(|ec_id| { - validate_marker( - value.expose(), - settings, - registry, - ec_id, - current_timestamp(), - ) - }); - *state = valid_until.map_or(PullSyncMarkerState::Invalid, |expires_at| { - PullSyncMarkerState::Valid { expires_at } - }); -} - -/// Reconciles browser marker state against finalized authoritative KV state. -pub(crate) fn reconcile_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: Option<&str>, - snapshot: &EcKvSnapshot, - state: &mut PullSyncMarkerState, - response: &mut Response, -) { - let pull_partners = sorted_pull_partner_domains(registry); - if pull_partners.is_empty() { - expire_if_present(state, response); - return; - } - - let Some(ec_id) = ec_id else { - expire_if_present(state, response); - return; - }; - - if !matches!(snapshot, EcKvSnapshot::NotRead) && !snapshot.belongs_to(ec_id) { - expire_if_present(state, response); - return; - } - - match snapshot { - EcKvSnapshot::Present { .. } => { - let entry = snapshot - .entry_for(ec_id) - .expect("snapshot binding should be checked before marker reconciliation"); - if !entry.consent.ok { - expire_if_present(state, response); - } else if entry_is_pull_complete(entry, registry) { - if !state.is_valid() { - set_marker(settings, registry, ec_id, state, response); - } - } else { - expire_if_present(state, response); - } - } - EcKvSnapshot::Missing { .. } => { - expire_if_present(state, response); - } - EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => { - if matches!(state, PullSyncMarkerState::Invalid) { - expire_if_present(state, response); - } - } - } -} - -/// Expires the completeness marker regardless of KV state. -pub(crate) fn expire_marker(state: &mut PullSyncMarkerState, response: &mut Response) { - append_cookie(response, &format_marker_cookie("", 0)); - *state = PullSyncMarkerState::Absent; -} - -fn set_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - state: &mut PullSyncMarkerState, - response: &mut Response, -) { - let now = current_timestamp(); - let expires_at = now.saturating_add(MARKER_MAX_AGE_SECS); - let Some(value) = create_marker(settings, registry, ec_id, expires_at) else { - return; - }; - append_cookie(response, &format_marker_cookie(&value, MARKER_MAX_AGE_SECS)); - *state = PullSyncMarkerState::Valid { expires_at }; -} - -fn expire_if_present(state: &mut PullSyncMarkerState, response: &mut Response) { - if state.was_present() { - expire_marker(state, response); - } -} - -fn append_cookie(response: &mut Response, value: &str) { - match HeaderValue::from_str(value) { - Ok(value) => { - response.headers_mut().append(header::SET_COOKIE, value); - } - Err(err) => { - log::warn!("Skipping pull-sync marker cookie: invalid header value: {err}"); - } - } -} - -fn format_marker_cookie(value: &str, max_age: u64) -> String { - // Keep this cookie host-only. The signed payload does not include the - // request host, so a Domain attribute would allow replay across sibling - // hosts that share the domain-wide EC cookie and passphrase. - format!( - "{COOKIE_TS_EC_PULL_COMPLETE}={value}; Path=/; Secure; SameSite=Lax; Max-Age={max_age}; HttpOnly" - ) -} - -#[cfg(test)] -pub(crate) fn create_marker_for_test( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, -) -> String { - create_marker_for_test_with_expiry( - settings, - registry, - ec_id, - current_timestamp().saturating_add(MARKER_MAX_AGE_SECS), - ) -} - -#[cfg(test)] -pub(crate) fn create_marker_for_test_with_expiry( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - expires_at: u64, -) -> String { - create_marker(settings, registry, ec_id, expires_at) - .expect("should create marker for non-empty test registry") -} - -fn create_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - expires_at: u64, -) -> Option { - let fingerprint = partner_set_fingerprint(registry)?; - let payload = marker_payload(ec_id, expires_at, &fingerprint); - let key = marker_key(settings); - let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); - mac.update(payload.as_bytes()); - let tag = hex::encode(mac.finalize().into_bytes()); - Some(format!("{MARKER_VERSION}.{expires_at}.{fingerprint}.{tag}")) -} - -fn validate_marker( - value: &str, - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - now: u64, -) -> Option { - if value.len() > MAX_MARKER_LENGTH { - return None; - } - - let mut segments = value.split('.'); - let version = segments.next()?; - let expires = segments.next()?; - let fingerprint = segments.next()?; - let tag = segments.next()?; - if segments.next().is_some() || version != MARKER_VERSION { - return None; - } - - let expires_at = expires.parse::().ok()?; - let max_future = now - .saturating_add(MARKER_MAX_AGE_SECS) - .saturating_add(MARKER_CLOCK_SKEW_SECS); - if expires_at <= now || expires_at > max_future { - return None; - } - - let expected_fingerprint = partner_set_fingerprint(registry)?; - if fingerprint != expected_fingerprint { - return None; - } - - let tag = hex::decode(tag).ok()?; - if tag.len() != 32 { - return None; - } - - let payload = marker_payload(ec_id, expires_at, fingerprint); - let key = marker_key(settings); - let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); - mac.update(payload.as_bytes()); - mac.verify_slice(&tag).ok()?; - Some(expires_at) -} - -fn marker_key(settings: &Settings) -> [u8; 32] { - let mut mac = HmacSha256::new_from_slice(settings.ec.passphrase.expose().as_bytes()) - .expect("should create marker key HMAC"); - mac.update(MARKER_KEY_LABEL); - mac.finalize().into_bytes().into() -} - -fn marker_payload(ec_id: &str, expires_at: u64, fingerprint: &str) -> String { - format!("{MARKER_VERSION}\0{ec_id}\0{expires_at}\0{fingerprint}") -} - -fn partner_set_fingerprint(registry: &PartnerRegistry) -> Option { - let domains = sorted_pull_partner_domains(registry); - if domains.is_empty() { - return None; - } - - let mut hasher = Sha256::new(); - hasher.update(b"trusted-server/ec-pull-partners/v1\0"); - for domain in domains { - hasher.update((domain.len() as u64).to_be_bytes()); - hasher.update(domain.as_bytes()); - } - Some(hex::encode(hasher.finalize())) -} - -fn sorted_pull_partner_domains(registry: &PartnerRegistry) -> Vec { - let mut domains = registry - .pull_enabled_partners() - .into_iter() - .map(|partner| partner.source_domain.clone()) - .collect::>(); - domains.sort(); - domains -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ec::kv_types::KvEntry; - use crate::redacted::Redacted; - use crate::settings::{EcPartner, Settings}; - use crate::test_support::tests::create_test_settings; - - const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; - - fn pull_partner(source_domain: &str) -> EcPartner { - EcPartner { - name: format!("Partner {source_domain}"), - source_domain: source_domain.to_owned(), - openrtb_atype: EcPartner::default_openrtb_atype(), - bidstream_enabled: true, - api_token: Some(Redacted::new(format!( - "token-{source_domain}-32-bytes-minimum-value" - ))), - batch_rate_limit: EcPartner::default_batch_rate_limit(), - pull_sync_enabled: true, - pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), - pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], - pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), - ts_pull_token: Some(Redacted::new("pull-token".to_owned())), - } - } - - fn settings_and_registry(domains: &[&str]) -> (Settings, PartnerRegistry) { - let mut settings = create_test_settings(); - settings.ec.partners = domains.iter().map(|domain| pull_partner(domain)).collect(); - let registry = - PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); - (settings, registry) - } - - fn empty_response() -> Response { - Response::builder() - .status(200) - .body(EdgeBody::empty()) - .expect("should build response") - } - - fn live_snapshot(ec_id: &str, domains: &[&str]) -> EcKvSnapshot { - let mut entry = KvEntry::tombstone(1_000); - entry.consent.ok = true; - for domain in domains { - entry.ids.insert( - (*domain).to_owned(), - crate::ec::kv_types::KvPartnerId::single(format!("uid-{domain}")), - ); - } - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: Some(1), - } - } - - fn marker_cookies(response: &Response) -> Vec<&str> { - response - .headers() - .get_all(header::SET_COOKIE) - .iter() - .filter_map(|value| value.to_str().ok()) - .collect() - } - - fn assert_expired(state: &PullSyncMarkerState, response: &Response) { - assert!(matches!(state, PullSyncMarkerState::Absent)); - assert_eq!( - marker_cookies(response), - vec!["ts-ec-pull-complete=; Path=/; Secure; SameSite=Lax; Max-Age=0; HttpOnly"], - "should emit the exact host-only expiration cookie" - ); - } - - #[test] - fn marker_round_trip_binds_ec_and_partner_set() { - let (settings, registry) = settings_and_registry(&["a.example.com", "b.example.com"]); - let now = 1_000; - let marker = - create_marker(&settings, ®istry, EC_ID, now + 3_600).expect("should create marker"); - - assert_eq!( - validate_marker(&marker, &settings, ®istry, EC_ID, now), - Some(now + 3_600), - "should validate the issued marker" - ); - assert!( - validate_marker(&marker, &settings, ®istry, "wrong", now).is_none(), - "should reject a marker for another EC" - ); - } - - #[test] - fn marker_fingerprint_is_order_independent_and_set_sensitive() { - let (settings, first) = settings_and_registry(&["a.example.com", "b.example.com"]); - let (_, reordered) = settings_and_registry(&["b.example.com", "a.example.com"]); - let (_, changed) = settings_and_registry(&["a.example.com", "c.example.com"]); - let marker = create_marker(&settings, &first, EC_ID, 4_600).expect("should create marker"); - - assert!( - validate_marker(&marker, &settings, &reordered, EC_ID, 1_000).is_some(), - "config ordering should not change the marker" - ); - assert!( - validate_marker(&marker, &settings, &changed, EC_ID, 1_000).is_none(), - "partner-set changes should invalidate the marker" - ); - } - - #[test] - fn fingerprint_changes_for_enable_disable_add_and_remove() { - let (_, enabled_one) = settings_and_registry(&["a.example.com"]); - let (_, enabled_two) = settings_and_registry(&["a.example.com", "b.example.com"]); - let mut disabled_config = pull_partner("a.example.com"); - disabled_config.pull_sync_enabled = false; - let disabled = PartnerRegistry::from_config(&[disabled_config]) - .expect("should build disabled registry"); - let removed = PartnerRegistry::empty(); - - let base = partner_set_fingerprint(&enabled_one); - assert_ne!( - base, - partner_set_fingerprint(&enabled_two), - "adding a pull partner should change the fingerprint" - ); - assert_ne!( - base, - partner_set_fingerprint(&disabled), - "disabling a pull partner should change the fingerprint" - ); - assert_ne!( - base, - partner_set_fingerprint(&removed), - "removing a pull partner should change the fingerprint" - ); - assert_ne!( - partner_set_fingerprint(&disabled), - partner_set_fingerprint(&enabled_one), - "enabling a partner should change the fingerprint" - ); - } - - #[test] - fn reconcile_rejects_snapshot_bound_to_different_ec() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let snapshot = live_snapshot("different-ec", &["a.example.com"]); - let mut valid_state = PullSyncMarkerState::Valid { expires_at: 4_600 }; - let mut valid_response = empty_response(); - - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut valid_state, - &mut valid_response, - ); - assert_expired(&valid_state, &valid_response); - - let mut absent_state = PullSyncMarkerState::Absent; - let mut absent_response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut absent_state, - &mut absent_response, - ); - assert!(marker_cookies(&absent_response).is_empty()); - assert!(matches!(absent_state, PullSyncMarkerState::Absent)); - } - - #[test] - fn authoritative_incomplete_states_expire_valid_marker() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let mut tombstone = KvEntry::tombstone(1_000); - tombstone.ids.insert( - "a.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("stale".to_owned()), - ); - let snapshots = [ - live_snapshot(EC_ID, &[]), - EcKvSnapshot::Missing { - ec_id: EC_ID.to_owned(), - }, - EcKvSnapshot::Present { - ec_id: EC_ID.to_owned(), - entry: Box::new(tombstone), - generation: Some(1), - }, - ]; - - for snapshot in snapshots { - let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; - let mut response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut state, - &mut response, - ); - assert_expired(&state, &response); - } - } - - #[test] - fn non_authoritative_states_preserve_valid_fixed_expiry_marker() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let snapshots = [ - EcKvSnapshot::Failed { - ec_id: EC_ID.to_owned(), - }, - EcKvSnapshot::NotRead, - ]; - - for snapshot in snapshots { - let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; - let mut response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut state, - &mut response, - ); - assert!(matches!( - state, - PullSyncMarkerState::Valid { expires_at: 4_600 } - )); - assert!(marker_cookies(&response).is_empty()); - } - } - - #[test] - fn steady_states_leave_the_marker_cookie_untouched() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let complete = live_snapshot(EC_ID, &["a.example.com"]); - let incomplete = live_snapshot(EC_ID, &[]); - let cases = [ - (PullSyncMarkerState::Valid { expires_at: 4_600 }, &complete), - (PullSyncMarkerState::Absent, &incomplete), - ]; - - for (mut state, snapshot) in cases { - let mut response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - snapshot, - &mut state, - &mut response, - ); - assert!( - marker_cookies(&response).is_empty(), - "a steady marker state should not add a Set-Cookie header" - ); - } - } - - #[test] - fn invalid_marker_is_cleared_without_authoritative_snapshot() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let mut state = PullSyncMarkerState::Invalid; - let mut response = empty_response(); - - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &EcKvSnapshot::NotRead, - &mut state, - &mut response, - ); - - assert_expired(&state, &response); - } - - #[test] - fn marker_allows_sixty_seconds_of_issuer_clock_skew() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let marker = - create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); - - assert_eq!( - validate_marker(&marker, &settings, ®istry, EC_ID, 940), - Some(4_600), - "a marker from an issuer sixty seconds ahead should remain valid" - ); - } - - #[test] - fn marker_rejects_expired_overlong_and_tampered_values() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let marker = - create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); - let mut tampered = marker.clone(); - tampered.push('0'); - - assert!( - validate_marker(&marker, &settings, ®istry, EC_ID, 4_600).is_none(), - "expired marker should fail" - ); - assert!( - validate_marker(&marker, &settings, ®istry, EC_ID, 939).is_none(), - "marker beyond the lifetime and clock-skew allowance should fail" - ); - assert!( - validate_marker(&tampered, &settings, ®istry, EC_ID, 1_000).is_none(), - "tampering should fail" - ); - assert!( - validate_marker( - &"x".repeat(MAX_MARKER_LENGTH + 1), - &settings, - ®istry, - EC_ID, - 1_000 - ) - .is_none(), - "overlong marker should fail" - ); - } - - #[test] - fn marker_rejects_wrong_passphrase_and_empty_partner_set() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let marker = - create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); - let mut changed_settings = settings.clone(); - changed_settings.ec.passphrase = - Redacted::new("different-secret-key-32-bytes-minimum".to_owned()); - let empty = PartnerRegistry::empty(); - - assert!( - validate_marker(&marker, &changed_settings, ®istry, EC_ID, 1_000).is_none(), - "passphrase rotation should invalidate the marker" - ); - assert!( - create_marker(&settings, &empty, EC_ID, 4_600).is_none(), - "empty partner sets should not produce a marker" - ); - } - - #[test] - fn marker_cookie_is_host_only_and_secure() { - let cookie = format_marker_cookie("value", MARKER_MAX_AGE_SECS); - assert_eq!( - cookie, - "ts-ec-pull-complete=value; Path=/; Secure; SameSite=Lax; Max-Age=3600; HttpOnly" - ); - assert!(!cookie.contains("Domain="), "marker should be host-only"); - } -} diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 435e164b80..1fbd8f3cb7 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -14,6 +14,9 @@ use crate::settings::{Ec, EcPartner}; use super::partner::{hash_api_key, normalize_partner_source_domain}; +/// Provider kind under which pull-sync partners run as identity providers. +pub const PULL_SYNC_KIND: &str = "pull-sync"; + /// Minimum length for inbound partner Bearer API tokens. pub const MIN_API_TOKEN_LENGTH: usize = 32; @@ -223,7 +226,8 @@ impl PartnerRegistry { } /// Builds the registry from `[ec]` settings and marks every source claimed - /// by an enabled identity provider whose kind is registered as + /// by an enabled identity provider whose kind is registered, and every + /// pull-sync partner when the `pull-sync` kind is registered, as /// provider-owned. /// /// An enabled provider with an unregistered kind never runs, so its @@ -250,6 +254,12 @@ impl PartnerRegistry { registry.mark_provider_owned(source); } } + let pull_sync_registered = is_registered_kind(PULL_SYNC_KIND); + for partner in ec.partners.iter().filter(|partner| { + pull_sync_registered && partner.pull_sync_enabled && partner.pull_sync_url.is_some() + }) { + registry.mark_provider_owned(&partner.source_domain); + } Ok(registry) } @@ -305,16 +315,12 @@ impl PartnerRegistry { .is_some_and(|partner| partner.provider_owned) } - /// Returns all partners with `pull_sync_enabled = true` whose source no - /// identity provider owns. - /// - /// A provider writes full-fidelity records for the sources it owns, so a - /// pull-sync write there would overwrite them with a single UID. + /// Returns all partners with `pull_sync_enabled = true`. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { self.by_source_domain .values() - .filter(|p| p.pull_sync_enabled && !p.provider_owned) + .filter(|p| p.pull_sync_enabled) .collect() } @@ -514,9 +520,15 @@ mod tests { #[test] fn from_ec_marks_enabled_provider_sources_only() { let mut ec = crate::test_support::tests::create_test_settings().ec; + let mut pull = make_partner("pull.example.com", &valid_api_token("pull-partner")); + pull.pull_sync_enabled = true; + pull.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); + pull.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; + pull.ts_pull_token = Some(Redacted::new("example-pull-token".to_owned())); ec.partners = vec![ make_partner("ids.example.com", &valid_api_token("ids-partner")), make_partner("other.example.com", &valid_api_token("other-partner")), + pull, ]; ec.identity_providers = vec![ toml::from_str( @@ -529,8 +541,9 @@ mod tests { .expect("should parse disabled provider"), ]; - let registry = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") - .expect("should build registry"); + let registry = + PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k" || kind == PULL_SYNC_KIND) + .expect("should build registry"); assert!( registry.is_provider_owned("ids.example.com"), @@ -540,6 +553,10 @@ mod tests { !registry.is_provider_owned("other.example.com"), "should not mark disabled provider source" ); + assert!( + registry.is_provider_owned("pull.example.com"), + "should mark pull-sync partner source" + ); } #[test] @@ -704,25 +721,6 @@ mod tests { assert!(result.is_err(), "should reject invalid source domain"); } - #[test] - fn pull_enabled_partners_excludes_provider_owned_sources() { - let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); - pull_partner.api_token = None; - pull_partner.pull_sync_enabled = true; - pull_partner.pull_sync_url = Some("https://pull.example.com/sync".to_owned()); - pull_partner.pull_sync_allowed_domains = vec!["pull.example.com".to_owned()]; - pull_partner.ts_pull_token = Some(Redacted::new("outbound-token".to_owned())); - let mut registry = - PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); - - registry.mark_provider_owned("pull.example.com"); - - assert!( - registry.pull_enabled_partners().is_empty(), - "should not pull sources an identity provider owns" - ); - } - #[test] fn pull_enabled_partners_filters_correctly() { let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 17b3d624f6..5218b14388 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -87,7 +87,6 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("ec/mod.rs", include_str!("ec/mod.rs")), ("ec/partner.rs", include_str!("ec/partner.rs")), ("ec/prebid_eids.rs", include_str!("ec/prebid_eids.rs")), - ("ec/pull_sync.rs", include_str!("ec/pull_sync.rs")), ("ec/rate_limiter.rs", include_str!("ec/rate_limiter.rs")), ("ec/registry.rs", include_str!("ec/registry.rs")), ("edge_cookie.rs", include_str!("edge_cookie.rs")), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 2a68d2ab8f..89da618ce8 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -61,7 +61,7 @@ use crate::cache_policy::{ CachePolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, }; use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; -use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; +use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; use crate::cookies::template_cache_policy::{TemplateCookieDecision, evaluate_cookie_policy}; use crate::creative_opportunities::{ @@ -3450,10 +3450,6 @@ struct EcSnapshotPreloadInput { is_get: bool, has_ec_id: bool, has_kv: bool, - marker_valid: bool, - auction_needs_row: bool, - eid_cookie_may_need_persistence: bool, - privacy_needs_row: bool, snapshot_already_read: bool, } @@ -3464,12 +3460,7 @@ fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { if input.snapshot_already_read { return false; } - let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; - let marker_can_skip = input.marker_valid - && !input.auction_needs_row - && !input.eid_cookie_may_need_persistence - && !input.privacy_needs_row; - eligible && !marker_can_skip + input.is_navigation && input.is_get && input.has_ec_id && input.has_kv } /// Rewrites a downstream request into an outbound publisher-origin request. @@ -4968,9 +4959,6 @@ pub async fn handle_publisher_request( let ec_id_owned = active_ec_id_owned.clone().filter(|_| ec_allowed); let ec_id = ec_id_owned.as_deref(); let cookie_jar = handle_request_cookies(&req)?; - if let Some(registry) = auction.registry { - ec_context.validate_pull_sync_marker(settings, registry); - } let geo = ec_context.geo_info().cloned(); let parsed_origin = url::Url::parse(&settings.publisher.origin_url).change_context( @@ -5295,25 +5283,11 @@ pub async fn handle_publisher_request( let request_method = req.method().clone(); let mut origin_request = Some(req); - let eid_cookie_may_need_persistence = cookie_jar - .as_ref() - .is_some_and(|jar| jar.get(COOKIE_TS_EIDS).is_some() || jar.get(COOKIE_SHAREDID).is_some()); - // This decision also gates the concurrent origin send below. A marker skip - // cannot currently delay the origin behind an auction because marker - // validation requires a non-empty registry, and every such auction sets - // `auction_needs_row` and retains the preload. let should_preload_ec = should_preload_ec_snapshot(&EcSnapshotPreloadInput { is_navigation, is_get, has_ec_id: active_ec_id.is_some(), has_kv: kv.is_some(), - marker_valid: ec_context.pull_sync_marker().is_valid(), - auction_needs_row: should_run_auction - && auction - .registry - .is_some_and(|registry| !registry.is_empty()), - eid_cookie_may_need_persistence, - privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), // A freshly generated EC already holds an authoritative `Missing` // snapshot: its row is only created by this navigation's write. snapshot_already_read: !matches!( @@ -8398,10 +8372,6 @@ mod tests { is_get: true, has_ec_id: true, has_kv: true, - marker_valid: false, - auction_needs_row: false, - eid_cookie_may_need_persistence: false, - privacy_needs_row: false, snapshot_already_read: false, }; assert!(should_preload_ec_snapshot(&baseline)); @@ -8430,33 +8400,6 @@ mod tests { ); } - #[test] - fn valid_marker_skips_only_when_no_other_consumer_needs_snapshot() { - let marker_only = EcSnapshotPreloadInput { - is_navigation: true, - is_get: true, - has_ec_id: true, - has_kv: true, - marker_valid: true, - auction_needs_row: false, - eid_cookie_may_need_persistence: false, - privacy_needs_row: false, - snapshot_already_read: false, - }; - assert!(!should_preload_ec_snapshot(&marker_only)); - assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { - auction_needs_row: true, - ..marker_only - })); - assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { - eid_cookie_may_need_persistence: true, - ..marker_only - })); - assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { - privacy_needs_row: true, - ..marker_only - })); - } use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::consent::ConsentContext; use crate::ec::kv_backend::test_support::InMemoryEcKv; @@ -8827,170 +8770,6 @@ mod tests { } } - #[derive(Clone, Copy)] - enum MarkerProbe { - Valid, - Malformed, - WrongEc, - Expired, - PartnerSetMismatch, - } - - fn marker_partner(source_domain: &str) -> crate::settings::EcPartner { - crate::settings::EcPartner { - name: format!("Marker partner {source_domain}"), - source_domain: source_domain.to_owned(), - openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), - bidstream_enabled: true, - api_token: Some(crate::redacted::Redacted::new(format!( - "marker-{source_domain}-api-token-32-bytes-minimum" - ))), - batch_rate_limit: crate::settings::EcPartner::default_batch_rate_limit(), - pull_sync_enabled: true, - pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), - pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], - pull_sync_ttl_sec: crate::settings::EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: crate::settings::EcPartner::default_pull_sync_rate_limit(), - ts_pull_token: Some(crate::redacted::Redacted::new("pull-token".to_owned())), - } - } - - async fn run_marker_lookup_probe( - probe: MarkerProbe, - has_eid_cookie: bool, - has_matched_auction_slot: bool, - ) -> usize { - let mut settings = if has_matched_auction_slot { - scheduling_settings() - } else { - create_test_settings() - }; - settings.ec.partners = vec![marker_partner("ssp.example.com")]; - let registry = - PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); - let http = Arc::new(StubHttpClient::new()); - http.set_concurrent_fanout(true); - http.push_response(200, b"ok".to_vec()); - let lookups = Arc::new(AtomicUsize::new(0)); - let graph = KvIdentityGraph::new(OrderRecordingKv { - inner: InMemoryEcKv::new("marker-store"), - http: Arc::clone(&http), - http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), - lookups: Arc::clone(&lookups), - }); - let services = build_services_with_http_client( - Arc::clone(&http) as Arc - ); - let ec_id = format!("{}.CkId01", "b".repeat(64)); - let marker = match probe { - MarkerProbe::Valid => { - crate::ec::pull_sync_marker::create_marker_for_test(&settings, ®istry, &ec_id) - } - MarkerProbe::Malformed => "not-a-marker".to_owned(), - MarkerProbe::WrongEc => crate::ec::pull_sync_marker::create_marker_for_test( - &settings, - ®istry, - &format!("{}.Other1", "c".repeat(64)), - ), - MarkerProbe::Expired => { - crate::ec::pull_sync_marker::create_marker_for_test_with_expiry( - &settings, - ®istry, - &ec_id, - crate::ec::current_timestamp().saturating_sub(1), - ) - } - MarkerProbe::PartnerSetMismatch => { - let other_registry = - PartnerRegistry::from_config(&[marker_partner("other.example.com")]) - .expect("should build alternate registry"); - crate::ec::pull_sync_marker::create_marker_for_test( - &settings, - &other_registry, - &ec_id, - ) - } - }; - let mut ec_context = EcContext::new_for_test_with_ip( - Some(ec_id), - scheduling_consent(), - Some("203.0.113.7".to_owned()), - ); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::from_cookie(Some(marker)), - ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let slots = has_matched_auction_slot - .then(scheduling_slot) - .into_iter() - .collect::>(); - let mut request = navigation_request(); - if has_eid_cookie { - request - .headers_mut() - .insert(header::COOKIE, HeaderValue::from_static("ts-eids=present")); - } - - let result = handle_publisher_request( - &settings, - &services, - Some(&graph), - &mut ec_context, - AuctionDispatch { - integration_registry: &IntegrationRegistry::empty_for_tests(), - orchestrator: &orchestrator, - slots: &slots, - registry: Some(®istry), - }, - request, - EdgeCacheHeader::SurrogateControl, - ) - .await; - - assert!(result.is_ok(), "should proxy the origin response"); - lookups.load(Ordering::SeqCst) - } - - #[tokio::test] - async fn valid_completeness_marker_skips_pull_only_snapshot_lookup() { - assert_eq!( - run_marker_lookup_probe(MarkerProbe::Valid, false, false).await, - 0 - ); - } - - #[tokio::test] - async fn invalid_markers_fall_back_to_snapshot_lookup() { - for probe in [ - MarkerProbe::Malformed, - MarkerProbe::WrongEc, - MarkerProbe::Expired, - MarkerProbe::PartnerSetMismatch, - ] { - assert_eq!( - run_marker_lookup_probe(probe, false, false).await, - 1, - "invalid marker should retain the normal preload" - ); - } - } - - #[tokio::test] - async fn valid_marker_does_not_skip_eid_cookie_persistence_lookup() { - assert_eq!( - run_marker_lookup_probe(MarkerProbe::Valid, true, false).await, - 1 - ); - } - - #[tokio::test] - async fn valid_marker_does_not_skip_auction_snapshot_lookup() { - assert_eq!( - run_marker_lookup_probe(MarkerProbe::Valid, false, true).await, - 1 - ); - } - #[tokio::test] async fn concurrent_client_starts_origin_before_ec_lookup() { let outcome = run_scheduling_probe(true, true, true, true).await; diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 148d774624..4e0724e714 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -468,11 +468,8 @@ pub struct EcPartner { /// Allowlist of domains TS may call for this partner's pull sync. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub pull_sync_allowed_domains: Vec, - /// Legacy pull-sync refresh interval retained for config compatibility. - /// - /// EC identity entries no longer store per-partner sync timestamps, so - /// this value is not used by the current fill-missing-only pull sync - /// behavior. + /// Legacy pull-sync refresh interval; retained for config compatibility; + /// unused. #[serde( default = "EcPartner::default_pull_sync_ttl_sec", deserialize_with = "from_value_or_str" @@ -630,6 +627,31 @@ impl IdentityProviderConfig { true } + /// Builds an enabled page-phase config with no TCF vendor, no rate limit, + /// default secrets and empty provider settings. + #[must_use] + pub fn new( + id: String, + kind: String, + endpoint: String, + allowed_domains: Vec, + sources: Vec, + ) -> Self { + Self { + id, + kind, + enabled: true, + endpoint, + allowed_domains, + sources, + tcf_vendor_id: None, + collect: CollectTiming::default(), + rate_limit_per_hour: None, + secrets: ProviderSecrets::default(), + settings: serde_json::Map::new(), + } + } + /// The configured implementation kind. #[must_use] pub fn kind(&self) -> &str { @@ -678,7 +700,8 @@ pub struct Ec { #[serde(default)] pub ec_store: Option, - /// Maximum number of concurrent pull-sync requests. + /// Maximum number of concurrent pull-sync requests; retained for config + /// compatibility; unused. #[serde(default = "Ec::default_pull_sync_concurrency")] pub pull_sync_concurrency: usize, diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index 831ed20729..6e154ba267 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -14,9 +14,14 @@ doctest = false workspace = true [dependencies] +bytes = { workspace = true } error-stack = { workspace = true } +http = { workspace = true } log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } trusted-server-core = { workspace = true } +url = { workspace = true } [dev-dependencies] toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 17f083be0f..a941f02a2a 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -9,28 +9,34 @@ //! Dependencies point one way: this crate depends on core, and core never //! depends on it. +pub mod pull_sync; + +use std::num::NonZeroU32; use std::sync::Arc; use std::time::Duration; use error_stack::Report; use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; -use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; +use trusted_server_core::ec::provider::{ + IdentityProvider, ProviderConfigError, ProviderKind as _, build_kind, +}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::settings::{ CollectTiming, Ec, IdentityProviderConfig, ProviderSecrets, Settings, }; +use crate::pull_sync::PullSync; + /// Whether `kind` has a registered implementation. -/// -/// No kinds are registered yet. #[must_use] -pub fn is_registered_kind(_kind: &str) -> bool { - false +pub fn is_registered_kind(kind: &str) -> bool { + kind == PullSync::KIND } /// Builds the partner registry, marking only sources claimed by enabled -/// providers whose kind is registered as provider-owned. +/// providers whose kind is registered (including pull-sync partners) as +/// provider-owned. /// /// # Errors /// @@ -41,19 +47,46 @@ pub fn partner_registry(ec: &Ec) -> Result Result, Report> { - let kind = config.kind(); - Err(Report::new(ProviderConfigError::UnknownKind) - .attach(format!("kind `{kind}` is not registered"))) + match config.kind() { + PullSync::KIND => build_kind::(config, secrets), + kind => Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))), + } +} + +/// One `pull-sync` provider per partner with pull sync enabled, built from the +/// partner's existing fields with no config change. +/// +/// Pull sync runs after the response has been sent, as it did before it +/// became a provider, so a slow partner never delays the page and a new EC +/// whose row does not exist yet is never pulled. Its hourly rate limit is +/// the partner's `pull_sync_rate_limit` (`None` when it is 0). +pub fn pull_sync_providers(ec: &Ec) -> impl Iterator + '_ { + ec.partners + .iter() + .filter(|partner| partner.pull_sync_enabled) + .filter_map(|partner| { + let endpoint = partner.pull_sync_url.clone()?; + let mut config = IdentityProviderConfig::new( + format!("pull-sync:{}", partner.source_domain), + PullSync::KIND.to_owned(), + endpoint, + partner.pull_sync_allowed_domains.clone(), + vec![partner.source_domain.clone()], + ); + config.secrets.token = partner.ts_pull_token.clone(); + config.collect = CollectTiming::AfterResponse; + config.rate_limit_per_hour = NonZeroU32::new(partner.pull_sync_rate_limit); + Some(config) + }) } /// Builds the coordinator for enabled providers collected before the @@ -127,17 +160,20 @@ fn build_phase_coordinator( // Source ownership across both phases is checked when settings load // (`config::validate_identity_providers`), so only this phase's providers // are built and checked here. - let registered = settings + let configured = settings .ec .identity_providers .iter() - .filter(|config| { - config.enabled && config.collect == timing && is_registered_kind(config.kind()) - }) + .filter(|config| config.enabled && is_registered_kind(config.kind())) + .cloned(); + let registered = configured + .chain(pull_sync_providers(&settings.ec)) + .filter(|config| config.collect == timing) .map(|config| { - build(config, &config.secrets).map(|provider| { - let registered = RegisteredProvider::new(config.clone(), provider); - match config.rate_limit_per_hour { + build(&config, &config.secrets).map(|provider| { + let rate_limit = config.rate_limit_per_hour; + let registered = RegisteredProvider::new(config, provider); + match rate_limit { Some(per_hour) => registered.with_rate_limit(per_hour), None => registered, } @@ -207,6 +243,39 @@ mod tests { assert!(coordinator.is_none(), "should build no coordinator"); } + #[test] + fn pull_sync_partner_runs_after_the_response() { + let mut settings = create_test_settings(); + settings.ec.partners = vec![ + toml::from_str( + r#" + name = "Pull partner" + source_domain = "partner.example.com" + pull_sync_enabled = true + pull_sync_url = "https://sync.example.com/pull" + pull_sync_allowed_domains = ["sync.example.com"] + ts_pull_token = "example-pull-token-32-bytes-minimum" + "#, + ) + .expect("should parse partner"), + ]; + + let page = build_coordinator(&settings).expect("should build page providers"); + let partners = Arc::new(partner_registry(&settings.ec).expect("should build partners")); + let after_response = build_after_response_coordinator(&settings, partners) + .expect("should build pull-sync providers") + .expect("should build an after-response coordinator"); + + assert!( + page.is_none(), + "should not run pull sync before the response" + ); + assert!( + !after_response.is_empty(), + "should run pull sync after the response" + ); + } + #[test] fn enabled_unknown_kind_is_skipped_and_claims_no_sources() { let mut settings = create_test_settings(); diff --git a/crates/trusted-server-identity-providers/src/pull_sync.rs b/crates/trusted-server-identity-providers/src/pull_sync.rs new file mode 100644 index 0000000000..404dbfffc8 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/pull_sync.rs @@ -0,0 +1,234 @@ +//! The `pull-sync` provider: the existing per-partner pull sync, run as an +//! identity provider on page-load navigations. + +use std::borrow::Cow; +use std::num::NonZeroUsize; + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, + ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, + ProviderRequest, ProviderResponse, ProviderUid, SourceCapability, SourceInput, +}; +use trusted_server_core::ec::registry::PULL_SYNC_KIND; +use url::Url; + +/// `pull-sync` kind. +pub struct PullSync; + +/// `pull-sync` has no provider-specific settings; everything comes from the +/// partner's existing pull-sync fields. +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PullSyncSettings {} + +/// A built `pull-sync` provider for one partner. +pub struct PullSyncProvider { + endpoint: Url, + source: SourceDomain, + authorization: Option, + capabilities: ProviderCapabilities, +} + +#[derive(Deserialize)] +struct PullSyncBody<'a> { + #[serde(borrow)] + uid: Option>, +} + +impl ProviderKind for PullSync { + const KIND: &'static str = PULL_SYNC_KIND; + type Settings = PullSyncSettings; + type Provider = PullSyncProvider; + + fn build( + common: &ProviderCommon<'_>, + _settings: PullSyncSettings, + ) -> Result> { + let [source] = common.sources else { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("pull-sync claims exactly one source")); + }; + let authorization = common + .secrets + .token + .as_ref() + .map(|token| { + let mut value = http::HeaderValue::try_from(format!("Bearer {}", token.expose())) + .map_err(|_| Report::new(ProviderConfigError::Invalid))?; + value.set_sensitive(true); + Ok::<_, Report>(value) + }) + .transpose()?; + let one = NonZeroUsize::MIN; + Ok(PullSyncProvider { + endpoint: common.endpoint.clone(), + source: source.clone(), + authorization, + capabilities: ProviderCapabilities { + sources: Box::new([SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }]), + shape: EidShape { + max_eids: one, + max_uids_per_eid: one, + eid_level_fields: EidLevelFields::Ignored, + }, + }, + }) + } +} + +impl IdentityProvider for PullSyncProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + let mut url = self.endpoint.clone(); + url.query_pairs_mut().append_pair("ec_id", lookup.ec_id); + let mut headers = http::HeaderMap::new(); + if let Some(value) = &self.authorization { + headers.insert(http::header::AUTHORIZATION, value.clone()); + } + Ok(ProviderRequest { + method: http::Method::GET, + url, + headers, + body: bytes::Bytes::new(), + context: None, + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let body: PullSyncBody<'body> = serde_json::from_slice(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + let Some(uid) = body.uid.filter(|uid| !uid.trim().is_empty()) else { + return Ok(LookupOutcome::NoMatch); + }; + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Owned(self.source.as_str().to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: uid, + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }])) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries}; + use trusted_server_core::redacted::Redacted; + use trusted_server_core::settings::ProviderSecrets; + + fn provider() -> PullSyncProvider { + let endpoint = Url::parse("https://sync.example.com/pull?x=1").expect("should parse"); + let sources = [SourceDomain::try_from("partner.example.com").expect("should parse")]; + let secrets = ProviderSecrets { + token: Some(Redacted::new("example-token".to_owned())), + }; + PullSync::build( + &ProviderCommon { + endpoint: &endpoint, + sources: &sources, + secrets: &secrets, + }, + PullSyncSettings {}, + ) + .expect("should build") + } + + #[test] + fn request_carries_ec_id_and_sensitive_bearer_token() { + let provider = provider(); + let consent = ConsentContext::default(); + let due = [SourceDomain::try_from("partner.example.com").expect("should parse")]; + + let request = provider + .build_request(&LookupInput { + ec_id: "abc.DEF123", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due, + current: OwnedEntries::empty(), + hashed_email: None, + }) + .expect("should build request"); + + assert_eq!( + request.url.as_str(), + "https://sync.example.com/pull?x=1&ec_id=abc.DEF123", + "should append ec_id" + ); + let auth = request + .headers + .get(http::header::AUTHORIZATION) + .expect("should send token"); + assert_eq!(auth, "Bearer example-token", "should use bearer auth"); + assert!(auth.is_sensitive(), "should mark the token sensitive"); + } + + #[test] + fn parses_uid_and_treats_null_as_no_match() { + let provider = provider(); + let headers = http::HeaderMap::new(); + let respond = |body: &'static [u8]| ProviderResponse { + status: http::StatusCode::OK, + headers: &headers, + body, + context: None, + }; + + let found = provider + .parse_response(respond(b"{\"uid\":\"u-1\"}")) + .expect("should parse"); + let none = provider + .parse_response(respond(b"{\"uid\":null}")) + .expect("should parse"); + let empty = provider + .parse_response(respond(b"{\"uid\":\" \"}")) + .expect("should parse"); + let invalid = provider.parse_response(respond(b"not json")); + + let LookupOutcome::Found(eids) = found else { + panic!("should find a UID") + }; + assert_eq!( + eids[0].source, "partner.example.com", + "should use the partner source" + ); + assert_eq!(eids[0].uids[0].id, "u-1", "should borrow the UID"); + assert!( + matches!(none, LookupOutcome::NoMatch), + "should treat null as no match" + ); + assert!( + matches!(empty, LookupOutcome::NoMatch), + "should treat an empty UID as no match" + ); + assert!(invalid.is_err(), "should reject invalid JSON"); + } +} From f2b5d4cce830231a49dbe779e369cf34b12a892d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:55:54 +0530 Subject: [PATCH 02/12] Mirror provider-owned bidstream UIDs to a JS-readable cookie Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/constants.rs | 4 +- crates/trusted-server-core/src/ec/cookies.rs | 2 +- .../src/ec/coordinator/mod.rs | 19 ++ crates/trusted-server-core/src/ec/finalize.rs | 207 ++++++++++++- crates/trusted-server-core/src/ec/mirror.rs | 280 ++++++++++++++++++ crates/trusted-server-core/src/ec/mod.rs | 28 +- crates/trusted-server-core/src/ec/registry.rs | 34 +++ docs/guide/edge-cookies.md | 20 +- 8 files changed, 575 insertions(+), 19 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/mirror.rs diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index 1fd3c2382c..6c0dd1a7f6 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -1,11 +1,13 @@ use http::header::HeaderName; pub const COOKIE_TS_EC: &str = "ts-ec"; -/// Short-lived signed proof that the current EC row has every pull-partner UID. +/// Retired pull-sync completeness marker; leftover cookies are expired. pub const COOKIE_TS_EC_PULL_COMPLETE: &str = "ts-ec-pull-complete"; /// Cookie written by the Trusted Server JS SDK containing a standard-base64-encoded /// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; +/// JS-readable mirror of provider-owned bidstream UIDs (spec §10). +pub const COOKIE_TS_EIDS_MIRROR: &str = "__Host-ts-eids-mirror"; pub const COOKIE_TS_TESTER: &str = "ts-tester"; pub const COOKIE_SHAREDID: &str = "sharedId"; diff --git a/crates/trusted-server-core/src/ec/cookies.rs b/crates/trusted-server-core/src/ec/cookies.rs index 71e5acb109..d45f1073d5 100644 --- a/crates/trusted-server-core/src/ec/cookies.rs +++ b/crates/trusted-server-core/src/ec/cookies.rs @@ -22,7 +22,7 @@ use crate::constants::COOKIE_TS_EC; use crate::settings::Settings; /// Maximum age for the EC cookie (1 year in seconds). -const COOKIE_MAX_AGE: i32 = 365 * 24 * 60 * 60; +pub(crate) const COOKIE_MAX_AGE: i32 = 365 * 24 * 60 * 60; fn is_allowed_ec_id_char(c: char) -> bool { c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_') diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 11f4c41855..3afd653bbe 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -26,6 +26,7 @@ use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; use crate::ec::kv_types::KvEntry; +use crate::ec::mirror::{MirrorCookie, build_mirror}; use crate::ec::page_load::RowChange; use crate::ec::provider::{ ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, @@ -70,6 +71,15 @@ pub trait IdentityCoordinator { /// /// Returns `None` when the row would not change, so no write is made. fn plan_write(&self, input: &WritePlanInput<'_>) -> Option; + + /// Builds the mirror cookie from the row as it stands after this + /// navigation: the committed row, or the snapshot when no write landed. + fn mirror( + &self, + row: Option<&KvEntry>, + consent: &ConsentContext, + now: UnixSeconds, + ) -> MirrorCookie; } /// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. @@ -632,6 +642,15 @@ impl IdentityCoordinator for ProviderCoordinator

{ fn plan_write(&self, input: &WritePlanInput<'_>) -> Option { plan_write(input) } + + fn mirror( + &self, + row: Option<&KvEntry>, + consent: &ConsentContext, + now: UnixSeconds, + ) -> MirrorCookie { + build_mirror(row, consent, &self.partners, now) + } } fn log_outcome(lookup: &ProviderLookup) { diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index f87c47bee3..253cbc587c 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -10,15 +10,16 @@ use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; use crate::consent::{ConsentContext, allows_eid_persistence}; -use crate::constants::COOKIE_TS_EC_PULL_COMPLETE; +use crate::constants::{COOKIE_TS_EC_PULL_COMPLETE, COOKIE_TS_EIDS_MIRROR}; use crate::settings::Settings; use super::EcContext; -use super::cookies::{append_cookie, expire_ec_cookie, set_ec_cookie}; +use super::cookies::{COOKIE_MAX_AGE, append_cookie, expire_ec_cookie, set_ec_cookie}; use super::eid_types::UnixSeconds; use super::generation::generate_ec_id; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; +use super::mirror::{MirrorCookie, build_mirror, mirror_consent}; use super::page_load::{Generation, PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; use super::registry::PartnerRegistry; @@ -59,7 +60,7 @@ pub fn ec_finalize_response( sharedid_cookie: Option<&str>, response: &mut Response, ) { - finalize_response_inner( + let navigation = finalize_response_inner( settings, ec_context, kv, @@ -68,9 +69,56 @@ pub fn ec_finalize_response( sharedid_cookie, response, ); + apply_mirror(ec_context, registry, navigation, response); expire_retired_pull_marker(ec_context, response); } +/// Sets or expires the mirror cookie (spec §10). +/// +/// Navigations rebuild it from the row as it stands after the page-load +/// write: the committed row is stored as the `Present` snapshot, so +/// [`super::EcKvSnapshot::entry_for`] returns it, or the read snapshot when no +/// write landed. Without EC and EID consent it is expired on navigations and on any +/// response whose request carried it. +fn apply_mirror( + ec_context: &EcContext, + registry: &PartnerRegistry, + navigation: bool, + response: &mut Response, +) { + let decision = if !mirror_consent(ec_context.consent()) { + MirrorCookie::Expire + } else if navigation { + let row = ec_context + .ec_value() + .and_then(|ec_id| ec_context.kv_snapshot().entry_for(ec_id)); + build_mirror( + row, + ec_context.consent(), + registry, + UnixSeconds::new(current_timestamp()), + ) + } else { + return; + }; + match decision { + MirrorCookie::Set(value) => append_cookie( + response, + &format!( + "{COOKIE_TS_EIDS_MIRROR}={}; Path=/; Secure; SameSite=Lax; Max-Age={COOKIE_MAX_AGE}", + value.as_str() + ), + ), + MirrorCookie::Expire if ec_context.mirror_cookie_present() || navigation => { + append_cookie( + response, + &format!("{COOKIE_TS_EIDS_MIRROR}=; Path=/; Secure; SameSite=Lax; Max-Age=0"), + ); + } + MirrorCookie::Expire => {} + } +} + /// Expires a leftover pull-sync completeness marker; the marker is retired. fn expire_retired_pull_marker(ec_context: &EcContext, response: &mut Response) { if ec_context.pull_marker_present() { @@ -91,11 +139,12 @@ fn finalize_response_inner( eids_cookie: Option<&str>, sharedid_cookie: Option<&str>, response: &mut Response, -) { +) -> bool { let (permit, after_response) = ec_context .take_navigation_permit() .and_then(|permit| permit.confirm_with_after_response(response.status())) .unzip(); + let navigation = permit.is_some(); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); @@ -109,7 +158,7 @@ fn finalize_response_inner( consent_withdrawn, response, ); - return; + return navigation; } if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { @@ -150,6 +199,7 @@ fn finalize_response_inner( // mint a browser cookie with no backing identity-graph row. log::info!("Skipping generated EC response write because KV graph is unavailable"); } + navigation } /// Shared inputs for the navigation's single write. @@ -890,7 +940,7 @@ mod tests { ); assert!( - get_header(&response, "set-cookie").is_none(), + !ec_cookie_emitted(&response), "should keep the EC cookie when the tombstone write fails" ); assert!( @@ -1932,7 +1982,7 @@ mod tests { ); assert!( - get_header(&response, "set-cookie").is_none(), + !ec_cookie_emitted(&response), "must not emit a cookie without an authoritative backing row" ); } @@ -2008,6 +2058,134 @@ mod tests { // Orphan-recovery gating and two-ID withdrawal // ----------------------------------------------------------------------- + fn mirror_cookies(response: &Response) -> Vec { + response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .filter(|cookie| cookie.starts_with("__Host-ts-eids-mirror=")) + .map(str::to_owned) + .collect() + } + + fn provider_found(source: &str) -> LookupResults { + let uid = KvUid::new( + UidValue::try_from("provider-uid").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + // Made for the EC ID `run_mirror_navigation` seeds. + LookupResults::new( + &sample_ec_id("mirr01"), + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + }], + ) + } + + fn run_mirror_navigation(lookups: LookupResults) -> Response { + let settings = create_test_settings(); + let ec_id = sample_ec_id("mirr01"); + let graph = KvIdentityGraph::in_memory("mirror_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + ec_context.set_lookup_results(lookups); + let mut registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) + .expect("should build registry"); + registry.mark_provider_owned("ids.example.com"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + None, + &mut response, + ); + response + } + + #[test] + fn navigation_sets_mirror_from_committed_row() { + let response = run_mirror_navigation(provider_found("ids.example.com")); + + let cookies = mirror_cookies(&response); + assert_eq!(cookies.len(), 1, "should set one mirror cookie"); + assert!( + !cookies[0].contains("Max-Age=0") && cookies[0].contains("Max-Age="), + "should set the mirror with a non-zero Max-Age" + ); + assert!( + !cookies[0].contains("HttpOnly") && !cookies[0].contains("Domain="), + "should be JS-readable and host-only" + ); + } + + #[test] + fn navigation_without_provider_uids_expires_mirror() { + let response = run_mirror_navigation(LookupResults::default()); + + let cookies = mirror_cookies(&response); + assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); + assert!( + cookies[0].contains("Max-Age=0"), + "should expire the mirror when no provider UID qualifies" + ); + } + + #[test] + fn withdrawn_consent_expires_mirror_on_any_response() { + let settings = create_test_settings(); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = make_context_with_consent(None, None, false, false, consent); + ec_context.set_mirror_cookie_present_for_test(true); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + None, + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let cookies = mirror_cookies(&response); + assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); + assert!( + cookies[0].contains("Max-Age=0"), + "should expire the mirror without a navigation" + ); + } + fn granting_consent() -> ConsentContext { ConsentContext { jurisdiction: Jurisdiction::NonRegulated, @@ -2036,6 +2214,17 @@ mod tests { ec } + /// Whether the response sets or expires the `ts-ec` cookie; other + /// cookies, such as the mirror, are ignored. + fn ec_cookie_emitted(response: &Response) -> bool { + response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .any(|cookie| cookie.starts_with("ts-ec=")) + } + fn assert_did_not_rotate(ec_context: &EcContext, orphan: &str, response: &Response) { assert_eq!( ec_context.ec_value(), @@ -2044,7 +2233,7 @@ mod tests { ); assert!(!ec_context.ec_generated(), "must not mark a rotated EC"); assert!( - get_header(response, "set-cookie").is_none(), + !ec_cookie_emitted(response), "must not emit a replacement cookie" ); } @@ -2632,7 +2821,7 @@ mod tests { ); assert!( - get_header(&response, "set-cookie").is_none(), + !ec_cookie_emitted(&response), "should not issue an EC whose row was not created" ); } diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs new file mode 100644 index 0000000000..72d678e05c --- /dev/null +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -0,0 +1,280 @@ +//! The JS-readable mirror of provider-owned bidstream UIDs (spec §10). + +use base64::Engine as _; +use serde_json::{Map, Value, json}; + +use crate::consent::ConsentContext; +use crate::consent::allows_eid_persistence; +use crate::ec::consent::ec_consent_granted; +use crate::ec::eid_types::{Expiry, UnixSeconds}; +use crate::ec::kv_types::{KvEid, KvEntry, KvUid}; +use crate::ec::registry::PartnerRegistry; + +/// Largest encoded mirror value. +pub const MAX_MIRROR_BYTES: usize = 3584; + +/// What to do with the mirror cookie on this response. +#[derive(Debug, PartialEq, Eq)] +pub enum MirrorCookie { + /// Set the cookie to this value. + Set(MirrorValue), + /// Expire the cookie. + Expire, +} + +/// An encoded mirror value within [`MAX_MIRROR_BYTES`]. +#[derive(Debug, PartialEq, Eq)] +pub struct MirrorValue(String); + +impl MirrorValue { + /// The encoded cookie value. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// Builds the mirror from the row as it stands after this navigation. +/// +/// Holds live, provider-written UIDs of `bidstream_enabled` partners, in +/// partner config order, as unpadded base64url JSON +/// `{"v":1,"eids":[...]}`. UIDs that would push the value past +/// [`MAX_MIRROR_BYTES`] are skipped. Returns [`MirrorCookie::Expire`] when +/// consent does not allow the EC or EID persistence, the row is missing or withdrawn, +/// or nothing qualifies. +/// +/// # Performance +/// +/// Re-encodes once per kept UID; the size cap bounds this to a few dozen +/// small encodes per navigation. +#[must_use] +pub fn build_mirror( + row: Option<&KvEntry>, + consent: &ConsentContext, + partners: &PartnerRegistry, + now: UnixSeconds, +) -> MirrorCookie { + let Some(row) = row.filter(|row| row.consent.ok && mirror_consent(consent)) else { + return MirrorCookie::Expire; + }; + let mut eids: Vec = Vec::new(); + let mut encoded: Option = None; + for partner in partners + .in_config_order() + .filter(|partner| partner.bidstream_enabled) + { + let Some(record) = row.ids.get(&partner.source_domain) else { + continue; + }; + for eid in record.eids() { + let entry_index = eids.len(); + eids.push(eid_json(&partner.source_domain, eid)); + let mut kept = 0_usize; + for uid in eid + .uids() + .iter() + .filter(|uid| uid.owner().is_some() && uid.is_live(now)) + { + uids_mut(&mut eids[entry_index]).push(uid_json(uid)); + if let Some(value) = encode(&eids) { + encoded = Some(value); + kept += 1; + } else { + uids_mut(&mut eids[entry_index]).pop(); + } + } + if kept == 0 { + eids.pop(); + } + } + } + match encoded { + Some(value) if !eids.is_empty() => MirrorCookie::Set(MirrorValue(value)), + _ => MirrorCookie::Expire, + } +} + +/// Whether consent allows the mirror: EC consent and EID persistence. +#[must_use] +pub(crate) fn mirror_consent(consent: &ConsentContext) -> bool { + ec_consent_granted(consent) && allows_eid_persistence(consent) +} + +fn uids_mut(eid: &mut Value) -> &mut Vec { + eid["uids"] + .as_array_mut() + .expect("should hold a uids array") +} + +fn encode(eids: &[Value]) -> Option { + let json = serde_json::to_vec(&json!({ "v": 1, "eids": eids })).ok()?; + let value = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json); + (value.len() <= MAX_MIRROR_BYTES).then_some(value) +} + +fn eid_json(source: &str, eid: &KvEid) -> Value { + let fields = eid.fields(); + let mut object = Map::new(); + object.insert("source".to_owned(), Value::from(source)); + if let Some(inserter) = &fields.inserter { + object.insert("inserter".to_owned(), Value::from(inserter.as_ref())); + } + if let Some(matcher) = &fields.matcher { + object.insert("matcher".to_owned(), Value::from(matcher.as_ref())); + } + if let Some(mm) = fields.mm { + object.insert("mm".to_owned(), Value::from(i32::from(mm))); + } + if let Some(ext) = &fields.ext { + object.insert("ext".to_owned(), ext.to_value()); + } + object.insert("uids".to_owned(), Value::Array(Vec::new())); + Value::Object(object) +} + +fn uid_json(uid: &KvUid) -> Value { + let mut object = Map::new(); + object.insert("id".to_owned(), Value::from(uid.id().as_str())); + object.insert("atype".to_owned(), Value::from(uid.atype().get())); + if let Some(ext) = uid.ext() { + object.insert("ext".to_owned(), ext.to_value()); + } + if let Expiry::At(at) = uid.expiry() { + object.insert("exp".to_owned(), Value::from(at.get())); + } + Value::Object(object) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::eid_types::{AgentType, EidFields, ProviderId, UidValue}; + use crate::ec::kv_types::KvPartnerId; + + fn partners(bidstream: bool) -> PartnerRegistry { + let partner = serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": bidstream + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn uid(id: &str, expiry: Expiry) -> KvUid { + KvUid::new( + UidValue::try_from(id).expect("should build"), + AgentType::try_from(1).expect("should build"), + None, + Some(ProviderId::new("p")), + expiry, + ) + } + + fn row(uids: Vec) -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + let record = KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), uids)], + UnixSeconds::new(0), + ) + .expect("should build record"); + entry.ids.insert("ids.example.com".to_owned(), record); + entry + } + + fn granting() -> ConsentContext { + ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + } + } + + fn decode(value: &MirrorValue) -> Value { + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(value.as_str()) + .expect("should decode"); + serde_json::from_slice(&bytes).expect("should parse") + } + + #[test] + fn mirrors_live_provider_uids_with_exp() { + let entry = row(vec![ + uid("live", Expiry::At(UnixSeconds::new(2000))), + uid("old", Expiry::At(UnixSeconds::new(10))), + ]); + + let MirrorCookie::Set(value) = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ) else { + panic!("should set the mirror"); + }; + + assert_eq!( + decode(&value), + json!({ "v": 1, "eids": [{ "source": "ids.example.com", "uids": [{ "id": "live", "atype": 1, "exp": 2000 }] }] }), + "should hold only the live UID with its exp" + ); + } + + #[test] + fn expires_without_consent_bidstream_or_uids() { + let entry = row(vec![uid("live", Expiry::Never)]); + let now = UnixSeconds::new(1000); + + assert!( + matches!( + build_mirror( + Some(&entry), + &ConsentContext::default(), + &partners(true), + now + ), + MirrorCookie::Expire + ), + "should expire without consent" + ); + assert!( + matches!( + build_mirror(Some(&entry), &granting(), &partners(false), now), + MirrorCookie::Expire + ), + "should expire for non-bidstream sources" + ); + assert!( + matches!( + build_mirror(None, &granting(), &partners(true), now), + MirrorCookie::Expire + ), + "should expire without a row" + ); + } + + #[test] + fn skips_uids_that_do_not_fit() { + let big: Vec = (0..20) + .map(|i| uid(&format!("{i:02}{}", "x".repeat(300)), Expiry::Never)) + .collect(); + let entry = row(big); + + let MirrorCookie::Set(value) = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ) else { + panic!("should set the mirror"); + }; + + assert!( + value.as_str().len() <= MAX_MIRROR_BYTES, + "should fit the size limit" + ); + let kept = decode(&value)["eids"][0]["uids"] + .as_array() + .expect("should hold uids") + .len(); + assert!(kept > 0 && kept < 20, "should keep a prefix that fits"); + } +} diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index b426e869b9..f079449890 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -33,6 +33,7 @@ //! - [`write_plan`] — Plans a navigation's single row change //! - [`provider`] — Identity provider interface and configuration //! - [`coordinator`] — Server-side identity lookup coordination +//! - [`mirror`] — JS-readable mirror of provider-owned bidstream UIDs mod auth; @@ -51,6 +52,7 @@ pub mod identify; pub mod kv; pub mod kv_backend; pub mod kv_types; +pub mod mirror; pub mod page_load; pub mod partner; pub mod prebid_eids; @@ -82,7 +84,7 @@ use error_stack::Report; use http::Request; use crate::consent::{self as consent_mod, ConsentContext, ConsentPipelineInput}; -use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_PULL_COMPLETE}; +use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_PULL_COMPLETE, COOKIE_TS_EIDS_MIRROR}; use crate::cookies::handle_request_cookies; use crate::ec::cookies::ec_id_has_only_allowed_chars; use crate::error::TrustedServerError; @@ -194,6 +196,8 @@ struct RequestEc { cookie_ec: Option, /// Whether the retired pull-sync completeness marker is present. pull_marker_present: bool, + /// Whether the mirror cookie is present. + mirror_cookie_present: bool, /// The parsed cookie jar (retained for consent pipeline input). jar: Option, } @@ -213,10 +217,14 @@ fn parse_ec_from_request(req: &Request) -> Result>, /// Whether the request carried the retired pull-sync marker cookie. pull_marker_present: bool, + /// Whether the request carried the mirror cookie. + mirror_cookie_present: bool, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, /// Navigation permit issued by the router for page-load navigations. @@ -402,6 +412,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: parsed.pull_marker_present, + mirror_cookie_present: parsed.mirror_cookie_present, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -597,6 +608,18 @@ impl EcContext { self.pull_marker_present } + /// Whether the request carried the mirror cookie. + #[must_use] + pub(crate) fn mirror_cookie_present(&self) -> bool { + self.mirror_cookie_present + } + + /// Marks the mirror cookie as present, for tests. + #[cfg(test)] + pub(crate) fn set_mirror_cookie_present_for_test(&mut self, present: bool) { + self.mirror_cookie_present = present; + } + /// Marks the retired pull-sync marker cookie as present, for tests. #[cfg(test)] pub(crate) fn set_pull_marker_present_for_test(&mut self, present: bool) { @@ -707,6 +730,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, + mirror_cookie_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -736,6 +760,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, + mirror_cookie_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -768,6 +793,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, + mirror_cookie_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 1fbd8f3cb7..afb7b4082e 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -64,6 +64,8 @@ pub struct PartnerConfig { pub struct PartnerRegistry { by_source_domain: HashMap, by_api_key_hash: HashMap, + /// Normalized source domains in `[[ec.partners]]` order. + order: Vec, } impl PartnerRegistry { @@ -149,6 +151,7 @@ impl PartnerRegistry { pub fn from_config(partners: &[EcPartner]) -> Result> { let mut by_source_domain = HashMap::with_capacity(partners.len()); let mut by_api_key_hash = HashMap::with_capacity(partners.len()); + let mut order = Vec::with_capacity(partners.len()); for partner in partners { let normalized_source = normalize_partner_source_domain(&partner.source_domain) @@ -202,12 +205,14 @@ impl PartnerRegistry { if let Some(api_key_hash) = api_key_hash { by_api_key_hash.insert(api_key_hash, normalized_source.clone()); } + order.push(normalized_source.clone()); by_source_domain.insert(normalized_source, config); } Ok(Self { by_source_domain, by_api_key_hash, + order, }) } @@ -269,6 +274,7 @@ impl PartnerRegistry { Self { by_source_domain: HashMap::new(), by_api_key_hash: HashMap::new(), + order: Vec::new(), } } @@ -324,6 +330,13 @@ impl PartnerRegistry { .collect() } + /// Partners in `[[ec.partners]]` order. + pub fn in_config_order(&self) -> impl Iterator { + self.order + .iter() + .filter_map(|source| self.by_source_domain.get(source)) + } + /// Returns an iterator over all configured partners. /// /// Iteration order is unspecified; callers that need determinism should @@ -587,6 +600,27 @@ mod tests { ); } + #[test] + fn in_config_order_preserves_partner_order() { + let partners = vec![ + make_partner("c.example.com", &valid_api_token("token-c")), + make_partner("a.example.com", &valid_api_token("token-a")), + make_partner("b.example.com", &valid_api_token("token-b")), + ]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + + let order: Vec<&str> = registry + .in_config_order() + .map(|partner| partner.source_domain.as_str()) + .collect(); + + assert_eq!( + order, + ["c.example.com", "a.example.com", "b.example.com"], + "should keep config order" + ); + } + #[test] fn empty_config_builds_empty_registry() { let registry = PartnerRegistry::from_config(&[]).expect("should build empty registry"); diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 527268a57f..573155516b 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -65,7 +65,6 @@ sequenceDiagram TS-->>B: Response
(no cookie refresh) end - Note over TS,KV: Phase 4: Post-send (background)
Dispatch pull-sync to partners ``` ### Response Finalization @@ -202,7 +201,7 @@ sequenceDiagram participant DSP as Downstream Partners / DSPs alt Pull sync seeds partner UID - TS->>DSP: Background pull sync request
(EC ID + consent context) + TS->>DSP: Pull sync lookup on a page navigation
(EC ID + consent context) DSP-->>TS: Partner UID for EC TS->>KV: Upsert ids[source_domain] = UID else Prebid sync seeds browser EIDs @@ -264,15 +263,22 @@ The relevant OpenRTB structure forwarded to Prebid Server and downstream partner Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. Browser-supplied EIDs, from the `/auction` request body, the `ts-eids` cookie, or the `sharedId` cookie, only fill partner UIDs that are missing from KV; a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). -### Pull-Sync Completeness Marker +### Pull Sync on Navigations -When the identity graph contains a UID for every pull-enabled partner, Trusted Server sets a signed, host-only `ts-ec-pull-complete` cookie. The cookie contains no partner UID or EC ID. It authenticates a one-hour expiration and a fingerprint of the current pull-partner source-domain set, bound to the active EC ID with key material derived from `ec.passphrase`. +Pull sync runs as an identity provider during top-level page navigations, within the shared identity lookup budget, and its result lands in the navigation's single KV write. A pull-sync partner's source is provider-owned: browser EID ingestion skips it and batch sync rejects its mappings with `provider_owned`. -A valid marker avoids a KV lookup only when pull-sync completeness is the sole reason to inspect the row. Auctions that need stored EIDs, browser EID-cookie ingestion, explicit withdrawal, and generation continue to use KV. The marker acts as recent proof that the row existed, so deletion of a previously complete row is not detected until the marker expires, at most one hour after issuance. Partner-set changes, passphrase rotation, malformed values, and expiration invalidate the marker and restore the normal lookup and orphan-recovery path. +The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. -Pull sync runs after response delivery, so a partner response that fills the last missing UID cannot set the marker on that already-sent response. A later eligible request verifies the completed row and issues the marker. Explicit withdrawal expires both `ts-ec` and any present `ts-ec-pull-complete` marker even when KV is unavailable. +### EID Mirror Cookie -Issuing or expiring the marker adds `Set-Cookie` to the outgoing response. Cache-privacy handling makes an otherwise shareable response private when that happens. A valid marker is not refreshed on each request, so this cost is limited to responses that establish or clear marker state in exchange for avoiding later KV reads. +Trusted Server mirrors provider-written UIDs for `bidstream_enabled` partners into a JavaScript-readable cookie so client-side code can use them: + +- **Name**: `__Host-ts-eids-mirror` +- **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` +- **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out +- **Updates**: every page navigation rebuilds it from the row after that navigation's write, and expires it when no live provider UID qualifies. Without EC and EID consent it is expired on navigations and on any response whose request carried it. + +Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. ## Configuration From de1877bd20c073fe0650f351a8fc315259853462 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 13:07:08 +0530 Subject: [PATCH 03/12] Make mirroring best effort and fix pull sync validation Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 16 +- crates/trusted-server-core/src/config.rs | 43 ++- .../src/ec/coordinator/mod.rs | 222 ++++++++++++---- .../src/ec/coordinator/results.rs | 4 + crates/trusted-server-core/src/ec/finalize.rs | 107 +++++++- crates/trusted-server-core/src/ec/mirror.rs | 244 +++++++++++++++--- crates/trusted-server-core/src/ec/provider.rs | 3 +- crates/trusted-server-core/src/settings.rs | 42 ++- .../src/lib.rs | 6 +- docs/guide/configuration.md | 2 +- docs/guide/edge-cookies.md | 3 +- 11 files changed, 575 insertions(+), 117 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 3faa735398..647a69e721 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -363,8 +363,6 @@ pub(crate) struct EcFinalizeState { pub(crate) use_finalize_kv: bool, pub(crate) eids_cookie: Option, pub(crate) sharedid_cookie: Option, - #[cfg_attr(not(test), allow(dead_code))] - pub(crate) is_real_browser: bool, /// Per-request services carried to the entry point so the pull-sync /// dispatcher can reuse the same platform HTTP client. pub(crate) services: RuntimeServices, @@ -566,7 +564,6 @@ impl EcRequestState { use_finalize_kv: self.finalize_kv_graph.is_some(), eids_cookie: self.eids_cookie, sharedid_cookie: self.sharedid_cookie, - is_real_browser: self.is_real_browser, services: self.services, } } @@ -942,8 +939,6 @@ async fn run_named_route( /// graph + partner registry + rate limiter, with a default EC context for /// response finalization. fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> Response { - let device_signals = device_signals_for(&req); - let is_real_browser = device_signals.looks_like_browser(); let eids_cookie = crate::extract_cookie_value(&req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); @@ -961,7 +956,6 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> use_finalize_kv: false, eids_cookie, sharedid_cookie, - is_real_browser, services: services.clone(), }); response @@ -3213,7 +3207,10 @@ mod tests { .get::() .expect("fallback response should carry EcFinalizeState"); assert!( - finalize.is_real_browser, + finalize + .ec_context + .device_signals() + .is_some_and(DeviceSignals::looks_like_browser), "browser-shaped JA4/H2 signals from the request extension must mark the request as a real browser" ); } @@ -3232,7 +3229,10 @@ mod tests { .get::() .expect("fallback response should carry EcFinalizeState"); assert!( - !finalize.is_real_browser, + !finalize + .ec_context + .device_signals() + .is_some_and(DeviceSignals::looks_like_browser), "a request without captured device signals should not be classified as a real browser" ); } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 44a592f1f3..b63ef1a0a1 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -363,7 +363,24 @@ fn validate_identity_providers(ec: &Ec) -> Result<(), Report report.change_context(TrustedServerError::Configuration { message: format!("ec.identity_providers: {detail}"), }) - }) + })?; + // Pull-sync partners run as providers too; a source claimed by both would + // fail the coordinator build on every request. + for config in ec.identity_providers.iter().filter(|config| config.enabled) { + if let Some(source) = config.sources.iter().find(|source| { + partners + .find_by_source_domain(source) + .is_some_and(|partner| partner.pull_sync_enabled && partner.pull_sync_url.is_some()) + }) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.identity_providers: source `{source}` of provider `{}` is also claimed by a pull-sync partner", + config.id + ), + })); + } + } + Ok(()) } fn validate_js_asset_proxy_config(settings: &Settings) -> Result<(), Report> { @@ -1399,6 +1416,30 @@ gam_network_id = "99999" ); } + #[test] + fn runtime_validation_rejects_source_claimed_by_provider_and_pull_sync() { + let mut settings = settings_with_identity_provider(serde_json::json!({})); + settings.ec.partners = vec![ + serde_json::from_value(serde_json::json!({ + "name": "ids", + "source_domain": "ids.example.com", + "pull_sync_enabled": true, + "pull_sync_url": "https://sync.example.com/pull", + "pull_sync_allowed_domains": ["sync.example.com"], + "ts_pull_token": "example-pull-token-32-bytes-minimum", + })) + .expect("should build partner"), + ]; + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject a source claimed twice"); + + assert!( + err.to_string().contains("pull-sync partner"), + "should explain the double claim: {err:?}" + ); + } + #[test] fn runtime_validation_rejects_provider_source_without_partner() { let settings = settings_with_identity_provider(serde_json::json!({ diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 3afd653bbe..7a54fe883b 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -9,7 +9,7 @@ pub mod transport; mod validate; use std::collections::HashSet; -use std::num::{NonZeroU16, NonZeroU32}; +use std::num::{NonZeroU16, NonZeroU32, NonZeroUsize}; use std::sync::Arc; use std::time::Duration; @@ -25,8 +25,7 @@ use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; -use crate::ec::kv_types::KvEntry; -use crate::ec::mirror::{MirrorCookie, build_mirror}; +use crate::ec::kv_types::{KvEid, KvEntry}; use crate::ec::page_load::RowChange; use crate::ec::provider::{ ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, @@ -34,7 +33,7 @@ use crate::ec::provider::{ validate_enabled_configs, }; use crate::ec::rate_limiter::RateLimiter; -use crate::ec::registry::PartnerRegistry; +use crate::ec::registry::{PULL_SYNC_KIND, PartnerRegistry}; use crate::ec::write_plan::{WritePlanInput, plan_write}; use crate::settings::IdentityProviderConfig; @@ -71,15 +70,6 @@ pub trait IdentityCoordinator { /// /// Returns `None` when the row would not change, so no write is made. fn plan_write(&self, input: &WritePlanInput<'_>) -> Option; - - /// Builds the mirror cookie from the row as it stands after this - /// navigation: the committed row, or the snapshot when no write landed. - fn mirror( - &self, - row: Option<&KvEntry>, - consent: &ConsentContext, - now: UnixSeconds, - ) -> MirrorCookie; } /// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. @@ -193,6 +183,9 @@ struct ProviderPolicy { allowed_domains: Box<[Box]>, tcf_vendor_id: Option, rate_limit_per_hour: Option, + /// Whether this is a pull-sync partner, counted against + /// [`ProviderCoordinator::with_pull_sync_concurrency`]. + pull_sync: bool, } /// The production coordinator, built per request from settings. @@ -204,6 +197,8 @@ pub struct ProviderCoordinator

> { /// Partner config: `openrtb_atype` for validation. partners: Arc, budget: Duration, + /// Most pull-sync lookups dispatched per navigation; `None` is unlimited. + pull_sync_concurrency: Option, } impl ProviderCoordinator

{ @@ -245,9 +240,19 @@ impl ProviderCoordinator

{ providers, partners, budget, + pull_sync_concurrency: None, }) } + /// Caps the pull-sync lookups dispatched per navigation + /// (`ec.pull_sync_concurrency`). Due pull-sync partners past the cap are + /// skipped as rate limited and retried on a later navigation. + #[must_use] + pub fn with_pull_sync_concurrency(mut self, max: Option) -> Self { + self.pull_sync_concurrency = max; + self + } + fn register( registered: RegisteredProvider

, ) -> Result, Report> { @@ -280,6 +285,7 @@ impl ProviderCoordinator

{ .collect(), tcf_vendor_id: config.tcf_vendor_id, rate_limit_per_hour, + pull_sync: config.kind() == PULL_SYNC_KIND, }, provider, }) @@ -299,7 +305,8 @@ impl ProviderCoordinator

{ } /// Claimed sources lacking a live provider-written UID (a browser - /// fallback does not count), minus sources needing input V1 + /// fallback does not count; for pull sync, a live owned UID), minus + /// sources needing input V1 /// never has; the flag reports whether any source was removed for input. fn due_sources( registered: &Registered

, @@ -310,16 +317,31 @@ impl ProviderCoordinator

{ let mut input_blocked = false; let mut due = Vec::new(); for source in ®istered.policy.sources { - let record = entry - .and_then(|entry| entry.ids.get(source.as_str())) - .filter(|record| record.is_provider_written()); + let record = entry.and_then(|entry| entry.ids.get(source.as_str())); // A UID whose provider asked for a refresh makes its source due; // the stored refresh state is the lookup's input. - if record.is_some_and(|record| record.refresh_due_for(®istered.id, now)) { + if record + .filter(|record| record.is_provider_written()) + .is_some_and(|record| record.refresh_due_for(®istered.id, now)) + { due.push(source.clone()); continue; } - if record.is_some_and(|record| record.has_live_uid(now)) { + // Legacy pull sync wrote single-UID records with no owner; a + // pull-sync provider treats them as its own and refreshes them so + // they gain an owner and are mirrored. + let held = record.is_some_and(|record| { + if registered.policy.pull_sync { + record + .eids() + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.owner().is_some() && uid.is_live(now)) + } else { + record.is_provider_written() && record.has_live_uid(now) + } + }); + if held { continue; } let needs_input = capabilities @@ -491,23 +513,28 @@ impl IdentityCoordinator for ProviderCoordinator

{ if self.providers.is_empty() { return pending; } + // A failed read cannot rule out a withdrawal tombstone, so no provider + // is contacted. + let snapshot_failed = matches!(lookup.snapshot, EcKvSnapshot::Failed { .. }); let entry = lookup.snapshot.entry_for(lookup.ec_id); - // Missing EC or EID consent, a withdrawal tombstone, or an unreadable - // row (which may hide a tombstone) skips every provider: a lookup - // discloses the EC ID before finalize could decline it. - let readable = !matches!(lookup.snapshot, EcKvSnapshot::Failed { .. }); - let consented = readable - && ec_consent_granted(lookup.consent) + // Missing EC or EID consent, or a withdrawal tombstone, skips every + // provider: a lookup discloses the EC ID before finalize could decline it. + let consented = ec_consent_granted(lookup.consent) && allows_eid_persistence(lookup.consent) && entry.is_none_or(|entry| entry.consent.ok); let deadline = Deadline::after(self.budget); pending.deadline = Some(deadline); + let mut pull_sync_dispatched = 0_usize; for (index, registered) in self.providers.iter().enumerate() { let provider = ProviderIndex::new(index); let skip = |reason| Tagged { provider, value: LookupStatus::Skipped(reason), }; + if snapshot_failed { + pending.early.push(skip(SkipReason::SnapshotUnavailable)); + continue; + } let vendor_denied = registered .policy .tcf_vendor_id @@ -531,6 +558,14 @@ impl IdentityCoordinator for ProviderCoordinator

{ })); continue; } + if registered.policy.pull_sync + && self + .pull_sync_concurrency + .is_some_and(|max| pull_sync_dispatched >= max.get()) + { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } let input = LookupInput { ec_id: lookup.ec_id, consent: lookup.consent, @@ -578,6 +613,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ let context = request.context.take(); match transport.start(request, deadline).await { Ok(value) => { + pull_sync_dispatched += usize::from(registered.policy.pull_sync); pending.requests.push(Tagged { provider, value }); if let Some(value) = context { pending.contexts.push(Tagged { provider, value }); @@ -642,15 +678,6 @@ impl IdentityCoordinator for ProviderCoordinator

{ fn plan_write(&self, input: &WritePlanInput<'_>) -> Option { plan_write(input) } - - fn mirror( - &self, - row: Option<&KvEntry>, - consent: &ConsentContext, - now: UnixSeconds, - ) -> MirrorCookie { - build_mirror(row, consent, &self.partners, now) - } } fn log_outcome(lookup: &ProviderLookup) { @@ -1019,6 +1046,115 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + fn pull_sync_pair(max: Option) -> ProviderCoordinator { + let sources = ["a.example.com", "b.example.com"]; + let partners: Vec = sources + .iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, "source_domain": source + })) + .expect("should build partner") + }) + .collect(); + let registered = sources + .iter() + .map(|source| { + let config: IdentityProviderConfig = toml::from_str(&format!( + "id = \"pull-sync:{source}\"\nkind = \"{PULL_SYNC_KIND}\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"{source}\"]" + )) + .expect("should parse config"); + let mut provider = echo_provider(SourceInput::Anonymous); + provider.0.sources = Box::new([SourceCapability { + source: SourceDomain::try_from(*source).expect("should parse"), + input: SourceInput::Anonymous, + }]); + RegisteredProvider::new(config, provider) + }) + .collect(); + ProviderCoordinator::new( + registered, + Arc::new(PartnerRegistry::from_config(&partners).expect("should build registry")), + Duration::from_millis(200), + ) + .expect("should build coordinator") + .with_pull_sync_concurrency(max) + } + + #[test] + fn pull_sync_concurrency_caps_dispatched_lookups() { + let coordinator = pull_sync_pair(NonZeroUsize::new(1)); + let transport = CannedTransport::new(not_found_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!(transport.sent.borrow().len(), 1, "should send one lookup"); + let statuses: Vec = results.iter().map(|lookup| lookup.status).collect(); + assert_eq!( + statuses[1], + LookupStatus::Skipped(SkipReason::RateLimited), + "should skip the pull-sync lookup past the cap" + ); + } + + #[test] + fn pull_sync_refreshes_ownerless_legacy_uids() { + let coordinator = pull_sync_pair(None); + let transport = CannedTransport::new(not_found_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + for source in ["a.example.com", "b.example.com"] { + entry + .ids + .insert(source.to_owned(), KvPartnerId::single("legacy")); + } + + run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + assert_eq!( + transport.sent.borrow().len(), + 2, + "should look up sources holding only ownerless legacy UIDs" + ); + } + + #[test] + fn pull_sync_concurrency_none_is_unlimited() { + let coordinator = pull_sync_pair(None); + let transport = CannedTransport::new(not_found_reply); + + run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!(transport.sent.borrow().len(), 2, "should send both lookups"); + } + + #[test] + fn failed_snapshot_skips_every_provider() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &EcKvSnapshot::Failed { + ec_id: EC_ID.to_owned(), + }, + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::SnapshotUnavailable), + "should skip when the snapshot read failed" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + #[test] fn tombstoned_row_skips_for_consent() { let coordinator = coordinator(""); @@ -1086,24 +1222,6 @@ mod tests { } } - #[test] - fn failed_snapshot_skips_every_provider() { - let coordinator = coordinator(""); - let transport = CannedTransport::new(ok_reply); - let snapshot = EcKvSnapshot::Failed { - ec_id: EC_ID.to_owned(), - }; - - let results = run(&coordinator, &transport, &snapshot, &granting()); - - assert_eq!( - only_status(&results), - LookupStatus::Skipped(SkipReason::Consent), - "should skip when the row could not be read" - ); - assert!(transport.sent.borrow().is_empty(), "should send nothing"); - } - #[test] fn tcf_vendor_id_does_not_gate_traffic_without_tcf() { let coordinator = coordinator("tcf_vendor_id = 9999"); diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index c8e6578f71..4ad5ce93bd 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -66,6 +66,9 @@ pub enum SkipReason { /// The vendor reports an opt-out; the provider's stored UIDs are /// removed. OptedOut, + /// The row snapshot read failed, so consent state (such as a withdrawal + /// tombstone) is unknown. + SnapshotUnavailable, } /// One outcome per provider per navigation. @@ -108,6 +111,7 @@ impl LookupStatus { Self::Skipped(SkipReason::NotDue) => "skipped_not_due", Self::Skipped(SkipReason::Input) => "skipped_input", Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", + Self::Skipped(SkipReason::SnapshotUnavailable) => "skipped_snapshot_unavailable", Self::Skipped(SkipReason::Destination) => "destination_rejected", Self::Skipped(SkipReason::OptedOut) => "skipped_opted_out", } diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 253cbc587c..97e701b70f 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -73,13 +73,17 @@ pub fn ec_finalize_response( expire_retired_pull_marker(ec_context, response); } -/// Sets or expires the mirror cookie (spec §10). +/// Sets or expires the mirror cookie (spec §10). Best effort. /// /// Navigations rebuild it from the row as it stands after the page-load /// write: the committed row is stored as the `Present` snapshot, so /// [`super::EcKvSnapshot::entry_for`] returns it, or the read snapshot when no -/// write landed. Without EC and EID consent it is expired on navigations and on any -/// response whose request carried it. +/// write landed. Without EC and EID consent it is expired on any response +/// whose request carried it. +/// +/// The cookie is only expired when the request carried it, so responses to +/// browsers without a mirror stay free of `Set-Cookie`. When the row could +/// not be read authoritatively (KV failure), the mirror is left untouched. fn apply_mirror( ec_context: &EcContext, registry: &PartnerRegistry, @@ -89,9 +93,10 @@ fn apply_mirror( let decision = if !mirror_consent(ec_context.consent()) { MirrorCookie::Expire } else if navigation { - let row = ec_context - .ec_value() - .and_then(|ec_id| ec_context.kv_snapshot().entry_for(ec_id)); + let Some(row) = mirror_row(ec_context) else { + log::debug!("Leaving the mirror cookie untouched: identity row is unavailable"); + return; + }; build_mirror( row, ec_context.consent(), @@ -109,7 +114,7 @@ fn apply_mirror( value.as_str() ), ), - MirrorCookie::Expire if ec_context.mirror_cookie_present() || navigation => { + MirrorCookie::Expire if ec_context.mirror_cookie_present() => { append_cookie( response, &format!("{COOKIE_TS_EIDS_MIRROR}=; Path=/; Secure; SameSite=Lax; Max-Age=0"), @@ -119,6 +124,26 @@ fn apply_mirror( } } +/// The row the mirror is built from. +/// +/// Returns `Some(None)` when there is authoritatively no row (no EC or a +/// missing row), and `None` when the row is unknown because the read failed +/// or never happened. +fn mirror_row(ec_context: &EcContext) -> Option> { + let Some(ec_id) = ec_context.ec_value() else { + return Some(None); + }; + let snapshot = ec_context.kv_snapshot(); + match snapshot { + EcKvSnapshot::Present { .. } | EcKvSnapshot::Missing { .. } + if snapshot.belongs_to(ec_id) => + { + Some(snapshot.entry_for(ec_id)) + } + _ => None, + } +} + /// Expires a leftover pull-sync completeness marker; the marker is retired. fn expire_retired_pull_marker(ec_context: &EcContext, response: &mut Response) { if ec_context.pull_marker_present() { @@ -2095,7 +2120,11 @@ mod tests { ) } - fn run_mirror_navigation(lookups: LookupResults) -> Response { + fn run_mirror_navigation( + lookups: LookupResults, + mirror_present: bool, + kv_read_failed: bool, + ) -> Response { let settings = create_test_settings(); let ec_id = sample_ec_id("mirr01"); let graph = KvIdentityGraph::in_memory("mirror_store"); @@ -2108,7 +2137,15 @@ mod tests { graph .create(&ec_id, &live) .expect("should seed the live row"); - let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + let snapshot = if kv_read_failed { + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + } + } else { + graph.load_snapshot(&ec_id) + }; + let mut ec_context = returning_user_context(&ec_id, snapshot, true); + ec_context.set_mirror_cookie_present_for_test(mirror_present); ec_context.set_lookup_results(lookups); let mut registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) .expect("should build registry"); @@ -2129,7 +2166,7 @@ mod tests { #[test] fn navigation_sets_mirror_from_committed_row() { - let response = run_mirror_navigation(provider_found("ids.example.com")); + let response = run_mirror_navigation(provider_found("ids.example.com"), false, false); let cookies = mirror_cookies(&response); assert_eq!(cookies.len(), 1, "should set one mirror cookie"); @@ -2145,7 +2182,7 @@ mod tests { #[test] fn navigation_without_provider_uids_expires_mirror() { - let response = run_mirror_navigation(LookupResults::default()); + let response = run_mirror_navigation(LookupResults::default(), true, false); let cookies = mirror_cookies(&response); assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); @@ -2155,6 +2192,54 @@ mod tests { ); } + #[test] + fn navigation_without_mirror_cookie_emits_no_expiry() { + let response = run_mirror_navigation(LookupResults::default(), false, false); + + assert!( + mirror_cookies(&response).is_empty(), + "should not expire a mirror the request did not carry" + ); + } + + #[test] + fn kv_read_failure_leaves_mirror_untouched() { + let response = run_mirror_navigation(provider_found("ids.example.com"), true, true); + + assert!( + mirror_cookies(&response).is_empty(), + "should leave the mirror untouched when the row could not be read" + ); + } + + #[test] + fn withdrawn_consent_without_mirror_cookie_emits_nothing() { + let settings = create_test_settings(); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = make_context_with_consent(None, None, false, false, consent); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + None, + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + mirror_cookies(&response).is_empty(), + "should not expire a mirror the request did not carry" + ); + } + #[test] fn withdrawn_consent_expires_mirror_on_any_response() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index 72d678e05c..e5537a01e0 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -1,7 +1,7 @@ //! The JS-readable mirror of provider-owned bidstream UIDs (spec §10). use base64::Engine as _; -use serde_json::{Map, Value, json}; +use serde_json::{Map, Value}; use crate::consent::ConsentContext; use crate::consent::allows_eid_persistence; @@ -39,14 +39,15 @@ impl MirrorValue { /// Holds live, provider-written UIDs of `bidstream_enabled` partners, in /// partner config order, as unpadded base64url JSON /// `{"v":1,"eids":[...]}`. UIDs that would push the value past -/// [`MAX_MIRROR_BYTES`] are skipped. Returns [`MirrorCookie::Expire`] when -/// consent does not allow the EC or EID persistence, the row is missing or withdrawn, -/// or nothing qualifies. +/// [`MAX_MIRROR_BYTES`] are skipped, as are UIDs that fail to serialize. +/// Returns [`MirrorCookie::Expire`] when consent does not allow the EC or EID +/// persistence, the row is missing or withdrawn, or nothing qualifies. /// /// # Performance /// -/// Re-encodes once per kept UID; the size cap bounds this to a few dozen -/// small encodes per navigation. +/// Linear in the row size: each EID header and UID is serialized once, the +/// encoded length is tracked arithmetically, and the value is base64-encoded +/// once at the end. #[must_use] pub fn build_mirror( row: Option<&KvEntry>, @@ -57,8 +58,7 @@ pub fn build_mirror( let Some(row) = row.filter(|row| row.consent.ok && mirror_consent(consent)) else { return MirrorCookie::Expire; }; - let mut eids: Vec = Vec::new(); - let mut encoded: Option = None; + let mut mirror = MirrorBuilder::default(); for partner in partners .in_config_order() .filter(|partner| partner.bidstream_enabled) @@ -66,50 +66,150 @@ pub fn build_mirror( let Some(record) = row.ids.get(&partner.source_domain) else { continue; }; + let source = partner.source_domain.as_str(); + // Ownerless UIDs of a pull-sync partner's source were written by + // legacy pull sync, so they belong to that partner's provider. + let pull_sync_owned = + partner.provider_owned && partner.pull_sync_enabled && partner.pull_sync_url.is_some(); + if record.eids().is_empty() { + if pull_sync_owned && !record.uid.is_empty() { + let header = legacy_header(source); + let uid = serde_json::to_string(&serde_json::json!({ + "id": record.uid, + "atype": partner.openrtb_atype, + })); + mirror.push_eid(source, header, [uid]); + } + continue; + } for eid in record.eids() { - let entry_index = eids.len(); - eids.push(eid_json(&partner.source_domain, eid)); - let mut kept = 0_usize; - for uid in eid + let uids = eid .uids() .iter() - .filter(|uid| uid.owner().is_some() && uid.is_live(now)) - { - uids_mut(&mut eids[entry_index]).push(uid_json(uid)); - if let Some(value) = encode(&eids) { - encoded = Some(value); - kept += 1; - } else { - uids_mut(&mut eids[entry_index]).pop(); - } + .filter(|uid| (pull_sync_owned || uid.owner().is_some()) && uid.is_live(now)) + .map(|uid| serde_json::to_string(&uid_json(uid))); + mirror.push_eid(source, eid_header(source, eid), uids); + } + } + let MirrorBuilder { + mut json, + eid_count, + } = mirror; + if eid_count == 0 { + return MirrorCookie::Expire; + } + json.push_str(JSON_SUFFIX); + let value = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json); + debug_assert!( + value.len() <= MAX_MIRROR_BYTES, + "should respect the size cap" + ); + MirrorCookie::Set(MirrorValue(value)) +} + +/// Accumulates the mirror JSON, tracking its encoded size so each part is +/// serialized once. +struct MirrorBuilder { + json: String, + eid_count: usize, +} + +impl Default for MirrorBuilder { + fn default() -> Self { + Self { + json: String::from(JSON_PREFIX), + eid_count: 0, + } + } +} + +impl MirrorBuilder { + /// Appends one EID with every UID that still fits; drops the EID when + /// none does. Parts that fail to serialize are skipped and logged. + fn push_eid( + &mut self, + source: &str, + header: Option, + uids: impl IntoIterator>, + ) { + let Some(header) = header else { + log::warn!("Skipping mirror EID for '{source}': header does not serialize"); + return; + }; + let eid_start = self.json.len(); + let mut kept = 0_usize; + for uid in uids { + let Ok(uid) = uid else { + log::warn!("Skipping mirror UID for '{source}': does not serialize"); + continue; + }; + let added = if kept == 0 { + usize::from(self.eid_count > 0) + header.len() + uid.len() + EID_SUFFIX.len() + } else { + 1 + uid.len() + }; + if encoded_len(self.json.len() + added + JSON_SUFFIX.len()) > MAX_MIRROR_BYTES { + continue; } if kept == 0 { - eids.pop(); + if self.eid_count > 0 { + self.json.push(','); + } + self.json.push_str(&header); + } else { + self.json.push(','); } + self.json.push_str(&uid); + kept += 1; + } + if kept > 0 { + self.json.push_str(EID_SUFFIX); + self.eid_count += 1; + } else { + self.json.truncate(eid_start); } - } - match encoded { - Some(value) if !eids.is_empty() => MirrorCookie::Set(MirrorValue(value)), - _ => MirrorCookie::Expire, } } -/// Whether consent allows the mirror: EC consent and EID persistence. -#[must_use] -pub(crate) fn mirror_consent(consent: &ConsentContext) -> bool { - ec_consent_granted(consent) && allows_eid_persistence(consent) +/// Header for a legacy single-UID record: the source only. +fn legacy_header(source: &str) -> Option { + let mut object = Map::new(); + object.insert("source".to_owned(), Value::from(source)); + object.insert("uids".to_owned(), Value::Array(Vec::new())); + open_uids_array(serde_json::to_string(&Value::Object(object)).ok()?) } -fn uids_mut(eid: &mut Value) -> &mut Vec { - eid["uids"] - .as_array_mut() - .expect("should hold a uids array") +const JSON_PREFIX: &str = r#"{"v":1,"eids":["#; +const JSON_SUFFIX: &str = "]}"; +const EID_SUFFIX: &str = "]}"; + +/// Unpadded base64 length of `bytes` input bytes. +fn encoded_len(bytes: usize) -> usize { + (bytes * 4).div_ceil(3) } -fn encode(eids: &[Value]) -> Option { - let json = serde_json::to_vec(&json!({ "v": 1, "eids": eids })).ok()?; - let value = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json); - (value.len() <= MAX_MIRROR_BYTES).then_some(value) +/// Serializes an EID without its UIDs, ending in an open `"uids":[`. +/// +/// `uids` sorts last among the EID's keys, so the empty array closes the +/// serialized object. +fn eid_header(source: &str, eid: &KvEid) -> Option { + open_uids_array(serde_json::to_string(&eid_json(source, eid)).ok()?) +} + +/// Turns a serialized EID ending in `"uids":[]}` into one ending in an open +/// `"uids":[`. +fn open_uids_array(mut header: String) -> Option { + let open = header.strip_suffix(EID_SUFFIX)?.len(); + header.ends_with(r#""uids":[]}"#).then(|| { + header.truncate(open); + header + }) +} + +/// Whether consent allows the mirror: EC consent and EID persistence. +#[must_use] +pub(crate) fn mirror_consent(consent: &ConsentContext) -> bool { + ec_consent_granted(consent) && allows_eid_persistence(consent) } fn eid_json(source: &str, eid: &KvEid) -> Value { @@ -150,6 +250,7 @@ mod tests { use super::*; use crate::ec::eid_types::{AgentType, EidFields, ProviderId, UidValue}; use crate::ec::kv_types::KvPartnerId; + use serde_json::json; fn partners(bidstream: bool) -> PartnerRegistry { let partner = serde_json::from_value(json!({ @@ -251,6 +352,73 @@ mod tests { ); } + #[test] + fn mirrors_legacy_uid_of_pull_sync_partner_only() { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + let pull_partner = serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true, + "pull_sync_enabled": true, "pull_sync_url": "https://sync.example.com/pull", + "pull_sync_allowed_domains": ["sync.example.com"], + "ts_pull_token": "example-pull-token-32-bytes-minimum" + })) + .expect("should build partner"); + let mut pull = + PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); + pull.mark_provider_owned("ids.example.com"); + let now = UnixSeconds::new(1000); + + let MirrorCookie::Set(value) = build_mirror(Some(&entry), &granting(), &pull, now) else { + panic!("should mirror the legacy pull-sync UID"); + }; + + assert_eq!( + decode(&value)["eids"][0]["uids"][0]["id"], + json!("legacy"), + "should attribute the ownerless UID to the pull-sync partner" + ); + assert_eq!( + build_mirror(Some(&entry), &granting(), &partners(true), now), + MirrorCookie::Expire, + "should not mirror ownerless UIDs of other partners" + ); + } + + #[test] + fn mirrors_several_eids_as_valid_json() { + let mut entry = row(vec![uid("one", Expiry::Never), uid("two", Expiry::Never)]); + let record = KvPartnerId::from_eids( + vec![ + KvEid::new(EidFields::default(), vec![uid("a", Expiry::Never)]), + KvEid::new(EidFields::default(), vec![uid("b", Expiry::Never)]), + ], + UnixSeconds::new(0), + ) + .expect("should build record"); + entry.ids.insert("ids.example.com".to_owned(), record); + + let MirrorCookie::Set(value) = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ) else { + panic!("should set the mirror"); + }; + + assert_eq!( + decode(&value), + json!({ "v": 1, "eids": [ + { "source": "ids.example.com", "uids": [{ "id": "a", "atype": 1 }] }, + { "source": "ids.example.com", "uids": [{ "id": "b", "atype": 1 }] } + ] }), + "should join EIDs into valid JSON" + ); + } + #[test] fn skips_uids_that_do_not_fit() { let big: Vec = (0..20) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index e0e50a9841..a13c6c8487 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -464,11 +464,12 @@ pub(crate) fn parse_sources( } /// Whether `url` is `https` and its host is one of `allowed_domains`, -/// compared ASCII case-insensitively. +/// compared ASCII case-insensitively, ignoring a trailing dot on the host. #[must_use] pub(crate) fn destination_allowed>(url: &Url, allowed_domains: &[S]) -> bool { url.scheme() == "https" && url.host_str().is_some_and(|host| { + let host = host.strip_suffix('.').unwrap_or(host); allowed_domains .iter() .any(|allowed| allowed.as_ref().eq_ignore_ascii_case(host)) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 4e0724e714..a0feee0027 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -700,8 +700,8 @@ pub struct Ec { #[serde(default)] pub ec_store: Option, - /// Maximum number of concurrent pull-sync requests; retained for config - /// compatibility; unused. + /// Maximum number of pull-sync lookups dispatched per navigation; 0 + /// means no limit. #[serde(default = "Ec::default_pull_sync_concurrency")] pub pull_sync_concurrency: usize, @@ -756,6 +756,19 @@ impl Ec { "replace-with-random-ec-passphrase", ]; + /// Normalizes every partner's `pull_sync_allowed_domains`: trimmed, + /// lowercased and without a trailing dot, matching how request hosts are + /// compared. + fn normalize_pull_sync_domains(&mut self) { + for domain in self + .partners + .iter_mut() + .flat_map(|partner| partner.pull_sync_allowed_domains.iter_mut()) + { + *domain = domain.trim().trim_end_matches('.').to_ascii_lowercase(); + } + } + /// Default maximum concurrent pull-sync requests. #[must_use] pub const fn default_pull_sync_concurrency() -> usize { @@ -3505,6 +3518,7 @@ impl Settings { self.image_optimizer.normalize()?; self.debug.auction_html_comment_options.normalize(); self.tinybird.normalize(); + self.ec.normalize_pull_sync_domains(); self.integrations .remove_legacy_static_secret_store_selectors(); self.consent.validate(); @@ -4365,6 +4379,30 @@ mod tests { use std::collections::BTreeSet; use std::sync::Arc; + #[test] + fn pull_sync_allowed_domains_are_normalized_at_load() { + let toml = format!( + r#"{} + [[ec.partners]] + name = "Pull partner" + source_domain = "partner.example.com" + pull_sync_enabled = true + pull_sync_url = "https://sync.example.com/pull" + pull_sync_allowed_domains = [" Sync.Example.COM. "] + ts_pull_token = "example-pull-token-32-bytes-minimum" + "#, + crate_test_settings_str() + ); + + let settings = Settings::from_toml(&toml).expect("should parse settings"); + + assert_eq!( + settings.ec.partners[0].pull_sync_allowed_domains, + vec!["sync.example.com".to_owned()], + "should trim, lowercase and drop the trailing dot" + ); + } + #[test] fn html_finalization_timeout_defaults_and_boundaries() { let parsed = Settings::from_toml(&crate_test_settings_str()) diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index a941f02a2a..266c4607dd 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -11,7 +11,7 @@ pub mod pull_sync; -use std::num::NonZeroU32; +use std::num::{NonZeroU32, NonZeroUsize}; use std::sync::Arc; use std::time::Duration; @@ -183,7 +183,9 @@ fn build_phase_coordinator( if registered.is_empty() { return Ok(None); } - ProviderCoordinator::new(registered, partners, budget).map(Some) + let pull_sync_concurrency = NonZeroUsize::new(settings.ec.pull_sync_concurrency); + ProviderCoordinator::new(registered, partners, budget) + .map(|coordinator| Some(coordinator.with_pull_sync_concurrency(pull_sync_concurrency))) } #[cfg(test)] diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 50fa54276f..b336b2c81e 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -753,7 +753,7 @@ be at least 32 bytes. Keep it stable to preserve EC identifier continuity. | ----------------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | | `passphrase` | String | Yes | Publisher passphrase used as HMAC key | | `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | -| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | +| `pull_sync_concurrency` | Integer | No | Maximum pull-sync lookups dispatched per navigation (0 = no limit) | | `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | | `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | | `partners` | Array | No | Static partner registry entries | diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 573155516b..432a062e2a 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -276,7 +276,8 @@ Trusted Server mirrors provider-written UIDs for `bidstream_enabled` partners in - **Name**: `__Host-ts-eids-mirror` - **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` - **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out -- **Updates**: every page navigation rebuilds it from the row after that navigation's write, and expires it when no live provider UID qualifies. Without EC and EID consent it is expired on navigations and on any response whose request carried it. +- **Contents source**: live UIDs written by an identity provider. UIDs that legacy pull sync stored without an owner count as the pull-sync partner's own. +- **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. From 42d5dd08004f50f3df2d943dfde8eafc788d751e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 3 Nov 2026 20:09:23 +0530 Subject: [PATCH 04/12] Stop mirroring ownerless UIDs and re-pulling held pull-sync sources The mirror cookie now holds only UIDs with a provider owner, so browser, batch and legacy pull-sync records are never mirrored, and its size budget counts the pending EID suffix for later UIDs. A pull-sync source with any live UID counts as held, as legacy pull sync did, so ownerless records no longer send the EC ID to the partner on every navigation. Config load warns when a pull-sync partner also sets a batch api_token, whose mappings batch sync rejects. Restore a test that an oversized pull-sync UID is dropped, and correct the guide on browser fallback ingest. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 82 +++++++++++++------ crates/trusted-server-core/src/ec/mirror.rs | 80 +++++++++--------- crates/trusted-server-core/src/ec/registry.rs | 48 ++++++++++- docs/guide/ec-setup-guide.md | 7 ++ docs/guide/edge-cookies.md | 4 +- 5 files changed, 151 insertions(+), 70 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 7a54fe883b..3413d2a778 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -25,7 +25,7 @@ use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; -use crate::ec::kv_types::{KvEid, KvEntry}; +use crate::ec::kv_types::KvEntry; use crate::ec::page_load::RowChange; use crate::ec::provider::{ ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, @@ -305,7 +305,7 @@ impl ProviderCoordinator

{ } /// Claimed sources lacking a live provider-written UID (a browser - /// fallback does not count; for pull sync, a live owned UID), minus + /// fallback does not count; for pull sync, any live UID counts), minus /// sources needing input V1 /// never has; the flag reports whether any source was removed for input. fn due_sources( @@ -327,19 +327,14 @@ impl ProviderCoordinator

{ due.push(source.clone()); continue; } - // Legacy pull sync wrote single-UID records with no owner; a - // pull-sync provider treats them as its own and refreshes them so - // they gain an owner and are mirrored. + // A pull-sync source counts any live UID, owned or not, as held: + // like legacy pull sync, it pulls only for sources with no UID, so + // an ownerless browser, batch or legacy record does not trigger a + // lookup (and an EC ID send) on every navigation. Sources with no + // UID stay bounded by the partner's hourly pull rate limit. let held = record.is_some_and(|record| { - if registered.policy.pull_sync { - record - .eids() - .iter() - .flat_map(KvEid::uids) - .any(|uid| uid.owner().is_some() && uid.is_live(now)) - } else { - record.is_provider_written() && record.has_live_uid(now) - } + (registered.policy.pull_sync || record.is_provider_written()) + && record.has_live_uid(now) }); if held { continue; @@ -1098,29 +1093,62 @@ mod tests { } #[test] - fn pull_sync_refreshes_ownerless_legacy_uids() { + fn pull_sync_holds_ownerless_uids_without_repulling() { let coordinator = pull_sync_pair(None); let transport = CannedTransport::new(not_found_reply); let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; - for source in ["a.example.com", "b.example.com"] { - entry - .ids - .insert(source.to_owned(), KvPartnerId::single("legacy")); - } + entry + .ids + .insert("a.example.com".to_owned(), KvPartnerId::single("legacy")); + let snapshot = seeded_snapshot(&entry); - run( - &coordinator, - &transport, - &seeded_snapshot(&entry), - &granting(), - ); + for _ in 0..2 { + run(&coordinator, &transport, &snapshot, &granting()); + } assert_eq!( transport.sent.borrow().len(), 2, - "should look up sources holding only ownerless legacy UIDs" + "should look up only the empty source, once per navigation, after a no-match" + ); + assert!( + transport + .sent + .borrow() + .iter() + .all(|url| url.ends_with("due=1")), + "should never include the ownerless source" + ); + } + + #[test] + fn pull_sync_drops_oversized_uid() { + let config: IdentityProviderConfig = toml::from_str(&format!( + "id = \"pull-sync:ids.example.com\"\nkind = \"{PULL_SYNC_KIND}\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]" + )) + .expect("should parse config"); + let coordinator = coordinator_with(RegisteredProvider::new( + config, + echo_provider(SourceInput::Anonymous), + )); + let transport = CannedTransport::new(|| { + Ok(TransportResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from("x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1)), + }) + }); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::NoMatch, + "should treat a pull with only an oversized UID as no match" ); + assert!(lookup.entries.is_empty(), "should store no oversized UID"); } #[test] diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index e5537a01e0..c41cb4d218 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -67,26 +67,14 @@ pub fn build_mirror( continue; }; let source = partner.source_domain.as_str(); - // Ownerless UIDs of a pull-sync partner's source were written by - // legacy pull sync, so they belong to that partner's provider. - let pull_sync_owned = - partner.provider_owned && partner.pull_sync_enabled && partner.pull_sync_url.is_some(); - if record.eids().is_empty() { - if pull_sync_owned && !record.uid.is_empty() { - let header = legacy_header(source); - let uid = serde_json::to_string(&serde_json::json!({ - "id": record.uid, - "atype": partner.openrtb_atype, - })); - mirror.push_eid(source, header, [uid]); - } - continue; - } + // Only provider-written UIDs carry an owner. Ownerless records come + // from browser ingest, batch sync or legacy pull sync, so they are + // never mirrored. for eid in record.eids() { let uids = eid .uids() .iter() - .filter(|uid| (pull_sync_owned || uid.owner().is_some()) && uid.is_live(now)) + .filter(|uid| uid.owner().is_some() && uid.is_live(now)) .map(|uid| serde_json::to_string(&uid_json(uid))); mirror.push_eid(source, eid_header(source, eid), uids); } @@ -146,7 +134,7 @@ impl MirrorBuilder { let added = if kept == 0 { usize::from(self.eid_count > 0) + header.len() + uid.len() + EID_SUFFIX.len() } else { - 1 + uid.len() + 1 + uid.len() + EID_SUFFIX.len() }; if encoded_len(self.json.len() + added + JSON_SUFFIX.len()) > MAX_MIRROR_BYTES { continue; @@ -171,14 +159,6 @@ impl MirrorBuilder { } } -/// Header for a legacy single-UID record: the source only. -fn legacy_header(source: &str) -> Option { - let mut object = Map::new(); - object.insert("source".to_owned(), Value::from(source)); - object.insert("uids".to_owned(), Value::Array(Vec::new())); - open_uids_array(serde_json::to_string(&Value::Object(object)).ok()?) -} - const JSON_PREFIX: &str = r#"{"v":1,"eids":["#; const JSON_SUFFIX: &str = "]}"; const EID_SUFFIX: &str = "]}"; @@ -353,12 +333,12 @@ mod tests { } #[test] - fn mirrors_legacy_uid_of_pull_sync_partner_only() { + fn does_not_mirror_browser_ingested_uid_of_pull_sync_source() { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; entry .ids - .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + .insert("ids.example.com".to_owned(), KvPartnerId::single("browser")); let pull_partner = serde_json::from_value(json!({ "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true, "pull_sync_enabled": true, "pull_sync_url": "https://sync.example.com/pull", @@ -369,21 +349,13 @@ mod tests { let mut pull = PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); pull.mark_provider_owned("ids.example.com"); - let now = UnixSeconds::new(1000); - let MirrorCookie::Set(value) = build_mirror(Some(&entry), &granting(), &pull, now) else { - panic!("should mirror the legacy pull-sync UID"); - }; + let mirror = build_mirror(Some(&entry), &granting(), &pull, UnixSeconds::new(1000)); assert_eq!( - decode(&value)["eids"][0]["uids"][0]["id"], - json!("legacy"), - "should attribute the ownerless UID to the pull-sync partner" - ); - assert_eq!( - build_mirror(Some(&entry), &granting(), &partners(true), now), + mirror, MirrorCookie::Expire, - "should not mirror ownerless UIDs of other partners" + "should not mirror an ownerless browser-ingested UID" ); } @@ -445,4 +417,36 @@ mod tests { .len(); assert!(kept > 0 && kept < 20, "should keep a prefix that fits"); } + + #[test] + fn several_uids_of_one_eid_respect_the_cap_at_every_boundary() { + // Three fixed UIDs fill most of the budget; growing the last UID one + // byte at a time crosses the cap at every possible offset. + for tail in 1..400 { + let mut uids: Vec = (0..3) + .map(|i| uid(&format!("{i}{}", "x".repeat(800)), Expiry::Never)) + .collect(); + uids.push(uid(&"y".repeat(tail), Expiry::Never)); + let entry = row(uids); + + let MirrorCookie::Set(value) = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ) else { + panic!("should set the mirror"); + }; + + assert!( + value.as_str().len() <= MAX_MIRROR_BYTES, + "should fit the size limit with tail {tail}" + ); + let kept = decode(&value)["eids"][0]["uids"] + .as_array() + .expect("should hold uids") + .len(); + assert!(kept >= 3, "should keep the fixed UIDs with tail {tail}"); + } + } } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index afb7b4082e..1e428c13d0 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -260,9 +260,18 @@ impl PartnerRegistry { } } let pull_sync_registered = is_registered_kind(PULL_SYNC_KIND); - for partner in ec.partners.iter().filter(|partner| { - pull_sync_registered && partner.pull_sync_enabled && partner.pull_sync_url.is_some() - }) { + for partner in ec + .partners + .iter() + .filter(|partner| pull_sync_registered && runs_pull_sync(partner)) + { + if batch_sync_rejected(partner) { + log::warn!( + "Partner '{}' runs pull sync and has a batch sync api_token; its source is \ + provider-owned, so batch sync rejects every mapping for it", + partner.source_domain + ); + } registry.mark_provider_owned(&partner.source_domain); } Ok(registry) @@ -358,6 +367,18 @@ impl PartnerRegistry { } } +/// Whether the partner runs pull sync as a provider, making its source +/// provider-owned. +fn runs_pull_sync(partner: &EcPartner) -> bool { + partner.pull_sync_enabled && partner.pull_sync_url.is_some() +} + +/// Whether a pull-sync partner also has a batch `api_token`, whose mappings +/// batch sync then rejects because the source is provider-owned. +fn batch_sync_rejected(partner: &EcPartner) -> bool { + runs_pull_sync(partner) && partner.api_token.is_some() +} + fn validate_api_token( source_domain: &str, api_token: &str, @@ -572,6 +593,27 @@ mod tests { ); } + #[test] + fn batch_token_on_pull_sync_partner_is_flagged() { + let mut pull = make_partner("pull.example.com", &valid_api_token("pull-partner")); + + assert!( + !batch_sync_rejected(&pull), + "should not flag a batch-only partner" + ); + pull.pull_sync_enabled = true; + pull.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); + assert!( + batch_sync_rejected(&pull), + "should flag a pull-sync partner with a batch token" + ); + pull.api_token = None; + assert!( + !batch_sync_rejected(&pull), + "should not flag a pull-sync partner without a batch token" + ); + } + #[test] fn from_ec_skips_providers_with_unregistered_kinds() { let mut ec = crate::test_support::tests::create_test_settings().ec; diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index cccb04daee..16207049c9 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -42,6 +42,13 @@ do not call either API may omit it. Provision high-entropy values under `ec_passphrase` and `partner_api_token`; see [Configuration](/guide/configuration#secret-store-migration). +A partner with pull sync enabled (`pull_sync_enabled` and `pull_sync_url`) +owns its source through the pull-sync provider, so batch sync rejects every +mapping for it with `provider_owned`. Trusted Server logs a warning at config +load when such a partner also sets `api_token`; use either pull sync or batch +sync for a partner, not both. See +[Pull Sync on Navigations](/guide/edge-cookies#pull-sync-on-navigations). + Required behavior assumptions: - The value stored under `ec_passphrase` is long-lived HMAC-SHA256 keying material for EC ID derivation; use a high-entropy random value of at least 32 characters diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 432a062e2a..3b16e3938a 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -265,7 +265,7 @@ Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source ### Pull Sync on Navigations -Pull sync runs as an identity provider during top-level page navigations, within the shared identity lookup budget, and its result lands in the navigation's single KV write. A pull-sync partner's source is provider-owned: browser EID ingestion skips it and batch sync rejects its mappings with `provider_owned`. +Pull sync runs as an identity provider during top-level page navigations, within the shared identity lookup budget, and its result lands in the navigation's single KV write. A pull-sync partner's source is provider-owned: browser EID ingestion stores a fallback UID for it only when the row holds no live UID for that source, and batch sync rejects its mappings with `provider_owned`. Trusted Server logs a warning at config load when a pull-sync partner also has a batch `api_token`, since that token's mappings are all rejected. A source that already holds any live UID, including an ownerless fallback, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. @@ -276,7 +276,7 @@ Trusted Server mirrors provider-written UIDs for `bidstream_enabled` partners in - **Name**: `__Host-ts-eids-mirror` - **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` - **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out -- **Contents source**: live UIDs written by an identity provider. UIDs that legacy pull sync stored without an owner count as the pull-sync partner's own. +- **Contents source**: live UIDs written by an identity provider. Ownerless UIDs, from browser ingest, batch sync or legacy pull sync, are never mirrored. - **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. From 96a248f995a2d0f45894c7985961ca27f55a0bd7 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 20:09:49 +0530 Subject: [PATCH 05/12] Keep batch sync for pull-sync partners Batch sync rejects only sources owned by a configured identity provider. A source owned only by its partner's pull sync accepts batch mappings again, as it did before pull sync became a provider. Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/batch_sync.rs | 71 ++++++++++++++++++- crates/trusted-server-core/src/ec/registry.rs | 67 +++++++++-------- docs/guide/configuration.md | 2 +- docs/guide/ec-setup-guide.md | 6 +- docs/guide/edge-cookies.md | 2 +- 5 files changed, 110 insertions(+), 38 deletions(-) diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 66a894dad8..48944c12c0 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -171,7 +171,7 @@ fn handle_batch_sync_with_writer( let (accepted, errors) = process_mappings_for_partner( writer, &partner.source_domain, - registry.is_provider_owned(&partner.source_domain), + registry.rejects_batch_sync(&partner.source_domain), &body.mappings, ); @@ -964,6 +964,75 @@ mod tests { ); } + #[test] + fn handle_batch_sync_accepts_mappings_for_a_pull_sync_owned_source() { + // Arrange: pull sync owns the source, but no configured provider does. + let mut registry = test_registry(); + assert!( + registry.mark_provider_owned("ssp.example.com"), + "should mark the pull-sync source" + ); + let limiter = MockRateLimiter { + should_exceed: false, + }; + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let writer = MockWriter::new(vec![Ok(UpsertResult::Written)]); + let body = + format!(r#"{{"mappings":[{{"ec_id":"{ec_id}","partner_uid":"one","timestamp":1}}]}}"#); + + // Act + let response = handle_batch_sync_with_writer( + &writer, + ®istry, + &limiter, + authorized_batch_request(&body), + ) + .expect("should return success response"); + + // Assert + assert_eq!( + response.status(), + StatusCode::OK, + "should accept batch mappings for a pull-sync partner" + ); + assert_eq!( + response_json(response)["accepted"], + 1, + "should accept the mapping" + ); + } + + #[test] + fn handle_batch_sync_rejects_mappings_for_a_configured_provider_source() { + let mut registry = test_registry(); + assert!( + registry.reject_batch_sync("ssp.example.com"), + "should mark the configured provider source" + ); + let limiter = MockRateLimiter { + should_exceed: false, + }; + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let writer = MockWriter::new(vec![Ok(UpsertResult::Written)]); + let body = + format!(r#"{{"mappings":[{{"ec_id":"{ec_id}","partner_uid":"one","timestamp":1}}]}}"#); + + let response = handle_batch_sync_with_writer( + &writer, + ®istry, + &limiter, + authorized_batch_request(&body), + ) + .expect("should return response"); + + let json = response_json(response); + assert_eq!(json["accepted"], 0, "should accept nothing"); + assert_eq!( + json["errors"][0]["reason"], REASON_PROVIDER_OWNED, + "should report provider_owned" + ); + } + #[test] fn handle_batch_sync_accepts_every_member_of_a_written_group() { let registry = test_registry(); diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 1e428c13d0..0cdc726549 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -51,8 +51,13 @@ pub struct PartnerConfig { /// Outbound bearer token for pull sync requests. pub ts_pull_token: Option>, /// Whether an enabled identity provider claims this source. Browser - /// ingest skips it and batch sync rejects it. + /// ingest skips it. pub provider_owned: bool, + /// Whether batch sync rejects this source's mappings: an enabled + /// configured provider owns it. A source owned only by its partner's pull + /// sync keeps accepting batch mappings, as it did before pull sync became + /// a provider. + pub batch_sync_rejected: bool, } /// In-memory partner registry with O(1) lookups by source domain and API key hash. @@ -257,6 +262,7 @@ impl PartnerRegistry { } for source in &provider.sources { registry.mark_provider_owned(source); + registry.reject_batch_sync(source); } } let pull_sync_registered = is_registered_kind(PULL_SYNC_KIND); @@ -265,13 +271,6 @@ impl PartnerRegistry { .iter() .filter(|partner| pull_sync_registered && runs_pull_sync(partner)) { - if batch_sync_rejected(partner) { - log::warn!( - "Partner '{}' runs pull sync and has a batch sync api_token; its source is \ - provider-owned, so batch sync rejects every mapping for it", - partner.source_domain - ); - } registry.mark_provider_owned(&partner.source_domain); } Ok(registry) @@ -330,6 +329,30 @@ impl PartnerRegistry { .is_some_and(|partner| partner.provider_owned) } + /// Marks `source` as owned by an enabled configured provider, so batch + /// sync rejects its mappings. Returns whether `source` is registered. + pub fn reject_batch_sync(&mut self, source: &str) -> bool { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return false; + }; + match self.by_source_domain.get_mut(&normalized) { + Some(partner) => { + partner.batch_sync_rejected = true; + true + } + None => false, + } + } + + /// Whether batch sync rejects mappings for `source` (spec §8.5): an + /// enabled configured provider owns it. Pull-sync ownership alone does + /// not reject batch mappings. + #[must_use] + pub fn rejects_batch_sync(&self, source: &str) -> bool { + self.find_by_source_domain(source) + .is_some_and(|partner| partner.batch_sync_rejected) + } + /// Returns all partners with `pull_sync_enabled = true`. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { @@ -373,12 +396,6 @@ fn runs_pull_sync(partner: &EcPartner) -> bool { partner.pull_sync_enabled && partner.pull_sync_url.is_some() } -/// Whether a pull-sync partner also has a batch `api_token`, whose mappings -/// batch sync then rejects because the source is provider-owned. -fn batch_sync_rejected(partner: &EcPartner) -> bool { - runs_pull_sync(partner) && partner.api_token.is_some() -} - fn validate_api_token( source_domain: &str, api_token: &str, @@ -421,6 +438,7 @@ fn build_partner_config( pull_sync_rate_limit: partner.pull_sync_rate_limit, ts_pull_token: partner.ts_pull_token.clone(), provider_owned: false, + batch_sync_rejected: false, } } @@ -591,26 +609,13 @@ mod tests { registry.is_provider_owned("pull.example.com"), "should mark pull-sync partner source" ); - } - - #[test] - fn batch_token_on_pull_sync_partner_is_flagged() { - let mut pull = make_partner("pull.example.com", &valid_api_token("pull-partner")); - - assert!( - !batch_sync_rejected(&pull), - "should not flag a batch-only partner" - ); - pull.pull_sync_enabled = true; - pull.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); assert!( - batch_sync_rejected(&pull), - "should flag a pull-sync partner with a batch token" + registry.rejects_batch_sync("ids.example.com"), + "should reject batch sync for a configured provider's source" ); - pull.api_token = None; assert!( - !batch_sync_rejected(&pull), - "should not flag a pull-sync partner without a batch token" + !registry.rejects_batch_sync("pull.example.com"), + "should keep batch sync for a source owned only by pull sync" ); } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index b336b2c81e..8886220e48 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -805,7 +805,7 @@ EID persistence, and, when `tcf_vendor_id` is set and TCF applies to the request, only with TCF vendor consent for that vendor. A provider owns its `sources`: its results replace whatever the row holds for -them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs +them, and batch sync rejects them. Browser-resolved IDs (`ts-eids`, `sharedId`) are never stored for an owned source, so a source a provider cleared on an opt-out is not refilled from the browser. diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index 16207049c9..1410e3deb3 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -43,10 +43,8 @@ do not call either API may omit it. Provision high-entropy values under [Configuration](/guide/configuration#secret-store-migration). A partner with pull sync enabled (`pull_sync_enabled` and `pull_sync_url`) -owns its source through the pull-sync provider, so batch sync rejects every -mapping for it with `provider_owned`. Trusted Server logs a warning at config -load when such a partner also sets `api_token`; use either pull sync or batch -sync for a partner, not both. See +owns its source through the pull-sync provider, so browser EID ingestion skips +it. Batch sync keeps accepting its mappings, so a partner may use both. See [Pull Sync on Navigations](/guide/edge-cookies#pull-sync-on-navigations). Required behavior assumptions: diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 3b16e3938a..ffd466aff7 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -265,7 +265,7 @@ Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source ### Pull Sync on Navigations -Pull sync runs as an identity provider during top-level page navigations, within the shared identity lookup budget, and its result lands in the navigation's single KV write. A pull-sync partner's source is provider-owned: browser EID ingestion stores a fallback UID for it only when the row holds no live UID for that source, and batch sync rejects its mappings with `provider_owned`. Trusted Server logs a warning at config load when a pull-sync partner also has a batch `api_token`, since that token's mappings are all rejected. A source that already holds any live UID, including an ownerless fallback, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. +Pull sync runs as an identity provider on top-level page navigations, after the response has been sent, as it did before it became a provider. It uses the after-response budget (`identity_after_response_budget_ms`), so a slow partner never delays the page, and its result reaches the row in the follow-up write and is served from the next page load. Like other after-response lookups, it never creates a row. A pull-sync partner's source is provider-owned, so browser EID ingestion skips it. Batch sync still accepts the partner's mappings, as before. A source that already holds any live UID, including a batch-synced one, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. From f0dc376046109cadc42d01134f0931477359e4c7 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:32:27 +0530 Subject: [PATCH 06/12] Set the mirror cookie only when its value changes The mirror was re-set on every consented navigation with a provider UID, and any Set-Cookie makes the response private and uncacheable, so returning users lost all caching of publisher HTML. The request's mirror value is now kept and the cookie is set only when the new value differs. An unchanged mirror is not re-extended; once it lapses the next navigation sets it again. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/finalize.rs | 53 +++++++++++++++---- crates/trusted-server-core/src/ec/mod.rs | 37 +++++++------ 2 files changed, 65 insertions(+), 25 deletions(-) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 97e701b70f..002f744914 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -81,9 +81,12 @@ pub fn ec_finalize_response( /// write landed. Without EC and EID consent it is expired on any response /// whose request carried it. /// -/// The cookie is only expired when the request carried it, so responses to -/// browsers without a mirror stay free of `Set-Cookie`. When the row could -/// not be read authoritatively (KV failure), the mirror is left untouched. +/// The cookie is only expired when the request carried it, and only set when +/// its value differs from the one the request carried, so responses that +/// leave the mirror as it was stay free of `Set-Cookie` and keep their +/// cacheability. An unchanged mirror is therefore not re-extended; once it +/// lapses, the next navigation sets it again. When the row could not be read +/// authoritatively (KV failure), the mirror is left untouched. fn apply_mirror( ec_context: &EcContext, registry: &PartnerRegistry, @@ -107,6 +110,7 @@ fn apply_mirror( return; }; match decision { + MirrorCookie::Set(value) if ec_context.mirror_cookie() == Some(value.as_str()) => {} MirrorCookie::Set(value) => append_cookie( response, &format!( @@ -2122,7 +2126,7 @@ mod tests { fn run_mirror_navigation( lookups: LookupResults, - mirror_present: bool, + mirror: Option<&str>, kv_read_failed: bool, ) -> Response { let settings = create_test_settings(); @@ -2145,7 +2149,7 @@ mod tests { graph.load_snapshot(&ec_id) }; let mut ec_context = returning_user_context(&ec_id, snapshot, true); - ec_context.set_mirror_cookie_present_for_test(mirror_present); + ec_context.set_mirror_cookie_for_test(mirror); ec_context.set_lookup_results(lookups); let mut registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) .expect("should build registry"); @@ -2166,7 +2170,7 @@ mod tests { #[test] fn navigation_sets_mirror_from_committed_row() { - let response = run_mirror_navigation(provider_found("ids.example.com"), false, false); + let response = run_mirror_navigation(provider_found("ids.example.com"), None, false); let cookies = mirror_cookies(&response); assert_eq!(cookies.len(), 1, "should set one mirror cookie"); @@ -2180,9 +2184,37 @@ mod tests { ); } + #[test] + fn navigation_with_an_unchanged_mirror_emits_no_cookie() { + // Arrange + let first = run_mirror_navigation(provider_found("ids.example.com"), None, false); + let set = mirror_cookies(&first); + let value = set[0] + .strip_prefix("__Host-ts-eids-mirror=") + .and_then(|rest| rest.split(';').next()) + .expect("should set a mirror value"); + + // Act + let unchanged = + run_mirror_navigation(provider_found("ids.example.com"), Some(value), false); + let changed = + run_mirror_navigation(provider_found("ids.example.com"), Some("stale"), false); + + // Assert + assert!( + mirror_cookies(&unchanged).is_empty(), + "should not re-set a mirror whose value is unchanged" + ); + assert_eq!( + mirror_cookies(&changed).len(), + 1, + "should set the mirror when its value changed" + ); + } + #[test] fn navigation_without_provider_uids_expires_mirror() { - let response = run_mirror_navigation(LookupResults::default(), true, false); + let response = run_mirror_navigation(LookupResults::default(), Some("stale"), false); let cookies = mirror_cookies(&response); assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); @@ -2194,7 +2226,7 @@ mod tests { #[test] fn navigation_without_mirror_cookie_emits_no_expiry() { - let response = run_mirror_navigation(LookupResults::default(), false, false); + let response = run_mirror_navigation(LookupResults::default(), None, false); assert!( mirror_cookies(&response).is_empty(), @@ -2204,7 +2236,8 @@ mod tests { #[test] fn kv_read_failure_leaves_mirror_untouched() { - let response = run_mirror_navigation(provider_found("ids.example.com"), true, true); + let response = + run_mirror_navigation(provider_found("ids.example.com"), Some("stale"), true); assert!( mirror_cookies(&response).is_empty(), @@ -2250,7 +2283,7 @@ mod tests { ..Default::default() }; let mut ec_context = make_context_with_consent(None, None, false, false, consent); - ec_context.set_mirror_cookie_present_for_test(true); + ec_context.set_mirror_cookie_for_test(Some("stale")); let mut response = empty_response(); ec_finalize_response( diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index f079449890..34209a3887 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -196,8 +196,8 @@ struct RequestEc { cookie_ec: Option, /// Whether the retired pull-sync completeness marker is present. pull_marker_present: bool, - /// Whether the mirror cookie is present. - mirror_cookie_present: bool, + /// The mirror cookie's value, when present. + mirror_cookie: Option, /// The parsed cookie jar (retained for consent pipeline input). jar: Option, } @@ -217,14 +217,15 @@ fn parse_ec_from_request(req: &Request) -> Result>, /// Whether the request carried the retired pull-sync marker cookie. pull_marker_present: bool, - /// Whether the request carried the mirror cookie. - mirror_cookie_present: bool, + /// The mirror cookie's value, when the request carried it. + mirror_cookie: Option, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, /// Navigation permit issued by the router for page-load navigations. @@ -412,7 +413,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: parsed.pull_marker_present, - mirror_cookie_present: parsed.mirror_cookie_present, + mirror_cookie: parsed.mirror_cookie, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -611,13 +612,19 @@ impl EcContext { /// Whether the request carried the mirror cookie. #[must_use] pub(crate) fn mirror_cookie_present(&self) -> bool { - self.mirror_cookie_present + self.mirror_cookie.is_some() + } + + /// The mirror cookie's value, when the request carried it. + #[must_use] + pub(crate) fn mirror_cookie(&self) -> Option<&str> { + self.mirror_cookie.as_deref() } - /// Marks the mirror cookie as present, for tests. + /// Sets the mirror cookie the request carried, for tests. #[cfg(test)] - pub(crate) fn set_mirror_cookie_present_for_test(&mut self, present: bool) { - self.mirror_cookie_present = present; + pub(crate) fn set_mirror_cookie_for_test(&mut self, value: Option<&str>) { + self.mirror_cookie = value.map(str::to_owned); } /// Marks the retired pull-sync marker cookie as present, for tests. @@ -730,7 +737,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, - mirror_cookie_present: false, + mirror_cookie: None, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -760,7 +767,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, - mirror_cookie_present: false, + mirror_cookie: None, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -793,7 +800,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, - mirror_cookie_present: false, + mirror_cookie: None, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), From 7850c890c84e25217ca33b885cb7308b284c818c Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:07:54 +0530 Subject: [PATCH 07/12] Skip identity lookups when the rate limiter cannot count Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 3413d2a778..c0686025f3 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -354,7 +354,7 @@ impl ProviderCoordinator

{ } /// Whether the provider's hourly per-EC limit is reached. Limiter errors - /// fail open, matching pull sync. + /// fail closed, as pull sync always has: an unknown count skips the lookup. fn rate_limited(registered: &Registered

, lookup: &LookupContext<'_>) -> bool { let Some(limit) = registered.policy.rate_limit_per_hour else { return false; @@ -365,10 +365,10 @@ impl ProviderCoordinator

{ Ok(exceeded) => exceeded, Err(report) => { log::warn!( - "Identity lookup: provider={} rate limiter failed, allowing lookup: {report:?}", + "Identity lookup: provider={} rate limiter failed, skipping lookup: {report:?}", registered.id ); - false + true } } } @@ -1365,6 +1365,45 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + struct FailingLimiter; + + impl RateLimiter for FailingLimiter { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Err(Report::new(TrustedServerError::KvStore { + store_name: "example-counter".to_owned(), + message: "unavailable".to_owned(), + })) + } + } + + #[test] + fn limiter_failure_skips_the_lookup() { + let coordinator = coordinator_with( + RegisteredProvider::new(provider_config(""), echo_provider(SourceInput::Anonymous)) + .with_rate_limit(NonZeroU32::new(1).expect("should be non-zero")), + ); + let transport = CannedTransport::new(ok_reply); + + let results = run_with( + &coordinator, + &transport, + &missing(), + &granting(), + &FailingLimiter, + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::RateLimited), + "should skip when the limiter cannot count" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + #[test] fn built_url_outside_allowlist_is_rejected() { let coordinator = coordinator(""); From 53e829fe1eab1f91a7dadc59ccb753fc32699445 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:09:03 +0530 Subject: [PATCH 08/12] Reject the pull-sync kind in identity provider config Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/config.rs | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index b63ef1a0a1..6a020c1451 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use validator::{Validate, ValidationError, ValidationErrors}; use crate::ec::provider::validate_enabled_configs; -use crate::ec::registry::PartnerRegistry; +use crate::ec::registry::{PULL_SYNC_KIND, PartnerRegistry}; use crate::error::TrustedServerError; use crate::integrations::{ adserver_mock::AdServerMockConfig, @@ -352,6 +352,20 @@ fn validate_identity_providers(ec: &Ec) -> Result<(), Report if ec.identity_providers.is_empty() { return Ok(()); } + // Pull-sync providers come only from partner config, which carries the + // pull token and rate limit; configuring the kind directly bypasses both. + if let Some(config) = ec + .identity_providers + .iter() + .find(|config| config.kind() == PULL_SYNC_KIND) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.identity_providers: provider `{}` uses kind `{PULL_SYNC_KIND}`; configure pull sync on its [[ec.partners]] entry instead", + config.id + ), + })); + } let partners = PartnerRegistry::from_ec(ec)?; validate_enabled_configs(&ec.identity_providers, &partners).map_err(|report| { let detail = report @@ -1440,6 +1454,21 @@ gam_network_id = "99999" ); } + #[test] + fn runtime_validation_rejects_a_directly_configured_pull_sync_provider() { + let settings = settings_with_identity_provider(serde_json::json!({ + "kind": PULL_SYNC_KIND, + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject the pull-sync kind in identity providers"); + + assert!( + err.to_string().contains("[[ec.partners]]"), + "should point to partner config: {err:?}" + ); + } + #[test] fn runtime_validation_rejects_provider_source_without_partner() { let settings = settings_with_identity_provider(serde_json::json!({ From c671688192ac9ca9d239da45ff4414e7d091c41a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:09:40 +0530 Subject: [PATCH 09/12] Refuse to key the template cache on the EID mirror cookie Signed-off-by: dhruv8sh --- .../src/cookies/template_cache_policy.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/cookies/template_cache_policy.rs b/crates/trusted-server-core/src/cookies/template_cache_policy.rs index 61c3b9139c..3997604534 100644 --- a/crates/trusted-server-core/src/cookies/template_cache_policy.rs +++ b/crates/trusted-server-core/src/cookies/template_cache_policy.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; -use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EC, COOKIE_TS_EIDS}; +use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_TS_EIDS_MIRROR}; use crate::platform::TemplateCookieValue; /// Whether the request can share a template and its explicit cookie dimensions. @@ -32,7 +32,13 @@ pub(crate) fn validate_cookie_names( return Err(format!("{field} contains invalid cookie name `{name}`")); } if field == "template_cache_key_cookies" - && [COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_SHAREDID].contains(&name.as_str()) + && [ + COOKIE_TS_EC, + COOKIE_TS_EIDS, + COOKIE_TS_EIDS_MIRROR, + COOKIE_SHAREDID, + ] + .contains(&name.as_str()) { return Err(format!( "{field} must not key on Trusted Server identity cookie `{name}`" @@ -232,7 +238,12 @@ mod tests { #[test] fn template_cookie_policy_rejects_identity_keys_but_allows_bypass() { - for name in [COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_SHAREDID] { + for name in [ + COOKIE_TS_EC, + COOKIE_TS_EIDS, + COOKIE_TS_EIDS_MIRROR, + COOKIE_SHAREDID, + ] { let error = validate_cookie_names(&names(&[name]), &[]) .expect_err("should reject identity cookie keys"); assert!( From 62f83c3350ad176669fc967a7bce2cdd738d41d8 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:11:02 +0530 Subject: [PATCH 10/12] Rotate pull-sync partners hourly so every partner gets a turn under the cap Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index c0686025f3..4d9d5c8b01 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -244,6 +244,22 @@ impl ProviderCoordinator

{ }) } + /// The order providers are dispatched in: configured providers first, + /// then pull-sync partners rotated by the hour, so that under + /// `pull_sync_concurrency` every partner gets a turn, as pull sync did + /// before it became a provider. + fn dispatch_order(&self, now: UnixSeconds) -> Vec { + let (mut order, mut pull_sync): (Vec, Vec) = + (0..self.providers.len()).partition(|&index| !self.providers[index].policy.pull_sync); + if !pull_sync.is_empty() { + let hour = usize::try_from(now.get() / 3600).unwrap_or(0); + let offset = hour % pull_sync.len(); + pull_sync.rotate_left(offset); + } + order.append(&mut pull_sync); + order + } + /// Caps the pull-sync lookups dispatched per navigation /// (`ec.pull_sync_concurrency`). Due pull-sync partners past the cap are /// skipped as rate limited and retried on a later navigation. @@ -520,7 +536,8 @@ impl IdentityCoordinator for ProviderCoordinator

{ let deadline = Deadline::after(self.budget); pending.deadline = Some(deadline); let mut pull_sync_dispatched = 0_usize; - for (index, registered) in self.providers.iter().enumerate() { + for index in self.dispatch_order(lookup.now) { + let registered = &self.providers[index]; let provider = ProviderIndex::new(index); let skip = |reason| Tagged { provider, @@ -1092,6 +1109,23 @@ mod tests { ); } + #[test] + fn pull_sync_rotates_the_partner_past_the_cap_by_the_hour() { + let coordinator = pull_sync_pair(NonZeroUsize::new(1)); + + let first_hour = coordinator.dispatch_order(UnixSeconds::new(0)); + let next_hour = coordinator.dispatch_order(UnixSeconds::new(3600)); + + assert_ne!( + first_hour.first(), + next_hour.first(), + "should give the other partner the first slot the next hour" + ); + let mut sorted = next_hour.clone(); + sorted.sort_unstable(); + assert_eq!(sorted, vec![0, 1], "should still dispatch every provider"); + } + #[test] fn pull_sync_holds_ownerless_uids_without_repulling() { let coordinator = pull_sync_pair(None); From 39309ffc43465dc95980f92a1ce6e25c607daf37 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:11:02 +0530 Subject: [PATCH 11/12] Document pull-sync rate limit rounding, the per-navigation cap and mirror exposure Signed-off-by: dhruv8sh --- docs/guide/edge-cookies.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index ffd466aff7..1698960da8 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -265,7 +265,7 @@ Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source ### Pull Sync on Navigations -Pull sync runs as an identity provider on top-level page navigations, after the response has been sent, as it did before it became a provider. It uses the after-response budget (`identity_after_response_budget_ms`), so a slow partner never delays the page, and its result reaches the row in the follow-up write and is served from the next page load. Like other after-response lookups, it never creates a row. A pull-sync partner's source is provider-owned, so browser EID ingestion skips it. Batch sync still accepts the partner's mappings, as before. A source that already holds any live UID, including a batch-synced one, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. +Pull sync runs as an identity provider on top-level page navigations, after the response has been sent, as it did before it became a provider. It uses the after-response budget (`identity_after_response_budget_ms`), so a slow partner never delays the page, and its result reaches the row in the follow-up write and is served from the next page load. Like other after-response lookups, it never creates a row. A pull-sync partner's source is provider-owned, so browser EID ingestion skips it. Batch sync still accepts the partner's mappings, as before. A source that already holds any live UID, including a batch-synced one, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. On Fastly that limit is enforced as a per-minute budget of at least one, so values below 60 behave like 60 per hour. At most `pull_sync_concurrency` partners are pulled per navigation (0 means no limit), unlike before, when every eligible partner was pulled in batches of that size. The partner order rotates every hour, so partners past the cap still get turns. The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. @@ -277,6 +277,7 @@ Trusted Server mirrors provider-written UIDs for `bidstream_enabled` partners in - **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` - **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out - **Contents source**: live UIDs written by an identity provider. Ownerless UIDs, from browser ingest, batch sync or legacy pull sync, are never mirrored. +- **Pull-sync UIDs**: pull sync now writes provider-owned UIDs, so a `bidstream_enabled` pull-sync partner's UIDs are mirrored. Before, they stayed in KV only; now any script on the page can read them, and the cookie is sent on every same-origin request. Set `bidstream_enabled = false` on a partner whose UIDs must not reach the page. - **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. From 9f1e764a19d2c7970dcc2a76f367c1710160780d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:12:12 +0530 Subject: [PATCH 12/12] Test mirroring of pull-sync results, non-navigations and withdrawn rows Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/finalize.rs | 47 ++++++++++++++++++- crates/trusted-server-core/src/ec/mirror.rs | 18 +++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 002f744914..9c91a53ba0 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -2099,11 +2099,15 @@ mod tests { } fn provider_found(source: &str) -> LookupResults { + provider_found_by(source, "example-provider") + } + + fn provider_found_by(source: &str, owner: &str) -> LookupResults { let uid = KvUid::new( UidValue::try_from("provider-uid").expect("should build uid"), AgentType::try_from(1).expect("should build atype"), None, - Some(ProviderId::new("example-provider")), + Some(ProviderId::new(owner)), Expiry::Never, ); // Made for the EC ID `run_mirror_navigation` seeds. @@ -2128,6 +2132,15 @@ mod tests { lookups: LookupResults, mirror: Option<&str>, kv_read_failed: bool, + ) -> Response { + run_mirror_request(lookups, mirror, kv_read_failed, true) + } + + fn run_mirror_request( + lookups: LookupResults, + mirror: Option<&str>, + kv_read_failed: bool, + navigation: bool, ) -> Response { let settings = create_test_settings(); let ec_id = sample_ec_id("mirr01"); @@ -2148,7 +2161,7 @@ mod tests { } else { graph.load_snapshot(&ec_id) }; - let mut ec_context = returning_user_context(&ec_id, snapshot, true); + let mut ec_context = returning_user_context(&ec_id, snapshot, navigation); ec_context.set_mirror_cookie_for_test(mirror); ec_context.set_lookup_results(lookups); let mut registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) @@ -2184,6 +2197,36 @@ mod tests { ); } + #[test] + fn pull_sync_result_is_mirrored_with_its_owner() { + let response = run_mirror_navigation( + provider_found_by("ids.example.com", "pull-sync:ids.example.com"), + None, + false, + ); + + assert_eq!( + mirror_cookies(&response).len(), + 1, + "should mirror a provider-owned pull-sync UID" + ); + } + + #[test] + fn consented_non_navigation_leaves_the_mirror_untouched() { + let response = run_mirror_request( + provider_found("ids.example.com"), + Some("stale"), + false, + false, + ); + + assert!( + mirror_cookies(&response).is_empty(), + "should neither set nor expire the mirror outside a navigation" + ); + } + #[test] fn navigation_with_an_unchanged_mirror_emits_no_cookie() { // Arrange diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index c41cb4d218..f2e9761538 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -359,6 +359,24 @@ mod tests { ); } + #[test] + fn tombstoned_row_expires_the_mirror_even_with_consent_on_the_request() { + let mut entry = row(vec![uid("live", Expiry::Never)]); + entry.consent.ok = false; + + let decision = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ); + + assert!( + matches!(decision, MirrorCookie::Expire), + "should expire the mirror for a withdrawn row" + ); + } + #[test] fn mirrors_several_eids_as_valid_json() { let mut entry = row(vec![uid("one", Expiry::Never), uid("two", Expiry::Never)]);