From 5b867588dd4b84154d64194dbf5cd5bc48711797 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 30 Oct 2026 07:03:15 +0530 Subject: [PATCH 01/38] Store and serve full-fidelity provider EIDs Add EID value types, an additive KV schema for provider-written EIDs with owner and expiry, EID-level fields on the OpenRTB Eid, and a resolver that serves only live UIDs to auctions, x-ts-eids, identify and the admin auction preview. Auction EIDs merge per (source, inserter, matcher, mm, ext) with first-occurrence UID dedupe. KV writes over 16 KiB are refused. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 246 +++++--- .../src/auction/formats.rs | 8 +- .../src/auction/test_support.rs | 8 +- crates/trusted-server-core/src/ec/admin.rs | 86 ++- .../trusted-server-core/src/ec/eid_types.rs | 594 ++++++++++++++++++ crates/trusted-server-core/src/ec/eids.rs | 213 ++++++- crates/trusted-server-core/src/ec/finalize.rs | 36 +- crates/trusted-server-core/src/ec/identify.rs | 111 +++- crates/trusted-server-core/src/ec/kv.rs | 119 ++-- crates/trusted-server-core/src/ec/kv_types.rs | 415 +++++++++++- crates/trusted-server-core/src/ec/mod.rs | 2 + .../trusted-server-core/src/ec/prebid_eids.rs | 37 +- .../trusted-server-core/src/ec/pull_sync.rs | 12 +- .../src/ec/pull_sync_marker.rs | 8 +- .../src/integrations/prebid.rs | 24 +- crates/trusted-server-core/src/openrtb.rs | 73 ++- 16 files changed, 1733 insertions(+), 259 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/eid_types.rs diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 086515fa20..b0121f7490 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -14,7 +14,8 @@ use crate::constants::COOKIE_TS_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; use crate::ec::EcKvSnapshot; -use crate::ec::eids::{resolve_partner_ids, to_eids}; +use crate::ec::current_timestamp; +use crate::ec::eid_types::UnixSeconds; use crate::ec::kv::KvIdentityGraph; use crate::ec::kv_types::MAX_UID_LENGTH; use crate::ec::prebid_eids::parse_prebid_eids_cookie; @@ -535,8 +536,8 @@ pub(crate) fn resolve_auction_eids( return Some(Vec::new()); } - let resolved = resolve_partner_ids(registry, entry); - Some(to_eids(&resolved)) + let now = UnixSeconds::new(current_timestamp()); + Some(entry.live_eids(registry, now).map(Eid::from).collect()) } pub(crate) fn resolve_client_auction_eids( @@ -600,7 +601,7 @@ fn parse_client_auction_eids(raw: Option<&JsonValue>) -> Option> { continue; } - eids.push(Eid { source, uids }); + eids.push(Eid::new(source, uids)); } if eids.is_empty() { None } else { Some(eids) } @@ -635,8 +636,9 @@ pub(crate) fn merge_auction_eids( client_eids: Option>, resolved_eids: Option>, ) -> Option> { - let mut merged = Vec::new(); + let mut merged: Vec = Vec::new(); + // Server-resolved entries come first so their fields win on duplicates. for eid in resolved_eids .into_iter() .flatten() @@ -645,50 +647,55 @@ pub(crate) fn merge_auction_eids( if eid.source.is_empty() { continue; } - - let source_index = match merged - .iter() - .position(|existing: &Eid| existing.source == eid.source) - { - Some(index) => index, - None => { - merged.push(Eid { - source: eid.source.clone(), - uids: Vec::new(), - }); - merged.len() - 1 - } - }; - - for uid in eid.uids { + let Eid { + source, + inserter, + matcher, + mm, + ext, + uids, + } = eid; + let mut kept = Vec::with_capacity(uids.len()); + for uid in uids { if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { continue; } - - if let Some(existing_uid) = merged[source_index] - .uids - .iter_mut() - .find(|existing| existing.id == uid.id) - { - if existing_uid.atype.is_none() { - existing_uid.atype = uid.atype; - } - if existing_uid.ext.is_none() { - existing_uid.ext = uid.ext; - } - } else { - merged[source_index].uids.push(uid); + let duplicate = merged + .iter() + .filter(|existing| existing.source == source) + .flat_map(|existing| existing.uids.iter()) + .chain(kept.iter()) + .any(|existing: &Uid| existing.id == uid.id); + if !duplicate { + kept.push(uid); } } + if kept.is_empty() { + continue; + } + // Entries are distinct when any EID-level field differs, including + // `ext` (compared by JSON value), so no entry's fields are lost. + let same_entry = merged.iter_mut().find(|existing| { + existing.source == source + && existing.inserter == inserter + && existing.matcher == matcher + && existing.mm == mm + && existing.ext == ext + }); + match same_entry { + Some(existing) => existing.uids.extend(kept), + None => merged.push(Eid { + source, + inserter, + matcher, + mm, + ext, + uids: kept, + }), + } } - merged.retain(|eid| !eid.uids.is_empty()); - - if merged.is_empty() { - None - } else { - Some(merged) - } + (!merged.is_empty()).then_some(merged) } #[cfg(test)] @@ -2058,33 +2065,132 @@ mod tests { ); } + #[test] + fn merge_auction_eids_keeps_distinct_eid_level_entries_per_source() { + let resolved = Some(vec![Eid { + inserter: Some("a.example.com".to_owned()), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + let client = Some(vec![ + Eid { + inserter: Some("b.example.com".to_owned()), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u2".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }, + Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(2), + ext: None, + }], + ), + ]); + + let merged = merge_auction_eids(client, resolved).expect("should merge"); + + assert_eq!( + merged.len(), + 2, + "should keep entries with different inserters" + ); + assert_eq!( + merged[0].uids.len(), + 1, + "should drop duplicate u1 from client" + ); + assert_eq!( + merged[0].uids[0].atype, + Some(1), + "should keep server fields for duplicate" + ); + assert_eq!( + merged[1].uids[0].id, "u2", + "should keep distinct client uid" + ); + } + + #[test] + fn merge_auction_eids_keeps_entries_with_different_ext_separate() { + let resolved = Some(vec![Eid { + ext: Some(json!({ "k": "server" })), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + let client = Some(vec![Eid { + ext: Some(json!({ "k": "client" })), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u2".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + + let merged = merge_auction_eids(client, resolved).expect("should merge"); + + assert_eq!(merged.len(), 2, "should keep entries with different ext"); + assert_eq!( + merged[0].ext, + Some(json!({ "k": "server" })), + "should keep server entry ext" + ); + assert_eq!( + merged[1].ext, + Some(json!({ "k": "client" })), + "should keep client entry ext" + ); + assert_eq!(merged[1].uids[0].id, "u2", "should keep client uid"); + } + #[test] fn merge_auction_eids_deduplicates_client_and_resolved_ids() { - let client_eids = Some(vec![Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "id5-sync.com".to_string(), + vec![Uid { id: "ID5_abc".to_string(), atype: Some(1), ext: None, }], - }]); + )]); let resolved_eids = Some(vec![ - Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_string(), + vec![Uid { id: "ID5_abc".to_string(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_string(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_string(), + vec![Uid { id: "LR_xyz".to_string(), atype: Some(3), ext: None, }], - }, + ), ]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2098,22 +2204,22 @@ mod tests { #[test] fn merge_auction_eids_preserves_multiple_uids_per_source() { - let client_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "sharedid.org".to_string(), + vec![Uid { id: "shared_client".to_string(), atype: None, ext: None, }], - }]); - let resolved_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { + )]); + let resolved_eids = Some(vec![Eid::new( + "sharedid.org".to_string(), + vec![Uid { id: "shared_server".to_string(), atype: Some(3), ext: None, }], - }]); + )]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2125,22 +2231,22 @@ mod tests { #[test] fn merge_auction_eids_prefers_server_resolved_metadata_on_conflict() { - let client_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { id: "shared_uid".to_string(), atype: Some(1), ext: Some(json!({ "provider": "client" })), }], - }]); - let resolved_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { + )]); + let resolved_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { id: "shared_uid".to_string(), atype: Some(3), ext: Some(json!({ "provider": "server" })), }], - }]); + )]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index b684c80dde..15076adb6a 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -850,14 +850,14 @@ mod tests { #[test] fn response_includes_eid_headers_when_eids_present() { let mut request = make_auction_request(); - request.user.eids = Some(vec![Eid { - source: "ssp.com".to_owned(), - uids: vec![Uid { + request.user.eids = Some(vec![Eid::new( + "ssp.com".to_owned(), + vec![Uid { id: "uid-1".to_owned(), atype: Some(3), ext: None, }], - }]); + )]); let settings = make_settings(); let result = make_empty_result(); diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index a83f8899a8..c839a582eb 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -92,14 +92,14 @@ pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { raw_ac_string: Some("fictional-ac".to_string()), ..Default::default() }), - eids: Some(vec![Eid { - source: "identity.example".to_string(), - uids: vec![Uid { + eids: Some(vec![Eid::new( + "identity.example".to_string(), + vec![Uid { id: "fictional-uid".to_string(), atype: Some(1), ext: None, }], - }]), + )]), }, device: Some(DeviceInfo { user_agent: Some("Fictional Browser".to_string()), diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index e21fab702c..383edddf15 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -31,7 +31,8 @@ use crate::cookies::extract_cookie_value; use crate::error::TrustedServerError; use crate::openrtb::Eid; -use super::eids::{resolve_partner_ids, to_eids}; +use super::current_timestamp; +use super::eid_types::UnixSeconds; use super::generation::is_valid_ec_id; use super::kv::KvIdentityGraph; use super::kv_backend::EcKvLookup; @@ -251,8 +252,8 @@ struct AuctionEidsView { struct SkippedPartnerId { /// Partner namespace key in the entry's `ids` map. source_domain: String, - /// Why the auction resolution skips it: `empty_uid`, `not_in_registry`, - /// or `bidstream_disabled`. + /// Why the auction resolution skips it: `empty_uid`, `expired` (provider + /// record with no live UID), `not_in_registry`, or `bidstream_disabled`. reason: &'static str, } @@ -479,22 +480,25 @@ fn iso_timestamp(unix_seconds: u64) -> Option { .map(|datetime| datetime.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) } -/// Derives the auction EID view for a valid entry, mirroring the filters in -/// [`resolve_partner_ids`] and reporting why each stored ID was skipped. +/// Derives the auction EID view for a valid entry through the same +/// [`KvEntry::live_eids`] resolution the live auction uses, reporting why each +/// stored ID was skipped. fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { - let resolved = resolve_partner_ids(registry, entry); - let eids = to_eids(&resolved); + let now = UnixSeconds::new(current_timestamp()); + let eids = entry.live_eids(registry, now).map(Eid::from).collect(); let mut skipped = Vec::new(); for (source_domain, partner_uid) in &entry.ids { - let reason = if partner_uid.uid.is_empty() { - "empty_uid" - } else { + let reason = if partner_uid.has_live_uid(now) { match registry.get(source_domain) { None => "not_in_registry", Some(partner) if !partner.bidstream_enabled => "bidstream_disabled", Some(_) => continue, } + } else if partner_uid.is_provider_written() { + "expired" + } else { + "empty_uid" }; skipped.push(SkippedPartnerId { source_domain: source_domain.clone(), @@ -682,9 +686,10 @@ mod tests { use crate::ec::kv::TombstoneOutcome; use super::*; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue}; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; - use crate::ec::kv_types::KvPartnerId; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -778,15 +783,11 @@ mod tests { let mut entry = KvEntry::minimal("bidstream.example", "uid-live", 1_741_824_000); entry.ids.insert( "disabled.example".to_owned(), - KvPartnerId { - uid: "uid-disabled".to_owned(), - }, + KvPartnerId::single("uid-disabled".to_owned()), ); entry.ids.insert( "unknown.example".to_owned(), - KvPartnerId { - uid: "uid-unknown".to_owned(), - }, + KvPartnerId::single("uid-unknown".to_owned()), ); entry } @@ -1533,4 +1534,55 @@ mod tests { assert_eq!(matched[0]["source_domain"], "sharedid.org"); assert_eq!(matched[0]["uid"], "shared-uid-123"); } + + #[test] + fn auction_view_matches_live_auction_for_provider_records() { + let registry = PartnerRegistry::from_config(&[ + make_test_partner("expired.example", true), + make_test_partner("live.example", true), + ]) + .expect("should build registry"); + let provider_record = |id: &str, expiry: Expiry| { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(0), + ) + .expect("should build record") + }; + let mut entry = KvEntry::tombstone(1); + entry.consent.ok = true; + entry.ids.insert( + "expired.example".to_owned(), + provider_record("old", Expiry::At(UnixSeconds::new(1))), + ); + entry.ids.insert( + "live.example".to_owned(), + provider_record("new", Expiry::Never), + ); + + let view = build_auction_view(®istry, &entry); + + let view_json = serde_json::to_value(&view.eids).expect("should serialize view eids"); + let live: Vec = entry + .live_eids(®istry, UnixSeconds::new(current_timestamp())) + .map(Eid::from) + .collect(); + assert_eq!( + view_json, + serde_json::to_value(&live).expect("should serialize live eids"), + "should show exactly what the live auction resolves" + ); + assert_eq!(view.eids.len(), 1, "should omit the expired provider UID"); + assert_eq!(view.eids[0].uids[0].id, "new", "should keep the live UID"); + assert_eq!(view.skipped.len(), 1, "should report the expired record"); + assert_eq!(view.skipped[0].source_domain, "expired.example"); + assert_eq!(view.skipped[0].reason, "expired", "should report expiry"); + } } diff --git a/crates/trusted-server-core/src/ec/eid_types.rs b/crates/trusted-server-core/src/ec/eid_types.rs new file mode 100644 index 0000000000..9b1a647f59 --- /dev/null +++ b/crates/trusted-server-core/src/ec/eid_types.rs @@ -0,0 +1,594 @@ +//! Value types for full-fidelity EIDs stored in the EC identity graph. +//! +//! Provider output is converted into these types at the trust boundary +//! (coordinator validation); past that point an invalid UID, agent type or +//! match method cannot be represented. + +use core::fmt; +use std::sync::Arc; + +use serde::de::{self, Deserializer, Visitor}; +use serde::{Deserialize, Serialize, Serializer}; +use serde_json::value::RawValue; + +use super::kv_types::MAX_UID_LENGTH; +use super::partner::normalize_partner_source_domain; + +/// Maximum serialized size of an `ext` object, in bytes. +pub const MAX_EXT_BYTES: usize = 1024; + +/// Maximum size of `inserter` and `matcher`, in bytes. +pub const MAX_EID_FIELD_BYTES: usize = 255; + +/// Why a provider value was rejected; counted per reason, never logged with +/// the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum EidValidationError { + /// The UID is blank. + #[display("UID is empty")] + EmptyUid, + /// The UID exceeds [`MAX_UID_LENGTH`]. + #[display("UID is too long")] + UidTooLong, + /// The agent type is negative. + #[display("agent type is negative")] + NegativeAgentType, + /// The match method is not an `OpenRTB` 2.6 value. + #[display("match method is unknown")] + UnknownMatchMethod, + /// An `ext` is not a JSON object within [`MAX_EXT_BYTES`]. + #[display("ext is not a JSON object within the size limit")] + InvalidExt, + /// An `inserter` or `matcher` exceeds [`MAX_EID_FIELD_BYTES`]. + #[display("EID field is too long")] + FieldTooLong, + /// The source is not a valid domain or not claimed by the provider. + #[display("source is not claimed by this provider")] + UnclaimedSource, + /// The expiry is not after the request time. + #[display("expiry is in the past")] + Expired, +} + +impl core::error::Error for EidValidationError {} + +/// Unix time in whole seconds. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + derive_more::Display, +)] +#[serde(transparent)] +pub struct UnixSeconds(u64); + +impl UnixSeconds { + /// Wraps a Unix timestamp in seconds. + #[must_use] + pub const fn new(seconds: u64) -> Self { + Self(seconds) + } + + /// Returns the timestamp in seconds. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } + + /// Adds `seconds`, saturating at `u64::MAX`. + #[must_use] + pub const fn saturating_add_secs(self, seconds: u64) -> Self { + Self(self.0.saturating_add(seconds)) + } +} + +struct ArcStrVisitor; + +impl<'de> Visitor<'de> for ArcStrVisitor { + type Value = Arc; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_str(self, value: &str) -> Result { + Ok(Arc::from(value)) + } +} + +/// Operator-chosen identifier of a configured provider instance. +/// +/// Clones share one allocation. Serialized as a plain string. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct ProviderId(Arc); + +impl ProviderId { + /// Creates a provider ID from configuration. + #[must_use] + pub fn new(id: &str) -> Self { + Self(Arc::from(id)) + } + + /// Returns the ID as a string slice. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Serialize for ProviderId { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for ProviderId { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_str(ArcStrVisitor).map(Self) + } +} + +/// Normalized EID source domain. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct SourceDomain(Arc); + +impl SourceDomain { + /// Returns the normalized domain. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom<&str> for SourceDomain { + type Error = EidValidationError; + + fn try_from(value: &str) -> Result { + normalize_partner_source_domain(value) + .map(|normalized| Self(Arc::from(normalized))) + .map_err(|_| EidValidationError::UnclaimedSource) + } +} + +impl Serialize for SourceDomain { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for SourceDomain { + fn deserialize>(deserializer: D) -> Result { + let raw = deserializer.deserialize_str(ArcStrVisitor)?; + Self::try_from(raw.as_ref()).map_err(de::Error::custom) + } +} + +/// A non-blank UID of at most [`MAX_UID_LENGTH`] bytes, kept verbatim. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(try_from = "String")] +pub struct UidValue(Box); + +impl UidValue { + /// Returns the UID. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + fn check(value: &str) -> Result<(), EidValidationError> { + if value.trim().is_empty() { + return Err(EidValidationError::EmptyUid); + } + if value.len() > MAX_UID_LENGTH { + return Err(EidValidationError::UidTooLong); + } + Ok(()) + } +} + +impl TryFrom<&str> for UidValue { + type Error = EidValidationError; + + fn try_from(value: &str) -> Result { + Self::check(value)?; + Ok(Self(Box::from(value))) + } +} + +impl TryFrom for UidValue { + type Error = EidValidationError; + + fn try_from(value: String) -> Result { + Self::check(&value)?; + Ok(Self(value.into_boxed_str())) + } +} + +/// Non-negative `OpenRTB` agent type, including vendor-specific values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "i32", into = "i32")] +pub struct AgentType(i32); + +impl AgentType { + /// Returns the raw agent type. + #[must_use] + pub const fn get(self) -> i32 { + self.0 + } +} + +impl TryFrom for AgentType { + type Error = EidValidationError; + + fn try_from(value: i32) -> Result { + if value < 0 { + return Err(EidValidationError::NegativeAgentType); + } + Ok(Self(value)) + } +} + +impl From for i32 { + fn from(value: AgentType) -> Self { + value.0 + } +} + +/// `OpenRTB` 2.6 EID match method (`mm`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "i32", into = "i32")] +pub enum MatchMethod { + /// `0`: unknown. + Unknown, + /// `1`: no matching. + NoMatching, + /// `2`: cookie sync. + CookieSync, + /// `3`: authenticated. + Authenticated, + /// `4`: observed. + Observed, + /// `5`: inference. + Inference, +} + +impl TryFrom for MatchMethod { + type Error = EidValidationError; + + fn try_from(value: i32) -> Result { + match value { + 0 => Ok(Self::Unknown), + 1 => Ok(Self::NoMatching), + 2 => Ok(Self::CookieSync), + 3 => Ok(Self::Authenticated), + 4 => Ok(Self::Observed), + 5 => Ok(Self::Inference), + _ => Err(EidValidationError::UnknownMatchMethod), + } + } +} + +impl From for i32 { + fn from(value: MatchMethod) -> Self { + match value { + MatchMethod::Unknown => 0, + MatchMethod::NoMatching => 1, + MatchMethod::CookieSync => 2, + MatchMethod::Authenticated => 3, + MatchMethod::Observed => 4, + MatchMethod::Inference => 5, + } + } +} + +/// When a stored UID stops being served. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Expiry { + /// Served until replaced or removed. + #[default] + Never, + /// Never served at or after this time. + At(UnixSeconds), +} + +impl Expiry { + /// Returns whether a UID with this expiry may be served at `now`. + #[must_use] + pub fn is_live(self, now: UnixSeconds) -> bool { + match self { + Self::Never => true, + Self::At(expires_at) => now < expires_at, + } + } + + /// Returns whether this is [`Expiry::Never`]; used to omit `exp`. + #[must_use] + pub fn is_never(&self) -> bool { + matches!(self, Self::Never) + } +} + +/// Serde adapter storing [`Expiry`] as an optional `exp` number. +pub mod optional_exp { + use serde::{Deserialize as _, Deserializer, Serialize as _, Serializer}; + + use super::{Expiry, UnixSeconds}; + + /// Serializes `At` as its seconds; `Never` is skipped by the caller. + /// + /// # Errors + /// + /// Returns the serializer's error. + pub fn serialize(expiry: &Expiry, serializer: S) -> Result { + match expiry { + Expiry::At(expires_at) => expires_at.serialize(serializer), + Expiry::Never => serializer.serialize_none(), + } + } + + /// Reads an optional number; absent or `null` means [`Expiry::Never`]. + /// + /// # Errors + /// + /// Returns the deserializer's error for a non-numeric value. + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Ok(Option::::deserialize(deserializer)?.map_or(Expiry::Never, Expiry::At)) + } +} + +/// An opaque JSON object of at most [`MAX_EXT_BYTES`], compared by bytes. +/// +/// Deserializing applies the same checks as [`RawJson::try_from`], so a +/// stored row cannot carry an oversized or non-object `ext`. +#[derive(Debug, Clone)] +pub struct RawJson(Box); + +impl RawJson { + /// Returns the raw JSON text. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.get() + } + + /// Parses the raw JSON into a [`serde_json::Value`] for `OpenRTB` output. + /// + /// # Panics + /// + /// Never in practice: the bytes were validated as JSON when stored. + #[must_use] + pub fn to_value(&self) -> serde_json::Value { + serde_json::from_str(self.0.get()).expect("should parse stored raw JSON") + } +} + +impl PartialEq for RawJson { + fn eq(&self, other: &Self) -> bool { + self.0.get() == other.0.get() + } +} + +impl RawJson { + fn check(value: &RawValue) -> Result<(), EidValidationError> { + let text = value.get(); + if text.len() > MAX_EXT_BYTES || !text.trim_start().starts_with('{') { + return Err(EidValidationError::InvalidExt); + } + Ok(()) + } +} + +impl TryFrom<&RawValue> for RawJson { + type Error = EidValidationError; + + fn try_from(value: &RawValue) -> Result { + Self::check(value)?; + Ok(Self(value.to_owned())) + } +} + +impl TryFrom> for RawJson { + type Error = EidValidationError; + + fn try_from(value: Box) -> Result { + Self::check(&value)?; + Ok(Self(value)) + } +} + +impl Serialize for RawJson { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for RawJson { + fn deserialize>(deserializer: D) -> Result { + let raw = Box::::deserialize(deserializer)?; + Self::try_from(raw).map_err(de::Error::custom) + } +} + +/// EID-level fields shared by all UIDs of one stored entry. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct EidFields { + /// `OpenRTB` `inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option>, + /// `OpenRTB` `matcher`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option>, + /// `OpenRTB` `mm`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// `OpenRTB` EID `ext`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn uid_value_rejects_blank_and_oversized() { + assert!(UidValue::try_from(" ").is_err(), "should reject blank UID"); + let long = "a".repeat(MAX_UID_LENGTH + 1); + assert!( + UidValue::try_from(long.as_str()).is_err(), + "should reject oversized UID" + ); + let uid = UidValue::try_from("ID-123").expect("should accept valid UID"); + assert_eq!(uid.as_str(), "ID-123", "should keep UID verbatim"); + } + + #[test] + fn agent_type_rejects_negative() { + assert!( + AgentType::try_from(-1).is_err(), + "should reject negative atype" + ); + assert_eq!( + AgentType::try_from(571_187) + .expect("should accept vendor atype") + .get(), + 571_187, + "should keep vendor atype" + ); + } + + #[test] + fn match_method_round_trips_openrtb_values() { + for value in 0..=5 { + let method = MatchMethod::try_from(value).expect("should accept OpenRTB mm"); + assert_eq!(i32::from(method), value, "should round-trip mm {value}"); + } + assert!( + MatchMethod::try_from(6).is_err(), + "should reject unknown mm" + ); + } + + #[test] + fn source_domain_normalizes() { + let source = SourceDomain::try_from("IDS.Example.COM").expect("should accept domain"); + assert_eq!( + source.as_str(), + "ids.example.com", + "should lowercase source domain" + ); + } + + #[test] + fn provider_id_round_trips_as_string() { + let id = ProviderId::new("example-provider"); + let encoded = serde_json::to_string(&id).expect("should serialize provider id"); + assert_eq!( + encoded, "\"example-provider\"", + "should serialize as plain string" + ); + let decoded: ProviderId = + serde_json::from_str(&encoded).expect("should deserialize provider id"); + assert_eq!(decoded, id, "should round-trip provider id"); + } + + #[test] + fn expiry_liveness() { + let now = UnixSeconds::new(100); + assert!( + Expiry::Never.is_live(now), + "should serve UIDs without expiry" + ); + assert!( + Expiry::At(UnixSeconds::new(101)).is_live(now), + "should serve before expiry" + ); + assert!( + !Expiry::At(UnixSeconds::new(100)).is_live(now), + "should not serve at expiry" + ); + } + + #[test] + fn raw_json_accepts_small_objects_only() { + let object = + serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode object"); + let raw = RawJson::try_from(object.as_ref()).expect("should accept object"); + assert_eq!(raw.as_str(), r#"{"k":1}"#, "should keep raw bytes"); + let array = serde_json::value::to_raw_value(&json!([1])).expect("should encode array"); + assert!( + RawJson::try_from(array.as_ref()).is_err(), + "should reject non-object" + ); + let big = serde_json::value::to_raw_value(&json!({ "k": "x".repeat(MAX_EXT_BYTES) })) + .expect("should encode big object"); + assert!( + RawJson::try_from(big.as_ref()).is_err(), + "should reject oversized ext" + ); + } + + #[test] + fn raw_json_compares_by_bytes() { + let a = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); + let b = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); + assert_eq!( + RawJson::try_from(a.as_ref()).expect("should accept"), + RawJson::try_from(b.as_ref()).expect("should accept"), + "should compare equal bytes as equal" + ); + } + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct WithExpiry { + #[serde( + default, + rename = "exp", + with = "optional_exp", + skip_serializing_if = "Expiry::is_never" + )] + expiry: Expiry, + } + + #[test] + fn optional_exp_omits_never_and_reads_missing() { + let never = serde_json::to_value(WithExpiry { + expiry: Expiry::Never, + }) + .expect("should serialize"); + assert_eq!(never, json!({}), "should omit exp for Never"); + let at: WithExpiry = + serde_json::from_value(json!({ "exp": 42 })).expect("should deserialize exp"); + assert_eq!( + at.expiry, + Expiry::At(UnixSeconds::new(42)), + "should read exp" + ); + let missing: WithExpiry = + serde_json::from_value(json!({})).expect("should deserialize empty"); + assert_eq!(missing.expiry, Expiry::Never, "should default to Never"); + } + + #[test] + fn raw_json_deserialize_applies_same_checks() { + let ok: RawJson = serde_json::from_str(r#"{"k":1}"#).expect("should accept small object"); + assert_eq!(ok.as_str(), r#"{"k":1}"#, "should keep raw bytes"); + assert!( + serde_json::from_str::("[1]").is_err(), + "should reject non-object on deserialize" + ); + let big = serde_json::to_string(&json!({ "k": "x".repeat(MAX_EXT_BYTES) })) + .expect("should encode big object"); + assert!( + serde_json::from_str::(&big).is_err(), + "should reject oversized ext on deserialize" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 1359f75091..904594c934 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -10,7 +10,8 @@ use error_stack::{Report, ResultExt as _}; use crate::error::TrustedServerError; use crate::openrtb::{Eid, Uid}; -use super::kv_types::KvEntry; +use super::eid_types::{RawJson, UnixSeconds}; +use super::kv_types::{KvEid, KvEntry, KvPartnerId}; use super::registry::PartnerRegistry; /// Maximum size (in bytes) for the base64-encoded `x-ts-eids` header value. @@ -64,17 +65,120 @@ pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec Vec { resolved .iter() - .map(|item| Eid { - source: item.source_domain.clone(), - uids: vec![Uid { - id: item.uid.clone(), - atype: Some(item.openrtb_atype), - ext: None, - }], + .map(|item| { + Eid::new( + item.source_domain.clone(), + vec![Uid { + id: item.uid.clone(), + atype: Some(item.openrtb_atype), + ext: None, + }], + ) }) .collect() } +/// A borrowed EID entry: a single legacy UID or a full-fidelity provider entry. +#[derive(Debug, Clone, Copy)] +pub enum EidView<'row> { + /// A record holding one UID and no EID-level fields. + Single { + /// Source domain. + source: &'row str, + /// The UID. + uid: &'row str, + /// The partner's configured agent type. + atype: i32, + }, + /// A provider-written entry; only live UIDs are emitted. + Full { + /// Source domain. + source: &'row str, + /// The stored entry. + eid: &'row KvEid, + /// Request time used to filter UIDs. + now: UnixSeconds, + }, +} + +impl From> for Eid { + fn from(view: EidView<'_>) -> Self { + match view { + EidView::Single { source, uid, atype } => Eid::new( + source, + vec![Uid { + id: uid.to_owned(), + atype: Some(atype), + ext: None, + }], + ), + EidView::Full { source, eid, now } => { + let fields = eid.fields(); + let uids = eid + .uids() + .iter() + .filter(|uid| uid.is_live(now)) + .map(|uid| Uid { + id: uid.id().as_str().to_owned(), + atype: Some(uid.atype().get()), + ext: uid.ext().map(RawJson::to_value), + }) + .collect(); + Eid { + inserter: fields.inserter.as_deref().map(str::to_owned), + matcher: fields.matcher.as_deref().map(str::to_owned), + mm: fields.mm.map(i32::from), + ext: fields.ext.as_ref().map(RawJson::to_value), + ..Eid::new(source, uids) + } + } + } + } +} + +/// Yields the live views of one stored record. +/// +/// Legacy records yield one [`EidView::Single`]; provider records yield one +/// [`EidView::Full`] per entry that still has a live UID. +pub fn source_views<'row>( + source: &'row str, + record: &'row KvPartnerId, + default_atype: i32, + now: UnixSeconds, +) -> impl Iterator> + 'row { + let single = + (!record.is_provider_written() && !record.uid.is_empty()).then_some(EidView::Single { + source, + uid: record.uid.as_str(), + atype: default_atype, + }); + let full = record + .eids() + .iter() + .filter(move |eid| eid.uids().iter().any(|uid| uid.is_live(now))) + .map(move |eid| EidView::Full { source, eid, now }); + single.into_iter().chain(full) +} + +impl KvEntry { + /// Iterates live entries for registered, `bidstream_enabled` sources in + /// source order, borrowing from the row. + pub fn live_eids<'row>( + &'row self, + registry: &'row PartnerRegistry, + now: UnixSeconds, + ) -> impl Iterator> + 'row { + self.ids.iter().flat_map(move |(source, record)| { + let partner = registry + .get(source) + .filter(|partner| partner.bidstream_enabled); + partner + .into_iter() + .flat_map(move |partner| source_views(source, record, partner.openrtb_atype, now)) + }) + } +} + /// Builds a base64-encoded EID header value, truncating if needed. /// /// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds @@ -145,6 +249,8 @@ pub fn encode_eids_header(eids: &[Eid]) -> Result<(String, bool), Report PartnerRegistry { + let mut partner = make_test_partner(source); + partner.bidstream_enabled = bidstream; + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn provider_record(ids: &[(&str, Expiry)]) -> KvPartnerId { + let uids = ids + .iter() + .map(|(id, expiry)| { + KvUid::new( + UidValue::try_from(*id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + *expiry, + ) + }) + .collect(); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + ..EidFields::default() + }; + KvPartnerId::from_eids(vec![KvEid::new(fields, uids)], UnixSeconds::new(0)) + .expect("should build record") + } + + #[test] + fn live_eids_skip_expired_uids_and_keep_eid_fields() { + let registry = registry_with("ids.example.com", true); + let mut entry = KvEntry::minimal("ids.example.com", "placeholder", 1); + entry.ids.insert( + "ids.example.com".to_owned(), + provider_record(&[ + ("old", Expiry::At(UnixSeconds::new(10))), + ("new", Expiry::Never), + ]), + ); + + let eids: Vec = entry + .live_eids(®istry, UnixSeconds::new(20)) + .map(Eid::from) + .collect(); + + assert_eq!(eids.len(), 1, "should emit one entry"); + assert_eq!( + eids[0].inserter.as_deref(), + Some("inserter.example.com"), + "should keep inserter" + ); + let ids: Vec<&str> = eids[0].uids.iter().map(|uid| uid.id.as_str()).collect(); + assert_eq!(ids, vec!["new"], "should drop expired uid"); + } + + #[test] + fn live_eids_respect_bidstream_flag() { + let registry = registry_with("ids.example.com", false); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); + assert_eq!( + entry.live_eids(®istry, UnixSeconds::new(0)).count(), + 0, + "should hide non-bidstream source" + ); + } + + #[test] + fn live_eids_emit_legacy_single_uid() { + let registry = registry_with("ids.example.com", true); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); + let eids: Vec = entry + .live_eids(®istry, UnixSeconds::new(0)) + .map(Eid::from) + .collect(); + assert_eq!(eids[0].uids[0].id, "v", "should emit legacy uid"); + assert_eq!(eids[0].uids[0].atype, Some(3), "should use partner atype"); + } } diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 019ef2aa40..ce4c831602 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1115,14 +1115,14 @@ mod tests { ); ec_context.set_recovery_eligible(true); ec_context.set_eid_sync_source(EidSyncSource::Navigation); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_should_not_persist".to_owned(), atype: Some(1), ext: None, }], - }]); + )]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1517,22 +1517,22 @@ mod tests { .expect("should serialize test cookie payload"), ); ec_context.set_client_eids(vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_from_body".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_from_body".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]); let partners = vec![make_partner("id5-sync.com"), make_partner("liveramp.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); @@ -1595,14 +1595,14 @@ mod tests { ec_id: ec_id.clone(), }); ec_context.set_eid_sync_source(EidSyncSource::Auction); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_should_not_persist".to_owned(), atype: Some(1), ext: None, }], - }]); + )]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -2315,9 +2315,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "partner-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("partner-uid".to_owned()), ); ec_context.set_kv_snapshot(EcKvSnapshot::Present { ec_id, diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 94464d1a57..26a6e195e4 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -12,13 +12,15 @@ use url::Url; use super::auth::authenticate_bearer; use super::consent::ec_consent_granted; use crate::error::TrustedServerError; -use crate::openrtb::{Eid, Uid}; +use crate::openrtb::Eid; use crate::settings::Settings; -use super::EcContext; +use super::eid_types::UnixSeconds; +use super::eids::source_views; use super::kv::KvIdentityGraph; use super::log_id; use super::registry::PartnerRegistry; +use super::{EcContext, current_timestamp}; /// Handles `GET /_ts/api/v1/identify`. /// @@ -84,7 +86,7 @@ pub fn handle_identify( }; let mut degraded = false; - let mut uid: Option = None; + let mut eid: Option = None; let mut cluster_size: Option = None; match kv.get(ec_id) { @@ -95,12 +97,16 @@ pub fn handle_identify( // write back with the live-entry TTL. log::trace!("Identify found tombstone for '{}'", log_id(ec_id)); } else { - // Extract only this partner's UID. - if let Some(partner_uid) = entry.ids.get(&partner.source_domain) - && !partner_uid.uid.is_empty() - { - uid = Some(partner_uid.uid.clone()); - } + // Extract only this partner's first live entry. + let now = UnixSeconds::new(current_timestamp()); + eid = entry + .ids + .get(&partner.source_domain) + .and_then(|record| { + source_views(&partner.source_domain, record, partner.openrtb_atype, now) + .next() + }) + .map(Eid::from); // Evaluate cluster size lazily for identify responses. Existing // stored cluster_size values are reused without a prefix-list call. @@ -124,14 +130,10 @@ pub fn handle_identify( } } - let eid = uid.as_ref().map(|u| Eid { - source: partner.source_domain.clone(), - uids: vec![Uid { - id: u.clone(), - atype: Some(partner.openrtb_atype), - ext: None, - }], - }); + let uid = eid + .as_ref() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id.clone()); let body = IdentifyResponse { ec: ec_id.to_owned(), @@ -334,6 +336,8 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; + use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; use crate::ec::registry::PartnerRegistry; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -756,4 +760,77 @@ mod tests { "should vary on identity request inputs for preflight" ); } + + fn provider_record(id: &str, expiry: Expiry) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + ..EidFields::default() + }; + KvPartnerId::from_eids(vec![KvEid::new(fields, vec![uid])], UnixSeconds::new(0)) + .expect("should build record") + } + + fn identify_body_for_record(record: KvPartnerId) -> serde_json::Value { + let settings = create_test_settings(); + let kv = KvIdentityGraph::in_memory("identify_store"); + let partners = vec![make_test_partner("ssp.example.com", VALID_API_TOKEN)]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = KvEntry::minimal("other.example.com", "other-uid", 1); + entry.ids.insert("ssp.example.com".to_owned(), record); + kv.create(&ec_id, &entry).expect("should seed KV entry"); + let req = Request::builder() + .method("GET") + .uri("https://edge.test-publisher.com/identify") + .header("authorization", format!("Bearer {VALID_API_TOKEN}")) + .body(EdgeBody::empty()) + .expect("should build test request"); + let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + + let response = handle_identify(&settings, &kv, ®istry, &req, &ec_context) + .expect("should construct identify response"); + + assert_eq!(response.status(), StatusCode::OK, "should return 200"); + serde_json::from_slice::( + &response.into_body().into_bytes().unwrap_or_default(), + ) + .expect("should decode identify response JSON") + } + + #[test] + fn handle_identify_omits_expired_provider_uid() { + let body = + identify_body_for_record(provider_record("old", Expiry::At(UnixSeconds::new(1)))); + + assert!( + body.get("uid").is_none(), + "should omit expired provider uid" + ); + assert!( + body.get("eid").is_none(), + "should omit eid without a live uid" + ); + } + + #[test] + fn handle_identify_returns_live_provider_eid_with_eid_fields() { + let body = identify_body_for_record(provider_record("live", Expiry::Never)); + + assert_eq!(body["uid"], "live", "should return live provider uid"); + assert_eq!( + body["eid"]["inserter"], "inserter.example.com", + "should keep provider EID-level fields" + ); + assert_eq!( + body["eid"]["uids"][0]["atype"], 1, + "should use the provider atype" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index b0eb83d3fa..9daaac243c 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -21,7 +21,7 @@ use crate::error::TrustedServerError; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; -use super::kv_types::{KvEntry, KvMetadata, KvNetwork}; +use super::kv_types::{KvEntry, KvMetadata, KvNetwork, MAX_ENTRY_BYTES}; use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. @@ -157,9 +157,7 @@ fn apply_cookie_partner_id_updates( entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); changed = true; } @@ -205,9 +203,7 @@ pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerId entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); changed = true; } @@ -284,6 +280,15 @@ impl KvIdentityGraph { store_name: store_name.to_owned(), message: "Failed to serialize KV entry body".to_owned(), })?; + if body.len() > MAX_ENTRY_BYTES { + return Err(Report::new(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: format!( + "Refusing to write KV entry of {} bytes (limit {MAX_ENTRY_BYTES})", + body.len() + ), + })); + } let meta = KvMetadata::from_entry(entry); let meta_str = serde_json::to_string(&meta).change_context(TrustedServerError::KvStore { @@ -920,9 +925,7 @@ impl KvIdentityGraph { entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; @@ -992,9 +995,7 @@ impl KvIdentityGraph { entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; @@ -1618,7 +1619,9 @@ impl KvIdentityGraph { #[cfg(test)] mod tests { use super::*; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; use crate::ec::kv_backend::test_support::InMemoryEcKv; + use crate::ec::kv_types::{KvEid, KvUid}; /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition /// and deletes the key, simulating a concurrent withdrawal that removes the @@ -1737,9 +1740,9 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::single( + "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), + ), ); let body = serde_json::to_vec(&entry).expect("should serialize invalid entry payload"); @@ -1772,9 +1775,9 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::single( + "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), + ), ); let err = KvIdentityGraph::serialize_entry(&entry, "test-store") @@ -1786,6 +1789,48 @@ mod tests { ); } + #[test] + fn create_refuses_oversized_provider_entry_before_write() { + let graph = KvIdentityGraph::in_memory("test-store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let uids = (0..40) + .map(|index| { + let id = format!("{index:03}{}", "x".repeat(480)); + KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ) + }) + .collect(); + let record = crate::ec::kv_types::KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), uids)], + UnixSeconds::new(0), + ) + .expect("should build record"); + let mut entry = live_entry(); + entry.ids.insert("ids.example.com".to_owned(), record); + assert!( + entry.validate().is_ok(), + "should be a valid entry apart from its size" + ); + + let err = graph + .create(&ec_id, &entry) + .expect_err("should refuse oversized entry"); + + assert!( + format!("{err}").contains("Refusing to write KV entry"), + "should report the size limit" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "should not write the oversized entry" + ); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; @@ -1796,9 +1841,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "concurrent.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("concurrent-uid".to_owned()), ); entry } @@ -2131,9 +2174,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-1".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid-1".to_owned()), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "uid-1")]; @@ -2159,9 +2200,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "old-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("old-uid".to_owned()), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "new-uid")]; @@ -2191,9 +2230,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "original".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("original".to_owned()), ); let updates = vec![ PartnerIdUpdate::new("ssp_x", "intermediate"), @@ -2215,9 +2252,7 @@ mod tests { let mut concurrent = live_entry(); concurrent.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "desired-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("desired-uid".to_owned()), ); let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -2333,9 +2368,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "latest-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("latest-uid".to_owned()), ); let updates = [ PartnerIdUpdate::new("ssp_x", "older-uid"), @@ -2354,9 +2387,7 @@ mod tests { let mut concurrent = live_entry(); concurrent.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("concurrent-uid".to_owned()), ); let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -2439,9 +2470,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("protected-uid".to_owned()), ); graph.create(&ec_id, &entry).expect("should seed entry"); @@ -2466,9 +2495,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("protected-uid".to_owned()), ); graph.create(&ec_id, &entry).expect("should seed entry"); diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 589f280c11..2a638dc219 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -10,6 +10,10 @@ use std::collections::{BTreeMap, BTreeSet}; use serde::{Deserialize, Serialize}; +use super::eid_types::{ + AgentType, EidFields, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, RawJson, UidValue, + UnixSeconds, optional_exp, +}; use crate::consent::ConsentContext; use crate::geo::GeoInfo; @@ -97,11 +101,233 @@ pub struct KvGeo { pub dma: Option, } -/// A partner user ID within a KV entry. +/// Maximum serialized size of a KV entry body written by a page load. +pub const MAX_ENTRY_BYTES: usize = 16 * 1024; + +/// A partner ID record within a KV entry, keyed by source domain. +/// +/// `uid` is always present so older readers keep working. Records written by +/// an identity provider also carry full-fidelity `eids`; new code reads +/// `eids` whenever it is non-empty and treats `uid` as derived. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct KvPartnerId { - /// The partner's user identifier. + /// First live UID of `eids`, or the single UID for records written by + /// browser ingest, batch sync or older versions. pub uid: String, + /// Full-fidelity entries written by a provider. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + eids: Vec, +} + +impl KvPartnerId { + /// Builds a single-UID record (browser ingest, batch sync, pull sync). + #[must_use] + pub fn single(uid: impl Into) -> Self { + Self { + uid: uid.into(), + eids: Vec::new(), + } + } + + /// Builds a provider record, keeping only entries with live UIDs. + /// + /// Returns `None` when no live UID remains. + #[must_use] + pub fn from_eids(eids: Vec, now: UnixSeconds) -> Option { + let mut record = Self { + uid: String::new(), + eids, + }; + record.prune_expired(now); + (!record.eids.is_empty()).then_some(record) + } + + /// Full-fidelity provider entries; empty for single-UID records. + #[must_use] + pub fn eids(&self) -> &[KvEid] { + &self.eids + } + + /// Whether this record was written by an identity provider. + #[must_use] + pub fn is_provider_written(&self) -> bool { + !self.eids.is_empty() + } + + /// Whether the record holds a UID that may be served at `now`. + #[must_use] + pub fn has_live_uid(&self, now: UnixSeconds) -> bool { + if self.eids.is_empty() { + return !self.uid.is_empty(); + } + self.eids + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.is_live(now)) + } + + /// Drops expired UIDs and empty entries, then resyncs `uid`. + /// + /// Returns whether anything changed. Single-UID records never change. + pub fn prune_expired(&mut self, now: UnixSeconds) -> bool { + if self.eids.is_empty() && !self.uid.is_empty() { + return false; + } + let before = self.clone(); + for eid in &mut self.eids { + eid.uids.retain(|uid| uid.is_live(now)); + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + *self != before + } +} + +/// One stored EID entry with its EID-level fields. +/// +/// Stored flat (`{"inserter": .., "uids": [..]}`). Serialization flattens +/// [`EidFields`]; deserialization goes through a flat private representation +/// because serde's `flatten` cannot deserialize the `RawValue` behind `ext`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(from = "KvEidRepr")] +pub struct KvEid { + #[serde(flatten)] + fields: EidFields, + uids: Vec, +} + +/// Flat stored shape of [`KvEid`], used only to deserialize it. +#[derive(Deserialize)] +struct KvEidRepr { + #[serde(default)] + inserter: Option>, + #[serde(default)] + matcher: Option>, + #[serde(default)] + mm: Option, + #[serde(default)] + ext: Option, + uids: Vec, +} + +impl From for KvEid { + fn from(repr: KvEidRepr) -> Self { + let KvEidRepr { + inserter, + matcher, + mm, + ext, + uids, + } = repr; + Self { + fields: EidFields { + inserter, + matcher, + mm, + ext, + }, + uids, + } + } +} + +impl KvEid { + /// Builds an entry from validated parts. + #[must_use] + pub fn new(fields: EidFields, uids: Vec) -> Self { + Self { fields, uids } + } + + /// EID-level fields. + #[must_use] + pub fn fields(&self) -> &EidFields { + &self.fields + } + + /// Stored UIDs, live or not. + #[must_use] + pub fn uids(&self) -> &[KvUid] { + &self.uids + } +} + +/// One stored UID with its owner and expiry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvUid { + id: UidValue, + atype: AgentType, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + owner: Option, + #[serde( + default, + rename = "exp", + with = "optional_exp", + skip_serializing_if = "Expiry::is_never" + )] + expiry: Expiry, +} + +impl KvUid { + /// Builds a UID from validated parts. + #[must_use] + pub fn new( + id: UidValue, + atype: AgentType, + ext: Option, + owner: Option, + expiry: Expiry, + ) -> Self { + Self { + id, + atype, + ext, + owner, + expiry, + } + } + + /// The UID value. + #[must_use] + pub fn id(&self) -> &UidValue { + &self.id + } + + /// The agent type. + #[must_use] + pub fn atype(&self) -> AgentType { + self.atype + } + + /// The UID-level `ext`. + #[must_use] + pub fn ext(&self) -> Option<&RawJson> { + self.ext.as_ref() + } + + /// The provider that wrote this UID. + #[must_use] + pub fn owner(&self) -> Option<&ProviderId> { + self.owner.as_ref() + } + + /// When the UID stops being served. + #[must_use] + pub fn expiry(&self) -> Expiry { + self.expiry + } + + /// Whether the UID may be served at `now`. + #[must_use] + pub fn is_live(&self, now: UnixSeconds) -> bool { + self.expiry.is_live(now) + } } /// Publisher property metadata captured when an EC entry is created. @@ -303,12 +529,7 @@ impl KvEntry { #[must_use] pub fn minimal(partner_id: &str, uid: &str, now: u64) -> Self { let mut ids = BTreeMap::new(); - ids.insert( - partner_id.to_owned(), - KvPartnerId { - uid: uid.to_owned(), - }, - ); + ids.insert(partner_id.to_owned(), KvPartnerId::single(uid.to_owned())); Self { v: SCHEMA_VERSION, created: now, @@ -387,6 +608,23 @@ impl KvEntry { partner_uid.uid.len() )); } + let oversized_field = partner_uid.eids().iter().any(|eid| { + let fields = eid.fields(); + [fields.inserter.as_deref(), fields.matcher.as_deref()] + .into_iter() + .flatten() + .any(|field| field.len() > MAX_EID_FIELD_BYTES) + }); + if oversized_field { + return Err(format!( + "partner ID '{partner_id}' has an EID inserter or matcher over MAX_EID_FIELD_BYTES" + )); + } + if partner_uid.eids().iter().any(|eid| eid.uids().is_empty()) { + return Err(format!( + "partner ID '{partner_id}' has an EID entry with no UIDs" + )); + } } if let Some(pub_properties) = &self.pub_properties { @@ -463,6 +701,9 @@ impl KvGeo { #[cfg(test)] mod tests { use super::*; + use crate::ec::eid_types::{ + AgentType, EidFields, Expiry, MatchMethod, ProviderId, RawJson, UidValue, UnixSeconds, + }; fn sample_consent_context() -> ConsentContext { ConsentContext { @@ -492,9 +733,7 @@ mod tests { let mut entry = KvEntry::new(&consent, Some(&geo), 1_741_824_000, "example.com"); entry.ids.insert( "liveramp".to_owned(), - KvPartnerId { - uid: "LR_xyz".to_owned(), - }, + KvPartnerId::single("LR_xyz".to_owned()), ); let json = serde_json::to_string(&entry).expect("should serialize KvEntry"); @@ -737,9 +976,7 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - KvPartnerId { - uid: "x".repeat(MAX_UID_LENGTH + 1), - }, + KvPartnerId::single("x".repeat(MAX_UID_LENGTH + 1)), ); let err = entry @@ -1206,4 +1443,154 @@ mod tests { "known_browser should default to None" ); } + + fn provider_uid(id: &str, expiry: Expiry) -> KvUid { + KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ) + } + + #[test] + fn legacy_partner_id_round_trips_without_eids() { + let legacy: KvPartnerId = serde_json::from_value(serde_json::json!({ "uid": "abc" })) + .expect("should read legacy record"); + assert_eq!(legacy, KvPartnerId::single("abc"), "should read legacy uid"); + assert_eq!( + serde_json::to_value(&legacy).expect("should serialize"), + serde_json::json!({ "uid": "abc" }), + "should not add eids to legacy records" + ); + } + + #[test] + fn from_eids_sets_uid_to_first_live_uid() { + let now = UnixSeconds::new(100); + let eid = KvEid::new( + EidFields::default(), + vec![ + provider_uid("expired", Expiry::At(UnixSeconds::new(50))), + provider_uid("live", Expiry::Never), + ], + ); + let record = KvPartnerId::from_eids(vec![eid], now).expect("should keep live uid"); + assert_eq!(record.uid, "live", "should mirror first live uid into uid"); + assert!(record.is_provider_written(), "should mark provider record"); + } + + #[test] + fn prune_expired_drops_dead_uids_and_resyncs_uid() { + let eid = KvEid::new( + EidFields::default(), + vec![ + provider_uid("first", Expiry::At(UnixSeconds::new(150))), + provider_uid("second", Expiry::Never), + ], + ); + let mut record = + KvPartnerId::from_eids(vec![eid], UnixSeconds::new(100)).expect("should build"); + assert_eq!(record.uid, "first", "should start on first uid"); + + let changed = record.prune_expired(UnixSeconds::new(200)); + + assert!(changed, "should report pruning"); + assert_eq!(record.eids()[0].uids().len(), 1, "should drop expired uid"); + assert_eq!(record.uid, "second", "should resync uid"); + } + + #[test] + fn provider_record_round_trips_with_owner_and_exp() { + let eid = KvEid::new( + EidFields::default(), + vec![provider_uid("v", Expiry::At(UnixSeconds::new(500)))], + ); + let record = KvPartnerId::from_eids(vec![eid], UnixSeconds::new(1)).expect("should build"); + let json = serde_json::to_value(&record).expect("should serialize"); + assert_eq!( + json, + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "owner": "example-provider", "exp": 500 }] }] }), + "should store owner and exp" + ); + let back: KvPartnerId = serde_json::from_value(json).expect("should deserialize"); + assert_eq!(back, record, "should round-trip"); + } + + #[test] + fn validate_rejects_empty_provider_eid() { + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + let record: KvPartnerId = + serde_json::from_value(serde_json::json!({ "uid": "v", "eids": [{ "uids": [] }] })) + .expect("should parse"); + entry.ids.insert("ids.example.com".to_owned(), record); + assert!(entry.validate().is_err(), "should reject EID with no UIDs"); + } + + #[test] + fn provider_record_round_trips_eid_level_fields() { + let ext = serde_json::value::to_raw_value(&serde_json::json!({ "k": 1 })) + .expect("should encode ext"); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + matcher: Some("matcher.example.com".into()), + mm: Some(MatchMethod::Authenticated), + ext: Some(RawJson::try_from(ext.as_ref()).expect("should accept ext")), + }; + let eid = KvEid::new(fields, vec![provider_uid("v", Expiry::Never)]); + let record = + KvPartnerId::from_eids(vec![eid], UnixSeconds::new(1)).expect("should build record"); + + let json = serde_json::to_string(&record).expect("should serialize"); + let back: KvPartnerId = serde_json::from_str(&json).expect("should deserialize"); + + assert_eq!( + serde_json::from_str::(&json).expect("should parse"), + serde_json::json!({ "uid": "v", "eids": [{ + "inserter": "inserter.example.com", + "matcher": "matcher.example.com", + "mm": 3, + "ext": { "k": 1 }, + "uids": [{ "id": "v", "atype": 1, "owner": "example-provider" }] + }] }), + "should store EID-level fields flat beside uids" + ); + assert_eq!(back, record, "should round-trip EID-level fields"); + } + + #[test] + fn provider_record_rejects_invalid_stored_ext() { + let big = "x".repeat(crate::ec::eid_types::MAX_EXT_BYTES); + let cases = [ + serde_json::json!({ "uid": "v", "eids": [{ "ext": [1], "uids": [{ "id": "v", "atype": 1 }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "ext": { "k": big }, "uids": [{ "id": "v", "atype": 1 }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "ext": { "k": big } }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "ext": "text" }] }] }), + ]; + + for case in cases { + let text = serde_json::to_string(&case).expect("should encode case"); + assert!( + serde_json::from_str::(&text).is_err(), + "should reject invalid stored ext: {text}" + ); + } + } + + #[test] + fn validate_rejects_oversized_inserter_and_matcher() { + let long = "a".repeat(crate::ec::eid_types::MAX_EID_FIELD_BYTES + 1); + for field in ["inserter", "matcher"] { + let mut eid = serde_json::json!({ "uids": [{ "id": "v", "atype": 1 }] }); + eid[field] = serde_json::Value::String(long.clone()); + let record: KvPartnerId = + serde_json::from_value(serde_json::json!({ "uid": "v", "eids": [eid] })) + .expect("should parse record"); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry.ids.insert("ids.example.com".to_owned(), record); + + assert!(entry.validate().is_err(), "should reject oversized {field}"); + } + } } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index f24c489edd..e815c43a19 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -26,6 +26,7 @@ //! - [`rate_limiter`] — Rate limiting abstraction (implemented by adapters) //! - [`identify`] — Identity read endpoint (`GET /_ts/api/v1/identify`) //! - [`eids`] — Shared EID resolution and formatting helpers +//! - [`eid_types`] — Value types for full-fidelity provider EIDs //! - [`batch_sync`] — S2S batch sync endpoint (`POST /_ts/api/v1/batch-sync`) //! - [`pull_sync`] — Background pull-sync dispatcher for organic routes @@ -36,6 +37,7 @@ pub mod batch_sync; pub mod consent; pub mod cookies; pub mod device; +pub mod eid_types; pub mod eids; pub mod finalize; pub mod generation; diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index d823cb32fb..3e2fbee219 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -294,10 +294,7 @@ fn structured_cookie_eids_to_openrtb(entries: Vec) -> Vec) -> Vec { entries .into_iter() .filter(|entry| !entry.source.is_empty() && !entry.id.is_empty()) - .map(|entry| Eid { - source: entry.source, - uids: vec![Uid { - id: entry.id, - atype: (entry.atype >= 0).then_some(entry.atype), - ext: None, - }], + .map(|entry| { + Eid::new( + entry.source, + vec![Uid { + id: entry.id, + atype: (entry.atype >= 0).then_some(entry.atype), + ext: None, + }], + ) }) .collect() } @@ -636,22 +635,22 @@ mod tests { {"source": "id5-sync.com", "uids": [{"id": "ID5_stale", "atype": 1}]} ])); let body_eids = vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_fresh".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]; let updates = collect_eid_updates(Some(&eids_cookie), None, Some(&body_eids), ®istry); diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 191e6c6d24..784c217e51 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -652,9 +652,7 @@ mod tests { if let EcKvSnapshot::Present { entry, .. } = &mut snapshot { entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid".to_owned()), ); } ec_context.set_kv_snapshot(snapshot); @@ -702,9 +700,7 @@ mod tests { entry.ids.insert( "b.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-b".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid-b".to_owned()), ); assert!( @@ -1213,9 +1209,7 @@ mod tests { entry.consent.ok = true; entry.ids.insert( "alpha.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "already-known".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("already-known".to_owned()), ); graph .create(&ec_id, &entry) diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs index 11df47296f..7b3c853074 100644 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ b/crates/trusted-server-core/src/ec/pull_sync_marker.rs @@ -375,9 +375,7 @@ mod tests { for domain in domains { entry.ids.insert( (*domain).to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: format!("uid-{domain}"), - }, + crate::ec::kv_types::KvPartnerId::single(format!("uid-{domain}")), ); } EcKvSnapshot::Present { @@ -510,9 +508,7 @@ mod tests { let mut tombstone = KvEntry::tombstone(1_000); tombstone.ids.insert( "a.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "stale".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("stale".to_owned()), ); let snapshots = [ live_snapshot(EC_ID, &[]), diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index c13260704f..33018ca519 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -7015,30 +7015,30 @@ external_bundle_sri = "sha384-AAAA" let provider = PrebidAuctionProvider::new(base_config()); let mut auction_request = create_test_auction_request(); auction_request.user.eids = Some(vec![ - crate::openrtb::Eid { - source: "liveramp.com".to_owned(), - uids: vec![crate::openrtb::Uid { + crate::openrtb::Eid::new( + "liveramp.com".to_owned(), + vec![crate::openrtb::Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, - crate::openrtb::Eid { - source: "id5-sync.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "id5-sync.com".to_owned(), + vec![crate::openrtb::Uid { id: "ID5_abc".to_owned(), atype: Some(1), ext: None, }], - }, - crate::openrtb::Eid { - source: "google.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "google.com".to_owned(), + vec![crate::openrtb::Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }, + ), ]); let settings = make_settings(); diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 67cd7f6950..645031a7b0 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -70,10 +70,37 @@ pub struct ConsentedProvidersSettings { pub struct Eid { /// Identity provider domain (e.g. `"id5-sync.com"`). pub source: String, + /// `OpenRTB` `inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option, + /// `OpenRTB` `matcher`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option, + /// `OpenRTB` `mm` (match method). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// EID-level extension data. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, /// One or more user IDs from this provider. pub uids: Vec, } +impl Eid { + /// Builds an entry with no EID-level fields. + #[must_use] + pub fn new(source: impl Into, uids: Vec) -> Self { + Self { + source: source.into(), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids, + } + } +} + /// A single user identifier within an [`Eid`] entry. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -405,16 +432,46 @@ mod tests { ); } + #[test] + fn eid_serializes_eid_level_fields_only_when_present() { + let plain = Eid::new("ids.example.com", vec![]); + assert_eq!( + serde_json::to_value(&plain).expect("should serialize"), + serde_json::json!({ "source": "ids.example.com", "uids": [] }), + "should omit absent EID-level fields" + ); + let full = Eid { + inserter: Some("inserter.example.com".to_owned()), + matcher: Some("matcher.example.com".to_owned()), + mm: Some(3), + ext: Some(serde_json::json!({ "k": 1 })), + ..Eid::new("ids.example.com", vec![]) + }; + let value = serde_json::to_value(&full).expect("should serialize"); + assert_eq!( + value["inserter"], + serde_json::json!("inserter.example.com"), + "should write inserter" + ); + assert_eq!(value["mm"], serde_json::json!(3), "should write mm"); + let back: Eid = serde_json::from_value(value).expect("should accept EID-level fields"); + assert_eq!( + back.matcher.as_deref(), + Some("matcher.example.com"), + "should read matcher" + ); + } + #[test] fn eid_serializes_correctly() { - let eid = Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5-abc123".to_owned(), atype: Some(1), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); assert_eq!(serialized["source"], "id5-sync.com", "source should match"); @@ -431,14 +488,14 @@ mod tests { #[test] fn eid_serializes_vendor_specific_atype() { - let eid = Eid { - source: "google.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "google.com".to_owned(), + vec![Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); From 00fb3b652411fbc55adc9e45ba78cdc4e2f4acb1 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 30 Oct 2026 07:33:40 +0530 Subject: [PATCH 02/38] Make every browser-driven EC write a single page-load write Top-level document navigations on the publisher route receive a single-use write permit; finalize plans one row change (browser ingest, provider results, prune, size cap) and commits it once without retry. Non-navigation requests and redirects never write, integration proxies no longer mint ECs, and new ECs are created by the page-load write instead of at generation. Withdrawal tombstones only the active EC and keeps the EC cookie when the tombstone does not land; the withdrawal completion marker is the one deliberate extra write. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 76 + .../src/auction/endpoints.rs | 32 +- .../src/ec/coordinator/mod.rs | 3 + .../src/ec/coordinator/results.rs | 147 + crates/trusted-server-core/src/ec/finalize.rs | 1204 +++++---- crates/trusted-server-core/src/ec/kv.rs | 2372 +++-------------- crates/trusted-server-core/src/ec/kv_types.rs | 15 + crates/trusted-server-core/src/ec/mod.rs | 277 +- .../trusted-server-core/src/ec/page_load.rs | 320 +++ .../trusted-server-core/src/ec/prebid_eids.rs | 5 +- .../trusted-server-core/src/ec/pull_sync.rs | 1 + crates/trusted-server-core/src/ec/registry.rs | 27 + .../trusted-server-core/src/ec/write_plan.rs | 354 +++ .../src/integrations/registry.rs | 22 +- crates/trusted-server-core/src/publisher.rs | 3 + 15 files changed, 2154 insertions(+), 2704 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/coordinator/mod.rs create mode 100644 crates/trusted-server-core/src/ec/coordinator/results.rs create mode 100644 crates/trusted-server-core/src/ec/page_load.rs create mode 100644 crates/trusted-server-core/src/ec/write_plan.rs diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index ce06ce7e30..7f75267188 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -114,6 +114,7 @@ use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; @@ -850,6 +851,11 @@ async fn dispatch_fallback( let is_navigation = is_navigation_request(&req); if is_navigation { ec.ec_context.set_eid_sync_source(EidSyncSource::Navigation); + // The page-load write permit uses the stricter Fetch Metadata test + // (spec §3.1); requests without it never write KV. + if let Some(permit) = NavigationPermit::classify(&req) { + ec.ec_context.set_navigation_permit(permit); + } } let is_publisher_navigation = ec.is_real_browser && is_navigation; if is_publisher_navigation @@ -2578,6 +2584,76 @@ mod tests { } } + fn navigation_request(path: &str) -> Request { + let mut request = browser_request(Method::GET, path, "document"); + request + .headers_mut() + .insert("sec-fetch-mode", HeaderValue::from_static("navigate")); + request + } + + fn take_finalize_state(response: &mut Response) -> super::EcFinalizeState { + response + .extensions_mut() + .remove::() + .expect("should attach finalize state") + } + + #[test] + fn publisher_navigation_receives_navigation_permit() { + let router = test_router(); + + let mut response = route(&router, navigation_request("/article")); + + let mut state = take_finalize_state(&mut response); + assert!( + state.ec_context.take_navigation_permit_for_test().is_some(), + "should issue a permit" + ); + } + + #[test] + fn publisher_request_without_fetch_metadata_navigation_gets_no_permit() { + let router = test_router(); + let mut prefetch = navigation_request("/article"); + prefetch + .headers_mut() + .insert("sec-purpose", HeaderValue::from_static("prefetch")); + + for request in [ + browser_request(Method::GET, "/article", "document"), + prefetch, + ] { + let mut response = route(&router, request); + + let mut state = take_finalize_state(&mut response); + assert!( + state.ec_context.take_navigation_permit_for_test().is_none(), + "should not issue a permit without a top-level navigation" + ); + } + } + + #[test] + fn integration_proxy_never_generates_ec() { + let router = test_router(); + + let mut response = route( + &router, + navigation_request("/integrations/prebid/bundle.js"), + ); + + let mut state = take_finalize_state(&mut response); + assert!( + !state.ec_context.ec_generated(), + "should not mint an EC on an integration proxy" + ); + assert!( + state.ec_context.take_navigation_permit_for_test().is_none(), + "should not issue a permit to an integration proxy" + ); + } + #[test] fn browser_device_signals_from_extension_reach_ec_finalize_state() { // Regression guard for the EdgeZero JA4/H2 signal loss: `edgezero_main` diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index b0121f7490..b261add7b3 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -789,10 +789,9 @@ mod tests { #[tokio::test] async fn auction_endpoint_snapshot_is_reused_by_response_finalization() { // `/auction` loads the identity-graph row to resolve server-side EIDs. - // Finalization runs afterwards on the same EC context and ingests - // `ts-eids`/`sharedId` updates. Both must be served by a single billable - // read: before the snapshot was shared, finalization saw `NotRead` and - // paid for a second lookup. + // Finalization runs afterwards on the same EC context. It must not pay + // for a second lookup, and it never writes KV: only the page load's + // navigation may write (spec D1). let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -876,21 +875,17 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should exist"); - assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("shared-cookie-id"), - "the sharedId update must still be ingested from the shared snapshot" + assert!( + stored.ids.is_empty(), + "an auction request must leave the row unchanged" ); } #[tokio::test] - async fn auction_body_eids_reach_kv_even_when_the_ts_eids_cookie_is_absent() { - // Regression test for #1184: `/auction` sends every EID it has in the - // request body, but response finalization used to ingest identity - // graph updates only from the `ts-eids` cookie — which the browser - // caps in size and may not have sent at all. `handle_auction` must - // hand its parsed body EIDs to `ec_context` so finalization ingests - // them regardless of what the cookie carried. + async fn auction_body_eids_reach_context_but_never_kv() { + // `handle_auction` hands its parsed body EIDs to `ec_context`, but body + // EIDs go to bidders on that request only: finalization of an + // `/auction` request never writes them to KV (spec §5.1, D1). let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -969,10 +964,9 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should exist"); - assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), - Some("ID5_from_body"), - "the body's EID must be ingested into KV without a ts-eids cookie" + assert!( + stored.ids.is_empty(), + "the body's EIDs must not be persisted from an auction request" ); } diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs new file mode 100644 index 0000000000..7fe8da9015 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -0,0 +1,3 @@ +//! Server-side identity lookup coordination. + +pub mod results; diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs new file mode 100644 index 0000000000..e9bc32a2d1 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -0,0 +1,147 @@ +//! Per-provider outcomes of one navigation's lookups. + +use crate::ec::eid_types::{EidValidationError, ProviderId, SourceDomain}; +use crate::ec::kv_types::KvEid; + +/// Validation drops for one response, by reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DropCounts { + /// Blank or oversized UIDs, or negative agent types. + pub invalid_uid: u16, + /// Invalid `ext`, `inserter`, `matcher` or `mm`. + pub invalid_field: u16, + /// Entries for sources the provider does not claim. + pub unclaimed_source: u16, + /// UIDs whose expiry is not after the request time. + pub expired: u16, + /// Entries or UIDs beyond the provider's shape limits. + pub over_limit: u16, + /// Duplicate UIDs within one source. + pub duplicate: u16, +} + +impl DropCounts { + /// Counts one validation failure. + pub fn record(&mut self, reason: EidValidationError) { + let counter = match reason { + EidValidationError::EmptyUid + | EidValidationError::UidTooLong + | EidValidationError::NegativeAgentType => &mut self.invalid_uid, + EidValidationError::UnknownMatchMethod + | EidValidationError::InvalidExt + | EidValidationError::FieldTooLong => &mut self.invalid_field, + EidValidationError::UnclaimedSource => &mut self.unclaimed_source, + EidValidationError::Expired => &mut self.expired, + }; + *counter = counter.saturating_add(1); + } + + /// Total drops. + #[must_use] + pub fn total(&self) -> u16 { + self.invalid_uid + .saturating_add(self.invalid_field) + .saturating_add(self.unclaimed_source) + .saturating_add(self.expired) + .saturating_add(self.over_limit) + .saturating_add(self.duplicate) + } +} + +/// Why a provider was not called. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkipReason { + /// The provider is disabled. + Disabled, + /// Consent does not permit this provider. + Consent, + /// None of its sources is due. + NotDue, + /// Every due source needs input V1 never has (hashed email). + Input, + /// The built URL is not `https` or not allowlisted. + Destination, + /// The provider's hourly rate limit is reached. + RateLimited, +} + +/// One outcome per provider per navigation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LookupStatus { + /// Valid entries were returned. + Found { + /// UIDs kept after validation. + kept: u16, + /// UIDs or entries dropped by validation. + dropped: DropCounts, + }, + /// The provider knows no ID for this visitor. + NoMatch, + /// No response before the shared deadline. + Timeout, + /// A non-success HTTP status. + HttpError(http::StatusCode), + /// The request could not be sent or the body exceeded the limit. + TransportFailed, + /// The body could not be parsed at all. + InvalidResponse, + /// The provider was not called. + Skipped(SkipReason), +} + +impl LookupStatus { + /// Metric name for logs (spec §12.2). + #[must_use] + pub fn metric_name(&self) -> &'static str { + match self { + Self::Found { .. } => "found", + Self::NoMatch => "no_match", + Self::Timeout => "timeout", + Self::HttpError(_) => "http_error", + Self::TransportFailed => "transport_error", + Self::InvalidResponse => "invalid_response", + Self::Skipped(SkipReason::Disabled) => "skipped_disabled", + Self::Skipped(SkipReason::Consent) => "skipped_consent", + Self::Skipped(SkipReason::NotDue) => "skipped_not_due", + Self::Skipped(SkipReason::Input) => "skipped_input", + Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", + Self::Skipped(SkipReason::Destination) => "destination_rejected", + } + } +} + +/// One provider's outcome and validated entries. +#[derive(Debug, Clone)] +pub struct ProviderLookup { + /// The provider. + pub provider: ProviderId, + /// What happened. + pub status: LookupStatus, + /// Validated entries per returned source; empty unless `Found`. + pub entries: Vec<(SourceDomain, Vec)>, +} + +/// Validated results of one navigation's lookups. +#[derive(Debug, Clone, Default)] +pub struct LookupResults { + lookups: Vec, +} + +impl LookupResults { + /// Wraps per-provider results in provider-config order. + #[must_use] + pub fn new(lookups: Vec) -> Self { + Self { lookups } + } + + /// Iterates results in provider-config order. + pub fn iter(&self) -> impl Iterator { + self.lookups.iter() + } + + /// Whether no provider produced a result. + #[must_use] + pub fn is_empty(&self) -> bool { + self.lookups.is_empty() + } +} diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index ce4c831602..eba972a8a6 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1,9 +1,9 @@ //! EC response finalization. //! //! Centralizes post-routing EC behavior so all handlers get consistent cookie -//! and KV semantics. - -use std::collections::HashSet; +//! and KV semantics. Every browser-driven KV write goes through the +//! navigation's single page-load write ([`KvIdentityGraph::commit_page_load`]), +//! so a request without a navigation permit never writes. use edgezero_core::body::Body as EdgeBody; use http::Response; @@ -14,16 +14,16 @@ use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; -use super::generation::{generate_ec_id, is_valid_ec_id}; -use super::kv::{ - CreateIfAbsentOutcome, EidCookieSyncOutcome, KvIdentityGraph, PartnerIdUpdate, - apply_partner_id_updates, -}; +use super::eid_types::UnixSeconds; +use super::generation::generate_ec_id; +use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; +use super::page_load::{Generation, PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; -use super::{EcKvSnapshot, EidSyncSource, current_timestamp, log_id}; +use super::write_plan::{WritePlanInput, plan_write}; +use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; /// TS-managed response headers tied to EC identity output. const EC_RESPONSE_HEADERS: &[&str] = &[ @@ -35,12 +35,18 @@ const EC_RESPONSE_HEADERS: &[&str] = &[ /// Finalizes EC response behavior for all routes. /// -/// Applies withdrawal handling, last-seen updates, cookie reconciliation, -/// Prebid EID ingestion, and cookie writes for new EC generation. +/// Applies withdrawal handling, browser EID ingestion, provider lookup +/// results, orphan recovery and cookie writes for new EC generation. +/// +/// KV is written at most once, and only when the request carries a +/// navigation permit (see [`super::page_load`]) and the final response is not +/// a redirect. Without a permit a newly generated EC is not issued and a +/// withdrawal only strips EC headers: the EC cookie is kept so the next +/// navigation can write the tombstone. /// -/// On consent withdrawal, the browser response clears the EC cookie -/// immediately and the EC identity-graph KV tombstone is the authoritative -/// revocation marker. There is no separate consent KV store to clean up. +/// On consent withdrawal during a navigation, the EC identity-graph KV +/// tombstone is the authoritative revocation marker and the EC cookie is +/// expired once the tombstone is in place (or the row is proven absent). /// /// `eids_cookie` should be the raw value of the `ts-eids` cookie extracted /// from the request *before* routing consumes it. @@ -54,6 +60,9 @@ pub fn ec_finalize_response( response: &mut Response, ) { ec_context.validate_pull_sync_marker(settings, registry); + let permit = ec_context + .take_navigation_permit() + .and_then(|permit| permit.confirm(response.status())); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); @@ -69,159 +78,175 @@ pub fn ec_finalize_response( ec_context, kv, registry, + permit, consent_withdrawn, response, ); return; } - // Returning user: consent is granted and EC came from request. - if ec_context.ec_was_present() && !ec_context.ec_generated() && consent_allows_ec { - if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { - let source = ec_context.eid_sync_source(); - let collected = source - .map(|_| { - collect_eid_updates( - eids_cookie, - sharedid_cookie, - ec_context.client_eids(), - registry, - ) - }) - .unwrap_or_default(); - let had_eid_updates = !collected.is_empty(); - let updates = gate_eid_updates_by_consent(collected, ec_context.consent()); - // `sync_eid_cookie_updates` early-returns without refreshing the - // snapshot when `updates` is empty, which is correct when there - // was never anything to write (an unconfigured registry, no sync - // source, or no EID cookies/body this request — the case - // `finalize_not_read_snapshot_does_not_rotate` covers). But when - // consent gating (e.g. TCF Purpose 4 denial) is what emptied - // `updates`, the request *did* have EID data, and orphan recovery - // below still needs an actual `Missing` read to detect an - // orphaned cookie. An unread `NotRead` snapshot proves nothing, so - // consent gating must not also skip that one read on a - // recovery-eligible request. - if updates.is_empty() - && had_eid_updates - && ec_context.recovery_eligible() - && matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead) - { - let snapshot = graph.load_snapshot(&ec_id); - ec_context.set_kv_snapshot(snapshot); - } else if let Some(source) = source { - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, source); - } - if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) - && ec_context.recovery_eligible() - { - confirm_then_recover_orphaned_ec( - settings, ec_context, graph, &ec_id, &updates, response, - ); - } - } - - reconcile_pull_sync_marker(settings, registry, ec_context, response); - - // Ordinary returning-user page views no longer refresh the browser - // cookie, emit the EC header, or update KV TTL. - return; - } - - // Newly generated EC in this request. Do not emit a generated EC when - // there is no KV graph: that would mint a browser cookie with no backing - // identity-graph row, producing a phantom ID on later requests. - if ec_context.ec_generated() { - let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { - log::info!("Skipping generated EC response write because KV graph is unavailable"); - reconcile_pull_sync_marker(settings, registry, ec_context, response); - return; - }; - - let updates = gate_eid_updates_by_consent( + if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { + let collected = if ec_context.eid_sync_source().is_some() || ec_context.ec_generated() { collect_eid_updates( eids_cookie, sharedid_cookie, ec_context.client_eids(), registry, - ), - ec_context.consent(), - ); - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, EidSyncSource::NewEc); - if ec_context.kv_snapshot().entry_for(&ec_id).is_some() { - set_ec_cookie_on_response(settings, ec_context, response); + ) } else { - log::warn!("Skipping generated EC cookie because backing row is not authoritative"); + Vec::new() + }; + let had_eid_updates = !collected.is_empty(); + let updates = gate_eid_updates_by_consent(collected, ec_context.consent()); + let commit = PageLoadCommit { + settings, + graph, + registry, + ec_id: &ec_id, + updates: &updates, + had_eid_updates, + }; + if ec_context.ec_generated() { + commit_generated(&commit, ec_context, permit, response); + } else if ec_context.ec_was_present() { + commit_returning(&commit, ec_context, permit, response); } + } else if ec_context.ec_generated() { + // Do not emit a generated EC when there is no KV graph: that would + // mint a browser cookie with no backing identity-graph row. + log::info!("Skipping generated EC response write because KV graph is unavailable"); } reconcile_pull_sync_marker(settings, registry, ec_context, response); } -fn sync_eid_cookie_updates( - graph: &KvIdentityGraph, - ec_context: &mut EcContext, - ec_id: &str, - updates: &[PartnerIdUpdate], - source: EidSyncSource, -) { - if updates.is_empty() { - return; +/// Shared inputs for the navigation's single write. +struct PageLoadCommit<'a> { + settings: &'a Settings, + graph: &'a KvIdentityGraph, + registry: &'a PartnerRegistry, + ec_id: &'a str, + /// Browser ingest, already consent-gated. + updates: &'a [PartnerIdUpdate], + /// Whether the request carried EID data before consent gating. + had_eid_updates: bool, +} + +impl PageLoadCommit<'_> { + /// Plans the row change for `ec_id` against `snapshot`. + fn plan( + &self, + ec_context: &EcContext, + ec_id: &str, + snapshot: &EcKvSnapshot, + new_entry: Option<&KvEntry>, + ) -> Option { + plan_write(&WritePlanInput { + ec_id, + snapshot, + new_entry, + browser_ingest: self.updates, + lookups: ec_context.lookup_results(), + registry: self.registry, + now: UnixSeconds::new(current_timestamp()), + }) } +} - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - ec_id, - updates, - ec_context.kv_snapshot().clone(), - ); - ec_context.set_kv_snapshot(snapshot); - record_eid_sync_terminal(source, outcome); +fn present_snapshot(ec_id: &str, entry: KvEntry) -> EcKvSnapshot { + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry), + generation: None, + } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct EidSyncMeasurement { - source: EidSyncSource, - outcome: EidCookieSyncOutcome, - already_matched: u8, - written: u8, - conflict_duplicate: u8, - deferred: u8, +/// Creates the row of a newly generated EC and issues its cookie on success. +fn commit_generated( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: Option, + response: &mut Response, +) { + let Some(new_entry) = ec_context.take_pending_entry() else { + log::warn!("Skipping generated EC cookie because no pending row was built"); + return; + }; + let Some(permit) = permit else { + log::info!("Skipping generated EC: not a page-load navigation"); + return; + }; + let Some(change) = commit.plan( + ec_context, + commit.ec_id, + ec_context.kv_snapshot(), + Some(&new_entry), + ) else { + log::warn!("Skipping generated EC: no row change was planned"); + return; + }; + match commit.graph.commit_page_load(permit, commit.ec_id, change) { + Ok(committed) => { + ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); + set_ec_cookie_on_response(commit.settings, ec_context, response); + } + Err(report) => { + log::warn!( + "Page-load write failed for new EC '{}': {}", + log_id(commit.ec_id), + report.current_context() + ); + } + } } -impl EidSyncMeasurement { - fn new(source: EidSyncSource, outcome: EidCookieSyncOutcome) -> Self { - Self { - source, - outcome, - already_matched: u8::from(matches!(outcome, EidCookieSyncOutcome::AlreadyMatched)), - written: u8::from(matches!( - outcome, - EidCookieSyncOutcome::Written | EidCookieSyncOutcome::WrittenWithDeferredFreshness - )), - conflict_duplicate: u8::from(matches!(outcome, EidCookieSyncOutcome::ConflictMatched)), - deferred: u8::from(matches!( - outcome, - EidCookieSyncOutcome::WrittenWithDeferredFreshness - | EidCookieSyncOutcome::DeferredConflict - | EidCookieSyncOutcome::DeferredFreshness - | EidCookieSyncOutcome::DeferredStaleRead - )), +/// Applies a returning EC's page-load write, or recovers an orphaned EC. +fn commit_returning( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: Option, + response: &mut Response, +) { + let Some(permit) = permit else { + return; + }; + // An unread snapshot is read only when this navigation has something to + // write, or when consent gating emptied real EID data on a request that + // may still need orphan recovery. A navigation with nothing to do costs + // no KV operation. + let has_changes = !commit.updates.is_empty() || !ec_context.lookup_results().is_empty(); + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead) + && (has_changes || (commit.had_eid_updates && ec_context.recovery_eligible())) + { + let snapshot = commit.graph.load_snapshot(commit.ec_id); + ec_context.set_kv_snapshot(snapshot); + } + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) { + if ec_context.recovery_eligible() { + confirm_then_recover_orphaned_ec(commit, ec_context, permit, response); } + return; } + update_returning(commit, ec_context, permit); } -fn record_eid_sync_terminal(source: EidSyncSource, outcome: EidCookieSyncOutcome) { - let measurement = EidSyncMeasurement::new(source, outcome); - log::info!( - "EID sync measurement: source={} outcome={} attempted=1 already_matched={} written={} \ - conflict_duplicate={} deferred={}", - measurement.source, - measurement.outcome, - measurement.already_matched, - measurement.written, - measurement.conflict_duplicate, - measurement.deferred, - ); +/// Writes the planned update for a returning EC whose row snapshot is held. +fn update_returning( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: PageLoadWrite, +) { + let Some(change) = commit.plan(ec_context, commit.ec_id, ec_context.kv_snapshot(), None) else { + return; + }; + match commit.graph.commit_page_load(permit, commit.ec_id, change) { + Ok(committed) => ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)), + Err(report) => log::info!( + "Page-load write skipped for '{}': {}", + log_id(commit.ec_id), + report.current_context() + ), + } } fn reconcile_pull_sync_marker( @@ -252,12 +277,11 @@ fn reconcile_pull_sync_marker( /// the outbound `/auction` bid request, evaluated here through /// [`allows_eid_persistence`]. Without this, a user who denies Purpose 4 /// would have their EIDs correctly stripped from the bid request but still -/// written to the identity graph from the `ts-eids`/`sharedId` cookies or the -/// `/auction` request body. +/// written to the identity graph from the `ts-eids`/`sharedId` cookies. fn gate_eid_updates_by_consent( - updates: Vec, + updates: Vec, consent: &ConsentContext, -) -> Vec { +) -> Vec { if updates.is_empty() || allows_eid_persistence(consent) { return updates; } @@ -268,75 +292,60 @@ fn gate_eid_updates_by_consent( Vec::new() } +/// Replaces a proven-orphaned EC with a new ID whose row is created by this +/// navigation's single write. One attempt: a collision leaves the orphan in +/// place for a later navigation. fn recover_orphaned_ec( - settings: &Settings, + commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, - graph: &KvIdentityGraph, - updates: &[super::kv::PartnerIdUpdate], + permit: PageLoadWrite, response: &mut Response, ) { - // Snapshot the orphaned ID once so every fail-closed exit binds the failed - // snapshot to the same key. - let Some(orphan_id) = ec_context.ec_value().map(str::to_owned) else { - return; - }; + let orphan_id = commit.ec_id.to_owned(); let Some(client_ip) = ec_context.client_ip().map(str::to_owned) else { log::warn!("Orphan EC recovery skipped because client IP is unavailable"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); return; }; + let new_id = match generate_ec_id(commit.settings, &client_ip) { + Ok(ec_id) => ec_id, + Err(err) => { + log::warn!("Orphan EC recovery ID generation failed: {err:?}"); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); + return; + } + }; + let mut new_entry = KvEntry::new( + ec_context.consent(), + ec_context.geo_info(), + current_timestamp(), + &commit.settings.publisher.domain, + ); + new_entry.device = ec_context + .device_signals() + .map(super::device::DeviceSignals::to_kv_device); - const MAX_RECOVERY_ATTEMPTS: usize = 5; - for _attempt in 0..MAX_RECOVERY_ATTEMPTS { - let ec_id = match generate_ec_id(settings, &client_ip) { - Ok(ec_id) => ec_id, - Err(err) => { - log::warn!("Orphan EC recovery ID generation failed: {err:?}"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); - return; - } - }; - let mut entry = KvEntry::new( - ec_context.consent(), - ec_context.geo_info(), - current_timestamp(), - &settings.publisher.domain, - ); - entry.device = ec_context - .device_signals() - .map(super::device::DeviceSignals::to_kv_device); - apply_partner_id_updates(&mut entry, updates); - - match graph.create_if_absent(&ec_id, &entry) { - Ok(CreateIfAbsentOutcome::Written) => { - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: None, - }; - ec_context.replace_with_generated(ec_id, snapshot); - set_ec_cookie_on_response(settings, ec_context, response); - return; - } - Ok(CreateIfAbsentOutcome::AlreadyExists) => continue, - Err(err) => { - log::warn!("Orphan EC recovery failed: {err:?}"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); - return; - } + let missing = EcKvSnapshot::Missing { + ec_id: new_id.clone(), + }; + let Some(change) = commit.plan(ec_context, &new_id, &missing, Some(&new_entry)) else { + log::warn!("Orphan EC recovery skipped: no row change was planned"); + return; + }; + match commit.graph.commit_page_load(permit, &new_id, change) { + Ok(committed) => { + let snapshot = present_snapshot(&new_id, committed); + ec_context.replace_with_generated(new_id, snapshot); + set_ec_cookie_on_response(commit.settings, ec_context, response); + } + Err(report) => { + log::warn!( + "Orphan EC recovery write failed: {}", + report.current_context() + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); } } - - log::warn!("Orphan EC recovery exhausted collision retries"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); } /// Proves an orphaned cookie is genuinely absent before rotating it. @@ -348,8 +357,8 @@ fn recover_orphaned_ec( /// POP that has not converged. Two point reads do not fix that — both can be /// stale — so absence has to be *proved*, not observed twice: /// -/// - a row that became visible after the origin round trip is adopted, with any -/// pending updates merged, and is never rotated; +/// - a row that became visible after the origin round trip is adopted, and the +/// navigation's single write is planned against it; it is never rotated; /// - a second miss is escalated to /// [`key_exists_confirmed`](KvIdentityGraph::key_exists_confirmed), which /// reads the primary data source. Only a proven-absent key rotates; @@ -358,23 +367,22 @@ fn recover_orphaned_ec( /// - neither a read failure nor a failed existence check is a miss, and neither /// rotates. fn confirm_then_recover_orphaned_ec( - settings: &Settings, + commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, - graph: &KvIdentityGraph, - ec_id: &str, - updates: &[super::kv::PartnerIdUpdate], + permit: PageLoadWrite, response: &mut Response, ) { - let confirmed = graph.load_snapshot(ec_id); + let confirmed = commit.graph.load_snapshot(commit.ec_id); match confirmed { EcKvSnapshot::Present { .. } => { // The row became visible after the origin round trip: adopt it and - // merge any pending updates rather than rotating a valid identity. + // plan the navigation's write against it rather than rotating a + // valid identity. ec_context.set_kv_snapshot(confirmed); - sync_eid_cookie_updates(graph, ec_context, ec_id, updates, EidSyncSource::Navigation); + update_returning(commit, ec_context, permit); } - EcKvSnapshot::Missing { .. } => match graph.key_exists_confirmed(ec_id) { - Ok(false) => recover_orphaned_ec(settings, ec_context, graph, updates, response), + EcKvSnapshot::Missing { .. } => match commit.graph.key_exists_confirmed(commit.ec_id) { + Ok(false) => recover_orphaned_ec(commit, ec_context, permit, response), Ok(true) => { log::warn!( "Orphan EC recovery skipped: both point reads missed a row the store still \ @@ -439,15 +447,18 @@ pub fn clear_ec_on_response(settings: &Settings, response: &mut Response, registry: &PartnerRegistry, + permit: Option, consent_withdrawn: bool, response: &mut Response, ) { @@ -456,69 +467,111 @@ fn finalize_unusable_consent( if !(consent_withdrawn && ec_context.cookie_was_present()) { return; } - - expire_ec_cookie(settings, response); - - // Compute once for the authoritative identity-graph tombstones. - let ids_to_withdraw = withdrawal_ec_ids(ec_context); - - // The identity-graph tombstone is the authoritative withdrawal marker - // for subsequent EC behavior. - if let Some(graph) = kv { - apply_withdrawal_tombstones(&ids_to_withdraw, |ec_id| { - // The graph hands back the post-withdrawal snapshot rather than - // leaving the caller to rebuild it, so post-send work that reads - // the context — pull sync discloses the raw EC ID to partners — - // sees the tombstone that was just written. Only the active ID has - // a snapshot in the context to correct. - let initial = if ec_context.kv_snapshot().belongs_to(ec_id) { - ec_context.kv_snapshot().clone() - } else { - EcKvSnapshot::NotRead - }; - let outcome = graph.tombstone_existing_from_snapshot(ec_id, initial); - // The browser cookie is already cleared, so a failed tombstone - // leaves a live row that server-side consumers still read as - // consented. Report every failure, including the non-active cookie - // ID whose outcome is not retained on the request context. - if matches!(outcome, EcKvSnapshot::Failed { .. }) { - log::warn!( - "EC withdrawal tombstone failed for '{}': the identity-graph row may \ - still be live with consent granted", - log_id(ec_id) - ); - } - if ec_context.ec_value() == Some(ec_id) { - ec_context.set_kv_snapshot(outcome); - } - }); + let Some(permit) = permit else { + // Not a page-load navigation: serve nothing, keep the EC cookie so the + // next navigation can write the tombstone (spec §3.1). + return; + }; + let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { + // No identity graph (or no valid active EC) holds a row to tombstone, + // so the cookie is the only identity state left to clear. + expire_ec_cookie(settings, response); + return; + }; + if withdraw_active_ec(graph, ec_context, &ec_id, permit) { + expire_ec_cookie(settings, response); } } -fn withdrawal_ec_ids(ec_context: &EcContext) -> HashSet { - let mut hashes = HashSet::new(); - - if let Some(cookie_ec_id) = ec_context.existing_cookie_ec_id() - && is_valid_ec_id(cookie_ec_id) - { - hashes.insert(cookie_ec_id.to_owned()); +/// Tombstones the active EC's row with the navigation's single write. +/// +/// Returns whether the withdrawal is complete: the row is now a tombstone or +/// is proven absent. A snapshot that is not a usable live row with a CAS +/// generation is re-read once; a miss is confirmed against the primary data +/// source so a stale read never clears the cookie of a live row. +fn withdraw_active_ec( + graph: &KvIdentityGraph, + ec_context: &mut EcContext, + ec_id: &str, + permit: PageLoadWrite, +) -> bool { + let usable = match ec_context.kv_snapshot().entry_for(ec_id) { + Some(entry) => { + !entry.consent.ok || Generation::of(ec_context.kv_snapshot(), ec_id).is_some() + } + None => false, + }; + if !usable { + let snapshot = graph.load_snapshot(ec_id); + ec_context.set_kv_snapshot(snapshot); } - if let Some(active_ec_id) = ec_context.ec_value() - && is_valid_ec_id(active_ec_id) - { - hashes.insert(active_ec_id.to_owned()); + let snapshot = ec_context.kv_snapshot(); + if let Some(entry) = snapshot.entry_for(ec_id) { + if !entry.consent.ok { + return true; + } + let Some(expected) = Generation::of(snapshot, ec_id) else { + log::warn!( + "EC withdrawal tombstone for '{}' skipped (no_generation); keeping the cookie", + log_id(ec_id) + ); + return false; + }; + return match graph.commit_page_load(permit, ec_id, RowChange::Tombstone { expected }) { + Ok(tombstone) => { + ec_context.set_kv_snapshot(present_snapshot(ec_id, tombstone)); + true + } + Err(report) => { + log::warn!( + "EC withdrawal tombstone failed for '{}': {}; keeping the cookie", + log_id(ec_id), + report.current_context() + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }); + false + } + }; } - - hashes -} - -fn apply_withdrawal_tombstones(ec_ids: &HashSet, mut write_tombstone: F) -where - F: FnMut(&str), -{ - for ec_id in ec_ids { - write_tombstone(ec_id); + if !matches!(snapshot, EcKvSnapshot::Missing { .. }) { + return false; + } + // A still-valid completion marker proves an earlier withdrawal finished, + // so the root existence check can be skipped. + match graph.withdrawal_marker_exists(ec_id, checked_current_timestamp()) { + Ok(true) => return true, + Ok(false) => {} + Err(err) => log::warn!( + "EC withdrawal completion marker lookup failed for '{}': {err:?}", + log_id(ec_id) + ), + } + match graph.key_exists_confirmed(ec_id) { + Ok(false) => true, + Ok(true) => { + log::warn!( + "EC withdrawal for '{}': point read missed a row the store still lists; \ + keeping the cookie for a later navigation", + log_id(ec_id) + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }); + false + } + Err(err) => { + log::warn!( + "EC withdrawal for '{}': existence check failed: {err:?}", + log_id(ec_id) + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }); + false + } } } @@ -531,6 +584,8 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::EidSyncSource; + use crate::ec::page_load::NavigationPermit; use crate::openrtb::{Eid, Uid}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -620,90 +675,6 @@ mod tests { } } - #[test] - fn withdrawal_ec_ids_returns_cookie_ec_only_when_active_missing() { - let cookie_ec = sample_ec_id("cook1e"); - let ec_context = make_context(None, Some(&cookie_ec), true, false, Jurisdiction::Unknown); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should include exactly one EC ID"); - assert!( - ids.contains(&cookie_ec), - "should include the cookie EC value" - ); - } - - #[test] - fn withdrawal_ec_ids_deduplicates_matching_cookie_and_active_ec() { - let ec_id = sample_ec_id("same01"); - let ec_context = make_context( - Some(&ec_id), - Some(&ec_id), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should deduplicate identical EC IDs"); - assert!(ids.contains(&ec_id), "should retain the shared EC ID"); - } - - #[test] - fn withdrawal_ec_ids_includes_both_cookie_and_active_when_different() { - let active_ec = sample_ec_id("activ1"); - let cookie_ec = sample_ec_id("cook1e"); - let ec_context = make_context( - Some(&active_ec), - Some(&cookie_ec), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 2, "should include both distinct EC IDs"); - assert!(ids.contains(&active_ec), "should include active EC ID"); - assert!(ids.contains(&cookie_ec), "should include cookie EC ID"); - } - - #[test] - fn withdrawal_ec_ids_filters_invalid_values() { - let valid_ec = sample_ec_id("valid1"); - let ec_context = make_context( - Some(&valid_ec), - Some("not-an-ec-id"), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should ignore malformed EC values"); - assert!(ids.contains(&valid_ec), "should keep the valid EC ID"); - } - - #[test] - fn apply_withdrawal_tombstones_invokes_writer_for_each_ec_id() { - let first = sample_ec_id("first1"); - let second = sample_ec_id("second"); - let mut ids = HashSet::new(); - ids.insert(first.clone()); - ids.insert(second.clone()); - - let mut written = Vec::new(); - apply_withdrawal_tombstones(&ids, |ec_id| written.push(ec_id.to_owned())); - written.sort(); - - let mut expected = vec![first, second]; - expected.sort(); - assert_eq!(written, expected, "should write a tombstone for each EC ID"); - } - #[test] fn clear_ec_on_response_removes_headers_and_expires_cookie() { let settings = create_test_settings(); @@ -755,6 +726,7 @@ mod tests { let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); let kv = KvIdentityGraph::in_memory("test-store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); @@ -807,6 +779,7 @@ mod tests { ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. } )); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); @@ -837,10 +810,10 @@ mod tests { } #[test] - fn withdrawal_still_expires_the_cookie_when_the_store_is_unavailable() { - // Cookie expiry is the primary enforcement, so it has to survive a - // store that cannot answer at all — the case where the identity-graph - // marker is exactly what goes missing. + fn withdrawal_keeps_the_cookie_when_the_tombstone_write_fails() { + // A failed tombstone write is not retried. The EC cookie is kept so the + // next navigation can still write the tombstone; clearing it would + // orphan a live row (spec §9.2). let settings = create_test_settings(); let ec_id = sample_ec_id("dead01"); let consent = ConsentContext { @@ -856,6 +829,7 @@ mod tests { entry: Box::new(live_entry()), generation: Some(1), }); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let kv = KvIdentityGraph::failing("test-store"); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); @@ -871,10 +845,9 @@ mod tests { &mut response, ); - let set_cookie = get_header_str(&response, "set-cookie").unwrap_or_default(); assert!( - set_cookie.contains("Max-Age=0"), - "should expire the EC cookie even when the store is unavailable: {set_cookie}" + get_header(&response, "set-cookie").is_none(), + "should keep the EC cookie when the tombstone write fails" ); assert!( get_header(&response, "x-ts-ec").is_none(), @@ -898,6 +871,7 @@ mod tests { }; let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); set_header(&mut response, "x-ts-eids", "[]"); @@ -1065,6 +1039,7 @@ mod tests { ec_id: orphaned_ec.clone(), }); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1126,6 +1101,7 @@ mod tests { let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1181,6 +1157,7 @@ mod tests { partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1248,10 +1225,22 @@ mod tests { } #[test] - fn finalize_navigation_routes_persist_returning_user_eid_updates() { - for (source, suffix, cookie_id) in [ - (EidSyncSource::Navigation, "naveid", "navigation-cookie-id"), - (EidSyncSource::PageBids, "spaeid", "page-bids-cookie-id"), + fn finalize_persists_returning_user_eid_updates_only_on_navigation() { + // `/_ts/page-bids` belongs to a page load whose navigation already + // made the write, so it never receives a permit (spec D1). + for (source, suffix, cookie_id, permit) in [ + ( + EidSyncSource::Navigation, + "naveid", + "navigation-cookie-id", + true, + ), + ( + EidSyncSource::PageBids, + "spaeid", + "page-bids-cookie-id", + false, + ), ] { let settings = create_test_settings(); let ec_id = sample_ec_id(suffix); @@ -1265,7 +1254,9 @@ mod tests { graph .create(&ec_id, &live) .expect("should seed the live row"); - let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + // A recovery-eligible context is a navigation and carries a permit. + let mut ec_context = + returning_user_context(&ec_id, graph.load_snapshot(&ec_id), permit); ec_context.set_eid_sync_source(source); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); @@ -1287,8 +1278,8 @@ mod tests { .expect("row should remain"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some(cookie_id), - "{source} should persist the returning-user EID cookie" + permit.then_some(cookie_id), + "{source} should persist the returning-user EID cookie only with a permit" ); } } @@ -1296,20 +1287,17 @@ mod tests { #[test] fn finalize_generated_ec_persists_eid_updates() { let settings = create_test_settings(); - let ec_id = sample_ec_id("geneid"); let graph = KvIdentityGraph::in_memory("test_store"); - let live = KvEntry::new( - &granting_consent(), + let mut ec_context = EcContext::new_for_test_with_ip( None, - current_timestamp(), - &settings.publisher.domain, + granting_consent(), + Some("192.0.2.10".to_owned()), ); - graph - .create(&ec_id, &live) - .expect("should seed generated row"); - let mut ec_context = - make_context(Some(&ec_id), None, false, true, Jurisdiction::NonRegulated); - ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint a new EC"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1327,114 +1315,18 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should create the row"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("generated-cookie-id") - ); - } - - #[test] - fn eid_sync_measurement_dimensions_are_bounded_and_identity_free() { - let sources = [ - EidSyncSource::Navigation, - EidSyncSource::Auction, - EidSyncSource::PageBids, - EidSyncSource::NewEc, - ]; - let outcomes = [ - EidCookieSyncOutcome::AlreadyMatched, - EidCookieSyncOutcome::Written, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - EidCookieSyncOutcome::ConflictMatched, - EidCookieSyncOutcome::DeferredConflict, - EidCookieSyncOutcome::DeferredFreshness, - EidCookieSyncOutcome::DeferredStaleRead, - EidCookieSyncOutcome::Missing, - EidCookieSyncOutcome::ConsentWithdrawn, - EidCookieSyncOutcome::Failed, - ]; - - assert_eq!( - sources.map(|source| source.to_string()), - ["navigation", "auction", "page_bids", "new_ec"] - ); - assert_eq!( - outcomes.map(|outcome| outcome.to_string()), - [ - "already_matched", - "written", - "written_with_deferred_freshness", - "conflict_matched", - "deferred_conflict", - "deferred_freshness", - "deferred_stale_read", - "missing", - "consent_withdrawn", - "failed", - ] - ); - - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Navigation, - EidCookieSyncOutcome::AlreadyMatched, - ), - EidSyncMeasurement { - source: EidSyncSource::Navigation, - outcome: EidCookieSyncOutcome::AlreadyMatched, - already_matched: 1, - written: 0, - conflict_duplicate: 0, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Auction, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - ), - EidSyncMeasurement { - source: EidSyncSource::Auction, - outcome: EidCookieSyncOutcome::WrittenWithDeferredFreshness, - already_matched: 0, - written: 1, - conflict_duplicate: 0, - deferred: 1, - } - ); - assert_eq!( - EidSyncMeasurement::new(EidSyncSource::NewEc, EidCookieSyncOutcome::ConflictMatched,), - EidSyncMeasurement { - source: EidSyncSource::NewEc, - outcome: EidCookieSyncOutcome::ConflictMatched, - already_matched: 0, - written: 0, - conflict_duplicate: 1, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::PageBids, - EidCookieSyncOutcome::DeferredStaleRead, - ), - EidSyncMeasurement { - source: EidSyncSource::PageBids, - outcome: EidCookieSyncOutcome::DeferredStaleRead, - already_matched: 0, - written: 0, - conflict_duplicate: 0, - deferred: 1, - } + Some("generated-cookie-id"), + "should ingest the browser EID in the create" ); } #[test] - fn finalize_auction_transient_miss_still_persists_eid_updates() { - // `/auction` saves its first lookup into the context and is never - // recovery eligible, so a stale miss there has no later chance to - // retry. Finalization must revalidate before dropping collected IDs. + fn finalize_auction_transient_miss_does_not_persist_eid_updates() { + // `/auction` never writes KV (spec D1): browser EIDs are persisted only + // by the page load's navigation write, whatever its snapshot says. let settings = create_test_settings(); let ec_id = sample_ec_id("named1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1474,20 +1366,17 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should remain"); - assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("shared-cookie-id"), - "a stale endpoint miss must not suppress EID persistence" + assert!( + stored.ids.is_empty(), + "an auction request must leave the row unchanged" ); } #[test] - fn finalize_persists_every_configured_partner_from_client_eids_over_a_trimmed_cookie() { - // Regression test for #1184: the `ts-eids` cookie only carries what - // the browser could fit under its size cap, but `/auction` also - // hands finalization the full EID set from the request body via - // `EcContext::set_client_eids`. That full set must land in KV even - // when the cookie alone would have dropped a configured partner. + fn finalize_never_persists_auction_body_eids() { + // `/auction` body EIDs go to bidders on that request only and are + // never persisted (spec §5.1); neither is the `ts-eids` cookie on a + // non-navigation request. let settings = create_test_settings(); let ec_id = sample_ec_id("cleids1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1551,16 +1440,10 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should exist after ingestion"); - assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), - Some("ID5_from_body"), - "the request body's EID should win over the cookie's stale value" - ); - assert_eq!( - stored.ids.get("liveramp.com").map(|id| id.uid.as_str()), - Some("LR_from_body"), - "a partner the cookie trimmed must still be ingested from the body" + .expect("row should remain"); + assert!( + stored.ids.is_empty(), + "should persist neither the auction body nor the cookie EIDs" ); } @@ -1591,10 +1474,9 @@ mod tests { false, purpose1_only_consent, ); - ec_context.set_kv_snapshot(EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }); - ec_context.set_eid_sync_source(EidSyncSource::Auction); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_navigation_permit(NavigationPermit::for_test()); ec_context.set_client_eids(vec![Eid::new( "id5-sync.com".to_owned(), vec![Uid { @@ -1870,6 +1752,7 @@ mod tests { ec_id: generated_ec, }); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1978,6 +1861,11 @@ mod tests { Some("192.0.2.10".to_owned()), ); ec.set_recovery_eligible(recovery_eligible); + if recovery_eligible { + // Only a top-level navigation is recovery eligible, and only a + // navigation carries the page-load write permit. + ec.set_navigation_permit(NavigationPermit::for_test()); + } ec.set_kv_snapshot(snapshot); ec } @@ -2121,6 +2009,7 @@ mod tests { .create(&active_ec, &live_entry()) .expect("should seed active row"); ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -2148,7 +2037,7 @@ mod tests { } #[test] - fn finalize_withdrawal_tombstones_both_present_ids_once() { + fn finalize_withdrawal_tombstones_only_the_active_id_once() { let settings = create_test_settings(); let active_ec = sample_ec_id("activ3"); let cookie_ec = sample_ec_id("cook3e"); @@ -2174,6 +2063,7 @@ mod tests { ) .expect("should seed cookie row"); ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -2190,10 +2080,10 @@ mod tests { .get(&active_ec) .expect("should read active row") .expect("should retain active tombstone"); - let (cookie_tombstone, cookie_generation) = graph + let (cookie_row, cookie_generation) = graph .get(&cookie_ec) .expect("should read cookie row") - .expect("should retain cookie tombstone"); + .expect("should retain cookie row"); assert!( !active_tombstone.consent.ok, "active row should be withdrawn" @@ -2203,14 +2093,11 @@ mod tests { "active IDs should be cleared" ); assert!( - !cookie_tombstone.consent.ok, - "cookie row should be withdrawn" - ); - assert!( - cookie_tombstone.ids.is_empty(), - "cookie IDs should be cleared" + cookie_row.consent.ok, + "the navigation's single write should tombstone only the active row" ); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut repeated_response = empty_response(); ec_finalize_response( &settings, @@ -2235,15 +2122,15 @@ mod tests { graph .get(&cookie_ec) .expect("should read cookie row") - .expect("should retain cookie tombstone") + .expect("should retain cookie row") .1, cookie_generation, - "repeated finalization should not rewrite cookie tombstone" + "repeated finalization should not rewrite the cookie row" ); } #[test] - fn finalize_withdrawal_keeps_cookie_deletion_on_kv_failure() { + fn finalize_withdrawal_keeps_ec_cookie_but_expires_marker_on_kv_failure() { let settings = create_test_settings(); let ec_id = sample_ec_id("failw1"); let consent = ConsentContext { @@ -2257,6 +2144,7 @@ mod tests { ec_context.set_pull_sync_marker_for_test( crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, ); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let graph = KvIdentityGraph::failing("unavailable-store"); let mut response = empty_response(); @@ -2282,10 +2170,8 @@ mod tests { "KV failure should not change response status" ); assert!( - cookies - .iter() - .any(|cookie| { cookie.starts_with("ts-ec=;") && cookie.contains("Max-Age=0") }), - "KV failure should not prevent EC cookie deletion" + cookies.iter().all(|cookie| !cookie.starts_with("ts-ec=;")), + "KV failure should keep the EC cookie so a later navigation can tombstone the row" ); assert!( cookies.iter().any(|cookie| { @@ -2419,6 +2305,276 @@ mod tests { ); } + fn registry_for(source: &str) -> PartnerRegistry { + PartnerRegistry::from_config(&[make_partner(source)]).expect("should build registry") + } + + fn eids_cookie(source: &str, uid: &str) -> String { + base64::engine::general_purpose::STANDARD.encode( + serde_json::json!([{ "source": source, "uids": [{ "id": uid, "atype": 1 }] }]) + .to_string(), + ) + } + + #[test] + fn auction_request_never_writes_browser_eids() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("auc001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Auction); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!( + entry.ids.is_empty(), + "should not write from a non-navigation request" + ); + } + + #[test] + fn navigation_writes_browser_eids_once() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("nav001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert_eq!( + entry.ids["partner.example.com"].uid, "u-1", + "should persist browser uid" + ); + assert!( + ec_context.take_navigation_permit().is_none(), + "should spend the permit" + ); + } + + #[test] + fn redirect_navigation_does_not_write() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("red001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + *response.status_mut() = http::StatusCode::FOUND; + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!(entry.ids.is_empty(), "should not write on a redirect"); + } + + #[test] + fn generated_ec_without_permit_gets_no_cookie_or_row() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + graph.get(&ec_id).expect("should read").is_none(), + "should not create without a permit" + ); + assert!( + get_header(&response, "set-cookie").is_none(), + "should not issue the EC" + ); + } + + #[test] + fn generated_ec_with_permit_creates_row_and_sets_cookie() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + graph.get(&ec_id).expect("should read").is_some(), + "should create the row" + ); + assert!( + get_header(&response, "set-cookie").is_some(), + "should issue the EC" + ); + } + + #[test] + fn generated_ec_collision_is_not_retried_or_issued() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed a colliding row"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + get_header(&response, "set-cookie").is_none(), + "should not issue an EC whose row was not created" + ); + } + + #[test] + fn withdrawal_without_permit_keeps_row_and_cookie() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + let mut response = empty_response(); + set_header(&mut response, "x-ts-ec", "stale"); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!( + entry.consent.ok, + "should not tombstone outside a navigation" + ); + assert!( + get_header(&response, "set-cookie").is_none(), + "should keep the EC cookie" + ); + assert!( + get_header(&response, "x-ts-ec").is_none(), + "should still strip EC response headers" + ); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 9daaac243c..434c38bbb9 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -22,7 +22,8 @@ use crate::error::TrustedServerError; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; use super::kv_types::{KvEntry, KvMetadata, KvNetwork, MAX_ENTRY_BYTES}; -use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; +use super::page_load::{KvWriteError, PageLoadWrite, RowChange}; +use super::{EcKvSnapshot, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. const MAX_CAS_RETRIES: u32 = 5; @@ -72,16 +73,16 @@ pub enum CreateIfAbsentOutcome { /// Partner UID update to apply to a KV identity graph entry. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PartnerIdUpdate { - /// Partner namespace key in [`KvEntry::ids`]. - pub(crate) partner_id: String, - /// Partner-scoped user ID value. - pub(crate) uid: String, +pub struct PartnerIdUpdate { + /// Source domain key in [`KvEntry::ids`]. + pub partner_id: String, + /// UID to fill in. + pub uid: String, } impl PartnerIdUpdate { /// Creates a partner UID update. - pub(crate) fn new(partner_id: impl Into, uid: impl Into) -> Self { + pub fn new(partner_id: impl Into, uid: impl Into) -> Self { Self { partner_id: partner_id.into(), uid: uid.into(), @@ -89,102 +90,6 @@ impl PartnerIdUpdate { } } -/// Terminal result of one browser EID-cookie persistence attempt. -#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] -pub(crate) enum EidCookieSyncOutcome { - /// The stored values already matched without a write. - #[display("already_matched")] - AlreadyMatched, - /// The one conditional write succeeded. - #[display("written")] - Written, - /// The write added missing IDs while deferring different values of unknown freshness. - #[display("written_with_deferred_freshness")] - WrittenWithDeferredFreshness, - /// A conflicting writer persisted every desired value. - #[display("conflict_matched")] - ConflictMatched, - /// A conflict left at least one desired value absent or different. - #[display("deferred_conflict")] - DeferredConflict, - /// A different stored value had unknown freshness. - #[display("deferred_freshness")] - DeferredFreshness, - /// An eventually consistent read missed a row already proven to exist. - #[display("deferred_stale_read")] - DeferredStaleRead, - /// The identity graph row was missing. - #[display("missing")] - Missing, - /// Consent had been withdrawn in the identity graph. - #[display("consent_withdrawn")] - ConsentWithdrawn, - /// KV or serialization failed. - #[display("failed")] - Failed, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CookieUpdateApplication { - AlreadyMatched, - Changed { deferred: bool }, - DeferredFreshness, -} - -fn apply_cookie_partner_id_updates( - entry: &mut KvEntry, - updates: &[PartnerIdUpdate], -) -> CookieUpdateApplication { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - let mut changed = false; - let mut deferred = false; - for (partner_id, uid) in latest_updates { - match entry.ids.get(partner_id) { - Some(existing) if existing.uid == uid => continue, - // Browser EID cookies carry no value-owned sequence or timestamp. - // A different value therefore has unknown freshness and must not - // replace the stored value. - Some(_) => { - deferred = true; - continue; - } - None => {} - } - - entry.ids.insert( - partner_id.to_owned(), - super::kv_types::KvPartnerId::single(uid.to_owned()), - ); - changed = true; - } - - if changed { - CookieUpdateApplication::Changed { deferred } - } else if deferred { - CookieUpdateApplication::DeferredFreshness - } else { - CookieUpdateApplication::AlreadyMatched - } -} - -fn partner_id_updates_match(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> bool { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - latest_updates.into_iter().all(|(partner_id, uid)| { - entry - .ids - .get(partner_id) - .is_some_and(|existing| existing.uid == uid) - }) -} - pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { @@ -465,6 +370,67 @@ impl KvIdentityGraph { ) } + /// Applies the navigation's single row change in one attempt. + /// + /// The `permit` is consumed, so a navigation cannot write twice. A + /// successful tombstone write also records the existing + /// withdrawal-completion marker: that second insert is the one deliberate + /// exception to the one-write-per-page-load rule. It predates the rule and + /// is kept because later withdrawals rely on it to skip the strongly + /// consistent existence check of an already-withdrawn row; it never changes + /// the identity row itself. + /// + /// # Errors + /// + /// Returns [`KvWriteError::Conflict`] when the row changed since it was + /// read, [`KvWriteError::AlreadyExists`] when a create finds a row, or + /// [`KvWriteError::Store`] on a serialization or store failure, including + /// a row over [`MAX_ENTRY_BYTES`]. Callers must not retry. + pub fn commit_page_load( + &self, + permit: PageLoadWrite, + ec_id: &str, + change: RowChange, + ) -> Result> { + permit.spend(); + let is_tombstone = matches!(change, RowChange::Tombstone { .. }); + let (entry, ttl, mode) = match change { + RowChange::Create(entry) => (entry, ENTRY_TTL, EcKvWriteMode::Add), + RowChange::Update { entry, expected } => ( + entry, + ENTRY_TTL, + EcKvWriteMode::IfGenerationMatch(expected.get()), + ), + RowChange::Tombstone { expected } => ( + KvEntry::tombstone(current_timestamp()), + TOMBSTONE_TTL, + EcKvWriteMode::IfGenerationMatch(expected.get()), + ), + }; + let (body, metadata) = + Self::serialize_entry(&entry, self.store_name()).change_context(KvWriteError::Store)?; + let outcome = self + .write_entry(ec_id, &body, &metadata, ttl, mode) + .change_context(KvWriteError::Store)?; + match outcome { + EcKvWriteOutcome::Written => { + if is_tombstone { + // Deliberate exception to the one-write-per-page-load rule: + // the withdrawal-completion marker is a separate key that + // predates the rule. Later withdrawals use it to avoid a + // strongly consistent existence check on a row already + // withdrawn, and it never touches the identity row. + self.record_withdrawal_completion(ec_id, entry.consent.updated); + } + Ok(entry) + } + EcKvWriteOutcome::PreconditionFailed => Err(Report::new(match mode { + EcKvWriteMode::Add => KvWriteError::AlreadyExists, + _ => KvWriteError::Conflict, + })), + } + } + /// Creates a new entry, or overwrites an existing tombstone on re-consent. /// /// Three-way behavior: @@ -560,154 +526,6 @@ impl KvIdentityGraph { ))) } - /// Persists browser EID cookies with one conditional write and one conflict read. - /// - /// Browser cookies do not carry a value-owned version, so a different - /// existing value has unknown freshness and is always deferred. After a CAS - /// conflict this method never writes again. A live follow-up becomes the - /// authoritative snapshot; a missing or failed follow-up retains the live - /// pre-write snapshot as proof and defers the update. - pub(crate) fn sync_eid_cookie_updates_from_snapshot( - &self, - ec_id: &str, - updates: &[PartnerIdUpdate], - snapshot: EcKvSnapshot, - ) -> (EcKvSnapshot, EidCookieSyncOutcome) { - if updates.is_empty() { - return (snapshot, EidCookieSyncOutcome::AlreadyMatched); - } - - let proven = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - .. - } if snapshot_id == ec_id => Some(snapshot.clone()), - _ => None, - }; - - let current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - EcKvSnapshot::Failed { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => return (snapshot, EidCookieSyncOutcome::Failed), - _ => self.load_snapshot(ec_id), - }; - - let (mut entry, generation) = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - generation: Some(generation), - } if snapshot_id == ec_id => (entry.as_ref().clone(), generation), - EcKvSnapshot::Missing { .. } => { - let outcome = if proven.is_some() { - EidCookieSyncOutcome::DeferredStaleRead - } else { - EidCookieSyncOutcome::Missing - }; - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, outcome); - } - EcKvSnapshot::Failed { .. } => { - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, EidCookieSyncOutcome::Failed); - } - EcKvSnapshot::Present { .. } | EcKvSnapshot::NotRead => { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - } - }; - - if !entry.consent.ok { - return (current, EidCookieSyncOutcome::ConsentWithdrawn); - } - let deferred_freshness = match apply_cookie_partner_id_updates(&mut entry, updates) { - CookieUpdateApplication::AlreadyMatched => { - return (current, EidCookieSyncOutcome::AlreadyMatched); - } - CookieUpdateApplication::DeferredFreshness => { - return (current, EidCookieSyncOutcome::DeferredFreshness); - } - CookieUpdateApplication::Changed { deferred } => deferred, - }; - - let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => ( - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: None, - }, - if deferred_freshness { - EidCookieSyncOutcome::WrittenWithDeferredFreshness - } else { - EidCookieSyncOutcome::Written - }, - ), - Ok(EcKvWriteOutcome::PreconditionFailed) => { - let refreshed = self.load_snapshot(ec_id); - let Some(refreshed_entry) = refreshed.entry_for(ec_id) else { - // The failed CAS proved `current`'s generation stale. Keep - // only its existence proof so later writes must reread. - let proof = match current { - EcKvSnapshot::Present { - ec_id: proven_id, - entry, - .. - } => EcKvSnapshot::Present { - ec_id: proven_id, - entry, - generation: None, - }, - other => other, - }; - let kept = Self::keep_proven(ec_id, refreshed, Some(&proof)); - return (kept, EidCookieSyncOutcome::DeferredConflict); - }; - if !refreshed_entry.consent.ok { - return (refreshed, EidCookieSyncOutcome::ConsentWithdrawn); - } - let outcome = if partner_id_updates_match(refreshed_entry, updates) { - EidCookieSyncOutcome::ConflictMatched - } else { - EidCookieSyncOutcome::DeferredConflict - }; - (refreshed, outcome) - } - Err(err) => { - log::warn!("EID cookie sync write failed: {err:?}"); - ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ) - } - } - } - /// Merges partner IDs using request-scoped persisted state as the first CAS input. /// /// A caller-supplied `Present` snapshot is *proof* that the row exists — @@ -1052,7 +870,12 @@ impl KvIdentityGraph { ) } - fn withdrawal_marker_exists( + /// Whether a still-valid withdrawal-completion marker exists for `ec_id`. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] when the marker listing fails. + pub(crate) fn withdrawal_marker_exists( &self, ec_id: &str, now: Option, @@ -1231,6 +1054,7 @@ impl KvIdentityGraph { self.overwrite_withdrawal_tombstone(ec_id).map(Some) } + #[cfg(test)] fn overwrite_withdrawal_tombstone( &self, ec_id: &str, @@ -1254,200 +1078,6 @@ impl KvIdentityGraph { }) } - /// Resolves a tombstone attempt whose point read reported the row absent. - /// - /// A completion marker proves an earlier withdrawal finished and avoids a - /// second root existence check. Otherwise, a proven-absent key is a no-op: - /// there is nothing to withdraw, and a forged cookie must not mint a row. A - /// key that provably exists is tombstoned unconditionally because no CAS - /// generation is available after a missed read. Marker-check failure falls - /// back to that privacy write, as does an unusable clock; root-existence - /// failure leaves withdrawal unresolved rather than silently dropped. - fn tombstone_unproven_missing( - &self, - ec_id: &str, - missing: EcKvSnapshot, - now: Option, - ) -> EcKvSnapshot { - match self.withdrawal_marker_exists(ec_id, now) { - Ok(true) => { - log::debug!( - "withdrawal tombstone for '{}': completion marker already exists", - log_id(ec_id) - ); - return missing; - } - Ok(false) => {} - Err(err) => { - // Marker failure must not weaken withdrawal. Fall back to the - // existing root existence check and unconditional privacy write. - log::warn!( - "withdrawal completion marker lookup failed for '{}': {err:?}", - log_id(ec_id) - ); - } - } - - match self.key_exists_confirmed(ec_id) { - Ok(false) => missing, - Ok(true) => { - log::warn!( - "withdrawal tombstone for '{}': point read missed a row the store still \ - lists; writing an unconditional tombstone", - log_id(ec_id) - ); - match self.overwrite_withdrawal_tombstone(ec_id) { - Ok(tombstone) => { - self.record_withdrawal_completion(ec_id, tombstone.consent.updated); - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(tombstone), - generation: None, - } - } - Err(err) => { - log::warn!( - "unconditional withdrawal tombstone failed for '{}': {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - } - } - Err(err) => { - log::warn!( - "withdrawal tombstone for '{}': existence check failed, cannot confirm \ - absence: {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - } - } - - /// Writes a tombstone only when an existing row can be confirmed. - /// - /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` - /// cookie must not mint a row. But a *point read* cannot prove absence on - /// an eventually-consistent store, and dropping a withdrawal is worse than - /// a redundant read, so absence is established in two stages: - /// - /// 1. Any snapshot that is not a usable `Present` for this EC ID — a - /// publisher preload that read `Missing`, a read that `Failed`, or one - /// lacking a CAS generation — is re-read. On the publisher path that - /// re-read is separated from the preload by the full origin round trip, - /// which gives replication time to converge. - /// 2. A re-read that still reports the row absent is checked against - /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads - /// the primary data source. - /// - /// Resolving the initial snapshot happens outside the retry counter, so all - /// [`MAX_CAS_RETRIES`] iterations stay available for the tombstone write. - pub(crate) fn tombstone_existing_from_snapshot( - &self, - ec_id: &str, - snapshot: EcKvSnapshot, - ) -> EcKvSnapshot { - let mut current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - .. - } if snapshot_id == ec_id && !entry.consent.ok => return snapshot, - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - _ => self.load_snapshot(ec_id), - }; - - for _attempt in 0..MAX_CAS_RETRIES { - let generation = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - .. - } if snapshot_id == ec_id && !entry.consent.ok => return current, - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(generation), - .. - } if snapshot_id == ec_id => generation, - // A missing row (including one that disappeared mid-retry) is - // only a no-op once absence is proven against the primary data - // source. - EcKvSnapshot::Missing { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => { - return self.tombstone_unproven_missing( - ec_id, - current, - checked_current_timestamp(), - ); - } - // A refreshed read that failed (or any other unusable state) - // fails closed rather than silently dropping the withdrawal. - _ => { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - }; - - let tombstone = KvEntry::tombstone(current_timestamp()); - let Ok((body, meta_str)) = Self::serialize_entry(&tombstone, self.store_name()) else { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - TOMBSTONE_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => { - self.record_withdrawal_completion(ec_id, tombstone.consent.updated); - return EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(tombstone), - generation: None, - }; - } - Ok(EcKvWriteOutcome::PreconditionFailed) => { - current = self.load_snapshot(ec_id); - } - Err(err) => { - log::warn!( - "conditional withdrawal tombstone failed for '{}': {err:?}", - log_id(ec_id) - ); - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - } - } - - // Withdrawal enforcement lost every CAS race, so the row can still be - // live with consent granted while the browser cookie is cleared. That - // divergence is only visible to operators if it is logged here. - log::warn!( - "withdrawal tombstone for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; the \ - identity-graph row may still be live with consent granted", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - /// Counts the number of keys sharing the same EC hash prefix. /// /// Uses the platform KV list API with a prefix filter, limited to @@ -1552,7 +1182,7 @@ impl KvIdentityGraph { /// Hard-deletes the entry and any withdrawal completion marker. /// /// Reserved for the IAB data deletion framework (deferred). Consent - /// withdrawal uses [`Self::tombstone_existing_from_snapshot`] instead. + /// withdrawal uses a page-load tombstone via [`Self::commit_page_load`] instead. /// /// # Errors /// @@ -1619,91 +1249,14 @@ impl KvIdentityGraph { #[cfg(test)] mod tests { use super::*; + use crate::ec::checked_current_timestamp; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_types::{KvEid, KvUid}; + use crate::ec::page_load::{Generation, NavigationPermit}; - /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition - /// and deletes the key, simulating a concurrent withdrawal that removes the - /// row between this writer's read and its write. - struct DisappearOnConflictEcKv { - inner: InMemoryEcKv, - conflicts_remaining: std::sync::Mutex, - } - - impl DisappearOnConflictEcKv { - fn new(conflicts: u32) -> Self { - Self { - inner: InMemoryEcKv::new("disappear-store"), - conflicts_remaining: std::sync::Mutex::new(conflicts), - } - } - - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize seeded entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed live entry"); - } - } - - impl EcKvStore for DisappearOnConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - let mut remaining = self - .conflicts_remaining - .lock() - .expect("should lock conflict counter"); - if *remaining > 0 { - *remaining -= 1; - self.inner.delete(key).expect("should delete on conflict"); - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - } - self.inner.insert(key, write) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - - fn snapshot_ec_id() -> String { - format!("{}.ABC123", "a".repeat(64)) + fn snapshot_ec_id() -> String { + format!("{}.ABC123", "a".repeat(64)) } #[test] @@ -1871,15 +1424,6 @@ mod tests { } } - fn with_partner_update_on_conflict(conflicts: u32) -> Self { - Self { - inner: InMemoryEcKv::new("partner-conflict-store"), - conflicts_remaining: std::sync::Mutex::new(conflicts), - revive_on_conflict: true, - partner_update_on_conflict: true, - } - } - fn seed_tombstone(&self, ec_id: &str) { let (body, meta) = KvIdentityGraph::serialize_entry( &KvEntry::tombstone(1000), @@ -1986,131 +1530,6 @@ mod tests { } } - /// Store that replaces the row during the first EID CAS write and records - /// the request's reads and conditional writes. - struct EidConflictEcKv { - inner: InMemoryEcKv, - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - follow_up_miss: bool, - miss_next_lookup: std::sync::atomic::AtomicBool, - } - - impl EidConflictEcKv { - fn new( - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - ) -> Self { - Self { - inner: InMemoryEcKv::new("eid-conflict-store"), - concurrent_entry, - lookups, - conditional_writes, - follow_up_miss: false, - miss_next_lookup: std::sync::atomic::AtomicBool::new(false), - } - } - - fn with_follow_up_miss(mut self) -> Self { - self.follow_up_miss = true; - self - } - - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize initial entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed initial entry"); - } - } - - impl EcKvStore for EidConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.lookups - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if self - .miss_next_lookup - .swap(false, std::sync::atomic::Ordering::Relaxed) - { - return Ok(None); - } - self.inner.lookup(key) - } - - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - self.conditional_writes - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let (body, meta) = KvIdentityGraph::serialize_entry( - &self.concurrent_entry, - self.inner.store_name(), - ) - .expect("should serialize concurrent entry"); - self.inner - .insert( - key, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Overwrite, - }, - ) - .expect("should write concurrent entry"); - if self.follow_up_miss { - self.miss_next_lookup - .store(true, std::sync::atomic::Ordering::Relaxed); - } - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - self.inner.insert(key, write) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - - fn count_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result> { - self.inner.count_keys_with_prefix(prefix, limit) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - #[test] fn create_or_revive_retries_cas_conflict_and_succeeds() { let store = ConflictInjectingEcKv::new(2, false); @@ -2247,353 +1666,82 @@ mod tests { } #[test] - fn eid_cookie_sync_conflict_matching_value_stops_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("desired-uid".to_owned()), - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); + fn evaluate_cluster_returns_stored_value_without_store_io() { + let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = live_entry(); + entry.network = Some(KvNetwork { + cluster_size: Some(5), + }); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); + let cluster_size = kv + .evaluate_cluster(&ec_id, &entry, 0) + .expect("should not touch store when cluster_size is already known"); - assert_eq!(outcome, EidCookieSyncOutcome::ConflictMatched); - assert_eq!( - snapshot - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("desired-uid"), - "the authoritative follow-up snapshot should replace stale request state" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" + cluster_size, + Some(5), + "should return stored cluster_size without re-listing keys" ); } #[test] - fn eid_cookie_sync_keeps_live_proof_when_conflict_follow_up_misses() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - live_entry(), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ) - .with_follow_up_miss(); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); + fn evaluate_cluster_skips_tombstone_without_store_io() { + let kv = KvIdentityGraph::failing("nonexistent_store_for_tombstone_cluster_test"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = KvEntry::tombstone(1000); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); + let cluster_size = kv + .evaluate_cluster(&ec_id, &entry, 0) + .expect("should not touch store for tombstone entries"); - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the live pre-write row should prevent conflict deferral from entering orphan recovery" - ); - assert_eq!( - snapshot.generation_for(&ec_id), - None, - "a failed CAS must not return its rejected generation" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "the initial refresh and conflict follow-up should be the only reads" - ); assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "the conflict must remain the request's only conditional write" + cluster_size, None, + "should not evaluate or write cluster_size for tombstones" ); } #[test] - fn eid_cookie_sync_reports_proven_refresh_miss_as_deferred() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("empty-store"); - let proven = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; + fn create_then_get_roundtrips_entry() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = live_entry(); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - proven, - ); + kv.create(&ec_id, &entry).expect("should create new entry"); + let (loaded, generation) = kv + .get(&ec_id) + .expect("should read entry back") + .expect("should find created entry"); - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the add-confirmed row should survive an eventually consistent miss" - ); - assert_eq!( - outcome, - EidCookieSyncOutcome::DeferredStaleRead, - "a row already proven to exist should report a deferred stale read" - ); + assert!(loaded.consent.ok, "should preserve consent state"); + assert!(generation > 0, "should expose a generation marker"); } #[test] - fn partner_id_conflict_match_uses_last_duplicate_value() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("latest-uid".to_owned()), - ); - let updates = [ - PartnerIdUpdate::new("ssp_x", "older-uid"), - PartnerIdUpdate::new("ssp_x", "latest-uid"), - ]; + fn create_rejects_existing_key() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = live_entry(); + kv.create(&ec_id, &entry).expect("should create new entry"); + let err = kv + .create(&ec_id, &entry) + .expect_err("should reject duplicate create"); assert!( - partner_id_updates_match(&entry, &updates), - "conflict matching should use the same last-value-wins rule as writes" + format!("{err}").contains("already exists"), + "should report duplicate key" ); } #[test] - fn eid_cookie_sync_conflicting_value_defers_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("concurrent-uid".to_owned()), - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "stale-uid")], - snapshot, - ); + fn create_if_absent_reports_written_and_collision() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read concurrent row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "concurrent-uid"); - } - - #[test] - fn eid_cookie_sync_preserves_concurrent_withdrawal_after_conflict() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - KvEntry::tombstone(2_000), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::ConsentWithdrawn); - assert!( - snapshot - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "the refreshed withdrawal tombstone must remain authoritative" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - } - - #[test] - fn eid_cookie_sync_defers_different_values_without_value_owned_freshness() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("protected-uid".to_owned()), - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "incoming-uid")], - graph.load_snapshot(&ec_id), - ); - - assert_eq!(outcome, EidCookieSyncOutcome::DeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read protected row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); - } - - #[test] - fn eid_cookie_sync_adds_missing_ids_while_deferring_different_values() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("mixed-freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("protected-uid".to_owned()), - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[ - PartnerIdUpdate::new("ssp_x", "different-uid"), - PartnerIdUpdate::new("ssp_y", "new-uid"), - ], - graph.load_snapshot(&ec_id), - ); - - assert_eq!(outcome, EidCookieSyncOutcome::WrittenWithDeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read updated row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); - assert_eq!(stored.ids["ssp_y"].uid, "new-uid"); - } - - #[test] - fn evaluate_cluster_returns_stored_value_without_store_io() { - let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut entry = live_entry(); - entry.network = Some(KvNetwork { - cluster_size: Some(5), - }); - - let cluster_size = kv - .evaluate_cluster(&ec_id, &entry, 0) - .expect("should not touch store when cluster_size is already known"); - - assert_eq!( - cluster_size, - Some(5), - "should return stored cluster_size without re-listing keys" - ); - } - - #[test] - fn evaluate_cluster_skips_tombstone_without_store_io() { - let kv = KvIdentityGraph::failing("nonexistent_store_for_tombstone_cluster_test"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = KvEntry::tombstone(1000); - - let cluster_size = kv - .evaluate_cluster(&ec_id, &entry, 0) - .expect("should not touch store for tombstone entries"); - - assert_eq!( - cluster_size, None, - "should not evaluate or write cluster_size for tombstones" - ); - } - - #[test] - fn create_then_get_roundtrips_entry() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = live_entry(); - - kv.create(&ec_id, &entry).expect("should create new entry"); - let (loaded, generation) = kv - .get(&ec_id) - .expect("should read entry back") - .expect("should find created entry"); - - assert!(loaded.consent.ok, "should preserve consent state"); - assert!(generation > 0, "should expose a generation marker"); - } - - #[test] - fn create_rejects_existing_key() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = live_entry(); - - kv.create(&ec_id, &entry).expect("should create new entry"); - let err = kv - .create(&ec_id, &entry) - .expect_err("should reject duplicate create"); - assert!( - format!("{err}").contains("already exists"), - "should report duplicate key" - ); - } - - #[test] - fn create_if_absent_reports_written_and_collision() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should create absent entry"), - CreateIfAbsentOutcome::Written + kv.create_if_absent(&ec_id, &live_entry()) + .expect("should create absent entry"), + CreateIfAbsentOutcome::Written ); assert_eq!( kv.create_if_absent(&ec_id, &live_entry()) @@ -2661,270 +1809,91 @@ mod tests { } #[test] - fn create_or_revive_clears_withdrawal_marker() { - let kv = KvIdentityGraph::in_memory("test_store"); + fn withdrawal_marker_validity_requires_a_usable_clock_before_expiry() { + let graph = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + graph + .write_withdrawal_marker(&ec_id, 1000) + .expect("should seed marker"); + let valid_until = 1000 + TOMBSTONE_TTL.as_secs(); + + for (now, expected) in [ + (Some(valid_until - 1), true), + (Some(valid_until), false), + (Some(valid_until + 1), false), + (None, false), + ] { + assert_eq!( + graph + .withdrawal_marker_exists(&ec_id, now) + .expect("should check marker validity"), + expected, + "marker validity should honor clock availability and exclusive expiry at {now:?}" + ); + } + } + + #[test] + fn withdrawal_marker_existence_rejects_malformed_expiry_suffix() { let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "withdrawal should record completion" + let marker_key = KvIdentityGraph::withdrawal_marker_key( + &ec_id, + current_timestamp() + TOMBSTONE_TTL.as_secs(), ); + let longer_key = format!("{marker_key}-longer"); + let store = InMemoryEcKv::new("test_store"); + store + .insert( + &longer_key, + EcKvWrite { + body: "1", + metadata: "{}", + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed longer marker key"); + let kv = KvIdentityGraph::new(store); - kv.create_or_revive(&ec_id, &live_entry()) - .expect("should revive tombstone"); - - let (loaded, _) = kv - .get(&ec_id) - .expect("should read revived entry") - .expect("should find revived entry"); - assert!(loaded.consent.ok, "should be live after revive"); assert!( !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "revival should clear stale withdrawal completion" + .expect("should validate withdrawal marker expiry"), + "a malformed expiry suffix must not prove withdrawal completion" ); } #[test] - fn stale_completion_marker_does_not_suppress_withdrawal_after_root_recreation() { + fn clear_absent_withdrawal_marker_lists_once_without_delete() { let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 1, - )); + let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); let ec_id = snapshot_ec_id(); - let expired_updated = current_timestamp().saturating_sub(TOMBSTONE_TTL.as_secs() + 1); - let expired_tombstone = KvEntry::tombstone(expired_updated); - graph - .create(&ec_id, &expired_tombstone) - .expect("should seed old tombstone"); - graph - .write_withdrawal_marker(&ec_id, expired_updated) - .expect("should seed old completion marker"); - // Model the root expiring just before its later-written completion - // marker, followed by recreation of the same key. - graph.store.delete(&ec_id).expect("should expire root row"); - assert_eq!( - graph - .create_if_absent(&ec_id, &live_entry()) - .expect("should recreate expired root"), - CreateIfAbsentOutcome::Written, - "should make the same key live again after root expiry" - ); + kv.clear_withdrawal_marker(&ec_id) + .expect("should accept an absent marker"); - let outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, + assert_eq!( + operations.list_count(), + 1, + "absent marker guard should list once" ); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "an expired completion marker must not suppress withdrawal of a recreated live row" + assert_eq!( + operations.delete_count(), + 0, + "absent marker guard should avoid a delete" ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read recreated row") - .expect("should retain recreated row"); - assert!(!stored.consent.ok, "recreated row should be tombstoned"); } #[test] - fn withdrawal_failed_clock_stale_miss_tombstones_live_root() { + fn writing_an_existing_withdrawal_marker_is_idempotent() { let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 1, - )); + let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &concurrent_live_entry()) - .expect("should seed a live row with partner IDs"); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed a completion marker left after root recreation"); - operations.reset(); - let missing = graph.load_snapshot(&ec_id); - assert!( - matches!(missing, EcKvSnapshot::Missing { .. }), - "should reproduce a stale point-read miss for the live row" - ); + let updated = current_timestamp(); - // None is the checked clock's failure result, not Unix epoch zero. - let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); - - let (stored, generation) = graph - .get(&ec_id) - .expect("should read back persisted withdrawal") - .expect("should retain the root"); - assert!( - !stored.consent.ok, - "an unusable clock must not let a marker suppress withdrawal" - ); - assert!( - stored.ids.is_empty(), - "withdrawal should clear stored partner IDs" - ); - assert_eq!( - generation, 2, - "withdrawal should write the existing root once" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "clock failure should not fail a strongly confirmed withdrawal" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "should strongly check the root" - ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::Overwrite, - ttl: TOMBSTONE_TTL - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL - }, - ], - "should write only the root tombstone and its completion marker" - ); - } - - #[test] - fn withdrawal_failed_clock_does_not_create_absent_root() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed a marker without a root"); - operations.reset(); - let missing = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "absent root should remain missing" - ); - assert!( - graph - .get(&ec_id) - .expect("should read absent root") - .is_none(), - "clock failure must not mint an unknown root" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "should prove root absence despite the marker" - ); - assert!( - operations.inserts().is_empty(), - "absent identity should cause no writes" - ); - } - - #[test] - fn withdrawal_marker_validity_requires_a_usable_clock_before_expiry() { - let graph = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed marker"); - let valid_until = 1000 + TOMBSTONE_TTL.as_secs(); - - for (now, expected) in [ - (Some(valid_until - 1), true), - (Some(valid_until), false), - (Some(valid_until + 1), false), - (None, false), - ] { - assert_eq!( - graph - .withdrawal_marker_exists(&ec_id, now) - .expect("should check marker validity"), - expected, - "marker validity should honor clock availability and exclusive expiry at {now:?}" - ); - } - } - - #[test] - fn withdrawal_marker_existence_rejects_malformed_expiry_suffix() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let marker_key = KvIdentityGraph::withdrawal_marker_key( - &ec_id, - current_timestamp() + TOMBSTONE_TTL.as_secs(), - ); - let longer_key = format!("{marker_key}-longer"); - let store = InMemoryEcKv::new("test_store"); - store - .insert( - &longer_key, - EcKvWrite { - body: "1", - metadata: "{}", - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed longer marker key"); - let kv = KvIdentityGraph::new(store); - - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should validate withdrawal marker expiry"), - "a malformed expiry suffix must not prove withdrawal completion" - ); - } - - #[test] - fn clear_absent_withdrawal_marker_lists_once_without_delete() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - - kv.clear_withdrawal_marker(&ec_id) - .expect("should accept an absent marker"); - - assert_eq!( - operations.list_count(), - 1, - "absent marker guard should list once" - ); - assert_eq!( - operations.delete_count(), - 0, - "absent marker guard should avoid a delete" - ); - } - - #[test] - fn writing_an_existing_withdrawal_marker_is_idempotent() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - let updated = current_timestamp(); - - kv.write_withdrawal_marker(&ec_id, updated) - .expect("should write first completion marker"); - kv.write_withdrawal_marker(&ec_id, updated) - .expect("should accept completion marker add collision"); + kv.write_withdrawal_marker(&ec_id, updated) + .expect("should write first completion marker"); + kv.write_withdrawal_marker(&ec_id, updated) + .expect("should accept completion marker add collision"); assert_eq!( operations.inserts(), @@ -3015,99 +1984,6 @@ mod tests { ); } - #[test] - fn clear_withdrawal_marker_accepts_concurrent_removal_after_delete_error() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, true)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - kv.clear_withdrawal_marker(&ec_id) - .expect("should accept a marker removed by another request"); - - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should confirm marker removal"), - "completion marker should remain absent" - ); - } - - #[test] - fn clear_withdrawal_marker_preserves_delete_error_when_marker_remains() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, false)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - kv.clear_withdrawal_marker(&ec_id).is_err(), - "a marker that remains after delete failure should block revival" - ); - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should confirm marker remains"), - "completion marker should remain present" - ); - } - - #[test] - fn withdrawal_marker_is_excluded_from_hash_prefix_count() { - let hash = "a".repeat(64); - let ec_id = format!("{hash}.ABC123"); - let kv = KvIdentityGraph::in_memory("test_store"); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count live root"), - 1, - "the root should be the only hash-prefix key" - ); - - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "withdrawal should record completion" - ); - assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count tombstoned root"), - 1, - "the marker namespace must not affect cluster counts" - ); - } - - #[test] - fn delete_removes_withdrawal_marker() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - kv.delete(&ec_id).expect("should delete entry and marker"); - - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "hard delete should remove the root" - ); - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "hard delete should remove withdrawal completion" - ); - } - #[test] fn upsert_partner_id_if_exists_reports_missing_key() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -3234,48 +2110,6 @@ mod tests { assert!(kv.get(&ec_id).expect("should read store").is_none()); } - #[test] - fn tombstone_existing_from_snapshot_never_creates_missing_key() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let snapshot = EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }; - - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "withdrawal should preserve the missing snapshot for an absent identity" - ); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "withdrawal must not create a tombstone for an absent key" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_uses_existing_generation() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); - let snapshot = kv.load_snapshot(&ec_id); - - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "should return the persisted tombstone" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "should persist withdrawal state"); - } - #[test] fn write_withdrawal_tombstone_overwrites_live_entry() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -3344,14 +2178,6 @@ mod tests { self.deletes.load(std::sync::atomic::Ordering::Relaxed) } - fn operation_count(&self) -> usize { - self.lookup_count() - + self.exact_check_count() - + self.inserts().len() - + self.list_count() - + self.delete_count() - } - fn inserts(&self) -> Vec { self.inserts .lock() @@ -3405,55 +2231,12 @@ mod tests { } } - fn completed_with_root_check_failure( + fn with_marker_delete_failure( operations: Arc, - ec_id: &str, + remove_before_error: bool, ) -> Self { - let store = Self { - root_check_fails: true, - stale_lookups_remaining: std::sync::Mutex::new(u32::MAX), - ..Self::new(operations) - }; - let tombstone = KvEntry::tombstone(current_timestamp()); - let (body, metadata) = - KvIdentityGraph::serialize_entry(&tombstone, store.inner.store_name()) - .expect("should serialize tombstone"); - store - .inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &metadata, - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed tombstone"); - store - .inner - .insert( - &KvIdentityGraph::withdrawal_marker_key( - ec_id, - tombstone.consent.updated + TOMBSTONE_TTL.as_secs(), - ), - EcKvWrite { - body: "1", - metadata: "{}", - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed completion marker"); - store - } - - fn with_marker_delete_failure( - operations: Arc, - remove_before_error: bool, - ) -> Self { - Self { - marker_delete_failure_removes_key: Some(remove_before_error), + Self { + marker_delete_failure_removes_key: Some(remove_before_error), ..Self::new(operations) } } @@ -3794,562 +2577,87 @@ mod tests { let ec_id = snapshot_ec_id(); let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a confirmed miss must stay missing" - ); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "must not create a root entry for a missing key" - ); - } - - #[test] - fn snapshot_upsert_failed_snapshot_is_not_revalidated() { - // A lookup that already errored is not retried on the hot path, even - // though the row exists. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a failed lookup must not be retried by partner enrichment" - ); - } - - #[test] - fn snapshot_upsert_rejects_tombstone() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &KvEntry::tombstone(1000)) - .expect("should seed tombstone"); - let snapshot = kv.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| entry.ids.is_empty()), - "a tombstone must reject partner enrichment" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("tombstone should remain"); - assert!(stored.ids.is_empty(), "no update should reach the store"); - } - - #[test] - fn snapshot_upsert_store_failure_returns_failed_not_request_local() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a store write failure must not claim request-local IDs were persisted" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_skips_backend_for_authoritative_tombstone() { - for generation in [Some(7), None] { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation, - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); - - assert_eq!( - outcome, snapshot, - "should preserve authoritative tombstone state" - ); - assert_eq!( - operations.operation_count(), - 0, - "an authoritative tombstone should not access the backend" - ); - } - } - - #[test] - fn tombstone_existing_from_snapshot_repeated_request_preserves_first_write() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let live_snapshot = graph.load_snapshot(&ec_id); - operations.reset(); - - graph.tombstone_existing_from_snapshot(&ec_id, live_snapshot); - - assert_eq!( - operations.lookup_count(), - 0, - "usable generation should avoid a read" - ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - ], - "first withdrawal should write the root and its completion marker" - ); - let first_snapshot = graph.load_snapshot(&ec_id); - let (first_entry, first_generation) = match &first_snapshot { - EcKvSnapshot::Present { - entry, generation, .. - } => (entry.as_ref().clone(), *generation), - other => panic!("should load first tombstone, got {other:?}"), - }; - operations.reset(); - - let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, first_snapshot); - - assert_eq!( - operations.operation_count(), - 0, - "repeated withdrawal should not access the backend" - ); - assert_eq!( - second_outcome.generation_for(&ec_id), - first_generation, - "repeated withdrawal should preserve the stored generation" - ); - assert_eq!( - second_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - Some(first_entry.consent.updated), - "repeated withdrawal should preserve the first tombstone timestamp" - ); - } - - #[test] - fn tombstone_existing_from_repeated_stale_miss_preserves_first_write() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 2, - )); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let first_outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - let first_updated = first_outcome - .entry_for(&ec_id) - .expect("should return first tombstone") - .consent - .updated; - assert_eq!( - operations.lookup_count(), - 1, - "first stale miss should retry its point read once" - ); - assert_eq!( - operations.list_count(), - 1, - "first stale miss should list completion markers once" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "first stale miss should confirm root existence once" - ); - assert_eq!( - operations.inserts().len(), - 2, - "first stale miss should write the root and completion marker" - ); - operations.reset(); - - graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert_eq!( - operations.lookup_count(), - 1, - "a repeated stale miss should retry its point read once" - ); - assert_eq!( - operations.exact_check_count(), - 0, - "a valid completion marker should avoid a root existence check" - ); - assert_eq!( - operations.list_count(), - 1, - "a repeated stale miss should list completion markers once" - ); - assert_eq!( - operations.delete_count(), - 0, - "a repeated withdrawal should not delete marker state" - ); - assert!( - operations.inserts().is_empty(), - "a repeated stale miss should not rewrite the completed tombstone" - ); - let (stored, generation) = graph - .get(&ec_id) - .expect("should read stored tombstone") - .expect("should preserve tombstone"); - assert_eq!( - generation, 2, - "only the first withdrawal should advance the root generation" - ); - assert_eq!( - stored.consent.updated, first_updated, - "repeated withdrawal should preserve the first tombstone timestamp" - ); - } - - #[test] - fn tombstone_existing_from_stale_parallel_snapshot_stops_after_conflict() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let stale_snapshot = graph.load_snapshot(&ec_id); - operations.reset(); - - let first_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot.clone()); - let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot); - - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - ], - "parallel loser should attempt stale CAS once and never replace the winner" - ); - assert_eq!( - operations.lookup_count(), - 1, - "parallel loser should reread exactly once after its conflict" - ); - assert_eq!( - second_outcome.generation_for(&ec_id), - Some(2), - "parallel loser should return the winner's stored generation" - ); - assert_eq!( - second_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - first_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - "parallel loser should preserve the winner's tombstone" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_succeeds_without_backend_for_tombstone() { - let graph = KvIdentityGraph::failing("unavailable-store"); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation: Some(3), - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); - - assert_eq!( - outcome, snapshot, - "authoritative tombstone should not touch unavailable backend" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_non_authoritative_states_reread_live_row() { - let ec_id = snapshot_ec_id(); - let states = [ - EcKvSnapshot::NotRead, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }, - EcKvSnapshot::Present { - ec_id: "different-ec-id".to_owned(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation: Some(9), - }, - ]; - - for state in states { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, state); - - assert_eq!( - operations.lookup_count(), - 1, - "state should force one reread" - ); - assert_eq!( - operations.inserts().len(), - 2, - "live reread should write the root and completion marker" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "reread live row should be tombstoned" - ); - } - } - - #[test] - fn tombstone_stale_miss_valid_marker_skips_root_check_and_writes() { - let ec_id = snapshot_ec_id(); - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::completed_with_root_check_failure( - Arc::clone(&operations), - &ec_id, - )); - - let outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a completion marker should resolve a repeated stale miss" - ); - assert_eq!( - operations.lookup_count(), - 1, - "should retry the stale point read once" - ); - assert_eq!( - operations.list_count(), - 1, - "should strongly list completion markers once" - ); - assert_eq!( - operations.exact_check_count(), - 0, - "valid marker should bypass the failing root check" - ); - assert!( - operations.inserts().is_empty(), - "valid marker should prevent redundant writes" - ); - } - - #[test] - fn tombstone_stale_miss_still_writes_when_marker_operations_fail() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_marker_failures( - Arc::clone(&operations), - 1, - )); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let outcome = graph.tombstone_existing_from_snapshot( + let outcome = kv.upsert_partner_ids_from_snapshot( &ec_id, + &updates, EcKvSnapshot::Missing { ec_id: ec_id.clone(), }, ); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "marker failures must not suppress the withdrawal write" + matches!(outcome, EcKvSnapshot::Missing { .. }), + "a confirmed miss must stay missing" ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::Overwrite, - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - ], - "failed completion recording should still be attempted after the root write" + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "must not create a root entry for a missing key" ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read stored row") - .expect("should preserve the root"); - assert!(!stored.consent.ok, "root should remain tombstoned"); } #[test] - fn tombstone_existing_from_snapshot_retries_cas_conflict() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); + fn snapshot_upsert_failed_snapshot_is_not_revalidated() { + // A lookup that already errored is not retried on the hot path, even + // though the row exists. + let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + ); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "should retry the conflict and persist the tombstone" + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a failed lookup must not be retried by partner enrichment" ); } #[test] - fn tombstone_gen_unavailable_survives_four_conflicts_then_writes() { - // A generation-unavailable snapshot refreshes once before its CAS. That - // refresh must not spend a CAS attempt, so a withdrawal tombstone still - // persists after four conflicts and a successful fifth write. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); + fn snapshot_upsert_rejects_tombstone() { + let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; + kv.create(&ec_id, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + let snapshot = kv.load_snapshot(&ec_id); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); assert!( outcome .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "the fifth CAS attempt must persist the tombstone after a refresh and four conflicts" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_returns_failed_after_cas_exhaustion() { - // Every CAS attempt loses its race, so the row stays live with consent - // granted while the browser cookie is already cleared. The caller must - // see a failure it can report rather than a silent no-op. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(MAX_CAS_RETRIES, false)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "CAS exhaustion must report a failed withdrawal" + .is_some_and(|entry| entry.ids.is_empty()), + "a tombstone must reject partner enrichment" ); - let (stored, _) = graph + let (stored, _) = kv .get(&ec_id) .expect("should read store") - .expect("row should remain"); - assert!( - stored.consent.ok, - "the row is still live, which is exactly why the failure must be reported" - ); + .expect("tombstone should remain"); + assert!(stored.ids.is_empty(), "no update should reach the store"); } #[test] - fn tombstone_existing_from_snapshot_overrides_concurrent_live_update() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::with_partner_update_on_conflict(1)); + fn snapshot_upsert_store_failure_returns_failed_not_request_local() { + let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let snapshot = graph.load_snapshot(&ec_id); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - let entry = outcome - .entry_for(&ec_id) - .expect("should return persisted tombstone"); - assert!(!entry.consent.ok, "withdrawal should win after retry"); assert!( - entry.ids.is_empty(), - "withdrawal should clear concurrent partner IDs" + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a store write failure must not claim request-local IDs were persisted" ); } @@ -4375,73 +2683,6 @@ mod tests { ); } - #[test] - fn tombstone_existing_from_snapshot_store_failure_returns_failed() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a failed tombstone write should return failed snapshot state" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_noop_when_row_disappears_on_retry() { - let store = DisappearOnConflictEcKv::new(1); - store.seed_live(&snapshot_ec_id()); - let graph = KvIdentityGraph::new(store); - let ec_id = snapshot_ec_id(); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a row that disappears during retry becomes a no-op" - ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "must not recreate the disappeared key" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_reretries_failed_snapshot_read() { - // A prior request-scoped read failed, so the snapshot is `Failed`. A - // withdrawal must not silently drop consent removal: re-read the store - // and tombstone the row if it is authoritatively present. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - - let outcome = kv.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "a failed snapshot must re-read and persist the tombstone" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "withdrawal must reach the store"); - } - #[test] fn key_exists_confirmed_distinguishes_absence_from_a_stale_point_read() { let graph = KvIdentityGraph::stale_lookup("stale-store", 1); @@ -4883,4 +3124,131 @@ mod tests { "a different suffix under the same hash is a different identity" ); } + + fn tombstone_via_page_load(kv: &KvIdentityGraph, ec_id: &str) { + let snapshot = kv.load_snapshot(ec_id); + let expected = Generation::of(&snapshot, ec_id).expect("should read generation"); + let permit = NavigationPermit::for_test() + .confirm(http::StatusCode::OK) + .expect("should confirm permit"); + kv.commit_page_load(permit, ec_id, RowChange::Tombstone { expected }) + .expect("should write tombstone"); + } + + #[test] + fn create_or_revive_clears_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "withdrawal should record completion" + ); + + kv.create_or_revive(&ec_id, &live_entry()) + .expect("should revive tombstone"); + + let (loaded, _) = kv + .get(&ec_id) + .expect("should read revived entry") + .expect("should find revived entry"); + assert!(loaded.consent.ok, "should be live after revive"); + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "revival should clear stale withdrawal completion" + ); + } + + #[test] + fn clear_withdrawal_marker_accepts_concurrent_removal_after_delete_error() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, true)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + + kv.clear_withdrawal_marker(&ec_id) + .expect("should accept a marker removed by another request"); + + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should confirm marker removal"), + "completion marker should remain absent" + ); + } + + #[test] + fn clear_withdrawal_marker_preserves_delete_error_when_marker_remains() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, false)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + + assert!( + kv.clear_withdrawal_marker(&ec_id).is_err(), + "a marker that remains after delete failure should block revival" + ); + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should confirm marker remains"), + "completion marker should remain present" + ); + } + + #[test] + fn withdrawal_marker_is_excluded_from_hash_prefix_count() { + let hash = "a".repeat(64); + let ec_id = format!("{hash}.ABC123"); + let kv = KvIdentityGraph::in_memory("test_store"); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + assert_eq!( + kv.count_hash_prefix_keys(&hash) + .expect("should count live root"), + 1, + "the root should be the only hash-prefix key" + ); + + tombstone_via_page_load(&kv, &ec_id); + + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "withdrawal should record completion" + ); + assert_eq!( + kv.count_hash_prefix_keys(&hash) + .expect("should count tombstoned root"), + 1, + "the marker namespace must not affect cluster counts" + ); + } + + #[test] + fn delete_removes_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + + kv.delete(&ec_id).expect("should delete entry and marker"); + + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "hard delete should remove the root" + ); + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "hard delete should remove withdrawal completion" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 2a638dc219..45ac8b6df6 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -186,6 +186,21 @@ impl KvPartnerId { .unwrap_or_default(); *self != before } + + /// Removes the last UID of the last entry and resyncs `uid`; used by the + /// write planner's size cap. + pub(crate) fn drop_last_uid(&mut self) { + if let Some(eid) = self.eids.last_mut() { + eid.uids.pop(); + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + } } /// One stored EID entry with its EID-level fields. diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index e815c43a19..33981b49a5 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -7,9 +7,10 @@ //! context. This is called pre-routing on every request. //! //! 2. **Generate** — [`EcContext::generate_if_needed`] creates a new EC ID -//! when none exists and consent allows it. This is called only in organic -//! handlers (publisher proxy, integration proxy) — never in read-only -//! endpoints like `/_ts/api/v1/identify`. +//! when none exists and consent allows it. This is called only for +//! publisher navigations — never in integration proxies or read-only +//! endpoints like `/_ts/api/v1/identify`. The new row is written by the +//! navigation's single page-load write in finalize ([`page_load`]). //! //! # Module structure //! @@ -29,6 +30,9 @@ //! - [`eid_types`] — Value types for full-fidelity provider EIDs //! - [`batch_sync`] — S2S batch sync endpoint (`POST /_ts/api/v1/batch-sync`) //! - [`pull_sync`] — Background pull-sync dispatcher for organic routes +//! - [`page_load`] — Single-use page-load write permit and row changes +//! - [`write_plan`] — Plans a navigation's single row change +//! - [`coordinator`] — Server-side identity lookup coordination mod auth; @@ -36,6 +40,7 @@ pub mod admin; pub mod batch_sync; pub mod consent; pub mod cookies; +pub mod coordinator; pub mod device; pub mod eid_types; pub mod eids; @@ -45,12 +50,14 @@ pub mod identify; pub mod kv; pub mod kv_backend; pub mod kv_types; +pub mod page_load; pub mod partner; pub mod prebid_eids; pub mod pull_sync; pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; +pub mod write_plan; /// Characters of an identifier kept when redacting it for a log. const LOG_ID_PREFIX_CHARS: usize = 8; @@ -85,8 +92,10 @@ use crate::platform::RuntimeServices; use crate::settings::Settings; use device::DeviceSignals; -use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph}; +use self::coordinator::results::LookupResults; +use self::kv::KvIdentityGraph; use self::kv_types::KvEntry; +use self::page_load::NavigationPermit; use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; /// Bounded request classifications that may persist browser EID cookies. @@ -242,6 +251,17 @@ pub fn get_ec_id(req: &Request) -> Result, Report); + +impl Clone for PermitSlot { + fn clone(&self) -> Self { + Self(None) + } +} + /// Captures the EC state for a single request lifecycle. /// /// Created via [`read_from_request`](Self::read_from_request) during @@ -284,6 +304,12 @@ pub struct EcContext { pull_sync_marker: PullSyncMarkerState, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, + /// Navigation permit issued by the router for page-load navigations. + navigation_permit: PermitSlot, + /// Validated provider lookup results collected for this navigation. + lookup_results: LookupResults, + /// New row built by [`Self::generate_if_needed`], written by finalize. + pending_entry: Option>, } impl EcContext { @@ -366,14 +392,20 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, + navigation_permit: PermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, }) } /// Generates a new EC ID if none exists and consent allows it. /// - /// This is the second phase of the EC lifecycle. Call this only in - /// organic handlers (publisher proxy, integration proxy, auction) — - /// never in read-only endpoints. + /// This is the second phase of the EC lifecycle. Call this only for + /// publisher navigations — never in read-only endpoints. + /// + /// Never writes KV. When `kv` is available, the new row is held as the + /// pending entry and created by the navigation's single page-load write + /// in finalize; the EC is issued only if that write succeeds. /// /// If an EC ID already exists (from the request), this is a no-op. /// If consent does not permit EC creation, this is a no-op. @@ -405,62 +437,32 @@ impl EcContext { }) })?; - const MAX_CREATE_ATTEMPTS: usize = 5; - for attempt in 0..MAX_CREATE_ATTEMPTS { - let ec_id = generation::generate_ec_id(settings, client_ip)?; - log::info!("Generated new EC ID: {}", log_id(&ec_id)); - let now = current_timestamp(); - let mut entry = KvEntry::new( - &self.consent, - self.geo_info.as_ref(), - now, - &settings.publisher.domain, - ); - entry.device = self - .device_signals - .as_ref() - .map(DeviceSignals::to_kv_device); - - if let Some(graph) = kv { - match graph.create_if_absent(&ec_id, &entry) { - Ok(CreateIfAbsentOutcome::Written) => { - self.kv_snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: None, - }; - } - Ok(CreateIfAbsentOutcome::AlreadyExists) => { - log::warn!( - "Generated EC ID collision on attempt {}/{MAX_CREATE_ATTEMPTS}", - attempt + 1 - ); - continue; - } - Err(err) => { - log::error!( - "Failed to create EC entry for id '{}' after generation: {err:?}", - log_id(&ec_id), - ); - return Err(err.change_context(TrustedServerError::EdgeCookie { - message: "Failed to persist generated EC ID to KV identity graph" - .to_string(), - })); - } - } - } - - self.ec_value = Some(ec_id); - self.ec_generated = true; - self.pull_sync_marker.invalidate_for_replaced_ec(); - return Ok(()); + let ec_id = generation::generate_ec_id(settings, client_ip)?; + log::info!("Generated new EC ID: {}", log_id(&ec_id)); + let now = current_timestamp(); + let mut entry = KvEntry::new( + &self.consent, + self.geo_info.as_ref(), + now, + &settings.publisher.domain, + ); + entry.device = self + .device_signals + .as_ref() + .map(DeviceSignals::to_kv_device); + + if kv.is_some() { + // The row is created by the navigation's single page-load write in + // finalize; a collision there leaves this EC unissued. + self.pending_entry = Some(Box::new(entry)); + self.kv_snapshot = EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }; } - - Err(Report::new(TrustedServerError::EdgeCookie { - message: format!( - "Failed to allocate a unique EC ID after {MAX_CREATE_ATTEMPTS} attempts" - ), - })) + self.ec_value = Some(ec_id); + self.ec_generated = true; + self.pull_sync_marker.invalidate_for_replaced_ec(); + Ok(()) } /// Returns the EC ID value, if present (either from request or generated). @@ -609,6 +611,37 @@ impl EcContext { self.pull_sync_marker = state; } + /// Stores the navigation permit issued by the router. + pub fn set_navigation_permit(&mut self, permit: NavigationPermit) { + self.navigation_permit = PermitSlot(Some(permit)); + } + + /// Takes the navigation permit; returns `None` after the first call. + pub(crate) fn take_navigation_permit(&mut self) -> Option { + self.navigation_permit.0.take() + } + + /// Takes the navigation permit, for adapter tests. + #[cfg(any(test, feature = "test-utils"))] + pub fn take_navigation_permit_for_test(&mut self) -> Option { + self.take_navigation_permit() + } + + /// Stores validated provider lookup results for finalize. + pub fn set_lookup_results(&mut self, results: LookupResults) { + self.lookup_results = results; + } + + /// Provider lookup results collected for this navigation. + pub(crate) fn lookup_results(&self) -> &LookupResults { + &self.lookup_results + } + + /// Takes the row built by [`Self::generate_if_needed`]. + pub(crate) fn take_pending_entry(&mut self) -> Option { + self.pending_entry.take().map(|entry| *entry) + } + /// Replaces an orphaned active ID after its new backing row is persisted. pub(crate) fn replace_with_generated(&mut self, ec_id: String, snapshot: EcKvSnapshot) { self.ec_value = Some(ec_id); @@ -667,6 +700,9 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } @@ -692,6 +728,9 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } @@ -720,6 +759,9 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } } @@ -768,69 +810,9 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; - use crate::ec::kv_backend::test_support::InMemoryEcKv; - use crate::ec::kv_backend::{ - EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, - }; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; - /// [`EcKvStore`] wrapper whose first `collisions` `Add` writes report a - /// precondition failure, forcing generation to retry with a fresh suffix. - struct AddCollidingEcKv { - inner: InMemoryEcKv, - collisions_remaining: std::sync::Mutex, - } - - impl AddCollidingEcKv { - fn new(collisions: u32) -> Self { - Self { - inner: InMemoryEcKv::new("add-colliding-store"), - collisions_remaining: std::sync::Mutex::new(collisions), - } - } - } - - impl EcKvStore for AddCollidingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::Add) { - let mut remaining = self - .collisions_remaining - .lock() - .expect("should lock collision counter"); - if *remaining > 0 { - *remaining -= 1; - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - } - self.inner.insert(key, write) - } - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - fn granting_consent() -> ConsentContext { ConsentContext { jurisdiction: Jurisdiction::NonRegulated, @@ -840,37 +822,48 @@ mod tests { } #[test] - fn generate_if_needed_retries_id_collision_then_persists() { + fn generate_if_needed_defers_row_creation_to_the_page_load_write() { let settings = create_test_settings(); - let mut ec = - EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.5".to_owned())); - let graph = KvIdentityGraph::new(AddCollidingEcKv::new(2)); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + let graph = KvIdentityGraph::in_memory("gen"); - ec.generate_if_needed(&settings, Some(&graph)) - .expect("should generate after bounded collisions"); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint EC"); - assert!(ec.ec_value().is_some(), "should allocate a fresh EC ID"); - assert!(ec.ec_generated(), "should mark the EC as generated"); + let ec_id = ec_context.ec_value().expect("should set EC").to_owned(); + assert!( + graph.get(&ec_id).expect("should read").is_none(), + "should not write during generation" + ); + assert!( + ec_context.take_pending_entry().is_some(), + "should hold the new row for finalize" + ); assert!( - matches!(ec.kv_snapshot(), EcKvSnapshot::Present { .. }), - "generation should seed a present snapshot" + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }), + "should record missing row" ); } #[test] - fn generate_if_needed_errors_after_collision_exhaustion() { - let settings = create_test_settings(); - let mut ec = - EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.6".to_owned())); - // Collide on every attempt so the bounded retry is exhausted. - let graph = KvIdentityGraph::new(AddCollidingEcKv::new(u32::MAX)); + fn cloned_context_never_carries_the_navigation_permit() { + let mut ec_context = EcContext::default(); + ec_context.set_navigation_permit(NavigationPermit::for_test()); - let result = ec.generate_if_needed(&settings, Some(&graph)); + let mut clone = ec_context.clone(); - assert!(result.is_err(), "should fail after exhausting attempts"); assert!( - ec.ec_value().is_none() && !ec.ec_generated(), - "must not activate an EC ID it could not persist" + clone.take_navigation_permit().is_none(), + "should drop permit on clone" + ); + assert!( + ec_context.take_navigation_permit().is_some(), + "should keep permit on original" ); } diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs new file mode 100644 index 0000000000..212034c3f4 --- /dev/null +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -0,0 +1,320 @@ +//! The one-KV-write-per-page-load rule, enforced by types. +//! +//! A request classified as a top-level document navigation receives a +//! [`NavigationPermit`]. Once the final response status is known it becomes a +//! [`PageLoadWrite`], which [`KvIdentityGraph::commit_page_load`] consumes. +//! Neither type is `Clone`, so one navigation makes at most one write. +//! +//! [`KvIdentityGraph::commit_page_load`]: super::kv::KvIdentityGraph::commit_page_load + +use edgezero_core::body::Body as EdgeBody; +use http::{Method, Request, StatusCode}; + +use super::EcKvSnapshot; +use super::kv_types::KvEntry; + +/// Issued to a request classified as a page-load navigation. +#[derive(Debug)] +pub struct NavigationPermit(()); + +impl NavigationPermit { + /// Returns a permit when `request` is a top-level document navigation: + /// `GET`, `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate`, and no + /// `prefetch` token in `Sec-Purpose`. + /// + /// Call once per request, from the publisher-proxy route only. + #[must_use] + pub fn classify(request: &Request) -> Option { + let headers = request.headers(); + let header_is = |name: &str, expected: &str| { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case(expected)) + }; + let prefetch = headers + .get("sec-purpose") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split([';', ',']) + .any(|token| token.trim().eq_ignore_ascii_case("prefetch")) + }); + let navigation = request.method() == Method::GET + && header_is("sec-fetch-dest", "document") + && header_is("sec-fetch-mode", "navigate") + && !prefetch; + navigation.then_some(Self(())) + } + + /// Confirms the permit once the final response status is known. + /// + /// Returns `None` for a redirect (301, 302, 303, 307, 308): the redirect + /// target is the page load. Other 3xx statuses, such as 304 Not Modified, + /// are the page load itself and keep the permit. + #[must_use] + pub fn confirm(self, status: StatusCode) -> Option { + let redirect = matches!( + status, + StatusCode::MOVED_PERMANENTLY + | StatusCode::FOUND + | StatusCode::SEE_OTHER + | StatusCode::TEMPORARY_REDIRECT + | StatusCode::PERMANENT_REDIRECT + ); + (!redirect).then_some(PageLoadWrite(())) + } + + /// Builds a permit without a request, for tests. + #[cfg(test)] + #[must_use] + pub fn for_test() -> Self { + Self(()) + } +} + +/// Proof that the current request may make the page load's single KV write. +#[derive(Debug)] +pub struct PageLoadWrite(()); + +impl PageLoadWrite { + /// Consumes the permit. Called only by the write itself. + pub(crate) fn spend(self) {} +} + +/// KV generation a conditional write expects. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Generation(u64); + +impl Generation { + /// Reads the generation captured in `snapshot` for `ec_id`. + /// + /// Returns `None` when the snapshot has no generation, so no blind + /// overwrite can be planned. + #[must_use] + pub fn of(snapshot: &EcKvSnapshot, ec_id: &str) -> Option { + snapshot.generation_for(ec_id).map(Self) + } + + /// The raw generation. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } +} + +/// The single row change of a navigation, with its CAS precondition. +#[derive(Debug)] +pub enum RowChange { + /// Create a row that must not exist (new EC or orphan recovery). + Create(KvEntry), + /// Replace the row only if it is still at `expected`. + Update { + /// The full new row. + entry: KvEntry, + /// Generation the row must still have. + expected: Generation, + }, + /// Replace a live row with a withdrawal tombstone. + Tombstone { + /// Generation the row must still have. + expected: Generation, + }, +} + +/// Why the single write did not land. +#[derive(Debug, derive_more::Display)] +pub enum KvWriteError { + /// The row changed since it was read. + #[display("row changed since it was read")] + Conflict, + /// A create found an existing row. + #[display("row already exists")] + AlreadyExists, + /// The KV store rejected the write. + #[display("KV store rejected the write")] + Store, +} + +impl core::error::Error for KvWriteError {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::KvEntry; + + fn request(headers: &[(&str, &str)]) -> http::Request { + let mut builder = http::Request::builder() + .method(http::Method::GET) + .uri("https://www.example.com/"); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + builder + .body(EdgeBody::empty()) + .expect("should build request") + } + + const DOCUMENT: [(&str, &str); 2] = [ + ("sec-fetch-dest", "document"), + ("sec-fetch-mode", "navigate"), + ]; + + #[test] + fn classify_accepts_top_level_document_navigation() { + assert!( + NavigationPermit::classify(&request(&DOCUMENT)).is_some(), + "should permit navigation" + ); + } + + #[test] + fn classify_rejects_missing_fetch_metadata_iframes_and_prefetch() { + assert!( + NavigationPermit::classify(&request(&[])).is_none(), + "should reject without fetch metadata" + ); + assert!( + NavigationPermit::classify(&request(&[ + ("sec-fetch-dest", "iframe"), + ("sec-fetch-mode", "navigate") + ])) + .is_none(), + "should reject iframe" + ); + let mut prefetch = DOCUMENT.to_vec(); + prefetch.push(("sec-purpose", "prefetch;prerender")); + assert!( + NavigationPermit::classify(&request(&prefetch)).is_none(), + "should reject prefetch" + ); + } + + #[test] + fn confirm_rejects_redirects() { + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::FOUND) + .is_none(), + "should not write on redirect" + ); + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::NOT_FOUND) + .is_some(), + "should write on error page" + ); + } + + #[test] + fn confirm_keeps_not_modified_and_drops_redirects() { + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::NOT_MODIFIED) + .is_some(), + "should write on 304 Not Modified" + ); + for status in [ + http::StatusCode::FOUND, + http::StatusCode::TEMPORARY_REDIRECT, + ] { + assert!( + NavigationPermit::for_test().confirm(status).is_none(), + "should not write on redirect {status}" + ); + } + } + + fn permit() -> PageLoadWrite { + NavigationPermit::for_test() + .confirm(http::StatusCode::OK) + .expect("should confirm") + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + #[test] + fn commit_create_reports_existing_row() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .commit_page_load(permit(), &ec_id, RowChange::Create(live_entry())) + .expect("should create row"); + + let error = graph + .commit_page_load(permit(), &ec_id, RowChange::Create(live_entry())) + .expect_err("should refuse second create"); + + assert!( + matches!(error.current_context(), KvWriteError::AlreadyExists), + "should report existing row" + ); + } + + #[test] + fn commit_update_with_stale_generation_conflicts_without_retry() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let snapshot = graph.load_snapshot(&ec_id); + let expected = Generation::of(&snapshot, &ec_id).expect("should read generation"); + graph + .create_or_revive(&ec_id, &live_entry()) + .expect("should no-op on live row"); + graph + .commit_page_load( + permit(), + &ec_id, + RowChange::Update { + entry: live_entry(), + expected, + }, + ) + .expect("should write at current generation"); + + let error = graph + .commit_page_load( + permit(), + &ec_id, + RowChange::Update { + entry: live_entry(), + expected, + }, + ) + .expect_err("should reject stale generation"); + + assert!( + matches!(error.current_context(), KvWriteError::Conflict), + "should report conflict" + ); + } + + #[test] + fn commit_tombstone_marks_consent_withdrawn() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let expected = + Generation::of(&graph.load_snapshot(&ec_id), &ec_id).expect("should read generation"); + + let committed = graph + .commit_page_load(permit(), &ec_id, RowChange::Tombstone { expected }) + .expect("should write tombstone"); + + assert!(!committed.consent.ok, "should return tombstone"); + let (stored, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!(!stored.consent.ok, "should persist tombstone"); + } +} diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 3e2fbee219..7be48e1e6d 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -157,7 +157,10 @@ pub(crate) fn collect_eid_updates( { updates.push(update); } - dedupe_partner_updates(updates) + let mut updates = dedupe_partner_updates(updates); + // Provider-owned sources are written only by their identity provider. + updates.retain(|update| !registry.is_provider_owned(&update.partner_id)); + updates } pub(crate) fn collect_prebid_eid_updates( diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 784c217e51..4aa8617f62 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -560,6 +560,7 @@ mod tests { pull_sync_ttl_sec: ttl_sec, pull_sync_rate_limit: 20, ts_pull_token: Some(Redacted::new("token".to_owned())), + provider_owned: false, } } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index c4637b4310..b138ecbe16 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -47,6 +47,9 @@ pub struct PartnerConfig { pub pull_sync_rate_limit: u32, /// Outbound bearer token for pull sync requests. pub ts_pull_token: Option>, + /// Whether an enabled identity provider claims this source. Browser + /// ingest skips it and batch sync rejects it. + pub provider_owned: bool, } /// In-memory partner registry with O(1) lookups by source domain and API key hash. @@ -234,6 +237,29 @@ impl PartnerRegistry { self.get(domain) } + /// Marks `source` as owned by an enabled identity provider. + /// + /// Returns `false` when the source has no `[[ec.partners]]` entry. + pub fn mark_provider_owned(&mut self, source: &str) -> bool { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return false; + }; + match self.by_source_domain.get_mut(&normalized) { + Some(partner) => { + partner.provider_owned = true; + true + } + None => false, + } + } + + /// Whether an enabled identity provider owns `source`. + #[must_use] + pub fn is_provider_owned(&self, source: &str) -> bool { + self.find_by_source_domain(source) + .is_some_and(|partner| partner.provider_owned) + } + /// Returns all partners with `pull_sync_enabled = true`. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { @@ -305,6 +331,7 @@ fn build_partner_config( pull_sync_ttl_sec: partner.pull_sync_ttl_sec, pull_sync_rate_limit: partner.pull_sync_rate_limit, ts_pull_token: partner.ts_pull_token.clone(), + provider_owned: false, } } diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs new file mode 100644 index 0000000000..075b14b59e --- /dev/null +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -0,0 +1,354 @@ +//! Plans a navigation's single row change. +//! +//! Combines browser ingest (fill missing, never provider-owned sources) and +//! validated provider results (replace the provider's sources), prunes +//! expired UIDs and enforces [`MAX_ENTRY_BYTES`]. Pure: no I/O. + +use super::EcKvSnapshot; +use super::coordinator::results::LookupResults; +use super::eid_types::{SourceDomain, UnixSeconds}; +use super::kv::PartnerIdUpdate; +use super::kv_types::{KvEntry, KvPartnerId, MAX_ENTRY_BYTES}; +use super::page_load::{Generation, RowChange}; +use super::registry::PartnerRegistry; + +/// Inputs to [`plan_write`]. +#[derive(Debug, Clone, Copy)] +pub struct WritePlanInput<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Row snapshot read for this request. + pub snapshot: &'a EcKvSnapshot, + /// The new row when this navigation creates one (new EC or recovery). + pub new_entry: Option<&'a KvEntry>, + /// Browser updates from `ts-eids` and `sharedId`, already reduced by the + /// ingest rules and consent-gated. + pub browser_ingest: &'a [PartnerIdUpdate], + /// Validated provider results. + pub lookups: &'a LookupResults, + /// Partner registry, for provider ownership. + pub registry: &'a PartnerRegistry, + /// Request time. + pub now: UnixSeconds, +} + +/// Plans the navigation's single row change. +/// +/// Returns `None` when nothing would change, when the row is a tombstone, or +/// when an update has no generation to guard it. +#[must_use] +pub fn plan_write(input: &WritePlanInput<'_>) -> Option { + let (base, expected) = match input.new_entry { + Some(entry) => (entry.clone(), None), + None => { + let entry = input.snapshot.entry_for(input.ec_id)?; + if !entry.consent.ok { + return None; + } + let Some(expected) = Generation::of(input.snapshot, input.ec_id) else { + log::info!( + "EC write planner: skipping update for '{}' (no_generation)", + super::log_id(input.ec_id) + ); + return None; + }; + (entry.clone(), Some(expected)) + } + }; + + let mut entry = base.clone(); + apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); + let written = apply_lookups(&mut entry, input.lookups, input.now); + for record in entry.ids.values_mut() { + record.prune_expired(input.now); + } + entry.ids.retain(|_, record| !record.uid.is_empty()); + enforce_size_cap(&mut entry, &written); + + match expected { + None => Some(RowChange::Create(entry)), + Some(_) if entry == base => None, + Some(expected) => Some(RowChange::Update { entry, expected }), + } +} + +fn apply_browser_ingest( + entry: &mut KvEntry, + updates: &[PartnerIdUpdate], + registry: &PartnerRegistry, +) { + for update in updates { + if registry.is_provider_owned(&update.partner_id) { + continue; + } + entry + .ids + .entry(update.partner_id.clone()) + .or_insert_with(|| KvPartnerId::single(update.uid.clone())); + } +} + +fn apply_lookups( + entry: &mut KvEntry, + lookups: &LookupResults, + now: UnixSeconds, +) -> Vec { + let mut written = Vec::new(); + for lookup in lookups.iter() { + for (source, eids) in &lookup.entries { + if let Some(record) = KvPartnerId::from_eids(eids.clone(), now) { + entry.ids.insert(source.as_str().to_owned(), record); + written.push(source.clone()); + } + } + } + written +} + +fn serialized_len(entry: &KvEntry) -> usize { + serde_json::to_vec(entry).map_or(usize::MAX, |bytes| bytes.len()) +} + +fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { + while serialized_len(entry) > MAX_ENTRY_BYTES { + let Some(source) = written.iter().rev().find(|source| { + entry + .ids + .get(source.as_str()) + .is_some_and(KvPartnerId::is_provider_written) + }) else { + log::warn!("EC write planner: row exceeds size cap with no provider UIDs left to drop"); + return; + }; + let record = entry + .ids + .get_mut(source.as_str()) + .expect("should hold a written source"); + record.drop_last_uid(); + if record.uid.is_empty() { + entry.ids.remove(source.as_str()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; + use crate::settings::EcPartner; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn partner(source: &str) -> EcPartner { + serde_json::from_value(serde_json::json!({ "name": source, "source_domain": source, "bidstream_enabled": true })) + .expect("should build partner") + } + + fn registry(owned: &[&str]) -> PartnerRegistry { + let mut registry = PartnerRegistry::from_config(&[ + partner("browser.example.com"), + partner("ids.example.com"), + ]) + .expect("should build registry"); + for source in owned { + assert!(registry.mark_provider_owned(source), "should mark {source}"); + } + registry + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + fn present(entry: &KvEntry) -> EcKvSnapshot { + let graph = KvIdentityGraph::in_memory("plan"); + graph.create(EC_ID, entry).expect("should seed row"); + graph.load_snapshot(EC_ID) + } + + fn found(source: &str, ids: &[&str]) -> LookupResults { + let uids = ids + .iter() + .map(|id| { + KvUid::new( + UidValue::try_from(*id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ) + }) + .collect(); + LookupResults::new(vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), uids)], + )], + }]) + } + + fn input<'a>( + snapshot: &'a EcKvSnapshot, + new_entry: Option<&'a KvEntry>, + ingest: &'a [PartnerIdUpdate], + lookups: &'a LookupResults, + registry: &'a PartnerRegistry, + ) -> WritePlanInput<'a> { + WritePlanInput { + ec_id: EC_ID, + snapshot, + new_entry, + browser_ingest: ingest, + lookups, + registry, + now: UnixSeconds::new(2000), + } + } + + #[test] + fn new_ec_creates_row_with_ingest_and_lookups() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let ingest = [ + PartnerIdUpdate::new("browser.example.com", "b-1"), + PartnerIdUpdate::new("ids.example.com", "client"), + ]; + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &ingest, + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert_eq!( + entry.ids["browser.example.com"].uid, "b-1", + "should ingest browser uid" + ); + assert_eq!( + entry.ids["ids.example.com"].uid, "p-1", + "should store provider uid, not client uid" + ); + } + + #[test] + fn unchanged_returning_row_plans_no_write() { + let registry = registry(&[]); + let mut entry = live_entry(); + entry + .ids + .insert("browser.example.com".to_owned(), KvPartnerId::single("b-1")); + let snapshot = present(&entry); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-2")]; + let lookups = LookupResults::default(); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite an existing browser uid" + ); + } + + #[test] + fn lookup_replaces_provider_source_with_cas_update() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + let snapshot = present(&entry); + let lookups = found("ids.example.com", &["p-1", "p-2"]); + + let change = plan_write(&input(&snapshot, None, &[], &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert_eq!( + entry.ids["ids.example.com"].eids()[0].uids().len(), + 2, + "should keep both provider uids" + ); + } + + #[test] + fn tombstoned_row_plans_nothing() { + let registry = registry(&[]); + let snapshot = present(&KvEntry::tombstone(1000)); + let lookups = found("ids.example.com", &["p-1"]); + assert!( + plan_write(&input(&snapshot, None, &[], &lookups, ®istry)).is_none(), + "should never write over a tombstone" + ); + } + + #[test] + fn snapshot_without_generation_plans_nothing() { + let registry = registry(&[]); + let snapshot = EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(live_entry()), + generation: None, + }; + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-1")]; + let lookups = LookupResults::default(); + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite blindly" + ); + } + + #[test] + fn oversized_row_drops_provider_uids_until_it_fits() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let big: Vec = (0..40) + .map(|i| format!("{i:03}{}", "x".repeat(500))) + .collect(); + let ids: Vec<&str> = big.iter().map(String::as_str).collect(); + let lookups = found("ids.example.com", &ids); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &[], + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create") + }; + let size = serde_json::to_vec(&entry).expect("should serialize").len(); + assert!( + size <= MAX_ENTRY_BYTES, + "should fit the row cap, got {size}" + ); + let kept = entry.ids["ids.example.com"].eids()[0].uids(); + assert!(kept.len() < 40, "should drop some uids"); + assert_eq!(kept[0].id().as_str(), ids[0], "should drop from the end"); + } +} diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 15a6486fd8..fc40711240 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -14,7 +14,6 @@ use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::error::TrustedServerError; use crate::geo::GeoInfo; -use crate::http_util::is_navigation_request; use crate::platform::RuntimeServices; use crate::settings::Settings; use crate::streaming_processor::StreamProcessor; @@ -1018,7 +1017,8 @@ impl IntegrationRegistry { /// Dispatch a proxy request when an integration handles the path. /// /// This method removes any caller-supplied `x-ts-ec` before proxying. - /// Response-side cookie mutation is centralized in EC finalize. + /// Response-side cookie mutation is centralized in EC finalize. It never + /// mints an EC: only a publisher navigation may create the EC row. #[must_use] pub async fn handle_proxy( &self, @@ -1028,24 +1028,14 @@ impl IntegrationRegistry { method, path, settings, - kv, - ec_context, + kv: _, + ec_context: _, services, mut req, } = input; if let Some((proxy, _)) = self.find_route(method, path) { - // Organic proxy handler: generate if needed (best effort). - // Only generate for document navigations — subresource requests - // may lack consent signals such as the Sec-GPC header. - if is_navigation_request(&req) { - if let Err(err) = ec_context.generate_if_needed(settings, kv) { - log::warn!("EC generation failed for integration proxy: {err:?}"); - } - } else { - log::debug!( - "EC generation skipped for integration proxy: non-document request (path={path})", - ); - } + // Integration proxies never mint an EC: only the publisher + // navigation may create the row (one write per page load). // Remove any caller-supplied EC header rather than forwarding it. req.headers_mut().remove(HEADER_X_TS_EC.clone()); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index a670d31416..b3e2e6e20d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -16350,6 +16350,9 @@ mod tests { scheduling_consent() }; let mut ec_context = EcContext::new_for_test(Some(identity.clone()), consent); + // A publisher page load: only a navigation may write the tombstone. + ec_context + .set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); assert_eq!( ec_context.ec_allowed(), !withdrawn, From 3f36d00d409205734b24515bf160c6e84b5eb139 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:31:32 +0530 Subject: [PATCH 03/38] Collect identity provider lookups on publisher navigations Add the IdentityProvider/ProviderKind interface and [[ec.identity_providers]] config, response validation, a deadline-bounded LookupTransport over select_until, the IdentityCoordinator with consent, allowlist and budget checks, the trusted-server-identity-providers crate (no kinds yet), and Fastly wiring that dispatches before the origin fetch and collects before the single page-load write. Signed-off-by: dhruv8sh --- .cargo/config.toml | 4 +- Cargo.lock | 10 + Cargo.toml | 2 + crates/trusted-server-adapter-axum/src/app.rs | 3 +- .../src/app.rs | 2 +- .../trusted-server-adapter-fastly/Cargo.toml | 1 + .../trusted-server-adapter-fastly/src/app.rs | 369 +++++- .../trusted-server-adapter-fastly/src/main.rs | 38 +- crates/trusted-server-adapter-spin/src/app.rs | 2 +- crates/trusted-server-core/src/config.rs | 160 ++- .../trusted-server-core/src/config_payload.rs | 41 + crates/trusted-server-core/src/consent/mod.rs | 37 + .../src/ec/coordinator/mod.rs | 1064 ++++++++++++++++- .../src/ec/coordinator/transport.rs | 459 +++++++ .../src/ec/coordinator/validate.rs | 354 ++++++ crates/trusted-server-core/src/ec/mod.rs | 2 + crates/trusted-server-core/src/ec/provider.rs | 681 +++++++++++ .../src/ec/pull_sync_marker.rs | 4 +- crates/trusted-server-core/src/ec/registry.rs | 51 +- crates/trusted-server-core/src/publisher.rs | 23 +- crates/trusted-server-core/src/settings.rs | 117 ++ .../Cargo.toml | 21 + .../src/lib.rs | 146 +++ docs/guide/configuration.md | 49 + trusted-server.example.toml | 23 + 25 files changed, 3635 insertions(+), 28 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/coordinator/transport.rs create mode 100644 crates/trusted-server-core/src/ec/coordinator/validate.rs create mode 100644 crates/trusted-server-core/src/ec/provider.rs create mode 100644 crates/trusted-server-identity-providers/Cargo.toml create mode 100644 crates/trusted-server-identity-providers/src/lib.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index aea0df2d59..0c81704c8b 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -28,8 +28,8 @@ test_details = "test --target aarch64-apple-darwin" # (native) are simply not listed. build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-identity-providers -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-identity-providers -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. diff --git a/Cargo.lock b/Cargo.lock index 70316712ac..d9cf35f9f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6604,6 +6604,7 @@ dependencies = [ "serde_json", "toml", "trusted-server-core", + "trusted-server-identity-providers", "url", "urlencoding", "web-time", @@ -6737,6 +6738,15 @@ dependencies = [ "web-time", ] +[[package]] +name = "trusted-server-identity-providers" +version = "0.1.0" +dependencies = [ + "error-stack", + "toml", + "trusted-server-core", +] + [[package]] name = "trusted-server-integration-tests" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 139cf4f610..f431012308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/trusted-server-adapter-spin", "crates/trusted-server-cli", "crates/trusted-server-core", + "crates/trusted-server-identity-providers", "crates/trusted-server-integration-tests", "crates/trusted-server-js", "crates/trusted-server-openrtb", @@ -121,6 +122,7 @@ toml_edit = "0.23.10" tower = "0.4" tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } +trusted-server-identity-providers = { path = "crates/trusted-server-identity-providers" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } url = "2.5.8" diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 0b1858e208..314580e79a 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -499,8 +499,7 @@ fn named_route_handler( Ok(admin_ec_lookup_not_supported()) } NamedRouteHandler::AdminEidsLookup => { - let partner_registry = - PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; handle_admin_eids_lookup(&partner_registry, &req) } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 668b025ccb..54029b36c7 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -655,7 +655,7 @@ fn build_router(state: &Arc) -> RouterService { .get( "/_ts/admin/eids", make_handler(Arc::clone(&state), |s, _services, req| async move { - let partner_registry = PartnerRegistry::from_config(&s.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&s.settings.ec)?; handle_admin_eids_lookup(&partner_registry, &req) }), ) diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 6c7121d0d6..0e5d6aaceb 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -37,6 +37,7 @@ log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-identity-providers = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } web-time = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 7f75267188..3b454d9725 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -86,6 +86,8 @@ //! [`load_settings_from_config_store`]; if settings loading itself fails, they //! are returned without geo or TS headers. +use std::cell::RefCell; +use std::net::IpAddr; use std::sync::Arc; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; @@ -111,12 +113,18 @@ use trusted_server_core::ec::admin::{ }; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; +use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; +use trusted_server_core::ec::coordinator::{ + IdentityCoordinator as _, LookupContext, PendingLookups, ProviderCoordinator, +}; use trusted_server_core::ec::device::DeviceSignals; +use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; +use trusted_server_core::ec::provider::ClientSignals; use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::ec::{EcContext, EidSyncSource}; +use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::is_navigation_request; use trusted_server_core::integrations::{ @@ -124,7 +132,8 @@ use trusted_server_core::integrations::{ RequestFilterRegistryOutcome, }; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformGeo as _, PlatformKvStore, RuntimeServices, StoreName, + ClientInfo, GeoInfo, PlatformGeo as _, PlatformKvStore, PlatformPendingRequest, + RuntimeServices, StoreName, }; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, @@ -143,6 +152,7 @@ use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; +use trusted_server_identity_providers::build_coordinator; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -335,6 +345,95 @@ pub(crate) struct EcFinalizeState { pub(crate) services: RuntimeServices, } +/// Identity lookups dispatched for this navigation, collected before finalize. +/// +/// In-flight platform requests are not `Send`, so they cannot ride in +/// response extensions with [`EcFinalizeState`]. They travel through a +/// request-local slot instead: the entry point clears it before routing and +/// takes it after the router returns. +pub(crate) struct IdentityLookups { + pub(crate) coordinator: ProviderCoordinator, + pub(crate) pending: PendingLookups, +} + +thread_local! { + static PENDING_IDENTITY_LOOKUPS: RefCell> = const { RefCell::new(None) }; +} + +/// Stores this request's dispatched identity lookups for the entry point. +fn stash_identity_lookups(lookups: IdentityLookups) { + PENDING_IDENTITY_LOOKUPS.with(|slot| *slot.borrow_mut() = Some(lookups)); +} + +/// Takes the identity lookups dispatched for the current request, if any. +/// +/// The entry point calls this before routing (discarding anything a prior +/// request left behind) and after the router returns. +pub(crate) fn take_identity_lookups() -> Option { + PENDING_IDENTITY_LOOKUPS.with(|slot| slot.borrow_mut().take()) +} + +/// Starts identity-provider lookups for a permitted publisher navigation. +/// +/// Returns `None` when no provider is enabled, the request has no EC, or the +/// identity graph is unavailable. A provider configuration that cannot be +/// built is logged and disables lookups for this request. +async fn dispatch_identity_lookups( + state: &AppState, + services: &RuntimeServices, + req: &Request, + ec: &mut EcRequestState, +) -> Option { + let coordinator = match build_coordinator(&state.settings) { + Ok(Some(coordinator)) => coordinator, + Ok(None) => return None, + Err(report) => { + log::error!("Identity providers disabled for this request: {report:?}"); + return None; + } + }; + let graph = ec.kv_graph.as_ref()?; + let ec_id = ec.ec_context.ec_value()?.to_owned(); + if matches!(ec.ec_context.kv_snapshot(), EcKvSnapshot::NotRead) { + let snapshot = graph.load_snapshot(&ec_id); + ec.ec_context.set_kv_snapshot(snapshot); + } + let transport = PlatformLookupTransport::new(services, state.settings.proxy.certificate_check); + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + let client_ip = ec + .ec_context + .client_ip() + .and_then(|ip| ip.parse::().ok()); + let user_agent = req + .headers() + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()); + let pending = coordinator + .dispatch( + &transport, + &LookupContext { + ec_id: &ec_id, + snapshot: ec.ec_context.kv_snapshot(), + consent: ec.ec_context.consent(), + client: ClientSignals { + ip: client_ip, + user_agent, + }, + publisher_domain: &state.settings.publisher.domain, + rate_limiter: &limiter, + now: UnixSeconds::new(now), + }, + ) + .await; + Some(IdentityLookups { + coordinator, + pending, + }) +} + /// Per-request EC identity state built before dispatch, mirroring the /// pre-routing prelude of the legacy `route_request` (device signals, bot /// gate, cookie capture, consent/geo-aware [`EcContext`], and KV graph @@ -580,7 +679,7 @@ async fn execute_named( handler, NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup ) { - let response = PartnerRegistry::from_config(&state.settings.ec.partners) + let response = PartnerRegistry::from_ec(&state.settings.ec) .and_then(|registry| match handler { NamedRouteHandler::AdminEcLookup => { // Deliberately do not use an EC request-state graph: that @@ -661,7 +760,7 @@ async fn run_named_route( cors_preflight_identify(&state.settings, &req) } else { let kv = crate::require_identity_graph(&state.settings)?; - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; handle_identify( &state.settings, &kv, @@ -675,7 +774,7 @@ async fn run_named_route( NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; let registry_ref = if partner_registry.is_empty() { None } else { @@ -699,7 +798,7 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; let registry_ref = if partner_registry.is_empty() { None } else { @@ -746,7 +845,7 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); handle_batch_sync(&kv, &partner_registry, &limiter, req) }); @@ -849,12 +948,14 @@ async fn dispatch_fallback( // Only for document navigations by recognised browsers; subresource // requests may lack consent signals such as Sec-GPC. let is_navigation = is_navigation_request(&req); + let mut has_permit = false; if is_navigation { ec.ec_context.set_eid_sync_source(EidSyncSource::Navigation); // The page-load write permit uses the stricter Fetch Metadata test // (spec §3.1); requests without it never write KV. if let Some(permit) = NavigationPermit::classify(&req) { ec.ec_context.set_navigation_permit(permit); + has_permit = true; } } let is_publisher_navigation = ec.is_real_browser && is_navigation; @@ -866,6 +967,15 @@ async fn dispatch_fallback( log::warn!("EC generation failed for publisher proxy: {err:?}"); } + // Identity lookups start before the origin fetch so they overlap it; + // the entry point collects them before EC finalization. + if is_publisher_navigation + && has_permit + && let Some(lookups) = dispatch_identity_lookups(state, services, &req, &mut ec).await + { + stash_identity_lookups(lookups); + } + // Run the server-side auction with the configured creative- // opportunity slots and collect dispatched bids from the lazy // publisher body stream. `handle_publisher_request` matches the @@ -873,7 +983,7 @@ async fn dispatch_fallback( // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. let slots = state.settings.creative_opportunity_slots(); - match PartnerRegistry::from_config(&state.settings.ec.partners) { + match PartnerRegistry::from_ec(&state.settings.ec) { Ok(partner_registry) => { let auction = AuctionDispatch { integration_registry: &state.registry, @@ -1427,8 +1537,24 @@ mod tests { use error_stack::Report; use futures::StreamExt as _; use futures::executor::block_on; + use std::borrow::Cow; + use std::num::NonZeroUsize; + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::consent::jurisdiction::Jurisdiction; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; + use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; + use trusted_server_core::ec::coordinator::{ + IdentityCoordinator as _, LookupContext, ProviderCoordinator, RegisteredProvider, + }; use trusted_server_core::ec::device::DeviceSignals; + use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; + use trusted_server_core::ec::provider::{ + ClientSignals, EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, + ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, + ProviderResponse, ProviderUid, SourceCapability, SourceInput, + }; + use trusted_server_core::ec::rate_limiter::RateLimiter; + use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::{ HeaderMutation, IntegrationRegistry, IntegrationRequestFilter, RequestFilterDecision, @@ -1441,7 +1567,7 @@ mod tests { RuntimeServices, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, }; - use trusted_server_core::settings::Settings; + use trusted_server_core::settings::{EcPartner, IdentityProviderConfig, Settings}; use trusted_server_core::test_support::nextjs_auction; #[test] @@ -2612,6 +2738,231 @@ mod tests { ); } + #[test] + fn publisher_navigation_without_providers_dispatches_nothing() { + let router = test_router(); + + let mut response = route(&router, navigation_request("/article")); + + let _state = take_finalize_state(&mut response); + assert!( + super::take_identity_lookups().is_none(), + "should dispatch no identity lookups without enabled providers" + ); + } + + /// Returns `p-1` for the lookup it was sent, then reports nothing left. + struct LookupHttpClient; + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for LookupHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + Ok(PlatformPendingRequest::new(()).with_backend_name(request.backend_name)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn select_until( + &self, + mut pending_requests: Vec, + _deadline: web_time::Instant, + ) -> Result, Report> { + let Some(ready) = pending_requests.pop() else { + return Ok(None); + }; + let backend_name = ready + .backend_name() + .expect("should carry the backend name") + .to_owned(); + let response = edgezero_core::http::response_builder() + .status(StatusCode::OK) + .body(Body::from("p-1")) + .map_err(|_| Report::new(PlatformError::HttpClient))?; + Ok(Some(PlatformSelectResult { + ready: Ok(PlatformResponse::new(response).with_backend_name(backend_name)), + remaining: pending_requests, + failed_backend_name: None, + })) + } + } + + /// Looks up `ids.example.com` and returns the response body as its UID. + struct EchoProvider(ProviderCapabilities); + + impl IdentityProvider for EchoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Ok(ProviderRequest { + method: Method::GET, + url: url::Url::parse("https://ids.example.com/lookup").expect("should parse"), + headers: edgezero_core::http::HeaderMap::new(), + body: Bytes::new(), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let id = std::str::from_utf8(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + }], + }])) + } + } + + struct AllowAll; + + impl RateLimiter for AllowAll { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Ok(false) + } + } + + #[test] + fn collected_results_reach_the_page_load_write() { + let provider_config: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", + ) + .expect("should parse provider config"); + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true + })) + .expect("should build partner"); + let mut settings = test_settings(); + settings.ec.ec_store = Some("ec_identity_store".to_owned()); + settings.ec.partners = vec![partner]; + settings.ec.identity_providers = vec![provider_config.clone()]; + let router = TrustedServerApp::routes_for_state(&app_state_for_settings(settings.clone())); + let mut request = navigation_request("/article"); + // Viceroy geolocates loopback to a US privacy state; a US Privacy + // string without a sale opt-out lets the navigation mint an EC. + request.extensions_mut().insert(ClientInfo { + client_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ..ClientInfo::default() + }); + request + .headers_mut() + .insert(header::COOKIE, HeaderValue::from_static("us_privacy=1YNN")); + let mut response = route(&router, request); + let mut state = take_finalize_state(&mut response); + let ec_id = state + .ec_context + .ec_value() + .expect("should generate an EC") + .to_owned(); + state.services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(LookupHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + let provider: Box = Box::new(EchoProvider(ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse source"), + input: SourceInput::Anonymous, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + })); + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new(provider_config, provider)], + Arc::new(PartnerRegistry::from_ec(&settings.ec).expect("should build registry")), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = PlatformLookupTransport::new(&state.services, false); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }; + let pending = block_on(coordinator.dispatch( + &transport, + &LookupContext { + ec_id: &ec_id, + snapshot: state.ec_context.kv_snapshot(), + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "test-publisher.com", + rate_limiter: &AllowAll, + now: UnixSeconds::new(1_700_000_000), + }, + )); + + let result = crate::apply_edgezero_ec_finalize( + &settings, + &mut state, + Some(super::IdentityLookups { + coordinator, + pending, + }), + &mut response, + ); + + result.expect("should finalize"); + assert!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .any(|value| value.to_str().is_ok_and(|cookie| cookie.contains(&ec_id))), + "should set the EC cookie after the page-load write" + ); + let graph = crate::require_identity_graph(&settings).expect("should open identity graph"); + let snapshot = graph.load_snapshot(&ec_id); + let entry = snapshot.entry_for(&ec_id).expect("should store the row"); + let row = serde_json::to_string(entry).expect("should serialize row"); + assert!( + row.contains("ids.example.com") && row.contains("p-1"), + "should write the provider EID into the row: {row}" + ); + } + #[test] fn publisher_request_without_fetch_metadata_navigation_gets_no_permit() { let router = test_router(); diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 3cc13fdfa8..a6ac48f369 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -13,6 +13,8 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; +use trusted_server_core::ec::coordinator::IdentityCoordinator as _; +use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; @@ -44,8 +46,8 @@ mod template_cache; mod tinybird; use crate::app::{ - AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, - load_settings_from_config_store, + AppState, EcFinalizeState, IdentityLookups, RuntimeStoreConfig, TrustedServerApp, + load_settings_from_config_store, take_identity_lookups, }; use crate::ec_kv::FastlyEcKvStore; use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; @@ -491,6 +493,8 @@ fn edgezero_main( core_req.extensions_mut().insert(device_signals); } core_req.extensions_mut().insert(client_info); + // Drop lookups a previous request on this sandbox left behind. + drop(take_identity_lookups()); match futures::executor::block_on(app.router().oneshot(core_req)) { Ok(response) => response, Err(error) => edge_error_response(error), @@ -517,6 +521,9 @@ fn edgezero_main( // Pop response extensions before the Fastly conversion, which drops them. let ec_state = response.extensions_mut().remove::(); + // Identity lookups dispatched on a publisher navigation are collected in + // EC finalization, after the router's executor has returned. + let identity = take_identity_lookups(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); @@ -544,7 +551,7 @@ fn edgezero_main( if let Some(mut ec_state) = ec_state { if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize(settings, &mut ec_state, identity, &mut response) { Ok(partner_registry) => { send_edgezero_response( response, @@ -563,7 +570,12 @@ fn edgezero_main( } else { match load_settings_from_config_store(&runtime_stores) { Ok(settings) => { - match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize( + &settings, + &mut ec_state, + identity, + &mut response, + ) { Ok(partner_registry) => { send_edgezero_response( response, @@ -636,12 +648,28 @@ fn apply_entry_point_finalize_headers( apply_finalize_headers(settings, geo_info.as_ref(), response); } +/// Runs EC finalization for a routed response. +/// +/// Collects `identity` lookups first. This must run outside the router's +/// `block_on`: collection drives its own executor, and nesting executors +/// panics. fn apply_edgezero_ec_finalize( settings: &Settings, ec_state: &mut EcFinalizeState, + identity: Option, response: &mut HttpResponse, ) -> Result> { - let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; + if let Some(IdentityLookups { + coordinator, + pending, + }) = identity + { + let transport = + PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); + let results = futures::executor::block_on(coordinator.collect(&transport, pending)); + ec_state.ec_context.set_lookup_results(results); + } + let partner_registry = PartnerRegistry::from_ec(&settings.ec)?; let finalize_kv_graph = if ec_state.use_finalize_kv { maybe_identity_graph(settings) } else { diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 9edce69fa3..f5fec2efc7 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -653,7 +653,7 @@ fn build_router(state: &Arc) -> RouterService { let s = Arc::clone(&s); async move { let req = ctx.into_request(); - let result = PartnerRegistry::from_config(&s.settings.ec.partners) + let result = PartnerRegistry::from_ec(&s.settings.ec) .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); Ok::(result.unwrap_or_else(|e| http_error(&e))) } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 059d500092..85bfd5cf0f 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -13,6 +13,7 @@ use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use validator::{Validate, ValidationError, ValidationErrors}; +use crate::ec::provider::validate_enabled_configs; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; use crate::integrations::{ @@ -32,7 +33,7 @@ use crate::integrations::{ sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; -use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; +use crate::settings::{AssetOriginAuth, Ec, IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] @@ -167,6 +168,16 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { ], true, ), + field( + vec![ + object("ec"), + optional_object("identity_providers"), + SecretPathSegment::ArrayEach, + optional_object("secrets"), + object("token"), + ], + true, + ), field( vec![ object("handlers"), @@ -268,6 +279,7 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report Result<(), Report> { + if ec.identity_providers.is_empty() { + return Ok(()); + } + let partners = PartnerRegistry::from_ec(ec)?; + validate_enabled_configs(&ec.identity_providers, &partners).map_err(|report| { + let detail = report + .frames() + .filter_map(|frame| frame.downcast_ref::()) + .map(String::as_str) + .collect::>() + .join("; "); + report.change_context(TrustedServerError::Configuration { + message: format!("ec.identity_providers: {detail}"), + }) + }) +} + fn validate_js_asset_proxy_config(settings: &Settings) -> Result<(), Report> { let Some(raw_config) = settings.integrations.get(JS_ASSET_PROXY_INTEGRATION_ID) else { return Ok(()); @@ -447,6 +482,15 @@ fn validate_secret_key_references(settings: &Settings) -> Result<(), Report Settings { + let mut settings = valid_settings(); + settings.ec.partners = vec![ + serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + ]; + let mut config = serde_json::json!({ + "id": "example-provider", + "kind": "example-kind", + "endpoint": "https://ids.example.com/v1/lookup", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + }); + if let (Some(base), serde_json::Value::Object(overrides)) = + (config.as_object_mut(), provider) + { + base.extend(overrides); + } + settings.ec.identity_providers = + vec![serde_json::from_value(config).expect("should build provider config")]; + settings + } + + #[test] + fn runtime_validation_accepts_valid_identity_provider() { + let settings = settings_with_identity_provider(serde_json::json!({})); + + validate_settings_for_runtime(&settings).expect("should accept valid identity provider"); + } + + #[test] + fn runtime_validation_rejects_provider_endpoint_outside_allowlist() { + let settings = settings_with_identity_provider(serde_json::json!({ + "endpoint": "https://other.example.com/v1/lookup" + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject endpoint outside allowlist"); + + assert!( + err.to_string().contains("allowed_domains"), + "should explain the allowlist rule: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_provider_source_without_partner() { + let settings = settings_with_identity_provider(serde_json::json!({ + "sources": ["unknown.example.com"] + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject source without partner"); + + assert!( + err.to_string().contains("[[ec.partners]]"), + "should explain the partner rule: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_source_claimed_by_two_providers() { + let mut settings = settings_with_identity_provider(serde_json::json!({})); + let mut second = settings.ec.identity_providers[0].clone(); + second.id = "second-provider".to_owned(); + settings.ec.identity_providers.push(second); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject double-claimed source"); + + assert!( + err.to_string() + .contains("claimed by another enabled provider"), + "should explain the claim rule: {err:?}" + ); + } + + #[test] + fn deploy_validation_rejects_empty_provider_token_reference() { + let settings = settings_with_identity_provider(serde_json::json!({ + "secrets": { "token": " " } + })); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject empty provider token reference"); + + assert!( + err.to_string() + .contains("ec.identity_providers[0].secrets.token"), + "should identify the provider token reference: {err:?}" + ); + } + + #[test] + fn app_config_keeps_provider_token_key_reference() { + let settings = settings_with_identity_provider(serde_json::json!({ + "secrets": { "token": "example_provider_token" } + })); + + let app_config = + TrustedServerAppConfig::new(settings).expect("should validate provider token key name"); + + let serialized = serde_json::to_string(&app_config).expect("should serialize app config"); + assert!( + serialized.contains("example_provider_token"), + "should keep the provider token key reference" + ); + } + #[test] fn trace_config_requires_enabled_on_both_validation_paths() { for raw in [ diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index 34a4143cf2..dd7545198b 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -205,6 +205,7 @@ mod tests { "s3-session-key" => "resolved-session-token", "partner-api-token-key" => "resolved-partner-api-token-32-bytes-ok", "partner-pull-token-key" => "resolved-partner-pull-token-32-bytes-ok", + "provider-token-key" => "resolved-provider-token", "trusted-client-ip-key" => "resolved-trusted-client-ip-secret-32-bytes", "trusted-forwarder-key" => "resolved-trusted-forwarder-secret-32-bytes", _ => key, @@ -332,6 +333,46 @@ mod tests { ); } + #[test] + fn resolves_identity_provider_token_from_the_mapped_default_store() { + let mut original = test_settings(); + original.ec.partners.push( + serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + ); + original.ec.identity_providers.push( + serde_json::from_value(serde_json::json!({ + "id": "example-provider", + "kind": "example-kind", + "endpoint": "https://ids.example.com/v1/lookup", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + "secrets": { "token": "provider-token-key" }, + })) + .expect("should build provider config"), + ); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should resolve the provider token"); + + assert_eq!( + reconstructed.ec.identity_providers[0] + .secrets + .token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-provider-token"), + "should resolve the provider token key" + ); + } + #[test] fn resolves_all_static_credentials_from_the_mapped_default_store() { let mut original = test_settings(); diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index 76aa6ad859..83bf3f33db 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -41,6 +41,7 @@ pub use types::{ ConsentContext, ConsentSource, PrivacyFlag, RawConsentSignals, TcfConsent, UsPrivacy, }; +use std::num::NonZeroU16; use std::sync::atomic::{AtomicBool, Ordering}; use web_time::{SystemTime, UNIX_EPOCH}; @@ -287,6 +288,14 @@ fn effective_tcf(ctx: &ConsentContext) -> Option<&types::TcfConsent> { }) } +/// Whether the effective TCF string grants vendor consent to `vendor_id`. +/// +/// Returns `false` when no TCF string applies. +#[must_use] +pub fn has_tcf_vendor_consent(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bool { + effective_tcf(ctx).is_some_and(|tcf| tcf.has_vendor_consent(vendor_id.get())) +} + /// Returns whether a server-side auction may be dispatched for this request. /// /// Fails closed for GDPR-relevant traffic: when an EU TCF signal is present @@ -685,6 +694,7 @@ mod tests { ConsentPipelineInput, allows_ec_creation, allows_eid_persistence, apply_expiration_check, apply_tcf_conflict_resolution, build_consent_context, build_context_from_signals, consent_allows_server_side_auction, gate_eids_by_consent, has_explicit_ec_withdrawal, + has_tcf_vendor_consent, }; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ @@ -692,6 +702,7 @@ mod tests { }; use crate::consent_config::{ConflictMode, ConsentConfig, ConsentMode}; use crate::cookies::parse_cookies_to_jar; + use std::num::NonZeroU16; fn build_request() -> Request { Request::builder() @@ -755,6 +766,32 @@ mod tests { } } + #[test] + fn tcf_vendor_consent_reads_the_effective_tcf_string() { + let mut tcf = TcfBuilder::new().with_storage(true).build(); + tcf.vendor_consents = vec![9999]; + let context = ConsentContext { + gdpr_applies: true, + tcf: Some(tcf), + ..ConsentContext::default() + }; + let granted = NonZeroU16::new(9999).expect("should be non-zero"); + let other = NonZeroU16::new(1234).expect("should be non-zero"); + + assert!( + has_tcf_vendor_consent(&context, granted), + "should grant consented vendor" + ); + assert!( + !has_tcf_vendor_consent(&context, other), + "should deny other vendor" + ); + assert!( + !has_tcf_vendor_consent(&ConsentContext::default(), granted), + "should deny when no TCF string applies" + ); + } + fn make_tcf(last_updated_ds: u64, allows_eids: bool) -> TcfConsent { TcfBuilder::new() .with_storage(true) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 7fe8da9015..28d9c98caa 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -1,3 +1,1065 @@ -//! Server-side identity lookup coordination. +//! Server-side identity lookup coordination (spec §8). +//! +//! [`ProviderCoordinator`] owns eligibility, consent, destination checks, a +//! deadline-bounded fan-out through a [`LookupTransport`], validation and +//! planning of the navigation's single row change. Providers stay pure. pub mod results; +pub mod transport; +mod validate; + +use std::collections::HashSet; +use std::num::{NonZeroU16, NonZeroU32}; +use std::sync::Arc; +use std::time::Duration; + +use error_stack::Report; + +use self::results::{LookupResults, LookupStatus, ProviderLookup, SkipReason}; +use self::transport::{ + Deadline, LookupTransport, ProviderIndex, Tagged, TransportError, TransportResponse, +}; +use self::validate::{ValidatedEntries, ValidationPolicy, validate}; +use crate::consent::ConsentContext; +use crate::consent::{allows_eid_persistence, has_tcf_vendor_consent}; +use crate::ec::EcKvSnapshot; +use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::kv_types::KvEntry; +use crate::ec::page_load::RowChange; +use crate::ec::provider::{ + ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, + ProviderResponse, SourceInput, destination_allowed, parse_sources, validate_enabled_configs, +}; +use crate::ec::rate_limiter::RateLimiter; +use crate::ec::registry::PartnerRegistry; +use crate::ec::write_plan::{WritePlanInput, plan_write}; +use crate::settings::IdentityProviderConfig; + +/// Coordinates server-side identity lookups for one navigation. +/// +/// Implementations own eligibility, consent, destination checks, the shared +/// time budget and validation. Providers never see KV, cookies, HTTP or the +/// clock. Implementations never write KV: they return the change and the +/// caller spends the [`crate::ec::page_load::PageLoadWrite`] permit. +#[allow(async_fn_in_trait)] +pub trait IdentityCoordinator { + /// Starts lookups for every eligible provider without waiting. + /// + /// Returns empty pending lookups when no provider is enabled; skipped + /// providers are recorded and reported by [`Self::collect`]. + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups; + + /// Waits for dispatched lookups until all finish or the deadline passes, + /// then parses and validates each response. + /// + /// Never fails: per-provider failures are recorded in [`LookupResults`]. + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults; + + /// Merges browser ingest and lookup results into the navigation's single + /// row change. + /// + /// Returns `None` when the row would not change, so no write is made. + fn plan_write(&self, input: &WritePlanInput<'_>) -> Option; +} + +/// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. +pub struct LookupContext<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Row snapshot read before dispatch. + pub snapshot: &'a EcKvSnapshot, + /// Request consent. + pub consent: &'a ConsentContext, + /// Client signals lent to providers. + pub client: ClientSignals<'a>, + /// Publisher domain. + pub publisher_domain: &'a str, + /// Existing EC rate limiter, used only for `rate_limit_per_hour`. + pub rate_limiter: &'a dyn RateLimiter, + /// Request time. + pub now: UnixSeconds, +} + +/// Lookups in flight for one navigation. +/// +/// Only the coordinator module can create a non-empty value. `Default` +/// (empty) and `Debug` (counts only) are implemented by hand so they do not +/// require bounds on `Pending`. +pub struct PendingLookups { + requests: Vec>, + deadline: Option, + /// Request time, for validating `exp` and resolving relative expiry. + now: UnixSeconds, + /// Providers that finished at dispatch (skipped or failed to send). + early: Vec>, +} + +impl Default for PendingLookups { + fn default() -> Self { + Self { + requests: Vec::new(), + deadline: None, + now: UnixSeconds::new(0), + early: Vec::new(), + } + } +} + +impl std::fmt::Debug for PendingLookups { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingLookups") + .field("requests", &self.requests.len()) + .field("early", &self.early.len()) + .finish_non_exhaustive() + } +} + +impl PendingLookups { + /// Whether nothing was dispatched or skipped. + #[must_use] + pub fn is_empty(&self) -> bool { + self.requests.is_empty() && self.early.is_empty() + } +} + +/// A configured provider handed to [`ProviderCoordinator::new`]. +pub struct RegisteredProvider

{ + /// The provider's configuration entry. + pub config: IdentityProviderConfig, + /// The built provider. + pub provider: P, + rate_limit_per_hour: Option, +} + +impl

RegisteredProvider

{ + /// Pairs a configuration entry with its built provider. + #[must_use] + pub fn new(config: IdentityProviderConfig, provider: P) -> Self { + Self { + config, + provider, + rate_limit_per_hour: None, + } + } + + /// Applies an hourly per-EC lookup limit (carried over from pull sync). + #[must_use] + pub fn with_rate_limit(mut self, per_hour: NonZeroU32) -> Self { + self.rate_limit_per_hour = Some(per_hour); + self + } +} + +/// A provider with the operator policy the coordinator applies to it. +struct Registered

{ + id: ProviderId, + policy: ProviderPolicy, + provider: P, +} + +/// Operator policy for one provider, validated when the coordinator is built. +struct ProviderPolicy { + sources: Box<[SourceDomain]>, + allowed_domains: Box<[Box]>, + tcf_vendor_id: Option, + rate_limit_per_hour: Option, +} + +/// The production coordinator, built per request from settings. +/// +/// Generic over the provider type: production uses the boxed default, tests +/// and single-provider builds use a concrete type. +pub struct ProviderCoordinator

> { + providers: Box<[Registered

]>, + /// Partner config: `openrtb_atype` for validation. + partners: Arc, + budget: Duration, +} + +impl ProviderCoordinator

{ + /// Validates and registers the enabled providers. + /// + /// Disabled providers are dropped: they make no requests. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when an enabled provider's + /// endpoint is not `https` with an allowlisted host, a source has no + /// partner entry, a source is outside the provider's capabilities, a + /// source is claimed by two enabled providers, or more than `u16::MAX` + /// providers are enabled. + pub fn new( + registered: Vec>, + partners: Arc, + budget: Duration, + ) -> Result> { + let enabled: Vec> = registered + .into_iter() + .filter(|registered| registered.config.enabled) + .collect(); + if u16::try_from(enabled.len()).is_err() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("too many identity providers are enabled")); + } + validate_enabled_configs( + enabled.iter().map(|registered| ®istered.config), + &partners, + )?; + + let providers = enabled + .into_iter() + .map(Self::register) + .collect::, Report>>()?; + + Ok(Self { + providers, + partners, + budget, + }) + } + + fn register( + registered: RegisteredProvider

, + ) -> Result, Report> { + let RegisteredProvider { + config, + provider, + rate_limit_per_hour, + } = registered; + let sources = parse_sources(&config)?; + let capable: HashSet<&SourceDomain> = provider + .capabilities() + .sources + .iter() + .map(|capability| &capability.source) + .collect(); + if let Some(source) = sources.iter().find(|source| !capable.contains(source)) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` is outside its kind's capabilities", + config.id + ))); + } + Ok(Registered { + id: ProviderId::new(&config.id), + policy: ProviderPolicy { + sources, + allowed_domains: config + .allowed_domains + .iter() + .map(|domain| Box::from(domain.as_str())) + .collect(), + tcf_vendor_id: config.tcf_vendor_id, + rate_limit_per_hour, + }, + provider, + }) + } + + /// Whether no provider is enabled. + #[must_use] + pub fn is_empty(&self) -> bool { + self.providers.is_empty() + } + + /// Claimed sources lacking a live UID, minus sources needing input V1 + /// never has; the flag reports whether any source was removed for input. + fn due_sources( + registered: &Registered

, + entry: Option<&KvEntry>, + now: UnixSeconds, + ) -> (Vec, bool) { + let capabilities = registered.provider.capabilities(); + let mut input_blocked = false; + let mut due = Vec::new(); + for source in ®istered.policy.sources { + let held = entry + .and_then(|entry| entry.ids.get(source.as_str())) + .is_some_and(|record| record.has_live_uid(now)); + if held { + continue; + } + let needs_input = capabilities + .sources + .iter() + .find(|capability| capability.source == *source) + .is_some_and(|capability| capability.input != SourceInput::Anonymous); + if needs_input { + input_blocked = true; + continue; + } + due.push(source.clone()); + } + (due, input_blocked) + } + + /// Whether the provider's hourly per-EC limit is reached. Limiter errors + /// fail open, matching pull sync. + fn rate_limited(registered: &Registered

, lookup: &LookupContext<'_>) -> bool { + let Some(limit) = registered.policy.rate_limit_per_hour else { + return false; + }; + let hash = lookup.ec_id.split('.').next().unwrap_or(lookup.ec_id); + let key = format!("identity:{}:{hash}", registered.id); + match lookup.rate_limiter.exceeded(&key, limit.get()) { + Ok(exceeded) => exceeded, + Err(report) => { + log::warn!( + "Identity lookup: provider={} rate limiter failed, allowing lookup: {report:?}", + registered.id + ); + false + } + } + } + + fn lookup( + &self, + provider: ProviderIndex, + status: LookupStatus, + entries: ValidatedEntries, + ) -> ProviderLookup { + ProviderLookup { + provider: self.providers[provider.get()].id.clone(), + status, + entries, + } + } + + fn interpret( + &self, + provider: ProviderIndex, + response: Result>, + now: UnixSeconds, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let response = match response { + Ok(response) => response, + Err(report) => { + let status = match report.current_context() { + TransportError::Timeout => LookupStatus::Timeout, + TransportError::Send | TransportError::BodyTooLarge => { + log::debug!( + "Identity lookup: provider={} transport failed: {report:?}", + registered.id + ); + LookupStatus::TransportFailed + } + }; + return self.lookup(provider, status, Vec::new()); + } + }; + if matches!( + response.status, + http::StatusCode::NOT_FOUND | http::StatusCode::NO_CONTENT + ) { + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + if !response.status.is_success() { + return self.lookup( + provider, + LookupStatus::HttpError(response.status), + Vec::new(), + ); + } + let parsed = registered.provider.parse_response(ProviderResponse { + status: response.status, + headers: &response.headers, + body: &response.body, + }); + let eids = match parsed { + Ok(LookupOutcome::Found(eids)) => eids, + Ok(LookupOutcome::NoMatch) => { + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + Err(report) => { + log::debug!( + "Identity lookup: provider={} response not parsed: {}", + registered.id, + report.current_context() + ); + return self.lookup(provider, LookupStatus::InvalidResponse, Vec::new()); + } + }; + let policy = ValidationPolicy { + provider: ®istered.id, + claimed: ®istered.policy.sources, + shape: registered.provider.capabilities().shape, + partners: &self.partners, + now, + }; + let (entries, dropped) = validate(eids, &policy); + let kept = entries + .iter() + .flat_map(|(_, eids)| eids) + .map(|eid| eid.uids().len()) + .sum::(); + if kept == 0 { + log::info!( + "Identity lookup: provider={} dropped every UID ({} drops)", + registered.id, + dropped.total() + ); + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + let kept = u16::try_from(kept).unwrap_or(u16::MAX); + self.lookup(provider, LookupStatus::Found { kept, dropped }, entries) + } +} + +impl IdentityCoordinator for ProviderCoordinator

{ + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups { + let mut pending = PendingLookups { + now: lookup.now, + ..PendingLookups::default() + }; + if self.providers.is_empty() { + return pending; + } + let entry = lookup.snapshot.entry_for(lookup.ec_id); + // A withdrawal tombstone or missing EID consent skips every provider. + let consented = + allows_eid_persistence(lookup.consent) && entry.is_none_or(|entry| entry.consent.ok); + let deadline = Deadline::after(self.budget); + pending.deadline = Some(deadline); + for (index, registered) in self.providers.iter().enumerate() { + let provider = ProviderIndex::new(index); + let skip = |reason| Tagged { + provider, + value: LookupStatus::Skipped(reason), + }; + let vendor_denied = registered + .policy + .tcf_vendor_id + .is_some_and(|vendor| !has_tcf_vendor_consent(lookup.consent, vendor)); + if !consented || vendor_denied { + pending.early.push(skip(SkipReason::Consent)); + continue; + } + let (due, input_blocked) = Self::due_sources(registered, entry, lookup.now); + if due.is_empty() { + pending.early.push(skip(if input_blocked { + SkipReason::Input + } else { + SkipReason::NotDue + })); + continue; + } + if Self::rate_limited(registered, lookup) { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } + let input = LookupInput { + ec_id: lookup.ec_id, + consent: lookup.consent, + client: lookup.client, + publisher_domain: lookup.publisher_domain, + due_sources: &due, + current: OwnedEntries::new(entry, ®istered.policy.sources), + hashed_email: None, + }; + let request = match registered.provider.build_request(&input) { + Ok(request) => request, + Err(report) => { + log::debug!( + "Identity lookup: provider={} request not built: {}", + registered.id, + report.current_context() + ); + pending.early.push(skip(SkipReason::Input)); + continue; + } + }; + if !destination_allowed(&request.url, ®istered.policy.allowed_domains) { + pending.early.push(skip(SkipReason::Destination)); + continue; + } + match transport.start(request, deadline).await { + Ok(value) => pending.requests.push(Tagged { provider, value }), + Err(report) => { + log::warn!( + "Identity lookup: provider={} send failed: {}", + registered.id, + report.current_context() + ); + pending.early.push(Tagged { + provider, + value: LookupStatus::TransportFailed, + }); + } + } + } + pending + } + + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults { + let PendingLookups { + requests, + deadline, + now, + early, + } = pending; + let responses = match deadline { + Some(deadline) if !requests.is_empty() => transport.gather(requests, deadline).await, + _ => Vec::new(), + }; + let mut slots: Vec> = + (0..self.providers.len()).map(|_| None).collect(); + for Tagged { provider, value } in early { + slots[provider.get()] = Some(self.lookup(provider, value, Vec::new())); + } + for Tagged { provider, value } in responses { + slots[provider.get()] = Some(self.interpret(provider, value, now)); + } + let lookups: Vec = slots.into_iter().flatten().collect(); + for lookup in &lookups { + log_outcome(lookup); + } + LookupResults::new(lookups) + } + + fn plan_write(&self, input: &WritePlanInput<'_>) -> Option { + plan_write(input) + } +} + +fn log_outcome(lookup: &ProviderLookup) { + let (kept, dropped) = match lookup.status { + LookupStatus::Found { kept, dropped } => (kept, dropped.total()), + _ => (0, 0), + }; + log::info!( + "Identity lookup: provider={} outcome={} kept={kept} dropped={dropped}", + lookup.provider, + lookup.status.metric_name() + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::consent::jurisdiction::Jurisdiction; + use crate::consent::types::TcfConsent; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::KvPartnerId; + use crate::ec::provider::{ + EidLevelFields, EidShape, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, + ProviderRequest, ProviderUid, SourceCapability, + }; + use crate::error::TrustedServerError; + use crate::settings::EcPartner; + use std::borrow::Cow; + use std::cell::RefCell; + use std::num::NonZeroUsize; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + type Reply = fn() -> Result>; + + struct EchoProvider(ProviderCapabilities); + + impl IdentityProvider for EchoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + Ok(ProviderRequest { + method: http::Method::GET, + url: url::Url::parse(&format!( + "https://ids.example.com/lookup?due={}", + lookup.due_sources.len() + )) + .expect("should parse"), + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let id = std::str::from_utf8(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + }], + }])) + } + } + + /// Answers every request with a canned result, recording the URLs sent. + struct CannedTransport { + reply: Reply, + sent: RefCell>, + } + + impl CannedTransport { + fn new(reply: Reply) -> Self { + Self { + reply, + sent: RefCell::default(), + } + } + } + + impl LookupTransport for CannedTransport { + type Pending = (); + + async fn start( + &self, + request: ProviderRequest, + _deadline: Deadline, + ) -> Result<(), Report> { + self.sent.borrow_mut().push(request.url.to_string()); + Ok(()) + } + + async fn gather( + &self, + pending: Vec>, + _deadline: Deadline, + ) -> Vec>>> { + pending + .into_iter() + .map(|tagged| Tagged { + provider: tagged.provider, + value: (self.reply)(), + }) + .collect() + } + } + + struct Limiter(bool); + + impl RateLimiter for Limiter { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Ok(self.0) + } + } + + fn reply(status: http::StatusCode, body: &'static [u8]) -> TransportResponse { + TransportResponse { + status, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from_static(body), + } + } + + fn ok_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"p-1")) + } + + fn timeout_reply() -> Result> { + Err(Report::new(TransportError::Timeout)) + } + + fn not_found_reply() -> Result> { + Ok(reply(http::StatusCode::NOT_FOUND, b"")) + } + + fn server_error_reply() -> Result> { + Ok(reply(http::StatusCode::BAD_GATEWAY, b"")) + } + + fn invalid_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"\xff")) + } + + fn partners() -> Arc { + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true + })) + .expect("should build partner"); + Arc::new(PartnerRegistry::from_config(&[partner]).expect("should build registry")) + } + + fn provider_config(extra: &str) -> IdentityProviderConfig { + toml::from_str(&format!( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]\n{extra}" + )) + .expect("should parse config") + } + + fn echo_provider(input: SourceInput) -> EchoProvider { + EchoProvider(ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse"), + input, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + }) + } + + fn coordinator_with( + registered: RegisteredProvider, + ) -> ProviderCoordinator { + ProviderCoordinator::new(vec![registered], partners(), Duration::from_millis(200)) + .expect("should build coordinator") + } + + fn coordinator(extra: &str) -> ProviderCoordinator { + coordinator_with(RegisteredProvider::new( + provider_config(extra), + echo_provider(SourceInput::Anonymous), + )) + } + + fn granting() -> ConsentContext { + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + } + } + + fn missing() -> EcKvSnapshot { + EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + } + } + + fn run_with( + coordinator: &ProviderCoordinator, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + limiter: &dyn RateLimiter, + ) -> LookupResults { + let context = LookupContext { + ec_id: EC_ID, + snapshot, + consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + rate_limiter: limiter, + now: UnixSeconds::new(1000), + }; + futures::executor::block_on(async { + let pending = coordinator.dispatch(transport, &context).await; + coordinator.collect(transport, pending).await + }) + } + + fn run( + coordinator: &ProviderCoordinator, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + ) -> LookupResults { + run_with(coordinator, transport, snapshot, consent, &Limiter(false)) + } + + fn only_status(results: &LookupResults) -> LookupStatus { + results + .iter() + .next() + .expect("should report the provider") + .status + } + + fn seeded_snapshot(entry: &KvEntry) -> EcKvSnapshot { + let graph = KvIdentityGraph::in_memory("coord"); + graph.create(EC_ID, entry).expect("should seed"); + graph.load_snapshot(EC_ID) + } + + #[test] + fn due_source_is_looked_up_and_validated() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep one UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0] + .owner() + .map(ProviderId::as_str), + Some("example-provider"), + "should record the owner" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + #[test] + fn source_with_live_uid_is_not_due() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn tombstoned_row_skips_for_consent() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&KvEntry::tombstone(1000)), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip a tombstoned row" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn missing_tcf_vendor_consent_skips_before_building_a_request() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn granted_tcf_vendor_consent_allows_the_lookup() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + let consent = ConsentContext { + gdpr_applies: true, + tcf: Some(TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents: vec![9999], + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + }; + + let results = run(&coordinator, &transport, &missing(), &consent); + + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should look up with vendor consent" + ); + } + + #[test] + fn hashed_email_sources_are_skipped_for_input() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + echo_provider(SourceInput::HashedEmail), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Input), + "should skip sources needing hashed email" + ); + } + + #[test] + fn rate_limited_provider_is_skipped() { + let coordinator = coordinator_with( + RegisteredProvider::new(provider_config(""), echo_provider(SourceInput::Anonymous)) + .with_rate_limit(NonZeroU32::new(1).expect("should be non-zero")), + ); + let transport = CannedTransport::new(ok_reply); + + let results = run_with( + &coordinator, + &transport, + &missing(), + &granting(), + &Limiter(true), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::RateLimited), + "should skip a rate-limited provider" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn built_url_outside_allowlist_is_rejected() { + let coordinator = coordinator(""); + // The echo provider builds its URL on the configured host; narrow the + // policy afterwards so the built URL falls outside the allowlist. + let coordinator = ProviderCoordinator { + providers: coordinator + .providers + .into_vec() + .into_iter() + .map(|mut registered| { + registered.policy.allowed_domains = Box::new([Box::from("other.example.com")]); + registered + }) + .collect(), + ..coordinator + }; + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Destination), + "should reject the destination" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn transport_and_http_outcomes_map_to_statuses() { + let cases: [(Reply, LookupStatus); 4] = [ + (timeout_reply, LookupStatus::Timeout), + (not_found_reply, LookupStatus::NoMatch), + ( + server_error_reply, + LookupStatus::HttpError(http::StatusCode::BAD_GATEWAY), + ), + (invalid_reply, LookupStatus::InvalidResponse), + ]; + for (reply, expected) in cases { + let coordinator = coordinator(""); + let transport = CannedTransport::new(reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report"); + assert_eq!(lookup.status, expected, "should map the outcome"); + assert!(lookup.entries.is_empty(), "should carry no entries"); + } + } + + #[test] + fn disabled_provider_is_not_registered() { + let coordinator = coordinator("enabled = false"); + + assert!(coordinator.is_empty(), "should drop disabled providers"); + } + + #[test] + fn endpoint_outside_allowlist_is_rejected_at_build() { + let mut config = provider_config(""); + config.endpoint = "https://other.example.com/".to_owned(); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new( + config, + echo_provider(SourceInput::Anonymous), + )], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject endpoint"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn source_outside_capabilities_is_rejected_at_build() { + let provider = EchoProvider(ProviderCapabilities { + sources: Box::new([]), + ..echo_provider(SourceInput::Anonymous).0 + }); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new(provider_config(""), provider)], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject source outside capabilities"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/transport.rs b/crates/trusted-server-core/src/ec/coordinator/transport.rs new file mode 100644 index 0000000000..e01f2c6a13 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/transport.rs @@ -0,0 +1,459 @@ +//! Sends provider lookups and gathers responses before a shared deadline. + +use std::cell::Cell; +use std::time::Duration; + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; + +use crate::ec::provider::ProviderRequest; +use crate::platform::{ + PlatformBackendSpec, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, + RuntimeServices, +}; + +/// Largest provider response body accepted. +pub const MAX_PROVIDER_RESPONSE_BYTES: usize = 64 * 1024; + +/// Smallest transport timeout registered for a lookup backend, in +/// milliseconds; the shared deadline still bounds collection. +const MIN_TRANSPORT_TIMEOUT_MS: u32 = 50; + +/// Sends provider lookups and gathers their responses before a deadline. +#[allow(async_fn_in_trait)] +pub trait LookupTransport { + /// An in-flight request. + type Pending; + + /// Starts one request without waiting for its response. + /// + /// # Errors + /// + /// Returns [`TransportError::Send`] when the request cannot be started. + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result>; + + /// Waits for in-flight requests until all finish or `deadline` passes. + /// + /// Requests still in flight at the deadline are abandoned and reported as + /// [`TransportError::Timeout`]. + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>>; +} + +/// A value labelled with the provider it belongs to. +#[derive(Debug)] +pub struct Tagged { + /// The provider's position in the coordinator. + pub provider: ProviderIndex, + /// The labelled value. + pub value: T, +} + +/// Position of a provider in the coordinator's configured list. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ProviderIndex(u16); + +impl ProviderIndex { + /// Wraps a position; the coordinator rejects more than `u16::MAX` providers. + /// + /// # Panics + /// + /// Panics when `index` does not fit in a `u16`. + pub(crate) fn new(index: usize) -> Self { + Self(u16::try_from(index).expect("should hold fewer than 65536 providers")) + } + + /// The position. + #[must_use] + pub fn get(self) -> usize { + usize::from(self.0) + } +} + +/// An owned response; providers parse it by borrowing its body. +#[derive(Debug)] +pub struct TransportResponse { + /// Response status. + pub status: http::StatusCode, + /// Response headers. + pub headers: http::HeaderMap, + /// Response body, at most [`MAX_PROVIDER_RESPONSE_BYTES`]. + pub body: bytes::Bytes, +} + +/// End of the shared lookup budget. +#[derive(Debug, Clone, Copy)] +pub struct Deadline(web_time::Instant); + +impl Deadline { + /// A deadline `budget` from now. + #[must_use] + pub fn after(budget: Duration) -> Self { + Self(web_time::Instant::now() + budget) + } + + /// The deadline instant. + #[must_use] + pub fn instant(self) -> web_time::Instant { + self.0 + } + + /// Time left before the deadline; zero once it has passed. + #[must_use] + pub fn remaining(self) -> Duration { + self.0.saturating_duration_since(web_time::Instant::now()) + } +} + +/// Why a lookup produced no response. +#[derive(Debug, derive_more::Display)] +pub enum TransportError { + /// The request could not be sent. + #[display("lookup request could not be sent")] + Send, + /// No response before the deadline. + #[display("lookup did not complete before the deadline")] + Timeout, + /// The body exceeded [`MAX_PROVIDER_RESPONSE_BYTES`]. + #[display("lookup response exceeded the body limit")] + BodyTooLarge, +} + +impl core::error::Error for TransportError {} + +/// [`LookupTransport`] over the platform HTTP client. +/// +/// Each request gets its own backend name (a per-transport sequence number is +/// the backend discriminator), so a ready response maps back to its provider +/// by backend name. Names repeat across requests and the transport timeout is +/// canonicalized, keeping the number of registered dynamic backends bounded. +pub struct PlatformLookupTransport<'s> { + services: &'s RuntimeServices, + certificate_check: bool, + next: Cell, +} + +impl<'s> PlatformLookupTransport<'s> { + /// Borrows the request's runtime services. + #[must_use] + pub fn new(services: &'s RuntimeServices, certificate_check: bool) -> Self { + Self { + services, + certificate_check, + next: Cell::new(0), + } + } + + fn transport_timeout(&self, deadline: Deadline) -> Duration { + let remaining_ms = u32::try_from(deadline.remaining().as_millis()).unwrap_or(u32::MAX); + let canonical = self + .services + .backend() + .canonicalize_transport_timeout_ms(remaining_ms, u32::MAX); + Duration::from_millis(u64::from(canonical.max(MIN_TRANSPORT_TIMEOUT_MS))) + } +} + +impl LookupTransport for PlatformLookupTransport<'_> { + type Pending = PlatformPendingRequest; + + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result> { + let ProviderRequest { + method, + url, + headers, + body, + } = request; + let host = url + .host_str() + .ok_or_else(|| Report::new(TransportError::Send))? + .to_owned(); + let timeout = self.transport_timeout(deadline); + let sequence = self.next.get(); + self.next.set(sequence.wrapping_add(1)); + let backend = self + .services + .backend() + .ensure(&PlatformBackendSpec { + scheme: url.scheme().to_owned(), + host, + port: url.port(), + host_header_override: None, + certificate_check: self.certificate_check, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(format!("ts-identity-{sequence}")), + }) + .change_context(TransportError::Send)?; + let mut builder = http::Request::builder().method(method).uri(url.as_str()); + if let Some(target) = builder.headers_mut() { + *target = headers; + } + let edge_request = builder + .body(EdgeBody::from(body.to_vec())) + .change_context(TransportError::Send)?; + self.services + .http_client() + .send_async(PlatformHttpRequest::new(edge_request, backend).with_cache_bypass()) + .await + .change_context(TransportError::Send) + } + + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>> { + // Names are captured before the first select: some adapters drop + // them from the requests that remain in flight. + let mut outstanding: Vec<(String, ProviderIndex)> = Vec::with_capacity(pending.len()); + let mut remaining = Vec::with_capacity(pending.len()); + let mut results = Vec::with_capacity(pending.len()); + for Tagged { provider, value } in pending { + match value.backend_name() { + Some(name) => { + outstanding.push((name.to_owned(), provider)); + remaining.push(value); + } + None => results.push(Tagged { + provider, + value: Err(Report::new(TransportError::Send)), + }), + } + } + while !remaining.is_empty() { + match self + .services + .http_client() + .select_until(remaining, deadline.instant()) + .await + { + Ok(Some(selected)) => { + remaining = selected.remaining; + let (name, value) = match selected.ready { + Ok(response) => { + (response.backend_name.clone(), read_response(response).await) + } + Err(report) => ( + selected.failed_backend_name, + Err(report.change_context(TransportError::Send)), + ), + }; + match name.and_then(|name| take_provider(&mut outstanding, &name)) { + Some(provider) => results.push(Tagged { provider, value }), + None => { + log::warn!("Identity lookup: dropping response with unknown backend"); + } + } + } + Ok(None) => break, + Err(report) => { + log::warn!("Identity lookup: select failed: {report:?}"); + break; + } + } + } + // Every request not yet answered timed out or was abandoned. + results.extend(outstanding.into_iter().map(|(_, provider)| Tagged { + provider, + value: Err(Report::new(TransportError::Timeout)), + })); + results + } +} + +/// Removes the first outstanding request with backend `name`. +fn take_provider( + outstanding: &mut Vec<(String, ProviderIndex)>, + name: &str, +) -> Option { + let index = outstanding + .iter() + .position(|(candidate, _)| candidate == name)?; + Some(outstanding.remove(index).1) +} + +async fn read_response( + response: PlatformResponse, +) -> Result> { + let (parts, body) = response.response.into_parts(); + let body = match body { + EdgeBody::Once(bytes) => bytes, + stream => stream + .into_bytes_bounded(MAX_PROVIDER_RESPONSE_BYTES) + .await + .change_context(TransportError::BodyTooLarge)?, + }; + if body.len() > MAX_PROVIDER_RESPONSE_BYTES { + return Err(Report::new(TransportError::BodyTooLarge)); + } + Ok(TransportResponse { + status: parts.status, + headers: parts.headers, + body, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use std::sync::Arc; + + fn request(path: &str) -> ProviderRequest { + ProviderRequest { + method: http::Method::GET, + url: url::Url::parse(&format!("https://ids.example.com/{path}")).expect("should parse"), + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + } + } + + fn gather_one( + transport: &PlatformLookupTransport<'_>, + deadline: Deadline, + ) -> Vec>>> { + futures::executor::block_on(async { + let pending = transport + .start(request("a"), deadline) + .await + .expect("should start"); + transport + .gather( + vec![Tagged { + provider: ProviderIndex::new(0), + value: pending, + }], + deadline, + ) + .await + }) + } + + #[test] + fn gather_returns_tagged_responses() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, b"{\"a\":1}".to_vec()); + client.push_response(204, Vec::new()); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services, true); + let deadline = Deadline::after(Duration::from_millis(200)); + + let results = futures::executor::block_on(async { + let first = transport + .start(request("a"), deadline) + .await + .expect("should start"); + let second = transport + .start(request("b"), deadline) + .await + .expect("should start"); + transport + .gather( + vec![ + Tagged { + provider: ProviderIndex::new(0), + value: first, + }, + Tagged { + provider: ProviderIndex::new(1), + value: second, + }, + ], + deadline, + ) + .await + }); + + assert_eq!(results.len(), 2, "should return one result per request"); + let first = results + .iter() + .find(|result| result.provider == ProviderIndex::new(0)) + .expect("should tag provider 0"); + let first = first.value.as_ref().expect("should succeed"); + assert_eq!(first.status, http::StatusCode::OK, "should map provider 0"); + assert_eq!(first.body.as_ref(), b"{\"a\":1}", "should carry the body"); + let second = results + .iter() + .find(|result| result.provider == ProviderIndex::new(1)) + .expect("should tag provider 1"); + assert_eq!( + second.value.as_ref().expect("should succeed").status, + http::StatusCode::NO_CONTENT, + "should map provider 1" + ); + assert_eq!( + client.recorded_request_uris(), + [ + "https://ids.example.com/a".to_owned(), + "https://ids.example.com/b".to_owned() + ], + "should send both requests" + ); + } + + #[test] + fn gather_reports_timeout_for_requests_past_the_deadline() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, Vec::new()); + client.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services, true); + + let results = gather_one(&transport, Deadline::after(Duration::from_millis(1))); + + let error = results[0].value.as_ref().expect_err("should time out"); + assert!( + matches!(error.current_context(), TransportError::Timeout), + "should report timeout" + ); + } + + #[test] + fn gather_rejects_oversized_bodies() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, vec![b'x'; MAX_PROVIDER_RESPONSE_BYTES + 1]); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services, true); + + let results = gather_one(&transport, Deadline::after(Duration::from_millis(200))); + + let error = results[0] + .value + .as_ref() + .expect_err("should reject the body"); + assert!( + matches!(error.current_context(), TransportError::BodyTooLarge), + "should report an oversized body" + ); + } + + #[test] + fn gather_with_no_requests_returns_nothing() { + let client = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services, true); + + let results = futures::executor::block_on( + transport.gather(Vec::new(), Deadline::after(Duration::from_millis(1))), + ); + + assert!(results.is_empty(), "should return nothing"); + assert_eq!( + client.recorded_select_count(), + 0, + "should not select an empty list" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs new file mode 100644 index 0000000000..25cd7332f1 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -0,0 +1,354 @@ +//! Turns a provider's parsed response into stored entries (spec §8.2). + +use std::borrow::Cow; + +use crate::ec::coordinator::results::DropCounts; +use crate::ec::eid_types::{ + AgentType, EidFields, EidValidationError, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, + RawJson, SourceDomain, UidValue, UnixSeconds, +}; +use crate::ec::kv_types::{KvEid, KvUid}; +use crate::ec::provider::{EidLevelFields, EidShape, ProviderEid, ProviderExpiry, ProviderUid}; +use crate::ec::registry::PartnerRegistry; + +/// Policy one response is validated against. +pub(crate) struct ValidationPolicy<'a> { + /// Owner recorded on every kept UID. + pub provider: &'a ProviderId, + /// Sources the provider claims in config. + pub claimed: &'a [SourceDomain], + /// The provider's declared shape limits. + pub shape: EidShape, + /// Partner config, for the default `atype`. + pub partners: &'a PartnerRegistry, + /// Request time. + pub now: UnixSeconds, +} + +/// Validated entries grouped by source, in response order. +pub(crate) type ValidatedEntries = Vec<(SourceDomain, Vec)>; + +/// Validates parsed entries, keeping valid ones grouped by source in +/// response order. +pub(crate) fn validate( + eids: Vec>, + policy: &ValidationPolicy<'_>, +) -> (ValidatedEntries, DropCounts) { + let mut dropped = DropCounts::default(); + let mut grouped: ValidatedEntries = Vec::new(); + let mut kept_eids = 0_usize; + + for eid in eids { + let Some(source) = claimed_source(&eid.source, policy.claimed) else { + dropped.record(EidValidationError::UnclaimedSource); + continue; + }; + if kept_eids == policy.shape.max_eids.get() { + dropped.over_limit = dropped.over_limit.saturating_add(1); + continue; + } + let fields = match eid_fields(&eid, policy.shape.eid_level_fields) { + Ok(fields) => fields, + Err(reason) => { + dropped.record(reason); + continue; + } + }; + let default_atype = policy + .partners + .find_by_source_domain(source.as_str()) + .map_or(1, |partner| partner.openrtb_atype); + let slot = match grouped.iter().position(|(existing, _)| existing == source) { + Some(index) => index, + None => { + grouped.push((source.clone(), Vec::new())); + grouped.len() - 1 + } + }; + let max_uids = policy.shape.max_uids_per_eid.get(); + let mut uids = Vec::with_capacity(eid.uids.len().min(max_uids)); + for uid in eid.uids { + if uids.len() == max_uids { + dropped.over_limit = dropped.over_limit.saturating_add(1); + continue; + } + match stored_uid(uid, default_atype, policy) { + Ok(uid) if is_duplicate(&grouped[slot].1, &uids, &uid) => { + dropped.duplicate = dropped.duplicate.saturating_add(1); + } + Ok(uid) => uids.push(uid), + Err(reason) => dropped.record(reason), + } + } + if !uids.is_empty() { + grouped[slot].1.push(KvEid::new(fields, uids)); + kept_eids += 1; + } + } + + grouped.retain(|(_, eids)| !eids.is_empty()); + (grouped, dropped) +} + +fn claimed_source<'p>(raw: &str, claimed: &'p [SourceDomain]) -> Option<&'p SourceDomain> { + let source = SourceDomain::try_from(raw).ok()?; + claimed.iter().find(|candidate| **candidate == source) +} + +fn bounded_field(value: Option<&Cow<'_, str>>) -> Result>, EidValidationError> { + match value { + Some(text) if text.len() > MAX_EID_FIELD_BYTES => Err(EidValidationError::FieldTooLong), + Some(text) => Ok(Some(Box::from(text.as_ref()))), + None => Ok(None), + } +} + +fn eid_fields( + eid: &ProviderEid<'_>, + level: EidLevelFields, +) -> Result { + if level == EidLevelFields::Ignored { + return Ok(EidFields::default()); + } + Ok(EidFields { + inserter: bounded_field(eid.inserter.as_ref())?, + matcher: bounded_field(eid.matcher.as_ref())?, + mm: eid.mm.map(MatchMethod::try_from).transpose()?, + ext: eid.ext.map(RawJson::try_from).transpose()?, + }) +} + +fn stored_uid( + uid: ProviderUid<'_>, + default_atype: i32, + policy: &ValidationPolicy<'_>, +) -> Result { + let id = UidValue::try_from(uid.id.into_owned())?; + let atype = AgentType::try_from(uid.atype.unwrap_or(default_atype))?; + let ext = uid.ext.map(RawJson::try_from).transpose()?; + let expiry = match uid.expiry { + ProviderExpiry::Unspecified => Expiry::Never, + ProviderExpiry::At(at) => Expiry::At(at), + ProviderExpiry::After(after) => Expiry::At(policy.now.saturating_add_secs(after.as_secs())), + }; + if !expiry.is_live(policy.now) { + return Err(EidValidationError::Expired); + } + Ok(KvUid::new( + id, + atype, + ext, + Some(policy.provider.clone()), + expiry, + )) +} + +fn is_duplicate(earlier: &[KvEid], current: &[KvUid], uid: &KvUid) -> bool { + earlier + .iter() + .flat_map(KvEid::uids) + .chain(current) + .any(|existing| existing.id() == uid.id()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::settings::EcPartner; + use serde_json::value::RawValue; + use std::num::NonZeroUsize; + use std::time::Duration; + + fn registry() -> PartnerRegistry { + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "openrtb_atype": 3, "bidstream_enabled": true + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn uid(id: &str) -> ProviderUid<'_> { + ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + } + } + + fn eid<'a>(source: &'a str, uids: Vec>) -> ProviderEid<'a> { + ProviderEid { + source: Cow::Borrowed(source), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids, + } + } + + fn policy<'a>( + provider: &'a ProviderId, + claimed: &'a [SourceDomain], + partners: &'a PartnerRegistry, + ) -> ValidationPolicy<'a> { + ValidationPolicy { + provider, + claimed, + shape: EidShape { + max_eids: NonZeroUsize::new(2).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(2).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + partners, + now: UnixSeconds::new(1000), + } + } + + fn claimed() -> [SourceDomain; 1] { + [SourceDomain::try_from("ids.example.com").expect("should parse")] + } + + #[test] + fn keeps_valid_uids_and_fills_owner_atype_and_expiry() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut timed = uid("u-2"); + timed.expiry = ProviderExpiry::After(Duration::from_secs(60)); + + let (entries, dropped) = validate( + vec![eid("IDS.example.com", vec![uid("u-1"), timed])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + let uids = entries[0].1[0].uids(); + assert_eq!(uids[0].atype().get(), 3, "should fill atype from partner"); + assert_eq!(uids[0].owner(), Some(&provider), "should record owner"); + assert_eq!( + uids[1].expiry(), + Expiry::At(UnixSeconds::new(1060)), + "should resolve relative expiry" + ); + } + + #[test] + fn drops_invalid_values_by_reason() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let long = "x".repeat(600); + let mut expired = uid("u-old"); + expired.expiry = ProviderExpiry::At(UnixSeconds::new(1000)); + let array_ext = RawValue::from_string("[1]".to_owned()).expect("should build raw"); + let mut bad_ext = uid("u-ext"); + bad_ext.ext = Some(&array_ext); + + let (entries, dropped) = validate( + vec![ + eid( + "ids.example.com", + vec![uid(" "), uid(&long), expired, bad_ext, uid("ok"), uid("ok")], + ), + eid("unclaimed.example.com", vec![uid("u-3")]), + ], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped.invalid_uid, 2, "should drop blank and long UIDs"); + assert_eq!(dropped.expired, 1, "should drop expired UID"); + assert_eq!( + dropped.invalid_field, 1, + "should drop UID with non-object ext" + ); + assert_eq!(dropped.duplicate, 1, "should drop duplicate UID"); + assert_eq!(dropped.unclaimed_source, 1, "should drop unclaimed source"); + assert_eq!(entries.len(), 1, "should keep the claimed source only"); + assert_eq!(entries[0].1[0].uids().len(), 1, "should keep one valid UID"); + } + + #[test] + fn enforces_shape_limits_from_the_end() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + + let (entries, dropped) = validate( + vec![ + eid("ids.example.com", vec![uid("a"), uid("b"), uid("c")]), + eid("ids.example.com", vec![uid("d")]), + eid("ids.example.com", vec![uid("e")]), + ], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!( + dropped.over_limit, 2, + "should drop third entry and third UID" + ); + let ids: Vec<&str> = entries[0] + .1 + .iter() + .flat_map(KvEid::uids) + .map(|uid| uid.id().as_str()) + .collect(); + assert_eq!( + ids, + ["a", "b", "d"], + "should keep the first entries and UIDs" + ); + } + + #[test] + fn ignored_eid_level_fields_are_stripped_not_rejected() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut with_fields = eid("ids.example.com", vec![uid("a")]); + with_fields.inserter = Some(Cow::Borrowed("inserter.example.com")); + with_fields.mm = Some(99); + let mut policy = policy(&provider, &claimed, &partners); + policy.shape.eid_level_fields = EidLevelFields::Ignored; + + let (entries, dropped) = validate(vec![with_fields], &policy); + + assert_eq!( + dropped, + DropCounts::default(), + "should not count ignored fields" + ); + assert_eq!( + entries[0].1[0].fields(), + &EidFields::default(), + "should strip EID-level fields" + ); + } + + #[test] + fn preserved_eid_level_fields_are_validated() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut bad_mm = eid("ids.example.com", vec![uid("a")]); + bad_mm.mm = Some(99); + let mut kept = eid("ids.example.com", vec![uid("b")]); + kept.inserter = Some(Cow::Borrowed("inserter.example.com")); + kept.mm = Some(3); + + let (entries, dropped) = + validate(vec![bad_mm, kept], &policy(&provider, &claimed, &partners)); + + assert_eq!(dropped.invalid_field, 1, "should drop unknown mm entry"); + assert_eq!( + entries[0].1[0].fields().inserter.as_deref(), + Some("inserter.example.com"), + "should keep valid inserter" + ); + assert_eq!( + entries[0].1[0].fields().mm, + Some(MatchMethod::Authenticated), + "should keep valid mm" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 33981b49a5..1663cd5d6b 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -32,6 +32,7 @@ //! - [`pull_sync`] — Background pull-sync dispatcher for organic routes //! - [`page_load`] — Single-use page-load write permit and row changes //! - [`write_plan`] — Plans a navigation's single row change +//! - [`provider`] — Identity provider interface and configuration //! - [`coordinator`] — Server-side identity lookup coordination mod auth; @@ -53,6 +54,7 @@ pub mod kv_types; pub mod page_load; pub mod partner; pub mod prebid_eids; +pub mod provider; pub mod pull_sync; pub(crate) mod pull_sync_marker; pub mod rate_limiter; diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs new file mode 100644 index 0000000000..d2be63fb7a --- /dev/null +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -0,0 +1,681 @@ +//! Identity provider interface (spec §7). +//! +//! Providers are pure: they build a lookup request and parse its response. +//! They never perform HTTP, KV, cookie or clock work; the coordinator in +//! [`crate::ec::coordinator`] executes requests and applies consent, +//! transport, validation, expiry and persistence policy. +//! +//! Provider implementations live in the `trusted-server-identity-providers` +//! crate, which depends on core; core never depends on it. + +use core::fmt; +use std::borrow::Cow; +use std::collections::HashSet; +use std::net::IpAddr; +use std::num::NonZeroUsize; +use std::time::Duration; + +use error_stack::{Report, ResultExt as _}; +use serde::Deserialize as _; +use serde::de::DeserializeOwned; +use serde_json::value::RawValue; +use url::Url; + +use crate::consent::ConsentContext; +use crate::ec::eid_types::{SourceDomain, UnixSeconds}; +use crate::ec::kv_types::{KvEid, KvEntry}; +use crate::ec::registry::PartnerRegistry; +use crate::settings::{IdentityProviderConfig, ProviderSecrets}; + +/// Builds lookup requests and parses lookup responses for one identity provider. +/// +/// Implementations must not perform I/O or read the clock; the coordinator +/// executes requests and applies consent, transport, validation, expiry and +/// persistence policy. +pub trait IdentityProvider { + /// Describes which sources and EID shapes this provider can return. + fn capabilities(&self) -> &ProviderCapabilities; + + /// Builds the lookup request for the given due sources. + /// + /// # Errors + /// + /// Returns [`ProviderError::Request`] when the input cannot form a request. + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result>; + + /// Parses a provider response, borrowing from its body. + /// + /// # Errors + /// + /// Returns [`ProviderError::InvalidResponse`] when the body cannot be + /// parsed at all. Individually malformed entries are returned and dropped + /// by the coordinator's validation, not reported as an error. + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report>; +} + +impl IdentityProvider for Box

{ + fn capabilities(&self) -> &ProviderCapabilities { + (**self).capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + (**self).build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + (**self).parse_response(response) + } +} + +/// What a provider can return, used by the coordinator to apply shared policy. +#[derive(Debug, Clone)] +pub struct ProviderCapabilities { + /// Sources this implementation can return and the input each needs. + pub sources: Box<[SourceCapability]>, + /// Limits and fields accepted from one response. + pub shape: EidShape, +} + +/// One source a provider can return. +#[derive(Debug, Clone)] +pub struct SourceCapability { + /// The source domain. + pub source: SourceDomain, + /// Visitor input the source needs. + pub input: SourceInput, +} + +/// Visitor input a source needs before it can be looked up. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceInput { + /// Works for any visitor. + Anonymous, + /// Needs a hashed email. Always skipped in V1. + HashedEmail, +} + +/// Limits a provider declares for one response. +#[derive(Debug, Clone, Copy)] +pub struct EidShape { + /// Maximum EID entries kept from one response. + pub max_eids: NonZeroUsize, + /// Maximum UIDs kept per EID entry. + pub max_uids_per_eid: NonZeroUsize, + /// Whether EID-level fields are kept. + pub eid_level_fields: EidLevelFields, +} + +/// Whether `inserter`, `matcher`, `mm` and EID `ext` are kept. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EidLevelFields { + /// The fields are validated and stored. + Preserved, + /// The fields are dropped without rejecting the entry. + Ignored, +} + +/// Request-scoped data the coordinator lends to a provider. +#[derive(Debug, Clone, Copy)] +pub struct LookupInput<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Request consent. + pub consent: &'a ConsentContext, + /// Client signals. + pub client: ClientSignals<'a>, + /// Publisher domain. + pub publisher_domain: &'a str, + /// Sources due for this provider, already filtered by consent and input. + pub due_sources: &'a [SourceDomain], + /// This provider's currently stored entries, borrowed from the row. + pub current: OwnedEntries<'a>, + /// Always `None` in V1. + pub hashed_email: Option<&'a HashedEmail>, +} + +/// Client signals lent to providers. +#[derive(Debug, Clone, Copy)] +pub struct ClientSignals<'a> { + /// Client IP address. + pub ip: Option, + /// Client `User-Agent`. + pub user_agent: Option<&'a str>, +} + +/// This provider's stored entries, borrowed from the row. +#[derive(Debug, Clone, Copy)] +pub struct OwnedEntries<'a> { + entry: Option<&'a KvEntry>, + sources: &'a [SourceDomain], +} + +impl<'a> OwnedEntries<'a> { + /// Views the entries of `entry` for `sources`. + #[must_use] + pub fn new(entry: Option<&'a KvEntry>, sources: &'a [SourceDomain]) -> Self { + Self { entry, sources } + } + + /// Iterates `(source, entry)` pairs in source order. + pub fn iter(&self) -> impl Iterator + 'a { + let entry = self.entry; + self.sources.iter().flat_map(move |source| { + entry + .and_then(|entry| entry.ids.get_key_value(source.as_str())) + .into_iter() + .flat_map(|(key, record)| record.eids().iter().map(move |eid| (key.as_str(), eid))) + }) + } +} + +/// A validated lowercase hex SHA-256 email digest. V1 never constructs one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HashedEmail(Box); + +impl HashedEmail { + /// Returns the digest. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// An outbound request description. The coordinator, not the provider, sends it. +/// +/// `Debug` is implemented by hand and prints only the method and host, because +/// headers and query strings may carry secrets. Providers mark secret headers +/// with `HeaderValue::set_sensitive(true)`. +pub struct ProviderRequest { + /// HTTP method. + pub method: http::Method, + /// Request URL; must be `https` with an allowlisted host. + pub url: Url, + /// Request headers. + pub headers: http::HeaderMap, + /// Request body. + pub body: bytes::Bytes, +} + +impl fmt::Debug for ProviderRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderRequest") + .field("method", &self.method) + .field("host", &self.url.host_str()) + .finish_non_exhaustive() + } +} + +/// A provider response, borrowed from the transport. +#[derive(Debug, Clone, Copy)] +pub struct ProviderResponse<'body> { + /// Response status. + pub status: http::StatusCode, + /// Response headers. + pub headers: &'body http::HeaderMap, + /// Response body. + pub body: &'body [u8], +} + +/// A parsed provider response. +#[derive(Debug)] +pub enum LookupOutcome<'body> { + /// Entries were returned. + Found(Vec>), + /// The provider knows no ID for this visitor. + NoMatch, +} + +/// One entry as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderEid<'body> { + /// EID source domain. + pub source: Cow<'body, str>, + /// `OpenRTB` `inserter`. + pub inserter: Option>, + /// `OpenRTB` `matcher`. + pub matcher: Option>, + /// `OpenRTB` `mm`. + pub mm: Option, + /// EID-level `ext`. + pub ext: Option<&'body RawValue>, + /// UIDs of the entry. + pub uids: Vec>, +} + +/// One UID as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderUid<'body> { + /// UID value. + pub id: Cow<'body, str>, + /// Agent type; filled from partner config when missing. + pub atype: Option, + /// UID-level `ext`. + pub ext: Option<&'body RawValue>, + /// Expiry as the provider states it. + pub expiry: ProviderExpiry, +} + +/// Expiry as a provider states it. The coordinator turns it into +/// [`crate::ec::eid_types::Expiry`] using its own clock, so providers never +/// read time. +#[derive(Debug, Clone, Copy)] +pub enum ProviderExpiry { + /// No expiry was stated. + Unspecified, + /// Expires at this time. + At(UnixSeconds), + /// Expires this long after the request. + After(Duration), +} + +/// Why a provider could not build a request or parse a response. +#[derive(Debug, derive_more::Display)] +pub enum ProviderError { + /// The input cannot form a request. + #[display("provider request could not be built")] + Request, + /// The body cannot be parsed at all. + #[display("provider response could not be parsed")] + InvalidResponse, +} + +impl core::error::Error for ProviderError {} + +/// Typed constructor for one provider implementation. +pub trait ProviderKind { + /// Value of `kind` in `[[ec.identity_providers]]`. + const KIND: &'static str; + /// The provider-specific `settings` table. + type Settings: DeserializeOwned; + /// The constructed provider. + type Provider: IdentityProvider + 'static; + + /// Builds the provider from shared and provider-specific configuration. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when the settings are unusable. + fn build( + common: &ProviderCommon<'_>, + settings: Self::Settings, + ) -> Result>; +} + +/// Shared configuration every provider receives at build time. +#[derive(Debug, Clone, Copy)] +pub struct ProviderCommon<'a> { + /// Configured lookup endpoint. + pub endpoint: &'a Url, + /// Configured sources. + pub sources: &'a [SourceDomain], + /// Resolved secrets. + pub secrets: &'a ProviderSecrets, +} + +/// Why a provider configuration was rejected. +#[derive(Debug, derive_more::Display)] +pub enum ProviderConfigError { + /// No implementation is registered for the configured `kind`. + #[display("provider kind is not registered")] + UnknownKind, + /// The configuration is unusable. + #[display("provider configuration is invalid")] + Invalid, +} + +impl core::error::Error for ProviderConfigError {} + +/// Parses `config.settings` for `K` and builds a boxed provider. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] when the settings do not +/// deserialize into `K::Settings`, a source or the endpoint is invalid, or +/// [`ProviderKind::build`] rejects them. +pub fn build_kind( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report> { + let settings = K::Settings::deserialize(serde_json::Value::Object(config.settings.clone())) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("settings for provider `{}` are invalid", config.id))?; + let endpoint = Url::parse(&config.endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + let sources = parse_sources(config)?; + let common = ProviderCommon { + endpoint: &endpoint, + sources: &sources, + secrets, + }; + let provider = K::build(&common, settings)?; + Ok(Box::new(provider)) +} + +/// Parses and normalizes a provider's configured sources. +pub(crate) fn parse_sources( + config: &IdentityProviderConfig, +) -> Result, Report> { + config + .sources + .iter() + .map(|source| SourceDomain::try_from(source.as_str())) + .collect::, _>>() + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("provider `{}` has an invalid source", config.id)) +} + +/// Whether `url` is `https` and its host is one of `allowed_domains`, +/// compared ASCII case-insensitively. +#[must_use] +pub(crate) fn destination_allowed>(url: &Url, allowed_domains: &[S]) -> bool { + url.scheme() == "https" + && url.host_str().is_some_and(|host| { + allowed_domains + .iter() + .any(|allowed| allowed.as_ref().eq_ignore_ascii_case(host)) + }) +} + +/// Checks the configuration rules core owns for every enabled provider +/// (spec §12.1): the endpoint is an `https` URL whose host is in +/// `allowed_domains`, every source has an `[[ec.partners]]` entry, and no +/// source is claimed by two enabled providers. +/// +/// Unknown kinds and capability checks need the provider crate and are +/// enforced when the coordinator is built. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] naming the first offending +/// provider. +pub fn validate_enabled_configs<'a>( + configs: impl IntoIterator, + partners: &PartnerRegistry, +) -> Result<(), Report> { + let mut claimed: HashSet = HashSet::new(); + for config in configs.into_iter().filter(|config| config.enabled) { + let endpoint = Url::parse(&config.endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + if !destination_allowed(&endpoint, &config.allowed_domains) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "endpoint for provider `{}` must be https with a host in allowed_domains", + config.id + ))); + } + for source in parse_sources(config)? { + if partners.find_by_source_domain(source.as_str()).is_none() { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` has no [[ec.partners]] entry", + config.id + ))); + } + if !claimed.insert(source.clone()) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` is claimed by another enabled provider", + config.id + ))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + use serde_json::json; + + struct Fixed(ProviderCapabilities); + + impl IdentityProvider for Fixed { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request)) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Ok(LookupOutcome::NoMatch) + } + } + + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct FixedSettings { + max_eids: usize, + } + + struct FixedKind; + + impl ProviderKind for FixedKind { + const KIND: &'static str = "fixed"; + type Settings = FixedSettings; + type Provider = Fixed; + + fn build( + common: &ProviderCommon<'_>, + settings: FixedSettings, + ) -> Result> { + let max_eids = NonZeroUsize::new(settings.max_eids) + .ok_or_else(|| Report::new(ProviderConfigError::Invalid))?; + Ok(Fixed(ProviderCapabilities { + sources: common + .sources + .iter() + .map(|source| SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }) + .collect(), + shape: EidShape { + max_eids, + max_uids_per_eid: max_eids, + eid_level_fields: EidLevelFields::Preserved, + }, + })) + } + } + + fn config(settings: &serde_json::Value) -> IdentityProviderConfig { + serde_json::from_value(json!({ + "id": "example-provider", + "kind": FixedKind::KIND, + "endpoint": "https://ids.example.com/v1", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + "settings": settings, + })) + .expect("should parse provider config") + } + + fn partners() -> PartnerRegistry { + let partner = serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn assert_invalid(error: &Report) { + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn build_kind_parses_typed_settings_and_boxes_provider() { + let provider = build_kind::( + &config(&json!({ "max_eids": 2 })), + &ProviderSecrets::default(), + ) + .expect("should build"); + + assert_eq!( + provider.capabilities().sources.len(), + 1, + "should claim configured source" + ); + assert_eq!( + provider.capabilities().shape.max_eids.get(), + 2, + "should apply typed settings" + ); + } + + #[test] + fn build_kind_rejects_unknown_settings() { + let error = build_kind::( + &config(&json!({ "unexpected": true })), + &ProviderSecrets::default(), + ) + .err() + .expect("should reject"); + + assert_invalid(&error); + } + + #[test] + fn config_parses_from_toml_with_a_settings_table() { + let parsed: IdentityProviderConfig = toml::from_str( + "id = \"p\"\nkind = \"fixed\"\nendpoint = \"https://ids.example.com/\"\n[settings]\nmax_eids = 3", + ) + .expect("should parse TOML config"); + + assert_eq!(parsed.kind(), "fixed", "should keep kind"); + assert!(parsed.enabled, "should default to enabled"); + assert_eq!( + parsed.settings.get("max_eids"), + Some(&json!(3)), + "should keep provider settings" + ); + } + + #[test] + fn request_debug_hides_query_and_headers() { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::AUTHORIZATION, + http::HeaderValue::from_static("Bearer example-secret"), + ); + let request = ProviderRequest { + method: http::Method::GET, + url: Url::parse("https://ids.example.com/v1?token=example-secret") + .expect("should parse"), + headers, + body: bytes::Bytes::new(), + }; + + let printed = format!("{request:?}"); + + assert!( + !printed.contains("example-secret"), + "should not print secrets" + ); + assert!(printed.contains("ids.example.com"), "should print host"); + } + + #[test] + fn validate_enabled_configs_accepts_valid_config() { + validate_enabled_configs([&config(&json!({}))], &partners()) + .expect("should accept valid config"); + } + + #[test] + fn validate_enabled_configs_rejects_endpoint_outside_allowlist() { + let mut outside = config(&json!({})); + outside.endpoint = "https://other.example.com/".to_owned(); + let mut plain_http = config(&json!({})); + plain_http.endpoint = "http://ids.example.com/".to_owned(); + + for rejected in [outside, plain_http] { + let error = validate_enabled_configs([&rejected], &partners()) + .expect_err("should reject endpoint"); + assert_invalid(&error); + } + } + + #[test] + fn validate_enabled_configs_rejects_source_without_partner() { + let mut unknown = config(&json!({})); + unknown.sources = vec!["unknown.example.com".to_owned()]; + + let error = validate_enabled_configs([&unknown], &partners()) + .expect_err("should reject source without partner"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_rejects_source_claimed_twice() { + let first = config(&json!({})); + let mut second = config(&json!({})); + second.id = "second-provider".to_owned(); + + let error = validate_enabled_configs([&first, &second], &partners()) + .expect_err("should reject double claim"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_ignores_disabled_providers() { + let first = config(&json!({})); + let mut disabled = config(&json!({})); + disabled.enabled = false; + disabled.endpoint = "https://other.example.com/".to_owned(); + + validate_enabled_configs([&first, &disabled], &partners()) + .expect("should ignore disabled provider"); + } + + #[test] + fn owned_entries_yields_only_configured_sources() { + let entry: KvEntry = serde_json::from_value(json!({ + "v": 1, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "ids.example.com": { "uid": "u-1", "eids": [{ "uids": [{ "id": "u-1", "atype": 3 }] }] }, + "other.example.com": { "uid": "u-2", "eids": [{ "uids": [{ "id": "u-2", "atype": 3 }] }] } + } + })) + .expect("should build entry"); + let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + + let owned: Vec<&str> = OwnedEntries::new(Some(&entry), &sources) + .iter() + .map(|(source, _)| source) + .collect(); + + assert_eq!( + owned, + ["ids.example.com"], + "should yield owned sources only" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs index 7b3c853074..7cc86898b2 100644 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ b/crates/trusted-server-core/src/ec/pull_sync_marker.rs @@ -357,8 +357,8 @@ mod tests { fn settings_and_registry(domains: &[&str]) -> (Settings, PartnerRegistry) { let mut settings = create_test_settings(); settings.ec.partners = domains.iter().map(|domain| pull_partner(domain)).collect(); - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); + let registry = + PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); (settings, registry) } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index b138ecbe16..ed0d2fbdb7 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -10,7 +10,7 @@ use error_stack::{Report, ResultExt as _}; use crate::error::TrustedServerError; use crate::redacted::Redacted; -use crate::settings::EcPartner; +use crate::settings::{Ec, EcPartner}; use super::partner::{hash_api_key, normalize_partner_source_domain}; @@ -208,6 +208,25 @@ impl PartnerRegistry { }) } + /// Builds the registry from `[ec]` settings and marks every source claimed + /// by an enabled identity provider as provider-owned. + /// + /// # Errors + /// + /// Returns the same errors as [`Self::from_config`]. + pub fn from_ec(ec: &Ec) -> Result> { + let mut registry = Self::from_config(&ec.partners)?; + for source in ec + .identity_providers + .iter() + .filter(|provider| provider.enabled) + .flat_map(|provider| &provider.sources) + { + registry.mark_provider_owned(source); + } + Ok(registry) + } + /// Returns an empty registry (no partners configured). #[must_use] pub fn empty() -> Self { @@ -462,6 +481,36 @@ mod tests { } } + #[test] + fn from_ec_marks_enabled_provider_sources_only() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![ + make_partner("ids.example.com", &valid_api_token("ids-partner")), + make_partner("other.example.com", &valid_api_token("other-partner")), + ]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"a\"\nkind = \"k\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + toml::from_str( + "id = \"b\"\nkind = \"k\"\nenabled = false\nendpoint = \"https://ids.example.com/\"\nsources = [\"other.example.com\"]", + ) + .expect("should parse disabled provider"), + ]; + + let registry = PartnerRegistry::from_ec(&ec).expect("should build registry"); + + assert!( + registry.is_provider_owned("ids.example.com"), + "should mark enabled provider source" + ); + assert!( + !registry.is_provider_owned("other.example.com"), + "should not mark disabled provider source" + ); + } + #[test] fn empty_config_builds_empty_registry() { let registry = PartnerRegistry::from_config(&[]).expect("should build empty registry"); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b3e2e6e20d..b70fad1355 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3454,9 +3454,15 @@ struct EcSnapshotPreloadInput { auction_needs_row: bool, eid_cookie_may_need_persistence: bool, privacy_needs_row: bool, + snapshot_already_read: bool, } fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { + // An earlier step of this request (identity-provider dispatch) already + // read the row for an existing EC, so a second read adds nothing. + if input.snapshot_already_read { + return false; + } let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; let marker_can_skip = input.marker_valid && !input.auction_needs_row @@ -5307,6 +5313,10 @@ pub async fn handle_publisher_request( .is_some_and(|registry| !registry.is_empty()), eid_cookie_may_need_persistence, privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), + // A freshly generated EC keeps its refresh read: the Add-confirmed + // snapshot may still upgrade to the stored row. + snapshot_already_read: !ec_context.ec_generated() + && !matches!(ec_context.kv_snapshot(), crate::ec::EcKvSnapshot::NotRead), }); let mut pending_origin = None; // A shared-template hit skips the origin entirely, and an in-flight request @@ -8389,6 +8399,7 @@ mod tests { auction_needs_row: false, eid_cookie_may_need_persistence: false, privacy_needs_row: false, + snapshot_already_read: false, }; assert!(should_preload_ec_snapshot(&baseline)); assert!(!should_preload_ec_snapshot(&EcSnapshotPreloadInput { @@ -8407,6 +8418,13 @@ mod tests { has_kv: false, ..baseline })); + assert!( + !should_preload_ec_snapshot(&EcSnapshotPreloadInput { + snapshot_already_read: true, + ..baseline + }), + "should skip the preload when the snapshot was already read" + ); } #[test] @@ -8420,6 +8438,7 @@ mod tests { auction_needs_row: false, eid_cookie_may_need_persistence: false, privacy_needs_row: false, + snapshot_already_read: false, }; assert!(!should_preload_ec_snapshot(&marker_only)); assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { @@ -8844,8 +8863,8 @@ mod tests { create_test_settings() }; settings.ec.partners = vec![marker_partner("ssp.example.com")]; - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); + let registry = + PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); let http = Arc::new(StubHttpClient::new()); http.set_concurrent_fanout(true); http.push_response(200, b"ok".to_vec()); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 754acfc262..af233cee06 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; +use std::num::NonZeroU16; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; @@ -579,6 +580,66 @@ impl EcPartner { } } +/// One `[[ec.identity_providers]]` entry. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct IdentityProviderConfig { + /// Unique provider ID; recorded as the owner of the UIDs it writes. + pub id: String, + /// Implementation kind, matched by the provider crate's `build`. + kind: String, + /// Whether lookups run. A disabled provider owns no sources. + #[serde( + default = "IdentityProviderConfig::default_enabled", + deserialize_with = "from_value_or_str" + )] + pub enabled: bool, + /// Lookup endpoint; must be `https` with a host in `allowed_domains`. + pub endpoint: String, + /// Exact hosts the coordinator may call for this provider. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub allowed_domains: Vec, + /// Source domains this provider owns; each needs an `[[ec.partners]]` entry. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub sources: Vec, + /// TCF vendor ID that must have vendor consent, when set. + #[serde(default)] + pub tcf_vendor_id: Option, + /// Secret Store key references, resolved at startup. + #[serde(default)] + pub secrets: ProviderSecrets, + /// Provider-specific settings, parsed by the provider kind. + /// + /// Held as a JSON object because settings travel through the JSON app + /// config pipeline. + #[serde(default)] + pub settings: serde_json::Map, +} + +impl IdentityProviderConfig { + const fn default_enabled() -> bool { + true + } + + /// The configured implementation kind. + #[must_use] + pub fn kind(&self) -> &str { + &self.kind + } +} + +/// Secret values for one identity provider. +/// +/// `EdgeZero` secret paths address fixed field names, so V1 supports one +/// optional bearer token per provider. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProviderSecrets { + /// Secret Store key whose value is the provider's token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token: Option>, +} + /// Edge Cookie (EC) configuration. /// /// Mapped from the `[ec]` TOML section. Controls EC identity generation, @@ -616,6 +677,18 @@ pub struct Ec { #[serde(default, deserialize_with = "vec_from_seq_or_map")] #[validate(nested)] pub partners: Vec, + + /// Shared time budget, in milliseconds, for all provider lookups on one + /// navigation. + #[serde( + default = "Ec::default_identity_budget_ms", + deserialize_with = "from_value_or_str" + )] + pub identity_budget_ms: u64, + + /// Server-side identity providers. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub identity_providers: Vec, } impl Ec { @@ -646,6 +719,12 @@ impl Ec { 3600 } + /// Default shared identity lookup budget (200 ms). + #[must_use] + pub const fn default_identity_budget_ms() -> u64 { + 200 + } + /// Returns `true` if `passphrase` matches a known placeholder value /// (case-insensitive). #[must_use] @@ -3510,6 +3589,19 @@ impl Settings { )); } } + for provider in &self.ec.identity_providers { + if provider + .secrets + .token + .as_ref() + .is_some_and(|token| EcPartner::is_placeholder_api_token(token.expose())) + { + insecure_fields.push(format!( + "ec.identity_providers[{}].secrets.token", + provider.id + )); + } + } for handler in &self.handlers { if Handler::is_placeholder_password(handler.password.expose()) { insecure_fields.push(format!("handlers[{}].password", handler.path)); @@ -6591,6 +6683,31 @@ source_domain = "partner.example.com" .expect("should accept a realistic partner pull token"); } + #[test] + fn reject_placeholder_secrets_includes_identity_provider_tokens() { + let mut settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); + settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); + let mut provider: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse provider config"); + provider.secrets.token = Some(Redacted::new( + "partner-api-token-32-bytes-minimum".to_owned(), + )); + settings.ec.identity_providers = vec![provider]; + + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject placeholder provider token"); + + assert!( + format!("{err:?}").contains("ec.identity_providers[example-provider].secrets.token"), + "should mention the provider token field" + ); + } + #[test] fn reject_placeholder_secrets_reports_both_partner_tokens() { let mut settings = diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml new file mode 100644 index 0000000000..1ff50dedf8 --- /dev/null +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "trusted-server-identity-providers" +description = "Identity provider implementations for Trusted Server." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +error-stack = { workspace = true } +trusted-server-core = { workspace = true } + +[dev-dependencies] +toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs new file mode 100644 index 0000000000..2b1154104a --- /dev/null +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -0,0 +1,146 @@ +//! Identity provider implementations for Trusted Server. +//! +//! Each provider kind is a module implementing +//! [`trusted_server_core::ec::provider::ProviderKind`]. [`build`] maps a +//! configured `kind` to its implementation; [`build_coordinator`] builds the +//! coordinator for every enabled provider. +//! +//! Dependencies point one way: this crate depends on core, and core never +//! depends on it. + +use std::sync::Arc; +use std::time::Duration; + +use error_stack::Report; +use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; +use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; +use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets, Settings}; + +/// Builds the provider for a configured `kind`. +/// +/// No kinds are registered yet, so every kind is rejected. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::UnknownKind`] for an unregistered kind, or +/// any error from [`trusted_server_core::ec::provider::build_kind`]. +pub fn build( + config: &IdentityProviderConfig, + _secrets: &ProviderSecrets, +) -> Result, Report> { + let kind = config.kind(); + Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))) +} + +/// Builds the coordinator for every enabled provider in `settings`. +/// +/// Returns `None` when no provider is enabled. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] when a provider cannot be built or fails +/// the coordinator's configuration checks. +pub fn build_coordinator( + settings: &Settings, +) -> Result, Report> { + let enabled: Vec<&IdentityProviderConfig> = settings + .ec + .identity_providers + .iter() + .filter(|config| config.enabled) + .collect(); + if enabled.is_empty() { + return Ok(None); + } + let registered = enabled + .into_iter() + .map(|config| { + build(config, &config.secrets) + .map(|provider| RegisteredProvider::new(config.clone(), provider)) + }) + .collect::, Report>>()?; + let partners = PartnerRegistry::from_ec(&settings.ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + let budget = Duration::from_millis(settings.ec.identity_budget_ms); + ProviderCoordinator::new(registered, Arc::new(partners), budget).map(Some) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_settings() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "example-admin-password" + + [publisher] + domain = "www.example.com" + cookie_domain = ".example.com" + origin_url = "https://origin.example.com" + proxy_secret = "example-proxy-secret" + + [ec] + passphrase = "example-passphrase-32-bytes-minimum" + "#, + ) + .expect("should parse test settings") + } + + #[test] + fn unknown_kind_is_rejected() { + let config: IdentityProviderConfig = toml::from_str( + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nallowed_domains = [\"ids.example.com\"]", + ) + .expect("should parse"); + + let error = build(&config, &ProviderSecrets::default()) + .err() + .expect("should reject unknown kind"); + + assert!( + matches!(error.current_context(), ProviderConfigError::UnknownKind), + "should report unknown kind" + ); + } + + #[test] + fn no_enabled_provider_builds_no_coordinator() { + let mut settings = create_test_settings(); + settings.ec.identity_providers = vec![ + toml::from_str( + "id = \"p\"\nkind = \"missing\"\nenabled = false\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse"), + ]; + + let coordinator = build_coordinator(&settings).expect("should skip disabled providers"); + + assert!(coordinator.is_none(), "should build no coordinator"); + } + + #[test] + fn enabled_unknown_kind_fails_the_build() { + let mut settings = create_test_settings(); + settings.ec.identity_providers = vec![ + toml::from_str( + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse"), + ]; + + let error = build_coordinator(&settings) + .err() + .expect("should reject unknown kind"); + + assert!( + matches!(error.current_context(), ProviderConfigError::UnknownKind), + "should report unknown kind" + ); + } +} diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 8e87e6fa86..60f47a9be6 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -59,6 +59,7 @@ publisher, trusted-client-IP, EC, handler, Tinybird, DataDome, and S3 fields: - `ec.passphrase` - `ec.partners[*].api_token`, when inbound identify or batch sync is used - `ec.partners[*].ts_pull_token`, when pull sync is enabled +- `ec.identity_providers[*].secrets.token`, when a provider needs an API token - `handlers[*].password` - `tinybird.auction_token_secret`, when Tinybird auction telemetry is enabled - `integrations.datadome.server_side_key_secret_name`, when protection is enabled @@ -756,6 +757,8 @@ be at least 32 bytes. Keep it stable to preserve EC identifier continuity. | `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | | `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | | `partners` | Array | No | Static partner registry entries | +| `identity_budget_ms` | Integer | No | Shared time budget for identity provider lookups (default 200) | +| `identity_providers` | Array | No | Server-side identity providers (see below) | ::: tip Partner keying `source_domain` is the canonical partner key. It matches incoming OpenRTB EID `source` values and is also used as the EC KV `ids` map key. @@ -788,6 +791,52 @@ TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase TRUSTED_SERVER__EC__EC_STORE=ec_identity_store ``` +### Identity providers + +`[[ec.identity_providers]]` entries look up a visitor's IDs from a provider's +server on publisher page navigations. Lookups start before the origin fetch, +share the `identity_budget_ms` budget, and are collected before the response is +finalized. Results go into the navigation's single EC row write. A lookup that +times out or fails leaves the row unchanged. + +Lookups run only for an existing or newly issued EC, only when consent allows +EID persistence, and, when `tcf_vendor_id` is set, only with TCF vendor consent +for that vendor. + +| Field | Type | Required | Description | +| ----------------- | ------- | -------- | ----------------------------------------------------------------------- | +| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | +| `kind` | String | Yes | Provider implementation built into this release | +| `enabled` | Boolean | No | Whether the provider runs (default `true`) | +| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | +| `allowed_domains` | Array | Yes | Hosts the provider may call | +| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | +| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | +| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | +| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | + +**Example**: + +```toml +[ec] +identity_budget_ms = 200 + +[[ec.identity_providers]] +id = "example-provider" +kind = "example-kind" +endpoint = "https://ids.example.com/v1/lookup" +allowed_domains = ["ids.example.com"] +sources = ["ids.example.com"] + +[ec.identity_providers.secrets] +token = "example_provider_token" +``` + +**Validation**: configuration validation fails when an enabled provider's +`endpoint` is not HTTPS or its host is not in `allowed_domains`, when a source +has no `[[ec.partners]]` entry, or when two enabled providers claim the same +source. Deploy validation also rejects an empty `secrets.token` key name. + ### Field Details #### `passphrase` diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 99afa869dd..e0cec612cf 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -107,6 +107,29 @@ pull_sync_concurrency = 3 # batch_rate_limit = 60 # max batch-sync requests/min (default 60) # pull_sync_enabled = false # default false +# Optional server-side identity providers, looked up on publisher navigations +# before the origin fetch. Every enabled provider shares one time budget: +# identity_budget_ms = 200 # default 200 +# +# [[ec.identity_providers]] +# id = "example-provider" +# Must name a provider kind built into this release. +# kind = "example-kind" +# enabled = false +# endpoint = "https://ids.example.com/v1/lookup" +# The endpoint host must be listed here. +# allowed_domains = ["ids.example.com"] +# Each source needs an [[ec.partners]] entry and must be claimed by only one +# enabled provider. +# sources = ["ids.example.com"] +# Optional: require TCF vendor consent for this provider. +# tcf_vendor_id = 9999 +# Secret Store key name, resolved like other secret references. +# [ec.identity_providers.secrets] +# token = "example_provider_token" +# Provider-specific settings, parsed by the provider kind. +# [ec.identity_providers.settings] + # ============================================================================= # OPTIONAL — Core features (disabled/omitted by default) From b64bd1501f2e7d4413504dfb08c89ffe3135195a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 12:53:06 +0530 Subject: [PATCH 04/38] Fix provider ownership, orphan attribution and size cap issues Signed-off-by: dhruv8sh --- Cargo.lock | 1 + .../trusted-server-adapter-fastly/src/app.rs | 20 +-- .../trusted-server-adapter-fastly/src/main.rs | 12 +- .../src/auction/endpoints.rs | 56 +++++++- .../src/ec/coordinator/mod.rs | 7 +- .../src/ec/coordinator/results.rs | 21 ++- .../trusted-server-core/src/ec/page_load.rs | 37 ++++- crates/trusted-server-core/src/ec/registry.rs | 73 ++++++++-- .../trusted-server-core/src/ec/write_plan.rs | 130 ++++++++++++++---- .../Cargo.toml | 1 + .../src/lib.rs | 49 +++++-- 11 files changed, 342 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9cf35f9f0..116d45a14c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6743,6 +6743,7 @@ name = "trusted-server-identity-providers" version = "0.1.0" dependencies = [ "error-stack", + "log", "toml", "trusted-server-core", ] diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 3b454d9725..175898ae53 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -123,7 +123,6 @@ use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::provider::ClientSignals; -use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::is_navigation_request; @@ -152,7 +151,7 @@ use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; -use trusted_server_identity_providers::build_coordinator; +use trusted_server_identity_providers::{build_coordinator, partner_registry}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -679,7 +678,7 @@ async fn execute_named( handler, NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup ) { - let response = PartnerRegistry::from_ec(&state.settings.ec) + let response = partner_registry(&state.settings.ec) .and_then(|registry| match handler { NamedRouteHandler::AdminEcLookup => { // Deliberately do not use an EC request-state graph: that @@ -760,7 +759,7 @@ async fn run_named_route( cors_preflight_identify(&state.settings, &req) } else { let kv = crate::require_identity_graph(&state.settings)?; - let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; + let partner_registry = partner_registry(&state.settings.ec)?; handle_identify( &state.settings, &kv, @@ -774,7 +773,7 @@ async fn run_named_route( NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; + let partner_registry = partner_registry(&state.settings.ec)?; let registry_ref = if partner_registry.is_empty() { None } else { @@ -798,7 +797,7 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; + let partner_registry = partner_registry(&state.settings.ec)?; let registry_ref = if partner_registry.is_empty() { None } else { @@ -845,7 +844,7 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; + let partner_registry = partner_registry(&state.settings.ec)?; let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); handle_batch_sync(&kv, &partner_registry, &limiter, req) }); @@ -983,7 +982,7 @@ async fn dispatch_fallback( // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. let slots = state.settings.creative_opportunity_slots(); - match PartnerRegistry::from_ec(&state.settings.ec) { + match partner_registry(&state.settings.ec) { Ok(partner_registry) => { let auction = AuctionDispatch { integration_registry: &state.registry, @@ -2909,7 +2908,10 @@ mod tests { })); let coordinator = ProviderCoordinator::new( vec![RegisteredProvider::new(provider_config, provider)], - Arc::new(PartnerRegistry::from_ec(&settings.ec).expect("should build registry")), + Arc::new( + PartnerRegistry::from_ec_with_kinds(&settings.ec, |_| true) + .expect("should build registry"), + ), Duration::from_millis(200), ) .expect("should build coordinator"); diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index a6ac48f369..efff0d4952 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -18,6 +18,7 @@ use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::pull_sync::{ PullSyncContext, build_pull_sync_context, dispatch_pull_sync, }; @@ -30,6 +31,7 @@ use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; use trusted_server_core::trace::{TraceTerminalResponse, is_trace_path}; use trusted_server_core::version_header::{apply_git_version_header, git_version_header_value}; +use trusted_server_identity_providers::partner_registry; mod app; mod backend; @@ -659,6 +661,14 @@ fn apply_edgezero_ec_finalize( identity: Option, response: &mut HttpResponse, ) -> Result> { + // A redirect skips the write, so do not wait for lookups it would discard. + let identity = identity.filter(|_| { + let redirect = NavigationPermit::is_redirect(response.status()); + if redirect { + log::debug!("Identity lookups dropped: redirect response skips the write"); + } + !redirect + }); if let Some(IdentityLookups { coordinator, pending, @@ -669,7 +679,7 @@ fn apply_edgezero_ec_finalize( let results = futures::executor::block_on(coordinator.collect(&transport, pending)); ec_state.ec_context.set_lookup_results(results); } - let partner_registry = PartnerRegistry::from_ec(&settings.ec)?; + let partner_registry = partner_registry(&settings.ec)?; let finalize_kv_graph = if ec_state.use_finalize_kv { maybe_identity_graph(settings) } else { diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index b261add7b3..50fdd263f8 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -660,14 +660,24 @@ pub(crate) fn merge_auction_eids( if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { continue; } + // A duplicate keeps the earlier UID's fields but fills its + // missing `atype` and `ext` from the later one. let duplicate = merged - .iter() + .iter_mut() .filter(|existing| existing.source == source) - .flat_map(|existing| existing.uids.iter()) - .chain(kept.iter()) - .any(|existing: &Uid| existing.id == uid.id); - if !duplicate { - kept.push(uid); + .flat_map(|existing| existing.uids.iter_mut()) + .chain(kept.iter_mut()) + .find(|existing: &&mut Uid| existing.id == uid.id); + match duplicate { + Some(existing) => { + if existing.atype.is_none() { + existing.atype = uid.atype; + } + if existing.ext.is_none() { + existing.ext = uid.ext; + } + } + None => kept.push(uid), } } if kept.is_empty() { @@ -2258,6 +2268,40 @@ mod tests { ); } + #[test] + fn merge_auction_eids_fills_missing_atype_and_ext_from_duplicate() { + let client_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { + id: "shared_uid".to_string(), + atype: Some(1), + ext: Some(json!({ "provider": "client" })), + }], + )]); + let resolved_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { + id: "shared_uid".to_string(), + atype: None, + ext: None, + }], + )]); + + let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); + + assert_eq!(merged[0].uids.len(), 1, "should deduplicate duplicate uid"); + assert_eq!( + merged[0].uids[0].atype, + Some(1), + "should fill missing atype from the duplicate" + ); + assert_eq!( + merged[0].uids[0].ext, + Some(json!({ "provider": "client" })), + "should fill missing ext from the duplicate" + ); + } + #[test] fn auction_rejects_oversized_body() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 28d9c98caa..cd95bf667a 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -96,6 +96,8 @@ pub struct LookupContext<'a> { pub struct PendingLookups { requests: Vec>, deadline: Option, + /// EC ID the lookups were dispatched for. + ec_id: String, /// Request time, for validating `exp` and resolving relative expiry. now: UnixSeconds, /// Providers that finished at dispatch (skipped or failed to send). @@ -107,6 +109,7 @@ impl Default for PendingLookups { Self { requests: Vec::new(), deadline: None, + ec_id: String::new(), now: UnixSeconds::new(0), early: Vec::new(), } @@ -421,6 +424,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ lookup: &LookupContext<'_>, ) -> PendingLookups { let mut pending = PendingLookups { + ec_id: lookup.ec_id.to_owned(), now: lookup.now, ..PendingLookups::default() }; @@ -511,6 +515,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ let PendingLookups { requests, deadline, + ec_id, now, early, } = pending; @@ -530,7 +535,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ for lookup in &lookups { log_outcome(lookup); } - LookupResults::new(lookups) + LookupResults::new(&ec_id, lookups) } fn plan_write(&self, input: &WritePlanInput<'_>) -> Option { diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index e9bc32a2d1..53e675ad57 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -124,14 +124,29 @@ pub struct ProviderLookup { /// Validated results of one navigation's lookups. #[derive(Debug, Clone, Default)] pub struct LookupResults { + /// EC ID the lookups were made for; `None` when nothing was looked up. + ec_id: Option, lookups: Vec, } impl LookupResults { - /// Wraps per-provider results in provider-config order. + /// Wraps per-provider results, made for `ec_id`, in provider-config + /// order. #[must_use] - pub fn new(lookups: Vec) -> Self { - Self { lookups } + pub fn new(ec_id: &str, lookups: Vec) -> Self { + Self { + ec_id: Some(ec_id.to_owned()), + lookups, + } + } + + /// Whether these results were looked up for `ec_id`. + /// + /// A navigation can switch EC IDs after dispatch (orphan recovery mints a + /// new one), so results for another ID must not be attributed to it. + #[must_use] + pub fn is_for(&self, ec_id: &str) -> bool { + self.ec_id.as_deref() == Some(ec_id) } /// Iterates results in provider-config order. diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs index 212034c3f4..7453d93e7f 100644 --- a/crates/trusted-server-core/src/ec/page_load.rs +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -54,15 +54,24 @@ impl NavigationPermit { /// are the page load itself and keep the permit. #[must_use] pub fn confirm(self, status: StatusCode) -> Option { - let redirect = matches!( + (!Self::is_redirect(status)).then_some(PageLoadWrite(())) + } + + /// Whether `status` is a redirect (301, 302, 303, 307, 308) on which the + /// page load's write is skipped. + /// + /// Callers use this to drop work, such as pending identity lookups, whose + /// only purpose is that write. + #[must_use] + pub fn is_redirect(status: StatusCode) -> bool { + matches!( status, StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND | StatusCode::SEE_OTHER | StatusCode::TEMPORARY_REDIRECT | StatusCode::PERMANENT_REDIRECT - ); - (!redirect).then_some(PageLoadWrite(())) + ) } /// Builds a permit without a request, for tests. @@ -226,6 +235,28 @@ mod tests { } } + #[test] + fn is_redirect_matches_only_write_skipping_statuses() { + for status in [ + http::StatusCode::MOVED_PERMANENTLY, + http::StatusCode::FOUND, + http::StatusCode::SEE_OTHER, + http::StatusCode::TEMPORARY_REDIRECT, + http::StatusCode::PERMANENT_REDIRECT, + ] { + assert!( + NavigationPermit::is_redirect(status), + "should treat {status} as a redirect" + ); + } + for status in [http::StatusCode::OK, http::StatusCode::NOT_MODIFIED] { + assert!( + !NavigationPermit::is_redirect(status), + "should not treat {status} as a redirect" + ); + } + } + fn permit() -> PageLoadWrite { NavigationPermit::for_test() .confirm(http::StatusCode::OK) diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index ed0d2fbdb7..05bfbb92ed 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -208,21 +208,45 @@ impl PartnerRegistry { }) } - /// Builds the registry from `[ec]` settings and marks every source claimed - /// by an enabled identity provider as provider-owned. + /// Builds the registry from `[ec]` settings without provider ownership. + /// + /// For adapters with no identity-provider implementations: no source is + /// provider-owned, so browser-supplied IDs are ingested for every source. /// /// # Errors /// /// Returns the same errors as [`Self::from_config`]. pub fn from_ec(ec: &Ec) -> Result> { + Self::from_ec_with_kinds(ec, |_| false) + } + + /// Builds the registry from `[ec]` settings and marks every source claimed + /// by an enabled identity provider whose kind is registered as + /// provider-owned. + /// + /// An enabled provider with an unregistered kind never runs, so its + /// sources stay open to browser ingest and a warning is logged. + /// + /// # Errors + /// + /// Returns the same errors as [`Self::from_config`]. + pub fn from_ec_with_kinds( + ec: &Ec, + is_registered_kind: impl Fn(&str) -> bool, + ) -> Result> { let mut registry = Self::from_config(&ec.partners)?; - for source in ec - .identity_providers - .iter() - .filter(|provider| provider.enabled) - .flat_map(|provider| &provider.sources) - { - registry.mark_provider_owned(source); + for provider in ec.identity_providers.iter().filter(|p| p.enabled) { + if !is_registered_kind(provider.kind()) { + log::warn!( + "Identity provider '{}' has unregistered kind '{}'; its sources stay browser-ingested", + provider.id, + provider.kind() + ); + continue; + } + for source in &provider.sources { + registry.mark_provider_owned(source); + } } Ok(registry) } @@ -499,7 +523,8 @@ mod tests { .expect("should parse disabled provider"), ]; - let registry = PartnerRegistry::from_ec(&ec).expect("should build registry"); + let registry = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect("should build registry"); assert!( registry.is_provider_owned("ids.example.com"), @@ -511,6 +536,34 @@ mod tests { ); } + #[test] + fn from_ec_skips_providers_with_unregistered_kinds() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![make_partner( + "ids.example.com", + &valid_api_token("ids-partner"), + )]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"a\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + ]; + + let with_kinds = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect("should build registry"); + let plain = PartnerRegistry::from_ec(&ec).expect("should build registry"); + + assert!( + !with_kinds.is_provider_owned("ids.example.com"), + "should not claim sources for an unregistered kind" + ); + assert!( + !plain.is_provider_owned("ids.example.com"), + "should not claim sources without provider implementations" + ); + } + #[test] fn empty_config_builds_empty_registry() { let registry = PartnerRegistry::from_config(&[]).expect("should build empty registry"); diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 075b14b59e..1ccc721246 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -58,7 +58,17 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { let mut entry = base.clone(); apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); - let written = apply_lookups(&mut entry, input.lookups, input.now); + let written = if input.lookups.is_for(input.ec_id) { + apply_lookups(&mut entry, input.lookups, input.now) + } else { + if !input.lookups.is_empty() { + log::info!( + "EC write planner: discarding lookups made for another EC ID than '{}'", + super::log_id(input.ec_id) + ); + } + Vec::new() + }; for record in entry.ids.values_mut() { record.prune_expired(input.now); } @@ -111,23 +121,35 @@ fn serialized_len(entry: &KvEntry) -> usize { fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { while serialized_len(entry) > MAX_ENTRY_BYTES { - let Some(source) = written.iter().rev().find(|source| { + if let Some(source) = written.iter().rev().find(|source| { entry .ids .get(source.as_str()) .is_some_and(KvPartnerId::is_provider_written) - }) else { - log::warn!("EC write planner: row exceeds size cap with no provider UIDs left to drop"); + }) { + let record = entry + .ids + .get_mut(source.as_str()) + .expect("should hold a written source"); + record.drop_last_uid(); + if record.uid.is_empty() { + entry.ids.remove(source.as_str()); + } + continue; + } + // No provider UIDs from this navigation are left: drop whole records, + // largest first, so every later write still fits the cap. + let Some(largest) = entry + .ids + .iter() + .max_by_key(|(_, record)| serde_json::to_vec(record).map_or(usize::MAX, |b| b.len())) + .map(|(source, _)| source.clone()) + else { + log::warn!("EC write planner: row exceeds size cap with no partner IDs left to drop"); return; }; - let record = entry - .ids - .get_mut(source.as_str()) - .expect("should hold a written source"); - record.drop_last_uid(); - if record.uid.is_empty() { - entry.ids.remove(source.as_str()); - } + log::warn!("EC write planner: dropping '{largest}' to fit the row size cap"); + entry.ids.remove(&largest); } } @@ -184,17 +206,20 @@ mod tests { ) }) .collect(); - LookupResults::new(vec![ProviderLookup { - provider: ProviderId::new("example-provider"), - status: LookupStatus::Found { - kept: 1, - dropped: DropCounts::default(), - }, - entries: vec![( - SourceDomain::try_from(source).expect("should build source"), - vec![KvEid::new(EidFields::default(), uids)], - )], - }]) + LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), uids)], + )], + }], + ) } fn input<'a>( @@ -250,6 +275,36 @@ mod tests { ); } + #[test] + fn lookups_for_another_ec_id_are_discarded() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let lookups_for_old = LookupResults::new( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.OLD123", + found("ids.example.com", &["p-1"]).iter().cloned().collect(), + ); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &[], + &lookups_for_old, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should not attribute the old EC ID's lookups to the new row" + ); + } + #[test] fn unchanged_returning_row_plans_no_write() { let registry = registry(&[]); @@ -317,6 +372,35 @@ mod tests { ); } + #[test] + fn oversized_row_without_provider_uids_drops_records_until_it_fits() { + let registry = registry(&[]); + let mut entry = live_entry(); + for i in 0..40 { + entry.ids.insert( + format!("source{i:02}.example.com"), + KvPartnerId::single(format!("{i:03}{}", "x".repeat(500))), + ); + } + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let lookups = LookupResults::default(); + + let change = plan_write(&input(&snapshot, Some(&entry), &[], &lookups, ®istry)) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create") + }; + let size = serde_json::to_vec(&entry).expect("should serialize").len(); + assert!( + size <= MAX_ENTRY_BYTES, + "should fit the row cap, got {size}" + ); + assert!(!entry.ids.is_empty(), "should keep the records that fit"); + } + #[test] fn oversized_row_drops_provider_uids_until_it_fits() { let registry = registry(&["ids.example.com"]); diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index 1ff50dedf8..831ed20729 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] error-stack = { workspace = true } +log = { workspace = true } trusted-server-core = { workspace = true } [dev-dependencies] diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 2b1154104a..dd6c086f43 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -15,7 +15,26 @@ use error_stack::Report; use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets, Settings}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::settings::{Ec, IdentityProviderConfig, ProviderSecrets, Settings}; + +/// Whether `kind` has a registered implementation. +/// +/// No kinds are registered yet. +#[must_use] +pub fn is_registered_kind(_kind: &str) -> bool { + false +} + +/// Builds the partner registry, marking only sources claimed by enabled +/// providers whose kind is registered as provider-owned. +/// +/// # Errors +/// +/// Returns the same errors as [`PartnerRegistry::from_ec`]. +pub fn partner_registry(ec: &Ec) -> Result> { + PartnerRegistry::from_ec_with_kinds(ec, is_registered_kind) +} /// Builds the provider for a configured `kind`. /// @@ -50,6 +69,17 @@ pub fn build_coordinator( .identity_providers .iter() .filter(|config| config.enabled) + .filter(|config| { + let registered = is_registered_kind(config.kind()); + if !registered { + log::warn!( + "Identity provider '{}' skipped: kind '{}' is not registered", + config.id, + config.kind() + ); + } + registered + }) .collect(); if enabled.is_empty() { return Ok(None); @@ -61,7 +91,7 @@ pub fn build_coordinator( .map(|provider| RegisteredProvider::new(config.clone(), provider)) }) .collect::, Report>>()?; - let partners = PartnerRegistry::from_ec(&settings.ec) + let partners = partner_registry(&settings.ec) .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; let budget = Duration::from_millis(settings.ec.identity_budget_ms); ProviderCoordinator::new(registered, Arc::new(partners), budget).map(Some) @@ -125,22 +155,23 @@ mod tests { } #[test] - fn enabled_unknown_kind_fails_the_build() { + fn enabled_unknown_kind_is_skipped_and_claims_no_sources() { let mut settings = create_test_settings(); settings.ec.identity_providers = vec![ toml::from_str( - "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"", + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", ) .expect("should parse"), ]; - let error = build_coordinator(&settings) - .err() - .expect("should reject unknown kind"); + let coordinator = + build_coordinator(&settings).expect("should skip unregistered provider kinds"); + let registry = partner_registry(&settings.ec).expect("should build registry"); + assert!(coordinator.is_none(), "should build no coordinator"); assert!( - matches!(error.current_context(), ProviderConfigError::UnknownKind), - "should report unknown kind" + !registry.is_provider_owned("ids.example.com"), + "should leave unregistered provider sources to browser ingest" ); } } From 49b73b217d9c0b9aecf30cc7bf8d057d304c683f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 14:07:28 +0530 Subject: [PATCH 05/38] Skip partner token length checks in deploy-time provider validation Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/config.rs | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 85bfd5cf0f..44a592f1f3 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -33,6 +33,7 @@ use crate::integrations::{ sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; +use crate::redacted::Redacted; use crate::settings::{AssetOriginAuth, Ec, IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; @@ -279,7 +280,7 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report Ec { + let mut ec = ec.clone(); + for (index, partner) in ec.partners.iter_mut().enumerate() { + if partner.api_token.is_some() { + partner.api_token = Some(Redacted::new(format!("{index:0>64}"))); + } + if partner.ts_pull_token.is_some() { + partner.ts_pull_token = Some(Redacted::new(format!("{index:1>64}"))); + } + } + ec +} + /// Checks the identity-provider rules core owns (spec §12.1). /// /// Unknown kinds and capability checks need the provider crate and are @@ -1355,6 +1374,16 @@ gam_network_id = "99999" validate_settings_for_runtime(&settings).expect("should accept valid identity provider"); } + #[test] + fn deploy_validation_accepts_provider_with_unresolved_partner_token() { + let mut settings = settings_with_identity_provider(serde_json::json!({})); + settings.ec.partners[0].api_token = + Some(Redacted::new("example_partner_api_token".to_owned())); + + validate_settings_for_deploy(&settings) + .expect("should not length-check a partner token secret key name"); + } + #[test] fn runtime_validation_rejects_provider_endpoint_outside_allowlist() { let settings = settings_with_identity_provider(serde_json::json!({ From 9e2a2d72e0130932532496e5edf087ecc9d6c379 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 14:13:00 +0530 Subject: [PATCH 06/38] Stop warning about unregistered provider kinds where kinds are unknown Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/registry.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 05bfbb92ed..5fa2677167 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -212,12 +212,14 @@ impl PartnerRegistry { /// /// For adapters with no identity-provider implementations: no source is /// provider-owned, so browser-supplied IDs are ingested for every source. + /// Kind registration is unknown here, so no unregistered-kind warning is + /// logged; use [`Self::from_ec_with_kinds`] where kinds are known. /// /// # Errors /// /// Returns the same errors as [`Self::from_config`]. pub fn from_ec(ec: &Ec) -> Result> { - Self::from_ec_with_kinds(ec, |_| false) + Self::from_config(&ec.partners) } /// Builds the registry from `[ec]` settings and marks every source claimed From c559b8924e41c21c0c723383a837e25275e30bf1 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 17:46:35 +0530 Subject: [PATCH 07/38] Write orphan-recovery lookups to the replacement EC row Orphan recovery minted a new EC ID and then discarded the navigation's provider results because they were looked up under the orphaned ID. Lookups depend on the browser, not the EC ID, and the orphan has no row, so retarget them to the replacement as the spec requires. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/results.rs | 12 +++ crates/trusted-server-core/src/ec/finalize.rs | 73 +++++++++++++++++++ crates/trusted-server-core/src/ec/mod.rs | 6 ++ 3 files changed, 91 insertions(+) diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index 53e675ad57..7fff308ea9 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -149,6 +149,18 @@ impl LookupResults { self.ec_id.as_deref() == Some(ec_id) } + /// Retargets results made for `orphan` to `replacement`. + /// + /// Orphan recovery replaces a proven-absent EC ID with a new one for the + /// same browser. Lookups depend on the browser, not on the EC ID, and the + /// orphan has no row left, so its results belong to the replacement row. + /// Results made for any other ID are left untouched. + pub(crate) fn retarget_orphan(&mut self, orphan: &str, replacement: &str) { + if self.is_for(orphan) { + self.ec_id = Some(replacement.to_owned()); + } + } + /// Iterates results in provider-config order. pub fn iter(&self) -> impl Iterator { self.lookups.iter() diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index eba972a8a6..9e092f2433 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -325,6 +325,9 @@ fn recover_orphaned_ec( .device_signals() .map(super::device::DeviceSignals::to_kv_device); + // Spec: the recovered row is created with this navigation's provider + // results, which were looked up under the orphaned ID. + ec_context.retarget_orphan_lookups(&orphan_id, &new_id); let missing = EcKvSnapshot::Missing { ec_id: new_id.clone(), }; @@ -585,6 +588,11 @@ mod tests { use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; use crate::ec::EidSyncSource; + use crate::ec::coordinator::results::{ + DropCounts, LookupResults, LookupStatus, ProviderLookup, + }; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv_types::{KvEid, KvUid}; use crate::ec::page_load::NavigationPermit; use crate::openrtb::{Eid, Uid}; use crate::redacted::Redacted; @@ -1020,6 +1028,71 @@ mod tests { ); } + #[test] + fn finalize_recovery_writes_lookups_made_for_the_orphan() { + let settings = create_test_settings(); + let orphaned_ec = sample_ec_id("orphn2"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test_with_ip( + Some(orphaned_ec.clone()), + consent, + Some("192.0.2.10".to_owned()), + ); + ec_context.set_recovery_eligible(true); + ec_context.set_kv_snapshot(EcKvSnapshot::Missing { + ec_id: orphaned_ec.clone(), + }); + let uid = KvUid::new( + UidValue::try_from("provider-uid-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + ec_context.set_lookup_results(LookupResults::new( + &orphaned_ec, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from("provider.example.com").expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + }], + )); + let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let replacement = ec_context.ec_value().expect("should rotate orphan"); + assert_ne!(replacement, orphaned_ec, "should mint a replacement EC ID"); + let (entry, _) = graph + .get(replacement) + .expect("should read replacement") + .expect("should create replacement row"); + assert!( + entry.ids.contains_key("provider.example.com"), + "should write the orphan's provider results to the replacement row" + ); + } + #[test] fn finalize_rotates_orphaned_cookie_to_new_backed_ec() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 1663cd5d6b..2a832aebdd 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -639,6 +639,12 @@ impl EcContext { &self.lookup_results } + /// Retargets this navigation's lookups from an orphaned EC ID to its + /// replacement; see [`LookupResults::retarget_orphan`]. + pub(crate) fn retarget_orphan_lookups(&mut self, orphan: &str, replacement: &str) { + self.lookup_results.retarget_orphan(orphan, replacement); + } + /// Takes the row built by [`Self::generate_if_needed`]. pub(crate) fn take_pending_entry(&mut self) -> Option { self.pending_entry.take().map(|entry| *entry) From 4ab0a1c31e4b1a92095ff2918450a227016349e6 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 17:49:49 +0530 Subject: [PATCH 08/38] Keep responses whose headers arrive in an expired deadline snapshot Collection that starts after its deadline takes one nonblocking snapshot. A request still in the headers state moved to the body state without reading bytes, which counted as no progress, so a response that had fully arrived was abandoned unread and reported as a timeout. Count entering the body state as progress. Signed-off-by: dhruv8sh --- .../src/platform/deadline.rs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index 6a8842d180..e4333c193f 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -262,9 +262,10 @@ pub(super) fn select_until( for index in 0..pending.len() { loop { let previous = pending[index].buffered_bytes(); + let was_headers = matches!(pending[index].state, Some(PendingState::Headers(_))); let result = pending[index].poll(); - let progressed = - matches!(result, Ok(None)) && pending[index].buffered_bytes() > previous; + let progressed = matches!(result, Ok(None)) + && snapshot_progressed(was_headers, previous, &pending[index]); match result { Ok(None) if progressed => {} Ok(None) => { @@ -337,6 +338,16 @@ pub(super) fn select_until( } } +/// Whether a nonblocking poll that returned no response advanced `item`. +/// +/// Arriving headers move a request into [`PendingState::Body`] without +/// reading any bytes; that is progress, or a response that fully arrived +/// before an expired collection would be abandoned unread. +fn snapshot_progressed(was_headers: bool, previous_bytes: usize, item: &DeadlinePending) -> bool { + let entered_body = was_headers && matches!(item.state, Some(PendingState::Body { .. })); + entered_body || item.buffered_bytes() > previous_bytes +} + fn enforce_aggregate_budget(pending: &mut [DeadlinePending], buffered_bytes: &mut usize) { if *buffered_bytes <= MAX_TOTAL_BUFFERED_BODY_BYTES { return; @@ -543,6 +554,20 @@ mod tests { ); } + #[test] + fn snapshot_counts_arriving_headers_as_progress() { + let pending = DeadlinePending::from_platform(buffered_pending(Vec::new(), "example")) + .expect("should resume a buffered pending request"); + assert!( + super::snapshot_progressed(true, 0, &pending), + "should keep polling a request whose headers just arrived" + ); + assert!( + !super::snapshot_progressed(false, 0, &pending), + "should stop polling a body that read nothing" + ); + } + #[test] fn expired_deadline_preserves_an_already_ready_body() { let pending = buffered_pending(b"ready".to_vec(), "example"); From e43ea9a93bda9a6002fde8976299dbc8776afb88 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 18:03:23 +0530 Subject: [PATCH 09/38] Surface arrived bodies once in an expired deadline snapshot A response body the host already holds can report not ready until the host services async I/O once, so an expired snapshot abandoned identity responses that arrived well before the deadline and reported them as timeouts. Give each pending body one host select capped at 1 ms before abandoning it. Signed-off-by: dhruv8sh --- .../src/platform/deadline.rs | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index e4333c193f..6b074f385c 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -16,6 +16,9 @@ use super::{ // Bound payload retained across all partial bodies, as well as each response. const MAX_TOTAL_BUFFERED_BODY_BYTES: usize = MAX_PLATFORM_RESPONSE_BODY_BYTES; +/// Longest host wait an expired snapshot spends surfacing one arrived body. +const SNAPSHOT_SURFACE_TIMEOUT_MS: u32 = 1; + // Keep partially buffered responses alongside requests still awaiting headers so // an early header with a slow body cannot hold up a completed sibling response. struct DeadlinePending { @@ -215,7 +218,9 @@ fn body_is_ready(body: &BodyHandle) -> Result> { /// Dropping response bodies closes their host handles. Fastly has no pending /// request cancellation ABI: unfinished header requests are abandoned and /// never waited on past the auction deadline. If collection starts after the -/// deadline, one nonblocking snapshot preserves already available responses. +/// deadline, one snapshot preserves already available responses; it may spend +/// at most [`SNAPSHOT_SURFACE_TIMEOUT_MS`] per pending body letting the host +/// surface bytes that already arrived. /// /// Partial bodies share a 10 MiB payload budget. If a chunk exceeds that /// capacity, the largest partial response fails with an attributed HTTP client @@ -260,12 +265,22 @@ pub(super) fn select_until( // Take one nonblocking snapshot. Fully buffered results are cached; // responses that would require a wait are abandoned, never retried. for index in 0..pending.len() { + let mut surfaced = false; loop { let previous = pending[index].buffered_bytes(); let was_headers = matches!(pending[index].state, Some(PendingState::Headers(_))); let result = pending[index].poll(); let progressed = matches!(result, Ok(None)) && snapshot_progressed(was_headers, previous, &pending[index]); + if matches!(result, Ok(None)) + && !progressed + && !surfaced + && let Some(PendingState::Body { body, .. }) = &pending[index].state + { + surfaced = true; + surface_arrived_body(body); + continue; + } match result { Ok(None) if progressed => {} Ok(None) => { @@ -338,6 +353,34 @@ pub(super) fn select_until( } } +/// Lets the host surface body bytes that have already arrived. +/// +/// A body whose bytes the host holds can still report not ready until the +/// host services async I/O once, so an expired snapshot would abandon a +/// response that fully arrived before the deadline. One select capped at +/// [`SNAPSHOT_SURFACE_TIMEOUT_MS`] runs at most once per pending body; a +/// failure leaves the body not ready. +fn surface_arrived_body(body: &BodyHandle) { + let handles = [ + // SAFETY: the live owned body retains the handle while select borrows it. + unsafe { body.as_u32() }, + ]; + let mut ready_index = 0; + // SAFETY: handles is a live slice, and ready_index is writable. The + // timeout is positive: the host interprets zero as an unbounded wait. + let status = unsafe { + fastly_sys::fastly_async_io::select( + handles.as_ptr(), + handles.len(), + SNAPSHOT_SURFACE_TIMEOUT_MS, + &mut ready_index, + ) + }; + if let Err(status) = status.result() { + log::debug!("Expired snapshot: body surface select failed: {status:?}"); + } +} + /// Whether a nonblocking poll that returned no response advanced `item`. /// /// Arriving headers move a request into [`PendingState::Body`] without From dfd160ab656a035853f257a3e850363a7084ca68 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 18:30:57 +0530 Subject: [PATCH 10/38] Fall back to browser IDs for provider-owned sources the provider did not fill Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 53 +++- crates/trusted-server-core/src/ec/finalize.rs | 117 +++++++++ .../trusted-server-core/src/ec/prebid_eids.rs | 7 +- .../trusted-server-core/src/ec/write_plan.rs | 236 +++++++++++++++++- docs/guide/configuration.md | 8 + 5 files changed, 402 insertions(+), 19 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index cd95bf667a..8fa406cbcb 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -272,7 +272,8 @@ impl ProviderCoordinator

{ self.providers.is_empty() } - /// Claimed sources lacking a live UID, minus sources needing input V1 + /// Claimed sources lacking a live provider-written UID (a browser + /// fallback does not count), minus sources needing input V1 /// never has; the flag reports whether any source was removed for input. fn due_sources( registered: &Registered

, @@ -285,7 +286,7 @@ impl ProviderCoordinator

{ for source in ®istered.policy.sources { let held = entry .and_then(|entry| entry.ids.get(source.as_str())) - .is_some_and(|record| record.has_live_uid(now)); + .is_some_and(|record| record.is_provider_written() && record.has_live_uid(now)); if held { continue; } @@ -560,8 +561,9 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::TcfConsent; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; use crate::ec::kv::KvIdentityGraph; - use crate::ec::kv_types::KvPartnerId; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; use crate::ec::provider::{ EidLevelFields, EidShape, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, ProviderUid, SourceCapability, @@ -829,6 +831,49 @@ mod tests { assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); } + fn provider_record(id: &str) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + #[test] + fn source_with_browser_fallback_uid_is_still_due() { + // Arrange + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("fallback"), + ); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should look up a source holding only a browser fallback" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + #[test] fn source_with_live_uid_is_not_due() { let coordinator = coordinator(""); @@ -837,7 +882,7 @@ mod tests { entry.consent.ok = true; entry .ids - .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + .insert("ids.example.com".to_owned(), provider_record("p-1")); let results = run( &coordinator, diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 9e092f2433..1fc5c1a074 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1583,6 +1583,123 @@ mod tests { ); } + #[test] + fn finalize_withholds_provider_owned_fallback_when_purpose_four_is_denied() { + // Arrange + let settings = create_test_settings(); + let ec_id = sample_ec_id("purp4o"); + let graph = KvIdentityGraph::in_memory("test_store"); + let purpose1_only_consent = purpose_one_only_consent(); + let live = KvEntry::new( + &purpose1_only_consent, + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row this request updates"); + let mut ec_context = EcContext::new_for_test_with_cookie( + Some(ec_id.clone()), + Some(ec_id.clone()), + true, + false, + purpose1_only_consent, + ); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { + id: "ID5_should_not_persist".to_owned(), + atype: Some(1), + ext: None, + }], + )]); + let mut registry = PartnerRegistry::from_config(&[make_partner("id5-sync.com")]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("id5-sync.com"), + "should mark the provider source" + ); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + None, + &mut response, + ); + + // Assert + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert!( + !stored.ids.contains_key("id5-sync.com"), + "should keep the provider-owned fallback out of KV without Purpose 4" + ); + } + + #[test] + fn finalize_stores_browser_fallback_for_provider_owned_source() { + // Arrange + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint a new EC"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut registry = PartnerRegistry::from_config(&[make_partner("sharedid.org")]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("sharedid.org"), + "should mark the provider source" + ); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("fallback-cookie-id"), + &mut response, + ); + + // Assert + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should create the row"); + let record = stored + .ids + .get("sharedid.org") + .expect("should store the browser fallback"); + assert_eq!( + record.uid, "fallback-cookie-id", + "should store the browser uid" + ); + assert!( + !record.is_provider_written(), + "should mark the fallback as not provider-written" + ); + } + #[test] fn finalize_auction_confirmed_miss_does_not_create_a_row() { // The same path with a genuinely absent row must stay a no-op: a route diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 7be48e1e6d..29db7122fc 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -157,10 +157,9 @@ pub(crate) fn collect_eid_updates( { updates.push(update); } - let mut updates = dedupe_partner_updates(updates); - // Provider-owned sources are written only by their identity provider. - updates.retain(|update| !registry.is_provider_owned(&update.partner_id)); - updates + // Provider-owned sources stay in the list: the write planner ingests them + // only as a fallback when the provider left the source empty. + dedupe_partner_updates(updates) } pub(crate) fn collect_prebid_eid_updates( diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 1ccc721246..60c60eb009 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -1,8 +1,14 @@ //! Plans a navigation's single row change. //! -//! Combines browser ingest (fill missing, never provider-owned sources) and -//! validated provider results (replace the provider's sources), prunes -//! expired UIDs and enforces [`MAX_ENTRY_BYTES`]. Pure: no I/O. +//! Combines browser ingest (fill missing) and validated provider results +//! (replace the provider's sources), prunes expired UIDs and enforces +//! [`MAX_ENTRY_BYTES`]. Pure: no I/O. +//! +//! A provider-owned source takes a browser UID only as a fallback: when the +//! row holds no live UID for it and the provider kept nothing for it on this +//! navigation. The fallback is a single-UID record without provider `eids`, +//! so the coordinator still treats the source as due and a later provider +//! result replaces it. use super::EcKvSnapshot; use super::coordinator::results::LookupResults; @@ -57,8 +63,26 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { }; let mut entry = base.clone(); - apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); - let written = if input.lookups.is_for(input.ec_id) { + let lookups_apply = input.lookups.is_for(input.ec_id); + let provider_filled: Vec<&SourceDomain> = if lookups_apply { + input + .lookups + .iter() + .flat_map(|lookup| lookup.entries.iter().map(|(source, _)| source)) + .collect() + } else { + Vec::new() + }; + apply_browser_ingest( + &mut entry, + input.browser_ingest, + &BrowserIngestRules { + registry: input.registry, + provider_filled: &provider_filled, + now: input.now, + }, + ); + let written = if lookups_apply { apply_lookups(&mut entry, input.lookups, input.now) } else { if !input.lookups.is_empty() { @@ -82,19 +106,46 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } } +/// What browser ingest needs to decide provider-owned fallbacks. +struct BrowserIngestRules<'a> { + registry: &'a PartnerRegistry, + /// Sources the provider returned entries for on this navigation. + provider_filled: &'a [&'a SourceDomain], + now: UnixSeconds, +} + fn apply_browser_ingest( entry: &mut KvEntry, updates: &[PartnerIdUpdate], - registry: &PartnerRegistry, + rules: &BrowserIngestRules<'_>, ) { for update in updates { - if registry.is_provider_owned(&update.partner_id) { + if !rules.registry.is_provider_owned(&update.partner_id) { + entry + .ids + .entry(update.partner_id.clone()) + .or_insert_with(|| KvPartnerId::single(update.uid.clone())); continue; } - entry + let provider_filled = rules + .provider_filled + .iter() + .any(|source| source.as_str() == update.partner_id); + let held = entry .ids - .entry(update.partner_id.clone()) - .or_insert_with(|| KvPartnerId::single(update.uid.clone())); + .get(&update.partner_id) + .is_some_and(|record| record.has_live_uid(rules.now)); + if provider_filled || held { + continue; + } + log::debug!( + "EC write planner: ingesting browser fallback for provider-owned source '{}'", + update.partner_id + ); + entry.ids.insert( + update.partner_id.clone(), + KvPartnerId::single(update.uid.clone()), + ); } } @@ -156,7 +207,7 @@ fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { #[cfg(test)] mod tests { use super::*; - use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup, SkipReason}; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; use crate::ec::kv::KvIdentityGraph; use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; @@ -435,4 +486,167 @@ mod tests { assert!(kept.len() < 40, "should drop some uids"); assert_eq!(kept[0].id().as_str(), ids[0], "should drop from the end"); } + + fn ingest_owned() -> [PartnerIdUpdate; 1] { + [PartnerIdUpdate::new("ids.example.com", "browser-id")] + } + + fn failed_lookup(status: LookupStatus) -> LookupResults { + LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status, + entries: Vec::new(), + }], + ) + } + + fn provider_entry(ids: &[&str]) -> KvEntry { + let mut entry = live_entry(); + let lookups = found("ids.example.com", ids); + let (_, eids) = lookups.iter().next().expect("should hold a lookup").entries[0].clone(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids(eids, UnixSeconds::new(1000)).expect("should build record"), + ); + entry + } + + #[test] + fn provider_owned_source_takes_browser_fallback_when_lookup_fails() { + // Arrange + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + + for status in [ + LookupStatus::NoMatch, + LookupStatus::Skipped(SkipReason::NotDue), + ] { + let lookups = failed_lookup(status); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!( + record.uid, "browser-id", + "should store the browser fallback" + ); + assert!( + !record.is_provider_written(), + "should mark the fallback as not provider-written" + ); + } + } + + #[test] + fn provider_owned_source_takes_browser_fallback_without_lookups() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let ingest = ingest_owned(); + let lookups = LookupResults::default(); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &ingest, + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert_eq!( + entry.ids["ids.example.com"].uid, "browser-id", + "should store the browser fallback when no provider ran" + ); + } + + #[test] + fn browser_fallback_is_not_stored_when_provider_found_the_source() { + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!(record.uid, "p-1", "should store the provider uid"); + assert!(record.is_provider_written(), "should keep provider eids"); + } + + #[test] + fn browser_fallback_never_overwrites_a_live_provider_uid() { + let registry = registry(&["ids.example.com"]); + let snapshot = present(&provider_entry(&["p-1"])); + let ingest = ingest_owned(); + let lookups = failed_lookup(LookupStatus::NoMatch); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should leave the live provider uid untouched" + ); + } + + #[test] + fn browser_fallback_does_not_overwrite_an_existing_fallback() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("older")); + let snapshot = present(&entry); + let ingest = ingest_owned(); + let lookups = failed_lookup(LookupStatus::NoMatch); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite an existing browser uid" + ); + } + + #[test] + fn later_provider_result_replaces_browser_fallback() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("browser-id"), + ); + let snapshot = present(&entry); + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input( + &snapshot, + None, + &ingest_owned(), + &lookups, + ®istry, + )) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!(record.uid, "p-1", "should replace the fallback"); + assert!(record.is_provider_written(), "should store provider eids"); + } } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 60f47a9be6..8922920b21 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -803,6 +803,14 @@ Lookups run only for an existing or newly issued EC, only when consent allows EID persistence, and, when `tcf_vendor_id` is set, only with TCF vendor consent for that vendor. +A provider owns its `sources`: its results replace whatever the row holds for +them, and batch sync rejects them. Browser-resolved IDs (`ts-eids`, `sharedId`, +`/auction` body EIDs) for an owned source are stored only as a fallback, when +the row has no live provider-written UID for the source and the provider kept +nothing for it on that navigation. The fallback follows the normal browser +ingest rules, never overwrites an existing value, and does not stop later +lookups: the next successful lookup replaces it. + | Field | Type | Required | Description | | ----------------- | ------- | -------- | ----------------------------------------------------------------------- | | `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | From 1b84e3c07e09f00775bf06f8d5159bca243b80db Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 19:33:03 +0530 Subject: [PATCH 11/38] Use an example host in the identify record test Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/identify.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 26a6e195e4..18346249c7 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -788,7 +788,7 @@ mod tests { kv.create(&ec_id, &entry).expect("should seed KV entry"); let req = Request::builder() .method("GET") - .uri("https://edge.test-publisher.com/identify") + .uri("https://edge.example.com/identify") .header("authorization", format!("Bearer {VALID_API_TOKEN}")) .body(EdgeBody::empty()) .expect("should build test request"); From 7db9e8817489fd254f97f8853e89358241d0940a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 19:59:10 +0530 Subject: [PATCH 12/38] Send sec-fetch-mode navigate from the EC integration test client Page-load writes only run on real navigations, which browsers mark with both sec-fetch-dest: document and sec-fetch-mode: navigate. Without the mode header the consent withdrawal scenario never got a write permit, so the EC cookie was kept. Signed-off-by: dhruv8sh --- crates/trusted-server-integration-tests/tests/common/ec.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/trusted-server-integration-tests/tests/common/ec.rs b/crates/trusted-server-integration-tests/tests/common/ec.rs index 0a1f149c3b..1846fb9602 100644 --- a/crates/trusted-server-integration-tests/tests/common/ec.rs +++ b/crates/trusted-server-integration-tests/tests/common/ec.rs @@ -113,6 +113,7 @@ impl EcTestClient { .client .get(format!("{}{path}", self.base_url)) .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") .header("accept", "text/html"); for (key, value) in headers { builder = builder.header(*key, *value); From 7c35af40cfe0889cb86e04932b76c29aeaa9f6a6 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 3 Nov 2026 19:59:12 +0530 Subject: [PATCH 13/38] Skip identity lookups without EC consent A lookup sends the EC ID to the provider before finalize can decline the EC, so GPC, sale opt-out and unknown-jurisdiction requests must not dispatch. Matches the gate pull sync already applies. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 8fa406cbcb..75e92b7446 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -23,6 +23,7 @@ use self::validate::{ValidatedEntries, ValidationPolicy, validate}; use crate::consent::ConsentContext; use crate::consent::{allows_eid_persistence, has_tcf_vendor_consent}; use crate::ec::EcKvSnapshot; +use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; use crate::ec::kv_types::KvEntry; use crate::ec::page_load::RowChange; @@ -433,9 +434,11 @@ impl IdentityCoordinator for ProviderCoordinator

{ return pending; } let entry = lookup.snapshot.entry_for(lookup.ec_id); - // A withdrawal tombstone or missing EID consent skips every provider. - let consented = - allows_eid_persistence(lookup.consent) && entry.is_none_or(|entry| entry.consent.ok); + // Missing EC or EID consent, or a withdrawal tombstone, skips every + // provider: a lookup discloses the EC ID before finalize could decline it. + let consented = ec_consent_granted(lookup.consent) + && allows_eid_persistence(lookup.consent) + && entry.is_none_or(|entry| entry.consent.ok); let deadline = Deadline::after(self.budget); pending.deadline = Some(deadline); for (index, registered) in self.providers.iter().enumerate() { @@ -919,6 +922,53 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + #[test] + fn missing_ec_consent_skips_every_provider() { + let coordinator = coordinator(""); + let cases = [ + ( + "GPC opt-out in a US state", + ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + ..Default::default() + }, + ), + ( + "unknown jurisdiction", + ConsentContext { + jurisdiction: Jurisdiction::Unknown, + ..Default::default() + }, + ), + ( + "GDPR without a TC string", + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ]; + + for (label, consent) in cases { + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + + let results = run(&coordinator, &transport, &seeded_snapshot(&entry), &consent); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip for {label}" + ); + assert!( + transport.sent.borrow().is_empty(), + "should send nothing for {label}" + ); + } + } + #[test] fn missing_tcf_vendor_consent_skips_before_building_a_request() { let coordinator = coordinator("tcf_vendor_id = 9999"); @@ -942,6 +992,7 @@ mod tests { purposes[0] = true; purposes[3] = true; let consent = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, gdpr_applies: true, tcf: Some(TcfConsent { version: 2, From 2de6d0d1022591e8ba9bee4aea746196c3da933b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 3 Nov 2026 18:24:59 +0530 Subject: [PATCH 14/38] Raise the UID length limit to 1024 bytes and drop oversized IDs on KV read LiveRamp envelopes are about 700 bytes, so the 512-byte limit dropped every one from KV and the auction. Rows written under a higher limit now load with the oversized IDs removed instead of failing as a whole. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/admin.rs | 2 +- .../src/ec/coordinator/validate.rs | 2 +- crates/trusted-server-core/src/ec/kv.rs | 31 ++-- crates/trusted-server-core/src/ec/kv_types.rs | 132 +++++++++++++++++- .../trusted-server-core/src/ec/pull_sync.rs | 4 +- docs/guide/integrations/prebid.md | 30 ++-- 6 files changed, 169 insertions(+), 32 deletions(-) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 383edddf15..7014721f0e 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -1417,7 +1417,7 @@ mod tests { #[test] fn eids_lookup_reports_configured_source_without_valid_uid() { - let oversized_uid = "x".repeat(513); + let oversized_uid = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); let cookie = eids_cookie_for(&serde_json::json!([{ "source": "bidstream.example", "uids": [ diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index 25cd7332f1..e7c8107ea9 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -238,7 +238,7 @@ mod tests { let partners = registry(); let provider = ProviderId::new("example-provider"); let claimed = claimed(); - let long = "x".repeat(600); + let long = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); let mut expired = uid("u-old"); expired.expiry = ProviderExpiry::At(UnixSeconds::new(1000)); let array_ext = RawValue::from_string("[1]".to_owned()).expect("should build raw"); diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 434c38bbb9..9a3f9492ee 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -267,12 +267,20 @@ impl KvIdentityGraph { ec_id: &str, body_bytes: &[u8], ) -> Result> { - let entry: KvEntry = + let mut entry: KvEntry = serde_json::from_slice(body_bytes).change_context(TrustedServerError::KvStore { store_name: store_name.to_owned(), message: format!("Failed to deserialize entry for key '{}'", log_id(ec_id)), })?; + let dropped = entry.drop_unreadable_ids(); + if dropped > 0 { + log::warn!( + "Dropped {dropped} unreadable partner IDs loading key '{}'", + log_id(ec_id) + ); + } + entry.validate().map_err(|message| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_owned(), @@ -1289,7 +1297,7 @@ mod tests { } #[test] - fn deserialize_entry_rejects_invalid_legacy_values() { + fn deserialize_entry_drops_oversized_ids() { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), @@ -1297,14 +1305,19 @@ mod tests { "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), ), ); - let body = serde_json::to_vec(&entry).expect("should serialize invalid entry payload"); + entry.ids.insert( + "ssp_y".to_owned(), + crate::ec::kv_types::KvPartnerId::single("kept"), + ); + let body = serde_json::to_vec(&entry).expect("should serialize oversized entry payload"); - let err = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) - .expect_err("should reject invalid legacy entry values"); - let err_text = format!("{err}"); - assert!( - err_text.contains("Loaded invalid entry"), - "should report validation failure for loaded entries" + let loaded = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) + .expect("should load an entry holding an oversized ID"); + + assert_eq!( + loaded.ids.keys().collect::>(), + vec!["ssp_y"], + "should drop only the oversized ID" ); } diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 45ac8b6df6..b97a37892e 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -6,9 +6,11 @@ //! //! The schema is versioned (`v: 1`) to allow future migrations. +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, de}; +use serde_json::value::RawValue; use super::eid_types::{ AgentType, EidFields, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, RawJson, UidValue, @@ -34,9 +36,10 @@ pub const MAX_SEEN_DOMAINS: usize = 50; pub const MAX_STORED_DOMAIN_LENGTH: usize = 255; /// Maximum allowed length (in bytes) for a partner UID across all sync -/// mechanisms (pixel, batch, pull). Defined centrally to ensure consistent -/// validation. -pub const MAX_UID_LENGTH: usize = 512; +/// mechanisms (pixel, batch, pull) and identity providers. Sized for +/// `LiveRamp` envelopes (about 700 bytes). Defined centrally to ensure +/// consistent validation. +pub const MAX_UID_LENGTH: usize = 1024; /// Full KV entry stored as the body of an EC identity graph record. /// @@ -187,6 +190,24 @@ impl KvPartnerId { *self != before } + /// Drops entries left without UIDs and resyncs `uid`. + /// + /// Returns whether the record still holds a UID within + /// [`MAX_UID_LENGTH`]. + fn retain_readable(&mut self) -> bool { + if self.eids.is_empty() { + return self.uid.len() <= MAX_UID_LENGTH; + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + !self.eids.is_empty() + } + /// Removes the last UID of the last entry and resyncs `uid`; used by the /// write planner's size cap. pub(crate) fn drop_last_uid(&mut self) { @@ -227,9 +248,39 @@ struct KvEidRepr { mm: Option, #[serde(default)] ext: Option, + #[serde(deserialize_with = "deserialize_readable_uids")] uids: Vec, } +/// The `id` of a stored UID, read without validating the rest of it. +#[derive(Deserialize)] +struct StoredId<'a> { + #[serde(borrow)] + id: Cow<'a, str>, +} + +/// Deserializes stored UIDs, skipping any whose `id` exceeds +/// [`MAX_UID_LENGTH`]. +/// +/// A row written under a higher limit then still loads; an entry left +/// without UIDs is removed by [`KvEntry::drop_unreadable_ids`]. Any other +/// invalid UID still fails the row. +fn deserialize_readable_uids<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let raw = Vec::>::deserialize(deserializer)?; + let mut uids = Vec::with_capacity(raw.len()); + for value in &raw { + let oversized = serde_json::from_str::>(value.get()) + .is_ok_and(|stored| stored.id.len() > MAX_UID_LENGTH); + if oversized { + continue; + } + uids.push(serde_json::from_str(value.get()).map_err(de::Error::custom)?); + } + Ok(uids) +} + impl From for KvEid { fn from(repr: KvEidRepr) -> Self { let KvEidRepr { @@ -504,6 +555,17 @@ pub(crate) fn validated_stored_domain(domain: &str) -> Option { } impl KvEntry { + /// Removes partner records holding no UID this build can serve. + /// + /// Lets this build load rows written by one with a higher + /// [`MAX_UID_LENGTH`] instead of rejecting the whole row. Returns the + /// number of records removed. + pub fn drop_unreadable_ids(&mut self) -> usize { + let before = self.ids.len(); + self.ids.retain(|_, record| record.retain_readable()); + before - self.ids.len() + } + /// Creates a new live entry from the current request context. /// /// `domain` is the publisher's apex domain (e.g. `"example.com"`), @@ -1543,6 +1605,68 @@ mod tests { assert!(entry.validate().is_err(), "should reject EID with no UIDs"); } + #[test] + fn validate_accepts_liveramp_sized_uid() { + let entry = KvEntry::minimal("liveramp.example.com", &"x".repeat(703), 1); + assert!( + entry.validate().is_ok(), + "should accept a 703-byte envelope-sized UID" + ); + } + + #[test] + fn drop_unreadable_ids_removes_oversized_single_uid() { + let mut entry = KvEntry::minimal("ids.example.com", "kept", 1); + entry.ids.insert( + "long.example.com".to_owned(), + KvPartnerId::single("x".repeat(MAX_UID_LENGTH + 1)), + ); + + let dropped = entry.drop_unreadable_ids(); + + assert_eq!(dropped, 1, "should drop the oversized record"); + assert_eq!( + entry.ids.keys().collect::>(), + vec!["ids.example.com"], + "should keep the readable record" + ); + assert!(entry.validate().is_ok(), "should validate after dropping"); + } + + #[test] + fn deserialize_drops_oversized_provider_uids() { + let long = "x".repeat(MAX_UID_LENGTH + 1); + let json = serde_json::json!({ + "v": SCHEMA_VERSION, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "partial.example.com": { "uid": long, "eids": [ + { "uids": [{ "id": long, "atype": 1 }] }, + { "uids": [{ "id": "short", "atype": 1 }] } + ] }, + "gone.example.com": { "uid": long, "eids": [ + { "uids": [{ "id": long, "atype": 1 }] } + ] } + } + }) + .to_string(); + let mut entry: KvEntry = + serde_json::from_str(&json).expect("should parse a row with oversized UIDs"); + + let dropped = entry.drop_unreadable_ids(); + + assert_eq!(dropped, 1, "should drop the record left without UIDs"); + let partial = entry + .ids + .get("partial.example.com") + .expect("should keep the record with a readable UID"); + assert_eq!(partial.uid, "short", "should resync uid to the kept UID"); + assert_eq!(partial.eids().len(), 1, "should drop the emptied entry"); + assert!(entry.validate().is_ok(), "should validate after dropping"); + } + #[test] fn provider_record_round_trips_eid_level_fields() { let ext = serde_json::value::to_raw_value(&serde_json::json!({ "k": 1 })) diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 4aa8617f62..24993c18f1 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -798,12 +798,12 @@ mod tests { #[test] fn extract_pull_uid_rejects_oversized_uid() { - let long_uid = "x".repeat(513); + let long_uid = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); let body = format!("{{\"uid\":\"{long_uid}\"}}"); let response = make_response(200, body.as_bytes()); let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject uid exceeding 512 bytes"); + assert!(uid.is_none(), "should reject uid exceeding MAX_UID_LENGTH"); } #[test] diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index b7a05ac7be..3b314f5bec 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -902,20 +902,20 @@ server-to-server ATS API. ### Degraded behavior -| Condition | Result | -| -------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| TCF Purpose 1 or LiveRamp vendor consent is denied | Default `tcfControl` blocks IdentityLink resolution and storage | -| TCF Purpose 3 or 4 alone is denied | Resolution/storage continues under defaults; publisher rules may differ | -| The user opts out under a US state signal | No LiveRamp EID is forwarded; the auction continues | -| LiveRamp cannot recognize the browser | IdentityLink yields no EID; the auction continues | -| LiveRamp network resolution fails | The current auction continues without RampID | -| The managed module is missing from the bundle | Existing diagnostics report the missing module; auctions continue | -| The origin is not approved by LiveRamp | Resolution yields no usable EID; the auction continues | -| EC/KV is unavailable | A current-request EID can still reach `/auction`; persistence degrades | -| The resolved envelope exceeds the 512-byte EID value cap | The envelope is dropped from both the `/auction` payload and EC persistence; the auction continues | -| The CMP is callable but never returns a settled result | The managed entry is never seeded; no vendor call, no identity storage, and the auction continues | -| Two managed entries address one Prebid submodule | The later entry is dropped with a logged error; the first entry's configuration takes effect | -| A silent CMP stub is replaced by a working CMP | The shim re-subscribes to the replacement; a CMP that already answered once keeps its own wait | +| Condition | Result | +| --------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| TCF Purpose 1 or LiveRamp vendor consent is denied | Default `tcfControl` blocks IdentityLink resolution and storage | +| TCF Purpose 3 or 4 alone is denied | Resolution/storage continues under defaults; publisher rules may differ | +| The user opts out under a US state signal | No LiveRamp EID is forwarded; the auction continues | +| LiveRamp cannot recognize the browser | IdentityLink yields no EID; the auction continues | +| LiveRamp network resolution fails | The current auction continues without RampID | +| The managed module is missing from the bundle | Existing diagnostics report the missing module; auctions continue | +| The origin is not approved by LiveRamp | Resolution yields no usable EID; the auction continues | +| EC/KV is unavailable | A current-request EID can still reach `/auction`; persistence degrades | +| The resolved envelope exceeds the 1024-byte EID value cap | The envelope is dropped from both the `/auction` payload and EC persistence; the auction continues | +| The CMP is callable but never returns a settled result | The managed entry is never seeded; no vendor call, no identity storage, and the auction continues | +| Two managed entries address one Prebid submodule | The later entry is dropped with a logged error; the first entry's configuration takes effect | +| A silent CMP stub is replaced by a working CMP | The shim re-subscribes to the replacement; a CMP that already answered once keeps its own wait | The TCF rows assume either the managed-ID automatic setup described above or a publisher-owned Prebid GDPR configuration. A CMP API and its policy remain @@ -932,7 +932,7 @@ domain, booleans, source names, counts, and status codes: `identityLink` entry with the test Placement ID. 2. With positive consent, confirm `idl_env` is created or refreshed. 3. Record the byte length of the resolved envelope — the length only, never the - value — and confirm it is at or below the 512-byte EID value cap. Anything + value — and confirm it is at or below the 1024-byte EID value cap. Anything above it is dropped as described in the degraded-behavior table. 4. Confirm `pbjs.getUserIdsAsEids()` reports source `liveramp.com` without recording its value. From ff5113dfa8c22b6b61fb6097e9e6ec46efb7eb02 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:31:47 +0530 Subject: [PATCH 15/38] Gate TCF vendor consent only where TCF applies and skip lookups on unreadable rows Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/consent/mod.rs | 9 +++ .../src/ec/coordinator/mod.rs | 71 +++++++++++++++---- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index 83bf3f33db..5d232df8a9 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -296,6 +296,15 @@ pub fn has_tcf_vendor_consent(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bo effective_tcf(ctx).is_some_and(|tcf| tcf.has_vendor_consent(vendor_id.get())) } +/// Whether TCF rules apply and withhold vendor consent from `vendor_id`. +/// +/// Returns `false` when no TCF string applies and GDPR does not, so a TCF +/// vendor requirement only gates traffic that TCF governs. +#[must_use] +pub fn tcf_vendor_consent_denied(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bool { + (ctx.gdpr_applies || effective_tcf(ctx).is_some()) && !has_tcf_vendor_consent(ctx, vendor_id) +} + /// Returns whether a server-side auction may be dispatched for this request. /// /// Fails closed for GDPR-relevant traffic: when an EU TCF signal is present diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 75e92b7446..1dec09044c 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -21,7 +21,7 @@ use self::transport::{ }; use self::validate::{ValidatedEntries, ValidationPolicy, validate}; use crate::consent::ConsentContext; -use crate::consent::{allows_eid_persistence, has_tcf_vendor_consent}; +use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; @@ -434,9 +434,12 @@ impl IdentityCoordinator for ProviderCoordinator

{ return pending; } let entry = lookup.snapshot.entry_for(lookup.ec_id); - // Missing EC or EID consent, or a withdrawal tombstone, skips every - // provider: a lookup discloses the EC ID before finalize could decline it. - let consented = ec_consent_granted(lookup.consent) + // Missing EC or EID consent, a withdrawal tombstone, or an unreadable + // row (which may hide a tombstone) skips every provider: a lookup + // discloses the EC ID before finalize could decline it. + let readable = !matches!(lookup.snapshot, EcKvSnapshot::Failed { .. }); + let consented = readable + && ec_consent_granted(lookup.consent) && allows_eid_persistence(lookup.consent) && entry.is_none_or(|entry| entry.consent.ok); let deadline = Deadline::after(self.budget); @@ -450,7 +453,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ let vendor_denied = registered .policy .tcf_vendor_id - .is_some_and(|vendor| !has_tcf_vendor_consent(lookup.consent, vendor)); + .is_some_and(|vendor| tcf_vendor_consent_denied(lookup.consent, vendor)); if !consented || vendor_denied { pending.early.push(skip(SkipReason::Consent)); continue; @@ -970,28 +973,42 @@ mod tests { } #[test] - fn missing_tcf_vendor_consent_skips_before_building_a_request() { - let coordinator = coordinator("tcf_vendor_id = 9999"); + fn failed_snapshot_skips_every_provider() { + let coordinator = coordinator(""); let transport = CannedTransport::new(ok_reply); + let snapshot = EcKvSnapshot::Failed { + ec_id: EC_ID.to_owned(), + }; - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &snapshot, &granting()); assert_eq!( only_status(&results), LookupStatus::Skipped(SkipReason::Consent), - "should skip" + "should skip when the row could not be read" ); assert!(transport.sent.borrow().is_empty(), "should send nothing"); } #[test] - fn granted_tcf_vendor_consent_allows_the_lookup() { + fn tcf_vendor_id_does_not_gate_traffic_without_tcf() { let coordinator = coordinator("tcf_vendor_id = 9999"); let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert!( + !matches!(only_status(&results), LookupStatus::Skipped(_)), + "should look up when no TCF string applies outside GDPR" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + fn tcf_consent(vendor_consents: Vec) -> ConsentContext { let mut purposes = vec![false; 24]; purposes[0] = true; purposes[3] = true; - let consent = ConsentContext { + ConsentContext { jurisdiction: Jurisdiction::Gdpr, gdpr_applies: true, tcf: Some(TcfConsent { @@ -1006,14 +1023,40 @@ mod tests { last_updated_ds: 0, purpose_consents: purposes, purpose_legitimate_interests: vec![false; 24], - vendor_consents: vec![9999], + vendor_consents, vendor_legitimate_interests: Vec::new(), special_feature_opt_ins: vec![false; 12], }), ..ConsentContext::default() - }; + } + } + + #[test] + fn missing_tcf_vendor_consent_skips_before_building_a_request() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &tcf_consent(vec![1])); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn granted_tcf_vendor_consent_allows_the_lookup() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &consent); + let results = run( + &coordinator, + &transport, + &missing(), + &tcf_consent(vec![9999]), + ); assert!( matches!(only_status(&results), LookupStatus::Found { .. }), From f0e2fb065dfaf272875f3d2f2445af2489b83287 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:33:15 +0530 Subject: [PATCH 16/38] Mint a new EC only on navigations that may write its row Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 175898ae53..d3bdafb189 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -958,7 +958,10 @@ async fn dispatch_fallback( } } let is_publisher_navigation = ec.is_real_browser && is_navigation; + // A new EC is minted only when this request may also write its row, + // so a client without Fetch Metadata never receives a throwaway EC. if is_publisher_navigation + && has_permit && let Err(err) = ec .ec_context .generate_if_needed(&state.settings, ec.kv_graph.as_ref()) @@ -2737,6 +2740,58 @@ mod tests { ); } + fn consented_navigation(mut request: Request) -> Request { + // Viceroy geolocates loopback to a US privacy state; a US Privacy + // string without a sale opt-out lets the navigation mint an EC. + request.extensions_mut().insert(ClientInfo { + client_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ..ClientInfo::default() + }); + request + .headers_mut() + .insert(header::COOKIE, HeaderValue::from_static("us_privacy=1YNN")); + request + } + + #[test] + fn navigation_without_fetch_metadata_mints_no_ec() { + let router = test_router(); + let mut request = consented_navigation(empty_request(Method::GET, "/article")); + request + .headers_mut() + .insert(header::ACCEPT, HeaderValue::from_static("text/html")); + request.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + + let mut response = route(&router, request); + + let state = take_finalize_state(&mut response); + assert!( + state.ec_context.ec_value().is_none(), + "should not mint an EC that no row can back" + ); + } + + #[test] + fn navigation_with_fetch_metadata_mints_an_ec() { + let router = test_router(); + + let mut response = route( + &router, + consented_navigation(navigation_request("/article")), + ); + + let state = take_finalize_state(&mut response); + assert!( + state.ec_context.ec_value().is_some(), + "should mint an EC on a page-load navigation" + ); + } + #[test] fn publisher_navigation_without_providers_dispatches_nothing() { let router = test_router(); From 97e680600572b24e1af18485e51fc99cac4ded33 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:35:07 +0530 Subject: [PATCH 17/38] Reject batch sync records for provider-owned sources Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/batch_sync.rs | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 4903589b29..66a894dad8 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -33,6 +33,7 @@ const REASON_INVALID_EC_ID: &str = "invalid_ec_id"; const REASON_INVALID_PARTNER_UID: &str = "invalid_partner_uid"; const REASON_INELIGIBLE: &str = "ineligible"; const REASON_KV_UNAVAILABLE: &str = "kv_unavailable"; +const REASON_PROVIDER_OWNED: &str = "provider_owned"; /// Maximum number of mappings allowed in a single batch request. const MAX_BATCH_SIZE: usize = 1000; @@ -167,7 +168,12 @@ fn handle_batch_sync_with_writer( } // 4. Process mappings with per-item validation and rejection reasons. - let (accepted, errors) = process_mappings(writer, &partner.source_domain, &body.mappings); + let (accepted, errors) = process_mappings_for_partner( + writer, + &partner.source_domain, + registry.is_provider_owned(&partner.source_domain), + &body.mappings, + ); let rejected = errors.len(); let status = if rejected > 0 { @@ -193,6 +199,30 @@ fn content_length_exceeds_limit(req: &Request, max_body_size: usize) - .is_some_and(|content_length| content_length > max_body_size) } +/// Processes mappings for one partner, rejecting all of them when an enabled +/// identity provider owns the partner's source (spec §8.5). +fn process_mappings_for_partner( + writer: &dyn BatchSyncWriter, + partner_id: &str, + provider_owned: bool, + mappings: &[SyncMapping], +) -> (usize, Vec) { + if !provider_owned { + return process_mappings(writer, partner_id, mappings); + } + log::info!( + "Batch sync: rejecting {} mappings for provider-owned source '{partner_id}'", + mappings.len() + ); + let errors = (0..mappings.len()) + .map(|index| MappingError { + index, + reason: REASON_PROVIDER_OWNED, + }) + .collect(); + (0, errors) +} + /// Validates all mappings, then processes each normalized EC ID once. /// /// Successful and eligibility outcomes fan out to every valid input in a @@ -544,6 +574,27 @@ mod tests { assert_eq!(errors[1].reason, REASON_INVALID_PARTNER_UID); } + #[test] + fn provider_owned_source_rejects_every_mapping() { + let writer = MockWriter::new(vec![Ok(UpsertResult::Written), Ok(UpsertResult::Written)]); + let mappings = vec![ + mapping(&format!("{}.ABC123", "a".repeat(64)), "u1", 1), + mapping(&format!("{}.ABC123", "b".repeat(64)), "u2", 1), + ]; + + let (accepted, errors) = + process_mappings_for_partner(&writer, "ids.example.com", true, &mappings); + + assert_eq!(accepted, 0, "should accept nothing"); + assert_eq!(errors.len(), 2, "should reject each mapping"); + assert!( + errors + .iter() + .all(|error| error.reason == REASON_PROVIDER_OWNED), + "should report provider_owned" + ); + } + #[test] fn process_mappings_aborts_on_kv_unavailable() { let writer = MockWriter::new(vec![ From 665f8d3bb414d8b2f8319899c32bec9fe22d964b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:34:20 +0530 Subject: [PATCH 18/38] Skip pull sync for provider-owned sources Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/registry.rs | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 5fa2677167..435e164b80 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -305,12 +305,16 @@ impl PartnerRegistry { .is_some_and(|partner| partner.provider_owned) } - /// Returns all partners with `pull_sync_enabled = true`. + /// Returns all partners with `pull_sync_enabled = true` whose source no + /// identity provider owns. + /// + /// A provider writes full-fidelity records for the sources it owns, so a + /// pull-sync write there would overwrite them with a single UID. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { self.by_source_domain .values() - .filter(|p| p.pull_sync_enabled) + .filter(|p| p.pull_sync_enabled && !p.provider_owned) .collect() } @@ -700,6 +704,25 @@ mod tests { assert!(result.is_err(), "should reject invalid source domain"); } + #[test] + fn pull_enabled_partners_excludes_provider_owned_sources() { + let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); + pull_partner.api_token = None; + pull_partner.pull_sync_enabled = true; + pull_partner.pull_sync_url = Some("https://pull.example.com/sync".to_owned()); + pull_partner.pull_sync_allowed_domains = vec!["pull.example.com".to_owned()]; + pull_partner.ts_pull_token = Some(Redacted::new("outbound-token".to_owned())); + let mut registry = + PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); + + registry.mark_provider_owned("pull.example.com"); + + assert!( + registry.pull_enabled_partners().is_empty(), + "should not pull sources an identity provider owns" + ); + } + #[test] fn pull_enabled_partners_filters_correctly() { let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); From c4684f6a1e3a70f26b1ba5f67a82e63960f49403 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:34:50 +0530 Subject: [PATCH 19/38] Document navigation-only browser EID writes and the narrowed TCF vendor gate Signed-off-by: dhruv8sh --- docs/guide/configuration.md | 8 ++++---- docs/guide/edge-cookies.md | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 0ff626d2e5..3cc9402a78 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -800,12 +800,12 @@ finalized. Results go into the navigation's single EC row write. A lookup that times out or fails leaves the row unchanged. Lookups run only for an existing or newly issued EC, only when consent allows -EID persistence, and, when `tcf_vendor_id` is set, only with TCF vendor consent -for that vendor. +EID persistence, and, when `tcf_vendor_id` is set and TCF applies to the +request, only with TCF vendor consent for that vendor. A provider owns its `sources`: its results replace whatever the row holds for -them, and batch sync rejects them. Browser-resolved IDs (`ts-eids`, `sharedId`, -`/auction` body EIDs) for an owned source are stored only as a fallback, when +them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs +(`ts-eids`, `sharedId`) for an owned source are stored only as a fallback, when the row has no live provider-written UID for the source and the provider kept nothing for it on that navigation. The fallback follows the normal browser ingest rules, never overwrites an existing value, and does not stop later diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 113066cdaf..527268a57f 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -148,13 +148,12 @@ flowchart LR ### Prebid EID Flow -Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a response. It collects matched partner UIDs from three request-local sources, applied in this order so that, when two sources carry the same partner, the later source's UID is the candidate offered to KV: +Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a top-level document navigation, the only request that writes KV. It collects matched partner UIDs from two request-local sources, applied in this order so that, when both carry the same partner, the later source's UID is the candidate offered to KV: 1. the `ts-eids` cookie, -2. the `eids` array in the `/auction` request body, -3. the `sharedId` cookie. +2. the `sharedId` cookie. -The `/auction` body carries the full EID set Prebid.js resolved, so `/auction` ingestion is not limited by the size cap TSJS applies to the `ts-eids` cookie. The cookie still bridges Prebid user ID modules with the identity graph on requests that carry no EID body, such as `GET /_ts/page-bids` and page navigations. +The `eids` array in the `/auction` request body goes to bidders on that request only and is never stored. The `ts-eids` cookie bridges Prebid user ID modules with the identity graph on the next navigation. These Prebid EID writes follow the same consent rule as bidstream EID forwarding: when a TCF signal is present, Purpose 1 (store/access device) and Purpose 4 (personalized ads) must both be consented, and when GDPR applies without a TCF signal the writes are withheld. A user who consents to Purpose 1 but not Purpose 4 keeps their EC, but no EIDs from the request are persisted. @@ -173,8 +172,8 @@ sequenceDiagram TSJS->>TSJS: Base64 encode full OpenRTB-style EID array
[{source, uids:[{id, atype, ext?}]}] TSJS->>B: document.cookie = "ts-eids=..." - Note over B,TS: Next eligible EID-sync request - B->>TS: Document navigation, POST /auction,
or admitted GET /_ts/page-bids + Note over B,TS: Next page load + B->>TS: Top-level document navigation TS->>TS: Base64 decode → parse OpenRTB-style EIDs
match source domains to partners TS->>KV: Add missing partner IDs in one conditional write
skip when UIDs already match
(requires TCF Purpose 1 + 4 under GDPR) ``` @@ -183,11 +182,11 @@ Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shap The `sharedId` cookie follows a similar path but is written directly by Prebid's SharedID module rather than by TSJS. The server reads it separately and maps it via the `sharedid.org` source domain. -Returning-user cookie persistence runs only on publisher document navigations, `POST /auction`, and admitted `GET /_ts/page-bids` SPA navigations. Static assets, analytics, integration requests, filter short circuits, and other subresources do not decode or persist these cookies. New EC creation remains eligible regardless of route because its backing row must include the request's initial IDs. +Cookie persistence runs only on top-level document navigations (`Sec-Fetch-Dest: document` and `Sec-Fetch-Mode: navigate`). `POST /auction`, `GET /_ts/page-bids`, static assets, analytics, integration requests, filter short circuits, and other subresources do not persist these cookies. -Each eligible request attempts at most one conditional cookie update. If another writer wins, Trusted Server reads the row once and does not write again from that request. A matching value completes the sync; an absent or different value is deferred. +Each navigation attempts at most one conditional write. If another writer wins, Trusted Server reads the row once and does not write again from that request. A matching value completes the sync; an absent or different value is deferred. -Browser cookies and the `/auction` request body do not carry a trustworthy value timestamp or sequence. Trusted Server therefore adds missing partner IDs but does not replace a different stored UID from a browser-supplied EID, even on later eligible requests. Pull- or push-enabled partners can replace that value through their authoritative synchronization path. A cookie-only partner retains the stored UID until an authoritative freshness rule is introduced or the EC row expires. +Browser cookies do not carry a trustworthy value timestamp or sequence. Trusted Server therefore adds missing partner IDs but does not replace a different stored UID from a browser-supplied EID, even on later eligible requests. Pull- or push-enabled partners can replace that value through their authoritative synchronization path. A cookie-only partner retains the stored UID until an authoritative freshness rule is introduced or the EC row expires. ### EID Seeding and Prebid Bidstream Forwarding From 09cad520c0d7dde93aaa7e380a26bd8122162a30 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:35:53 +0530 Subject: [PATCH 20/38] Skip the snapshot preload for a freshly generated EC Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/publisher.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b70fad1355..2a68d2ab8f 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3458,8 +3458,9 @@ struct EcSnapshotPreloadInput { } fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { - // An earlier step of this request (identity-provider dispatch) already - // read the row for an existing EC, so a second read adds nothing. + // An earlier step of this request already settled the row snapshot + // (identity-provider dispatch, or EC generation's authoritative miss), so + // a read adds nothing. if input.snapshot_already_read { return false; } @@ -5313,10 +5314,12 @@ pub async fn handle_publisher_request( .is_some_and(|registry| !registry.is_empty()), eid_cookie_may_need_persistence, privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), - // A freshly generated EC keeps its refresh read: the Add-confirmed - // snapshot may still upgrade to the stored row. - snapshot_already_read: !ec_context.ec_generated() - && !matches!(ec_context.kv_snapshot(), crate::ec::EcKvSnapshot::NotRead), + // A freshly generated EC already holds an authoritative `Missing` + // snapshot: its row is only created by this navigation's write. + snapshot_already_read: !matches!( + ec_context.kv_snapshot(), + crate::ec::EcKvSnapshot::NotRead + ), }); let mut pending_origin = None; // A shared-template hit skips the origin entirely, and an in-flight request From 17072f45c57b56e90a2bc99adb3171b0afbe074a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:40:55 +0530 Subject: [PATCH 21/38] Build the partner registry once per app state Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 79 ++++++++++++++++--- .../trusted-server-adapter-fastly/src/main.rs | 17 +++- .../src/lib.rs | 47 +++++------ 3 files changed, 106 insertions(+), 37 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index d3bdafb189..f9187ab0d5 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -88,7 +88,7 @@ use std::cell::RefCell; use std::net::IpAddr; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; use edgezero_adapter_fastly::context::FastlyRequestContext; @@ -123,6 +123,7 @@ use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::provider::ClientSignals; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::is_navigation_request; @@ -151,7 +152,7 @@ use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; -use trusted_server_identity_providers::{build_coordinator, partner_registry}; +use trusted_server_identity_providers::{build_coordinator_with, partner_registry}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -200,6 +201,27 @@ pub(crate) struct AppState { pub(crate) registry: Arc, pub(crate) default_kv_store: Arc, pub(crate) auction_telemetry_sink: Arc, + /// Partner registry built on first use and reused for the state's lifetime. + pub(crate) partner_registry: OnceLock>, +} + +impl AppState { + /// Returns the partner registry, building it on first use. + /// + /// A failed build is not cached, so every caller sees its error. + /// + /// # Errors + /// + /// Returns the same errors as [`partner_registry`]. + pub(crate) fn partner_registry( + &self, + ) -> Result, Report> { + if let Some(registry) = self.partner_registry.get() { + return Ok(Arc::clone(registry)); + } + let registry = Arc::new(partner_registry(&self.settings.ec)?); + Ok(Arc::clone(self.partner_registry.get_or_init(|| registry))) + } } /// Build the application state, loading settings and constructing all per-application components. @@ -245,6 +267,7 @@ pub(crate) fn build_state_from_settings( registry: Arc::new(registry), default_kv_store, auction_telemetry_sink, + partner_registry: OnceLock::new(), })) } @@ -383,7 +406,23 @@ async fn dispatch_identity_lookups( req: &Request, ec: &mut EcRequestState, ) -> Option { - let coordinator = match build_coordinator(&state.settings) { + if !state + .settings + .ec + .identity_providers + .iter() + .any(|config| config.enabled) + { + return None; + } + let partners = match state.partner_registry() { + Ok(partners) => partners, + Err(report) => { + log::error!("Identity providers disabled for this request: {report:?}"); + return None; + } + }; + let coordinator = match build_coordinator_with(&state.settings, partners) { Ok(Some(coordinator)) => coordinator, Ok(None) => return None, Err(report) => { @@ -678,7 +717,8 @@ async fn execute_named( handler, NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup ) { - let response = partner_registry(&state.settings.ec) + let response = state + .partner_registry() .and_then(|registry| match handler { NamedRouteHandler::AdminEcLookup => { // Deliberately do not use an EC request-state graph: that @@ -759,7 +799,7 @@ async fn run_named_route( cors_preflight_identify(&state.settings, &req) } else { let kv = crate::require_identity_graph(&state.settings)?; - let partner_registry = partner_registry(&state.settings.ec)?; + let partner_registry = state.partner_registry()?; handle_identify( &state.settings, &kv, @@ -773,11 +813,11 @@ async fn run_named_route( NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partner_registry = partner_registry(&state.settings.ec)?; + let partner_registry = state.partner_registry()?; let registry_ref = if partner_registry.is_empty() { None } else { - Some(&partner_registry) + Some(&*partner_registry) }; handle_auction( &state.settings, @@ -797,11 +837,11 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - let partner_registry = partner_registry(&state.settings.ec)?; + let partner_registry = state.partner_registry()?; let registry_ref = if partner_registry.is_empty() { None } else { - Some(&partner_registry) + Some(&*partner_registry) }; let auction = AuctionDispatch { integration_registry: &state.registry, @@ -844,7 +884,7 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = partner_registry(&state.settings.ec)?; + let partner_registry = state.partner_registry()?; let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); handle_batch_sync(&kv, &partner_registry, &limiter, req) }); @@ -985,13 +1025,13 @@ async fn dispatch_fallback( // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. let slots = state.settings.creative_opportunity_slots(); - match partner_registry(&state.settings.ec) { + match state.partner_registry() { Ok(partner_registry) => { let auction = AuctionDispatch { integration_registry: &state.registry, orchestrator: &state.orchestrator, slots, - registry: Some(&partner_registry), + registry: Some(&*partner_registry), }; match handle_publisher_request( &state.settings, @@ -1749,6 +1789,7 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, + partner_registry: std::sync::OnceLock::new(), }); TrustedServerApp::routes_for_state(&state) } @@ -2753,6 +2794,19 @@ mod tests { request } + #[test] + fn partner_registry_is_built_once_per_state() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + + let first = state.partner_registry().expect("should build the registry"); + let second = state.partner_registry().expect("should reuse the registry"); + + assert!( + Arc::ptr_eq(&first, &second), + "should return the cached registry" + ); + } + #[test] fn navigation_without_fetch_metadata_mints_no_ec() { let router = test_router(); @@ -2993,6 +3047,7 @@ mod tests { let result = crate::apply_edgezero_ec_finalize( &settings, + None, &mut state, Some(super::IdentityLookups { coordinator, diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index efff0d4952..7c8340bdb8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -553,7 +553,13 @@ fn edgezero_main( if let Some(mut ec_state) = ec_state { if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, identity, &mut response) { + match apply_edgezero_ec_finalize( + settings, + app_state.as_deref(), + &mut ec_state, + identity, + &mut response, + ) { Ok(partner_registry) => { send_edgezero_response( response, @@ -574,6 +580,7 @@ fn edgezero_main( Ok(settings) => { match apply_edgezero_ec_finalize( &settings, + None, &mut ec_state, identity, &mut response, @@ -657,10 +664,11 @@ fn apply_entry_point_finalize_headers( /// panics. fn apply_edgezero_ec_finalize( settings: &Settings, + state: Option<&AppState>, ec_state: &mut EcFinalizeState, identity: Option, response: &mut HttpResponse, -) -> Result> { +) -> Result, Report> { // A redirect skips the write, so do not wait for lookups it would discard. let identity = identity.filter(|_| { let redirect = NavigationPermit::is_redirect(response.status()); @@ -679,7 +687,10 @@ fn apply_edgezero_ec_finalize( let results = futures::executor::block_on(coordinator.collect(&transport, pending)); ec_state.ec_context.set_lookup_results(results); } - let partner_registry = partner_registry(&settings.ec)?; + let partner_registry = match state { + Some(state) => state.partner_registry()?, + None => Arc::new(partner_registry(&settings.ec)?), + }; let finalize_kv_graph = if ec_state.use_finalize_kv { maybe_identity_graph(settings) } else { diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index dd6c086f43..d7206687ed 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -64,37 +64,40 @@ pub fn build( pub fn build_coordinator( settings: &Settings, ) -> Result, Report> { - let enabled: Vec<&IdentityProviderConfig> = settings + let partners = partner_registry(&settings.ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + build_coordinator_with(settings, Arc::new(partners)) +} + +/// Builds the coordinator for every enabled provider in `settings` against an +/// already built partner registry. +/// +/// Enabled providers with an unregistered kind are skipped; building the +/// registry with [`partner_registry`] already warned about them. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] when a provider cannot be built or fails +/// the coordinator's configuration checks. +pub fn build_coordinator_with( + settings: &Settings, + partners: Arc, +) -> Result, Report> { + let registered = settings .ec .identity_providers .iter() - .filter(|config| config.enabled) - .filter(|config| { - let registered = is_registered_kind(config.kind()); - if !registered { - log::warn!( - "Identity provider '{}' skipped: kind '{}' is not registered", - config.id, - config.kind() - ); - } - registered - }) - .collect(); - if enabled.is_empty() { - return Ok(None); - } - let registered = enabled - .into_iter() + .filter(|config| config.enabled && is_registered_kind(config.kind())) .map(|config| { build(config, &config.secrets) .map(|provider| RegisteredProvider::new(config.clone(), provider)) }) .collect::, Report>>()?; - let partners = partner_registry(&settings.ec) - .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + if registered.is_empty() { + return Ok(None); + } let budget = Duration::from_millis(settings.ec.identity_budget_ms); - ProviderCoordinator::new(registered, Arc::new(partners), budget).map(Some) + ProviderCoordinator::new(registered, partners, budget).map(Some) } #[cfg(test)] From 54079ae99c6c33e472e6fa112f234dce173977db Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:42:52 +0530 Subject: [PATCH 22/38] Remove EID resolution, create-if-absent and proxy inputs left unused Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-axum/src/app.rs | 3 - .../src/app.rs | 3 - .../trusted-server-adapter-fastly/src/app.rs | 2 - crates/trusted-server-adapter-spin/src/app.rs | 3 - crates/trusted-server-core/src/ec/eids.rs | 197 +++--------------- crates/trusted-server-core/src/ec/kv.rs | 54 ----- .../src/integrations/registry.rs | 30 +-- 7 files changed, 29 insertions(+), 263 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 314580e79a..be6589a103 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -224,15 +224,12 @@ async fn dispatch_fallback( } if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); return state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services, req, }) diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 54029b36c7..b0c9599e34 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -535,15 +535,12 @@ fn build_router(state: &Arc) -> RouterService { EdgeCacheHeader::CloudflareCdnCacheControl, ) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, }) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index f9187ab0d5..675a1fc898 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -959,8 +959,6 @@ async fn dispatch_fallback( method: &method, path: &path, settings: &state.settings, - kv: ec.kv_graph.as_ref(), - ec_context: &mut ec.ec_context, services, req, }) diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index f5fec2efc7..9a35062fc7 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -817,15 +817,12 @@ fn build_router(state: &Arc) -> RouterService { let result = if method == Method::GET && path.starts_with("/static/tsjs=") { handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SMaxageFallback) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, }) diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 904594c934..8bd39614a8 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -1,8 +1,8 @@ //! Shared EID resolution and formatting helpers. //! -//! Used by both `/_ts/api/v1/identify` and `/auction` to resolve source-domain -//! keyed IDs from KV entries, convert them to `OpenRTB` EID structures, and -//! build base64-encoded response headers. +//! Used by both `/_ts/api/v1/identify` and `/auction` to view source-domain +//! keyed IDs from KV entries as `OpenRTB` EID structures, and to build +//! base64-encoded response headers. use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use error_stack::{Report, ResultExt as _}; @@ -17,67 +17,6 @@ use super::registry::PartnerRegistry; /// Maximum size (in bytes) for the base64-encoded `x-ts-eids` header value. pub const MAX_EIDS_HEADER_BYTES: usize = 4096; -/// A source-domain keyed ID resolved from a KV entry against the partner registry. -/// -/// Only includes partners with `bidstream_enabled = true` and a non-empty UID. -pub struct ResolvedPartnerId { - /// The partner's identity source domain and EC KV `ids` key. - pub source_domain: String, - /// The synced user ID value. - pub uid: String, - /// `OpenRTB` agent type for this partner's identifiers. - pub openrtb_atype: i32, -} - -/// Resolves source-domain keyed IDs from a KV entry against the partner registry. -/// -/// Filters to partners with `bidstream_enabled = true` and non-empty UIDs, -/// sorted deterministically by source domain. -#[must_use] -pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec { - let mut resolved = Vec::new(); - - for (source_domain, partner_uid) in &entry.ids { - if partner_uid.uid.is_empty() { - continue; - } - - let Some(partner) = registry.get(source_domain) else { - continue; - }; - if !partner.bidstream_enabled { - continue; - } - - resolved.push(ResolvedPartnerId { - source_domain: partner.source_domain.clone(), - uid: partner_uid.uid.clone(), - openrtb_atype: partner.openrtb_atype, - }); - } - - resolved.sort_by(|a, b| a.source_domain.cmp(&b.source_domain)); - resolved -} - -/// Converts resolved partner IDs to `OpenRTB` `Eid` entries. -#[must_use] -pub fn to_eids(resolved: &[ResolvedPartnerId]) -> Vec { - resolved - .iter() - .map(|item| { - Eid::new( - item.source_domain.clone(), - vec![Uid { - id: item.uid.clone(), - atype: Some(item.openrtb_atype), - ext: None, - }], - ) - }) - .collect() -} - /// A borrowed EID entry: a single legacy UID or a full-fidelity provider entry. #[derive(Debug, Clone, Copy)] pub enum EidView<'row> { @@ -179,28 +118,11 @@ impl KvEntry { } } -/// Builds a base64-encoded EID header value, truncating if needed. -/// -/// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds -/// [`MAX_EIDS_HEADER_BYTES`] after base64 encoding, partners are removed -/// from the end of the deterministic partner ordering until it fits. -/// -/// # Errors -/// -/// Returns an error if JSON serialization fails. -pub fn build_eids_header( - resolved: &[ResolvedPartnerId], -) -> Result<(String, bool), Report> { - let eids = to_eids(resolved); - encode_eids_header(&eids) -} - /// Encodes a pre-built EID slice into a base64 header value with truncation. /// -/// Like [`build_eids_header`] but operates on already-constructed `Eid` values -/// (e.g., from `UserInfo.eids` in the auction response path). -/// -/// Returns `(encoded_value, was_truncated)`. +/// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds +/// [`MAX_EIDS_HEADER_BYTES`] after base64 encoding, EIDs are removed from the +/// end until it fits. /// /// # Errors /// @@ -273,89 +195,30 @@ mod tests { } } - #[test] - fn resolve_partner_ids_sorts_by_source_domain() { - let partners = vec![ - make_test_partner("zeta.example.com"), - make_test_partner("alpha.example.com"), - ]; - let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); - - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - entry.ids.insert( - "zeta.example.com".to_owned(), - super::super::kv_types::KvPartnerId::single("uid-z".to_owned()), - ); - entry.ids.insert( - "alpha.example.com".to_owned(), - super::super::kv_types::KvPartnerId::single("uid-a".to_owned()), - ); - - let resolved = resolve_partner_ids(®istry, &entry); - let source_domains: Vec<&str> = resolved - .iter() - .map(|item| item.source_domain.as_str()) - .collect(); - - assert_eq!( - source_domains, - vec!["alpha.example.com", "zeta.example.com"], - "should sort deterministically by source domain" - ); + fn header_eid(source_domain: String, uid: String) -> Eid { + Eid::new( + source_domain, + vec![Uid { + id: uid, + atype: Some(3), + ext: None, + }], + ) } #[test] - fn to_eids_maps_resolved_ids_correctly() { - let resolved = vec![ - ResolvedPartnerId { - uid: "LR_xyz".to_owned(), - source_domain: "liveramp.com".to_owned(), - openrtb_atype: 3, - }, - ResolvedPartnerId { - uid: "ID5_abc".to_owned(), - source_domain: "id5-sync.com".to_owned(), - openrtb_atype: 1, - }, - ResolvedPartnerId { - uid: "pair-id".to_owned(), - source_domain: "google.com".to_owned(), - openrtb_atype: 571187, - }, - ]; - - let eids = to_eids(&resolved); - - assert_eq!(eids.len(), 3, "should produce one EID per resolved partner"); - assert_eq!(eids[0].source, "liveramp.com"); - assert_eq!(eids[0].uids[0].id, "LR_xyz"); - assert_eq!(eids[0].uids[0].atype, Some(3)); - assert_eq!(eids[1].source, "id5-sync.com"); - assert_eq!(eids[1].uids[0].id, "ID5_abc"); - assert_eq!(eids[1].uids[0].atype, Some(1)); - assert_eq!(eids[2].source, "google.com", "should preserve PAIR source"); - assert_eq!(eids[2].uids[0].id, "pair-id", "should preserve PAIR ID"); - assert_eq!( - eids[2].uids[0].atype, - Some(571187), - "should preserve PAIR vendor-specific atype" - ); - } - - #[test] - fn build_eids_header_truncates_when_too_large() { - let mut resolved = Vec::new(); - for idx in 0..64 { - resolved.push(ResolvedPartnerId { - uid: format!("uid_{}", "x".repeat(100)), - source_domain: format!("partner-{idx}.example.com"), - openrtb_atype: 3, - }); - } + fn encode_eids_header_truncates_when_too_large() { + let eids: Vec = (0..64) + .map(|idx| { + header_eid( + format!("partner-{idx}.example.com"), + format!("uid_{}", "x".repeat(100)), + ) + }) + .collect(); let (encoded, truncated) = - build_eids_header(&resolved).expect("should build truncated header"); + encode_eids_header(&eids).expect("should build truncated header"); assert!(truncated, "should report truncation for large payload"); assert!( @@ -365,15 +228,11 @@ mod tests { } #[test] - fn build_eids_header_fits_without_truncation() { - let resolved = vec![ResolvedPartnerId { - uid: "u1".to_owned(), - source_domain: "ssp.com".to_owned(), - openrtb_atype: 3, - }]; + fn encode_eids_header_fits_without_truncation() { + let eids = vec![header_eid("ssp.example.com".to_owned(), "u1".to_owned())]; let (encoded, truncated) = - build_eids_header(&resolved).expect("should build header without truncation"); + encode_eids_header(&eids).expect("should build header without truncation"); assert!(!truncated, "should not truncate small payload"); assert!(!encoded.is_empty(), "should produce non-empty value"); diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 9a3f9492ee..7d074d1145 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -62,15 +62,6 @@ pub enum UpsertResult { Unchanged, } -/// Outcome of atomically creating an identity-graph root when absent. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CreateIfAbsentOutcome { - /// The candidate entry was persisted. - Written, - /// A row already exists for the candidate key. - AlreadyExists, -} - /// Partner UID update to apply to a KV identity graph entry. #[derive(Debug, Clone, PartialEq, Eq)] pub struct PartnerIdUpdate { @@ -341,23 +332,6 @@ impl KvIdentityGraph { } } - /// Atomically creates an entry while preserving collision as normal control flow. - /// - /// # Errors - /// - /// Returns [`TrustedServerError::KvStore`] when serialization or store I/O fails. - pub fn create_if_absent( - &self, - ec_id: &str, - entry: &KvEntry, - ) -> Result> { - let (body, meta_str) = Self::serialize_entry(entry, self.store_name())?; - match self.write_entry(ec_id, &body, &meta_str, ENTRY_TTL, EcKvWriteMode::Add)? { - EcKvWriteOutcome::Written => Ok(CreateIfAbsentOutcome::Written), - EcKvWriteOutcome::PreconditionFailed => Ok(CreateIfAbsentOutcome::AlreadyExists), - } - } - /// Low-level write with shared error context. fn write_entry( &self, @@ -1746,34 +1720,6 @@ mod tests { ); } - #[test] - fn create_if_absent_reports_written_and_collision() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should create absent entry"), - CreateIfAbsentOutcome::Written - ); - assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should report collision"), - CreateIfAbsentOutcome::AlreadyExists - ); - } - - #[test] - fn create_if_absent_propagates_store_error() { - let kv = KvIdentityGraph::failing("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - assert!( - kv.create_if_absent(&ec_id, &live_entry()).is_err(), - "should preserve store failures instead of reporting a collision" - ); - } - #[test] fn create_or_revive_revives_tombstone() { let kv = KvIdentityGraph::in_memory("test_store"); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index fc40711240..0460984238 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -10,8 +10,6 @@ use matchit::Router; use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; -use crate::ec::EcContext; -use crate::ec::kv::KvIdentityGraph; use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::platform::RuntimeServices; @@ -785,15 +783,11 @@ impl IntegrationMetadata { /// Inputs to [`IntegrationRegistry::handle_proxy`]. /// -/// Bundled into a struct so the dispatch surface stays within the project's -/// 7-argument cap; `ec_context` and `req` participate in the borrow so the -/// whole thing shares one lifetime. +/// Bundled into a struct so the dispatch surface shares one lifetime. pub struct ProxyDispatchInput<'a> { pub method: &'a Method, pub path: &'a str, pub settings: &'a Settings, - pub kv: Option<&'a KvIdentityGraph>, - pub ec_context: &'a mut EcContext, pub services: &'a RuntimeServices, pub req: Request, } @@ -1028,8 +1022,6 @@ impl IntegrationRegistry { method, path, settings, - kv: _, - ec_context: _, services, mut req, } = input; @@ -1825,14 +1817,10 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { method: &http::Method::GET, path: "/integrations/test/echo", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &noop_services(), req, })) @@ -2179,8 +2167,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("some-ec-value"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = noop_services(); // Call handle_proxy (uses futures executor in test environment) @@ -2188,8 +2174,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })); @@ -2229,8 +2213,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("evil;injected"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2238,8 +2220,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })) @@ -2278,8 +2258,6 @@ mod tests { )) .expect("should build Cookie header"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2287,8 +2265,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })) @@ -2324,8 +2300,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("some-ec-value"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2333,8 +2307,6 @@ mod tests { method: &Method::POST, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })); From a987581b9e7defec4c3383b304bc6b7dc5fbf19f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:44:03 +0530 Subject: [PATCH 23/38] Track the row size incrementally while enforcing the size cap Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/write_plan.rs | 76 ++++++++++++++----- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 60c60eb009..5046bf82da 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -170,37 +170,73 @@ fn serialized_len(entry: &KvEntry) -> usize { serde_json::to_vec(entry).map_or(usize::MAX, |bytes| bytes.len()) } +/// Serialized bytes one `ids` member adds to the row: its record, its +/// escaped key, the colon and a separating comma. +fn member_len(source: &str, record: &KvPartnerId) -> usize { + let key = serde_json::to_vec(source).map_or(usize::MAX, |bytes| bytes.len()); + let value = serde_json::to_vec(record).map_or(usize::MAX, |bytes| bytes.len()); + key.saturating_add(value).saturating_add(2) +} + +/// Shrinks `entry` until it serializes within [`MAX_ENTRY_BYTES`]. +/// +/// # Performance +/// +/// Tracks the row size by the bytes each drop removes and serializes the +/// whole row again only when that estimate reaches the cap, so the cost is +/// linear in the number of dropped UIDs and records. fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { - while serialized_len(entry) > MAX_ENTRY_BYTES { - if let Some(source) = written.iter().rev().find(|source| { + let mut size = serialized_len(entry); + // Drop this navigation's provider UIDs first, newest source last-in first. + while size > MAX_ENTRY_BYTES { + let Some(source) = written.iter().rev().find(|source| { entry .ids .get(source.as_str()) .is_some_and(KvPartnerId::is_provider_written) - }) { - let record = entry - .ids - .get_mut(source.as_str()) - .expect("should hold a written source"); - record.drop_last_uid(); - if record.uid.is_empty() { - entry.ids.remove(source.as_str()); - } - continue; - } - // No provider UIDs from this navigation are left: drop whole records, - // largest first, so every later write still fits the cap. - let Some(largest) = entry + }) else { + break; + }; + let record = entry .ids - .iter() - .max_by_key(|(_, record)| serde_json::to_vec(record).map_or(usize::MAX, |b| b.len())) - .map(|(source, _)| source.clone()) - else { + .get_mut(source.as_str()) + .expect("should hold a written source"); + let before = member_len(source.as_str(), record); + record.drop_last_uid(); + let after = if record.uid.is_empty() { + entry.ids.remove(source.as_str()); + 0 + } else { + member_len(source.as_str(), record) + }; + size = size.saturating_sub(before.saturating_sub(after)); + if size <= MAX_ENTRY_BYTES { + size = serialized_len(entry); + } + } + if size <= MAX_ENTRY_BYTES { + return; + } + // No provider UIDs from this navigation are left: drop whole records, + // largest first, so every later write still fits the cap. + let mut by_size: Vec<(usize, String)> = entry + .ids + .iter() + .map(|(source, record)| (member_len(source, record), source.clone())) + .collect(); + // A stable ascending sort pops the last of equally large records first. + by_size.sort_by_key(|(len, _)| *len); + while size > MAX_ENTRY_BYTES { + let Some((len, largest)) = by_size.pop() else { log::warn!("EC write planner: row exceeds size cap with no partner IDs left to drop"); return; }; log::warn!("EC write planner: dropping '{largest}' to fit the row size cap"); entry.ids.remove(&largest); + size = size.saturating_sub(len); + if size <= MAX_ENTRY_BYTES { + size = serialized_len(entry); + } } } From e681663a8d2d2579fa70231f66f59b6a42a2c1c8 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:45:11 +0530 Subject: [PATCH 24/38] Reuse the core clock for lookups and use should-style expect messages Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-fastly/src/app.rs | 4 +--- crates/trusted-server-core/src/ec/finalize.rs | 12 ++++++------ crates/trusted-server-core/src/ec/kv.rs | 6 +++--- crates/trusted-server-core/src/ec/mod.rs | 3 ++- crates/trusted-server-core/src/ec/pull_sync.rs | 2 +- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 675a1fc898..aef63f6009 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -446,9 +446,7 @@ async fn dispatch_identity_lookups( .headers() .get(header::USER_AGENT) .and_then(|value| value.to_str().ok()); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_secs()); + let now = trusted_server_core::ec::current_timestamp(); let pending = coordinator .dispatch( &transport, diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 1fc5c1a074..41105abe24 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1290,7 +1290,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("sharedid.org"), "a subresource response must not persist request EID cookies" @@ -1348,7 +1348,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), permit.then_some(cookie_id), @@ -1438,7 +1438,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( stored.ids.is_empty(), "an auction request must leave the row unchanged" @@ -1513,7 +1513,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( stored.ids.is_empty(), "should persist neither the auction body nor the cookie EIDs" @@ -1575,7 +1575,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("id5-sync.com"), "denying TCF Purpose 4 must keep the EID out of KV even though Purpose 1 \ @@ -1640,7 +1640,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("id5-sync.com"), "should keep the provider-owned fallback out of KV without Purpose 4" diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 7d074d1145..2adcba566f 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -1931,7 +1931,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read root after rejected hard delete") - .expect("rejected hard delete should retain root"); + .expect("should retain the root after a rejected hard delete"); assert!( !stored.consent.ok, "rejected hard delete should leave the tombstoned root in place" @@ -2520,7 +2520,7 @@ mod tests { let (stored, _) = kv .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert_eq!( stored.ids.get("ssp_x").map(|id| id.uid.as_str()), Some("uid-1"), @@ -2597,7 +2597,7 @@ mod tests { let (stored, _) = kv .get(&ec_id) .expect("should read store") - .expect("tombstone should remain"); + .expect("should keep the tombstone"); assert!(stored.ids.is_empty(), "no update should reach the store"); } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 2a832aebdd..f90940d5be 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -780,7 +780,8 @@ impl EcContext { /// native and `wasm32-wasip1` targets and to a JS-backed clock on /// `wasm32-unknown-unknown` (Cloudflare Workers), where `std::time` is not /// available. -pub(crate) fn current_timestamp() -> u64 { +#[must_use] +pub fn current_timestamp() -> u64 { checked_current_timestamp().unwrap_or(0) } diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 24993c18f1..4fac813dd5 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -1186,7 +1186,7 @@ mod tests { let (entry, _) = graph .get(&ec_id) .expect("should read store") - .expect("tombstone should remain"); + .expect("should keep the tombstone"); assert!( entry.ids.is_empty(), "no partner UID may be written back onto a tombstoned row" From ef7ad61aada55347724a9c7e9097f9d5072a4b36 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 01:22:25 +0530 Subject: [PATCH 25/38] Collect provider IDs after the response and keep provider refresh state Providers set to collect = "after_response" look up after the page is sent and store their results with one compare-and-swap follow-up write (Fastly only). Providers may keep private refresh state with a UID, generate IDs, pass request context to their reply parser and report an opt-out. Refresh is due only while the UID is live. Unusable refresh state is dropped without dropping the UID. The admin EC view redacts refresh state. Operators can bound lookups per provider with rate_limit_per_hour. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 121 ++++-- .../trusted-server-adapter-fastly/src/main.rs | 102 ++++- crates/trusted-server-core/src/ec/admin.rs | 47 +++ .../src/ec/after_response.rs | 369 ++++++++++++++++++ .../src/ec/coordinator/mod.rs | 292 +++++++++++++- .../src/ec/coordinator/results.rs | 7 + .../src/ec/coordinator/transport.rs | 2 + .../src/ec/coordinator/validate.rs | 91 ++++- crates/trusted-server-core/src/ec/finalize.rs | 133 ++++++- crates/trusted-server-core/src/ec/kv.rs | 64 ++- crates/trusted-server-core/src/ec/kv_types.rs | 87 +++++ crates/trusted-server-core/src/ec/mod.rs | 32 +- .../trusted-server-core/src/ec/page_load.rs | 43 +- crates/trusted-server-core/src/ec/provider.rs | 45 +++ .../trusted-server-core/src/ec/write_plan.rs | 50 ++- crates/trusted-server-core/src/settings.rs | 37 +- .../src/lib.rs | 109 +++++- 17 files changed, 1517 insertions(+), 114 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/after_response.rs diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index aef63f6009..5364bfc182 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -148,7 +148,7 @@ use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, handle_verify_signature, }; -use trusted_server_core::settings::{ProxyAssetRoute, Settings}; +use trusted_server_core::settings::{CollectTiming, ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; @@ -368,31 +368,97 @@ pub(crate) struct EcFinalizeState { } /// Identity lookups dispatched for this navigation, collected before finalize. -/// -/// In-flight platform requests are not `Send`, so they cannot ride in -/// response extensions with [`EcFinalizeState`]. They travel through a -/// request-local slot instead: the entry point clears it before routing and -/// takes it after the router returns. pub(crate) struct IdentityLookups { pub(crate) coordinator: ProviderCoordinator, pub(crate) pending: PendingLookups, } -thread_local! { - static PENDING_IDENTITY_LOOKUPS: RefCell> = const { RefCell::new(None) }; +/// What a navigation's after-response identity lookups need once the response +/// has been sent. +/// +/// The router owns the request, so the request signals the providers read are +/// copied here. The coordinator is built and the lookups dispatched only after +/// the response, so neither delays the page. +pub(crate) struct AfterResponseLookups { + pub(crate) ec_id: String, + pub(crate) client_ip: Option, + pub(crate) user_agent: Option, +} + +/// Identity work a navigation hands to the entry point. +/// +/// In-flight platform requests are not `Send`, so they cannot ride in +/// response extensions with [`EcFinalizeState`]. They travel through one +/// request-local slot instead: the entry point clears it before routing and +/// takes it after the router returns. +#[derive(Default)] +pub(crate) struct NavigationLookups { + /// Page-phase lookups, collected before EC finalization. + pub(crate) page: Option, + /// After-response lookups, dispatched once the response has been sent. + pub(crate) after_response: Option, } -/// Stores this request's dispatched identity lookups for the entry point. -fn stash_identity_lookups(lookups: IdentityLookups) { - PENDING_IDENTITY_LOOKUPS.with(|slot| *slot.borrow_mut() = Some(lookups)); +thread_local! { + static NAVIGATION_LOOKUPS: RefCell = RefCell::new(NavigationLookups::default()); } -/// Takes the identity lookups dispatched for the current request, if any. +/// Takes the identity work stored for the current request. /// /// The entry point calls this before routing (discarding anything a prior /// request left behind) and after the router returns. -pub(crate) fn take_identity_lookups() -> Option { - PENDING_IDENTITY_LOOKUPS.with(|slot| slot.borrow_mut().take()) +pub(crate) fn take_navigation_lookups() -> NavigationLookups { + NAVIGATION_LOOKUPS.with(|slot| std::mem::take(&mut *slot.borrow_mut())) +} + +/// Request signals lent to identity providers. +struct ProviderRequestSignals<'a> { + client_ip: Option, + user_agent: Option<&'a str>, +} + +impl<'a> ProviderRequestSignals<'a> { + fn read(req: &'a Request, ec: &EcRequestState) -> Self { + Self { + client_ip: ec + .ec_context + .client_ip() + .and_then(|ip| ip.parse::().ok()), + user_agent: req + .headers() + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()), + } + } +} + +/// Captures the request signals for after-response identity lookups on a +/// permitted publisher navigation. +/// +/// Returns `None` when no after-response provider is enabled, the request has +/// no EC, or the identity graph is unavailable. +fn prepare_after_response_lookups( + state: &AppState, + req: &Request, + ec: &EcRequestState, +) -> Option { + let has_after_response_provider = state + .settings + .ec + .identity_providers + .iter() + .any(|config| config.enabled && config.collect == CollectTiming::AfterResponse); + if !has_after_response_provider { + return None; + } + ec.kv_graph.as_ref()?; + let ec_id = ec.ec_context.ec_value()?.to_owned(); + let signals = ProviderRequestSignals::read(req, ec); + Some(AfterResponseLookups { + ec_id, + client_ip: signals.client_ip, + user_agent: signals.user_agent.map(str::to_owned), + }) } /// Starts identity-provider lookups for a permitted publisher navigation. @@ -438,14 +504,7 @@ async fn dispatch_identity_lookups( } let transport = PlatformLookupTransport::new(services, state.settings.proxy.certificate_check); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - let client_ip = ec - .ec_context - .client_ip() - .and_then(|ip| ip.parse::().ok()); - let user_agent = req - .headers() - .get(header::USER_AGENT) - .and_then(|value| value.to_str().ok()); + let signals = ProviderRequestSignals::read(req, ec); let now = trusted_server_core::ec::current_timestamp(); let pending = coordinator .dispatch( @@ -455,8 +514,8 @@ async fn dispatch_identity_lookups( snapshot: ec.ec_context.kv_snapshot(), consent: ec.ec_context.consent(), client: ClientSignals { - ip: client_ip, - user_agent, + ip: signals.client_ip, + user_agent: signals.user_agent, }, publisher_domain: &state.settings.publisher.domain, rate_limiter: &limiter, @@ -1011,7 +1070,15 @@ async fn dispatch_fallback( && has_permit && let Some(lookups) = dispatch_identity_lookups(state, services, &req, &mut ec).await { - stash_identity_lookups(lookups); + NAVIGATION_LOOKUPS.with(|slot| slot.borrow_mut().page = Some(lookups)); + } + // After-response providers only capture request signals here; they + // are built and dispatched once the response has been sent. + if is_publisher_navigation + && has_permit + && let Some(lookups) = prepare_after_response_lookups(state, &req, &ec) + { + NAVIGATION_LOOKUPS.with(|slot| slot.borrow_mut().after_response = Some(lookups)); } // Run the server-side auction with the configured creative- @@ -2850,7 +2917,7 @@ mod tests { let _state = take_finalize_state(&mut response); assert!( - super::take_identity_lookups().is_none(), + super::take_navigation_lookups().page.is_none(), "should dispatch no identity lookups without enabled providers" ); } @@ -2922,6 +2989,7 @@ mod tests { url: url::Url::parse("https://ids.example.com/lookup").expect("should parse"), headers: edgezero_core::http::HeaderMap::new(), body: Bytes::new(), + context: None, }) } @@ -2942,6 +3010,7 @@ mod tests { atype: None, ext: None, expiry: ProviderExpiry::Unspecified, + refresh: None, }], }])) } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 7c8340bdb8..6a59e836fd 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -13,12 +13,16 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; +use trusted_server_core::ec::after_response::commit_after_response_lookups; use trusted_server_core::ec::coordinator::IdentityCoordinator as _; +use trusted_server_core::ec::coordinator::LookupContext; use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; use trusted_server_core::ec::device::DeviceSignals; +use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; +use trusted_server_core::ec::provider::ClientSignals; use trusted_server_core::ec::pull_sync::{ PullSyncContext, build_pull_sync_context, dispatch_pull_sync, }; @@ -31,7 +35,7 @@ use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; use trusted_server_core::trace::{TraceTerminalResponse, is_trace_path}; use trusted_server_core::version_header::{apply_git_version_header, git_version_header_value}; -use trusted_server_identity_providers::partner_registry; +use trusted_server_identity_providers::{build_after_response_coordinator, partner_registry}; mod app; mod backend; @@ -48,8 +52,8 @@ mod template_cache; mod tinybird; use crate::app::{ - AppState, EcFinalizeState, IdentityLookups, RuntimeStoreConfig, TrustedServerApp, - load_settings_from_config_store, take_identity_lookups, + AfterResponseLookups, AppState, EcFinalizeState, IdentityLookups, NavigationLookups, + RuntimeStoreConfig, TrustedServerApp, load_settings_from_config_store, take_navigation_lookups, }; use crate::ec_kv::FastlyEcKvStore; use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; @@ -496,7 +500,7 @@ fn edgezero_main( } core_req.extensions_mut().insert(client_info); // Drop lookups a previous request on this sandbox left behind. - drop(take_identity_lookups()); + drop(take_navigation_lookups()); match futures::executor::block_on(app.router().oneshot(core_req)) { Ok(response) => response, Err(error) => edge_error_response(error), @@ -525,7 +529,10 @@ fn edgezero_main( let ec_state = response.extensions_mut().remove::(); // Identity lookups dispatched on a publisher navigation are collected in // EC finalization, after the router's executor has returned. - let identity = take_identity_lookups(); + let NavigationLookups { + page: identity, + after_response, + } = take_navigation_lookups(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); @@ -567,6 +574,12 @@ fn edgezero_main( counters.as_ref(), ); run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); + run_after_response_lookups( + settings, + &partner_registry, + &mut ec_state, + after_response, + ); return; } Err(e) => { @@ -596,6 +609,12 @@ fn edgezero_main( &partner_registry, &ec_state, ); + run_after_response_lookups( + &settings, + &partner_registry, + &mut ec_state, + after_response, + ); return; } Err(e) => { @@ -735,6 +754,79 @@ fn run_edgezero_pull_sync_after_send( ); } +/// Runs after-response identity lookups and applies their follow-up write. +/// +/// Called once the response has been sent, so provider latency never delays +/// the page. Skipped unless finalize confirmed the navigation (it issues the +/// follow-up permit) and the EC is still the one the lookups were prepared +/// for. +fn run_after_response_lookups( + settings: &Settings, + partner_registry: &PartnerRegistry, + ec_state: &mut EcFinalizeState, + lookups: Option, +) { + let Some(lookups) = lookups else { + return; + }; + let Some(permit) = ec_state.ec_context.take_after_response_permit() else { + log::debug!( + "After-response lookups skipped: no permit (consent, redirect or failed page-load write)" + ); + return; + }; + if ec_state.ec_context.ec_value() != Some(lookups.ec_id.as_str()) { + log::debug!("After-response lookups skipped: EC changed during the request"); + return; + } + let coordinator = match build_after_response_coordinator(settings) { + Ok(Some(coordinator)) => coordinator, + Ok(None) => return, + Err(report) => { + log::error!("After-response identity providers disabled for this request: {report:?}"); + return; + } + }; + let Some(graph) = maybe_identity_graph(settings) else { + return; + }; + let snapshot = graph.load_snapshot(&lookups.ec_id); + let transport = + PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()); + let pending = futures::executor::block_on(coordinator.dispatch( + &transport, + &LookupContext { + ec_id: &lookups.ec_id, + snapshot: &snapshot, + consent: ec_state.ec_context.consent(), + client: ClientSignals { + ip: lookups.client_ip, + user_agent: lookups.user_agent.as_deref(), + }, + publisher_domain: &settings.publisher.domain, + rate_limiter: &limiter, + now: UnixSeconds::new(now), + }, + )); + if pending.is_empty() { + return; + } + let results = futures::executor::block_on(coordinator.collect(&transport, pending)); + let outcome = commit_after_response_lookups( + &graph, + permit, + partner_registry, + &lookups.ec_id, + &snapshot, + &results, + ); + log::info!("After-response lookups: write outcome={outcome}"); +} + fn prepare_pull_sync_after_send( context: Option, graph_factory: F, diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 7014721f0e..a0d8151a36 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -395,6 +395,7 @@ fn build_lookup_response( match serde_json::from_slice::(&lookup.body) { Ok(mut entry_json) => { add_iso_timestamp_companions(&mut entry_json); + redact_refresh_state(&mut entry_json); payload.entry = Some(entry_json); match serde_json::from_slice::(&lookup.body) { @@ -471,6 +472,30 @@ fn add_iso_timestamp_companions(entry_json: &mut JsonValue) { } } +/// Placeholder served in place of a provider's private refresh state. +const REDACTED: &str = "[redacted]"; + +/// Replaces every provider refresh `state` in the entry with [`REDACTED`]. +/// +/// Refresh state can hold a provider credential (for example a refresh +/// token), so the admin view shows only that it exists and when it is due. +fn redact_refresh_state(value: &mut JsonValue) { + match value { + JsonValue::Object(object) => { + if let Some(state) = object + .get_mut("refresh") + .and_then(JsonValue::as_object_mut) + .and_then(|refresh| refresh.get_mut("state")) + { + *state = JsonValue::String(REDACTED.to_owned()); + } + object.values_mut().for_each(redact_refresh_state); + } + JsonValue::Array(items) => items.iter_mut().for_each(redact_refresh_state), + _ => {} + } +} + /// Formats a unix-seconds timestamp as ISO 8601 (`yyyy-MM-ddTHH:mm:ss.SSSZ`). /// /// Returns `None` for values outside the representable date range. @@ -1585,4 +1610,26 @@ mod tests { assert_eq!(view.skipped[0].source_domain, "expired.example"); assert_eq!(view.skipped[0].reason, "expired", "should report expiry"); } + + #[test] + fn redacts_provider_refresh_state() { + let mut entry = serde_json::json!({ + "ids": {"token.example": {"eids": [{"source": "token.example", "uids": [ + {"id": "token", "refresh": {"state": "secret-refresh", "from": 1_000}} + ]}]}} + }); + + redact_refresh_state(&mut entry); + + let refresh = &entry["ids"]["token.example"]["eids"][0]["uids"][0]["refresh"]; + assert_eq!(refresh["state"], REDACTED, "should hide the refresh state"); + assert_eq!( + refresh["from"], 1_000, + "should keep when the refresh is due" + ); + assert_eq!( + entry["ids"]["token.example"]["eids"][0]["uids"][0]["id"], "token", + "should keep the UID" + ); + } } diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs new file mode 100644 index 0000000000..c6ecba1031 --- /dev/null +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -0,0 +1,369 @@ +//! The follow-up write of after-response identity lookups. +//! +//! Providers configured with `collect = "after_response"` are dispatched only +//! after the navigation's response has been sent, so their latency never +//! delays the page. Their validated results reach the row in one follow-up +//! update, guarded by the row's current generation, and are served from the +//! next page load. Browser ingest never runs here: it belongs to the +//! navigation's own write. + +use super::coordinator::results::LookupResults; +use super::eid_types::UnixSeconds; +use super::kv::KvIdentityGraph; +use super::page_load::{AfterResponseWrite, RowChange}; +use super::registry::PartnerRegistry; +use super::write_plan::{WritePlanInput, plan_write}; +use super::{EcKvSnapshot, current_timestamp, log_id}; + +/// What the follow-up write did. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum AfterResponseOutcome { + /// The row was updated with the after-response results. + #[display("written")] + Written, + /// The results would not change the row. + #[display("unchanged")] + Unchanged, + /// The row is missing, withdrawn or unreadable, so nothing was written. + #[display("no_row")] + NoRow, + /// The row changed since it was read; the results are dropped. + #[display("conflict")] + Conflict, + /// The store rejected the write. + #[display("failed")] + Failed, +} + +/// Applies after-response lookup results to the row of `ec_id`. +/// +/// Plans an update from `lookups` alone against `snapshot`, the row read +/// after the response was sent, and writes it once, guarded by the snapshot's +/// generation. The `permit` is consumed whether or not a write is +/// made. +/// +/// # Performance +/// +/// At most one conditional KV write; the row is read by the caller. +/// Runs after the response has been sent, so it adds no page latency. +#[must_use] +pub fn commit_after_response_lookups( + graph: &KvIdentityGraph, + permit: AfterResponseWrite, + registry: &PartnerRegistry, + ec_id: &str, + snapshot: &EcKvSnapshot, + lookups: &LookupResults, +) -> AfterResponseOutcome { + if lookups.is_empty() { + return AfterResponseOutcome::Unchanged; + } + if !matches!(snapshot, EcKvSnapshot::Present { .. }) { + log::info!( + "After-response lookups: no row for '{}', skipping write", + log_id(ec_id) + ); + return AfterResponseOutcome::NoRow; + } + let change = plan_write(&WritePlanInput { + ec_id, + snapshot, + new_entry: None, + browser_ingest: &[], + lookups, + registry, + now: UnixSeconds::new(current_timestamp()), + }); + let Some(change) = change else { + // A withdrawn row also plans no change. + return match snapshot.entry_for(ec_id) { + Some(entry) if entry.consent.ok => AfterResponseOutcome::Unchanged, + _ => AfterResponseOutcome::NoRow, + }; + }; + let RowChange::Update { entry, expected } = change else { + log::warn!( + "After-response lookups: unexpected row change for '{}', skipping write", + log_id(ec_id) + ); + return AfterResponseOutcome::NoRow; + }; + match graph.commit_after_response(permit, ec_id, &entry, expected) { + Ok(()) => AfterResponseOutcome::Written, + Err(report) => { + log::warn!( + "After-response write failed for '{}': {}", + log_id(ec_id), + report.current_context() + ); + match report.current_context() { + super::page_load::KvWriteError::Conflict => AfterResponseOutcome::Conflict, + _ => AfterResponseOutcome::Failed, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; + use crate::settings::EcPartner; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn registry() -> PartnerRegistry { + let partners: Vec = ["browser.example.com", "ids.example.com"] + .iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, + "source_domain": source, + "bidstream_enabled": true, + })) + .expect("should build partner") + }) + .collect(); + let mut registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + assert!( + registry.mark_provider_owned("ids.example.com"), + "should mark the provider source as owned" + ); + registry + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + fn found(ec_id: &str, id: &str) -> LookupResults { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + LookupResults::new( + ec_id, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from("ids.example.com").expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + }], + ) + } + + fn stored_uid(graph: &KvIdentityGraph, source: &str) -> Option { + graph + .load_snapshot(EC_ID) + .entry_for(EC_ID) + .and_then(|entry| entry.ids.get(source).map(|record| record.uid.clone())) + } + + #[test] + fn writes_after_response_results_to_an_existing_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + let mut entry = live_entry(); + entry.ids.insert( + "browser.example.com".to_owned(), + KvPartnerId::single("b-1".to_owned()), + ); + graph.create(EC_ID, &entry).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should write the row" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should store the provider UID" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-1"), + "should keep the browser UID written by the page load" + ); + } + + #[test] + fn replaces_a_browser_fallback_for_an_owned_source() { + let graph = KvIdentityGraph::in_memory("after-response"); + let mut entry = live_entry(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("from-browser".to_owned()), + ); + graph.create(EC_ID, &entry).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should write the row" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should replace the browser fallback with the provider UID" + ); + } + + #[test] + fn skips_the_write_when_the_row_already_holds_the_results() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let first = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + assert_eq!(first, AfterResponseOutcome::Written, "should write first"); + + let second = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + second, + AfterResponseOutcome::Unchanged, + "should not rewrite identical results" + ); + } + + #[test] + fn writes_nothing_without_a_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::NoRow, + "should not create a row" + ); + assert!( + graph.load_snapshot(EC_ID).entry_for(EC_ID).is_none(), + "should leave the row absent" + ); + } + + #[test] + fn writes_nothing_to_a_withdrawn_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph + .create(EC_ID, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::NoRow, + "should not revive a tombstone" + ); + assert!( + stored_uid(&graph, "ids.example.com").is_none(), + "should not store the provider UID on a tombstone" + ); + } + + #[test] + fn drops_results_made_for_another_ec() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let other = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ABC123"; + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(other, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should discard results dispatched for another EC" + ); + assert!( + stored_uid(&graph, "ids.example.com").is_none(), + "should not store results for another EC" + ); + } + + #[test] + fn skips_the_write_without_results() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &LookupResults::new(EC_ID, Vec::new()), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should make no write without results" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 1dec09044c..40de27dc8b 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -29,7 +29,8 @@ use crate::ec::kv_types::KvEntry; use crate::ec::page_load::RowChange; use crate::ec::provider::{ ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, - ProviderResponse, SourceInput, destination_allowed, parse_sources, validate_enabled_configs, + ProviderEid, ProviderResponse, SourceInput, destination_allowed, parse_sources, + validate_enabled_configs, }; use crate::ec::rate_limiter::RateLimiter; use crate::ec::registry::PartnerRegistry; @@ -103,6 +104,10 @@ pub struct PendingLookups { now: UnixSeconds, /// Providers that finished at dispatch (skipped or failed to send). early: Vec>, + /// Providers whose IDs were generated at dispatch, validated. + generated: Vec>, + /// Provider state to hand back with each in-flight request's response. + contexts: Vec>, } impl Default for PendingLookups { @@ -113,6 +118,8 @@ impl Default for PendingLookups { ec_id: String::new(), now: UnixSeconds::new(0), early: Vec::new(), + generated: Vec::new(), + contexts: Vec::new(), } } } @@ -122,15 +129,16 @@ impl std::fmt::Debug for PendingLookups { f.debug_struct("PendingLookups") .field("requests", &self.requests.len()) .field("early", &self.early.len()) + .field("generated", &self.generated.len()) .finish_non_exhaustive() } } impl PendingLookups { - /// Whether nothing was dispatched or skipped. + /// Whether nothing was dispatched, generated or skipped. #[must_use] pub fn is_empty(&self) -> bool { - self.requests.is_empty() && self.early.is_empty() + self.requests.is_empty() && self.early.is_empty() && self.generated.is_empty() } } @@ -285,10 +293,16 @@ impl ProviderCoordinator

{ let mut input_blocked = false; let mut due = Vec::new(); for source in ®istered.policy.sources { - let held = entry + let record = entry .and_then(|entry| entry.ids.get(source.as_str())) - .is_some_and(|record| record.is_provider_written() && record.has_live_uid(now)); - if held { + .filter(|record| record.is_provider_written()); + // A UID whose provider asked for a refresh makes its source due; + // the stored refresh state is the lookup's input. + if record.is_some_and(|record| record.refresh_due(now)) { + due.push(source.clone()); + continue; + } + if record.is_some_and(|record| record.has_live_uid(now)) { continue; } let needs_input = capabilities @@ -331,10 +345,17 @@ impl ProviderCoordinator

{ status: LookupStatus, entries: ValidatedEntries, ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let cleared = if status == LookupStatus::Skipped(SkipReason::OptedOut) { + registered.policy.sources.to_vec() + } else { + Vec::new() + }; ProviderLookup { - provider: self.providers[provider.get()].id.clone(), + provider: registered.id.clone(), status, entries, + cleared, } } @@ -342,6 +363,7 @@ impl ProviderCoordinator

{ &self, provider: ProviderIndex, response: Result>, + context: Option<&str>, now: UnixSeconds, ) -> ProviderLookup { let registered = &self.providers[provider.get()]; @@ -378,12 +400,20 @@ impl ProviderCoordinator

{ status: response.status, headers: &response.headers, body: &response.body, + context, }); let eids = match parsed { Ok(LookupOutcome::Found(eids)) => eids, Ok(LookupOutcome::NoMatch) => { return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); } + Ok(LookupOutcome::OptedOut) => { + return self.lookup( + provider, + LookupStatus::Skipped(SkipReason::OptedOut), + Vec::new(), + ); + } Err(report) => { log::debug!( "Identity lookup: provider={} response not parsed: {}", @@ -393,6 +423,17 @@ impl ProviderCoordinator

{ return self.lookup(provider, LookupStatus::InvalidResponse, Vec::new()); } }; + self.validated(provider, eids, now) + } + + /// Validates a provider's entries and reports them as one lookup. + fn validated( + &self, + provider: ProviderIndex, + eids: Vec>, + now: UnixSeconds, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; let policy = ValidationPolicy { provider: ®istered.id, claimed: ®istered.policy.sources, @@ -467,10 +508,6 @@ impl IdentityCoordinator for ProviderCoordinator

{ })); continue; } - if Self::rate_limited(registered, lookup) { - pending.early.push(skip(SkipReason::RateLimited)); - continue; - } let input = LookupInput { ec_id: lookup.ec_id, consent: lookup.consent, @@ -480,7 +517,18 @@ impl IdentityCoordinator for ProviderCoordinator

{ current: OwnedEntries::new(entry, ®istered.policy.sources), hashed_email: None, }; - let request = match registered.provider.build_request(&input) { + if let Some(eids) = registered.provider.generate(&input) { + pending.generated.push(Tagged { + provider, + value: self.validated(provider, eids, lookup.now), + }); + continue; + } + if Self::rate_limited(registered, lookup) { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } + let mut request = match registered.provider.build_request(&input) { Ok(request) => request, Err(report) => { log::debug!( @@ -496,8 +544,14 @@ impl IdentityCoordinator for ProviderCoordinator

{ pending.early.push(skip(SkipReason::Destination)); continue; } + let context = request.context.take(); match transport.start(request, deadline).await { - Ok(value) => pending.requests.push(Tagged { provider, value }), + Ok(value) => { + pending.requests.push(Tagged { provider, value }); + if let Some(value) = context { + pending.contexts.push(Tagged { provider, value }); + } + } Err(report) => { log::warn!( "Identity lookup: provider={} send failed: {}", @@ -525,6 +579,8 @@ impl IdentityCoordinator for ProviderCoordinator

{ ec_id, now, early, + generated, + contexts, } = pending; let responses = match deadline { Some(deadline) if !requests.is_empty() => transport.gather(requests, deadline).await, @@ -535,8 +591,15 @@ impl IdentityCoordinator for ProviderCoordinator

{ for Tagged { provider, value } in early { slots[provider.get()] = Some(self.lookup(provider, value, Vec::new())); } + for Tagged { provider, value } in generated { + slots[provider.get()] = Some(value); + } for Tagged { provider, value } in responses { - slots[provider.get()] = Some(self.interpret(provider, value, now)); + let context = contexts + .iter() + .find(|tagged| tagged.provider == provider) + .map(|tagged| tagged.value.as_str()); + slots[provider.get()] = Some(self.interpret(provider, value, context, now)); } let lookups: Vec = slots.into_iter().flatten().collect(); for lookup in &lookups { @@ -569,7 +632,7 @@ mod tests { use crate::consent::types::TcfConsent; use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; use crate::ec::kv::KvIdentityGraph; - use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvRefresh, KvUid}; use crate::ec::provider::{ EidLevelFields, EidShape, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, ProviderUid, SourceCapability, @@ -584,6 +647,9 @@ mod tests { type Reply = fn() -> Result>; + /// Context the echo provider sets on its request and expects back. + const ECHO_CONTEXT: &str = "echo-context"; + struct EchoProvider(ProviderCapabilities); impl IdentityProvider for EchoProvider { @@ -604,6 +670,7 @@ mod tests { .expect("should parse"), headers: http::HeaderMap::new(), body: bytes::Bytes::new(), + context: Some(ECHO_CONTEXT.to_owned()), }) } @@ -611,8 +678,14 @@ mod tests { &self, response: ProviderResponse<'body>, ) -> Result, Report> { + if response.context != Some(ECHO_CONTEXT) { + return Err(Report::new(ProviderError::InvalidResponse)); + } let id = std::str::from_utf8(response.body) .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + if id == "optout" { + return Ok(LookupOutcome::OptedOut); + } Ok(LookupOutcome::Found(vec![ProviderEid { source: Cow::Borrowed("ids.example.com"), inserter: None, @@ -624,6 +697,7 @@ mod tests { atype: None, ext: None, expiry: ProviderExpiry::Unspecified, + refresh: None, }], }])) } @@ -767,8 +841,8 @@ mod tests { } } - fn run_with( - coordinator: &ProviderCoordinator, + fn run_with( + coordinator: &ProviderCoordinator

, transport: &CannedTransport, snapshot: &EcKvSnapshot, consent: &ConsentContext, @@ -792,8 +866,8 @@ mod tests { }) } - fn run( - coordinator: &ProviderCoordinator, + fn run( + coordinator: &ProviderCoordinator

, transport: &CannedTransport, snapshot: &EcKvSnapshot, consent: &ConsentContext, @@ -1206,4 +1280,184 @@ mod tests { "should report invalid config" ); } + + fn refreshable_record(from: u64) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from("p-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(5000)), + ) + .with_refresh(KvRefresh { + state: "refresh-state".to_owned(), + from: UnixSeconds::new(from), + }); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + fn snapshot_holding(record: KvPartnerId) -> EcKvSnapshot { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.ids.insert("ids.example.com".to_owned(), record); + seeded_snapshot(&entry) + } + + #[test] + fn a_due_refresh_makes_a_live_source_due() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should refresh a live UID whose refresh time has passed" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send the refresh"); + } + + #[test] + fn a_future_refresh_leaves_the_source_held() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(2000)), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should not refresh before the refresh time" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + fn optout_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"optout")) + } + + #[test] + fn an_opted_out_reply_clears_the_sources() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(optout_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should report the opt-out" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should clear the provider's sources" + ); + } + + /// An echo provider that generates its ID instead of looking it up. + struct GeneratingProvider(EchoProvider); + + impl IdentityProvider for GeneratingProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn generate(&self, _lookup: &LookupInput<'_>) -> Option>> { + Some(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed("generated-1"), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }]) + } + } + + fn generating_coordinator() -> ProviderCoordinator { + ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + GeneratingProvider(echo_provider(SourceInput::Anonymous)), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator") + } + + #[test] + fn a_generated_id_replaces_the_lookup_for_a_due_source() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep the generated UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0].id().as_str(), + "generated-1", + "should store the generated ID" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn a_held_source_is_not_regenerated() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(provider_record("p-1")), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should keep the held ID" + ); + } } diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index 7fff308ea9..c8e6578f71 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -63,6 +63,9 @@ pub enum SkipReason { Destination, /// The provider's hourly rate limit is reached. RateLimited, + /// The vendor reports an opt-out; the provider's stored UIDs are + /// removed. + OptedOut, } /// One outcome per provider per navigation. @@ -106,6 +109,7 @@ impl LookupStatus { Self::Skipped(SkipReason::Input) => "skipped_input", Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", Self::Skipped(SkipReason::Destination) => "destination_rejected", + Self::Skipped(SkipReason::OptedOut) => "skipped_opted_out", } } } @@ -119,6 +123,9 @@ pub struct ProviderLookup { pub status: LookupStatus, /// Validated entries per returned source; empty unless `Found`. pub entries: Vec<(SourceDomain, Vec)>, + /// Sources whose stored records are removed, because the user opted + /// out of the provider; empty unless `Skipped(OptedOut)`. + pub cleared: Vec, } /// Validated results of one navigation's lookups. diff --git a/crates/trusted-server-core/src/ec/coordinator/transport.rs b/crates/trusted-server-core/src/ec/coordinator/transport.rs index e01f2c6a13..9402d6aab2 100644 --- a/crates/trusted-server-core/src/ec/coordinator/transport.rs +++ b/crates/trusted-server-core/src/ec/coordinator/transport.rs @@ -174,6 +174,7 @@ impl LookupTransport for PlatformLookupTransport<'_> { url, headers, body, + context: _, } = request; let host = url .host_str() @@ -317,6 +318,7 @@ mod tests { url: url::Url::parse(&format!("https://ids.example.com/{path}")).expect("should parse"), headers: http::HeaderMap::new(), body: bytes::Bytes::new(), + context: None, } } diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index e7c8107ea9..a189fcd25a 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -7,10 +7,13 @@ use crate::ec::eid_types::{ AgentType, EidFields, EidValidationError, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, RawJson, SourceDomain, UidValue, UnixSeconds, }; -use crate::ec::kv_types::{KvEid, KvUid}; +use crate::ec::kv_types::{KvEid, KvRefresh, KvUid}; use crate::ec::provider::{EidLevelFields, EidShape, ProviderEid, ProviderExpiry, ProviderUid}; use crate::ec::registry::PartnerRegistry; +/// Largest private refresh state a provider may keep with one UID. +const MAX_REFRESH_STATE_BYTES: usize = 2048; + /// Policy one response is validated against. pub(crate) struct ValidationPolicy<'a> { /// Owner recorded on every kept UID. @@ -126,21 +129,43 @@ fn stored_uid( let id = UidValue::try_from(uid.id.into_owned())?; let atype = AgentType::try_from(uid.atype.unwrap_or(default_atype))?; let ext = uid.ext.map(RawJson::try_from).transpose()?; - let expiry = match uid.expiry { - ProviderExpiry::Unspecified => Expiry::Never, - ProviderExpiry::At(at) => Expiry::At(at), - ProviderExpiry::After(after) => Expiry::At(policy.now.saturating_add_secs(after.as_secs())), + let expiry = match resolve_expiry(uid.expiry, policy.now) { + Some(at) => Expiry::At(at), + None => Expiry::Never, }; if !expiry.is_live(policy.now) { return Err(EidValidationError::Expired); } - Ok(KvUid::new( - id, - atype, - ext, - Some(policy.provider.clone()), - expiry, - )) + let stored = KvUid::new(id, atype, ext, Some(policy.provider.clone()), expiry); + let Some(refresh) = uid.refresh else { + return Ok(stored); + }; + // Unusable refresh state is dropped, never the UID: a refresh with no + // stated time, or with empty or oversized state, is simply not stored. + let usable = !refresh.state.is_empty() && refresh.state.len() <= MAX_REFRESH_STATE_BYTES; + match resolve_expiry(refresh.from, policy.now) { + Some(from) if usable => Ok(stored.with_refresh(KvRefresh { + state: refresh.state.into_owned(), + from, + })), + _ => { + log::debug!( + "Identity lookup: provider={} drops unusable refresh state", + policy.provider + ); + Ok(stored) + } + } +} + +/// Resolves a provider-stated time against `now`. `None` when no time was +/// stated. +fn resolve_expiry(expiry: ProviderExpiry, now: UnixSeconds) -> Option { + match expiry { + ProviderExpiry::Unspecified => None, + ProviderExpiry::At(at) => Some(at), + ProviderExpiry::After(after) => Some(now.saturating_add_secs(after.as_secs())), + } } fn is_duplicate(earlier: &[KvEid], current: &[KvUid], uid: &KvUid) -> bool { @@ -154,6 +179,7 @@ fn is_duplicate(earlier: &[KvEid], current: &[KvUid], uid: &KvUid) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::ec::provider::ProviderRefresh; use crate::settings::EcPartner; use serde_json::value::RawValue; use std::num::NonZeroUsize; @@ -173,6 +199,7 @@ mod tests { atype: None, ext: None, expiry: ProviderExpiry::Unspecified, + refresh: None, } } @@ -233,6 +260,46 @@ mod tests { ); } + #[test] + fn keeps_the_uid_when_its_refresh_state_is_unusable() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let in_a_minute = ProviderExpiry::After(Duration::from_secs(60)); + let cases = [ + ( + "state".to_owned(), + in_a_minute, + Some(UnixSeconds::new(1060)), + ), + (String::new(), in_a_minute, None), + ("x".repeat(MAX_REFRESH_STATE_BYTES + 1), in_a_minute, None), + ("state".to_owned(), ProviderExpiry::Unspecified, None), + ]; + + for (state, from, expected) in cases { + let mut refreshed = uid("u-1"); + refreshed.refresh = Some(ProviderRefresh { + state: Cow::Owned(state), + from, + }); + + let (entries, dropped) = validate( + vec![eid("ids.example.com", vec![refreshed])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop no UID"); + assert_eq!( + entries[0].1[0].uids()[0] + .refresh() + .map(|refresh| refresh.from), + expected, + "should keep only usable refresh state" + ); + } + } + #[test] fn drops_invalid_values_by_reason() { let partners = registry(); diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 41105abe24..055e4b2950 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -60,9 +60,10 @@ pub fn ec_finalize_response( response: &mut Response, ) { ec_context.validate_pull_sync_marker(settings, registry); - let permit = ec_context + let (permit, after_response) = ec_context .take_navigation_permit() - .and_then(|permit| permit.confirm(response.status())); + .and_then(|permit| permit.confirm_with_after_response(response.status())) + .unzip(); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); @@ -106,10 +107,17 @@ pub fn ec_finalize_response( updates: &updates, had_eid_updates, }; - if ec_context.ec_generated() { - commit_generated(&commit, ec_context, permit, response); + let page_write_ok = if ec_context.ec_generated() { + commit_generated(&commit, ec_context, permit, response) } else if ec_context.ec_was_present() { - commit_returning(&commit, ec_context, permit, response); + commit_returning(&commit, ec_context, permit, response) + } else { + false + }; + // The follow-up write is allowed only once consent allows the EC and + // the page-load write succeeded or had nothing to change. + if page_write_ok && let Some(after_response) = after_response { + ec_context.set_after_response_permit(after_response); } } else if ec_context.ec_generated() { // Do not emit a generated EC when there is no KV graph: that would @@ -162,19 +170,21 @@ fn present_snapshot(ec_id: &str, entry: KvEntry) -> EcKvSnapshot { } /// Creates the row of a newly generated EC and issues its cookie on success. +/// +/// Returns whether the row was created. fn commit_generated( commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, permit: Option, response: &mut Response, -) { +) -> bool { let Some(new_entry) = ec_context.take_pending_entry() else { log::warn!("Skipping generated EC cookie because no pending row was built"); - return; + return false; }; let Some(permit) = permit else { log::info!("Skipping generated EC: not a page-load navigation"); - return; + return false; }; let Some(change) = commit.plan( ec_context, @@ -183,12 +193,13 @@ fn commit_generated( Some(&new_entry), ) else { log::warn!("Skipping generated EC: no row change was planned"); - return; + return false; }; match commit.graph.commit_page_load(permit, commit.ec_id, change) { Ok(committed) => { ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); set_ec_cookie_on_response(commit.settings, ec_context, response); + true } Err(report) => { log::warn!( @@ -196,19 +207,23 @@ fn commit_generated( log_id(commit.ec_id), report.current_context() ); + false } } } /// Applies a returning EC's page-load write, or recovers an orphaned EC. +/// +/// Returns whether the row is live afterwards: written, recovered, or left +/// unchanged because there was nothing to write. fn commit_returning( commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, permit: Option, response: &mut Response, -) { +) -> bool { let Some(permit) = permit else { - return; + return false; }; // An unread snapshot is read only when this navigation has something to // write, or when consent gating emptied real EID data on a request that @@ -225,27 +240,38 @@ fn commit_returning( if ec_context.recovery_eligible() { confirm_then_recover_orphaned_ec(commit, ec_context, permit, response); } - return; + return matches!(ec_context.kv_snapshot(), EcKvSnapshot::Present { .. }); } - update_returning(commit, ec_context, permit); + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Failed { .. }) { + return false; + } + update_returning(commit, ec_context, permit) } /// Writes the planned update for a returning EC whose row snapshot is held. +/// +/// Returns `false` only when a planned write failed. fn update_returning( commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, permit: PageLoadWrite, -) { +) -> bool { let Some(change) = commit.plan(ec_context, commit.ec_id, ec_context.kv_snapshot(), None) else { - return; + return true; }; match commit.graph.commit_page_load(permit, commit.ec_id, change) { - Ok(committed) => ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)), - Err(report) => log::info!( - "Page-load write skipped for '{}': {}", - log_id(commit.ec_id), - report.current_context() - ), + Ok(committed) => { + ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); + true + } + Err(report) => { + log::info!( + "Page-load write skipped for '{}': {}", + log_id(commit.ec_id), + report.current_context() + ); + false + } } } @@ -815,6 +841,10 @@ mod tests { .expect("should replace the stale miss with a tombstone snapshot"); assert!(!snapshot_entry.consent.ok && snapshot_entry.ids.is_empty()); assert_eq!(ec_context.kv_snapshot().generation_for(&ec_id), None); + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups after a withdrawal" + ); } #[test] @@ -1065,6 +1095,7 @@ mod tests { SourceDomain::try_from("provider.example.com").expect("should build source"), vec![KvEid::new(EidFields::default(), vec![uid])], )], + cleared: Vec::new(), }], )); let graph = KvIdentityGraph::in_memory("test_store"); @@ -2770,4 +2801,64 @@ mod tests { entry.consent.ok = true; entry } + + #[test] + fn finalize_allows_after_response_lookups_on_a_live_row() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after1"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create( + &ec_id, + &crate::ec::kv_types::KvEntry::minimal("p.example", "uid", 1), + ) + .expect("should seed the identity"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_some(), + "should allow after-response lookups when the page write had nothing to change" + ); + } + + #[test] + fn finalize_withholds_after_response_lookups_when_the_row_read_failed() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after2"); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + true, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups when the page phase failed" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 2adcba566f..b0dc976387 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -22,7 +22,7 @@ use crate::error::TrustedServerError; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; use super::kv_types::{KvEntry, KvMetadata, KvNetwork, MAX_ENTRY_BYTES}; -use super::page_load::{KvWriteError, PageLoadWrite, RowChange}; +use super::page_load::{AfterResponseWrite, Generation, KvWriteError, PageLoadWrite, RowChange}; use super::{EcKvSnapshot, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. @@ -389,23 +389,61 @@ impl KvIdentityGraph { EcKvWriteMode::IfGenerationMatch(expected.get()), ), }; + self.write_row(ec_id, &entry, ttl, mode)?; + if is_tombstone { + // Deliberate exception to the one-write-per-page-load rule: + // the withdrawal-completion marker is a separate key that + // predates the rule. Later withdrawals use it to avoid a + // strongly consistent existence check on a row already + // withdrawn, and it never touches the identity row. + self.record_withdrawal_completion(ec_id, entry.consent.updated); + } + Ok(entry) + } + + /// Applies a navigation's follow-up update after its response was sent. + /// + /// The row is replaced only while it is still at `expected`, so a + /// concurrent write is never overwritten. The `permit` is consumed. + /// + /// # Errors + /// + /// Returns [`KvWriteError::Conflict`] when the row changed since it was + /// read, or [`KvWriteError::Store`] on a serialization or store failure, + /// including a row over [`MAX_ENTRY_BYTES`]. Callers must not retry. + pub fn commit_after_response( + &self, + permit: AfterResponseWrite, + ec_id: &str, + entry: &KvEntry, + expected: Generation, + ) -> Result<(), Report> { + permit.spend(); + self.write_row( + ec_id, + entry, + ENTRY_TTL, + EcKvWriteMode::IfGenerationMatch(expected.get()), + ) + } + + /// Serializes `entry` and writes it with `mode`, mapping a failed + /// precondition to [`KvWriteError::AlreadyExists`] for a create and + /// [`KvWriteError::Conflict`] otherwise. + fn write_row( + &self, + ec_id: &str, + entry: &KvEntry, + ttl: Duration, + mode: EcKvWriteMode, + ) -> Result<(), Report> { let (body, metadata) = - Self::serialize_entry(&entry, self.store_name()).change_context(KvWriteError::Store)?; + Self::serialize_entry(entry, self.store_name()).change_context(KvWriteError::Store)?; let outcome = self .write_entry(ec_id, &body, &metadata, ttl, mode) .change_context(KvWriteError::Store)?; match outcome { - EcKvWriteOutcome::Written => { - if is_tombstone { - // Deliberate exception to the one-write-per-page-load rule: - // the withdrawal-completion marker is a separate key that - // predates the rule. Later withdrawals use it to avoid a - // strongly consistent existence check on a row already - // withdrawn, and it never touches the identity row. - self.record_withdrawal_completion(ec_id, entry.consent.updated); - } - Ok(entry) - } + EcKvWriteOutcome::Written => Ok(()), EcKvWriteOutcome::PreconditionFailed => Err(Report::new(match mode { EcKvWriteMode::Add => KvWriteError::AlreadyExists, _ => KvWriteError::Conflict, diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index b97a37892e..c6f401f40d 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -157,6 +157,15 @@ impl KvPartnerId { !self.eids.is_empty() } + /// Whether a UID in the record asks its provider for a refresh by `now`. + #[must_use] + pub fn refresh_due(&self, now: UnixSeconds) -> bool { + self.eids + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.refresh_due(now)) + } + /// Whether the record holds a UID that may be served at `now`. #[must_use] pub fn has_live_uid(&self, now: UnixSeconds) -> bool { @@ -338,6 +347,29 @@ pub struct KvUid { skip_serializing_if = "Expiry::is_never" )] expiry: Expiry, + /// Private refresh state; never served, mirrored or forwarded. + #[serde(default, skip_serializing_if = "Option::is_none")] + refresh: Option, +} + +/// Refresh state a provider keeps with one of its UIDs. +/// +/// It stays in the row only: outbound EIDs are built from the UID's `id`, +/// `atype` and `ext`, so this state never reaches bidders or cookies. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct KvRefresh { + /// Provider-defined state, opaque to the framework. + pub state: String, + /// When the owning provider should be asked to refresh the UID. + pub from: UnixSeconds, +} + +impl core::fmt::Debug for KvRefresh { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("KvRefresh") + .field("from", &self.from) + .finish_non_exhaustive() + } } impl KvUid { @@ -356,9 +388,36 @@ impl KvUid { ext, owner, expiry, + refresh: None, } } + /// Attaches the owning provider's private refresh state. + #[must_use] + pub fn with_refresh(mut self, refresh: KvRefresh) -> Self { + self.refresh = Some(refresh); + self + } + + /// The owning provider's private refresh state. + #[must_use] + pub fn refresh(&self) -> Option<&KvRefresh> { + self.refresh.as_ref() + } + + /// Whether the owning provider asked to refresh this live UID by `now`. + /// + /// An expired UID is never refresh-due: its source is due anyway, and a + /// refresh that keeps failing must not outlive the UID. + #[must_use] + pub fn refresh_due(&self, now: UnixSeconds) -> bool { + self.is_live(now) + && self + .refresh + .as_ref() + .is_some_and(|refresh| refresh.from <= now) + } + /// The UID value. #[must_use] pub fn id(&self) -> &UidValue { @@ -1732,4 +1791,32 @@ mod tests { assert!(entry.validate().is_err(), "should reject oversized {field}"); } } + + #[test] + fn refresh_is_due_only_while_the_uid_is_live() { + let uid = KvUid::new( + UidValue::try_from("p-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + None, + Expiry::At(UnixSeconds::new(5000)), + ) + .with_refresh(KvRefresh { + state: "refresh-state".to_owned(), + from: UnixSeconds::new(1000), + }); + + assert!( + !uid.refresh_due(UnixSeconds::new(999)), + "should wait for `from`" + ); + assert!( + uid.refresh_due(UnixSeconds::new(1000)), + "should be due at `from`" + ); + assert!( + !uid.refresh_due(UnixSeconds::new(5000)), + "should stop refreshing once the UID expires" + ); + } } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index f90940d5be..84c3147393 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -38,6 +38,7 @@ mod auth; pub mod admin; +pub mod after_response; pub mod batch_sync; pub mod consent; pub mod cookies; @@ -97,7 +98,7 @@ use device::DeviceSignals; use self::coordinator::results::LookupResults; use self::kv::KvIdentityGraph; use self::kv_types::KvEntry; -use self::page_load::NavigationPermit; +use self::page_load::{AfterResponseWrite, NavigationPermit}; use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; /// Bounded request classifications that may persist browser EID cookies. @@ -264,6 +265,17 @@ impl Clone for PermitSlot { } } +/// Holds the navigation's after-response permit. Like [`PermitSlot`], +/// cloning an [`EcContext`] never copies it. +#[derive(Debug, Default)] +struct AfterResponsePermitSlot(Option); + +impl Clone for AfterResponsePermitSlot { + fn clone(&self) -> Self { + Self(None) + } +} + /// Captures the EC state for a single request lifecycle. /// /// Created via [`read_from_request`](Self::read_from_request) during @@ -308,6 +320,9 @@ pub struct EcContext { eid_sync_source: Option, /// Navigation permit issued by the router for page-load navigations. navigation_permit: PermitSlot, + /// Permit for the navigation's follow-up write after its response, set + /// by finalize when it confirms the navigation permit. + after_response_permit: AfterResponsePermitSlot, /// Validated provider lookup results collected for this navigation. lookup_results: LookupResults, /// New row built by [`Self::generate_if_needed`], written by finalize. @@ -395,6 +410,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), lookup_results: LookupResults::default(), pending_entry: None, }) @@ -623,6 +639,17 @@ impl EcContext { self.navigation_permit.0.take() } + /// Stores the after-response permit confirmed with the navigation permit. + pub(crate) fn set_after_response_permit(&mut self, permit: AfterResponseWrite) { + self.after_response_permit = AfterResponsePermitSlot(Some(permit)); + } + + /// Takes the after-response permit; returns `None` when finalize did not + /// confirm a navigation, or after the first call. + pub fn take_after_response_permit(&mut self) -> Option { + self.after_response_permit.0.take() + } + /// Takes the navigation permit, for adapter tests. #[cfg(any(test, feature = "test-utils"))] pub fn take_navigation_permit_for_test(&mut self) -> Option { @@ -709,6 +736,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), lookup_results: LookupResults::default(), pending_entry: None, } @@ -737,6 +765,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), lookup_results: LookupResults::default(), pending_entry: None, } @@ -768,6 +797,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), lookup_results: LookupResults::default(), pending_entry: None, } diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs index 7453d93e7f..5eddc555d5 100644 --- a/crates/trusted-server-core/src/ec/page_load.rs +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -3,7 +3,12 @@ //! A request classified as a top-level document navigation receives a //! [`NavigationPermit`]. Once the final response status is known it becomes a //! [`PageLoadWrite`], which [`KvIdentityGraph::commit_page_load`] consumes. -//! Neither type is `Clone`, so one navigation makes at most one write. +//! Neither type is `Clone`, so one navigation makes at most one write before +//! its response. +//! +//! The one exception is [`AfterResponseWrite`], confirmed alongside the page +//! load's permit. It allows one follow-up update after the response has been +//! sent, carrying only the results of after-response identity lookups. //! //! [`KvIdentityGraph::commit_page_load`]: super::kv::KvIdentityGraph::commit_page_load @@ -57,6 +62,19 @@ impl NavigationPermit { (!Self::is_redirect(status)).then_some(PageLoadWrite(())) } + /// Confirms the permit like [`Self::confirm`] and also issues the + /// navigation's [`AfterResponseWrite`]. + /// + /// Returns `None` for a redirect, which makes neither write. + #[must_use] + pub fn confirm_with_after_response( + self, + status: StatusCode, + ) -> Option<(PageLoadWrite, AfterResponseWrite)> { + self.confirm(status) + .map(|permit| (permit, AfterResponseWrite(()))) + } + /// Whether `status` is a redirect (301, 302, 303, 307, 308) on which the /// page load's write is skipped. /// @@ -91,6 +109,29 @@ impl PageLoadWrite { pub(crate) fn spend(self) {} } +/// Proof that the current navigation may make one follow-up update after its +/// response has been sent. +/// +/// Issued only by [`NavigationPermit::confirm_with_after_response`] and +/// consumed by [`KvIdentityGraph::commit_after_response`]. It is not `Clone`, +/// so a navigation makes at most one follow-up write. +/// +/// [`KvIdentityGraph::commit_after_response`]: super::kv::KvIdentityGraph::commit_after_response +#[derive(Debug)] +pub struct AfterResponseWrite(()); + +impl AfterResponseWrite { + /// Consumes the permit. Called only by the write itself. + pub(crate) fn spend(self) {} + + /// Builds a permit without a navigation, for tests. + #[cfg(test)] + #[must_use] + pub fn for_test() -> Self { + Self(()) + } +} + /// KV generation a conditional write expects. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Generation(u64); diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index d2be63fb7a..28b2970180 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -57,6 +57,17 @@ pub trait IdentityProvider { &self, response: ProviderResponse<'body>, ) -> Result, Report>; + + /// Creates IDs for due sources without a lookup, for a provider whose + /// IDs the operator owns rather than a vendor server. + /// + /// Runs when a source is due. A returned entry goes through the same + /// validation and write plan as a lookup result, and no request is built. + /// The default creates nothing. + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + let _ = lookup; + None + } } impl IdentityProvider for Box

{ @@ -77,6 +88,10 @@ impl IdentityProvider for Box

{ ) -> Result, Report> { (**self).parse_response(response) } + + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + (**self).generate(lookup) + } } /// What a provider can return, used by the coordinator to apply shared policy. @@ -206,6 +221,9 @@ pub struct ProviderRequest { pub headers: http::HeaderMap, /// Request body. pub body: bytes::Bytes, + /// Provider state returned with the response in + /// [`ProviderResponse::context`]; never sent. + pub context: Option, } impl fmt::Debug for ProviderRequest { @@ -226,6 +244,8 @@ pub struct ProviderResponse<'body> { pub headers: &'body http::HeaderMap, /// Response body. pub body: &'body [u8], + /// The [`ProviderRequest::context`] of the request this answers. + pub context: Option<&'body str>, } /// A parsed provider response. @@ -235,6 +255,8 @@ pub enum LookupOutcome<'body> { Found(Vec>), /// The provider knows no ID for this visitor. NoMatch, + /// The visitor opted out with the provider; its stored UIDs are removed. + OptedOut, } /// One entry as returned by a provider, before validation. @@ -265,6 +287,28 @@ pub struct ProviderUid<'body> { pub ext: Option<&'body RawValue>, /// Expiry as the provider states it. pub expiry: ProviderExpiry, + /// Private refresh state kept with the UID in the row and never served. + pub refresh: Option>, +} + +/// Refresh state a provider asks the framework to keep with a UID. +/// +/// The stored UID's source becomes due again at `from`, and the state is +/// handed back through [`LookupInput::current`]. `Debug` hides the state, +/// which may hold a credential. +pub struct ProviderRefresh<'body> { + /// Provider-defined state, such as a refresh token. + pub state: Cow<'body, str>, + /// When the provider should be asked to refresh the UID. + pub from: ProviderExpiry, +} + +impl fmt::Debug for ProviderRefresh<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderRefresh") + .field("from", &self.from) + .finish_non_exhaustive() + } } /// Expiry as a provider states it. The coordinator turns it into @@ -587,6 +631,7 @@ mod tests { .expect("should parse"), headers, body: bytes::Bytes::new(), + context: None, }; let printed = format!("{request:?}"); diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 5046bf82da..fb00d8ca38 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -1,7 +1,7 @@ //! Plans a navigation's single row change. //! //! Combines browser ingest (fill missing) and validated provider results -//! (replace the provider's sources), prunes expired UIDs and enforces +//! (replace the provider's sources, or remove them on an opt-out), prunes expired UIDs and enforces //! [`MAX_ENTRY_BYTES`]. Pure: no I/O. //! //! A provider-owned source takes a browser UID only as a fallback: when the @@ -68,7 +68,13 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { input .lookups .iter() - .flat_map(|lookup| lookup.entries.iter().map(|(source, _)| source)) + .flat_map(|lookup| { + lookup + .entries + .iter() + .map(|(source, _)| source) + .chain(&lookup.cleared) + }) .collect() } else { Vec::new() @@ -162,6 +168,14 @@ fn apply_lookups( written.push(source.clone()); } } + for source in &lookup.cleared { + if entry.ids.remove(source.as_str()).is_some() { + log::info!( + "EC write planner: removing '{source}' after provider '{}' opt-out", + lookup.provider + ); + } + } } written } @@ -305,6 +319,7 @@ mod tests { SourceDomain::try_from(source).expect("should build source"), vec![KvEid::new(EidFields::default(), uids)], )], + cleared: Vec::new(), }], ) } @@ -534,10 +549,41 @@ mod tests { provider: ProviderId::new("example-provider"), status, entries: Vec::new(), + cleared: Vec::new(), }], ) } + #[test] + fn opt_out_removes_the_provider_sources_and_blocks_browser_fallback() { + // Arrange + let registry = registry(&["ids.example.com"]); + let snapshot = present(&provider_entry(&["p-1"])); + let ingest = ingest_owned(); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::OptedOut), + entries: Vec::new(), + cleared: vec![SourceDomain::try_from("ids.example.com").expect("should parse")], + }], + ); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should remove the opted-out source without a browser fallback" + ); + } + fn provider_entry(ids: &[&str]) -> KvEntry { let mut entry = live_entry(); let lookups = found("ids.example.com", ids); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index af233cee06..8ea999d82a 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; -use std::num::NonZeroU16; +use std::num::{NonZeroU16, NonZeroU32}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; @@ -605,6 +605,12 @@ pub struct IdentityProviderConfig { /// TCF vendor ID that must have vendor consent, when set. #[serde(default)] pub tcf_vendor_id: Option, + /// When the provider's lookup runs on a navigation. + #[serde(default)] + pub collect: CollectTiming, + /// Most lookups per EC per hour, when set; unset means unlimited. + #[serde(default)] + pub rate_limit_per_hour: Option, /// Secret Store key references, resolved at startup. #[serde(default)] pub secrets: ProviderSecrets, @@ -628,6 +634,20 @@ impl IdentityProviderConfig { } } +/// When an identity provider's lookup runs on a navigation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CollectTiming { + /// Before the response, within `identity_budget_ms`; results reach the + /// navigation's own write and response. + #[default] + Page, + /// After the response has been sent, within + /// `identity_after_response_budget_ms`; results reach the row in one + /// follow-up write and are served from the next page load. + AfterResponse, +} + /// Secret values for one identity provider. /// /// `EdgeZero` secret paths address fixed field names, so V1 supports one @@ -686,6 +706,15 @@ pub struct Ec { )] pub identity_budget_ms: u64, + /// Time budget, in milliseconds, for lookups of providers with + /// `collect = "after_response"`, measured from their dispatch after the + /// response has been sent. + #[serde( + default = "Ec::default_identity_after_response_budget_ms", + deserialize_with = "from_value_or_str" + )] + pub identity_after_response_budget_ms: u64, + /// Server-side identity providers. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub identity_providers: Vec, @@ -725,6 +754,12 @@ impl Ec { 200 } + /// Default after-response identity lookup budget (2 s). + #[must_use] + pub const fn default_identity_after_response_budget_ms() -> u64 { + 2000 + } + /// Returns `true` if `passphrase` matches a known placeholder value /// (case-insensitive). #[must_use] diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index d7206687ed..cb6e08118e 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -2,8 +2,9 @@ //! //! Each provider kind is a module implementing //! [`trusted_server_core::ec::provider::ProviderKind`]. [`build`] maps a -//! configured `kind` to its implementation; [`build_coordinator`] builds the -//! coordinator for every enabled provider. +//! configured `kind` to its implementation; [`build_coordinator`] and +//! [`build_after_response_coordinator`] build the coordinators for enabled +//! providers collected before and after the response. //! //! Dependencies point one way: this crate depends on core, and core never //! depends on it. @@ -16,7 +17,9 @@ use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvid use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; -use trusted_server_core::settings::{Ec, IdentityProviderConfig, ProviderSecrets, Settings}; +use trusted_server_core::settings::{ + CollectTiming, Ec, IdentityProviderConfig, ProviderSecrets, Settings, +}; /// Whether `kind` has a registered implementation. /// @@ -53,14 +56,16 @@ pub fn build( .attach(format!("kind `{kind}` is not registered"))) } -/// Builds the coordinator for every enabled provider in `settings`. +/// Builds the coordinator for enabled providers collected before the +/// response (`collect = "page"`, the default). /// -/// Returns `None` when no provider is enabled. +/// Returns `None` when no such provider is enabled. /// /// # Errors /// -/// Returns [`ProviderConfigError`] when a provider cannot be built or fails -/// the coordinator's configuration checks. +/// Returns [`ProviderConfigError`] when a provider cannot be built or the +/// enabled providers, taken together, fail the coordinator's configuration +/// checks. pub fn build_coordinator( settings: &Settings, ) -> Result, Report> { @@ -69,8 +74,8 @@ pub fn build_coordinator( build_coordinator_with(settings, Arc::new(partners)) } -/// Builds the coordinator for every enabled provider in `settings` against an -/// already built partner registry. +/// Builds the page-phase coordinator against an already built partner +/// registry. /// /// Enabled providers with an unregistered kind are skipped; building the /// registry with [`partner_registry`] already warned about them. @@ -83,20 +88,65 @@ pub fn build_coordinator_with( settings: &Settings, partners: Arc, ) -> Result, Report> { + build_phase_coordinator( + settings, + partners, + CollectTiming::Page, + Duration::from_millis(settings.ec.identity_budget_ms), + ) +} + +/// Builds the coordinator for enabled providers collected after the response +/// has been sent (`collect = "after_response"`). +/// +/// Its budget is `identity_after_response_budget_ms`. Returns `None` when no +/// such provider is enabled. +/// +/// # Errors +/// +/// Returns the same errors as [`build_coordinator`]. +pub fn build_after_response_coordinator( + settings: &Settings, +) -> Result, Report> { + let partners = partner_registry(&settings.ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + build_phase_coordinator( + settings, + Arc::new(partners), + CollectTiming::AfterResponse, + Duration::from_millis(settings.ec.identity_after_response_budget_ms), + ) +} + +fn build_phase_coordinator( + settings: &Settings, + partners: Arc, + timing: CollectTiming, + budget: Duration, +) -> Result, Report> { + // Source ownership across both phases is checked when settings load + // (`config::validate_identity_providers`), so only this phase's providers + // are built and checked here. let registered = settings .ec .identity_providers .iter() - .filter(|config| config.enabled && is_registered_kind(config.kind())) + .filter(|config| { + config.enabled && config.collect == timing && is_registered_kind(config.kind()) + }) .map(|config| { - build(config, &config.secrets) - .map(|provider| RegisteredProvider::new(config.clone(), provider)) + build(config, &config.secrets).map(|provider| { + let registered = RegisteredProvider::new(config.clone(), provider); + match config.rate_limit_per_hour { + Some(per_hour) => registered.with_rate_limit(per_hour), + None => registered, + } + }) }) .collect::, Report>>()?; if registered.is_empty() { return Ok(None); } - let budget = Duration::from_millis(settings.ec.identity_budget_ms); ProviderCoordinator::new(registered, partners, budget).map(Some) } @@ -177,4 +227,37 @@ mod tests { "should leave unregistered provider sources to browser ingest" ); } + + #[test] + fn rejects_an_unknown_collect_timing() { + toml::from_str::( + "id = \"example-provider\"\nkind = \"example-kind\"\ncollect = \"later\"", + ) + .expect_err("should reject an unknown `collect` value"); + } + + #[test] + fn parses_an_optional_hourly_rate_limit() { + let limited: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 2", + ) + .expect("should parse a rate limit"); + let unset: IdentityProviderConfig = + toml::from_str("id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"") + .expect("should parse without a rate limit"); + + assert_eq!( + limited.rate_limit_per_hour.map(std::num::NonZeroU32::get), + Some(2), + "should keep the hourly limit" + ); + assert!( + unset.rate_limit_per_hour.is_none(), + "should leave the limit unset by default" + ); + toml::from_str::( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 0", + ) + .expect_err("should reject a zero limit"); + } } From d45b5087543e893c8bf7e5954584b023c9507f7f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 19:29:02 +0530 Subject: [PATCH 26/38] Never store browser UIDs for provider-owned sources Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/finalize.rs | 14 +- .../trusted-server-core/src/ec/prebid_eids.rs | 3 +- .../trusted-server-core/src/ec/write_plan.rs | 120 ++++++------------ docs/guide/configuration.md | 7 +- 4 files changed, 48 insertions(+), 96 deletions(-) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 055e4b2950..2e28f92a23 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1679,7 +1679,7 @@ mod tests { } #[test] - fn finalize_stores_browser_fallback_for_provider_owned_source() { + fn finalize_skips_browser_uid_for_provider_owned_source() { // Arrange let settings = create_test_settings(); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1717,17 +1717,9 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("should create the row"); - let record = stored - .ids - .get("sharedid.org") - .expect("should store the browser fallback"); - assert_eq!( - record.uid, "fallback-cookie-id", - "should store the browser uid" - ); assert!( - !record.is_provider_written(), - "should mark the fallback as not provider-written" + !stored.ids.contains_key("sharedid.org"), + "should not store a browser UID for a provider-owned source" ); } diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 29db7122fc..768a431188 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -157,8 +157,7 @@ pub(crate) fn collect_eid_updates( { updates.push(update); } - // Provider-owned sources stay in the list: the write planner ingests them - // only as a fallback when the provider left the source empty. + // Provider-owned sources stay in the list; the write planner skips them. dedupe_partner_updates(updates) } diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index fb00d8ca38..c1b566a54a 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -4,11 +4,9 @@ //! (replace the provider's sources, or remove them on an opt-out), prunes expired UIDs and enforces //! [`MAX_ENTRY_BYTES`]. Pure: no I/O. //! -//! A provider-owned source takes a browser UID only as a fallback: when the -//! row holds no live UID for it and the provider kept nothing for it on this -//! navigation. The fallback is a single-UID record without provider `eids`, -//! so the coordinator still treats the source as due and a later provider -//! result replaces it. +//! Browser ingest skips provider-owned sources entirely (spec §8.4), so a +//! source the provider cleared on an opt-out is never refilled from the +//! browser's `ts-eids` cookie. use super::EcKvSnapshot; use super::coordinator::results::LookupResults; @@ -64,30 +62,7 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { let mut entry = base.clone(); let lookups_apply = input.lookups.is_for(input.ec_id); - let provider_filled: Vec<&SourceDomain> = if lookups_apply { - input - .lookups - .iter() - .flat_map(|lookup| { - lookup - .entries - .iter() - .map(|(source, _)| source) - .chain(&lookup.cleared) - }) - .collect() - } else { - Vec::new() - }; - apply_browser_ingest( - &mut entry, - input.browser_ingest, - &BrowserIngestRules { - registry: input.registry, - provider_filled: &provider_filled, - now: input.now, - }, - ); + apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); let written = if lookups_apply { apply_lookups(&mut entry, input.lookups, input.now) } else { @@ -112,46 +87,24 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } } -/// What browser ingest needs to decide provider-owned fallbacks. -struct BrowserIngestRules<'a> { - registry: &'a PartnerRegistry, - /// Sources the provider returned entries for on this navigation. - provider_filled: &'a [&'a SourceDomain], - now: UnixSeconds, -} - +/// Fills missing sources from the browser, skipping provider-owned ones. fn apply_browser_ingest( entry: &mut KvEntry, updates: &[PartnerIdUpdate], - rules: &BrowserIngestRules<'_>, + registry: &PartnerRegistry, ) { for update in updates { - if !rules.registry.is_provider_owned(&update.partner_id) { - entry - .ids - .entry(update.partner_id.clone()) - .or_insert_with(|| KvPartnerId::single(update.uid.clone())); + if registry.is_provider_owned(&update.partner_id) { + log::trace!( + "EC write planner: skipping browser UID for provider-owned source '{}'", + update.partner_id + ); continue; } - let provider_filled = rules - .provider_filled - .iter() - .any(|source| source.as_str() == update.partner_id); - let held = entry + entry .ids - .get(&update.partner_id) - .is_some_and(|record| record.has_live_uid(rules.now)); - if provider_filled || held { - continue; - } - log::debug!( - "EC write planner: ingesting browser fallback for provider-owned source '{}'", - update.partner_id - ); - entry.ids.insert( - update.partner_id.clone(), - KvPartnerId::single(update.uid.clone()), - ); + .entry(update.partner_id.clone()) + .or_insert_with(|| KvPartnerId::single(update.uid.clone())); } } @@ -596,7 +549,7 @@ mod tests { } #[test] - fn provider_owned_source_takes_browser_fallback_when_lookup_fails() { + fn browser_ingest_skips_a_provider_owned_source() { // Arrange let registry = registry(&["ids.example.com"]); let snapshot = present(&live_entry()); @@ -604,32 +557,43 @@ mod tests { for status in [ LookupStatus::NoMatch, + LookupStatus::Timeout, LookupStatus::Skipped(SkipReason::NotDue), + LookupStatus::Skipped(SkipReason::OptedOut), ] { let lookups = failed_lookup(status); // Act - let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) - .expect("should plan update"); + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)); // Assert - let RowChange::Update { entry, .. } = change else { - panic!("should update existing row") - }; - let record = &entry.ids["ids.example.com"]; - assert_eq!( - record.uid, "browser-id", - "should store the browser fallback" - ); assert!( - !record.is_provider_written(), - "should mark the fallback as not provider-written" + change.is_none(), + "should not store a browser UID for a provider-owned source" ); } } #[test] - fn provider_owned_source_takes_browser_fallback_without_lookups() { + fn opted_out_source_is_not_refilled_from_the_browser_on_the_next_navigation() { + // Arrange: the row after an opt-out holds nothing for the source. + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + let lookups = LookupResults::default(); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not restore an opted-out provider UID from the browser cookie" + ); + } + + #[test] + fn new_row_skips_browser_uid_for_a_provider_owned_source() { let registry = registry(&["ids.example.com"]); let snapshot = EcKvSnapshot::Missing { ec_id: EC_ID.to_owned(), @@ -650,9 +614,9 @@ mod tests { let RowChange::Create(entry) = change else { panic!("should create a new row") }; - assert_eq!( - entry.ids["ids.example.com"].uid, "browser-id", - "should store the browser fallback when no provider ran" + assert!( + !entry.ids.contains_key("ids.example.com"), + "should not store a browser UID when creating the row" ); } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 3cc9402a78..e52648eae8 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -805,11 +805,8 @@ request, only with TCF vendor consent for that vendor. A provider owns its `sources`: its results replace whatever the row holds for them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs -(`ts-eids`, `sharedId`) for an owned source are stored only as a fallback, when -the row has no live provider-written UID for the source and the provider kept -nothing for it on that navigation. The fallback follows the normal browser -ingest rules, never overwrites an existing value, and does not stop later -lookups: the next successful lookup replaces it. +(`ts-eids`, `sharedId`) are never stored for an owned source, so a source a +provider cleared on an opt-out is not refilled from the browser. | Field | Type | Required | Description | | ----------------- | ------- | -------- | ----------------------------------------------------------------------- | From e171d1d89a5ae3fc991d3b7b469c948f1e578503 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 19:31:19 +0530 Subject: [PATCH 27/38] Hand refresh state only to the provider that wrote it Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 37 ++++++++++- crates/trusted-server-core/src/ec/kv_types.rs | 9 ++- crates/trusted-server-core/src/ec/provider.rs | 61 ++++++++++++++++--- 3 files changed, 93 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 40de27dc8b..3fd940b030 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -298,7 +298,7 @@ impl ProviderCoordinator

{ .filter(|record| record.is_provider_written()); // A UID whose provider asked for a refresh makes its source due; // the stored refresh state is the lookup's input. - if record.is_some_and(|record| record.refresh_due(now)) { + if record.is_some_and(|record| record.refresh_due_for(®istered.id, now)) { due.push(source.clone()); continue; } @@ -514,7 +514,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ client: lookup.client, publisher_domain: lookup.publisher_domain, due_sources: &due, - current: OwnedEntries::new(entry, ®istered.policy.sources), + current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), hashed_email: None, }; if let Some(eids) = registered.provider.generate(&input) { @@ -1282,11 +1282,15 @@ mod tests { } fn refreshable_record(from: u64) -> KvPartnerId { + refreshable_record_owned_by("example-provider", from) + } + + fn refreshable_record_owned_by(owner: &str, from: u64) -> KvPartnerId { let uid = KvUid::new( UidValue::try_from("p-1").expect("should build uid"), AgentType::try_from(1).expect("should build atype"), None, - Some(ProviderId::new("example-provider")), + Some(ProviderId::new(owner)), Expiry::At(UnixSeconds::new(5000)), ) .with_refresh(KvRefresh { @@ -1346,6 +1350,33 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + #[test] + fn another_providers_refresh_state_is_never_handed_over() { + // Arrange: a live UID with due refresh state written by a provider that + // no longer owns the source. + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record_owned_by("former-provider", 900)), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should not refresh a UID another provider wrote" + ); + assert!( + transport.sent.borrow().is_empty(), + "should not send another provider's refresh state" + ); + } + fn optout_reply() -> Result> { Ok(reply(http::StatusCode::OK, b"optout")) } diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index c6f401f40d..c709f066bb 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -157,13 +157,16 @@ impl KvPartnerId { !self.eids.is_empty() } - /// Whether a UID in the record asks its provider for a refresh by `now`. + /// Whether a UID `owner` wrote asks it for a refresh by `now`. + /// + /// UIDs another provider wrote never make the record refresh-due for + /// `owner`, so their refresh state is never handed to it. #[must_use] - pub fn refresh_due(&self, now: UnixSeconds) -> bool { + pub fn refresh_due_for(&self, owner: &ProviderId, now: UnixSeconds) -> bool { self.eids .iter() .flat_map(KvEid::uids) - .any(|uid| uid.refresh_due(now)) + .any(|uid| uid.owner() == Some(owner) && uid.refresh_due(now)) } /// Whether the record holds a UID that may be served at `now`. diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 28b2970180..89f328e06a 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -22,7 +22,7 @@ use serde_json::value::RawValue; use url::Url; use crate::consent::ConsentContext; -use crate::ec::eid_types::{SourceDomain, UnixSeconds}; +use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; use crate::ec::kv_types::{KvEid, KvEntry}; use crate::ec::registry::PartnerRegistry; use crate::settings::{IdentityProviderConfig, ProviderSecrets}; @@ -170,27 +170,46 @@ pub struct ClientSignals<'a> { } /// This provider's stored entries, borrowed from the row. +/// +/// Only entries whose every UID this provider wrote are visible, so another +/// provider's refresh state never reaches it. #[derive(Debug, Clone, Copy)] pub struct OwnedEntries<'a> { entry: Option<&'a KvEntry>, sources: &'a [SourceDomain], + owner: &'a ProviderId, } impl<'a> OwnedEntries<'a> { - /// Views the entries of `entry` for `sources`. + /// Views the entries of `entry` for `sources` that `owner` wrote. #[must_use] - pub fn new(entry: Option<&'a KvEntry>, sources: &'a [SourceDomain]) -> Self { - Self { entry, sources } + pub fn new( + entry: Option<&'a KvEntry>, + sources: &'a [SourceDomain], + owner: &'a ProviderId, + ) -> Self { + Self { + entry, + sources, + owner, + } } /// Iterates `(source, entry)` pairs in source order. pub fn iter(&self) -> impl Iterator + 'a { let entry = self.entry; + let owner = self.owner; self.sources.iter().flat_map(move |source| { entry .and_then(|entry| entry.ids.get_key_value(source.as_str())) .into_iter() - .flat_map(|(key, record)| record.eids().iter().map(move |eid| (key.as_str(), eid))) + .flat_map(move |(key, record)| { + record + .eids() + .iter() + .filter(move |eid| eid.uids().iter().all(|uid| uid.owner() == Some(owner))) + .map(move |eid| (key.as_str(), eid)) + }) }) } } @@ -705,14 +724,15 @@ mod tests { "consent": { "ok": true, "updated": 1 }, "geo": { "country": "US" }, "ids": { - "ids.example.com": { "uid": "u-1", "eids": [{ "uids": [{ "id": "u-1", "atype": 3 }] }] }, - "other.example.com": { "uid": "u-2", "eids": [{ "uids": [{ "id": "u-2", "atype": 3 }] }] } + "ids.example.com": { "uid": "u-1", "eids": [{ "uids": [{ "id": "u-1", "atype": 3, "owner": "example-provider" }] }] }, + "other.example.com": { "uid": "u-2", "eids": [{ "uids": [{ "id": "u-2", "atype": 3, "owner": "example-provider" }] }] } } })) .expect("should build entry"); let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + let owner = ProviderId::new("example-provider"); - let owned: Vec<&str> = OwnedEntries::new(Some(&entry), &sources) + let owned: Vec<&str> = OwnedEntries::new(Some(&entry), &sources, &owner) .iter() .map(|(source, _)| source) .collect(); @@ -723,4 +743,29 @@ mod tests { "should yield owned sources only" ); } + + #[test] + fn owned_entries_hides_entries_another_provider_wrote() { + let entry: KvEntry = serde_json::from_value(json!({ + "v": 1, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "ids.example.com": { "uid": "u-1", "eids": [ + { "uids": [{ "id": "u-1", "atype": 3, "owner": "former-provider" }] }, + { "uids": [{ "id": "u-2", "atype": 3 }] } + ] } + } + })) + .expect("should build entry"); + let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + let owner = ProviderId::new("example-provider"); + + let count = OwnedEntries::new(Some(&entry), &sources, &owner) + .iter() + .count(); + + assert_eq!(count, 0, "should hide entries this provider did not write"); + } } From cbe3a2bc63bff8be8ee50ebffdfc3f7170f80f32 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 19:33:51 +0530 Subject: [PATCH 28/38] Remember a provider opt-out so the provider is not called again for 30 days Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 53 +++++++++++++++ crates/trusted-server-core/src/ec/kv_types.rs | 38 +++++++++++ .../trusted-server-core/src/ec/write_plan.rs | 64 ++++++++++++++++++- docs/guide/configuration.md | 4 ++ 4 files changed, 156 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 3fd940b030..0705e8d14f 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -499,6 +499,12 @@ impl IdentityCoordinator for ProviderCoordinator

{ pending.early.push(skip(SkipReason::Consent)); continue; } + // A remembered opt-out keeps the provider from being called until + // it lapses. + if entry.is_some_and(|entry| entry.opted_out(®istered.id, lookup.now)) { + pending.early.push(skip(SkipReason::OptedOut)); + continue; + } let (due, input_blocked) = Self::due_sources(registered, entry, lookup.now); if due.is_empty() { pending.early.push(skip(if input_blocked { @@ -1350,6 +1356,53 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + #[test] + fn remembered_opt_out_skips_the_provider_until_it_lapses() { + // Arrange + let coordinator = coordinator(""); + let provider = ProviderId::new("example-provider"); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.opt_outs.insert(provider.clone(), 1001); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::OptedOut), + "should skip a provider whose opt-out is remembered" + ); + assert!( + transport.sent.borrow().is_empty(), + "should not call a provider the visitor opted out of" + ); + + // Arrange: the opt-out lapsed. + entry.opt_outs.insert(provider, 1000); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should call the provider again once the opt-out lapses" + ); + } + #[test] fn another_providers_refresh_state_is_never_handed_over() { // Arrange: a live UID with due refresh state written by a provider that diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index c709f066bb..b24e91e6c0 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -41,6 +41,10 @@ pub const MAX_STORED_DOMAIN_LENGTH: usize = 255; /// consistent validation. pub const MAX_UID_LENGTH: usize = 1024; +/// How long a provider opt-out keeps that provider from being called again, +/// in seconds. Matches the longest lifetime a provider UID may have. +pub const OPT_OUT_MEMORY_SECONDS: u64 = 30 * 24 * 60 * 60; + /// Full KV entry stored as the body of an EC identity graph record. /// /// **KV key:** Full EC ID (`{64hex}.{6alnum}`). @@ -69,6 +73,10 @@ pub struct KvEntry { /// Populated by pixel sync, batch sync, and pull sync operations. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub ids: BTreeMap, + /// Provider ID → Unix time until which that provider's opt-out is + /// honoured without calling the provider again. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub opt_outs: BTreeMap, } /// Consent state within a KV entry. @@ -658,7 +666,35 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + opt_outs: BTreeMap::new(), + } + } + + /// Whether `provider` opted out and must not be called again by `now`. + #[must_use] + pub fn opted_out(&self, provider: &ProviderId, now: UnixSeconds) -> bool { + self.opt_outs + .get(provider) + .is_some_and(|until| *until > now.get()) + } + + /// Remembers an opt-out by `provider` for [`OPT_OUT_MEMORY_SECONDS`]. + /// + /// An opt-out already remembered keeps its end time, so a provider skipped + /// because of it is asked again once the time passes. + pub fn remember_opt_out(&mut self, provider: &ProviderId, now: UnixSeconds) { + if self.opted_out(provider, now) { + return; } + self.opt_outs.insert( + provider.clone(), + now.saturating_add_secs(OPT_OUT_MEMORY_SECONDS).get(), + ); + } + + /// Forgets opt-outs whose time has passed. + pub fn prune_opt_outs(&mut self, now: UnixSeconds) { + self.opt_outs.retain(|_, until| *until > now.get()); } /// Creates a minimal live entry. @@ -688,6 +724,7 @@ impl KvEntry { device: None, network: None, ids, + opt_outs: BTreeMap::new(), } } @@ -720,6 +757,7 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + opt_outs: BTreeMap::new(), } } diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index c1b566a54a..0a6e087521 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -9,7 +9,7 @@ //! browser's `ts-eids` cookie. use super::EcKvSnapshot; -use super::coordinator::results::LookupResults; +use super::coordinator::results::{LookupResults, LookupStatus, SkipReason}; use super::eid_types::{SourceDomain, UnixSeconds}; use super::kv::PartnerIdUpdate; use super::kv_types::{KvEntry, KvPartnerId, MAX_ENTRY_BYTES}; @@ -77,6 +77,7 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { for record in entry.ids.values_mut() { record.prune_expired(input.now); } + entry.prune_opt_outs(input.now); entry.ids.retain(|_, record| !record.uid.is_empty()); enforce_size_cap(&mut entry, &written); @@ -121,6 +122,9 @@ fn apply_lookups( written.push(source.clone()); } } + if lookup.status == LookupStatus::Skipped(SkipReason::OptedOut) { + entry.remember_opt_out(&lookup.provider, now); + } for source in &lookup.cleared { if entry.ids.remove(source.as_str()).is_some() { log::info!( @@ -213,7 +217,7 @@ mod tests { use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup, SkipReason}; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; use crate::ec::kv::KvIdentityGraph; - use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid, OPT_OUT_MEMORY_SECONDS}; use crate::settings::EcPartner; const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; @@ -537,6 +541,61 @@ mod tests { ); } + #[test] + fn opt_out_is_remembered_without_extending_it() { + // Arrange + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let provider = ProviderId::new("example-provider"); + let lookups = failed_lookup(LookupStatus::Skipped(SkipReason::OptedOut)); + + // Act: the first opt-out is remembered. + let first = plan_write(&input( + &present(&provider_entry(&["p-1"])), + None, + &ingest, + &lookups, + ®istry, + )) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = first else { + panic!("should update existing row") + }; + let until = entry.opt_outs[&provider]; + assert_eq!( + until, + 2000 + OPT_OUT_MEMORY_SECONDS, + "should remember the opt-out for the memory period" + ); + + // Act: a skip because of the remembered opt-out changes nothing. + let again = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + // Assert + assert!(again.is_none(), "should not extend a remembered opt-out"); + } + + #[test] + fn lapsed_opt_out_is_pruned() { + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut entry = live_entry(); + entry + .opt_outs + .insert(ProviderId::new("example-provider"), 1000); + let lookups = LookupResults::default(); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!(entry.opt_outs.is_empty(), "should forget a lapsed opt-out"); + } + fn provider_entry(ids: &[&str]) -> KvEntry { let mut entry = live_entry(); let lookups = found("ids.example.com", ids); @@ -559,7 +618,6 @@ mod tests { LookupStatus::NoMatch, LookupStatus::Timeout, LookupStatus::Skipped(SkipReason::NotDue), - LookupStatus::Skipped(SkipReason::OptedOut), ] { let lookups = failed_lookup(status); diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index e52648eae8..5574ef53d7 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -808,6 +808,10 @@ them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs (`ts-eids`, `sharedId`) are never stored for an owned source, so a source a provider cleared on an opt-out is not refilled from the browser. +When a provider reports an opt-out, its sources are cleared and the row +remembers the opt-out for 30 days. The provider is not called for that EC +again until the 30 days pass. + | Field | Type | Required | Description | | ----------------- | ------- | -------- | ----------------------------------------------------------------------- | | `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | From ca915dbb3340d69c5044524fa044ce52db53a2a6 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 19:45:07 +0530 Subject: [PATCH 29/38] Add an empty owned-entries view for providers with nothing stored Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/provider.rs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 89f328e06a..302b120b25 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -177,7 +177,7 @@ pub struct ClientSignals<'a> { pub struct OwnedEntries<'a> { entry: Option<&'a KvEntry>, sources: &'a [SourceDomain], - owner: &'a ProviderId, + owner: Option<&'a ProviderId>, } impl<'a> OwnedEntries<'a> { @@ -191,7 +191,17 @@ impl<'a> OwnedEntries<'a> { Self { entry, sources, - owner, + owner: Some(owner), + } + } + + /// A view holding no entries, for a provider with nothing stored. + #[must_use] + pub fn empty() -> Self { + Self { + entry: None, + sources: &[], + owner: None, } } @@ -207,7 +217,7 @@ impl<'a> OwnedEntries<'a> { record .eids() .iter() - .filter(move |eid| eid.uids().iter().all(|uid| uid.owner() == Some(owner))) + .filter(move |eid| eid.uids().iter().all(|uid| uid.owner() == owner)) .map(move |eid| (key.as_str(), eid)) }) }) @@ -744,6 +754,15 @@ mod tests { ); } + #[test] + fn empty_owned_entries_yields_nothing() { + assert_eq!( + OwnedEntries::empty().iter().count(), + 0, + "should yield no entries" + ); + } + #[test] fn owned_entries_hides_entries_another_provider_wrote() { let entry: KvEntry = serde_json::from_value(json!({ From 97a69f1e828a5d89a1944b83d62b62aa52f4968d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 20:09:49 +0530 Subject: [PATCH 30/38] Run after-response lookups only for an existing consented row An EC minted on the navigation, or one whose row is missing or withdrawn when read after the response, is skipped before any provider is called, so no vendor is sent an EC ID whose row cannot take the result. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/main.rs | 12 ++++- .../src/ec/after_response.rs | 50 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 6a59e836fd..0256c8d9e9 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -13,7 +13,9 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; -use trusted_server_core::ec::after_response::commit_after_response_lookups; +use trusted_server_core::ec::after_response::{ + after_response_lookups_allowed, commit_after_response_lookups, +}; use trusted_server_core::ec::coordinator::IdentityCoordinator as _; use trusted_server_core::ec::coordinator::LookupContext; use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; @@ -791,6 +793,14 @@ fn run_after_response_lookups( return; }; let snapshot = graph.load_snapshot(&lookups.ec_id); + if !after_response_lookups_allowed( + &snapshot, + &lookups.ec_id, + ec_state.ec_context.ec_generated(), + ) { + log::debug!("After-response lookups skipped: new EC or no live row"); + return; + } let transport = PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index c6ecba1031..37952eaadc 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -35,6 +35,24 @@ pub enum AfterResponseOutcome { Failed, } +/// Whether after-response lookups may be dispatched for `ec_id`. +/// +/// They run only for an EC that existed before this navigation and whose row, +/// read after the response was sent, is live and consented. An EC minted on +/// this navigation is skipped: no partner can know it yet, and its new row +/// may not be readable so soon. Its lookups run from the next navigation. +#[must_use] +pub fn after_response_lookups_allowed( + snapshot: &EcKvSnapshot, + ec_id: &str, + ec_generated: bool, +) -> bool { + !ec_generated + && snapshot + .entry_for(ec_id) + .is_some_and(|entry| entry.consent.ok) +} + /// Applies after-response lookup results to the row of `ec_id`. /// /// Plans an update from `lookups` alone against `snapshot`, the row read @@ -366,4 +384,36 @@ mod tests { "should make no write without results" ); } + + #[test] + fn lookups_run_only_for_an_existing_consented_row() { + // Arrange + let graph = KvIdentityGraph::in_memory("after_response_allowed"); + let missing = graph.load_snapshot(EC_ID); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let present = graph.load_snapshot(EC_ID); + let withdrawn_graph = KvIdentityGraph::in_memory("after_response_withdrawn"); + withdrawn_graph + .create(EC_ID, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + let withdrawn = withdrawn_graph.load_snapshot(EC_ID); + + // Act + Assert + assert!( + after_response_lookups_allowed(&present, EC_ID, false), + "should run for a returning EC with a live row" + ); + assert!( + !after_response_lookups_allowed(&present, EC_ID, true), + "should not run for an EC minted on this navigation" + ); + assert!( + !after_response_lookups_allowed(&missing, EC_ID, false), + "should not run when the row does not exist" + ); + assert!( + !after_response_lookups_allowed(&withdrawn, EC_ID, false), + "should not run for a withdrawn row" + ); + } } From a6adc2610367b909827fff1b30d1b0b5f7d9088f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 21:22:13 +0530 Subject: [PATCH 31/38] Re-read the row just before the after-response write The follow-up write was guarded by the generation read before the lookups were dispatched, up to the full lookup budget earlier, so any write from another request in that window dropped the results. The row is now read just before the write, so only the gap between that read and the write can conflict. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/main.rs | 10 +-- .../src/ec/after_response.rs | 65 ++++++++++++++----- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 0256c8d9e9..1d4c9057cb 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -826,14 +826,8 @@ fn run_after_response_lookups( return; } let results = futures::executor::block_on(coordinator.collect(&transport, pending)); - let outcome = commit_after_response_lookups( - &graph, - permit, - partner_registry, - &lookups.ec_id, - &snapshot, - &results, - ); + let outcome = + commit_after_response_lookups(&graph, permit, partner_registry, &lookups.ec_id, &results); log::info!("After-response lookups: write outcome={outcome}"); } diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index 37952eaadc..f7e9206ce3 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -55,27 +55,28 @@ pub fn after_response_lookups_allowed( /// Applies after-response lookup results to the row of `ec_id`. /// -/// Plans an update from `lookups` alone against `snapshot`, the row read -/// after the response was sent, and writes it once, guarded by the snapshot's -/// generation. The `permit` is consumed whether or not a write is -/// made. +/// Re-reads the row just before writing, plans an update from `lookups` +/// alone against it, and writes it once, guarded by that read's generation. +/// Reading here rather than before dispatch keeps the window in which another +/// write can conflict to the gap between this read and the write. The +/// `permit` is consumed whether or not a write is made. /// /// # Performance /// -/// At most one conditional KV write; the row is read by the caller. -/// Runs after the response has been sent, so it adds no page latency. +/// One KV read and at most one conditional KV write. Runs after the response +/// has been sent, so it adds no page latency. #[must_use] pub fn commit_after_response_lookups( graph: &KvIdentityGraph, permit: AfterResponseWrite, registry: &PartnerRegistry, ec_id: &str, - snapshot: &EcKvSnapshot, lookups: &LookupResults, ) -> AfterResponseOutcome { if lookups.is_empty() { return AfterResponseOutcome::Unchanged; } + let snapshot = &graph.load_snapshot(ec_id); if !matches!(snapshot, EcKvSnapshot::Present { .. }) { log::info!( "After-response lookups: no row for '{}', skipping write", @@ -205,7 +206,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -226,6 +226,48 @@ mod tests { ); } + #[test] + fn writes_against_the_row_as_read_just_before_the_write() { + // Arrange + let graph = KvIdentityGraph::in_memory("after-response-reread"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read_before_dispatch = graph.load_snapshot(EC_ID); + graph + .upsert_partner_id(EC_ID, "browser.example.com", "b-2") + .expect("should rewrite the row as another request would"); + assert_ne!( + graph.load_snapshot(EC_ID), + read_before_dispatch, + "should change the row after the dispatch read" + ); + + // Act + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + // Assert + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should not conflict with a write made while the lookups ran" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-2"), + "should keep the other request's write" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should store the provider UID" + ); + } + #[test] fn replaces_a_browser_fallback_for_an_owned_source() { let graph = KvIdentityGraph::in_memory("after-response"); @@ -241,7 +283,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -266,7 +307,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); assert_eq!(first, AfterResponseOutcome::Written, "should write first"); @@ -276,7 +316,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -296,7 +335,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -323,7 +361,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -349,7 +386,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(other, "p-1"), ); @@ -374,7 +410,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &LookupResults::new(EC_ID, Vec::new()), ); From eb2c868a3aaa86249d4a775173e736553d522c52 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 21:22:13 +0530 Subject: [PATCH 32/38] Drop provider refresh state that is already due A refresh time at or before now made the source due on every navigation until the UID expired. Such state is now dropped like other unusable refresh state; the UID itself is kept. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/validate.rs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index a189fcd25a..367e049e55 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -141,10 +141,11 @@ fn stored_uid( return Ok(stored); }; // Unusable refresh state is dropped, never the UID: a refresh with no - // stated time, or with empty or oversized state, is simply not stored. + // stated time, with empty or oversized state, or due already (which + // would make the source due on every navigation) is simply not stored. let usable = !refresh.state.is_empty() && refresh.state.len() <= MAX_REFRESH_STATE_BYTES; match resolve_expiry(refresh.from, policy.now) { - Some(from) if usable => Ok(stored.with_refresh(KvRefresh { + Some(from) if usable && from > policy.now => Ok(stored.with_refresh(KvRefresh { state: refresh.state.into_owned(), from, })), @@ -275,6 +276,21 @@ mod tests { (String::new(), in_a_minute, None), ("x".repeat(MAX_REFRESH_STATE_BYTES + 1), in_a_minute, None), ("state".to_owned(), ProviderExpiry::Unspecified, None), + ( + "state".to_owned(), + ProviderExpiry::After(Duration::ZERO), + None, + ), + ( + "state".to_owned(), + ProviderExpiry::At(UnixSeconds::new(1000)), + None, + ), + ( + "state".to_owned(), + ProviderExpiry::At(UnixSeconds::new(999)), + None, + ), ]; for (state, from, expected) in cases { From aade0ffda349e932fa2f5b18df6c4f312e9ad7bb Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:53:24 +0530 Subject: [PATCH 33/38] Document collect timing, the after-response budget and rate limit granularity Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/settings.rs | 3 ++ docs/guide/configuration.md | 55 +++++++++++++--------- trusted-server.example.toml | 9 ++++ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 8ea999d82a..148d774624 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -609,6 +609,9 @@ pub struct IdentityProviderConfig { #[serde(default)] pub collect: CollectTiming, /// Most lookups per EC per hour, when set; unset means unlimited. + /// + /// The Fastly limiter enforces this as a per-minute budget of at least + /// one, so values below 60 behave like 60 per hour. #[serde(default)] pub rate_limit_per_hour: Option, /// Secret Store key references, resolved at startup. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 5574ef53d7..50fa54276f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -749,16 +749,17 @@ Legacy consent-store records are not read or migrated into `ec.ec_store`. Their `passphrase` is a key name in `trusted_server_secrets`; the resolved value must be at least 32 bytes. Keep it stable to preserve EC identifier continuity. -| Field | Type | Required | Description | -| ------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | -| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | -| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | -| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | -| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | -| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | -| `partners` | Array | No | Static partner registry entries | -| `identity_budget_ms` | Integer | No | Shared time budget for identity provider lookups (default 200) | -| `identity_providers` | Array | No | Server-side identity providers (see below) | +| Field | Type | Required | Description | +| ----------------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | +| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | +| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | +| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | +| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | +| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | +| `partners` | Array | No | Static partner registry entries | +| `identity_budget_ms` | Integer | No | Shared time budget for identity provider lookups (default 200) | +| `identity_after_response_budget_ms` | Integer | No | Time budget for `collect = "after_response"` lookups (default 2000) | +| `identity_providers` | Array | No | Server-side identity providers (see below) | ::: tip Partner keying `source_domain` is the canonical partner key. It matches incoming OpenRTB EID `source` values and is also used as the EC KV `ids` map key. @@ -812,17 +813,19 @@ When a provider reports an opt-out, its sources are cleared and the row remembers the opt-out for 30 days. The provider is not called for that EC again until the 30 days pass. -| Field | Type | Required | Description | -| ----------------- | ------- | -------- | ----------------------------------------------------------------------- | -| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | -| `kind` | String | Yes | Provider implementation built into this release | -| `enabled` | Boolean | No | Whether the provider runs (default `true`) | -| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | -| `allowed_domains` | Array | Yes | Hosts the provider may call | -| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | -| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | -| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | -| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | +| Field | Type | Required | Description | +| --------------------- | ------- | -------- | ----------------------------------------------------------------------- | +| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | +| `kind` | String | Yes | Provider implementation built into this release | +| `enabled` | Boolean | No | Whether the provider runs (default `true`) | +| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | +| `allowed_domains` | Array | Yes | Hosts the provider may call | +| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | +| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | +| `collect` | String | No | `"page"` (default) or `"after_response"`; see below | +| `rate_limit_per_hour` | Integer | No | Most lookups per EC per hour; unset means unlimited. See the note below | +| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | +| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | **Example**: @@ -841,6 +844,16 @@ sources = ["ids.example.com"] token = "example_provider_token" ``` +`collect = "after_response"` lookups run after the response is sent, within +`identity_after_response_budget_ms`, and only for an EC whose row already +existed before the navigation. Their results reach the row in a follow-up write. + +::: warning Rate limit granularity +On Fastly, `rate_limit_per_hour` is enforced as a per-minute budget rounded up +to at least one lookup per minute, so any value below 60 behaves like 60 per +hour. Values of 60 and above are enforced as `ceil(value / 60)` per minute. +::: + **Validation**: configuration validation fails when an enabled provider's `endpoint` is not HTTPS or its host is not in `allowed_domains`, when a source has no `[[ec.partners]]` entry, or when two enabled providers claim the same diff --git a/trusted-server.example.toml b/trusted-server.example.toml index e0cec612cf..6418f21eaf 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -110,6 +110,9 @@ pull_sync_concurrency = 3 # Optional server-side identity providers, looked up on publisher navigations # before the origin fetch. Every enabled provider shares one time budget: # identity_budget_ms = 200 # default 200 +# Budget for providers with collect = "after_response", which run after the +# response is sent and write their results in a follow-up write: +# identity_after_response_budget_ms = 2000 # # [[ec.identity_providers]] # id = "example-provider" @@ -124,6 +127,12 @@ pull_sync_concurrency = 3 # sources = ["ids.example.com"] # Optional: require TCF vendor consent for this provider. # tcf_vendor_id = 9999 +# "page" (default) collects before the response; "after_response" collects +# after it is sent, only for an EC whose row already exists. +# collect = "page" +# Optional: most lookups per EC per hour. On Fastly this is enforced per +# minute with a minimum of one, so values below 60 behave like 60. +# rate_limit_per_hour = 60 # Secret Store key name, resolved like other secret references. # [ec.identity_providers.secrets] # token = "example_provider_token" From 49ec524ef0fa4791a23ad1d949e0ac82e44fe5f1 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:54:44 +0530 Subject: [PATCH 34/38] Generate IDs only for due sources without a live UID Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 0705e8d14f..11f4c41855 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -281,6 +281,13 @@ impl ProviderCoordinator

{ self.providers.is_empty() } + /// Whether the row holds a live provider-written UID for `source`. + fn holds_live_uid(entry: Option<&KvEntry>, source: &str, now: UnixSeconds) -> bool { + entry + .and_then(|entry| entry.ids.get(source)) + .is_some_and(|record| record.is_provider_written() && record.has_live_uid(now)) + } + /// Claimed sources lacking a live provider-written UID (a browser /// fallback does not count), minus sources needing input V1 /// never has; the flag reports whether any source was removed for input. @@ -523,12 +530,20 @@ impl IdentityCoordinator for ProviderCoordinator

{ current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), hashed_email: None, }; - if let Some(eids) = registered.provider.generate(&input) { - pending.generated.push(Tagged { - provider, - value: self.validated(provider, eids, lookup.now), + if let Some(mut eids) = registered.provider.generate(&input) { + // Only a due source without a live UID may take a generated + // ID: a refresh-due source keeps its UID and goes to lookup. + eids.retain(|eid| { + due.iter().any(|source| source.as_str() == eid.source) + && !Self::holds_live_uid(entry, &eid.source, lookup.now) }); - continue; + if !eids.is_empty() { + pending.generated.push(Tagged { + provider, + value: self.validated(provider, eids, lookup.now), + }); + continue; + } } if Self::rate_limited(registered, lookup) { pending.early.push(skip(SkipReason::RateLimited)); @@ -1526,6 +1541,35 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send no lookup"); } + #[test] + fn a_refresh_due_live_source_is_looked_up_not_regenerated() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + ); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + lookup + .entries + .iter() + .flat_map(|(_, eids)| eids) + .flat_map(KvEid::uids) + .all(|uid| uid.id().as_str() != "generated-1"), + "should not replace the live UID with a generated one" + ); + assert_eq!( + transport.sent.borrow().len(), + 1, + "should send the refresh lookup instead" + ); + } + #[test] fn a_held_source_is_not_regenerated() { let coordinator = generating_coordinator(); From dbb9fa60465a8771de885b5b08fca50f6934da6f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:56:00 +0530 Subject: [PATCH 35/38] Reuse the partner registry and core clock for after-response lookups Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-fastly/src/main.rs | 10 +++++----- crates/trusted-server-identity-providers/src/lib.rs | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 1d4c9057cb..134889694b 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -19,6 +19,7 @@ use trusted_server_core::ec::after_response::{ use trusted_server_core::ec::coordinator::IdentityCoordinator as _; use trusted_server_core::ec::coordinator::LookupContext; use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; +use trusted_server_core::ec::current_timestamp; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::finalize::ec_finalize_response; @@ -764,7 +765,7 @@ fn run_edgezero_pull_sync_after_send( /// for. fn run_after_response_lookups( settings: &Settings, - partner_registry: &PartnerRegistry, + partner_registry: &Arc, ec_state: &mut EcFinalizeState, lookups: Option, ) { @@ -781,7 +782,8 @@ fn run_after_response_lookups( log::debug!("After-response lookups skipped: EC changed during the request"); return; } - let coordinator = match build_after_response_coordinator(settings) { + let coordinator = match build_after_response_coordinator(settings, Arc::clone(partner_registry)) + { Ok(Some(coordinator)) => coordinator, Ok(None) => return, Err(report) => { @@ -804,9 +806,7 @@ fn run_after_response_lookups( let transport = PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_secs()); + let now = current_timestamp(); let pending = futures::executor::block_on(coordinator.dispatch( &transport, &LookupContext { diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index cb6e08118e..17f083be0f 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -97,22 +97,22 @@ pub fn build_coordinator_with( } /// Builds the coordinator for enabled providers collected after the response -/// has been sent (`collect = "after_response"`). +/// has been sent (`collect = "after_response"`), against an already built +/// partner registry. /// /// Its budget is `identity_after_response_budget_ms`. Returns `None` when no /// such provider is enabled. /// /// # Errors /// -/// Returns the same errors as [`build_coordinator`]. +/// Returns the same errors as [`build_coordinator_with`]. pub fn build_after_response_coordinator( settings: &Settings, + partners: Arc, ) -> Result, Report> { - let partners = partner_registry(&settings.ec) - .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; build_phase_coordinator( settings, - Arc::new(partners), + partners, CollectTiming::AfterResponse, Duration::from_millis(settings.ec.identity_after_response_budget_ms), ) From 1826524b7b9c650f7bd3ad42a118315675a4e7f4 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:57:05 +0530 Subject: [PATCH 36/38] Hide the body and context from provider response debug output Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/provider.rs | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 302b120b25..e0e50a9841 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -265,7 +265,10 @@ impl fmt::Debug for ProviderRequest { } /// A provider response, borrowed from the transport. -#[derive(Debug, Clone, Copy)] +/// +/// Its `Debug` output omits the body, headers and context, which can hold +/// identifiers or a reply-decryption key. +#[derive(Clone, Copy)] pub struct ProviderResponse<'body> { /// Response status. pub status: http::StatusCode, @@ -277,6 +280,16 @@ pub struct ProviderResponse<'body> { pub context: Option<&'body str>, } +impl fmt::Debug for ProviderResponse<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderResponse") + .field("status", &self.status) + .field("body_len", &self.body.len()) + .field("has_context", &self.context.is_some()) + .finish_non_exhaustive() + } +} + /// A parsed provider response. #[derive(Debug)] pub enum LookupOutcome<'body> { @@ -672,6 +685,29 @@ mod tests { assert!(printed.contains("ids.example.com"), "should print host"); } + #[test] + fn response_debug_hides_body_and_context() { + let headers = http::HeaderMap::new(); + let response = ProviderResponse { + status: http::StatusCode::OK, + headers: &headers, + body: b"example-uid", + context: Some("example-key"), + }; + + let printed = format!("{response:?}"); + + assert!( + !printed.contains("example-uid"), + "should not print the body" + ); + assert!( + !printed.contains("example-key"), + "should not print the context" + ); + assert!(printed.contains("200"), "should print the status"); + } + #[test] fn validate_enabled_configs_accepts_valid_config() { validate_enabled_configs([&config(&json!({}))], &partners()) From bc65fb9b1c4a83e6fd2136a519cd81de2f4e17c7 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:58:42 +0530 Subject: [PATCH 37/38] Plan no row write when pruning is the only change Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/write_plan.rs | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 0a6e087521..3b543e2a56 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -74,6 +74,8 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } Vec::new() }; + // Pruning alone never triggers a write; it rides along with a real change. + let changed = entry != base; for record in entry.ids.values_mut() { record.prune_expired(input.now); } @@ -83,7 +85,7 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { match expected { None => Some(RowChange::Create(entry)), - Some(_) if entry == base => None, + Some(_) if !changed || entry == base => None, Some(expected) => Some(RowChange::Update { entry, expected }), } } @@ -578,7 +580,7 @@ mod tests { } #[test] - fn lapsed_opt_out_is_pruned() { + fn lapsed_opt_out_alone_plans_no_write() { let registry = registry(&["ids.example.com"]); let ingest: [PartnerIdUpdate; 0] = []; let mut entry = live_entry(); @@ -587,6 +589,21 @@ mod tests { .insert(ProviderId::new("example-provider"), 1000); let lookups = LookupResults::default(); + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + assert!(change.is_none(), "should not write only to prune"); + } + + #[test] + fn lapsed_opt_out_is_pruned_with_a_real_change() { + let registry = registry(&["ids.example.com"]); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-1")]; + let mut entry = live_entry(); + entry + .opt_outs + .insert(ProviderId::new("example-provider"), 1000); + let lookups = LookupResults::default(); + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)) .expect("should plan update"); @@ -596,6 +613,44 @@ mod tests { assert!(entry.opt_outs.is_empty(), "should forget a lapsed opt-out"); } + #[test] + fn not_due_skip_with_an_expired_uid_plans_no_write() { + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut entry = live_entry(); + let expired = KvUid::new( + UidValue::try_from("old-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(1500)), + ); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![expired])], + UnixSeconds::new(1000), + ) + .expect("should build provider record"), + ); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::NotDue), + entries: Vec::new(), + cleared: Vec::new(), + }], + ); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + assert!( + change.is_none(), + "should not write only to prune an expired UID" + ); + } + fn provider_entry(ids: &[&str]) -> KvEntry { let mut entry = live_entry(); let lookups = found("ids.example.com", ids); From b334e75e4ce756d75f3ef2f594420d7c8a97e7bc Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:00:44 +0530 Subject: [PATCH 38/38] Test after-response write conflicts and a row withdrawn before the write Signed-off-by: dhruv8sh --- .../src/ec/after_response.rs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index f7e9206ce3..59abf86484 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -128,7 +128,9 @@ mod tests { use super::*; use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv::TombstoneOutcome; use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; + use crate::ec::page_load::{Generation, KvWriteError}; use crate::settings::EcPartner; const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; @@ -268,6 +270,72 @@ mod tests { ); } + #[test] + fn follow_up_write_conflicts_when_the_row_changed_after_its_read() { + // Arrange + let graph = KvIdentityGraph::in_memory("after-response-conflict"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read = graph.load_snapshot(EC_ID); + let expected = Generation::of(&read, EC_ID).expect("should read a generation"); + graph + .upsert_partner_id(EC_ID, "browser.example.com", "b-2") + .expect("should rewrite the row as another request would"); + + // Act + let result = graph.commit_after_response( + AfterResponseWrite::for_test(), + EC_ID, + &live_entry(), + expected, + ); + + // Assert + let report = result.expect_err("should refuse a stale generation"); + assert!( + matches!(report.current_context(), KvWriteError::Conflict), + "should report a conflict" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-2"), + "should keep the other request's write" + ); + } + + #[test] + fn follow_up_write_never_revives_a_row_withdrawn_after_its_read() { + // Arrange: scenario 23, the row is tombstoned between read and CAS. + let graph = KvIdentityGraph::in_memory("after-response-withdrawn"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read = graph.load_snapshot(EC_ID); + let expected = Generation::of(&read, EC_ID).expect("should read a generation"); + assert_eq!( + graph + .write_withdrawal_tombstone(EC_ID, drop) + .expect("should withdraw the row"), + TombstoneOutcome::Written, + "should tombstone the row" + ); + + // Act + let result = graph.commit_after_response( + AfterResponseWrite::for_test(), + EC_ID, + &live_entry(), + expected, + ); + + // Assert + assert!(result.is_err(), "should refuse to overwrite the tombstone"); + assert!( + graph + .load_snapshot(EC_ID) + .entry_for(EC_ID) + .is_none_or(|entry| !entry.consent.ok), + "should leave the row withdrawn" + ); + } + #[test] fn replaces_a_browser_fallback_for_an_owned_source() { let graph = KvIdentityGraph::in_memory("after-response");