diff --git a/.github/workflows/build-daemon.yml b/.github/workflows/build-daemon.yml index 19fcebfb8..d4ba65a27 100644 --- a/.github/workflows/build-daemon.yml +++ b/.github/workflows/build-daemon.yml @@ -94,7 +94,7 @@ jobs: os: macos-15-intel platform: darwin-x64 - target: aarch64-apple-darwin - os: macos-14 + os: macos-15 platform: darwin-arm64 runs-on: ${{ matrix.os }} # Nothing in this job has a bounded runtime of its own, and GitHub's default diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f922da05b..854df0e28 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -842,7 +842,7 @@ jobs: platform: linux-arm64 - os: macos-15-intel platform: darwin-x64 - - os: macos-14 + - os: macos-15 platform: darwin-arm64 runs-on: ${{ matrix.os }} timeout-minutes: 20 diff --git a/CHANGELOG.md b/CHANGELOG.md index d79aa292b..bb68329ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 1.0.14-beta.0 — 2026-11-10 + +### Fixes + +- Keep telemetry retryable through prolonged network and server outages, retain interrupted acknowledgements, and preserve permanent rejections for inspection. (#997) +- Build the macOS ARM daemon on the supported macos-15 hosted runner. (#997) +- Count partial acknowledgements separately from outage attempts so a recovered endpoint can finish delivery. (#997) +- Keep credential-refused batches recoverable after an outage without exhausting the skipped-acknowledgement budget. (#997) +- Run the ARM installation check on macos-15 so a release can complete. (#997) +- Keep CLI delivery-status filename parsing aligned with the independent retry counters. (#997) + +### Docs + +- Clarify ingestion retries and give every public OpenAPI operation a unique ID. (#997) +- Explain manual OTLP credential-refusal recovery within the original organisation. (#997) + +### Dependencies + +- Update the Python SDK test dependency oauthlib to 4.0.1 to fix GHSA-3jp2-3mrg-2vxc and unblock the security gate. (#997) + ## 1.0.14-beta.0 — 2026-11-02 ### Docs diff --git a/__tests__/hooks/delivery-health.test.ts b/__tests__/hooks/delivery-health.test.ts index ccec651d8..f38b55512 100644 --- a/__tests__/hooks/delivery-health.test.ts +++ b/__tests__/hooks/delivery-health.test.ts @@ -65,6 +65,23 @@ describe("hooks/delivery-health parseParkedName", () => { expect(parsed.base).toBe("claude-0"); }); + it.each([ + ["claude-0.a99.s1.r2.c401.jsonl", 99, 401, false], + ["claude-0.a5.s1.jsonl", 5, undefined, false], + ["claude-0.a2.r1.c403.jsonl", 2, 403, false], + ["claude-0.a9.s3.r2.c422.jsonl.poison", 9, 422, true], + ])("reads independent retry budgets in %s", (name, attempt, clientStatus, poison) => { + expect(parseParkedName(name as string)).toEqual({ base: "claude-0", attempt, clientStatus, poison }); + }); + + it.each(["claude-0.a5.soon.jsonl", "claude-0.a5.r-1.jsonl", "claude-0.a5.r4294967296.jsonl"])( + "leaves an invalid retry suffix in %s alone", + (name) => { + expect(parseParkedName(name).base).toBe(name.slice(0, -".jsonl".length)); + expect(parseParkedName(name).attempt).toBe(0); + }, + ); + it("leaves a non-numeric .c/.a in the base alone", () => { // Mirrors the Rust comment: parsing rather than merely matching the prefix // is what keeps a real spool name safe. diff --git a/crates/fpai-collect/src/delivery.rs b/crates/fpai-collect/src/delivery.rs index 817298a14..69ed7d974 100644 --- a/crates/fpai-collect/src/delivery.rs +++ b/crates/fpai-collect/src/delivery.rs @@ -628,7 +628,7 @@ mod tests { .await; let failed = tmpdir("redrive-failed"); let spool = tmpdir("redrive-spool"); - let refused_again = failed.join("a-0.a1.c401.jsonl"); + let refused_again = failed.join("a-0.a1.r1.c401.jsonl"); let resendable = failed.join("b-0.a1.c403.jsonl"); let not_credential = failed.join("c-0.a1.c404.jsonl"); std::fs::write(&refused_again, "{\"refused\":1}\n").unwrap(); @@ -656,12 +656,12 @@ mod tests { "refused again by that key: renamed into another park" ); assert_eq!( - std::fs::read(failed.join("a-0.a2.c401.jsonl")).unwrap(), + std::fs::read(failed.join("a-0.a2.r2.c401.jsonl")).unwrap(), b"{\"refused\":1}\n" ); delivery.redrive_refused().await; assert_eq!( - std::fs::read(failed.join("a-0.a3.c401.jsonl.poison")).unwrap(), + std::fs::read(failed.join("a-0.a3.r3.c401.jsonl.poison")).unwrap(), b"{\"refused\":1}\n", "exhausting retries must preserve the last local copy too" ); @@ -770,7 +770,7 @@ mod tests { )); delivery.deliver(fresh).await; assert!(old.exists(), "a key that is refused re-sends nothing"); - assert!(failed.join("hooks-a-1-0.a1.c401.jsonl").exists()); + assert!(failed.join("hooks-a-1-0.a1.r1.c401.jsonl").exists()); std::fs::remove_dir_all(failed).unwrap(); std::fs::remove_dir_all(spool).unwrap(); } diff --git a/crates/fpai-collect/src/otlp.rs b/crates/fpai-collect/src/otlp.rs index 56f8afc45..9e267e54c 100644 --- a/crates/fpai-collect/src/otlp.rs +++ b/crates/fpai-collect/src/otlp.rs @@ -1,6 +1,14 @@ //! Opaque OTLP/HTTP relay. No translation, decompression or event-uploader reuse. //! Each batch is a JSON metadata line followed by the exact request bytes. //! SDK exporters publish the same format; only atomic .jsonl files are read. +//! +//! Manual recovery of credential refusals (401/403): stop the daemon and verify +//! a working `events:add` key for the SAME Cloud origin and organization. Back up +//! the affected files under `/state/failed/otlp/`, then move only +//! those batches into `state/spool-otlp/` with fresh `otlp-.jsonl` names, +//! never overwriting another file or changing their metadata/body bytes. Restart +//! the daemon and confirm delivery before removing the backup. Never recover +//! queues archived by `org_boundary` into a different organization's spool. use std::fs::{self, File, OpenOptions}; use std::io::{self, Write}; use std::net::{Ipv4Addr, SocketAddr}; @@ -175,9 +183,21 @@ impl Relay { // Bound response size too; an ingest proxy need not be trusted. let mut response = response; let mut result = Vec::new(); - while let Ok(Some(chunk)) = response.chunk().await { + loop { + let chunk = match response.chunk().await { + Ok(Some(chunk)) => chunk, + Ok(None) => break, + Err(_) => { + self.report( + None, + Some("response interrupted; retained".into()), + ); + return Outcome::Unreachable; + } + }; if result.len() + chunk.len() > 64 * 1024 { - break; + self.park(path, Some(code)).await; + return Outcome::Settled; } result.extend_from_slice(&chunk); } @@ -190,7 +210,7 @@ impl Relay { return Outcome::Settled; } self.report(None, Some(format!("{code} parked"))); - if !matches!(code, 429 | 502 | 503 | 504) { + if !retryable_status(code) { break; } delay = retry_after(response.headers()).unwrap_or(delay); @@ -219,9 +239,12 @@ impl Relay { return; }; let mut parked = ParkedName::parse(name); - parked.attempt += 1; - parked.client_status = status.filter(|c| (400..500).contains(c) && *c != 429); - parked.poison = parked.attempt >= 3; + parked.attempt = parked.attempt.saturating_add(1); + parked.client_status = status.filter(|c| !retryable_status(*c)); + if !status.is_some_and(retryable_status) { + parked.client_attempts = parked.client_attempts.saturating_add(1); + } + parked.poison = !status.is_some_and(retryable_status) && parked.client_attempts >= 3; let target = self.failed.join(parked.render()); if tokio::fs::rename(path, target).await.is_ok() { let _ = File::open(&self.failed).and_then(|f| f.sync_all()); @@ -229,6 +252,10 @@ impl Relay { } } +fn retryable_status(code: u16) -> bool { + matches!(code, 408 | 429 | 500..=599) +} + fn retry_after(headers: &HeaderMap) -> Option { let value = headers.get("retry-after")?.to_str().ok()?; if let Ok(seconds) = value.parse::() { @@ -454,7 +481,10 @@ async fn deliver_loop(relay: Arc, sd: Shutdown) -> Result<(), TaskError> let mut delivered = 0; while let Ok(Some(entry)) = entries.next_entry().await { let name = entry.file_name().to_string_lossy().to_string(); - if !name.starts_with("otlp-") || !name.ends_with(".jsonl") { + if !name.starts_with("otlp-") + || !name.ends_with(".jsonl") + || !ParkedName::parse(&name).is_auto_retryable() + { continue; } if entry.file_type().await.is_ok_and(|t| t.is_file()) { @@ -679,13 +709,127 @@ mod tests { parked .to_str() .unwrap() - .contains(&format!(".a1.c{code}.jsonl")) + .contains(&format!(".a1.r1.c{code}.jsonl")) ); assert_eq!(fs::read(&parked).unwrap(), original); + assert!( + !ParkedName::parse(parked.file_name().unwrap().to_str().unwrap()) + .is_auto_retryable() + ); + fs::remove_dir_all(home).unwrap(); + } + } + + #[tokio::test] + async fn temporary_http_failures_remain_retryable_after_four_parks() { + for status in [429, 500, 503] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(status)) + .expect(20) + .mount(&server) + .await; + let home = temp(); + let mut relay = Relay::new( + &home, + &Ingest { + url: server.uri(), + key: "test".into(), + }, + ) + .unwrap(); + relay.retry_base = Duration::from_millis(1); + let mut file = batch(&relay, b"kept"); + let original = fs::read(&file).unwrap(); + for _ in 0..4 { + relay.deliver(&file, &shutdown()).await; + file = fs::read_dir(&relay.failed) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + assert!( + ParkedName::parse(file.file_name().unwrap().to_str().unwrap()) + .is_auto_retryable() + ); + assert_eq!(fs::read(&file).unwrap(), original); + } + server.verify().await; + server.reset().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(401)) + .expect(1) + .mount(&server) + .await; + relay.deliver(&file, &shutdown()).await; + file = fs::read_dir(&relay.failed) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let state = ParkedName::parse(file.file_name().unwrap().to_str().unwrap()); + assert_eq!(state.client_status, Some(401)); + assert!(!state.poison); + assert!(!state.is_auto_retryable()); + assert_eq!(fs::read(&file).unwrap(), original); + server.verify().await; + server.reset().await; + Mock::given(method("POST")) + .and(header("authorization", "Bearer replacement")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + // Follow the documented manual requeue after fixing same-org credentials. + relay.key = "replacement".into(); + let requeued = relay.spool.join(format!("otlp-{}.jsonl", new_request_id())); + fs::rename(&file, &requeued).unwrap(); + assert_eq!(fs::read(&requeued).unwrap(), original); + assert!( + ParkedName::parse(requeued.file_name().unwrap().to_str().unwrap()) + .is_auto_retryable() + ); + file = requeued; + relay.deliver(&file, &shutdown()).await; + assert!(!file.exists()); + server.verify().await; fs::remove_dir_all(home).unwrap(); } } + #[tokio::test] + async fn interrupted_success_response_keeps_the_spool_batch() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = [0; 4096]; + assert!(stream.read(&mut request).await.unwrap() > 0); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{").await.unwrap(); + }); + let home = temp(); + let relay = Relay::new( + &home, + &Ingest { + url, + key: "key".into(), + }, + ) + .unwrap(); + let file = batch(&relay, b"kept"); + let original = fs::read(&file).unwrap(); + assert!(matches!( + relay.deliver(&file, &shutdown()).await, + Outcome::Unreachable + )); + assert_eq!(fs::read(&file).unwrap(), original); + server.await.unwrap(); + fs::remove_dir_all(home).unwrap(); + } + #[tokio::test] async fn transport_failures_never_poison_and_the_batch_delivers_once_reachable() { let reserved = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); diff --git a/crates/fpai-collect/src/uploader.rs b/crates/fpai-collect/src/uploader.rs index c953069c9..0a25beea9 100644 --- a/crates/fpai-collect/src/uploader.rs +++ b/crates/fpai-collect/src/uploader.rs @@ -21,8 +21,8 @@ //! //! **`failed/` is a retry queue, not a graveyard.** A batch there is data the //! server does not have, and it is the last copy. Batches are retried in place -//! with the attempt count encoded in the filename, and only ever parked as -//! `.poison` — never deleted. +//! with the attempt count encoded in the filename. Temporary failures never +//! exhaust into `.poison`; rejected batches are retained, never deleted. //! //! **Every attempt says who it is.** Three headers go with each POST so a //! failure can be found in the server's logs: `x-request-id` (new per attempt, @@ -59,7 +59,7 @@ pub const DEFAULT_MAX_RETRIES: u32 = 5; pub const DEFAULT_RETRY_BASE: Duration = Duration::from_millis(1000); /// Per-read timeout. Bounds progress, not total transfer size. pub const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(120); -/// Attempts a parked batch gets before it is marked poison. +/// Separate budgets for skipped acknowledgements and definitive client rejections. pub const DEFAULT_FAILED_RETRIES_MAX: u32 = 3; /// Header names. The server validates the request id as a W3C trace id (32 /// lowercase hex) and mints its own when it is not one. @@ -379,7 +379,17 @@ impl Uploader { // and returning Ok here deletes the spool file behind it. // Park instead, non-retryable — it will parse-fail // identically until the URL is fixed. - match resp.json::().await { + let ack_bytes = match resp.bytes().await { + Ok(bytes) => bytes, + Err(error) => { + self.park(path, None, &request_id, true).await; + return Err(UploadError::Network { + attempts: attempt, + detail: error_chain(&error), + }); + } + }; + match serde_json::from_slice::(&ack_bytes) { Ok(ack) => { // `record_ack`'s own contract: "A 200 that stored // nothing is an error, not a success." It said so @@ -397,7 +407,7 @@ impl Uploader { // `.poison` at `failed_retries_max`, after which it // is kept forever and never retried again. if self.record_ack(path, &ack, attempt, &request_id) { - self.park(path, None, &request_id).await; + self.park(path, None, &request_id, false).await; return Err(if ack.accepted == 0 { UploadError::StoredNothing { skipped: ack.skipped, @@ -413,7 +423,7 @@ impl Uploader { } Err(_) => { let code = status.as_u16(); - self.park(path, Some(code), &request_id).await; + self.park(path, Some(code), &request_id, false).await; return Err(UploadError::Client { status: code }); } } @@ -428,11 +438,11 @@ impl Uploader { if is_credential_status(code) { self.credential_refusals.store(true, Ordering::SeqCst); } - self.park(path, Some(code), &request_id).await; + self.park(path, Some(code), &request_id, false).await; return Err(UploadError::Client { status: code }); } if attempt >= self.max_retries { - self.park(path, None, &request_id).await; + self.park(path, None, &request_id, true).await; return Err(UploadError::Server { status: code, attempts: attempt, @@ -455,7 +465,7 @@ impl Uploader { } Err(err) => { if attempt >= self.max_retries { - self.park(path, None, &request_id).await; + self.park(path, None, &request_id, true).await; return Err(UploadError::Network { attempts: attempt, detail: error_chain(&err), @@ -503,11 +513,11 @@ impl Uploader { // Keyed on the FILE NAME, not the retry counter: `upload_file`'s own // `attempt` restarts at 1 for every call, including each call made from // `retry_parked`. The parked name is where the batch's history actually - // lives (`.aN`), so a file already carrying one has been counted. + // lives (`.sN`); outage attempts must not hide the first skipped ack. let first_attempt = path .file_name() .and_then(|n| n.to_str()) - .map(|n| ParkedName::parse(n).attempt == 0) + .map(|n| ParkedName::parse(n).skipped_attempts == 0) .unwrap_or(true); if first_attempt { self.metrics @@ -586,8 +596,17 @@ impl Uploader { /// not written beside the batch: the filename is this module's only /// metadata store (see `ParkedName`), and a sidecar is exactly what that /// design rejects. - async fn park(&self, path: &Path, client_status: Option, request_id: &str) { - if let Err(err) = self.park_inner(path, client_status, request_id).await { + async fn park( + &self, + path: &Path, + client_status: Option, + request_id: &str, + retryable: bool, + ) { + if let Err(err) = self + .park_inner(path, client_status, request_id, retryable) + .await + { tracing::error!( file = %path.display(), request_id, @@ -602,6 +621,7 @@ impl Uploader { path: &Path, client_status: Option, request_id: &str, + retryable: bool, ) -> Result<(), std::io::Error> { tokio::fs::create_dir_all(&self.failed_dir).await?; let name = path @@ -610,11 +630,21 @@ impl Uploader { .unwrap_or("batch.jsonl"); let mut parked = ParkedName::parse(name); - parked.attempt += 1; - if client_status.is_some() { - parked.client_status = client_status; + parked.attempt = parked.attempt.saturating_add(1); + parked.client_status = client_status; + if !retryable && client_status.is_none() { + parked.skipped_attempts = parked.skipped_attempts.saturating_add(1); + } + if !retryable && client_status.is_some() { + parked.client_attempts = parked.client_attempts.saturating_add(1); } - parked.poison = parked.attempt >= self.failed_retries_max.max(1); + // Outages spend neither rejection budget; repeated credential refusals stay bounded. + let rejections = if client_status.is_some() { + parked.client_attempts + } else { + parked.skipped_attempts + }; + parked.poison = !retryable && rejections >= self.failed_retries_max.max(1); let dest = unique_destination(&self.failed_dir, &parked.render()).await; tracing::warn!( @@ -823,7 +853,7 @@ pub fn error_chain(err: &dyn std::error::Error) -> String { } /// Retry state carried in a parked batch's filename: -/// `.a[.c].jsonl[.poison]`. +/// `.a[.s][.r][.c].jsonl[.poison]`. /// /// The filename IS the database. A sidecar would need its own atomicity story /// and could desynchronise from the batch it describes; a rename cannot. @@ -831,6 +861,8 @@ pub fn error_chain(err: &dyn std::error::Error) -> String { pub struct ParkedName { pub base: String, pub attempt: u32, + pub skipped_attempts: u32, + pub client_attempts: u32, pub client_status: Option, pub poison: bool, } @@ -845,9 +877,11 @@ impl ParkedName { let mut base = stem.to_string(); let mut attempt = 0; + let mut skipped_attempts = 0; + let mut client_attempts = 0; let mut client_status = None; - // Suffixes are stripped right to left: `.cNNN` then `.aN`. Parsing + // Suffixes are stripped right to left: `.cNNN`, `.rN`, `.sN`, `.aN`. Parsing // rather than merely matching the prefix is what keeps a real spool // name safe — `claude-...-0` has no `.c`/`.a` to find, and anything // that does but is not numeric is left as part of the base. @@ -857,6 +891,18 @@ impl ParkedName { client_status = Some(code); base.truncate(idx); } + if let Some(idx) = base.rfind(".r") + && let Ok(n) = base[idx + 2..].parse::() + { + client_attempts = n; + base.truncate(idx); + } + if let Some(idx) = base.rfind(".s") + && let Ok(n) = base[idx + 2..].parse::() + { + skipped_attempts = n; + base.truncate(idx); + } if let Some(idx) = base.rfind(".a") && let Ok(n) = base[idx + 2..].parse::() { @@ -867,6 +913,8 @@ impl ParkedName { ParkedName { base, attempt, + skipped_attempts, + client_attempts, client_status, poison, } @@ -874,6 +922,12 @@ impl ParkedName { pub fn render(&self) -> String { let mut s = format!("{}.a{}", self.base, self.attempt); + if self.skipped_attempts > 0 { + s.push_str(&format!(".s{}", self.skipped_attempts)); + } + if self.client_attempts > 0 { + s.push_str(&format!(".r{}", self.client_attempts)); + } if let Some(code) = self.client_status { s.push_str(&format!(".c{code}")); } @@ -962,6 +1016,8 @@ mod tests { let p = ParkedName { base: "hooks-s1-123-0".into(), attempt: 2, + skipped_attempts: 0, + client_attempts: 0, client_status: Some(404), poison: false, }; @@ -974,6 +1030,8 @@ mod tests { let p = ParkedName { base: "claude-s-1-0".into(), attempt: 3, + skipped_attempts: 0, + client_attempts: 0, client_status: None, poison: true, }; @@ -992,11 +1050,22 @@ mod tests { // one that has already been retried. let p = ParkedName::parse("claude-3ee9c788-1785741108180149712-0.jsonl"); assert_eq!(p.attempt, 0); + assert_eq!(p.skipped_attempts, 0); assert_eq!(p.client_status, None); assert!(!p.poison); assert_eq!(p.base, "claude-3ee9c788-1785741108180149712-0"); } + #[test] + fn skipped_ack_state_survives_renames_and_old_names_start_with_no_skips() { + let old = ParkedName::parse("batch.a99.jsonl"); + assert_eq!(old.skipped_attempts, 0); + let mixed = ParkedName::parse("batch.a99.s2.c422.jsonl"); + assert_eq!(mixed.base, "batch"); + assert_eq!((mixed.attempt, mixed.skipped_attempts), (99, 2)); + assert_eq!(mixed.render(), "batch.a99.s2.c422.jsonl"); + } + #[test] fn only_batches_with_no_definitive_rejection_are_auto_retried() { let park = |n: &str| ParkedName::parse(n); diff --git a/crates/fpai-collect/tests/uploader.rs b/crates/fpai-collect/tests/uploader.rs index ba3be6d7b..8c4a4a476 100644 --- a/crates/fpai-collect/tests/uploader.rs +++ b/crates/fpai-collect/tests/uploader.rs @@ -92,7 +92,7 @@ async fn a_later_chunk_credential_refusal_preserves_all_bytes_and_can_be_resent( assert_eq!(requests[0].body, b"{ \"n\": 1 }\n"); assert_eq!(requests[1].body, b"{ \"n\": 2 }\n"); let names = parked(&failed); - assert_eq!(names, [format!("original.a2.c{status}.jsonl")]); + assert_eq!(names, [format!("original.a2.r1.c{status}.jsonl")]); let retained = failed.join(&names[0]); assert_eq!( fs::read(&retained).unwrap(), @@ -927,3 +927,246 @@ async fn an_unset_machine_id_is_sent_as_unknown() { fs::remove_dir_all(&spool).ok(); fs::remove_dir_all(&failed).ok(); } + +#[tokio::test] +async fn temporary_rejections_outlive_the_old_poison_budget_and_recover() { + use fpai_collect::uploader::ParkedName; + for status in [408, 429, 500, 503] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(status)) + .expect(20) + .mount(&server) + .await; + let spool = tmpdir("long-outage"); + let failed = tmpdir("long-outage-failed"); + let mut file = write_batch(&spool, "outage.jsonl", 2); + let original = fs::read(&file).unwrap(); + let up = uploader(&server, &failed); + for round in 1..=4 { + assert!(up.upload_file(&file).await.is_err()); + let names = parked(&failed); + assert_eq!(names.len(), 1); + let state = ParkedName::parse(&names[0]); + assert_eq!(state.attempt, round); + assert!(state.is_auto_retryable(), "{status}: {state:?}"); + file = failed.join(&names[0]); + assert_eq!(fs::read(&file).unwrap(), original); + } + server.verify().await; + server.reset().await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({"accepted": 2, "skipped": 0})), + ) + .expect(1) + .mount(&server) + .await; + up.upload_file(&file).await.unwrap(); + assert!(!file.exists()); + server.verify().await; + fs::remove_dir_all(spool).unwrap(); + fs::remove_dir_all(failed).unwrap(); + } +} + +#[tokio::test] +async fn outages_do_not_spend_the_partial_ack_budget() { + use fpai_collect::uploader::ParkedName; + for recover in [true, false] { + let server = MockServer::start().await; + let spool = tmpdir("outage-partial"); + let failed = tmpdir("outage-partial-failed"); + let mut file = write_batch(&spool, "mixed.jsonl", 2); + let original = fs::read(&file).unwrap(); + let up = uploader(&server, &failed); + for skipped in 1..=3 { + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(500)) + .expect(20) + .mount(&server) + .await; + for _ in 0..4 { + up.upload_file(&file).await.unwrap_err(); + let names = parked(&failed); + assert_eq!(names.len(), 1); + file = failed.join(&names[0]); + let state = ParkedName::parse(&names[0]); + assert!(state.is_auto_retryable()); + assert_eq!(state.skipped_attempts, skipped - 1); + assert_eq!(fs::read(&file).unwrap(), original); + } + server.verify().await; + server.reset().await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({"accepted": 1, "skipped": 1})), + ) + .expect(1) + .mount(&server) + .await; + assert!(matches!( + up.upload_file(&file).await, + Err(UploadError::PartiallySkipped { .. }) + )); + let names = parked(&failed); + assert_eq!(names.len(), 1); + file = failed.join(&names[0]); + let state = ParkedName::parse(&names[0]); + assert_eq!(state.skipped_attempts, skipped); + assert_eq!(state.poison, skipped == 3); + assert_eq!(fs::read(&file).unwrap(), original); + assert_eq!(up.metrics().skipped_total.load(Ordering::Relaxed), 1); + server.verify().await; + server.reset().await; + if recover { + assert!(state.is_auto_retryable()); + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({"accepted": 2, "skipped": 0})), + ) + .expect(1) + .mount(&server) + .await; + up.upload_file(&file).await.unwrap(); + assert!(!file.exists()); + server.verify().await; + break; + } + } + fs::remove_dir_all(spool).unwrap(); + fs::remove_dir_all(failed).unwrap(); + } +} + +#[tokio::test] +async fn outages_followed_by_a_client_rejection_keep_credential_recovery_available() { + use fpai_collect::uploader::ParkedName; + for status in [401, 403, 422] { + let server = MockServer::start().await; + let failed = tmpdir("outage-client"); + // Legacy outage state has no skipped-ack history, even at saturation. + let file = write_batch(&failed, "mixed.a4294967295.jsonl", 2); + let original = fs::read(&file).unwrap(); + let up = uploader(&server, &failed); + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(status)) + .expect(1) + .mount(&server) + .await; + assert!( + matches!(up.upload_file(&file).await, Err(UploadError::Client { status: s }) if s == status) + ); + let names = parked(&failed); + assert_eq!(names.len(), 1); + let state = ParkedName::parse(&names[0]); + assert_eq!(state.attempt, u32::MAX); + assert_eq!(state.client_status, Some(status)); + assert_eq!(state.client_attempts, 1); + assert!(!state.poison); + assert!(!state.is_auto_retryable()); + assert_eq!(fs::read(failed.join(&names[0])).unwrap(), original); + server.verify().await; + server.reset().await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({"accepted": 2, "skipped": 0})), + ) + .expect(1) + .mount(&server) + .await; + up.upload_file(&failed.join(&names[0])).await.unwrap(); + assert!(parked(&failed).is_empty()); + server.verify().await; + fs::remove_dir_all(failed).unwrap(); + } +} + +#[tokio::test] +async fn network_failures_outlive_the_poison_budget_without_losing_bytes() { + use fpai_collect::uploader::ParkedName; + let spool = tmpdir("network-outage"); + let failed = tmpdir("network-outage-failed"); + let mut file = write_batch(&spool, "network.jsonl", 1); + let original = fs::read(&file).unwrap(); + let up = Uploader::new( + "http://127.0.0.1:1/events".into(), + "key".into(), + failed.clone(), + ) + .unwrap() + .with_retry_base(Duration::from_millis(1)); + for _ in 0..4 { + assert!(matches!( + up.upload_file(&file).await, + Err(UploadError::Network { .. }) + )); + let names = parked(&failed); + assert!(ParkedName::parse(&names[0]).is_auto_retryable()); + file = failed.join(&names[0]); + assert_eq!(fs::read(&file).unwrap(), original); + } + fs::remove_dir_all(spool).unwrap(); + fs::remove_dir_all(failed).unwrap(); +} + +#[tokio::test] +async fn oversized_event_rejection_is_retained_without_automatic_retries() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(422)) + .expect(1) + .mount(&server) + .await; + let spool = tmpdir("oversized"); + let failed = tmpdir("oversized-failed"); + let file = write_batch(&spool, "oversized.jsonl", 1); + let original = fs::read(&file).unwrap(); + assert!(matches!( + uploader(&server, &failed).upload_file(&file).await, + Err(UploadError::Client { status: 422 }) + )); + let names = parked(&failed); + assert!(!fpai_collect::uploader::ParkedName::parse(&names[0]).is_auto_retryable()); + assert_eq!(fs::read(failed.join(&names[0])).unwrap(), original); + server.verify().await; + fs::remove_dir_all(spool).unwrap(); + fs::remove_dir_all(failed).unwrap(); +} + +#[tokio::test] +async fn interrupted_success_acknowledgements_remain_retryable() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/events", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + for _ in 0..4 { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = [0; 4096]; + assert!(stream.read(&mut request).await.unwrap() > 0); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{\"accepted\":").await.unwrap(); + } + }); + let spool = tmpdir("interrupted-ack"); + let failed = tmpdir("interrupted-ack-failed"); + let mut file = write_batch(&spool, "ack.jsonl", 1); + let original = fs::read(&file).unwrap(); + let up = Uploader::new(url, "key".into(), failed.clone()).unwrap(); + for _ in 0..4 { + assert!(matches!( + up.upload_file(&file).await, + Err(UploadError::Network { .. }) + )); + let names = parked(&failed); + assert!(fpai_collect::uploader::ParkedName::parse(&names[0]).is_auto_retryable()); + file = failed.join(&names[0]); + assert_eq!(fs::read(&file).unwrap(), original); + } + server.await.unwrap(); + fs::remove_dir_all(spool).unwrap(); + fs::remove_dir_all(failed).unwrap(); +} diff --git a/docs/agenteye/opentelemetry.mdx b/docs/agenteye/opentelemetry.mdx index b2444f070..fe30700ed 100644 --- a/docs/agenteye/opentelemetry.mdx +++ b/docs/agenteye/opentelemetry.mdx @@ -119,6 +119,16 @@ For an agent you configure manually, use its own content-capture controls: ## Check or undo setup +The relay retains telemetry through temporary network and server failures and +retries after recovery. A permanent `400` rejection, including an event that is +too large to store, requires correcting the payload. The relay keeps the +rejected batch for inspection rather than repeatedly sending it. + +After a `401` or `403`, replace the credential with a key granting `events:add` +for the same organisation. Reconnecting does not automatically resend retained +OTLP exports refused for credentials; ask your operator to recover those batches. +Keep them in their original organisation. + ```bash failproofai otel status failproofai otel disable all diff --git a/docs/reference/http-api.mdx b/docs/reference/http-api.mdx index 106bc94f6..cbd60fc8e 100644 --- a/docs/reference/http-api.mdx +++ b/docs/reference/http-api.mdx @@ -69,6 +69,15 @@ The current specification has complete route, method, parameter, permission, and Use `Content-Type: application/json` for JSON writes. Treat `401` as missing or invalid authentication, `403` as a valid identity without the required permission, `404` as a missing or organization-inaccessible resource, `409` as a state conflict, and `422` as an invalid field or permission value. Error responses include a human-readable message; permission failures also name the required grant. +Event ingestion uses newline-delimited JSON: one complete object per line. Check +both `accepted` and `skipped` in a successful response. Retry the entire batch +after a network failure or an HTTP 408, 429, or 5xx response; replaying the same +events is safe. Historical batches can span many months. If ingestion returns `422` with +`error: "event_too_large"`, reduce the offending event's payload before retrying. +Splitting a batch into smaller requests does not shrink an individual event. +For file imports, reuse the original `existing_before` cutoff on retries so a +partially stored session can finish. + ## Request IDs Every response carries an `X-Request-Id` header, and every JSON error body includes the same value as `request_id`. Quote it when you contact support: it identifies that one request. diff --git a/docs/reference/openapi.json b/docs/reference/openapi.json index ab319110f..8975065d7 100644 --- a/docs/reference/openapi.json +++ b/docs/reference/openapi.json @@ -1229,7 +1229,7 @@ "Auth" ], "description": "Describe the API key making this request: the organization it belongs to and the permissions it holds. Requires no permission of its own — any valid key may describe itself, and only itself. Intended to be called once when provisioning a machine, so a bad key, a missing permission and a wrong organization can be told apart before anything is sent. A key that has just been revoked or has just expired may still be reported as valid for up to a minute.", - "operationId": "handler", + "operationId": "get_auth_introspect", "responses": { "200": { "description": "The key is valid. Reports its organization and effective permissions." @@ -2031,7 +2031,7 @@ ], "summary": "The distinct environment names seen in this organization's evaluations.", "description": "Returns a bare JSON array of strings, suitable for an environment filter.\nRead from the evaluations dataset rather than the event store, so a key with\n`evaluations:read` but not `events:read` can still populate the filter.\nCached per organization for 60 seconds, so a brand-new environment can take\nup to a minute to show up here.", - "operationId": "list_environments", + "operationId": "get_evaluations_environments", "responses": { "200": { "description": "A JSON array of environment names, sorted." @@ -2217,7 +2217,7 @@ ], "summary": "Ingest a batch of events.", "description": "The body is newline-delimited JSON — one event object per line, not a JSON\narray. Malformed lines are SKIPPED, not rejected: the response reports\n`accepted` and `skipped` counts and the status is 200 even when every line\nwas bad. A 5xx means \"retry the whole batch\"; duplicate events are collapsed\nserver-side, so retrying is safe.", - "operationId": "handler", + "operationId": "post_events", "requestBody": { "description": "One JSON event object per line.", "content": { @@ -2241,6 +2241,9 @@ }, "500": { "description": "Storage write failed — retry the batch." + }, + "422": { + "description": "An event exceeds the storage JSON row limit; reduce its payload." } }, "security": [ @@ -2283,7 +2286,7 @@ ], "summary": "List the environment names this organization has actually sent events from.", "description": "A JSON array of strings, alphabetically sorted — the values accepted by the\n`environment` filter on `/events`, `/events/summary` and `/sessions`. Derived\nfrom stored events, so an environment appears only once something has been\ningested under it, and results are cached for up to a minute: a brand-new\nenvironment can take that long to show up.\n\nEither `events:read` or `evaluations:read` grants this, so session-only keys\ncan populate the same filter.", - "operationId": "list_environments", + "operationId": "get_events_environments", "responses": { "200": { "description": "The organization's environment names, sorted." @@ -2869,7 +2872,7 @@ ], "summary": "Liveness probe.", "description": "The only endpoint on `/v1` that needs no credential — it reports that the\nprocess is answering, nothing about your data.", - "operationId": "handler", + "operationId": "get_health", "responses": { "200": { "description": "The server is answering." @@ -4017,7 +4020,7 @@ ], "summary": "Describe the tables a query may read.", "description": "Returns `{schema, tables: [{name, columns: [{name, type}]}]}` — the same\nlisting the query editor's autocomplete and schema browser are built on.\nTypes are normalised display names (`string`, `int`, `float`, `timestamp`,\n`uuid`, `bool`), with a trailing `?` when the column is nullable.\n\nThe shape is the same for every organization; only the rows a query over\nthese tables returns are yours.", - "operationId": "get_schema", + "operationId": "get_queries_schema", "responses": { "200": { "description": "The queryable tables and their columns." @@ -4528,7 +4531,7 @@ ], "summary": "The setting catalogue — key, type, and constraints, with no values.", "description": "Registry metadata only, so it reads nothing per-organization and returns the\nsame document for every caller. Use it to render typed inputs without\nre-declaring the catalogue; use `GET /settings` when you also need values.", - "operationId": "get_schema", + "operationId": "get_settings_schema", "responses": { "200": { "description": "The setting definitions, under `settings`." @@ -4946,6 +4949,103 @@ } ] } + }, + "/events/import": { + "post": { + "tags": [ + "Events" + ], + "summary": "Import one file of agent events into the caller's organization.", + "description": "The body is the file itself, optionally gzipped (detected from its bytes).\nThe format is detected from the content unless `format` names it: our event\nJSONL, a Claude Code transcript, or a Codex rollout. Transcripts are\nconverted exactly as the failproofaid collector converts them live, secrets\nare redacted, and every event is stamped `ingest_source: \"upload\"` and\n`uploaded_by: `. Sessions already present in the organization are\nskipped whole. `dry_run=true` reports what would happen and stores nothing.", + "operationId": "post_events_import", + "parameters": [ + { + "name": "format", + "in": "query", + "description": "`auto` (default), `agenteye`, `claude` or `codex`.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "path", + "in": "query", + "description": "The file's path relative to what was dropped, e.g.\n`-home-me-repo/0b5c….jsonl`. Claude and Codex take the session id from\nit, and a Claude subagent its parent session. A trailing `.gz` is ignored.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "environment", + "in": "query", + "description": "Environment for events that do not name one. Default `upload`.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "dry_run", + "in": "query", + "description": "Convert and report; store nothing.", + "required": false, + "schema": { + "type": "boolean" + } + }, + { + "name": "existing_before", + "in": "query", + "description": "Unix milliseconds. A session counts as already present only when it has\nrows ingested before this. Pass the dry run's `checked_at`, so the later\nfiles of a session this import created are not skipped as \"existing\".\nDefault: now.", + "required": false, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "requestBody": { + "description": "One file: event JSONL, a Claude Code transcript or a Codex rollout. Gzip is accepted.", + "content": { + "application/octet-stream": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "File processed (or, with `dry_run`, previewed)." + }, + "400": { + "description": "Bad `format`, `path` or `environment`." + }, + "401": { + "description": "Missing, unknown, or disabled key." + }, + "403": { + "description": "The key lacks `events:add`." + }, + "413": { + "description": "The file exceeds MAX_BODY_BYTES after decompression." + }, + "422": { + "description": "Unrecognized file format, or a converted event exceeds the storage JSON row limit." + }, + "500": { + "description": "Storage failed — retry the file." + } + }, + "security": [ + { + "api_key": [] + } + ] + } } }, "components": { diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 06f127817..a0c9a0e88 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -4,6 +4,12 @@ description: "Diagnose missing sessions, missing policies, failed delivery, and icon: "wrench" --- +Temporary network failures and HTTP 408, 429, or 5xx responses leave telemetry +eligible for automatic retry, even after a long outage. The daemon preserves the +batch until delivery is acknowledged. An `event_too_large` rejection needs a +smaller event payload before retrying. Files already marked `.poison` need +manual recovery; updating does not automatically resend them. + diff --git a/docs/sessions/overview.mdx b/docs/sessions/overview.mdx index c03551eb1..6f30e6abf 100644 --- a/docs/sessions/overview.mdx +++ b/docs/sessions/overview.mdx @@ -16,6 +16,10 @@ Follow one agent run from its goal through model calls, tools, and final respons ## Find a session +If a session was held in cold storage after you reached your run allowance, +raising the allowance within the same billing window releases eligible sessions +oldest first. Delivery retries do not make a session consume allowance twice. + 1. In the Cloud sidebar, go to **Observe → Sessions**. diff --git a/sdk/python/skill/SKILL.md b/sdk/python/skill/SKILL.md index e12ec1bd2..50683b35e 100644 --- a/sdk/python/skill/SKILL.md +++ b/sdk/python/skill/SKILL.md @@ -391,6 +391,11 @@ platform is the `fp-cloud-cli` skill's job, from a separate environment. ## 6. Production — the collector has to agree with you +Temporary network failures and HTTP 408, 429, or 5xx responses leave the daemon's +telemetry batches eligible for retry. A `422 event_too_large` rejection requires +reducing the event payload; splitting a batch between complete JSON objects +cannot shrink a single event. Existing `.poison` batches need manual recovery. + The SDK writes files. It never talks to the network, so from its point of view a completely unshipped integration looks perfect. diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index 160f7da89..494ad9402 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -2744,11 +2744,11 @@ wheels = [ [[package]] name = "oauthlib" -version = "4.0.0" +version = "4.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7a/d8/a1bcc8ba112a627f8ffbdc212a78ce18d3ac07e91a5ca65d27918eee25a1/oauthlib-4.0.0.tar.gz", hash = "sha256:efb274799819440f95b4ab3b818869f1ce9ae26c5beacba0201d1a1b76b54f86", size = 187232, upload-time = "2026-09-28T06:01:18.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/17/2bb072de0b990303c985f26869fd47bb179024a051f47e7f9d208eb92f8e/oauthlib-4.0.1.tar.gz", hash = "sha256:3d73febdb6faa00f277c81fb04be7b7586fb96fc3b7f0b9c772dada69a25ede7", size = 191219, upload-time = "2026-10-31T09:52:58.008Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/f4/78229a1066068ca14fc60fb26cf7381cabe4382261392b90e5f9552722d4/oauthlib-4.0.0-py3-none-any.whl", hash = "sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd", size = 159715, upload-time = "2026-09-28T06:01:17.008Z" }, + { url = "https://files.pythonhosted.org/packages/df/ef/3b8adde1b32a6f540758bff734d3442c621734c83708e5f02b0ba2f43b84/oauthlib-4.0.1-py3-none-any.whl", hash = "sha256:498a298876e996301950a5b053d12c31628bb7b895db03666430c69ce182eb61", size = 161934, upload-time = "2026-10-31T09:52:56.206Z" }, ] [[package]] diff --git a/src/hooks/delivery-health.ts b/src/hooks/delivery-health.ts index b32da8c73..77f78b724 100644 --- a/src/hooks/delivery-health.ts +++ b/src/hooks/delivery-health.ts @@ -66,7 +66,7 @@ export interface ParkedName { /** * Mirrors `ParkedName::parse` in `crates/fpai-collect/src/uploader.rs`. * - * Suffixes are stripped right to left (`.cNNN` then `.aN`), and each is parsed + * Suffixes are stripped right to left (`.cNNN`, `.rN`, `.sN`, `.aN`), and each is parsed * rather than merely matched, which is what keeps a real spool name safe: a * base like `claude-...-0` has no numeric `.c`/`.a` to find, and anything that * looks like one but is not numeric stays part of the base. @@ -90,6 +90,16 @@ export function parseParkedName(name: string): ParkedName { base = base.slice(0, cIdx); } } + // The CLI needs status and total attempts; the two rejection budgets are Rust-owned. + for (const suffix of [".r", ".s"]) { + const idx = base.lastIndexOf(suffix); + if (idx === -1) continue; + const raw = base.slice(idx + 2); + const n = Number(raw); + if (/^\+?\d+$/.test(raw) && Number.isInteger(n) && n >= 0 && n <= 0xffffffff) { + base = base.slice(0, idx); + } + } const aIdx = base.lastIndexOf(".a"); if (aIdx !== -1) { const n = Number(base.slice(aIdx + 2));